diff --git a/.claude/settings.json b/.claude/settings.json index fcd49c3140..cf8bfdd284 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -110,7 +110,6 @@ ] }, "enabledPlugins": { - "rust-analyzer-lsp@claude-plugins-official": true, "typescript-lsp@claude-plugins-official": true, "code-review@claude-plugins-official": true } diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml new file mode 100644 index 0000000000..ca9ce2aaac --- /dev/null +++ b/.github/workflows/backend-test-windows.yml @@ -0,0 +1,165 @@ +name: Backend integration tests (Windows) + +on: + workflow_dispatch: + push: + branches: + - "ci-windows-tests" + +env: + CARGO_INCREMENTAL: 0 + SQLX_OFFLINE: true + DISABLE_EMBEDDING: true + +jobs: + cargo_test_windows: + runs-on: blacksmith-16vcpu-windows-2025 + steps: + - uses: actions/checkout@v4 + + - name: Read EE repo commit hash + shell: pwsh + run: | + $ee_repo_ref = Get-Content .\backend\ee-repo-ref.txt + echo "ee_repo_ref=$ee_repo_ref" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Checkout windmill-ee-private repository + uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + - name: Substitute EE code + shell: bash + run: | + ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - name: Setup PostgreSQL + uses: ikalnytskyi/action-setup-postgres@v6 + with: + username: postgres + password: changeme + database: windmill + port: 5432 + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: backend + toolchain: 1.93.0 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "9.0.x" + + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + + - uses: actions/setup-go@v2 + with: + go-version: 1.21.5 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.10 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - uses: astral-sh/setup-uv@v6.2.1 + with: + version: "0.9.24" + + - uses: shivammathur/setup-php@v2 + with: + php-version: "8.3" + tools: composer + + - name: Install windmill CLI + shell: bash + run: | + cd cli + bash gen_wm_client.sh + bun install + mkdir -p "$HOME/.local/bin" + printf '#!/bin/sh\nexec bun run "%s/cli/src/main.ts" "$@"\n' "$GITHUB_WORKSPACE" > "$HOME/.local/bin/wmill" + chmod +x "$HOME/.local/bin/wmill" + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install OpenSSL via vcpkg + run: | + vcpkg.exe install openssl-windows:x64-windows + vcpkg.exe install openssl:x64-windows-static + vcpkg.exe integrate install + + - name: Get runtime paths + id: runtime-paths + shell: pwsh + run: | + echo "DENO_PATH=$($(Get-Command deno).Source)" >> $env:GITHUB_OUTPUT + echo "BUN_PATH=$($(Get-Command bun).Source)" >> $env:GITHUB_OUTPUT + echo "NODE_BIN_PATH=$($(Get-Command node).Source)" >> $env:GITHUB_OUTPUT + echo "GO_PATH=$($(Get-Command go).Source)" >> $env:GITHUB_OUTPUT + echo "UV_PATH=$($(Get-Command uv).Source)" >> $env:GITHUB_OUTPUT + echo "PHP_PATH=$($(Get-Command php).Source)" >> $env:GITHUB_OUTPUT + echo "COMPOSER_PATH=$($(Get-Command composer).Source)" >> $env:GITHUB_OUTPUT + echo "POWERSHELL_PATH=$($(Get-Command pwsh).Source)" >> $env:GITHUB_OUTPUT + echo "DOTNET_PATH=$($(Get-Command dotnet).Source)" >> $env:GITHUB_OUTPUT + + - name: Build DuckDB FFI module + working-directory: backend/windmill-duckdb-ffi-internal + timeout-minutes: 30 + run: | + cargo build --release -p windmill_duckdb_ffi_internal + New-Item -ItemType Directory -Path ..\target\debug -Force + Copy-Item target\release\windmill_duckdb_ffi_internal.dll ..\target\debug\ + + - name: Print runtime versions and env + shell: pwsh + run: | + deno --version + bun -v + node --version + go version + python3 --version + php --version + pwsh --version + dotnet --version + echo "TEMP=$env:TEMP" + echo "TMP=$env:TMP" + echo "USERPROFILE=$env:USERPROFILE" + echo "HOME=$env:HOME" + + - name: cargo test + working-directory: backend + timeout-minutes: 60 + env: + DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill + RUST_LOG: "off" + RUST_LOG_STYLE: never + CARGO_NET_GIT_FETCH_WITH_CLI: true + CARGO_BUILD_JOBS: 12 + VCPKGRS_DYNAMIC: 1 + OPENSSL_DIR: ${{ env.VCPKG_INSTALLATION_ROOT }}\installed\x64-windows-static + DENO_PATH: ${{ steps.runtime-paths.outputs.DENO_PATH }} + BUN_PATH: ${{ steps.runtime-paths.outputs.BUN_PATH }} + NODE_BIN_PATH: ${{ steps.runtime-paths.outputs.NODE_BIN_PATH }} + GO_PATH: ${{ steps.runtime-paths.outputs.GO_PATH }} + UV_PATH: ${{ steps.runtime-paths.outputs.UV_PATH }} + PHP_PATH: ${{ steps.runtime-paths.outputs.PHP_PATH }} + COMPOSER_PATH: ${{ steps.runtime-paths.outputs.COMPOSER_PATH }} + POWERSHELL_PATH: ${{ steps.runtime-paths.outputs.POWERSHELL_PATH }} + DOTNET_PATH: ${{ steps.runtime-paths.outputs.DOTNET_PATH }} + WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1 + WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1 + WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1 + run: > + cargo test + --no-fail-fast + --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,csharp,php,quickjs,mcp,run_inline + --all + -- --nocapture --test-threads=10 diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 8ef5eecea9..be537abb11 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -1,6 +1,7 @@ name: Backend only integration tests on: + workflow_dispatch: push: branches: - "main" diff --git a/.webmux.yaml b/.webmux.yaml new file mode 100644 index 0000000000..84f82d4a11 --- /dev/null +++ b/.webmux.yaml @@ -0,0 +1,105 @@ +# Project display name in the dashboard +name: Windmill + +workspace: + mainBranch: main + worktreeRoot: ../windmill__worktrees + defaultAgent: claude + +startupEnvs: + CARGO_FEATURES: "quickjs" + WM_CLONE_DB: false + USE_RUST_PLUGIN: false + +lifecycleHooks: + postCreate: bash ./scripts/post-create.sh + preRemove: bash ./scripts/pre-remove.sh + +auto_name: + provider: claude + model: haiku + +# Each service defines a port env var that webmux injects into pane and agent +# process environments when creating a worktree. Ports are auto-assigned: +# base + (slot x step). +services: + - name: backend + portEnv: BACKEND_PORT + portStart: 8000 + portStep: 10 + - name: frontend + portEnv: FRONTEND_PORT + portStart: 3000 + portStep: 10 + +profiles: + full: + runtime: host + yolo: true + envPassthrough: [] + systemPrompt: > + You are running inside a tmux session with other panes running services. + Pane layout (current window): + - Pane 0: this pane (claude agent) + - Pane 1: backend (cargo watch -x run) + - Pane 2: frontend (npm run dev) + To check logs, use: \`tmux capture-pane -t .1 -p -S -50\` (backend) or \`tmux capture-pane -t .2 -p -S -50\` (frontend). + When restarting backend or frontend, make sure to use ${BACKEND_PORT} and ${FRONTEND_PORT}. + To connect to the database, use this connection string: ${DATABASE_URL} + Because we are running backend with cargo watch, to verify your changes, just check the logs in the backend pane. No need for cargo check. + IMPORTANT: Read docs/autonomous-mode.md before starting any work. + panes: + - id: agent + kind: agent + focus: true + - id: backend + kind: command + split: right + command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/backend" && PORT=${BACKEND_PORT:-8000} cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}" + - id: frontend + kind: command + split: bottom + command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/frontend" && npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0 + + frontendOnly: + runtime: host + yolo: true + envPassthrough: [] + systemPrompt: > + You are running inside a tmux session with other panes running services. + Pane layout (current window): + - Pane 0: this pane (claude agent) + - Pane 1: frontend (npm run dev) + To check logs, use: \`tmux capture-pane -t .1 -p -S -50\` (frontend). + When restarting frontend, make sure to use ${FRONTEND_PORT}. + To connect to the database, use this connection string: ${DATABASE_URL} + Because we are running frontend with npm run dev, to verify your changes, just check the logs in the frontend pane. No need for npm run build. + IMPORTANT: Read docs/autonomous-mode.md before starting any work. + panes: + - id: agent + kind: agent + focus: true + - id: frontend + kind: command + split: right + command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/frontend" && npm run generate-backend-client && npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0 + + agentOnly: + runtime: host + yolo: true + envPassthrough: [] + systemPrompt: > + IMPORTANT: Read docs/autonomous-mode.md before starting any work. + panes: + - id: agent + kind: agent + focus: true + +integrations: + github: + linkedRepos: + - repo: windmill-labs/windmill-ee-private + alias: ee-private + dir: ../windmill-ee-private__worktrees + linear: + enabled: true diff --git a/.wmdev.yaml b/.wmdev.yaml deleted file mode 100644 index c028c8f3bf..0000000000 --- a/.wmdev.yaml +++ /dev/null @@ -1,111 +0,0 @@ -name: Windmill - -startupEnvs: - CARGO_FEATURES: "quickjs" - -services: - - name: BE - portEnv: BACKEND_PORT - - name: FE - portEnv: FRONTEND_PORT - -profiles: - default: - name: default - - sandbox: - name: sandbox - image: windmill-sandbox - envPassthrough: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - R2_ENDPOINT - - R2_BUCKET - - R2_PUBLIC_URL - extraMounts: - - hostPath: ~/.ssh - guestPath: /root/.ssh - writable: true - - hostPath: ~/.codex - guestPath: /root/.codex - writable: true - - hostPath: ~/windmill-ee-private - writable: true - - hostPath: ~/windmill-ee-private__worktrees - writable: true - systemPrompt: > - You are running inside a sandboxed container with full permissions. - This worktree is configured with the following ports: - - - Backend: port ${BACKEND_PORT}. - Start with: cd backend && PORT=${BACKEND_PORT} - DATABASE_URL=postgres://postgres:changeme@localhost:5432/windmill - cargo watch -x run - - - Frontend: port ${FRONTEND_PORT}. - Start with: cd frontend && REMOTE=http://localhost:${BACKEND_PORT} - npm run dev -- --port ${FRONTEND_PORT} --host 0.0.0.0 - - --- Screenshots --- - You can take screenshots of the frontend UI and upload them to R2 - for use in PR descriptions. - 1) Take a screenshot: - bunx playwright screenshot --browser chromium - http://localhost:${FRONTEND_PORT}/path/to/page /tmp/screenshot.png - 2) Upload to R2: - aws s3 cp /tmp/screenshot.png - "s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/screenshot.png" - --endpoint-url "$(printenv R2_ENDPOINT)" - 3) The public URL will be: - $(printenv R2_PUBLIC_URL)//screenshot.png - 4) Include in PR descriptions using markdown image syntax. - - --- Terminal Recordings (asciinema) --- - You can record terminal sessions and upload them for sharing. - asciinema is available on PATH. - - 1) Write a shell script with the commands to demo. Add sleep - delays for readable pacing: - - 0.5s after printing a "$ command" line (lets viewer read it) - - 1.5-2s after command output (lets viewer absorb the result) - - Set GIT_PAGER=cat and PAGER=cat to prevent pager hangs - - 2) Record headlessly: - asciinema rec --headless --overwrite \ - -c "bash /tmp/demo.sh" \ - --window-size 120x50 \ - --title "Description of demo" \ - /tmp/demo.cast - - 3) Upload to asciinema.org: - XDG_DATA_HOME=/tmp/.local/share \ - asciinema upload --server-url https://asciinema.org /tmp/demo.cast - - --- Mermaid Diagrams --- - You can render Mermaid diagrams to SVG using the pre-installed mmdc CLI. - The puppeteer config (no-sandbox + Chromium path) is at /root/.puppeteerrc.json. - - 1) Write a .mmd file with your diagram: - cat > /tmp/diagram.mmd << 'EOF' - graph TD - A[Start] --> B[End] - EOF - - 2) Render to SVG (the -p flag is required): - mmdc -i /tmp/diagram.mmd -o /tmp/diagram.svg -p /root/.puppeteerrc.json - - 3) Upload to R2: - aws s3 cp /tmp/diagram.svg - "s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/diagram.svg" - --endpoint-url "$(printenv R2_ENDPOINT)" - - 4) The public URL will be: - $(printenv R2_PUBLIC_URL)//diagram.svg - - 5) Include in PR descriptions using markdown image syntax. - - IMPORTANT: Read docs/autonomous-mode.md before starting any work. - -linkedRepos: - - repo: windmill-labs/windmill-ee-private - alias: ee diff --git a/CHANGELOG.md b/CHANGELOG.md index 15fca3e649..61457b258d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,135 @@ # Changelog +## [1.654.0](https://github.com/windmill-labs/windmill/compare/v1.653.0...v1.654.0) (2026-03-10) + + +### Features + +* add git sync support for workspace dependencies ([#8144](https://github.com/windmill-labs/windmill/issues/8144)) ([4f29e05](https://github.com/windmill-labs/windmill/commit/4f29e05e3ae725e0be7ab797f8fa2186d8c5c0a5)) +* add kafka trigger offset reset and auto.offset.reset config ([#8283](https://github.com/windmill-labs/windmill/issues/8283)) ([b02f9e5](https://github.com/windmill-labs/windmill/commit/b02f9e5c2426bff2356e1aaaa18e05b18c5efc6b)) +* add preprocessor support for dedicated workers and bunnative scripts ([#8284](https://github.com/windmill-labs/windmill/issues/8284)) ([dc0e59f](https://github.com/windmill-labs/windmill/commit/dc0e59f432a0e3a53606adb8ac76d2dd2d365ace)) +* add Vertex AI support for Google Gemini models ([#8303](https://github.com/windmill-labs/windmill/issues/8303)) ([cb349cb](https://github.com/windmill-labs/windmill/commit/cb349cb3d1b7561fb70a8c23fa83dc1c9441821c)) +* **frontend:** replace flat sugiyama with recursive compound layout for flow graph ([#8204](https://github.com/windmill-labs/windmill/issues/8204)) ([cad4436](https://github.com/windmill-labs/windmill/commit/cad44365ac17029a2257f12cef061219b0265570)) + + +### Bug Fixes + +* **cli:** fail when passing an invalid --workspace arg ([#8294](https://github.com/windmill-labs/windmill/issues/8294)) ([f291b1c](https://github.com/windmill-labs/windmill/commit/f291b1cc19689e69e7aa008c19ce747e9c56240e)) +* debounce webhook arg accumulation with max_count/max_time limits ([#8307](https://github.com/windmill-labs/windmill/issues/8307)) ([83be59e](https://github.com/windmill-labs/windmill/commit/83be59e0e866ebd091f1e27c0571710a989fd2e4)) +* delete debounce_key on post-preprocessing limit exceeded ([#8299](https://github.com/windmill-labs/windmill/issues/8299)) ([438f609](https://github.com/windmill-labs/windmill/commit/438f609a78325ee5c2493079ca27bf587fa0d5ff)) +* explicilty fail when --base-url --token --workspace are invalid ([#8302](https://github.com/windmill-labs/windmill/issues/8302)) ([5baeb8c](https://github.com/windmill-labs/windmill/commit/5baeb8c842a392c21457b7561e30b385e02a6a48)) +* handle missing schema in RunnableByPath during wmill.d.ts generation ([#8300](https://github.com/windmill-labs/windmill/issues/8300)) ([b841e0a](https://github.com/windmill-labs/windmill/commit/b841e0a0384941079f37374f8fbbe2dd7fb51897)) +* optimize flow lock generation and add rt.d.ts guidance for TS resource types ([#8295](https://github.com/windmill-labs/windmill/issues/8295)) ([b40cf80](https://github.com/windmill-labs/windmill/commit/b40cf80fdd62cbc31db0872ada551ce213b9dac8)) +* preserve teams oauth tenant on settings page reload ([#8308](https://github.com/windmill-labs/windmill/issues/8308)) ([dbfa271](https://github.com/windmill-labs/windmill/commit/dbfa271b8962fe7b3d2aa8bf494e9557047fc8b3)) +* resync custom_instance_user password on startup ([#8297](https://github.com/windmill-labs/windmill/issues/8297)) ([53ac43f](https://github.com/windmill-labs/windmill/commit/53ac43f5ee34570a9bb7b3441c73095e23690300)) +* show meaningful error messages in database manager schema fetch ([#8296](https://github.com/windmill-labs/windmill/issues/8296)) ([cda8439](https://github.com/windmill-labs/windmill/commit/cda843922dcfd9a02ef9926751cbf8f544d2d4b6)) +* skip loading flow preview history for new flows ([#8293](https://github.com/windmill-labs/windmill/issues/8293)) ([ac8c668](https://github.com/windmill-labs/windmill/commit/ac8c668cb93e56bc2a247bbdbbec14e5608125d2)) +* teams selection not sticking in workspace settings ([#8309](https://github.com/windmill-labs/windmill/issues/8309)) ([fefc8c6](https://github.com/windmill-labs/windmill/commit/fefc8c62a00fe7a39f3104091e08087cd7c37afb)) + +## [1.653.0](https://github.com/windmill-labs/windmill/compare/v1.652.0...v1.653.0) (2026-03-10) + + +### Features + +* add indexer time window setting (default 7 days) ([#8290](https://github.com/windmill-labs/windmill/issues/8290)) ([0c4d72c](https://github.com/windmill-labs/windmill/commit/0c4d72cfe38d61cf3f6e9bc31056005f1adb494d)) +* add slack connection fields to workspace settings export/import ([#8287](https://github.com/windmill-labs/windmill/issues/8287)) ([39e77ec](https://github.com/windmill-labs/windmill/commit/39e77ecd002b41630fa8d146ee0f15369656acda)) + + +### Performance Improvements + +* optimize job_stats storage for timestamps and zero-memory jobs ([#8289](https://github.com/windmill-labs/windmill/issues/8289)) ([2d8335d](https://github.com/windmill-labs/windmill/commit/2d8335dc43a7cb182eb5a058119d8b0be067cdfd)) + +## [1.652.0](https://github.com/windmill-labs/windmill/compare/v1.651.1...v1.652.0) (2026-03-09) + + +### Features + +* add secretKeyRef support for package registry and storage credentials ([#8275](https://github.com/windmill-labs/windmill/issues/8275)) ([73d27e9](https://github.com/windmill-labs/windmill/commit/73d27e92dd6ced1602f6328f245fec0fa96860e1)) +* expose OTEL trace context as env vars in job execution ([#8277](https://github.com/windmill-labs/windmill/issues/8277)) ([93f75ad](https://github.com/windmill-labs/windmill/commit/93f75ada5e49036f0d998e3d3d53de4dc2c2e83f)) +* workflow-as-code (WAC) v2 ([#8172](https://github.com/windmill-labs/windmill/issues/8172)) ([a6d4390](https://github.com/windmill-labs/windmill/commit/a6d4390790d21d535df1e9d525bffd577c50d8dc)) + + +### Bug Fixes + +* cli: support deleting linked resources-variables without throwing ([#8248](https://github.com/windmill-labs/windmill/issues/8248)) ([7859bca](https://github.com/windmill-labs/windmill/commit/7859bca6ae80d32a73a46910960afc6812e64115)) +* Database studio fixes ([#8251](https://github.com/windmill-labs/windmill/issues/8251)) ([1d78589](https://github.com/windmill-labs/windmill/commit/1d785899404e8636a206cda9a2914df32a1a5269)) +* **frontend:** unsaved changes dialog when flow already saved ([#8259](https://github.com/windmill-labs/windmill/issues/8259)) ([0330993](https://github.com/windmill-labs/windmill/commit/0330993cb66cdabffcd6e552a0f85a9a3931c62d)) +* gracefully handle uninitialized OTEL tracing proxy port ([#8274](https://github.com/windmill-labs/windmill/issues/8274)) ([8b1fe8f](https://github.com/windmill-labs/windmill/commit/8b1fe8f9de7b0c03655558d0c46cfff71a4b2047)) +* guard iteration picker VirtualList against empty items array ([#8273](https://github.com/windmill-labs/windmill/issues/8273)) ([c97cf60](https://github.com/windmill-labs/windmill/commit/c97cf604ab4a902d89fe873b90dbeb9dabc940eb)), closes [#8272](https://github.com/windmill-labs/windmill/issues/8272) +* mask secrets in OAuth config debug/log output ([#8269](https://github.com/windmill-labs/windmill/issues/8269)) ([e75763d](https://github.com/windmill-labs/windmill/commit/e75763dbe5ffe08e6cde082203596d510c2c3b29)) +* parallel branchall hang on bad stop_after_all_iters_if + results.x.length null ([#8276](https://github.com/windmill-labs/windmill/issues/8276)) ([41e523f](https://github.com/windmill-labs/windmill/commit/41e523f827c4e3d5db525a1f14e24936b0b8af46)) +* redact secrets in set_global_setting log line ([#8270](https://github.com/windmill-labs/windmill/issues/8270)) ([6a0473c](https://github.com/windmill-labs/windmill/commit/6a0473c5783dc0fef2ae82dc5345a5f0596f124d)) +* remove $bindable() fallback values causing props_invalid_value error in oauth settings ([#8265](https://github.com/windmill-labs/windmill/issues/8265)) ([037035e](https://github.com/windmill-labs/windmill/commit/037035e094937827305dad29bd76a495d78bc46f)) +* skip down migrations in potentially_stale checksum comparison ([#8271](https://github.com/windmill-labs/windmill/issues/8271)) ([5ba4029](https://github.com/windmill-labs/windmill/commit/5ba4029d8692b2e6054fca7f45ed4cfded4738ef)) +* sql input horizontal scroll missing after switching flow steps ([#8249](https://github.com/windmill-labs/windmill/issues/8249)) ([ce8ac9c](https://github.com/windmill-labs/windmill/commit/ce8ac9cf52dc17061673b9b72556279c48c26f8e)) +* wmill workspace whoami output ([#8246](https://github.com/windmill-labs/windmill/issues/8246)) ([1ac391a](https://github.com/windmill-labs/windmill/commit/1ac391a795585747fe5911ac41b157556569fedb)) + +## [1.651.1](https://github.com/windmill-labs/windmill/compare/v1.651.0...v1.651.1) (2026-03-05) + + +### Bug Fixes + +* prevent slow loading toast interval from leaking on promise cancellation ([#8240](https://github.com/windmill-labs/windmill/issues/8240)) ([2e582b1](https://github.com/windmill-labs/windmill/commit/2e582b1bc1c299388a3c97cfddff9d0eb92858f2)) +* suppress unused variable warnings on windows builds ([#8241](https://github.com/windmill-labs/windmill/issues/8241)) ([2d58382](https://github.com/windmill-labs/windmill/commit/2d583826dc065c05684d4cd1d1510f0d1f2d9ae9)) + +## [1.651.0](https://github.com/windmill-labs/windmill/compare/v1.650.0...v1.651.0) (2026-03-05) + + +### Features + +* add sandbox annotations, volume mounts, for AI sandbox starting with claude ([#8058](https://github.com/windmill-labs/windmill/issues/8058)) ([5f0ef93](https://github.com/windmill-labs/windmill/commit/5f0ef936d1d5d07d01c8e07e26ec254feebef8fb)) +* hash-based MCP tool names for long paths ([#8133](https://github.com/windmill-labs/windmill/issues/8133)) ([ce041e8](https://github.com/windmill-labs/windmill/commit/ce041e8a5e7ff105df389875d9981f3843d4ce39)) + + +### Bug Fixes + +* **python-client:** add delete_s3_object ([#8216](https://github.com/windmill-labs/windmill/issues/8216)) ([90f4c64](https://github.com/windmill-labs/windmill/commit/90f4c64ee12e1d04ce846ff88d6658f667e194e0)) +* update CLI bun template to match UI template ([#8238](https://github.com/windmill-labs/windmill/issues/8238)) ([a8cbe93](https://github.com/windmill-labs/windmill/commit/a8cbe9396ffc51140dce5582d57f4dc59873304e)) +* write fallback package.json for codebase mode nsjail ([#8239](https://github.com/windmill-labs/windmill/issues/8239)) ([d46913b](https://github.com/windmill-labs/windmill/commit/d46913b74a0ffd41d2323e0355cc81954f09e29d)) + +## [1.650.0](https://github.com/windmill-labs/windmill/compare/v1.649.0...v1.650.0) (2026-03-05) + + +### Features + +* add move, delete, and duplicate to flow node context menu ([#8050](https://github.com/windmill-labs/windmill/issues/8050)) ([c0c9388](https://github.com/windmill-labs/windmill/commit/c0c9388415716ce77d841bd08a46f94e0a529685)) +* add variable and resource types to flow env variables ([#8214](https://github.com/windmill-labs/windmill/issues/8214)) ([164e499](https://github.com/windmill-labs/windmill/commit/164e499c64dc5eb76fcfb0f8cefbad2df244f610)) +* Ducklake typechecker ([#8118](https://github.com/windmill-labs/windmill/issues/8118)) ([53caecf](https://github.com/windmill-labs/windmill/commit/53caecf1da8d76e246178dfb9b86d330f0ec52fd)) +* make WINDMILL_DIR configurable via environment variable ([#8215](https://github.com/windmill-labs/windmill/issues/8215)) ([424ca59](https://github.com/windmill-labs/windmill/commit/424ca59dfe3e730f5388d9cac4ea7e69773614d3)) +* make WM_END_USER_EMAIL display users from different workspaces ([#8208](https://github.com/windmill-labs/windmill/issues/8208)) ([baf2bcf](https://github.com/windmill-labs/windmill/commit/baf2bcf14da0c8c95bdbbf511fcaee48be33948b)) +* persistent Db manager state in URI ([#8134](https://github.com/windmill-labs/windmill/issues/8134)) ([4bf827b](https://github.com/windmill-labs/windmill/commit/4bf827bea4d44aca8c5ff7aa67ad449dbcf00673)) +* replace hub error toasts with warning alerts and add disable hub setting ([#8225](https://github.com/windmill-labs/windmill/issues/8225)) ([63ebae8](https://github.com/windmill-labs/windmill/commit/63ebae8829a6dc47a4e23c8670b514f042c9d4be)) +* token expiration notifications ([#8190](https://github.com/windmill-labs/windmill/issues/8190)) ([e56ccd2](https://github.com/windmill-labs/windmill/commit/e56ccd200be29e6ac8ea2b04a341b1ce78a307f6)) + + +### Bug Fixes + +* handle multipart stream errors gracefully instead of panicking ([#8226](https://github.com/windmill-labs/windmill/issues/8226)) ([19c065b](https://github.com/windmill-labs/windmill/commit/19c065bed5468c484c8e7a50a6b79ab90153cc0e)) +* improve windows compatibility ([077779e](https://github.com/windmill-labs/windmill/commit/077779ec52f7d3e5fcc93951544bf47bd6dc30b6)) +* wrap set_encryption_key in a single database transaction ([#8212](https://github.com/windmill-labs/windmill/issues/8212)) ([62382fd](https://github.com/windmill-labs/windmill/commit/62382fd2869ea0190dd0c0b714f9cbd35ceddd7a)) + +## [1.649.0](https://github.com/windmill-labs/windmill/compare/v1.648.0...v1.649.0) (2026-03-03) + + +### Features + +* **frontend:** add script recorder for offline replay ([#8200](https://github.com/windmill-labs/windmill/issues/8200)) ([c97d8b4](https://github.com/windmill-labs/windmill/commit/c97d8b4715f86ea83ab2c0223ba859ced690829a)) +* move index management out of /srch/, add storage size reporting ([#8169](https://github.com/windmill-labs/windmill/issues/8169)) ([ee01acd](https://github.com/windmill-labs/windmill/commit/ee01acd9a6a2cd68a3f226988bfb46f6a6e64c08)) + + +### Bug Fixes + +* clean up slow-load toast interval on component destroy ([#8207](https://github.com/windmill-labs/windmill/issues/8207)) ([26f4f2b](https://github.com/windmill-labs/windmill/commit/26f4f2b399b828185b553289d6560e12261030a3)) +* **frontend:** prevent subflow expansion from hiding all insertion points ([#8203](https://github.com/windmill-labs/windmill/issues/8203)) ([e97da86](https://github.com/windmill-labs/windmill/commit/e97da860672171e33054a77d71f4824bb09e540d)) +* gracefully handle malformed OAuth entries in instance config ([#8205](https://github.com/windmill-labs/windmill/issues/8205)) ([cac4bdd](https://github.com/windmill-labs/windmill/commit/cac4bdd54f0c3ea80844ac31f7597f418ff7d8ae)) +* skip stop_after_if evaluation for skipped (identity) flow steps ([#8201](https://github.com/windmill-labs/windmill/issues/8201)) ([e6f7775](https://github.com/windmill-labs/windmill/commit/e6f7775d4d9a052aefc37260c6ed161146841cd7)) +* use exact matching for python requirements directive parsing ([#8199](https://github.com/windmill-labs/windmill/issues/8199)) ([2b2be38](https://github.com/windmill-labs/windmill/commit/2b2be38f129bbe58b6bb3815c4bd94aa03a3da90)) + + +### Performance Improvements + +* use two-step query in input history to leverage v2_job index ([#8197](https://github.com/windmill-labs/windmill/issues/8197)) ([50defdd](https://github.com/windmill-labs/windmill/commit/50defdded113b4d2cf0991b3fb642d1cd9a462b7)) + ## [1.648.0](https://github.com/windmill-labs/windmill/compare/v1.647.2...v1.648.0) (2026-03-02) diff --git a/CLAUDE.md b/CLAUDE.md index 4e7afeba8a..fe22fae0f7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,7 @@ Open-source platform for internal tools, workflows, API integrations, background - **Validation**: `docs/validation.md` — what checks to run based on what you changed - **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow - **Backend patterns**: use the `rust-backend` skill when writing Rust code -- **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code +- **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill. - **Domain guides**: `.claude/skills/native-trigger/` and `frontend/tutorial-system-guide.mdc` - **Brand/UI guidelines**: `frontend/brand-guidelines.md` @@ -26,6 +26,29 @@ Open-source platform for internal tools, workflows, API integrations, background - **Login**: `admin@windmill.dev` / `changeme` - **Instance settings**: navigate to `/#superadmin-settings` +## Banned Patterns + +### `$bindable(default_value)` on optional props + +Using `$bindable(default_value)` on props that can be `undefined` is **banned**. This pattern causes subtle bugs because the default value masks the `undefined` state. + +**Bad:** + +```svelte +let { my_prop = $bindable(default_value) }: { my_prop?: string } = $props() +``` + +**Correct alternatives:** + +1. **Use `$derived` with nullish coalescing** — handle the potential `undefined` at the usage site: + + ```svelte + let { my_prop = $bindable() }: { my_prop?: string } = $props() + let effective_value = $derived(my_prop ?? default_value) + ``` + +2. **Create a `useMyPropState()` helper** — encapsulate the undefined-handling logic in a reusable function and call it higher in the component tree, so the child component always receives a defined value. + ## Core Principles - Search for existing code to reuse before writing new code diff --git a/Dockerfile b/Dockerfile index 7cca6ab329..70a28b2f96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -262,11 +262,17 @@ COPY --from=oven/bun:1.3.10 /usr/local/bin/bun /usr/bin/bun RUN bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill -COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php -COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer +# Install Claude Code CLI (used by claude sandbox scripts) +# The installer puts the binary in ~/.local/bin/claude (symlink to ~/.local/share/claude/versions/*) +# Copy it to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + +COPY --from=php:8.3.30-cli /usr/local/bin/php /usr/bin/php +COPY --from=composer:2.9.5 /usr/bin/composer /usr/bin/composer # add the docker client to call docker from a worker if enabled -COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ +COPY --from=docker:29-dind /usr/local/bin/docker /usr/local/bin/ ENV RUSTUP_HOME="/tmp/windmill/cache/rustup" ENV CARGO_HOME="/tmp/windmill/cache/cargo" diff --git a/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json b/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json new file mode 100644 index 0000000000..d9d9793cd1 --- /dev/null +++ b/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = now() + interval '60 seconds'\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f" +} diff --git a/backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json b/backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json similarity index 51% rename from backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json rename to backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json index 84b10ccba6..409faa032f 100644 --- a/backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json +++ b/backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json @@ -1,16 +1,17 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT token\n FROM token\n WHERE token LIKE concat($1::text, '%')\n LIMIT 1\n ", + "query": "SELECT group_ FROM usr_to_group WHERE usr = $1 AND workspace_id = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "token", + "name": "group_", "type_info": "Varchar" } ], "parameters": { "Left": [ + "Text", "Text" ] }, @@ -18,5 +19,5 @@ false ] }, - "hash": "90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492" + "hash": "015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11" } diff --git a/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json b/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json new file mode 100644 index 0000000000..52fb375962 --- /dev/null +++ b/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT large_file_storage->>'volume_storage' FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e" +} diff --git a/backend/.sqlx/query-0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451.json b/backend/.sqlx/query-0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451.json new file mode 100644 index 0000000000..afc74d8e7a --- /dev/null +++ b/backend/.sqlx/query-0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451" +} diff --git a/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json b/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json new file mode 100644 index 0000000000..6a6b77e650 --- /dev/null +++ b/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "extra_perms", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4" +} diff --git a/backend/.sqlx/query-0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699.json b/backend/.sqlx/query-0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699.json new file mode 100644 index 0000000000..b8e52cdbe7 --- /dev/null +++ b/backend/.sqlx/query-0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET canceled_by = $2, canceled_reason = $3 WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699" +} diff --git a/backend/.sqlx/query-0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f.json b/backend/.sqlx/query-0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f.json new file mode 100644 index 0000000000..10141b3a78 --- /dev/null +++ b/backend/.sqlx/query-0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT j.id, j.runnable_path, j.args, j.kind::text AS \"kind!\"\n FROM v2_job j\n JOIN v2_job_queue q ON j.id = q.id\n WHERE j.runnable_path = $1\n AND j.kind = 'deploymentcallback'\n ORDER BY j.created_at DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "args", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "kind!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + null + ] + }, + "hash": "0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f" +} diff --git a/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json b/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json new file mode 100644 index 0000000000..0140324406 --- /dev/null +++ b/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750" +} diff --git a/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json b/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json new file mode 100644 index 0000000000..3f39982319 --- /dev/null +++ b/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6" +} diff --git a/backend/.sqlx/query-12631fecee6aa11a45cf5c8d101c0dd8de50ac9b57e68198f637038d344fdd46.json b/backend/.sqlx/query-12631fecee6aa11a45cf5c8d101c0dd8de50ac9b57e68198f637038d344fdd46.json new file mode 100644 index 0000000000..c452c33018 --- /dev/null +++ b/backend/.sqlx/query-12631fecee6aa11a45cf5c8d101c0dd8de50ac9b57e68198f637038d344fdd46.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n auto_offset_reset = $5,\n script_path = $6,\n path = $7,\n is_flow = $8,\n edited_by = $9,\n email = $10,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $13,\n error_handler_args = $14,\n retry = $15\n WHERE\n workspace_id = $11 AND path = $12\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "VarcharArray", + "JsonbArray", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "12631fecee6aa11a45cf5c8d101c0dd8de50ac9b57e68198f637038d344fdd46" +} diff --git a/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json b/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json new file mode 100644 index 0000000000..9a6ae60a49 --- /dev/null +++ b/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6" +} diff --git a/backend/.sqlx/query-16c96166ffa6b9aec65c6072b204b52b87e3c2f3d76e47eb173fc78721355066.json b/backend/.sqlx/query-16c96166ffa6b9aec65c6072b204b52b87e3c2f3d76e47eb173fc78721355066.json deleted file mode 100644 index 7fd747c125..0000000000 --- a/backend/.sqlx/query-16c96166ffa6b9aec65c6072b204b52b87e3c2f3d76e47eb173fc78721355066.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH _ AS (\n UPDATE debounce_key\n SET debounced_times = 0, -- reset debounced_times\n first_started_at = now(), -- rest\n previous_job_id = NULL\n WHERE job_id = $1\n )\n UPDATE v2_job_debounce_batch \n SET debounce_batch = nextval('debounce_batch_seq') -- move to new batch\n WHERE id = $1\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "16c96166ffa6b9aec65c6072b204b52b87e3c2f3d76e47eb173fc78721355066" -} diff --git a/backend/.sqlx/query-19a7ebb2e7e8e57b6e7c974da8eb7c6841a5c4ff12ba7c12c73d691c49dd99ed.json b/backend/.sqlx/query-19a7ebb2e7e8e57b6e7c974da8eb7c6841a5c4ff12ba7c12c73d691c49dd99ed.json new file mode 100644 index 0000000000..18ad13d90f --- /dev/null +++ b/backend/.sqlx/query-19a7ebb2e7e8e57b6e7c974da8eb7c6841a5c4ff12ba7c12c73d691c49dd99ed.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email FROM token WHERE token = $1 AND (expiration > NOW() OR expiration IS NULL)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "19a7ebb2e7e8e57b6e7c974da8eb7c6841a5c4ff12ba7c12c73d691c49dd99ed" +} diff --git a/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json b/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json new file mode 100644 index 0000000000..9514010409 --- /dev/null +++ b/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e" +} diff --git a/backend/.sqlx/query-1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636.json b/backend/.sqlx/query-1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636.json deleted file mode 100644 index aaa1ae945b..0000000000 --- a/backend/.sqlx/query-1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636" -} diff --git a/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json b/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json new file mode 100644 index 0000000000..fa67a0797d --- /dev/null +++ b/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at)\n VALUES ($1, $2, $3, $4, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET size_bytes = $3, last_used_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab" +} diff --git a/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json b/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json new file mode 100644 index 0000000000..8897e8a7de --- /dev/null +++ b/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc" +} diff --git a/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json b/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json new file mode 100644 index 0000000000..2010b40667 --- /dev/null +++ b/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98" +} diff --git a/backend/.sqlx/query-2c8b8ed14647332491846ae3fa8b0ab8113d52ae8ae613a810c2b452e0972d05.json b/backend/.sqlx/query-2c8b8ed14647332491846ae3fa8b0ab8113d52ae8ae613a810c2b452e0972d05.json new file mode 100644 index 0000000000..a47db0ecab --- /dev/null +++ b/backend/.sqlx/query-2c8b8ed14647332491846ae3fa8b0ab8113d52ae8ae613a810c2b452e0972d05.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2c8b8ed14647332491846ae3fa8b0ab8113d52ae8ae613a810c2b452e0972d05" +} diff --git a/backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json b/backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json new file mode 100644 index 0000000000..8c5f43ab07 --- /dev/null +++ b/backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n CASE\n WHEN flow_version.id IS NOT NULL THEN\n flow_version.value -> 'flow_env' -> $3\n ELSE\n root_job.raw_flow -> 'flow_env' -> $3\n END AS \"flow_env: sqlx::types::Json>\"\n FROM\n v2_job current_job\n JOIN\n v2_job root_job ON root_job.id = COALESCE(current_job.root_job, current_job.flow_innermost_root_job, current_job.parent_job, current_job.id)\n AND root_job.workspace_id = current_job.workspace_id\n LEFT JOIN\n flow_version ON flow_version.id = root_job.runnable_id\n AND flow_version.path = root_job.runnable_path\n AND flow_version.workspace_id = root_job.workspace_id\n WHERE\n current_job.id = $1 AND\n current_job.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flow_env: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943" +} diff --git a/backend/.sqlx/query-36b95bc7956eb7bba7cd6fa9cd829980a0bf4970b919cabad1daab16627404fc.json b/backend/.sqlx/query-36b95bc7956eb7bba7cd6fa9cd829980a0bf4970b919cabad1daab16627404fc.json new file mode 100644 index 0000000000..e625a747f7 --- /dev/null +++ b/backend/.sqlx/query-36b95bc7956eb7bba7cd6fa9cd829980a0bf4970b919cabad1daab16627404fc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (config->>'native_mode')::boolean FROM config WHERE name = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "bool", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "36b95bc7956eb7bba7cd6fa9cd829980a0bf4970b919cabad1daab16627404fc" +} diff --git a/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json b/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json new file mode 100644 index 0000000000..2fdb1ae80d --- /dev/null +++ b/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), updated_by = $5, last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int4", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34" +} diff --git a/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json b/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json new file mode 100644 index 0000000000..c73c00c2aa --- /dev/null +++ b/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json @@ -0,0 +1,76 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n name as \"name!\",\n size_bytes as \"size_bytes!\",\n file_count as \"file_count!\",\n created_at as \"created_at!\",\n created_by as \"created_by!\",\n updated_at,\n updated_by,\n description as \"description!\",\n last_used_at,\n extra_perms as \"extra_perms!\"\n FROM (\n SELECT\n COALESCE(v.name, a.path) as name,\n COALESCE(v.size_bytes, 0) as size_bytes,\n COALESCE(v.file_count, 0) as file_count,\n COALESCE(v.created_at, a.min_created_at) as created_at,\n COALESCE(v.created_by, 'unknown') as created_by,\n v.updated_at,\n v.updated_by,\n COALESCE(v.description, '') as description,\n v.last_used_at,\n COALESCE(v.extra_perms, '{}'::jsonb) as extra_perms\n FROM (\n SELECT path, MIN(created_at) as min_created_at\n FROM asset\n WHERE workspace_id = $1 AND kind = 'volume'\n GROUP BY path\n ) a\n FULL OUTER JOIN volume v ON v.workspace_id = $1 AND v.name = a.path\n WHERE v.workspace_id = $1 OR a.path IS NOT NULL\n ) combined\n ORDER BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "size_bytes!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "file_count!", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "created_at!", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "updated_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "updated_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "description!", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "last_used_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "extra_perms!", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + true, + true, + null, + true, + null + ] + }, + "hash": "40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468" +} diff --git a/backend/.sqlx/query-aed5439aa6dad950e505f9f8f6914fa5ca21319c501b2822c9bc751ddfc9a0a4.json b/backend/.sqlx/query-4b5a711986017654bdd495893a16ddc6ab09c98cd8723865cbd341404bc6a02f.json similarity index 59% rename from backend/.sqlx/query-aed5439aa6dad950e505f9f8f6914fa5ca21319c501b2822c9bc751ddfc9a0a4.json rename to backend/.sqlx/query-4b5a711986017654bdd495893a16ddc6ab09c98cd8723865cbd341404bc6a02f.json index 1d165a0d13..d90a467380 100644 --- a/backend/.sqlx/query-aed5439aa6dad950e505f9f8f6914fa5ca21319c501b2822c9bc751ddfc9a0a4.json +++ b/backend/.sqlx/query-4b5a711986017654bdd495893a16ddc6ab09c98cd8723865cbd341404bc6a02f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n script_path,\n is_flow,\n mode,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), $12, $13, $14\n )\n ", + "query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n auto_offset_reset,\n script_path,\n is_flow,\n mode,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $15\n )\n ", "describe": { "columns": [], "parameters": { @@ -12,6 +12,7 @@ "VarcharArray", "JsonbArray", "Varchar", + "Varchar", "Bool", { "Custom": { @@ -34,5 +35,5 @@ }, "nullable": [] }, - "hash": "aed5439aa6dad950e505f9f8f6914fa5ca21319c501b2822c9bc751ddfc9a0a4" + "hash": "4b5a711986017654bdd495893a16ddc6ab09c98cd8723865cbd341404bc6a02f" } diff --git a/backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json b/backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json deleted file mode 100644 index 6b47103c3a..0000000000 --- a/backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5" -} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json b/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json new file mode 100644 index 0000000000..2eda021880 --- /dev/null +++ b/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM volume WHERE workspace_id = $1 AND name = $2\n AND (lease_until IS NULL OR lease_until < now())\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa" +} diff --git a/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json b/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json new file mode 100644 index 0000000000..dd4a011ee1 --- /dev/null +++ b/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by, extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "extra_perms", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147" +} diff --git a/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json b/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json new file mode 100644 index 0000000000..746c306c8f --- /dev/null +++ b/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM volume WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6" +} diff --git a/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json b/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json new file mode 100644 index 0000000000..3df9c6c195 --- /dev/null +++ b/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int4", + "Text" + ] + }, + "nullable": [] + }, + "hash": "75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca" +} diff --git a/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json b/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json new file mode 100644 index 0000000000..0bf454028b --- /dev/null +++ b/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = now() + interval '60 seconds'\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f" +} diff --git a/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json b/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json new file mode 100644 index 0000000000..7b85bd9315 --- /dev/null +++ b/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM volume WHERE workspace_id = $1 AND name = $2 AND lease_until > now() AND leased_by = $3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5" +} diff --git a/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json b/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json new file mode 100644 index 0000000000..8cbe7146b6 --- /dev/null +++ b/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33" +} diff --git a/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json b/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json new file mode 100644 index 0000000000..594fcf2960 --- /dev/null +++ b/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b" +} diff --git a/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json b/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json new file mode 100644 index 0000000000..c60239fbce --- /dev/null +++ b/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4" +} diff --git a/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json b/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json new file mode 100644 index 0000000000..b364fcd4ac --- /dev/null +++ b/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET extra_perms = extra_perms - $1\n WHERE workspace_id = $2 AND name = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d" +} diff --git a/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json b/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json new file mode 100644 index 0000000000..728141923a --- /dev/null +++ b/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2 AND lease_until > now()", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "leased_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a" +} diff --git a/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json b/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json new file mode 100644 index 0000000000..26c254eb0b --- /dev/null +++ b/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a" +} diff --git a/backend/.sqlx/query-72d3ebb05ac1ffeb0e8d0a3146d95bb5b90e7c4d1dc2c8a6ef06eddf6678f230.json b/backend/.sqlx/query-9229d9a9ff389cf26e480b604b83900e2d362ee934ef27284ef39f4eed440e59.json similarity index 62% rename from backend/.sqlx/query-72d3ebb05ac1ffeb0e8d0a3146d95bb5b90e7c4d1dc2c8a6ef06eddf6678f230.json rename to backend/.sqlx/query-9229d9a9ff389cf26e480b604b83900e2d362ee934ef27284ef39f4eed440e59.json index 0a78509ca3..9eb47caf8a 100644 --- a/backend/.sqlx/query-72d3ebb05ac1ffeb0e8d0a3146d95bb5b90e7c4d1dc2c8a6ef06eddf6678f230.json +++ b/backend/.sqlx/query-9229d9a9ff389cf26e480b604b83900e2d362ee934ef27284ef39f4eed440e59.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2", + "query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2", "describe": { "columns": [ { @@ -73,5 +73,5 @@ false ] }, - "hash": "72d3ebb05ac1ffeb0e8d0a3146d95bb5b90e7c4d1dc2c8a6ef06eddf6678f230" + "hash": "9229d9a9ff389cf26e480b604b83900e2d362ee934ef27284ef39f4eed440e59" } diff --git a/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json b/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json new file mode 100644 index 0000000000..ee64e97d12 --- /dev/null +++ b/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7" +} diff --git a/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json b/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json new file mode 100644 index 0000000000..fc33fd7373 --- /dev/null +++ b/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89" +} diff --git a/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json b/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json new file mode 100644 index 0000000000..86a99d2eaf --- /dev/null +++ b/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de" +} diff --git a/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json b/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json new file mode 100644 index 0000000000..9051a88a50 --- /dev/null +++ b/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT extra_perms, created_by FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7" +} diff --git a/backend/.sqlx/query-a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4.json b/backend/.sqlx/query-a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4.json new file mode 100644 index 0000000000..aedbbf424e --- /dev/null +++ b/backend/.sqlx/query-a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4" +} diff --git a/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json b/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json new file mode 100644 index 0000000000..1f28a0a5a7 --- /dev/null +++ b/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, 0, $3)\n ON CONFLICT (workspace_id, name) DO NOTHING\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6" +} diff --git a/backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json b/backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json new file mode 100644 index 0000000000..af35d619fa --- /dev/null +++ b/backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token_expiry_notification (token, expiration) VALUES ($1, $2) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437" +} diff --git a/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json b/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json new file mode 100644 index 0000000000..7a45e6c402 --- /dev/null +++ b/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5" +} diff --git a/backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json b/backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json new file mode 100644 index 0000000000..00604f4bc9 --- /dev/null +++ b/backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token_expiry_notification WHERE expiration <= now()", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4" +} diff --git a/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json b/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json new file mode 100644 index 0000000000..4fa871c594 --- /dev/null +++ b/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e" +} diff --git a/backend/.sqlx/query-a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327.json b/backend/.sqlx/query-a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327.json new file mode 100644 index 0000000000..dc77dbc402 --- /dev/null +++ b/backend/.sqlx/query-a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Float4" + ] + }, + "nullable": [] + }, + "hash": "a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327" +} diff --git a/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json b/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json new file mode 100644 index 0000000000..3c7c1ad52a --- /dev/null +++ b/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935" +} diff --git a/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json b/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json new file mode 100644 index 0000000000..8d09036772 --- /dev/null +++ b/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1 RETURNING suspend", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "suspend", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a" +} diff --git a/backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json b/backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json new file mode 100644 index 0000000000..9085383617 --- /dev/null +++ b/backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token WHERE expiration <= now()\n RETURNING substring(token for 10) as token_prefix, label, email, workspace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token_prefix", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "label", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + true, + true, + true + ] + }, + "hash": "bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f" +} diff --git a/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json b/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json new file mode 100644 index 0000000000..442d55ed25 --- /dev/null +++ b/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes, last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb" +} diff --git a/backend/.sqlx/query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json b/backend/.sqlx/query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json new file mode 100644 index 0000000000..697e49ab9d --- /dev/null +++ b/backend/.sqlx/query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker)\n SELECT q.workspace_id, q.id, q.started_at,\n COALESCE((EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000, 0)::bigint,\n $2::jsonb, r.memory_peak, 'failure'::job_status, q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_runtime r ON r.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb" +} diff --git a/backend/.sqlx/query-c23bea7db9623a60683596b7d6e689e2c0100c1569436a01b207876aaa470154.json b/backend/.sqlx/query-c23bea7db9623a60683596b7d6e689e2c0100c1569436a01b207876aaa470154.json deleted file mode 100644 index be352ce88e..0000000000 --- a/backend/.sqlx/query-c23bea7db9623a60683596b7d6e689e2c0100c1569436a01b207876aaa470154.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n CASE\n WHEN flow_version.id IS NOT NULL THEN\n (flow_version.value -> 'flow_env' -> $3) #> $4\n ELSE\n (root_job.raw_flow -> 'flow_env' -> $3) #> $4\n END AS \"flow_env: sqlx::types::Json>\"\n FROM\n v2_job current_job\n JOIN\n v2_job root_job ON root_job.id = COALESCE(current_job.root_job, current_job.flow_innermost_root_job, current_job.parent_job, current_job.id)\n AND root_job.workspace_id = current_job.workspace_id\n LEFT JOIN\n flow_version ON flow_version.id = root_job.runnable_id\n AND flow_version.path = root_job.runnable_path\n AND flow_version.workspace_id = root_job.workspace_id\n WHERE\n current_job.id = $1 AND\n current_job.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "flow_env: sqlx::types::Json>", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Text", - "TextArray" - ] - }, - "nullable": [ - null - ] - }, - "hash": "c23bea7db9623a60683596b7d6e689e2c0100c1569436a01b207876aaa470154" -} diff --git a/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json b/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json new file mode 100644 index 0000000000..efd03ae26e --- /dev/null +++ b/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n workflow_as_code_status,\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3 AND workflow_as_code_status IS NOT NULL\n RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS \"job_ids: serde_json::Value\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_ids: serde_json::Value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340" +} diff --git a/backend/.sqlx/query-c944e384c4b4c6455b431978adc54d176f294b661675392cf92561c0e6e02e6e.json b/backend/.sqlx/query-c944e384c4b4c6455b431978adc54d176f294b661675392cf92561c0e6e02e6e.json new file mode 100644 index 0000000000..45afd37651 --- /dev/null +++ b/backend/.sqlx/query-c944e384c4b4c6455b431978adc54d176f294b661675392cf92561c0e6e02e6e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "c944e384c4b4c6455b431978adc54d176f294b661675392cf92561c0e6e02e6e" +} diff --git a/backend/.sqlx/query-c9530931f670eab1208c4a284a55afdc3fcbb0eb5f98fd63e2ec89442becbfaa.json b/backend/.sqlx/query-c9530931f670eab1208c4a284a55afdc3fcbb0eb5f98fd63e2ec89442becbfaa.json deleted file mode 100644 index 488d3c42bd..0000000000 --- a/backend/.sqlx/query-c9530931f670eab1208c4a284a55afdc3fcbb0eb5f98fd63e2ec89442becbfaa.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH _ AS (\n UPDATE debounce_key\n SET debounced_times = 0,\n first_started_at = now(),\n previous_job_id = NULL\n WHERE job_id = $1\n )\n UPDATE v2_job_debounce_batch\n SET debounce_batch = nextval('debounce_batch_seq')\n WHERE id = $1\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "c9530931f670eab1208c4a284a55afdc3fcbb0eb5f98fd63e2ec89442becbfaa" -} diff --git a/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json b/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json new file mode 100644 index 0000000000..0400e6992d --- /dev/null +++ b/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, name, size_bytes, created_by, last_used_at\n FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true + ] + }, + "hash": "d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c" +} diff --git a/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json b/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json new file mode 100644 index 0000000000..16fc965f6a --- /dev/null +++ b/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "leased_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04" +} diff --git a/backend/.sqlx/query-d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc.json b/backend/.sqlx/query-d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc.json deleted file mode 100644 index c071272ff9..0000000000 --- a/backend/.sqlx/query-d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text", - "Float4" - ] - }, - "nullable": [] - }, - "hash": "d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc" -} diff --git a/backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json b/backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json new file mode 100644 index 0000000000..015aa7b05a --- /dev/null +++ b/backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token_expiry_notification n\n USING token t\n WHERE n.token = t.token\n AND n.expiration > now()\n AND n.expiration <= now() + interval '7 days'\n RETURNING substring(t.token for 10) as token_prefix, t.label, t.email, t.workspace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token_prefix", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "label", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + true, + true, + true + ] + }, + "hash": "d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6" +} diff --git a/backend/.sqlx/query-1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82.json b/backend/.sqlx/query-dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353.json similarity index 65% rename from backend/.sqlx/query-1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82.json rename to backend/.sqlx/query-dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353.json index 5b82c5288f..5791090fc1 100644 --- a/backend/.sqlx/query-1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82.json +++ b/backend/.sqlx/query-dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n WITH job_info AS (\n SELECT id, kind::text AS kind, parent_job\n FROM v2_job\n WHERE id = $1\n )\n SELECT\n q.id AS \"id!\",\n s.flow_status,\n q.suspend AS \"suspend!\",\n j.runnable_path AS script_path,\n j.permissioned_as_email AS email,\n (ji.kind IN ('flow', 'flowpreview')) AS \"is_flow_level!\"\n FROM job_info ji\n JOIN v2_job_queue q ON q.id = CASE\n WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id\n ELSE ji.parent_job\n END\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_status s ON s.id = q.id\n FOR UPDATE OF q\n ", + "query": "\n WITH job_info AS (\n SELECT id, kind::text AS kind, parent_job\n FROM v2_job\n WHERE id = $1\n )\n SELECT\n q.id AS \"id!\",\n s.flow_status,\n q.suspend AS \"suspend!\",\n j.runnable_path AS script_path,\n j.permissioned_as_email AS email,\n (ji.kind IN ('flow', 'flowpreview')) AS \"is_flow_level!\",\n (ji.kind NOT IN ('flow', 'flowpreview') AND q.id = ji.id) AS \"is_wac!\"\n FROM job_info ji\n JOIN v2_job_queue q ON q.id = CASE\n WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id\n ELSE COALESCE(ji.parent_job, ji.id)\n END\n JOIN v2_job j ON j.id = q.id\n LEFT JOIN v2_job_status s ON s.id = q.id\n FOR UPDATE OF q\n ", "describe": { "columns": [ { @@ -32,6 +32,11 @@ "ordinal": 5, "name": "is_flow_level!", "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "is_wac!", + "type_info": "Bool" } ], "parameters": { @@ -45,8 +50,9 @@ false, true, false, + null, null ] }, - "hash": "1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82" + "hash": "dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353" } diff --git a/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json b/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json new file mode 100644 index 0000000000..91d89df9cd --- /dev/null +++ b/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, size_bytes FROM volume WHERE workspace_id = $1 ORDER BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "size_bytes", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865" +} diff --git a/backend/.sqlx/query-e2921e44c70cf6c76c55177f2b56985e84c59ecb3e1a13fcf27d5f7ae5f8d84c.json b/backend/.sqlx/query-e2921e44c70cf6c76c55177f2b56985e84c59ecb3e1a13fcf27d5f7ae5f8d84c.json deleted file mode 100644 index ccf11a0ac1..0000000000 --- a/backend/.sqlx/query-e2921e44c70cf6c76c55177f2b56985e84c59ecb3e1a13fcf27d5f7ae5f8d84c.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE\n workspace_id = $10 AND path = $11\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "VarcharArray", - "JsonbArray", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "e2921e44c70cf6c76c55177f2b56985e84c59ecb3e1a13fcf27d5f7ae5f8d84c" -} diff --git a/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json b/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json new file mode 100644 index 0000000000..c4a1d4cd07 --- /dev/null +++ b/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2::bool), true)\n WHERE workspace_id = $3 AND name = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d" +} diff --git a/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json new file mode 100644 index 0000000000..c96961eac4 --- /dev/null +++ b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT token as \"token!\"\n FROM token\n WHERE token LIKE concat($1::text, '%')\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06" +} diff --git a/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json b/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json new file mode 100644 index 0000000000..649ee4c387 --- /dev/null +++ b/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT permissioned_as FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "permissioned_as", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0" +} diff --git a/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json b/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json new file mode 100644 index 0000000000..72acab6120 --- /dev/null +++ b/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20" +} diff --git a/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json b/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json new file mode 100644 index 0000000000..e1ca416938 --- /dev/null +++ b/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes, file_count, leased_by, lease_until\n FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "file_count", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "leased_by", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "lease_until", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + true + ] + }, + "hash": "f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06" +} diff --git a/backend/.sqlx/query-ad8487a797713b3a6c10fb399c9fb8dcd940bb92e998145e250f28ccfe1c7033.json b/backend/.sqlx/query-fbccafe6d34093a723b9d5a6ee8d618a80ceba6de2d39202e6293ef5207c31f6.json similarity index 67% rename from backend/.sqlx/query-ad8487a797713b3a6c10fb399c9fb8dcd940bb92e998145e250f28ccfe1c7033.json rename to backend/.sqlx/query-fbccafe6d34093a723b9d5a6ee8d618a80ceba6de2d39202e6293ef5207c31f6.json index a703945270..da56a367b6 100644 --- a/backend/.sqlx/query-ad8487a797713b3a6c10fb399c9fb8dcd940bb92e998145e250f28ccfe1c7033.json +++ b/backend/.sqlx/query-fbccafe6d34093a723b9d5a6ee8d618a80ceba6de2d39202e6293ef5207c31f6.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO audit\n (workspace_id, username, operation, action_kind, resource, parameters, email, span)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + "query": "INSERT INTO audit_partitioned\n (workspace_id, username, operation, action_kind, resource, parameters, email, span)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "describe": { "columns": [], "parameters": { @@ -29,5 +29,5 @@ }, "nullable": [] }, - "hash": "ad8487a797713b3a6c10fb399c9fb8dcd940bb92e998145e250f28ccfe1c7033" + "hash": "fbccafe6d34093a723b9d5a6ee8d618a80ceba6de2d39202e6293ef5207c31f6" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index a2a27a9413..7f10019e62 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -860,9 +860,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9" +checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf" dependencies = [ "aws-lc-sys", "zeroize", @@ -870,9 +870,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.37.1" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" +checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e" dependencies = [ "cc", "cmake", @@ -1334,9 +1334,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.14" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b53543b4b86ed43f051644f704a98c7291b3618b67adf057ee77a366fa52fcaa" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" dependencies = [ "xmlparser", ] @@ -1900,7 +1900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" dependencies = [ "once_cell", - "proc-macro-crate 3.4.0", + "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.117", @@ -2550,6 +2550,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cooked-waker" version = "5.0.0" @@ -6173,20 +6182,20 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", "wasip2", "wasip3", ] @@ -7094,7 +7103,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.2", + "socket2 0.6.3", "system-configuration", "tokio", "tower-service", @@ -7421,9 +7430,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "ipnetwork" @@ -7965,9 +7974,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.182" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libffi" @@ -8093,9 +8102,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.24" +version = "1.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4735e9cbde5aac84a5ce588f6b23a90b9b0b528f6c5a8db8a4aff300463a0839" +checksum = "d52f4c29e2a68ac30c9087e1b772dc9f44a2b66ed44edf2266cf2be9b03dafc1" dependencies = [ "cc", "libc", @@ -8683,7 +8692,7 @@ dependencies = [ "darling 0.20.11", "heck 0.5.0", "num-bigint", - "proc-macro-crate 3.4.0", + "proc-macro-crate", "proc-macro-error2", "proc-macro2", "quote", @@ -9252,7 +9261,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ - "proc-macro-crate 3.4.0", + "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.117", @@ -10331,16 +10340,6 @@ dependencies = [ "elliptic-curve", ] -[[package]] -name = "proc-macro-crate" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" -dependencies = [ - "once_cell", - "toml_edit 0.19.15", -] - [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -10665,7 +10664,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.35", - "socket2 0.6.2", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -10674,9 +10673,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "aws-lc-rs", "bytes", @@ -10703,16 +10702,16 @@ dependencies = [ "cfg_aliases 0.2.1", "libc", "once_cell", - "socket2 0.6.2", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -10723,6 +10722,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radium" version = "0.7.0" @@ -11088,9 +11093,12 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "relative-path" -version = "1.9.3" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0" +dependencies = [ + "serde", +] [[package]] name = "rend" @@ -11384,9 +11392,9 @@ dependencies = [ [[package]] name = "rquickjs" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16661bff09e9ed8e01094a188b463de45ec0693ade55b92ed54027d7ba7c40c" +checksum = "c50dc6d6c587c339edb4769cf705867497a2baf0eca8b4645fa6ecd22f02c77a" dependencies = [ "rquickjs-core", "rquickjs-macro", @@ -11394,26 +11402,27 @@ dependencies = [ [[package]] name = "rquickjs-core" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8db6379e204ef84c0811e90e7cc3e3e4d7688701db68a00d14a6db6849087b" +checksum = "b8bf7840285c321c3ab20e752a9afb95548c75cd7f4632a0627cea3507e310c1" dependencies = [ "async-lock", + "hashbrown 0.16.0", "relative-path", "rquickjs-sys", ] [[package]] name = "rquickjs-macro" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6041104330c019fcd936026ae05e2446f5e8a2abef329d924f25424b7052a2f3" +checksum = "7106215ff41a5677b104906a13e1a440b880f4b6362b5dc4f3978c267fad2b80" dependencies = [ - "convert_case 0.6.0", + "convert_case 0.10.0", "fnv", "ident_case", "indexmap 2.11.1", - "proc-macro-crate 1.3.1", + "proc-macro-crate", "proc-macro2", "quote", "rquickjs-core", @@ -11422,9 +11431,9 @@ dependencies = [ [[package]] name = "rquickjs-sys" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bc352c6b663604c3c186c000cfcc6c271f4b50bc135a285dd6d4f2a42f9790a" +checksum = "27344601ef27460e82d6a4e1ecb9e7e99f518122095f3c51296da8e9be2b9d83" dependencies = [ "cc", ] @@ -11978,9 +11987,9 @@ checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -12668,12 +12677,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -13850,7 +13859,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" dependencies = [ "fastrand", - "getrandom 0.4.1", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -14453,7 +14462,7 @@ dependencies = [ "indexmap 2.11.1", "toml_datetime 0.7.0", "toml_parser", - "winnow 0.7.14", + "winnow 0.7.15", ] [[package]] @@ -14462,7 +14471,7 @@ version = "1.0.9+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" dependencies = [ - "winnow 0.7.14", + "winnow 0.7.15", ] [[package]] @@ -15732,7 +15741,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-nats", @@ -15764,6 +15773,7 @@ dependencies = [ "sql-builder", "sqlx", "strum 0.27.2", + "tar", "tempfile", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", @@ -15789,14 +15799,16 @@ dependencies = [ "windmill-queue", "windmill-runtime-nativets", "windmill-test-utils", + "windmill-types", "windmill-worker", + "windmill-worker-volumes", "windows-service", "windows-sys 0.52.0", ] [[package]] name = "windmill-alerting" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15809,7 +15821,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "argon2", @@ -15837,6 +15849,7 @@ dependencies = [ "dashmap 6.1.0", "datafusion", "ed25519-dalek", + "eventsource-stream", "flate2", "futures", "git-version", @@ -15928,6 +15941,7 @@ dependencies = [ "windmill-parser-py", "windmill-parser-py-imports", "windmill-parser-sql", + "windmill-parser-sql-asset", "windmill-parser-ts", "windmill-queue", "windmill-store", @@ -15943,11 +15957,12 @@ dependencies = [ "windmill-trigger-websocket", "windmill-types", "windmill-worker", + "windmill-worker-volumes", ] [[package]] name = "windmill-api-agent-workers" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15970,7 +15985,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15983,7 +15998,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16009,7 +16024,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.648.0" +version = "1.654.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -16019,7 +16034,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16036,7 +16051,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "base64 0.22.1", @@ -16059,7 +16074,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16082,7 +16097,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16098,7 +16113,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16118,7 +16133,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16138,7 +16153,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16152,7 +16167,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-nats", @@ -16179,7 +16194,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16204,7 +16219,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "flate2", @@ -16222,7 +16237,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16243,7 +16258,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16263,7 +16278,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16293,7 +16308,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16320,7 +16335,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.648.0" +version = "1.654.0" dependencies = [ "lazy_static", "serde", @@ -16332,7 +16347,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.648.0" +version = "1.654.0" dependencies = [ "argon2", "axum 0.7.9", @@ -16355,7 +16370,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16369,7 +16384,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.648.0" +version = "1.654.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16377,6 +16392,7 @@ dependencies = [ "http 1.4.0", "hyper 1.8.1", "lazy_static", + "magic-crypt", "regex", "serde", "serde_json", @@ -16399,7 +16415,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.648.0" +version = "1.654.0" dependencies = [ "chrono", "lazy_static", @@ -16413,7 +16429,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16432,7 +16448,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.648.0" +version = "1.654.0" dependencies = [ "aes-gcm", "anyhow", @@ -16531,7 +16547,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.648.0" +version = "1.654.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -16550,7 +16566,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.648.0" +version = "1.654.0" dependencies = [ "regex", "serde", @@ -16565,7 +16581,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16589,7 +16605,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "futures", @@ -16606,7 +16622,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.648.0" +version = "1.654.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16622,7 +16638,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-trait", @@ -16643,7 +16659,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-trait", @@ -16674,7 +16690,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-oauth2", @@ -16698,7 +16714,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-stream", @@ -16732,7 +16748,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "futures", @@ -16750,7 +16766,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.648.0" +version = "1.654.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -16759,7 +16775,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "lazy_static", @@ -16771,7 +16787,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "serde_json", @@ -16783,7 +16799,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "gosyn", @@ -16795,7 +16811,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "lazy_static", @@ -16807,7 +16823,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "serde_json", @@ -16819,7 +16835,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "nu-parser", @@ -16830,7 +16846,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16841,7 +16857,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16849,12 +16865,22 @@ dependencies = [ "rustpython-parser", "serde_json", "windmill-parser", - "windmill-parser-sql", +] + +[[package]] +name = "windmill-parser-py-asset" +version = "1.653.0" +dependencies = [ + "anyhow", + "rustpython-ast", + "rustpython-parser", + "windmill-parser", + "windmill-parser-sql-asset", ] [[package]] name = "windmill-parser-py-imports" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-recursion", @@ -16878,7 +16904,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "lazy_static", @@ -16892,7 +16918,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16909,7 +16935,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "lazy_static", @@ -16917,6 +16943,17 @@ dependencies = [ "regex-lite", "serde", "serde_json", + "windmill-parser", + "windmill-types", +] + +[[package]] +name = "windmill-parser-sql-asset" +version = "1.653.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", "sqlparser 0.59.0", "windmill-parser", "windmill-types", @@ -16924,7 +16961,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "lazy_static", @@ -16938,12 +16975,43 @@ dependencies = [ "triomphe", "wasm-bindgen", "windmill-parser", - "windmill-parser-sql", +] + +[[package]] +name = "windmill-parser-ts-asset" +version = "1.653.0" +dependencies = [ + "anyhow", + "serde-wasm-bindgen", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", + "swc_ecma_visit", + "triomphe", + "wasm-bindgen", + "windmill-parser", + "windmill-parser-sql-asset", +] + +[[package]] +name = "windmill-parser-wac" +version = "1.654.0" +dependencies = [ + "anyhow", + "rustpython-ast", + "rustpython-parser", + "serde", + "serde_json", + "sha2 0.10.9", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", + "swc_ecma_visit", ] [[package]] name = "windmill-parser-yaml" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "serde", @@ -16954,7 +17022,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-recursion", @@ -16991,7 +17059,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "const_format", @@ -17029,7 +17097,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.648.0" +version = "1.654.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17040,7 +17108,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-recursion", @@ -17069,7 +17137,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -17092,7 +17160,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-trait", @@ -17125,7 +17193,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-trait", @@ -17145,7 +17213,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-trait", @@ -17179,7 +17247,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-trait", @@ -17214,7 +17282,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-trait", @@ -17237,7 +17305,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-trait", @@ -17261,7 +17329,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-nats", @@ -17285,7 +17353,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-trait", @@ -17320,7 +17388,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-trait", @@ -17348,7 +17416,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-trait", @@ -17371,7 +17439,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17389,7 +17457,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.648.0" +version = "1.654.0" dependencies = [ "anyhow", "async-once-cell", @@ -17489,9 +17557,28 @@ dependencies = [ "windmill-queue", "windmill-runtime-nativets", "windmill-types", + "windmill-worker-volumes", "yaml-rust", ] +[[package]] +name = "windmill-worker-volumes" +version = "1.654.0" +dependencies = [ + "bytes", + "futures", + "lazy_static", + "md-5 0.10.6", + "object_store", + "regex", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "windmill-common", +] + [[package]] name = "windows" version = "0.56.0" @@ -18076,9 +18163,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" dependencies = [ "memchr", ] @@ -18359,18 +18446,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" dependencies = [ "proc-macro2", "quote", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 2d964c45b4..1b0924c2fa 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.648.0" +version = "1.654.0" authors.workspace = true edition.workspace = true @@ -59,6 +59,7 @@ members = [ "./windmill-oauth", "./parsers/windmill-parser", "./parsers/windmill-parser-ts", + "./parsers/windmill-parser-ts-asset", "./parsers/windmill-parser-go", "./parsers/windmill-parser-rust", "./parsers/windmill-parser-csharp", @@ -67,16 +68,21 @@ members = [ "./parsers/windmill-parser-ruby", "./parsers/windmill-parser-bash", "./parsers/windmill-parser-py", + "./parsers/windmill-parser-py-asset", "./parsers/windmill-parser-py-imports", + "./parsers/windmill-parser-wac", + "./parsers/windmill-parser-sql", + "./parsers/windmill-parser-sql-asset", "./parsers/windmill-sql-datatype-parser-wasm", "./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu", + "./windmill-worker-volumes", "./windmill-test-utils", "./windmill-api-integration-tests", ] exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.648.0" +version = "1.654.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -250,10 +256,13 @@ reqwest.workspace = true windmill-queue = { workspace = true, features = ["failpoints"] } windmill-dep-map.workspace = true windmill-test-utils.workspace = true +windmill-worker-volumes.workspace = true +windmill-types.workspace = true axum.workspace = true serde.workspace = true windmill-api-client.workspace = true tempfile.workspace = true +tar.workspace = true windmill-parser-ts.workspace = true rumqttc.workspace = true rdkafka.workspace = true @@ -267,6 +276,7 @@ aws-credential-types.workspace = true windmill-api = { path = "./windmill-api", default-features = false } windmill-queue = { path = "./windmill-queue" } windmill-worker = { path = "./windmill-worker" } +windmill-worker-volumes = { path = "./windmill-worker-volumes" } windmill-dep-map = { path = "./windmill-dep-map" } windmill-types = { path = "./windmill-types" } windmill-common = { path = "./windmill-common", default-features = false } @@ -314,7 +324,9 @@ windmill-api-workers = { path = "./windmill-api-workers" } windmill-store = { path = "./windmill-store" } windmill-parser = { path = "./parsers/windmill-parser" } windmill-parser-ts = { path = "./parsers/windmill-parser-ts" } +windmill-parser-ts-asset = { path = "./parsers/windmill-parser-ts-asset" } windmill-parser-py = { path = "./parsers/windmill-parser-py" } +windmill-parser-py-asset = { path = "./parsers/windmill-parser-py-asset" } windmill-parser-py-imports = { path = "./parsers/windmill-parser-py-imports" } windmill-parser-go = { path = "./parsers/windmill-parser-go" } windmill-parser-rust = { path = "./parsers/windmill-parser-rust" } @@ -325,8 +337,10 @@ windmill-parser-ruby = { path = "./parsers/windmill-parser-ruby" } windmill-parser-nu = { path = "./parsers/windmill-parser-nu" } windmill-parser-bash = { path = "./parsers/windmill-parser-bash" } windmill-parser-sql = { path = "./parsers/windmill-parser-sql" } +windmill-parser-sql-asset = { path = "./parsers/windmill-parser-sql-asset" } windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" } windmill-parser-php = { path = "./parsers/windmill-parser-php" } +windmill-parser-wac = { path = "./parsers/windmill-parser-wac" } windmill-jseval = { path = "./windmill-jseval" } windmill-runtime-nativets = { path = "./windmill-runtime-nativets" } windmill-api-client = { path = "./windmill-api-client" } @@ -439,6 +453,7 @@ base64 = "^0.22.1" base32 = "^0" hmac = "0.12.1" sha2 = "0.10.6" +md-5 = "0.10.6" sha1 = "0.10.6" sqlx = { version = "0.8.0", features = [ "macros", @@ -512,7 +527,7 @@ nu-parser = { version = "0.101.0", default-features = false } globset = "0.4.16" croner = "2.2.0" rmcp = { version = "=0.15.0", features = ["client", "transport-streamable-http-client", "transport-streamable-http-client-reqwest"] } -rquickjs = { version = "0.8", features = ["futures", "parallel", "macro"] } +rquickjs = { version = "0.11", features = ["futures", "parallel", "macro"] } process-wrap = { version = "8.2.1", features = ["tokio1"] } systemstat = "0.2.4" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index e0d990384a..06277182aa 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -9b3339730eb4bb0b564c7c56ac546f33fb3d8905 \ No newline at end of file +2f52c015bc6c81391234fa87b27ee1d4cd3a48a3 \ No newline at end of file diff --git a/backend/migrations/20260226000000_add_volumes.down.sql b/backend/migrations/20260226000000_add_volumes.down.sql new file mode 100644 index 0000000000..33dc3804be --- /dev/null +++ b/backend/migrations/20260226000000_add_volumes.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS volume; diff --git a/backend/migrations/20260226000000_add_volumes.up.sql b/backend/migrations/20260226000000_add_volumes.up.sql new file mode 100644 index 0000000000..00f40c9768 --- /dev/null +++ b/backend/migrations/20260226000000_add_volumes.up.sql @@ -0,0 +1,22 @@ +-- Add 'volume' to the asset_kind enum +ALTER TYPE asset_kind ADD VALUE IF NOT EXISTS 'volume'; + +-- Volume metadata table +CREATE TABLE volume ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + size_bytes BIGINT NOT NULL DEFAULT 0, + file_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by VARCHAR(255) NOT NULL, + updated_at TIMESTAMPTZ, + updated_by VARCHAR(255), + description TEXT NOT NULL DEFAULT '', + lease_until TIMESTAMPTZ, + leased_by VARCHAR(255), + last_used_at TIMESTAMPTZ, + extra_perms JSONB NOT NULL DEFAULT '{}', + PRIMARY KEY (workspace_id, name) +); + +CREATE INDEX idx_volume_last_used ON volume(workspace_id, last_used_at); diff --git a/backend/migrations/20260302000000_add_token_expiry_notified.down.sql b/backend/migrations/20260302000000_add_token_expiry_notified.down.sql new file mode 100644 index 0000000000..ab827c5de5 --- /dev/null +++ b/backend/migrations/20260302000000_add_token_expiry_notified.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS token_expiry_notification; diff --git a/backend/migrations/20260302000000_add_token_expiry_notified.up.sql b/backend/migrations/20260302000000_add_token_expiry_notified.up.sql new file mode 100644 index 0000000000..61883f070d --- /dev/null +++ b/backend/migrations/20260302000000_add_token_expiry_notified.up.sql @@ -0,0 +1,8 @@ +-- Tracks pending expiry notifications: row exists = not yet notified. +-- Deleted once the notification is sent. Orphaned rows are harmless (filtered out by the join). +CREATE TABLE token_expiry_notification ( + token VARCHAR(255) PRIMARY KEY, + expiration TIMESTAMPTZ NOT NULL +); + +CREATE INDEX idx_token_expiry_notification_expiration ON token_expiry_notification (expiration); diff --git a/backend/migrations/20260309000000_kafka_offset_reset.down.sql b/backend/migrations/20260309000000_kafka_offset_reset.down.sql new file mode 100644 index 0000000000..d6df00484e --- /dev/null +++ b/backend/migrations/20260309000000_kafka_offset_reset.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE kafka_trigger DROP COLUMN auto_offset_reset; +ALTER TABLE kafka_trigger DROP COLUMN reset_offset; diff --git a/backend/migrations/20260309000000_kafka_offset_reset.up.sql b/backend/migrations/20260309000000_kafka_offset_reset.up.sql new file mode 100644 index 0000000000..7bcb23226d --- /dev/null +++ b/backend/migrations/20260309000000_kafka_offset_reset.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE kafka_trigger ADD COLUMN auto_offset_reset VARCHAR(10) NOT NULL DEFAULT 'latest'; +ALTER TABLE kafka_trigger ADD COLUMN reset_offset BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/backend/migrations/20260311000000_optimize_job_stats_timestamps.down.sql b/backend/migrations/20260311000000_optimize_job_stats_timestamps.down.sql new file mode 100644 index 0000000000..e526b89b11 --- /dev/null +++ b/backend/migrations/20260311000000_optimize_job_stats_timestamps.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE job_stats DROP COLUMN IF EXISTS timeseries_start; +ALTER TABLE job_stats DROP COLUMN IF EXISTS offsets_cs; diff --git a/backend/migrations/20260311000000_optimize_job_stats_timestamps.up.sql b/backend/migrations/20260311000000_optimize_job_stats_timestamps.up.sql new file mode 100644 index 0000000000..f851c7887e --- /dev/null +++ b/backend/migrations/20260311000000_optimize_job_stats_timestamps.up.sql @@ -0,0 +1,5 @@ +-- Store timeseries timestamps as a start time + integer centisecond offsets +-- instead of full TIMESTAMPTZ[] arrays. Saves ~4 bytes per data point. +-- i32 centiseconds gives ~248 days of range with 10ms precision. +ALTER TABLE job_stats ADD COLUMN IF NOT EXISTS timeseries_start TIMESTAMPTZ; +ALTER TABLE job_stats ADD COLUMN IF NOT EXISTS offsets_cs INTEGER[]; diff --git a/backend/migrations/20260311100000_audit_partitioning.down.sql b/backend/migrations/20260311100000_audit_partitioning.down.sql new file mode 100644 index 0000000000..7eb63bbdba --- /dev/null +++ b/backend/migrations/20260311100000_audit_partitioning.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS audit_partitioned CASCADE; diff --git a/backend/migrations/20260311100000_audit_partitioning.up.sql b/backend/migrations/20260311100000_audit_partitioning.up.sql new file mode 100644 index 0000000000..c221af8d38 --- /dev/null +++ b/backend/migrations/20260311100000_audit_partitioning.up.sql @@ -0,0 +1,58 @@ +-- Create a new daily-partitioned audit table alongside the existing one. +-- New inserts go to audit_partitioned; reads UNION ALL both tables. +-- The old audit table empties out naturally via retention cleanup. + +CREATE TABLE audit_partitioned ( + workspace_id VARCHAR(50) NOT NULL, + id BIGINT NOT NULL DEFAULT nextval('audit_id_seq'), + timestamp TIMESTAMPTZ NOT NULL DEFAULT now(), + username VARCHAR(255) NOT NULL, + operation VARCHAR(50) NOT NULL, + action_kind ACTION_KIND NOT NULL, + resource VARCHAR(255), + parameters JSONB, + email VARCHAR(255), + span VARCHAR(255), + PRIMARY KEY (id, timestamp) +) PARTITION BY RANGE (timestamp); + +-- Create daily partitions for today + 3 days +DO $$ +DECLARE + curr_date DATE := CURRENT_DATE; + end_date DATE := CURRENT_DATE + INTERVAL '3 days'; +BEGIN + WHILE curr_date <= end_date LOOP + EXECUTE format( + 'CREATE TABLE %I PARTITION OF audit_partitioned FOR VALUES FROM (%L) TO (%L)', + 'audit_' || to_char(curr_date, 'YYYYMMDD'), + curr_date, + curr_date + INTERVAL '1 day' + ); + curr_date := curr_date + INTERVAL '1 day'; + END LOOP; +END $$; + +-- Indexes (auto-propagated to all current and future partitions) +CREATE INDEX ix_audit_partitioned_timestamps ON audit_partitioned (timestamp DESC); +CREATE INDEX idx_audit_partitioned_workspace ON audit_partitioned (workspace_id, timestamp DESC); +CREATE INDEX idx_audit_partitioned_recent_login_activities + ON audit_partitioned (timestamp, username) + WHERE operation IN ('users.login', 'oauth.login', 'users.token.refresh'); + +-- Grants (match the old audit table) +GRANT ALL ON audit_partitioned TO windmill_user; +GRANT ALL ON audit_partitioned TO windmill_admin; + +-- RLS (match the old audit table) +ALTER TABLE audit_partitioned ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON audit_partitioned FOR ALL TO windmill_admin USING (true); +CREATE POLICY see_own ON audit_partitioned FOR ALL TO windmill_user + USING ((username)::text = current_setting('session.user'::text)); +CREATE POLICY schedule ON audit_partitioned FOR INSERT TO windmill_user + WITH CHECK ((username)::text ~~ 'schedule-%'::text); +CREATE POLICY schedule_audit ON audit_partitioned FOR INSERT TO windmill_user + WITH CHECK ((parameters ->> 'end_user'::text) ~~ 'schedule-%'::text); +CREATE POLICY webhook ON audit_partitioned FOR INSERT TO windmill_user + WITH CHECK ((username)::text ~~ 'webhook-%'::text); diff --git a/backend/parsers/windmill-parser-py-asset/Cargo.toml b/backend/parsers/windmill-parser-py-asset/Cargo.toml new file mode 100644 index 0000000000..ddd6446079 --- /dev/null +++ b/backend/parsers/windmill-parser-py-asset/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "windmill-parser-py-asset" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_py_asset" +path = "./src/lib.rs" + +[dependencies] +windmill-parser.workspace = true +windmill-parser-sql-asset.workspace = true +rustpython-parser.workspace = true +rustpython-ast = { version = "0.4.0", features = ["visitor"] } +anyhow.workspace = true diff --git a/backend/parsers/windmill-parser-py/src/asset_parser.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs similarity index 99% rename from backend/parsers/windmill-parser-py/src/asset_parser.rs rename to backend/parsers/windmill-parser-py-asset/src/lib.rs index 5ebca6d250..10b93a998b 100644 --- a/backend/parsers/windmill-parser-py/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -215,7 +215,7 @@ impl AssetsFinder { _ => return Err(()), }; // We use the SQL parser to detect RW, specific tables, etc. - let sql_assets = windmill_parser_sql::parse_wmill_sdk_sql_assets( + let sql_assets = windmill_parser_sql_asset::parse_wmill_sdk_sql_assets( *kind, path, schema.as_deref(), diff --git a/backend/parsers/windmill-parser-py/Cargo.toml b/backend/parsers/windmill-parser-py/Cargo.toml index 2a4f95e49a..0c50e0f8d8 100644 --- a/backend/parsers/windmill-parser-py/Cargo.toml +++ b/backend/parsers/windmill-parser-py/Cargo.toml @@ -10,7 +10,6 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true -windmill-parser-sql.workspace = true rustpython-parser.workspace = true itertools.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index c299bff4af..c6852bd351 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -21,11 +21,8 @@ use rustpython_parser::{ Parse, }; -pub mod asset_parser; pub mod pydantic_parser; -pub use asset_parser::parse_assets; - const FUNCTION_CALL: &str = ""; /// Cheap string-based check to see if code might contain Pydantic models or dataclasses. @@ -296,11 +293,14 @@ pub fn parse_python_signature( // Check if main function was found if params.is_none() { + let is_wac_v2 = (code.contains("@workflow") || code.contains("workflow(")) + && (code.contains("@task") || code.contains("task(")) + && (code.contains("import wmill") || code.contains("from wmill")); return Ok(MainArgSignature { star_args: false, star_kwargs: false, args: vec![], - no_main_func: Some(true), + no_main_func: Some(!is_wac_v2), has_preprocessor: Some(has_preprocessor), }); } diff --git a/backend/parsers/windmill-parser-sql-asset/Cargo.toml b/backend/parsers/windmill-parser-sql-asset/Cargo.toml new file mode 100644 index 0000000000..73835b2c06 --- /dev/null +++ b/backend/parsers/windmill-parser-sql-asset/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "windmill-parser-sql-asset" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_sql_asset" +path = "./src/lib.rs" + +[dependencies] +windmill-parser.workspace = true +windmill-types.workspace = true +anyhow.workspace = true +serde_json.workspace = true +serde.workspace = true +sqlparser = { version = "0.59.0", features = ["visitor"] } diff --git a/backend/parsers/windmill-parser-sql/src/asset_parser.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs similarity index 100% rename from backend/parsers/windmill-parser-sql/src/asset_parser.rs rename to backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs diff --git a/backend/parsers/windmill-parser-sql/src/asset_parser_utils.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs similarity index 100% rename from backend/parsers/windmill-parser-sql/src/asset_parser_utils.rs rename to backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs diff --git a/backend/parsers/windmill-parser-sql-asset/src/lib.rs b/backend/parsers/windmill-parser-sql-asset/src/lib.rs new file mode 100644 index 0000000000..65d8242a80 --- /dev/null +++ b/backend/parsers/windmill-parser-sql-asset/src/lib.rs @@ -0,0 +1,4 @@ +mod asset_parser; +mod asset_parser_utils; +pub use asset_parser::parse_assets; +pub use asset_parser_utils::parse_wmill_sdk_sql_assets; diff --git a/backend/parsers/windmill-parser-sql/Cargo.toml b/backend/parsers/windmill-parser-sql/Cargo.toml index be92d1133b..e84888280c 100644 --- a/backend/parsers/windmill-parser-sql/Cargo.toml +++ b/backend/parsers/windmill-parser-sql/Cargo.toml @@ -20,5 +20,4 @@ windmill-types.workspace = true anyhow.workspace = true lazy_static.workspace = true serde_json.workspace = true -serde.workspace = true -sqlparser = { version = "0.59.0", features = ["visitor"] } \ No newline at end of file +serde.workspace = true \ No newline at end of file diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index cc86ea595c..13fd1a3199 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -20,11 +20,6 @@ pub use windmill_parser::{Arg, MainArgSignature, ObjectType, Typ}; pub const SANITIZED_ENUM_STR: &str = "__sanitized_enum__"; pub const SANITIZED_RAW_STRING_STR: &str = "__sanitized_raw_string__"; -mod asset_parser; -mod asset_parser_utils; -pub use asset_parser::parse_assets; -pub use asset_parser_utils::parse_wmill_sdk_sql_assets; - pub fn parse_mysql_sig(code: &str) -> anyhow::Result { let parsed = parse_mysql_file(&code)?; if let Some(x) = parsed { @@ -238,7 +233,7 @@ lazy_static::lazy_static! { // used for `unsafe` sql interpolation // -- %%name%% (type) = default - static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%\s*([\s\w\/]+)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); + static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%[ \t]*([\w][\w \t\/]*)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); } fn parsed_default(parsed_typ: &Typ, default: String) -> Option { @@ -1547,4 +1542,36 @@ SELECT $1::integer; Ok(()) } + + #[test] + fn test_parse_pgsql_safe_interpolated_args() -> anyhow::Result<()> { + // There was a bug where enum would be "angrycreative"/"bishop"/"test SELECT x" + let code = r#" +-- %%table_name%% angrycreative/bishop/test +SELECT x +"#; + assert_eq!( + parse_pgsql_sig(code)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![Arg { + otyp: Some("__sanitized_enum__".to_string()), + name: "table_name".to_string(), + typ: Typ::Str(Some(vec![ + "angrycreative".to_string(), + "bishop".to_string(), + "test".to_string() + ])), + default: None, + has_default: false, + oidx: None, + },], + no_main_func: None, + has_preprocessor: None + } + ); + + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-ts-asset/Cargo.toml b/backend/parsers/windmill-parser-ts-asset/Cargo.toml new file mode 100644 index 0000000000..f1c668d4eb --- /dev/null +++ b/backend/parsers/windmill-parser-ts-asset/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "windmill-parser-ts-asset" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_ts_asset" +path = "./src/lib.rs" + +[target.'cfg(target_arch = "wasm32")'.dependencies] +wasm-bindgen.workspace = true +serde-wasm-bindgen.workspace = true + +[dependencies] +windmill-parser.workspace = true +windmill-parser-sql-asset.workspace = true +swc_common.workspace = true +triomphe.workspace = true +swc_ecma_parser.workspace = true +swc_ecma_ast.workspace = true +swc_ecma_visit.workspace = true +anyhow.workspace = true diff --git a/backend/parsers/windmill-parser-ts/src/asset_parser.rs b/backend/parsers/windmill-parser-ts-asset/src/lib.rs similarity index 99% rename from backend/parsers/windmill-parser-ts/src/asset_parser.rs rename to backend/parsers/windmill-parser-ts-asset/src/lib.rs index d4a655858e..aea77de814 100644 --- a/backend/parsers/windmill-parser-ts/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-ts-asset/src/lib.rs @@ -259,7 +259,7 @@ impl Visit for AssetsFinder { }); // We use the SQL parser to detect RW, specific tables, etc. - let sql_assets = windmill_parser_sql::parse_wmill_sdk_sql_assets( + let sql_assets = windmill_parser_sql_asset::parse_wmill_sdk_sql_assets( *kind, asset_name, schema.as_deref(), diff --git a/backend/parsers/windmill-parser-ts/Cargo.toml b/backend/parsers/windmill-parser-ts/Cargo.toml index b8d577b79f..5f04b14f3b 100644 --- a/backend/parsers/windmill-parser-ts/Cargo.toml +++ b/backend/parsers/windmill-parser-ts/Cargo.toml @@ -15,7 +15,6 @@ serde-wasm-bindgen.workspace = true [dependencies] windmill-parser.workspace = true -windmill-parser-sql.workspace = true swc_common.workspace = true triomphe.workspace = true swc_ecma_parser.workspace = true diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 04dd345b2f..54e9d465fb 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -211,8 +211,6 @@ pub enum TypeDecl { Interface(TsInterfaceDecl), Alias(TsTypeAliasDecl), } -pub mod asset_parser; -pub use asset_parser::parse_assets; /// skip_params is a micro optimization for when we just want to find the main /// function without parsing all the params. @@ -261,7 +259,9 @@ pub fn parse_deno_signature( for specifier in &named_export.specifiers { if let swc_ecma_ast::ExportSpecifier::Named(spec) = specifier { let export_name = match &spec.exported { - Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => ident.sym.as_ref(), + Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => { + ident.sym.as_ref() + } Some(swc_ecma_ast::ModuleExportName::Str(s)) => s.value.as_ref(), None => match &spec.orig { swc_ecma_ast::ModuleExportName::Ident(ident) => ident.sym.as_ref(), @@ -315,7 +315,11 @@ pub fn parse_deno_signature( let mut c: u16 = 0; - let no_main_func = entrypoint_params.is_none(); + let is_wac_v2 = entrypoint_params.is_none() + && code.contains("workflow(") + && code.contains("task(") + && code.contains("windmill-client"); + let no_main_func = entrypoint_params.is_none() && !is_wac_v2; let mut type_resolver = HashMap::new(); let r = MainArgSignature { star_args: false, @@ -833,7 +837,9 @@ fn tstype_to_typ( false, ), symbol @ _ if symbol.starts_with("DynMultiselect_") => ( - Typ::DynMultiselect(symbol.strip_prefix("DynMultiselect_").unwrap().to_string()), + Typ::DynMultiselect( + symbol.strip_prefix("DynMultiselect_").unwrap().to_string(), + ), false, ), symbol @ _ => { diff --git a/backend/parsers/windmill-parser-wac/Cargo.toml b/backend/parsers/windmill-parser-wac/Cargo.toml new file mode 100644 index 0000000000..d354b1c653 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "windmill-parser-wac" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_wac" +path = "./src/lib.rs" + +[dependencies] +rustpython-parser.workspace = true +rustpython-ast = { version = "0.4.0", features = ["visitor"] } +swc_common.workspace = true +swc_ecma_parser.workspace = true +swc_ecma_ast.workspace = true +swc_ecma_visit.workspace = true +serde.workspace = true +serde_json.workspace = true +anyhow.workspace = true +sha2.workspace = true diff --git a/backend/parsers/windmill-parser-wac/src/dag.rs b/backend/parsers/windmill-parser-wac/src/dag.rs new file mode 100644 index 0000000000..662f5a06b6 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/dag.rs @@ -0,0 +1,44 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct WorkflowDag { + pub nodes: Vec, + pub edges: Vec, + pub params: Vec, + pub source_hash: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct Param { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub typ: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct DagNode { + pub id: String, + pub node_type: DagNodeType, + pub label: String, + pub line: usize, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(tag = "type")] +pub enum DagNodeType { + Step { name: String, script: String }, + Branch { condition_source: String }, + ParallelStart, + ParallelEnd, + LoopStart { iter_source: String }, + LoopEnd, + Return, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct DagEdge { + pub from: String, + pub to: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, +} diff --git a/backend/parsers/windmill-parser-wac/src/lib.rs b/backend/parsers/windmill-parser-wac/src/lib.rs new file mode 100644 index 0000000000..1496b9d6cf --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/lib.rs @@ -0,0 +1,32 @@ +pub mod dag; +pub mod python; +pub mod typescript; +pub mod validation; + +use dag::WorkflowDag; +use validation::CompileError; + +#[derive(Debug, serde::Serialize)] +#[serde(tag = "type")] +pub enum ParseResult { + #[serde(rename = "success")] + Success(WorkflowDag), + #[serde(rename = "error")] + Error { errors: Vec }, +} + +pub fn parse_workflow(code: &str, language: &str) -> ParseResult { + let result = match language { + "python" | "python3" | "py" => python::parse_python_workflow(code), + "typescript" | "ts" | "deno" | "bun" => typescript::parse_ts_workflow(code), + _ => Err(vec![CompileError { + message: format!("Unsupported language: {language}"), + line: 0, + }]), + }; + + match result { + Ok(dag) => ParseResult::Success(dag), + Err(errors) => ParseResult::Error { errors }, + } +} diff --git a/backend/parsers/windmill-parser-wac/src/python.rs b/backend/parsers/windmill-parser-wac/src/python.rs new file mode 100644 index 0000000000..74f0117376 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/python.rs @@ -0,0 +1,717 @@ +use std::collections::HashMap; + +use rustpython_parser::{ + ast::{ + Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry, + StmtTryStar, StmtWhile, + }, + Parse, +}; + +use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag}; +use crate::validation::{self, CompileError}; + +struct LineIndex { + newline_offsets: Vec, +} + +impl LineIndex { + fn new(source: &str) -> Self { + let mut offsets = vec![0]; + for (i, c) in source.char_indices() { + if c == '\n' { + offsets.push(i + 1); + } + } + Self { newline_offsets: offsets } + } + + fn line_of(&self, byte_offset: usize) -> usize { + match self.newline_offsets.binary_search(&byte_offset) { + Ok(line) => line + 1, + Err(line) => line, + } + } +} + +/// Maps task function name → optional external path (from `@task(path="...")`) +type TaskFunctions = HashMap>; + +/// First pass: scan top-level `@task async def foo(...)` declarations. +fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { + let mut tasks = HashMap::new(); + for stmt in stmts { + if let Stmt::AsyncFunctionDef(func) = stmt { + for dec in &func.decorator_list { + match dec { + // @task (bare decorator) + Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => { + tasks.insert(func.name.to_string(), None); + } + // @task(path="...") + Expr::Call(call) => { + if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { + if id.as_str() == "task" { + let path = extract_task_path_kwarg(call); + tasks.insert(func.name.to_string(), path); + } + } + } + _ => {} + } + } + } + } + tasks +} + +/// Extract the `path=` keyword argument from a `@task(path="...")` call. +fn extract_task_path_kwarg(call: &ExprCall) -> Option { + for kw in &call.keywords { + if let Some(ref arg) = kw.arg { + if arg.as_str() == "path" { + if let Expr::Constant(c) = &kw.value { + if let rustpython_parser::ast::Constant::Str(s) = &c.value { + return Some(s.to_string()); + } + } + } + } + } + None +} + +struct WacWalker { + nodes: Vec, + edges: Vec, + errors: Vec, + node_counter: usize, + line_index: LineIndex, + task_functions: TaskFunctions, + in_try: bool, + in_while: bool, + in_nested_func: bool, + in_comprehension: bool, +} + +impl WacWalker { + fn new(source: &str, task_functions: TaskFunctions) -> Self { + Self { + nodes: Vec::new(), + edges: Vec::new(), + errors: Vec::new(), + node_counter: 0, + line_index: LineIndex::new(source), + task_functions, + in_try: false, + in_while: false, + in_nested_func: false, + in_comprehension: false, + } + } + + fn next_id(&mut self) -> String { + let id = format!("step_{}", self.node_counter); + self.node_counter += 1; + id + } + + fn add_node(&mut self, node: DagNode) -> String { + let id = node.id.clone(); + self.nodes.push(node); + id + } + + fn add_edge(&mut self, from: &str, to: &str, label: Option) { + self.edges + .push(DagEdge { from: from.to_string(), to: to.to_string(), label }); + } + + fn line_of_expr(&self, expr: &Expr) -> usize { + let offset = match expr { + Expr::Call(c) => c.range.start().to_usize(), + Expr::Await(a) => a.range.start().to_usize(), + Expr::Attribute(a) => a.range.start().to_usize(), + Expr::Name(n) => n.range.start().to_usize(), + _ => 0, + }; + self.line_index.line_of(offset) + } + + fn line_of_stmt(&self, stmt: &Stmt) -> usize { + let offset = match stmt { + Stmt::If(s) => s.range.start().to_usize(), + Stmt::For(s) => s.range.start().to_usize(), + Stmt::While(s) => s.range.start().to_usize(), + Stmt::Return(s) => s.range.start().to_usize(), + Stmt::Expr(s) => s.range.start().to_usize(), + Stmt::Try(s) => s.range.start().to_usize(), + Stmt::TryStar(s) => s.range.start().to_usize(), + Stmt::Assign(s) => s.range.start().to_usize(), + Stmt::AnnAssign(s) => s.range.start().to_usize(), + Stmt::FunctionDef(s) => s.range.start().to_usize(), + Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(), + _ => 0, + }; + self.line_index.line_of(offset) + } + + /// Check if an expression is a call to a known @task function + fn is_task_fn_call(&self, expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { + return self.task_functions.contains_key(id.as_str()); + } + } + false + } + + /// Check if an expression is `asyncio.gather(...)` call + fn is_asyncio_gather_call(expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) = + call.func.as_ref() + { + if attr.as_str() == "gather" { + if let Expr::Name(ExprName { id, .. }) = value.as_ref() { + return id.as_str() == "asyncio"; + } + } + } + } + false + } + + /// Extract step name and script from a task function call. + /// Name = function name, script = task_path or function name. + fn extract_step_info_from_task_call(&self, call: &ExprCall) -> Option<(String, String)> { + if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { + let name = id.to_string(); + let script = self + .task_functions + .get(id.as_str()) + .and_then(|p| p.clone()) + .unwrap_or_else(|| name.clone()); + Some((name, script)) + } else { + None + } + } + + fn expr_to_source(expr: &Expr) -> String { + match expr { + Expr::Compare(c) => { + let left = Self::expr_to_source(&c.left); + if let Some(comparator) = c.comparators.first() { + let right = Self::expr_to_source(comparator); + let op = match c.ops.first() { + Some(rustpython_parser::ast::CmpOp::Gt) => ">", + Some(rustpython_parser::ast::CmpOp::Lt) => "<", + Some(rustpython_parser::ast::CmpOp::GtE) => ">=", + Some(rustpython_parser::ast::CmpOp::LtE) => "<=", + Some(rustpython_parser::ast::CmpOp::Eq) => "==", + Some(rustpython_parser::ast::CmpOp::NotEq) => "!=", + Some(rustpython_parser::ast::CmpOp::In) => "in", + Some(rustpython_parser::ast::CmpOp::NotIn) => "not in", + Some(rustpython_parser::ast::CmpOp::Is) => "is", + Some(rustpython_parser::ast::CmpOp::IsNot) => "is not", + None => "?", + }; + format!("{left} {op} {right}") + } else { + left + } + } + Expr::Subscript(s) => { + let value = Self::expr_to_source(&s.value); + let slice = Self::expr_to_source(&s.slice); + format!("{value}[{slice}]") + } + Expr::Attribute(a) => { + let value = Self::expr_to_source(&a.value); + format!("{value}.{}", a.attr) + } + Expr::Name(n) => n.id.to_string(), + Expr::Constant(c) => match &c.value { + rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""), + rustpython_parser::ast::Constant::Int(i) => i.to_string(), + rustpython_parser::ast::Constant::Float(f) => f.to_string(), + rustpython_parser::ast::Constant::Bool(b) => b.to_string(), + rustpython_parser::ast::Constant::None => "None".to_string(), + _ => "...".to_string(), + }, + _ => "...".to_string(), + } + } + + /// Check if a statement body contains any task function calls (recursively) + fn body_contains_step(&self, body: &[Stmt]) -> bool { + for stmt in body { + if self.stmt_contains_step(stmt) { + return true; + } + } + false + } + + fn stmt_contains_step(&self, stmt: &Stmt) -> bool { + match stmt { + Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value), + Stmt::Assign(a) => self.expr_contains_step(&a.value), + Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), + Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), + Stmt::While(s) => { + self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse) + } + Stmt::Try(s) => { + self.body_contains_step(&s.body) + || self.body_contains_step(&s.orelse) + || self.body_contains_step(&s.finalbody) + || s.handlers.iter().any(|h| match h { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + self.body_contains_step(&eh.body) + } + }) + } + Stmt::TryStar(s) => { + self.body_contains_step(&s.body) + || self.body_contains_step(&s.orelse) + || self.body_contains_step(&s.finalbody) + || s.handlers.iter().any(|h| match h { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + self.body_contains_step(&eh.body) + } + }) + } + Stmt::Return(_) => false, + _ => false, + } + } + + fn expr_contains_step(&self, expr: &Expr) -> bool { + if self.is_task_fn_call(expr) { + return true; + } + match expr { + Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value), + Expr::Call(call) => { + if self.is_task_fn_call(&Expr::Call(call.clone())) { + return true; + } + if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) { + return call.args.iter().any(|a| self.expr_contains_step(a)); + } + false + } + _ => false, + } + } + + /// Walk a list of statements, returning (first_node_id, last_node_id) + fn walk_body(&mut self, body: &[Stmt]) -> Option<(String, String)> { + let mut first_id: Option = None; + let mut prev_id: Option = None; + + for stmt in body { + if let Some((stmt_first, stmt_last)) = self.walk_stmt(stmt) { + if let Some(ref prev) = prev_id { + self.add_edge(prev, &stmt_first, None); + } + if first_id.is_none() { + first_id = Some(stmt_first); + } + prev_id = Some(stmt_last); + } + } + + match (first_id, prev_id) { + (Some(f), Some(l)) => Some((f, l)), + _ => None, + } + } + + fn walk_stmt(&mut self, stmt: &Stmt) -> Option<(String, String)> { + match stmt { + Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value), + Stmt::Assign(a) => self.walk_expr_stmt(&a.value), + Stmt::If(if_stmt) => self.walk_if(if_stmt), + Stmt::For(for_stmt) => self.walk_for(for_stmt), + Stmt::While(while_stmt) => self.walk_while(while_stmt), + Stmt::Try(try_stmt) => self.walk_try(try_stmt), + Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt), + Stmt::Return(ret) => self.walk_return(ret), + Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => { + if self.stmt_contains_step(stmt) { + self.errors.push(validation::error_step_in_nested_function( + self.line_of_stmt(stmt), + )); + } + None + } + _ => None, + } + } + + fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> { + // await task_fn(...) + if let Expr::Await(ExprAwait { value, .. }) = expr { + // await task_fn(...) + if let Expr::Call(call) = value.as_ref() { + if self.is_task_fn_call(&Expr::Call(call.clone())) { + return self.emit_step(call, expr); + } + } + // await asyncio.gather(task_fn(...), task_fn(...), ...) + if Self::is_asyncio_gather_call(value) { + if let Expr::Call(gather_call) = value.as_ref() { + return self.emit_parallel(gather_call, expr); + } + } + } + + // Bare task_fn() without await — validation error + if self.is_task_fn_call(expr) { + self.errors + .push(validation::error_missing_await(self.line_of_expr(expr))); + } + + None + } + + fn emit_step(&mut self, call: &ExprCall, expr: &Expr) -> Option<(String, String)> { + if self.in_try { + self.errors + .push(validation::error_step_in_try(self.line_of_expr(expr))); + return None; + } + if self.in_while { + self.errors + .push(validation::error_step_in_while(self.line_of_expr(expr))); + return None; + } + if self.in_nested_func { + self.errors.push(validation::error_step_in_nested_function( + self.line_of_expr(expr), + )); + return None; + } + if self.in_comprehension { + self.errors.push(validation::error_step_in_comprehension( + self.line_of_expr(expr), + )); + return None; + } + + let (name, script) = self + .extract_step_info_from_task_call(call) + .unwrap_or(("unknown".into(), "unknown".into())); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Step { name: name.clone(), script }, + label: name, + line: self.line_of_expr(expr), + }); + Some((node_id.clone(), node_id)) + } + + fn emit_parallel(&mut self, gather_call: &ExprCall, expr: &Expr) -> Option<(String, String)> { + if self.in_try { + self.errors + .push(validation::error_step_in_try(self.line_of_expr(expr))); + return None; + } + if self.in_while { + self.errors + .push(validation::error_step_in_while(self.line_of_expr(expr))); + return None; + } + + let line = self.line_of_expr(expr); + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::ParallelStart, + label: "parallel".to_string(), + line, + }); + + let mut step_ids = Vec::new(); + for arg in &gather_call.args { + // Each arg should be task_fn(...) + if let Expr::Call(call) = arg { + if self.is_task_fn_call(&Expr::Call(call.clone())) { + let (name, script) = self + .extract_step_info_from_task_call(call) + .unwrap_or(("unknown".into(), "unknown".into())); + let step_id = self.next_id(); + let node_id = self.add_node(DagNode { + id: step_id.clone(), + node_type: DagNodeType::Step { name: name.clone(), script }, + label: name, + line: self.line_of_expr(arg), + }); + self.add_edge(&start_node_id, &node_id, None); + step_ids.push(node_id); + } + } + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::ParallelEnd, + label: "join".to_string(), + line, + }); + + for step_id in &step_ids { + self.add_edge(step_id, &end_node_id, None); + } + + Some((start_node_id, end_node_id)) + } + + fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> { + let has_steps_in_body = self.body_contains_step(&if_stmt.body); + let has_steps_in_else = self.body_contains_step(&if_stmt.orelse); + + if !has_steps_in_body && !has_steps_in_else { + return None; + } + + let line = self.line_index.line_of(if_stmt.range.start().to_usize()); + let condition_source = Self::expr_to_source(&if_stmt.test); + + let branch_id = self.next_id(); + let branch_node_id = self.add_node(DagNode { + id: branch_id.clone(), + node_type: DagNodeType::Branch { condition_source }, + label: "if".to_string(), + line, + }); + + let merge_id = format!("{branch_id}_merge"); + + let mut last_ids = Vec::new(); + + if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) { + self.add_edge(&branch_node_id, &true_first, Some("true".to_string())); + last_ids.push(true_last); + } else { + last_ids.push(branch_node_id.clone()); + } + + if !if_stmt.orelse.is_empty() { + if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) { + self.add_edge(&branch_node_id, &else_first, Some("false".to_string())); + last_ids.push(else_last); + } else { + last_ids.push(branch_node_id.clone()); + } + } + + if last_ids.len() == 1 { + Some((branch_node_id, last_ids.into_iter().next().unwrap())) + } else { + Some((branch_node_id, merge_id)) + } + } + + fn walk_for(&mut self, for_stmt: &StmtFor) -> Option<(String, String)> { + if !self.body_contains_step(&for_stmt.body) { + return None; + } + + let line = self.line_index.line_of(for_stmt.range.start().to_usize()); + let iter_source = Self::expr_to_source(&for_stmt.iter); + + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::LoopStart { iter_source }, + label: "for".to_string(), + line, + }); + + if let Some((body_first, body_last)) = self.walk_body(&for_stmt.body) { + self.add_edge(&start_node_id, &body_first, None); + self.add_edge(&body_last, &start_node_id, Some("next".to_string())); + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::LoopEnd, + label: "end for".to_string(), + line, + }); + self.add_edge(&start_node_id, &end_node_id, Some("done".to_string())); + + Some((start_node_id, end_node_id)) + } + + fn walk_while(&mut self, while_stmt: &StmtWhile) -> Option<(String, String)> { + if self.body_contains_step(&while_stmt.body) { + let line = self.line_index.line_of(while_stmt.range.start().to_usize()); + self.errors.push(validation::error_step_in_while(line)); + } + None + } + + fn walk_try(&mut self, try_stmt: &StmtTry) -> Option<(String, String)> { + let has_steps = self.body_contains_step(&try_stmt.body) + || self.body_contains_step(&try_stmt.orelse) + || self.body_contains_step(&try_stmt.finalbody) + || try_stmt.handlers.iter().any(|h| match h { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + self.body_contains_step(&eh.body) + } + }); + + if has_steps { + let line = self.line_index.line_of(try_stmt.range.start().to_usize()); + self.errors.push(validation::error_step_in_try(line)); + } + None + } + + fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> { + let has_steps = self.body_contains_step(&try_stmt.body) + || self.body_contains_step(&try_stmt.orelse) + || self.body_contains_step(&try_stmt.finalbody) + || try_stmt.handlers.iter().any(|h| match h { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + self.body_contains_step(&eh.body) + } + }); + + if has_steps { + let line = self.line_index.line_of(try_stmt.range.start().to_usize()); + self.errors.push(validation::error_step_in_try(line)); + } + None + } + + fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> { + let line = self.line_index.line_of(ret.range.start().to_usize()); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Return, + label: "return".to_string(), + line, + }); + Some((node_id.clone(), node_id)) + } +} + +/// Extract workflow function parameters (no longer skips ctx) +fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec { + let mut params = Vec::new(); + for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) { + let name = arg_with_default.def.arg.to_string(); + let typ = arg_with_default + .def + .annotation + .as_ref() + .map(|ann| WacWalker::expr_to_source(ann)); + params.push(Param { name, typ }); + } + params +} + +pub fn parse_python_workflow(code: &str) -> Result> { + let ast = rustpython_parser::ast::Suite::parse(code, "") + .map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?; + + // First pass: collect @task functions + let task_functions = collect_task_functions(&ast); + + // Find the @workflow async def + let workflow_fn = ast.iter().find_map(|stmt| { + if let Stmt::AsyncFunctionDef(func) = stmt { + let has_workflow_decorator = func.decorator_list.iter().any(|dec| { + if let Expr::Name(ExprName { id, .. }) = dec { + id.as_str() == "workflow" + } else { + false + } + }); + if has_workflow_decorator { + return Some(func); + } + } + // Also check non-async for error reporting + if let Stmt::FunctionDef(func) = stmt { + let has_workflow_decorator = func.decorator_list.iter().any(|dec| { + if let Expr::Name(ExprName { id, .. }) = dec { + id.as_str() == "workflow" + } else { + false + } + }); + if has_workflow_decorator { + return None; // Will be reported as not-async below + } + } + None + }); + + // Check for non-async workflow function + let non_async_workflow = ast.iter().find_map(|stmt| { + if let Stmt::FunctionDef(func) = stmt { + let has_workflow_decorator = func.decorator_list.iter().any(|dec| { + if let Expr::Name(ExprName { id, .. }) = dec { + id.as_str() == "workflow" + } else { + false + } + }); + if has_workflow_decorator { + let line_index = LineIndex::new(code); + return Some(line_index.line_of(func.range.start().to_usize())); + } + } + None + }); + + if let Some(line) = non_async_workflow { + if workflow_fn.is_none() { + return Err(vec![validation::error_not_async(line)]); + } + } + + let workflow_fn = workflow_fn.ok_or_else(|| { + vec![CompileError { message: "No @workflow async function found.".to_string(), line: 0 }] + })?; + + let params = extract_params(&workflow_fn.args); + let source_hash = compute_source_hash(code); + + let mut walker = WacWalker::new(code, task_functions); + walker.walk_body(&workflow_fn.body); + + if !walker.errors.is_empty() { + return Err(walker.errors); + } + + Ok(WorkflowDag { nodes: walker.nodes, edges: walker.edges, params, source_hash }) +} + +fn compute_source_hash(code: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(code.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +trait ToUsize { + fn to_usize(self) -> usize; +} + +impl ToUsize for rustpython_parser::text_size::TextSize { + fn to_usize(self) -> usize { + u32::from(self) as usize + } +} diff --git a/backend/parsers/windmill-parser-wac/src/typescript.rs b/backend/parsers/windmill-parser-wac/src/typescript.rs new file mode 100644 index 0000000000..bd777749ea --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/typescript.rs @@ -0,0 +1,739 @@ +use std::collections::HashMap; + +use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned}; +use swc_ecma_ast::*; +use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax}; + +use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag}; +use crate::validation::{self, CompileError}; + +/// Maps task function name → optional external path (from `task("f/path", ...)`) +type TaskFunctions = HashMap>; + +/// First pass: scan top-level `const foo = task(async (...) => {})` or +/// `const foo = task("f/path", async (...) => {})` declarations. +fn collect_task_functions(module: &Module) -> TaskFunctions { + let mut tasks = HashMap::new(); + for item in &module.body { + // const foo = task(async (...) => { ... }) + // const foo = task("f/path", async (...) => { ... }) + if let ModuleItem::Stmt(Stmt::Decl(Decl::Var(var_decl))) = item { + for decl in &var_decl.decls { + if let (Some(name), Some(init)) = (extract_var_name(&decl.name), &decl.init) { + if let Some(path) = extract_task_call_info(init) { + tasks.insert(name, path); + } + } + } + } + // export const foo = task(...) + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(export)) = item { + if let Decl::Var(var_decl) = &export.decl { + for decl in &var_decl.decls { + if let (Some(name), Some(init)) = (extract_var_name(&decl.name), &decl.init) { + if let Some(path) = extract_task_call_info(init) { + tasks.insert(name, path); + } + } + } + } + } + } + tasks +} + +/// Extract variable name from a pattern (simple ident case) +fn extract_var_name(pat: &Pat) -> Option { + if let Pat::Ident(BindingIdent { id, .. }) = pat { + Some(id.sym.to_string()) + } else { + None + } +} + +/// Check if expr is `task(async fn)` or `task("path", async fn)`. +/// Returns Some(optional_path) if it is a task() call. +fn extract_task_call_info(expr: &Expr) -> Option> { + if let Expr::Call(call) = expr { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Ident(ident) = callee.as_ref() { + if ident.sym.as_ref() == "task" { + // task("f/path", async fn) or task(async fn) + if call.args.len() == 2 { + // task("f/path", async fn) + let path = extract_string_lit(&call.args[0].expr); + return Some(path); + } else if call.args.len() == 1 { + // task(async fn) + return Some(None); + } + } + } + } + } + None +} + +struct TsWacWalker { + nodes: Vec, + edges: Vec, + errors: Vec, + node_counter: usize, + cm: Lrc, + task_functions: TaskFunctions, + in_try: bool, + in_while: bool, + in_nested_func: bool, +} + +impl TsWacWalker { + fn new(cm: Lrc, task_functions: TaskFunctions) -> Self { + Self { + nodes: Vec::new(), + edges: Vec::new(), + errors: Vec::new(), + node_counter: 0, + cm, + task_functions, + in_try: false, + in_while: false, + in_nested_func: false, + } + } + + fn next_id(&mut self) -> String { + let id = format!("step_{}", self.node_counter); + self.node_counter += 1; + id + } + + fn add_node(&mut self, node: DagNode) -> String { + let id = node.id.clone(); + self.nodes.push(node); + id + } + + fn add_edge(&mut self, from: &str, to: &str, label: Option) { + self.edges + .push(DagEdge { from: from.to_string(), to: to.to_string(), label }); + } + + fn span_line(&self, span: swc_common::Span) -> usize { + let loc = self.cm.lookup_char_pos(span.lo); + loc.line + } + + /// Check if expr is a call to a known task function + fn is_task_call(&self, expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Ident(ident) = callee.as_ref() { + return self.task_functions.contains_key(ident.sym.as_ref()); + } + } + } + false + } + + /// Check if expr is `Promise.all([...])` + fn is_promise_all(expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Member(MemberExpr { obj, prop: MemberProp::Ident(prop), .. }) = + callee.as_ref() + { + if prop.sym.as_ref() == "all" { + if let Expr::Ident(ident) = obj.as_ref() { + return ident.sym.as_ref() == "Promise"; + } + } + } + } + } + false + } + + /// Extract step name and script from a task function call. + /// Name = function name, script = task_path or function name. + fn extract_step_info_from_task_call(&self, call: &CallExpr) -> Option<(String, String)> { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Ident(ident) = callee.as_ref() { + let name = ident.sym.to_string(); + let script = self + .task_functions + .get(ident.sym.as_ref()) + .and_then(|p| p.clone()) + .unwrap_or_else(|| name.clone()); + return Some((name, script)); + } + } + None + } + + fn expr_to_source(&self, expr: &Expr) -> String { + let span = expr.span(); + self.cm + .span_to_snippet(span) + .unwrap_or_else(|_| "...".to_string()) + } + + fn body_contains_step(&self, stmts: &[Stmt]) -> bool { + stmts.iter().any(|s| self.stmt_contains_step(s)) + } + + fn stmt_contains_step(&self, stmt: &Stmt) -> bool { + match stmt { + Stmt::Expr(expr_stmt) => self.expr_contains_step(&expr_stmt.expr), + Stmt::Decl(Decl::Var(var_decl)) => var_decl.decls.iter().any(|d| { + d.init + .as_ref() + .map_or(false, |init| self.expr_contains_step(init)) + }), + Stmt::If(if_stmt) => { + self.stmt_contains_step(&if_stmt.cons) + || if_stmt + .alt + .as_ref() + .map_or(false, |alt| self.stmt_contains_step(alt)) + } + Stmt::Block(block) => self.body_contains_step(&block.stmts), + Stmt::For(for_stmt) => self.stmt_contains_step(&for_stmt.body), + Stmt::ForIn(for_in) => self.stmt_contains_step(&for_in.body), + Stmt::ForOf(for_of) => self.stmt_contains_step(&for_of.body), + Stmt::While(while_stmt) => self.stmt_contains_step(&while_stmt.body), + Stmt::Try(try_stmt) => { + self.body_contains_step(&try_stmt.block.stmts) + || try_stmt + .handler + .as_ref() + .map_or(false, |h| self.body_contains_step(&h.body.stmts)) + || try_stmt + .finalizer + .as_ref() + .map_or(false, |f| self.body_contains_step(&f.stmts)) + } + Stmt::Return(ret) => ret + .arg + .as_ref() + .map_or(false, |arg| self.expr_contains_step(arg)), + _ => false, + } + } + + fn expr_contains_step(&self, expr: &Expr) -> bool { + if self.is_task_call(expr) { + return true; + } + match expr { + Expr::Await(await_expr) => self.expr_contains_step(&await_expr.arg), + Expr::Call(call) => { + if Self::is_promise_all(&Expr::Call(call.clone())) { + return call.args.iter().any(|a| self.expr_contains_step(&a.expr)); + } + false + } + Expr::Paren(p) => self.expr_contains_step(&p.expr), + _ => false, + } + } + + fn walk_body(&mut self, stmts: &[Stmt]) -> Option<(String, String)> { + let mut first_id: Option = None; + let mut prev_id: Option = None; + + for stmt in stmts { + if let Some((stmt_first, stmt_last)) = self.walk_stmt(stmt) { + if let Some(ref prev) = prev_id { + self.add_edge(prev, &stmt_first, None); + } + if first_id.is_none() { + first_id = Some(stmt_first); + } + prev_id = Some(stmt_last); + } + } + + match (first_id, prev_id) { + (Some(f), Some(l)) => Some((f, l)), + _ => None, + } + } + + fn walk_stmt(&mut self, stmt: &Stmt) -> Option<(String, String)> { + match stmt { + Stmt::Expr(expr_stmt) => self.walk_expr_stmt(&expr_stmt.expr), + Stmt::Decl(Decl::Var(var_decl)) => { + // const result = await task_fn(...) + for decl in &var_decl.decls { + if let Some(init) = &decl.init { + if let Some(result) = self.walk_expr_stmt(init) { + return Some(result); + } + } + } + None + } + Stmt::If(if_stmt) => self.walk_if(if_stmt), + Stmt::For(for_stmt) => self.walk_for_stmt(for_stmt), + Stmt::ForIn(for_in) => self.walk_for_in(for_in), + Stmt::ForOf(for_of) => self.walk_for_of(for_of), + Stmt::While(while_stmt) => self.walk_while(while_stmt), + Stmt::Try(try_stmt) => self.walk_try(try_stmt), + Stmt::Block(block) => self.walk_body(&block.stmts), + Stmt::Return(ret) => self.walk_return(ret), + Stmt::Decl(Decl::Fn(_)) => { + if self.stmt_contains_step(stmt) { + self.errors.push(validation::error_step_in_nested_function( + self.span_line(stmt.span()), + )); + } + None + } + _ => None, + } + } + + fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> { + // await task_fn(...) + if let Expr::Await(await_expr) = expr { + if let Expr::Call(call) = await_expr.arg.as_ref() { + if self.is_task_call(&Expr::Call(call.clone())) { + return self.emit_step(call, expr); + } + } + // await Promise.all([task_fn(...), ...]) + if Self::is_promise_all(&await_expr.arg) { + if let Expr::Call(promise_call) = await_expr.arg.as_ref() { + return self.emit_parallel(promise_call, expr); + } + } + } + + // Bare task_fn() without await + if self.is_task_call(expr) { + self.errors + .push(validation::error_missing_await(self.span_line(expr.span()))); + } + + None + } + + fn emit_step(&mut self, call: &CallExpr, expr: &Expr) -> Option<(String, String)> { + if self.in_try { + self.errors + .push(validation::error_step_in_catch(self.span_line(expr.span()))); + return None; + } + if self.in_while { + self.errors + .push(validation::error_step_in_while(self.span_line(expr.span()))); + return None; + } + if self.in_nested_func { + self.errors.push(validation::error_step_in_nested_function( + self.span_line(expr.span()), + )); + return None; + } + + let (name, script) = self + .extract_step_info_from_task_call(call) + .unwrap_or(("unknown".into(), "unknown".into())); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Step { name: name.clone(), script }, + label: name, + line: self.span_line(expr.span()), + }); + Some((node_id.clone(), node_id)) + } + + fn emit_parallel(&mut self, promise_call: &CallExpr, expr: &Expr) -> Option<(String, String)> { + if self.in_try { + self.errors + .push(validation::error_step_in_catch(self.span_line(expr.span()))); + return None; + } + if self.in_while { + self.errors + .push(validation::error_step_in_while(self.span_line(expr.span()))); + return None; + } + + let line = self.span_line(expr.span()); + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::ParallelStart, + label: "parallel".to_string(), + line, + }); + + let mut step_ids = Vec::new(); + + // Promise.all takes an array as first argument + if let Some(first_arg) = promise_call.args.first() { + if let Expr::Array(ArrayLit { elems, .. }) = first_arg.expr.as_ref() { + for elem in elems.iter().flatten() { + if let Expr::Call(call) = elem.expr.as_ref() { + if self.is_task_call(&Expr::Call(call.clone())) { + let (name, script) = self + .extract_step_info_from_task_call(call) + .unwrap_or(("unknown".into(), "unknown".into())); + let step_id = self.next_id(); + let node_id = self.add_node(DagNode { + id: step_id.clone(), + node_type: DagNodeType::Step { name: name.clone(), script }, + label: name, + line: self.span_line(elem.expr.span()), + }); + self.add_edge(&start_node_id, &node_id, None); + step_ids.push(node_id); + } + } + } + } + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::ParallelEnd, + label: "join".to_string(), + line, + }); + + for step_id in &step_ids { + self.add_edge(step_id, &end_node_id, None); + } + + Some((start_node_id, end_node_id)) + } + + fn walk_if(&mut self, if_stmt: &IfStmt) -> Option<(String, String)> { + let has_steps_cons = self.stmt_contains_step(&if_stmt.cons); + let has_steps_alt = if_stmt + .alt + .as_ref() + .map_or(false, |a| self.stmt_contains_step(a)); + + if !has_steps_cons && !has_steps_alt { + return None; + } + + let line = self.span_line(if_stmt.span); + let condition_source = self.expr_to_source(&if_stmt.test); + + let branch_id = self.next_id(); + let branch_node_id = self.add_node(DagNode { + id: branch_id.clone(), + node_type: DagNodeType::Branch { condition_source }, + label: "if".to_string(), + line, + }); + + let mut last_ids = Vec::new(); + + // True branch + if let Some((true_first, true_last)) = self.walk_stmt(&if_stmt.cons) { + self.add_edge(&branch_node_id, &true_first, Some("true".to_string())); + last_ids.push(true_last); + } else { + last_ids.push(branch_node_id.clone()); + } + + // False branch + if let Some(alt) = &if_stmt.alt { + if let Some((else_first, else_last)) = self.walk_stmt(alt) { + self.add_edge(&branch_node_id, &else_first, Some("false".to_string())); + last_ids.push(else_last); + } else { + last_ids.push(branch_node_id.clone()); + } + } + + if last_ids.len() == 1 { + Some((branch_node_id, last_ids.into_iter().next().unwrap())) + } else { + let merge_id = format!("{branch_id}_merge"); + Some((branch_node_id, merge_id)) + } + } + + fn walk_for_stmt(&mut self, for_stmt: &ForStmt) -> Option<(String, String)> { + if !self.stmt_contains_step(&for_stmt.body) { + return None; + } + self.walk_loop_body(&for_stmt.body, for_stmt.span, "for") + } + + fn walk_for_in(&mut self, for_in: &ForInStmt) -> Option<(String, String)> { + if !self.stmt_contains_step(&for_in.body) { + return None; + } + let iter_source = self.expr_to_source(&for_in.right); + self.walk_loop_body_with_iter(&for_in.body, for_in.span, &iter_source) + } + + fn walk_for_of(&mut self, for_of: &ForOfStmt) -> Option<(String, String)> { + if !self.stmt_contains_step(&for_of.body) { + return None; + } + let iter_source = self.expr_to_source(&for_of.right); + self.walk_loop_body_with_iter(&for_of.body, for_of.span, &iter_source) + } + + fn walk_loop_body( + &mut self, + body: &Stmt, + span: swc_common::Span, + _label: &str, + ) -> Option<(String, String)> { + self.walk_loop_body_with_iter(body, span, "...") + } + + fn walk_loop_body_with_iter( + &mut self, + body: &Stmt, + span: swc_common::Span, + iter_source: &str, + ) -> Option<(String, String)> { + let line = self.span_line(span); + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::LoopStart { iter_source: iter_source.to_string() }, + label: "for".to_string(), + line, + }); + + if let Some((body_first, body_last)) = self.walk_stmt(body) { + self.add_edge(&start_node_id, &body_first, None); + self.add_edge(&body_last, &start_node_id, Some("next".to_string())); + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::LoopEnd, + label: "end for".to_string(), + line, + }); + self.add_edge(&start_node_id, &end_node_id, Some("done".to_string())); + + Some((start_node_id, end_node_id)) + } + + fn walk_while(&mut self, while_stmt: &WhileStmt) -> Option<(String, String)> { + if self.stmt_contains_step(&while_stmt.body) { + self.errors.push(validation::error_step_in_while( + self.span_line(while_stmt.span), + )); + } + None + } + + fn walk_try(&mut self, try_stmt: &TryStmt) -> Option<(String, String)> { + let has_steps = self.body_contains_step(&try_stmt.block.stmts) + || try_stmt + .handler + .as_ref() + .map_or(false, |h| self.body_contains_step(&h.body.stmts)) + || try_stmt + .finalizer + .as_ref() + .map_or(false, |f| self.body_contains_step(&f.stmts)); + + if has_steps { + self.errors.push(validation::error_step_in_catch( + self.span_line(try_stmt.span), + )); + } + None + } + + fn walk_return(&mut self, ret: &ReturnStmt) -> Option<(String, String)> { + let line = self.span_line(ret.span); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Return, + label: "return".to_string(), + line, + }); + Some((node_id.clone(), node_id)) + } +} + +/// Extract workflow function params (no longer skips ctx) +fn extract_ts_params(params: &[swc_ecma_ast::Param], cm: &Lrc) -> Vec { + let mut result = Vec::new(); + for param in params { + let (name, typ) = match ¶m.pat { + Pat::Ident(BindingIdent { id, type_ann, .. }) => { + let name = id.sym.to_string(); + let typ = type_ann.as_ref().map(|ann| { + cm.span_to_snippet(ann.type_ann.span()) + .unwrap_or_else(|_| "unknown".to_string()) + }); + (name, typ) + } + _ => continue, + }; + result.push(Param { name, typ }); + } + result +} + +pub fn parse_ts_workflow(code: &str) -> Result> { + let cm: Lrc = Default::default(); + let fm = cm.new_source_file(FileName::Custom("workflow.ts".into()).into(), code.into()); + let lexer = Lexer::new( + Syntax::Typescript(TsSyntax::default()), + Default::default(), + StringInput::from(&*fm), + None, + ); + + let mut parser = Parser::new_from(lexer); + let module = parser + .parse_module() + .map_err(|e| vec![CompileError { message: format!("Parse error: {e:?}"), line: 0 }])?; + + // First pass: collect task functions + let task_functions = collect_task_functions(&module); + + // Find: export default workflow(async (...) => { ... }) + // or: export default workflow(async function(...) { ... }) + let mut workflow_body: Option<(&[Stmt], Vec)> = None; + + for item in &module.body { + // export default workflow(async (...) => { ... }) + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultExpr(export)) = item { + if let Some(result) = find_workflow_call(&export.expr, &cm) { + workflow_body = Some(result); + break; + } + } + // const wf = workflow(async (...) => { ... }); export default wf; + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultDecl(export)) = item { + if let DefaultDecl::Fn(_) = &export.decl { + // `export default async function(...) { ... }` — not wrapped in workflow(), skip + } + } + if let ModuleItem::Stmt(Stmt::Decl(Decl::Var(var_decl))) = item { + for decl in &var_decl.decls { + if let Some(init) = &decl.init { + if let Some(result) = find_workflow_call(init, &cm) { + workflow_body = Some(result); + break; + } + } + } + } + } + + let (stmts, params) = workflow_body.ok_or_else(|| { + vec![CompileError { + message: "No workflow() wrapped async function found.".to_string(), + line: 0, + }] + })?; + + let source_hash = compute_source_hash(code); + + let mut walker = TsWacWalker::new(cm, task_functions); + walker.walk_body(stmts); + + if !walker.errors.is_empty() { + return Err(walker.errors); + } + + Ok(WorkflowDag { nodes: walker.nodes, edges: walker.edges, params, source_hash }) +} + +/// Find workflow(async (...) => { ... }) or workflow(async function(...) { ... }) +fn find_workflow_call<'a>(expr: &'a Expr, cm: &Lrc) -> Option<(&'a [Stmt], Vec)> { + if let Expr::Call(call) = expr { + // Check if callee is `workflow` + let is_workflow = match &call.callee { + Callee::Expr(callee_expr) => { + if let Expr::Ident(ident) = callee_expr.as_ref() { + ident.sym.as_ref() == "workflow" + } else { + false + } + } + _ => false, + }; + + if is_workflow { + if let Some(first_arg) = call.args.first() { + return extract_async_fn_body(&first_arg.expr, cm); + } + } + } + None +} + +fn extract_async_fn_body<'a>( + expr: &'a Expr, + cm: &Lrc, +) -> Option<(&'a [Stmt], Vec)> { + match expr { + Expr::Arrow(arrow) if arrow.is_async => { + let params = extract_arrow_params(&arrow.params, cm); + match &*arrow.body { + BlockStmtOrExpr::BlockStmt(block) => Some((&block.stmts, params)), + _ => None, + } + } + Expr::Fn(fn_expr) if fn_expr.function.is_async => { + let params = extract_ts_params(&fn_expr.function.params, cm); + fn_expr + .function + .body + .as_ref() + .map(|body| (body.stmts.as_slice(), params)) + } + Expr::Paren(p) => extract_async_fn_body(&p.expr, cm), + _ => None, + } +} + +/// Extract arrow function params (no longer skips ctx) +fn extract_arrow_params(pats: &[Pat], cm: &Lrc) -> Vec { + let mut result = Vec::new(); + for pat in pats { + match pat { + Pat::Ident(BindingIdent { id, type_ann, .. }) => { + let name = id.sym.to_string(); + let typ = type_ann.as_ref().map(|ann| { + cm.span_to_snippet(ann.type_ann.span()) + .unwrap_or_else(|_| "unknown".to_string()) + }); + result.push(Param { name, typ }); + } + _ => {} + } + } + result +} + +fn extract_string_lit(expr: &Expr) -> Option { + match expr { + Expr::Lit(Lit::Str(s)) => Some(s.value.to_string()), + Expr::Tpl(tpl) if tpl.exprs.is_empty() && tpl.quasis.len() == 1 => { + tpl.quasis.first().map(|q| q.raw.to_string()) + } + _ => None, + } +} + +fn compute_source_hash(code: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(code.as_bytes()); + format!("{:x}", hasher.finalize()) +} diff --git a/backend/parsers/windmill-parser-wac/src/validation.rs b/backend/parsers/windmill-parser-wac/src/validation.rs new file mode 100644 index 0000000000..e3a57c6811 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/validation.rs @@ -0,0 +1,64 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct CompileError { + pub message: String, + pub line: usize, +} + +impl std::fmt::Display for CompileError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "line {}: {}", self.line, self.message) + } +} + +pub fn error_step_in_try(line: usize) -> CompileError { + CompileError { + message: + "Task calls inside try/except are not allowed. Steps have built-in error handling." + .to_string(), + line, + } +} + +pub fn error_step_in_while(line: usize) -> CompileError { + CompileError { + message: "Task calls inside while loops are not allowed. Use for loops instead." + .to_string(), + line, + } +} + +pub fn error_step_in_nested_function(line: usize) -> CompileError { + CompileError { + message: "Task calls inside nested functions, closures, or lambdas are not allowed." + .to_string(), + line, + } +} + +pub fn error_step_in_comprehension(line: usize) -> CompileError { + CompileError { message: "Task calls inside comprehensions are not allowed.".to_string(), line } +} + +pub fn error_not_async(line: usize) -> CompileError { + CompileError { message: "Workflow function must be async.".to_string(), line } +} + +pub fn error_missing_await(line: usize) -> CompileError { + CompileError { + message: + "Task calls must be awaited directly or used inside asyncio.gather()/Promise.all()." + .to_string(), + line, + } +} + +pub fn error_step_in_catch(line: usize) -> CompileError { + CompileError { + message: + "Task calls inside catch blocks are not allowed. Steps have built-in error handling." + .to_string(), + line, + } +} diff --git a/backend/parsers/windmill-parser-wac/tests/python_tests.rs b/backend/parsers/windmill-parser-wac/tests/python_tests.rs new file mode 100644 index 0000000000..59f0b59f5c --- /dev/null +++ b/backend/parsers/windmill-parser-wac/tests/python_tests.rs @@ -0,0 +1,266 @@ +use windmill_parser_wac::dag::DagNodeType; +use windmill_parser_wac::python::parse_python_workflow; + +#[test] +fn test_simple_sequential_workflow() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(url: str): ... +@task +async def load_data(data: list): ... + +@workflow +async def my_etl(url: str): + raw = await extract_data(url=url) + await load_data(data=raw) + return {"status": "done"} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 3); // 2 steps + 1 return + assert_eq!(dag.edges.len(), 2); // step0->step1, step1->return + + // Check params (url — no ctx to skip) + assert_eq!(dag.params.len(), 1); + assert_eq!(dag.params[0].name, "url"); + assert_eq!(dag.params[0].typ.as_deref(), Some("str")); + + // Check first step + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "extract_data"); + assert_eq!(script, "extract_data"); + } + _ => panic!("expected Step node"), + } + + // Check second step + match &dag.nodes[1].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "load_data"); + assert_eq!(script, "load_data"); + } + _ => panic!("expected Step node"), + } + + // Check return + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Return)); + + // Check source hash is non-empty + assert!(!dag.source_hash.is_empty()); +} + +#[test] +fn test_parallel_workflow() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(url: str): ... +@task +async def clean_data(data: list): ... +@task +async def compute_stats(data: list): ... +@task +async def load_to_warehouse(rows: list): ... + +@workflow +async def my_etl(url: str): + raw = await extract_data(url=url) + cleaned, stats = await asyncio.gather( + clean_data(data=raw), + compute_stats(data=raw), + ) + await load_to_warehouse(rows=cleaned) + return {"status": "done"} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + + // extract, ParallelStart, clean, stats, ParallelEnd, load, return = 7 + assert_eq!(dag.nodes.len(), 7); + + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::ParallelStart)); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[3].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[4].node_type, DagNodeType::ParallelEnd)); + assert!(matches!(dag.nodes[5].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[6].node_type, DagNodeType::Return)); +} + +#[test] +fn test_conditional_workflow() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def send_alert(msg: str): ... +@task +async def load_data(): ... + +@workflow +async def my_etl(count: int): + if count > 100: + await send_alert(msg="large") + await load_data() + return {"done": True} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + // Branch, notify step, load step, return = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. })); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. })); +} + +#[test] +fn test_for_loop_workflow() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def process_item(item: str): ... + +@workflow +async def my_etl(items: list): + for item in items: + await process_item(item=item) + return {"done": True} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + // LoopStart, step, LoopEnd, return = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!( + dag.nodes[0].node_type, + DagNodeType::LoopStart { .. } + )); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::LoopEnd)); +} + +#[test] +fn test_reject_step_in_try() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(): ... + +@workflow +async def my_etl(): + try: + await extract_data() + except Exception: + pass +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("try/except")); +} + +#[test] +fn test_reject_step_in_while() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(): ... + +@workflow +async def my_etl(): + while True: + await extract_data() +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("while")); +} + +#[test] +fn test_reject_non_async() { + let code = r#" +from wmill import workflow + +@workflow +def my_etl(): + pass +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("async")); +} + +#[test] +fn test_reject_missing_await() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(): ... + +@workflow +async def my_etl(): + extract_data() +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("awaited")); +} + +#[test] +fn test_no_workflow_function() { + let code = r#" +async def my_func(): + pass +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("No @workflow")); +} + +#[test] +fn test_task_with_external_path() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task(path="f/external_script") +async def run_external(x: int): ... + +@workflow +async def my_wf(x: int): + result = await run_external(x=x) + return result +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 2); // 1 step + 1 return (bare `return` is not a step node but walk_return creates one) + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "run_external"); + assert_eq!(script, "f/external_script"); + } + _ => panic!("expected Step node"), + } +} diff --git a/backend/parsers/windmill-parser-wac/tests/ts_tests.rs b/backend/parsers/windmill-parser-wac/tests/ts_tests.rs new file mode 100644 index 0000000000..949f326b90 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/tests/ts_tests.rs @@ -0,0 +1,245 @@ +use windmill_parser_wac::dag::DagNodeType; +use windmill_parser_wac::typescript::parse_ts_workflow; + +#[test] +fn test_simple_sequential_ts_workflow() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async (url: string) => {}); +const load_data = task(async (data: any) => {}); + +export default workflow(async (url: string) => { + const raw = await extract_data(url); + await load_data(raw); + return { status: "done" }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 3); // 2 steps + 1 return + assert_eq!(dag.edges.len(), 2); + + // Check params (url — no ctx to skip) + assert_eq!(dag.params.len(), 1); + assert_eq!(dag.params[0].name, "url"); + assert_eq!(dag.params[0].typ.as_deref(), Some("string")); + + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "extract_data"); + assert_eq!(script, "extract_data"); + } + _ => panic!("expected Step node"), + } + + match &dag.nodes[1].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "load_data"); + assert_eq!(script, "load_data"); + } + _ => panic!("expected Step node"), + } + + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Return)); + assert!(!dag.source_hash.is_empty()); +} + +#[test] +fn test_parallel_ts_workflow() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async (url: string) => {}); +const clean_data = task(async (data: any) => {}); +const compute_stats = task(async (data: any) => {}); +const load_to_warehouse = task(async (rows: any) => {}); + +export default workflow(async (url: string) => { + const raw = await extract_data(url); + + const [cleaned, stats] = await Promise.all([ + clean_data(raw), + compute_stats(raw), + ]); + + await load_to_warehouse(cleaned); + return { status: "done", rows: stats.rowCount }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + // extract, ParallelStart, clean, stats, ParallelEnd, load, return = 7 + assert_eq!(dag.nodes.len(), 7); + + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::ParallelStart)); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[3].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[4].node_type, DagNodeType::ParallelEnd)); + assert!(matches!(dag.nodes[5].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[6].node_type, DagNodeType::Return)); +} + +#[test] +fn test_conditional_ts_workflow() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const send_alert = task(async (msg: string) => {}); +const load_data = task(async () => {}); + +export default workflow(async (count: number) => { + if (count > 100) { + await send_alert("large"); + } + await load_data(); + return { done: true }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + // Branch, notify, load, return = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. })); +} + +#[test] +fn test_for_of_ts_workflow() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const process_item = task(async (item: string) => {}); + +export default workflow(async (items: string[]) => { + for (const item of items) { + await process_item(item); + } + return { done: true }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + // LoopStart, step, LoopEnd, return = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!( + dag.nodes[0].node_type, + DagNodeType::LoopStart { .. } + )); +} + +#[test] +fn test_reject_step_in_try_catch() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async () => {}); + +export default workflow(async () => { + try { + await extract_data(); + } catch (e) { + console.log(e); + } +}); +"#; + + let result = parse_ts_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("catch")); +} + +#[test] +fn test_reject_step_in_while_ts() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async () => {}); + +export default workflow(async () => { + while (true) { + await extract_data(); + } +}); +"#; + + let result = parse_ts_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("while")); +} + +#[test] +fn test_reject_missing_await_ts() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async () => {}); + +export default workflow(async () => { + extract_data(); +}); +"#; + + let result = parse_ts_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("awaited")); +} + +#[test] +fn test_no_workflow_wrapper() { + let code = r#" +export default async function main(ctx: any) { + return {}; +} +"#; + + let result = parse_ts_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("No workflow()")); +} + +#[test] +fn test_variable_declaration_with_step() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const compute = task(async () => {}); + +export default workflow(async () => { + const result = await compute(); + return result; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 2); // step + return + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. })); +} + +#[test] +fn test_task_with_external_path() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const run_external = task("f/external_script", async (x: number) => {}); + +export default workflow(async (x: number) => { + const result = await run_external(x); + return result; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 2); // step + return + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "run_external"); + assert_eq!(script, "f/external_script"); + } + _ => panic!("expected Step node"), + } +} diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 1a0d425704..8372b1838c 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -38,6 +38,8 @@ csharp-parser = [ "dep:windmill-parser-csharp"] nu-parser = [ "dep:windmill-parser-nu"] java-parser = [ "dep:windmill-parser-java"] ruby-parser = [ "dep:windmill-parser-ruby"] +wac-parser = [ "dep:windmill-parser-wac"] +asset-parser = [ "dep:windmill-parser-ts-asset", "dep:windmill-parser-py-asset", "dep:windmill-parser-sql-asset"] [dependencies] anyhow.workspace = true @@ -55,6 +57,10 @@ windmill-parser-csharp = { workspace = true, optional = true } windmill-parser-nu = { workspace = true, optional = true } windmill-parser-java = { workspace = true, optional = true } windmill-parser-ruby = { workspace = true, optional = true } +windmill-parser-wac = { workspace = true, optional = true } +windmill-parser-ts-asset = { workspace = true, optional = true } +windmill-parser-py-asset = { workspace = true, optional = true } +windmill-parser-sql-asset = { workspace = true, optional = true } wasm-bindgen.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-wasm/build.nu b/backend/parsers/windmill-parser-wasm/build.nu index d954366113..4e08ce95d3 100755 --- a/backend/parsers/windmill-parser-wasm/build.nu +++ b/backend/parsers/windmill-parser-wasm/build.nu @@ -56,6 +56,17 @@ const targets = [ features: "ruby-parser", env: "tree-sitter", }, + { + ident: "wac", + desc: "Workflow-as-Code", + features: "wac-parser", + env: "default", + }, { + ident: "asset", + desc: "Asset parsers (TS, Python, SQL) with SQL AST", + features: "asset-parser", + env: "default", + }, # ^^^ Add new entry here ^^^ ]; # NOTE: This is legacy command for building all, but it is not more used diff --git a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh index e8215f8fa4..80f31650f3 100755 --- a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh +++ b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh @@ -33,3 +33,6 @@ popd pushd "pkg-java" && npm publish ${args} popd + +pushd "pkg-asset" && npm publish ${args} +popd diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 61cc11d81d..2af0a3bc9a 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -187,28 +187,28 @@ pub fn parse_ruby(code: &str) -> String { wrap_sig(windmill_parser_ruby::parse_ruby_signature(code)) } -#[cfg(feature = "sql-parser")] +#[cfg(feature = "asset-parser")] #[wasm_bindgen] pub fn parse_assets_sql(code: &str) -> String { - match windmill_parser_sql::parse_assets(code) { + match windmill_parser_sql_asset::parse_assets(code) { Ok(r) => serde_json::to_string(&r).unwrap(), Err(err) => format!("err: {:?}", err), } } -#[cfg(feature = "ts-parser")] +#[cfg(feature = "asset-parser")] #[wasm_bindgen] pub fn parse_assets_ts(code: &str) -> String { - match windmill_parser_ts::parse_assets(code) { + match windmill_parser_ts_asset::parse_assets(code) { Ok(r) => serde_json::to_string(&r).unwrap(), Err(err) => format!("err: {:?}", err), } } -#[cfg(feature = "py-parser")] +#[cfg(feature = "asset-parser")] #[wasm_bindgen] pub fn parse_assets_py(code: &str) -> String { - match windmill_parser_py::parse_assets(code) { + match windmill_parser_py_asset::parse_assets(code) { Ok(r) => serde_json::to_string(&r).unwrap(), Err(err) => format!("err: {:?}", err), } @@ -223,4 +223,11 @@ pub fn parse_assets_ansible(code: &str) -> String { } } +#[cfg(feature = "wac-parser")] +#[wasm_bindgen] +pub fn parse_workflow_as_code(code: &str, language: &str) -> String { + let result = windmill_parser_wac::parse_workflow(code, language); + serde_json::to_string(&result).unwrap_or_else(|_| "{\"type\": \"error\"}".to_string()) +} + // for related places search: ADD_NEW_LANG diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 2463995606..44388690e4 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -18,6 +18,7 @@ pub enum AssetKind { Resource, Ducklake, DataTable, + Volume, } #[derive(Serialize, Debug, PartialEq, Clone)] @@ -148,4 +149,5 @@ pub const ASSET_KINDS: &[(&str, AssetKind)] = &[ ("$res:", AssetKind::Resource), ("ducklake://", AssetKind::Ducklake), ("datatable://", AssetKind::DataTable), + ("volume://", AssetKind::Volume), ]; diff --git a/backend/src/db_connect.rs b/backend/src/db_connect.rs index 05bf7b6522..abb3378efc 100644 --- a/backend/src/db_connect.rs +++ b/backend/src/db_connect.rs @@ -20,6 +20,7 @@ pub async fn connect_db( server_mode: bool, indexer_mode: bool, worker_mode: bool, + num_workers: i32, #[cfg(feature = "private")] mut killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> anyhow::Result> { use anyhow::Context; @@ -34,13 +35,7 @@ pub async fn connect_db( } else if indexer_mode { DEFAULT_MAX_CONNECTIONS_INDEXER } else { - DEFAULT_MAX_CONNECTIONS_WORKER - + std::env::var("NUM_WORKERS") - .ok() - .map(|x| x.parse().ok()) - .flatten() - .unwrap_or(1) - - 1 + DEFAULT_MAX_CONNECTIONS_WORKER + (num_workers.max(1) as u32) - 1 } } }; @@ -103,7 +98,7 @@ pub async fn connect( use sqlx::Executor; use std::time::Duration; let mut pool_options = sqlx::postgres::PgPoolOptions::new() - .min_connections((max_connections / 5).clamp(1, max_connections)) + .min_connections(0) .max_connections(max_connections) .max_lifetime(Duration::from_secs(30 * 60)); // 30 mins if worker_mode { diff --git a/backend/src/main.rs b/backend/src/main.rs index ac8b3f30b2..d228342a1d 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -37,8 +37,9 @@ use windmill_common::ee_oss::{ use windmill_common::{ agent_workers::AgentConfig, global_settings::{ - APP_WORKSPACED_ROUTE_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, - CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + APP_WORKSPACED_ROUTE_SETTING, AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, + BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, @@ -62,7 +63,7 @@ use windmill_common::{ }, worker::{ is_native_mode_from_env, reload_custom_tags_setting, Connection, HUB_CACHE_DIR, - HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_DIR, TMP_LOGS_DIR, WORKER_GROUP, + HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, WINDMILL_DIR, WORKER_GROUP, }, KillpillSender, DEFAULT_HUB_BASE_URL, METRICS_ENABLED, }; @@ -97,8 +98,9 @@ use windmill_worker::{ use crate::monitor::{ initial_load, load_keep_job_dir, load_metrics_debug_enabled, load_require_preexisting_user, load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db, - reload_app_workspaced_route_setting, reload_base_url_setting, - reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting, + reload_app_workspaced_route_setting, reload_audit_log_retention_days_setting, + reload_base_url_setting, reload_bunfig_install_scopes_setting, + reload_critical_alert_mute_ui_setting, reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting, reload_extra_pip_index_url_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, @@ -238,8 +240,8 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { ) })?; - create_dir_all(HUB_CACHE_DIR)?; - create_dir_all(BUN_BUNDLE_CACHE_DIR)?; + create_dir_all(&*HUB_CACHE_DIR)?; + create_dir_all(&*BUN_BUNDLE_CACHE_DIR)?; for path in paths.values() { tracing::info!("Caching hub script at {path}"); @@ -249,7 +251,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { .as_ref() .is_some_and(|x| x == &ScriptLang::Deno) { - let job_dir = format!("{}/cache_init/{}", TMP_DIR, Uuid::new_v4()); + let job_dir = format!("{}/cache_init/{}", *WINDMILL_DIR, Uuid::new_v4()); create_dir_all(&job_dir)?; let _ = windmill_worker::generate_deno_lock( &Uuid::nil(), @@ -267,7 +269,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { tokio::fs::remove_dir_all(job_dir).await?; } else if res.language.as_ref().is_some_and(|x| x == &ScriptLang::Bun) { let job_id = Uuid::new_v4(); - let job_dir = format!("{}/cache_init/{}", TMP_DIR, job_id); + let job_dir = format!("{}/cache_init/{}", *WINDMILL_DIR, job_id); create_dir_all(&job_dir)?; if let Some(lock) = res.lockfile { let _ = windmill_worker::prepare_job_dir(&lock, &job_dir).await?; @@ -384,9 +386,9 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { println!("Fetched {} resource types from hub", resource_types.len()); - create_dir_all(HUB_RT_CACHE_DIR)?; + create_dir_all(&*HUB_RT_CACHE_DIR)?; - let cache_path = format!("{}/{}", HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); + let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); let content = serde_json::to_string_pretty(&resource_types) .with_context(|| "Failed to serialize resource types")?; @@ -398,7 +400,7 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { } pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyhow::Result<()> { - let cache_path = format!("{}/{}", HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); + let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); if tokio::fs::metadata(&cache_path).await.is_err() { tracing::info!( @@ -517,6 +519,51 @@ fn print_help() { println!("- At startup, Windmill logs currently set configuration keys for visibility."); } +async fn resync_custom_instance_user_pwd_if_needed(db: &Pool) { + use windmill_common::utils::get_custom_pg_instance_password; + use windmill_common::{get_database_url, PgDatabase}; + + let user_pwd = match get_custom_pg_instance_password(db).await { + Ok(pwd) => pwd, + Err(_) => { + // Setting doesn't exist yet (fresh install or pre-migration), skip check + return; + } + }; + + let mut pg_creds = match get_database_url().await { + Ok(url) => match PgDatabase::parse_uri(&url.as_str().await) { + Ok(creds) => creds, + Err(e) => { + tracing::warn!("Failed to parse database URL for custom_instance_user check: {e}"); + return; + } + }, + Err(e) => { + tracing::warn!("Failed to get database URL for custom_instance_user check: {e}"); + return; + } + }; + + pg_creds.user = Some("custom_instance_user".to_string()); + pg_creds.password = Some(user_pwd); + + match pg_creds.connect().await { + Ok(_) => { + tracing::info!("custom_instance_user password is in sync"); + } + Err(e) => { + tracing::warn!("custom_instance_user password is out of sync ({e}), refreshing..."); + if let Err(e) = windmill_api_settings::refresh_custom_instance_user_pwd_inner(db).await + { + tracing::error!("Failed to refresh custom_instance_user password: {e}"); + } else { + tracing::info!("Successfully refreshed custom_instance_user password"); + } + } + } +} + async fn windmill_main() -> anyhow::Result<()> { let (killpill_tx, mut killpill_rx) = KillpillSender::new(2); let mut monitor_killpill_rx = killpill_tx.subscribe(); @@ -647,6 +694,7 @@ async fn windmill_main() -> anyhow::Result<()> { let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer || mode == Mode::MCP { 0 } else if is_native_mode_from_env() { + NATIVE_MODE_RESOLVED.store(true, std::sync::atomic::Ordering::Relaxed); println!("Native mode enabled: forcing NUM_WORKERS=8"); 8 } else { @@ -819,6 +867,30 @@ async fn windmill_main() -> anyhow::Result<()> { } } + // Resolve native mode early (before connect_db) so connection pool size accounts for it. + // native_mode can come from env OR from the DB worker group config. + if worker_mode && !is_native_mode_from_env() { + if let Some(db) = conn.as_sql() { + let native_from_db: bool = sqlx::query_scalar!( + "SELECT (config->>'native_mode')::boolean FROM config WHERE name = $1", + format!("worker__{}", *windmill_common::worker::WORKER_GROUP) + ) + .fetch_optional(db) + .await + .ok() + .flatten() + .flatten() + .unwrap_or(false); + if native_from_db { + NATIVE_MODE_RESOLVED.store(true, std::sync::atomic::Ordering::Relaxed); + num_workers = 8; + tracing::info!( + "Native mode detected from worker config (early): forcing NUM_WORKERS=8" + ); + } + } + } + let conn = if mode == Mode::Agent { conn } else { @@ -831,6 +903,7 @@ async fn windmill_main() -> anyhow::Result<()> { server_mode, indexer_mode, worker_mode, + num_workers, #[cfg(feature = "private")] killpill_rx.resubscribe(), ) @@ -838,6 +911,11 @@ async fn windmill_main() -> anyhow::Result<()> { // NOTE: Variable/resource cache initialization moved to API server in windmill-api + // Check if custom_instance_user password is in sync + if server_mode { + resync_custom_instance_user_pwd_if_needed(&db).await; + } + Connection::Sql(db) }; @@ -930,16 +1008,6 @@ Windmill Community Edition {GIT_VERSION} ) .await; - // native_mode may also be set via DB worker group config (not just env). - // NATIVE_MODE_RESOLVED is updated by load_worker_config during initial_load. - if worker_mode - && !is_native_mode_from_env() - && NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed) - { - num_workers = 8; - tracing::info!("Native mode detected from worker config: forcing NUM_WORKERS=8"); - } - monitor_db( &conn, &base_internal_url, @@ -969,7 +1037,7 @@ Windmill Community Edition {GIT_VERSION} DirBuilder::new() .recursive(true) - .create("/tmp/windmill") + .create(&*WINDMILL_DIR) .expect("could not create initial server dir"); #[cfg(feature = "tantivy")] @@ -1614,6 +1682,9 @@ async fn process_notify_event( } TIMEOUT_WAIT_RESULT_SETTING => reload_timeout_wait_result_setting(conn).await, RETENTION_PERIOD_SECS_SETTING => reload_retention_period_setting(conn).await, + AUDIT_LOG_RETENTION_DAYS_SETTING => { + reload_audit_log_retention_days_setting(conn).await + } MONITOR_LOGS_ON_OBJECT_STORE_SETTING => { reload_delete_logs_periodically_setting(conn).await } @@ -1717,6 +1788,11 @@ async fn process_notify_event( tracing::error!(error = %e, "Could not reload critical alert UI setting"); } } + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING => { + if let Err(e) = reload_critical_alerts_on_token_expiry_setting(conn).await { + tracing::error!(error = %e, "Could not reload critical alerts on token expiry setting"); + } + } "workspace_telemetry_enabled" => { // Read the new value from the database and log it let enabled = sqlx::query_scalar!( @@ -1794,27 +1870,27 @@ pub async fn run_workers( let mut handles = Vec::with_capacity(num_workers as usize); for x in [ - TMP_LOGS_DIR, - UV_CACHE_DIR, - DENO_CACHE_DIR, - DENO_CACHE_DIR_DEPS, - DENO_CACHE_DIR_NPM, - BUN_CACHE_DIR, - PY310_CACHE_DIR, - PY311_CACHE_DIR, - PY312_CACHE_DIR, - PY313_CACHE_DIR, - BUN_BUNDLE_CACHE_DIR, - GO_CACHE_DIR, - GO_BIN_CACHE_DIR, - RUST_CACHE_DIR, - CSHARP_CACHE_DIR, - NU_CACHE_DIR, - HUB_CACHE_DIR, - POWERSHELL_CACHE_DIR, - JAVA_CACHE_DIR, - RUBY_CACHE_DIR, - TAR_JAVA_CACHE_DIR, // for related places search: ADD_NEW_LANG + &*TMP_LOGS_DIR, + &*UV_CACHE_DIR, + &*DENO_CACHE_DIR, + &*DENO_CACHE_DIR_DEPS, + &*DENO_CACHE_DIR_NPM, + &*BUN_CACHE_DIR, + &*PY310_CACHE_DIR, + &*PY311_CACHE_DIR, + &*PY312_CACHE_DIR, + &*PY313_CACHE_DIR, + &*BUN_BUNDLE_CACHE_DIR, + &*GO_CACHE_DIR, + &*GO_BIN_CACHE_DIR, + &*RUST_CACHE_DIR, + &*CSHARP_CACHE_DIR, + &*NU_CACHE_DIR, + &*HUB_CACHE_DIR, + &*POWERSHELL_CACHE_DIR, + &*JAVA_CACHE_DIR, + &*RUBY_CACHE_DIR, + &*TAR_JAVA_CACHE_DIR, // for related places search: ADD_NEW_LANG ] { DirBuilder::new() .recursive(true) @@ -1824,7 +1900,7 @@ pub async fn run_workers( tracing::info!( "Starting {num_workers} workers and SLEEP_QUEUE={}ms", - *windmill_worker::SLEEP_QUEUE + windmill_worker::sleep_queue() ); for i in 1..(num_workers + 1) { diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 895fbd3117..156f93b7c1 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -44,10 +44,12 @@ use windmill_common::{ apps::APP_WORKSPACED_ROUTE, auth::create_token_for_owner, ee_oss::CriticalErrorChannel, + email_oss::send_email_if_possible, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ - BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, + AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, + CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, @@ -73,13 +75,14 @@ use windmill_common::{ load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env, load_worker_config, reload_custom_tags_setting, store_pull_query, store_suspended_pull_query, Connection, WorkerConfig, DEFAULT_TAGS_PER_WORKSPACE, - DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR, + DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, WORKER_GROUP, }, - KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERT_MUTE_UI_ENABLED, - CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, - METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, - OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, + KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, + DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, + MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, + SERVICE_LOG_RETENTION_SECS, }; use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING}; #[cfg(feature = "parquet")] @@ -207,6 +210,10 @@ pub async fn initial_load( tracing::error!("Error loading critical alert mute ui setting: {e:#}"); } + if let Err(e) = reload_critical_alerts_on_token_expiry_setting(conn).await { + tracing::error!("Error loading critical alerts on token expiry setting: {e:#}"); + } + if let Some(db) = conn.as_sql() { if let Err(e) = load_tag_per_workspace_enabled(db).await { tracing::error!("Error loading default tag per workpsace: {e:#}"); @@ -244,9 +251,8 @@ pub async fn initial_load( .map(|x| x.tags.clone()) .unwrap_or_default(); // we only check from env as native_mode is not stored in the token + // NATIVE_MODE_RESOLVED is already set in main.rs during startup let native_mode = windmill_common::worker::is_native_mode_from_env(); - windmill_common::worker::NATIVE_MODE_RESOLVED - .store(native_mode, std::sync::atomic::Ordering::Relaxed); *config = WorkerConfig { worker_tags, env_vars: load_env_vars( @@ -317,9 +323,15 @@ pub async fn initial_load( if server_mode { reload_retention_period_setting(&conn).await; + reload_audit_log_retention_days_setting(&conn).await; reload_request_size(&conn).await; reload_saml_metadata_setting(&conn).await; reload_scim_token_setting(&conn).await; + + // Ensure audit partitions exist before any requests arrive + if let Some(db) = conn.as_sql() { + manage_audit_partitions(&db, audit_log_retention_days().await).await; + } } if worker_mode { @@ -477,6 +489,21 @@ pub async fn reload_critical_alert_mute_ui_setting(conn: &Connection) -> error:: Ok(()) } +pub async fn reload_critical_alerts_on_token_expiry_setting( + conn: &Connection, +) -> error::Result<()> { + if let Ok(Some(serde_json::Value::Bool(t))) = load_value_from_global_settings_with_conn( + conn, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, + true, + ) + .await + { + CRITICAL_ALERTS_ON_TOKEN_EXPIRY.store(t, Ordering::Relaxed); + } + Ok(()) +} + pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> { let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await; @@ -595,7 +622,7 @@ async fn sleep_until_next_minute_start_plus_one_s() { use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE; async fn find_two_highest_files(hostname: &str) -> (Option, Option) { - let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname); + let log_dir = format!("{}/{}/", *TMP_WINDMILL_LOGS_SERVICE, hostname); let rd_dir = tokio::fs::read_dir(log_dir).await; if let Ok(mut log_files) = rd_dir { let mut highest_file: Option = None; @@ -614,7 +641,8 @@ async fn find_two_highest_files(hostname: &str) -> (Option, Option, + label: Option, + email: Option, + workspace_id: Option, +} + +/// When updating this filter, also update: +/// - `register_token_expiry_notification` in windmill-api-auth/src/lib.rs +/// - `isUserToken` in frontend/src/lib/components/settings/TokensTable.svelte +fn is_user_token(label: Option<&str>) -> bool { + match label { + None => true, + Some(l) => { + l != "session" + && !l.starts_with("ephemeral") + && !l.starts_with("Ephemeral") + && l != "debugger-token" + && !l.starts_with("mcp-oauth-") + } + } +} + +async fn report_token_expiration(db: &DB, token: &TokenRow, expired: bool) { + if !is_user_token(token.label.as_deref()) { + return; + } + let prefix = token.token_prefix.as_deref().unwrap_or("??????????"); + let email_addr = token.email.as_deref().unwrap_or("unknown"); + let token_desc = match token.label.as_deref() { + Some(l) if !l.is_empty() => format!("'{l}' ({prefix}****)"), + _ => format!("{prefix}****"), + }; + + let (alert_message, email_subject, email_body) = if expired { + ( + format!( + "API token {token_desc} of '{email_addr}' has expired and been deleted" + ), + "Windmill: Your API token has expired", + format!( + "Your API token {token_desc} has expired and been deleted.\n\nPlease create a new token if you still need API access." + ), + ) + } else { + ( + format!("API token {token_desc} of '{email_addr}' is expiring soon"), + "Windmill: Your API token is expiring soon", + format!( + "Your API token {token_desc} is expiring soon.\n\nPlease rotate or renew your token to avoid service disruption." + ), + ) + }; + + tracing::info!("{}", alert_message); + if CRITICAL_ALERTS_ON_TOKEN_EXPIRY.load(Ordering::Relaxed) { + report_critical_error( + alert_message, + db.clone(), + token.workspace_id.as_deref(), + None, + ) + .await; + } + if let Some(email) = &token.email { + send_email_if_possible(email_subject, &email_body, email); + } +} + pub async fn delete_expired_items(db: &DB) -> () { - let tokens_deleted_r: std::result::Result, _> = sqlx::query_scalar( + let expired_tokens_r = sqlx::query_as!( + TokenRow, "DELETE FROM token WHERE expiration <= now() RETURNING concat(token_prefix, '*****')", ) .fetch_all(db) .await; - match tokens_deleted_r { + match expired_tokens_r { Ok(tokens) => { - if tokens.len() > 0 { - tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens) + if !tokens.is_empty() { + tracing::info!("deleted {} expired tokens", tokens.len()); + for t in &tokens { + report_token_expiration(db, t, true).await; + } } } Err(e) => tracing::error!("Error deleting token: {}", e.to_string()), @@ -935,18 +1036,16 @@ pub async fn delete_expired_items(db: &DB) -> () { .iter() .map(|f| format!("{}/{}", f.hostname, f.file_path)) .collect(); - delete_log_files_from_disk_and_store(paths, TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await; + delete_log_files_from_disk_and_store(paths, &*TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await; } Err(e) => tracing::error!("Error deleting log file: {:?}", e), } - #[cfg(not(feature = "enterprise"))] - let audit_retention_secs = 1 * 60 * 60 * 24 * 14; - - #[cfg(feature = "enterprise")] - let audit_retention_secs = 1 * 60 * 60 * 24 * 365; + let audit_retention_days = audit_log_retention_days().await; + let audit_retention_secs: i64 = audit_retention_days * 60 * 60 * 24; + // Clean up old (non-partitioned) audit table — will eventually be empty and dropped if let Err(e) = sqlx::query_scalar!( "DELETE FROM audit WHERE timestamp <= now() - ($1::bigint::text || ' s')::interval", audit_retention_secs, @@ -954,7 +1053,7 @@ pub async fn delete_expired_items(db: &DB) -> () { .fetch_all(db) .await { - tracing::error!("Error deleting audit log on CE: {:?}", e); + tracing::error!("Error deleting audit log: {:?}", e); } if let Err(e) = sqlx::query_scalar!( @@ -1064,6 +1163,41 @@ pub async fn delete_expired_items(db: &DB) -> () { } } +pub async fn check_expiring_tokens(db: &DB) { + // Find tokens expiring within 7 days that still have a pending notification row + let expiring_tokens_r = sqlx::query_as!( + TokenRow, + "DELETE FROM token_expiry_notification n + USING token t + WHERE n.token = t.token + AND n.expiration > now() + AND n.expiration <= now() + interval '7 days' + RETURNING substring(t.token for 10) as token_prefix, t.label, t.email, t.workspace_id", + ) + .fetch_all(db) + .await; + + match expiring_tokens_r { + Ok(tokens) => { + for t in &tokens { + report_token_expiration(db, t, false).await; + } + if !tokens.is_empty() { + tracing::info!("Sent expiration warnings for {} token(s)", tokens.len()); + } + } + Err(e) => tracing::error!("Error checking expiring tokens: {}", e), + } + + // Clean up notification rows whose expiration has passed + if let Err(e) = sqlx::query!("DELETE FROM token_expiry_notification WHERE expiration <= now()") + .execute(db) + .await + { + tracing::error!("Error cleaning up expired token notifications: {}", e); + } +} + /// Delete a batch of expired jobs with LIMIT and SKIP LOCKED for high-scale environments. /// Uses a single transaction per batch to minimize lock duration. /// Returns the number of jobs deleted in this batch. @@ -1140,7 +1274,7 @@ async fn delete_expired_jobs_batch( .filter_map(|opt| opt) .flat_map(|inner_vec| inner_vec.into_iter()) .collect(); - delete_log_files_from_disk_and_store(paths, TMP_DIR, "").await; + delete_log_files_from_disk_and_store(paths, &*WINDMILL_DIR, "").await; } Err(e) => tracing::error!("Error deleting job logs: {:?}", e), } @@ -1367,7 +1501,7 @@ pub async fn reload_maven_settings_xml_setting(conn: &Connection) { let settings_xml = MAVEN_SETTINGS_XML.read().await.clone(); match settings_xml { Some(ref content) if !content.trim().is_empty() => { - let m2_dir = format!("{JAVA_HOME_DIR}/.m2"); + let m2_dir = format!("{}/.m2", *JAVA_HOME_DIR); if let Err(e) = tokio::fs::create_dir_all(&m2_dir).await { tracing::error!("Failed to create .m2 directory: {e:#}"); return; @@ -1378,7 +1512,7 @@ pub async fn reload_maven_settings_xml_setting(conn: &Connection) { } } _ => { - let settings_path = format!("{JAVA_HOME_DIR}/.m2/settings.xml"); + let settings_path = format!("{}/.m2/settings.xml", *JAVA_HOME_DIR); let _ = tokio::fs::remove_file(&settings_path).await; } } @@ -1444,6 +1578,22 @@ pub async fn reload_retention_period_setting(conn: &Connection) { tracing::error!("Error reloading retention period: {:?}", e) } } + +pub async fn reload_audit_log_retention_days_setting(conn: &Connection) { + if let Err(e) = reload_setting( + conn, + AUDIT_LOG_RETENTION_DAYS_SETTING, + "AUDIT_LOG_RETENTION_DAYS", + 0, // 0 means use default: 365 for EE, 14 for CE + AUDIT_LOG_RETENTION_DAYS.clone(), + |x| x, + ) + .await + { + tracing::error!("Error reloading audit log retention days: {:?}", e) + } +} + pub async fn reload_delete_logs_periodically_setting(conn: &Connection) { if let Err(e) = reload_setting( conn, @@ -2051,6 +2201,25 @@ pub async fn monitor_db( } }; + // Run every hour (10 iterations * 30s = 5 minutes) + // Check for tokens expiring within 7 days and send alerts + let check_expiring_tokens_f = async { + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(10) { + if let Some(db) = conn.as_sql() { + check_expiring_tokens(&db).await; + } + } + }; + + // run every hour (120 iterations * 30s = 3600s) + let manage_audit_partitions_f = async { + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(120) { + if let Some(db) = conn.as_sql() { + manage_audit_partitions(&db, audit_log_retention_days().await).await; + } + } + }; + join!( expired_items_f, zombie_jobs_f, @@ -2072,6 +2241,8 @@ pub async fn monitor_db( cleanup_worker_group_stats_f, native_triggers_sync_f, cleanup_notify_events_f, + check_expiring_tokens_f, + manage_audit_partitions_f, ); } @@ -2693,7 +2864,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n &windmill_queue::MiniCompletedJob::from(job), memory_peak, None, - error::Error::ExecutionErr(error_message), + error::Error::ExecutionErr(error_message.clone()), matches!(error_kind, ErrorMessage::SameWorker), // unrecoverable if the job is a same worker zombie Some(&same_worker_tx_never_used), "", @@ -2704,10 +2875,74 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n &mut windmill_common::bench::BenchmarkIter::new(), ) .await; + + // If handle_job_error failed (e.g. schedule push failure rolled back the tx), + // the job is still in the queue. Force-complete it to prevent infinite zombie loops. + if let Err(e) = force_complete_zombie_job(db, &job_id, &error_message).await { + tracing::error!("Failed to force-complete zombie job {}: {e:#}", job_id); + } } } } +/// Force-complete a zombie job that handle_job_error failed to complete. +/// This is a minimal fallback: it inserts a failed completed job and deletes +/// from the queue in a single transaction, without schedule pushing or +/// error handler logic that could cause the completion to fail. +async fn force_complete_zombie_job( + db: &Pool, + job_id: &Uuid, + error_message: &str, +) -> error::Result<()> { + let still_queued = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1)", + job_id + ) + .fetch_one(db) + .await? + .unwrap_or(false); + + if !still_queued { + return Ok(()); + } + + tracing::error!( + "Zombie job {job_id} was not completed by handle_job_error, force-completing it" + ); + + let error_value = serde_json::json!({ + "message": error_message, + "name": "ExecutionErr", + }); + + let mut tx = db.begin().await?; + + sqlx::query!( + "INSERT INTO v2_job_completed + (workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker) + SELECT q.workspace_id, q.id, q.started_at, + COALESCE((EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000, 0)::bigint, + $2::jsonb, r.memory_peak, 'failure'::job_status, q.worker + FROM v2_job_queue q + LEFT JOIN v2_job_runtime r ON r.id = q.id + WHERE q.id = $1 + ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb", + job_id, + error_value, + ) + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM v2_job_queue WHERE id = $1", job_id) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + tracing::info!("Force-completed zombie job {job_id}"); + Ok(()) +} + async fn cleanup_concurrency_counters_orphaned_keys(db: &DB) -> error::Result<()> { let result = sqlx::query!( " @@ -3236,3 +3471,72 @@ RETURNING job_id } Ok(()) } + +async fn audit_log_retention_days() -> i64 { + let v = *AUDIT_LOG_RETENTION_DAYS.read().await; + if v > 0 { + v + } else if cfg!(feature = "enterprise") { + 365 + } else { + 14 + } +} + +async fn manage_audit_partitions(db: &DB, retention_days: i64) { + let today = chrono::Utc::now().date_naive(); + + // Create partitions for today and the next 3 days + for days_ahead in 0..=3i64 { + let date = today + chrono::Duration::days(days_ahead); + let next_date = date + chrono::Duration::days(1); + let partition_name = format!("audit_{}", date.format("%Y%m%d")); + let quoted_name = format!("\"{}\"", partition_name.replace('"', "\"\"")); + let sql = format!( + "CREATE TABLE IF NOT EXISTS {quoted_name} PARTITION OF audit_partitioned \ + FOR VALUES FROM ('{date}') TO ('{next_date}')" + ); + if let Err(e) = sqlx::query(&sql).execute(db).await { + if !e.to_string().contains("already exists") { + tracing::error!("Error creating audit partition {partition_name}: {e:?}"); + } + } + } + + // Drop expired partitions + let cutoff_date = today - chrono::Duration::days(retention_days); + + let partitions = sqlx::query_scalar::<_, String>( + "SELECT c.relname::text \ + FROM pg_inherits i \ + JOIN pg_class c ON c.oid = i.inhrelid \ + WHERE i.inhparent = 'audit_partitioned'::regclass", + ) + .fetch_all(db) + .await; + + match partitions { + Ok(partitions) => { + for partition_name in partitions { + if let Some(date_str) = partition_name.strip_prefix("audit_") { + if let Ok(date) = chrono::NaiveDate::parse_from_str(date_str, "%Y%m%d") { + if date < cutoff_date { + let quoted_name = + format!("\"{}\"", partition_name.replace('"', "\"\"")); + let sql = format!("DROP TABLE IF EXISTS {quoted_name}"); + match sqlx::query(&sql).execute(db).await { + Ok(_) => tracing::info!( + "Dropped expired audit partition {partition_name}" + ), + Err(e) => tracing::error!( + "Error dropping audit partition {partition_name}: {e:?}" + ), + } + } + } + } + } + } + Err(e) => tracing::error!("Error listing audit partitions: {e:?}"), + } +} diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 01db7590f5..3ece1ce7ef 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -151,6 +151,8 @@ sqs_trigger: path(char), queue_url(char), aws_resource_path(char), message_attri FK: (workspace_id) -> workspace(id) token: token(char), label(char), expiration(ts), workspace_id(char), owner(char), email(char), super_admin(bool), created_at(ts), last_used_at(ts), scopes(text[]), job(uuid) FK: (workspace_id) -> workspace(id) +token_expiry_notification: token(char), expiration(ts) + INDEX: idx_token_expiry_notification_expiration (expiration) tutorial_progress: email(char), progress(bit64), skipped_all(bool) unique_ext_jwt_token: jwt_hash(bigint), last_used_at(ts) usage: id(char), is_workspace(bool), month_(int), usage(int) diff --git a/backend/test_wac_e2e.sh b/backend/test_wac_e2e.sh new file mode 100755 index 0000000000..4379ffde42 --- /dev/null +++ b/backend/test_wac_e2e.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# E2E test for WAC v2 workflow-as-code suspend/resume lifecycle +set -euo pipefail + +BASE_URL="${BASE_URL:-http://localhost:8070}" +TOKEN="${WM_TOKEN:-}" +WORKSPACE="dev" +TIMEOUT=60 # seconds + +# Get auth token if not set +if [ -z "$TOKEN" ]; then + TOKEN=$(curl -s "${BASE_URL}/api/auth/login" \ + -H "Content-Type: application/json" \ + -d '{"email":"admin@windmill.dev","password":"changeme"}' | tr -d '"') +fi + +echo "=== WAC v2 E2E Test ===" +echo "Base URL: $BASE_URL" +echo "" + +WAC_CODE='import { task, workflow } from "windmill-client@1.999.19"; + +const double = task(async (x: number): Promise => { + console.log("[double] START at " + new Date().toISOString()); + await new Promise(r => setTimeout(r, 2000)); + console.log("[double] END at " + new Date().toISOString()); + return x * 2; +}); + +const increment = task(async (x: number): Promise => { + console.log("[increment] START at " + new Date().toISOString()); + await new Promise(r => setTimeout(r, 2000)); + console.log("[increment] END at " + new Date().toISOString()); + return x + 1; +}); + +export const main = workflow(async (x: number = 10) => { + const [doubled, incremented] = await Promise.all([ + double(x), + increment(x), + ]); + const final_result = await double(incremented); + return { doubled, incremented, final_result }; +});' + +echo "Step 1: Submitting preview job..." +JOB_ID=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs/run/preview" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOKEN" \ + -d "$(jq -n --arg code "$WAC_CODE" '{ + content: $code, + language: "bun", + args: {"x": 10} + }')" | tr -d '"') + +echo "Job ID: $JOB_ID" + +if [ -z "$JOB_ID" ] || [ "$JOB_ID" = "null" ]; then + echo "FAIL: Could not create job" + exit 1 +fi + +echo "" +echo "Step 2: Polling for completion (timeout: ${TIMEOUT}s)..." + +START=$SECONDS +LAST_STATUS="" +while true; do + ELAPSED=$((SECONDS - START)) + if [ $ELAPSED -gt $TIMEOUT ]; then + echo "FAIL: Timed out after ${TIMEOUT}s" + # Dump job state for debugging + echo "" + echo "=== Debug info ===" + echo "Parent job queue state:" + source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local + psql "$DATABASE_URL" -c "SELECT id, running, suspend, suspend_until, canceled_by FROM v2_job_queue WHERE id = '$JOB_ID'::uuid" 2>/dev/null + echo "Child jobs:" + psql "$DATABASE_URL" -c "SELECT id, running, suspend, created_at FROM v2_job_queue WHERE parent_job = '$JOB_ID'::uuid ORDER BY created_at" 2>/dev/null + echo "Completed children:" + psql "$DATABASE_URL" -c "SELECT id FROM completed_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null + echo "Checkpoint:" + psql "$DATABASE_URL" -c "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = '$JOB_ID'::uuid" 2>/dev/null + echo "Total child count:" + psql "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null + exit 1 + fi + + # Check completed job + RESULT=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/completed/get_result/${JOB_ID}" \ + -H "Authorization: Bearer $TOKEN" 2>/dev/null) + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/completed/get_result/${JOB_ID}" \ + -H "Authorization: Bearer $TOKEN" 2>/dev/null) + + if [ "$HTTP_CODE" = "200" ]; then + echo "Job completed in ${ELAPSED}s!" + echo "" + echo "Step 3: Checking result..." + echo "Result: $RESULT" + + # Validate + DOUBLED=$(echo "$RESULT" | jq -r '.doubled // empty') + INCREMENTED=$(echo "$RESULT" | jq -r '.incremented // empty') + FINAL=$(echo "$RESULT" | jq -r '.final_result // empty') + + PASS=true + if [ "$DOUBLED" != "20" ]; then + echo "FAIL: doubled = $DOUBLED, expected 20" + PASS=false + fi + if [ "$INCREMENTED" != "11" ]; then + echo "FAIL: incremented = $INCREMENTED, expected 11" + PASS=false + fi + if [ "$FINAL" != "22" ]; then + echo "FAIL: final_result = $FINAL, expected 22" + PASS=false + fi + + if $PASS; then + echo "PASS: All values correct!" + # Check no excessive child jobs + source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local 2>/dev/null + CHILD_COUNT=$(psql -tA "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null) + echo "Total child jobs created: $CHILD_COUNT (expected: 3)" + if [ "$CHILD_COUNT" -gt "3" ]; then + echo "WARN: More children than expected ($CHILD_COUNT > 3)" + fi + exit 0 + else + exit 1 + fi + fi + + # Show progress + STATUS=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/get/${JOB_ID}" \ + -H "Authorization: Bearer $TOKEN" 2>/dev/null | jq -r '.type // empty') + if [ "$STATUS" != "$LAST_STATUS" ]; then + echo " [${ELAPSED}s] Status: $STATUS" + LAST_STATUS="$STATUS" + fi + + # Check for runaway child creation + source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local 2>/dev/null + CHILD_COUNT=$(psql -tA "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null) + if [ "$CHILD_COUNT" -gt "10" ]; then + echo "FAIL: Runaway child creation detected! $CHILD_COUNT children (expected 3)" + echo "" + echo "=== Debug info ===" + psql "$DATABASE_URL" -c "SELECT id, running, suspend, suspend_until FROM v2_job_queue WHERE id = '$JOB_ID'::uuid" 2>/dev/null + psql "$DATABASE_URL" -c "SELECT id, running, suspend, created_at FROM v2_job_queue WHERE parent_job = '$JOB_ID'::uuid ORDER BY created_at LIMIT 20" 2>/dev/null + psql "$DATABASE_URL" -c "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = '$JOB_ID'::uuid" 2>/dev/null + exit 1 + fi + + sleep 1 +done diff --git a/backend/tests/agent_workers.rs b/backend/tests/agent_workers.rs index b6414dc24e..4f12d5f42c 100644 --- a/backend/tests/agent_workers.rs +++ b/backend/tests/agent_workers.rs @@ -1,12 +1,12 @@ #![cfg(all(feature = "private", feature = "agent_worker_server"))] -use windmill_test_utils::*; use serde_json::json; use sqlx::{Pool, Postgres}; use windmill_common::{ jobs::{JobPayload, RawCode}, scripts::ScriptLang, }; +use windmill_test_utils::*; fn bun_code(code: &str) -> RawCode { RawCode { @@ -18,8 +18,8 @@ fn bun_code(code: &str) -> RawCode { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - concurrency_settings: - windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), } } @@ -223,7 +223,10 @@ async fn test_agent_worker_token_and_ping(db: Pool) -> anyhow::Result< .fetch_one(&db) .await?; - assert!(worker_count > 0, "worker ping should be recorded in database"); + assert!( + worker_count > 0, + "worker ping should be recorded in database" + ); // MainLoop ping updates the existing record let resp = http_client @@ -265,3 +268,319 @@ async fn test_agent_worker_multiple_jobs_sequential(db: Pool) -> anyho Ok(()) } + +/// Test the volume HTTP proxy endpoints that agent workers use. +/// +/// Exercises the full volume lifecycle via HTTP: +/// 1. Configure workspace S3 storage (FilesystemStorage) +/// 2. Pre-populate a volume with a file +/// 3. POST /begin — acquire lease, get manifest +/// 4. GET /file/* — download existing file +/// 5. PUT /file/* — upload a new file +/// 6. POST /commit — finalize with stats, release lease +/// 7. Verify DB state and storage +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> { + let (client, _port, _server) = init_client_agent_mode(db.clone()).await; + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file + let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + let base = client.baseurl(); + let http = client.client(); + let vol_base = format!("{base}/w/test-workspace/volumes/test-vol"); + + // 3. POST /begin — acquire lease, get manifest + permissions + let resp = http + .post(format!("{vol_base}/begin")) + .json(&json!({ + "worker_name": "test-worker-1", + "permissioned_as": "u/test-user" + })) + .send() + .await?; + assert!( + resp.status().is_success(), + "begin should succeed, got: {}", + resp.status() + ); + + let begin_body: serde_json::Value = resp.json().await?; + assert!( + begin_body["writable"].as_bool().unwrap(), + "should be writable" + ); + let manifest = begin_body["manifest"].as_object().unwrap(); + assert!( + manifest.contains_key("hello.txt"), + "manifest should contain hello.txt, got: {manifest:?}" + ); + + // 4. GET /file/* — download the existing file + let resp = http + .get(format!("{vol_base}/file/hello.txt")) + .send() + .await?; + assert!( + resp.status().is_success(), + "file download should succeed, got: {}", + resp.status() + ); + let file_bytes = resp.bytes().await?; + assert_eq!( + file_bytes.as_ref(), + b"hello from volume", + "downloaded file content should match" + ); + + // 5. PUT /file/* — upload a new file + let resp = http + .put(format!("{vol_base}/file/output.txt")) + .body(b"written by agent worker".to_vec()) + .send() + .await?; + assert!( + resp.status().is_success(), + "file upload should succeed, got: {}", + resp.status() + ); + + // 6. POST /commit — finalize: report stats, release lease + let resp = http + .post(format!("{vol_base}/commit")) + .json(&json!({ + "worker_name": "test-worker-1", + "deleted_keys": [], + "symlinks": {}, + "file_count": 2, + "size_bytes": 39 + })) + .send() + .await?; + assert!( + resp.status().is_success(), + "commit should succeed, got: {}", + resp.status() + ); + + // 7. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert_eq!(vol_row.file_count, 2, "file_count should be 2"); + assert_eq!(vol_row.size_bytes, 39, "size_bytes should match"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + assert!( + vol_row.lease_until.is_none() || vol_row.lease_until.unwrap() < chrono::Utc::now(), + "lease_until should be cleared or in the past" + ); + + // 8. Verify the uploaded file was persisted in storage + let output_path = vol_dir.join("output.txt"); + assert!(output_path.exists(), "output.txt should be in storage"); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by agent worker"); + + Ok(()) +} + +/// Full E2E test: agent worker in HTTP mode runs a Bun script with a volume mount. +/// +/// The worker pulls the job via HTTP, downloads volume files via the server-side +/// volume proxy endpoints, executes the script, and syncs changes back. +#[cfg(all(feature = "parquet", feature = "enterprise"))] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_http_worker_e2e(db: Pool) -> anyhow::Result<()> { + let (_client, port, _server) = init_client_agent_mode(db.clone()).await; + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file + let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + // 3. Push the job, then run worker with HTTP connection (bun tag) + let code = r#"// volume: test-vol /tmp/data +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("/tmp/data/hello.txt", "utf-8"); + writeFileSync("/tmp/data/output.txt", "written by agent worker"); + return { + read_content: content, + output_exists: existsSync("/tmp/data/output.txt"), + }; +}"#; + + let uuid = RunJob::from(JobPayload::Code(bun_code(code))) + .push(&db) + .await; + let listener = listen_for_completed_jobs(&db).await; + + let conn = testing_http_connection_with_tags( + port, + vec!["bun".into(), "flow".into(), "dependency".into()], + ) + .await; + + in_test_worker(conn, listener.find(&uuid), port).await; + + let result = completed_job(uuid, &db).await; + + assert!(result.success, "job should succeed: {:?}", result.result); + let json = result.json_result().expect("should have JSON result"); + assert_eq!(json["read_content"], json!("hello from volume")); + assert_eq!(json["output_exists"], json!(true)); + + // 4. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert!( + vol_row.file_count >= 2, + "should have at least 2 files (hello.txt + output.txt), got: {}", + vol_row.file_count + ); + assert!(vol_row.size_bytes > 0, "size_bytes should be > 0"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + assert!( + vol_row.lease_until.is_none() || vol_row.lease_until.unwrap() < chrono::Utc::now(), + "lease_until should be cleared or in the past" + ); + + // 5. Verify the new file was written back to the storage + let output_path = vol_dir.join("output.txt"); + assert!( + output_path.exists(), + "output.txt should be synced back to storage" + ); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by agent worker"); + + Ok(()) +} + +/// Test the volume release endpoint (error/cancel path). +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_release(db: Pool) -> anyhow::Result<()> { + let (client, _port, _server) = init_client_agent_mode(db.clone()).await; + + // Set up filesystem storage + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + let base = client.baseurl(); + let http = client.client(); + let vol_base = format!("{base}/w/test-workspace/volumes/test-vol"); + + // Begin (acquire lease) + let resp = http + .post(format!("{vol_base}/begin")) + .json(&json!({ + "worker_name": "test-worker-2", + "permissioned_as": "u/test-user" + })) + .send() + .await?; + assert!(resp.status().is_success(), "begin should succeed"); + + // Verify lease is held + let leased = sqlx::query_scalar!( + "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await? + .flatten(); + assert_eq!(leased.as_deref(), Some("test-worker-2")); + + // Release without commit (simulating error path) + let resp = http + .post(format!("{vol_base}/release")) + .json(&json!({ "worker_name": "test-worker-2" })) + .send() + .await?; + assert!(resp.status().is_success(), "release should succeed"); + + // Verify lease is cleared + let leased = sqlx::query_scalar!( + "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await? + .flatten(); + assert!(leased.is_none(), "lease should be released"); + + Ok(()) +} diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index 15a2b3c27c..c69300028d 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -1,5 +1,6 @@ use sqlx::postgres::Postgres; use sqlx::Pool; +use uuid::Uuid; use windmill_common::jobs::{JobPayload, RawCode}; use windmill_common::scripts::ScriptLang; use windmill_test_utils::*; @@ -887,7 +888,7 @@ mod dedicated_worker_protocol { if bundle_for_node { // For Node.js: bundle to JavaScript first (like production's build_loader with LoaderMode::Node) - let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.js", None); + let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.js", None, None); std::fs::write(dir.join("wrapper.mjs"), wrapper).unwrap(); // Use the exact same build_loader function as production @@ -924,7 +925,7 @@ mod dedicated_worker_protocol { output_path } else { // For Bun: use TypeScript directly (like production) - let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.ts", None); + let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.ts", None, None); let wrapper_path = dir.join("wrapper.mjs"); std::fs::write(&wrapper_path, wrapper).unwrap(); wrapper_path @@ -1448,3 +1449,240 @@ export function main() { return { a, b }; } ); } } + +// ============================================================================ +// Codebase Mode Tests +// ============================================================================ + +/// Create a TAR archive in memory containing a single `main.js` file. +fn create_codebase_tar(main_js_content: &str) -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + let content = main_js_content.as_bytes(); + let mut header = tar::Header::new_gnu(); + header.set_path("main.js").unwrap(); + header.set_size(content.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append(&header, content).unwrap(); + builder.into_inner().unwrap() +} + +/// Place a TAR codebase at the expected cache path for the given job ID and hash. +fn place_codebase_in_cache(job_id: &Uuid, tar_bytes: &[u8], is_esm: bool) { + let codebase_id = if is_esm { + format!("{}.esm.tar", job_id) + } else { + format!("{}.tar", job_id) + }; + let bundle_path = format!("script_bundle/test-workspace/{}", codebase_id); + let cache_path = format!( + "{}/{}.tar", + *windmill_common::worker::ROOT_CACHE_NOMOUNT_DIR, + bundle_path, + ); + let parent = std::path::Path::new(&cache_path).parent().unwrap(); + std::fs::create_dir_all(parent).unwrap(); + std::fs::write(&cache_path, tar_bytes).unwrap(); +} + +#[sqlx::test(fixtures("base"))] +async fn test_cjs_codebase_tar(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let main_js = r#" +module.exports.main = function() { + return "cjs codebase ok"; +}; +"#; + let inner_content = r#"export function main() { return "cjs codebase ok"; }"#; + + let job_id = Uuid::new_v4(); + let tar_bytes = create_codebase_tar(main_js); + place_codebase_in_cache(&job_id, &tar_bytes, false); + + let job = JobPayload::Code(RawCode { + hash: Some(-43), // PREVIEW_IS_TAR_CODEBASE_HASH + content: inner_content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: Default::default(), + debouncing_settings: Default::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = RunJob::from(job) + .job_id(job_id) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("cjs codebase ok")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_esm_codebase_tar(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let main_js = r#" +export function main() { + return "esm codebase ok"; +} +"#; + let inner_content = r#"export function main() { return "esm codebase ok"; }"#; + + let job_id = Uuid::new_v4(); + let tar_bytes = create_codebase_tar(main_js); + place_codebase_in_cache(&job_id, &tar_bytes, true); + + let job = JobPayload::Code(RawCode { + hash: Some(-45), // PREVIEW_IS_TAR_ESM_CODEBASE_HASH + content: inner_content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: Default::default(), + debouncing_settings: Default::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = RunJob::from(job) + .job_id(job_id) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("esm codebase ok")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_cjs_codebase_tar_nsjail(db: Pool) -> anyhow::Result<()> { + if std::process::Command::new("nsjail") + .arg("--help") + .output() + .is_err() + { + eprintln!("nsjail not found, skipping test"); + return Ok(()); + } + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let main_js = r#" +module.exports.main = function() { + return "cjs nsjail ok"; +}; +"#; + let inner_content = r#"export function main() { return "cjs nsjail ok"; }"#; + + let job_id = Uuid::new_v4(); + let tar_bytes = create_codebase_tar(main_js); + place_codebase_in_cache(&job_id, &tar_bytes, false); + + let job = JobPayload::Code(RawCode { + hash: Some(-43), + content: inner_content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: Default::default(), + debouncing_settings: Default::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + use std::sync::atomic::Ordering; + windmill_worker::JOB_ISOLATION.store( + windmill_worker::JobIsolationLevel::NsjailSandboxing as u8, + Ordering::Relaxed, + ); + + let result = RunJob::from(job) + .job_id(job_id) + .run_until_complete(&db, false, port) + .await; + + windmill_worker::JOB_ISOLATION.store( + windmill_worker::JobIsolationLevel::Undefined as u8, + Ordering::Relaxed, + ); + + let json = result.json_result().unwrap(); + assert_eq!(json, serde_json::json!("cjs nsjail ok")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_esm_codebase_tar_nsjail(db: Pool) -> anyhow::Result<()> { + if std::process::Command::new("nsjail") + .arg("--help") + .output() + .is_err() + { + eprintln!("nsjail not found, skipping test"); + return Ok(()); + } + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let main_js = r#" +export function main() { + return "esm nsjail ok"; +} +"#; + let inner_content = r#"export function main() { return "esm nsjail ok"; }"#; + + let job_id = Uuid::new_v4(); + let tar_bytes = create_codebase_tar(main_js); + place_codebase_in_cache(&job_id, &tar_bytes, true); + + let job = JobPayload::Code(RawCode { + hash: Some(-45), + content: inner_content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: Default::default(), + debouncing_settings: Default::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + use std::sync::atomic::Ordering; + windmill_worker::JOB_ISOLATION.store( + windmill_worker::JobIsolationLevel::NsjailSandboxing as u8, + Ordering::Relaxed, + ); + + let result = RunJob::from(job) + .job_id(job_id) + .run_until_complete(&db, false, port) + .await; + + windmill_worker::JOB_ISOLATION.store( + windmill_worker::JobIsolationLevel::Undefined as u8, + Ordering::Relaxed, + ); + + let json = result.json_result().unwrap(); + assert_eq!(json, serde_json::json!("esm nsjail ok")); + Ok(()) +} diff --git a/backend/tests/end_user_email.rs b/backend/tests/end_user_email.rs new file mode 100644 index 0000000000..1a60f8672c --- /dev/null +++ b/backend/tests/end_user_email.rs @@ -0,0 +1,323 @@ +//! Tests for WM_END_USER_EMAIL environment variable. +//! +//! These tests verify that WM_END_USER_EMAIL is populated with the authenticated +//! user's email when executing app components. +//! +//! TODO: Add tests for scripts and flows once public execution endpoints are identified. +//! Currently only apps support non-workspace-member execution via OptAuthed + token lookup. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::worker::Connection; +use windmill_test_utils::*; + +const SAME_WS_TOKEN: &str = "SECRET_TOKEN"; +const OTHER_WS_TOKEN: &str = "OTHER_WS_TOKEN"; +const NO_WS_TOKEN: &str = "NO_WS_TOKEN"; + +const SAME_WS_EMAIL: &str = "test@windmill.dev"; +const OTHER_WS_EMAIL: &str = "other-ws@windmill.dev"; +const NO_WS_EMAIL: &str = "no-ws@windmill.dev"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +// TODO: Script tests - need to identify public execution endpoints for non-workspace-members +// async fn run_script(port: u16, token: &str) -> anyhow::Result { +// let url = format!( +// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/p/f/test/get_end_user_email", +// port +// ); +// let resp = authed(client().post(&url), token) +// .json(&json!({})) +// .send() +// .await?; +// if !resp.status().is_success() { +// anyhow::bail!("script run failed: {} - {}", resp.status(), resp.text().await?); +// } +// Ok(resp.json::().await? +// .as_str().unwrap_or("").to_string()) +// } + +// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members +// async fn run_flow(port: u16, token: &str) -> anyhow::Result { +// let url = format!( +// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/f/f/test/get_end_user_email_flow", +// port +// ); +// let resp = authed(client().post(&url), token) +// .json(&json!({})) +// .send() +// .await?; +// if !resp.status().is_success() { +// anyhow::bail!("flow run failed: {} - {}", resp.status(), resp.text().await?); +// } +// Ok(resp.json::().await? +// .as_str().unwrap_or("").to_string()) +// } + +/// Create an app with inline script via API +async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<()> { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps/create", + port + ); + let resp = authed(client().post(&url), SAME_WS_TOKEN) + .json(&json!({ + "path": path, + "summary": "Test app for WM_END_USER_EMAIL", + "value": { + "type": "app", + "grid": [], + "subgrids": {}, + "hiddenInlineScripts": [{ + "name": "get_email", + "language": "deno", + "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", + "path": "f/test/email_app/get_email" + }] + }, + "policy": { + "execution_mode": "anonymous", + "on_behalf_of": null, + "on_behalf_of_email": null, + "triggerables_v2": { + "get_email": { + "static_inputs": {}, + "one_of_inputs": {} + }, + // SHA256 hash of raw_code content for anonymous execution + "rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": { + "static_inputs": {}, + "one_of_inputs": {} + } + } + } + })) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!("create app failed: {} - {}", resp.status(), resp.text().await?); + } + Ok(()) +} + +/// Create a raw app with inline script via API (uses regular app endpoint with rawapp type) +async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<()> { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps/create", + port + ); + let resp = authed(client().post(&url), SAME_WS_TOKEN) + .json(&json!({ + "path": path, + "summary": "Test raw app for WM_END_USER_EMAIL", + "value": { + "type": "rawapp", + "css": "", + "inlineScripts": [{ + "name": "get_email", + "language": "deno", + "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }" + }] + }, + "policy": { + "execution_mode": "anonymous", + "on_behalf_of": null, + "on_behalf_of_email": null, + "triggerables_v2": { + "get_email": { + "static_inputs": {}, + "one_of_inputs": {} + }, + // SHA256 hash of raw_code content for anonymous execution + "rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": { + "static_inputs": {}, + "one_of_inputs": {} + } + } + } + })) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!("create raw app failed: {} - {}", resp.status(), resp.text().await?); + } + Ok(()) +} + +async fn run_app_inline_script(port: u16, token: &str, app_path: &str, force_viewer: bool) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps_u/execute_component/{}", + port, app_path + ); + let mut payload = json!({ + "args": {}, + "component": "get_email", + "raw_code": { + "language": "deno", + "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", + "path": format!("{}/get_email", app_path) + } + }); + if force_viewer { + payload["force_viewer_static_fields"] = json!({}); + } + let resp = authed(client().post(&url), token) + .json(&payload) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!("app inline script run failed: {} - {}", resp.status(), resp.text().await?); + } + let job_id = resp.text().await?; + wait_for_job_result(port, token, &job_id).await +} + +async fn run_raw_app_inline_script(port: u16, token: &str, app_path: &str, force_viewer: bool) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps_u/execute_component/{}", + port, app_path + ); + let mut payload = json!({ + "args": {}, + "component": "get_email", + "raw_code": { + "language": "deno", + "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }" + } + }); + if force_viewer { + payload["force_viewer_static_fields"] = json!({}); + } + let resp = authed(client().post(&url), token) + .json(&payload) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!("raw app inline script run failed: {} - {}", resp.status(), resp.text().await?); + } + let job_id = resp.text().await?; + wait_for_job_result(port, token, &job_id).await +} + +async fn wait_for_job_result(port: u16, token: &str, job_id: &str) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/jobs_u/completed/get_result/{}", + port, job_id + ); + for _ in 0..100 { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + let resp = authed(client().get(&url), token).send().await?; + if resp.status().is_success() { + return Ok(resp.json::().await? + .as_str().unwrap_or("").to_string()); + } + } + anyhow::bail!("timeout waiting for job result") +} + +// TODO: Script tests - need to identify public execution endpoints for non-workspace-members +// #[cfg(feature = "deno_core")] +// #[sqlx::test(fixtures("base", "end_user_email"))] +// async fn test_script_wm_end_user_email(db: Pool) -> anyhow::Result<()> { +// initialize_tracing().await; +// set_jwt_secret().await; +// let server = ApiServer::start(db.clone()).await?; +// let port = server.addr.port(); +// +// in_test_worker(Connection::Sql(db.clone()), async move { +// let result = run_script(port, SAME_WS_TOKEN).await?; +// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); +// Ok::<(), anyhow::Error>(()) +// }, port).await?; +// +// Ok(()) +// } + +// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members +// #[cfg(feature = "deno_core")] +// #[sqlx::test(fixtures("base", "end_user_email"))] +// async fn test_flow_wm_end_user_email(db: Pool) -> anyhow::Result<()> { +// initialize_tracing().await; +// set_jwt_secret().await; +// let server = ApiServer::start(db.clone()).await?; +// let port = server.addr.port(); +// +// in_test_worker(Connection::Sql(db.clone()), async move { +// let result = run_flow(port, SAME_WS_TOKEN).await?; +// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); +// Ok::<(), anyhow::Error>(()) +// }, port).await?; +// +// Ok(()) +// } + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_app_wm_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let app_path = "f/test/email_app"; + + in_test_worker(Connection::Sql(db.clone()), async move { + // Create the app with inline script first + create_app_with_inline_script(port, app_path).await?; + + // Same workspace user (force_viewer mode works for workspace members) + let result = run_app_inline_script(port, SAME_WS_TOKEN, app_path, true).await?; + assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); + + // Other workspace user (uses app's anonymous policy + token lookup) + let result = run_app_inline_script(port, OTHER_WS_TOKEN, app_path, false).await?; + assert_eq!(result, OTHER_WS_EMAIL, "other workspace user should get their email"); + + // No workspace user (uses app's anonymous policy + token lookup) + let result = run_app_inline_script(port, NO_WS_TOKEN, app_path, false).await?; + assert_eq!(result, NO_WS_EMAIL, "no workspace user should get their email"); + + Ok::<(), anyhow::Error>(()) + }, port).await?; + + Ok(()) +} + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_raw_app_wm_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let app_path = "f/test/email_raw_app"; + + in_test_worker(Connection::Sql(db.clone()), async move { + // Create the raw app with inline script first + create_raw_app_with_inline_script(port, app_path).await?; + + // Same workspace user (force_viewer mode works for workspace members) + let result = run_raw_app_inline_script(port, SAME_WS_TOKEN, app_path, true).await?; + assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); + + // Other workspace user (uses app's anonymous policy + token lookup) + let result = run_raw_app_inline_script(port, OTHER_WS_TOKEN, app_path, false).await?; + assert_eq!(result, OTHER_WS_EMAIL, "other workspace user should get their email"); + + // No workspace user (uses app's anonymous policy + token lookup) + let result = run_raw_app_inline_script(port, NO_WS_TOKEN, app_path, false).await?; + assert_eq!(result, NO_WS_EMAIL, "no workspace user should get their email"); + + Ok::<(), anyhow::Error>(()) + }, port).await?; + + Ok(()) +} diff --git a/backend/tests/fixtures/end_user_email.sql b/backend/tests/fixtures/end_user_email.sql new file mode 100644 index 0000000000..654ad93680 --- /dev/null +++ b/backend/tests/fixtures/end_user_email.sql @@ -0,0 +1,63 @@ +-- Fixture for WM_END_USER_EMAIL tests +-- Sets up 3 users with different workspace memberships: +-- 1. test@windmill.dev - in test-workspace (from base.sql) +-- 2. other-ws@windmill.dev - in other-workspace only +-- 3. no-ws@windmill.dev - not in any workspace + +-- Second workspace for cross-workspace user +INSERT INTO workspace (id, name, owner) +VALUES ('other-workspace', 'other-workspace', 'other-ws-user'); + +INSERT INTO workspace_key(workspace_id, kind, key) +VALUES ('other-workspace', 'cloud', 'other-key'); + +INSERT INTO workspace_settings (workspace_id) +VALUES ('other-workspace'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) +VALUES ('other-workspace', 'all', 'All users', '{}'); + +-- User in other-workspace only (not in test-workspace) +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) +VALUES ('other-ws@windmill.dev', 'hash', 'password', false, true, 'Other WS User'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) +VALUES ('other-workspace', 'other-ws@windmill.dev', 'other-ws-user', true, 'Admin'); + +INSERT INTO token(token, email, label, super_admin) +VALUES ('OTHER_WS_TOKEN', 'other-ws@windmill.dev', 'other ws token', false); + +-- User not in any workspace +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) +VALUES ('no-ws@windmill.dev', 'hash', 'password', false, true, 'No WS User'); + +INSERT INTO token(token, email, label, super_admin) +VALUES ('NO_WS_TOKEN', 'no-ws@windmill.dev', 'no ws token', false); + +-- Script that returns WM_END_USER_EMAIL (public via extra_perms) +INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, extra_perms) +VALUES ( + 'test-workspace', 'test-user', + 'export function main() { return Deno.env.get("WM_END_USER_EMAIL") || ""; }', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'deno', '', 'script', + '{"g/all": true}' +); + +-- Flow that returns WM_END_USER_EMAIL (public via extra_perms) +INSERT INTO flow (workspace_id, summary, description, path, versions, schema, value, edited_by, extra_perms) +VALUES ( + 'test-workspace', 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email_flow', '{900002}', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + 'test-user', + '{"g/all": true}' +); + +INSERT INTO flow_version (id, workspace_id, path, schema, value, created_by) +VALUES ( + 900002, 'test-workspace', 'f/test/get_end_user_email_flow', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + 'test-user' +); diff --git a/backend/tests/fixtures/hello.sql b/backend/tests/fixtures/hello.sql index f273cdfeaa..962f6441d9 100644 --- a/backend/tests/fixtures/hello.sql +++ b/backend/tests/fixtures/hello.sql @@ -30,6 +30,89 @@ export async function main(foo: string, bar: string) { '', 'f/system/hello_with_preprocessor', 123413, 'deno', ''); +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'system', +' +export async function preprocessor(foo: string, bar: string) { + return { foo: foo + "_preprocessed", bar: bar + "_preprocessed" }; +} + +export async function main(foo: string, bar: string) { + return "Hello " + foo + " " + bar; +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}', +'', +'', +'f/system/hello_preprocessor_dedicated_bun', 123414, 'bun', E'{}\n//bun.lock\n{}'); + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'system', +' +def preprocessor(foo: str, bar: str): + return {"foo": foo + "_preprocessed", "bar": bar + "_preprocessed"} + +def main(foo: str, bar: str): + return "Hello " + foo + " " + bar +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}', +'', +'', +'f/system/hello_preprocessor_dedicated_python', 123415, 'python3', ''); + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'system', +' +export async function preprocessor(foo: string, bar: string) { + return { foo: foo + "_preprocessed", bar: bar + "_preprocessed" }; +} + +export async function main(foo: string, bar: string) { + return "Hello " + foo + " " + bar; +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}', +'', +'', +'f/system/hello_preprocessor_dedicated_deno', 123416, 'deno', ''); + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'system', +'//native +export async function preprocessor(foo: string, bar: string) { + return { foo: foo + "_preprocessed", bar: bar + "_preprocessed" }; +} + +export async function main(foo: string, bar: string) { + return "Hello " + foo + " " + bar; +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}', +'', +'', +'f/system/hello_preprocessor_bunnative', 123417, 'bunnative', E'{}\n//bun.lock\n{}'); + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'system', +'//native +export async function preprocessor(foo: string, bar: string) { + return { foo: foo + "_preprocessed", bar: bar + "_preprocessed" }; +} + +export async function main(foo: string, bar: string) { + return "Hello " + foo + " " + bar; +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}', +'', +'', +'f/system/hello_preprocessor_dedicated_bunnative', 123418, 'bunnative', E'{}\n//bun.lock\n{}'); + INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( 'test-workspace', '', diff --git a/backend/tests/job_payload.rs b/backend/tests/job_payload.rs index 87253807be..4a8355b78d 100644 --- a/backend/tests/job_payload.rs +++ b/backend/tests/job_payload.rs @@ -1,16 +1,16 @@ mod job_payload { use serde_json::json; use sqlx::{Pool, Postgres}; + use windmill_common::flow_status::RestartedFrom; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowValue}; use windmill_common::jobs::JobPayload; use windmill_common::scripts::{ScriptHash, ScriptLang}; - use windmill_common::flow_status::RestartedFrom; - use windmill_test_utils::*; use windmill_common::min_version::{ MIN_VERSION, MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440, }; + use windmill_test_utils::*; pub async fn initialize_tracing() { use std::sync::Once; @@ -305,7 +305,7 @@ mod job_payload { path: "f/system/hello_with_nodes_flow".to_string(), dedicated_worker: None, version: 1443253234253454, - debouncing_settings: Default::default(), + debouncing_settings: Default::default(), }) .run_until_complete(&db, false, port) .await @@ -768,4 +768,256 @@ mod job_payload { .await; Ok(()) } + + #[sqlx::test(fixtures("base", "hello"))] + async fn test_dedicated_worker_preprocessor_bun(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let test = || async { + let db = &db; + let job = RunJob::from(JobPayload::ScriptHash { + hash: ScriptHash(123414), + path: "f/system/hello_preprocessor_dedicated_bun".to_string(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Bun, + priority: None, + apply_preprocessor: true, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: + windmill_common::runnable_settings::DebouncingSettings::default(), + }) + .arg("foo", json!("hello")) + .arg("bar", json!("world")) + .run_until_complete_with(db, false, port, |id| async move { + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(false)); + }) + .await; + + let args = job.args.as_ref().unwrap(); + assert_eq!(args.get("foo"), Some(&json!("hello_preprocessed"))); + assert_eq!(args.get("bar"), Some(&json!("world_preprocessed"))); + assert_eq!( + job.json_result().unwrap(), + json!("Hello hello_preprocessed world_preprocessed") + ); + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", job.id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(true)); + }; + test_for_versions(VERSION_FLAGS.iter().copied(), test).await; + Ok(()) + } + + #[sqlx::test(fixtures("base", "hello"))] + async fn test_dedicated_worker_preprocessor_python(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let test = || async { + let db = &db; + let job = RunJob::from(JobPayload::ScriptHash { + hash: ScriptHash(123415), + path: "f/system/hello_preprocessor_dedicated_python".to_string(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Python3, + priority: None, + apply_preprocessor: true, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: + windmill_common::runnable_settings::DebouncingSettings::default(), + }) + .arg("foo", json!("hello")) + .arg("bar", json!("world")) + .run_until_complete_with(db, false, port, |id| async move { + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(false)); + }) + .await; + + let args = job.args.as_ref().unwrap(); + assert_eq!(args.get("foo"), Some(&json!("hello_preprocessed"))); + assert_eq!(args.get("bar"), Some(&json!("world_preprocessed"))); + assert_eq!( + job.json_result().unwrap(), + json!("Hello hello_preprocessed world_preprocessed") + ); + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", job.id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(true)); + }; + test_for_versions(VERSION_FLAGS.iter().copied(), test).await; + Ok(()) + } + + #[sqlx::test(fixtures("base", "hello"))] + async fn test_dedicated_worker_preprocessor_deno(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let test = || async { + let db = &db; + let job = RunJob::from(JobPayload::ScriptHash { + hash: ScriptHash(123416), + path: "f/system/hello_preprocessor_dedicated_deno".to_string(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Deno, + priority: None, + apply_preprocessor: true, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: + windmill_common::runnable_settings::DebouncingSettings::default(), + }) + .arg("foo", json!("hello")) + .arg("bar", json!("world")) + .run_until_complete_with(db, false, port, |id| async move { + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(false)); + }) + .await; + + let args = job.args.as_ref().unwrap(); + assert_eq!(args.get("foo"), Some(&json!("hello_preprocessed"))); + assert_eq!(args.get("bar"), Some(&json!("world_preprocessed"))); + assert_eq!( + job.json_result().unwrap(), + json!("Hello hello_preprocessed world_preprocessed") + ); + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", job.id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(true)); + }; + test_for_versions(VERSION_FLAGS.iter().copied(), test).await; + Ok(()) + } + + #[sqlx::test(fixtures("base", "hello"))] + async fn test_bunnative_preprocessor(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let test = || async { + let db = &db; + let job = RunJob::from(JobPayload::ScriptHash { + hash: ScriptHash(123417), + path: "f/system/hello_preprocessor_bunnative".to_string(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Bunnative, + priority: None, + apply_preprocessor: true, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: + windmill_common::runnable_settings::DebouncingSettings::default(), + }) + .arg("foo", json!("hello")) + .arg("bar", json!("world")) + .run_until_complete_with(db, false, port, |id| async move { + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(false)); + }) + .await; + + let args = job.args.as_ref().unwrap(); + assert_eq!(args.get("foo"), Some(&json!("hello_preprocessed"))); + assert_eq!(args.get("bar"), Some(&json!("world_preprocessed"))); + assert_eq!( + job.json_result().unwrap(), + json!("Hello hello_preprocessed world_preprocessed") + ); + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", job.id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(true)); + }; + test_for_versions(VERSION_FLAGS.iter().copied(), test).await; + Ok(()) + } + + #[sqlx::test(fixtures("base", "hello"))] + async fn test_dedicated_worker_preprocessor_bunnative( + db: Pool, + ) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let test = || async { + let db = &db; + let job = RunJob::from(JobPayload::ScriptHash { + hash: ScriptHash(123418), + path: "f/system/hello_preprocessor_dedicated_bunnative".to_string(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Bunnative, + priority: None, + apply_preprocessor: true, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: + windmill_common::runnable_settings::DebouncingSettings::default(), + }) + .arg("foo", json!("hello")) + .arg("bar", json!("world")) + .run_until_complete_with(db, false, port, |id| async move { + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(false)); + }) + .await; + + let args = job.args.as_ref().unwrap(); + assert_eq!(args.get("foo"), Some(&json!("hello_preprocessed"))); + assert_eq!(args.get("bar"), Some(&json!("world_preprocessed"))); + assert_eq!( + job.json_result().unwrap(), + json!("Hello hello_preprocessed world_preprocessed") + ); + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", job.id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(true)); + }; + test_for_versions(VERSION_FLAGS.iter().copied(), test).await; + Ok(()) + } } diff --git a/backend/tests/nativets_dedicated.rs b/backend/tests/nativets_dedicated.rs index 4b730b32c7..f13759e5bc 100644 --- a/backend/tests/nativets_dedicated.rs +++ b/backend/tests/nativets_dedicated.rs @@ -80,8 +80,13 @@ mod prewarmed_isolate_tests { let mut results = Vec::new(); for job_args in &jobs { - let mut isolate = - PrewarmedIsolate::spawn("".to_string(), js.clone(), ann.clone(), arg_names.clone()); + let mut isolate = PrewarmedIsolate::spawn( + "".to_string(), + js.clone(), + ann.clone(), + arg_names.clone(), + None, + ); isolate.wait_ready().await.expect("isolate failed to warm"); let args = serde_json::to_string(job_args).unwrap(); @@ -180,8 +185,13 @@ export function main(n: number): number { let ann = default_annotation(); // Pre-warm first isolate - let mut warm = - PrewarmedIsolate::spawn("".to_string(), js.clone(), ann.clone(), arg_names.clone()); + let mut warm = PrewarmedIsolate::spawn( + "".to_string(), + js.clone(), + ann.clone(), + arg_names.clone(), + None, + ); warm.wait_ready() .await .expect("first isolate failed to warm"); @@ -193,8 +203,13 @@ export function main(n: number): number { let executing = warm.start_execution(args); // Pipeline: start pre-warming next isolate while current one runs - warm = - PrewarmedIsolate::spawn("".to_string(), js.clone(), ann.clone(), arg_names.clone()); + warm = PrewarmedIsolate::spawn( + "".to_string(), + js.clone(), + ann.clone(), + arg_names.clone(), + None, + ); let prewarmed_result = executing.wait().await.expect("isolate execution failed"); match prewarmed_result.result { diff --git a/backend/tests/nativets_stress.rs b/backend/tests/nativets_stress.rs index 9717403830..082a43be2c 100644 --- a/backend/tests/nativets_stress.rs +++ b/backend/tests/nativets_stress.rs @@ -206,7 +206,7 @@ fn spawn_workers( std::fs::DirBuilder::new() .recursive(true) - .create(windmill_worker::GO_BIN_CACHE_DIR) + .create(&*windmill_worker::GO_BIN_CACHE_DIR) .expect("could not create initial worker dir"); let (tx, _) = KillpillSender::new(n + 1); diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index dd494de007..22c2313b91 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -1,7 +1,7 @@ -use windmill_test_utils::*; use sqlx::postgres::Postgres; use sqlx::Pool; use windmill_common::scripts::ScriptLang; +use windmill_test_utils::*; #[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] @@ -188,7 +188,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, @@ -207,14 +208,14 @@ def main(): #[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_python_global_site_packages(db: Pool) -> anyhow::Result<()> { - use windmill_common::{cache::concatcp, worker::ROOT_CACHE_DIR}; + use windmill_common::worker::ROOT_CACHE_DIR; initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); // Shared for all 3.12.* - let path = concatcp!(ROOT_CACHE_DIR, "python_3_12/global-site-packages").to_owned(); + let path = format!("{}python_3_12/global-site-packages", *ROOT_CACHE_DIR); std::fs::create_dir_all(&path).unwrap(); std::fs::write(path + "/my_global_site_package_3_12_any.py", "").unwrap(); @@ -237,7 +238,9 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( + ) + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, @@ -271,7 +274,9 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( + ) + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, @@ -310,7 +315,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, @@ -347,7 +353,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, diff --git a/backend/tests/scripts/test_volume_with_claude.ts b/backend/tests/scripts/test_volume_with_claude.ts new file mode 100644 index 0000000000..b6230a19ee --- /dev/null +++ b/backend/tests/scripts/test_volume_with_claude.ts @@ -0,0 +1,102 @@ +// volume: agent-memory .claude +// sandbox + +import Anthropic from "@anthropic-ai/sdk"; +import * as fs from "fs"; +import * as path from "path"; + +type Anthropic = { + api_key: string; + model?: string; +}; + +export async function main(anthropic_resource: Anthropic) { + const claudeDir = ".claude"; + const results: Record = {}; + + // --- Step 1: Verify volume is mounted at the relative path --- + results["volume_exists"] = fs.existsSync(claudeDir); + if (!results["volume_exists"]) { + fs.mkdirSync(claudeDir, { recursive: true }); + } + + const testFile = path.join(claudeDir, "mount-check.txt"); + fs.writeFileSync(testFile, "volume mount verified"); + results["volume_writable"] = fs.readFileSync(testFile, "utf-8") === "volume mount verified"; + + // --- Step 2: Create memory directory structure --- + const memoryDir = path.join(claudeDir, "memory"); + fs.mkdirSync(memoryDir, { recursive: true }); + + const memoryFile = path.join(memoryDir, "MEMORY.md"); + fs.writeFileSync(memoryFile, "# Agent Memory\n\nThis file persists across runs.\n"); + results["memory_file_created"] = fs.existsSync(memoryFile); + + // --- Step 3: Call Claude to generate structured content --- + const client = new Anthropic({ apiKey: anthropic_resource.api_key }); + const model = anthropic_resource.model ?? "claude-sonnet-4-20250514"; + + const response = await client.messages.create({ + model, + max_tokens: 256, + messages: [ + { + role: "user", + content: + 'Return a JSON object with exactly these keys: "greeting" (a short hello), "timestamp" (current ISO date you estimate), "items" (array of 3 random fruit names). Only return the JSON, no markdown.', + }, + ], + }); + + const assistantText = + response.content[0].type === "text" ? response.content[0].text : ""; + results["claude_responded"] = assistantText.length > 0; + results["claude_model"] = response.model; + results["claude_stop_reason"] = response.stop_reason; + + let parsed: Record = {}; + try { + parsed = JSON.parse(assistantText); + results["claude_valid_json"] = true; + results["claude_has_greeting"] = "greeting" in parsed; + results["claude_has_items"] = + Array.isArray(parsed.items) && parsed.items.length === 3; + } catch { + results["claude_valid_json"] = false; + } + + // --- Step 4: Write Claude's response to volume --- + const responsePath = path.join(claudeDir, "claude-response.json"); + fs.writeFileSync(responsePath, JSON.stringify(parsed, null, 2)); + results["response_written"] = fs.existsSync(responsePath); + + // --- Step 5: Read back and verify --- + const readBack = fs.readFileSync(responsePath, "utf-8"); + const readParsed = JSON.parse(readBack); + results["readback_matches"] = + JSON.stringify(readParsed) === JSON.stringify(parsed); + + // --- Step 6: List all volume contents --- + const volumeContents = fs.readdirSync(claudeDir); + results["volume_files"] = volumeContents; + results["volume_file_count"] = volumeContents.length; + + // --- Step 7: Verify memory file persists --- + const memoryContent = fs.readFileSync(memoryFile, "utf-8"); + results["memory_persisted"] = memoryContent.includes("Agent Memory"); + + // --- Summary --- + const allChecks = [ + results["volume_exists"] || true, + results["volume_writable"], + results["claude_responded"], + results["claude_valid_json"], + results["response_written"], + results["readback_matches"], + results["memory_file_created"], + results["memory_persisted"], + ]; + results["all_passed"] = allChecks.every(Boolean); + + return results; +} diff --git a/backend/tests/volume_tests.rs b/backend/tests/volume_tests.rs new file mode 100644 index 0000000000..9df30c5615 --- /dev/null +++ b/backend/tests/volume_tests.rs @@ -0,0 +1,637 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::jobs::{JobPayload, RawCode}; +use windmill_common::scripts::ScriptLang; +use windmill_test_utils::*; + +#[sqlx::test(fixtures("base"))] +async fn test_volume_insert(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "test-volume", + 1024_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT workspace_id, name, size_bytes, created_by, last_used_at + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-volume" + ) + .fetch_one(&db) + .await?; + + assert_eq!(row.workspace_id, "test-workspace"); + assert_eq!(row.name, "test-volume"); + assert_eq!(row.size_bytes, 1024); + assert_eq!(row.created_by, "test-user"); + assert!(row.last_used_at.is_none()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_upsert_size(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET size_bytes = $3, last_used_at = now()", + "test-workspace", + "upsert-vol", + 500_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT size_bytes FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "upsert-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.size_bytes, 500); + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET size_bytes = $3, last_used_at = now()", + "test-workspace", + "upsert-vol", + 2048_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT size_bytes, last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "upsert-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.size_bytes, 2048); + assert!(row.last_used_at.is_some()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_update_last_used(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "used-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .fetch_one(&db) + .await?; + assert!(row.last_used_at.is_none()); + + sqlx::query!( + "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .fetch_one(&db) + .await?; + assert!(row.last_used_at.is_some()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_update_nonexistent_noop(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let result = sqlx::query!( + "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "nonexistent-vol" + ) + .execute(&db) + .await?; + + assert_eq!(result.rows_affected(), 0); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_list_multiple(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + for i in 0..5 { + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + format!("vol-{}", i), + (i * 100) as i64, + "test-user" + ) + .execute(&db) + .await?; + } + + let rows = sqlx::query!( + "SELECT name, size_bytes FROM volume WHERE workspace_id = $1 ORDER BY name", + "test-workspace" + ) + .fetch_all(&db) + .await?; + + assert_eq!(rows.len(), 5); + assert_eq!(rows[0].name, "vol-0"); + assert_eq!(rows[0].size_bytes, 0); + assert_eq!(rows[4].name, "vol-4"); + assert_eq!(rows[4].size_bytes, 400); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_delete(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "deleteme", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .fetch_one(&db) + .await?; + assert_eq!(count, Some(1)); + + sqlx::query!( + "DELETE FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .fetch_one(&db) + .await?; + assert_eq!(count, Some(0)); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_workspace_fk_constraint(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let result = sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "nonexistent-workspace", + "vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("foreign key"), + "Expected foreign key violation, got: {}", + err + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_primary_key_uniqueness(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "unique-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let result = sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "unique-vol", + 200_i64, + "another-user" + ) + .execute(&db) + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("duplicate key") || err.contains("unique"), + "Expected unique violation, got: {}", + err + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_extra_perms(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Insert volume with default (empty) extra_perms + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "perms-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + // Default extra_perms should be empty object + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.extra_perms, serde_json::json!({})); + + // Set extra_perms via jsonb_set (same pattern as granular_acls.rs) + sqlx::query!( + "UPDATE volume SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2::bool), true) + WHERE workspace_id = $3 AND name = $4", + &vec!["u/alice".to_string()], + true, + "test-workspace", + "perms-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + let perms = row.extra_perms.as_object().unwrap(); + assert_eq!(perms.get("u/alice").and_then(|v| v.as_bool()), Some(true)); + + // Remove a permission entry + sqlx::query!( + "UPDATE volume SET extra_perms = extra_perms - $1 + WHERE workspace_id = $2 AND name = $3", + "u/alice", + "test-workspace", + "perms-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.extra_perms, serde_json::json!({})); + + Ok(()) +} + +#[test] +fn test_parse_volume_annotations_python() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = r#"# sandbox +# volume: training-data /tmp/training +# volume: models /opt/models + +def main(): + pass +"#; + let volumes = parse_volume_annotations(content, "#"); + assert_eq!(volumes.len(), 2); + assert_eq!(volumes[0].name, "training-data"); + assert_eq!(volumes[0].target, "/tmp/training"); + assert_eq!(volumes[1].name, "models"); + assert_eq!(volumes[1].target, "/opt/models"); +} + +#[test] +fn test_parse_volume_annotations_typescript() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = r#"// sandbox +// volume: datasets /tmp/datasets + +export async function main() { + return "hello"; +} +"#; + let volumes = parse_volume_annotations(content, "//"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "datasets"); + assert_eq!(volumes[0].target, "/tmp/datasets"); +} + +#[test] +fn test_parse_volume_annotations_no_prefix_match() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "def main():\n pass"; + let volumes = parse_volume_annotations(content, "#"); + assert!(volumes.is_empty()); +} + +#[test] +fn test_parse_volume_annotations_empty_script() { + use windmill_worker_volumes::parse_volume_annotations; + + let volumes = parse_volume_annotations("", "#"); + assert!(volumes.is_empty()); +} + +#[test] +fn test_sandbox_annotation_python() { + use windmill_common::worker::PythonAnnotations; + + let content = "# sandbox\n# volume: data /tmp/data\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); +} + +#[test] +fn test_sandbox_annotation_typescript() { + use windmill_common::worker::TypeScriptAnnotations; + + let content = "// sandbox\n// volume: data /tmp/data\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); +} + +#[test] +fn test_volume_comment_prefix_selection() { + use windmill_common::scripts::ScriptLang; + + let get_prefix = |lang: &ScriptLang| -> &str { + match lang { + ScriptLang::Python3 + | ScriptLang::Bash + | ScriptLang::Powershell + | ScriptLang::Ansible + | ScriptLang::Ruby => "#", + ScriptLang::Deno + | ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Go => "//", + _ => "", + } + }; + + assert_eq!(get_prefix(&ScriptLang::Python3), "#"); + assert_eq!(get_prefix(&ScriptLang::Bash), "#"); + assert_eq!(get_prefix(&ScriptLang::Powershell), "#"); + assert_eq!(get_prefix(&ScriptLang::Ansible), "#"); + assert_eq!(get_prefix(&ScriptLang::Ruby), "#"); + assert_eq!(get_prefix(&ScriptLang::Deno), "//"); + assert_eq!(get_prefix(&ScriptLang::Bun), "//"); + assert_eq!(get_prefix(&ScriptLang::Bunnative), "//"); + assert_eq!(get_prefix(&ScriptLang::Nativets), "//"); + assert_eq!(get_prefix(&ScriptLang::Go), "//"); +} + +#[test] +fn test_volume_mount_struct() { + use windmill_worker_volumes::VolumeMount; + + let mount = VolumeMount { name: "test-vol".to_string(), target: "/mnt/data".to_string() }; + assert_eq!(mount.name, "test-vol"); + assert_eq!(mount.target, "/mnt/data"); +} + +#[test] +fn test_parse_volume_relative_path() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "// volume: agent-memory .claude\nexport function main() {}"; + let volumes = parse_volume_annotations(content, "//"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "agent-memory"); + assert_eq!(volumes[0].target, ".claude"); +} + +#[test] +fn test_parse_volume_relative_nested_path() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "# volume: data data/models\ndef main():\n pass"; + let volumes = parse_volume_annotations(content, "#"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "data"); + assert_eq!(volumes[0].target, "data/models"); +} + +#[cfg(feature = "private")] +#[test] +fn test_volume_nsjail_mount() { + use std::path::Path; + use windmill_worker_volumes::volume_nsjail_mount; + + let result = volume_nsjail_mount(Path::new("/tmp/volumes/data"), "/mnt/data"); + assert!(result.contains("src: \"/tmp/volumes/data\"")); + assert!(result.contains("dst: \"/mnt/data\"")); + assert!(result.contains("is_bind: true")); + assert!(result.contains("rw: true")); +} + +#[test] +fn test_sync_stats_default() { + use windmill_worker_volumes::SyncStats; + + let stats = SyncStats { new_size_bytes: 0, file_count: 0, uploaded: 0, skipped: 0 }; + assert_eq!(stats.new_size_bytes, 0); + assert_eq!(stats.file_count, 0); + assert_eq!(stats.uploaded, 0); + assert_eq!(stats.skipped, 0); +} + +#[test] +fn test_asset_kind_volume_variant() { + use windmill_types::assets::AssetKind; + + let kind = AssetKind::Volume; + let serialized = serde_json::to_string(&kind).unwrap(); + assert_eq!(serialized, "\"volume\""); + + let deserialized: AssetKind = serde_json::from_str("\"volume\"").unwrap(); + assert!(matches!(deserialized, AssetKind::Volume)); +} + +/// E2E test: run a bun script with volume mount through a SQL-connected worker. +/// Pre-populates the volume in filesystem storage, verifies the script can read +/// files and write new ones, then checks sync-back to storage and DB state. +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_volume_sql_worker_e2e(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null, + "volume_storage": "primary" + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file (workspace-namespaced path) + let vol_dir = storage_dir + .path() + .join("volumes") + .join("test-workspace") + .join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + // 3. Push the job and run with SQL-connected worker + let code = r#"// volume: test-vol /tmp/data + +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("/tmp/data/hello.txt", "utf-8"); + writeFileSync("/tmp/data/output.txt", "written by sql worker"); + return { + read_content: content, + output_exists: existsSync("/tmp/data/output.txt"), + }; +}"#; + + let job = JobPayload::Code(RawCode { + hash: None, + content: code.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port).await; + + assert!(result.success, "job should succeed: {:?}", result.result); + let json = result.json_result().expect("should have JSON result"); + assert_eq!(json["read_content"], json!("hello from volume")); + assert_eq!(json["output_exists"], json!(true)); + + // 4. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert!( + vol_row.file_count >= 2, + "should have at least 2 files (hello.txt + output.txt), got: {}", + vol_row.file_count + ); + assert!(vol_row.size_bytes > 0, "size_bytes should be > 0"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + + // 5. Verify the new file was written back to storage + let output_path = vol_dir.join("output.txt"); + assert!( + output_path.exists(), + "output.txt should be synced back to storage" + ); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by sql worker"); + + Ok(()) +} diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 9548986a5a..32fada32fb 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3548,3 +3548,170 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: Ok(()) } + +#[cfg(all(feature = "quickjs", feature = "python"))] +#[sqlx::test(fixtures("base"))] +async fn test_stop_after_all_iters_if_bad_expr_parallel_branchall( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let port = 123; + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "id": "a", + "value": { + "branches": [ + {"modules": [{ + "id": "b", + "value": { + "input_transforms": { "n": { "type": "javascript", "expr": "flow_input.n" } }, + "type": "rawscript", + "language": "python3", + "content": "def main(n): return n", + }, + }]} + ], + "type": "branchall", + "parallel": true, + }, + "stop_after_all_iters_if": { + "expr": "invalid!!!syntax", + "skip_if_stopped": false, + }, + }, + ], + })) + .unwrap(); + let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; + + let cjob = RunJob::from(job) + .arg("n", json!(42)) + .run_until_complete(&db, false, port) + .await; + + assert!( + !cjob.success, + "flow should fail when stop_after_all_iters_if has bad expression" + ); + + let result = cjob.json_result().unwrap(); + let error_msg = result["error"]["message"].as_str().unwrap_or(""); + assert!( + error_msg.contains("stop_after_all_iters_if"), + "error should mention stop_after_all_iters_if, got: {error_msg}" + ); + + Ok(()) +} + +#[cfg(all(feature = "quickjs", feature = "python"))] +#[sqlx::test(fixtures("base"))] +async fn test_stop_after_all_iters_if_bad_expr_parallel_forloop( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let port = 123; + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "id": "a", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "result.items" }, + "skip_failures": false, + "parallel": true, + "modules": [{ + "value": { + "input_transforms": { + "n": { "type": "javascript", "expr": "flow_input.iter.value" }, + }, + "type": "rawscript", + "language": "python3", + "content": "def main(n): return n", + }, + }], + }, + "stop_after_all_iters_if": { + "expr": "invalid!!!syntax", + "skip_if_stopped": false, + }, + }, + ], + })) + .unwrap(); + let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; + + let cjob = RunJob::from(job) + .arg("items", json!([1, 2, 3])) + .run_until_complete(&db, false, port) + .await; + + assert!( + !cjob.success, + "flow should fail when stop_after_all_iters_if has bad expression" + ); + + let result = cjob.json_result().unwrap(); + let error_msg = result["error"]["message"].as_str().unwrap_or(""); + assert!( + error_msg.contains("stop_after_all_iters_if"), + "error should mention stop_after_all_iters_if, got: {error_msg}" + ); + + Ok(()) +} + +#[cfg(all(feature = "quickjs", feature = "python"))] +#[sqlx::test(fixtures("base"))] +async fn test_results_length_in_input_transform(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Step a returns a list, step b accesses results.a.length via input transform. + // This tests that the handle_full_regex fast path falls through to QuickJS + // when the SQL JSON path operator can't resolve JS properties like .length. + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "a", + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(): return [10, 20, 30]", + }, + }, + { + "id": "b", + "value": { + "input_transforms": { + "v": { "type": "javascript", "expr": "results.a.length" }, + }, + "type": "rawscript", + "language": "python3", + "content": "def main(v): return v", + }, + }, + ], + })) + .unwrap(); + + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!( + result, + json!(3), + "results.a.length should resolve to 3, not null" + ); + + Ok(()) +} diff --git a/backend/windmill-api-agent-workers/src/lib.rs b/backend/windmill-api-agent-workers/src/lib.rs index d2913a217c..b02a7098b7 100644 --- a/backend/windmill-api-agent-workers/src/lib.rs +++ b/backend/windmill-api-agent-workers/src/lib.rs @@ -51,4 +51,12 @@ impl AgentCache { pub fn new() -> Self { AgentCache {} } + + pub async fn extract_worker_name( + &self, + _token: &str, + _db: &windmill_common::DB, + ) -> Option { + None + } } diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs index 05cff4ff38..30a6b241b2 100644 --- a/backend/windmill-api-auth/src/auth.rs +++ b/backend/windmill-api-auth/src/auth.rs @@ -37,7 +37,45 @@ use windmill_common::{ lazy_static::lazy_static! { // Global auth cache accessible from main.rs for direct invalidation pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300); + // Cache for token -> email lookups (for non-workspace-member authenticated users) + static ref TOKEN_EMAIL_CACHE: Cache> = Cache::new(500); +} +/// Get email from a valid token, with caching. +/// Used for WM_END_USER_EMAIL when user is authenticated but not a workspace member. +async fn get_email_from_token(db: &DB, token: &str) -> Option { + if let Some(cached) = TOKEN_EMAIL_CACHE.get(token) { + return cached; + } + + let email = sqlx::query_scalar!( + "SELECT email FROM token WHERE token = $1 AND (expiration > NOW() OR expiration IS NULL)", + token + ) + .fetch_optional(db) + .await + .ok() + .flatten() + .flatten(); // email column is nullable, so we get Option> + + TOKEN_EMAIL_CACHE.insert(token.to_string(), email.clone()); + email +} + +/// Get end user email from authenticated user or token. +/// Returns email if user is authenticated (workspace member) or has valid instance token. +pub async fn get_end_user_email( + db: &DB, + opt_authed: Option<&ApiAuthed>, + token: Option<&str>, +) -> Option { + if let Some(authed) = opt_authed { + return Some(authed.email.clone()); + } + if let Some(token) = token { + return get_email_from_token(db, token).await; + } + None } // Global function to invalidate tokens from cache by prefix pub fn invalidate_token_from_cache(token_prefix: &str) { diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index f3f42b6a17..50175b8686 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -32,8 +32,8 @@ use scopes::ScopeDefinition; // Re-export key auth types and functions pub use auth::{ - invalidate_token_from_cache, AuthCache, ExpiringAuthCache, OptTokened, Tokened, - TruncatedTokenWithEmail, AUTH_CACHE, + get_end_user_email, invalidate_token_from_cache, AuthCache, ExpiringAuthCache, OptTokened, + Tokened, TruncatedTokenWithEmail, AUTH_CACHE, }; // ------------ ApiAuthed & OptJobAuthed types ------------ @@ -573,6 +573,14 @@ pub async fn create_token_internal( )); } + register_token_expiry_notification( + &mut *tx, + &token, + token_config.label.as_deref(), + token_config.expiration, + ) + .await; + audit_log( &mut *tx, authed, @@ -588,6 +596,39 @@ pub async fn create_token_internal( Ok(token) } +/// Insert a pending expiry notification row for user tokens that have an expiration. +/// When updating this filter, also update: +/// - `is_user_token` in src/monitor.rs +/// - `isUserToken` in frontend/src/lib/components/settings/TokensTable.svelte +pub async fn register_token_expiry_notification( + tx: &mut sqlx::PgConnection, + token: &str, + label: Option<&str>, + expiration: Option>, +) { + let Some(expiration) = expiration else { return }; + if label == Some("session") + || label.is_some_and(|l| { + l.starts_with("ephemeral") + || l.starts_with("Ephemeral") + || l == "debugger-token" + || l.starts_with("mcp-oauth-") + }) + { + return; + } + if let Err(e) = sqlx::query!( + "INSERT INTO token_expiry_notification (token, expiration) VALUES ($1, $2) ON CONFLICT DO NOTHING", + token, + expiration, + ) + .execute(&mut *tx) + .await + { + tracing::error!("Failed to register token expiry notification: {}", e); + } +} + // ------------ Permission helpers ------------ pub fn get_perm_in_extra_perms_for_authed( diff --git a/backend/windmill-api-configs/src/lib.rs b/backend/windmill-api-configs/src/lib.rs index f4e91e554f..a05f2dbaa8 100644 --- a/backend/windmill-api-configs/src/lib.rs +++ b/backend/windmill-api-configs/src/lib.rs @@ -129,11 +129,12 @@ async fn update_config( #[cfg(not(feature = "enterprise"))] let config = if name.starts_with("worker__") { - // In CE, only allow setting worker_tags, cache_clear, and init_bash + // In CE, only allow setting worker_tags, cache_clear, init_bash, and native_mode serde_json::json!({ "worker_tags": config.get("worker_tags"), "cache_clear": config.get("cache_clear"), - "init_bash": config.get("init_bash") + "init_bash": config.get("init_bash"), + "native_mode": config.get("native_mode") }) } else { config diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index 048a4120f5..da9267419d 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -24,7 +24,7 @@ use windmill_common::{ utils::{not_found_if_none, StripPath}, }; -const KINDS: [&str; 18] = [ +const KINDS: [&str; 19] = [ "script", "group_", "resource", @@ -43,6 +43,7 @@ const KINDS: [&str; 18] = [ "gcp_trigger", "sqs_trigger", "email_trigger", + "volume", ]; pub fn workspaced_service() -> Router { @@ -77,7 +78,7 @@ async fn add_granular_acl( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" || kind == "folder" { + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { "name" } else { "path" @@ -89,6 +90,22 @@ async fn add_granular_acl( } else if kind == "group_" { crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) .await?; + } else if kind == "volume" { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("volume '{path}' not found")))?; + // created_by is stored with u/ prefix (from job.permissioned_as) + let owner_username = created_by.strip_prefix("u/").unwrap_or(&created_by); + if owner_username != authed.username { + return Err(Error::NotAuthorized( + "Only the volume owner or an admin can modify permissions".to_string(), + )); + } } else { require_owner_of_path(&authed, path)?; } @@ -243,6 +260,22 @@ async fn remove_granular_acl( } else if kind == "group_" { crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) .await?; + } else if kind == "volume" { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("volume '{path}' not found")))?; + // created_by is stored with u/ prefix (from job.permissioned_as) + let owner_username = created_by.strip_prefix("u/").unwrap_or(&created_by); + if owner_username != authed.username { + return Err(Error::NotAuthorized( + "Only the volume owner or an admin can modify permissions".to_string(), + )); + } } else { require_owner_of_path(&authed, path)?; } @@ -250,7 +283,7 @@ async fn remove_granular_acl( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" || kind == "folder" { + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { "name" } else { "path" @@ -380,7 +413,11 @@ async fn get_granular_acls( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" { "name" } else { "path" }; + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { + "name" + } else { + "path" + }; let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!( "SELECT extra_perms from {kind} WHERE {identifier} = $1 AND workspace_id = $2" )) diff --git a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs new file mode 100644 index 0000000000..72a102dfd8 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs @@ -0,0 +1,452 @@ +/*! + * Integration tests for workspace dependencies git sync. + * + * These tests verify that creating, archiving, and deleting workspace dependencies + * triggers deployment callback jobs with the correct arguments for git sync. + * + * Run with enterprise features: + * ```bash + * cargo test --test workspace_dependencies_git_sync --features enterprise,private + * ``` + */ + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use std::time::Duration; + +use windmill_test_utils::*; + +/// Row shape for querying deployment callback jobs from v2_job_queue +#[derive(Debug)] +#[allow(dead_code)] +struct DeploymentCallbackJob { + id: uuid::Uuid, + runnable_path: Option, + args: Option, + kind: String, +} + +/// Poll for deployment callback jobs in the queue for a given script path +async fn get_deployment_callback_jobs( + db: &Pool, + script_path: &str, + timeout: Duration, +) -> anyhow::Result> { + let deadline = tokio::time::Instant::now() + timeout; + loop { + let rows = sqlx::query_as!( + DeploymentCallbackJob, + r#" + SELECT j.id, j.runnable_path, j.args, j.kind::text AS "kind!" + FROM v2_job j + JOIN v2_job_queue q ON j.id = q.id + WHERE j.runnable_path = $1 + AND j.kind = 'deploymentcallback' + ORDER BY j.created_at DESC + "#, + script_path, + ) + .fetch_all(db) + .await?; + + if !rows.is_empty() { + return Ok(rows); + } + + if tokio::time::Instant::now() >= deadline { + // Return empty if timeout - caller will handle assertion + return Ok(vec![]); + } + + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +/// Configure git sync for the test workspace with workspace dependencies enabled +async fn setup_git_sync_config(db: &Pool, sync_script_path: &str) -> anyhow::Result<()> { + let git_sync_config = json!({ + "include_type": ["workspacedependencies"], + "include_path": ["**"], + "repositories": [{ + "script_path": sync_script_path, + "git_repo_resource_path": "$res:u/test-user/test_git_repo", + "use_individual_branch": false, + "group_by_folder": false + }] + }); + + sqlx::query!( + "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", + git_sync_config, + "test-workspace" + ) + .execute(db) + .await?; + + Ok(()) +} + +/// Create a git repository resource for testing +async fn create_git_repo_resource(db: &Pool) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by) + VALUES ('test-workspace', 'u/test-user/test_git_repo', $1::jsonb, 'git_repository', '{}'::jsonb, 'test-user') + ON CONFLICT (workspace_id, path) DO NOTHING + "#, + ) + .bind(json!({ + "url": "https://github.com/test/test.git", + "branch": "main", + "token": "test-token" + })) + .execute(db) + .await?; + + Ok(()) +} + +/// Create a dummy sync script for testing (with version >= 28103 for debouncing support) +async fn create_sync_script(db: &Pool, path: &str) -> anyhow::Result { + let hash: i64 = rand::random::().unsigned_abs() as i64; + sqlx::query( + r#" + INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, kind, lock) + VALUES ('test-workspace', $1, $2, 'sync script', '', + 'export function main(items: any[]) { return { synced: items.length }; }', + 'test-user', 'bun', 'script', '') + "#, + ) + .bind(hash) + .bind(path) + .execute(db) + .await?; + Ok(hash) +} + +/// Create a folder for the versioned script path +async fn create_folder(db: &Pool, name: &str) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by) + VALUES ('test-workspace', $1, $1, ARRAY['u/test-user'], '{}'::jsonb, 'test-user') + ON CONFLICT (workspace_id, name) DO NOTHING + "#, + ) + .bind(name) + .execute(db) + .await?; + Ok(()) +} + +// ============================================================================ +// Tests +// ============================================================================ + +/// Test that creating a workspace dependency triggers a git sync deployment callback +/// with the correct path_type and path arguments. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_create_workspace_dependencies_triggers_git_sync( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + // Setup: Create folder, git repo resource, sync script, and configure git sync + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script"; + create_sync_script(&db, sync_script_path).await?; + setup_git_sync_config(&db, sync_script_path).await?; + + // Start API server + let (client, _port, _server) = init_client(db.clone()).await; + + // Create workspace dependency via API + let response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/create", + client.baseurl() + )) + .json(&json!({ + "workspace_id": "test-workspace", + "language": "python3", + "name": "test-deps", + "content": "requests==2.28.0\nnumpy==1.24.0" + })) + .send() + .await?; + + assert!( + response.status().is_success(), + "Failed to create workspace dependency: {:?}", + response.text().await + ); + + // Wait for deployment callback job to be created + tokio::time::sleep(Duration::from_millis(500)).await; + + // Query for deployment callback jobs + let jobs = get_deployment_callback_jobs(&db, sync_script_path, Duration::from_secs(5)).await?; + + assert!( + !jobs.is_empty(), + "Expected at least one deployment callback job to be created" + ); + + // Verify the job arguments + let job = &jobs[0]; + let args = job.args.as_ref().expect("Job should have args"); + + // Check that path_type is "workspace_dependencies" (or check items array) + // The exact structure depends on whether debouncing is enabled + if let Some(items) = args.get("items") { + // Debounced format: items is an array + let items_arr = items.as_array().expect("items should be an array"); + assert!(!items_arr.is_empty(), "items array should not be empty"); + + let item = &items_arr[0]; + assert_eq!( + item.get("path_type").and_then(|v| v.as_str()), + Some("workspace_dependencies"), + "path_type should be 'workspace_dependencies'" + ); + + // Path should be "dependencies/test-deps.requirements.in" or similar + let path = item.get("path").and_then(|v| v.as_str()).unwrap_or(""); + assert!( + path.contains("dependencies") || path.contains("requirements"), + "path should contain dependencies or requirements: got {}", + path + ); + } else if let Some(path_type) = args.get("path_type") { + // Non-debounced format: path_type is a direct field + assert_eq!( + path_type.as_str(), + Some("workspace_dependencies"), + "path_type should be 'workspace_dependencies'" + ); + } else { + panic!( + "Job args should contain either 'items' array or 'path_type' field: {:?}", + args + ); + } + + Ok(()) +} + +/// Test that archiving a workspace dependency triggers a git sync deployment callback +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_archive_workspace_dependencies_triggers_git_sync( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + // Setup + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script_archive"; + create_sync_script(&db, sync_script_path).await?; + setup_git_sync_config(&db, sync_script_path).await?; + + let (client, _port, _server) = init_client(db.clone()).await; + + // First create a workspace dependency + let create_response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/create", + client.baseurl() + )) + .json(&json!({ + "workspace_id": "test-workspace", + "language": "python3", + "name": "archive-test-deps", + "content": "flask==2.0.0" + })) + .send() + .await?; + + assert!(create_response.status().is_success()); + + // Wait a bit for the create job to be processed + tokio::time::sleep(Duration::from_millis(300)).await; + + // Now archive it + let archive_response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/archive/python3?name=archive-test-deps", + client.baseurl() + )) + .send() + .await?; + + assert!( + archive_response.status().is_success(), + "Failed to archive workspace dependency: {:?}", + archive_response.text().await + ); + + // Wait for deployment callback job + tokio::time::sleep(Duration::from_millis(500)).await; + + // Verify at least one deployment callback job exists + // (create test already validates create triggers git sync; this test validates archive does too) + let jobs = get_deployment_callback_jobs(&db, sync_script_path, Duration::from_secs(5)).await?; + + assert!( + !jobs.is_empty(), + "Expected at least one deployment callback job after archive" + ); + + Ok(()) +} + +/// Test that workspace dependencies are NOT synced when workspacedependencies type is excluded +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_workspace_dependencies_respects_include_type_filter( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + // Setup with git sync that EXCLUDES workspacedependencies + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script_filter"; + create_sync_script(&db, sync_script_path).await?; + + // Configure git sync to only include scripts (not workspace dependencies) + let git_sync_config = json!({ + "include_type": ["script"], // Note: workspacedependencies is NOT included + "include_path": ["**"], + "repositories": [{ + "script_path": sync_script_path, + "git_repo_resource_path": "$res:u/test-user/test_git_repo", + "use_individual_branch": false, + "group_by_folder": false + }] + }); + + sqlx::query!( + "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", + git_sync_config, + "test-workspace" + ) + .execute(&db) + .await?; + + let (client, _port, _server) = init_client(db.clone()).await; + + // Create workspace dependency + let response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/create", + client.baseurl() + )) + .json(&json!({ + "workspace_id": "test-workspace", + "language": "python3", + "name": "filtered-deps", + "content": "django==4.0.0" + })) + .send() + .await?; + + assert!(response.status().is_success()); + + // Wait a bit + tokio::time::sleep(Duration::from_secs(1)).await; + + // Should NOT have any deployment callback jobs because workspacedependencies is filtered out + let jobs = + get_deployment_callback_jobs(&db, sync_script_path, Duration::from_millis(500)).await?; + + assert!( + jobs.is_empty(), + "Expected NO deployment callback jobs when workspacedependencies is not in include_type, got {}", + jobs.len() + ); + + Ok(()) +} + +/// Test that the commit message is correctly generated for workspace dependencies +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_workspace_dependencies_commit_message(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Setup + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script_msg"; + create_sync_script(&db, sync_script_path).await?; + setup_git_sync_config(&db, sync_script_path).await?; + + let (client, _port, _server) = init_client(db.clone()).await; + + // Create workspace dependency + let response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/create", + client.baseurl() + )) + .json(&json!({ + "workspace_id": "test-workspace", + "language": "bun", + "name": null, // unnamed/default dependency + "content": "lodash: ^4.17.21" + })) + .send() + .await?; + + assert!(response.status().is_success()); + + tokio::time::sleep(Duration::from_millis(500)).await; + + let jobs = get_deployment_callback_jobs(&db, sync_script_path, Duration::from_secs(5)).await?; + assert!(!jobs.is_empty()); + + let job = &jobs[0]; + let args = job.args.as_ref().expect("Job should have args"); + + // Check commit message format + if let Some(items) = args.get("items") { + let items_arr = items.as_array().expect("items should be an array"); + if !items_arr.is_empty() { + let commit_msg = items_arr[0] + .get("commit_msg") + .and_then(|v| v.as_str()) + .unwrap_or(""); + + assert!( + commit_msg.contains("[WM]"), + "Commit message should contain '[WM]' prefix: {}", + commit_msg + ); + assert!( + commit_msg.to_lowercase().contains("workspace") + || commit_msg.to_lowercase().contains("dependency") + || commit_msg.to_lowercase().contains("deployed"), + "Commit message should mention workspace dependency or deployed: {}", + commit_msg + ); + } + } else if let Some(commit_msg) = args.get("commit_msg").and_then(|v| v.as_str()) { + assert!( + commit_msg.contains("[WM]"), + "Commit message should contain '[WM]' prefix: {}", + commit_msg + ); + } + + Ok(()) +} diff --git a/backend/windmill-api-jobs/src/job_metrics.rs b/backend/windmill-api-jobs/src/job_metrics.rs index aba8a5f25c..ad316150c0 100644 --- a/backend/windmill-api-jobs/src/job_metrics.rs +++ b/backend/windmill-api-jobs/src/job_metrics.rs @@ -79,7 +79,7 @@ async fn get_job_metrics( >, ) -> error::JsonResult { let records = sqlx::query_as::<_, JobStatsRecord>( - "SELECT * FROM job_stats where workspace_id = $1 and job_id = $2", + "SELECT workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float, timeseries_start, offsets_cs FROM job_stats WHERE workspace_id = $1 AND job_id = $2", ) .bind(w_id) .bind(job_id) @@ -91,7 +91,7 @@ async fn get_job_metrics( let mut timeseries_metrics: Vec = vec![]; for record in records { - let metric_id = record.metric_id; + let metric_id = record.metric_id.clone(); match record.metric_kind { MetricKind::ScalarInt => { let value = record.scalar_int.unwrap_or_default() as f64; @@ -102,47 +102,43 @@ async fn get_job_metrics( scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value }); } MetricKind::TimeseriesInt => { - if record.timestamps.clone().unwrap_or_default().len() - != record.timeseries_int.clone().unwrap_or_default().len() - { - tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_int) + let timestamps = resolve_timestamps(&record); + let timeseries_int = record.timeseries_int.unwrap_or_default(); + if timestamps.len() != timeseries_int.len() { + tracing::warn!("Timeseries metric {} has an invalid shape. timestamps: {}, measurements: {}", metric_id, timestamps.len(), timeseries_int.len()); } let (timestamps, timeseries_int) = timeseries_sample( from_timestamp, to_timestamp, timeseries_max_datapoints, - record.timestamps.unwrap_or_default(), - record.timeseries_int.unwrap_or_default(), + timestamps, + timeseries_int, ); - let mut values: Vec = vec![]; - for (idx, value) in timeseries_int.iter().enumerate() { - values.push(DataPoint { - timestamp: timestamps[idx], - value: value.to_owned() as f64, - }); - } + let values: Vec = timestamps + .iter() + .zip(timeseries_int.iter()) + .map(|(ts, v)| DataPoint { timestamp: *ts, value: *v as f64 }) + .collect(); timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); } MetricKind::TimeseriesFloat => { - if record.timestamps.clone().unwrap_or_default().len() - != record.timeseries_int.clone().unwrap_or_default().len() - { - tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_float) + let timestamps = resolve_timestamps(&record); + let timeseries_float = record.timeseries_float.unwrap_or_default(); + if timestamps.len() != timeseries_float.len() { + tracing::warn!("Timeseries metric {} has an invalid shape. timestamps: {}, measurements: {}", metric_id, timestamps.len(), timeseries_float.len()); } let (timestamps, timeseries_float) = timeseries_sample( from_timestamp, to_timestamp, timeseries_max_datapoints, - record.timestamps.unwrap_or_default(), - record.timeseries_float.unwrap_or_default(), + timestamps, + timeseries_float, ); - let mut values: Vec = vec![]; - for (idx, value) in timeseries_float.iter().enumerate() { - values.push(DataPoint { - timestamp: timestamps[idx], - value: value.to_owned() as f64, - }); - } + let values: Vec = timestamps + .iter() + .zip(timeseries_float.iter()) + .map(|(ts, v)| DataPoint { timestamp: *ts, value: *v as f64 }) + .collect(); timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); } }; @@ -152,6 +148,21 @@ async fn get_job_metrics( let response = JobStatsResponse { metrics_metadata, scalar_metrics, timeseries_metrics }; Ok(Json(response)) } + +/// Reconstruct full timestamps from `timeseries_start` + `offsets_cs` if available, +/// otherwise fall back to legacy `timestamps` column. +fn resolve_timestamps(record: &JobStatsRecord) -> Vec> { + if let (Some(start), Some(offsets)) = (record.timeseries_start, &record.offsets_cs) { + if !offsets.is_empty() { + return offsets + .iter() + .map(|&cs| start + chrono::Duration::milliseconds(cs as i64 * 10)) + .collect(); + } + } + // Legacy fallback: use the full timestamps column + record.timestamps.clone().unwrap_or_default() +} #[derive(Deserialize)] struct JobProgressSetRequest { percent: i32, diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 04a1eb617b..252c21580e 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -43,8 +43,8 @@ use windmill_common::{ get_database_url, global_settings::{ APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, - CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, - ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, + EMAIL_DOMAIN_SETTING, ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, }, instance_config::{self, ApplyMode, InstanceConfig}, server::Smtp, @@ -316,7 +316,11 @@ pub async fn set_global_setting_internal( ) .execute(db) .await?; - tracing::info!("Set global setting {} to {}", key, v); + tracing::info!( + "Set global setting {} to {}", + key, + instance_config::format_setting_value(&key, &v) + ); } }; @@ -519,6 +523,7 @@ pub async fn get_global_setting( && key != DEFAULT_TAGS_WORKSPACES_SETTING && key != HUB_BASE_URL_SETTING && key != HUB_ACCESSIBLE_URL_SETTING + && key != DISABLE_HUB_SETTING && key != EMAIL_DOMAIN_SETTING && key != APP_WORKSPACED_ROUTE_SETTING { @@ -565,6 +570,7 @@ pub async fn send_stats(Extension(db): Extension, authed: ApiAuthed) -> Resu &HTTP_CLIENT, &db, windmill_common::stats_oss::SendStatsReason::Manual, + false, ) .await?; @@ -577,6 +583,7 @@ pub async fn get_stats(Extension(db): Extension, authed: ApiAuthed) -> Resul let stats = windmill_common::stats_oss::get_stats_payload( &db, &windmill_common::stats_oss::SendStatsReason::Manual, + false, ) .await?; let encrypted = windmill_common::stats_oss::encrypt_stats(&stats)?; @@ -796,11 +803,7 @@ async fn list_custom_instance_pg_databases( return Ok(Json(result)); } -async fn refresh_custom_instance_user_pwd( - authed: ApiAuthed, - Extension(db): Extension, -) -> JsonResult<()> { - require_super_admin(&db, &authed.email).await?; +pub async fn refresh_custom_instance_user_pwd_inner(db: &DB) -> Result<()> { // 20251208123907_safety_custom_instance_db_user_pwd.up let query = r#" DO $$ @@ -808,7 +811,7 @@ async fn refresh_custom_instance_user_pwd( pwd text; BEGIN SELECT gen_random_uuid()::text INTO pwd; - + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_user') THEN EXECUTE format('ALTER USER custom_instance_user WITH PASSWORD %L', pwd); RAISE NOTICE 'Updated password for existing user custom_instance_user'; @@ -833,7 +836,16 @@ async fn refresh_custom_instance_user_pwd( END $$; "#; - sqlx::query(query).execute(&db).await?; + sqlx::query(query).execute(db).await?; + Ok(()) +} + +async fn refresh_custom_instance_user_pwd( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult<()> { + require_super_admin(&db, &authed.email).await?; + refresh_custom_instance_user_pwd_inner(&db).await?; Ok(Json(())) } @@ -1085,7 +1097,7 @@ async fn sync_cached_resource_types( require_super_admin(&db, &authed.email).await?; use windmill_common::worker::HUB_RT_CACHE_DIR; - let cache_path = format!("{}/resource_types.json", HUB_RT_CACHE_DIR); + let cache_path = format!("{}/resource_types.json", *HUB_RT_CACHE_DIR); let content = tokio::fs::read_to_string(&cache_path).await.map_err(|e| { error::Error::NotFound(format!( diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 675ce45e7a..3c5fa40026 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -393,7 +393,7 @@ async fn list_users_as_super_admin( let rows = if active_only.is_some_and(|x| x) { sqlx::query_as!( GlobalUserInfo, - "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')), + "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')), authors as (SELECT distinct email FROM usr WHERE usr.operator IS false) SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user FROM password @@ -1878,6 +1878,14 @@ async fn impersonate( .execute(&mut *tx) .await?; + windmill_api_auth::register_token_expiry_notification( + &mut *tx, + &token, + new_token.label.as_deref(), + new_token.expiration, + ) + .await; + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index b698426d53..a03bb3a490 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -29,6 +29,7 @@ windmill-dep-map.workspace = true axum.workspace = true chrono.workspace = true hex.workspace = true +magic-crypt.workspace = true http.workspace = true hyper.workspace = true lazy_static.workspace = true diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index d6e05b098b..2b3146a6fb 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -31,7 +31,9 @@ use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::users::username_to_permissioned_as; -use windmill_common::variables::{build_crypt, decrypt, encrypt, WORKSPACE_CRYPT_CACHE}; +use windmill_common::variables::{ + build_crypt, decrypt, encrypt, SECRET_SALT, WORKSPACE_CRYPT_CACHE, +}; use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; #[cfg(feature = "enterprise")] use windmill_common::workspaces::GitRepositorySettings; @@ -300,6 +302,8 @@ struct LargeFileStorageWithSecondary { large_file_storage: LargeFileStorage, #[serde(default)] secondary_storage: HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + volume_storage: Option, } #[derive(Deserialize, Debug)] struct EditLargeFileStorageConfig { @@ -2418,20 +2422,28 @@ async fn set_encryption_key( )); } + // Build the previous cipher before the transaction (reads from cache/pool) let previous_encryption_key = build_crypt(&db, w_id.as_str()).await?; + let mut tx = db.begin().await?; + sqlx::query!( "UPDATE workspace_key SET key = $1 WHERE workspace_id = $2", request.new_key.clone(), w_id ) - .execute(&db) + .execute(&mut *tx) .await?; - WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); - if !request.skip_reencrypt.unwrap_or(false) { - let new_encryption_key = build_crypt(&db, w_id.as_str()).await?; + // Build the new cipher directly from the key string, since the transaction + // hasn't committed yet and build_crypt() would read the old key from the pool. + let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { + format!("{}{}", request.new_key, salt) + } else { + request.new_key.clone() + }; + let new_encryption_key = magic_crypt::new_magic_crypt!(crypt_key, 256); let mut truncated_new_key = request.new_key.clone(); truncated_new_key.truncate(8); @@ -2445,7 +2457,7 @@ async fn set_encryption_key( "SELECT path, value, is_secret FROM variable WHERE workspace_id = $1", w_id ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; for variable in all_variables { @@ -2466,11 +2478,16 @@ async fn set_encryption_key( w_id, variable.path ) - .execute(&db) + .execute(&mut *tx) .await?; } } + tx.commit().await?; + + // Invalidate the cache only after the transaction has committed + WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); + // Trigger git sync for encryption key changes handle_deployment_metadata( &authed.email, diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 7af8137630..e6760ea711 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] @@ -63,6 +63,7 @@ windmill-object-store.workspace = true windmill-audit.workspace = true windmill-parser.workspace = true windmill-parser-sql.workspace = true +windmill-parser-sql-asset.workspace = true windmill-parser-ts.workspace = true windmill-parser-py = { workspace = true, optional = true } windmill-parser-py-imports = { workspace = true, optional = true } @@ -70,6 +71,7 @@ windmill-git-sync.workspace = true windmill-indexer = { workspace = true, optional = true } windmill-autoscaling = { workspace = true, optional = true } windmill-worker = { workspace = true, optional = true } +windmill-worker-volumes.workspace = true windmill-dep-map.workspace = true tokio.workspace = true tokio-stream.workspace = true @@ -173,6 +175,7 @@ aws-sdk-bedrock = { workspace = true, optional = true } aws-sdk-bedrockruntime = { workspace = true, optional = true } aws-smithy-types = { workspace = true, optional = true } async-trait.workspace = true +eventsource-stream.workspace = true windmill-jseval.workspace = true tar.workspace = true flate2.workspace = true diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 1a036f6dc9..7f04a7287e 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -8857,9 +8857,8 @@ paths: type: boolean flow_env: type: object - description: Environment variables available to all steps - additionalProperties: - type: string + description: "Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource)." + additionalProperties: {} priority: type: number description: Execution priority (higher numbers run first) @@ -14644,9 +14643,8 @@ paths: type: boolean flow_env: type: object - description: Environment variables available to all steps - additionalProperties: - type: string + description: "Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource)." + additionalProperties: {} priority: type: number description: Execution priority (higher numbers run first) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4dc1d252c4..5fbeacfddf 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.648.0 + version: 1.654.0 title: Windmill API contact: @@ -12004,6 +12004,19 @@ paths: schema: type: string + /w/{workspace}/kafka_triggers/reset_offsets/{path}: + post: + summary: reset kafka trigger offsets to earliest + operationId: resetKafkaOffsets + tags: + - kafka_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: kafka trigger offsets reset successfully + /w/{workspace}/nats_triggers/create: post: summary: create nats trigger @@ -15198,6 +15211,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] responses: "200": @@ -15243,6 +15257,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] requestBody: description: acl to add @@ -15299,6 +15314,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] requestBody: description: acl to add @@ -16873,6 +16889,9 @@ paths: lost_lock_ownership: description: Is the current indexer service being replaced type: boolean + max_index_time_window_secs: + description: Maximum time window in seconds for indexing + type: number /srch/index/search/service_logs: get: @@ -17282,7 +17301,90 @@ paths: path: type: string description: The asset path - + + + /w/{workspace}/volumes/list: + get: + summary: List all volumes in the workspace + operationId: listVolumes + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: list of volumes + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Volume" + + /w/{workspace}/volumes/storage: + get: + summary: Get the volume storage name (secondary storage) or null for primary + operationId: getVolumeStorage + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: volume storage name or null + content: + application/json: + schema: + type: string + nullable: true + + /w/{workspace}/volumes/create: + post: + summary: Create a new volume + operationId: createVolume + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - name + properties: + name: + type: string + responses: + "200": + description: volume created + content: + text/plain: + schema: + type: string + + /w/{workspace}/volumes/delete/{name}: + delete: + summary: Delete a volume (admin only) + operationId: deleteVolume + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: name + in: path + required: true + schema: + type: string + responses: + "200": + description: volume deleted + content: + text/plain: + schema: + type: string /mcp/w/{workspace}/list_tools: get: @@ -18370,6 +18472,7 @@ components: - trigger - settings - key + - workspacedependencies AIProviderModel: type: object @@ -18717,7 +18820,6 @@ components: required: - path - summary - - description - content - language @@ -21972,7 +22074,7 @@ components: description: Path to the Kafka resource containing connection configuration group_id: type: string - description: Kafka consumer group ID for this trigger + description: Kafka consumer group ID for this trigger topics: type: array items: @@ -21989,6 +22091,13 @@ components: required: - key - value + auto_offset_reset: + type: string + enum: + - latest + - earliest + default: latest + description: "Initial offset behavior when consumer group has no committed offset. 'latest' starts from new messages only, 'earliest' starts from the beginning." server_id: type: string description: ID of the server currently handling this trigger (internal) @@ -22049,6 +22158,13 @@ components: required: - key - value + auto_offset_reset: + type: string + enum: + - latest + - earliest + default: latest + description: "Initial offset behavior when consumer group has no committed offset." mode: $ref: "#/components/schemas/TriggerMode" error_handler_path: @@ -22101,6 +22217,13 @@ components: required: - key - value + auto_offset_reset: + type: string + enum: + - latest + - earliest + default: latest + description: "Initial offset behavior when consumer group has no committed offset." path: type: string description: The unique path identifier for this trigger @@ -23997,6 +24120,7 @@ components: - resource - ducklake - datatable + - volume Asset: type: object properties: @@ -24005,6 +24129,38 @@ components: kind: $ref: "#/components/schemas/AssetKind" required: [path, kind] + Volume: + type: object + required: + - name + - size_bytes + - file_count + - created_at + - created_by + properties: + name: + type: string + size_bytes: + type: integer + format: int64 + file_count: + type: integer + created_at: + type: string + format: date-time + created_by: + type: string + updated_at: + type: string + format: date-time + nullable: true + last_used_at: + type: string + format: date-time + nullable: true + extra_perms: + type: object + additionalProperties: true ProtectionRuleset: type: object description: A workspace protection rule defining restrictions and bypass permissions diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 730df2a875..1dca68f219 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use std::time::Duration; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::ai_providers::{ - empty_string_as_none, AIProvider, ProviderConfig, ProviderModel, + empty_string_as_none, AIPlatform, AIProvider, ProviderConfig, ProviderModel, }; use windmill_common::error::{to_anyhow, Error, Result}; use windmill_common::utils::configure_client; @@ -29,7 +29,7 @@ const AI_TIMEOUT_MAX_SECS: u64 = 86400; // 24 hours const AI_TIMEOUT_DEFAULT_SECS: u64 = 3600; // 1 hour const HTTP_POOL_MAX_IDLE_PER_HOST: usize = 10; const HTTP_POOL_IDLE_TIMEOUT_SECS: u64 = 90; -const KEEPALIVE_INTERVAL_SECS: u64 = 15; +pub(crate) const KEEPALIVE_INTERVAL_SECS: u64 = 15; lazy_static::lazy_static! { /// AI request timeout in seconds. @@ -87,7 +87,7 @@ lazy_static::lazy_static! { } }; - static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() + pub(crate) static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() .timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS)) .pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST) .pool_idle_timeout(Some(std::time::Duration::from_secs(HTTP_POOL_IDLE_TIMEOUT_SECS))) @@ -135,15 +135,6 @@ struct AIOAuthResource { user: Option, } -/// Platform for Anthropic API -#[derive(Deserialize, Debug, Clone, Default, PartialEq)] -#[serde(rename_all = "snake_case")] -enum AnthropicPlatform { - #[default] - Standard, - GoogleVertexAi, -} - #[derive(Deserialize, Debug)] struct AIStandardResource { #[serde(alias = "baseUrl", default, deserialize_with = "empty_string_as_none")] @@ -172,9 +163,9 @@ struct AIStandardResource { deserialize_with = "empty_string_as_none" )] aws_session_token: Option, - /// Platform for Anthropic API (standard or google_vertex_ai) + /// Platform (standard or google_vertex_ai) #[serde(default)] - platform: AnthropicPlatform, + platform: AIPlatform, /// Enable 1M context window for Anthropic #[serde(alias = "enable_1M_context", default)] enable_1m_context: bool, @@ -207,7 +198,7 @@ struct AIRequestConfig { pub aws_secret_access_key: Option, #[allow(dead_code)] pub aws_session_token: Option, - pub platform: AnthropicPlatform, + pub platform: AIPlatform, pub enable_1m_context: bool, } @@ -301,7 +292,7 @@ impl AIRequestConfig { None, None, None, - AnthropicPlatform::Standard, + AIPlatform::Standard, false, ) } @@ -374,16 +365,11 @@ impl AIRequestConfig { let is_azure = provider.is_azure_openai(base_url); let is_anthropic = matches!(provider, AIProvider::Anthropic); let is_anthropic_vertex = - is_anthropic && self.platform == AnthropicPlatform::GoogleVertexAi; + is_anthropic && self.platform == AIPlatform::GoogleVertexAi; let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some(); let is_google_ai = matches!(provider, AIProvider::GoogleAI); - // GoogleAI uses OpenAI-compatible endpoint in the proxy (for the chat), but not for the ai agent - let base_url = if is_google_ai { - format!("{}/openai", base_url) - } else { - base_url.to_string() - }; + let base_url = base_url.to_string(); let base_url = base_url.as_str(); // Build URL based on provider @@ -428,6 +414,11 @@ impl AIRequestConfig { if let Some(api_key) = self.api_key { if is_azure { request = request.header("api-key", api_key.clone()) + } else if is_google_ai { + // Note: GoogleAI requests are intercepted earlier (see the GoogleAI + // handler block above) and never reach this code path. This branch + // is kept as a safety net for the standard Gemini API auth format. + request = request.header("x-goog-api-key", api_key.clone()) } else { request = request.header("authorization", format!("Bearer {}", api_key.clone())) } @@ -611,7 +602,7 @@ fn is_sse_response(headers: &HeaderMap) -> bool { .unwrap_or(false) } -fn inject_keepalives( +pub(crate) fn inject_keepalives( upstream: S, interval: Duration, ) -> impl futures::Stream> @@ -830,6 +821,39 @@ async fn proxy( ai_path = chat_path; } + // Handle GoogleAI (Gemini) using the native Gemini API + if matches!(provider, AIProvider::GoogleAI) { + let api_key = request_config.api_key.as_deref().unwrap_or(""); + let base_url = request_config.base_url.trim_end_matches('/'); + let is_vertex = request_config.platform == AIPlatform::GoogleVertexAi; + + let mut tx = db.begin().await?; + audit_log( + &mut *tx, + &authed, + "ai.request", + ActionKind::Execute, + &w_id, + Some(&authed.email), + Some([("ai_config_path", &format!("{:?}", ai_path)[..])].into()), + ) + .await?; + tx.commit().await?; + + return match ai_path.as_str() { + "chat/completions" => { + crate::google::handle_google_ai_chat(&body, api_key, base_url, is_vertex).await + } + "models" => { + crate::google::handle_google_ai_models(api_key, base_url, is_vertex).await + } + _ => Err(Error::BadRequest(format!( + "Unsupported Google AI path: {}", + ai_path + ))), + }; + } + // Handle Bedrock-specific logic when the feature is enabled #[cfg(feature = "bedrock")] { diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index d156dc9c88..9846912192 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -8,7 +8,7 @@ use std::{collections::HashMap, sync::Arc}; * LICENSE-AGPL for a copy of the license. */ use crate::{ - auth::OptTokened, + auth::{get_end_user_email, OptTokened}, db::{ApiAuthed, DB}, jobs::RunJobQuery, users::{require_owner_of_path, OptAuthed}, @@ -993,9 +993,18 @@ macro_rules! process_app_multipart { let mut uploaded_js = false; let mut multipart = $multipart; - while let Some(field) = multipart.next_field().await.unwrap() { - let name = field.name().unwrap().to_string(); - let data = field.bytes().await.unwrap(); + while let Some(field) = multipart + .next_field() + .await + .map_err(|e| Error::BadRequest(format!("failed to read multipart field: {e}")))? + { + let name = field + .name() + .ok_or_else(|| Error::BadRequest("multipart field missing name".to_string()))? + .to_string(); + let data = field.bytes().await.map_err(|e| { + Error::BadRequest(format!("failed to read multipart stream: {e}")) + })?; if name == "app" { let app = serde_json::from_slice(&data).map_err(to_anyhow)?; let (ntx, npath, nid) = $internal_fn( @@ -2149,7 +2158,8 @@ async fn execute_component( (email.as_str(), permissioned_as) }; - let end_user_email = opt_authed.as_ref().map(|a| a.email.clone()); + let end_user_email = + get_end_user_email(&db, opt_authed.as_ref(), tokened.token.as_deref()).await; let (uuid, mut tx) = push( &db, diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 144703ba5c..66a67c5f97 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -1,4 +1,5 @@ pub use windmill_api_auth::auth::{ - invalidate_token_from_cache, list_tokens_internal, transform_old_scope_to_new_scope, AuthCache, - ExpiringAuthCache, OptTokened, Tokened, TruncatedTokenWithEmail, + get_end_user_email, invalidate_token_from_cache, list_tokens_internal, + transform_old_scope_to_new_scope, AuthCache, ExpiringAuthCache, OptTokened, Tokened, + TruncatedTokenWithEmail, }; diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 8b0fb44dfe..c8ed841e19 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -284,6 +284,9 @@ pub async fn migrate( 20260207000004, ]; for m in migrator.migrations.iter() { + if m.migration_type.is_down_migration() { + continue; + } if potentially_stale.contains(&m.version) { if let Err(err) = sqlx::query("DELETE FROM _sqlx_migrations WHERE version = $1 AND checksum != $2") diff --git a/backend/windmill-api/src/google.rs b/backend/windmill-api/src/google.rs new file mode 100644 index 0000000000..dd2a47bca3 --- /dev/null +++ b/backend/windmill-api/src/google.rs @@ -0,0 +1,354 @@ +//! Google AI (Gemini API) handler for the AI chat proxy. +//! +//! Handles POST `chat/completions` requests using the native Gemini API, +//! converting from/to OpenAI format so the existing frontend parsers continue to work. +//! +//! Supports both standard Google AI (generativelanguage.googleapis.com) and +//! Google Vertex AI ({region}-aiplatform.googleapis.com) endpoints. +//! +//! Used by `windmill-api/src/ai.rs` when the provider is `GoogleAI`. +//! Shared conversion logic lives in `windmill_common::ai_google`. + +use axum::body::Body; +use bytes::Bytes; +use eventsource_stream::Eventsource; +use futures::StreamExt; +use serde::Deserialize; +use serde_json::json; +use windmill_common::{ + ai_google::{ + gemini_event_to_openai_sse_chunks, gemini_response_to_openai, openai_messages_to_gemini, + parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google, + GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiTextRequest, GeminiTool, + }, + ai_types::OpenAIMessage, + error::{Error, Result}, +}; + +use crate::ai::{inject_keepalives, HTTP_CLIENT, KEEPALIVE_INTERVAL_SECS}; + +// ============================================================================ +// Request type (OpenAI format received from the frontend) +// ============================================================================ + +#[derive(Deserialize, Debug)] +struct ChatRequest { + model: String, + messages: Vec, + #[serde(default)] + stream: bool, + #[serde(default)] + temperature: Option, + #[serde(default)] + max_tokens: Option, + #[serde(default)] + tools: Option>, +} + +#[derive(Deserialize, Debug)] +struct ChatRequestTool { + function: ChatRequestToolFunction, +} + +#[derive(Deserialize, Debug)] +struct ChatRequestToolFunction { + name: String, + #[serde(default)] + description: Option, + #[serde(default)] + parameters: Option, +} + +// ============================================================================ +// Helpers for Vertex AI vs standard Google AI URL/auth +// ============================================================================ + +/// Build the endpoint URL for a model action (streamGenerateContent, generateContent, predict). +/// +/// - Standard: `{base_url}/models/{model}:{action}` +/// - Vertex AI: `{base_url}/{model}:{action}` (base_url already contains .../publishers/google/models) +fn build_model_endpoint(base_url: &str, model: &str, action: &str, is_vertex: bool) -> String { + if is_vertex { + format!("{}/{}:{}", base_url, model, action) + } else { + format!("{}/models/{}:{}", base_url, model, action) + } +} + +/// Set the appropriate auth header on a request builder. +/// +/// - Standard: `x-goog-api-key` header +/// - Vertex AI: `Authorization: Bearer` header +fn set_auth( + request: reqwest::RequestBuilder, + api_key: &str, + is_vertex: bool, +) -> reqwest::RequestBuilder { + if is_vertex { + request.header("Authorization", format!("Bearer {}", api_key)) + } else { + request.header("x-goog-api-key", api_key) + } +} + +// ============================================================================ +// Public handler +// ============================================================================ + +/// Handle a `chat/completions` POST request using the native Gemini API. +/// +/// Converts the incoming OpenAI-format body to a `GeminiTextRequest`, sends it +/// to the appropriate Gemini endpoint, and converts the response back to the +/// OpenAI SSE or JSON format that the frontend expects. +pub async fn handle_google_ai_chat( + body: &Bytes, + api_key: &str, + base_url: &str, + is_vertex: bool, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + let request: ChatRequest = serde_json::from_slice(body) + .map_err(|e| Error::BadRequest(format!("Failed to parse request body: {}", e)))?; + + let (contents, system_instruction) = openai_messages_to_gemini(&request.messages); + + let generation_config = + if request.temperature.is_some() || request.max_tokens.is_some() { + Some(GeminiGenerationConfig { + temperature: request.temperature, + max_output_tokens: request.max_tokens, + response_mime_type: None, + response_schema: None, + }) + } else { + None + }; + + let gemini_tools = request.tools.as_ref().map(|tools| { + let declarations: Vec = tools + .iter() + .map(|t| { + let mut params = t.function.parameters.clone().unwrap_or(json!({})); + sanitize_schema_for_google(&mut params); + GeminiFunctionDeclaration { + name: t.function.name.clone(), + description: t.function.description.clone(), + parameters: params, + } + }) + .collect(); + vec![GeminiTool { + function_declarations: Some(declarations), + google_search: None, + }] + }); + + let gemini_request = GeminiTextRequest { + contents, + tools: gemini_tools, + tool_config: None, + system_instruction, + generation_config, + }; + + let request_body = serde_json::to_string(&gemini_request) + .map_err(|e| Error::internal_err(format!("Failed to serialize Gemini request: {}", e)))?; + + let base_url = base_url.trim_end_matches('/'); + + if request.stream { + handle_streaming(&request.model, request_body, api_key, base_url, is_vertex).await + } else { + handle_non_streaming(&request.model, request_body, api_key, base_url, is_vertex).await + } +} + +// ============================================================================ +// Streaming path +// ============================================================================ + +async fn handle_streaming( + model: &str, + request_body: String, + api_key: &str, + base_url: &str, + is_vertex: bool, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + let endpoint = format!( + "{}?alt=sse", + build_model_endpoint(base_url, model, "streamGenerateContent", is_vertex) + ); + + let request = HTTP_CLIENT + .post(&endpoint) + .header("content-type", "application/json") + .body(request_body); + let request = set_auth(request, api_key, is_vertex); + + let response = request.send().await.map_err(|e| { + Error::internal_err(format!("Failed to send request to Gemini API: {}", e)) + })?; + + if let Err(e) = response.error_for_status_ref() { + let status = e.status().map(|s| s.to_string()).unwrap_or_default(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::AIError(format!("{}: {}", status, body))); + } + + let id = format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()); + let model_str = model.to_string(); + + let gemini_sse_stream = response.bytes_stream().eventsource(); + let openai_sse_stream = async_stream::stream! { + tokio::pin!(gemini_sse_stream); + let mut tool_call_index: usize = 0; + while let Some(event) = gemini_sse_stream.next().await { + match event { + Ok(event) => match parse_gemini_sse_event(&event.data) { + Ok(Some(parsed)) => { + for chunk in gemini_event_to_openai_sse_chunks( + &parsed, &id, &model_str, &mut tool_call_index, + ) { + yield Ok::(Bytes::from(chunk)); + } + } + Ok(None) => {} + Err(e) => tracing::error!("Error parsing Gemini SSE event: {}", e), + }, + Err(e) => tracing::error!("Error reading Gemini SSE stream: {}", e), + } + } + yield Ok::(Bytes::from("data: [DONE]\n\n")); + }; + + let mut headers = http::HeaderMap::new(); + headers.insert("content-type", "text/event-stream".parse().unwrap()); + headers.insert("cache-control", "no-cache".parse().unwrap()); + headers.insert("connection", "keep-alive".parse().unwrap()); + + Ok(( + http::StatusCode::OK, + headers, + Body::from_stream(inject_keepalives( + Box::pin(openai_sse_stream), + std::time::Duration::from_secs(KEEPALIVE_INTERVAL_SECS), + )), + )) +} + +// ============================================================================ +// Model listing +// ============================================================================ + +/// List available Gemini models and convert to OpenAI format. +/// +/// - Standard: `GET {base_url}/models` — returns `{ models: [...] }` +/// - Vertex AI: `GET {base_url}` — returns `{ models: [...] }` (base_url already ends with .../models) +pub async fn handle_google_ai_models( + api_key: &str, + base_url: &str, + is_vertex: bool, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + #[derive(Deserialize)] + struct GeminiModel { + name: String, + #[serde(rename = "displayName", default)] + display_name: String, + } + + #[derive(Deserialize)] + struct GeminiModelsResponse { + #[serde(default)] + models: Vec, + } + + let base_url = base_url.trim_end_matches('/'); + let endpoint = if is_vertex { + // Vertex AI: base_url is .../publishers/google/models + base_url.to_string() + } else { + // Standard: append /models + format!("{}/models", base_url) + }; + + let request = HTTP_CLIENT.get(&endpoint); + let request = set_auth(request, api_key, is_vertex); + + let response = request.send().await.map_err(|e| { + Error::internal_err(format!("Failed to fetch Gemini models: {}", e)) + })?; + + if let Err(e) = response.error_for_status_ref() { + let status = e.status().map(|s| s.to_string()).unwrap_or_default(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::AIError(format!("{}: {}", status, body))); + } + + let gemini_resp: GeminiModelsResponse = response.json().await.map_err(|e| { + Error::internal_err(format!("Failed to parse Gemini models response: {}", e)) + })?; + + let data: Vec = gemini_resp + .models + .into_iter() + .map(|m| { + json!({ + "id": m.name, + "object": "model", + "display_name": m.display_name, + }) + }) + .collect(); + + let body_bytes = serde_json::to_vec(&json!({ "data": data })) + .map_err(|e| Error::internal_err(format!("Failed to serialize models: {}", e)))?; + + let mut headers = http::HeaderMap::new(); + headers.insert("content-type", "application/json".parse().unwrap()); + + Ok((http::StatusCode::OK, headers, Body::from(body_bytes))) +} + +// ============================================================================ +// Non-streaming path +// ============================================================================ + +async fn handle_non_streaming( + model: &str, + request_body: String, + api_key: &str, + base_url: &str, + is_vertex: bool, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + let endpoint = build_model_endpoint(base_url, model, "generateContent", is_vertex); + + let request = HTTP_CLIENT + .post(&endpoint) + .header("content-type", "application/json") + .body(request_body); + let request = set_auth(request, api_key, is_vertex); + + let response = request.send().await.map_err(|e| { + Error::internal_err(format!("Failed to send request to Gemini API: {}", e)) + })?; + + if let Err(e) = response.error_for_status_ref() { + let status = e.status().map(|s| s.to_string()).unwrap_or_default(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::AIError(format!("{}: {}", status, body))); + } + + let body = response.bytes().await.map_err(|e| { + Error::internal_err(format!("Failed to read Gemini response body: {}", e)) + })?; + + let parsed = parse_gemini_response(&body)?; + let openai_response = gemini_response_to_openai(&parsed, model); + + let body_bytes = serde_json::to_vec(&openai_response) + .map_err(|e| Error::internal_err(format!("Failed to serialize response: {}", e)))?; + + let mut headers = http::HeaderMap::new(); + headers.insert("content-type", "application/json".parse().unwrap()); + + Ok((http::StatusCode::OK, headers, Body::from(body_bytes))) +} diff --git a/backend/windmill-api/src/health.rs b/backend/windmill-api/src/health.rs index d861f338ef..60bd290f66 100644 --- a/backend/windmill-api/src/health.rs +++ b/backend/windmill-api/src/health.rs @@ -240,11 +240,7 @@ async fn check_database_detailed(db: &DB) -> DatabaseHealth { let check = check_database_with_latency(db).await; let pool = get_pool_stats(db); - DatabaseHealth { - healthy: check.healthy, - latency_ms: check.latency_ms, - pool, - } + DatabaseHealth { healthy: check.healthy, latency_ms: check.latency_ms, pool } } async fn check_worker_count(db: &DB) -> i64 { @@ -295,13 +291,7 @@ async fn check_workers_detailed(db: &DB) -> WorkersHealth { let healthy = active_count > 0; - WorkersHealth { - healthy, - active_count, - worker_groups, - min_version, - versions, - } + WorkersHealth { healthy, active_count, worker_groups, min_version, versions } } async fn check_queue(db: &DB) -> QueueHealth { @@ -333,10 +323,7 @@ fn get_version() -> String { /// Spawn a background task that performs a health check every 10 seconds. /// Updates the cache and prometheus metrics continuously. -pub fn start_health_check_loop( - db: DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { +pub fn start_health_check_loop(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(10)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -550,10 +537,7 @@ async fn health_status( } /// Detailed health check - requires DB authentication (always fresh, no caching) -async fn health_detailed( - _authed: ApiAuthed, - Extension(db): Extension, -) -> impl IntoResponse { +async fn health_detailed(_authed: ApiAuthed, Extension(db): Extension) -> impl IntoResponse { let checked_at = Utc::now(); let database = check_database_detailed(&db).await; let readiness = check_readiness(); @@ -564,12 +548,7 @@ async fn health_detailed( status: HealthStatus::Unhealthy, checked_at, version: get_version(), - checks: HealthChecks { - database, - workers: None, - queue: None, - readiness, - }, + checks: HealthChecks { database, workers: None, queue: None, readiness }, }; return (StatusCode::SERVICE_UNAVAILABLE, Json(response)); } @@ -587,12 +566,7 @@ async fn health_detailed( status, checked_at, version: get_version(), - checks: HealthChecks { - database, - workers: Some(workers), - queue: Some(queue), - readiness, - }, + checks: HealthChecks { database, workers: Some(workers), queue: Some(queue), readiness }, }; let status_code = if status == HealthStatus::Unhealthy { diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 01f5d67bd3..23f20459c3 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -12,15 +12,15 @@ use windmill_types::s3::StorageResourceType; #[cfg(all(feature = "parquet", not(feature = "private")))] use crate::db::{ApiAuthed, OptJobAuthed, DB}; #[cfg(all(feature = "parquet", not(feature = "private")))] -use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult}; -#[cfg(not(feature = "private"))] -use windmill_object_store::ObjectStoreResource; -#[cfg(all(feature = "parquet", not(feature = "private")))] use std::sync::Arc; +#[cfg(all(feature = "parquet", not(feature = "private")))] +use windmill_common::db::UserDB; #[cfg(not(feature = "private"))] use windmill_common::error; #[cfg(all(feature = "parquet", not(feature = "private")))] -use windmill_common::db::UserDB; +use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult}; +#[cfg(not(feature = "private"))] +use windmill_object_store::ObjectStoreResource; #[cfg(all(feature = "parquet", not(feature = "private")))] use bytes::Bytes; diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 420bbf24b9..a9cb930b8f 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -47,7 +47,7 @@ use windmill_common::runtime_assets::{register_runtime_asset, InsertRuntimeAsset use windmill_common::scripts::ScriptRunnableSettingsInline; use windmill_common::triggers::TriggerMetadata; use windmill_common::utils::{RunnableKind, WarnAfterExt}; -use windmill_common::worker::{Connection, CLOUD_HOSTED, TMP_DIR}; +use windmill_common::worker::{Connection, CLOUD_HOSTED, WINDMILL_DIR}; use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES, }; @@ -448,14 +448,15 @@ async fn get_flow_env_by_flow_job_id( Path((w_id, flow_job_id, var_name)): Path<(String, Uuid, String)>, Query(JsonPath { json_path, .. }): Query, ) -> windmill_common::error::JsonResult> { - let flow_env = sqlx::query_scalar!( + // Fetch raw value (without json_path) to check for $var:/$res: references + let raw_value = sqlx::query_scalar!( r#" SELECT CASE WHEN flow_version.id IS NOT NULL THEN - (flow_version.value -> 'flow_env' -> $3) #> $4 + flow_version.value -> 'flow_env' -> $3 ELSE - (root_job.raw_flow -> 'flow_env' -> $3) #> $4 + root_job.raw_flow -> 'flow_env' -> $3 END AS "flow_env: sqlx::types::Json>" FROM v2_job current_job @@ -472,16 +473,86 @@ async fn get_flow_env_by_flow_job_id( flow_job_id, w_id, var_name, - json_path - .as_ref() - .map(|x| x.split(".").collect::>()) - .unwrap_or_default() as Vec<&str>, ) .fetch_optional(&db) .await? - .map(|r| r.map(|x| x.0)) - .flatten() - .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + .and_then(|r| r.map(|x| x.0)); + + // Resolve $var:/$res: references if present + let resolved = if let Some(raw) = raw_value { + let raw_str = raw.get(); + let db_authed = windmill_common::db::DbWithOptAuthed::::from_authed( + &authed, + db.clone(), + None, + ); + if let Some(path) = raw_str + .strip_prefix("\"$var:") + .and_then(|s| s.strip_suffix("\"")) + { + match windmill_store::variables::get_value_internal(&db_authed, &w_id, path, false) + .await + { + Ok(val) => to_raw_value(&serde_json::Value::String(val)), + Err(e) => { + tracing::warn!("Failed to resolve flow_env variable $var:{path}: {e}"); + raw + } + } + } else if let Some(path) = raw_str + .strip_prefix("\"$res:") + .and_then(|s| s.strip_suffix("\"")) + { + match windmill_store::resources::get_resource_value_interpolated_internal( + &db_authed, + &w_id, + path, + Some(flow_job_id), + Some(&tokened.token), + false, + ) + .await + { + Ok(Some(val)) => to_raw_value(&val), + Ok(None) => { + tracing::warn!( + "Failed to resolve flow_env resource $res:{path}: resource not found" + ); + raw + } + Err(e) => { + tracing::warn!("Failed to resolve flow_env resource $res:{path}: {e}"); + raw + } + } + } else { + raw + } + } else { + to_raw_value(&serde_json::Value::Null) + }; + + // Apply json_path navigation on the (possibly resolved) value + let flow_env = if let Some(ref jp) = json_path { + let mut value: serde_json::Value = + serde_json::from_str(resolved.get()).unwrap_or(serde_json::Value::Null); + for part in jp.split('.') { + value = match value { + serde_json::Value::Object(ref mut map) => { + map.remove(part).unwrap_or(serde_json::Value::Null) + } + serde_json::Value::Array(ref arr) => part + .parse::() + .ok() + .and_then(|i| arr.get(i).cloned()) + .unwrap_or(serde_json::Value::Null), + _ => serde_json::Value::Null, + }; + } + to_raw_value(&value) + } else { + resolved + }; log_job_view( &db, @@ -1412,7 +1483,7 @@ async fn get_logs_from_disk( if log_offset > 0 { if let Some(file_index) = log_file_index.clone() { for file_p in &file_index { - if !tokio::fs::metadata(format!("{TMP_DIR}/{file_p}")) + if !tokio::fs::metadata(format!("{}/{file_p}", *WINDMILL_DIR)) .await .is_ok() { @@ -1427,7 +1498,7 @@ async fn get_logs_from_disk( "#.to_string(), )); for file_p in file_index.clone() { - let mut file = tokio::fs::File::open(format!("{TMP_DIR}/{file_p}")).await.map_err(to_anyhow)?; + let mut file = tokio::fs::File::open(format!("{}/{file_p}", *WINDMILL_DIR)).await.map_err(to_anyhow)?; let mut buffer = Vec::new(); file.read_to_end(&mut buffer).await.map_err(to_anyhow)?; yield Ok(bytes::Bytes::from(buffer)) as anyhow::Result; @@ -2184,12 +2255,13 @@ async fn resume_suspended_job_internal( let value = value.unwrap_or(serde_json::Value::Null); verify_suspended_secret(&w_id, &db, job_id, resume_id, &approver, secret).await?; - // Get flow info - works for both step-level (job_id is a step) and flow-level (job_id is the flow) - let (flow_info, is_flow_level) = get_flow_info_for_resume(job_id, &db).await?; + // Get flow info - works for step-level, flow-level, and WAC approval + let (flow_info, is_flow_level, is_wac) = get_flow_info_for_resume(job_id, &db).await?; // For step-level resumes, verify user auth and flow status // For flow-level resumes (pre-approvals), the flow might not be at a suspended step yet - if !is_flow_level { + // For WAC approvals, skip flow status checks (there is no flow) + if !is_flow_level && !is_wac { let parent_flow = GetQuery::new() .without_logs() .without_code() @@ -2251,6 +2323,16 @@ async fn resume_suspended_job_internal( ) .execute(&mut *tx) .await?; + } else if is_wac { + // WAC approval: decrement suspend counter directly on the WAC parent job + if flow_info.suspend > 0 { + sqlx::query!( + "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1", + flow_info.id, + ) + .execute(&mut *tx) + .await?; + } } else if is_flow_level { // For flow-level resumes, decrement the suspend counter if the flow is currently suspended // The approval will be matched when the worker checks for resumes (both step-level and flow-level) @@ -2408,10 +2490,15 @@ struct FlowInfo { email: Option, } -/// Get flow info from either a step job (by looking up its parent) or a flow job directly. -/// Returns (FlowInfo, is_flow_level) where is_flow_level indicates if job_id was a flow job. -async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowInfo, bool)> { - // Single query that determines if job_id is a flow or step, and fetches the appropriate flow info +/// Get flow info from either a step job (by looking up its parent), a flow job directly, +/// or a WAC workflow job (self-suspended for approval). +/// Returns (FlowInfo, is_flow_level, is_wac) where: +/// - is_flow_level: job_id was a flow job (pre-approval) +/// - is_wac: job_id is a WAC workflow suspended for approval (target is itself) +async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowInfo, bool, bool)> { + // Single query that determines if job_id is a flow, step, or WAC job, + // and fetches the appropriate suspended job info. + // For WAC jobs (no parent, not a flow), the job itself is the suspended target. let result = sqlx::query!( r#" WITH job_info AS ( @@ -2425,14 +2512,15 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI q.suspend AS "suspend!", j.runnable_path AS script_path, j.permissioned_as_email AS email, - (ji.kind IN ('flow', 'flowpreview')) AS "is_flow_level!" + (ji.kind IN ('flow', 'flowpreview')) AS "is_flow_level!", + (ji.kind NOT IN ('flow', 'flowpreview') AND q.id = ji.id) AS "is_wac!" FROM job_info ji JOIN v2_job_queue q ON q.id = CASE WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id - ELSE ji.parent_job + ELSE COALESCE(ji.parent_job, ji.id) END JOIN v2_job j ON j.id = q.id - JOIN v2_job_status s ON s.id = q.id + LEFT JOIN v2_job_status s ON s.id = q.id FOR UPDATE OF q "#, job_id, @@ -2449,7 +2537,7 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI email: Some(result.email), }; - Ok((flow_info, result.is_flow_level)) + Ok((flow_info, result.is_flow_level, result.is_wac)) } async fn get_suspended_flow_info<'c>( @@ -4745,7 +4833,7 @@ fn register_potential_assets_on_inline_execution( preview: &PreviewInline, ) { let assets = if preview.language == ScriptLang::DuckDb { - Some(windmill_parser_sql::parse_assets(&preview.content).map(|a| a.assets)) + Some(windmill_parser_sql_asset::parse_assets(&preview.content).map(|a| a.assets)) } else if preview.language == ScriptLang::Postgresql { let datatable = preview .args @@ -4763,7 +4851,7 @@ fn register_potential_assets_on_inline_execution( (None, None) }; let content = content.as_deref().unwrap_or(&preview.content); - windmill_parser_sql::parse_wmill_sdk_sql_assets( + windmill_parser_sql_asset::parse_wmill_sdk_sql_assets( AssetKind::DataTable, datatable, schema.as_deref(), @@ -5888,7 +5976,7 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R )); } - let local_file = format!("{TMP_DIR}/logs/{file_p}"); + let local_file = format!("{}/logs/{file_p}", *WINDMILL_DIR); if tokio::fs::metadata(&local_file).await.is_ok() { let mut file = tokio::fs::File::open(local_file).await.map_err(to_anyhow)?; let mut buffer = Vec::new(); @@ -5934,10 +6022,10 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R } #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - return Err(error::Error::NotFound(format!( - "File not found on server logs volume /tmp/windmill/logs and no distributed logs s3 storage for {}", - file_p - ))); + return Err(error::Error::NotFound(format!( + "File not found on server logs volume {}/logs and no distributed logs s3 storage for {}", + *WINDMILL_DIR, file_p + ))); } async fn get_job_update( diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 9b713cad0a..a25a431dba 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -64,6 +64,7 @@ use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; mod ai; +mod google; mod apps; pub mod args; mod audit; @@ -170,6 +171,9 @@ pub mod users_ee; mod users_oss; mod utils; mod variables; +#[cfg(feature = "private")] +pub mod volumes_ee; +mod volumes_oss; pub mod webhook_util; mod workspaces; #[cfg(feature = "private")] @@ -248,6 +252,74 @@ type IndexReader = windmill_indexer::completed_runs_oss::IndexReader; #[cfg(feature = "tantivy")] type ServiceLogIndexReader = windmill_indexer::service_logs_oss::ServiceLogIndexReader; +/// Worker name derived from the agent JWT token, used to authenticate volume operations. +/// Defined unconditionally so volume endpoint handlers can reference it regardless of +/// whether agent_worker_server is enabled (the extension is only populated on the agent path). +#[derive(Clone)] +pub struct AgentWorkerName(pub String); + +/// Middleware that injects a synthetic `ApiAuthed` and JWT-derived worker name +/// into request extensions. +/// +/// Used for volume proxy endpoints under the agent_workers path, where the +/// agent JWT auth layer has already validated the request. The volume handlers +/// need `ApiAuthed` to resolve the workspace S3 client, but the agent JWT +/// format is incompatible with the standard auth extractor. +/// +/// The worker name is extracted from the JWT claims rather than trusting +/// self-reported values in request bodies/query params. +#[cfg(feature = "agent_worker_server")] +async fn inject_agent_authed( + request: axum::extract::Request, + next: axum::middleware::Next, +) -> Response { + let mut request = request; + + // Extract worker name from agent JWT via AgentCache + // (OSS returns None; EE decodes the JWT and returns the worker name) + { + let extracted = { + let token = request + .headers() + .get(axum::http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.strip_prefix("Bearer ").map(|t| t.to_string())); + let cache = request.extensions().get::>().cloned(); + let db = request.extensions().get::().cloned(); + match (token, cache, db) { + (Some(token), Some(cache), Some(db)) => Some((token, cache, db)), + _ => None, + } + }; + + if let Some((token, cache, db)) = extracted { + if let Some(worker_name) = cache.extract_worker_name(&token, &db).await { + request + .extensions_mut() + .insert(AgentWorkerName(worker_name)); + } + } + } + + request + .extensions_mut() + .insert(windmill_api_auth::OptJobAuthed { + authed: ApiAuthed { + email: "agent-worker@windmill.dev".to_string(), + username: "agent-worker".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + token_prefix: None, + }, + job_id: None, + }); + next.run(request).await +} + pub async fn run_server( db: DB, job_index_reader: Option, @@ -262,7 +334,7 @@ pub async fn run_server( ) -> anyhow::Result<()> { let user_db = UserDB::new(db.clone()); - for x in [HUB_CACHE_DIR] { + for x in [&*HUB_CACHE_DIR] { DirBuilder::new() .recursive(true) .create(x) @@ -513,6 +585,7 @@ pub async fn run_server( users::workspaced_service().layer(Extension(argon2.clone())), ) .nest("/variables", variables::workspaced_service()) + .nest("/volumes", volumes_oss::workspaced_service()) .nest("/workers", windmill_api_workers::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) .nest("/oidc", oidc_oss::workspaced_service()) @@ -626,7 +699,13 @@ pub async fn run_server( .nest("/w/:workspace_id/agent_workers", { #[cfg(feature = "agent_worker_server")] { - agent_workers_router.layer(Extension(agent_cache.clone())) + agent_workers_router + .nest( + "/volumes", + volumes_oss::agent_workspaced_service() + .layer(axum::middleware::from_fn(inject_agent_authed)), + ) + .layer(Extension(agent_cache.clone())) } #[cfg(not(feature = "agent_worker_server"))] { diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 604db2dc3e..9a323deef6 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -64,9 +64,10 @@ impl McpBackend for WindmillBackend { auth: &ApiAuthed, workspace_id: &str, favorites_only: bool, + path_prefix: Option<&str>, ) -> BackendResult> { let scope_type = if favorites_only { "favorites" } else { "all" }; - get_items::(&self.user_db, auth, workspace_id, scope_type, "script") + get_items::(&self.user_db, auth, workspace_id, scope_type, "script", path_prefix) .await .map_err(|e| ErrorData::internal_error(e.message, None)) } @@ -76,9 +77,10 @@ impl McpBackend for WindmillBackend { auth: &ApiAuthed, workspace_id: &str, favorites_only: bool, + path_prefix: Option<&str>, ) -> BackendResult> { let scope_type = if favorites_only { "favorites" } else { "all" }; - get_items::(&self.user_db, auth, workspace_id, scope_type, "flow") + get_items::(&self.user_db, auth, workspace_id, scope_type, "flow", path_prefix) .await .map_err(|e| ErrorData::internal_error(e.message, None)) } diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index d168f8fc48..1cfbfd7085 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -136,6 +136,7 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen workspace_id: &str, scope_type: &str, item_type: &str, + path_prefix: Option<&str>, ) -> Result, ErrorData> { let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); let fields = vec!["o.path", "o.summary", "o.description", "o.schema"]; @@ -153,6 +154,11 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)"); } + if let Some(prefix) = path_prefix { + let escaped = prefix.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_"); + sqlb.and_where("o.path LIKE ? ESCAPE '\\'".bind(&format!("{}%", escaped))); + } + sqlb.order_by( if item_type == "flow" { "o.edited_at" diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs index 20d2b58d0e..c83bb21f2c 100644 --- a/backend/windmill-api/src/service_logs.rs +++ b/backend/windmill-api/src/service_logs.rs @@ -102,7 +102,11 @@ async fn get_log_file( #[cfg(feature = "parquet")] if let Some(s3_client) = s3_client { let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); - let file = s3_client.get(&windmill_object_store::object_store_reexports::Path::from(path)).await; + let file = s3_client + .get(&windmill_object_store::object_store_reexports::Path::from( + path, + )) + .await; match file { Ok(file) => { let bytes = file.bytes().await; @@ -126,7 +130,7 @@ async fn get_log_file( } } } - let file = tokio::fs::read(format!("{}{}", TMP_WINDMILL_LOGS_SERVICE, path)).await; + let file = tokio::fs::read(format!("{}{}", *TMP_WINDMILL_LOGS_SERVICE, path)).await; if let Ok(bytes) = file { Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))) } else { diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index f3ec05348a..9d95ead840 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -369,7 +369,11 @@ async fn route_job( let s3_object = s3_client.get(&path).await; let s3_object = match s3_object { - Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { .. }) if trigger.is_static_website => { + Err( + windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { + .. + }, + ) if trigger.is_static_website => { // fallback to index.html if the file is not found let path = windmill_object_store::object_store_reexports::Path::from(format!( "{}/index.html", diff --git a/backend/windmill-api/src/volumes_oss.rs b/backend/windmill-api/src/volumes_oss.rs new file mode 100644 index 0000000000..26b1c2cd46 --- /dev/null +++ b/backend/windmill-api/src/volumes_oss.rs @@ -0,0 +1,17 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::volumes_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn workspaced_service() -> Router { + Router::new() +} + +#[cfg(not(feature = "private"))] +#[allow(dead_code)] +pub fn agent_workspaced_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/workspace_dependencies.rs b/backend/windmill-api/src/workspace_dependencies.rs index d887923f5b..e5c9377b01 100644 --- a/backend/windmill-api/src/workspace_dependencies.rs +++ b/backend/windmill-api/src/workspace_dependencies.rs @@ -16,6 +16,7 @@ use windmill_common::{ use windmill_dep_map::workspace_dependencies::{ trigger_dependents_to_recompute_dependencies_in_the_background, NewWorkspaceDependencies, }; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; use crate::db::ApiAuthed; @@ -37,21 +38,36 @@ async fn create( ) -> error::Result<(StatusCode, String)> { tracing::info!(workspace_id = %nwd.workspace_id, name = ?nwd.name, language = ?nwd.language, "create workspace dependencies"); require_admin(authed.is_admin, &authed.username)?; - Ok(( - StatusCode::CREATED, - format!( - "{}", - nwd.create( - ( - authed.email, - username_to_permissioned_as(&authed.username), - authed.username, - ), - db - ) - .await? - ), - )) + + let dep_path = WorkspaceDependencies::to_path(&nwd.name, nwd.language)?; + let w_id = nwd.workspace_id.clone(); + let email = authed.email.clone(); + let username = authed.username.clone(); + + let id = nwd + .create( + ( + authed.email, + username_to_permissioned_as(&authed.username), + authed.username, + ), + db.clone(), + ) + .await?; + + handle_deployment_metadata( + &email, + &username, + &db, + &w_id, + DeployedObject::WorkspaceDependencies { path: dep_path }, + None, + true, + None, + ) + .await?; + + Ok((StatusCode::CREATED, format!("{}", id))) } #[axum::debug_handler] @@ -92,8 +108,21 @@ async fn archive( tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "archive workspace dependencies"); require_admin(authed.is_admin, &authed.username)?; let db = &db; + let dep_path = WorkspaceDependencies::to_path(¶ms.name, language)?; WorkspaceDependencies::archive(params.name.clone(), language, &w_id, db).await?; + handle_deployment_metadata( + &authed.email, + &authed.username, + db, + &w_id, + DeployedObject::WorkspaceDependencies { path: dep_path.clone() }, + None, + true, + None, + ) + .await?; + trigger_dependents_to_recompute_dependencies_in_the_background( params.name.is_none(), w_id, @@ -103,7 +132,7 @@ async fn archive( username_to_permissioned_as(&authed.username), authed.username, ), - WorkspaceDependencies::to_path(¶ms.name, language)?, + dep_path, db.clone(), ) .await; @@ -121,8 +150,21 @@ async fn delete( tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "delete workspace dependencies"); require_admin(authed.is_admin, &authed.username)?; let db = &db; + let dep_path = WorkspaceDependencies::to_path(¶ms.name, language)?; WorkspaceDependencies::delete(params.name.clone(), language, &w_id, db).await?; + handle_deployment_metadata( + &authed.email, + &authed.username, + db, + &w_id, + DeployedObject::WorkspaceDependencies { path: dep_path.clone() }, + None, + true, + None, + ) + .await?; + trigger_dependents_to_recompute_dependencies_in_the_background( params.name.is_none(), w_id, @@ -132,7 +174,7 @@ async fn delete( username_to_permissioned_as(&authed.username), authed.username, ), - WorkspaceDependencies::to_path(¶ms.name, language)?, + dep_path, db.clone(), ) .await; diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index c881e9ae9b..cd32587648 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -43,6 +43,7 @@ use windmill_common::runnable_settings::{ConcurrencySettings, DebouncingSettings use windmill_common::scripts::ScriptRunnableSettingsHandle; use windmill_common::utils::require_admin; use windmill_common::variables::decrypt; +use windmill_common::worker::WINDMILL_DIR; use windmill_common::{ db::UserDB, error::{to_anyhow, Error, Result}, @@ -281,6 +282,12 @@ struct SimplifiedSettings { color: Option, #[serde(skip_serializing_if = "Option::is_none")] operator_settings: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_team_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_command_script: Option, } // V1 format: Legacy flat format for backward compatibility (matches main branch exactly) @@ -315,6 +322,12 @@ struct SimplifiedSettingsLegacy { color: Option, #[serde(skip_serializing_if = "Option::is_none")] operator_settings: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_team_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_command_script: Option, } // Internal struct for querying database @@ -334,6 +347,9 @@ struct SettingsRow { mute_critical_alerts: Option, color: Option, operator_settings: Option, + slack_team_id: Option, + slack_name: Option, + slack_command_script: Option, } pub(crate) async fn tarball_workspace( @@ -372,7 +388,7 @@ pub(crate) async fn tarball_workspace( let mut tx = user_db.begin(&authed).await?; - let tmp_dir = TempDir::new_in("/tmp/windmill/")?; + let tmp_dir = TempDir::new_in(&*WINDMILL_DIR)?; let name = match archive_type.as_deref() { Some("tar") | None => Ok(format!("windmill-{w_id}.tar")), @@ -938,7 +954,10 @@ pub(crate) async fn tarball_workspace( workspace.name as name, mute_critical_alerts, color, - operator_settings + operator_settings, + slack_team_id, + slack_name, + slack_command_script FROM workspace_settings LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id WHERE workspace_id = $1"#, @@ -964,6 +983,9 @@ pub(crate) async fn tarball_workspace( mute_critical_alerts: row.mute_critical_alerts, color: row.color.clone(), operator_settings: row.operator_settings.clone(), + slack_team_id: row.slack_team_id.clone(), + slack_name: row.slack_name.clone(), + slack_command_script: row.slack_command_script.clone(), }; serde_json::to_value(settings) .map(|v| serde_json::to_string_pretty(&v).ok()) @@ -1023,6 +1045,9 @@ pub(crate) async fn tarball_workspace( mute_critical_alerts: row.mute_critical_alerts, color: row.color, operator_settings: row.operator_settings, + slack_team_id: row.slack_team_id, + slack_name: row.slack_name, + slack_command_script: row.slack_command_script, }; serde_json::to_value(settings) .map(|v| serde_json::to_string_pretty(&v).ok()) diff --git a/backend/windmill-common/src/ai_google.rs b/backend/windmill-common/src/ai_google.rs new file mode 100644 index 0000000000..ccf34685e5 --- /dev/null +++ b/backend/windmill-common/src/ai_google.rs @@ -0,0 +1,726 @@ +//! Shared Google AI (Gemini API) types and conversion utilities. +//! +//! This module provides: +//! - Gemini request/response types +//! - OpenAI → Gemini message conversion +//! - Gemini SSE event parsing +//! +//! Used by both windmill-api (chat proxy) and windmill-worker (AI agent). + +use serde::{Deserialize, Serialize}; + +use crate::ai_types::{ContentPart, ExtraContent, GoogleExtraContent, OpenAIContent, OpenAIMessage, ToolDef, UrlCitation}; +use crate::error::Error; + +// ============================================================================ +// Request / Content Types +// ============================================================================ + +/// Inline data for binary content (images). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiInlineData { + #[serde(rename = "mimeType")] + pub mime_type: String, + pub data: String, +} + +/// A part of content — text, inline data, function call, or function response. +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(untagged)] +pub enum GeminiPart { + Text { + text: String, + }, + InlineData { + #[serde(rename = "inlineData")] + inline_data: GeminiInlineData, + }, + FunctionCall { + #[serde(rename = "functionCall")] + function_call: GeminiFunctionCall, + /// Thought signature for Gemini 3+ models — required when replaying function calls. + #[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")] + thought_signature: Option, + }, + FunctionResponse { + #[serde(rename = "functionResponse")] + function_response: GeminiFunctionResponse, + }, +} + +/// A function call from the model. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiFunctionCall { + pub name: String, + pub args: serde_json::Value, +} + +/// A function response sent back to the model. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiFunctionResponse { + pub name: String, + pub response: serde_json::Value, +} + +/// Content message with an optional role and a list of parts. +#[derive(Serialize, Clone, Debug)] +pub struct GeminiContentMessage { + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, + pub parts: Vec, +} + +/// Main request body for `generateContent` / `streamGenerateContent`. +#[derive(Serialize)] +pub struct GeminiTextRequest { + pub contents: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option>, + #[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")] + pub tool_config: Option, + #[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")] + pub system_instruction: Option, + #[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")] + pub generation_config: Option, +} + +/// Tool definition — function declarations and/or Google Search grounding. +#[derive(Serialize)] +pub struct GeminiTool { + #[serde(rename = "functionDeclarations", skip_serializing_if = "Option::is_none")] + pub function_declarations: Option>, + #[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")] + pub google_search: Option, +} + +/// A single function declaration. +/// +/// `parameters` holds a pre-serialized (and, for the worker, pre-sanitized) JSON Schema. +#[derive(Serialize)] +pub struct GeminiFunctionDeclaration { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + pub parameters: serde_json::Value, +} + +/// Tool configuration controlling when and how functions are called. +#[derive(Serialize)] +pub struct GeminiToolConfig { + #[serde(rename = "functionCallingConfig")] + pub function_calling_config: GeminiFunctionCallingConfig, +} + +/// Function calling mode and optional allow-list. +#[derive(Serialize)] +pub struct GeminiFunctionCallingConfig { + pub mode: String, + #[serde(rename = "allowedFunctionNames", skip_serializing_if = "Option::is_none")] + pub allowed_function_names: Option>, +} + +/// Generation parameters (temperature, token limits, structured output). +#[derive(Serialize)] +pub struct GeminiGenerationConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, + #[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")] + pub response_mime_type: Option, + #[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")] + pub response_schema: Option, +} + +// ============================================================================ +// Image Generation Types +// ============================================================================ + +/// Request body for Imagen / Gemini image generation. +#[derive(Serialize)] +pub struct GeminiImageRequest { + #[serde(skip_serializing_if = "Option::is_none")] + pub contents: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub instances: Option>, +} + +/// Content wrapper used in `generateContent` image requests. +#[derive(Serialize)] +pub struct GeminiImageContent { + pub parts: Vec, +} + +/// Prompt wrapper for Imagen `predict` endpoint. +#[derive(Serialize)] +pub struct GeminiPredictContent { + pub prompt: String, +} + +/// Top-level response from Gemini/Imagen image generation. +#[derive(Deserialize)] +pub struct GeminiImageResponse { + pub candidates: Option>, + pub predictions: Option>, +} + +#[derive(Deserialize)] +pub struct GeminiImageCandidate { + pub content: GeminiImageCandidateContent, +} + +#[derive(Deserialize)] +pub struct GeminiImageCandidateContent { + pub parts: Vec, +} + +#[derive(Deserialize)] +pub struct GeminiImageCandidatePart { + #[serde(rename = "inlineData")] + pub inline_data: Option, +} + +#[derive(Deserialize)] +pub struct GeminiPredictCandidate { + #[serde(rename = "bytesBase64Encoded")] + pub bytes_base64_encoded: String, +} + +// ============================================================================ +// SSE Response Types +// ============================================================================ + +/// One part inside a streaming candidate — text, function call, or thought signature. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEPart { + #[serde(default)] + pub text: Option, + #[serde(rename = "functionCall")] + pub function_call: Option, + /// Thought signature for Gemini 3+ models. + #[serde(rename = "thoughtSignature")] + pub thought_signature: Option, +} + +/// Function call contained in a streaming part. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEFunctionCall { + pub name: String, + pub args: serde_json::Value, +} + +/// Content block inside a streaming candidate. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEContent { + pub parts: Option>, +} + +/// Web source from a Gemini grounding chunk. +#[derive(Deserialize, Debug)] +pub struct GeminiGroundingChunkWeb { + pub uri: String, + #[serde(default)] + pub title: Option, +} + +/// One grounding chunk (search result) from Gemini web search. +#[derive(Deserialize, Debug)] +pub struct GeminiGroundingChunk { + pub web: Option, +} + +/// Grounding metadata attached to a streaming candidate. +#[derive(Deserialize, Debug)] +pub struct GeminiGroundingMetadata { + #[serde(rename = "groundingChunks", default)] + pub grounding_chunks: Vec, + #[serde(rename = "webSearchQueries", default)] + pub web_search_queries: Vec, +} + +/// One candidate inside a streaming Gemini response. +#[derive(Deserialize, Debug)] +pub struct GeminiSSECandidate { + pub content: Option, + #[serde(rename = "finishReason")] + pub finish_reason: Option, + #[serde(rename = "groundingMetadata")] + pub grounding_metadata: Option, +} + +/// Token usage from the `usageMetadata` field of a Gemini SSE event. +#[derive(Deserialize, Debug, Clone)] +pub struct GeminiUsageMetadata { + #[serde(rename = "promptTokenCount", default)] + pub prompt_token_count: Option, + #[serde(rename = "candidatesTokenCount", default)] + pub candidates_token_count: Option, + #[serde(rename = "totalTokenCount", default)] + pub total_token_count: Option, +} + +/// Top-level structure of one Gemini SSE event. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEEvent { + pub candidates: Option>, + #[serde(rename = "usageMetadata")] + pub usage_metadata: Option, +} + +// ============================================================================ +// Parsed Event Result +// ============================================================================ + +/// A single function call extracted from a Gemini SSE event. +#[derive(Debug)] +pub struct GeminiToolCallEvent { + pub name: String, + pub args: serde_json::Value, + pub thought_signature: Option, +} + +impl GeminiToolCallEvent { + /// Convert the thought signature (if present) into an [`ExtraContent`]. + pub fn to_extra_content(&self) -> Option { + self.thought_signature.as_ref().map(|sig| ExtraContent { + google: Some(GoogleExtraContent { thought_signature: Some(sig.clone()) }), + }) + } +} + +/// Structured result of parsing a Gemini response (streaming SSE event or non-streaming body). +#[derive(Debug, Default)] +pub struct GeminiParsedEvent { + pub text: Option, + pub tool_calls: Vec, + pub annotations: Vec, + pub used_websearch: bool, + pub usage: Option, + pub finish_reason: Option, +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +/// Parse a data URL into `(mime_type, base64_data)`. +/// +/// Expected format: `data:;base64,`. +pub fn parse_data_url(url: &str) -> Option<(String, String)> { + let rest = url.strip_prefix("data:")?; + let (header, data) = rest.split_once(',')?; + let media_type = header.strip_suffix(";base64")?; + Some((media_type.to_string(), data.to_string())) +} + +/// Find the function name associated with a `tool_call_id` by scanning prior messages. +pub fn find_gemini_function_name(messages: &[OpenAIMessage], tool_call_id: &str) -> String { + messages + .iter() + .filter_map(|msg| msg.tool_calls.as_ref()) + .flatten() + .find(|tc| tc.id == tool_call_id) + .map(|tc| tc.function.name.clone()) + .unwrap_or_else(|| "unknown_function".to_string()) +} + +/// Convert an [`OpenAIContent`] value to a list of [`GeminiPart`]s. +/// +/// Handles text and `image_url` (data URLs). `S3Object` variants are skipped here; +/// the worker handles them by downloading and injecting inline data beforehand. +pub fn convert_content_to_gemini_parts(content: &OpenAIContent) -> Vec { + match content { + OpenAIContent::Text(text) if !text.is_empty() => { + vec![GeminiPart::Text { text: text.clone() }] + } + OpenAIContent::Text(_) => vec![], + OpenAIContent::Parts(parts) => parts + .iter() + .filter_map(|part| match part { + ContentPart::Text { text } if !text.is_empty() => { + Some(GeminiPart::Text { text: text.clone() }) + } + ContentPart::ImageUrl { image_url } => { + parse_data_url(&image_url.url).map(|(mime_type, data)| { + GeminiPart::InlineData { + inline_data: GeminiInlineData { mime_type, data }, + } + }) + } + // S3Objects are handled by the worker + _ => None, + }) + .collect(), + } +} + +/// Convert OpenAI-format messages to Gemini `contents` and an optional `systemInstruction`. +/// +/// Returns `(contents, system_instruction)`. +/// +/// `S3Object` images in content parts are skipped (the worker pre-converts them). +/// Tool call history is preserved correctly for multi-turn agent conversations. +pub fn openai_messages_to_gemini( + messages: &[OpenAIMessage], +) -> (Vec, Option) { + let mut contents: Vec = Vec::new(); + let mut system_instruction: Option = None; + + for msg in messages { + match msg.role.as_str() { + "system" => { + if let Some(content) = &msg.content { + let parts = convert_content_to_gemini_parts(content); + if !parts.is_empty() { + system_instruction = + Some(GeminiContentMessage { role: None, parts }); + } + } + } + "tool" => { + if let (Some(tool_call_id), Some(content)) = + (&msg.tool_call_id, &msg.content) + { + let func_name = find_gemini_function_name(messages, tool_call_id); + let response_text = match content { + OpenAIContent::Text(text) => text.clone(), + OpenAIContent::Parts(parts) => parts + .iter() + .filter_map(|p| { + if let ContentPart::Text { text } = p { + Some(text.as_str()) + } else { + None + } + }) + .collect::>() + .join(" "), + }; + contents.push(GeminiContentMessage { + role: Some("user".to_string()), + parts: vec![GeminiPart::FunctionResponse { + function_response: GeminiFunctionResponse { + name: func_name, + response: serde_json::json!({ "result": response_text }), + }, + }], + }); + } + } + role => { + let gemini_role = if role == "assistant" { "model" } else { "user" }; + let mut parts: Vec = Vec::new(); + + if let Some(content) = &msg.content { + parts.extend(convert_content_to_gemini_parts(content)); + } + + if let Some(tool_calls) = &msg.tool_calls { + for tc in tool_calls { + let args: serde_json::Value = + serde_json::from_str(&tc.function.arguments).unwrap_or_default(); + let thought_signature = tc + .extra_content + .as_ref() + .and_then(|ec| ec.google.as_ref()) + .and_then(|g| g.thought_signature.clone()); + parts.push(GeminiPart::FunctionCall { + function_call: GeminiFunctionCall { + name: tc.function.name.clone(), + args, + }, + thought_signature, + }); + } + } + + if !parts.is_empty() { + contents.push(GeminiContentMessage { + role: Some(gemini_role.to_string()), + parts, + }); + } + } + } + } + + (contents, system_instruction) +} + +/// Convert OpenAI tool definitions to Gemini format. +/// +/// `tool_params` must be pre-serialized (and, for the worker, pre-sanitized for Google) +/// JSON schema values, one per entry in `tools` in the same order. +pub fn openai_tools_to_gemini( + tools: &[ToolDef], + tool_params: &[serde_json::Value], + has_websearch: bool, +) -> Option> { + let mut gemini_tools: Vec = Vec::new(); + + let declarations: Vec = tools + .iter() + .zip(tool_params.iter()) + .map(|(t, params)| GeminiFunctionDeclaration { + name: t.function.name.clone(), + description: t.function.description.clone(), + parameters: params.clone(), + }) + .collect(); + + if !declarations.is_empty() { + gemini_tools.push(GeminiTool { + function_declarations: Some(declarations), + google_search: None, + }); + } + + if has_websearch { + gemini_tools.push(GeminiTool { + function_declarations: None, + google_search: Some(serde_json::json!({})), + }); + } + + if gemini_tools.is_empty() { + None + } else { + Some(gemini_tools) + } +} + +/// Parse one Gemini SSE data line into a [`GeminiParsedEvent`]. +/// +/// Returns `Ok(None)` for empty data or unrecognised payloads (e.g. `"[DONE]"`). +/// Logs a warning and returns `Ok(None)` on JSON parse errors rather than propagating. +pub fn parse_gemini_sse_event(data: &str) -> Result, Error> { + if data.is_empty() || data == "[DONE]" { + return Ok(None); + } + + let event: GeminiSSEEvent = match serde_json::from_str(data) { + Ok(e) => e, + Err(e) => { + tracing::error!("Failed to parse Gemini SSE event {}: {}", data, e); + return Ok(None); + } + }; + + let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() }; + + let Some(candidates) = event.candidates else { + return Ok(Some(parsed)); + }; + + extract_candidates_into(&candidates, &mut parsed); + + Ok(Some(parsed)) +} + +/// Parse a non-streaming Gemini `generateContent` response body. +pub fn parse_gemini_response(data: &[u8]) -> Result { + let event: GeminiSSEEvent = serde_json::from_slice(data) + .map_err(|e| Error::internal_err(format!("Failed to parse Gemini response: {}", e)))?; + + let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() }; + + if let Some(candidates) = event.candidates { + extract_candidates_into(&candidates, &mut parsed); + } + + Ok(parsed) +} + +// ============================================================================ +// Gemini → OpenAI Format Conversion +// ============================================================================ + +/// Convert a `GeminiParsedEvent` from a non-streaming response to an OpenAI chat completion JSON. +pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> serde_json::Value { + let content = parsed.text.as_deref().unwrap_or_default(); + + let tool_calls: Vec = parsed + .tool_calls + .iter() + .enumerate() + .map(|(i, tc)| { + serde_json::json!({ + "index": i, + "id": format!("call_{}", uuid::Uuid::new_v4().simple()), + "type": "function", + "function": { + "name": tc.name, + "arguments": serde_json::to_string(&tc.args).unwrap_or_default() + } + }) + }) + .collect(); + + let finish_reason = parsed + .finish_reason + .as_deref() + .map(|r| r.to_lowercase()) + .unwrap_or_else(|| "stop".to_string()); + + let usage = parsed.usage.as_ref().map(|u| { + serde_json::json!({ + "prompt_tokens": u.prompt_token_count.unwrap_or(0), + "completion_tokens": u.candidates_token_count.unwrap_or(0), + "total_tokens": u.total_token_count.unwrap_or(0), + }) + }); + + let mut message = serde_json::json!({ + "role": "assistant", + "content": content, + }); + if !tool_calls.is_empty() { + message["tool_calls"] = serde_json::json!(tool_calls); + } + + serde_json::json!({ + "id": format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()), + "object": "chat.completion", + "model": model, + "choices": [{ + "index": 0, + "message": message, + "finish_reason": finish_reason, + }], + "usage": usage, + }) +} + +/// Convert a `GeminiParsedEvent` from a streaming SSE event into OpenAI-format SSE lines. +/// +/// Returns the serialized `"data: {...}\n\n"` lines ready to be written to the response stream. +/// `tool_call_index` is mutated to track the running index across multiple SSE events. +pub fn gemini_event_to_openai_sse_chunks( + parsed: &GeminiParsedEvent, + id: &str, + model: &str, + tool_call_index: &mut usize, +) -> Vec { + let mut chunks = Vec::new(); + + if let Some(text) = &parsed.text { + let chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": model, + "choices": [{ + "index": 0, + "delta": { "content": text }, + "finish_reason": null, + }] + }); + chunks.push(format!("data: {}\n\n", chunk)); + } + + for tc in &parsed.tool_calls { + let args_str = serde_json::to_string(&tc.args).unwrap_or_default(); + let call_id = format!("call_{}", uuid::Uuid::new_v4().simple()); + let chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": model, + "choices": [{ + "index": 0, + "delta": { + "tool_calls": [{ + "index": *tool_call_index, + "id": call_id, + "type": "function", + "function": { + "name": tc.name, + "arguments": args_str, + } + }] + }, + "finish_reason": null, + }] + }); + chunks.push(format!("data: {}\n\n", chunk)); + *tool_call_index += 1; + } + + chunks +} + +/// Recursively remove JSON Schema fields unsupported by the Gemini API. +pub fn sanitize_schema_for_google(value: &mut serde_json::Value) { + const UNSUPPORTED: &[&str] = &[ + "additionalProperties", + "strict", + "$schema", + "default", + "exclusiveMinimum", + "exclusiveMaximum", + "const", + "multipleOf", + ]; + + if let Some(obj) = value.as_object_mut() { + for field in UNSUPPORTED { + obj.remove(*field); + } + for v in obj.values_mut() { + sanitize_schema_for_google(v); + } + } else if let Some(arr) = value.as_array_mut() { + for v in arr.iter_mut() { + sanitize_schema_for_google(v); + } + } +} + +// ============================================================================ +// Internal Helpers +// ============================================================================ + +fn extract_candidates_into(candidates: &[GeminiSSECandidate], parsed: &mut GeminiParsedEvent) { + for candidate in candidates { + if let Some(content) = &candidate.content { + if let Some(parts) = &content.parts { + for part in parts { + if let Some(text) = &part.text { + if !text.is_empty() { + match parsed.text.as_mut() { + Some(existing) => existing.push_str(text), + None => parsed.text = Some(text.clone()), + } + } + } + + if let Some(function_call) = &part.function_call { + parsed.tool_calls.push(GeminiToolCallEvent { + name: function_call.name.clone(), + args: function_call.args.clone(), + thought_signature: part.thought_signature.clone(), + }); + } + } + } + } + + if candidate.finish_reason.is_some() { + parsed.finish_reason = candidate.finish_reason.clone(); + } + + if let Some(grounding) = &candidate.grounding_metadata { + if !grounding.web_search_queries.is_empty() || !grounding.grounding_chunks.is_empty() { + parsed.used_websearch = true; + } + for chunk in &grounding.grounding_chunks { + if let Some(web) = &chunk.web { + parsed.annotations.push(UrlCitation { + start_index: 0, + end_index: 0, + url: web.uri.clone(), + title: web.title.clone(), + }); + } + } + } + } +} diff --git a/backend/windmill-common/src/ai_providers.rs b/backend/windmill-common/src/ai_providers.rs index 323e10a128..e3e1e95eee 100644 --- a/backend/windmill-common/src/ai_providers.rs +++ b/backend/windmill-common/src/ai_providers.rs @@ -29,6 +29,15 @@ pub const GOOGLE_AI_BASE_URL: &str = "https://generativelanguage.googleapis.com/ /// (e.g., AWS_REGION or AWS_DEFAULT_REGION env vars, or ~/.aws/config) pub const USE_ENV_REGION: &str = ""; +/// Platform variant for providers that support Google Vertex AI (Anthropic, GoogleAI). +#[derive(Deserialize, Debug, Clone, Default, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum AIPlatform { + #[default] + Standard, + GoogleVertexAi, +} + #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] #[serde(rename_all = "lowercase")] pub enum AIProvider { diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 669614d64a..2968493ecc 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -72,6 +72,7 @@ pub fn asset_kind_from_parser(parser_kind: windmill_parser::asset_parser::AssetK windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, windmill_parser::asset_parser::AssetKind::Ducklake => AssetKind::Ducklake, windmill_parser::asset_parser::AssetKind::DataTable => AssetKind::DataTable, + windmill_parser::asset_parser::AssetKind::Volume => AssetKind::Volume, } } diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index 08d3f2a516..3d2c58ce95 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -101,6 +101,51 @@ impl PermsCache { } } +/// Check a user's access level against an `extra_perms` JSONB object. +/// +/// Returns `None` if the user has no matching entry (no access). +/// Returns `Some(true)` if the user (or any of their groups) has write access. +/// Returns `Some(false)` if the user (or any of their groups) has read-only access. +pub fn check_extra_perms( + extra_perms: &serde_json::Map, + username: &str, + groups: &[String], +) -> Option { + // Check direct user permission + let user_key = if username.starts_with("u/") { + username.to_string() + } else { + format!("u/{username}") + }; + if let Some(v) = extra_perms.get(&user_key) { + return Some(v.as_bool().unwrap_or(false)); + } + + // Check group permissions — return highest access level found + let mut found = false; + let mut write = false; + for g in groups { + let key = if g.starts_with("g/") { + g.to_string() + } else { + format!("g/{g}") + }; + if let Some(v) = extra_perms.get(&key) { + found = true; + if v.as_bool().unwrap_or(false) { + write = true; + break; + } + } + } + + if found { + Some(write) + } else { + None + } +} + pub fn has_expired(expiration_time: DateTime, take: Option) -> bool { let now = Utc::now(); @@ -300,7 +345,7 @@ async fn fetch_authed_from_permissioned_as_inner( if let Some(r) = r { (r.is_admin, r.operator) } else { - return Err(Error::internal_err(format!( + return Err(Error::NotFound(format!( "user {name} not found in workspace {w_id}" ))); } diff --git a/backend/windmill-common/src/bench.rs b/backend/windmill-common/src/bench.rs index b87872119b..1ed2b9aa56 100644 --- a/backend/windmill-common/src/bench.rs +++ b/backend/windmill-common/src/bench.rs @@ -1,5 +1,5 @@ use crate::{ - worker::{write_file, TMP_DIR}, + worker::{write_file, WINDMILL_DIR}, DB, }; use serde::Serialize; @@ -113,7 +113,8 @@ impl BenchmarkInfo { "Writing benchmark {path}, duration of benchmark: {total_duration}ms and RPS: {}{pool_info}", self.iters as f64 / total_duration as f64 * 1000.0 ); - write_file(TMP_DIR, path, &serde_json::to_string(&self).unwrap()).expect("write profiling"); + write_file(&WINDMILL_DIR, path, &serde_json::to_string(&self).unwrap()) + .expect("write profiling"); Ok(()) } } diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index eaf7d857c1..49dcf25450 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -78,6 +78,8 @@ pub enum Error { AIError(String), #[error("{0}")] AlreadyCompleted(String), + #[error("WAC job suspended: {0}")] + WacSuspended(String), #[error("Find python error: {0}")] FindPythonError(String), #[error("Problem with arguments: {0}")] @@ -108,6 +110,7 @@ impl Error { Self::JsonErr(_) => "JsonErr", Self::AIError(_) => "AIError", Self::AlreadyCompleted(_) => "AlreadyCompleted", + Self::WacSuspended(_) => "WacSuspended", Self::FindPythonError(_) => "FindPythonError", Self::ArgumentErr(_) => "ArgumentErr", Self::Generic(_, _) => "Generic", diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 3347127303..6d2f225479 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -4,6 +4,7 @@ pub const DEFAULT_TAGS_WORKSPACES_SETTING: &str = "default_tags_workspaces"; pub const BASE_URL_SETTING: &str = "base_url"; pub const OAUTH_SETTING: &str = "oauths"; pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs"; +pub const AUDIT_LOG_RETENTION_DAYS_SETTING: &str = "audit_log_retention_days"; pub const MONITOR_LOGS_ON_OBJECT_STORE_SETTING: &str = "monitor_logs_on_s3"; pub const JOB_DEFAULT_TIMEOUT_SECS_SETTING: &str = "job_default_timeout"; pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb"; @@ -44,9 +45,11 @@ pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; +pub const DISABLE_HUB_SETTING: &str = "disable_hub"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize"; +pub const CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING: &str = "critical_alerts_on_token_expiry"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs index 45a924748d..23bde13795 100644 --- a/backend/windmill-common/src/indexer.rs +++ b/backend/windmill-common/src/indexer.rs @@ -13,6 +13,7 @@ pub struct TantivyIndexerSettings { pub refresh_index_period: u64, pub refresh_log_index_period: u64, pub max_indexed_job_log_size: usize, + pub max_index_time_window_secs: i64, pub should_clear_job_index: bool, pub should_clear_log_index: bool, } @@ -26,6 +27,7 @@ impl Default for TantivyIndexerSettings { refresh_index_period: 300, refresh_log_index_period: 300, max_indexed_job_log_size: 1_000_000, + max_index_time_window_secs: 60 * 60 * 24 * 7, // 7 days should_clear_job_index: false, should_clear_log_index: false, } @@ -39,6 +41,7 @@ pub struct TantivyIndexerSettingsOpt { pub refresh_index_period: Option, pub refresh_log_index_period: Option, pub max_indexed_job_log_size: Option, + pub max_index_time_window_secs: Option, pub should_clear_job_index: Option, pub should_clear_log_index: Option, } @@ -58,6 +61,7 @@ pub async fn load_indexer_config(db: &DB) -> error::Result error::Result TantivyIndexerSettings { if let Some(b) = get_env_var("TANTIVY_MAX_INDEXED_JOB_LOG_SIZE__KB") { settings.max_indexed_job_log_size = (b * BYTES_PER_KB) as usize; } + if let Some(b) = get_env_var("TANTIVY_MAX_INDEX_TIME_WINDOW__S") { + settings.max_index_time_window_secs = b as i64; + } settings } diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 4de90be5c5..fb70db90ae 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -44,7 +44,7 @@ pub struct EnvRefWrapper { /// /// `Literal` serializes back to a plain JSON string, preserving backwards /// compatibility with existing consumers. -#[derive(Deserialize, Serialize, Clone, Debug)] +#[derive(Deserialize, Serialize, Clone)] #[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))] #[serde(untagged)] pub enum StringOrSecretRef { @@ -53,6 +53,16 @@ pub enum StringOrSecretRef { EnvRef(EnvRefWrapper), } +impl fmt::Debug for StringOrSecretRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Literal(_) => f.write_str("Literal(****)"), + Self::SecretRef(w) => f.debug_tuple("SecretRef").field(w).finish(), + Self::EnvRef(w) => f.debug_tuple("EnvRef").field(w).finish(), + } + } +} + impl StringOrSecretRef { /// Returns the literal string value, or `None` if this is an unresolved ref. pub fn as_literal(&self) -> Option<&str> { @@ -230,6 +240,8 @@ pub struct GlobalSettings { pub no_default_maven: Option, #[serde(skip_serializing_if = "Option::is_none")] pub default_tags_per_workspace: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub disable_hub: Option, // String settings #[serde(skip_serializing_if = "Option::is_none")] @@ -253,25 +265,25 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub instance_python_version: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub pip_index_url: Option, + pub pip_index_url: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub pip_extra_index_url: Option, + pub pip_extra_index_url: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub npm_config_registry: Option, + pub npm_config_registry: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub bunfig_install_scopes: Option, + pub bunfig_install_scopes: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub npmrc: Option, + pub npmrc: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub nuget_config: Option, + pub nuget_config: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub maven_repos: Option, + pub maven_repos: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub ruby_repos: Option, + pub ruby_repos: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub powershell_repo_url: Option, + pub powershell_repo_url: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub powershell_repo_pat: Option, + pub powershell_repo_pat: Option, // Array settings #[serde(skip_serializing_if = "Option::is_none")] @@ -922,7 +934,7 @@ fn redact_string(s: &str) -> String { } } -fn format_setting_value(key: &str, value: &serde_json::Value) -> String { +pub fn format_setting_value(key: &str, value: &serde_json::Value) -> String { if SENSITIVE_SETTINGS.contains(&key) { return match value { serde_json::Value::String(s) => format!("\"{}\"", redact_string(s)), @@ -2207,6 +2219,25 @@ mod tests { assert_eq!(v, *"world"); } + #[test] + fn string_or_secret_ref_debug_masks_literal() { + let v = StringOrSecretRef::Literal("super-secret-value".to_string()); + let debug = format!("{v:?}"); + assert_eq!(debug, "Literal(****)"); + assert!(!debug.contains("super-secret-value")); + } + + #[test] + fn format_setting_value_redacts_oauth_secrets() { + let val = serde_json::json!({ + "google": {"id": "client-id", "secret": "my-super-secret-12345"} + }); + let formatted = format_setting_value("oauths", &val); + assert!(!formatted.contains("my-super-secret-12345")); + assert!(formatted.contains("client-id")); + assert!(formatted.contains("****")); + } + #[test] #[should_panic(expected = "literal_value() called on unresolved secret ref")] fn string_or_secret_ref_literal_value_panics_on_ref() { diff --git a/backend/windmill-common/src/job_metrics.rs b/backend/windmill-common/src/job_metrics.rs index d416e4d44b..c49185de72 100644 --- a/backend/windmill-common/src/job_metrics.rs +++ b/backend/windmill-common/src/job_metrics.rs @@ -15,9 +15,11 @@ pub struct JobStatsRecord { pub timestamps: Option>>, pub timeseries_int: Option>, pub timeseries_float: Option>, + pub timeseries_start: Option>, + pub offsets_cs: Option>, } -#[derive(sqlx::Type, Debug, PartialEq, Deserialize, Serialize)] +#[derive(sqlx::Type, Debug, Clone, PartialEq, Deserialize, Serialize)] #[sqlx(type_name = "METRIC_KIND", rename_all = "snake_case")] pub enum MetricKind { ScalarInt, @@ -52,29 +54,21 @@ pub async fn register_metric_for_job( return Ok(metric_id); } - let (scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) = match metric_kind - { + let is_timeseries = matches!( + metric_kind, + MetricKind::TimeseriesInt | MetricKind::TimeseriesFloat + ); + + let (scalar_int, scalar_float, timeseries_int, timeseries_float) = match metric_kind { MetricKind::ScalarInt | MetricKind::ScalarFloat => { - (None as Option, None as Option, None, None, None) + (None as Option, None as Option, None, None) } - MetricKind::TimeseriesInt => ( - None, - None, - Some(&[] as &[chrono::DateTime]), - Some(&[] as &[i32]), - None, - ), - MetricKind::TimeseriesFloat => ( - None, - None, - Some(&[] as &[chrono::DateTime]), - None, - Some(&[] as &[f32]), - ), + MetricKind::TimeseriesInt => (None, None, Some(&[] as &[i32]), None), + MetricKind::TimeseriesFloat => (None, None, None, Some(&[] as &[f32])), }; sqlx::query( - "INSERT INTO job_stats (workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + "INSERT INTO job_stats (workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timeseries_int, timeseries_float, timeseries_start, offsets_cs) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, CASE WHEN $10 THEN now() ELSE NULL END, CASE WHEN $10 THEN ARRAY[]::int[] ELSE NULL END)", ) .bind(workspace_id) .bind(job_id) @@ -83,9 +77,9 @@ pub async fn register_metric_for_job( .bind(metric_kind) .bind(scalar_int) .bind(scalar_float) - .bind(timestamps) .bind(timeseries_int) .bind(timeseries_float) + .bind(is_timeseries) .execute(db) .warn_after_seconds(1) .await?; @@ -117,6 +111,30 @@ pub async fn record_metric( } let metric_kind = metric_kind_opt.unwrap(); + record_metric_impl(db, workspace_id, job_id, metric_id, value, metric_kind).await +} + +/// Record a timeseries metric value without the extra SELECT to look up metric_kind. +/// Use this when the caller already knows the metric kind (e.g. the worker that registered it). +pub async fn record_timeseries_value( + db: &DB, + workspace_id: String, + job_id: Uuid, + metric_id: String, + value: MetricNumericValue, + metric_kind: MetricKind, +) -> error::Result<()> { + record_metric_impl(db, workspace_id, job_id, metric_id, value, metric_kind).await +} + +async fn record_metric_impl( + db: &DB, + workspace_id: String, + job_id: Uuid, + metric_id: String, + value: MetricNumericValue, + metric_kind: MetricKind, +) -> error::Result<()> { let (value_int, value_float) = match value { MetricNumericValue::Integer(val) => { if metric_kind != MetricKind::TimeseriesInt && metric_kind != MetricKind::ScalarInt { @@ -160,7 +178,7 @@ pub async fn record_metric( } MetricKind::TimeseriesInt => { sqlx::query!( - "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", + "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", &workspace_id, &job_id, &metric_id, @@ -169,7 +187,7 @@ pub async fn record_metric( } MetricKind::TimeseriesFloat => { sqlx::query!( - "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", + "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", &workspace_id, &job_id, &metric_id, diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index bc40e4cd75..583914c573 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -18,7 +18,7 @@ use crate::{ scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, users::username_to_permissioned_as, utils::{StripPath, HTTP_CLIENT}, - worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, TMP_DIR}, + worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, WINDMILL_DIR}, FlowVersionInfo, ScriptHashInfo, Tag, }; @@ -225,7 +225,7 @@ pub async fn get_logs_from_disk( if log_offset > 0 { if let Some(file_index) = log_file_index.clone() { for file_p in &file_index { - if !tokio::fs::metadata(format!("{TMP_DIR}/{file_p}")) + if !tokio::fs::metadata(format!("{}/{file_p}", *WINDMILL_DIR)) .await .is_ok() { @@ -236,7 +236,7 @@ pub async fn get_logs_from_disk( let logs = logs.to_string(); let stream = async_stream::stream! { for file_p in file_index.clone() { - let mut file = tokio::fs::File::open(format!("{TMP_DIR}/{file_p}")).await.map_err(to_anyhow)?; + let mut file = tokio::fs::File::open(format!("{}/{file_p}", *WINDMILL_DIR)).await.map_err(to_anyhow)?; let mut buffer = Vec::new(); file.read_to_end(&mut buffer).await.map_err(to_anyhow)?; yield Ok(bytes::Bytes::from(buffer)) as anyhow::Result; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 643e08fc82..edebdaa974 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -29,6 +29,7 @@ use sqlx::{Acquire, Postgres}; pub mod agent_workers; #[cfg(feature = "bedrock")] pub mod ai_bedrock; +pub mod ai_google; pub mod ai_providers; pub mod ai_types; pub mod apps; @@ -193,6 +194,7 @@ lazy_static::lazy_static! { pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false); pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false); + pub static ref CRITICAL_ALERTS_ON_TOKEN_EXPIRY: AtomicBool = AtomicBool::new(false); pub static ref BASE_URL: Arc> = Arc::new(RwLock::new("".to_string())); pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); @@ -203,6 +205,7 @@ lazy_static::lazy_static! { pub static ref CRITICAL_ALERTS_ON_DB_OVERSIZE: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_RETENTION_SECS: Arc> = Arc::new(RwLock::new(0)); + pub static ref AUDIT_LOG_RETENTION_DAYS: Arc> = Arc::new(RwLock::new(0)); pub static ref MONITOR_LOGS_ON_OBJECT_STORE: Arc> = Arc::new(RwLock::new(false)); diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 68f77a2303..cfff28dbcb 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -206,12 +206,12 @@ pub async fn get_full_hub_script_by_path( let version = path_iterator .next() .ok_or_else(|| Error::internal_err(format!("expected hub path to have version number")))?; - let cache_path = format!("{HUB_CACHE_DIR}/{version}"); + let cache_path = format!("{}/{version}", *HUB_CACHE_DIR); let script; if tokio::fs::metadata(&cache_path).await.is_err() { script = get_full_hub_script_by_path_inner(path, http_client, db).await?; if let Err(e) = crate::worker::write_file( - HUB_CACHE_DIR, + &HUB_CACHE_DIR, &version, &serde_json::to_string(&script).map_err(to_anyhow)?, ) { diff --git a/backend/windmill-common/src/stats_oss.rs b/backend/windmill-common/src/stats_oss.rs index 161c78f81b..d3cc2e2b44 100644 --- a/backend/windmill-common/src/stats_oss.rs +++ b/backend/windmill-common/src/stats_oss.rs @@ -32,6 +32,7 @@ pub async fn send_stats( _http_client: &reqwest::Client, _db: &DB, _reason: SendStatsReason, + _minimal: bool, ) -> Result<()> { // stats details are closed source Ok(()) @@ -56,7 +57,11 @@ pub async fn get_user_usage<'c, E: sqlx::Executor<'c, Database = Postgres>>( pub struct Stats {} #[cfg(not(feature = "private"))] -pub async fn get_stats_payload(_db: &DB, _reason: &SendStatsReason) -> Result { +pub async fn get_stats_payload( + _db: &DB, + _reason: &SendStatsReason, + _minimal: bool, +) -> Result { // stats details are closed source Ok(Stats {}) } diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 701a0887d0..f4892b67c9 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -6,8 +6,6 @@ * LICENSE-AGPL for a copy of the license. */ -use const_format::concatcp; - use std::{ collections::HashMap, sync::{Arc, RwLock}, @@ -61,7 +59,9 @@ fn create_targets_filter(default_env_filter: LevelFilter) -> Targets { pub const LOGS_SERVICE: &str = "logs/services/"; -pub const TMP_WINDMILL_LOGS_SERVICE: &str = concatcp!("/tmp/windmill/", LOGS_SERVICE); +lazy_static::lazy_static! { + pub static ref TMP_WINDMILL_LOGS_SERVICE: String = format!("{}/{}", *crate::worker::WINDMILL_DIR, LOGS_SERVICE); +} pub fn initialize_tracing( hostname: &str, @@ -108,7 +108,7 @@ pub fn initialize_tracing( use tracing_appender::rolling::{RollingFileAppender, Rotation}; - let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname); + let log_dir = format!("{}/{}/", *TMP_WINDMILL_LOGS_SERVICE, hostname); std::fs::create_dir_all(&log_dir).unwrap(); let file_appender = RollingFileAppender::builder() .rotation(Rotation::MINUTELY) diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index e4c5315cba..10b1a3b408 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -314,14 +314,14 @@ pub async fn create_directory_async(directory_path: &str) { .recursive(true) .create(directory_path) .await - .expect("could not create dir"); + .unwrap_or_else(|e| panic!("could not create dir '{}': {}", directory_path, e)); } pub fn create_directory_sync(directory_path: &str) { SyncDirBuilder::new() .recursive(true) .create(directory_path) - .expect("could not create dir"); + .unwrap_or_else(|e| panic!("could not create dir '{}': {}", directory_path, e)); } #[track_caller] diff --git a/backend/windmill-common/src/webhook.rs b/backend/windmill-common/src/webhook.rs index f1f8508f74..4677084fbe 100644 --- a/backend/windmill-common/src/webhook.rs +++ b/backend/windmill-common/src/webhook.rs @@ -39,29 +39,115 @@ pub enum WebhookPayload { #[serde(tag = "type")] pub enum WebhookMessage { // See https://serde.rs/enum-representations.html#internally-tagged for how this looks in JSON - CreateApp { workspace: String, path: String }, - DeleteApp { workspace: String, path: String }, - UpdateApp { workspace: String, old_path: String, new_path: String }, - CreateFlow { workspace: String, path: String }, - UpdateFlow { workspace: String, old_path: String, new_path: String }, - ArchiveFlow { workspace: String, path: String }, - DeleteFlow { workspace: String, path: String }, - CreateFolder { workspace: String, name: String }, - UpdateFolder { workspace: String, name: String }, - DeleteFolder { workspace: String, name: String }, - DeleteResource { workspace: String, path: String }, - CreateResource { workspace: String, path: String }, - UpdateResource { workspace: String, old_path: String, new_path: String }, - CreateResourceType { name: String }, - DeleteResourceType { name: String }, - UpdateResourceType { name: String }, - CreateScript { workspace: String, path: String, hash: String }, - UpdateScript { workspace: String, path: String, hash: String }, - DeleteScript { workspace: String, hash: String }, - DeleteScriptPath { workspace: String, path: String }, - CreateVariable { workspace: String, path: String }, - UpdateVariable { workspace: String, old_path: String, new_path: String }, - DeleteVariable { workspace: String, path: String }, + CreateApp { + workspace: String, + path: String, + }, + DeleteApp { + workspace: String, + path: String, + }, + UpdateApp { + workspace: String, + old_path: String, + new_path: String, + }, + CreateFlow { + workspace: String, + path: String, + }, + UpdateFlow { + workspace: String, + old_path: String, + new_path: String, + }, + ArchiveFlow { + workspace: String, + path: String, + }, + DeleteFlow { + workspace: String, + path: String, + }, + CreateFolder { + workspace: String, + name: String, + }, + UpdateFolder { + workspace: String, + name: String, + }, + DeleteFolder { + workspace: String, + name: String, + }, + DeleteResource { + workspace: String, + path: String, + }, + CreateResource { + workspace: String, + path: String, + }, + UpdateResource { + workspace: String, + old_path: String, + new_path: String, + }, + CreateResourceType { + name: String, + }, + DeleteResourceType { + name: String, + }, + UpdateResourceType { + name: String, + }, + CreateScript { + workspace: String, + path: String, + hash: String, + }, + UpdateScript { + workspace: String, + path: String, + hash: String, + }, + DeleteScript { + workspace: String, + hash: String, + }, + DeleteScriptPath { + workspace: String, + path: String, + }, + CreateVariable { + workspace: String, + path: String, + }, + UpdateVariable { + workspace: String, + old_path: String, + new_path: String, + }, + DeleteVariable { + workspace: String, + path: String, + }, + TokenExpiringSoon { + workspace: String, + token_prefix: String, + label: String, + owner: String, + expires_at: String, + days_remaining: i64, + }, + TokenExpired { + workspace: String, + token_prefix: String, + label: String, + owner: String, + }, } #[derive(Clone)] @@ -267,6 +353,20 @@ mod tests { new_path: "n".into(), }, WebhookMessage::DeleteVariable { workspace: "w".into(), path: "p".into() }, + WebhookMessage::TokenExpiringSoon { + workspace: "w".into(), + token_prefix: "abc1234567".into(), + label: "my-token".into(), + owner: "user@example.com".into(), + expires_at: "2026-03-10T00:00:00Z".into(), + days_remaining: 7, + }, + WebhookMessage::TokenExpired { + workspace: "w".into(), + token_prefix: "abc1234567".into(), + label: "my-token".into(), + owner: "user@example.com".into(), + }, ]; for msg in &messages { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 1a2d55b896..b3876fbcfa 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -2,7 +2,6 @@ use anyhow::anyhow; use axum::http::HeaderMap; use bytes::Bytes; -use const_format::concatcp; use itertools::Itertools; use regex::Regex; use reqwest_middleware::ClientWithMiddleware; @@ -274,7 +273,9 @@ lazy_static::lazy_static! { pub static ref ROOT_STANDALONE_BUNDLE_DIR: String = format!("{}/.windmill/standalone_bundle", std::env::var("HOME").unwrap_or_else(|_| "/root".to_string())); } -pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); +lazy_static::lazy_static! { + pub static ref ROOT_CACHE_NOMOUNT_DIR: String = format!("{}/cache_nomount/", *WINDMILL_DIR); +} /// Whether native mode is forced by the environment (NATIVE_MODE=true env var or WORKER_GROUP=native). /// This does NOT account for native_mode set in the DB worker group config — for that, read @@ -353,6 +354,45 @@ impl HttpClient { ))) } } + + pub async fn get_bytes(&self, url: &str) -> anyhow::Result { + let base_url = self.base_internal_url.clone(); + let response = self + .client + .get(format!("{}{}", base_url, url)) + .send() + .await + .map_err(|e| anyhow::anyhow!(e))?; + if response.status().is_success() { + Ok(response.bytes().await?) + } else { + Err(anyhow::anyhow!( + "HTTP agent request GET {} failed {}", + url, + response.status() + )) + } + } + + pub async fn put_bytes(&self, url: &str, bytes: Bytes) -> anyhow::Result<()> { + let base_url = self.base_internal_url.clone(); + let response = self + .client + .put(format!("{}{}", base_url, url)) + .body(bytes) + .send() + .await + .map_err(|e| anyhow::anyhow!(e))?; + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "HTTP agent request PUT {} failed {}", + url, + response.status() + )) + } + } } #[derive(Clone)] @@ -490,13 +530,33 @@ pub async fn store_pull_query(wc: &WorkerConfig) { *l = queries; } -pub const TMP_DIR: &str = "/tmp/windmill"; -pub const TMP_LOGS_DIR: &str = concatcp!(TMP_DIR, "/logs"); - -pub const HUB_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "hub"); -pub const HUB_RT_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "hub_rt"); - -pub const ROOT_CACHE_DIR: &str = concatcp!(TMP_DIR, "/cache/"); +lazy_static::lazy_static! { + pub static ref WINDMILL_DIR: String = { + let dir = std::env::var("WINDMILL_DIR") + .unwrap_or_else(|_| { + #[cfg(not(windows))] + { "/tmp/windmill".to_string() } + #[cfg(windows)] + { + let temp = std::env::temp_dir(); + let temp_str = temp.to_string_lossy(); + let normalized = temp_str.trim_end_matches(&['/', '\\'][..]).replace('\\', "/"); + format!("{}/windmill", normalized) + } + }); + if dir.is_empty() { + panic!("WINDMILL_DIR must not be empty"); + } + if dir.ends_with('/') || dir.ends_with('\\') { + panic!("WINDMILL_DIR must not end with a trailing slash, got: {dir}"); + } + dir + }; + pub static ref TMP_LOGS_DIR: String = format!("{}/logs", *WINDMILL_DIR); + pub static ref ROOT_CACHE_DIR: String = format!("{}/cache/", *WINDMILL_DIR); + pub static ref HUB_CACHE_DIR: String = format!("{}hub", *ROOT_CACHE_DIR); + pub static ref HUB_RT_CACHE_DIR: String = format!("{}hub_rt", *ROOT_CACHE_DIR); +} pub fn write_file(dir: &str, path: &str, content: &str) -> error::Result { let path = format!("{}/{}", dir, path); @@ -677,6 +737,7 @@ pub struct PythonAnnotations { pub py311: bool, pub py312: bool, pub py313: bool, + pub sandbox: bool, } #[annotations("//")] @@ -690,6 +751,7 @@ pub struct TypeScriptAnnotations { pub nodejs: bool, pub native: bool, pub nobundling: bool, + pub sandbox: bool, } #[annotations("--")] @@ -2148,4 +2210,62 @@ mod tests { ); assert_ne!(a, b); } + + #[test] + fn test_python_sandbox_annotation() { + let content = "# sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_python_sandbox_annotation_with_other_annotations() { + let content = "# no_cache\n# sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + assert!(annotations.no_cache); + } + + #[test] + fn test_python_no_sandbox_annotation() { + let content = "# no_cache\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(!annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_annotation() { + let content = "// sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_annotation_with_other_annotations() { + let content = "// npm\n// sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + assert!(annotations.npm); + } + + #[test] + fn test_typescript_no_sandbox_annotation() { + let content = "// npm\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(!annotations.sandbox); + } + + #[test] + fn test_python_sandbox_no_space() { + let content = "#sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_no_space() { + let content = "//sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + } } diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index f56648f61e..5aba7162e1 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -146,6 +146,7 @@ pub enum ObjectType { Trigger, Settings, Key, + WorkspaceDependencies, } #[derive(Serialize, Deserialize, Debug)] diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.lock b/backend/windmill-duckdb-ffi-internal/Cargo.lock index 07e428c633..559196a3c2 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.lock +++ b/backend/windmill-duckdb-ffi-internal/Cargo.lock @@ -2164,6 +2164,7 @@ version = "0.1.0" dependencies = [ "chrono", "duckdb", + "regex", "rust_decimal", "serde", "serde_json", diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.toml b/backend/windmill-duckdb-ffi-internal/Cargo.toml index 7043b33ee5..7eb6869ab9 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.toml +++ b/backend/windmill-duckdb-ffi-internal/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] chrono = "0.4.41" duckdb = { version = "1.4.4", features = ["bundled"] } +regex = "1" rust_decimal = "1.37.2" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "^1", features = ["preserve_order", "raw_value"] } diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index c5c819e60b..2701319d6e 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -1,12 +1,14 @@ use std::{ collections::HashMap, - ffi::{CStr, CString, c_char, c_uint}, + ffi::{c_char, c_uint, CStr, CString}, ptr::null_mut, + sync::LazyLock, }; -use duckdb::{Row, core::LogicalTypeId, params_from_iter, types::TimeUnit}; -use rust_decimal::{Decimal, prelude::FromPrimitive}; -use serde::Deserialize; +use duckdb::{core::LogicalTypeId, params_from_iter, types::TimeUnit, Row}; +use regex::Regex; +use rust_decimal::{prelude::FromPrimitive, Decimal}; +use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; #[derive(Deserialize, Clone, Debug, PartialEq, Default)] @@ -96,6 +98,218 @@ pub extern "C" fn run_duckdb_ffi( }) } +#[derive(Serialize, Debug)] +struct PrepareQueryColumnInfo { + name: String, + #[serde(rename = "type")] + type_name: String, +} + +#[derive(Serialize, Debug)] +struct PrepareQueryResult { + #[serde(skip_serializing_if = "Option::is_none")] + columns: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +fn is_setup_statement(query: &str) -> bool { + let trimmed = query.trim_start(); + let upper = trimmed.to_uppercase(); + upper.starts_with("ATTACH") + || upper.starts_with("USE") + || upper.starts_with("INSTALL") + || upper.starts_with("LOAD") + || upper.starts_with("SET") + || upper.starts_with("RESET") + || upper.starts_with("CREATE OR REPLACE SECRET") + || upper.starts_with("CREATE SECRET") +} + +/// Returns true if the query is expected to return a result set and can be wrapped with DESCRIBE. +fn is_describable_query(query: &str) -> bool { + let trimmed = query.trim_start(); + let upper = trimmed.to_uppercase(); + upper.starts_with("SELECT") + || upper.starts_with("WITH") + || upper.starts_with("VALUES") + || upper.starts_with("TABLE") + || upper.starts_with("FROM") +} + +static PARAM_RE: LazyLock = LazyLock::new(|| Regex::new(r"\$\d+").expect("invalid regex")); + +fn replace_params_with_null(query: &str) -> String { + PARAM_RE.replace_all(query, "NULL").to_string() +} + +#[unsafe(no_mangle)] +pub extern "C" fn prepare_duckdb_ffi( + query_block_list: *const *const c_char, + query_block_list_count: usize, + token: *const c_char, + base_internal_url: *const c_char, + w_id: *const c_char, +) -> *mut c_char { + let r = match convert_prepare_args( + query_block_list, + query_block_list_count, + token, + base_internal_url, + w_id, + ) + .and_then(|(query_block_list, token, base_internal_url, w_id)| { + prepare_duckdb_internal(query_block_list, token, base_internal_url, w_id) + }) { + Ok(result) => result, + Err(err) => { + let err = serde_json::to_string(&err) + .unwrap_or_else(|_| "Unknown error in duckdb ffi lib".to_string()); + format!("ERROR {}", err) + } + }; + + CString::new(r).map(|s| s.into_raw()).unwrap_or_else(|e| { + println!("Failed to allocate error string in duckdb ffi lib: {:?}", e); + null_mut() + }) +} + +fn setup_duckdb_connection( + conn: &duckdb::Connection, + token: &str, + base_internal_url: &str, + w_id: &str, +) -> Result<(), String> { + let (s3_access_key, s3_secret_key) = token.rsplit_once('.').unwrap_or(("", token)); + let (s3_endpoint_ssl, s3_endpoint) = base_internal_url + .split_once("://") + .unwrap_or(("http", &base_internal_url)); + let s3_endpoint_ssl = s3_endpoint_ssl == "https"; + + conn.execute_batch(&format!( + "INSTALL httpfs; LOAD httpfs; + INSTALL azure; LOAD azure; + CREATE OR REPLACE SECRET s3_secret ( + TYPE s3, + PROVIDER config, + KEY_ID '{s3_access_key}', + SECRET '{s3_secret_key}', + ENDPOINT '{s3_endpoint}/api/w/{w_id}/s3_proxy', + URL_STYLE path, + USE_SSL {s3_endpoint_ssl} + ); + CREATE OR REPLACE SECRET gcs_secret ( + TYPE gcs, + KEY_ID '{s3_access_key}', + SECRET '{s3_secret_key}', + ENDPOINT '{s3_endpoint}/api/w/{w_id}/s3_proxy', + USE_SSL {s3_endpoint_ssl} + ); + ", + )) + .map_err(|e| format!("Error setting up S3 secret: {}", e.to_string())) +} + +fn convert_prepare_args<'a>( + query_block_list: *const *const c_char, + query_block_list_count: usize, + token: *const c_char, + base_internal_url: *const c_char, + w_id: *const c_char, +) -> Result<(Vec<&'a str>, &'a str, &'a str, &'a str), String> { + let query_block_list = unsafe { + std::slice::from_raw_parts(query_block_list, query_block_list_count) + .iter() + .map(|q| { + CStr::from_ptr(*q).to_str().unwrap_or_else(|e| { + println!( + "Invalid query_block string pointer in duckdb ffi: {}", + e.to_string() + ); + "Invalid query_block string pointer in duckdb ffi" + }) + }) + .collect::>() + }; + let token = unsafe { CStr::from_ptr(token) } + .to_str() + .map_err(|e| format!("Invalid token string: {}", e.to_string()))?; + let base_internal_url = unsafe { CStr::from_ptr(base_internal_url) } + .to_str() + .map_err(|e| format!("Invalid base_internal_url string: {}", e.to_string()))?; + let w_id = unsafe { CStr::from_ptr(w_id) } + .to_str() + .map_err(|e| format!("Invalid w_id string: {}", e.to_string()))?; + Ok((query_block_list, token, base_internal_url, w_id)) +} + +fn prepare_duckdb_internal( + query_block_list: Vec<&str>, + token: &str, + base_internal_url: &str, + w_id: &str, +) -> Result { + let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?; + + setup_duckdb_connection(&conn, token, base_internal_url, w_id)?; + + let mut results: Vec = vec![]; + + // IMPORTANT: Setup statements (ATTACH, USE, INSTALL, etc.) are executed but intentionally + // do not produce a PrepareQueryResult entry. The frontend prepends these as connection setup + // before the actual user queries, and mapPrepareResults expects results.length to equal the + // number of user queries (not setup statements). If a new setup-like statement is added to + // the connection flow (e.g. in setup_duckdb_connection or transform_attach_ducklake) without + // also being caught by is_setup_statement, the result count will mismatch and the frontend + // will throw. + for query_block in &query_block_list { + if is_setup_statement(query_block) { + conn.execute_batch(query_block) + .map_err(|e| format!("Error executing setup statement: {}", e.to_string()))?; + continue; + } + + let modified_query = replace_params_with_null(query_block); + // Validate the query parses correctly by preparing it + if let Err(e) = conn.prepare(&modified_query) { + results.push(PrepareQueryResult { columns: None, error: Some(e.to_string()) }); + continue; + } + + // DESCRIBE only works on queries that return result sets (SELECT, WITH, VALUES, TABLE, + // FROM). For non-returning statements (INSERT, UPDATE, DELETE, CREATE, DROP, ALTER, etc.) + // we skip DESCRIBE and assume no columns. + if !is_describable_query(&modified_query) { + results.push(PrepareQueryResult { columns: Some(vec![]), error: None }); + continue; + } + + // Note: We have to use a DESCRIBE statement and cannot simply use the + // methods returned by .prepare() because they panic if the statement was + // not executed at least once (which we specifically do not want to do). + let describe_query = format!("DESCRIBE {}", modified_query); + match conn.prepare(&describe_query).and_then(|mut stmt| { + let rows = stmt.query_map([], |row| { + Ok(PrepareQueryColumnInfo { + name: row.get::<_, String>(0)?, + type_name: row.get::<_, String>(1)?, + }) + })?; + rows.collect::, _>>() + }) { + Ok(columns) => { + results.push(PrepareQueryResult { columns: Some(columns), error: None }); + } + Err(e) => { + results.push(PrepareQueryResult { columns: None, error: Some(e.to_string()) }); + } + } + } + + serde_json::to_string(&results).map_err(|e| e.to_string()) +} + fn convert_args<'a>( query_block_list: *const *const c_char, query_block_list_count: usize, @@ -170,38 +384,7 @@ fn run_duckdb_internal<'a>( ) -> Result<(String, Option>), String> { let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?; - let (s3_access_key, s3_secret_key) = token.split_at(token.rfind('.').unwrap_or(0)); - let s3_secret_key = &s3_secret_key[1..]; - let (s3_endpoint_ssl, s3_endpoint) = base_internal_url - .split_once("://") - .unwrap_or(("http", &base_internal_url)); - let s3_endpoint_ssl = match s3_endpoint_ssl { - "https" => true, - _ => false, - }; - - conn.execute_batch(&format!( - "INSTALL httpfs; LOAD httpfs; - INSTALL azure; LOAD azure; - CREATE OR REPLACE SECRET s3_secret ( - TYPE s3, - PROVIDER config, - KEY_ID '{s3_access_key}', - SECRET '{s3_secret_key}', - ENDPOINT '{s3_endpoint}/api/w/{w_id}/s3_proxy', - URL_STYLE path, - USE_SSL {s3_endpoint_ssl} - ); - CREATE OR REPLACE SECRET gcs_secret ( - TYPE gcs, - KEY_ID '{s3_access_key}', - SECRET '{s3_secret_key}', - ENDPOINT '{s3_endpoint}/api/w/{w_id}/s3_proxy', - USE_SSL {s3_endpoint_ssl} - ); - ", - )) - .map_err(|e| format!("Error setting up S3 secret: {}", e.to_string()))?; + setup_duckdb_connection(&conn, token, base_internal_url, w_id)?; let mut results: Vec>> = vec![]; let mut column_order = None; diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index 645e477a62..f9cecd46ce 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -12,6 +12,10 @@ use windmill_common::{scripts::ScriptHash, DB}; pub mod git_sync_ee; pub mod git_sync_oss; +#[cfg(feature = "private")] +pub use git_sync_ee::{handle_deployment_metadata, handle_fork_branch_creation}; + +#[cfg(not(feature = "private"))] pub use git_sync_oss::{handle_deployment_metadata, handle_fork_branch_creation}; #[derive(Clone, Debug)] @@ -38,6 +42,7 @@ pub enum DeployedObject { EmailTrigger { path: String, parent_path: Option }, Settings { setting_type: String }, Key { key_type: String }, + WorkspaceDependencies { path: String }, } impl DeployedObject { @@ -65,6 +70,7 @@ impl DeployedObject { DeployedObject::EmailTrigger { path, .. } => path.to_owned(), DeployedObject::Settings { .. } => "settings.yaml".to_string(), DeployedObject::Key { .. } => "encryption_key.yaml".to_string(), + DeployedObject::WorkspaceDependencies { path, .. } => path.to_owned(), } } @@ -74,7 +80,8 @@ impl DeployedObject { | Self::Group { .. } | Self::ResourceType { .. } | Self::Settings { .. } - | Self::Key { .. } => true, + | Self::Key { .. } + | Self::WorkspaceDependencies { .. } => true, _ => false, } } @@ -103,6 +110,7 @@ impl DeployedObject { DeployedObject::EmailTrigger { parent_path, .. } => parent_path.to_owned(), DeployedObject::Settings { .. } => None, DeployedObject::Key { .. } => None, + DeployedObject::WorkspaceDependencies { .. } => None, } } @@ -130,6 +138,244 @@ impl DeployedObject { DeployedObject::EmailTrigger { .. } => "email_trigger", DeployedObject::Settings { .. } => "settings", DeployedObject::Key { .. } => "key", - }.to_string() + DeployedObject::WorkspaceDependencies { .. } => "workspace_dependencies", + } + .to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use windmill_common::scripts::ScriptHash; + + // --- DeployedObject::get_path tests --- + + #[test] + fn test_get_path_script() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "f/folder/script".to_string(), + parent_path: None, + }; + assert_eq!(obj.get_path(), "f/folder/script"); + } + + #[test] + fn test_get_path_flow() { + let obj = DeployedObject::Flow { + path: "f/folder/flow".to_string(), + parent_path: Some("f/folder/old_flow".to_string()), + version: 1, + }; + assert_eq!(obj.get_path(), "f/folder/flow"); + } + + #[test] + fn test_get_path_user() { + let obj = DeployedObject::User { email: "user@example.com".to_string() }; + assert_eq!(obj.get_path(), "users/user@example.com"); + } + + #[test] + fn test_get_path_group() { + let obj = DeployedObject::Group { name: "admins".to_string() }; + assert_eq!(obj.get_path(), "groups/admins"); + } + + #[test] + fn test_get_path_settings() { + let obj = DeployedObject::Settings { setting_type: "error_handler".to_string() }; + assert_eq!(obj.get_path(), "settings.yaml"); + } + + #[test] + fn test_get_path_key() { + let obj = DeployedObject::Key { key_type: "encryption".to_string() }; + assert_eq!(obj.get_path(), "encryption_key.yaml"); + } + + #[test] + fn test_get_path_workspace_dependencies() { + let obj = DeployedObject::WorkspaceDependencies { + path: "workspace-dependencies/python".to_string(), + }; + assert_eq!(obj.get_path(), "workspace-dependencies/python"); + } + + // --- DeployedObject::get_ignore_regex_filter tests --- + + #[test] + fn test_ignore_regex_filter_user() { + let obj = DeployedObject::User { email: "user@example.com".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_group() { + let obj = DeployedObject::Group { name: "admins".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_resource_type() { + let obj = DeployedObject::ResourceType { path: "postgresql".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_settings() { + let obj = DeployedObject::Settings { setting_type: "error_handler".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_key() { + let obj = DeployedObject::Key { key_type: "encryption".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_workspace_dependencies() { + let obj = DeployedObject::WorkspaceDependencies { + path: "workspace-dependencies/python".to_string(), + }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_script() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "f/folder/script".to_string(), + parent_path: None, + }; + assert!(!obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_flow() { + let obj = DeployedObject::Flow { + path: "f/folder/flow".to_string(), + parent_path: None, + version: 1, + }; + assert!(!obj.get_ignore_regex_filter()); + } + + // --- DeployedObject::get_parent_path tests --- + + #[test] + fn test_get_parent_path_script_with_parent() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "f/folder/script".to_string(), + parent_path: Some("f/folder/old_script".to_string()), + }; + assert_eq!(obj.get_parent_path(), Some("f/folder/old_script".to_string())); + } + + #[test] + fn test_get_parent_path_script_without_parent() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "f/folder/script".to_string(), + parent_path: None, + }; + assert_eq!(obj.get_parent_path(), None); + } + + #[test] + fn test_get_parent_path_folder() { + let obj = DeployedObject::Folder { path: "f/folder".to_string() }; + assert_eq!(obj.get_parent_path(), None); + } + + #[test] + fn test_get_parent_path_workspace_dependencies() { + let obj = DeployedObject::WorkspaceDependencies { + path: "workspace-dependencies/python".to_string(), + }; + assert_eq!(obj.get_parent_path(), None); + } + + // --- DeployedObject::get_kind tests --- + + #[test] + fn test_get_kind_script() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "test".to_string(), + parent_path: None, + }; + assert_eq!(obj.get_kind(), "script"); + } + + #[test] + fn test_get_kind_flow() { + let obj = DeployedObject::Flow { + path: "test".to_string(), + parent_path: None, + version: 1, + }; + assert_eq!(obj.get_kind(), "flow"); + } + + #[test] + fn test_get_kind_app() { + let obj = DeployedObject::App { + path: "test".to_string(), + version: 1, + parent_path: None, + }; + assert_eq!(obj.get_kind(), "app"); + } + + #[test] + fn test_get_kind_workspace_dependencies() { + let obj = DeployedObject::WorkspaceDependencies { + path: "workspace-dependencies/python".to_string(), + }; + assert_eq!(obj.get_kind(), "workspace_dependencies"); + } + + #[test] + fn test_get_kind_all_triggers() { + assert_eq!( + DeployedObject::HttpTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "http_trigger" + ); + assert_eq!( + DeployedObject::WebsocketTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "websocket_trigger" + ); + assert_eq!( + DeployedObject::KafkaTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "kafka_trigger" + ); + assert_eq!( + DeployedObject::NatsTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "nats_trigger" + ); + assert_eq!( + DeployedObject::PostgresTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "postgres_trigger" + ); + assert_eq!( + DeployedObject::MqttTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "mqtt_trigger" + ); + assert_eq!( + DeployedObject::SqsTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "sqs_trigger" + ); + assert_eq!( + DeployedObject::GcpTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "gcp_trigger" + ); + assert_eq!( + DeployedObject::EmailTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "email_trigger" + ); } } diff --git a/backend/windmill-jseval/src/lib.rs b/backend/windmill-jseval/src/lib.rs index 89feaf7b3e..7172e8cfc8 100644 --- a/backend/windmill-jseval/src/lib.rs +++ b/backend/windmill-jseval/src/lib.rs @@ -152,6 +152,21 @@ pub fn try_exact_property_access( None } +/// JS runtime properties (not methods) that cannot be resolved by PostgreSQL's +/// #> JSON path operator. Function calls like .map(...) already don't match the +/// RE_FULL regex due to parentheses, so only property accesses need listing here. +const JS_ONLY_PROPERTIES: &[&str] = &["length"]; + +fn ends_with_js_only_property(rest: Option<&str>) -> bool { + match rest { + None => false, + Some(rest) => { + let last_segment = rest.rsplit('.').next().unwrap_or(""); + JS_ONLY_PROPERTIES.contains(&last_segment) + } + } +} + pub async fn handle_full_regex( expr: &str, authed_client: &AuthedClient, @@ -162,6 +177,13 @@ pub async fn handle_full_regex( let obj_key = captures.get(2).unwrap().as_str(); let idx_o = captures.get(3).map(|y| y.as_str()); let rest = captures.get(4).map(|y| y.as_str()); + + // Skip the SQL fast path when the expression accesses a JS runtime + // property (e.g. .length) that the PostgreSQL #> operator can't resolve. + if ends_with_js_only_property(rest) { + return None; + } + let query = if let Some(idx) = idx_o { match rest { Some(rest) => Some(format!("{}{}", idx, rest)), diff --git a/backend/windmill-mcp/src/common/mod.rs b/backend/windmill-mcp/src/common/mod.rs index d67c1718cc..60ad18a85c 100644 --- a/backend/windmill-mcp/src/common/mod.rs +++ b/backend/windmill-mcp/src/common/mod.rs @@ -11,6 +11,8 @@ pub mod types; pub use schema::convert_schema_to_schema_type; pub use scope::{is_resource_allowed, parse_mcp_scopes, McpScopeConfig}; pub use transform::{ - apply_key_transformation, reverse_transform, reverse_transform_key, transform_path, + apply_key_transformation, extract_hub_version_id_from_hashed, + extract_path_prefix_from_hashed, parse_tool_prefix, reverse_transform, reverse_transform_key, + transform_hub_path, transform_path, }; pub use types::*; diff --git a/backend/windmill-mcp/src/common/transform.rs b/backend/windmill-mcp/src/common/transform.rs index 9a6fa317b1..0a12c85892 100644 --- a/backend/windmill-mcp/src/common/transform.rs +++ b/backend/windmill-mcp/src/common/transform.rs @@ -4,9 +4,15 @@ //! to make them compatible with MCP tool naming requirements. use super::types::SchemaType; +use windmill_common::utils::calculate_hash; -/// MCP clients do not allow names longer than 60 characters -const MAX_PATH_LENGTH: usize = 60; +/// Max tool name length. The MCP spec allows 64 chars, but some clients +/// (e.g. Cursor) prepend the server name to the tool name, so we use 40 +/// to leave room for that prefix. +const MAX_PATH_LENGTH: usize = 40; + +/// Length of the SHA256 hash suffix used for hashed names +const HASH_LEN: usize = 16; /// Transform the path for workspace scripts/flows /// @@ -14,19 +20,133 @@ const MAX_PATH_LENGTH: usize = 60; /// path with the type prefix. This is used when listing, because we can't /// have names with slashes. Because we replace slashes with underscores, /// we also need to escape underscores. +/// +/// For short names (≤40 chars): `s-{escaped_path}` or `f-{escaped_path}` +/// For long names (>40 chars): `S-{escaped[:22]}{sha256[:16]}` or `F-{escaped[:22]}{sha256[:16]}` +/// +/// The uppercase prefix signals that the name is hashed. pub fn transform_path(path: &str, type_str: &str) -> String { let escaped_path = path.replace('_', "__").replace('/', "_"); - // first letter of type_str is used as prefix, only one letter to avoid reaching 60 char name limit - let transformed_path = format!("{}-{}", &type_str[..1], escaped_path); - if transformed_path.len() > MAX_PATH_LENGTH { - let suffix = "_TRUNC"; - return format!( - "{}{}", - &transformed_path[..MAX_PATH_LENGTH - suffix.len()], - suffix - ); + let prefix_char = &type_str[..1]; + let short_name = format!("{}-{}", prefix_char, escaped_path); + + if short_name.len() <= MAX_PATH_LENGTH { + return short_name; } - transformed_path + + let upper_prefix = prefix_char.to_uppercase(); + // Layout: "{Upper}-" (2 chars) + prefix_body (22 chars) + hash (16 chars) = 40 + let prefix_body_len = MAX_PATH_LENGTH - 2 - HASH_LEN; + let hash = calculate_hash(&short_name); + let hash_suffix = &hash[..HASH_LEN]; + let truncated = truncate_to_char_boundary(&escaped_path, prefix_body_len); + format!("{}-{}{}", upper_prefix, truncated, hash_suffix) +} + +/// Transform the path for hub scripts +/// +/// For short names (≤40 chars): `hs-{id}-{summary}` +/// For long names (>40 chars): `Hs-{id}-{summary[:N]}{sha256[:16]}` +pub fn transform_hub_path(version_id: u64, summary: &str) -> String { + let escaped_summary = summary.replace(' ', "_"); + let short_name = format!("hs-{}-{}", version_id, escaped_summary); + + if short_name.len() <= MAX_PATH_LENGTH { + return short_name; + } + + let hash = calculate_hash(&short_name); + let hash_suffix = &hash[..HASH_LEN]; + // "Hs-{id}-" prefix, then fill remaining with summary + hash + let fixed_prefix = format!("Hs-{}-", version_id); + let available = MAX_PATH_LENGTH - fixed_prefix.len() - HASH_LEN; + let truncated_summary = truncate_to_char_boundary(&escaped_summary, available); + format!("{}{}{}", fixed_prefix, truncated_summary, hash_suffix) +} + +/// Parse the prefix of any tool name (both short and hashed). +/// Returns `(type_str, is_hub, is_hashed)`. +/// Hashed names use an uppercase first character as the signal. +pub fn parse_tool_prefix(name: &str) -> Result<(&str, bool, bool), String> { + let is_hashed = name.chars().next().map(|c| c.is_ascii_uppercase()).unwrap_or(false); + let lower = name.to_ascii_lowercase(); + let (type_str, is_hub) = if lower.starts_with("hs-") { + ("script", true) + } else if lower.starts_with("s-") { + ("script", false) + } else if lower.starts_with("f-") { + ("flow", false) + } else { + return Err(format!("Invalid tool name prefix: {}", name)); + }; + Ok((type_str, is_hub, is_hashed)) +} + +/// Extract the hub version_id from a hashed hub script name like `Hs-{id}-...` +pub fn extract_hub_version_id_from_hashed(name: &str) -> Result { + let rest = name + .strip_prefix("Hs-") + .ok_or_else(|| format!("Not a hashed hub name: {}", name))?; + let id = rest + .split('-') + .next() + .ok_or_else(|| format!("No version_id in hashed hub name: {}", name))?; + if id.is_empty() { + return Err(format!("Empty version_id in hashed hub name: {}", name)); + } + Ok(id.to_string()) +} + +/// Extract a safe original-path prefix from a hashed tool name. +/// +/// Given `S-u_admin_engineering__te`, extracts the escaped prefix between +/// the type prefix (`S-`, `F-`, or `Hs-`) and the hash, un-escapes it, and +/// returns a prefix suitable for `WHERE path LIKE '{prefix}%'`. +/// +/// Returns `None` if the name is too short or has an unrecognized prefix. +pub fn extract_path_prefix_from_hashed(name: &str) -> Option { + let prefix_len = if name.starts_with("Hs-") { + 3 + } else if name.starts_with("S-") || name.starts_with("F-") { + 2 + } else { + return None; + }; + if name.len() <= prefix_len + HASH_LEN { + return None; + } + let escaped_prefix = &name[prefix_len..name.len() - HASH_LEN]; + if escaped_prefix.is_empty() { + return None; + } + + // Strip trailing underscores — they may be half of a `__` pair split by truncation + let trimmed = escaped_prefix.trim_end_matches('_'); + if trimmed.is_empty() { + return None; + } + + Some(unescape_path(trimmed)) +} + +/// Un-escape a mangled path segment: `__` → `_`, standalone `_` → `/`. +fn unescape_path(s: &str) -> String { + const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@"; + s.replace("__", TEMP_PLACEHOLDER) + .replace('_', "/") + .replace(TEMP_PLACEHOLDER, "_") +} + +/// Truncate a string to at most `max_len` bytes, ensuring we don't split a UTF-8 character. +fn truncate_to_char_boundary(s: &str, max_len: usize) -> &str { + if s.len() <= max_len { + return s; + } + let mut end = max_len; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + &s[..end] } /// Reverse the transformation of a path @@ -38,25 +158,22 @@ pub fn transform_path(path: &str, type_str: &str) -> String { /// This is used in call_tool to get the original path, and the type of the item. /// /// Returns: (type, original_path, is_hub) +/// +/// Note: This only works for non-hashed (short) names. Hashed names must be +/// resolved via `parse_tool_prefix` + path enumeration in the runner. pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool), String> { - let is_hub = transformed_path.starts_with("h"); - let transformed_path = if is_hub { - transformed_path[1..].to_string() - } else { - transformed_path.to_string() - }; - let type_str = if transformed_path.starts_with("s-") { - "script" - } else if transformed_path.starts_with("f-") { - "flow" - } else { - return Err(format!( - "Invalid prefix in transformed path: {}", - transformed_path - )); - }; + let (type_str, is_hub, is_hashed) = parse_tool_prefix(transformed_path)?; - let mangled_path = &transformed_path[2..]; + if is_hashed { + return Err( + "Hashed names cannot be reverse-transformed directly; use path enumeration instead" + .to_string(), + ); + } + + // Strip the prefix: "hs-" (3 chars) for hub, "s-"/"f-" (2 chars) for others + let prefix_len = if is_hub { 3 } else { 2 }; + let mangled_path = &transformed_path[prefix_len..]; let original_path = if is_hub { let parts = mangled_path.split("-").collect::>(); @@ -65,11 +182,7 @@ pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool), } parts[0].to_string() } else { - const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@"; - mangled_path - .replace("__", TEMP_PLACEHOLDER) - .replace('_', "/") - .replace(TEMP_PLACEHOLDER, "_") + unescape_path(mangled_path) }; Ok((type_str, original_path, is_hub)) @@ -97,16 +210,13 @@ pub fn reverse_transform_key(transformed_key: &str, schema_obj: &Option s, None => { - // No schema available, return the key as is (best guess) return transformed_key.to_string(); } }; for original_key_in_schema in schema_obj.properties.keys() { - // Apply the SAME forward transformation to the schema key let potential_transformed_key = apply_key_transformation(original_key_in_schema); - // If it matches the key we received, we found the likely original if potential_transformed_key == transformed_key { return original_key_in_schema.clone(); } @@ -120,7 +230,7 @@ mod tests { use super::*; #[test] - fn test_transform_path() { + fn test_transform_path_short() { assert_eq!( transform_path("u/admin/script", "script"), "s-u_admin_script" @@ -130,7 +240,108 @@ mod tests { } #[test] - fn test_reverse_transform() { + fn test_transform_path_long_is_hashed() { + let long_path = "u/engineering/team/automation/very_long_script_name_that_exceeds_limit"; + let result = transform_path(long_path, "script"); + assert_eq!(result.len(), MAX_PATH_LENGTH); + assert!(result.starts_with("S-")); + let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap(); + assert!(is_hashed); + } + + #[test] + fn test_transform_path_long_flow_is_hashed() { + let long_path = "f/engineering/team/automation/very_long_flow_name_that_exceeds_limit"; + let result = transform_path(long_path, "flow"); + assert_eq!(result.len(), MAX_PATH_LENGTH); + assert!(result.starts_with("F-")); + let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap(); + assert!(is_hashed); + } + + #[test] + fn test_transform_path_hashing_is_deterministic() { + let path = "u/engineering/team/automation/very_long_script_name_that_exceeds_limit"; + let a = transform_path(path, "script"); + let b = transform_path(path, "script"); + assert_eq!(a, b); + } + + #[test] + fn test_transform_path_different_long_paths_differ() { + let a = transform_path( + "u/engineering/team/automation/very_long_script_name_that_exceeds_limit_a", + "script", + ); + let b = transform_path( + "u/engineering/team/automation/very_long_script_name_that_exceeds_limit_b", + "script", + ); + assert_ne!(a, b); + } + + #[test] + fn test_transform_hub_path_short() { + let result = transform_hub_path(12345, "Send Slack Message"); + assert_eq!(result, "hs-12345-Send_Slack_Message"); + let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap(); + assert!(!is_hashed); + } + + #[test] + fn test_transform_hub_path_long_is_hashed() { + let result = transform_hub_path( + 12345, + "Send Slack Message To Channel With Very Long Description That Exceeds Limit", + ); + assert_eq!(result.len(), MAX_PATH_LENGTH); + assert!(result.starts_with("Hs-12345-")); + let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap(); + assert!(is_hashed); + } + + #[test] + fn test_extract_hub_version_id_from_hashed() { + let name = "Hs-12345-Send_Slack_Message_To_Ch9e8d7c6b5a4f3e2d"; + let id = extract_hub_version_id_from_hashed(name).unwrap(); + assert_eq!(id, "12345"); + } + + #[test] + fn test_parse_tool_prefix() { + let (t, hub, hashed) = parse_tool_prefix("S-something").unwrap(); + assert_eq!(t, "script"); + assert!(!hub); + assert!(hashed); + + let (t, hub, hashed) = parse_tool_prefix("F-something").unwrap(); + assert_eq!(t, "flow"); + assert!(!hub); + assert!(hashed); + + let (t, hub, hashed) = parse_tool_prefix("Hs-12345-something").unwrap(); + assert_eq!(t, "script"); + assert!(hub); + assert!(hashed); + + let (t, hub, hashed) = parse_tool_prefix("s-u_admin_script").unwrap(); + assert_eq!(t, "script"); + assert!(!hub); + assert!(!hashed); + + let (t, hub, hashed) = parse_tool_prefix("f-f_folder_flow").unwrap(); + assert_eq!(t, "flow"); + assert!(!hub); + assert!(!hashed); + + let (t, hub, hashed) = parse_tool_prefix("hs-12345-summary").unwrap(); + assert_eq!(t, "script"); + assert!(hub); + assert!(!hashed); + } + + #[test] + fn test_reverse_transform_short_names() { let (type_str, path, is_hub) = reverse_transform("s-u_admin_script").unwrap(); assert_eq!(type_str, "script"); assert_eq!(path, "u/admin/script"); @@ -142,6 +353,70 @@ mod tests { assert!(!is_hub); } + #[test] + fn test_extract_path_prefix_from_hashed() { + // Generate a real hashed name and verify prefix extraction + let long_path = "u/admin/engineering/team/automation/very_long_script"; + let hashed = transform_path(long_path, "script"); + let (_, _, is_hashed) = parse_tool_prefix(&hashed).unwrap(); + assert!(is_hashed); + + let prefix = extract_path_prefix_from_hashed(&hashed).unwrap(); + // The original path should start with the extracted prefix + assert!( + long_path.starts_with(&prefix), + "path '{}' should start with prefix '{}'", + long_path, + prefix + ); + } + + #[test] + fn test_extract_path_prefix_underscore_in_path() { + let long_path = "u/admin/my_team/automation/very_long_script_name_here"; + let hashed = transform_path(long_path, "script"); + let prefix = extract_path_prefix_from_hashed(&hashed).unwrap(); + assert!( + long_path.starts_with(&prefix), + "path '{}' should start with prefix '{}'", + long_path, + prefix + ); + } + + #[test] + fn test_extract_path_prefix_rejects_invalid_prefix() { + assert!(extract_path_prefix_from_hashed("x-something").is_none()); + assert!(extract_path_prefix_from_hashed("").is_none()); + assert!(extract_path_prefix_from_hashed("S-").is_none()); + } + + #[test] + fn test_extract_path_prefix_handles_hs_prefix() { + // Hs- is 3 chars, not 2 — ensure the prefix is stripped correctly + let hashed = transform_hub_path(12345, "a]very long hub script summary that exceeds the limit"); + let (_, is_hub, is_hashed) = parse_tool_prefix(&hashed).unwrap(); + assert!(is_hub); + assert!(is_hashed); + + let prefix = extract_path_prefix_from_hashed(&hashed); + // Should not start with 's' (leftover from Hs- if sliced at index 2) + if let Some(ref p) = prefix { + assert!( + !p.starts_with('s'), + "prefix '{}' should not start with 's' from mis-sliced Hs- prefix", + p + ); + } + } + + #[test] + fn test_reverse_transform_rejects_hashed_names() { + assert!(reverse_transform("S-something").is_err()); + assert!(reverse_transform("F-something").is_err()); + assert!(reverse_transform("Hs-12345-something").is_err()); + } + #[test] fn test_apply_key_transformation() { assert_eq!(apply_key_transformation("my key"), "my_key"); diff --git a/backend/windmill-mcp/src/common/types.rs b/backend/windmill-mcp/src/common/types.rs index 43fe223ded..6161ca7963 100644 --- a/backend/windmill-mcp/src/common/types.rs +++ b/backend/windmill-mcp/src/common/types.rs @@ -92,8 +92,10 @@ pub struct ItemSchema { /// Trait for objects that can be converted to MCP tools pub trait ToolableItem { - /// Get the path or identifier for this item (transformed for MCP compatibility) - fn get_path_or_id(&self) -> String; + /// Get the MCP-compatible tool name (path transformed with escaping/hashing) + fn get_transformed_path(&self) -> String; + /// Get the original full path of this item (for display in tool title) + fn get_full_path(&self) -> &str; /// Get the summary/title of this item fn get_summary(&self) -> &str; /// Get the description of this item diff --git a/backend/windmill-mcp/src/lib.rs b/backend/windmill-mcp/src/lib.rs index a75b545154..7df6ee9f39 100644 --- a/backend/windmill-mcp/src/lib.rs +++ b/backend/windmill-mcp/src/lib.rs @@ -14,9 +14,9 @@ pub mod client; // Re-export common types at crate root for convenience pub use common::{ - convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_path, FlowInfo, - HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo, ResourceType, SchemaType, - ScriptInfo, ToolableItem, WorkspaceId, + convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_hub_path, + transform_path, FlowInfo, HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo, + ResourceType, SchemaType, ScriptInfo, ToolableItem, WorkspaceId, }; // Re-export client types at crate root for backward compatibility diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs index fd01b2b26c..0b942353b3 100644 --- a/backend/windmill-mcp/src/server/backend.rs +++ b/backend/windmill-mcp/src/server/backend.rs @@ -55,20 +55,22 @@ pub trait McpBackend: Send + Sync + Clone + 'static { // Listing Operations // ───────────────────────────────────────────────────────────────── - /// List scripts, optionally filtered to favorites only + /// List scripts, optionally filtered to favorites only and/or by path prefix async fn list_scripts( &self, auth: &Self::Auth, workspace_id: &str, favorites_only: bool, + path_prefix: Option<&str>, ) -> BackendResult>; - /// List flows, optionally filtered to favorites only + /// List flows, optionally filtered to favorites only and/or by path prefix async fn list_flows( &self, auth: &Self::Auth, workspace_id: &str, favorites_only: bool, + path_prefix: Option<&str>, ) -> BackendResult>; /// List resource types in workspace diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs index dd65c96334..be8764fa2e 100644 --- a/backend/windmill-mcp/src/server/runner.rs +++ b/backend/windmill-mcp/src/server/runner.rs @@ -5,7 +5,10 @@ use crate::common::schema::extract_resource_types_from_schema; use crate::common::scope::parse_mcp_scopes; -use crate::common::transform::{reverse_transform, reverse_transform_key}; +use crate::common::transform::{ + extract_hub_version_id_from_hashed, extract_path_prefix_from_hashed, parse_tool_prefix, + reverse_transform, reverse_transform_key, +}; use crate::common::types::{ResourceInfo, ToolableItem, WorkspaceId}; use crate::server::backend::{McpAuth, McpBackend}; use crate::server::endpoints::endpoint_tool_to_mcp_tool; @@ -81,6 +84,13 @@ impl Runner { } } +fn find_matching_path(candidates: Vec, request_name: &str) -> Option { + candidates + .into_iter() + .find(|item| item.get_transformed_path() == request_name) + .map(|item| item.get_full_path().to_string()) +} + impl ServerHandler for Runner { fn get_info(&self) -> ServerInfo { ServerInfo { @@ -120,9 +130,9 @@ impl ServerHandler for Runner { // Fetch all items concurrently let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!( self.backend - .list_scripts(&auth, &workspace_id, favorites_only), + .list_scripts(&auth, &workspace_id, favorites_only, None), self.backend - .list_flows(&auth, &workspace_id, favorites_only), + .list_flows(&auth, &workspace_id, favorites_only, None), self.backend.list_resource_types(&auth, &workspace_id), async { if let Some(ref apps) = scope_config.hub_apps { @@ -231,17 +241,6 @@ impl ServerHandler for Runner { let scope_config = parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?; - // Handle truncated tool names - if request.name.ends_with("_TRUNC") { - return Ok(CallToolResult::error(vec![rmcp::model::Annotated::new( - rmcp::model::RawContent::Text(rmcp::model::RawTextContent { - text: "Tool path is too long. Consider shortening it to make it compatible with MCP.".to_string(), - meta: None, - }), - None, - )])); - } - let args = request.arguments.map(Value::Object).unwrap_or(Value::Null); // Check if this is an endpoint tool @@ -274,10 +273,58 @@ impl ServerHandler for Runner { } } - // Not an endpoint tool - parse as script/flow - let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| { - ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None) - })?; + // Resolve the tool name to (type, path, is_hub) + let (type_str, is_hub, is_hashed) = + parse_tool_prefix(&request.name).map_err(|e| { + ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None) + })?; + + let (tool_type, path, is_hub) = if !is_hashed { + reverse_transform(&request.name).map_err(|e| { + ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None) + })? + } else if is_hub { + let version_id = + extract_hub_version_id_from_hashed(&request.name).map_err(|e| { + ErrorData::internal_error( + format!("Failed to extract hub version_id: {}", e), + None, + ) + })?; + (type_str, version_id, true) + } else { + let path_prefix = extract_path_prefix_from_hashed(&request.name); + let favorites_only = scope_config.favorites; + let matched_path = if type_str == "script" { + find_matching_path( + self.backend + .list_scripts(&auth, &workspace_id, favorites_only, path_prefix.as_deref()) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))?, + &request.name, + ) + } else { + find_matching_path( + self.backend + .list_flows(&auth, &workspace_id, favorites_only, path_prefix.as_deref()) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))?, + &request.name, + ) + }; + + let matched_path = matched_path.ok_or_else(|| { + ErrorData::internal_error( + format!( + "No {} found matching hashed tool name '{}'", + type_str, request.name + ), + None, + ) + })?; + + (type_str, matched_path, false) + }; // Validate script/flow scope if !is_hub && scope_config.granular { diff --git a/backend/windmill-mcp/src/server/tools.rs b/backend/windmill-mcp/src/server/tools.rs index a9262cdd6f..50ed03426b 100644 --- a/backend/windmill-mcp/src/server/tools.rs +++ b/backend/windmill-mcp/src/server/tools.rs @@ -9,7 +9,7 @@ use std::collections::HashMap; use std::sync::Arc; use crate::common::schema::{convert_schema_to_schema_type, make_schema_compatible}; -use crate::common::transform::transform_path; +use crate::common::transform::{transform_hub_path, transform_path}; use crate::common::types::{ FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem, }; @@ -17,10 +17,14 @@ use crate::server::backend::McpBackend; /// Implementation of ToolableItem for ScriptInfo impl ToolableItem for ScriptInfo { - fn get_path_or_id(&self) -> String { + fn get_transformed_path(&self) -> String { transform_path(&self.path, "script") } + fn get_full_path(&self) -> &str { + &self.path + } + fn get_summary(&self) -> &str { self.summary.as_deref().unwrap_or("No summary") } @@ -48,10 +52,14 @@ impl ToolableItem for ScriptInfo { /// Implementation of ToolableItem for FlowInfo impl ToolableItem for FlowInfo { - fn get_path_or_id(&self) -> String { + fn get_transformed_path(&self) -> String { transform_path(&self.path, "flow") } + fn get_full_path(&self) -> &str { + &self.path + } + fn get_summary(&self) -> &str { self.summary.as_deref().unwrap_or("No summary") } @@ -79,10 +87,13 @@ impl ToolableItem for FlowInfo { /// Implementation of ToolableItem for HubScriptInfo impl ToolableItem for HubScriptInfo { - fn get_path_or_id(&self) -> String { - let id = self.version_id; + fn get_transformed_path(&self) -> String { let summary = self.summary.as_deref().unwrap_or("No summary"); - format!("hs-{}-{}", id, summary.replace(" ", "_")) + transform_hub_path(self.version_id, summary) + } + + fn get_full_path(&self) -> &str { + self.summary.as_deref().unwrap_or("No summary") } fn get_summary(&self) -> &str { @@ -124,7 +135,7 @@ pub fn create_tool_from_item( resources_types: &[ResourceType], ) -> Tool { let is_hub = item.is_hub(); - let path = item.get_path_or_id(); + let path = item.get_transformed_path(); let item_type = item.item_type(); let description = format!( "This is a {} named `{}` with the following description: `{}`.{}", @@ -170,15 +181,24 @@ pub fn create_tool_from_item( } }; + let title = { + let summary = item.get_summary(); + if summary == "No summary" { + item.get_full_path().to_string() + } else { + summary.to_string() + } + }; + Tool { name: Cow::Owned(path), description: Some(Cow::Owned(description)), input_schema: Arc::new(input_schema_map), - title: Some(item.get_summary().to_string()), + title: Some(title.clone()), output_schema: None, icons: None, annotations: Some(ToolAnnotations { - title: Some(item.get_summary().to_string()), + title: Some(title), read_only_hint: Some(false), // Can modify environment destructive_hint: Some(true), // Can potentially be destructive idempotent_hint: Some(false), // Are not guaranteed to be idempotent diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index e0c499c355..26d807dac3 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -94,7 +94,7 @@ pub struct OAuthConfig { } /// OAuth client credentials -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct OAuthClient { #[serde(default = "empty_string")] pub id: String, @@ -110,6 +110,21 @@ pub struct OAuthClient { pub grant_types: Vec, } +impl std::fmt::Debug for OAuthClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OAuthClient") + .field("id", &self.id) + .field("secret", &"***") + .field("display_name", &self.display_name) + .field("allowed_domains", &self.allowed_domains) + .field("connect_config", &self.connect_config) + .field("login_config", &self.login_config) + .field("tenant", &self.tenant) + .field("grant_types", &self.grant_types) + .finish() + } +} + fn empty_string() -> String { "".to_string() } @@ -608,7 +623,18 @@ pub async fn refresh_token<'c>( .await?; let account = windmill_common::utils::not_found_if_none(account, "Account", &id.to_string())?; - refresh_token_for_account(tx, path, w_id, id, db, account, oauth_clients, http_client, connect_configs_json).await + refresh_token_for_account( + tx, + path, + w_id, + id, + db, + account, + oauth_clients, + http_client, + connect_configs_json, + ) + .await } /// Refresh an OAuth token given pre-fetched account info (no additional SELECT). diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 806e3e99a2..324ea6c0ed 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -818,7 +818,7 @@ pub async fn add_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> Result<(Uuid, i64), Error> { +) -> Result<(Uuid, i64, Option), Error> { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); @@ -830,7 +830,7 @@ pub async fn add_completed_job( } let result_columns = result_columns.as_ref(); - let (opt_uuid, duration, _skip_downstream_error_handlers) = (|| { + let (opt_uuid, duration, _skip_downstream_error_handlers, wac_job_ids) = (|| { commit_completed_job( db, completed_job, @@ -866,7 +866,7 @@ pub async fn add_completed_job( // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout if let Some(job_id) = opt_uuid { - return Ok((job_id, duration)); + return Ok((job_id, duration, None)); } #[cfg(feature = "cloud")] @@ -887,7 +887,7 @@ pub async fn add_completed_job( // tracing::error!("4 {:?}", start.elapsed()); - Ok((completed_job.id, duration)) + Ok((completed_job.id, duration, wac_job_ids)) } async fn commit_completed_job( @@ -902,7 +902,7 @@ async fn commit_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> windmill_common::error::Result<(Option, i64, bool)> { +) -> windmill_common::error::Result<(Option, i64, bool, Option)> { // let start = std::time::Instant::now(); let mut tx = db.begin().warn_after_seconds(10).await?; @@ -1003,25 +1003,31 @@ async fn commit_completed_job( .map_err(|e| Error::InternalErr(format!("Could not update job labels: {e:#}")))?; } + let mut wac_job_ids: Option = None; if !completed_job.is_flow_step() { if let Some(parent_job) = completed_job.parent_job { - let _ = sqlx::query_scalar!( - "UPDATE v2_job_status SET + // Only update WAC parents (v1 or v2). The WHERE condition skips + // non-WAC parents entirely (error handlers, run_script children, etc.). + // Also returns pending_steps.job_ids so WAC v2 child completion + // doesn't need a separate read. + let row = sqlx::query_scalar!( + r#"UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( jsonb_set( - COALESCE(workflow_as_code_status, '{}'::jsonb), + workflow_as_code_status, array[$1], COALESCE(workflow_as_code_status->$1, '{}'::jsonb) ), array[$1, 'duration_ms'], to_jsonb($2::bigint) ) - WHERE id = $3", + WHERE id = $3 AND workflow_as_code_status IS NOT NULL + RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS "job_ids: serde_json::Value""#, &completed_job.id.to_string(), duration, parent_job ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .warn_after_seconds(10) .await .inspect_err(|e| { @@ -1029,7 +1035,10 @@ async fn commit_completed_job( "Could not update parent job `duration_ms` in workflow as code status: {}", e, ) - }); + }) + .ok() + .flatten(); + wac_job_ids = row.flatten(); } } // tracing::error!("Added completed job {:#?}", queued_job); @@ -1250,14 +1259,14 @@ async fn commit_completed_job( completed_job.id ); // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); - Ok((None, duration, _skip_downstream_error_handlers)) + Ok((None, duration, _skip_downstream_error_handlers, wac_job_ids)) } async fn check_result_size( db: &Pool, queued_job: &MiniCompletedJob, result: Json<&T>, -) -> Option, i64, bool), Error>> { +) -> Option, i64, bool, Option), Error>> { let result_size = result.size() / 1024 / 1024; if result_size > 2 { if result_size > *MAX_RESULT_SIZE_MB { @@ -2942,7 +2951,13 @@ impl PulledJobResult { .and_then(|x| x.get("triggered_by_relative_import")) .is_some(); - if (is_djob_to_debounce || debounce_delay_s.filter(|x| *x > 0).is_some()) + let has_debounce_args = debounce_args_to_accumulate + .as_ref() + .map_or(false, |v| !v.is_empty()); + + if (is_djob_to_debounce + || debounce_delay_s.filter(|x| *x > 0).is_some() + || has_debounce_args) && MIN_VERSION_SUPPORTS_DEBOUNCING.met().await && !*WMDEBUG_NO_DEBOUNCING { @@ -3031,11 +3046,18 @@ impl PulledJobResult { let new_value = to_raw_value(&accumulated_arg); + let original_value = j + .args + .as_ref() + .and_then(|a| a.get(arg_name_to_accumulate)) + .map(|v| v.get().to_string()) + .unwrap_or_else(|| "null".to_string()); + append_logs( &j_id, &j.workspace_id, format!( - "Substituting `{arg_name_to_accumulate}` with: {}\n\n", + "Accumulating debounced argument `{arg_name_to_accumulate}`:\n original: {original_value}\n accumulated: {}\n\n", &new_value ), &(db.into()), @@ -3046,6 +3068,18 @@ impl PulledJobResult { .get_or_insert(Json(Default::default())) .as_mut() .insert(arg_name_to_accumulate.to_owned(), new_value); + + // Persist accumulated args to v2_job so that flow steps + // re-reading from the DB (via get_mini_pulled_job) see them + if let Some(ref args) = j.args { + sqlx::query!( + "UPDATE v2_job SET args = $2 WHERE id = $1", + j_id, + args as &Json>>, + ) + .execute(db) + .await?; + } } // Handle dependency job debouncing cleanup when a job is pulled for execution @@ -3594,7 +3628,8 @@ pub async fn check_debouncing_within_limits( ); if allowed_amount - .map(|allowed_amount| current_amount > allowed_amount) + .filter(|&a| a > 0) + .map(|allowed_amount| current_amount + 1 >= allowed_amount) .unwrap_or_default() && no_legacy_compat { @@ -5478,10 +5513,11 @@ async fn push_inner<'c, 'd>( &mut *tx, ) .await - .map_err(|e| { - Error::internal_err(format!( + .map_err(|e| match e { + Error::NotFound(_) => e, + _ => Error::internal_err(format!( "Could not get permissions directly for job {job_id}: {e:#}" - )) + )), })? } }; diff --git a/backend/windmill-queue/tests/debounce_test.rs b/backend/windmill-queue/tests/debounce_test.rs index 0d7f390eff..a2cc664ddd 100644 --- a/backend/windmill-queue/tests/debounce_test.rs +++ b/backend/windmill-queue/tests/debounce_test.rs @@ -585,7 +585,9 @@ mod debounce { Ok(()) } - /// Test: max_total_debounces_amount limit - debounce batch resets when exceeded. + /// Test: max_total_debounces_amount limit - push-time debounce deletes key and + /// completes previous job when limit is reached. With max=2, the 2nd event + /// (debounced_times=1, total events=2) triggers the limit. #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_debounce_max_count_limit(db: Pool) -> anyhow::Result<()> { let settings = DebouncingSettings { @@ -596,8 +598,10 @@ mod debounce { }; let args_hm = empty_args(); - // Push 4 jobs: after the 3rd debounce (exceeding limit of 2), batch should reset + // With max=2: job1 debounced (dt=0), job2 triggers limit (dt=1, 1+1>=2), + // job3 debounced (fresh INSERT), job4 triggers limit (dt=1 again) let mut jobs = Vec::new(); + let mut scheduled_fors = Vec::new(); for _ in 0..4 { let job_id = Uuid::new_v4(); insert_noop_job(&db, job_id, "test-workspace").await; @@ -620,11 +624,44 @@ mod debounce { ) .await?; tx.commit().await?; + scheduled_fors.push(scheduled_for); } - // The debounce_key entry should still exist + // Job 1: debounced (scheduled_for set) + assert!( + scheduled_fors[0].is_some(), + "job1 should be debounced (scheduled_for set)" + ); + // Job 2: limit exceeded → scheduled_for cleared, previous job completed + assert!( + scheduled_fors[1].is_none(), + "job2 should execute immediately (limit exceeded)" + ); + assert!( + is_completed(&db, &jobs[0]).await, + "job1 should be completed (debounced by job2 at limit)" + ); + // Job 3: new batch (fresh INSERT after DELETE) + assert!( + scheduled_fors[2].is_some(), + "job3 should be debounced (new batch)" + ); + // Job 4: limit exceeded again + assert!( + scheduled_fors[3].is_none(), + "job4 should execute immediately (limit exceeded)" + ); + assert!( + is_completed(&db, &jobs[2]).await, + "job3 should be completed (debounced by job4 at limit)" + ); + + // The debounce_key entry should be deleted after the last limit exceeded let dk = get_debounce_key(&db, "count_limit_key").await; - assert!(dk.is_some(), "debounce_key entry should exist"); + assert!( + dk.is_none(), + "debounce_key entry should be deleted after limit exceeded" + ); Ok(()) } @@ -975,7 +1012,8 @@ mod debounce { Ok(()) } - /// Test: Post-preprocessing debounce with max count limit resets the batch. + /// Test: Post-preprocessing debounce with max count limit deletes the debounce_key entry + /// and completes the previous job. With max=2, the 2nd event triggers the limit. #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_post_preprocessing_debounce_max_count_resets( db: Pool, @@ -988,7 +1026,8 @@ mod debounce { }; let args_hm = empty_args(); - // Push 4 jobs. After 3rd debounce (exceeding limit of 2), batch should reset. + // With max=2: job1 debounced (dt=0), job2 limit exceeded (dt=1, 1+1>=2), + // job3 debounced (fresh INSERT), job4 limit exceeded (dt=1 again) let mut jobs = Vec::new(); let mut results = Vec::new(); for _ in 0..4 { @@ -1011,20 +1050,41 @@ mod debounce { results.push(result); } - // First job always gets scheduled_for + // Job 1: debounced (first in batch) assert!(results[0].is_some(), "first job should get scheduled_for"); - // Jobs 2 and 3 should also get scheduled_for (debouncing within limit) - assert!(results[1].is_some(), "second job should get scheduled_for"); - assert!(results[2].is_some(), "third job should get scheduled_for"); + // Job 2: limit exceeded → execute immediately, complete job1 + assert!( + results[1].is_none(), + "second job should execute immediately (limit exceeded)" + ); + assert!( + is_completed(&db, &jobs[0]).await, + "job1 should be completed (debounced by job2 at limit)" + ); - // Job 4 (the one that exceeds the limit): when limit is exceeded, - // the batch resets and the job executes immediately (no scheduled_for delay) - // The exact behavior depends on whether the limit check happens before or after the - // new job is counted. Let's just verify the debounce_key is reset. - let dk = get_debounce_key(&db, "pp_max_count_key").await.unwrap(); - // debounced_times should have been reset at some point - assert!(dk.0 == jobs[3], "debounce_key should point to last job"); + // Job 3: new batch (fresh INSERT after DELETE) + assert!( + results[2].is_some(), + "third job should get scheduled_for (new batch)" + ); + + // Job 4: limit exceeded again → execute immediately, complete job3 + assert!( + results[3].is_none(), + "fourth job should execute immediately (limit exceeded)" + ); + assert!( + is_completed(&db, &jobs[2]).await, + "job3 should be completed (debounced by job4 at limit)" + ); + + // debounce_key should be deleted after the last limit exceeded + let dk = get_debounce_key(&db, "pp_max_count_key").await; + assert!( + dk.is_none(), + "debounce_key entry should be deleted when limits exceeded" + ); Ok(()) } @@ -1390,20 +1450,23 @@ mod debounce { &db, ) .await?; - // When limit is exceeded, the function resets and returns None (execute immediately) + // When limit is exceeded, the function returns None (execute immediately) assert!( r2.is_none(), "should return None when time limit is exceeded" ); - // Verify the batch was reset: debounced_times should be 0 - let dk = get_debounce_key(&db, "pp_time_limit_key").await.unwrap(); - assert_eq!(dk.2, 0, "debounced_times should be reset to 0"); - - // Job 1 should NOT be completed (time limit reset skips debouncing the previous job) + // Verify the debounce_key entry is deleted (not just reset) + let dk = get_debounce_key(&db, "pp_time_limit_key").await; assert!( - is_queued(&db, &job1).await, - "job1 should still be queued (time limit reset doesn't debounce)" + dk.is_none(), + "debounce_key entry should be deleted when time limit exceeded" + ); + + // Job 1 should be completed (debounced by job2 when limit exceeded) + assert!( + is_completed(&db, &job1).await, + "job1 should be completed (debounced by job2 at time limit)" ); Ok(()) @@ -1466,16 +1529,31 @@ mod debounce { ) .await?; tx.commit().await?; - // scheduled_for is still set (push-time doesn't clear it on limit exceed) - // but the batch should be reset - let dk = get_debounce_key(&db, "push_time_limit_key").await.unwrap(); - assert_eq!(dk.2, 0, "debounced_times should be reset to 0"); + + // scheduled_for should be cleared (execute immediately when limit exceeded) + assert!( + sf2.is_none(), + "scheduled_for should be cleared when time limit exceeded" + ); + + // debounce_key should be deleted (not just reset) + let dk = get_debounce_key(&db, "push_time_limit_key").await; + assert!( + dk.is_none(), + "debounce_key entry should be deleted when time limit exceeded" + ); + + // Job 1 should be completed (debounced by job2 at time limit) + assert!( + is_completed(&db, &job1).await, + "job1 should be completed (debounced by job2 at time limit)" + ); Ok(()) } - /// Test: max_count boundary — at exactly the limit, debouncing still works. - /// One over the limit triggers reset. + /// Test: max_count boundary — with max=3, the 3rd event (debounced_times=2, + /// total events=3) triggers the limit. Events 1-2 debounce, event 3 launches. #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_post_preprocessing_max_count_exact_boundary( db: Pool, @@ -1490,7 +1568,8 @@ mod debounce { let mut jobs = Vec::new(); let mut results = Vec::new(); - // Push 5 jobs: job 1 (no debounce), jobs 2-4 (debounce, count 1-3), job 5 (count 4 > limit 3 → reset) + // With max=3: job1 dt=0 (1 event), job2 dt=1 (2 events), job3 dt=2 (3 events → limit), + // job4 dt=0 (new batch), job5 dt=1 (2 events) for _ in 0..5 { let id = Uuid::new_v4(); insert_flow_job(&db, id, "test-workspace", "f/test/flow").await; @@ -1511,31 +1590,39 @@ mod debounce { results.push(result); } - // Jobs 1-4 should return Some (scheduled_for) — debouncing within limit - for (i, r) in results.iter().enumerate().take(4) { - assert!( - r.is_some(), - "job {} should get scheduled_for (within limit)", - i + 1 - ); - } + // Jobs 1-2: debounced (within limit) + assert!(results[0].is_some(), "job 1 should get scheduled_for"); + assert!(results[1].is_some(), "job 2 should get scheduled_for"); - // Job 5 (debounced_times=4, exceeds limit=3) should return None (batch reset) + // Job 3: limit exceeded (dt=2, 2+1=3 >= 3) → execute immediately assert!( - results[4].is_none(), - "job 5 should return None (limit exceeded, batch reset)" + results[2].is_none(), + "job 3 should return None (limit exceeded)" ); - // After reset, debounced_times should be 0 - let dk = get_debounce_key(&db, "pp_count_boundary_key") - .await - .unwrap(); - assert_eq!(dk.2, 0, "debounced_times should be reset to 0 after limit"); + // Jobs 4-5: new batch after DELETE + assert!( + results[3].is_some(), + "job 4 should get scheduled_for (new batch)" + ); + assert!( + results[4].is_some(), + "job 5 should get scheduled_for (within limit)" + ); + + // debounce_key should exist (pointing to job5, within new batch) + let dk = get_debounce_key(&db, "pp_count_boundary_key").await; + assert!( + dk.is_some(), + "debounce_key entry should exist (new batch in progress)" + ); Ok(()) } - /// Test: after a max_count reset, a new batch starts fresh and debouncing works again. + /// Test: after a max_count limit exceeded, a new batch starts completely fresh. + /// The debounce_key entry is deleted, so the next cycle starts with a fresh INSERT. + /// With max=2: every 2 events forms a batch (1st debounced, 2nd triggers limit). #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_post_preprocessing_max_count_reset_new_batch( db: Pool, @@ -1548,9 +1635,8 @@ mod debounce { }; let args_hm = empty_args(); - // Limit check is `debounced_times > max`, so with max=2 we need 4 jobs - // to trigger reset (debounced_times=3 on the 4th job, 3>2=true). - // Cycle 1: jobs 1-4 (job 4 exceeds limit → reset) + // With max=2: job1 debounced, job2 triggers limit (completes job1), + // job3 debounced (new batch), job4 triggers limit (completes job3) let mut cycle1 = Vec::new(); for _ in 0..4 { let id = Uuid::new_v4(); @@ -1571,22 +1657,28 @@ mod debounce { .await?; cycle1_results.push(r); } - assert!(cycle1_results[0].is_some(), "cycle1 job1 scheduled"); - assert!(cycle1_results[1].is_some(), "cycle1 job2 scheduled"); + assert!(cycle1_results[0].is_some(), "cycle1 job1 debounced"); + assert!( + cycle1_results[1].is_none(), + "cycle1 job2 should execute immediately (limit)" + ); assert!( cycle1_results[2].is_some(), - "cycle1 job3 scheduled (at limit)" + "cycle1 job3 debounced (new batch)" ); assert!( cycle1_results[3].is_none(), - "cycle1 job4 should reset (over limit)" + "cycle1 job4 should execute immediately (limit)" ); - // Verify debounced_times is reset to 0 - let dk = get_debounce_key(&db, "pp_reset_cycle_key").await.unwrap(); - assert_eq!(dk.2, 0, "debounced_times should be 0 after reset"); + // Verify debounce_key entry is deleted after limit exceeded + let dk = get_debounce_key(&db, "pp_reset_cycle_key").await; + assert!( + dk.is_none(), + "debounce_key entry should be deleted after limit exceeded" + ); - // Cycle 2: jobs 5-8 (new batch, should debounce independently) + // Cycle 2: completely fresh batch since entry was deleted let mut cycle2 = Vec::new(); for _ in 0..4 { let id = Uuid::new_v4(); @@ -1607,19 +1699,22 @@ mod debounce { .await?; cycle2_results.push(r); } - // After cycle 1 reset, debounced_times=0. Cycle 2's first job hits ON CONFLICT - // and increments to 1 (unlike cycle 1's first job which was a fresh insert at 0). - // So cycle 2 reaches the limit one job sooner: - // job5: dt=1, job6: dt=2, job7: dt=3 (>2 → reset), job8: dt=1 - assert!(cycle2_results[0].is_some(), "cycle2 job1 scheduled (dt=1)"); - assert!(cycle2_results[1].is_some(), "cycle2 job2 scheduled (dt=2)"); + // Cycle 2 behaves identically: [debounced, limit, debounced, limit] assert!( - cycle2_results[2].is_none(), - "cycle2 job3 should reset (dt=3 > 2)" + cycle2_results[0].is_some(), + "cycle2 job1 debounced (fresh INSERT)" ); assert!( - cycle2_results[3].is_some(), - "cycle2 job4 scheduled (fresh after reset, dt=1)" + cycle2_results[1].is_none(), + "cycle2 job2 should execute immediately (limit)" + ); + assert!( + cycle2_results[2].is_some(), + "cycle2 job3 debounced (new batch)" + ); + assert!( + cycle2_results[3].is_none(), + "cycle2 job4 should execute immediately (limit)" ); Ok(()) @@ -1735,7 +1830,9 @@ mod debounce { Ok(()) } - /// Test: after a max_count reset, the new batch gets a different batch ID. + /// Test: the limit-triggered job stays in the same batch as its predecessors + /// (so args can be accumulated), and the next batch after reset is different. + /// With max=3: batch of 3 events (2 debounced + 1 limit trigger), then new batch. #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_post_preprocessing_batch_id_changes_on_reset( db: Pool, @@ -1743,14 +1840,14 @@ mod debounce { let settings = DebouncingSettings { debounce_delay_s: Some(5), debounce_key: Some("pp_batch_reset_id_test".to_string()), - max_total_debounces_amount: Some(2), + max_total_debounces_amount: Some(3), ..Default::default() }; let args_hm = empty_args(); - // Batch 1: jobs 1-4 (job 4 triggers reset at debounced_times=3 > 2) + // With max=3: jobs 1-2 debounced (dt=0,1), job 3 triggers limit (dt=2, 2+1>=3) let mut batch1_jobs = Vec::new(); - for _ in 0..4 { + for _ in 0..3 { let id = Uuid::new_v4(); insert_flow_job(&db, id, "test-workspace", "f/test/flow").await; batch1_jobs.push(id); @@ -1768,7 +1865,7 @@ mod debounce { .await?; } - // Batch 2: jobs 5-6 (new batch after reset) + // Batch 2: jobs 4-5 (new batch after DELETE, within limit) let mut batch2_jobs = Vec::new(); for _ in 0..2 { let id = Uuid::new_v4(); @@ -1795,17 +1892,18 @@ mod debounce { .fetch_one(&db) .await?; - // Job 4 (the one that triggered reset) should have a different batch from jobs 1-3 - let reset_batch: i64 = sqlx::query_scalar!( + // Job 3 (limit-triggered) stays in the same batch as jobs 1-2 + // so that maybe_apply_debouncing can accumulate args from all 3. + let trigger_batch: i64 = sqlx::query_scalar!( "SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1", - batch1_jobs[3], + batch1_jobs[2], ) .fetch_one(&db) .await?; - assert_ne!( - batch1_id, reset_batch, - "reset job should have a different batch ID" + assert_eq!( + batch1_id, trigger_batch, + "limit-triggered job should stay in the same batch for arg accumulation" ); // Batch 2 jobs should share the same batch but different from batch 1 @@ -2325,6 +2423,346 @@ mod debounce { Ok(()) } + /// Test: 5 webhook calls with max_total_debounces_amount=2 and debounce_args_to_accumulate. + /// Simulates the flow described by the user: debounce_delay_s=50, max=2, accumulate x. + /// + /// Expected behavior: + /// Call 1: debounced (first in batch, scheduled_for set) + /// Call 2: launched immediately (limit reached at 2 total events, completes call 1) + /// Call 3: debounced (new batch starts fresh) + /// Call 4: launched immediately (limit reached again, completes call 3) + /// Call 5: debounced (new batch, waiting for delay or more events) + #[sqlx::test(migrations = "../migrations", fixtures("base"))] + async fn test_post_preprocessing_webhook_5_calls_max_2( + db: Pool, + ) -> anyhow::Result<()> { + let settings = DebouncingSettings { + debounce_delay_s: Some(50), + debounce_key: Some("webhook_5calls_key".to_string()), + max_total_debounces_amount: Some(2), + debounce_args_to_accumulate: Some(vec!["x".to_string()]), + ..Default::default() + }; + + let mut jobs = Vec::new(); + let mut results = Vec::new(); + + for i in 0..5 { + let id = Uuid::new_v4(); + let args_val = serde_json::json!({"x": [i + 1]}); + insert_flow_job_with_args(&db, id, "test-workspace", "f/test/flow", &args_val).await; + jobs.push(id); + + let args_hm: HashMap> = serde_json::from_value(args_val).unwrap(); + let args = PushArgs::from(&args_hm); + let result = windmill_queue::jobs_ee::maybe_debounce_post_preprocessing( + &settings, + &Some("f/test/flow".to_string()), + "test-workspace", + id, + &args, + &db, + ) + .await?; + results.push(result); + } + + // Call 1: debounced (first in batch) + assert!( + results[0].is_some(), + "call 1 should be debounced (scheduled_for set)" + ); + + // Call 2: launched immediately (limit exceeded: dt=1, 1+1 >= 2) + assert!( + results[1].is_none(), + "call 2 should launch immediately (limit reached at 2 total events)" + ); + + // Call 3: debounced (new batch, fresh INSERT after key was deleted by call 2) + assert!( + results[2].is_some(), + "call 3 should be debounced (new batch started)" + ); + + // Call 4: launched immediately (limit exceeded again) + assert!( + results[3].is_none(), + "call 4 should launch immediately (limit reached again)" + ); + + // Call 5: debounced (new batch) + assert!( + results[4].is_some(), + "call 5 should be debounced (new batch, waiting for delay)" + ); + + // Verify final state after all 5 calls: + // Completed: jobs[0] (debounced by call 2), jobs[2] (debounced by call 4) + assert!( + is_completed(&db, &jobs[0]).await, + "job from call 1 should be completed (debounced by call 2)" + ); + assert!( + is_completed(&db, &jobs[2]).await, + "job from call 3 should be completed (debounced by call 4)" + ); + + // Queued: jobs[1] (launched immediately), jobs[3] (launched immediately), jobs[4] (debounced, waiting) + assert!( + is_queued(&db, &jobs[1]).await, + "call 2's job should be queued (launched immediately)" + ); + assert!( + is_queued(&db, &jobs[3]).await, + "call 4's job should be queued (launched immediately)" + ); + assert!( + is_queued(&db, &jobs[4]).await, + "call 5's job should be queued (debounced, waiting)" + ); + + // debounce_key should exist pointing to call 5's job (the active batch) + let dk = get_debounce_key(&db, "webhook_5calls_key").await; + assert!(dk.is_some(), "debounce_key should exist for call 5's batch"); + let (dk_job_id, _, dk_times) = dk.unwrap(); + assert_eq!(dk_job_id, jobs[4], "debounce_key should point to call 5"); + assert_eq!( + dk_times, 0, + "debounced_times should be 0 (first in new batch)" + ); + + Ok(()) + } + + /// Test: 5 webhook calls with max_total_debounces_amount=2 verifies both + /// debounce behavior AND accumulated arg values via maybe_apply_debouncing. + /// + /// Each call sends {x: [i]}. Expected: + /// Call 1 (x=[1]): debounced + /// Call 2 (x=[2]): fires immediately (limit), accumulated x=[1,2] + /// Call 3 (x=[3]): debounced (new batch) + /// Call 4 (x=[4]): fires immediately (limit), accumulated x=[3,4] + /// Call 5 (x=[5]): debounced (new batch), only x=[5] since batch has 1 job + #[sqlx::test(migrations = "../migrations", fixtures("base"))] + async fn test_post_preprocessing_max_count_accumulation( + db: Pool, + ) -> anyhow::Result<()> { + let settings = DebouncingSettings { + debounce_delay_s: Some(50), + debounce_key: Some("max_count_accum_key".to_string()), + max_total_debounces_amount: Some(2), + debounce_args_to_accumulate: Some(vec!["x".to_string()]), + ..Default::default() + }; + let rs_handle = setup_debouncing_settings(&db, &settings).await; + + let mut jobs = Vec::new(); + let mut results = Vec::new(); + + for i in 0..5 { + let id = Uuid::new_v4(); + let args_val = serde_json::json!({"x": [i + 1]}); + insert_flow_job_with_args(&db, id, "test-workspace", "f/test/accum_flow", &args_val) + .await; + jobs.push((id, args_val.clone())); + + let args_hm: HashMap> = serde_json::from_value(args_val).unwrap(); + let args = PushArgs::from(&args_hm); + let result = windmill_queue::jobs_ee::maybe_debounce_post_preprocessing( + &settings, + &Some("f/test/accum_flow".to_string()), + "test-workspace", + id, + &args, + &db, + ) + .await?; + results.push(result); + } + + // Verify debounce behavior + assert!(results[0].is_some(), "call 1 debounced"); + assert!(results[1].is_none(), "call 2 fires immediately"); + assert!(results[2].is_some(), "call 3 debounced"); + assert!(results[3].is_none(), "call 4 fires immediately"); + assert!(results[4].is_some(), "call 5 debounced"); + + // Call 2 fires immediately with MaxCountExceeded. + // Simulate worker: store runnable_settings_handle, then call maybe_apply_debouncing. + let survivor_2 = jobs[1].0; + sqlx::query!( + "UPDATE v2_job_queue SET runnable_settings_handle = $1 WHERE id = $2", + rs_handle, + survivor_2, + ) + .execute(&db) + .await?; + + let mut pulled_2 = make_pulled_job_result( + survivor_2, + "test-workspace", + "f/test/accum_flow", + &jobs[1].1, + JobKind::Flow, + "flow", + rs_handle, + ); + pulled_2.maybe_apply_debouncing(&db).await?; + assert_accumulated_items(&pulled_2, &[1, 2], "x"); + + // Call 4 fires immediately with MaxCountExceeded. + let survivor_4 = jobs[3].0; + sqlx::query!( + "UPDATE v2_job_queue SET runnable_settings_handle = $1 WHERE id = $2", + rs_handle, + survivor_4, + ) + .execute(&db) + .await?; + + let mut pulled_4 = make_pulled_job_result( + survivor_4, + "test-workspace", + "f/test/accum_flow", + &jobs[3].1, + JobKind::Flow, + "flow", + rs_handle, + ); + pulled_4.maybe_apply_debouncing(&db).await?; + assert_accumulated_items(&pulled_4, &[3, 4], "x"); + + // Call 5 is debounced (only job in its batch so far). + // When eventually pulled, it should only have its own args. + let survivor_5 = jobs[4].0; + sqlx::query!( + "UPDATE v2_job_queue SET runnable_settings_handle = $1 WHERE id = $2", + rs_handle, + survivor_5, + ) + .execute(&db) + .await?; + + let mut pulled_5 = make_pulled_job_result( + survivor_5, + "test-workspace", + "f/test/accum_flow", + &jobs[4].1, + JobKind::Flow, + "flow", + rs_handle, + ); + pulled_5.maybe_apply_debouncing(&db).await?; + assert_accumulated_items(&pulled_5, &[5], "x"); + + Ok(()) + } + + /// Test: same as above but triggered by max_total_debouncing_time instead of count. + /// Uses a very short time window so that the 2nd call exceeds it. + /// + /// Call 1 (x=[10]): debounced + /// -- sleep past max_total_debouncing_time -- + /// Call 2 (x=[20]): fires immediately (time exceeded), accumulated x=[10,20] + /// Call 3 (x=[30]): debounced (new batch) + #[sqlx::test(migrations = "../migrations", fixtures("base"))] + async fn test_post_preprocessing_max_time_accumulation( + db: Pool, + ) -> anyhow::Result<()> { + let settings = DebouncingSettings { + debounce_delay_s: Some(50), + debounce_key: Some("max_time_accum_key".to_string()), + max_total_debouncing_time: Some(1), // 1 second + debounce_args_to_accumulate: Some(vec!["x".to_string()]), + ..Default::default() + }; + let rs_handle = setup_debouncing_settings(&db, &settings).await; + + // Call 1: debounced + let id1 = Uuid::new_v4(); + let args1 = serde_json::json!({"x": [10]}); + insert_flow_job_with_args(&db, id1, "test-workspace", "f/test/time_accum", &args1).await; + + let args_hm: HashMap> = + serde_json::from_value(args1.clone()).unwrap(); + let r1 = windmill_queue::jobs_ee::maybe_debounce_post_preprocessing( + &settings, + &Some("f/test/time_accum".to_string()), + "test-workspace", + id1, + &PushArgs::from(&args_hm), + &db, + ) + .await?; + assert!(r1.is_some(), "call 1 should be debounced"); + + // Wait for time to exceed + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + + // Call 2: time exceeded, fires immediately + let id2 = Uuid::new_v4(); + let args2 = serde_json::json!({"x": [20]}); + insert_flow_job_with_args(&db, id2, "test-workspace", "f/test/time_accum", &args2).await; + + let args_hm2: HashMap> = + serde_json::from_value(args2.clone()).unwrap(); + let r2 = windmill_queue::jobs_ee::maybe_debounce_post_preprocessing( + &settings, + &Some("f/test/time_accum".to_string()), + "test-workspace", + id2, + &PushArgs::from(&args_hm2), + &db, + ) + .await?; + assert!( + r2.is_none(), + "call 2 should fire immediately (time exceeded)" + ); + + // Simulate worker: store handle and call maybe_apply_debouncing + sqlx::query!( + "UPDATE v2_job_queue SET runnable_settings_handle = $1 WHERE id = $2", + rs_handle, + id2, + ) + .execute(&db) + .await?; + + let mut pulled = make_pulled_job_result( + id2, + "test-workspace", + "f/test/time_accum", + &args2, + JobKind::Flow, + "flow", + rs_handle, + ); + pulled.maybe_apply_debouncing(&db).await?; + assert_accumulated_items(&pulled, &[10, 20], "x"); + + // Call 3: new batch, debounced + let id3 = Uuid::new_v4(); + let args3 = serde_json::json!({"x": [30]}); + insert_flow_job_with_args(&db, id3, "test-workspace", "f/test/time_accum", &args3).await; + + let args_hm3: HashMap> = + serde_json::from_value(args3.clone()).unwrap(); + let r3 = windmill_queue::jobs_ee::maybe_debounce_post_preprocessing( + &settings, + &Some("f/test/time_accum".to_string()), + "test-workspace", + id3, + &PushArgs::from(&args_hm3), + &db, + ) + .await?; + assert!(r3.is_some(), "call 3 should be debounced (new batch)"); + + Ok(()) + } + // ========================================================================= // Stress test for DB contention (run manually with --ignored) // ========================================================================= @@ -3074,6 +3512,29 @@ mod debounce { let other: String = serde_json::from_str(other_raw.get())?; assert_eq!(other, "x", "non-accumulated arg should be unchanged"); + // Verify accumulated args were persisted to v2_job (needed for flows + // where subsequent steps re-read args from the DB) + let db_args: Option = + sqlx::query_scalar!("SELECT args FROM v2_job WHERE id = $1", survivor_id,) + .fetch_one(&db) + .await?; + let db_args = db_args.expect("v2_job args should not be null after accumulation"); + let db_items = db_args + .get("items") + .expect("persisted args should contain 'items'"); + let db_items: Vec = + serde_json::from_value::>(db_items.clone())? + .iter() + .map(|v| v.as_i64().unwrap()) + .collect(); + let mut db_items_sorted = db_items.clone(); + db_items_sorted.sort(); + assert_eq!( + db_items_sorted, + vec![1, 2, 3, 4, 5, 6], + "persisted args in v2_job should contain all accumulated items" + ); + Ok(()) } @@ -3417,6 +3878,237 @@ mod debounce { Ok(()) } + /// Test: Push-time (script) debounce with max_total_debounces_amount=2. + /// 5 calls, each sending {x: [i]}. Expected: + /// Call 1: debounced (scheduled_for set) + /// Call 2: fires immediately (limit), accumulated x=[1,2] + /// Call 3: debounced (new batch) + /// Call 4: fires immediately (limit), accumulated x=[3,4] + /// Call 5: debounced (new batch), x=[5] + #[sqlx::test(migrations = "../migrations", fixtures("base"))] + async fn test_push_max_count_accumulation(db: Pool) -> anyhow::Result<()> { + let settings = DebouncingSettings { + debounce_delay_s: Some(50), + debounce_key: Some("push_count_accum_key".to_string()), + max_total_debounces_amount: Some(2), + debounce_args_to_accumulate: Some(vec!["x".to_string()]), + ..Default::default() + }; + let rs_handle = setup_debouncing_settings(&db, &settings).await; + + let mut jobs = Vec::new(); + let mut scheduled_fors = Vec::new(); + + for i in 0..5 { + let id = Uuid::new_v4(); + let args_val = serde_json::json!({"x": [i + 1]}); + insert_script_job_with_args(&db, id, "test-workspace", "f/test/push_script", &args_val) + .await; + sqlx::query!( + "UPDATE v2_job_queue SET runnable_settings_handle = $1 WHERE id = $2", + rs_handle, + id, + ) + .execute(&db) + .await?; + jobs.push((id, args_val.clone())); + + let args_hm: HashMap> = serde_json::from_value(args_val).unwrap(); + let push_args = PushArgs::from(&args_hm); + let mut scheduled_for = None; + let mut tx = db.begin().await?; + windmill_queue::jobs_ee::maybe_debounce( + &settings, + &mut scheduled_for, + &Some("f/test/push_script".to_string()), + "test-workspace", + JobKind::Script, + id, + &push_args, + &mut tx, + ) + .await?; + tx.commit().await?; + scheduled_fors.push(scheduled_for); + } + + // Verify debounce behavior + assert!(scheduled_fors[0].is_some(), "call 1 debounced"); + assert!(scheduled_fors[1].is_none(), "call 2 fires immediately"); + assert!(scheduled_fors[2].is_some(), "call 3 debounced"); + assert!(scheduled_fors[3].is_none(), "call 4 fires immediately"); + assert!(scheduled_fors[4].is_some(), "call 5 debounced"); + + // Call 2: accumulate args + let mut pulled_2 = make_pulled_job_result( + jobs[1].0, + "test-workspace", + "f/test/push_script", + &jobs[1].1, + JobKind::Script, + "deno", + rs_handle, + ); + pulled_2.maybe_apply_debouncing(&db).await?; + assert_accumulated_items(&pulled_2, &[1, 2], "x"); + + // Call 4: accumulate args + let mut pulled_4 = make_pulled_job_result( + jobs[3].0, + "test-workspace", + "f/test/push_script", + &jobs[3].1, + JobKind::Script, + "deno", + rs_handle, + ); + pulled_4.maybe_apply_debouncing(&db).await?; + assert_accumulated_items(&pulled_4, &[3, 4], "x"); + + // Call 5: only its own args + let mut pulled_5 = make_pulled_job_result( + jobs[4].0, + "test-workspace", + "f/test/push_script", + &jobs[4].1, + JobKind::Script, + "deno", + rs_handle, + ); + pulled_5.maybe_apply_debouncing(&db).await?; + assert_accumulated_items(&pulled_5, &[5], "x"); + + Ok(()) + } + + /// Test: Push-time (script) debounce with max_total_debouncing_time=1s. + /// Call 1 (x=[10]): debounced + /// -- sleep past max time -- + /// Call 2 (x=[20]): fires immediately (time exceeded), accumulated x=[10,20] + /// Call 3 (x=[30]): debounced (new batch) + #[sqlx::test(migrations = "../migrations", fixtures("base"))] + async fn test_push_max_time_accumulation(db: Pool) -> anyhow::Result<()> { + let settings = DebouncingSettings { + debounce_delay_s: Some(50), + debounce_key: Some("push_time_accum_key".to_string()), + max_total_debouncing_time: Some(1), + debounce_args_to_accumulate: Some(vec!["x".to_string()]), + ..Default::default() + }; + let rs_handle = setup_debouncing_settings(&db, &settings).await; + + // Call 1: debounced + let id1 = Uuid::new_v4(); + let args1 = serde_json::json!({"x": [10]}); + insert_script_job_with_args(&db, id1, "test-workspace", "f/test/push_time", &args1).await; + sqlx::query!( + "UPDATE v2_job_queue SET runnable_settings_handle = $1 WHERE id = $2", + rs_handle, + id1, + ) + .execute(&db) + .await?; + + let args_hm: HashMap> = + serde_json::from_value(args1.clone()).unwrap(); + let mut sf1 = None; + let mut tx = db.begin().await?; + windmill_queue::jobs_ee::maybe_debounce( + &settings, + &mut sf1, + &Some("f/test/push_time".to_string()), + "test-workspace", + JobKind::Script, + id1, + &PushArgs::from(&args_hm), + &mut tx, + ) + .await?; + tx.commit().await?; + assert!(sf1.is_some(), "call 1 should be debounced"); + + // Wait for time to exceed + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + + // Call 2: time exceeded, fires immediately + let id2 = Uuid::new_v4(); + let args2 = serde_json::json!({"x": [20]}); + insert_script_job_with_args(&db, id2, "test-workspace", "f/test/push_time", &args2).await; + sqlx::query!( + "UPDATE v2_job_queue SET runnable_settings_handle = $1 WHERE id = $2", + rs_handle, + id2, + ) + .execute(&db) + .await?; + + let args_hm2: HashMap> = + serde_json::from_value(args2.clone()).unwrap(); + let mut sf2 = None; + let mut tx = db.begin().await?; + windmill_queue::jobs_ee::maybe_debounce( + &settings, + &mut sf2, + &Some("f/test/push_time".to_string()), + "test-workspace", + JobKind::Script, + id2, + &PushArgs::from(&args_hm2), + &mut tx, + ) + .await?; + tx.commit().await?; + assert!( + sf2.is_none(), + "call 2 should fire immediately (time exceeded)" + ); + + // Verify accumulation + let mut pulled = make_pulled_job_result( + id2, + "test-workspace", + "f/test/push_time", + &args2, + JobKind::Script, + "deno", + rs_handle, + ); + pulled.maybe_apply_debouncing(&db).await?; + assert_accumulated_items(&pulled, &[10, 20], "x"); + + // Call 3: new batch, debounced + let id3 = Uuid::new_v4(); + let args3 = serde_json::json!({"x": [30]}); + insert_script_job_with_args(&db, id3, "test-workspace", "f/test/push_time", &args3).await; + sqlx::query!( + "UPDATE v2_job_queue SET runnable_settings_handle = $1 WHERE id = $2", + rs_handle, + id3, + ) + .execute(&db) + .await?; + + let args_hm3: HashMap> = + serde_json::from_value(args3.clone()).unwrap(); + let mut sf3 = None; + let mut tx = db.begin().await?; + windmill_queue::jobs_ee::maybe_debounce( + &settings, + &mut sf3, + &Some("f/test/push_time".to_string()), + "test-workspace", + JobKind::Script, + id3, + &PushArgs::from(&args_hm3), + &mut tx, + ) + .await?; + tx.commit().await?; + assert!(sf3.is_some(), "call 3 should be debounced (new batch)"); + + Ok(()) + } + /// Test: Flow (without preprocessor) debounce accumulation via push-time maybe_debounce. #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_flow_debounce_accumulation_no_preprocessor( @@ -3504,6 +4196,27 @@ mod debounce { assert_accumulated_items(&result, &[10, 20, 30, 40, 50], "items"); + // Verify accumulated args were persisted to v2_job + let db_args: Option = + sqlx::query_scalar!("SELECT args FROM v2_job WHERE id = $1", survivor_id,) + .fetch_one(&db) + .await?; + let db_items = db_args + .expect("v2_job args should not be null") + .get("items") + .expect("persisted args should contain 'items'") + .clone(); + let mut db_items: Vec = serde_json::from_value::>(db_items)? + .iter() + .map(|v| v.as_i64().unwrap()) + .collect(); + db_items.sort(); + assert_eq!( + db_items, + vec![10, 20, 30, 40, 50], + "persisted args in v2_job should contain all accumulated items" + ); + Ok(()) } @@ -3604,6 +4317,33 @@ mod debounce { let extra: String = serde_json::from_str(extra_raw.get())?; assert_eq!(extra, "v", "non-accumulated arg should be unchanged"); + // Verify accumulated args were persisted to v2_job + let db_args: Option = + sqlx::query_scalar!("SELECT args FROM v2_job WHERE id = $1", survivor_id,) + .fetch_one(&db) + .await?; + let db_args = db_args.expect("v2_job args should not be null"); + let db_items = db_args + .get("items") + .expect("persisted args should contain 'items'") + .clone(); + let mut db_items: Vec = serde_json::from_value::>(db_items)? + .iter() + .map(|v| v.as_i64().unwrap()) + .collect(); + db_items.sort(); + assert_eq!( + db_items, + vec![100, 200, 300, 400, 500, 600], + "persisted args in v2_job should contain all accumulated items" + ); + // "extra" should also be persisted unchanged + let db_extra = db_args.get("extra").unwrap().as_str().unwrap(); + assert_eq!( + db_extra, "v", + "persisted non-accumulated arg should be unchanged" + ); + Ok(()) } diff --git a/backend/windmill-runtime-nativets/src/dedicated.rs b/backend/windmill-runtime-nativets/src/dedicated.rs index 5533c36262..5838456de9 100644 --- a/backend/windmill-runtime-nativets/src/dedicated.rs +++ b/backend/windmill-runtime-nativets/src/dedicated.rs @@ -56,6 +56,7 @@ impl PrewarmedIsolate { js_code: String, ann: NativeAnnotation, arg_names: Vec, + entrypoint: Option, ) -> Self { let (args_tx, args_rx) = tokio::sync::oneshot::channel::(); let (result_tx, result_rx) = tokio::sync::oneshot::channel::(); @@ -98,7 +99,7 @@ impl PrewarmedIsolate { }); let exec_result = tokio::select! { - r = execute_main(&mut js_runtime, None, false, None) => r, + r = execute_main(&mut js_runtime, entrypoint.as_deref(), false, None) => r, _ = memory_limit_rx.recv() => { Err(ExecuteError::Script("Memory limit reached, killing isolate".to_string())) } diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs index a523106cbc..e81925e7bb 100644 --- a/backend/windmill-runtime-nativets/src/lib.rs +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -45,7 +45,7 @@ use uuid::Uuid; use windmill_common::error::Error; use windmill_common::result_stream::append_result_stream_db; -use windmill_common::worker::{write_file, Connection, TMP_DIR}; +use windmill_common::worker::{write_file, Connection, WINDMILL_DIR}; // ── Permission container ───────────────────────────────────────────── @@ -151,7 +151,9 @@ static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH pub(crate) const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js"); -const ERROR_DIR: &str = const_format::concatcp!(TMP_DIR, "/native_errors"); +lazy_static::lazy_static! { + static ref ERROR_DIR: String = format!("{}/native_errors", *WINDMILL_DIR); +} lazy_static! { static ref RE_PROXY: Regex = @@ -263,14 +265,14 @@ fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> { } fn write_error_expr(expr: &str, uuid: &Uuid) { - if let Err(e) = std::fs::create_dir_all(ERROR_DIR) { - tracing::error!("failed to create error dir {ERROR_DIR}: {e}"); + if let Err(e) = std::fs::create_dir_all(&*ERROR_DIR) { + tracing::error!("failed to create error dir {}: {e}", *ERROR_DIR); return; } - let dir_entries = match std::fs::read_dir(ERROR_DIR) { + let dir_entries = match std::fs::read_dir(&*ERROR_DIR) { Ok(entries) => entries.count(), Err(_) => { - tracing::error!("failed to read error dir {ERROR_DIR}"); + tracing::error!("failed to read error dir {}", *ERROR_DIR); return; } }; @@ -279,15 +281,16 @@ fn write_error_expr(expr: &str, uuid: &Uuid) { tracing::info!("native error for job {uuid}: {expr}"); } if dir_entries >= 100 { - tracing::info!("Too many error files in {ERROR_DIR}, skipping write"); + tracing::info!("Too many error files in {}, skipping write", *ERROR_DIR); return; } let path = format!("/{uuid}.js"); tracing::info!( - "nativets job {uuid} failed, writing error expr to {ERROR_DIR}/{path} for debugging: {path}" + "nativets job {uuid} failed, writing error expr to {}/{path} for debugging: {path}", + *ERROR_DIR ); - if let Err(e) = write_file(ERROR_DIR, &path, expr) { + if let Err(e) = write_file(&ERROR_DIR, &path, expr) { tracing::error!("failed to write error expr to file {path}: {e}"); } } diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index bcac53e7d2..6b414ca10f 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -47,7 +47,7 @@ use windmill_common::{ StripPath, }, variables, - worker::{CLOUD_HOSTED, TMP_DIR}, + worker::{CLOUD_HOSTED, WINDMILL_DIR}, PgDatabase, }; @@ -1752,7 +1752,7 @@ async fn write_ssh_file( var_path: &str, ) -> std::result::Result { let id_file_name = format!(".ssh_id_priv_{}", Uuid::new_v4()); - let loc = std::path::Path::new(TMP_DIR) + let loc = std::path::Path::new(&*WINDMILL_DIR) .join("ssh_ids") .join(id_file_name); diff --git a/backend/windmill-test-utils/Cargo.toml b/backend/windmill-test-utils/Cargo.toml index e805ee5a3d..d1729a7511 100644 --- a/backend/windmill-test-utils/Cargo.toml +++ b/backend/windmill-test-utils/Cargo.toml @@ -10,8 +10,8 @@ path = "src/lib.rs" [features] default = [] -private = [] -enterprise = [] +private = ["windmill-api/private"] +enterprise = ["windmill-api/enterprise"] python = ["windmill-common/python"] deno_core = ["dep:windmill-runtime-nativets"] agent_worker_server = ["dep:windmill-api-agent-workers"] diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index adbeb83ee7..5d5c80b470 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -66,6 +66,8 @@ fn next_worker_name() -> String { .unwrap_or(s) }) .unwrap_or("no thread name"); + // Replace colons because they are illegal in Windows directory names + let thread_name = thread_name.replace(':', "_"); format!("{id}/worker-{thread_name}") } @@ -149,6 +151,7 @@ pub struct RunJob { pub args: serde_json::Map, pub scheduled_for_o: Option>, pub email: String, + pub job_id: Option, } impl From for RunJob { @@ -158,6 +161,7 @@ impl From for RunJob { args: Default::default(), scheduled_for_o: None, email: "test@windmill.dev".to_string(), + job_id: None, } } } @@ -181,8 +185,13 @@ impl RunJob { self } + pub fn job_id(mut self, id: Uuid) -> Self { + self.job_id = Some(id); + self + } + pub async fn push(self, db: &Pool) -> Uuid { - let RunJob { payload, args, scheduled_for_o, email } = self; + let RunJob { payload, args, scheduled_for_o, email, job_id } = self; let mut hm_args = std::collections::HashMap::new(); for (k, v) in args { hm_args.insert(k, windmill_common::worker::to_raw_value(&v)); @@ -204,7 +213,7 @@ impl RunJob { /* parent_job */ None, /* root job */ None, /* flow_innermost_root_job */ None, - /* job_id */ None, + /* job_id */ job_id, /* is_flow_step */ false, /* same_worker */ false, None, @@ -228,7 +237,7 @@ impl RunJob { /// Push the job as a specific user (for testing permissions) pub async fn push_as(self, db: &Pool, username: &str, email: &str) -> Uuid { - let RunJob { payload, args, scheduled_for_o, .. } = self; + let RunJob { payload, args, scheduled_for_o, job_id, .. } = self; let mut hm_args = std::collections::HashMap::new(); for (k, v) in args { hm_args.insert(k, windmill_common::worker::to_raw_value(&v)); @@ -250,7 +259,7 @@ impl RunJob { /* parent_job */ None, /* root job */ None, /* flow_innermost_root_job */ None, - /* job_id */ None, + /* job_id */ job_id, /* is_flow_step */ false, /* same_worker */ false, None, @@ -380,7 +389,7 @@ pub fn spawn_test_worker( std::fs::DirBuilder::new() .recursive(true) - .create(windmill_worker::GO_BIN_CACHE_DIR) + .create(&*windmill_worker::GO_BIN_CACHE_DIR) .expect("could not create initial worker dir"); let (tx, rx) = KillpillSender::new(1); @@ -828,6 +837,15 @@ pub async fn run_preview_relative_imports( #[cfg(all(feature = "private", feature = "agent_worker_server"))] pub async fn testing_http_connection(port: u16) -> Connection { + testing_http_connection_with_tags( + port, + vec!["flow".into(), "python3".into(), "dependency".into()], + ) + .await +} + +#[cfg(all(feature = "private", feature = "agent_worker_server"))] +pub async fn testing_http_connection_with_tags(port: u16, tags: Vec) -> Connection { let suffix = windmill_common::utils::create_default_worker_suffix("test-agent-worker"); let agent_token = format!( "{}{}", @@ -835,7 +853,7 @@ pub async fn testing_http_connection(port: u16) -> Connection { windmill_common::jwt::encode_with_internal_secret(windmill_api_agent_workers::AgentAuth { worker_group: "testing-agent".to_owned(), suffix: Some(suffix.clone()), - tags: vec!["flow".into(), "python3".into(), "dependency".into()], + tags, exp: Some(usize::MAX), }) .await diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index cf86d83801..be20cfae3e 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -13,6 +13,7 @@ pub enum AssetKind { Variable, // Deprecated Ducklake, DataTable, + Volume, } #[derive( diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index bcbe0ede33..28d3a1e449 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -105,15 +105,15 @@ impl ScriptLang { pub fn is_native(&self) -> bool { matches!( self, - ScriptLang::Bunnative | - ScriptLang::Nativets | - ScriptLang::Postgresql | - ScriptLang::Mysql | - ScriptLang::Graphql | - ScriptLang::Snowflake | - ScriptLang::Mssql | - ScriptLang::Bigquery | - ScriptLang::OracleDB + ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Postgresql + | ScriptLang::Mysql + | ScriptLang::Graphql + | ScriptLang::Snowflake + | ScriptLang::Mssql + | ScriptLang::Bigquery + | ScriptLang::OracleDB ) } @@ -459,6 +459,7 @@ pub struct NewScript { pub path: String, pub parent_hash: Option, pub summary: String, + #[serde(default)] pub description: String, pub content: String, pub schema: Option, diff --git a/backend/windmill-worker-volumes/Cargo.toml b/backend/windmill-worker-volumes/Cargo.toml new file mode 100644 index 0000000000..08b9a554b2 --- /dev/null +++ b/backend/windmill-worker-volumes/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "windmill-worker-volumes" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_worker_volumes" +path = "src/lib.rs" + +[features] +enterprise = [] +private = [] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +object_store.workspace = true +tokio.workspace = true +tracing.workspace = true +bytes.workspace = true +futures.workspace = true +serde.workspace = true +serde_json.workspace = true +regex.workspace = true +lazy_static.workspace = true +md-5.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/backend/windmill-worker-volumes/src/lib.rs b/backend/windmill-worker-volumes/src/lib.rs new file mode 100644 index 0000000000..a9003a3712 --- /dev/null +++ b/backend/windmill-worker-volumes/src/lib.rs @@ -0,0 +1,544 @@ +#[cfg(feature = "private")] +mod volume_ee; +mod volume_oss; +pub use volume_oss::*; + +pub use object_store::ObjectStore as DynObjectStore; + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::PathBuf; + +pub const MAX_VOLUMES_PER_JOB: usize = 10; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FileEntry { + pub size: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub md5: Option, +} + +pub fn compute_md5_hex(data: &[u8]) -> String { + use md5::{Digest, Md5}; + let result = Md5::digest(data); + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut hex = String::with_capacity(32); + for &b in result.iter() { + hex.push(HEX[(b >> 4) as usize] as char); + hex.push(HEX[(b & 0x0f) as usize] as char); + } + hex +} + +/// Extract an MD5 hash from an S3 ETag, if it's a simple (non-multipart) ETag. +pub fn etag_to_md5(e_tag: Option<&str>) -> Option { + let tag = e_tag?.trim_matches('"'); + // Multipart ETags contain a '-' (e.g. "abc123-5"), skip those + if tag.contains('-') || tag.is_empty() { + return None; + } + Some(tag.to_string()) +} + +lazy_static::lazy_static! { + static ref ARGS_INTERPOLATION_RE: regex::Regex = + regex::Regex::new(r#"\$args\[((?:\w+\.)*\w+)\]"#).unwrap(); + static ref VALID_VOLUME_NAME_RE: regex::Regex = + regex::Regex::new(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,253}[a-zA-Z0-9]$").unwrap(); +} + +#[derive(Debug, Clone, PartialEq)] +pub struct VolumeMount { + pub name: String, + pub target: String, +} + +pub struct VolumeState { + pub mount: VolumeMount, + pub local_dir: PathBuf, + pub manifest: HashMap, + pub symlinks: HashMap, +} + +pub struct DownloadStats { + pub total_files: usize, + pub from_cache: usize, + pub downloaded: usize, +} + +pub struct SyncStats { + pub new_size_bytes: u64, + pub file_count: usize, + pub uploaded: usize, + pub skipped: usize, +} + +pub fn validate_volume_name(name: &str) -> Result<(), String> { + if name.contains("..") { + return Err(format!( + "Volume name '{}' contains '..' which is not allowed", + name + )); + } + if !VALID_VOLUME_NAME_RE.is_match(name) { + return Err(format!( + "Volume name '{}' is invalid. Names must be 2-255 characters, \ + start and end with alphanumeric, and contain only alphanumeric, '.', '_', or '-'", + name + )); + } + Ok(()) +} + +const ALLOWED_ABSOLUTE_PREFIXES: &[&str] = &["/tmp/", "/mnt/", "/opt/", "/home/", "/data/"]; + +pub fn validate_volume_target(target: &str) -> Result<(), String> { + if target.split('/').any(|seg| seg == "..") { + return Err(format!( + "Volume target '{target}' contains '..' segments which is not allowed" + )); + } + if target.starts_with('/') + && !ALLOWED_ABSOLUTE_PREFIXES + .iter() + .any(|p| target.starts_with(p)) + { + return Err(format!( + "Volume target '{target}' must be a relative path or start with one of: {}", + ALLOWED_ABSOLUTE_PREFIXES.join(", ") + )); + } + Ok(()) +} + +pub fn validate_volume_mounts(mounts: &[VolumeMount]) -> Result<(), String> { + if mounts.len() > MAX_VOLUMES_PER_JOB { + return Err(format!( + "Too many volume mounts ({}, max {})", + mounts.len(), + MAX_VOLUMES_PER_JOB + )); + } + let mut seen_names = HashSet::new(); + let mut seen_targets = HashSet::new(); + for v in mounts { + if !seen_names.insert(&v.name) { + return Err(format!("Duplicate volume name: '{}'", v.name)); + } + if !seen_targets.insert(&v.target) { + return Err(format!("Duplicate volume target: '{}'", v.target)); + } + } + Ok(()) +} + +pub fn interpolate_volume_name( + name: &str, + args: Option<&HashMap>>, + workspace_id: &str, +) -> String { + let name = name.replace("$workspace", workspace_id); + if !name.contains("$args[") { + return name; + } + let Some(args) = args else { + return name; + }; + let mut result = name.clone(); + for cap in ARGS_INTERPOLATION_RE.captures_iter(&name) { + let full_match = cap.get(0).unwrap().as_str(); + let arg_name = cap.get(1).unwrap().as_str(); + let arg_value = if arg_name.contains('.') { + let parts: Vec<&str> = arg_name.split('.').collect(); + let root = parts[0]; + let mut value = args + .get(root) + .map(|x| x.get().to_string()) + .unwrap_or_default(); + for part in parts.iter().skip(1) { + if let Ok(obj) = serde_json::from_str::(&value) { + value = obj + .get(part) + .map(|v| v.to_string()) + .unwrap_or_default() + .to_string(); + } else { + value = String::new(); + break; + } + } + value.trim_matches('"').to_string() + } else { + args.get(arg_name) + .map(|x| x.get().trim_matches('"').to_string()) + .unwrap_or_default() + }; + result = result.replace(full_match, &arg_value); + } + result +} + +pub fn parse_volume_annotations(content: &str, comment_prefix: &str) -> Vec { + let mut volumes = Vec::new(); + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + if !trimmed.starts_with(comment_prefix) { + break; + } + let after_prefix = trimmed[comment_prefix.len()..].trim(); + if let Some(rest) = after_prefix.strip_prefix("volume:") { + let rest = rest.trim(); + let mut parts = rest.splitn(2, char::is_whitespace); + if let (Some(name), Some(target)) = (parts.next(), parts.next()) { + let name = name.trim(); + let target = target.trim(); + if !name.is_empty() && !target.is_empty() { + volumes + .push(VolumeMount { name: name.to_string(), target: target.to_string() }); + } + } + } + } + volumes +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_python_single_volume() { + let content = "# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_typescript_single_volume() { + let content = "// volume: mydata /tmp/data\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_multiple_volumes() { + let content = "# volume: data1 /tmp/data1\n# volume: data2 /tmp/data2\n# volume: models /opt/models\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![ + VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }, + VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() }, + VolumeMount { name: "models".to_string(), target: "/opt/models".to_string() }, + ] + ); + } + + #[test] + fn parse_mixed_annotations_and_volumes() { + let content = "# sandbox\n# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_no_volumes() { + let content = "# sandbox\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert!(result.is_empty()); + } + + #[test] + fn parse_empty_content() { + let result = parse_volume_annotations("", "#"); + assert!(result.is_empty()); + } + + #[test] + fn parse_stops_at_non_comment_line() { + let content = + "# volume: data1 /tmp/data1\ndef main():\n # volume: data2 /tmp/data2\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }] + ); + } + + #[test] + fn parse_skips_blank_lines_in_header() { + let content = + "# volume: data1 /tmp/data1\n\n# volume: data2 /tmp/data2\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![ + VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }, + VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() }, + ] + ); + } + + #[test] + fn parse_ignores_malformed_volume_lines() { + let content = + "# volume:\n# volume: onlyname\n# volume: good /tmp/good\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "good".to_string(), target: "/tmp/good".to_string() }] + ); + } + + #[test] + fn parse_extra_whitespace() { + let content = "# volume: mydata /tmp/data \ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_target_with_spaces_in_path() { + let content = "// volume: mydata /tmp/my data dir\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { + name: "mydata".to_string(), + target: "/tmp/my data dir".to_string(), + }] + ); + } + + #[test] + fn parse_volume_with_dashes_and_underscores() { + let content = "# volume: my-data_v2 /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "my-data_v2".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn interpolate_workspace() { + let name = "$workspace-data"; + let result = interpolate_volume_name(name, None, "my_ws"); + assert_eq!(result, "my_ws-data"); + } + + #[test] + fn interpolate_args_simple() { + let mut args = HashMap::new(); + args.insert( + "env".to_string(), + serde_json::value::RawValue::from_string("\"prod\"".to_string()).unwrap(), + ); + let result = interpolate_volume_name("data-$args[env]", Some(&args), "ws"); + assert_eq!(result, "data-prod"); + } + + #[test] + fn interpolate_args_and_workspace() { + let mut args = HashMap::new(); + args.insert( + "env".to_string(), + serde_json::value::RawValue::from_string("\"staging\"".to_string()).unwrap(), + ); + let result = interpolate_volume_name("$workspace-$args[env]-cache", Some(&args), "acme"); + assert_eq!(result, "acme-staging-cache"); + } + + #[test] + fn interpolate_no_placeholders() { + let result = interpolate_volume_name("plain-name", None, "ws"); + assert_eq!(result, "plain-name"); + } + + #[test] + fn interpolate_missing_arg() { + let args = HashMap::new(); + let result = interpolate_volume_name("data-$args[missing]", Some(&args), "ws"); + assert_eq!(result, "data-"); + } + + #[test] + fn interpolate_nested_arg() { + let mut args = HashMap::new(); + args.insert( + "config".to_string(), + serde_json::value::RawValue::from_string( + r#"{"env": "prod", "region": "us-east"}"#.to_string(), + ) + .unwrap(), + ); + let result = interpolate_volume_name( + "data-$args[config.env]-$args[config.region]", + Some(&args), + "ws", + ); + assert_eq!(result, "data-prod-us-east"); + } + + #[test] + fn parse_wrong_prefix_returns_empty() { + let content = "# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "//"); + assert!(result.is_empty()); + } + + #[test] + fn parse_relative_path() { + let content = "// volume: agent-memory .claude\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { name: "agent-memory".to_string(), target: ".claude".to_string() }] + ); + } + + #[test] + fn parse_relative_nested_path() { + let content = "# volume: data data/models\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "data".to_string(), target: "data/models".to_string() }] + ); + } + + #[test] + fn validate_valid_names() { + assert!(validate_volume_name("mydata").is_ok()); + assert!(validate_volume_name("my-data_v2").is_ok()); + assert!(validate_volume_name("acme-staging-cache").is_ok()); + assert!(validate_volume_name("a1").is_ok()); + assert!(validate_volume_name("data.v2").is_ok()); + assert!(validate_volume_name("A0").is_ok()); + } + + #[test] + fn validate_rejects_path_traversal() { + assert!(validate_volume_name("../other-workspace").is_err()); + assert!(validate_volume_name("data/../secrets").is_err()); + assert!(validate_volume_name("a..b").is_err()); + } + + #[test] + fn validate_rejects_special_start_end() { + assert!(validate_volume_name("-data").is_err()); + assert!(validate_volume_name("data-").is_err()); + assert!(validate_volume_name(".data").is_err()); + assert!(validate_volume_name("data.").is_err()); + assert!(validate_volume_name("_data").is_err()); + } + + #[test] + fn validate_rejects_path_separators() { + assert!(validate_volume_name("data/secrets").is_err()); + assert!(validate_volume_name("data\\secrets").is_err()); + } + + #[test] + fn validate_rejects_too_short() { + assert!(validate_volume_name("").is_err()); + assert!(validate_volume_name("a").is_err()); + } + + #[test] + fn validate_rejects_too_long() { + let long_name = format!("a{}a", "b".repeat(254)); + assert!(validate_volume_name(&long_name).is_err()); + } + + #[test] + fn validate_rejects_spaces_and_special() { + assert!(validate_volume_name("my data").is_err()); + assert!(validate_volume_name("my@data").is_err()); + assert!(validate_volume_name("my$data").is_err()); + } + + #[test] + fn validate_target_allows_relative() { + assert!(validate_volume_target("data").is_ok()); + assert!(validate_volume_target("data/models").is_ok()); + assert!(validate_volume_target(".claude").is_ok()); + } + + #[test] + fn validate_target_allows_safe_absolute() { + assert!(validate_volume_target("/tmp/data").is_ok()); + assert!(validate_volume_target("/mnt/data").is_ok()); + assert!(validate_volume_target("/opt/models").is_ok()); + assert!(validate_volume_target("/home/user/data").is_ok()); + assert!(validate_volume_target("/data/cache").is_ok()); + } + + #[test] + fn validate_target_rejects_dangerous_absolute() { + assert!(validate_volume_target("/etc/passwd").is_err()); + assert!(validate_volume_target("/proc/self").is_err()); + assert!(validate_volume_target("/sys/fs").is_err()); + assert!(validate_volume_target("/dev/null").is_err()); + assert!(validate_volume_target("/usr/bin").is_err()); + assert!(validate_volume_target("/var/log").is_err()); + } + + #[test] + fn validate_target_rejects_traversal() { + assert!(validate_volume_target("../../etc").is_err()); + assert!(validate_volume_target("data/../../../etc").is_err()); + assert!(validate_volume_target("/tmp/../etc/passwd").is_err()); + } + + #[test] + fn validate_mounts_rejects_too_many() { + let mounts: Vec = (0..11) + .map(|i| VolumeMount { name: format!("v{:02}", i), target: format!("t{}", i) }) + .collect(); + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_rejects_duplicate_name() { + let mounts = vec![ + VolumeMount { name: "data".to_string(), target: "/tmp/a".to_string() }, + VolumeMount { name: "data".to_string(), target: "/tmp/b".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_rejects_duplicate_target() { + let mounts = vec![ + VolumeMount { name: "v1".to_string(), target: "/tmp/data".to_string() }, + VolumeMount { name: "v2".to_string(), target: "/tmp/data".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_ok() { + let mounts = vec![ + VolumeMount { name: "v1".to_string(), target: "/tmp/a".to_string() }, + VolumeMount { name: "v2".to_string(), target: "/tmp/b".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_ok()); + } +} diff --git a/backend/windmill-worker-volumes/src/volume_oss.rs b/backend/windmill-worker-volumes/src/volume_oss.rs new file mode 100644 index 0000000000..07cd6906d2 --- /dev/null +++ b/backend/windmill-worker-volumes/src/volume_oss.rs @@ -0,0 +1,116 @@ +#[cfg(feature = "private")] +pub use crate::volume_ee::*; + +#[cfg(not(feature = "private"))] +use crate::{DownloadStats, SyncStats, VolumeMount, VolumeState}; +#[cfg(not(feature = "private"))] +use object_store::ObjectStore; +#[cfg(not(feature = "private"))] +use std::path::Path; +#[cfg(not(feature = "private"))] +use std::sync::Arc; +#[cfg(not(feature = "private"))] +use windmill_common::error; + +#[cfg(not(feature = "private"))] +pub async fn download_volume( + _client: Arc, + _volume: &VolumeMount, + _job_dir: &str, + _workspace_id: &str, +) -> error::Result<(VolumeState, DownloadStats)> { + Err(error::Error::internal_err( + "Volumes are not available in this build".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +pub fn volume_nsjail_mount(_local_dir: &Path, _target: &str) -> String { + String::new() +} + +#[cfg(not(feature = "private"))] +pub async fn sync_volume_back( + _client: Arc, + _state: &VolumeState, + _workspace_id: &str, +) -> error::Result { + Err(error::Error::internal_err( + "Volumes are not available in this build".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +pub fn walk_dir(dir: &Path) -> std::io::Result> { + let mut result = Vec::new(); + walk_dir_inner(dir, &mut result)?; + Ok(result) +} + +#[cfg(not(feature = "private"))] +fn walk_dir_inner(dir: &Path, result: &mut Vec) -> std::io::Result<()> { + if !dir.is_dir() { + return Ok(()); + } + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + let meta = match std::fs::symlink_metadata(&path) { + Ok(m) => m, + Err(_) => continue, + }; + if meta.is_dir() { + walk_dir_inner(&path, result)?; + } else if meta.is_file() { + result.push(path); + } + } + Ok(()) +} + +#[cfg(not(feature = "private"))] +pub fn collect_symlinks(dir: &Path) -> std::collections::HashMap { + let mut symlinks = std::collections::HashMap::new(); + collect_symlinks_inner(dir, dir, &mut symlinks); + symlinks +} + +#[cfg(not(feature = "private"))] +fn collect_symlinks_inner( + base: &Path, + dir: &Path, + symlinks: &mut std::collections::HashMap, +) { + let entries = match std::fs::read_dir(dir) { + Ok(e) => e, + Err(_) => return, + }; + for entry in entries { + let entry = match entry { + Ok(e) => e, + Err(_) => continue, + }; + let path = entry.path(); + let meta = match std::fs::symlink_metadata(&path) { + Ok(m) => m, + Err(_) => continue, + }; + if meta.file_type().is_symlink() { + if let Ok(target) = std::fs::read_link(&path) { + let relative = path + .strip_prefix(base) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + symlinks.insert(relative, target.to_string_lossy().to_string()); + } + } else if meta.is_dir() { + collect_symlinks_inner(base, &path, symlinks); + } + } +} + +#[cfg(not(feature = "private"))] +pub fn restore_symlinks(_dir: &Path, _symlinks: &std::collections::HashMap) { + // No-op in OSS build +} diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 5cf0a4c26e..23927753ee 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -10,10 +10,10 @@ path = "src/lib.rs" [features] default = [] -private = [] +private = ["windmill-worker-volumes/private", "windmill-queue/private"] mcp = ["dep:windmill-mcp"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] -enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] +enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] mssql = ["dep:tiberius"] mssql-kerberos = ["mssql", "tiberius/integrated-auth-gssapi"] # Linux/Unix integrated auth mssql-winauth = ["mssql", "tiberius/winauth"] # Windows integrated auth @@ -47,6 +47,7 @@ windmill-audit.workspace = true # there isn't really a reason for audit-worth ac windmill-common = { workspace = true, default-features = false } windmill-types.workspace = true windmill-object-store.workspace = true +windmill-worker-volumes.workspace = true windmill-jseval.workspace = true windmill-runtime-nativets = { workspace = true, optional = true } windmill-mcp = { workspace = true, optional = true } diff --git a/backend/windmill-worker/loader.bun.windows.js b/backend/windmill-worker/loader.bun.windows.js new file mode 100644 index 0000000000..227c68a56c --- /dev/null +++ b/backend/windmill-worker/loader.bun.windows.js @@ -0,0 +1,123 @@ +// Windows-specific bun loader that uses a virtual "windmill-url" namespace instead +// of writing .url files to disk. This avoids Windows path issues (backslashes in +// resolve(), 8.3 short filenames, drive letter prefixes). The virtual namespace +// approach is likely better on all fronts but we keep the original .url-file loader +// on Linux to avoid breaking back-compat. +const p = { + name: "windmill-relative-resolver", + async setup(build) { + const { readFileSync } = await import("fs"); + const { resolve } = await import("node:path"); + + const base_internal_url = "BASE_INTERNAL_URL".replace( + "localhost", + "127.0.0.1" + ); + + const w_id = "W_ID"; + const current_path = "CURRENT_PATH"; + const token = "TOKEN"; + + const cdir = resolve("./"); + const cdirNoPrivate = cdir.replace(/^\/private/, ""); // for macos + // Normalize path to forward slashes to match Bun's resolver output on Windows + const cdirFwd = cdir.replace(/\\/g, "/"); + const cdirPosix = cdirFwd.replace(/^[a-zA-Z]:/, ""); + const filterResolve = new RegExp( + `^(?!\\.\/main\\.ts)(?!${cdirFwd}\/main\\.ts)(?!${cdirPosix}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.mjs).*\\.ts$` + ); + + let cdirNodeModules = `${cdirFwd}/node_modules/`; + + const filterLoad = new RegExp(`^${cdir}\/main\\.ts$`); + const transpiler = new Bun.Transpiler({ + loader: "ts", + }); + + function replaceRelativeImports(code) { + const imports = transpiler.scanImports(code); + for (const imp of imports) { + if (imp.kind == "import-statement") { + if ( + (imp.path.startsWith(".") || + imp.path.startsWith("/u/") || + imp.path.startsWith("/f/")) && + !imp.path.endsWith(".ts") + ) { + code = code.replaceAll(imp.path, imp.path + ".ts"); + } + } + } + return { + contents: code, + }; + } + + function normalizePath(rawPath) { + return rawPath.split("/").reduce((acc, seg) => { + if (seg === "..") acc.pop(); + else if (seg !== "." && seg !== "") acc.push(seg); + return acc; + }, []).join("/"); + } + + // Resolve a windmill script import path relative to an importer path. + // Bun on Windows may prefix args with "windmill-url:" or strip leading "/". + function resolveWindmillImport(importerPath, importPath) { + const path = importPath.replace(/^windmill-url:/, "").replace(/^\//, ""); + const isAbsolute = path.startsWith("f/") || path.startsWith("u/"); + const endExt = path.endsWith(".ts") ? "" : ".ts"; + const rawScriptPath = isAbsolute + ? `${path}${endExt}` + : `${importerPath}/../${path}${endExt}`; + return { path: normalizePath(rawScriptPath), namespace: "windmill-url" }; + } + + build.onLoad({ filter: filterLoad }, async (args) => { + const code = readFileSync(args.path, "utf8"); + return replaceRelativeImports(code); + }); + + // Load windmill scripts by fetching from the API + build.onLoad({ filter: /.*/, namespace: "windmill-url" }, async (args) => { + const path = args.path.replace(/^windmill-url:/, ""); + const url = `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${path}`; + const req = await fetch(url, { + method: "GET", + headers: { + Authorization: "Bearer " + token, + }, + }); + if (!req.ok) { + throw new Error( + `Failed to find relative import at ${url} (status ${req.status})` + ); + } + const contents = await req.text(); + return { + contents: replaceRelativeImports(contents).contents, + loader: "tsx", + }; + }); + + // Resolve windmill script imports from the file namespace (e.g. from main.ts) + build.onResolve({ filter: filterResolve }, (args) => { + const importerFwd = args.importer?.replace(/\\/g, "/") ?? ""; + if (importerFwd.startsWith(cdirNodeModules)) { + return undefined; + } + const isMainTs = + args.importer == "./main.ts" || importerFwd.endsWith("/main.ts"); + const file_path = isMainTs + ? current_path + : importerFwd.replace(cdirFwd + "/", ""); + return resolveWindmillImport(file_path, args.path); + }); + + // Resolve nested imports from within windmill-url modules + build.onResolve({ filter: /\.ts$/, namespace: "windmill-url" }, (args) => { + const importer = args.importer.replace(/^windmill-url:/, ""); + return resolveWindmillImport(importer, args.path); + }); + }, +}; diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index 8cbee3dec9..afd5c42ba9 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -14,6 +14,18 @@ clone_newnet: false clone_newuser: {CLONE_NEWUSER} clone_newcgroup: false +uidmap { + inside_id: "1000" + outside_id: "" + count: 1 +} + +gidmap { + inside_id: "1000" + outside_id: "" + count: 1 +} + skip_setsid: true keep_caps: false keep_env: true @@ -130,6 +142,13 @@ mount { rw: true } +mount { + src: "{JOB_DIR}/checkpoint.json" + dst: "/tmp/{LANG}/checkpoint.json" + is_bind: true + mandatory: false +} + mount { src: "{JOB_DIR}/result.json" dst: "/tmp/{LANG}/result.json" diff --git a/backend/windmill-worker/nsjail/run.powershell.config.proto b/backend/windmill-worker/nsjail/run.powershell.config.proto index 87b6abda21..afe9d5df9f 100644 --- a/backend/windmill-worker/nsjail/run.powershell.config.proto +++ b/backend/windmill-worker/nsjail/run.powershell.config.proto @@ -127,7 +127,7 @@ iface_no_lo: true mount { src: "{CACHE_DIR}" - dst: "/tmp/windmill/cache/powershell" + dst: "{CACHE_DIR}" is_bind: true rw: false mandatory: false diff --git a/backend/windmill-worker/src/ai/image_handler.rs b/backend/windmill-worker/src/ai/image_handler.rs index 7eb6bb6597..63d8aeaec3 100644 --- a/backend/windmill-worker/src/ai/image_handler.rs +++ b/backend/windmill-worker/src/ai/image_handler.rs @@ -2,8 +2,8 @@ use base64::Engine; use futures; use ulid; use windmill_common::{client::AuthedClient, error::Error}; -use windmill_types::s3::S3Object; use windmill_queue::MiniPulledJob; +use windmill_types::s3::S3Object; use crate::ai::types::*; diff --git a/backend/windmill-worker/src/ai/providers/anthropic.rs b/backend/windmill-worker/src/ai/providers/anthropic.rs index e42e1dc49b..847c706d71 100644 --- a/backend/windmill-worker/src/ai/providers/anthropic.rs +++ b/backend/windmill-worker/src/ai/providers/anthropic.rs @@ -1,14 +1,16 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; +use windmill_common::{ + ai_google::parse_data_url, ai_providers::AIProvider, client::AuthedClient, error::Error, +}; use crate::ai::{ image_handler::prepare_messages_for_api, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, sse::{AnthropicSSEParser, SSEParser}, types::*, - utils::{extract_text_content, parse_data_url, should_use_structured_output_tool}, + utils::{extract_text_content, should_use_structured_output_tool}, }; /// Anthropic API version for standard API @@ -353,21 +355,21 @@ pub struct AnthropicResponse { pub struct AnthropicQueryBuilder { #[allow(dead_code)] provider_kind: AIProvider, - platform: AnthropicPlatform, + platform: AIPlatform, enable_1m_context: bool, } impl AnthropicQueryBuilder { pub fn new( provider_kind: AIProvider, - platform: AnthropicPlatform, + platform: AIPlatform, enable_1m_context: bool, ) -> Self { Self { provider_kind, platform, enable_1m_context } } fn is_vertex(&self) -> bool { - self.platform == AnthropicPlatform::GoogleVertexAi + self.platform == AIPlatform::GoogleVertexAi } async fn build_text_request( diff --git a/backend/windmill-worker/src/ai/providers/google_ai.rs b/backend/windmill-worker/src/ai/providers/google_ai.rs index 62e6afff75..8ce0b5b951 100644 --- a/backend/windmill-worker/src/ai/providers/google_ai.rs +++ b/backend/windmill-worker/src/ai/providers/google_ai.rs @@ -1,254 +1,56 @@ use async_trait::async_trait; -use serde::{Deserialize, Serialize}; -use windmill_common::{client::AuthedClient, error::Error}; +use windmill_common::{ + ai_google::{ + openai_messages_to_gemini, openai_tools_to_gemini, GeminiGenerationConfig, + GeminiImageContent, GeminiImageRequest, GeminiImageResponse, GeminiInlineData, GeminiPart, + GeminiPredictContent, GeminiTextRequest, GeminiTool, + }, + client::AuthedClient, + error::Error, +}; use crate::ai::{ - image_handler::download_and_encode_s3_image, + image_handler::{download_and_encode_s3_image, prepare_messages_for_api}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, sse::{GeminiSSEParser, SSEParser}, types::*, - utils::parse_data_url, }; -// ============================================================================ -// Gemini API Types - Shared between text and image -// ============================================================================ - -/// Inline data for binary content (images) -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GeminiInlineData { - #[serde(rename = "mimeType")] - pub mime_type: String, - pub data: String, -} - -/// A part of content - can be text, inline data, function call, or function response -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(untagged)] -pub enum GeminiPart { - Text { - text: String, - }, - InlineData { - #[serde(rename = "inlineData")] - inline_data: GeminiInlineData, - }, - FunctionCall { - #[serde(rename = "functionCall")] - function_call: GeminiFunctionCall, - /// Thought signature for Gemini 3+ models - required for function calling - #[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")] - thought_signature: Option, - }, - FunctionResponse { - #[serde(rename = "functionResponse")] - function_response: GeminiFunctionResponse, - }, -} - -/// A function call from the model -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GeminiFunctionCall { - pub name: String, - pub args: serde_json::Value, -} - -/// A function response to send back to the model -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GeminiFunctionResponse { - pub name: String, - pub response: serde_json::Value, -} - -// ============================================================================ -// Gemini Text API Request Types -// ============================================================================ - -/// Main request structure for Gemini generateContent -#[derive(Serialize)] -pub struct GeminiTextRequest { - pub contents: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub tools: Option>, - #[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")] - pub tool_config: Option, - #[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")] - pub system_instruction: Option, - #[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")] - pub generation_config: Option, -} - -/// Content message with role and parts -#[derive(Serialize)] -pub struct GeminiContentMessage { - #[serde(skip_serializing_if = "Option::is_none")] - pub role: Option, - pub parts: Vec, -} - -/// Tool definition - either function declarations or Google Search -#[derive(Serialize)] -pub struct GeminiTool { - #[serde( - rename = "functionDeclarations", - skip_serializing_if = "Option::is_none" - )] - pub function_declarations: Option>, - #[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")] - pub google_search: Option, -} - -/// Function declaration for tool use -#[derive(Serialize)] -pub struct GeminiFunctionDeclaration { - pub name: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - pub parameters: OpenAPISchema, -} - -/// Tool configuration for controlling function calling behavior -#[derive(Serialize)] -pub struct GeminiToolConfig { - #[serde(rename = "functionCallingConfig")] - pub function_calling_config: GeminiFunctionCallingConfig, -} - -/// Function calling configuration -#[derive(Serialize)] -pub struct GeminiFunctionCallingConfig { - pub mode: String, - #[serde( - rename = "allowedFunctionNames", - skip_serializing_if = "Option::is_none" - )] - pub allowed_function_names: Option>, -} - -/// Generation configuration for output format -#[derive(Serialize)] -pub struct GeminiGenerationConfig { - #[serde(skip_serializing_if = "Option::is_none")] - pub temperature: Option, - #[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")] - pub max_output_tokens: Option, - #[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")] - pub response_mime_type: Option, - #[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")] - pub response_schema: Option, -} - -// ============================================================================ -// Gemini API Response Types -// ============================================================================ - -/// Grounding metadata from Google Search -#[derive(Deserialize)] -#[allow(dead_code)] -pub struct GeminiGroundingMetadata { - #[serde(rename = "webSearchQueries")] - pub web_search_queries: Option>, - #[serde(rename = "groundingChunks")] - pub grounding_chunks: Option>, -} - -// ============================================================================ -// Gemini Image API Types (for Imagen models) -// ============================================================================ - -/// Request for image generation (Imagen models) -#[derive(Serialize)] -pub struct GeminiImageRequest { - #[serde(skip_serializing_if = "Option::is_none")] - pub contents: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub instances: Option>, -} - -/// Content for image generation -#[derive(Serialize)] -pub struct GeminiImageContent { - pub parts: Vec, -} - -/// Content for Imagen predict endpoint -#[derive(Serialize)] -pub struct GeminiPredictContent { - pub prompt: String, -} - -/// Response for image generation -#[derive(Deserialize)] -pub struct GeminiImageResponse { - pub candidates: Option>, - pub predictions: Option>, -} - -/// Image candidate from generateContent -#[derive(Deserialize)] -pub struct GeminiImageCandidate { - pub content: GeminiImageCandidateContent, -} - -/// Content in image candidate -#[derive(Deserialize)] -pub struct GeminiImageCandidateContent { - pub parts: Vec, -} - -/// Part of image candidate -#[derive(Deserialize)] -pub struct GeminiImageCandidatePart { - #[serde(rename = "inlineData", skip_serializing_if = "Option::is_none")] - pub inline_data: Option, -} - -/// Prediction candidate from Imagen -#[derive(Deserialize)] -pub struct GeminiPredictCandidate { - #[serde(rename = "bytesBase64Encoded")] - pub bytes_base64_encoded: String, -} - // ============================================================================ // Query Builder Implementation // ============================================================================ -pub struct GoogleAIQueryBuilder; +pub struct GoogleAIQueryBuilder { + platform: AIPlatform, +} impl GoogleAIQueryBuilder { - pub fn new() -> Self { - Self + pub fn new(platform: AIPlatform) -> Self { + Self { platform } + } + + fn is_vertex(&self) -> bool { + self.platform == AIPlatform::GoogleVertexAi } - /// Build a text request using the native Gemini API format async fn build_text_request( &self, args: &BuildRequestArgs<'_>, client: &AuthedClient, workspace_id: &str, ) -> Result { - // Convert messages to Gemini format - let contents = self - .convert_messages_to_gemini(args.messages, client, workspace_id) - .await?; + let prepared_messages = + prepare_messages_for_api(args.messages, client, workspace_id).await?; + let (contents, system_instruction) = openai_messages_to_gemini(&prepared_messages); - // Build tools array let tools = self.convert_tools_to_gemini(args.tools, args.has_websearch); - // Build generation config let generation_config = self.build_generation_config(args); - // Build system instruction from system_prompt - let system_instruction = args.system_prompt.map(|s| GeminiContentMessage { - role: None, - parts: vec![GeminiPart::Text { text: s.to_string() }], - }); - let request = GeminiTextRequest { contents, tools, - tool_config: None, // Use AUTO mode by default + tool_config: None, system_instruction, generation_config, }; @@ -257,7 +59,6 @@ impl GoogleAIQueryBuilder { .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) } - /// Build an image generation request async fn build_image_request( &self, args: &BuildRequestArgs<'_>, @@ -267,7 +68,6 @@ impl GoogleAIQueryBuilder { let is_imagen = args.model.contains("imagen"); let request = if is_imagen { - // For Imagen models, use simple prompt format GeminiImageRequest { instances: Some(vec![GeminiPredictContent { prompt: args.user_message.trim().to_string(), @@ -275,7 +75,6 @@ impl GoogleAIQueryBuilder { contents: None, } } else { - // For Gemini models with image generation, build parts let mut parts = vec![GeminiPart::Text { text: args.user_message.trim().to_string() }]; if let Some(system_prompt) = args.system_prompt { @@ -285,7 +84,6 @@ impl GoogleAIQueryBuilder { ); } - // Add input images if provided if let Some(images) = args.images { for image in images.iter() { if !image.s3.is_empty() { @@ -308,218 +106,39 @@ impl GoogleAIQueryBuilder { .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) } - /// Convert OpenAI-format messages to Gemini format - async fn convert_messages_to_gemini( - &self, - messages: &[OpenAIMessage], - client: &AuthedClient, - workspace_id: &str, - ) -> Result, Error> { - let mut gemini_messages = Vec::new(); - - for msg in messages { - match msg.role.as_str() { - "system" => { - // Skip - handled via args.system_prompt in build_text_request - } - "tool" => { - // Handle tool responses - if let (Some(tool_call_id), Some(content)) = (&msg.tool_call_id, &msg.content) { - let func_name = self.find_function_name_by_id(messages, tool_call_id); - let response_text = match content { - OpenAIContent::Text(text) => text.clone(), - OpenAIContent::Parts(parts) => parts - .iter() - .filter_map(|p| match p { - ContentPart::Text { text } => Some(text.clone()), - _ => None, - }) - .collect::>() - .join(" "), - }; - - gemini_messages.push(GeminiContentMessage { - role: Some("user".to_string()), - parts: vec![GeminiPart::FunctionResponse { - function_response: GeminiFunctionResponse { - name: func_name, - response: serde_json::json!({ "result": response_text }), - }, - }], - }); - } - } - _ => { - // Handle user/assistant messages - let role = match msg.role.as_str() { - "assistant" => "model", - _ => "user", - }; - - let mut parts = Vec::new(); - - // Handle regular content - if let Some(content) = &msg.content { - let content_parts = self - .convert_content_to_parts(&Some(content.clone()), client, workspace_id) - .await?; - parts.extend(content_parts); - } - - // Handle tool calls from assistant - if let Some(tool_calls) = &msg.tool_calls { - for tc in tool_calls { - let args: serde_json::Value = - serde_json::from_str(&tc.function.arguments).unwrap_or_default(); - // Extract thought_signature from extra_content if present - let thought_signature = tc - .extra_content - .as_ref() - .and_then(|ec| ec.google.as_ref()) - .and_then(|g| g.thought_signature.clone()); - parts.push(GeminiPart::FunctionCall { - function_call: GeminiFunctionCall { - name: tc.function.name.clone(), - args, - }, - thought_signature, - }); - } - } - - if !parts.is_empty() { - gemini_messages - .push(GeminiContentMessage { role: Some(role.to_string()), parts }); - } - } - } - } - - Ok(gemini_messages) - } - - /// Convert OpenAI content to Gemini parts - async fn convert_content_to_parts( - &self, - content: &Option, - client: &AuthedClient, - workspace_id: &str, - ) -> Result, Error> { - let mut parts = Vec::new(); - - if let Some(content) = content { - match content { - OpenAIContent::Text(text) => { - if !text.is_empty() { - parts.push(GeminiPart::Text { text: text.clone() }); - } - } - OpenAIContent::Parts(content_parts) => { - for part in content_parts { - match part { - ContentPart::Text { text } => { - if !text.is_empty() { - parts.push(GeminiPart::Text { text: text.clone() }); - } - } - ContentPart::ImageUrl { image_url } => { - // Parse data URL format: data:mime_type;base64,data - if let Some((mime_type, data)) = parse_data_url(&image_url.url) { - parts.push(GeminiPart::InlineData { - inline_data: GeminiInlineData { mime_type, data }, - }); - } - } - ContentPart::S3Object { s3_object } => { - if !s3_object.s3.is_empty() { - let (mime_type, data) = download_and_encode_s3_image( - s3_object, - client, - workspace_id, - ) - .await?; - parts.push(GeminiPart::InlineData { - inline_data: GeminiInlineData { mime_type, data }, - }); - } - } - } - } - } - } - } - - Ok(parts) - } - - /// Find function name by tool call ID from previous messages - fn find_function_name_by_id(&self, messages: &[OpenAIMessage], tool_call_id: &str) -> String { - for msg in messages { - if let Some(tool_calls) = &msg.tool_calls { - for tc in tool_calls { - if tc.id == tool_call_id { - return tc.function.name.clone(); - } - } - } - } - "unknown_function".to_string() - } - - /// Convert OpenAI tools to Gemini format + /// Convert OpenAI tool definitions to Gemini format. + /// + /// Sanitizes each tool's JSON schema for Google compatibility before delegating + /// to the shared [`openai_tools_to_gemini`] function. fn convert_tools_to_gemini( &self, tools: Option<&[ToolDef]>, has_websearch: bool, ) -> Option> { - let mut gemini_tools = Vec::new(); - - // Add function declarations - if let Some(tool_defs) = tools { - let declarations: Vec = tool_defs - .iter() - .filter_map(|t| { - // Deserialize RawValue into OpenAPISchema, sanitize, then use - let mut schema: OpenAPISchema = - serde_json::from_str(t.function.parameters.get()).ok()?; - schema.sanitize_for_google(); - - Some(GeminiFunctionDeclaration { - name: t.function.name.clone(), - description: t.function.description.clone(), - parameters: schema, - }) - }) - .collect(); - - if !declarations.is_empty() { - gemini_tools.push(GeminiTool { - function_declarations: Some(declarations), - google_search: None, - }); + let Some(tool_defs) = tools else { + if has_websearch { + return Some(vec![GeminiTool { + function_declarations: None, + google_search: Some(serde_json::json!({})), + }]); } - } + return None; + }; - // Add Google Search tool if enabled - if has_websearch { - gemini_tools.push(GeminiTool { - function_declarations: None, - google_search: Some(serde_json::json!({})), - }); - } + let tool_params: Vec = tool_defs + .iter() + .map(|t| { + let mut schema: OpenAPISchema = + serde_json::from_str(t.function.parameters.get()).unwrap_or_default(); + schema.sanitize_for_google(); + serde_json::to_value(&schema).unwrap_or_default() + }) + .collect(); - if gemini_tools.is_empty() { - None - } else { - Some(gemini_tools) - } + openai_tools_to_gemini(tool_defs, &tool_params, has_websearch) } - /// Build generation config for structured output and other settings - fn build_generation_config( - &self, - args: &BuildRequestArgs<'_>, - ) -> Option { + fn build_generation_config(&self, args: &BuildRequestArgs<'_>) -> Option { let has_output_schema = args .output_schema .and_then(|s| s.properties.as_ref()) @@ -529,15 +148,11 @@ impl GoogleAIQueryBuilder { let (response_mime_type, response_schema) = if has_output_schema { let mut schema = args.output_schema.unwrap().clone(); schema.sanitize_for_google(); - ( - Some("application/json".to_string()), - serde_json::to_value(&schema).ok(), - ) + (Some("application/json".to_string()), serde_json::to_value(&schema).ok()) } else { (None, None) }; - // Only create config if there's something to configure if args.temperature.is_some() || args.max_tokens.is_some() || response_mime_type.is_some() { Some(GeminiGenerationConfig { temperature: args.temperature, @@ -554,7 +169,6 @@ impl GoogleAIQueryBuilder { #[async_trait] impl QueryBuilder for GoogleAIQueryBuilder { fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool { - // Google AI supports tools only for text output matches!(output_type, OutputType::Text) } @@ -578,7 +192,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { Error::internal_err(format!("Failed to parse Gemini image response: {}", e)) })?; - // First, check Gemini models (candidates -> content -> parts -> inline_data) let image_data_from_gemini = gemini_response.candidates.as_ref().and_then(|candidates| { candidates.iter().find_map(|candidate| { candidate @@ -589,13 +202,11 @@ impl QueryBuilder for GoogleAIQueryBuilder { }) }); - // Then, check Imagen models (predictions -> bytes_base64_encoded) let image_data_from_imagen = gemini_response .predictions .as_ref() .and_then(|predictions| predictions.first().map(|p| &p.bytes_base64_encoded)); - // Image data, preferring Gemini first then Imagen models let image_data = image_data_from_gemini.or(image_data_from_imagen); match image_data { @@ -627,7 +238,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { .. } = gemini_sse_parser; - // Send tool call arguments events for accumulated tool calls for tool_call in accumulated_tool_calls.values() { let event = StreamingEvent::ToolCallArguments { call_id: tool_call.id.clone(), @@ -637,7 +247,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { stream_event_processor.send(event, &mut events_str).await?; } - // Convert Gemini usage metadata to TokenUsage let usage = gemini_usage.map(|u| { TokenUsage::new( u.prompt_token_count, @@ -647,11 +256,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { }); Ok(ParsedResponse::Text { - content: if accumulated_content.is_empty() { - None - } else { - Some(accumulated_content) - }, + content: if accumulated_content.is_empty() { None } else { Some(accumulated_content) }, tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations, @@ -661,20 +266,30 @@ impl QueryBuilder for GoogleAIQueryBuilder { } fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { - match output_type { - OutputType::Text => { - format!( - "{}/models/{}:streamGenerateContent?alt=sse", - base_url, model - ) + let base_url = base_url.trim_end_matches('/'); + if self.is_vertex() { + // Vertex AI: base_url is .../publishers/google/models + match output_type { + OutputType::Text => { + format!("{}/{}:streamGenerateContent?alt=sse", base_url, model) + } + OutputType::Image => { + let url_suffix = + if model.contains("imagen") { "predict" } else { "generateContent" }; + format!("{}/{}:{}", base_url, model, url_suffix) + } } - OutputType::Image => { - let url_suffix = if model.contains("imagen") { - "predict" - } else { - "generateContent" - }; - format!("{}/models/{}:{}", base_url, model, url_suffix) + } else { + // Standard Google AI: base_url is generativelanguage.googleapis.com/v1beta + match output_type { + OutputType::Text => { + format!("{}/models/{}:streamGenerateContent?alt=sse", base_url, model) + } + OutputType::Image => { + let url_suffix = + if model.contains("imagen") { "predict" } else { "generateContent" }; + format!("{}/models/{}:{}", base_url, model, url_suffix) + } } } } @@ -685,7 +300,12 @@ impl QueryBuilder for GoogleAIQueryBuilder { _base_url: &str, _output_type: &OutputType, ) -> Vec<(&'static str, String)> { - // Native Gemini API always uses x-goog-api-key - vec![("x-goog-api-key", api_key.to_string())] + if self.is_vertex() { + // Vertex AI uses OAuth2 Bearer token + vec![("Authorization", format!("Bearer {}", api_key))] + } else { + // Standard Google AI uses API key + vec![("x-goog-api-key", api_key.to_string())] + } } } diff --git a/backend/windmill-worker/src/ai/query_builder.rs b/backend/windmill-worker/src/ai/query_builder.rs index 04f1b4b548..c1d0d79a05 100644 --- a/backend/windmill-worker/src/ai/query_builder.rs +++ b/backend/windmill-worker/src/ai/query_builder.rs @@ -1,7 +1,5 @@ use async_trait::async_trait; -use windmill_common::{ - client::AuthedClient, error::Error, worker::Connection, -}; +use windmill_common::{client::AuthedClient, error::Error, worker::Connection}; use windmill_queue::MiniPulledJob; use windmill_types::s3::S3Object; @@ -112,8 +110,10 @@ pub fn create_query_builder(provider: &ProviderWithResource) -> Box Box::new(GoogleAIQueryBuilder::new()), + // Google AI uses the Gemini API (with platform-specific handling for Vertex AI) + AIProvider::GoogleAI => Box::new(GoogleAIQueryBuilder::new( + provider.get_platform().clone(), + )), // OpenAI use the Responses API AIProvider::OpenAI => Box::new(OpenAIQueryBuilder::new(provider.kind.clone())), // Anthropic uses its own API format (with platform-specific handling for Vertex AI) diff --git a/backend/windmill-worker/src/ai/sse.rs b/backend/windmill-worker/src/ai/sse.rs index 77fba3140a..62f13f3494 100644 --- a/backend/windmill-worker/src/ai/sse.rs +++ b/backend/windmill-worker/src/ai/sse.rs @@ -5,15 +5,18 @@ use reqwest::Response; use serde::Deserialize; use serde_json; use tokio_stream::StreamExt; -use windmill_common::{error::Error, utils::rd_string}; +use windmill_common::{ + ai_google::{parse_gemini_sse_event, GeminiUsageMetadata}, + ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall}, + error::Error, + utils::rd_string, +}; use crate::ai::{ query_builder::StreamEventProcessor, types::{StreamingEvent, UrlCitation}, }; -use windmill_common::ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall}; - #[derive(Deserialize)] pub struct OpenAIChoiceDeltaToolCallFunction { pub name: Option, @@ -457,96 +460,19 @@ impl SSEParser for AnthropicSSEParser { // Gemini SSE Parser // ============================================================================ -/// Gemini streaming response part - can be text or function call -#[derive(Deserialize, Debug)] -pub struct GeminiSSEPart { - #[serde(default)] - pub text: Option, - #[serde(rename = "functionCall")] - pub function_call: Option, - /// Thought signature for Gemini 3+ models - required for function calling - #[serde(rename = "thoughtSignature")] - pub thought_signature: Option, -} - -/// Function call in Gemini streaming response -#[derive(Deserialize, Debug)] -pub struct GeminiSSEFunctionCall { - pub name: String, - pub args: serde_json::Value, -} - -/// Content in Gemini streaming candidate -#[derive(Deserialize, Debug)] -pub struct GeminiSSEContent { - pub parts: Option>, -} - -/// Web reference in Gemini grounding chunk -#[derive(Deserialize, Debug)] -pub struct GeminiGroundingChunkWeb { - pub uri: String, - #[serde(default)] - pub title: Option, -} - -/// Grounding chunk from Gemini web search -#[derive(Deserialize, Debug)] -pub struct GeminiGroundingChunk { - pub web: Option, -} - -/// Grounding metadata from Gemini web search -#[derive(Deserialize, Debug)] -pub struct GeminiGroundingMetadata { - #[serde(rename = "groundingChunks", default)] - pub grounding_chunks: Vec, - #[serde(rename = "webSearchQueries", default)] - pub web_search_queries: Vec, -} - -/// Candidate in Gemini streaming response -#[derive(Deserialize, Debug)] -pub struct GeminiSSECandidate { - pub content: Option, - #[serde(rename = "finishReason")] - #[allow(dead_code)] - pub finish_reason: Option, - #[serde(rename = "groundingMetadata")] - pub grounding_metadata: Option, -} - -/// Gemini usage metadata from SSE response -#[derive(Deserialize, Debug, Clone)] -pub struct GeminiUsageMetadata { - #[serde(rename = "promptTokenCount", default)] - pub prompt_token_count: Option, - #[serde(rename = "candidatesTokenCount", default)] - pub candidates_token_count: Option, - #[serde(rename = "totalTokenCount", default)] - pub total_token_count: Option, -} - -/// Gemini SSE event structure -#[derive(Deserialize, Debug)] -pub struct GeminiSSEEvent { - pub candidates: Option>, - #[serde(rename = "usageMetadata")] - pub usage_metadata: Option, -} - -/// Gemini SSE Parser for streaming responses +/// Accumulates Gemini streaming events and converts them into the worker's +/// internal [`OpenAIToolCall`] / [`StreamingEvent`] representation. +/// +/// The actual SSE parsing is delegated to [`parse_gemini_sse_event`] from +/// `windmill_common::ai_google` so the logic can be shared with the API proxy. pub struct GeminiSSEParser { pub accumulated_content: String, pub accumulated_tool_calls: HashMap, pub events_str: String, pub stream_event_processor: StreamEventProcessor, tool_call_index: i64, - /// Collected URL citation annotations from web search pub annotations: Vec, - /// Whether web search was used in this response pub used_websearch: bool, - /// Token usage from usageMetadata pub usage: Option, } @@ -567,101 +493,57 @@ impl GeminiSSEParser { impl SSEParser for GeminiSSEParser { async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { - let event: Option = serde_json::from_str(data) - .inspect_err(|e| { - tracing::error!("Failed to parse SSE as a Gemini event {}: {}", data, e); - }) - .ok(); + let Some(parsed) = parse_gemini_sse_event(data)? else { + return Ok(()); + }; - if let Some(event) = event { - if let Some(candidates) = event.candidates { - for candidate in candidates { - if let Some(content) = candidate.content { - if let Some(parts) = content.parts { - for part in parts { - // Handle text content - if let Some(text) = part.text { - if !text.is_empty() { - self.accumulated_content.push_str(&text); - let event = StreamingEvent::TokenDelta { content: text }; - self.stream_event_processor - .send(event, &mut self.events_str) - .await?; - } - } + if let Some(text) = parsed.text { + self.accumulated_content.push_str(&text); + self.stream_event_processor + .send(StreamingEvent::TokenDelta { content: text }, &mut self.events_str) + .await?; + } - // Handle function calls - if let Some(function_call) = part.function_call { - let call_id = format!("call_{}", rd_string(24)); - let idx = self.tool_call_index; - self.tool_call_index += 1; + for tool_call in parsed.tool_calls { + let call_id = format!("call_{}", rd_string(24)); + let idx = self.tool_call_index; + self.tool_call_index += 1; - // Send tool call start event - let event = StreamingEvent::ToolCall { - call_id: call_id.clone(), - function_name: function_call.name.clone(), - }; - self.stream_event_processor - .send(event, &mut self.events_str) - .await?; + self.stream_event_processor + .send( + StreamingEvent::ToolCall { + call_id: call_id.clone(), + function_name: tool_call.name.clone(), + }, + &mut self.events_str, + ) + .await?; - // Build extra_content with thought_signature if present - let extra_content = - part.thought_signature.map(|sig| ExtraContent { - google: Some(GoogleExtraContent { - thought_signature: Some(sig), - }), - }); + let extra_content = tool_call.thought_signature.map(|sig| ExtraContent { + google: Some(GoogleExtraContent { thought_signature: Some(sig) }), + }); - // Store accumulated tool call - self.accumulated_tool_calls.insert( - idx, - OpenAIToolCall { - id: call_id, - function: OpenAIFunction { - name: function_call.name, - arguments: serde_json::to_string( - &function_call.args, - ) - .unwrap_or_else(|_| "{}".to_string()), - }, - r#type: "function".to_string(), - extra_content, - }, - ); - } - } - } - } + self.accumulated_tool_calls.insert( + idx, + OpenAIToolCall { + id: call_id, + function: OpenAIFunction { + name: tool_call.name, + arguments: serde_json::to_string(&tool_call.args) + .unwrap_or_else(|_| "{}".to_string()), + }, + r#type: "function".to_string(), + extra_content, + }, + ); + } - // Handle grounding metadata (web search results) - if let Some(ref grounding_metadata) = candidate.grounding_metadata { - // Set used_websearch if there are search queries or grounding chunks - if !grounding_metadata.web_search_queries.is_empty() - || !grounding_metadata.grounding_chunks.is_empty() - { - self.used_websearch = true; - } - - // Extract citations from grounding chunks - for chunk in &grounding_metadata.grounding_chunks { - if let Some(ref web) = chunk.web { - self.annotations.push(UrlCitation { - start_index: 0, // Gemini doesn't provide character indices - end_index: 0, - url: web.uri.clone(), - title: web.title.clone(), - }); - } - } - } - } - } - - // Extract usage metadata - if let Some(usage_metadata) = event.usage_metadata { - self.usage = Some(usage_metadata); - } + self.annotations.extend(parsed.annotations); + if parsed.used_websearch { + self.used_websearch = true; + } + if let Some(usage) = parsed.usage { + self.usage = Some(usage); } Ok(()) diff --git a/backend/windmill-worker/src/ai/types.rs b/backend/windmill-worker/src/ai/types.rs index eebe09bc77..c11c92adb3 100644 --- a/backend/windmill-worker/src/ai/types.rs +++ b/backend/windmill-worker/src/ai/types.rs @@ -20,10 +20,11 @@ use windmill_common::{ flow_status::AgentAction, flows::FlowModule, }; -use windmill_types::s3::S3Object; use windmill_parser::Typ; +use windmill_types::s3::S3Object; -// Re-export shared types from windmill_common::ai_types +// Re-export shared types from windmill_common +pub use windmill_common::ai_providers::AIPlatform; pub use windmill_common::ai_types::{ ContentPart, ImageUrlData, OpenAIContent, OpenAIMessage, ToolDef, ToolDefFunction, UrlCitation, }; @@ -156,14 +157,6 @@ impl From for AIAgentArgs { } } -#[derive(Deserialize, Debug, Clone, Default, PartialEq)] -#[serde(rename_all = "snake_case")] -pub enum AnthropicPlatform { - #[default] - Standard, - GoogleVertexAi, -} - #[derive(Deserialize, Debug)] pub struct ProviderResource { #[serde(alias = "apiKey", default, deserialize_with = "empty_string_as_none")] @@ -194,9 +187,9 @@ pub struct ProviderResource { deserialize_with = "empty_string_as_none" )] pub aws_session_token: Option, - /// Platform for Anthropic API (standard or google_vertex_ai) + /// Platform (standard or google_vertex_ai) #[serde(default)] - pub platform: AnthropicPlatform, + pub platform: AIPlatform, /// Enable 1M context window for Anthropic #[serde(alias = "enable_1M_context", default)] pub enable_1m_context: bool, @@ -244,7 +237,7 @@ impl ProviderWithResource { self.resource.aws_session_token.as_deref() } - pub fn get_platform(&self) -> &AnthropicPlatform { + pub fn get_platform(&self) -> &AIPlatform { &self.resource.platform } @@ -1603,10 +1596,7 @@ mod tests { schema.sanitize_for_google(); - assert!( - schema.multiple_of.is_none(), - "multipleOf should be removed" - ); + assert!(schema.multiple_of.is_none(), "multipleOf should be removed"); } #[test] @@ -1639,7 +1629,10 @@ mod tests { assert!(schema.default.is_none()); let value_prop = schema.properties.as_ref().unwrap().get("value").unwrap(); - assert!(value_prop.default.is_none(), "nested default should be removed"); + assert!( + value_prop.default.is_none(), + "nested default should be removed" + ); assert!( value_prop.exclusive_minimum.is_none(), "nested exclusiveMinimum should be removed" @@ -1652,7 +1645,10 @@ mod tests { value_prop.multiple_of.is_none(), "nested multipleOf should be removed" ); - assert!(value_prop.r#const.is_none(), "nested const should be removed"); + assert!( + value_prop.r#const.is_none(), + "nested const should be removed" + ); assert!(schema.properties.is_some()); assert!(matches!(&schema.r#type, Some(SchemaType::Single(t)) if t == "object")); diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index a374db29c5..d095602b40 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -731,16 +731,3 @@ pub fn extract_text_content(content: &OpenAIContent) -> String { .join(""), } } - -/// Parse a data URL to extract media type and base64 data -/// Format: data:mime_type;base64,data -/// Returns (media_type, data) tuple if successful -pub fn parse_data_url(url: &str) -> Option<(String, String)> { - if !url.starts_with("data:") { - return None; - } - let rest = url.strip_prefix("data:")?; - let (header, data) = rest.split_once(",")?; - let media_type = header.strip_suffix(";base64")?; - Some((media_type.to_string(), data.to_string())) -} diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index aa250de4ab..253384f6f2 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -33,9 +33,9 @@ use crate::{ read_and_check_result, start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, + is_sandboxing_enabled, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, - is_sandboxing_enabled, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, - PY_INSTALL_DIR, TZ_ENV, + DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -1184,7 +1184,7 @@ mount {{ job_dir, "run.config.proto", &NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT - .replace("{PY_INSTALL_DIR}", PY_INSTALL_DIR) + .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 64270b4044..7d92607af5 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -26,7 +26,12 @@ use windmill_queue::{ }; lazy_static::lazy_static! { - pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string()); + pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| { + #[cfg(not(windows))] + { "/bin/bash".to_string() } + #[cfg(windows)] + { "bash".to_string() } + }); } const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto"); @@ -194,7 +199,7 @@ exit $exit_status .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; let mut cmd_args = vec![ @@ -210,7 +215,10 @@ exit $exit_status .current_dir(job_dir) .env_clear() .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bash).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .args(cmd_args) @@ -236,7 +244,10 @@ exit $exit_status .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bash).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index 2f748fe22c..b092e135b6 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -6,9 +6,9 @@ use reqwest::Client; use serde_json::{json, value::RawValue, Value}; use windmill_common::client::AuthedClient; use windmill_common::error::to_anyhow; -use windmill_object_store::convert_json_line_stream; use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; use windmill_common::{error::Error, worker::to_raw_value}; +use windmill_object_store::convert_json_line_stream; use windmill_parser_sql::{ parse_bigquery_sig, parse_db_resource, parse_s3_mode, parse_sql_blocks, parse_sql_statement_named_params, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 9c46db572d..2fd194229a 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -22,7 +22,7 @@ use crate::{ handle_child::handle_child, is_sandboxing_enabled, read_ee_registry, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, - NPMRC, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + NPMRC, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; use windmill_common::{ @@ -53,7 +53,14 @@ use windmill_object_store::attempt_fetch_bytes; use windmill_parser::Typ; +// The Windows loader uses a virtual "windmill-url" namespace instead of writing .url +// files to disk, which avoids Windows path issues. The virtual namespace approach is +// likely better on all fronts but we keep the original .url-file loader on Linux to +// avoid breaking back-compat. +#[cfg(not(windows))] pub const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.js"); +#[cfg(windows)] +pub const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.windows.js"); pub const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js"); @@ -73,10 +80,12 @@ pub const BUN_DEDICATED_WORKER_ARGS: &[&str] = &["run", "-i", "--prefer-offline" /// - `arg_names`: The argument names for the main function (e.g., ["x", "y"]) /// - `main_import`: The import path for the main module (e.g., "./main.ts") /// - `date_conversions`: Optional date conversion statements for Datetime args +/// - `preprocessor_spread`: If the script has a preprocessor function, the comma-separated arg names for it pub fn generate_dedicated_worker_wrapper( arg_names: &[&str], main_import: &str, date_conversions: Option<&str>, + preprocessor_spread: Option<&str>, ) -> String { let spread = arg_names.join(","); let dates = date_conversions.unwrap_or(""); @@ -87,6 +96,36 @@ pub fn generate_dedicated_worker_wrapper( "" }; + let preprocessor_logic = if let Some(pre_spread) = preprocessor_spread { + format!( + r#" + if (rawLine.startsWith("preprocess:")) {{ + const preInput = rawLine.slice("preprocess:".length); + const parsedArgs = JSON.parse(preInput); + if (Main.preprocessor === undefined || typeof Main.preprocessor !== 'function') {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: "preprocessor function is missing", name: "Error" }})); + continue; + }} + try {{ + function preArgsObjToArr({{ {pre_spread} }}) {{ + return [ {pre_spread} ]; + }} + const preprocessedArgs = await Main.preprocessor(...preArgsObjToArr(parsedArgs)); + console.log("wm_res[preprocessed_args]:" + JSON.stringify(preprocessedArgs ?? {{}}, (key, value) => typeof value === 'undefined' ? null : value)); + // Now call main with preprocessed args + const mainArgs = getArgs(JSON.stringify(preprocessedArgs ?? {{}})); + const res = await Main.main(...mainArgs); + console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value)); + }} catch (e) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: rawLine }})); + }} + continue; + }}"# + ) + } else { + String::new() + }; + format!( r#" import * as Main from "{main_import}"; @@ -107,15 +146,17 @@ function getArgs(line) {{ for await (const line of Readline.createInterface({{ input: process.stdin }})) {{ {print_lines} - if (line === "end") {{ + const rawLine = line; + if (rawLine === "end") {{ process.exit(0); }} + {preprocessor_logic} try {{ - const args = getArgs(line); + const args = getArgs(rawLine); const res = await Main.main(...args); console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value)); }} catch (e) {{ - console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: line }})); + console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: rawLine }})); }} }} "# @@ -527,6 +568,8 @@ pub async fn build_loader( current_path: &str, mode: LoaderMode, ) -> Result<()> { + // Use forward slashes in JS strings to avoid backslash escape issues on Windows + let job_dir_js = job_dir.replace('\\', "/"); let loader = RELATIVE_BUN_LOADER .replace("W_ID", w_id) .replace("BASE_INTERNAL_URL", base_internal_url) @@ -549,13 +592,13 @@ import {{ readdir }} from "node:fs/promises"; let fileNames = [] try {{ - fileNames = await readdir("{job_dir}/node_modules") + fileNames = await readdir("{job_dir_js}/node_modules") }} catch (e) {{ }} try {{ await Bun.build({{ - entrypoints: ["{job_dir}/wrapper.mjs"], + entrypoints: ["{job_dir_js}/wrapper.mjs"], outdir: "./", target: "node", plugins: [p], @@ -597,7 +640,7 @@ plugin(p) try {{ await Bun.build({{ - entrypoints: ["{job_dir}/main.ts"], + entrypoints: ["{job_dir_js}/main.ts"], outdir: "./", target: "{}", plugins: [p], @@ -737,7 +780,7 @@ async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result error::Result> { let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); + if annotation.sandbox && NSJAIL_AVAILABLE.is_none() { + return Err(error::Error::ExecutionErr( + "Script has //sandbox annotation but nsjail is not available on this worker. \ + Please ensure nsjail is installed or remove the //sandbox annotation." + .to_string(), + )); + } + let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if let (Some(lock), true) = ( maybe_lock.get_lock(), !annotation.nobundling && !*DISABLE_BUNDLING && codebase.is_none(), @@ -1019,6 +1070,12 @@ pub async fn handle_bun_job( let _ = write_file(job_dir, "main.ts", inner_content)?; } else if !annotation.native && codebase.is_none() { let _ = write_file(job_dir, "package.json", r#"{ "type": "module" }"#)?; + } else if codebase.is_some() { + // Write a valid fallback package.json for codebase mode. Without this, + // nsjail creates an empty 0-byte file (from the mandatory: false mount) + // which Node.js fails to parse as JSON (ERR_INVALID_PACKAGE_CONFIG). + // If the codebase TAR includes a package.json, it will overwrite this. + let _ = write_file(job_dir, "package.json", "{}")?; }; let common_bun_proc_envs: HashMap = @@ -1028,6 +1085,37 @@ pub async fn handle_bun_job( let apply_preprocessor = job.flow_step_id.as_deref() != Some("preprocessor") && job.preprocessed == Some(false); + let is_wac_v2 = main_override.is_none() && crate::wac_executor::is_wac_v2_ts(inner_content); + + // For WAC v2, inject variable names into unnamed task() calls so the + // runtime can use them for step naming (timeline, graph). + // `const double = task(async ...` → `const double = task("double", async ...` + // Also handles: export const, let, var, and optional generic type parameters. + // Skips calls that already have a string argument: `task("path", async ...` + let inner_content = if is_wac_v2 { + use regex::Regex; + use std::borrow::Cow; + lazy_static::lazy_static! { + static ref TASK_RE: Regex = + Regex::new(r#"(?m)((?:export\s+)?(?:const|let|var)\s+)(\w+)(\s*=\s*task\s*(?:<[^>]*>)?\s*\(\s*)(async\b)"#).unwrap(); + } + let replaced = TASK_RE.replace_all(inner_content, r#"${1}${2}${3}"${2}", ${4}"#); + match replaced { + Cow::Borrowed(_) => inner_content.to_string(), + Cow::Owned(s) => s, + } + } else { + inner_content.to_string() + }; + let inner_content = inner_content.as_str(); + + // WAC v2 scripts can't use bundle caching because the wrapper imports + // windmill-client from node_modules, which isn't available in bundle mode + if is_wac_v2 && has_bundle_cache { + has_bundle_cache = false; + let _ = write_file(job_dir, "main.ts", inner_content)?; + } + let mut format = BundleFormat::Cjs; if has_bundle_cache { let target; @@ -1152,18 +1240,29 @@ pub async fn handle_bun_job( init_logs = format!("\n{}{}", cache_logs, init_logs); } + if annotation.sandbox { + init_logs.push_str("sandbox mode (nsjail)\n"); + } + let write_wrapper_f = async { if !has_bundle_cache && annotation.native { return Ok(()) as error::Result<()>; } // let mut start = Instant::now(); - let args = windmill_parser_ts::parse_deno_signature( - inner_content, - true, - false, - main_override.map(ToString::to_string), - )? - .args; + let args = if is_wac_v2 { + // For WAC v2, try to parse "main" args; if that fails, try the default export + windmill_parser_ts::parse_deno_signature(inner_content, true, false, None) + .unwrap_or_default() + .args + } else { + windmill_parser_ts::parse_deno_signature( + inner_content, + true, + false, + main_override.map(ToString::to_string), + )? + .args + }; let pre_args = if apply_preprocessor { Some( @@ -1200,6 +1299,14 @@ pub async fn handle_bun_job( // we cannot use Bun.read and Bun.write because it results in an EBADF error on cloud let main_name = main_override.unwrap_or("main"); + // For WAC child jobs where the parser can't find params (task-wrapped consts), + // fall back to passing arg values directly (filtering out internal fields) + let child_spread = if spread.is_empty() && main_override.is_some() { + "Object.values(Object.fromEntries(Object.entries(args).filter(([k]) => !k.startsWith('_'))))".to_string() + } else { + "argsObjToArr(args)".to_string() + }; + let main_import = if codebase.is_some() || has_bundle_cache { "./main.js" } else { @@ -1223,8 +1330,116 @@ pub async fn handle_bun_job( "".to_string() }; - let wrapper_content = format!( - r#" + let wac_spread = if spread.is_empty() { + "Object.values(args)".to_string() + } else { + format!("argsObjToArr(args)") + }; + + let wrapper_content = if is_wac_v2 { + format!( + r#" +import * as Main from "{main_import}"; +import {{ WorkflowCtx, StepSuspend, setWorkflowCtx }} from "windmill-client"; + +import * as fs from "fs/promises"; + +let args = await fs.readFile('args.json', {{ encoding: 'utf8' }}).then(JSON.parse); +const checkpoint = JSON.parse(await fs.readFile('checkpoint.json', {{ encoding: 'utf8' }})); + +function argsObjToArr({{ {spread} }}) {{ + return [ {spread} ]; +}} + +BigInt.prototype.toJSON = function () {{ + return this.toString(); +}}; + +// Find the workflow entrypoint (export default) +let workflowFn = Main.default; +if (!workflowFn || !workflowFn._is_workflow) {{ + for (const key of Object.keys(Main)) {{ + if (Main[key]?._is_workflow) {{ + workflowFn = Main[key]; + break; + }} + }} +}} +if (!workflowFn) {{ + throw new Error("No workflow() entrypoint found. Wrap your main function with workflow()."); +}} + +async function run() {{ + {dates} + {preprocessor} + const argsArr = {wac_spread}; + + const ctx = new WorkflowCtx(checkpoint); + setWorkflowCtx(ctx); + + try {{ + const result = await workflowFn(...argsArr); + setWorkflowCtx(null); + // Flush any unawaited tasks (e.g. forgotten await on last statement) + const trailing = ctx._flushPending(); + if (trailing.length > 0) {{ + return {{ type: "dispatch", mode: trailing.length > 1 ? "parallel" : "sequential", steps: trailing }}; + }} + return {{ type: "complete", result: result ?? null }}; + }} catch (e) {{ + setWorkflowCtx(null); + if (e?.name === "StepSuspend" || e instanceof StepSuspend) {{ + const dispatch = e.dispatchInfo ?? e.dispatch_info ?? {{}}; + if (dispatch.mode === "step_complete") {{ + return {{ type: "complete", result: dispatch.result ?? null }}; + }} + if (dispatch.mode === "inline_checkpoint") {{ + return {{ type: "inline_checkpoint", key: dispatch.key, result: dispatch.result ?? null }}; + }} + if (dispatch.mode === "approval") {{ + return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form }}; + }} + if (dispatch.mode === "sleep") {{ + return {{ type: "sleep", key: dispatch.key, seconds: dispatch.seconds }}; + }} + return {{ type: "dispatch", mode: dispatch.mode ?? "sequential", steps: dispatch.steps ?? [] }}; + }} + throw e; + }} +}} + +try {{ + const output = await run(); + const output_json = JSON.stringify(output, (key, value) => + typeof value === 'undefined' ? null : value + ); + await fs.writeFile("result.json", output_json); + process.exit(0); +}} catch(e) {{ + console.error(e); + let err = {{ message: e.message, name: e.name, stack: e.stack }}; + let step_id = process.env.WM_FLOW_STEP_ID; + if (step_id) {{ + err["step_id"] = step_id; + }} + const extra = {{}}; + Object.getOwnPropertyNames(e).forEach((key) => {{ + if (['line', 'name', 'stack', 'column', 'message', 'sourceURL', 'originalLine', 'originalColumn'].includes(key)) {{ + return; + }} + extra[key] = e[key]; + }}); + if (Object.keys(extra).length > 0) {{ + err["extra"] = extra; + }} + await fs.writeFile("result.json", JSON.stringify(err)); + process.exit(1); +}} + "#, + ) + } else { + format!( + r#" import * as Main from "{main_import}"; import * as fs from "fs/promises"; @@ -1246,11 +1461,14 @@ BigInt.prototype.toJSON = function () {{ async function run() {{ {dates} {preprocessor} - const argsArr = argsObjToArr(args); + // If the entrypoint has no parsed params (spread is empty), pass values directly + // This handles WAC child jobs where tasks are const-wrapped functions + const argsArr = {child_spread}; if (Main.{main_name} === undefined || typeof Main.{main_name} !== 'function') {{ throw new Error("{main_name} function is missing"); }} - let res = await Main.{main_name}(...argsArr); + let entrypoint = Main.{main_name}; + let res = await entrypoint(...argsArr); if (isAsyncIterable(res)) {{ for await (const chunk of res) {{ console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); @@ -1284,7 +1502,8 @@ try {{ process.exit(1); }} "#, - ); + ) + }; write_file(job_dir, "wrapper.mjs", &wrapper_content)?; Ok(()) as error::Result<()> }; @@ -1309,6 +1528,7 @@ try {{ && !annotation.nobundling && !*DISABLE_BUNDLING && !codebase.is_some() + && !is_wac_v2 && (maybe_lock.get_lock().is_some() || annotation.native); let write_loader_f = async { @@ -1354,6 +1574,23 @@ try {{ write_wrapper_f, write_loader_f )?; + + // For WAC v2, write checkpoint.json before bun runs + if is_wac_v2 { + if let Connection::Sql(db) = conn { + let checkpoint = crate::wac_executor::load_checkpoint(db, &job.id).await?; + let checkpoint = + crate::wac_executor::prepare_checkpoint_for_resume(db, &job.id, checkpoint).await?; + + let checkpoint_json = serde_json::to_string(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + write_file(job_dir, "checkpoint.json", &checkpoint_json)?; + } else { + write_file(job_dir, "checkpoint.json", r#"{"completed_steps":{}}"#)?; + } + } + if !codebase.is_some() && !has_bundle_cache { if build_cache { generate_bun_bundle( @@ -1445,11 +1682,71 @@ try {{ append_logs(&job.id, &job.workspace_id, format!("{init_logs}\n"), conn).await; + if apply_preprocessor { + // First pass: run preprocessor function + let pre_result = crate::js_eval::eval_fetch_timeout( + env_code.clone(), + inner_content.to_string(), + js_code.clone(), + job_args, + Some("preprocessor".to_string()), + job.id, + job.timeout, + conn, + mem_peak, + canceled_by, + worker_name, + &job.workspace_id, + false, + occupancy_metrics, + None, + has_stream, + ) + .await?; + + let preprocessed: HashMap> = + serde_json::from_str(pre_result.get()).map_err(|e| { + error::Error::internal_err(format!( + "error deserializing preprocessed args: {e:#}" + )) + })?; + *new_args = Some(preprocessed.clone()); + + // Second pass: run main with preprocessed args + let preprocessed_json = sqlx::types::Json(preprocessed); + let stream_notifier = StreamNotifier::new(conn, job); + + let result = crate::js_eval::eval_fetch_timeout( + env_code, + inner_content.to_string(), + js_code, + Some(&preprocessed_json), + job.script_entrypoint_override.clone(), + job.id, + job.timeout, + conn, + mem_peak, + canceled_by, + worker_name, + &job.workspace_id, + false, + occupancy_metrics, + stream_notifier, + has_stream, + ) + .await?; + tracing::info!( + "Executed native code (with preprocessor) in {}ms", + started_at.elapsed().as_millis() + ); + return Ok(result); + } + let stream_notifier = StreamNotifier::new(conn, job); let result = crate::js_eval::eval_fetch_timeout( env_code, - inner_content.clone(), + inner_content.to_string(), js_code, job_args, job.script_entrypoint_override.clone(), @@ -1476,7 +1773,7 @@ try {{ append_logs(&job.id, &job.workspace_id, init_logs, conn).await; //do not cache local dependencies - let child = if is_sandboxing_enabled() { + let child = if is_sandboxing_enabled() || annotation.sandbox { let _ = write_file( job_dir, "run.config.proto", @@ -1495,7 +1792,7 @@ try {{ }, ), ) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; @@ -1537,7 +1834,9 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bun).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn).await?, + ) .envs(common_bun_proc_envs) .env("PATH", PATH_ENV.as_str()) .args(args) @@ -1555,7 +1854,10 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bun).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn) + .await?, + ) .envs(common_bun_proc_envs) .stdin(Stdio::null()) .stdout(Stdio::piped()) @@ -1586,7 +1888,10 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bun).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn) + .await?, + ) .envs(common_bun_proc_envs) .stdin(Stdio::null()) .stdout(Stdio::piped()) @@ -1644,7 +1949,766 @@ try {{ })?; *new_args = Some(args.clone()); } - read_result(job_dir, handle_result.result_stream).await + + let result = read_result(job_dir, handle_result.result_stream).await?; + + // WAC v2 post-execution: parse output and handle dispatch/suspend + if is_wac_v2 { + return handle_wac_v2_output(result, job, conn).await; + } + + Ok(result) +} + +/// Handle WAC v2 output after bun/python exits. Parse result as WacOutput, +/// dispatch child jobs on suspend, or return the final result. +pub async fn handle_wac_v2_output( + result: Box, + job: &MiniPulledJob, + conn: &Connection, +) -> error::Result> { + use crate::wac_executor::{ + add_completed_step, load_checkpoint, parse_wac_output, update_checkpoint_for_dispatch, + WacOutput, + }; + use serde_json::Value; + use windmill_common::get_latest_flow_version_info_for_path; + use windmill_common::jobs::{script_path_to_payload, JobKind, JobPayload, RawCode}; + use windmill_common::runnable_settings::{ + ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, + }; + use windmill_queue::{push, PushArgs, PushIsolationLevel}; + + let output = parse_wac_output(&result)?; + + match output { + WacOutput::Complete { result: value } => { + // Workflow completed — return the inner result value + let raw = serde_json::value::to_raw_value(&value).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize WAC result: {e}")) + })?; + Ok(raw) + } + WacOutput::Dispatch { mode, steps } => { + if steps.is_empty() { + return Err(error::Error::internal_err( + "WAC v2 dispatch with no steps — this is a bug in the workflow SDK".to_string(), + )); + } + let db = match conn { + Connection::Sql(db) => db, + _ => { + return Err(error::Error::internal_err( + "WAC v2 dispatch requires SQL connection".to_string(), + )) + } + }; + + let mut checkpoint = load_checkpoint(db, &job.id).await?; + + // Source hash validation: detect if code changed between replays + let current_hash = job.runnable_id.map(|h| h.0.to_string()).unwrap_or_default(); + if !current_hash.is_empty() { + if checkpoint.source_hash.is_empty() { + checkpoint.source_hash = current_hash.clone(); + } else if checkpoint.source_hash != current_hash { + return Err(error::Error::ExecutionErr( + "Workflow source code changed between replays. \ + Cannot safely resume from checkpoint — step keys may have shifted. \ + Please restart this workflow." + .to_string(), + )); + } + } + let num_steps = steps.len(); + + tracing::info!( + job_id = %job.id, + mode = %mode, + num_steps = num_steps, + steps = ?steps.iter().map(|s| &s.name).collect::>(), + "WAC v2 dispatching child jobs" + ); + + // Create child jobs for each step. + // Each child re-runs the full workflow with a checkpoint containing + // _executing_key = step_key, so only that step runs its inner function. + // + // IMPORTANT: To prevent a race condition where a fast child completes + // before the parent is suspended, we: + // 1. Pre-generate child UUIDs + // 2. Save checkpoint + suspend parent + seed child checkpoints + // 3. THEN push the child jobs (making them visible to workers) + + // Read the parent's original args for the child jobs + let parent_args: HashMap> = { + let stored: serde_json::Map = checkpoint.input_args.clone(); + if stored.is_empty() { + // First dispatch — read from the parent job's args + let row: Option = sqlx::query_scalar( + "SELECT args FROM v2_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(&job.id) + .bind(&job.workspace_id) + .fetch_optional(db) + .await?; + let args_val = row.unwrap_or(Value::Object(Default::default())); + if let Value::Object(map) = args_val { + // Store for future re-runs + checkpoint.input_args = map.clone(); + map.into_iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(&v).map_err(|e| { + error::Error::internal_err(format!( + "Failed to serialize arg '{k}': {e}" + )) + })?; + Ok((k, raw)) + }) + .collect::>>()? + } else { + HashMap::new() + } + } else { + stored + .into_iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(&v).map_err(|e| { + error::Error::internal_err(format!( + "Failed to serialize arg '{k}': {e}" + )) + })?; + Ok((k, raw)) + }) + .collect::>>()? + } + }; + + // Pre-generate child UUIDs so we can save them in the checkpoint + // before the children become visible to workers. + // Validate key uniqueness — duplicate keys would cause one child's + // UUID to be overwritten in the job_ids map, making it unmappable + // on completion (the parent would hang). + { + let mut seen_keys = std::collections::HashSet::new(); + for s in &steps { + if !seen_keys.insert(&s.key) { + return Err(error::Error::internal_err(format!( + "WAC v2 duplicate step key '{}' — each task call must produce a unique key", + s.key + ))); + } + } + } + let job_ids: Vec<(String, Uuid)> = steps + .iter() + .map(|s| (s.key.clone(), ulid::Ulid::new().into())) + .collect(); + + // Resolve job_payload once (same for all children since they re-run + // the parent script) + let job_payload_template = match job.kind { + JobKind::Script => { + if let Some(hash) = job.runnable_id { + Ok(JobPayload::ScriptHash { + hash, + path: job.runnable_path.clone().unwrap_or_default(), + cache_ttl: job.cache_ttl, + cache_ignore_s3_path: job.cache_ignore_s3_path, + dedicated_worker: None, + language: job.script_lang.unwrap_or(ScriptLang::Bun), + priority: job.priority, + apply_preprocessor: false, + concurrency_settings: ConcurrencySettings::default(), + debouncing_settings: DebouncingSettings::default(), + }) + } else { + Err(error::Error::internal_err( + "WAC v2 Script job missing runnable_id".to_string(), + )) + } + } + JobKind::Preview => { + let row: Option<(Option, Option)> = sqlx::query_as( + "SELECT raw_code, raw_lock FROM v2_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(&job.id) + .bind(&job.workspace_id) + .fetch_optional(db) + .await?; + let (code, lock) = row.unwrap_or_default(); + Ok(JobPayload::Code(RawCode { + content: code.unwrap_or_default(), + path: job.runnable_path.clone(), + hash: None, + language: job.script_lang.unwrap_or(ScriptLang::Bun), + lock: lock, + cache_ttl: job.cache_ttl, + cache_ignore_s3_path: job.cache_ignore_s3_path, + dedicated_worker: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), + })) + } + _ => Err(error::Error::internal_err(format!( + "WAC v2 unsupported job kind: {:?}", + job.kind + ))), + }?; + + // Step 1: Save checkpoint, suspend parent, and seed child checkpoints + // in a single transaction — all BEFORE children become visible. + { + let mut tx = db.begin().await?; + + // Update checkpoint with pending steps + update_checkpoint_for_dispatch(&mut checkpoint, &steps, &mode, &job_ids); + let status_json = serde_json::to_value(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(&job.id) + .bind(&status_json) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to save WAC checkpoint: {e}")) + })?; + + // Store per-child-job info for the WorkflowTimeline UI + for (step, (_, child_id)) in steps.iter().zip(job_ids.iter()) { + let child_id_str = child_id.to_string(); + let timeline_val = serde_json::json!({ + "scheduled_for": chrono::Utc::now().to_rfc3339(), + "name": step.key, + }); + sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + COALESCE(workflow_as_code_status, '{}'::jsonb), + ARRAY[$2], + $3 + ) WHERE id = $1", + ) + .bind(&job.id) + .bind(&child_id_str) + .bind(&timeline_val) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to update WAC timeline status: {e}" + )) + })?; + } + + // Suspend parent before children become visible. + // Keep running = true so the normal pull query ignores it. + // The suspended pull query picks it up when suspend reaches 0 + // (it checks: suspend_until IS NOT NULL AND suspend <= 0). + let suspend_count = num_steps as i32; + sqlx::query!( + "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1", + job.id, + suspend_count, + ) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to suspend WAC parent job {}: {e}", + job.id + )) + })?; + + tx.commit().await?; + } + + // Step 2: Push child jobs (now visible to workers). + // Parent is already suspended, so child completions are safe. + // Track successfully pushed children so we can cancel them on + // partial failure (e.g. pushing child 3 of 5 fails). + let mut pushed_ids: Vec = Vec::with_capacity(num_steps); + let push_result: error::Result<()> = async { + for (step, (_, child_uuid)) in steps.iter().zip(job_ids.iter()) { + // Resolve job payload based on dispatch_type + let (job_payload, child_args, is_external) = match step.dispatch_type.as_str() { + "script" => { + // Resolve script path to job payload (handles hash, lang, etc.) + let (payload, _, _, _, _) = script_path_to_payload( + &step.script, + None, // no authed db for background workers + db.clone(), + &job.workspace_id, + Some(true), // skip preprocessor + ) + .await?; + let step_args: HashMap> = step + .args + .iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(v).unwrap(); + (k.clone(), raw) + }) + .collect(); + (payload, step_args, true) + } + "flow" => { + let flow_info = get_latest_flow_version_info_for_path( + None, + db, + &job.workspace_id, + &step.script, + true, + ) + .await?; + let payload = JobPayload::Flow { + path: step.script.clone(), + dedicated_worker: flow_info.dedicated_worker, + apply_preprocessor: false, + version: flow_info.version, + }; + let step_args: HashMap> = step + .args + .iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(v).unwrap(); + (k.clone(), raw) + }) + .collect(); + (payload, step_args, true) + } + _ => { + // "inline" — re-run parent with _executing_key + (job_payload_template.clone(), parent_args.clone(), false) + } + }; + + let push_args = PushArgs { args: &child_args, extra: None }; + + // Apply step-level overrides to payload (cache, concurrency) + let mut job_payload = job_payload; + if let Some(cache_ttl) = step.cache_ttl { + match &mut job_payload { + JobPayload::ScriptHash { cache_ttl: ref mut ct, .. } => { + *ct = Some(cache_ttl) + } + JobPayload::Code(ref mut code) => code.cache_ttl = Some(cache_ttl), + _ => {} + } + } + if step.concurrent_limit.is_some() + || step.concurrency_key.is_some() + || step.concurrency_time_window_s.is_some() + { + match &mut job_payload { + JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. } => { + if let Some(limit) = step.concurrent_limit { + cs.concurrent_limit = Some(limit); + } + if let Some(ref key) = step.concurrency_key { + cs.concurrency_key = Some(key.clone()); + } + if let Some(window) = step.concurrency_time_window_s { + cs.concurrency_time_window_s = Some(window); + } + } + JobPayload::Code(ref mut code) => { + if let Some(limit) = step.concurrent_limit { + code.concurrency_settings.concurrent_limit = Some(limit); + } + if let Some(ref key) = step.concurrency_key { + code.concurrency_settings.custom_concurrency_key = + Some(key.clone()); + } + if let Some(window) = step.concurrency_time_window_s { + code.concurrency_settings.concurrency_time_window_s = + Some(window); + } + } + _ => {} + } + } + + let (_, mut tx) = push( + db, + PushIsolationLevel::IsolatedRoot(db.clone()), + &job.workspace_id, + job_payload, + push_args, + &job.created_by, + &job.permissioned_as_email, + job.permissioned_as.clone(), + None, + None, + None, + Some(job.id), // parent_job + job.root_job.or(Some(job.id)), // root_job + job.flow_innermost_root_job, + Some(*child_uuid), // pre-generated job_id + false, // is_flow_step + false, // same_worker + None, // pre_run_error + job.visible_to_owner, + step.tag.clone().or_else(|| Some(job.tag.clone())), + step.timeout.or(job.timeout), + None, // flow_step_id + step.priority, // priority_override + None, // authed + false, // running + None, // end_user_email + None, // trigger + None, // suspended_mode + ) + .await?; + + // Seed child checkpoint only for inline tasks (they need + // _executing_key to know which step to run). External + // scripts/flows don't need a WAC checkpoint. + if !is_external { + let child_checkpoint_json = serde_json::json!({ + "completed_steps": &checkpoint.completed_steps, + "_executing_key": &step.key, + }); + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(child_uuid) + .bind(&child_checkpoint_json) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to seed child checkpoint: {e}" + )) + })?; + } + + tx.commit().await.map_err(|e| { + error::Error::internal_err(format!("Failed to commit child push: {e}")) + })?; + + pushed_ids.push(*child_uuid); + + tracing::info!( + parent_job = %job.id, + child_job = %child_uuid, + step_name = %step.name, + step_key = %step.key, + "WAC v2 dispatched child job" + ); + } + Ok(()) + } + .await; + + if let Err(e) = push_result { + tracing::error!( + job_id = %job.id, + error = %e, + pushed_count = pushed_ids.len(), + total_count = num_steps, + "WAC v2 failed to push child jobs, cleaning up" + ); + + // Cancel already-pushed children so they don't complete and + // corrupt the checkpoint (they'd decrement suspend on a parent + // that's about to be unsuspended and re-run). + for child_id in &pushed_ids { + let _ = sqlx::query!( + "UPDATE v2_job_queue SET canceled_by = $2, canceled_reason = $3 WHERE id = $1", + child_id, + "system", + "WAC dispatch failed: not all children could be pushed", + ) + .execute(db) + .await; + } + + // Clear pending_steps from checkpoint so the parent doesn't + // think children are outstanding when it re-runs. + let _ = sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = \ + workflow_as_code_status #- '{_checkpoint,pending_steps}' \ + WHERE id = $1", + ) + .bind(&job.id) + .execute(db) + .await; + + // Unsuspend parent so the error propagates instead of a 14-day hang + let _ = sqlx::query!( + "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", + job.id, + ) + .execute(db) + .await; + return Err(e); + } + + tracing::info!( + job_id = %job.id, + num_steps = num_steps, + "WAC v2 parent job suspended" + ); + + Err(error::Error::WacSuspended(format!( + "WAC v2 job {} suspended waiting for {} child job(s)", + job.id, num_steps + ))) + } + WacOutput::Approval { key, timeout, form } => { + let db = match conn { + Connection::Sql(db) => db, + _ => { + return Err(error::Error::internal_err( + "WAC v2 approval requires SQL connection".to_string(), + )) + } + }; + + let mut checkpoint = load_checkpoint(db, &job.id).await?; + let timeout_secs = timeout.unwrap_or(1800) as f64; + + // Mark this step as pending approval + checkpoint.pending_steps = Some(crate::wac_executor::WacPendingSteps { + mode: "approval".to_string(), + keys: vec![key.clone()], + job_ids: serde_json::Map::new(), + }); + + let mut tx = db.begin().await?; + + // Save checkpoint + let status_json = serde_json::to_value(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(&job.id) + .bind(&status_json) + .execute(&mut *tx) + .await + .map_err(|e| error::Error::internal_err(format!("Failed to save checkpoint: {e}")))?; + + // Store approval form metadata for the approval page endpoint + let approval_meta = serde_json::json!({ + "key": key, + "form": form, + "timeout": timeout_secs as u32, + }); + sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + COALESCE(workflow_as_code_status, '{}'::jsonb), + '{_approval}', + $2::jsonb + ) WHERE id = $1", + ) + .bind(&job.id) + .bind(&approval_meta) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to save approval meta: {e}")) + })?; + + // Suspend parent with suspend=1 (waiting for 1 approval event) + sqlx::query!( + "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", + job.id, + timeout_secs, + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + tracing::info!( + job_id = %job.id, + approval_key = %key, + timeout_secs = timeout_secs, + "WAC v2 parent job suspended waiting for approval" + ); + + Err(error::Error::WacSuspended(format!( + "WAC v2 job {} suspended waiting for approval (key: {})", + job.id, key + ))) + } + WacOutput::Sleep { key, seconds } => { + let db = match conn { + Connection::Sql(db) => db, + _ => { + return Err(error::Error::internal_err( + "WAC v2 sleep requires SQL connection".to_string(), + )) + } + }; + + let mut checkpoint = load_checkpoint(db, &job.id).await?; + let sleep_secs = seconds.max(1) as f64; + + // Mark this step as pending sleep + checkpoint.pending_steps = Some(crate::wac_executor::WacPendingSteps { + mode: "sleep".to_string(), + keys: vec![key.clone()], + job_ids: serde_json::Map::new(), + }); + + let mut tx = db.begin().await?; + + // Save checkpoint + let status_json = serde_json::to_value(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(&job.id) + .bind(&status_json) + .execute(&mut *tx) + .await + .map_err(|e| error::Error::internal_err(format!("Failed to save checkpoint: {e}")))?; + + // Suspend parent — it will auto-resume when suspend_until passes. + // Use suspend=1 (not 0) so the suspended pull query only picks it up + // when `suspend_until <= now()`, not via `suspend <= 0`. + sqlx::query!( + "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", + job.id, + sleep_secs, + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + tracing::info!( + job_id = %job.id, + sleep_key = %key, + sleep_secs = sleep_secs, + "WAC v2 parent job sleeping for {}s", + sleep_secs + ); + + Err(error::Error::WacSuspended(format!( + "WAC v2 job {} sleeping for {}s (key: {})", + job.id, seconds, key + ))) + } + WacOutput::InlineCheckpoint { key, result: value } => { + let db = match conn { + Connection::Sql(db) => db, + _ => { + return Err(error::Error::internal_err( + "WAC v2 inline checkpoint requires SQL connection".to_string(), + )) + } + }; + + let mut checkpoint = load_checkpoint(db, &job.id).await?; + + // Source hash validation (same as Dispatch path) + let current_hash = job.runnable_id.map(|h| h.0.to_string()).unwrap_or_default(); + if !current_hash.is_empty() { + if checkpoint.source_hash.is_empty() { + checkpoint.source_hash = current_hash.clone(); + } else if checkpoint.source_hash != current_hash { + return Err(error::Error::ExecutionErr( + "Workflow source code changed between replays. \ + Cannot safely resume from checkpoint — step keys may have shifted. \ + Please restart this workflow." + .to_string(), + )); + } + } + + tracing::info!( + job_id = %job.id, + step_key = %key, + "WAC v2 inline checkpoint — persisting step result" + ); + + add_completed_step(&mut checkpoint, &key, value); + + // Save checkpoint + reset running in a single transaction + { + let mut tx = db.begin().await?; + let status_json = serde_json::to_value(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(&job.id) + .bind(&status_json) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to save WAC checkpoint: {e}")) + })?; + + // Reset running=false so the job is immediately eligible for pickup. + // Unlike dispatch (which sets suspend>0), inline checkpoints don't suspend — + // the job should be re-run right away to continue past the cached step. + sqlx::query!( + "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1", + job.id, + ) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to reset running state for inline checkpoint: {e}" + )) + })?; + + tx.commit().await?; + } + + Err(error::Error::WacSuspended(format!( + "WAC v2 job {} inline checkpoint for step {}", + job.id, key + ))) + } + } } pub async fn get_common_bun_proc_envs(base_internal_url: Option<&str>) -> HashMap { @@ -1765,13 +2829,35 @@ async fn handle_dedicated_bunnative( let env_code = env_code.to_string(); let js_code = js_code.to_string(); + let pre_arg_names: Option> = windmill_parser_ts::parse_deno_signature( + inner_content, + true, + false, + Some("preprocessor".to_string()), + ) + .ok() + .filter(|sig| !sig.args.is_empty()) + .map(|sig| sig.args.into_iter().map(|x| x.name).collect()); + let mut warm = PrewarmedIsolate::spawn( env_code.clone(), js_code.clone(), ann.clone(), arg_names.clone(), + None, ); + // Pre-warm preprocessor isolate if the script has a preprocessor + let mut pre_warm = pre_arg_names.as_ref().map(|pre_names| { + PrewarmedIsolate::spawn( + env_code.clone(), + js_code.clone(), + ann.clone(), + pre_names.clone(), + Some("preprocessor".to_string()), + ) + }); + let init_log = format!("dedicated worker nativets: {worker_name}\n\n"); let alive = true; let mut killpill_rx = killpill_rx; @@ -1804,108 +2890,138 @@ async fn handle_dedicated_bunnative( "{}".to_string() }; - if let Err(e) = warm.wait_ready().await { - tracing::error!("pre-warmed isolate failed during init: {e}"); - let result = Arc::new(to_raw_value(&serde_json::json!({ - "message": format!("isolate init failed: {e}"), - "name": "Error", - }))); - append_logs(&id, &job.workspace_id, init_log.clone(), &db.into()).await; - job_completed_tx.send_job(JobCompleted { - job: MiniCompletedJob::from(job), - result, - result_columns: None, - mem_peak: 0, - canceled_by: None, - success: false, - cached_res_path: None, - token: token.to_string(), - duration: None, - preprocessed_args: None, - has_stream: Some(false), - from_cache: None, - flow_runners, - done_tx, - }, true).await?; + // Run the job: preprocess if needed, then execute main. + // Uses a labeled block to unify error handling with a single JobCompleted send. + let (result, success, preprocessed_args, logs) = 'job: { + if let Err(e) = warm.wait_ready().await { + break 'job ( + Arc::new(to_raw_value(&serde_json::json!({"message": format!("isolate init failed: {e}"), "name": "Error"}))), + false, None, init_log.clone(), + ); + } + + let needs_preprocessing = job.preprocessed == Some(false); + + let (main_args, preprocessed) = if needs_preprocessing { + let Some(ref pre_names) = pre_arg_names else { + break 'job ( + Arc::new(to_raw_value(&serde_json::json!({"message": "preprocessor function is missing", "name": "Error"}))), + false, None, init_log.clone(), + ); + }; + + let mut pre_isolate = pre_warm.take().unwrap_or_else(|| { + PrewarmedIsolate::spawn( + env_code.clone(), + js_code.clone(), + ann.clone(), + pre_names.clone(), + Some("preprocessor".to_string()), + ) + }); + if let Err(e) = pre_isolate.wait_ready().await { + break 'job ( + Arc::new(to_raw_value(&serde_json::json!({"message": format!("preprocessor isolate init failed: {e}"), "name": "Error"}))), + false, None, init_log.clone(), + ); + } + + let pre_executing = pre_isolate.start_execution(args.clone()); + // Pipeline: start pre-warming the next preprocessor isolate + pre_warm = Some(PrewarmedIsolate::spawn( + env_code.clone(), + js_code.clone(), + ann.clone(), + pre_names.clone(), + Some("preprocessor".to_string()), + )); + + let pre_result = match pre_executing.wait().await { + Ok(r) => r, + Err(e) => break 'job ( + Arc::new(to_raw_value(&serde_json::json!({"message": format!("preprocessor failed: {e}"), "name": "Error"}))), + false, None, init_log.clone(), + ), + }; + if !pre_result.logs.is_empty() { + append_logs(&id, &job.workspace_id, pre_result.logs, &db.into()).await; + } + let raw = match pre_result.result { + Ok(r) => r, + Err(e) => break 'job ( + Arc::new(to_raw_value(&serde_json::json!({"message": format!("preprocessor failed: {e}"), "name": "Error"}))), + false, None, init_log.clone(), + ), + }; + + let preprocessed: HashMap> = match serde_json::from_str(raw.get()) { + Ok(v) => v, + Err(e) => break 'job ( + Arc::new(to_raw_value(&serde_json::json!({"message": format!("error deserializing preprocessed args: {e:#}"), "name": "Error"}))), + false, None, init_log.clone(), + ), + }; + let main_args = serde_json::to_string(&preprocessed) + .unwrap_or_else(|_| "{}".to_string()); + (main_args, Some(preprocessed)) + } else { + (args, None) + }; + + let executing = warm.start_execution(main_args); + // Pipeline: start pre-warming the next main isolate warm = PrewarmedIsolate::spawn( env_code.clone(), js_code.clone(), ann.clone(), arg_names.clone(), + None, ); - continue; - } - let executing = warm.start_execution(args); + let main_result = match executing.wait().await { + Ok(r) => r, + Err(e) => break 'job ( + Arc::new(to_raw_value(&serde_json::json!({"message": format!("{e}"), "name": "Error"}))), + false, preprocessed, init_log.clone(), + ), + }; - warm = PrewarmedIsolate::spawn( - env_code.clone(), - js_code.clone(), - ann.clone(), - arg_names.clone(), - ); - - match executing.wait().await { - Ok(prewarmed_result) => { - let mut logs = init_log.clone(); - if !prewarmed_result.logs.is_empty() { - logs.push_str(&prewarmed_result.logs); - } - append_logs(&id, &job.workspace_id, logs, &db.into()).await; - - let (result, success) = match prewarmed_result.result { - Ok(raw) => (Arc::new(raw), true), - Err(e) => ( - Arc::new(to_raw_value(&serde_json::json!({ - "message": e, - "name": "Error", - }))), - false, - ), - }; - - job_completed_tx.send_job(JobCompleted { - job: MiniCompletedJob::from(job), - result, - result_columns: None, - mem_peak: 0, - canceled_by: None, - success, - cached_res_path: None, - token: token.to_string(), - duration: None, - preprocessed_args: None, - has_stream: Some(false), - from_cache: None, - flow_runners, - done_tx, - }, true).await?; + let mut logs = init_log.clone(); + if !main_result.logs.is_empty() { + logs.push_str(&main_result.logs); } - Err(e) => { - tracing::error!("isolate execution failed: {e}"); - append_logs(&id, &job.workspace_id, init_log.clone(), &db.into()).await; - let result = Arc::new(to_raw_value(&serde_json::json!({ - "message": format!("{e}"), - "name": "Error", - }))); - job_completed_tx.send_job(JobCompleted { - job: MiniCompletedJob::from(job), - result, - result_columns: None, - mem_peak: 0, - canceled_by: None, - success: false, - cached_res_path: None, - token: token.to_string(), - duration: None, - preprocessed_args: None, - has_stream: Some(false), - from_cache: None, - flow_runners: None, - done_tx: None, - }, true).await?; - } - } + + let (result, success) = match main_result.result { + Ok(raw) => (Arc::new(raw), true), + Err(e) => ( + Arc::new(to_raw_value(&serde_json::json!({ + "message": e, + "name": "Error", + }))), + false, + ), + }; + + (result, success, preprocessed, logs) + }; + + append_logs(&id, &job.workspace_id, logs, &db.into()).await; + job_completed_tx.send_job(JobCompleted { + job: MiniCompletedJob::from(job), + result, + result_columns: None, + mem_peak: 0, + canceled_by: None, + success, + cached_res_path: None, + token: token.to_string(), + duration: None, + preprocessed_args, + has_stream: Some(false), + from_cache: None, + flow_runners, + done_tx, + }, true).await?; } else { tracing::debug!("job channel closed for nativets dedicated worker"); break; @@ -2145,6 +3261,18 @@ pub async fn start_worker( .join("\n"); let arg_names: Vec<&str> = args.iter().map(|x| x.name.as_str()).collect(); + + // Parse preprocessor signature if it exists + let pre_spread = windmill_parser_ts::parse_deno_signature( + inner_content, + true, + false, + Some("preprocessor".to_string()), + ) + .ok() + .filter(|sig| !sig.args.is_empty()) + .map(|sig| sig.args.into_iter().map(|x| x.name).join(",")); + // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str()); // we cannot use Bun.read and Bun.write because it results in an EBADF error on cloud @@ -2158,7 +3286,12 @@ pub async fn start_worker( } else { Some(dates.as_str()) }; - let wrapper_content = generate_dedicated_worker_wrapper(&arg_names, main_import, dates_opt); + let wrapper_content = generate_dedicated_worker_wrapper( + &arg_names, + main_import, + dates_opt, + pre_spread.as_deref(), + ); write_file(job_dir, "wrapper.mjs", &wrapper_content)?; } diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index e03f90525e..846c421024 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -15,10 +15,6 @@ use tokio::process::Command; use tokio::{fs::File, io::AsyncReadExt}; use windmill_common::flows::Step; -#[cfg(feature = "parquet")] -use windmill_types::s3::{LargeFileStorage, ObjectStoreResource, S3Object}; -#[cfg(feature = "parquet")] -use windmill_object_store::get_etag_or_empty; use windmill_common::variables::{build_crypt_with_key_suffix, decrypt}; use windmill_common::worker::{ to_raw_value, update_ping_for_failed_init_script_query, write_file, Connection, Ping, PingType, @@ -32,6 +28,10 @@ use windmill_common::{ utils::configure_client, variables::ContextualVariable, }; +#[cfg(feature = "parquet")] +use windmill_object_store::get_etag_or_empty; +#[cfg(feature = "parquet")] +use windmill_types::s3::{LargeFileStorage, ObjectStoreResource, S3Object}; use anyhow::{anyhow, Result}; use windmill_parser_sql::{s3_mode_extension, S3ModeArgs, S3ModeFormat}; @@ -886,7 +886,7 @@ pub async fn cached_result_path( } #[cfg(feature = "parquet")] -async fn get_workspace_s3_resource_path( +pub(crate) async fn get_workspace_s3_resource_path( db: &DB, client: &AuthedClient, workspace_id: &str, @@ -948,7 +948,11 @@ async fn get_workspace_s3_resource_path( ) } Some(LargeFileStorage::FilesystemStorage(fs)) => { - (StorageResourceType::Filesystem, fs.root_path.clone()) + return Ok(Some( + windmill_object_store::ObjectStoreResource::Filesystem( + windmill_object_store::FilesystemSettings { root_path: fs.root_path.clone() }, + ), + )); } None => { return Ok(None); @@ -1090,7 +1094,7 @@ fn tentatively_improve_error(err: Error, executable: &str) -> Error { pub async fn clean_cache() -> error::Result<()> { tracing::info!("Started cleaning cache"); - tokio::fs::remove_dir_all(ROOT_CACHE_DIR).await?; + tokio::fs::remove_dir_all(&*ROOT_CACHE_DIR).await?; tracing::info!("Finished cleaning cache"); Ok(()) } @@ -1557,4 +1561,3 @@ mod tests { assert!(result.is_err()); } } - diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index e4c32648d0..22f46e724b 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -13,14 +13,11 @@ use itertools::Itertools; #[cfg(feature = "csharp")] use tokio::{fs::File, io::AsyncReadExt, process::Command}; #[cfg(feature = "csharp")] -use windmill_common::{ - utils::calculate_hash, - worker::write_file, -}; +use windmill_common::{utils::calculate_hash, worker::write_file}; -use windmill_common::error::{self, Error}; #[cfg(feature = "csharp")] use crate::global_cache::save_cache; +use windmill_common::error::{self, Error}; #[cfg(feature = "csharp")] use windmill_queue::append_logs; @@ -105,8 +102,8 @@ pub async fn generate_nuget_lockfile( let mut gen_lockfile_cmd = Command::new(DOTNET_PATH.as_str()); gen_lockfile_cmd .current_dir(job_dir) - .env("DOTNET_CLI_HOME", CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{CSHARP_CACHE_DIR}/nuget")) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") .env("DOTNET_NOLOGO", "true") .env("MSBUILDDISABLENODEREUSE", "1") @@ -367,8 +364,8 @@ async fn build_cs_proj( .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .env("DOTNET_CLI_HOME", CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{CSHARP_CACHE_DIR}/nuget")) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") .env("DOTNET_NOLOGO", "true") .env("MSBUILDDISABLENODEREUSE", "1") @@ -434,7 +431,7 @@ async fn build_cs_proj( } } - let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR); + let bin_path = format!("{}/{hash}", *CSHARP_CACHE_DIR); #[cfg(unix)] let target = format!("{job_dir}/Main"); #[cfg(windows)] @@ -516,11 +513,10 @@ pub async fn handle_csharp_job( inner_content, requirements_o.unwrap_or(&String::new()) )); - let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR); + let bin_path = format!("{}/{hash}", *CSHARP_CACHE_DIR); let remote_path = format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"); - let (cache, cache_logs) = - crate::global_cache::load_cache(&bin_path, &remote_path, false).await; + let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { #[cfg(unix)] @@ -591,11 +587,11 @@ pub async fn handle_csharp_job( "run.config.proto", &NSJAIL_CONFIG_RUN_CSHARP_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", CSHARP_CACHE_DIR) + .replace("{CACHE_DIR}", &*CSHARP_CACHE_DIR) .replace("{CACHE_HASH}", &hash) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -604,12 +600,15 @@ pub async fn handle_csharp_job( .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) - .env("DOTNET_CLI_HOME", CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{CSHARP_CACHE_DIR}/nuget")) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") .env("DOTNET_NOLOGO", "true") .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) @@ -637,11 +636,14 @@ pub async fn handle_csharp_job( .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) - .env("DOTNET_CLI_HOME", CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{CSHARP_CACHE_DIR}/nuget")) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") .env("DOTNET_NOLOGO", "true") .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 66d8ae8673..d512184e39 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -17,6 +17,7 @@ use crate::{ NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, }; use windmill_common::client::AuthedClient; +use windmill_common::worker::TypeScriptAnnotations; use tokio::{fs::File, io::AsyncReadExt, process::Command}; use windmill_common::{error::Result, scripts::ScriptLang, worker::write_file, BASE_URL}; @@ -120,11 +121,13 @@ async fn get_common_deno_proc_envs( } // Add proxy envs (including OTEL tracing proxy if enabled for deno) - for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno) - .await - .unwrap_or_default() - { - deno_envs.insert(k.to_string(), v); + if let Some(conn) = conn { + for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno, job_id, w_id, conn) + .await + .unwrap_or_default() + { + deno_envs.insert(k.to_string(), v); + } } return deno_envs; @@ -231,8 +234,13 @@ pub async fn handle_deno_job( occupancy_metrics: &mut OccupancyMetrics, has_stream: &mut bool, ) -> error::Result> { + let annotations = TypeScriptAnnotations::parse(inner_content); + // let mut start = Instant::now(); - let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); + let mut logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); + if annotations.sandbox { + logs1.push_str("sandbox mode (nsjail)\n"); + } append_logs(&job.id, &job.workspace_id, logs1, conn).await; let main_override = job.script_entrypoint_override.as_deref(); @@ -443,14 +451,15 @@ try {{ } let allow_read = format!( - "--allow-read=./,/tmp/windmill/cache/deno/,{}", + "--allow-read=./,{}/,{}", + *DENO_CACHE_DIR, DENO_PATH.as_str() ); if let Some(deno_flags) = DENO_FLAGS.as_ref() { for flag in deno_flags { args.push(flag); } - } else if is_sandboxing_enabled() { + } else if is_sandboxing_enabled() || annotations.sandbox { args.push("--allow-net"); args.push("--allow-sys"); args.push(allow_read.as_str()); @@ -504,7 +513,8 @@ try {{ *has_stream = handle_result.result_stream.is_some(); // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str()); - if let Err(e) = tokio::fs::remove_dir_all(format!("{DENO_CACHE_DIR}/gen/file/{job_dir}")).await + if let Err(e) = + tokio::fs::remove_dir_all(format!("{}/gen/file/{job_dir}", *DENO_CACHE_DIR)).await { tracing::error!("failed to remove deno gen tmp cache dir: {}", e); } @@ -645,11 +655,61 @@ pub async fn start_worker( .join("\n"); let spread = args.into_iter().map(|x| x.name).join(","); + + // Parse preprocessor signature if it exists + let pre_spread = windmill_parser_ts::parse_deno_signature( + inner_content, + true, + false, + Some("preprocessor".to_string()), + ) + .ok() + .filter(|sig| !sig.args.is_empty()) + .map(|sig| sig.args.into_iter().map(|x| x.name).join(",")); + + let preprocessor_import = if pre_spread.is_some() { + r#"import { preprocessor } from "./main.ts";"# + } else { + "" + }; + + let preprocessor_logic = if let Some(ref pre_spread) = pre_spread { + format!( + r#" + if (line.startsWith("preprocess:")) {{ + const preInput = line.slice("preprocess:".length); + const parsedArgs = JSON.parse(preInput); + if (typeof preprocessor !== 'function') {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: "preprocessor function is missing", name: "Error" }}) + '\n'); + continue; + }} + try {{ + function preArgsObjToArr({{ {pre_spread} }}: any) {{ + return [ {pre_spread} ]; + }} + const preprocessedArgs: any = await preprocessor(...preArgsObjToArr(parsedArgs)); + console.log("wm_res[preprocessed_args]:" + JSON.stringify(preprocessedArgs ?? {{}}, (key, value) => typeof value === 'undefined' ? null : value) + '\n'); + // Now call main with preprocessed args + let {{ {spread} }} = preprocessedArgs ?? {{}}; + {dates} + let res: any = await main(...[ {spread} ]); + console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value) + '\n'); + }} catch (e) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: line }}) + '\n'); + }} + continue; + }}"# + ) + } else { + String::new() + }; + // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str()); // we cannot use Bun.read and Bun.write because it results in an EBADF error on cloud let wrapper_content: String = format!( r#" import {{ main }} from "./main.ts"; +{preprocessor_import} BigInt.prototype.toJSON = function () {{ return this.toString(); @@ -668,6 +728,7 @@ for await (const chunk of Deno.stdin.readable) {{ exit = true; break; }} + {preprocessor_logic} try {{ let {{ {spread} }} = JSON.parse(line) {dates} diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 73136e1cc0..45e2f647a5 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -161,6 +161,22 @@ pub async fn do_duckdb( let base_internal_url = client.base_internal_url.clone(); let w_id = job.workspace_id.clone(); + if annotations.prepare { + let result = tokio::task::spawn_blocking(move || { + prepare_duckdb_ffi_safe( + query_block_list.iter().map(String::as_str), + &token, + &base_internal_url, + &w_id, + ) + }) + .await + .map_err(|e| Error::from(to_anyhow(e))) + .and_then(|r| r)?; + + return Ok(result); + } + let result = tokio::task::spawn_blocking(move || { run_duckdb_ffi_safe( query_block_list.iter().map(String::as_str), @@ -248,6 +264,18 @@ struct DuckDbFfiLib { collect_first_row_only: bool, ) -> *mut c_char, >, + prepare_duckdb_ffi: Option< + Symbol< + 'static, + unsafe extern "C" fn( + query_block_list: *const *const c_char, + query_block_list_count: usize, + token: *const c_char, + base_internal_url: *const c_char, + w_id: *const c_char, + ) -> *mut c_char, + >, + >, free_cstr: Symbol<'static, unsafe extern "C" fn(string: *mut c_char) -> ()>, } @@ -307,8 +335,11 @@ impl DuckDbFfiLib { } } + let prepare_duckdb_ffi = unsafe { lib.get(b"prepare_duckdb_ffi").ok() }; + Ok(DuckDbFfiLib { run_duckdb_ffi: unsafe { lib.get(b"run_duckdb_ffi").map_err(to_anyhow)? }, + prepare_duckdb_ffi, free_cstr: unsafe { lib.get(b"free_cstr").map_err(to_anyhow)? }, }) } @@ -388,6 +419,56 @@ fn run_duckdb_ffi_safe<'a>( } } +fn prepare_duckdb_ffi_safe<'a>( + query_block_list: impl Iterator, + token: &str, + base_internal_url: &str, + w_id: &str, +) -> Result> { + let query_block_list = query_block_list + .map(|s| { + CString::new(s).map_err(|e| { + Error::ExecutionErr(format!("Failed CString conversion: {}", e.to_string())) + }) + }) + .collect::>>()?; + let query_block_list = query_block_list + .iter() + .map(|s| s.as_ptr()) + .collect::>(); + + let token = CString::new(token).map_err(to_anyhow)?; + let base_internal_url = CString::new(base_internal_url).map_err(to_anyhow)?; + let w_id = CString::new(w_id).map_err(to_anyhow)?; + + let lib = DuckDbFfiLib::get_singleton()?; + let prepare_fn = lib.prepare_duckdb_ffi.as_ref().ok_or_else(|| { + Error::InternalErr( + "prepare_duckdb_ffi not available in duckdb ffi library. Please update to the latest windmill_duckdb_ffi_lib.".to_string(), + ) + })?; + let free_cstr = &lib.free_cstr; + + let result_str = unsafe { + let ptr = prepare_fn( + query_block_list.as_ptr(), + query_block_list.len(), + token.as_ptr(), + base_internal_url.as_ptr(), + w_id.as_ptr(), + ); + let str = CStr::from_ptr(ptr).to_string_lossy().to_string(); + free_cstr(ptr); + str + }; + + if result_str.starts_with("ERROR") { + Err(Error::ExecutionErr(result_str[6..].to_string())) + } else { + Ok(serde_json::value::RawValue::from_string(result_str).map_err(to_anyhow)?) + } +} + struct ParsedAttachDbResource<'a> { resource_path: &'a str, name: &'a str, diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 812be2fc7a..ea5a9ded46 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -18,8 +18,8 @@ pub async fn build_tar_and_push( custom_folder_name: Option, platform_agnostic: bool, ) -> error::Result<()> { - use windmill_object_store::object_store_reexports::Path; use tokio::fs::create_dir_all; + use windmill_object_store::object_store_reexports::Path; use crate::TAR_PYBASE_CACHE_DIR; @@ -33,7 +33,7 @@ pub async fn build_tar_and_push( folder.split("/").last().unwrap().to_owned() }; - let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", lang); + let prefix = &format!("{}/{}", *TAR_PYBASE_CACHE_DIR, lang); let tar_path = format!("{prefix}/{folder_name}_tar.tar"); create_dir_all(prefix).await?; @@ -197,7 +197,9 @@ pub async fn exists_in_cache(bin_path: &str, _remote_path: &str) -> bool { #[cfg(all(feature = "enterprise", feature = "parquet"))] if let Some(os) = windmill_object_store::get_object_store().await { return os - .get(&windmill_object_store::object_store_reexports::Path::from(_remote_path)) + .get(&windmill_object_store::object_store_reexports::Path::from( + _remote_path, + )) .await .is_ok(); } @@ -221,7 +223,7 @@ pub async fn save_cache( let file_to_cache = if is_dir { let tar_path = format!( "{}/tar/{}_tar.tar", - windmill_common::worker::ROOT_CACHE_DIR, + *windmill_common::worker::ROOT_CACHE_DIR, local_cache_path .split("/") .last() diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 35307baa72..0315bf25c3 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -2,6 +2,7 @@ use crate::{common::MaybeLock, get_proxy_envs_for_lang}; use std::{collections::HashMap, fs::DirBuilder, process::Stdio}; use windmill_common::scripts::ScriptLang; +use crate::global_cache::save_cache; use itertools::Itertools; use serde_json::value::RawValue; use tokio::{ @@ -15,7 +16,6 @@ use windmill_common::{ utils::calculate_hash, worker::{write_file, Connection, GoAnnotations}, }; -use crate::global_cache::save_cache; use windmill_parser_go::{parse_go_imports, REQUIRE_PARSE}; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -53,8 +53,7 @@ fn get_windows_program_files() -> String { #[cfg(windows)] fn windows_gopath() -> String { - let tmp_dir = get_windows_tmp_dir(); - GO_CACHE_DIR.replace("/tmp", &tmp_dir).replace("/", r"\\") + GO_CACHE_DIR.replace('/', "\\") } #[cfg(windows)] @@ -108,10 +107,9 @@ pub async fn handle_go_job( .expect("could not create go job dir"); let hash = calculate_hash(&format!("{}{:?}v2", inner_content, &maybe_lock)); - let bin_path = format!("{}/{hash}", GO_BIN_CACHE_DIR); + let bin_path = format!("{}/{hash}", *GO_BIN_CACHE_DIR); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); - let (cache, cache_logs) = - crate::global_cache::load_cache(&bin_path, &remote_path, false).await; + let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let (skip_go_mod, skip_tidy) = if cache { (true, true) @@ -238,15 +236,15 @@ func Run(req Req) (interface{{}}, error){{ .env("GOPATH", { #[cfg(unix)] { - GO_CACHE_DIR + GO_CACHE_DIR.as_str() } #[cfg(windows)] { - windows_gopath() + &windows_gopath() } }) .env("HOME", HOME_ENV.as_str()) - .env("GOCACHE", GO_CACHE_DIR) + .env("GOCACHE", GO_CACHE_DIR.as_str()) .envs(PROXY_ENVS.clone()) .args(vec!["build", "main.go"]) .stdout(Stdio::piped()) @@ -347,7 +345,7 @@ func Run(req Req) (interface{{}}, error){{ .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -356,7 +354,7 @@ func Run(req Req) (interface{{}}, error){{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Go).await?) + .envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -377,18 +375,18 @@ func Run(req Req) (interface{{}}, error){{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Go).await?) + .envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("GOPATH", { #[cfg(unix)] { - GO_CACHE_DIR + GO_CACHE_DIR.as_str() } #[cfg(windows)] { - windows_gopath() + &windows_gopath() } }) .env("HOME", HOME_ENV.as_str()); @@ -508,7 +506,7 @@ pub async fn install_go_dependencies( #[cfg(windows)] child_cmd.env("GOPATH", windows_gopath()); #[cfg(unix)] - child_cmd.env("GOPATH", GO_CACHE_DIR); + child_cmd.env("GOPATH", GO_CACHE_DIR.as_str()); #[cfg(windows)] set_windows_env_vars(&mut child_cmd); @@ -591,11 +589,11 @@ pub async fn install_go_dependencies( .env("GOPATH", { #[cfg(unix)] { - GO_CACHE_DIR + GO_CACHE_DIR.as_str() } #[cfg(windows)] { - windows_gopath() + &windows_gopath() } }) .args(vec!["mod", mod_command]) diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 1a107990dc..205da6700a 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -737,21 +737,24 @@ where let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0; if update_job_row && job_id != Uuid::nil() { if let Connection::Sql(ref db) = conn { - // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs - if i == 2 { - memory_metric_id = job_metrics::register_metric_for_job( - &db, - w_id.to_string(), - job_id, - "memory_kb".to_string(), - job_metrics::MetricKind::TimeseriesInt, - Some("Job Memory Footprint (kB)".to_string()), - ) - .await; - } - if let Ok(ref metric_id) = memory_metric_id { - if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await { - tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); + // Only track memory when it's non-zero (avoids storing all-zero timeseries for jobs that don't report memory) + if current_mem > 0 { + // Register on first non-zero reading (deferred from i==2 to avoid metric for jobs with no memory reporting) + if memory_metric_id.is_err() { + memory_metric_id = job_metrics::register_metric_for_job( + &db, + w_id.to_string(), + job_id, + "memory_kb".to_string(), + job_metrics::MetricKind::TimeseriesInt, + Some("Job Memory Footprint (kB)".to_string()), + ) + .await; + } + if let Ok(ref metric_id) = memory_metric_id { + if let Err(err) = job_metrics::record_timeseries_value(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem), job_metrics::MetricKind::TimeseriesInt).await { + tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); + } } } } diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 49129848ce..6f558e3328 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, path::PathBuf, process::Stdio}; +use crate::global_cache::save_cache; use anyhow::{anyhow, bail}; use async_recursion::async_recursion; use itertools::Itertools; @@ -15,7 +16,6 @@ use windmill_common::{ utils::calculate_hash, worker::{copy_dir_recursively, write_file, Connection}, }; -use crate::global_cache::save_cache; use windmill_parser::Arg; use windmill_parser_java::parse_java_sig_meta; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -185,8 +185,8 @@ pub async fn resolve<'a>( cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) - .env("COURSIER_CACHE", COURSIER_CACHE_DIR) + .env("HOME", &*JAVA_HOME_DIR) + .env("COURSIER_CACHE", &*COURSIER_CACHE_DIR) .envs(PROXY_ENVS.clone()); // Configure proxies @@ -208,7 +208,7 @@ pub async fn resolve<'a>( cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val)); } } - cmd.arg(&format!("-Duser.home={}", JAVA_HOME_DIR)); + cmd.arg(&format!("-Duser.home={}", *JAVA_HOME_DIR)); if metadata(TRUST_STORE_PATH.clone()).await.is_ok() { cmd.args(&[ &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), @@ -223,7 +223,7 @@ pub async fn resolve<'a>( "--parallel", &format!("{}", *JAVA_CONCURRENT_DOWNLOADS), "--cache", - COURSIER_CACHE_DIR, + &*COURSIER_CACHE_DIR, ]) .args(&get_repos(job_id, w_id, conn).await) .args(&deps.split("\n").collect_vec()) @@ -276,7 +276,8 @@ async fn install<'a>( match (it.next(), it.next(), it.next()) { (Some(group_id), Some(artifact_id), Some(version)) => { let path = format!( - "{JAVA_REPOSITORY_DIR}/{}/{artifact_id}/{version}", + "{}/{}/{artifact_id}/{version}", + *JAVA_REPOSITORY_DIR, group_id.replace(".", "/") ); Ok(RequiredDependency { @@ -312,7 +313,7 @@ async fn install<'a>( metadata(TRUST_STORE_PATH.clone()).await, ); let job_dir = job_dir.to_owned(); - let fetch_dir = format!("{JAVA_CACHE_DIR}/tmp-fetch-{}", Uuid::new_v4()); + let fetch_dir = format!("{}/tmp-fetch-{}", *JAVA_CACHE_DIR, Uuid::new_v4()); let fetch_dir2 = fetch_dir.clone(); par_install_language_dependencies_all_at_once( deps, @@ -334,8 +335,8 @@ async fn install<'a>( cmd.env_clear() .current_dir(&job_dir) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) - .env("COURSIER_CACHE", COURSIER_CACHE_DIR) + .env("HOME", &*JAVA_HOME_DIR) + .env("COURSIER_CACHE", &*COURSIER_CACHE_DIR) .envs(PROXY_ENVS.clone()); // Configure proxies { @@ -357,7 +358,7 @@ async fn install<'a>( } } - cmd.arg(&format!("-Duser.home={}", JAVA_HOME_DIR)); + cmd.arg(&format!("-Duser.home={}", *JAVA_HOME_DIR)); if trust_store_metadata.is_ok() { cmd.args(&[ &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), @@ -400,7 +401,7 @@ async fn install<'a>( if depth == 3 { copy_dir_recursively( &PathBuf::from(path), - &PathBuf::from(JAVA_REPOSITORY_DIR), + &PathBuf::from(&*JAVA_REPOSITORY_DIR), )?; return Ok(()); @@ -465,7 +466,7 @@ async fn compile<'a>( let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?; let hash = compute_hash(inner_content, *requirements_o); - let bin_path = format!("{}/{hash}", JAVA_CACHE_DIR); + let bin_path = format!("{}/{hash}", *JAVA_CACHE_DIR); let remote_path = format!("java_jar/{hash}"); let (cache, ..) = crate::global_cache::load_cache(&bin_path, &remote_path, true).await; @@ -501,7 +502,7 @@ async fn compile<'a>( cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) + .env("HOME", &*JAVA_HOME_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) @@ -604,7 +605,7 @@ async fn run<'a>( "run.config.proto", &NSJAIL_CONFIG_RUN_JAVA_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", JAVA_CACHE_DIR) + .replace("{CACHE_DIR}", &*JAVA_CACHE_DIR) .replace("{SHARED_MOUNT}", &shared_mount) // .replace("{CACHED_TARGET}", &shared_mount) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), @@ -613,10 +614,11 @@ async fn run<'a>( cmd.env_clear() .current_dir(job_dir) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) + .env("HOME", &*JAVA_HOME_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)) .args(vec![ "--config", "run.config.proto", @@ -671,10 +673,11 @@ async fn run<'a>( cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) + .env("HOME", &*JAVA_HOME_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) - .envs(reserved_variables); + .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)); if metadata(TRUST_STORE_PATH.clone()).await.is_ok() { cmd.args(&[ &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 741458c0e0..2fe56f50b9 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -70,6 +70,10 @@ mod sanitized_sql_params; mod schema; pub mod sql_utils; mod universal_pkg_installer; +#[cfg(feature = "private")] +mod volume_ee; +mod volume_oss; +pub mod wac_executor; mod worker; mod worker_flow; mod worker_lockfiles; diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index a4320a3612..28ac27c925 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -16,8 +16,8 @@ use crate::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, - get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, - TRACING_PROXY_CA_CERT_PATH, + get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, + PATH_ENV, TRACING_PROXY_CA_CERT_PATH, }; use windmill_common::client::AuthedClient; use windmill_common::scripts::ScriptLang; @@ -253,7 +253,7 @@ async fn run<'a>( .replace("{NU_PATH}", &NU_PATH) .replace("{SHARED_MOUNT}", &shared_mount) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -264,7 +264,7 @@ async fn run<'a>( .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Nu).await?) + .envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?) .args(vec![ "--config", "run.config.proto", @@ -303,7 +303,7 @@ async fn run<'a>( .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Nu).await?) + .envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?) // TODO(v1): // "--plugins", // &format!( diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index e98755e555..df006fef7f 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -22,7 +22,7 @@ use crate::{ get_reserved_variables, read_result, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, - COMPOSER_CACHE_DIR, COMPOSER_PATH, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, + is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, }; use windmill_common::client::AuthedClient; @@ -316,6 +316,7 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)) .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .args(args) @@ -332,6 +333,7 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)) .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .stdin(Stdio::null()) diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index acae81a588..961a49a58a 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -159,7 +159,7 @@ try { async fn scan_module_directories() -> Result, Error> { let mut module_dirs = HashMap::new(); - let cache_dir = std::path::Path::new(POWERSHELL_CACHE_DIR); + let cache_dir = std::path::Path::new(&*POWERSHELL_CACHE_DIR); if let Ok(entries) = fs::read_dir(cache_dir) { for entry in entries { @@ -391,7 +391,7 @@ pub async fn handle_powershell_job( .join(", "); let install_string = generate_powershell_install_code() - .replace("{path}", POWERSHELL_CACHE_DIR) + .replace("{path}", &*POWERSHELL_CACHE_DIR) .replace("{job_id}", &job.id.to_string()) .replace("{has_private_repo}", &format!("${has_private_repo}")) .replace("{has_credentials}", &format!("${has_credentials}")) @@ -442,7 +442,7 @@ $PSModulePathBackup = $env:PSModulePath $env:PSModulePath = \"$PSHome/Modules\" Get-Module -ListAvailable | Import-Module $env:PSModulePath = \"{}:$PSModulePathBackup\"", - POWERSHELL_CACHE_DIR + *POWERSHELL_CACHE_DIR ); #[cfg(windows)] @@ -452,7 +452,7 @@ $PSModulePathBackup = $env:PSModulePath $env:PSModulePath = \"C:\\Program Files\\PowerShell\\7\\Modules\" Get-Module -ListAvailable | Import-Module $env:PSModulePath = \"{};$PSModulePathBackup\"", - POWERSHELL_CACHE_DIR + *POWERSHELL_CACHE_DIR ); // NOTE: powershell error handling / termination is quite tricky compared to bash @@ -525,7 +525,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) - .replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR), + .replace("{CACHE_DIR}", &*POWERSHELL_CACHE_DIR), )?; let cmd_args = vec![ "--config", diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 047521a0b4..3a040234b6 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -134,8 +134,8 @@ use crate::{ handle_child::handle_child, is_sandboxing_enabled, read_ee_registry, worker_utils::ping_job_status, - PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, - PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, + PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, + PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, UV_INDEX_STRATEGY, }; use windmill_common::client::AuthedClient; @@ -278,7 +278,7 @@ pub async fn uv_pip_compile( "requirements.txt", // Target to /tmp/windmill/cache/uv "--cache-dir", - UV_CACHE_DIR, + &*UV_CACHE_DIR, ]; args.extend(["-p", &py_version_str, "--python-preference", "only-managed"]); @@ -567,6 +567,17 @@ pub async fn handle_python_job( let annotations = PythonAnnotations::parse(inner_content); + let is_wac_v2 = job.script_entrypoint_override.is_none() + && crate::wac_executor::is_wac_v2_py(inner_content); + + if annotations.sandbox && NSJAIL_AVAILABLE.is_none() { + return Err(Error::ExecutionErr( + "Script has #sandbox annotation but nsjail is not available on this worker. \ + Please ensure nsjail is installed or remove the #sandbox annotation." + .to_string(), + )); + } + let (py_version, mut additional_python_paths) = handle_python_deps( job_dir, requirements_o, @@ -605,16 +616,14 @@ pub async fn handle_python_job( } { - append_logs( - &job.id, - &job.workspace_id, - format!( - "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", - py_version.clone().to_string() - ), - conn, - ) - .await; + let mut logs = format!( + "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", + py_version.clone().to_string() + ); + if annotations.sandbox { + logs.push_str("sandbox mode (nsjail)\n"); + } + append_logs(&job.id, &job.workspace_id, logs, conn).await; } let ( import_loader, @@ -671,8 +680,69 @@ pub async fn handle_python_job( String::new() }; let main_override = main_name.unwrap_or_else(|| "main".to_string()); - let wrapper_content: String = format!( - r#" + let res_to_json_body = python_res_to_json_body(postprocessor); + let wrapper_content: String = if is_wac_v2 { + format!( + r#" +import os +import json +{import_loader} +{import_base64} +{import_datetime} +import traceback +import sys +from {module_dir_dot} import {last} as inner_script +from wmill.client import _run_workflow + +with open("args.json") as f: + kwargs = json.load(f, strict=False) +args = {{}} +{transforms} + +with open("checkpoint.json") as f: + checkpoint = json.load(f, strict=False) + +result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json") + +# Find the @workflow-decorated function +workflow_fn = None +for name in dir(inner_script): + obj = getattr(inner_script, name) + if callable(obj) and getattr(obj, '_is_workflow', False): + workflow_fn = obj + break + +if workflow_fn is None: + raise ValueError("No @workflow function found in script") + +for k, v in list(args.items()): + if v == '': + del args[k] + +try: + output = _run_workflow(workflow_fn, checkpoint, args) + output_json = json.dumps(output, separators=(',', ':'), default=str) + with open(result_json, 'w') as f: + f.write(output_json) +except BaseException as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + tb = traceback.format_tb(exc_traceback) + with open(result_json, 'w') as f: + err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }} + extra = e.__dict__ + if extra and len(extra) > 0: + err['extra'] = extra + flow_node_id = os.environ.get('WM_FLOW_STEP_ID') + if flow_node_id: + err['step_id'] = flow_node_id + err_json = json.dumps(err, separators=(',', ':'), default=str).replace('\n', '') + f.write(err_json) + sys.exit(1) +"#, + ) + } else { + format!( + r#" import os import json {import_loader} @@ -699,19 +769,7 @@ replace_invalid_fields = re.compile(r'(?:\bNaN\b|\\*\\u0000|Infinity|\-Infinity) result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json") def res_to_json(res, typ): - if typ.__name__ == 'DataFrame': - if typ.__module__ == 'pandas.core.frame': - res = res.values.tolist() - elif typ.__module__ == 'polars.dataframe.frame': - res = res.rows() - elif typ.__name__ == 'bytes': - res = to_b_64(res) - elif typ.__name__ == 'dict': - for k, v in res.items(): - if type(v).__name__ == 'bytes': - res[k] = to_b_64(v) - unprocessed = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '') - return {postprocessor} +{res_to_json_body} try: {preprocessor} @@ -745,9 +803,26 @@ except BaseException as e: f.write(err_json) sys.exit(1) "#, - ); + ) + }; write_file(job_dir, "wrapper.py", &wrapper_content)?; + // For WAC v2, write checkpoint.json before python runs. + if is_wac_v2 { + if let Connection::Sql(db) = conn { + let checkpoint = crate::wac_executor::load_checkpoint(db, &job.id).await?; + let checkpoint = + crate::wac_executor::prepare_checkpoint_for_resume(db, &job.id, checkpoint).await?; + + let checkpoint_json = serde_json::to_string(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + write_file(job_dir, "checkpoint.json", &checkpoint_json)?; + } else { + write_file(job_dir, "checkpoint.json", r#"{"completed_steps":{}}"#)?; + } + } + tracing::debug!("Finished writing wrapper"); let mut reserved_variables = @@ -784,7 +859,7 @@ except BaseException as e: #[cfg(windows)] let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";"); - if is_sandboxing_enabled() { + if is_sandboxing_enabled() || annotations.sandbox { let shared_deps = additional_python_paths .into_iter() .map(|pp| { @@ -805,7 +880,7 @@ mount {{ "run.config.proto", &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{PY_INSTALL_DIR}", PY_INSTALL_DIR) + .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str()) @@ -815,7 +890,7 @@ mount {{ "{ADDITIONAL_PYTHON_PATHS}", additional_python_paths_folders.as_str(), ) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; } else { @@ -828,14 +903,17 @@ mount {{ job.id ); - let child = if is_sandboxing_enabled() { + let child = if is_sandboxing_enabled() || annotations.sandbox { let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .current_dir(job_dir) .env_clear() // inject PYTHONPATH here - for some reason I had to do it in nsjail conf .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Python3).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -861,7 +939,10 @@ mount {{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Python3).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -924,7 +1005,39 @@ mount {{ *new_args = Some(args.clone()); } - read_result(job_dir, handle_result.result_stream).await + let result = read_result(job_dir, handle_result.result_stream).await?; + + // WAC v2 post-execution: parse output and handle dispatch/suspend. + // Box::pin to avoid bloating handle_python_job's async state machine (stack overflow). + if is_wac_v2 { + return Box::pin(crate::bun_executor::handle_wac_v2_output(result, job, conn)).await; + } + + Ok(result) +} + +/// Generate Python code to spread preprocessor args from `kwargs` into `pre_args`. +/// The `indent` parameter controls the join separator for multi-arg spreads. +fn python_preprocessor_spread(sig: windmill_parser::MainArgSignature, indent: &str) -> String { + if sig.star_kwargs { + "pre_args = kwargs".to_string() + } else { + sig.args + .into_iter() + .map(|x| { + let name = &x.name; + if x.default.is_none() { + format!("pre_args[\"{name}\"] = kwargs.get(\"{name}\")") + } else { + format!( + r#"pre_args["{name}"] = kwargs.get("{name}") +{indent}if pre_args["{name}"] is None: +{indent} del pre_args["{name}"]"# + ) + } + }) + .join(&format!("\n{indent}")) + } } async fn prepare_wrapper( @@ -1093,31 +1206,7 @@ async fn prepare_wrapper( .join("\n ") }; - let pre_spread = if let Some(pre_sig) = pre_sig { - let spread = if pre_sig.star_kwargs { - "pre_args = kwargs".to_string() - } else { - pre_sig - .args - .into_iter() - .map(|x| { - let name = &x.name; - if x.default.is_none() { - format!("pre_args[\"{name}\"] = kwargs.get(\"{name}\")") - } else { - format!( - r#"pre_args["{name}"] = kwargs.get("{name}") - if pre_args["{name}"] is None: - del pre_args["{name}"]"# - ) - } - }) - .join("\n ") - }; - Some(spread) - } else { - None - }; + let pre_spread = pre_sig.map(|sig| python_preprocessor_spread(sig, " ")); let module_dir_dot = dirs.replace("/", ".").replace("-", "_"); @@ -1410,10 +1499,10 @@ async fn spawn_uv_install( &nsjail_proto, NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT .replace("{WORKER_DIR}", worker_dir) - .replace("{PY_INSTALL_DIR}", &PY_INSTALL_DIR) + .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{TARGET_DIR}", &venv_p) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .as_str(), )?; @@ -2238,6 +2327,26 @@ This is not normal behavior, please make sure all workers have enough memory.\n }; } +/// Python function body for `res_to_json(res, typ)`. +/// Handles DataFrame, bytes, dict coercion + JSON serialization. +fn python_res_to_json_body(postprocessor: &str) -> String { + format!( + r#" if typ.__name__ == 'DataFrame': + if typ.__module__ == 'pandas.core.frame': + res = res.values.tolist() + elif typ.__module__ == 'polars.dataframe.frame': + res = res.rows() + elif typ.__name__ == 'bytes': + res = to_b_64(res) + elif typ.__name__ == 'dict': + for k, v in res.items(): + if type(v).__name__ == 'bytes': + res[k] = to_b_64(v) + unprocessed = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '') + return {postprocessor}"# + ) +} + // Returns code snippet that needs to be injected into wrapper to post-process results or leave unprocessed fn get_result_postprocessor<'a>(skip: bool) -> &'a str { if skip { @@ -2333,6 +2442,16 @@ pub async fn start_worker( _, ) = prepare_wrapper(job_dir, None, None, None, inner_content, script_path).await?; + // Parse preprocessor signature if the script has one + let pre_spread = windmill_parser_py::parse_python_signature( + inner_content, + Some("preprocessor".to_string()), + false, + ) + .ok() + .filter(|sig| !sig.args.is_empty()) + .map(|sig| python_preprocessor_spread(sig, " ")); + { let postprocessor = get_result_postprocessor(annotations.skip_result_postprocessing); let indented_transforms = transforms @@ -2341,6 +2460,41 @@ pub async fn start_worker( .collect::>() .join("\n"); + let preprocessor_logic = if let Some(ref pre_spread) = pre_spread { + format!( + r#" + if line.startswith('preprocess:'): + pre_input = line[len('preprocess:'):] + kwargs = json.loads(pre_input, strict=False) + if not hasattr(inner_script, 'preprocessor') or not callable(inner_script.preprocessor): + err_json = json.dumps({{"message": "preprocessor function is missing", "name": "Error"}}, separators=(',', ':'), default=str).replace('\n', '') + sys.stdout.write("wm_res[error]:" + err_json + "\n") + sys.stdout.flush() + continue + try: + pre_args = {{}} + {pre_spread} + for k, v in list(pre_args.items()): + if v == '': + del pre_args[k] + preprocessed_kwargs = inner_script.preprocessor(**pre_args) + preprocessed_json = json.dumps(preprocessed_kwargs, separators=(',', ':'), default=str).replace('\n', '') + sys.stdout.write("wm_res[preprocessed_args]:" + preprocessed_json + "\n") + transform_and_run(preprocessed_kwargs) + except BaseException as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + tb = traceback.format_tb(exc_traceback) + err_json = json.dumps({{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }}, separators=(',', ':'), default=str).replace('\n', '') + sys.stdout.write("wm_res[error]:" + err_json + "\n") + sys.stdout.flush() + continue +"# + ) + } else { + String::new() + }; + + let res_to_json_body = python_res_to_json_body(postprocessor); let wrapper_content: String = format!( r#" import json @@ -2358,37 +2512,32 @@ def to_b_64(v: bytes): b64 = base64.b64encode(v) return b64.decode('ascii') -replace_invalid_fields = re.compile(r'(?:\bNaN\b|\\u0000|Infinity|\-Infinity)') -sys.stdout.write('start\n') +def res_to_json(res, typ): +{res_to_json_body} -for line in sys.stdin: - if line == 'end\n': - break - kwargs = json.loads(line, strict=False) +def transform_and_run(kwargs): args = {{}} {indented_transforms} {spread} for k, v in list(args.items()): if v == '': del args[k] + res = inner_script.main(**args) + typ = type(res) + res_json = res_to_json(res, typ) + sys.stdout.write("wm_res[success]:" + res_json + "\n") +replace_invalid_fields = re.compile(r'(?:\bNaN\b|\\u0000|Infinity|\-Infinity)') +sys.stdout.write('start\n') + +for line in sys.stdin: + if line == 'end\n': + break + line = line.strip() + {preprocessor_logic} + kwargs = json.loads(line, strict=False) try: - res = inner_script.main(**args) - typ = type(res) - if typ.__name__ == 'DataFrame': - if typ.__module__ == 'pandas.core.frame': - res = res.values.tolist() - elif typ.__module__ == 'polars.dataframe.frame': - res = res.rows() - elif typ.__name__ == 'bytes': - res = to_b_64(res) - elif typ.__name__ == 'dict': - for k, v in res.items(): - if type(v).__name__ == 'bytes': - res[k] = to_b_64(v) - unprocessed = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '') - res_json = {postprocessor} - sys.stdout.write("wm_res[success]:" + res_json + "\n") + transform_and_run(kwargs) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index 52636808a6..78b03deb07 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -18,6 +18,8 @@ use windmill_common::{ use anyhow::{anyhow, bail}; use windmill_queue::append_logs; +#[cfg(unix)] +use crate::python_executor::UV_PATH; use crate::{ common::{start_child_process, OccupancyMetrics}, handle_child::handle_child, @@ -25,8 +27,6 @@ use crate::{ HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, UV_CACHE_DIR, WIN_ENVS, }; -#[cfg(unix)] -use crate::python_executor::UV_PATH; impl From for PyVAlias { fn from(value: PyV) -> Self { @@ -234,7 +234,8 @@ impl PyV { pub(crate) fn to_cache_dir(&self, ignore_patch: bool) -> String { use windmill_common::worker::ROOT_CACHE_DIR; format!( - "{ROOT_CACHE_DIR}{}", + "{}{}", + *ROOT_CACHE_DIR, self.to_cache_dir_top_level(ignore_patch) ) } @@ -311,7 +312,7 @@ impl PyV { Command::new(uv_cmd) .env_clear() .envs(WIN_ENVS.to_vec()) - .env("UV_CACHE_DIR", UV_CACHE_DIR) + .env("UV_CACHE_DIR", &*UV_CACHE_DIR) .args([ "python", "list", @@ -539,8 +540,8 @@ impl PyV { ]) // TODO: Do we need these? .envs([ - ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), - ("UV_CACHE_DIR", UV_CACHE_DIR), + ("UV_PYTHON_INSTALL_DIR", &*PY_INSTALL_DIR), + ("UV_CACHE_DIR", &*UV_CACHE_DIR), ]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -630,11 +631,9 @@ impl PyV { "--system", "--python-preference=only-managed", ]) - .envs([ - ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), - ("UV_PYTHON_PREFERENCE", "only-managed"), - ("UV_CACHE_DIR", UV_CACHE_DIR), - ]) + .env("UV_PYTHON_INSTALL_DIR", &*PY_INSTALL_DIR) + .env("UV_PYTHON_PREFERENCE", "only-managed") + .env("UV_CACHE_DIR", &*UV_CACHE_DIR) // .stdout(Stdio::piped()) .stderr(Stdio::piped()) .output() diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index c237d8cc02..e26633fc9a 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -665,7 +665,7 @@ pub async fn process_completed_job( add_time!(bench, "pre add_completed_job"); - let (_, duration) = add_completed_job( + let (_, duration, wac_job_ids) = add_completed_job( db, &job, true, @@ -717,6 +717,29 @@ pub async fn process_completed_job( } return Ok(r); } + } else if let Some(parent_job) = parent_job { + // wac_job_ids is piggybacked from the duration write in + // add_completed_job — no extra query needed. + if let Some(job_ids) = wac_job_ids { + if let Ok(Some(_)) = handle_wac_child_completion( + db, + &job_id, + parent_job, + &workspace_id, + result, + true, + job_ids, + ) + .await + { + if let Some(done_tx) = done_tx { + done_tx + .send(()) + .expect("done receiver should still be alive"); + } + return Ok(None); + } + } } } else { let result = add_completed_job_error( @@ -770,11 +793,229 @@ pub async fn process_completed_job( } return Ok(r); } + } else if let Some(parent_job) = job.parent_job { + // WAC child failed — query job_ids from parent (errors are rare, + // so the extra read is acceptable here). + let job_ids_json: Option> = sqlx::query_scalar( + "SELECT workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' \ + FROM v2_job_status WHERE id = $1", + ) + .bind(&parent_job) + .fetch_optional(db) + .await?; + if let Some(Some(job_ids)) = job_ids_json { + let err_result = Arc::new(serde_json::value::to_raw_value(&result).unwrap()); + if let Ok(Some(_)) = handle_wac_child_completion( + db, + &job.id, + parent_job, + &job.workspace_id, + err_result, + false, + job_ids, + ) + .await + { + if let Some(done_tx) = done_tx { + done_tx + .send(()) + .expect("done receiver should still be alive"); + } + return Ok(None); + } + } } } return Ok(None); } +/// Handle a WAC v2 child job completion. +/// Returns Ok(Some(())) if the parent was a WAC job and was handled, +/// Ok(None) if the parent is not a WAC job (caller should fall through). +/// +/// CONCURRENCY: Multiple parallel children may complete simultaneously on +/// different workers. We use atomic SQL operations throughout: +/// - `completed_steps` is merged via `jsonb_set(... || jsonb_build_object(...))` +/// — PostgreSQL serialises concurrent UPDATEs on the same row, so each +/// worker sees the previous worker's writes. +/// - The suspend counter (set to N at dispatch time) is decremented atomically +/// with `RETURNING` to determine the "all done" condition. +pub(crate) async fn handle_wac_child_completion( + db: &DB, + child_job_id: &Uuid, + parent_job_id: Uuid, + workspace_id: &str, + result: Arc>, + success: bool, + job_ids_value: Value, +) -> error::Result> { + let job_ids = match job_ids_value { + Value::Object(m) => m, + _ => return Ok(None), // Not a WAC parent or no pending steps + }; + + let child_id_str = child_job_id.to_string(); + let step_key = job_ids.iter().find_map(|(key, val)| { + if val.as_str() == Some(&child_id_str) { + Some(key.clone()) + } else { + None + } + }); + + let step_key = match step_key { + Some(k) => k, + None => { + if !success { + // No step key and failed — can't store error, fail parent immediately + tracing::error!( + parent_job = %parent_job_id, + child_job = %child_job_id, + "WAC v2 child job failed but no step key found, failing parent" + ); + sqlx::query!( + "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", + parent_job_id, + ) + .execute(db) + .await?; + let parent_mini = get_mini_completed_job(&parent_job_id, workspace_id, db).await?; + if let Some(parent_mini) = parent_mini { + let child_err: Value = + serde_json::from_str(result.get()).unwrap_or(Value::Null); + let err_value = json!({ + "message": format!("WAC child job {} failed (no step key)", child_job_id), + "error": child_err, + }); + let _ = windmill_queue::add_completed_job_error( + db, + &parent_mini, + 0, + None, + err_value, + "wac_child_handler", + false, + None, + ) + .await; + } + return Ok(Some(())); + } + tracing::warn!( + parent_job = %parent_job_id, + child_job = %child_job_id, + "WAC v2 child completed but no matching step key found in checkpoint, decrementing suspend to avoid parent hang" + ); + // Still decrement suspend so the parent doesn't hang indefinitely + let _ = sqlx::query_scalar!( + "UPDATE v2_job_queue \ + SET suspend = GREATEST(suspend - 1, 0) \ + WHERE id = $1 \ + RETURNING suspend", + parent_job_id, + ) + .fetch_optional(db) + .await?; + return Ok(Some(())); + } + }; + + // Build result — wrap errors with _error marker so workflow try/catch can handle them + let result_value: Value = if success { + serde_json::from_str(result.get()).unwrap_or(Value::Null) + } else { + let child_err: Value = serde_json::from_str(result.get()).unwrap_or(Value::Null); + tracing::info!( + parent_job = %parent_job_id, + child_job = %child_job_id, + step_key = %step_key, + "WAC v2 child job failed, storing error for workflow try/catch" + ); + json!({ + "__wmill_error": true, + "message": format!("WAC task '{}' failed (child job {})", step_key, child_job_id), + "child_job_id": child_job_id.to_string(), + "step_key": step_key, + "result": child_err, + }) + }; + + tracing::info!( + parent_job = %parent_job_id, + child_job = %child_job_id, + step_key = %step_key, + success = success, + "WAC v2 child job completed" + ); + + // Use a transaction to ensure completed_steps merge + suspend decrement + // are atomic. Without this, a crash between the two could strand the parent. + let result_json = serde_json::to_value(&result_value) + .map_err(|e| error::Error::InternalErr(format!("Failed to serialize step result: {e}")))?; + + let mut tx = db.begin().await?; + + // Merge the completed step into the checkpoint. + // Uses `|| jsonb_build_object(key, value)` so concurrent children on + // different workers don't overwrite each other — PostgreSQL serialises + // concurrent UPDATEs on the same row and each sees the previous write. + sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + workflow_as_code_status, + '{_checkpoint,completed_steps}', + COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps', '{}'::jsonb) + || jsonb_build_object($2::text, $3::jsonb) + ) WHERE id = $1", + ) + .bind(&parent_job_id) + .bind(&step_key) + .bind(&result_json) + .execute(&mut *tx) + .await + .map_err(|e| error::Error::InternalErr(format!("Failed to add WAC completed step: {e}")))?; + + // Decrement the suspend counter. The counter was set to N (number of + // children) at dispatch time. When it reaches 0 all children are done. + // Keep suspend_until non-null so the suspended pull query + // (`WHERE suspend_until IS NOT NULL AND suspend <= 0`) picks up the parent. + let new_suspend: Option = sqlx::query_scalar!( + "UPDATE v2_job_queue \ + SET suspend = GREATEST(suspend - 1, 0) \ + WHERE id = $1 \ + RETURNING suspend", + parent_job_id, + ) + .fetch_optional(&mut *tx) + .await?; + + let all_done = new_suspend == Some(0); + + if all_done { + // Clear pending_steps from checkpoint since all children are complete. + // This is cosmetic — the next replay will overwrite it anyway — but + // keeps the checkpoint clean for frontend display. + let _ = sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = \ + workflow_as_code_status #- '{_checkpoint,pending_steps}' \ + WHERE id = $1", + ) + .bind(&parent_job_id) + .execute(&mut *tx) + .await; + } + + tx.commit().await?; + + if all_done { + tracing::info!( + parent_job = %parent_job_id, + "WAC v2 all child jobs completed, unsuspending parent" + ); + } + + Ok(Some(())) +} + pub async fn handle_non_flow_job_error( db: &DB, job: &MiniCompletedJob, diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index 869b58d7de..d283b2a1c4 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -1,7 +1,6 @@ use std::{collections::HashMap, process::Stdio}; use anyhow::anyhow; -use const_format::concatcp; use itertools::Itertools; use regex::Regex; use tokio::{ @@ -122,7 +121,7 @@ pub async fn prepare<'a>( .write_all(&wrap(inner_content)?.into_bytes()) .await?; - let mini_wm_path = format!("{RUBY_CACHE_DIR}/gems/windmill-internal/windmill"); + let mini_wm_path = format!("{}/gems/windmill-internal/windmill", *RUBY_CACHE_DIR); if !std::fs::metadata(&mini_wm_path).is_ok() { fs::create_dir_all(&mini_wm_path).await?; @@ -339,7 +338,7 @@ Your Gemfile syntax will continue to work as-is." &NSJAIL_CONFIG_LOCK_RUBY_CONTENT .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); @@ -588,7 +587,7 @@ async fn install<'a>( // 123...zx-activesupport-8.0.2 // ^^^^^^^^ hash based on source and type (GEM or GIT) let handle = format!("{}-{}-{}", hash, pkg, version); - let path = format!("{RUBY_CACHE_DIR}/gems/{}", &handle); + let path = format!("{}/gems/{}", *RUBY_CACHE_DIR, &handle); deps.push(RequiredDependency { path, @@ -632,7 +631,7 @@ async fn install<'a>( &NSJAIL_CONFIG_DOWNLOAD_RUBY_CONTENT .replace("{TARGET}", &dependency.path) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); @@ -741,9 +740,9 @@ async fn install<'a>( }; // Include builtin windmill client { - const WM_INTERNAL: &str = concatcp!(RUBY_CACHE_DIR, "/gems/windmill-internal"); - res.top_level_paths.push(WM_INTERNAL.to_owned()); - res.rubylib += format!(":{WM_INTERNAL}").as_str(); + let wm_internal = format!("{}/gems/windmill-internal", *RUBY_CACHE_DIR); + res.top_level_paths.push(wm_internal.clone()); + res.rubylib += format!(":{wm_internal}").as_str(); } Ok(res) } @@ -800,7 +799,7 @@ mount {{ .replace("{JOB_DIR}", job_dir) .replace("{SHARED_MOUNT}", &shared_mount) .replace("{SHARED_DEPENDENCIES}", &shared_deps) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; @@ -813,7 +812,10 @@ mount {{ .envs(envs) .envs(reserved_variables) .envs(RUBY_PROXY_ENVS.clone()) - .envs(get_proxy_envs_for_lang(&ScriptLang::Ruby).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn) + .await?, + ) .args(vec![ "--config", "run.config.proto", @@ -852,7 +854,10 @@ mount {{ .env("BASE_INTERNAL_URL", base_internal_url) .envs(reserved_variables) .envs(RUBY_PROXY_ENVS.clone()) - .envs(get_proxy_envs_for_lang(&ScriptLang::Ruby).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn) + .await?, + ) .envs(envs); cmd.stdin(Stdio::null()) diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 54d90550b0..dae8e08766 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -5,6 +5,7 @@ use std::{collections::HashMap, process::Stdio}; use uuid::Uuid; use windmill_parser_rust::parse_rust_deps_into_manifest; +use crate::global_cache::save_cache; use itertools::Itertools; use tokio::{ fs::{create_dir_all, File}, @@ -16,7 +17,6 @@ use windmill_common::{ utils::calculate_hash, worker::{write_file, Connection}, }; -use crate::global_cache::save_cache; use windmill_queue::MiniPulledJob; use windmill_queue::{append_logs, CanceledBy}; @@ -41,20 +41,30 @@ const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.co const NSJAIL_CONFIG_COMPILE_RUST_CONTENT: &str = include_str!("../nsjail/download.rust.config.proto"); +#[cfg(windows)] +const RUST_BIN_NAME: &str = "main.exe"; +#[cfg(not(windows))] +const RUST_BIN_NAME: &str = "main"; + fn find_cargo_path() -> String { if let Ok(p) = std::env::var("CARGO_PATH") { return p; } - let from_home = format!("{}/bin/cargo", CARGO_HOME.as_str()); - if std::path::Path::new(&from_home).exists() { - return from_home; - } - for p in ["/usr/local/cargo/bin/cargo", "/usr/bin/cargo"] { + let candidates = if cfg!(windows) { + vec![format!("{}\\bin\\cargo.exe", CARGO_HOME.as_str())] + } else { + vec![ + format!("{}/bin/cargo", CARGO_HOME.as_str()), + "/usr/local/cargo/bin/cargo".to_string(), + "/usr/bin/cargo".to_string(), + ] + }; + for p in &candidates { if std::path::Path::new(p).exists() { - return p.to_string(); + return p.clone(); } } - from_home + candidates.into_iter().next().unwrap() } #[cfg(not(windows))] @@ -71,7 +81,6 @@ fn find_preinstalled_dir(env_var: &str, candidates: &[&str]) -> String { } lazy_static::lazy_static! { - static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() }); static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() }); static ref CARGO_PATH: String = find_cargo_path(); @@ -81,14 +90,14 @@ lazy_static::lazy_static! { #[cfg(windows)] lazy_static::lazy_static! { - static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", *HOME_DIR); - static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR); + static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", HOME_ENV.as_str()); + static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", HOME_ENV.as_str()); } #[cfg(not(windows))] lazy_static::lazy_static! { - static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", *HOME_DIR); - static ref RUSTUP_HOME_DEFAULT: String = format!("{}/.rustup", *HOME_DIR); + static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", HOME_ENV.as_str()); + static ref RUSTUP_HOME_DEFAULT: String = format!("{}/.rustup", HOME_ENV.as_str()); } const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; @@ -97,11 +106,11 @@ const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; lazy_static::lazy_static! { static ref PREINSTALLED_CARGO: String = find_preinstalled_dir( "CARGO_PREINSTALL_DIR", - &["/usr/local/cargo", &format!("{}/.cargo", *HOME_DIR)], + &["/usr/local/cargo", &format!("{}/.cargo", HOME_ENV.as_str())], ); static ref PREINSTALLED_RUSTUP: String = find_preinstalled_dir( "RUSTUP_PREINSTALL_DIR", - &["/usr/local/rustup", &format!("{}/.rustup", *HOME_DIR)], + &["/usr/local/rustup", &format!("{}/.rustup", HOME_ENV.as_str())], ); } @@ -337,7 +346,7 @@ async fn get_build_dir( if !is_sandboxing_enabled() { // If nsjail is disabled then entire worker has shared build directory // It drastically improves cache hit-rate. - Some((format!("{RUST_CACHE_DIR}/build/{worker_name}"), true)) + Some((format!("{}/build/{worker_name}", *RUST_CACHE_DIR), true)) } else { // If nsjail is enabled, having global shared directory is vulnerability and target for an attack // Instead we either: @@ -345,7 +354,8 @@ async fn get_build_dir( // 2. If user is not known or something else goes wrong - use random build dir. This is equivalent to no cache at all. Some(( format!( - "{RUST_CACHE_DIR}/build/{}@{}@{}", + "{}/build/{}@{}@{}", + *RUST_CACHE_DIR, &job.workspace_id, p.replace('/', "."), &job.created_by @@ -355,7 +365,10 @@ async fn get_build_dir( } } }) - .unwrap_or((format!("{RUST_CACHE_DIR}/build/{}", Uuid::new_v4()), false)); + .unwrap_or(( + format!("{}/build/{}", *RUST_CACHE_DIR, Uuid::new_v4()), + false, + )); { let (t, r, g) = ( @@ -449,7 +462,7 @@ pub async fn build_rust_crate( is_preview: bool, ) -> error::Result { ensure_rust_runtime_dirs(); - let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); + let bin_path = format!("{}/{hash}", *RUST_CACHE_DIR); let build_dir = get_build_dir(job, job_dir, conn, worker_name, is_preview).await?; @@ -459,9 +472,9 @@ pub async fn build_rust_crate( "download.config.proto", &NSJAIL_CONFIG_COMPILE_RUST_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", RUST_CACHE_DIR) + .replace("{CACHE_DIR}", &*RUST_CACHE_DIR) .replace("{CARGO_HOME}", CARGO_HOME.as_str()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{BUILD}", &build_dir), )?; @@ -517,6 +530,13 @@ pub async fn build_rust_crate( std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), ); build_rust_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + // MSVC linker needs LIB and INCLUDE to find kernel32.lib etc. + if let Ok(lib) = std::env::var("LIB") { + build_rust_cmd.env("LIB", lib); + } + if let Ok(include) = std::env::var("INCLUDE") { + build_rust_cmd.env("INCLUDE", include); + } } start_child_process(build_rust_cmd, CARGO_PATH.as_str(), false).await? }; @@ -541,30 +561,29 @@ pub async fn build_rust_crate( tokio::fs::copy( &format!( - "{build_dir}/target/{}/main", + "{build_dir}/target/{}/{RUST_BIN_NAME}", if is_preview { "debug" } else { "release" }, ), - format! {"{job_dir}/main"}, + format!("{job_dir}/{RUST_BIN_NAME}"), ) .await .map_err(|e| { Error::ExecutionErr(format!( - "could not copy built binary from [...]/target/.../main to {job_dir}/main: {e:?}" + "could not copy built binary from [...]/target/.../{RUST_BIN_NAME} to {job_dir}/{RUST_BIN_NAME}: {e:?}" )) })?; match save_cache( &bin_path, &format!("{RUST_OBJECT_STORE_PREFIX}{hash}"), - &format!("{job_dir}/main"), + &format!("{job_dir}/{RUST_BIN_NAME}"), false, ) .await { Err(e) => { let em = format!( - "could not save {bin_path} to {} to rust cache: {e:?}", - format!("{job_dir}/main"), + "could not save {bin_path} to {job_dir}/{RUST_BIN_NAME} to rust cache: {e:?}", ); tracing::error!(em); Ok(em) @@ -605,26 +624,25 @@ pub async fn handle_rust_job( check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?; let hash = compute_rust_hash(inner_content, requirements_o); - let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); + let bin_path = format!("{}/{hash}", *RUST_CACHE_DIR); let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; - let (cache, cache_logs) = - crate::global_cache::load_cache(&bin_path, &remote_path, false).await; + let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { - let target = format!("{job_dir}/main"); + let target = format!("{job_dir}/{RUST_BIN_NAME}"); #[cfg(unix)] let symlink = std::os::unix::fs::symlink(&bin_path, &target); #[cfg(windows)] - let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + let symlink = std::os::windows::fs::symlink_file(&bin_path, &target); symlink.map_err(|e| { Error::ExecutionErr(format!( - "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" + "could not copy cached binary from {bin_path} to {target}: {e:?}" )) })?; @@ -669,10 +687,10 @@ pub async fn handle_rust_job( "run.config.proto", &NSJAIL_CONFIG_RUN_RUST_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", RUST_CACHE_DIR) + .replace("{CACHE_DIR}", &*RUST_CACHE_DIR) .replace("{CACHE_HASH}", &hash) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{SHARED_MOUNT}", shared_mount), )?; @@ -682,7 +700,10 @@ pub async fn handle_rust_job( .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Rust).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -691,14 +712,17 @@ pub async fn handle_rust_job( .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { - let compiled_executable_name = "./main"; + let compiled_executable_name = &format!("{job_dir}/{RUST_BIN_NAME}"); let mut run_rust = build_command_with_isolation(compiled_executable_name, &[]); run_rust .current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Rust).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 9f89f2fc44..90d06287bc 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -9,8 +9,8 @@ use serde_json::{json, value::RawValue, Value}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use windmill_common::error::to_anyhow; -use windmill_object_store::convert_json_line_stream; use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; +use windmill_object_store::convert_json_line_stream; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{ diff --git a/backend/windmill-worker/src/volume_oss.rs b/backend/windmill-worker/src/volume_oss.rs new file mode 100644 index 0000000000..3736e83485 --- /dev/null +++ b/backend/windmill-worker/src/volume_oss.rs @@ -0,0 +1,112 @@ +#[cfg(feature = "private")] +pub(crate) use crate::volume_ee::*; + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) struct LeaseRenewalGuard(pub Option>); + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +impl Drop for LeaseRenewalGuard { + fn drop(&mut self) { + if let Some(handle) = self.0.take() { + handle.abort(); + } + } +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) struct VolumeSetupResult { + pub states: Vec, + pub writable: Vec, + pub client: Option>, + pub lease_renewal: LeaseRenewalGuard, +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +#[allow(dead_code)] +pub(crate) fn setup_volume_mount_paths( + _volume: &windmill_worker_volumes::VolumeMount, + _state: &windmill_worker_volumes::VolumeState, + _job_dir: &str, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result<()> { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn setup_volumes_sql_worker( + _volume_mounts: &[windmill_worker_volumes::VolumeMount], + _db: &windmill_common::DB, + _workspace_id: &str, + _job_id: uuid::Uuid, + _permissioned_as: &str, + _worker_name: &str, + _job_dir: &str, + _client: &windmill_common::client::AuthedClient, + _conn: &windmill_common::worker::Connection, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn setup_volumes_http_worker( + _volume_mounts: &[windmill_worker_volumes::VolumeMount], + _http: &windmill_common::worker::HttpClient, + _workspace_id: &str, + _job_id: uuid::Uuid, + _permissioned_as: &str, + _canceled_by: &Option, + _worker_name: &str, + _job_dir: &str, + _conn: &windmill_common::worker::Connection, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn sync_volumes_sql_worker( + _volume_states: &[windmill_worker_volumes::VolumeState], + _volume_writable: &[bool], + _vol_client: &std::sync::Arc, + _db: &windmill_common::DB, + _workspace_id: &str, + _job_id: uuid::Uuid, + _worker_name: &str, + _conn: &windmill_common::worker::Connection, + _job_succeeded: bool, +) { +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn sync_volumes_http_worker( + _volume_states: &[windmill_worker_volumes::VolumeState], + _volume_writable: &[bool], + _http: &windmill_common::worker::HttpClient, + _workspace_id: &str, + _job_id: uuid::Uuid, + _worker_name: &str, + _conn: &windmill_common::worker::Connection, + _job_succeeded: bool, +) { +} diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs new file mode 100644 index 0000000000..7c350ac92b --- /dev/null +++ b/backend/windmill-worker/src/wac_executor.rs @@ -0,0 +1,339 @@ +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use serde_json::Value; +use uuid::Uuid; + +use windmill_common::error::{self, Error}; +use windmill_common::DB; + +/// Checkpoint state persisted across workflow invocations. +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +pub struct WacCheckpoint { + #[serde(default)] + pub source_hash: String, + #[serde(default)] + pub completed_steps: serde_json::Map, + #[serde(skip_serializing_if = "Option::is_none")] + pub pending_steps: Option, + #[serde(default)] + pub input_args: serde_json::Map, + /// Accumulated map of step_key → child job UUID across all dispatch rounds. + /// Unlike `pending_steps.job_ids` (cleared after completion), this persists + /// so the frontend can always resolve step keys to child job names. + #[serde(default)] + pub job_ids: serde_json::Map, + /// When set on a child job's checkpoint, indicates which step this child + /// should execute directly (instead of dispatching). + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub _executing_key: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct WacPendingSteps { + pub mode: String, + pub keys: Vec, + pub job_ids: serde_json::Map, +} + +/// Output from a single WAC invocation (parsed from result.json). +#[derive(Debug, Deserialize)] +#[serde(tag = "type")] +pub enum WacOutput { + #[serde(rename = "dispatch")] + Dispatch { mode: String, steps: Vec }, + #[serde(rename = "complete")] + Complete { result: Value }, + /// An inline step executed in the parent process — persist result to + /// checkpoint and re-run immediately (no child job, no suspend). + #[serde(rename = "inline_checkpoint")] + InlineCheckpoint { key: String, result: Value }, + /// Suspend the workflow waiting for an external approval event. + /// No child job is dispatched — the parent suspends directly and resumes + /// when a user hits the resume/cancel endpoint. + #[serde(rename = "approval")] + Approval { key: String, timeout: Option, form: Option }, + /// Server-side sleep — suspend the workflow for a duration without holding a worker. + #[serde(rename = "sleep")] + Sleep { key: String, seconds: u32 }, +} + +/// A step dispatched by the WAC SDK. +/// +/// `dispatch_type` determines how the child job is created: +/// - `"inline"` (default): re-runs the parent workflow with `_executing_key` set +/// - `"script"`: runs a separate Windmill script resolved from `script` path +/// - `"flow"`: runs a separate Windmill flow resolved from `script` path +#[derive(Debug, Deserialize, Clone)] +pub struct WacStepDispatch { + pub name: String, + pub script: String, + pub args: serde_json::Map, + pub key: String, + #[serde(default = "default_dispatch_type")] + pub dispatch_type: String, + // Per-task options forwarded to push() + #[serde(default)] + pub timeout: Option, + #[serde(default)] + pub tag: Option, + #[serde(default)] + pub cache_ttl: Option, + #[serde(default)] + pub priority: Option, + #[serde(default)] + pub concurrent_limit: Option, + #[serde(default)] + pub concurrency_key: Option, + #[serde(default)] + pub concurrency_time_window_s: Option, +} + +fn default_dispatch_type() -> String { + "inline".to_string() +} + +/// Load the WAC checkpoint from `v2_job_status.workflow_as_code_status._checkpoint`. +pub async fn load_checkpoint(db: &DB, job_id: &Uuid) -> error::Result { + let row: Option> = sqlx::query_scalar( + "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = $1", + ) + .bind(job_id) + .fetch_optional(db) + .await?; + + match row { + Some(Some(status)) => { + let checkpoint: WacCheckpoint = match serde_json::from_value(status) { + Ok(c) => c, + Err(e) => { + tracing::warn!( + job_id = %job_id, + error = %e, + "Failed to deserialize WAC checkpoint, resetting to empty" + ); + WacCheckpoint::default() + } + }; + Ok(checkpoint) + } + _ => Ok(WacCheckpoint::default()), + } +} + +/// Save the WAC checkpoint to `v2_job_status.workflow_as_code_status._checkpoint`. +/// The top level of workflow_as_code_status is reserved for per-child-job timeline data. +pub async fn save_checkpoint( + db: &DB, + job_id: &Uuid, + checkpoint: &WacCheckpoint, +) -> error::Result<()> { + let status_json = serde_json::to_value(checkpoint) + .map_err(|e| Error::InternalErr(format!("Failed to serialize checkpoint: {e}")))?; + + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(job_id) + .bind(&status_json) + .execute(db) + .await + .map_err(|e| Error::InternalErr(format!("Failed to save WAC checkpoint: {e}")))?; + + Ok(()) +} + +/// Parse the WAC result from result.json content. +pub fn parse_wac_output(result: &RawValue) -> error::Result { + serde_json::from_str(result.get()) + .map_err(|e| Error::InternalErr(format!("Failed to parse WAC output: {e}"))) +} + +/// Process a "dispatch" result: update checkpoint with pending steps info. +pub fn update_checkpoint_for_dispatch( + checkpoint: &mut WacCheckpoint, + steps: &[WacStepDispatch], + mode: &str, + job_ids: &[(String, Uuid)], +) { + let ids_map: serde_json::Map = job_ids + .iter() + .map(|(key, id)| (key.clone(), Value::String(id.to_string()))) + .collect(); + // Accumulate into persistent job_ids (survives pending_steps clearing) + for (k, v) in ids_map.iter() { + checkpoint.job_ids.insert(k.clone(), v.clone()); + } + let pending = WacPendingSteps { + mode: mode.to_string(), + keys: steps.iter().map(|s| s.key.clone()).collect(), + job_ids: ids_map, + }; + checkpoint.pending_steps = Some(pending); +} + +/// Process a completed child job result: add to checkpoint's completed_steps. +pub fn add_completed_step(checkpoint: &mut WacCheckpoint, step_key: &str, result: Value) { + checkpoint + .completed_steps + .insert(step_key.to_string(), result); + // If all pending steps are complete, clear pending + if let Some(ref pending) = checkpoint.pending_steps { + let all_done = pending + .keys + .iter() + .all(|k| checkpoint.completed_steps.contains_key(k)); + if all_done { + checkpoint.pending_steps = None; + } + } +} + +/// Check if all pending parallel steps are complete. +pub fn all_pending_complete(checkpoint: &WacCheckpoint) -> bool { + match &checkpoint.pending_steps { + None => true, + Some(pending) => pending + .keys + .iter() + .all(|k| checkpoint.completed_steps.contains_key(k)), + } +} + +/// If the checkpoint has a pending approval or sleep, inject the resume result +/// into `completed_steps` and save back to DB. Returns the (possibly modified) checkpoint. +/// +/// Called by both bun and python executors before writing checkpoint.json to disk. +pub async fn prepare_checkpoint_for_resume( + db: &DB, + job_id: &Uuid, + mut checkpoint: WacCheckpoint, +) -> error::Result { + let pending_mode = checkpoint.pending_steps.as_ref().map(|p| p.mode.as_str()); + + match pending_mode { + Some("approval") => { + let approval_key = checkpoint + .pending_steps + .as_ref() + .and_then(|p| p.keys.first().cloned()) + .unwrap_or_default(); + + let resume_row = sqlx::query_as::<_, (sqlx::types::Json>, Option, bool)>( + "SELECT value, approver, approved FROM resume_job WHERE job = $1 ORDER BY created_at ASC LIMIT 1", + ) + .bind(job_id) + .fetch_optional(db) + .await?; + + let approval_result = if let Some((value, approver, approved)) = resume_row { + serde_json::json!({ + "value": serde_json::from_str::(value.get()).unwrap_or(Value::Null), + "approver": approver.unwrap_or_else(|| "anonymous".to_string()), + "approved": approved, + }) + } else { + serde_json::json!({ + "value": null, + "approver": null, + "approved": false, + }) + }; + checkpoint + .completed_steps + .insert(approval_key.clone(), approval_result); + checkpoint.pending_steps = None; + save_checkpoint(db, job_id, &checkpoint).await?; + + tracing::info!( + job_id = %job_id, + approval_key = %approval_key, + "WAC v2 injected approval result into checkpoint" + ); + } + Some("sleep") => { + let sleep_key = checkpoint + .pending_steps + .as_ref() + .and_then(|p| p.keys.first().cloned()) + .unwrap_or_default(); + + checkpoint + .completed_steps + .insert(sleep_key.clone(), Value::Bool(true)); + checkpoint.pending_steps = None; + save_checkpoint(db, job_id, &checkpoint).await?; + + tracing::info!( + job_id = %job_id, + sleep_key = %sleep_key, + "WAC v2 resumed from sleep" + ); + } + _ => {} + } + + Ok(checkpoint) +} + +/// Detect WAC v2 patterns in TypeScript/Bun code. +/// Checks for `import ... from "windmill-client"` containing workflow/task, +/// skipping comment lines. +pub fn is_wac_v2_ts(code: &str) -> bool { + let mut has_wac_import = false; + let mut has_workflow = false; + let mut has_task = false; + for line in code.lines() { + let trimmed = line.trim(); + if trimmed.starts_with("//") { + continue; + } + if trimmed.contains("windmill-client") + && (trimmed.starts_with("import") || trimmed.starts_with("from")) + { + has_wac_import = true; + if trimmed.contains("workflow") { + has_workflow = true; + } + if trimmed.contains("task") { + has_task = true; + } + } + if trimmed.contains("export") && trimmed.contains("workflow(") { + has_workflow = true; + } + } + has_wac_import && has_workflow && has_task +} + +/// Detect WAC v2 patterns in Python code. +/// Checks for `@workflow` decorator and `@task` decorator with wmill import, +/// skipping comment lines. +pub fn is_wac_v2_py(code: &str) -> bool { + let mut has_wmill_import = false; + let mut has_workflow_decorator = false; + let mut has_task_decorator = false; + for line in code.lines() { + let trimmed = line.trim(); + if trimmed.starts_with('#') { + continue; + } + if trimmed.starts_with("import wmill") || trimmed.starts_with("from wmill") { + has_wmill_import = true; + } + if trimmed == "@workflow" || trimmed.starts_with("@workflow(") { + has_workflow_decorator = true; + } + if trimmed == "@task" || trimmed.starts_with("@task(") { + has_task_decorator = true; + } + } + has_wmill_import && has_workflow_decorator && has_task_decorator +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index c354d34ee2..2a9026b82e 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -37,7 +37,7 @@ use windmill_common::{ utils::{create_directory_async, WarnAfterExt}, worker::{ make_pull_query, write_file, Connection, HttpClient, MAX_TIMEOUT, - MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS, ROOT_CACHE_DIR, ROOT_CACHE_NOMOUNT_DIR, TMP_DIR, + MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS, ROOT_CACHE_DIR, ROOT_CACHE_NOMOUNT_DIR, WINDMILL_DIR, }, worker_group_job_stats::JobStatsMap, KillpillSender, @@ -47,7 +47,6 @@ use windmill_common::{ use windmill_common::ee_oss::LICENSE_KEY_VALID; use anyhow::Result; -use const_format::concatcp; #[cfg(feature = "prometheus")] use prometheus::IntCounter; @@ -196,45 +195,47 @@ use windmill_common::bench::{benchmark_init, benchmark_verify, BenchmarkInfo, Be use windmill_common::add_time; -pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_10"); -pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_11"); -pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_12"); -pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_13"); +lazy_static::lazy_static! { + pub static ref PY310_CACHE_DIR: String = format!("{}python_3_10", *ROOT_CACHE_DIR); + pub static ref PY311_CACHE_DIR: String = format!("{}python_3_11", *ROOT_CACHE_DIR); + pub static ref PY312_CACHE_DIR: String = format!("{}python_3_12", *ROOT_CACHE_DIR); + pub static ref PY313_CACHE_DIR: String = format!("{}python_3_13", *ROOT_CACHE_DIR); -pub const TAR_JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/java"); + pub static ref TAR_JAVA_CACHE_DIR: String = format!("{}tar/java", *ROOT_CACHE_DIR); -pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); -pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_runtime"); -pub const TAR_PYBASE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar"); -pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); -pub const DENO_CACHE_DIR_DEPS: &str = concatcp!(ROOT_CACHE_DIR, "deno/deps"); -pub const DENO_CACHE_DIR_NPM: &str = concatcp!(ROOT_CACHE_DIR, "deno/npm"); + pub static ref UV_CACHE_DIR: String = format!("{}uv", *ROOT_CACHE_DIR); + pub static ref PY_INSTALL_DIR: String = format!("{}py_runtime", *ROOT_CACHE_DIR); + pub static ref TAR_PYBASE_CACHE_DIR: String = format!("{}tar", *ROOT_CACHE_DIR); + pub static ref DENO_CACHE_DIR: String = format!("{}deno", *ROOT_CACHE_DIR); + pub static ref DENO_CACHE_DIR_DEPS: String = format!("{}deno/deps", *ROOT_CACHE_DIR); + pub static ref DENO_CACHE_DIR_NPM: String = format!("{}deno/npm", *ROOT_CACHE_DIR); -pub const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go"); -pub const RUST_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "rust"); -pub const NU_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "nu"); -pub const CSHARP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "csharp"); + pub static ref GO_CACHE_DIR: String = format!("{}go", *ROOT_CACHE_DIR); + pub static ref RUST_CACHE_DIR: String = format!("{}rust", *ROOT_CACHE_DIR); + pub static ref NU_CACHE_DIR: String = format!("{}nu", *ROOT_CACHE_DIR); + pub static ref CSHARP_CACHE_DIR: String = format!("{}csharp", *ROOT_CACHE_DIR); -// Java -pub const JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "java"); -pub const COURSIER_CACHE_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/coursier-cache"); -pub const JAVA_REPOSITORY_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/repository"); -pub const JAVA_HOME_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/home"); + // Java + pub static ref JAVA_CACHE_DIR: String = format!("{}java", *ROOT_CACHE_DIR); + pub static ref COURSIER_CACHE_DIR: String = format!("{}/coursier-cache", *JAVA_CACHE_DIR); + pub static ref JAVA_REPOSITORY_DIR: String = format!("{}/repository", *JAVA_CACHE_DIR); + pub static ref JAVA_HOME_DIR: String = format!("{}/home", *JAVA_CACHE_DIR); -// Ruby -pub const RUBY_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "ruby"); + // Ruby + pub static ref RUBY_CACHE_DIR: String = format!("{}ruby", *ROOT_CACHE_DIR); -// for related places search: ADD_NEW_LANG -pub const BUN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "bun"); -pub const BUN_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "bun"); -pub const BUN_CODEBASE_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "script_bundle"); + // for related places search: ADD_NEW_LANG + pub static ref BUN_CACHE_DIR: String = format!("{}bun", *ROOT_CACHE_NOMOUNT_DIR); + pub static ref BUN_BUNDLE_CACHE_DIR: String = format!("{}bun", *ROOT_CACHE_DIR); + pub static ref BUN_CODEBASE_BUNDLE_CACHE_DIR: String = format!("{}script_bundle", *ROOT_CACHE_NOMOUNT_DIR); -pub const GO_BIN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "gobin"); -pub const POWERSHELL_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "powershell"); -pub const COMPOSER_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "composer"); + pub static ref GO_BIN_CACHE_DIR: String = format!("{}gobin", *ROOT_CACHE_DIR); + pub static ref POWERSHELL_CACHE_DIR: String = format!("{}powershell", *ROOT_CACHE_DIR); + pub static ref COMPOSER_CACHE_DIR: String = format!("{}composer", *ROOT_CACHE_DIR); -pub const TRACING_PROXY_CA_CERT_PATH: &str = - concatcp!(ROOT_CACHE_NOMOUNT_DIR, "tracing_proxy_ca.pem"); + pub static ref TRACING_PROXY_CA_CERT_PATH: String = + format!("{}tracing_proxy_ca.pem", *ROOT_CACHE_NOMOUNT_DIR); +} const NUM_SECS_PING: u64 = 5; const NUM_SECS_READINGS: u64 = 60; @@ -302,7 +303,7 @@ pub struct PowershellRepo { lazy_static::lazy_static! { - pub static ref SLEEP_QUEUE: u64 = std::env::var("SLEEP_QUEUE") + static ref SLEEP_QUEUE_BASE: u64 = std::env::var("SLEEP_QUEUE") .ok() .and_then(|x| x.parse::().ok()) .unwrap_or_else(|| { @@ -562,7 +563,16 @@ lazy_static::lazy_static! { pub static ref DOTNET_PATH: String = std::env::var("DOTNET_PATH").unwrap_or_else(|_| DOTNET_DEFAULT_PATH.to_string()); pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new()); - pub static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()); + pub static ref HOME_ENV: String = { + #[cfg(not(windows))] + { std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()) } + #[cfg(windows)] + { + std::env::var("HOME") + .or_else(|_| std::env::var("USERPROFILE")) + .unwrap_or_else(|_| std::env::temp_dir().to_string_lossy().to_string()) + } + }; pub static ref GIT_PATH: String = std::env::var("GIT_PATH").unwrap_or_else(|_| "/usr/bin/git".to_string()); pub static ref NODE_PATH: Option = std::env::var("NODE_PATH").ok(); @@ -637,6 +647,14 @@ lazy_static::lazy_static! { pub static ref FLOW_RUNNER_RUNNING: Mutex = Mutex::new(false); } +pub fn sleep_queue() -> u64 { + if NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed) { + 300 + } else { + *SLEEP_QUEUE_BASE + } +} + type Envs = Vec<(String, String)>; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -730,26 +748,69 @@ pub async fn is_otel_tracing_proxy_enabled_for_lang(lang: &ScriptLang) -> bool { } } +/// Get OTEL trace context environment variables for a job (TRACEPARENT, OTEL_TRACE_ID, OTEL_SPAN_ID). +/// Returns an empty vec when OTEL tracing is not enabled or on non-enterprise builds. +pub fn get_otel_context_envs(job_id: &uuid::Uuid) -> Vec<(&'static str, String)> { + #[cfg(all(feature = "private", feature = "enterprise"))] + if windmill_common::OTEL_TRACING_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { + let trace_id = format!("{:032x}", job_id.as_u128()); + let span_id = format!("{:016x}", job_id.as_u64_pair().1); + let traceparent = format!("00-{}-{}-01", trace_id, span_id); + return vec![ + ("TRACEPARENT", traceparent), + ("OTEL_TRACE_ID", trace_id), + ("OTEL_SPAN_ID", span_id), + ]; + } + let _ = job_id; + vec![] +} + /// Get proxy environment variables for job execution for a specific language. /// When OTEL tracing proxy is enabled for this language, routes all traffic through the proxy. /// Otherwise, uses the standard HTTP_PROXY/HTTPS_PROXY from environment. pub async fn get_proxy_envs_for_lang( lang: &ScriptLang, + job_id: &uuid::Uuid, + w_id: &str, + conn: &Connection, ) -> anyhow::Result> { + #[allow(unused_mut)] + let mut envs; #[cfg(all(feature = "private", feature = "enterprise"))] if is_otel_tracing_proxy_enabled_for_lang(lang).await { - return get_otel_tracing_proxy_envs().await; + envs = get_otel_tracing_proxy_envs(job_id, w_id, conn).await?; + } else { + envs = PROXY_ENVS.clone(); } - let _ = lang; - Ok(PROXY_ENVS.clone()) + #[cfg(not(all(feature = "private", feature = "enterprise")))] + { + let _ = (lang, w_id, conn); + envs = PROXY_ENVS.clone(); + } + envs.extend(get_otel_context_envs(job_id)); + Ok(envs) } #[cfg(all(feature = "private", feature = "enterprise"))] -async fn get_otel_tracing_proxy_envs() -> anyhow::Result> { - let port = crate::otel_tracing_proxy_ee::TRACING_PROXY_PORT +async fn get_otel_tracing_proxy_envs( + job_id: &uuid::Uuid, + w_id: &str, + conn: &Connection, +) -> anyhow::Result> { + let port = match *crate::otel_tracing_proxy_ee::TRACING_PROXY_PORT .read() .await - .ok_or_else(|| anyhow::anyhow!("OTEL tracing proxy port not initialized"))?; + { + Some(p) => p, + None => { + let reason = "OTEL tracing proxy is enabled but not available (not initialized yet, or NUM_WORKERS > 1). \ + This job's HTTP requests will not be traced."; + tracing::warn!("{}", reason); + append_logs(job_id, w_id, format!("\n[warning] {reason}\n"), conn).await; + return Ok(PROXY_ENVS.clone()); + } + }; let proxy_url = format!("http://127.0.0.1:{}", port); Ok(vec![ ("HTTP_PROXY", proxy_url.clone()), @@ -1320,7 +1381,7 @@ fn start_interactive_worker_shell( { Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION) } - _ => Duration::from_millis(*SLEEP_QUEUE * 10), + _ => Duration::from_millis(sleep_queue() * 10), }; tokio::select! { _ = tokio::time::sleep(nap_time) => { @@ -1333,7 +1394,7 @@ fn start_interactive_worker_shell( Err(err) => { tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err); - tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE * 20)).await; + tokio::time::sleep(Duration::from_millis(sleep_queue() * 20)).await; } }; } @@ -1375,11 +1436,11 @@ pub async fn run_worker( let start_time = Instant::now(); - let worker_dir = format!("{TMP_DIR}/{worker_name}"); + let worker_dir = format!("{}/{worker_name}", *WINDMILL_DIR); tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir"); #[cfg(feature = "python")] - { + if !NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed) { let (conn, worker_name, hostname, worker_dir) = ( conn.clone(), worker_name.clone(), @@ -2646,7 +2707,7 @@ pub async fn run_worker( None }; - tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE)).await; + tokio::time::sleep(Duration::from_millis(sleep_queue())).await; #[cfg(feature = "benchmark")] { @@ -2667,7 +2728,7 @@ pub async fn run_worker( } Err(err) => { tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err); - tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE * 5)).await; + tokio::time::sleep(Duration::from_millis(sleep_queue() * 5)).await; } }; } @@ -3475,6 +3536,13 @@ pub async fn handle_queued_job( { return Ok(false); } + if result + .as_ref() + .is_err_and(|err| matches!(err, &Error::WacSuspended(_))) + { + // WAC v2 job suspended while waiting for child jobs — don't complete it + return Ok(true); + } process_result( cjob, result.map(|x| Arc::new(x)), @@ -3873,7 +3941,7 @@ pub async fn run_language_executor( run_inline: bool, ) -> error::Result> { if language == Some(ScriptLang::Postgresql) { - return do_postgresql( + return Box::pin(do_postgresql( job, &client, &code, @@ -3885,7 +3953,7 @@ pub async fn run_language_executor( occupancy_metrics, parent_runnable_path, run_inline, - ) + )) .await; } else if language == Some(ScriptLang::Mysql) { #[cfg(not(feature = "mysql"))] @@ -3900,7 +3968,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_mysql( + return Box::pin(do_mysql( job, &client, &code, @@ -3911,7 +3979,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::Bigquery) { @@ -3937,7 +4005,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_bigquery( + return Box::pin(do_bigquery( job, &client, &code, @@ -3948,7 +4016,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::Snowflake) { @@ -3966,7 +4034,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_snowflake( + return Box::pin(do_snowflake( job, &client, &code, @@ -3977,7 +4045,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::Mssql) { @@ -4003,7 +4071,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_mssql( + return Box::pin(do_mssql( job, &client, &code, @@ -4014,7 +4082,7 @@ pub async fn run_language_executor( occupancy_metrics, job_dir, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::OracleDB) { @@ -4040,7 +4108,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_oracledb( + return Box::pin(do_oracledb( job, &client, &code, @@ -4051,7 +4119,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::DuckDb) { @@ -4065,7 +4133,7 @@ pub async fn run_language_executor( #[cfg(feature = "duckdb")] { - return do_duckdb( + return Box::pin(do_duckdb( job, &client, &code, @@ -4077,7 +4145,7 @@ pub async fn run_language_executor( occupancy_metrics, parent_runnable_path, run_inline, - ) + )) .await; } } else if language == Some(ScriptLang::Graphql) { @@ -4086,7 +4154,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_graphql( + return Box::pin(do_graphql( job, &client, &code, @@ -4095,7 +4163,7 @@ pub async fn run_language_executor( canceled_by, worker_name, occupancy_metrics, - ) + )) .await; } else if language == Some(ScriptLang::Nativets) { if run_inline { @@ -4122,7 +4190,7 @@ pub async fn run_language_executor( .collect::>() .join("\n")); - let result = do_nativets( + let result = Box::pin(do_nativets( job, &client, env_code, @@ -4133,7 +4201,7 @@ pub async fn run_language_executor( worker_name, occupancy_metrics, has_stream, - ) + )) .await?; return Ok(result); } @@ -4151,7 +4219,8 @@ pub async fn run_language_executor( job.id ); - let shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) { + #[allow(unused_mut)] + let mut shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) { let folder = if job.script_lang == Some(ScriptLang::Go) { "/go" } else { @@ -4173,7 +4242,8 @@ mount {{ // println!("handle lang job {:?}", SystemTime::now()); - let envs = build_envs(envs.as_ref())?; + #[allow(unused_mut)] + let mut envs = build_envs(envs.as_ref())?; let Some(language) = language else { return Err(Error::ExecutionErr( @@ -4209,6 +4279,106 @@ mount {{ } } + // Volume mount setup (requires workspace S3 storage; CE has file count/size limits) + #[cfg(feature = "parquet")] + let volume_mounts = { + let comment_prefix = match language { + ScriptLang::Python3 + | ScriptLang::Bash + | ScriptLang::Powershell + | ScriptLang::Ansible + | ScriptLang::Ruby => "#", + ScriptLang::Deno + | ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Go => "//", + _ => "", + }; + let raw_mounts = windmill_worker_volumes::parse_volume_annotations(&code, comment_prefix); + let args_ref = job.args.as_ref().map(|a| &**a); + let mut interpolated = Vec::new(); + for mut v in raw_mounts { + v.name = windmill_worker_volumes::interpolate_volume_name( + &v.name, + args_ref, + &job.workspace_id, + ); + if let Err(e) = windmill_worker_volumes::validate_volume_name(&v.name) { + return Err(Error::ExecutionErr(e)); + } + if let Err(e) = windmill_worker_volumes::validate_volume_target(&v.target) { + return Err(Error::ExecutionErr(e)); + } + interpolated.push(v); + } + if let Err(e) = windmill_worker_volumes::validate_volume_mounts(&interpolated) { + return Err(Error::ExecutionErr(e)); + } + interpolated + }; + + #[cfg(feature = "parquet")] + let mut volume_setup = crate::volume_oss::VolumeSetupResult { + states: Vec::new(), + writable: Vec::new(), + client: None, + lease_renewal: crate::volume_oss::LeaseRenewalGuard(None), + }; + + #[cfg(feature = "parquet")] + if !volume_mounts.is_empty() { + let vol_summary: Vec = volume_mounts + .iter() + .map(|v| format!("'{}' -> {}", v.name, v.target)) + .collect(); + append_logs( + &job.id, + &job.workspace_id, + format!( + "\n--- VOLUME MOUNTS ---\nPulling {} volume(s): {}\n", + volume_mounts.len(), + vol_summary.join(", "), + ), + conn, + ) + .await; + + if let Connection::Sql(db) = conn { + volume_setup = crate::volume_oss::setup_volumes_sql_worker( + &volume_mounts, + db, + &job.workspace_id, + job.id, + &job.permissioned_as, + worker_name, + job_dir, + client, + conn, + language, + &mut envs, + &mut shared_mount, + ) + .await?; + } else if let Connection::Http(http) = conn { + volume_setup = crate::volume_oss::setup_volumes_http_worker( + &volume_mounts, + http, + &job.workspace_id, + job.id, + &job.permissioned_as, + &job.canceled_by, + worker_name, + job_dir, + conn, + language, + &mut envs, + &mut shared_mount, + ) + .await?; + } + } + // Box::pin all language handlers to prevent large match enum on stack let result: error::Result> = match language { ScriptLang::Python3 => { @@ -4620,6 +4790,62 @@ mount {{ // for related places search: ADD_NEW_LANG _ => panic!("unreachable, language is not supported: {language:#?}"), }; + // Volume sync-back and lease release + #[cfg(feature = "parquet")] + if !volume_setup.states.is_empty() { + // Stop lease renewal before sync-back + volume_setup.lease_renewal.0.take().map(|h| h.abort()); + + if let Some(ref vol_client) = volume_setup.client { + if let Connection::Sql(db) = conn { + crate::volume_oss::sync_volumes_sql_worker( + &volume_setup.states, + &volume_setup.writable, + vol_client, + db, + &job.workspace_id, + job.id, + worker_name, + conn, + result.is_ok(), + ) + .await; + } + } + + if let Connection::Http(http) = conn { + crate::volume_oss::sync_volumes_http_worker( + &volume_setup.states, + &volume_setup.writable, + http, + &job.workspace_id, + job.id, + worker_name, + conn, + result.is_ok(), + ) + .await; + } + + // Clean up absolute-path symlinks created by setup_volume_mount_paths + if !is_sandboxing_enabled() { + #[allow(unused_variables)] // state is only used on unix + for state in &volume_setup.states { + #[cfg(unix)] + if state.mount.target.starts_with('/') { + let target_path = std::path::Path::new(&state.mount.target); + if target_path + .symlink_metadata() + .map(|m| m.file_type().is_symlink()) + .unwrap_or(false) + { + std::fs::remove_file(target_path).ok(); + } + } + } + } + } + tracing::info!( workspace_id = %job.workspace_id, is_ok = result.is_ok(), diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index d0b7fc7a58..dfc2bbeed9 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -11,7 +11,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; -use crate::common::{cached_result_path, get_root_job_id, save_in_cache}; +use crate::common::{cached_result_path, get_root_job_id, save_in_cache, transform_json}; use crate::js_eval::{eval_timeout, IdContext}; use crate::worker_utils::get_tag_and_concurrency; use crate::{ @@ -53,7 +53,7 @@ use windmill_common::runnable_settings::{ use windmill_common::scripts::{ScriptHash, ScriptRunnableSettingsInline}; use windmill_common::users::username_to_permissioned_as; use windmill_common::utils::WarnAfterExt; -use windmill_common::worker::to_raw_value; +use windmill_common::worker::{to_raw_value, Connection}; use windmill_common::{ add_time, get_latest_flow_version_info_for_path, get_script_info_for_hash, FlowVersionInfo, ScriptHashInfo, DB, @@ -865,7 +865,7 @@ pub async fn update_flow_status_after_job_completion_internal( .and_then(|x| x.stop_after_all_iters_if.as_ref()) { let args = from_result_to_args(args.as_ref().await.get_ref())?; - evaluate_stop_after_all_iters_if( + if let Err(e) = evaluate_stop_after_all_iters_if( db, stop_after_all_iters_if, module_status, @@ -879,7 +879,16 @@ pub async fn update_flow_status_after_job_completion_internal( flow, &old_status, ) - .await?; + .await + { + tracing::error!("error evaluating stop_after_all_iters_if: {e:#}"); + stop_early = true; + skip_if_stop_early = false; + stop_early_err_msg = Some(format!( + "Error evaluating stop_after_all_iters_if expression `{}`: {e:#}", + stop_after_all_iters_if.expr + )); + } } let new_status = if @@ -1074,7 +1083,7 @@ pub async fn update_flow_status_after_job_completion_internal( { let args = from_result_to_args(args.as_ref().await.get_ref())?; - evaluate_stop_after_all_iters_if( + if let Err(e) = evaluate_stop_after_all_iters_if( db, stop_after_all_iters_if, module_status, @@ -1088,7 +1097,15 @@ pub async fn update_flow_status_after_job_completion_internal( flow, &old_status, ) - .await?; + .await + { + stop_early = true; + skip_if_stop_early = false; + stop_early_err_msg = Some(format!( + "Error evaluating stop_after_all_iters_if expression `{}`: {e:#}", + stop_after_all_iters_if.expr + )); + } } } @@ -1436,10 +1453,12 @@ pub async fn update_flow_status_after_job_completion_internal( } }; - // When debouncing is applied, store the flow's debouncing settings in - // the runnable_settings_handle so that maybe_apply_debouncing can find - // them after re-pull and perform argument accumulation. - let new_runnable_settings_handle: Option = if scheduled_for.is_some() { + // Store the flow's debouncing settings in the runnable_settings_handle + // so that maybe_apply_debouncing can find them after pull and perform + // argument accumulation. This is needed both when debounced (CanDebounce) + // and when firing immediately (MaxCountExceeded), since accumulation + // happens at pull time in both cases. + let new_runnable_settings_handle: Option = if has_debouncing { let debouncing_hash = flow_value .debouncing_settings .insert_cached(db) @@ -1705,8 +1724,8 @@ pub async fn update_flow_status_after_job_completion_internal( chat_ai_info.conversation_id, ) .await?; - let duration = if success { - let (_, duration) = add_completed_job( + let (duration, wac_job_ids) = if success { + let (_, duration, wac_job_ids) = add_completed_job( db, &cflow_job, true, @@ -1720,9 +1739,9 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + (duration, wac_job_ids) } else { - let (_, duration) = add_completed_job( + let (_, duration, wac_job_ids) = add_completed_job( db, &cflow_job, false, @@ -1740,11 +1759,30 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + (duration, wac_job_ids) }; flow_job_duration = flow_job .started_at .map(|x| FlowJobDuration { started_at: x, duration_ms: duration }); + + // If this flow is a WAC child (not a flow step, has parent), + // notify the WAC parent of completion. + if !flow_job.is_flow_step() { + if let Some(parent_job) = flow_job.parent_job { + if let Some(job_ids) = wac_job_ids { + let _ = crate::result_processor::handle_wac_child_completion( + db, + &flow_job.id, + parent_job, + &flow_job.workspace_id, + nresult.clone(), + success, + job_ids, + ) + .await; + } + } + } } true } else { @@ -2245,6 +2283,35 @@ pub async fn handle_flow( killpill_rx: &tokio::sync::broadcast::Receiver<()>, ) -> anyhow::Result<()> { let flow = flow_data.value(); + + // Resolve $var: and $res: references in flow_env. + // We resolve into a separate variable to avoid cloning the entire FlowValue + // (which includes modules, failure_module, etc.) just to replace flow_env. + let resolved_env; + let flow_env = if let Some(ref env) = flow.flow_env { + match transform_json( + client, + &flow_job.workspace_id, + env, + &flow_job, + &Connection::Sql(db.clone()), + ) + .await + { + Ok(Some(resolved)) => { + resolved_env = resolved; + Some(&resolved_env) + } + Ok(None) => flow.flow_env.as_ref(), + Err(e) => { + tracing::warn!("Failed to resolve flow_env references: {e}"); + flow.flow_env.as_ref() + } + } + } else { + None + }; + let status = flow_job .parse_flow_status() .with_context(|| "Unable to parse flow status")?; @@ -2348,6 +2415,7 @@ pub async fn handle_flow( flow_job, status, flow, + flow_env, db, client, last_result.clone(), @@ -2448,6 +2516,7 @@ async fn push_next_flow_job( flow_job: Arc, mut status: FlowStatus, flow: &FlowValue, + flow_env: Option<&HashMap>>, db: &sqlx::Pool, client: &AuthedClient, last_job_result: Option>>, @@ -2580,7 +2649,7 @@ async fn push_next_flow_job( let skip = compute_bool_from_expr( &skip_expr, arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, Arc::new(to_raw_value(&json!("{}"))), None, None, @@ -2705,7 +2774,7 @@ async fn push_next_flow_job( expr.to_string(), context, Some(arc_flow_job_args.clone()), - flow.flow_env.as_ref(), + flow_env, None, None, None @@ -2966,7 +3035,7 @@ async fn push_next_flow_job( &input_transform, arc_last_job_result.clone(), Some(arc_flow_job_args.clone()), - flow.flow_env.as_ref(), + flow_env, Some(client), None, ) @@ -3004,7 +3073,7 @@ async fn push_next_flow_job( &status.retry, arc_last_job_result.clone(), arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, Some(client), ) .await? @@ -3092,7 +3161,7 @@ async fn push_next_flow_job( compute_bool_from_expr( &skip_if.expr, arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), None, Some(&idcontext), @@ -3182,7 +3251,7 @@ async fn push_next_flow_job( }; transform_input( arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), input_transforms, resumes.clone(), @@ -3209,7 +3278,7 @@ async fn push_next_flow_job( let next_flow_transform = compute_next_flow_transform( arc_flow_job_args.clone(), arc_last_job_result.clone(), - flow.flow_env.as_ref(), + flow_env, &flow_job, &flow, transform_context, @@ -3373,7 +3442,7 @@ async fn push_next_flow_job( let ctx = get_transform_context(&flow_job, "", &status); let ti = transform_input( Marc::new(args), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), input_transforms, resumes.clone(), @@ -3428,7 +3497,7 @@ async fn push_next_flow_job( let ctx = get_transform_context(&flow_job, &previous_id, &status); let ti = transform_input( Marc::new(hm), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), input_transforms, resumes.clone(), @@ -3546,7 +3615,7 @@ async fn push_next_flow_job( timeout_transform, arc_last_job_result.clone(), Some(arc_flow_job_args.clone()), - flow.flow_env.as_ref(), + flow_env, Some(client), Some(&ctx), ) @@ -3625,7 +3694,7 @@ async fn push_next_flow_job( parallelism_transform, arc_last_job_result.clone(), Some(arc_flow_job_args.clone()), - flow.flow_env.as_ref(), + flow_env, Some(client), Some(&ctx), ) @@ -4461,7 +4530,7 @@ async fn compute_next_flow_transform( let pred = compute_bool_from_expr( &b.expr, arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), None, Some(&idcontext), diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 41204c956a..207d9ec7b8 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.648.0"; +export const VERSION = "v1.654.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/bootstrap/script_bootstrap.ts b/cli/bootstrap/script_bootstrap.ts index 5335e1741c..89d98a1927 100644 --- a/cli/bootstrap/script_bootstrap.ts +++ b/cli/bootstrap/script_bootstrap.ts @@ -39,7 +39,10 @@ export const scriptBootstrapCode = { } `, - bun: `export async function main() { + bun: `// there are multiple modes to add as header: //nobundling //native //npm //nodejs +// https://www.windmill.dev/docs/getting_started/scripts_quickstart/typescript#modes + +export async function main() { return "Hello world"; } `, diff --git a/cli/dev.nu b/cli/dev.nu new file mode 100644 index 0000000000..91a0faa8e3 --- /dev/null +++ b/cli/dev.nu @@ -0,0 +1,59 @@ +#!/usr/bin/env nu + +let cli_cache = "/tmp/windmill/cache_nomount/bun/" +let bundle_cache = "/tmp/windmill/cache/bun/" + +# Clean CLI package cache +def "main clean" [] { + rm -rf ($cli_cache ++ "windmill-cli@*") + rm -rf ($cli_cache ++ "windmill-cli/") + print "Cleaned CLI cache" +} + +# Clear bundle cache (forces hub scripts to re-bundle) +def "main clear-bundles" [] { + rm -rf ($bundle_cache ++ "*") + print "Cleared bundle cache" +} + +# Patch CLI cache with local build +def "main patch" [] { + print "Patching CLI cache..." + + let versions = (ls $cli_cache | where name =~ "windmill-cli@" | get name) + + if ($versions | is-empty) { + print "No CLI versions found in cache" + return + } + + for path in $versions { + rm -rf ($path ++ "/esm") + ^cp -r npm/esm ($path ++ "/esm") + ^cp npm/package.json ($path ++ "/package.json") + print $"Patched ($path | path basename)" + } + + print "Done!" +} + +# Build CLI, patch cache, and clear bundles +def main [ + --patch(-p) # Only patch existing cache (skip build) + --clean(-c) # Clean CLI cache first +] { + if $clean { + main clean + } + + if $patch { + main patch + } else { + print "Building CLI..." + bun run build + main patch + } + + # Always clear bundle cache so hub scripts use patched CLI + main clear-bundles +} diff --git a/cli/src/commands/gitsync-settings/converter.ts b/cli/src/commands/gitsync-settings/converter.ts index 89600fd5d1..6861603bb4 100644 --- a/cli/src/commands/gitsync-settings/converter.ts +++ b/cli/src/commands/gitsync-settings/converter.ts @@ -33,6 +33,7 @@ export class GitSyncSettingsConverter { includeGroups: includeTypes.includes("group"), includeSettings: includeTypes.includes("settings"), includeKey: includeTypes.includes("key"), + skipWorkspaceDependencies: !includeTypes.includes("workspacedependencies"), }; // Only include extraIncludes if it has content @@ -61,6 +62,7 @@ export class GitSyncSettingsConverter { if (opts.includeGroups) includeTypes.push("group"); if (opts.includeSettings) includeTypes.push("settings"); if (opts.includeKey) includeTypes.push("key"); + if (!opts.skipWorkspaceDependencies) includeTypes.push("workspacedependencies"); const result: BackendGitSyncSettings = { include_path: opts.includes || [], @@ -99,6 +101,7 @@ export class GitSyncSettingsConverter { includeGroups: opts.includeGroups ?? false, includeSettings: opts.includeSettings ?? false, includeKey: opts.includeKey ?? false, + skipWorkspaceDependencies: opts.skipWorkspaceDependencies ?? false, }; } @@ -122,6 +125,7 @@ export class GitSyncSettingsConverter { includeGroups: opts.includeGroups, includeSettings: opts.includeSettings, includeKey: opts.includeKey, + skipWorkspaceDependencies: opts.skipWorkspaceDependencies, }; } diff --git a/cli/src/commands/gitsync-settings/types.ts b/cli/src/commands/gitsync-settings/types.ts index 36b51aedbc..7bebb1f1a1 100644 --- a/cli/src/commands/gitsync-settings/types.ts +++ b/cli/src/commands/gitsync-settings/types.ts @@ -37,6 +37,7 @@ export const GIT_SYNC_FIELDS = [ "includeGroups", "includeSettings", "includeKey", + "skipWorkspaceDependencies", ] as const; export type GitSyncField = typeof GIT_SYNC_FIELDS[number]; @@ -57,6 +58,7 @@ export const INCLUDE_TYPE_MAPPINGS = { group: "includeGroups", settings: "includeSettings", key: "includeKey", + workspacedependencies: "skipWorkspaceDependencies", } as const; // Write mode for branch-based configuration diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 2d72bc35cf..826c1de090 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -51,10 +51,16 @@ export async function downloadZip( ); if (!zipResponse.ok) { - console.log( - colors.red("Failed to request tarball from API " + zipResponse.statusText) - ); - throw new Error(await zipResponse.text()); + const body = await zipResponse.text(); + if (zipResponse.status === 404 || body.includes("no rows returned")) { + log.info(colors.red(`Workspace '${workspace.workspaceId}' not found on ${workspace.remote}. Please check your --workspace and try again.`)); + } else { + log.info(colors.red(`Failed to request tarball from API: ${zipResponse.status} ${zipResponse.statusText}`)); + if (body) { + log.info(colors.red(body)); + } + } + return process.exit(1); } else { log.debug(`Downloaded zip/tarball successfully`); } diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 6d4a359600..7d48a14848 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -2626,6 +2626,7 @@ export async function push( let [_basePath, changes] = queue.shift()!; const promise = (async () => { const alreadySynced: string[] = []; + const deletedVarsResPaths: string[] = []; const isRawApp = isRawAppFile(changes[0].path); if (isRawApp) { const deleteRawApp = changes.find( @@ -2870,12 +2871,23 @@ export async function push( name: change.path.split(SEP)[1], }); break; - case "resource": - await wmill.deleteResource({ - workspace: workspaceId, - path: removeSuffix(target, ".resource.json"), - }); + case "resource": { + const resourcePath = removeSuffix(target, ".resource.json"); + try { + await wmill.deleteResource({ + workspace: workspaceId, + path: resourcePath, + }); + } catch (e: any) { + if (e?.status === 404 && deletedVarsResPaths.includes(resourcePath)) { + log.debug(`Resource ${resourcePath} already deleted by linked variable`); + } else { + throw e; + } + } + deletedVarsResPaths.push(resourcePath); break; + } case "resource-type": await wmill.deleteResourceType({ workspace: workspaceId, @@ -3012,12 +3024,23 @@ export async function push( }); break; } - case "variable": - await wmill.deleteVariable({ - workspace: workspaceId, - path: removeSuffix(target, ".variable.json"), - }); + case "variable": { + const variablePath = removeSuffix(target, ".variable.json"); + try { + await wmill.deleteVariable({ + workspace: workspaceId, + path: variablePath, + }); + } catch (e: any) { + if (e?.status === 404 && deletedVarsResPaths.includes(variablePath)) { + log.debug(`Variable ${variablePath} already deleted by linked resource`); + } else { + throw e; + } + } + deletedVarsResPaths.push(variablePath); break; + } case "user": { const users = await wmill.listUsers({ workspace: workspaceId, diff --git a/cli/src/commands/workspace/workspace.ts b/cli/src/commands/workspace/workspace.ts index b70469073c..6124bd1e1c 100644 --- a/cli/src/commands/workspace/workspace.ts +++ b/cli/src/commands/workspace/workspace.ts @@ -408,7 +408,8 @@ async function remove(_opts: GlobalOptions, name: string) { async function whoami(_opts: GlobalOptions) { await requireLogin(_opts); - log.info(await wmill.globalWhoami()); + const whoamiInfo = await wmill.globalWhoami(); + log.info(JSON.stringify(whoamiInfo, null, 2)); const activeName = await getActiveWorkspaceName(_opts); log.info("Active: " + colors.green.bold(activeName || "none")); } diff --git a/cli/src/core/auth.ts b/cli/src/core/auth.ts index 0be7571320..037555a1a9 100644 --- a/cli/src/core/auth.ts +++ b/cli/src/core/auth.ts @@ -7,6 +7,7 @@ import { GlobalUserInfo } from "../../gen/types.gen.ts"; import { loginInteractive, tryGetLoginInfo } from "./login.ts"; import { GlobalOptions } from "../types.ts"; + /** * Main authentication function - moved from context.ts to break circular dependencies * This function maintains the original API signature from context.ts @@ -35,6 +36,14 @@ export async function requireLogin( throw new Error(`Network error: Could not connect to Windmill server at ${workspace.remote}`); } + // If the user explicitly provided credentials via flags, fail immediately + // rather than falling back to interactive login — they expect their explicit + // credentials to work and should fix them if they don't. + if (opts.token || opts.baseUrl) { + log.info(colors.red("Could not authenticate with the provided credentials. Please check your --token and --base-url and try again.")); + return process.exit(1); + } + log.info( "! Could not reach API given existing credentials. Attempting to reauth..." ); diff --git a/cli/src/core/context.ts b/cli/src/core/context.ts index 86307646bf..2b7a2266e2 100644 --- a/cli/src/core/context.ts +++ b/cli/src/core/context.ts @@ -3,6 +3,7 @@ import * as log from "./log.ts"; import { Select } from "@cliffy/prompt/select"; import { Confirm } from "@cliffy/prompt/confirm"; import { Input } from "@cliffy/prompt/input"; +import { Table } from "@cliffy/table"; import { loginInteractive } from "./login.ts"; import { GlobalOptions } from "../types.ts"; @@ -24,6 +25,7 @@ import { isGitRepository, } from "../utils/git.ts"; import { WM_FORK_PREFIX } from "./constants.ts"; +import { levenshteinDistance } from "@jsr/std__text/levenshtein-distance"; // Helper function to select from multiple matching profiles async function selectFromMultipleProfiles( @@ -467,6 +469,30 @@ export async function resolveWorkspace( `Found an active workspace \`${workspace.name}\` but the branch name indicates this is a forked workspace. Ignoring active workspace and trying to resolve the correct workspace from the branch name \`${branch}\`` ); } + } else if (opts.workspace) { + // --workspace was explicitly provided but not found — fail immediately + const workspaces = await allWorkspaces(opts.configDir); + const names = workspaces.map((w) => w.name); + let msg = `Workspace "${opts.workspace}" not found.`; + const suggestions = names + .map((n) => ({ name: n, dist: levenshteinDistance(opts.workspace!, n) })) + .filter((s) => s.dist <= 3) + .sort((a, b) => a.dist - b.dist) + .slice(0, 3); + if (suggestions.length > 0) { + msg += ` Did you mean: ${suggestions.map((s) => `"${s.name}"`).join(", ")}?`; + } + log.info(colors.red.bold(msg)); + if (workspaces.length > 0) { + log.info("\nAvailable workspaces:"); + new Table() + .header(["name", "remote", "workspace id"]) + .padding(2) + .border(true) + .body(workspaces.map((w) => [w.name, w.remote, w.workspaceId])) + .render(); + } + return process.exit(-1); } // Try branch-based resolution (medium priority) diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 3ffddc4837..0f7acce4a9 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -53,6 +53,9 @@ export interface SimplifiedSettings { mute_critical_alerts?: boolean; color?: string; operator_settings?: any; + slack_team_id?: string; + slack_name?: string; + slack_command_script?: string; } // Legacy settings interface for reading old settings.yaml files @@ -77,6 +80,9 @@ interface LegacySimplifiedSettings { mute_critical_alerts?: boolean; color?: string; operator_settings?: any; + slack_team_id?: string; + slack_name?: string; + slack_command_script?: string; } // Helper to convert legacy flat settings to new grouped format @@ -94,6 +100,9 @@ export function migrateToGroupedFormat(settings: any): SimplifiedSettings { if (settings.mute_critical_alerts !== undefined) result.mute_critical_alerts = settings.mute_critical_alerts; if (settings.color !== undefined) result.color = settings.color; if (settings.operator_settings !== undefined) result.operator_settings = settings.operator_settings; + if (settings.slack_team_id !== undefined) result.slack_team_id = settings.slack_team_id; + if (settings.slack_name !== undefined) result.slack_name = settings.slack_name; + if (settings.slack_command_script !== undefined) result.slack_command_script = settings.slack_command_script; // Handle auto_invite: check if already grouped or needs migration if (settings.auto_invite && typeof settings.auto_invite === "object") { @@ -183,12 +192,18 @@ export async function pushWorkspaceSettings( mute_critical_alerts: remoteSettings.mute_critical_alerts, color: remoteSettings.color, operator_settings: remoteSettings.operator_settings, + slack_team_id: remoteSettings.slack_team_id, + slack_name: remoteSettings.slack_name, + slack_command_script: remoteSettings.slack_command_script, }; } catch (err) { throw new Error(`Failed to get workspace settings: ${err}`); } - if (isSuperset(localSettings, settings)) { + // Exclude read-only fields from comparison (slack_team_id and slack_name are set via OAuth only) + const { slack_team_id: _lst, slack_name: _lsn, ...comparableLocal } = localSettings; + const { slack_team_id: _rst, slack_name: _rsn, ...comparableRemote } = settings; + if (isSuperset(comparableLocal, comparableRemote)) { log.debug(`Workspace settings are up to date`); return; } @@ -366,6 +381,16 @@ export async function pushWorkspaceSettings( requestBody: localSettings.operator_settings, }); } + + if (localSettings.slack_command_script != settings.slack_command_script) { + log.debug(`Updating slack command script...`); + await wmill.editSlackCommand({ + workspace, + requestBody: { + slack_command_script: localSettings.slack_command_script, + }, + }); + } } export async function pushWorkspaceKey( diff --git a/cli/src/guidance/skills.ts b/cli/src/guidance/skills.ts index 4a16eb2f32..34e288c7d7 100644 --- a/cli/src/guidance/skills.ts +++ b/cli/src/guidance/skills.ts @@ -7,25 +7,25 @@ export interface SkillMetadata { } export const SKILLS: SkillMetadata[] = [ - { name: "write-script-python3", description: "MUST use when writing Python scripts.", languageKey: "python3" }, + { name: "write-script-go", description: "MUST use when writing Go scripts.", languageKey: "go" }, + { name: "write-script-java", description: "MUST use when writing Java scripts.", languageKey: "java" }, + { name: "write-script-graphql", description: "MUST use when writing GraphQL queries.", languageKey: "graphql" }, + { name: "write-script-rust", description: "MUST use when writing Rust scripts.", languageKey: "rust" }, + { name: "write-script-bunnative", description: "MUST use when writing Bun Native scripts.", languageKey: "bunnative" }, + { name: "write-script-postgresql", description: "MUST use when writing PostgreSQL queries.", languageKey: "postgresql" }, + { name: "write-script-php", description: "MUST use when writing PHP scripts.", languageKey: "php" }, + { name: "write-script-bigquery", description: "MUST use when writing BigQuery queries.", languageKey: "bigquery" }, { name: "write-script-bun", description: "MUST use when writing Bun/TypeScript scripts.", languageKey: "bun" }, + { name: "write-script-csharp", description: "MUST use when writing C# scripts.", languageKey: "csharp" }, + { name: "write-script-mssql", description: "MUST use when writing MS SQL Server queries.", languageKey: "mssql" }, + { name: "write-script-deno", description: "MUST use when writing Deno/TypeScript scripts.", languageKey: "deno" }, { name: "write-script-mysql", description: "MUST use when writing MySQL queries.", languageKey: "mysql" }, { name: "write-script-powershell", description: "MUST use when writing PowerShell scripts.", languageKey: "powershell" }, { name: "write-script-snowflake", description: "MUST use when writing Snowflake queries.", languageKey: "snowflake" }, - { name: "write-script-go", description: "MUST use when writing Go scripts.", languageKey: "go" }, - { name: "write-script-deno", description: "MUST use when writing Deno/TypeScript scripts.", languageKey: "deno" }, - { name: "write-script-bash", description: "MUST use when writing Bash scripts.", languageKey: "bash" }, - { name: "write-script-bunnative", description: "MUST use when writing Bun Native scripts.", languageKey: "bunnative" }, + { name: "write-script-python3", description: "MUST use when writing Python scripts.", languageKey: "python3" }, { name: "write-script-duckdb", description: "MUST use when writing DuckDB queries.", languageKey: "duckdb" }, + { name: "write-script-bash", description: "MUST use when writing Bash scripts.", languageKey: "bash" }, { name: "write-script-nativets", description: "MUST use when writing Native TypeScript scripts.", languageKey: "nativets" }, - { name: "write-script-bigquery", description: "MUST use when writing BigQuery queries.", languageKey: "bigquery" }, - { name: "write-script-rust", description: "MUST use when writing Rust scripts.", languageKey: "rust" }, - { name: "write-script-php", description: "MUST use when writing PHP scripts.", languageKey: "php" }, - { name: "write-script-mssql", description: "MUST use when writing MS SQL Server queries.", languageKey: "mssql" }, - { name: "write-script-postgresql", description: "MUST use when writing PostgreSQL queries.", languageKey: "postgresql" }, - { name: "write-script-graphql", description: "MUST use when writing GraphQL queries.", languageKey: "graphql" }, - { name: "write-script-csharp", description: "MUST use when writing C# scripts.", languageKey: "csharp" }, - { name: "write-script-java", description: "MUST use when writing Java scripts.", languageKey: "java" }, { name: "write-flow", description: "MUST use when creating flows." }, { name: "raw-app", description: "MUST use when creating raw apps." }, { name: "triggers", description: "MUST use when configuring triggers." }, @@ -36,6 +36,2562 @@ export const SKILLS: SkillMetadata[] = [ // Skill content for each skill (loaded inline for bundling) export const SKILL_CONTENT: Record = { + "write-script-go": `--- +name: write-script-go +description: MUST use when writing Go scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Go + +## Structure + +The file package must be \`inner\` and export a function called \`main\`: + +\`\`\`go +package inner + +func main(param1 string, param2 int) (map[string]interface{}, error) { + return map[string]interface{}{ + "result": param1, + "count": param2, + }, nil +} +\`\`\` + +**Important:** +- Package must be \`inner\` +- Return type must be \`({return_type}, error)\` +- Function name is \`main\` (lowercase) + +## Return Types + +The return type can be any Go type that can be serialized to JSON: + +\`\`\`go +package inner + +type Result struct { + Name string \`json:"name"\` + Count int \`json:"count"\` +} + +func main(name string, count int) (Result, error) { + return Result{ + Name: name, + Count: count, + }, nil +} +\`\`\` + +## Error Handling + +Return errors as the second return value: + +\`\`\`go +package inner + +import "errors" + +func main(value int) (string, error) { + if value < 0 { + return "", errors.New("value must be positive") + } + return "success", nil +} +\`\`\` +`, + "write-script-java": `--- +name: write-script-java +description: MUST use when writing Java scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Java + +The script must contain a Main public class with a \`public static main()\` method: + +\`\`\`java +public class Main { + public static Object main(String name, int count) { + java.util.Map result = new java.util.HashMap<>(); + result.put("name", name); + result.put("count", count); + return result; + } +} +\`\`\` + +**Important:** +- Class must be named \`Main\` +- Method must be \`public static Object main(...)\` +- Return type is \`Object\` or \`void\` + +## Maven Dependencies + +Add dependencies using comments at the top: + +\`\`\`java +//requirements: +//com.google.code.gson:gson:2.10.1 +//org.apache.httpcomponents:httpclient:4.5.14 + +import com.google.gson.Gson; + +public class Main { + public static Object main(String input) { + Gson gson = new Gson(); + return gson.fromJson(input, Object.class); + } +} +\`\`\` +`, + "write-script-graphql": `--- +name: write-script-graphql +description: MUST use when writing GraphQL queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# GraphQL + +## Structure + +Write GraphQL queries or mutations. Arguments can be added as query parameters: + +\`\`\`graphql +query GetUser($id: ID!) { + user(id: $id) { + id + name + email + } +} +\`\`\` + +## Variables + +Variables are passed as script arguments and automatically bound to the query: + +\`\`\`graphql +query SearchProducts($query: String!, $limit: Int = 10) { + products(search: $query, first: $limit) { + edges { + node { + id + name + price + } + } + } +} +\`\`\` + +## Mutations + +\`\`\`graphql +mutation CreateUser($input: CreateUserInput!) { + createUser(input: $input) { + id + name + createdAt + } +} +\`\`\` +`, + "write-script-rust": `--- +name: write-script-rust +description: MUST use when writing Rust scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Rust + +## Structure + +The script must contain a function called \`main\` with proper return type: + +\`\`\`rust +use anyhow::anyhow; +use serde::Serialize; + +#[derive(Serialize, Debug)] +struct ReturnType { + result: String, + count: i32, +} + +fn main(param1: String, param2: i32) -> anyhow::Result { + Ok(ReturnType { + result: param1, + count: param2, + }) +} +\`\`\` + +**Important:** +- Arguments should be owned types +- Return type must be serializable (\`#[derive(Serialize)]\`) +- Return type is \`anyhow::Result\` + +## Dependencies + +Packages must be specified with a partial cargo.toml at the beginning of the script: + +\`\`\`rust +//! \`\`\`cargo +//! [dependencies] +//! anyhow = "1.0.86" +//! reqwest = { version = "0.11", features = ["json"] } +//! tokio = { version = "1", features = ["full"] } +//! \`\`\` + +use anyhow::anyhow; +// ... rest of the code +\`\`\` + +**Note:** Serde is already included, no need to add it again. + +## Async Functions + +If you need to handle async functions (e.g., using tokio), keep the main function sync and create the runtime inside: + +\`\`\`rust +//! \`\`\`cargo +//! [dependencies] +//! anyhow = "1.0.86" +//! tokio = { version = "1", features = ["full"] } +//! reqwest = { version = "0.11", features = ["json"] } +//! \`\`\` + +use anyhow::anyhow; +use serde::Serialize; + +#[derive(Serialize, Debug)] +struct Response { + data: String, +} + +fn main(url: String) -> anyhow::Result { + let rt = tokio::runtime::Runtime::new()?; + rt.block_on(async { + let resp = reqwest::get(&url).await?.text().await?; + Ok(Response { data: resp }) + }) +} +\`\`\` +`, + "write-script-bunnative": `--- +name: write-script-bunnative +description: MUST use when writing Bun Native scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# TypeScript (Bun Native) + +Native TypeScript execution with fetch only - no external imports allowed. + +## Structure + +Export a single **async** function called \`main\`: + +\`\`\`typescript +export async function main(param1: string, param2: number) { + // Your code here + return { result: param1, count: param2 }; +} +\`\`\` + +Do not call the main function. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +Use the \`RT\` namespace for resource types: + +\`\`\`typescript +export async function main(stripe: RT.Stripe) { + // stripe contains API key and config from the resource +} +\`\`\` + +Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + +Before using a resource type, check the \`rt.d.ts\` file in the project root to see all available resource types and their fields. This file is generated by \`wmill resource-type generate-namespace\`. + +## Imports + +**No imports allowed.** Use the globally available \`fetch\` function: + +\`\`\`typescript +export async function main(url: string) { + const response = await fetch(url); + return await response.json(); +} +\`\`\` + +## Windmill Client + +The windmill client is not available in native TypeScript mode. Use fetch to call APIs directly. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: + +\`\`\`typescript +type Event = { + kind: + | "webhook" + | "http" + | "websocket" + | "kafka" + | "email" + | "nats" + | "postgres" + | "sqs" + | "mqtt" + | "gcp"; + body: any; + headers: Record; + query: Record; +}; + +export async function preprocessor(event: Event) { + return { + param1: event.body.field1, + param2: event.query.id, + }; +} +\`\`\` + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +### S3Object Type + +The S3Object type represents a file in S3 storage: + +\`\`\`typescript +type S3Object = { + s3: string; // Path within the bucket +}; +\`\`\` + +## TypeScript Operations + +\`\`\`typescript +import * as wmill from "windmill-client"; + +// Load file content from S3 +const content: Uint8Array = await wmill.loadS3File(s3object); + +// Load file as stream +const blob: Blob = await wmill.loadS3FileStream(s3object); + +// Write file to S3 +const result: S3Object = await wmill.writeS3File( + s3object, // Target path (or undefined to auto-generate) + fileContent, // string or Blob + s3ResourcePath // Optional: specific S3 resource to use +); +\`\`\` + + +# TypeScript SDK (windmill-client) + +Import: import * as wmill from 'windmill-client' + +/** + * Create a SQL template function for PostgreSQL/datatable queries + * @param name - Database/datatable name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age}::int + * \`.fetch() + */ +datatable(name: string = "main"): DatatableSqlTemplateFunction + +/** + * Create a SQL template function for DuckDB/ducklake queries + * @param name - DuckDB database name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age} + * \`.fetch() + */ +ducklake(name: string = "main"): SqlTemplateFunction + +/** + * Initialize the Windmill client with authentication token and base URL + * @param token - Authentication token (defaults to WM_TOKEN env variable) + * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) + */ +setClient(token?: string, baseUrl?: string): void + +/** + * Create a client configuration from env variables + * @returns client configuration + */ +getWorkspace(): string + +/** + * Get a resource value by path + * @param path path of the resource, default to internal state path + * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error + * @returns resource value + */ +async getResource(path?: string, undefinedIfEmpty?: boolean): Promise + +/** + * Get the true root job id + * @param jobId job id to get the root job id from (default to current job) + * @returns root job id + */ +async getRootJobId(jobId?: string): Promise + +/** + * @deprecated Use runScriptByPath or runScriptByHash instead + */ +async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its path and wait for the result + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its hash and wait for the result + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Append a text to the result stream + * @param text text to append to the result stream + */ +appendToResultStream(text: string): void + +/** + * Stream to the result stream + * @param stream stream to stream to the result stream + */ +async streamResult(stream: AsyncIterable): Promise + +/** + * Run a flow synchronously by its path and wait for the result + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param verbose - Enable verbose logging + * @returns Flow execution result + */ +async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Wait for a job to complete and return its result + * @param jobId - ID of the job to wait for + * @param verbose - Enable verbose logging + * @returns Job result when completed + */ +async waitJob(jobId: string, verbose: boolean = false): Promise + +/** + * Get the result of a completed job + * @param jobId - ID of the completed job + * @returns Job result + */ +async getResult(jobId: string): Promise + +/** + * Get the result of a job if completed, or its current status + * @param jobId - ID of the job + * @returns Object with started, completed, success, and result properties + */ +async getResultMaybe(jobId: string): Promise + +/** + * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead + */ +async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its path + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its hash + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a flow asynchronously by its path + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) + * @returns Job ID of the created job + */ +async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise + +/** + * Resolve a resource value in case the default value was picked because the input payload was undefined + * @param obj resource value or path of the resource under the format \`$res:path\` + * @returns resource value + */ +async resolveDefaultResource(obj: any): Promise + +/** + * Get the state file path from environment variables + * @returns State path string + */ +getStatePath(): string + +/** + * Set a resource value by path + * @param path path of the resource to set, default to state path + * @param value new value of the resource to set + * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type + */ +async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise + +/** + * Set the state + * @param state state to set + * @deprecated use setState instead + */ +async setInternalState(state: any): Promise + +/** + * Set the state + * @param state state to set + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async setState(state: any, path?: string): Promise + +/** + * Set the progress + * Progress cannot go back and limited to 0% to 99% range + * @param percent Progress to set in % + * @param jobId? Job to set progress for + */ +async setProgress(percent: number, jobId?: any): Promise + +/** + * Get the progress + * @param jobId? Job to get progress from + * @returns Optional clamped between 0 and 100 progress value + */ +async getProgress(jobId?: any): Promise + +/** + * Set a flow user state + * @param key key of the state + * @param value value of the state + */ +async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise + +/** + * Get a flow user state + * @param path path of the variable + */ +async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise + +/** + * Get the internal state + * @deprecated use getState instead + */ +async getInternalState(): Promise + +/** + * Get the state shared across executions + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async getState(path?: string): Promise + +/** + * Get a variable by path + * @param path path of the variable + * @returns variable value + */ +async getVariable(path: string): Promise + +/** + * Set a variable by path, create if not exist + * @param path path of the variable + * @param value value of the variable + * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) + * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") + */ +async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise + +/** + * Build a PostgreSQL connection URL from a database resource + * @param path - Path to the database resource + * @returns PostgreSQL connection URL string + */ +async databaseUrlFromResource(path: string): Promise + +async polarsConnectionSettings(s3_resource_path: string | undefined): Promise + +async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise + +/** + * Get S3 client settings from a resource or workspace default + * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) + * @returns S3 client configuration settings + */ +async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise + +/** + * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContent = await wmill.loadS3FileContent(inputFile) + * // if the file is a raw text file, it can be decoded and printed directly: + * const text = new TextDecoder().decode(fileContentStream) + * console.log(text); + * \`\`\` + */ +async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContentBlob = await wmill.loadS3FileStream(inputFile) + * // if the content is plain text, the blob can be read directly: + * console.log(await fileContentBlob.text()); + * \`\`\` + */ +async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * const s3object = await writeS3File(s3Object, "Hello Windmill!") + * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') + * console.log(fileContentAsUtf8Str) + * \`\`\` + */ +async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise + +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +async signS3Objects(s3objects: S3Object[]): Promise + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +async signS3Object(s3object: S3Object): Promise + +/** + * Generate a presigned public URL for an array of S3 objects. + * If an S3 object is not signed yet, it will be signed first. + * @param s3Objects s3 objects to sign + * @returns list of signed public URLs + */ +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. + * @param s3Object s3 object to sign + * @returns signed public URL + */ +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Get URLs needed for resuming a flow after this step + * @param approver approver name + * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. + * This allows pre-approvals that can be consumed by any later suspend step in the same flow. + * @returns approval page UI URL, resume and cancel API URLs for resuming the flow + */ +async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * @deprecated use getResumeUrls instead + */ +getResumeEndpoints(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) + * @param audience audience of the token + * @param expiresIn Optional number of seconds until the token expires + * @returns jwt token + */ +async getIdToken(audience: string, expiresIn?: number): Promise + +/** + * Convert a base64-encoded string to Uint8Array + * @param data - Base64-encoded string + * @returns Decoded Uint8Array + */ +base64ToUint8Array(data: string): Uint8Array + +/** + * Convert a Uint8Array to base64-encoded string + * @param arrayBuffer - Uint8Array to encode + * @returns Base64-encoded string + */ +uint8ArrayToBase64(arrayBuffer: Uint8Array): string + +/** + * Get email from workspace username + * This method is particularly useful for apps that require the email address of the viewer. + * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. + * @param username + * @returns email address + */ +async usernameToEmail(username: string): Promise + +/** + * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Slack approval request. + * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. + * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Slack approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * @param {string} [options.resumeButtonText] - Optional text for the resume button. + * @param {string} [options.cancelButtonText] - Optional text for the cancel button. + * + * @returns {Promise} Resolves when the Slack approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getSlackApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveSlackApproval({ + * slackResourcePath: "/u/alex/my_slack_resource", + * channelId: "admins-slack-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * resumeButtonText: "Resume", + * cancelButtonText: "Cancel", + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, resumeButtonText, cancelButtonText, }: SlackApprovalOptions): Promise + +/** + * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Teams approval request. + * @param {string} options.teamName - The Teams team name where the approval request will be sent. + * @param {string} options.channelName - The Teams channel name where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Teams approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Teams approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getTeamsApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveTeamsApproval({ + * teamName: "admins-teams", + * channelName: "admins-teams-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise + +/** + * Parse an S3 object from URI string or record format + * @param s3Object - S3 object as URI string (s3://storage/key) or record + * @returns S3 object record with storage and s3 key + */ +parseS3Object(s3Object: S3Object): S3ObjectRecord + +setWorkflowCtx(ctx: WorkflowCtx | null): void + +async sleep(seconds: number): Promise + +async step(name: string, fn: () => T | Promise): Promise + +/** + * Create a task that dispatches to a separate Windmill script. + * + * @example + * const extract = taskScript("f/data/extract"); + * // inside workflow: await extract({ url: "https://..." }) + */ +taskScript(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike + +/** + * Create a task that dispatches to a separate Windmill flow. + * + * @example + * const pipeline = taskFlow("f/etl/pipeline"); + * // inside workflow: await pipeline({ input: data }) + */ +taskFlow(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike + +/** + * Mark an async function as a workflow-as-code entry point. + * + * The function must be **deterministic**: given the same inputs it must call + * tasks in the same order on every replay. Branching on task results is fine + * (results are replayed from checkpoint), but branching on external state + * (current time, random values, external API calls) must use \`step()\` to + * checkpoint the value so replays see the same result. + */ +workflow(fn: (...args: any[]) => Promise): void + +/** + * Suspend the workflow and wait for an external approval. + * + * Use \`getResumeUrls()\` (wrapped in \`step()\`) to obtain resume/cancel/approvalPage + * URLs before calling this function. + * + * @example + * const urls = await step("urls", () => getResumeUrls()); + * await step("notify", () => sendEmail(urls.approvalPage)); + * const { value, approver } = await waitForApproval({ timeout: 3600 }); + */ +waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> + +/** + * Process items in parallel with optional concurrency control. + * + * Each item is processed by calling \`fn(item)\`, which should be a task(). + * Items are dispatched in batches of \`concurrency\` (default: all at once). + * + * @example + * const process = task(async (item: string) => { ... }); + * const results = await parallel(items, process, { concurrency: 5 }); + */ +async parallel(items: T[], fn: (item: T) => PromiseLike | R, options?: { concurrency?: number },): Promise +`, + "write-script-postgresql": `--- +name: write-script-postgresql +description: MUST use when writing PostgreSQL queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# PostgreSQL + +Arguments are obtained directly in the statement with \`$1::{type}\`, \`$2::{type}\`, etc. + +Name the parameters by adding comments at the beginning of the script (without specifying the type): + +\`\`\`sql +-- $1 name1 +-- $2 name2 = default_value +SELECT * FROM users WHERE name = $1::TEXT AND age > $2::INT; +\`\`\` +`, + "write-script-php": `--- +name: write-script-php +description: MUST use when writing PHP scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# PHP + +## Structure + +The script must start with \` $param1, "count" => $param2]; +} +\`\`\` + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +You need to **redefine** the type of the resources that are needed before the main function. Always check if the class already exists using \`class_exists\`: + +\`\`\`php + @name2; +\`\`\` +`, + "write-script-bun": `--- +name: write-script-bun +description: MUST use when writing Bun/TypeScript scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# TypeScript (Bun) + +Bun runtime with full npm ecosystem and fastest execution. + +## Structure + +Export a single **async** function called \`main\`: + +\`\`\`typescript +export async function main(param1: string, param2: number) { + // Your code here + return { result: param1, count: param2 }; +} +\`\`\` + +Do not call the main function. Libraries are installed automatically. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +Use the \`RT\` namespace for resource types: + +\`\`\`typescript +export async function main(stripe: RT.Stripe) { + // stripe contains API key and config from the resource +} +\`\`\` + +Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + +Before using a resource type, check the \`rt.d.ts\` file in the project root to see all available resource types and their fields. This file is generated by \`wmill resource-type generate-namespace\`. + +## Imports + +\`\`\`typescript +import Stripe from "stripe"; +import { someFunction } from "some-package"; +\`\`\` + +## Windmill Client + +Import the windmill client for platform interactions: + +\`\`\`typescript +import * as wmill from "windmill-client"; +\`\`\` + +See the SDK documentation for available methods. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: + +\`\`\`typescript +type Event = { + kind: + | "webhook" + | "http" + | "websocket" + | "kafka" + | "email" + | "nats" + | "postgres" + | "sqs" + | "mqtt" + | "gcp"; + body: any; + headers: Record; + query: Record; +}; + +export async function preprocessor(event: Event) { + return { + param1: event.body.field1, + param2: event.query.id, + }; +} +\`\`\` + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +### S3Object Type + +The S3Object type represents a file in S3 storage: + +\`\`\`typescript +type S3Object = { + s3: string; // Path within the bucket +}; +\`\`\` + +## TypeScript Operations + +\`\`\`typescript +import * as wmill from "windmill-client"; + +// Load file content from S3 +const content: Uint8Array = await wmill.loadS3File(s3object); + +// Load file as stream +const blob: Blob = await wmill.loadS3FileStream(s3object); + +// Write file to S3 +const result: S3Object = await wmill.writeS3File( + s3object, // Target path (or undefined to auto-generate) + fileContent, // string or Blob + s3ResourcePath // Optional: specific S3 resource to use +); +\`\`\` + + +# TypeScript SDK (windmill-client) + +Import: import * as wmill from 'windmill-client' + +/** + * Create a SQL template function for PostgreSQL/datatable queries + * @param name - Database/datatable name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age}::int + * \`.fetch() + */ +datatable(name: string = "main"): DatatableSqlTemplateFunction + +/** + * Create a SQL template function for DuckDB/ducklake queries + * @param name - DuckDB database name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age} + * \`.fetch() + */ +ducklake(name: string = "main"): SqlTemplateFunction + +/** + * Initialize the Windmill client with authentication token and base URL + * @param token - Authentication token (defaults to WM_TOKEN env variable) + * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) + */ +setClient(token?: string, baseUrl?: string): void + +/** + * Create a client configuration from env variables + * @returns client configuration + */ +getWorkspace(): string + +/** + * Get a resource value by path + * @param path path of the resource, default to internal state path + * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error + * @returns resource value + */ +async getResource(path?: string, undefinedIfEmpty?: boolean): Promise + +/** + * Get the true root job id + * @param jobId job id to get the root job id from (default to current job) + * @returns root job id + */ +async getRootJobId(jobId?: string): Promise + +/** + * @deprecated Use runScriptByPath or runScriptByHash instead + */ +async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its path and wait for the result + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its hash and wait for the result + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Append a text to the result stream + * @param text text to append to the result stream + */ +appendToResultStream(text: string): void + +/** + * Stream to the result stream + * @param stream stream to stream to the result stream + */ +async streamResult(stream: AsyncIterable): Promise + +/** + * Run a flow synchronously by its path and wait for the result + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param verbose - Enable verbose logging + * @returns Flow execution result + */ +async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Wait for a job to complete and return its result + * @param jobId - ID of the job to wait for + * @param verbose - Enable verbose logging + * @returns Job result when completed + */ +async waitJob(jobId: string, verbose: boolean = false): Promise + +/** + * Get the result of a completed job + * @param jobId - ID of the completed job + * @returns Job result + */ +async getResult(jobId: string): Promise + +/** + * Get the result of a job if completed, or its current status + * @param jobId - ID of the job + * @returns Object with started, completed, success, and result properties + */ +async getResultMaybe(jobId: string): Promise + +/** + * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead + */ +async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its path + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its hash + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a flow asynchronously by its path + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) + * @returns Job ID of the created job + */ +async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise + +/** + * Resolve a resource value in case the default value was picked because the input payload was undefined + * @param obj resource value or path of the resource under the format \`$res:path\` + * @returns resource value + */ +async resolveDefaultResource(obj: any): Promise + +/** + * Get the state file path from environment variables + * @returns State path string + */ +getStatePath(): string + +/** + * Set a resource value by path + * @param path path of the resource to set, default to state path + * @param value new value of the resource to set + * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type + */ +async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise + +/** + * Set the state + * @param state state to set + * @deprecated use setState instead + */ +async setInternalState(state: any): Promise + +/** + * Set the state + * @param state state to set + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async setState(state: any, path?: string): Promise + +/** + * Set the progress + * Progress cannot go back and limited to 0% to 99% range + * @param percent Progress to set in % + * @param jobId? Job to set progress for + */ +async setProgress(percent: number, jobId?: any): Promise + +/** + * Get the progress + * @param jobId? Job to get progress from + * @returns Optional clamped between 0 and 100 progress value + */ +async getProgress(jobId?: any): Promise + +/** + * Set a flow user state + * @param key key of the state + * @param value value of the state + */ +async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise + +/** + * Get a flow user state + * @param path path of the variable + */ +async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise + +/** + * Get the internal state + * @deprecated use getState instead + */ +async getInternalState(): Promise + +/** + * Get the state shared across executions + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async getState(path?: string): Promise + +/** + * Get a variable by path + * @param path path of the variable + * @returns variable value + */ +async getVariable(path: string): Promise + +/** + * Set a variable by path, create if not exist + * @param path path of the variable + * @param value value of the variable + * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) + * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") + */ +async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise + +/** + * Build a PostgreSQL connection URL from a database resource + * @param path - Path to the database resource + * @returns PostgreSQL connection URL string + */ +async databaseUrlFromResource(path: string): Promise + +async polarsConnectionSettings(s3_resource_path: string | undefined): Promise + +async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise + +/** + * Get S3 client settings from a resource or workspace default + * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) + * @returns S3 client configuration settings + */ +async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise + +/** + * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContent = await wmill.loadS3FileContent(inputFile) + * // if the file is a raw text file, it can be decoded and printed directly: + * const text = new TextDecoder().decode(fileContentStream) + * console.log(text); + * \`\`\` + */ +async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContentBlob = await wmill.loadS3FileStream(inputFile) + * // if the content is plain text, the blob can be read directly: + * console.log(await fileContentBlob.text()); + * \`\`\` + */ +async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * const s3object = await writeS3File(s3Object, "Hello Windmill!") + * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') + * console.log(fileContentAsUtf8Str) + * \`\`\` + */ +async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise + +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +async signS3Objects(s3objects: S3Object[]): Promise + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +async signS3Object(s3object: S3Object): Promise + +/** + * Generate a presigned public URL for an array of S3 objects. + * If an S3 object is not signed yet, it will be signed first. + * @param s3Objects s3 objects to sign + * @returns list of signed public URLs + */ +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. + * @param s3Object s3 object to sign + * @returns signed public URL + */ +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Get URLs needed for resuming a flow after this step + * @param approver approver name + * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. + * This allows pre-approvals that can be consumed by any later suspend step in the same flow. + * @returns approval page UI URL, resume and cancel API URLs for resuming the flow + */ +async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * @deprecated use getResumeUrls instead + */ +getResumeEndpoints(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) + * @param audience audience of the token + * @param expiresIn Optional number of seconds until the token expires + * @returns jwt token + */ +async getIdToken(audience: string, expiresIn?: number): Promise + +/** + * Convert a base64-encoded string to Uint8Array + * @param data - Base64-encoded string + * @returns Decoded Uint8Array + */ +base64ToUint8Array(data: string): Uint8Array + +/** + * Convert a Uint8Array to base64-encoded string + * @param arrayBuffer - Uint8Array to encode + * @returns Base64-encoded string + */ +uint8ArrayToBase64(arrayBuffer: Uint8Array): string + +/** + * Get email from workspace username + * This method is particularly useful for apps that require the email address of the viewer. + * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. + * @param username + * @returns email address + */ +async usernameToEmail(username: string): Promise + +/** + * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Slack approval request. + * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. + * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Slack approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * @param {string} [options.resumeButtonText] - Optional text for the resume button. + * @param {string} [options.cancelButtonText] - Optional text for the cancel button. + * + * @returns {Promise} Resolves when the Slack approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getSlackApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveSlackApproval({ + * slackResourcePath: "/u/alex/my_slack_resource", + * channelId: "admins-slack-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * resumeButtonText: "Resume", + * cancelButtonText: "Cancel", + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, resumeButtonText, cancelButtonText, }: SlackApprovalOptions): Promise + +/** + * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Teams approval request. + * @param {string} options.teamName - The Teams team name where the approval request will be sent. + * @param {string} options.channelName - The Teams channel name where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Teams approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Teams approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getTeamsApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveTeamsApproval({ + * teamName: "admins-teams", + * channelName: "admins-teams-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise + +/** + * Parse an S3 object from URI string or record format + * @param s3Object - S3 object as URI string (s3://storage/key) or record + * @returns S3 object record with storage and s3 key + */ +parseS3Object(s3Object: S3Object): S3ObjectRecord + +setWorkflowCtx(ctx: WorkflowCtx | null): void + +async sleep(seconds: number): Promise + +async step(name: string, fn: () => T | Promise): Promise + +/** + * Create a task that dispatches to a separate Windmill script. + * + * @example + * const extract = taskScript("f/data/extract"); + * // inside workflow: await extract({ url: "https://..." }) + */ +taskScript(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike + +/** + * Create a task that dispatches to a separate Windmill flow. + * + * @example + * const pipeline = taskFlow("f/etl/pipeline"); + * // inside workflow: await pipeline({ input: data }) + */ +taskFlow(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike + +/** + * Mark an async function as a workflow-as-code entry point. + * + * The function must be **deterministic**: given the same inputs it must call + * tasks in the same order on every replay. Branching on task results is fine + * (results are replayed from checkpoint), but branching on external state + * (current time, random values, external API calls) must use \`step()\` to + * checkpoint the value so replays see the same result. + */ +workflow(fn: (...args: any[]) => Promise): void + +/** + * Suspend the workflow and wait for an external approval. + * + * Use \`getResumeUrls()\` (wrapped in \`step()\`) to obtain resume/cancel/approvalPage + * URLs before calling this function. + * + * @example + * const urls = await step("urls", () => getResumeUrls()); + * await step("notify", () => sendEmail(urls.approvalPage)); + * const { value, approver } = await waitForApproval({ timeout: 3600 }); + */ +waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> + +/** + * Process items in parallel with optional concurrency control. + * + * Each item is processed by calling \`fn(item)\`, which should be a task(). + * Items are dispatched in batches of \`concurrency\` (default: all at once). + * + * @example + * const process = task(async (item: string) => { ... }); + * const results = await parallel(items, process, { concurrency: 5 }); + */ +async parallel(items: T[], fn: (item: T) => PromiseLike | R, options?: { concurrency?: number },): Promise +`, + "write-script-csharp": `--- +name: write-script-csharp +description: MUST use when writing C# scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# C# + +The script must contain a public static \`Main\` method inside a class: + +\`\`\`csharp +public class Script +{ + public static object Main(string name, int count) + { + return new { Name = name, Count = count }; + } +} +\`\`\` + +**Important:** +- Class name is irrelevant +- Method must be \`public static\` +- Return type can be \`object\` or specific type + +## NuGet Packages + +Add packages using the \`#r\` directive at the top: + +\`\`\`csharp +#r "nuget: Newtonsoft.Json, 13.0.3" +#r "nuget: RestSharp, 110.2.0" + +using Newtonsoft.Json; +using RestSharp; + +public class Script +{ + public static object Main(string url) + { + var client = new RestClient(url); + var request = new RestRequest(); + var response = client.Get(request); + return JsonConvert.DeserializeObject(response.Content); + } +} +\`\`\` +`, + "write-script-mssql": `--- +name: write-script-mssql +description: MUST use when writing MS SQL Server queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Microsoft SQL Server (MSSQL) + +Arguments use \`@P1\`, \`@P2\`, etc. + +Name the parameters by adding comments before the statement: + +\`\`\`sql +-- @P1 name1 (varchar) +-- @P2 name2 (int) = 0 +SELECT * FROM users WHERE name = @P1 AND age > @P2; +\`\`\` +`, + "write-script-deno": `--- +name: write-script-deno +description: MUST use when writing Deno/TypeScript scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# TypeScript (Deno) + +Deno runtime with npm support via \`npm:\` prefix and native Deno libraries. + +## Structure + +Export a single **async** function called \`main\`: + +\`\`\`typescript +export async function main(param1: string, param2: number) { + // Your code here + return { result: param1, count: param2 }; +} +\`\`\` + +Do not call the main function. Libraries are installed automatically. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +Use the \`RT\` namespace for resource types: + +\`\`\`typescript +export async function main(stripe: RT.Stripe) { + // stripe contains API key and config from the resource +} +\`\`\` + +Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + +Before using a resource type, check the \`rt.d.ts\` file in the project root to see all available resource types and their fields. This file is generated by \`wmill resource-type generate-namespace\`. + +## Imports + +\`\`\`typescript +// npm packages use npm: prefix +import Stripe from "npm:stripe"; +import { someFunction } from "npm:some-package"; + +// Deno standard library +import { serve } from "https://deno.land/std/http/server.ts"; +\`\`\` + +## Windmill Client + +Import the windmill client for platform interactions: + +\`\`\`typescript +import * as wmill from "windmill-client"; +\`\`\` + +See the SDK documentation for available methods. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: + +\`\`\`typescript +type Event = { + kind: + | "webhook" + | "http" + | "websocket" + | "kafka" + | "email" + | "nats" + | "postgres" + | "sqs" + | "mqtt" + | "gcp"; + body: any; + headers: Record; + query: Record; +}; + +export async function preprocessor(event: Event) { + return { + param1: event.body.field1, + param2: event.query.id, + }; +} +\`\`\` + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +### S3Object Type + +The S3Object type represents a file in S3 storage: + +\`\`\`typescript +type S3Object = { + s3: string; // Path within the bucket +}; +\`\`\` + +## TypeScript Operations + +\`\`\`typescript +import * as wmill from "windmill-client"; + +// Load file content from S3 +const content: Uint8Array = await wmill.loadS3File(s3object); + +// Load file as stream +const blob: Blob = await wmill.loadS3FileStream(s3object); + +// Write file to S3 +const result: S3Object = await wmill.writeS3File( + s3object, // Target path (or undefined to auto-generate) + fileContent, // string or Blob + s3ResourcePath // Optional: specific S3 resource to use +); +\`\`\` + + +# TypeScript SDK (windmill-client) + +Import: import * as wmill from 'windmill-client' + +/** + * Create a SQL template function for PostgreSQL/datatable queries + * @param name - Database/datatable name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age}::int + * \`.fetch() + */ +datatable(name: string = "main"): DatatableSqlTemplateFunction + +/** + * Create a SQL template function for DuckDB/ducklake queries + * @param name - DuckDB database name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age} + * \`.fetch() + */ +ducklake(name: string = "main"): SqlTemplateFunction + +/** + * Initialize the Windmill client with authentication token and base URL + * @param token - Authentication token (defaults to WM_TOKEN env variable) + * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) + */ +setClient(token?: string, baseUrl?: string): void + +/** + * Create a client configuration from env variables + * @returns client configuration + */ +getWorkspace(): string + +/** + * Get a resource value by path + * @param path path of the resource, default to internal state path + * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error + * @returns resource value + */ +async getResource(path?: string, undefinedIfEmpty?: boolean): Promise + +/** + * Get the true root job id + * @param jobId job id to get the root job id from (default to current job) + * @returns root job id + */ +async getRootJobId(jobId?: string): Promise + +/** + * @deprecated Use runScriptByPath or runScriptByHash instead + */ +async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its path and wait for the result + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its hash and wait for the result + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Append a text to the result stream + * @param text text to append to the result stream + */ +appendToResultStream(text: string): void + +/** + * Stream to the result stream + * @param stream stream to stream to the result stream + */ +async streamResult(stream: AsyncIterable): Promise + +/** + * Run a flow synchronously by its path and wait for the result + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param verbose - Enable verbose logging + * @returns Flow execution result + */ +async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Wait for a job to complete and return its result + * @param jobId - ID of the job to wait for + * @param verbose - Enable verbose logging + * @returns Job result when completed + */ +async waitJob(jobId: string, verbose: boolean = false): Promise + +/** + * Get the result of a completed job + * @param jobId - ID of the completed job + * @returns Job result + */ +async getResult(jobId: string): Promise + +/** + * Get the result of a job if completed, or its current status + * @param jobId - ID of the job + * @returns Object with started, completed, success, and result properties + */ +async getResultMaybe(jobId: string): Promise + +/** + * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead + */ +async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its path + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its hash + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a flow asynchronously by its path + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) + * @returns Job ID of the created job + */ +async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise + +/** + * Resolve a resource value in case the default value was picked because the input payload was undefined + * @param obj resource value or path of the resource under the format \`$res:path\` + * @returns resource value + */ +async resolveDefaultResource(obj: any): Promise + +/** + * Get the state file path from environment variables + * @returns State path string + */ +getStatePath(): string + +/** + * Set a resource value by path + * @param path path of the resource to set, default to state path + * @param value new value of the resource to set + * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type + */ +async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise + +/** + * Set the state + * @param state state to set + * @deprecated use setState instead + */ +async setInternalState(state: any): Promise + +/** + * Set the state + * @param state state to set + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async setState(state: any, path?: string): Promise + +/** + * Set the progress + * Progress cannot go back and limited to 0% to 99% range + * @param percent Progress to set in % + * @param jobId? Job to set progress for + */ +async setProgress(percent: number, jobId?: any): Promise + +/** + * Get the progress + * @param jobId? Job to get progress from + * @returns Optional clamped between 0 and 100 progress value + */ +async getProgress(jobId?: any): Promise + +/** + * Set a flow user state + * @param key key of the state + * @param value value of the state + */ +async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise + +/** + * Get a flow user state + * @param path path of the variable + */ +async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise + +/** + * Get the internal state + * @deprecated use getState instead + */ +async getInternalState(): Promise + +/** + * Get the state shared across executions + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async getState(path?: string): Promise + +/** + * Get a variable by path + * @param path path of the variable + * @returns variable value + */ +async getVariable(path: string): Promise + +/** + * Set a variable by path, create if not exist + * @param path path of the variable + * @param value value of the variable + * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) + * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") + */ +async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise + +/** + * Build a PostgreSQL connection URL from a database resource + * @param path - Path to the database resource + * @returns PostgreSQL connection URL string + */ +async databaseUrlFromResource(path: string): Promise + +async polarsConnectionSettings(s3_resource_path: string | undefined): Promise + +async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise + +/** + * Get S3 client settings from a resource or workspace default + * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) + * @returns S3 client configuration settings + */ +async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise + +/** + * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContent = await wmill.loadS3FileContent(inputFile) + * // if the file is a raw text file, it can be decoded and printed directly: + * const text = new TextDecoder().decode(fileContentStream) + * console.log(text); + * \`\`\` + */ +async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContentBlob = await wmill.loadS3FileStream(inputFile) + * // if the content is plain text, the blob can be read directly: + * console.log(await fileContentBlob.text()); + * \`\`\` + */ +async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * const s3object = await writeS3File(s3Object, "Hello Windmill!") + * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') + * console.log(fileContentAsUtf8Str) + * \`\`\` + */ +async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise + +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +async signS3Objects(s3objects: S3Object[]): Promise + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +async signS3Object(s3object: S3Object): Promise + +/** + * Generate a presigned public URL for an array of S3 objects. + * If an S3 object is not signed yet, it will be signed first. + * @param s3Objects s3 objects to sign + * @returns list of signed public URLs + */ +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. + * @param s3Object s3 object to sign + * @returns signed public URL + */ +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Get URLs needed for resuming a flow after this step + * @param approver approver name + * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. + * This allows pre-approvals that can be consumed by any later suspend step in the same flow. + * @returns approval page UI URL, resume and cancel API URLs for resuming the flow + */ +async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * @deprecated use getResumeUrls instead + */ +getResumeEndpoints(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) + * @param audience audience of the token + * @param expiresIn Optional number of seconds until the token expires + * @returns jwt token + */ +async getIdToken(audience: string, expiresIn?: number): Promise + +/** + * Convert a base64-encoded string to Uint8Array + * @param data - Base64-encoded string + * @returns Decoded Uint8Array + */ +base64ToUint8Array(data: string): Uint8Array + +/** + * Convert a Uint8Array to base64-encoded string + * @param arrayBuffer - Uint8Array to encode + * @returns Base64-encoded string + */ +uint8ArrayToBase64(arrayBuffer: Uint8Array): string + +/** + * Get email from workspace username + * This method is particularly useful for apps that require the email address of the viewer. + * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. + * @param username + * @returns email address + */ +async usernameToEmail(username: string): Promise + +/** + * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Slack approval request. + * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. + * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Slack approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * @param {string} [options.resumeButtonText] - Optional text for the resume button. + * @param {string} [options.cancelButtonText] - Optional text for the cancel button. + * + * @returns {Promise} Resolves when the Slack approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getSlackApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveSlackApproval({ + * slackResourcePath: "/u/alex/my_slack_resource", + * channelId: "admins-slack-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * resumeButtonText: "Resume", + * cancelButtonText: "Cancel", + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, resumeButtonText, cancelButtonText, }: SlackApprovalOptions): Promise + +/** + * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Teams approval request. + * @param {string} options.teamName - The Teams team name where the approval request will be sent. + * @param {string} options.channelName - The Teams channel name where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Teams approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Teams approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getTeamsApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveTeamsApproval({ + * teamName: "admins-teams", + * channelName: "admins-teams-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise + +/** + * Parse an S3 object from URI string or record format + * @param s3Object - S3 object as URI string (s3://storage/key) or record + * @returns S3 object record with storage and s3 key + */ +parseS3Object(s3Object: S3Object): S3ObjectRecord + +setWorkflowCtx(ctx: WorkflowCtx | null): void + +async sleep(seconds: number): Promise + +async step(name: string, fn: () => T | Promise): Promise + +/** + * Create a task that dispatches to a separate Windmill script. + * + * @example + * const extract = taskScript("f/data/extract"); + * // inside workflow: await extract({ url: "https://..." }) + */ +taskScript(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike + +/** + * Create a task that dispatches to a separate Windmill flow. + * + * @example + * const pipeline = taskFlow("f/etl/pipeline"); + * // inside workflow: await pipeline({ input: data }) + */ +taskFlow(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike + +/** + * Mark an async function as a workflow-as-code entry point. + * + * The function must be **deterministic**: given the same inputs it must call + * tasks in the same order on every replay. Branching on task results is fine + * (results are replayed from checkpoint), but branching on external state + * (current time, random values, external API calls) must use \`step()\` to + * checkpoint the value so replays see the same result. + */ +workflow(fn: (...args: any[]) => Promise): void + +/** + * Suspend the workflow and wait for an external approval. + * + * Use \`getResumeUrls()\` (wrapped in \`step()\`) to obtain resume/cancel/approvalPage + * URLs before calling this function. + * + * @example + * const urls = await step("urls", () => getResumeUrls()); + * await step("notify", () => sendEmail(urls.approvalPage)); + * const { value, approver } = await waitForApproval({ timeout: 3600 }); + */ +waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> + +/** + * Process items in parallel with optional concurrency control. + * + * Each item is processed by calling \`fn(item)\`, which should be a task(). + * Items are dispatched in batches of \`concurrency\` (default: all at once). + * + * @example + * const process = task(async (item: string) => { ... }); + * const results = await parallel(items, process, { concurrency: 5 }); + */ +async parallel(items: T[], fn: (item: T) => PromiseLike | R, options?: { concurrency?: number },): Promise +`, + "write-script-mysql": `--- +name: write-script-mysql +description: MUST use when writing MySQL queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# MySQL + +Arguments use \`?\` placeholders. + +Name the parameters by adding comments before the statement: + +\`\`\`sql +-- ? name1 (text) +-- ? name2 (int) = 0 +SELECT * FROM users WHERE name = ? AND age > ?; +\`\`\` +`, + "write-script-powershell": `--- +name: write-script-powershell +description: MUST use when writing PowerShell scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# PowerShell + +## Structure + +Arguments are obtained by calling the \`param\` function on the first line: + +\`\`\`powershell +param($Name, $Count = 0, [int]$Age) + +# Your code here +Write-Output "Processing $Name, count: $Count, age: $Age" + +# Return object +@{ + name = $Name + count = $Count + age = $Age +} +\`\`\` + +## Parameter Types + +You can specify types for parameters: + +\`\`\`powershell +param( + [string]$Name, + [int]$Count = 0, + [bool]$Enabled = $true, + [array]$Items +) + +@{ + name = $Name + count = $Count + enabled = $Enabled + items = $Items +} +\`\`\` + +## Return Values + +Return values by outputting them at the end of the script: + +\`\`\`powershell +param($Input) + +$result = @{ + processed = $true + data = $Input + timestamp = Get-Date -Format "o" +} + +$result +\`\`\` +`, + "write-script-snowflake": `--- +name: write-script-snowflake +description: MUST use when writing Snowflake queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Snowflake + +Arguments use \`?\` placeholders. + +Name the parameters by adding comments before the statement: + +\`\`\`sql +-- ? name1 (text) +-- ? name2 (number) = 0 +SELECT * FROM users WHERE name = ? AND age > ?; +\`\`\` +`, "write-script-python3": `--- name: write-script-python3 description: MUST use when writing Python scripts. @@ -459,6 +3015,16 @@ def load_s3_file_reader(s3object: S3Object | str, s3_resource_path: str | None) # ''' def write_s3_file(s3object: S3Object | str | None, file_content: BufferedReader | bytes, s3_resource_path: str | None, content_type: str | None = None, content_disposition: str | None = None) -> S3Object +# Permanently delete a file from the workspace S3 bucket. +# +# '''python +# from wmill import S3Object +# +# s3_obj = S3Object(s3="/path/to/my_file.txt") +# client.delete_s3_object(s3_obj) +# ''' +def delete_s3_object(s3object: S3Object | str, s3_resource_path: str | None = None) -> None + # Sign S3 objects for use by anonymous users in public apps. # # Args: @@ -677,18 +3243,6 @@ def boto3_connection_settings(s3_resource_path: str = '') -> Boto3ConnectionSett # State path string def get_state_path() -> str -# Decorator to mark a function as a workflow task. -# -# When executed inside a Windmill job, the decorated function runs as a -# separate workflow step. Outside Windmill, it executes normally. -# -# Args: -# tag: Optional worker tag for execution -# -# Returns: -# Decorated function -def task(*args, **kwargs) - # Parse resource syntax from string. def parse_resource_syntax(s: str) -> Optional[str] @@ -752,2047 +3306,96 @@ def infer_sql_type(value) -> str def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] -`, - "write-script-bun": `--- -name: write-script-bun -description: MUST use when writing Bun/TypeScript scripts. ---- +# Decorator that marks a function as a workflow task. +# +# Works in both WAC v1 (sync, HTTP-based dispatch) and WAC v2 +# (async, checkpoint/replay) modes: +# +# - **v2 (inside @workflow)**: dispatches as a checkpoint step. +# - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. +# - **Standalone**: executes the function body directly. +# +# Usage:: +# +# @task +# async def extract_data(url: str): ... +# +# @task(path="f/external_script", timeout=600, tag="gpu") +# async def run_external(x: int): ... +def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) + +# Create a task that dispatches to a separate Windmill script. +# +# Usage:: +# +# extract = task_script("f/data/extract", timeout=600) +# +# @workflow +# async def main(): +# data = await extract(url="https://...") +def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) + +# Create a task that dispatches to a separate Windmill flow. +# +# Usage:: +# +# pipeline = task_flow("f/etl/pipeline", priority=10) +# +# @workflow +# async def main(): +# result = await pipeline(input=data) +def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) + +# Decorator marking an async function as a workflow-as-code entry point. +# +# The function must be **deterministic**: given the same inputs it must call +# tasks in the same order on every replay. Branching on task results is fine +# (results are replayed from checkpoint), but branching on external state +# (current time, random values, external API calls) must use \`\`step()\`\` to +# checkpoint the value so replays see the same result. +def workflow(func) + +# Execute \`\`fn\`\` inline and checkpoint the result. +# +# On replay the cached value is returned without re-executing \`\`fn\`\`. +# Use for lightweight deterministic operations (timestamps, random IDs, +# config reads) that should not incur the overhead of a child job. +async def step(name: str, fn) + +# Server-side sleep — suspend the workflow for the given duration without holding a worker. +# +# Inside a @workflow, the parent job suspends and auto-resumes after \`\`seconds\`\`. +# Outside a workflow, falls back to \`\`asyncio.sleep\`\`. +async def sleep(seconds: int) + +# Suspend the workflow and wait for an external approval. +# +# Use \`\`get_resume_urls()\`\` (wrapped in \`\`step()\`\`) to obtain +# resume/cancel/approval URLs before calling this function. +# +# Returns a dict with \`\`value\`\` (form data), \`\`approver\`\`, and \`\`approved\`\`. +# +# Example:: +# +# urls = await step("urls", lambda: get_resume_urls()) +# await step("notify", lambda: send_email(urls["approvalPage"])) +# result = await wait_for_approval(timeout=3600) +async def wait_for_approval(timeout: int = 1800, form: dict | None = None) -> dict + +# Process items in parallel with optional concurrency control. +# +# Each item is processed by calling \`\`fn(item)\`\`, which should be a @task. +# Items are dispatched in batches of \`\`concurrency\`\` (default: all at once). +# +# Example:: +# +# @task +# async def process(item: str): +# ... +# +# results = await parallel(items, process, concurrency=5) +async def parallel(items, fn, concurrency: Optional[int] = None) -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# TypeScript (Bun) - -Bun runtime with full npm ecosystem and fastest execution. - -## Structure - -Export a single **async** function called \`main\`: - -\`\`\`typescript -export async function main(param1: string, param2: number) { - // Your code here - return { result: param1, count: param2 }; -} -\`\`\` - -Do not call the main function. Libraries are installed automatically. - -## Resource Types - -On Windmill, credentials and configuration are stored in resources and passed as parameters to main. - -Use the \`RT\` namespace for resource types: - -\`\`\`typescript -export async function main(stripe: RT.Stripe) { - // stripe contains API key and config from the resource -} -\`\`\` - -Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. - -## Imports - -\`\`\`typescript -import Stripe from "stripe"; -import { someFunction } from "some-package"; -\`\`\` - -## Windmill Client - -Import the windmill client for platform interactions: - -\`\`\`typescript -import * as wmill from "windmill-client"; -\`\`\` - -See the SDK documentation for available methods. - -## Preprocessor Scripts - -For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: - -\`\`\`typescript -type Event = { - kind: - | "webhook" - | "http" - | "websocket" - | "kafka" - | "email" - | "nats" - | "postgres" - | "sqs" - | "mqtt" - | "gcp"; - body: any; - headers: Record; - query: Record; -}; - -export async function preprocessor(event: Event) { - return { - param1: event.body.field1, - param2: event.query.id, - }; -} -\`\`\` - -## S3 Object Operations - -Windmill provides built-in support for S3-compatible storage operations. - -### S3Object Type - -The S3Object type represents a file in S3 storage: - -\`\`\`typescript -type S3Object = { - s3: string; // Path within the bucket -}; -\`\`\` - -## TypeScript Operations - -\`\`\`typescript -import * as wmill from "windmill-client"; - -// Load file content from S3 -const content: Uint8Array = await wmill.loadS3File(s3object); - -// Load file as stream -const blob: Blob = await wmill.loadS3FileStream(s3object); - -// Write file to S3 -const result: S3Object = await wmill.writeS3File( - s3object, // Target path (or undefined to auto-generate) - fileContent, // string or Blob - s3ResourcePath // Optional: specific S3 resource to use -); -\`\`\` - - -# TypeScript SDK (windmill-client) - -Import: import * as wmill from 'windmill-client' - -/** - * Initialize the Windmill client with authentication token and base URL - * @param token - Authentication token (defaults to WM_TOKEN env variable) - * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) - */ -setClient(token?: string, baseUrl?: string): void - -/** - * Create a client configuration from env variables - * @returns client configuration - */ -getWorkspace(): string - -/** - * Get a resource value by path - * @param path path of the resource, default to internal state path - * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error - * @returns resource value - */ -async getResource(path?: string, undefinedIfEmpty?: boolean): Promise - -/** - * Get the true root job id - * @param jobId job id to get the root job id from (default to current job) - * @returns root job id - */ -async getRootJobId(jobId?: string): Promise - -/** - * @deprecated Use runScriptByPath or runScriptByHash instead - */ -async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Run a script synchronously by its path and wait for the result - * @param path - Script path in Windmill - * @param args - Arguments to pass to the script - * @param verbose - Enable verbose logging - * @returns Script execution result - */ -async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Run a script synchronously by its hash and wait for the result - * @param hash_ - Script hash in Windmill - * @param args - Arguments to pass to the script - * @param verbose - Enable verbose logging - * @returns Script execution result - */ -async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Append a text to the result stream - * @param text text to append to the result stream - */ -appendToResultStream(text: string): void - -/** - * Stream to the result stream - * @param stream stream to stream to the result stream - */ -async streamResult(stream: AsyncIterable): Promise - -/** - * Run a flow synchronously by its path and wait for the result - * @param path - Flow path in Windmill - * @param args - Arguments to pass to the flow - * @param verbose - Enable verbose logging - * @returns Flow execution result - */ -async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Wait for a job to complete and return its result - * @param jobId - ID of the job to wait for - * @param verbose - Enable verbose logging - * @returns Job result when completed - */ -async waitJob(jobId: string, verbose: boolean = false): Promise - -/** - * Get the result of a completed job - * @param jobId - ID of the completed job - * @returns Job result - */ -async getResult(jobId: string): Promise - -/** - * Get the result of a job if completed, or its current status - * @param jobId - ID of the job - * @returns Object with started, completed, success, and result properties - */ -async getResultMaybe(jobId: string): Promise - -/** - * Wrap a function to execute as a Windmill task within a flow context - * @param f - Function to wrap as a task - * @returns Async wrapper function that executes as a Windmill job - */ -task(f: (_: P) => T): (_: P) => Promise - -/** - * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead - */ -async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise - -/** - * Run a script asynchronously by its path - * @param path - Script path in Windmill - * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) - * @returns Job ID of the created job - */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise - -/** - * Run a script asynchronously by its hash - * @param hash_ - Script hash in Windmill - * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) - * @returns Job ID of the created job - */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise - -/** - * Run a flow asynchronously by its path - * @param path - Flow path in Windmill - * @param args - Arguments to pass to the flow - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) - * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) - * @returns Job ID of the created job - */ -async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise - -/** - * Resolve a resource value in case the default value was picked because the input payload was undefined - * @param obj resource value or path of the resource under the format \`$res:path\` - * @returns resource value - */ -async resolveDefaultResource(obj: any): Promise - -/** - * Get the state file path from environment variables - * @returns State path string - */ -getStatePath(): string - -/** - * Set a resource value by path - * @param path path of the resource to set, default to state path - * @param value new value of the resource to set - * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type - */ -async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise - -/** - * Set the state - * @param state state to set - * @deprecated use setState instead - */ -async setInternalState(state: any): Promise - -/** - * Set the state - * @param state state to set - * @param path Optional state resource path override. Defaults to \`getStatePath()\`. - */ -async setState(state: any, path?: string): Promise - -/** - * Set the progress - * Progress cannot go back and limited to 0% to 99% range - * @param percent Progress to set in % - * @param jobId? Job to set progress for - */ -async setProgress(percent: number, jobId?: any): Promise - -/** - * Get the progress - * @param jobId? Job to get progress from - * @returns Optional clamped between 0 and 100 progress value - */ -async getProgress(jobId?: any): Promise - -/** - * Set a flow user state - * @param key key of the state - * @param value value of the state - */ -async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise - -/** - * Get a flow user state - * @param path path of the variable - */ -async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise - -/** - * Get the internal state - * @deprecated use getState instead - */ -async getInternalState(): Promise - -/** - * Get the state shared across executions - * @param path Optional state resource path override. Defaults to \`getStatePath()\`. - */ -async getState(path?: string): Promise - -/** - * Get a variable by path - * @param path path of the variable - * @returns variable value - */ -async getVariable(path: string): Promise - -/** - * Set a variable by path, create if not exist - * @param path path of the variable - * @param value value of the variable - * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) - * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") - */ -async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise - -/** - * Build a PostgreSQL connection URL from a database resource - * @param path - Path to the database resource - * @returns PostgreSQL connection URL string - */ -async databaseUrlFromResource(path: string): Promise - -async polarsConnectionSettings(s3_resource_path: string | undefined): Promise - -async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise - -/** - * Get S3 client settings from a resource or workspace default - * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) - * @returns S3 client configuration settings - */ -async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise - -/** - * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. - * - * \`\`\`typescript - * let fileContent = await wmill.loadS3FileContent(inputFile) - * // if the file is a raw text file, it can be decoded and printed directly: - * const text = new TextDecoder().decode(fileContentStream) - * console.log(text); - * \`\`\` - */ -async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise - -/** - * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. - * - * \`\`\`typescript - * let fileContentBlob = await wmill.loadS3FileStream(inputFile) - * // if the content is plain text, the blob can be read directly: - * console.log(await fileContentBlob.text()); - * \`\`\` - */ -async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise - -/** - * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. - * - * \`\`\`typescript - * const s3object = await writeS3File(s3Object, "Hello Windmill!") - * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') - * console.log(fileContentAsUtf8Str) - * \`\`\` - */ -async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise - -/** - * Sign S3 objects to be used by anonymous users in public apps - * @param s3objects s3 objects to sign - * @returns signed s3 objects - */ -async signS3Objects(s3objects: S3Object[]): Promise - -/** - * Sign S3 object to be used by anonymous users in public apps - * @param s3object s3 object to sign - * @returns signed s3 object - */ -async signS3Object(s3object: S3Object): Promise - -/** - * Generate a presigned public URL for an array of S3 objects. - * If an S3 object is not signed yet, it will be signed first. - * @param s3Objects s3 objects to sign - * @returns list of signed public URLs - */ -async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise - -/** - * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. - * @param s3Object s3 object to sign - * @returns signed public URL - */ -async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise - -/** - * Get URLs needed for resuming a flow after this step - * @param approver approver name - * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. - * This allows pre-approvals that can be consumed by any later suspend step in the same flow. - * @returns approval page UI URL, resume and cancel API URLs for resuming the flow - */ -async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ - approvalPage: string; - resume: string; - cancel: string; -}> - -/** - * @deprecated use getResumeUrls instead - */ -getResumeEndpoints(approver?: string): Promise<{ - approvalPage: string; - resume: string; - cancel: string; -}> - -/** - * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) - * @param audience audience of the token - * @param expiresIn Optional number of seconds until the token expires - * @returns jwt token - */ -async getIdToken(audience: string, expiresIn?: number): Promise - -/** - * Convert a base64-encoded string to Uint8Array - * @param data - Base64-encoded string - * @returns Decoded Uint8Array - */ -base64ToUint8Array(data: string): Uint8Array - -/** - * Convert a Uint8Array to base64-encoded string - * @param arrayBuffer - Uint8Array to encode - * @returns Base64-encoded string - */ -uint8ArrayToBase64(arrayBuffer: Uint8Array): string - -/** - * Get email from workspace username - * This method is particularly useful for apps that require the email address of the viewer. - * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. - * @param username - * @returns email address - */ -async usernameToEmail(username: string): Promise - -/** - * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. - * - * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** - * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). - * - * @param {Object} options - The configuration options for the Slack approval request. - * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. - * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. - * @param {string} [options.message] - Optional custom message to include in the Slack approval request. - * @param {string} [options.approver] - Optional user ID or name of the approver for the request. - * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. - * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. - * - * @returns {Promise} Resolves when the Slack approval request is successfully sent. - * - * @throws {Error} If the function is not called within a flow or flow preview. - * @throws {Error} If the \`JobService.getSlackApprovalPayload\` call fails. - * - * **Usage Example:** - * \`\`\`typescript - * await requestInteractiveSlackApproval({ - * slackResourcePath: "/u/alex/my_slack_resource", - * channelId: "admins-slack-channel", - * message: "Please approve this request", - * approver: "approver123", - * defaultArgsJson: { key1: "value1", key2: 42 }, - * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, - * }); - * \`\`\` - * - * **Note:** This function requires execution within a Windmill flow or flow preview. - */ -async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise - -/** - * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. - * - * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** - * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). - * - * @param {Object} options - The configuration options for the Teams approval request. - * @param {string} options.teamName - The Teams team name where the approval request will be sent. - * @param {string} options.channelName - The Teams channel name where the approval request will be sent. - * @param {string} [options.message] - Optional custom message to include in the Teams approval request. - * @param {string} [options.approver] - Optional user ID or name of the approver for the request. - * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. - * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. - * - * @returns {Promise} Resolves when the Teams approval request is successfully sent. - * - * @throws {Error} If the function is not called within a flow or flow preview. - * @throws {Error} If the \`JobService.getTeamsApprovalPayload\` call fails. - * - * **Usage Example:** - * \`\`\`typescript - * await requestInteractiveTeamsApproval({ - * teamName: "admins-teams", - * channelName: "admins-teams-channel", - * message: "Please approve this request", - * approver: "approver123", - * defaultArgsJson: { key1: "value1", key2: 42 }, - * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, - * }); - * \`\`\` - * - * **Note:** This function requires execution within a Windmill flow or flow preview. - */ -async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise - -/** - * Parse an S3 object from URI string or record format - * @param s3Object - S3 object as URI string (s3://storage/key) or record - * @returns S3 object record with storage and s3 key - */ -parseS3Object(s3Object: S3Object): S3ObjectRecord - -/** - * Create a SQL template function for PostgreSQL/datatable queries - * @param name - Database/datatable name (default: "main") - * @returns SQL template function for building parameterized queries - * @example - * let sql = wmill.datatable() - * let name = 'Robin' - * let age = 21 - * await sql\` - * SELECT * FROM friends - * WHERE name = \${name} AND age = \${age}::int - * \`.fetch() - */ -datatable(name: string = "main"): DatatableSqlTemplateFunction - -/** - * Create a SQL template function for DuckDB/ducklake queries - * @param name - DuckDB database name (default: "main") - * @returns SQL template function for building parameterized queries - * @example - * let sql = wmill.ducklake() - * let name = 'Robin' - * let age = 21 - * await sql\` - * SELECT * FROM friends - * WHERE name = \${name} AND age = \${age} - * \`.fetch() - */ -ducklake(name: string = "main"): SqlTemplateFunction -`, - "write-script-mysql": `--- -name: write-script-mysql -description: MUST use when writing MySQL queries. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# MySQL - -Arguments use \`?\` placeholders. - -Name the parameters by adding comments before the statement: - -\`\`\`sql --- ? name1 (text) --- ? name2 (int) = 0 -SELECT * FROM users WHERE name = ? AND age > ?; -\`\`\` -`, - "write-script-powershell": `--- -name: write-script-powershell -description: MUST use when writing PowerShell scripts. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# PowerShell - -## Structure - -Arguments are obtained by calling the \`param\` function on the first line: - -\`\`\`powershell -param($Name, $Count = 0, [int]$Age) - -# Your code here -Write-Output "Processing $Name, count: $Count, age: $Age" - -# Return object -@{ - name = $Name - count = $Count - age = $Age -} -\`\`\` - -## Parameter Types - -You can specify types for parameters: - -\`\`\`powershell -param( - [string]$Name, - [int]$Count = 0, - [bool]$Enabled = $true, - [array]$Items -) - -@{ - name = $Name - count = $Count - enabled = $Enabled - items = $Items -} -\`\`\` - -## Return Values - -Return values by outputting them at the end of the script: - -\`\`\`powershell -param($Input) - -$result = @{ - processed = $true - data = $Input - timestamp = Get-Date -Format "o" -} - -$result -\`\`\` -`, - "write-script-snowflake": `--- -name: write-script-snowflake -description: MUST use when writing Snowflake queries. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# Snowflake - -Arguments use \`?\` placeholders. - -Name the parameters by adding comments before the statement: - -\`\`\`sql --- ? name1 (text) --- ? name2 (number) = 0 -SELECT * FROM users WHERE name = ? AND age > ?; -\`\`\` -`, - "write-script-go": `--- -name: write-script-go -description: MUST use when writing Go scripts. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# Go - -## Structure - -The file package must be \`inner\` and export a function called \`main\`: - -\`\`\`go -package inner - -func main(param1 string, param2 int) (map[string]interface{}, error) { - return map[string]interface{}{ - "result": param1, - "count": param2, - }, nil -} -\`\`\` - -**Important:** -- Package must be \`inner\` -- Return type must be \`({return_type}, error)\` -- Function name is \`main\` (lowercase) - -## Return Types - -The return type can be any Go type that can be serialized to JSON: - -\`\`\`go -package inner - -type Result struct { - Name string \`json:"name"\` - Count int \`json:"count"\` -} - -func main(name string, count int) (Result, error) { - return Result{ - Name: name, - Count: count, - }, nil -} -\`\`\` - -## Error Handling - -Return errors as the second return value: - -\`\`\`go -package inner - -import "errors" - -func main(value int) (string, error) { - if value < 0 { - return "", errors.New("value must be positive") - } - return "success", nil -} -\`\`\` -`, - "write-script-deno": `--- -name: write-script-deno -description: MUST use when writing Deno/TypeScript scripts. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# TypeScript (Deno) - -Deno runtime with npm support via \`npm:\` prefix and native Deno libraries. - -## Structure - -Export a single **async** function called \`main\`: - -\`\`\`typescript -export async function main(param1: string, param2: number) { - // Your code here - return { result: param1, count: param2 }; -} -\`\`\` - -Do not call the main function. Libraries are installed automatically. - -## Resource Types - -On Windmill, credentials and configuration are stored in resources and passed as parameters to main. - -Use the \`RT\` namespace for resource types: - -\`\`\`typescript -export async function main(stripe: RT.Stripe) { - // stripe contains API key and config from the resource -} -\`\`\` - -Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. - -## Imports - -\`\`\`typescript -// npm packages use npm: prefix -import Stripe from "npm:stripe"; -import { someFunction } from "npm:some-package"; - -// Deno standard library -import { serve } from "https://deno.land/std/http/server.ts"; -\`\`\` - -## Windmill Client - -Import the windmill client for platform interactions: - -\`\`\`typescript -import * as wmill from "windmill-client"; -\`\`\` - -See the SDK documentation for available methods. - -## Preprocessor Scripts - -For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: - -\`\`\`typescript -type Event = { - kind: - | "webhook" - | "http" - | "websocket" - | "kafka" - | "email" - | "nats" - | "postgres" - | "sqs" - | "mqtt" - | "gcp"; - body: any; - headers: Record; - query: Record; -}; - -export async function preprocessor(event: Event) { - return { - param1: event.body.field1, - param2: event.query.id, - }; -} -\`\`\` - -## S3 Object Operations - -Windmill provides built-in support for S3-compatible storage operations. - -### S3Object Type - -The S3Object type represents a file in S3 storage: - -\`\`\`typescript -type S3Object = { - s3: string; // Path within the bucket -}; -\`\`\` - -## TypeScript Operations - -\`\`\`typescript -import * as wmill from "windmill-client"; - -// Load file content from S3 -const content: Uint8Array = await wmill.loadS3File(s3object); - -// Load file as stream -const blob: Blob = await wmill.loadS3FileStream(s3object); - -// Write file to S3 -const result: S3Object = await wmill.writeS3File( - s3object, // Target path (or undefined to auto-generate) - fileContent, // string or Blob - s3ResourcePath // Optional: specific S3 resource to use -); -\`\`\` - - -# TypeScript SDK (windmill-client) - -Import: import * as wmill from 'windmill-client' - -/** - * Initialize the Windmill client with authentication token and base URL - * @param token - Authentication token (defaults to WM_TOKEN env variable) - * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) - */ -setClient(token?: string, baseUrl?: string): void - -/** - * Create a client configuration from env variables - * @returns client configuration - */ -getWorkspace(): string - -/** - * Get a resource value by path - * @param path path of the resource, default to internal state path - * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error - * @returns resource value - */ -async getResource(path?: string, undefinedIfEmpty?: boolean): Promise - -/** - * Get the true root job id - * @param jobId job id to get the root job id from (default to current job) - * @returns root job id - */ -async getRootJobId(jobId?: string): Promise - -/** - * @deprecated Use runScriptByPath or runScriptByHash instead - */ -async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Run a script synchronously by its path and wait for the result - * @param path - Script path in Windmill - * @param args - Arguments to pass to the script - * @param verbose - Enable verbose logging - * @returns Script execution result - */ -async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Run a script synchronously by its hash and wait for the result - * @param hash_ - Script hash in Windmill - * @param args - Arguments to pass to the script - * @param verbose - Enable verbose logging - * @returns Script execution result - */ -async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Append a text to the result stream - * @param text text to append to the result stream - */ -appendToResultStream(text: string): void - -/** - * Stream to the result stream - * @param stream stream to stream to the result stream - */ -async streamResult(stream: AsyncIterable): Promise - -/** - * Run a flow synchronously by its path and wait for the result - * @param path - Flow path in Windmill - * @param args - Arguments to pass to the flow - * @param verbose - Enable verbose logging - * @returns Flow execution result - */ -async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Wait for a job to complete and return its result - * @param jobId - ID of the job to wait for - * @param verbose - Enable verbose logging - * @returns Job result when completed - */ -async waitJob(jobId: string, verbose: boolean = false): Promise - -/** - * Get the result of a completed job - * @param jobId - ID of the completed job - * @returns Job result - */ -async getResult(jobId: string): Promise - -/** - * Get the result of a job if completed, or its current status - * @param jobId - ID of the job - * @returns Object with started, completed, success, and result properties - */ -async getResultMaybe(jobId: string): Promise - -/** - * Wrap a function to execute as a Windmill task within a flow context - * @param f - Function to wrap as a task - * @returns Async wrapper function that executes as a Windmill job - */ -task(f: (_: P) => T): (_: P) => Promise - -/** - * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead - */ -async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise - -/** - * Run a script asynchronously by its path - * @param path - Script path in Windmill - * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) - * @returns Job ID of the created job - */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise - -/** - * Run a script asynchronously by its hash - * @param hash_ - Script hash in Windmill - * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) - * @returns Job ID of the created job - */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise - -/** - * Run a flow asynchronously by its path - * @param path - Flow path in Windmill - * @param args - Arguments to pass to the flow - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) - * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) - * @returns Job ID of the created job - */ -async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise - -/** - * Resolve a resource value in case the default value was picked because the input payload was undefined - * @param obj resource value or path of the resource under the format \`$res:path\` - * @returns resource value - */ -async resolveDefaultResource(obj: any): Promise - -/** - * Get the state file path from environment variables - * @returns State path string - */ -getStatePath(): string - -/** - * Set a resource value by path - * @param path path of the resource to set, default to state path - * @param value new value of the resource to set - * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type - */ -async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise - -/** - * Set the state - * @param state state to set - * @deprecated use setState instead - */ -async setInternalState(state: any): Promise - -/** - * Set the state - * @param state state to set - * @param path Optional state resource path override. Defaults to \`getStatePath()\`. - */ -async setState(state: any, path?: string): Promise - -/** - * Set the progress - * Progress cannot go back and limited to 0% to 99% range - * @param percent Progress to set in % - * @param jobId? Job to set progress for - */ -async setProgress(percent: number, jobId?: any): Promise - -/** - * Get the progress - * @param jobId? Job to get progress from - * @returns Optional clamped between 0 and 100 progress value - */ -async getProgress(jobId?: any): Promise - -/** - * Set a flow user state - * @param key key of the state - * @param value value of the state - */ -async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise - -/** - * Get a flow user state - * @param path path of the variable - */ -async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise - -/** - * Get the internal state - * @deprecated use getState instead - */ -async getInternalState(): Promise - -/** - * Get the state shared across executions - * @param path Optional state resource path override. Defaults to \`getStatePath()\`. - */ -async getState(path?: string): Promise - -/** - * Get a variable by path - * @param path path of the variable - * @returns variable value - */ -async getVariable(path: string): Promise - -/** - * Set a variable by path, create if not exist - * @param path path of the variable - * @param value value of the variable - * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) - * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") - */ -async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise - -/** - * Build a PostgreSQL connection URL from a database resource - * @param path - Path to the database resource - * @returns PostgreSQL connection URL string - */ -async databaseUrlFromResource(path: string): Promise - -async polarsConnectionSettings(s3_resource_path: string | undefined): Promise - -async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise - -/** - * Get S3 client settings from a resource or workspace default - * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) - * @returns S3 client configuration settings - */ -async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise - -/** - * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. - * - * \`\`\`typescript - * let fileContent = await wmill.loadS3FileContent(inputFile) - * // if the file is a raw text file, it can be decoded and printed directly: - * const text = new TextDecoder().decode(fileContentStream) - * console.log(text); - * \`\`\` - */ -async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise - -/** - * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. - * - * \`\`\`typescript - * let fileContentBlob = await wmill.loadS3FileStream(inputFile) - * // if the content is plain text, the blob can be read directly: - * console.log(await fileContentBlob.text()); - * \`\`\` - */ -async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise - -/** - * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. - * - * \`\`\`typescript - * const s3object = await writeS3File(s3Object, "Hello Windmill!") - * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') - * console.log(fileContentAsUtf8Str) - * \`\`\` - */ -async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise - -/** - * Sign S3 objects to be used by anonymous users in public apps - * @param s3objects s3 objects to sign - * @returns signed s3 objects - */ -async signS3Objects(s3objects: S3Object[]): Promise - -/** - * Sign S3 object to be used by anonymous users in public apps - * @param s3object s3 object to sign - * @returns signed s3 object - */ -async signS3Object(s3object: S3Object): Promise - -/** - * Generate a presigned public URL for an array of S3 objects. - * If an S3 object is not signed yet, it will be signed first. - * @param s3Objects s3 objects to sign - * @returns list of signed public URLs - */ -async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise - -/** - * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. - * @param s3Object s3 object to sign - * @returns signed public URL - */ -async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise - -/** - * Get URLs needed for resuming a flow after this step - * @param approver approver name - * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. - * This allows pre-approvals that can be consumed by any later suspend step in the same flow. - * @returns approval page UI URL, resume and cancel API URLs for resuming the flow - */ -async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ - approvalPage: string; - resume: string; - cancel: string; -}> - -/** - * @deprecated use getResumeUrls instead - */ -getResumeEndpoints(approver?: string): Promise<{ - approvalPage: string; - resume: string; - cancel: string; -}> - -/** - * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) - * @param audience audience of the token - * @param expiresIn Optional number of seconds until the token expires - * @returns jwt token - */ -async getIdToken(audience: string, expiresIn?: number): Promise - -/** - * Convert a base64-encoded string to Uint8Array - * @param data - Base64-encoded string - * @returns Decoded Uint8Array - */ -base64ToUint8Array(data: string): Uint8Array - -/** - * Convert a Uint8Array to base64-encoded string - * @param arrayBuffer - Uint8Array to encode - * @returns Base64-encoded string - */ -uint8ArrayToBase64(arrayBuffer: Uint8Array): string - -/** - * Get email from workspace username - * This method is particularly useful for apps that require the email address of the viewer. - * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. - * @param username - * @returns email address - */ -async usernameToEmail(username: string): Promise - -/** - * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. - * - * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** - * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). - * - * @param {Object} options - The configuration options for the Slack approval request. - * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. - * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. - * @param {string} [options.message] - Optional custom message to include in the Slack approval request. - * @param {string} [options.approver] - Optional user ID or name of the approver for the request. - * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. - * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. - * - * @returns {Promise} Resolves when the Slack approval request is successfully sent. - * - * @throws {Error} If the function is not called within a flow or flow preview. - * @throws {Error} If the \`JobService.getSlackApprovalPayload\` call fails. - * - * **Usage Example:** - * \`\`\`typescript - * await requestInteractiveSlackApproval({ - * slackResourcePath: "/u/alex/my_slack_resource", - * channelId: "admins-slack-channel", - * message: "Please approve this request", - * approver: "approver123", - * defaultArgsJson: { key1: "value1", key2: 42 }, - * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, - * }); - * \`\`\` - * - * **Note:** This function requires execution within a Windmill flow or flow preview. - */ -async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise - -/** - * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. - * - * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** - * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). - * - * @param {Object} options - The configuration options for the Teams approval request. - * @param {string} options.teamName - The Teams team name where the approval request will be sent. - * @param {string} options.channelName - The Teams channel name where the approval request will be sent. - * @param {string} [options.message] - Optional custom message to include in the Teams approval request. - * @param {string} [options.approver] - Optional user ID or name of the approver for the request. - * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. - * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. - * - * @returns {Promise} Resolves when the Teams approval request is successfully sent. - * - * @throws {Error} If the function is not called within a flow or flow preview. - * @throws {Error} If the \`JobService.getTeamsApprovalPayload\` call fails. - * - * **Usage Example:** - * \`\`\`typescript - * await requestInteractiveTeamsApproval({ - * teamName: "admins-teams", - * channelName: "admins-teams-channel", - * message: "Please approve this request", - * approver: "approver123", - * defaultArgsJson: { key1: "value1", key2: 42 }, - * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, - * }); - * \`\`\` - * - * **Note:** This function requires execution within a Windmill flow or flow preview. - */ -async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise - -/** - * Parse an S3 object from URI string or record format - * @param s3Object - S3 object as URI string (s3://storage/key) or record - * @returns S3 object record with storage and s3 key - */ -parseS3Object(s3Object: S3Object): S3ObjectRecord - -/** - * Create a SQL template function for PostgreSQL/datatable queries - * @param name - Database/datatable name (default: "main") - * @returns SQL template function for building parameterized queries - * @example - * let sql = wmill.datatable() - * let name = 'Robin' - * let age = 21 - * await sql\` - * SELECT * FROM friends - * WHERE name = \${name} AND age = \${age}::int - * \`.fetch() - */ -datatable(name: string = "main"): DatatableSqlTemplateFunction - -/** - * Create a SQL template function for DuckDB/ducklake queries - * @param name - DuckDB database name (default: "main") - * @returns SQL template function for building parameterized queries - * @example - * let sql = wmill.ducklake() - * let name = 'Robin' - * let age = 21 - * await sql\` - * SELECT * FROM friends - * WHERE name = \${name} AND age = \${age} - * \`.fetch() - */ -ducklake(name: string = "main"): SqlTemplateFunction -`, - "write-script-bash": `--- -name: write-script-bash -description: MUST use when writing Bash scripts. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# Bash - -## Structure - -Do not include \`#!/bin/bash\`. Arguments are obtained as positional parameters: - -\`\`\`bash -# Get arguments -var1="$1" -var2="$2" - -echo "Processing $var1 and $var2" - -# Return JSON by echoing to stdout -echo "{\\"result\\": \\"$var1\\", \\"count\\": $var2}" -\`\`\` - -**Important:** -- Do not include shebang (\`#!/bin/bash\`) -- Arguments are always strings -- Access with \`$1\`, \`$2\`, etc. - -## Output - -The script output is captured as the result. For structured data, output valid JSON: - -\`\`\`bash -name="$1" -count="$2" - -# Output JSON result -cat << EOF -{ - "name": "$name", - "count": $count, - "timestamp": "$(date -Iseconds)" -} -EOF -\`\`\` - -## Environment Variables - -Environment variables set in Windmill are available: - -\`\`\`bash -# Access environment variable -echo "Workspace: $WM_WORKSPACE" -echo "Job ID: $WM_JOB_ID" -\`\`\` -`, - "write-script-bunnative": `--- -name: write-script-bunnative -description: MUST use when writing Bun Native scripts. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# TypeScript (Bun Native) - -Native TypeScript execution with fetch only - no external imports allowed. - -## Structure - -Export a single **async** function called \`main\`: - -\`\`\`typescript -export async function main(param1: string, param2: number) { - // Your code here - return { result: param1, count: param2 }; -} -\`\`\` - -Do not call the main function. - -## Resource Types - -On Windmill, credentials and configuration are stored in resources and passed as parameters to main. - -Use the \`RT\` namespace for resource types: - -\`\`\`typescript -export async function main(stripe: RT.Stripe) { - // stripe contains API key and config from the resource -} -\`\`\` - -Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. - -## Imports - -**No imports allowed.** Use the globally available \`fetch\` function: - -\`\`\`typescript -export async function main(url: string) { - const response = await fetch(url); - return await response.json(); -} -\`\`\` - -## Windmill Client - -The windmill client is not available in native TypeScript mode. Use fetch to call APIs directly. - -## Preprocessor Scripts - -For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: - -\`\`\`typescript -type Event = { - kind: - | "webhook" - | "http" - | "websocket" - | "kafka" - | "email" - | "nats" - | "postgres" - | "sqs" - | "mqtt" - | "gcp"; - body: any; - headers: Record; - query: Record; -}; - -export async function preprocessor(event: Event) { - return { - param1: event.body.field1, - param2: event.query.id, - }; -} -\`\`\` - -## S3 Object Operations - -Windmill provides built-in support for S3-compatible storage operations. - -### S3Object Type - -The S3Object type represents a file in S3 storage: - -\`\`\`typescript -type S3Object = { - s3: string; // Path within the bucket -}; -\`\`\` - -## TypeScript Operations - -\`\`\`typescript -import * as wmill from "windmill-client"; - -// Load file content from S3 -const content: Uint8Array = await wmill.loadS3File(s3object); - -// Load file as stream -const blob: Blob = await wmill.loadS3FileStream(s3object); - -// Write file to S3 -const result: S3Object = await wmill.writeS3File( - s3object, // Target path (or undefined to auto-generate) - fileContent, // string or Blob - s3ResourcePath // Optional: specific S3 resource to use -); -\`\`\` - - -# TypeScript SDK (windmill-client) - -Import: import * as wmill from 'windmill-client' - -/** - * Initialize the Windmill client with authentication token and base URL - * @param token - Authentication token (defaults to WM_TOKEN env variable) - * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) - */ -setClient(token?: string, baseUrl?: string): void - -/** - * Create a client configuration from env variables - * @returns client configuration - */ -getWorkspace(): string - -/** - * Get a resource value by path - * @param path path of the resource, default to internal state path - * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error - * @returns resource value - */ -async getResource(path?: string, undefinedIfEmpty?: boolean): Promise - -/** - * Get the true root job id - * @param jobId job id to get the root job id from (default to current job) - * @returns root job id - */ -async getRootJobId(jobId?: string): Promise - -/** - * @deprecated Use runScriptByPath or runScriptByHash instead - */ -async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Run a script synchronously by its path and wait for the result - * @param path - Script path in Windmill - * @param args - Arguments to pass to the script - * @param verbose - Enable verbose logging - * @returns Script execution result - */ -async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Run a script synchronously by its hash and wait for the result - * @param hash_ - Script hash in Windmill - * @param args - Arguments to pass to the script - * @param verbose - Enable verbose logging - * @returns Script execution result - */ -async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Append a text to the result stream - * @param text text to append to the result stream - */ -appendToResultStream(text: string): void - -/** - * Stream to the result stream - * @param stream stream to stream to the result stream - */ -async streamResult(stream: AsyncIterable): Promise - -/** - * Run a flow synchronously by its path and wait for the result - * @param path - Flow path in Windmill - * @param args - Arguments to pass to the flow - * @param verbose - Enable verbose logging - * @returns Flow execution result - */ -async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Wait for a job to complete and return its result - * @param jobId - ID of the job to wait for - * @param verbose - Enable verbose logging - * @returns Job result when completed - */ -async waitJob(jobId: string, verbose: boolean = false): Promise - -/** - * Get the result of a completed job - * @param jobId - ID of the completed job - * @returns Job result - */ -async getResult(jobId: string): Promise - -/** - * Get the result of a job if completed, or its current status - * @param jobId - ID of the job - * @returns Object with started, completed, success, and result properties - */ -async getResultMaybe(jobId: string): Promise - -/** - * Wrap a function to execute as a Windmill task within a flow context - * @param f - Function to wrap as a task - * @returns Async wrapper function that executes as a Windmill job - */ -task(f: (_: P) => T): (_: P) => Promise - -/** - * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead - */ -async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise - -/** - * Run a script asynchronously by its path - * @param path - Script path in Windmill - * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) - * @returns Job ID of the created job - */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise - -/** - * Run a script asynchronously by its hash - * @param hash_ - Script hash in Windmill - * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) - * @returns Job ID of the created job - */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise - -/** - * Run a flow asynchronously by its path - * @param path - Flow path in Windmill - * @param args - Arguments to pass to the flow - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) - * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) - * @returns Job ID of the created job - */ -async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise - -/** - * Resolve a resource value in case the default value was picked because the input payload was undefined - * @param obj resource value or path of the resource under the format \`$res:path\` - * @returns resource value - */ -async resolveDefaultResource(obj: any): Promise - -/** - * Get the state file path from environment variables - * @returns State path string - */ -getStatePath(): string - -/** - * Set a resource value by path - * @param path path of the resource to set, default to state path - * @param value new value of the resource to set - * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type - */ -async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise - -/** - * Set the state - * @param state state to set - * @deprecated use setState instead - */ -async setInternalState(state: any): Promise - -/** - * Set the state - * @param state state to set - * @param path Optional state resource path override. Defaults to \`getStatePath()\`. - */ -async setState(state: any, path?: string): Promise - -/** - * Set the progress - * Progress cannot go back and limited to 0% to 99% range - * @param percent Progress to set in % - * @param jobId? Job to set progress for - */ -async setProgress(percent: number, jobId?: any): Promise - -/** - * Get the progress - * @param jobId? Job to get progress from - * @returns Optional clamped between 0 and 100 progress value - */ -async getProgress(jobId?: any): Promise - -/** - * Set a flow user state - * @param key key of the state - * @param value value of the state - */ -async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise - -/** - * Get a flow user state - * @param path path of the variable - */ -async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise - -/** - * Get the internal state - * @deprecated use getState instead - */ -async getInternalState(): Promise - -/** - * Get the state shared across executions - * @param path Optional state resource path override. Defaults to \`getStatePath()\`. - */ -async getState(path?: string): Promise - -/** - * Get a variable by path - * @param path path of the variable - * @returns variable value - */ -async getVariable(path: string): Promise - -/** - * Set a variable by path, create if not exist - * @param path path of the variable - * @param value value of the variable - * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) - * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") - */ -async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise - -/** - * Build a PostgreSQL connection URL from a database resource - * @param path - Path to the database resource - * @returns PostgreSQL connection URL string - */ -async databaseUrlFromResource(path: string): Promise - -async polarsConnectionSettings(s3_resource_path: string | undefined): Promise - -async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise - -/** - * Get S3 client settings from a resource or workspace default - * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) - * @returns S3 client configuration settings - */ -async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise - -/** - * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. - * - * \`\`\`typescript - * let fileContent = await wmill.loadS3FileContent(inputFile) - * // if the file is a raw text file, it can be decoded and printed directly: - * const text = new TextDecoder().decode(fileContentStream) - * console.log(text); - * \`\`\` - */ -async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise - -/** - * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. - * - * \`\`\`typescript - * let fileContentBlob = await wmill.loadS3FileStream(inputFile) - * // if the content is plain text, the blob can be read directly: - * console.log(await fileContentBlob.text()); - * \`\`\` - */ -async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise - -/** - * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. - * - * \`\`\`typescript - * const s3object = await writeS3File(s3Object, "Hello Windmill!") - * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') - * console.log(fileContentAsUtf8Str) - * \`\`\` - */ -async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise - -/** - * Sign S3 objects to be used by anonymous users in public apps - * @param s3objects s3 objects to sign - * @returns signed s3 objects - */ -async signS3Objects(s3objects: S3Object[]): Promise - -/** - * Sign S3 object to be used by anonymous users in public apps - * @param s3object s3 object to sign - * @returns signed s3 object - */ -async signS3Object(s3object: S3Object): Promise - -/** - * Generate a presigned public URL for an array of S3 objects. - * If an S3 object is not signed yet, it will be signed first. - * @param s3Objects s3 objects to sign - * @returns list of signed public URLs - */ -async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise - -/** - * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. - * @param s3Object s3 object to sign - * @returns signed public URL - */ -async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise - -/** - * Get URLs needed for resuming a flow after this step - * @param approver approver name - * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. - * This allows pre-approvals that can be consumed by any later suspend step in the same flow. - * @returns approval page UI URL, resume and cancel API URLs for resuming the flow - */ -async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ - approvalPage: string; - resume: string; - cancel: string; -}> - -/** - * @deprecated use getResumeUrls instead - */ -getResumeEndpoints(approver?: string): Promise<{ - approvalPage: string; - resume: string; - cancel: string; -}> - -/** - * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) - * @param audience audience of the token - * @param expiresIn Optional number of seconds until the token expires - * @returns jwt token - */ -async getIdToken(audience: string, expiresIn?: number): Promise - -/** - * Convert a base64-encoded string to Uint8Array - * @param data - Base64-encoded string - * @returns Decoded Uint8Array - */ -base64ToUint8Array(data: string): Uint8Array - -/** - * Convert a Uint8Array to base64-encoded string - * @param arrayBuffer - Uint8Array to encode - * @returns Base64-encoded string - */ -uint8ArrayToBase64(arrayBuffer: Uint8Array): string - -/** - * Get email from workspace username - * This method is particularly useful for apps that require the email address of the viewer. - * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. - * @param username - * @returns email address - */ -async usernameToEmail(username: string): Promise - -/** - * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. - * - * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** - * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). - * - * @param {Object} options - The configuration options for the Slack approval request. - * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. - * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. - * @param {string} [options.message] - Optional custom message to include in the Slack approval request. - * @param {string} [options.approver] - Optional user ID or name of the approver for the request. - * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. - * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. - * - * @returns {Promise} Resolves when the Slack approval request is successfully sent. - * - * @throws {Error} If the function is not called within a flow or flow preview. - * @throws {Error} If the \`JobService.getSlackApprovalPayload\` call fails. - * - * **Usage Example:** - * \`\`\`typescript - * await requestInteractiveSlackApproval({ - * slackResourcePath: "/u/alex/my_slack_resource", - * channelId: "admins-slack-channel", - * message: "Please approve this request", - * approver: "approver123", - * defaultArgsJson: { key1: "value1", key2: 42 }, - * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, - * }); - * \`\`\` - * - * **Note:** This function requires execution within a Windmill flow or flow preview. - */ -async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise - -/** - * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. - * - * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** - * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). - * - * @param {Object} options - The configuration options for the Teams approval request. - * @param {string} options.teamName - The Teams team name where the approval request will be sent. - * @param {string} options.channelName - The Teams channel name where the approval request will be sent. - * @param {string} [options.message] - Optional custom message to include in the Teams approval request. - * @param {string} [options.approver] - Optional user ID or name of the approver for the request. - * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. - * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. - * - * @returns {Promise} Resolves when the Teams approval request is successfully sent. - * - * @throws {Error} If the function is not called within a flow or flow preview. - * @throws {Error} If the \`JobService.getTeamsApprovalPayload\` call fails. - * - * **Usage Example:** - * \`\`\`typescript - * await requestInteractiveTeamsApproval({ - * teamName: "admins-teams", - * channelName: "admins-teams-channel", - * message: "Please approve this request", - * approver: "approver123", - * defaultArgsJson: { key1: "value1", key2: 42 }, - * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, - * }); - * \`\`\` - * - * **Note:** This function requires execution within a Windmill flow or flow preview. - */ -async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise - -/** - * Parse an S3 object from URI string or record format - * @param s3Object - S3 object as URI string (s3://storage/key) or record - * @returns S3 object record with storage and s3 key - */ -parseS3Object(s3Object: S3Object): S3ObjectRecord - -/** - * Create a SQL template function for PostgreSQL/datatable queries - * @param name - Database/datatable name (default: "main") - * @returns SQL template function for building parameterized queries - * @example - * let sql = wmill.datatable() - * let name = 'Robin' - * let age = 21 - * await sql\` - * SELECT * FROM friends - * WHERE name = \${name} AND age = \${age}::int - * \`.fetch() - */ -datatable(name: string = "main"): DatatableSqlTemplateFunction - -/** - * Create a SQL template function for DuckDB/ducklake queries - * @param name - DuckDB database name (default: "main") - * @returns SQL template function for building parameterized queries - * @example - * let sql = wmill.ducklake() - * let name = 'Robin' - * let age = 21 - * await sql\` - * SELECT * FROM friends - * WHERE name = \${name} AND age = \${age} - * \`.fetch() - */ -ducklake(name: string = "main"): SqlTemplateFunction `, "write-script-duckdb": `--- name: write-script-duckdb @@ -2858,6 +3461,69 @@ SELECT * FROM read_parquet('s3:///path/to/file.parquet'); -- JSON files SELECT * FROM read_json('s3:///path/to/file.json'); \`\`\` +`, + "write-script-bash": `--- +name: write-script-bash +description: MUST use when writing Bash scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Bash + +## Structure + +Do not include \`#!/bin/bash\`. Arguments are obtained as positional parameters: + +\`\`\`bash +# Get arguments +var1="$1" +var2="$2" + +echo "Processing $var1 and $var2" + +# Return JSON by echoing to stdout +echo "{\\"result\\": \\"$var1\\", \\"count\\": $var2}" +\`\`\` + +**Important:** +- Do not include shebang (\`#!/bin/bash\`) +- Arguments are always strings +- Access with \`$1\`, \`$2\`, etc. + +## Output + +The script output is captured as the result. For structured data, output valid JSON: + +\`\`\`bash +name="$1" +count="$2" + +# Output JSON result +cat << EOF +{ + "name": "$name", + "count": $count, + "timestamp": "$(date -Iseconds)" +} +EOF +\`\`\` + +## Environment Variables + +Environment variables set in Windmill are available: + +\`\`\`bash +# Access environment variable +echo "Workspace: $WM_WORKSPACE" +echo "Job ID: $WM_JOB_ID" +\`\`\` `, "write-script-nativets": `--- name: write-script-nativets @@ -2903,6 +3569,8 @@ export async function main(stripe: RT.Stripe) { Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. +Before using a resource type, check the \`rt.d.ts\` file in the project root to see all available resource types and their fields. This file is generated by \`wmill resource-type generate-namespace\`. + ## Imports **No imports allowed.** Use the globally available \`fetch\` function: @@ -2953,6 +3621,36 @@ export async function preprocessor(event: Event) { Import: import * as wmill from 'windmill-client' +/** + * Create a SQL template function for PostgreSQL/datatable queries + * @param name - Database/datatable name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age}::int + * \`.fetch() + */ +datatable(name: string = "main"): DatatableSqlTemplateFunction + +/** + * Create a SQL template function for DuckDB/ducklake queries + * @param name - DuckDB database name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age} + * \`.fetch() + */ +ducklake(name: string = "main"): SqlTemplateFunction + /** * Initialize the Windmill client with authentication token and base URL * @param token - Authentication token (defaults to WM_TOKEN env variable) @@ -3047,13 +3745,6 @@ async getResult(jobId: string): Promise */ async getResultMaybe(jobId: string): Promise -/** - * Wrap a function to execute as a Windmill task within a flow context - * @param f - Function to wrap as a task - * @returns Async wrapper function that executes as a Windmill job - */ -task(f: (_: P) => T): (_: P) => Promise - /** * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead */ @@ -3325,6 +4016,8 @@ async usernameToEmail(username: string): Promise * @param {string} [options.approver] - Optional user ID or name of the approver for the request. * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * @param {string} [options.resumeButtonText] - Optional text for the resume button. + * @param {string} [options.cancelButtonText] - Optional text for the cancel button. * * @returns {Promise} Resolves when the Slack approval request is successfully sent. * @@ -3340,12 +4033,14 @@ async usernameToEmail(username: string): Promise * approver: "approver123", * defaultArgsJson: { key1: "value1", key2: 42 }, * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * resumeButtonText: "Resume", + * cancelButtonText: "Cancel", * }); * \`\`\` * * **Note:** This function requires execution within a Windmill flow or flow preview. */ -async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise +async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, resumeButtonText, cancelButtonText, }: SlackApprovalOptions): Promise /** * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. @@ -3389,436 +4084,65 @@ async requestInteractiveTeamsApproval({ teamName, channelName, message, approver */ parseS3Object(s3Object: S3Object): S3ObjectRecord -/** - * Create a SQL template function for PostgreSQL/datatable queries - * @param name - Database/datatable name (default: "main") - * @returns SQL template function for building parameterized queries - * @example - * let sql = wmill.datatable() - * let name = 'Robin' - * let age = 21 - * await sql\` - * SELECT * FROM friends - * WHERE name = \${name} AND age = \${age}::int - * \`.fetch() - */ -datatable(name: string = "main"): DatatableSqlTemplateFunction +setWorkflowCtx(ctx: WorkflowCtx | null): void + +async sleep(seconds: number): Promise + +async step(name: string, fn: () => T | Promise): Promise /** - * Create a SQL template function for DuckDB/ducklake queries - * @param name - DuckDB database name (default: "main") - * @returns SQL template function for building parameterized queries + * Create a task that dispatches to a separate Windmill script. + * * @example - * let sql = wmill.ducklake() - * let name = 'Robin' - * let age = 21 - * await sql\` - * SELECT * FROM friends - * WHERE name = \${name} AND age = \${age} - * \`.fetch() + * const extract = taskScript("f/data/extract"); + * // inside workflow: await extract({ url: "https://..." }) */ -ducklake(name: string = "main"): SqlTemplateFunction -`, - "write-script-bigquery": `--- -name: write-script-bigquery -description: MUST use when writing BigQuery queries. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# BigQuery - -Arguments use \`@name\` syntax. - -Name the parameters by adding comments before the statement: - -\`\`\`sql --- @name1 (string) --- @name2 (int64) = 0 -SELECT * FROM users WHERE name = @name1 AND age > @name2; -\`\`\` -`, - "write-script-rust": `--- -name: write-script-rust -description: MUST use when writing Rust scripts. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# Rust - -## Structure - -The script must contain a function called \`main\` with proper return type: - -\`\`\`rust -use anyhow::anyhow; -use serde::Serialize; - -#[derive(Serialize, Debug)] -struct ReturnType { - result: String, - count: i32, -} - -fn main(param1: String, param2: i32) -> anyhow::Result { - Ok(ReturnType { - result: param1, - count: param2, - }) -} -\`\`\` - -**Important:** -- Arguments should be owned types -- Return type must be serializable (\`#[derive(Serialize)]\`) -- Return type is \`anyhow::Result\` - -## Dependencies - -Packages must be specified with a partial cargo.toml at the beginning of the script: - -\`\`\`rust -//! \`\`\`cargo -//! [dependencies] -//! anyhow = "1.0.86" -//! reqwest = { version = "0.11", features = ["json"] } -//! tokio = { version = "1", features = ["full"] } -//! \`\`\` - -use anyhow::anyhow; -// ... rest of the code -\`\`\` - -**Note:** Serde is already included, no need to add it again. - -## Async Functions - -If you need to handle async functions (e.g., using tokio), keep the main function sync and create the runtime inside: - -\`\`\`rust -//! \`\`\`cargo -//! [dependencies] -//! anyhow = "1.0.86" -//! tokio = { version = "1", features = ["full"] } -//! reqwest = { version = "0.11", features = ["json"] } -//! \`\`\` - -use anyhow::anyhow; -use serde::Serialize; - -#[derive(Serialize, Debug)] -struct Response { - data: String, -} - -fn main(url: String) -> anyhow::Result { - let rt = tokio::runtime::Runtime::new()?; - rt.block_on(async { - let resp = reqwest::get(&url).await?.text().await?; - Ok(Response { data: resp }) - }) -} -\`\`\` -`, - "write-script-php": `--- -name: write-script-php -description: MUST use when writing PHP scripts. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# PHP - -## Structure - -The script must start with \` $param1, "count" => $param2]; -} -\`\`\` - -## Resource Types - -On Windmill, credentials and configuration are stored in resources and passed as parameters to main. - -You need to **redefine** the type of the resources that are needed before the main function. Always check if the class already exists using \`class_exists\`: - -\`\`\`php - @P2; -\`\`\` -`, - "write-script-postgresql": `--- -name: write-script-postgresql -description: MUST use when writing PostgreSQL queries. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# PostgreSQL - -Arguments are obtained directly in the statement with \`$1::{type}\`, \`$2::{type}\`, etc. - -Name the parameters by adding comments at the beginning of the script (without specifying the type): - -\`\`\`sql --- $1 name1 --- $2 name2 = default_value -SELECT * FROM users WHERE name = $1::TEXT AND age > $2::INT; -\`\`\` -`, - "write-script-graphql": `--- -name: write-script-graphql -description: MUST use when writing GraphQL queries. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# GraphQL - -## Structure - -Write GraphQL queries or mutations. Arguments can be added as query parameters: - -\`\`\`graphql -query GetUser($id: ID!) { - user(id: $id) { - id - name - email - } -} -\`\`\` - -## Variables - -Variables are passed as script arguments and automatically bound to the query: - -\`\`\`graphql -query SearchProducts($query: String!, $limit: Int = 10) { - products(search: $query, first: $limit) { - edges { - node { - id - name - price - } - } - } -} -\`\`\` - -## Mutations - -\`\`\`graphql -mutation CreateUser($input: CreateUserInput!) { - createUser(input: $input) { - id - name - createdAt - } -} -\`\`\` -`, - "write-script-csharp": `--- -name: write-script-csharp -description: MUST use when writing C# scripts. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# C# - -The script must contain a public static \`Main\` method inside a class: - -\`\`\`csharp -public class Script -{ - public static object Main(string name, int count) - { - return new { Name = name, Count = count }; - } -} -\`\`\` - -**Important:** -- Class name is irrelevant -- Method must be \`public static\` -- Return type can be \`object\` or specific type - -## NuGet Packages - -Add packages using the \`#r\` directive at the top: - -\`\`\`csharp -#r "nuget: Newtonsoft.Json, 13.0.3" -#r "nuget: RestSharp, 110.2.0" - -using Newtonsoft.Json; -using RestSharp; - -public class Script -{ - public static object Main(string url) - { - var client = new RestClient(url); - var request = new RestRequest(); - var response = client.Get(request); - return JsonConvert.DeserializeObject(response.Content); - } -} -\`\`\` -`, - "write-script-java": `--- -name: write-script-java -description: MUST use when writing Java scripts. ---- - -## CLI Commands - -Place scripts in a folder. After writing, run: -- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files -- \`wmill sync push\` - Deploy to Windmill - -Use \`wmill resource-type list --schema\` to discover available resource types. - -# Java - -The script must contain a Main public class with a \`public static main()\` method: - -\`\`\`java -public class Main { - public static Object main(String name, int count) { - java.util.Map result = new java.util.HashMap<>(); - result.put("name", name); - result.put("count", count); - return result; - } -} -\`\`\` - -**Important:** -- Class must be named \`Main\` -- Method must be \`public static Object main(...)\` -- Return type is \`Object\` or \`void\` - -## Maven Dependencies - -Add dependencies using comments at the top: - -\`\`\`java -//requirements: -//com.google.code.gson:gson:2.10.1 -//org.apache.httpcomponents:httpclient:4.5.14 - -import com.google.gson.Gson; - -public class Main { - public static Object main(String input) { - Gson gson = new Gson(); - return gson.fromJson(input, Object.class); - } -} -\`\`\` +taskScript(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike + +/** + * Create a task that dispatches to a separate Windmill flow. + * + * @example + * const pipeline = taskFlow("f/etl/pipeline"); + * // inside workflow: await pipeline({ input: data }) + */ +taskFlow(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike + +/** + * Mark an async function as a workflow-as-code entry point. + * + * The function must be **deterministic**: given the same inputs it must call + * tasks in the same order on every replay. Branching on task results is fine + * (results are replayed from checkpoint), but branching on external state + * (current time, random values, external API calls) must use \`step()\` to + * checkpoint the value so replays see the same result. + */ +workflow(fn: (...args: any[]) => Promise): void + +/** + * Suspend the workflow and wait for an external approval. + * + * Use \`getResumeUrls()\` (wrapped in \`step()\`) to obtain resume/cancel/approvalPage + * URLs before calling this function. + * + * @example + * const urls = await step("urls", () => getResumeUrls()); + * await step("notify", () => sendEmail(urls.approvalPage)); + * const { value, approver } = await waitForApproval({ timeout: 3600 }); + */ +waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> + +/** + * Process items in parallel with optional concurrency control. + * + * Each item is processed by calling \`fn(item)\`, which should be a task(). + * Items are dispatched in batches of \`concurrency\` (default: all at once). + * + * @example + * const process = task(async (item: string) => { ... }); + * const results = await parallel(items, process, { concurrency: 5 }); + */ +async parallel(items: T[], fn: (item: T) => PromiseLike | R, options?: { concurrency?: number },): Promise `, "write-flow": `--- name: write-flow @@ -3832,7 +4156,7 @@ description: MUST use when creating flows. Create a folder ending with \`.flow\` and add a YAML file with the flow definition. For rawscript modules, use \`!inline path/to/script.ts\` for the content key. After writing: -- \`wmill flow generate-locks --yes\` - Generate lock files +- \`wmill flow generate-locks --yes\` - Generate lock files for the specific flow you modified (e.g. \`wmill flow generate-locks f/my_folder/my_flow.flow --yes\`) - \`wmill sync push\` - Deploy to Windmill ## OpenFlow Schema @@ -3945,7 +4269,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"number","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"number","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"number","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"flow_env":{"type":"object","description":"Environment variables available to all steps","additionalProperties":{"type":"string"}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","description":"Custom error message shown when stopping"}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_use":{"type":"boolean","description":"If true, this step's result is deleted after use to save memory"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_images":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of image references for vision-capable models.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'images/photo.jpg' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"number","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"number","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"number","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","description":"Custom error message shown when stopping"}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_use":{"type":"boolean","description":"If true, this step's result is deleted after use to save memory"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_images":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of image references for vision-capable models.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'images/photo.jpg' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, "raw-app": `--- name: raw-app description: MUST use when creating raw apps. @@ -4540,7 +4864,7 @@ description: MUST use when using the CLI. The Windmill CLI (\`wmill\`) provides commands for managing scripts, flows, apps, and other resources. -Current version: 1.642.0 +Current version: 1.651.1 ## Global Options @@ -4598,6 +4922,15 @@ Launch a dev server that will spawn a webserver with HMR **Options:** - \`--includes \` - Filter paths givena glob pattern or path +### docs + +Search Windmill documentation. Requires Enterprise Edition. + +**Arguments:** \`\` + +**Options:** +- \`--json\` - Output results as JSON. + ### flow flow related commands @@ -4646,7 +4979,7 @@ folder related commands - \`--json\` - Output as JSON (for piping to jq) - \`folder new \` - create a new folder locally - \`--summary \` - folder summary -- \`folder push \` - push a local folder to the remote by name. This overrides any remote versions. +- \`folder push \` - push a local folder to the remote by name. This overrides any remote versions. - \`folder add-missing\` - create default folder.meta.yaml for all subdirectories of f/ that are missing one - \`-y, --yes\` - skip confirmation prompt diff --git a/cli/src/main.ts b/cli/src/main.ts index 38b68bb0f4..73e1aeb8cb 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -67,7 +67,7 @@ export { workspaceAdd, }; -export const VERSION = "1.648.0"; +export const VERSION = "1.654.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/cli/test/gitsync_converter_unit.test.ts b/cli/test/gitsync_converter_unit.test.ts new file mode 100644 index 0000000000..e84c2ffa11 --- /dev/null +++ b/cli/test/gitsync_converter_unit.test.ts @@ -0,0 +1,227 @@ +/** + * Unit tests for GitSyncSettingsConverter. + * Tests conversion between backend format (include_type array) and + * SyncOptions format (boolean flags like skipWorkspaceDependencies). + */ + +import { expect, test, describe } from "bun:test"; +import { GitSyncSettingsConverter } from "../src/commands/gitsync-settings/converter.ts"; + +// ============================================================================= +// fromBackendFormat - converts backend include_type array to SyncOptions +// ============================================================================= + +describe("GitSyncSettingsConverter.fromBackendFormat", () => { + test("converts workspacedependencies in include_type to skipWorkspaceDependencies: false", () => { + const backend = { + include_path: ["f/**"], + include_type: ["script", "flow", "workspacedependencies"], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + expect(result.skipWorkspaceDependencies).toBe(false); + }); + + test("sets skipWorkspaceDependencies: true when workspacedependencies is absent", () => { + const backend = { + include_path: ["f/**"], + include_type: ["script", "flow"], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + expect(result.skipWorkspaceDependencies).toBe(true); + }); + + test("handles empty include_type array", () => { + const backend = { + include_path: [], + include_type: [], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + expect(result.skipWorkspaceDependencies).toBe(true); + expect(result.skipScripts).toBe(true); + expect(result.skipFlows).toBe(true); + }); + + test("converts all standard types correctly", () => { + const backend = { + include_path: ["f/**"], + include_type: ["script", "flow", "app", "folder", "variable", "resource", "resourcetype", "secret", "schedule", "trigger", "user", "group", "settings", "key", "workspacedependencies"], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + + expect(result.skipScripts).toBe(false); + expect(result.skipFlows).toBe(false); + expect(result.skipApps).toBe(false); + expect(result.skipFolders).toBe(false); + expect(result.skipVariables).toBe(false); + expect(result.skipResources).toBe(false); + expect(result.skipResourceTypes).toBe(false); + expect(result.skipSecrets).toBe(false); + expect(result.includeSchedules).toBe(true); + expect(result.includeTriggers).toBe(true); + expect(result.includeUsers).toBe(true); + expect(result.includeGroups).toBe(true); + expect(result.includeSettings).toBe(true); + expect(result.includeKey).toBe(true); + expect(result.skipWorkspaceDependencies).toBe(false); + }); +}); + +// ============================================================================= +// toBackendFormat - converts SyncOptions to backend include_type array +// ============================================================================= + +describe("GitSyncSettingsConverter.toBackendFormat", () => { + test("adds workspacedependencies when skipWorkspaceDependencies is false", () => { + const opts = { + includes: ["f/**"], + skipWorkspaceDependencies: false, + }; + const result = GitSyncSettingsConverter.toBackendFormat(opts); + expect(result.include_type).toContain("workspacedependencies"); + }); + + test("does not add workspacedependencies when skipWorkspaceDependencies is true", () => { + const opts = { + includes: ["f/**"], + skipWorkspaceDependencies: true, + }; + const result = GitSyncSettingsConverter.toBackendFormat(opts); + expect(result.include_type).not.toContain("workspacedependencies"); + }); + + test("adds workspacedependencies when skipWorkspaceDependencies is undefined (defaults to false)", () => { + const opts = { + includes: ["f/**"], + // skipWorkspaceDependencies not set + }; + const normalized = GitSyncSettingsConverter.normalize(opts); + const result = GitSyncSettingsConverter.toBackendFormat(normalized); + expect(result.include_type).toContain("workspacedependencies"); + }); + + test("converts all boolean flags to include_type correctly", () => { + const opts = { + includes: ["f/**"], + skipScripts: false, + skipFlows: false, + skipApps: false, + skipFolders: false, + skipVariables: false, + skipResources: false, + skipResourceTypes: false, + skipSecrets: false, + includeSchedules: true, + includeTriggers: true, + includeUsers: true, + includeGroups: true, + includeSettings: true, + includeKey: true, + skipWorkspaceDependencies: false, + }; + const result = GitSyncSettingsConverter.toBackendFormat(opts); + + expect(result.include_type).toContain("script"); + expect(result.include_type).toContain("flow"); + expect(result.include_type).toContain("app"); + expect(result.include_type).toContain("folder"); + expect(result.include_type).toContain("variable"); + expect(result.include_type).toContain("resource"); + expect(result.include_type).toContain("resourcetype"); + expect(result.include_type).toContain("secret"); + expect(result.include_type).toContain("schedule"); + expect(result.include_type).toContain("trigger"); + expect(result.include_type).toContain("user"); + expect(result.include_type).toContain("group"); + expect(result.include_type).toContain("settings"); + expect(result.include_type).toContain("key"); + expect(result.include_type).toContain("workspacedependencies"); + }); +}); + +// ============================================================================= +// normalize - applies defaults for undefined fields +// ============================================================================= + +describe("GitSyncSettingsConverter.normalize", () => { + test("defaults skipWorkspaceDependencies to false", () => { + const opts = { includes: ["f/**"] }; + const result = GitSyncSettingsConverter.normalize(opts); + expect(result.skipWorkspaceDependencies).toBe(false); + }); + + test("preserves explicit skipWorkspaceDependencies: true", () => { + const opts = { includes: ["f/**"], skipWorkspaceDependencies: true }; + const result = GitSyncSettingsConverter.normalize(opts); + expect(result.skipWorkspaceDependencies).toBe(true); + }); + + test("preserves explicit skipWorkspaceDependencies: false", () => { + const opts = { includes: ["f/**"], skipWorkspaceDependencies: false }; + const result = GitSyncSettingsConverter.normalize(opts); + expect(result.skipWorkspaceDependencies).toBe(false); + }); +}); + +// ============================================================================= +// Round-trip conversion tests +// ============================================================================= + +describe("GitSyncSettingsConverter round-trip", () => { + test("backend -> SyncOptions -> backend preserves workspacedependencies", () => { + const original = { + include_path: ["f/**"], + include_type: ["script", "flow", "workspacedependencies"], + }; + + const syncOpts = GitSyncSettingsConverter.fromBackendFormat(original); + const backAgain = GitSyncSettingsConverter.toBackendFormat(syncOpts); + + expect(backAgain.include_type).toContain("workspacedependencies"); + expect(backAgain.include_type).toContain("script"); + expect(backAgain.include_type).toContain("flow"); + }); + + test("backend without workspacedependencies -> SyncOptions -> backend still excludes it", () => { + const original = { + include_path: ["f/**"], + include_type: ["script", "flow"], + }; + + const syncOpts = GitSyncSettingsConverter.fromBackendFormat(original); + const backAgain = GitSyncSettingsConverter.toBackendFormat(syncOpts); + + expect(backAgain.include_type).not.toContain("workspacedependencies"); + expect(backAgain.include_type).toContain("script"); + expect(backAgain.include_type).toContain("flow"); + }); + + test("SyncOptions with defaults -> backend includes workspacedependencies", () => { + const opts = { + includes: ["f/**"], + skipScripts: false, + skipFlows: false, + // skipWorkspaceDependencies not set - should default to false + }; + + const normalized = GitSyncSettingsConverter.normalize(opts); + const backend = GitSyncSettingsConverter.toBackendFormat(normalized); + + expect(backend.include_type).toContain("workspacedependencies"); + }); +}); + +// ============================================================================= +// extractGitSyncFields +// ============================================================================= + +describe("GitSyncSettingsConverter.extractGitSyncFields", () => { + test("includes skipWorkspaceDependencies in extracted fields", () => { + const opts = { + includes: ["f/**"], + skipWorkspaceDependencies: true, + someOtherField: "ignored", + }; + const result = GitSyncSettingsConverter.extractGitSyncFields(opts); + expect(result.skipWorkspaceDependencies).toBe(true); + }); +}); diff --git a/cli/test/settings_unit.test.ts b/cli/test/settings_unit.test.ts index 2d6a5249ef..014456cfc0 100644 --- a/cli/test/settings_unit.test.ts +++ b/cli/test/settings_unit.test.ts @@ -193,5 +193,21 @@ describe("migrateToGroupedFormat", () => { expect("webhook" in result).toBe(false); expect("deploy_to" in result).toBe(false); expect("color" in result).toBe(false); + expect("slack_team_id" in result).toBe(false); + expect("slack_name" in result).toBe(false); + expect("slack_command_script" in result).toBe(false); + }); + + test("copies slack fields through", () => { + const settings = { + name: "ws", + slack_team_id: "T12345", + slack_name: "my-team", + slack_command_script: "u/admin/slack_handler", + }; + const result = migrateToGroupedFormat(settings); + expect(result.slack_team_id).toBe("T12345"); + expect(result.slack_name).toBe("my-team"); + expect(result.slack_command_script).toBe("u/admin/slack_handler"); }); }); diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 450dd68399..61a365e496 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -61,8 +61,13 @@ RUN ln -s /usr/bin/bun /usr/bin/node \ && bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# Copy to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + # add the docker client to call docker from a worker if enabled -COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ +COPY --from=docker:29-dind /usr/local/bin/docker /usr/local/bin/ # nsjail runtime deps and binary RUN apt-get update && apt-get install -y libprotobuf-dev libnl-route-3-dev \ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 7cc4dafa05..7ba85a0358 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -61,8 +61,13 @@ RUN ln -s /usr/bin/bun /usr/bin/node \ && bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# Copy to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + # add the docker client to call docker from a worker if enabled -COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ +COPY --from=docker:29-dind /usr/local/bin/docker /usr/local/bin/ # nsjail runtime deps and binary RUN apt-get update && apt-get install -y libprotobuf-dev libnl-route-3-dev \ diff --git a/frontend/BUGS.txt b/frontend/BUGS.txt new file mode 100644 index 0000000000..aae0762777 --- /dev/null +++ b/frontend/BUGS.txt @@ -0,0 +1,39 @@ +# Svelte 5 Migration - Bug Report +# Testing started: 2026-03-02 + +## Warnings (not blocking but worth fixing) + +1. [WARNING] binding_property_non_reactive in Grid.svelte:372:5 + - `bind:this={moveResizes[item.id]}` is binding to a non-reactive property + - File: src/lib/components/apps/svelte-grid/Grid.svelte + - Appears multiple times in App editor + - Status: NOT FIXED (non-blocking warning) + +2. [WARNING] legacy_recursive_reactive_block in RecomputeAllComponents.svelte + - Migrated `$:` reactive block that both accesses and updates the same reactive value + - File: src/lib/components/apps/editor/RecomputeAllComponents.svelte + - May cause recursive updates when converted to $effect + - Status: NOT FIXED (non-blocking warning) + +3. [WARNING] ownership_invalid_mutation in SchemaForm.svelte:70:16 + - Mutating unbound props (`schema`) is strongly discouraged + - Parent: src/lib/components/ApiConnectForm.svelte should use `bind:schema={...}` + - Appears when opening PostgreSQL resource creation form + - Status: NOT FIXED (non-blocking warning) + +4. [WARNING] ownership_invalid_binding in InputTransformSchemaForm.svelte + - Passes `schema` to InputTransformForm.svelte with `bind:`, but parent Pane.svelte didn't declare `schema` as binding + - Appears in flow editor when adding a TypeScript step + - Status: NOT FIXED (non-blocking warning) + +## Bugs + +1. [BUG] state_descriptors_fixed in Chart.svelte (Queue metrics drawer) + - Error: "Property descriptors defined on `$state` objects must contain `value` and always be `enumerable`, `configurable` and `writable`." + - Triggered by: Clicking "Queue metrics" on /workers page + - File: src/lib/components/chartjs-wrappers/Chart.svelte + - Root cause: Chart.js's `listenArrayEvents` calls Object.defineProperty on data arrays that are Svelte 5 $state proxies, which reject non-standard property descriptors + - Fix: Use $state.snapshot() to pass plain copies of data and options to Chart.js + - Status: FIXED + + diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 520148b1d6..aebb63af83 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.648.0", + "version": "1.654.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.648.0", + "version": "1.654.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -75,16 +75,17 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", + "windmill-parser-wasm-asset": "1.653.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", - "windmill-parser-wasm-py": "^1.628.3", - "windmill-parser-wasm-regex": "1.646.0", + "windmill-parser-wasm-py": "1.653.0", + "windmill-parser-wasm-regex": "1.653.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", - "windmill-parser-wasm-ts": "1.647.1", + "windmill-parser-wasm-ts": "1.653.0", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "^1.3.4", @@ -143,7 +144,7 @@ "svelte-range-slider-pips": "^2.3.1", "svelte-splitpanes": "^8.0.9", "tailwindcss": "^3.4.1", - "tar": "^7.4.3", + "tar": "^7.5.4", "tslib": "^2.6.1", "typescript": "^5.5.0", "vite": "^8.0.0-beta.16", @@ -5523,39 +5524,6 @@ "license": "MIT", "optional": true }, - "node_modules/fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "dev": true, - "license": "ISC", - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/fs-minipass/node_modules/minipass": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", - "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/fs-minipass/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, - "license": "ISC" - }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -5680,53 +5648,6 @@ "giget": "dist/cli.mjs" } }, - "node_modules/giget/node_modules/chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/giget/node_modules/minipass": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", - "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=8" - } - }, - "node_modules/giget/node_modules/minizlib": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", - "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", - "dev": true, - "license": "MIT", - "dependencies": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/giget/node_modules/minizlib/node_modules/minipass": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", - "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/giget/node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -5734,31 +5655,6 @@ "dev": true, "license": "MIT" }, - "node_modules/giget/node_modules/tar": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", - "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", - "dev": true, - "license": "ISC", - "dependencies": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", - "minipass": "^5.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/giget/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, - "license": "ISC" - }, "node_modules/github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", @@ -8265,19 +8161,6 @@ "node": ">= 18" } }, - "node_modules/mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true, - "license": "MIT", - "bin": { - "mkdirp": "bin/cmd.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", @@ -11651,11 +11534,11 @@ } }, "node_modules/tar": { - "version": "7.5.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.1.tgz", - "integrity": "sha512-nlGpxf+hv0v7GkWBK2V9spgactGOp0qvfWRxUMjqHyzrt3SgwE48DIv/FhqPHJYLHpgW1opq3nERbz5Anq7n1g==", + "version": "7.5.11", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", + "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", "dev": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", @@ -12698,6 +12581,11 @@ "node": ">=8" } }, + "node_modules/windmill-parser-wasm-asset": { + "version": "1.653.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.653.0.tgz", + "integrity": "sha512-Tc9smy79wZSEBxAbiad8D4NiydOVHl541amCRQ93yqqffsmFnyRr4tpBYV/dR5uQ4zdX7aTe5ROwz2jhsXODYQ==" + }, "node_modules/windmill-parser-wasm-csharp": { "version": "1.510.1", "resolved": "https://registry.npmjs.org/windmill-parser-wasm-csharp/-/windmill-parser-wasm-csharp-1.510.1.tgz", @@ -12724,14 +12612,14 @@ "integrity": "sha512-u2qaMkupSdhJibxvkLh3r/y36IARvnYNTLXWvOKxcQ0G/BPUB4+yF5o/yf47vv9zUV5WZv4mrdsKDt/pZDYeDg==" }, "node_modules/windmill-parser-wasm-py": { - "version": "1.628.3", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.628.3.tgz", - "integrity": "sha512-TlluqknZpg8cZ+A3m6JFLPseY2PpKtDsxdj26fAnCUzKPtse8TxQR+n0dwC80rfW5TwdWSulvNGRDgcNuf7CTw==" + "version": "1.653.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.653.0.tgz", + "integrity": "sha512-vMkSL3JpELpag7nmyGA8onhYNiAG3K1mkh2k4vwVHC3W5dUd12fSS9gsBco2FqPGUPnWv1gCaHwmjBrOBVGL1w==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.646.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.646.0.tgz", - "integrity": "sha512-sRDQBX9ML3VIcW9r1Ug8SOKkHog6uKBwCTE6OADU3K8o8C45NiJufMGCeKtBWoF/Ki08IEj+0h4dd6QEIKpeIQ==" + "version": "1.653.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.653.0.tgz", + "integrity": "sha512-LEvLhb8uCb/jGMzd+lwP886LcwxuTE+W04wdyqmdqy1JD9FBToKeiqW7CtZWqbWr1oI8yK9U3jmqSqyOcUCzRw==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", @@ -12744,9 +12632,9 @@ "integrity": "sha512-9yGLYZX2Hn9TdTqGY/5Fp50ftzgUsrfBkSK9vJkKJd5Amyg+yXLBGzd8pz6Org+4uxMenz/16wpsgijvo6uhhQ==" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.647.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.647.1.tgz", - "integrity": "sha512-64iSAUMU5W/WtePqE1vtDvglDqtkiZVndyieYBVDX0nl7UuovS+wPgH/P3TEoKbR+FwAPacki0CX3DsEzZ/Yxw==" + "version": "1.653.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.653.0.tgz", + "integrity": "sha512-zwBUy7ijo58ooAKcsYflISY/+xllCw3Aq34Kj1PED6uABWVbV6A8MWHqEsjiVzC6iWfihWNtZJpck8zsRr9DCg==" }, "node_modules/windmill-parser-wasm-yaml": { "version": "1.593.0", diff --git a/frontend/package.json b/frontend/package.json index 360003f0e3..45a7f26220 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.648.0", + "version": "1.654.0", "scripts": { "dev": "vite dev", "build": "vite build", @@ -67,7 +67,7 @@ "svelte-range-slider-pips": "^2.3.1", "svelte-splitpanes": "^8.0.9", "tailwindcss": "^3.4.1", - "tar": "^7.4.3", + "tar": "^7.5.4", "tslib": "^2.6.1", "typescript": "^5.5.0", "vite": "^8.0.0-beta.16", @@ -78,7 +78,8 @@ "overrides": { "monaco-graphql": { "monaco-editor": "$monaco-editor" - } + }, + "tar": "$tar" }, "type": "module", "dependencies": { @@ -147,16 +148,17 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", + "windmill-parser-wasm-asset": "1.653.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", - "windmill-parser-wasm-py": "^1.628.3", - "windmill-parser-wasm-regex": "1.646.0", + "windmill-parser-wasm-py": "1.653.0", + "windmill-parser-wasm-regex": "1.653.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", - "windmill-parser-wasm-ts": "1.647.1", + "windmill-parser-wasm-ts": "1.653.0", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "^1.3.4", @@ -287,6 +289,11 @@ "svelte": "./package/components/recording/FlowRecordingReplay.svelte", "default": "./package/components/recording/FlowRecordingReplay.svelte" }, + "./components/ScriptRecordingReplay.svelte": { + "types": "./package/components/recording/ScriptRecordingReplay.svelte.d.ts", + "svelte": "./package/components/recording/ScriptRecordingReplay.svelte", + "default": "./package/components/recording/ScriptRecordingReplay.svelte" + }, "./components/FlowWrapper.svelte": { "types": "./package/components/FlowWrapper.svelte.d.ts", "svelte": "./package/components/FlowWrapper.svelte", @@ -489,6 +496,9 @@ "components/FlowRecordingReplay.svelte": [ "./package/components/recording/FlowRecordingReplay.svelte.d.ts" ], + "components/ScriptRecordingReplay.svelte": [ + "./package/components/recording/ScriptRecordingReplay.svelte.d.ts" + ], "components/FlowBuilder.svelte": [ "./package/components/FlowBuilder.svelte.d.ts" ], diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index d8429a5ebf..8a79bd7109 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -15,6 +15,7 @@ import GitHubAppIntegration from './GitHubAppIntegration.svelte' import BedrockCredentialsCheck from './BedrockCredentialsCheck.svelte' import { isCloudHosted } from '$lib/cloud' + import ResourceGen from './copilot/ResourceGen.svelte' interface Props { resourceType: string @@ -149,6 +150,12 @@ }} class="as-json-toggle" /> + {#if resourceType == 'postgresql'} + import { run } from 'svelte/legacy' + import { createEventDispatcher } from 'svelte' import { Button, Drawer } from './common' import DrawerContent from './common/drawer/DrawerContent.svelte' @@ -6,24 +8,26 @@ import AppConnectInner from './AppConnectInner.svelte' import DarkModeObserver from './DarkModeObserver.svelte' - export let expressOAuthSetup = false - - let drawer: Drawer - let resourceType = '' - let step = 1 - let disabled = false - let isGoogleSignin = false - let manual = true - - let appConnectInner: AppConnectInner | undefined = undefined - - let rtToLoad: string | undefined = '' - export async function open(rt?: string) { - rtToLoad = rt - drawer.openDrawer?.() + interface Props { + expressOAuthSetup?: boolean } - $: appConnectInner && onRtToLoadChange(rtToLoad) + let { expressOAuthSetup = false }: Props = $props() + + let drawer: Drawer | undefined = $state() + let resourceType = $state('') + let step = $state(1) + let disabled = $state(false) + let isGoogleSignin = $state(false) + let manual = $state(true) + + let appConnectInner: AppConnectInner | undefined = $state(undefined) + + let rtToLoad: string | undefined = $state('') + export async function open(rt?: string) { + rtToLoad = rt + drawer?.openDrawer?.() + } function onRtToLoadChange(rtToLoad: string | undefined) { appConnectInner?.open(rtToLoad) @@ -31,7 +35,10 @@ const dispatch = createEventDispatcher() - let darkMode: boolean = false + let darkMode: boolean = $state(false) + run(() => { + appConnectInner && onRtToLoadChange(rtToLoad) + }) @@ -47,7 +54,7 @@ @@ -68,7 +75,7 @@ {/if} {#if isGoogleSignin} - diff --git a/frontend/src/lib/components/Description.svelte b/frontend/src/lib/components/Description.svelte index c3cd74c7d6..2078b74cd2 100644 --- a/frontend/src/lib/components/Description.svelte +++ b/frontend/src/lib/components/Description.svelte @@ -2,11 +2,19 @@ import { ExternalLink } from 'lucide-svelte' import { twMerge } from 'tailwind-merge' - export let link: string | undefined = undefined + + interface Props { + link?: string | undefined; + class?: string; + children?: import('svelte').Snippet; + } + + let { link = undefined, class: className = '', children }: Props = $props(); + -
- +
+ {@render children?.()} {#if link} Learn more initial) + if (untrackedInitial) { + if (untrackedInitial.type == 'script') { + replaceScript(untrackedInitial.script) + } else if (untrackedInitial.type == 'flow') { + replaceFlow(untrackedInitial.flow) } modeInitialized = true } @@ -596,9 +597,9 @@ } }) } - let token = $derived($page.url.searchParams.get('wm_token') ?? undefined) - let workspace = $derived($page.url.searchParams.get('workspace') ?? undefined) - let themeDarkRaw = $derived($page.url.searchParams.get('activeColorTheme')) + let token = $derived(page.url.searchParams.get('wm_token') ?? undefined) + let workspace = $derived(page.url.searchParams.get('workspace') ?? undefined) + let themeDarkRaw = $derived(page.url.searchParams.get('activeColorTheme')) let themeDark = $derived(themeDarkRaw == '2' || themeDarkRaw == '4') $effect.pre(() => { diff --git a/frontend/src/lib/components/DiffEditor.svelte b/frontend/src/lib/components/DiffEditor.svelte index 1d252d082a..3fb26334b9 100644 --- a/frontend/src/lib/components/DiffEditor.svelte +++ b/frontend/src/lib/components/DiffEditor.svelte @@ -102,8 +102,8 @@ }) } - if (defaultLang !== undefined) { - setupModel(defaultLang, defaultOriginal, defaultModified, defaultModifiedLang) + if (defaultLang !== undefined || defaultOriginal !== undefined || defaultModified !== undefined) { + setupModel(defaultLang ?? 'plaintext', defaultOriginal, defaultModified, defaultModifiedLang) } } diff --git a/frontend/src/lib/components/DropdownSubmenuItem.svelte b/frontend/src/lib/components/DropdownSubmenuItem.svelte index 45346aaa3c..ed08fd7abd 100644 --- a/frontend/src/lib/components/DropdownSubmenuItem.svelte +++ b/frontend/src/lib/components/DropdownSubmenuItem.svelte @@ -1,4 +1,5 @@ diff --git a/frontend/src/lib/components/DropdownV2.svelte b/frontend/src/lib/components/DropdownV2.svelte index 902de661aa..c6243f3529 100644 --- a/frontend/src/lib/components/DropdownV2.svelte +++ b/frontend/src/lib/components/DropdownV2.svelte @@ -76,7 +76,7 @@ ids: { menu: dropdownId } } = createDropdownMenu({ positioning: { - placement + placement: untrack(() => placement) }, loop: true, onOpenChange: ({ next }) => { diff --git a/frontend/src/lib/components/DropdownV2Inner.svelte b/frontend/src/lib/components/DropdownV2Inner.svelte index 21157bd6b2..b56a5cce70 100644 --- a/frontend/src/lib/components/DropdownV2Inner.svelte +++ b/frontend/src/lib/components/DropdownV2Inner.svelte @@ -62,6 +62,9 @@ {item.displayName}

{@render item.extra?.()} + {#if item.shortcut} + {item.shortcut} + {/if} {#if item.tooltip} {#snippet text()} diff --git a/frontend/src/lib/components/DucklakePicker.svelte b/frontend/src/lib/components/DucklakePicker.svelte index 9260c99bf9..e84ddb9060 100644 --- a/frontend/src/lib/components/DucklakePicker.svelte +++ b/frontend/src/lib/components/DucklakePicker.svelte @@ -1,6 +1,6 @@
@@ -49,7 +49,6 @@ {/if}
diff --git a/frontend/src/lib/components/DurationMs.svelte b/frontend/src/lib/components/DurationMs.svelte index 451925c0c6..60d2964ffe 100644 --- a/frontend/src/lib/components/DurationMs.svelte +++ b/frontend/src/lib/components/DurationMs.svelte @@ -4,9 +4,13 @@ import { Hourglass } from 'lucide-svelte' import WaitTimeWarning from './common/waitTimeWarning/WaitTimeWarning.svelte' - export let duration_ms: number - export let self_wait_time_ms: number | undefined = undefined - export let aggregate_wait_time_ms: number | undefined = undefined + interface Props { + duration_ms: number; + self_wait_time_ms?: number | undefined; + aggregate_wait_time_ms?: number | undefined; + } + + let { duration_ms, self_wait_time_ms = undefined, aggregate_wait_time_ms = undefined }: Props = $props();
diff --git a/frontend/src/lib/components/DynamicInput.svelte b/frontend/src/lib/components/DynamicInput.svelte index bd6cceaf99..692272d50e 100644 --- a/frontend/src/lib/components/DynamicInput.svelte +++ b/frontend/src/lib/components/DynamicInput.svelte @@ -113,10 +113,10 @@ } }) - let lastArgs = $state.snapshot(otherArgs) + let lastArgs = $state.snapshot(untrack(() => otherArgs)) let timeout: number | undefined = $state() - let nargs = $state($state.snapshot(otherArgs)) + let nargs = $state($state.snapshot(untrack(() => otherArgs))) $effect(() => { otherArgs untrack(() => clearTimeout(timeout)) diff --git a/frontend/src/lib/components/EditableSchemaForm.svelte b/frontend/src/lib/components/EditableSchemaForm.svelte index 99fe00bdb0..989ce6a2fe 100644 --- a/frontend/src/lib/components/EditableSchemaForm.svelte +++ b/frontend/src/lib/components/EditableSchemaForm.svelte @@ -286,7 +286,7 @@ } } - let jsonView: boolean = $state(customUi?.jsonOnly == true) + let jsonView: boolean = $state(untrack(() => customUi)?.jsonOnly == true) let schemaString: string = $state(JSON.stringify(schema, null, '\t')) let error: string | undefined = $state(undefined) let editor: SimpleEditor | undefined = $state(undefined) @@ -296,8 +296,8 @@ editor?.setCode(schemaString) } - const editTabDefaultSize = noPreview ? 100 : 50 - editPanelSize = editTab ? (editPanelInitialSize ?? editTabDefaultSize) : 0 + const editTabDefaultSize = untrack(() => noPreview) ? 100 : 50 + editPanelSize = untrack(() => editTab) ? (untrack(() => editPanelInitialSize) ?? editTabDefaultSize) : 0 let inputPanelSize = $state(100 - editPanelSize) let editPanelSizeSmooth = tweened(editPanelSize, { duration: 150 @@ -592,7 +592,7 @@ {argName} {#if !uiOnly}
- + {#snippet trigger()}
@@ -221,7 +224,7 @@
diff --git a/frontend/src/lib/components/FlowHistoryJobPicker.svelte b/frontend/src/lib/components/FlowHistoryJobPicker.svelte index 3a703d8bd7..26fa41f1af 100644 --- a/frontend/src/lib/components/FlowHistoryJobPicker.svelte +++ b/frontend/src/lib/components/FlowHistoryJobPicker.svelte @@ -12,13 +12,15 @@ selected?: string | undefined selectInitial?: boolean loading?: boolean + newFlow?: boolean } let { path, selected = undefined, selectInitial = false, - loading = $bindable(false) + loading = $bindable(false), + newFlow = false }: Props = $props() const dispatch = createEventDispatcher() @@ -41,7 +43,9 @@ } $effect(() => { - $workspaceStore && untrack(() => loadInitial()) + if ($workspaceStore && !newFlow) { + untrack(() => loadInitial()) + } }) diff --git a/frontend/src/lib/components/FlowInputViewer.svelte b/frontend/src/lib/components/FlowInputViewer.svelte index 94c25ffcc4..62aef2dd37 100644 --- a/frontend/src/lib/components/FlowInputViewer.svelte +++ b/frontend/src/lib/components/FlowInputViewer.svelte @@ -3,7 +3,11 @@ import FieldHeader from './FieldHeader.svelte' - export let schema: Schema | { [key: string]: unknown } | undefined + interface Props { + schema: Schema | { [key: string]: unknown } | undefined; + } + + let { schema }: Props = $props();
    diff --git a/frontend/src/lib/components/FlowLogViewerWrapper.svelte b/frontend/src/lib/components/FlowLogViewerWrapper.svelte index 4549fc8304..648dbc9f06 100644 --- a/frontend/src/lib/components/FlowLogViewerWrapper.svelte +++ b/frontend/src/lib/components/FlowLogViewerWrapper.svelte @@ -62,7 +62,7 @@ const timelineItems = $derived(timelineCompute?.items ?? undefined) const timelineNow = $derived(timelineCompute?.now ?? Date.now()) - let moduleTracker = new ChangeTracker($state.snapshot(job.raw_flow?.modules ?? [])) + let moduleTracker = new ChangeTracker($state.snapshot(untrack(() => job).raw_flow?.modules ?? [])) $effect(() => { readFieldsRecursively(job.raw_flow?.modules ?? []) untrack(() => moduleTracker.track($state.snapshot(job.raw_flow?.modules ?? []))) @@ -123,7 +123,7 @@ } let timelineAvailableWidths = $state>({}) - let lastJobId: string | undefined = $state(job.id) + let lastJobId: string | undefined = $state(untrack(() => job).id) const timelinelWidth = $derived.by(() => { const widths = Object.values(timelineAvailableWidths) diff --git a/frontend/src/lib/components/FlowPlugConnect.svelte b/frontend/src/lib/components/FlowPlugConnect.svelte index da70d882db..cd44f3d1bd 100644 --- a/frontend/src/lib/components/FlowPlugConnect.svelte +++ b/frontend/src/lib/components/FlowPlugConnect.svelte @@ -4,9 +4,13 @@ import AnimatedButton from './common/button/AnimatedButton.svelte' import { twMerge } from 'tailwind-merge' - export let connecting: boolean - export let id: undefined | string = undefined - export let wrapperClasses = '' + interface Props { + connecting: boolean; + id?: undefined | string; + wrapperClasses?: string; + } + + let { connecting, id = undefined, wrapperClasses = '' }: Props = $props(); diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 28546536bb..f702b84397 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -588,6 +588,7 @@ {/if} { if (!currentJobId) { currentJobId = jobId @@ -607,7 +608,9 @@ />
- + {#if jobId} + + {/if} {#if job}
@@ -670,6 +673,10 @@
Loading history...
+ {:else} +
+ Flow status will display here +
{/if}
diff --git a/frontend/src/lib/components/FlowStatusViewer.svelte b/frontend/src/lib/components/FlowStatusViewer.svelte index 5da8b31419..950eb7354f 100644 --- a/frontend/src/lib/components/FlowStatusViewer.svelte +++ b/frontend/src/lib/components/FlowStatusViewer.svelte @@ -62,7 +62,7 @@ showLogsWithResult = false }: Props = $props() - let lastJobId: string = jobId + let lastJobId: string = untrack(() => jobId) let retryStatus = $state({ val: {} }) let globalRefreshes: Record Promise)[]> = $state({}) @@ -71,11 +71,11 @@ flowState, suspendStatus, retryStatus, - hideDownloadInGraph, - hideNodeDefinition, - hideTimeline, - hideJobId, - hideDownloadLogs + hideDownloadInGraph: untrack(() => hideDownloadInGraph), + hideNodeDefinition: untrack(() => hideNodeDefinition), + hideTimeline: untrack(() => hideTimeline), + hideJobId: untrack(() => hideJobId), + hideDownloadLogs: untrack(() => hideDownloadLogs) }) function loadOwner(path: string) { diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 410ba822ab..25731f82c8 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -181,7 +181,7 @@ let resultStreams: Record = $state({}) - if (onResultStreamUpdate == undefined) { + if (untrack(() => onResultStreamUpdate) == undefined) { onResultStreamUpdate = ({ jobId, result_stream @@ -234,7 +234,7 @@ }) let jobResults: any[] = $state( - flowJobIds?.flowJobs?.map((x, id) => `iter #${id + 1} not loaded by frontend yet`) ?? [] + untrack(() => flowJobIds)?.flowJobs?.map((x, id) => `iter #${id + 1} not loaded by frontend yet`) ?? [] ) let retry_selected = $state('') @@ -805,7 +805,7 @@ let destroyed = false - updateRecursiveRefresh(jobId) + updateRecursiveRefresh(untrack(() => jobId)) async function updateJobId() { if (jobId !== job?.id || innerModules == undefined) { diff --git a/frontend/src/lib/components/FlowViewer.svelte b/frontend/src/lib/components/FlowViewer.svelte index 8a78637519..2355ddf68d 100644 --- a/frontend/src/lib/components/FlowViewer.svelte +++ b/frontend/src/lib/components/FlowViewer.svelte @@ -1,4 +1,5 @@ {#if tabular} diff --git a/frontend/src/lib/components/GraphqlSchemaViewer.svelte b/frontend/src/lib/components/GraphqlSchemaViewer.svelte index 7929083d9a..1ab4c68f00 100644 --- a/frontend/src/lib/components/GraphqlSchemaViewer.svelte +++ b/frontend/src/lib/components/GraphqlSchemaViewer.svelte @@ -5,10 +5,17 @@ import { onDestroy, onMount } from 'svelte' - let divEl: HTMLDivElement | null = null + let divEl: HTMLDivElement | null = $state(null) let editor: meditor.IStandaloneCodeEditor - export let code: string = '' + + interface Props { + code?: string; + class?: string; + } + + let { code = '', class: className = '' }: Props = $props(); + async function loadMonaco() { editor = meditor.create(divEl as HTMLDivElement, { @@ -43,4 +50,4 @@ }) -
+
diff --git a/frontend/src/lib/components/GroupEditor.svelte b/frontend/src/lib/components/GroupEditor.svelte index a8391bd8a6..5a8c4cb411 100644 --- a/frontend/src/lib/components/GroupEditor.svelte +++ b/frontend/src/lib/components/GroupEditor.svelte @@ -159,12 +159,14 @@ {/if} {#if members} - - - user - - - + + {#snippet headerRow()} + + user + + + + {/snippet} {#snippet body()} {#each members ?? [] as { member_name, role }} @@ -301,10 +303,12 @@ {#if instance_group?.emails}

Members from the instance group

- - - user - + + {#snippet headerRow()} + + user + + {/snippet} {#snippet body()} {#each instance_group?.emails ?? [] as email} diff --git a/frontend/src/lib/components/IdEditorInput.svelte b/frontend/src/lib/components/IdEditorInput.svelte index 7b85d8e52b..611dbc2d3e 100644 --- a/frontend/src/lib/components/IdEditorInput.svelte +++ b/frontend/src/lib/components/IdEditorInput.svelte @@ -1,4 +1,5 @@ {#if entries.length} diff --git a/frontend/src/lib/components/InstanceGroupEditor.svelte b/frontend/src/lib/components/InstanceGroupEditor.svelte index cf2a3ded0a..f3d8df33de 100644 --- a/frontend/src/lib/components/InstanceGroupEditor.svelte +++ b/frontend/src/lib/components/InstanceGroupEditor.svelte @@ -1,4 +1,6 @@
@@ -85,17 +91,20 @@
{#if members} - - user - - - - {#each members as { member_email }} - {member_email} - - - - {/each} - + > + + {/each} + + {/snippet} {:else}
diff --git a/frontend/src/lib/components/InstanceNameEditor.svelte b/frontend/src/lib/components/InstanceNameEditor.svelte index 31fc3d738b..cb710cddce 100644 --- a/frontend/src/lib/components/InstanceNameEditor.svelte +++ b/frontend/src/lib/components/InstanceNameEditor.svelte @@ -1,4 +1,7 @@
- +
+ {/snippet} diff --git a/frontend/src/lib/components/PageHeader.svelte b/frontend/src/lib/components/PageHeader.svelte index 07b02227d4..556ac1b0cc 100644 --- a/frontend/src/lib/components/PageHeader.svelte +++ b/frontend/src/lib/components/PageHeader.svelte @@ -1,11 +1,23 @@
@@ -31,9 +43,9 @@ {/if} - {#if $$slots.default} + {#if children}
- + {@render children?.()}
{/if}
diff --git a/frontend/src/lib/components/ParqetCsvTableRenderer.svelte b/frontend/src/lib/components/ParqetCsvTableRenderer.svelte index 07a8c267a9..262be32363 100644 --- a/frontend/src/lib/components/ParqetCsvTableRenderer.svelte +++ b/frontend/src/lib/components/ParqetCsvTableRenderer.svelte @@ -1,4 +1,6 @@ @@ -182,7 +189,7 @@
- mountGrid()}> diff --git a/frontend/src/lib/components/PermissionHistory.svelte b/frontend/src/lib/components/PermissionHistory.svelte index 82d646bd53..23f3ec151f 100644 --- a/frontend/src/lib/components/PermissionHistory.svelte +++ b/frontend/src/lib/components/PermissionHistory.svelte @@ -79,12 +79,14 @@

No permission changes recorded yet

{:else} - - Changed By - Change Type - Affected - Date - + {#snippet headerRow()} + + Changed By + Change Type + Affected + Date + + {/snippet} {#snippet body()} {#each history as change} diff --git a/frontend/src/lib/components/PersistentScriptDrawer.svelte b/frontend/src/lib/components/PersistentScriptDrawer.svelte index aa5c869f31..23fa122b7d 100644 --- a/frontend/src/lib/components/PersistentScriptDrawer.svelte +++ b/frontend/src/lib/components/PersistentScriptDrawer.svelte @@ -10,19 +10,19 @@ import { Hourglass, Loader2, Play, RefreshCw } from 'lucide-svelte' let dispatch = createEventDispatcher() - let drawer: Drawer + let drawer: Drawer | undefined = $state() - let script: Script - let loadQueuedJobs = true - let queuedJobsLoading = false + let script: Script | undefined = $state() + let loadQueuedJobs = $state(true) + let queuedJobsLoading = $state(false) let queuedJobs: { status: 'running' | 'queued' jobId: string scheduledFor: string scriptHash: string - }[] = [] + }[] = $state([]) - let cancellingInProgress = false + let cancellingInProgress = $state(false) async function continuouslyLoadQueuedJobs() { while (loadQueuedJobs) { @@ -40,7 +40,7 @@ let qjs = await JobService.listQueue({ workspace: $workspaceStore ?? '', orderDesc: false, - scriptPathExact: script.path + scriptPathExact: script?.path }) let loadingQueuedJobs: { status: 'running' | 'queued' @@ -71,12 +71,12 @@ cancellingInProgress = true await JobService.cancelPersistentQueuedJobs({ workspace: $workspaceStore ?? '', - path: script.path, + path: script?.path ?? '', requestBody: { reason: undefined } }) - sendUserToast(`All jobs cancelled for ${script.path}`) + sendUserToast(`All jobs cancelled for ${script?.path}`) cancellingInProgress = false } @@ -88,12 +88,12 @@ script = persistentScript! loadQueuedJobs = true continuouslyLoadQueuedJobs() - drawer.openDrawer?.() + drawer?.openDrawer?.() } async function exit() { loadQueuedJobs = false - drawer.closeDrawer?.() + drawer?.closeDrawer?.() } onDestroy(() => { @@ -117,51 +117,57 @@ >

- Queued jobs for {script.path} + Queued jobs for {script?.path}

- - Script Hash - Job ID - Status - Scheduled For - - - {#each queuedJobs as { jobId, status, scriptHash, scheduledFor }} - - - - {scriptHash} - - - - {jobId.substring(24)} - - - {#if status === 'running'} - - - - {:else} - - - - {/if} - - {scheduledFor} - - {/each} - + {#snippet headerRow()} + + Script Hash + Job ID + Status + Scheduled For + + {/snippet} + {#snippet body()} + + {#each queuedJobs as { jobId, status, scriptHash, scheduledFor }} + + + + {scriptHash} + + + + {jobId.substring(24)} + + + {#if status === 'running'} + + + + {:else} + + + + {/if} + + {scheduledFor} + + {/each} + + {/snippet} {#snippet actions()} diff --git a/frontend/src/lib/components/Popover.svelte b/frontend/src/lib/components/Popover.svelte index 1fa0a0d88d..07447b2e4a 100644 --- a/frontend/src/lib/components/Popover.svelte +++ b/frontend/src/lib/components/Popover.svelte @@ -43,10 +43,10 @@ onClick }: Props = $props() - const [popperRef, popperContent] = createPopperActions({ placement }) + const [popperRef, popperContent] = createPopperActions({ placement: untrack(() => placement) }) const popperOptions: PopperOptions<{}> = { - placement, + placement: untrack(() => placement), strategy: 'fixed', modifiers: [ { name: 'offset', options: { offset: [8, 8] } }, diff --git a/frontend/src/lib/components/PrefixedInput.svelte b/frontend/src/lib/components/PrefixedInput.svelte index c548418855..2a1dccf386 100644 --- a/frontend/src/lib/components/PrefixedInput.svelte +++ b/frontend/src/lib/components/PrefixedInput.svelte @@ -1,4 +1,5 @@ diff --git a/frontend/src/lib/components/RadioButton.svelte b/frontend/src/lib/components/RadioButton.svelte index efa9a62b40..c7e5dc8dce 100644 --- a/frontend/src/lib/components/RadioButton.svelte +++ b/frontend/src/lib/components/RadioButton.svelte @@ -1,13 +1,24 @@ @@ -28,7 +39,7 @@ class="sr-only" bind:group={value} aria-labelledby="memory-option-0-label" - on:click={() => dispatch('change', val)} + onclick={() => dispatch('change', val)} />

{#if typeof label !== 'string'} diff --git a/frontend/src/lib/components/Range.svelte b/frontend/src/lib/components/Range.svelte index 0e92ac72de..72129aada2 100644 --- a/frontend/src/lib/components/Range.svelte +++ b/frontend/src/lib/components/Range.svelte @@ -1,29 +1,47 @@

- +
{#if max <= min}
Impossible to display range: {`max (${max}) <= min (${min})`}
+ import { untrack } from 'svelte' import { GitSyncService } from '$lib/gen' import Select from './select/Select.svelte' @@ -32,7 +33,7 @@ }: Props = $props() // Track all loaded repositories across pages - let loadedRepositories = $state(initialRepositories) + let loadedRepositories = $state(untrack(() => initialRepositories)) let currentPage = $state(1) let isLoadingMore = $state(false) diff --git a/frontend/src/lib/components/Required.svelte b/frontend/src/lib/components/Required.svelte index 2163efa976..3ee1deec1a 100644 --- a/frontend/src/lib/components/Required.svelte +++ b/frontend/src/lib/components/Required.svelte @@ -1,12 +1,19 @@ {#if required} - * + * {:else if detail || detail != ''} - ({detail != '' ? `${detail}` : ''}) {/if} diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 5a5943390e..70aaf6a2ee 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -24,12 +24,12 @@ import GitHubAppIntegration from './GitHubAppIntegration.svelte' import Button from './common/button/Button.svelte' import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte' + import ResourceGen from './copilot/ResourceGen.svelte' interface Props { canSave?: boolean resource_type?: string | undefined path?: string - newResource?: boolean hidePath?: boolean onChange?: (args: { path: string; args: Record; description: string }) => void defaultValues?: Record | undefined @@ -39,7 +39,6 @@ canSave = $bindable(true), resource_type = $bindable(undefined), path = $bindable(''), - newResource = false, hidePath = false, onChange, defaultValues = undefined @@ -62,6 +61,7 @@ let resourceTypeInfo: ResourceType | undefined = $state(undefined) let editDescription = $state(false) let viewJsonSchema = $state(false) + let newResource = $derived(!path) const dispatch = createEventDispatcher() @@ -81,7 +81,7 @@ .map(([k, _]) => k) } - if (!newResource) { + if (!untrack(() => newResource)) { initEdit() } else if (resource_type) { loadResourceType() @@ -270,6 +270,13 @@ right: 'As JSON' }} /> + {#if resourceToEdit?.resource_type === 'nats' || resourceToEdit?.resource_type === 'kafka'} {:else} @@ -296,9 +303,17 @@ {#if loadingSchema} {:else if !viewJsonSchema && resourceTypeInfo?.is_fileset} -
- Fileset -
+
+
Fileset
+ +
{:else if !viewJsonSchema && resourceSchema && resourceSchema?.properties} {#if resourceTypeInfo?.format_extension} diff --git a/frontend/src/lib/components/ResourceEditorDrawer.svelte b/frontend/src/lib/components/ResourceEditorDrawer.svelte index 50db916f13..dae6943868 100644 --- a/frontend/src/lib/components/ResourceEditorDrawer.svelte +++ b/frontend/src/lib/components/ResourceEditorDrawer.svelte @@ -5,50 +5,44 @@ import { Loader2, Save } from 'lucide-svelte' - let drawer: Drawer - let canSave = true - let resource_type: string | undefined = undefined - let defaultValues: Record | undefined = undefined + let drawer: Drawer | undefined = $state() + let canSave = $state(true) + let resource_type: string | undefined = $state(undefined) + let defaultValues: Record | undefined = $state(undefined) let resourceEditor: { editResource: () => void; createResource: () => void } | undefined = - undefined + $state(undefined) - let path: string | undefined = undefined + let path: string | undefined = $state(undefined) - let newResource = false export async function initEdit(p: string): Promise { resource_type = undefined - newResource = false path = p - drawer.openDrawer?.() + drawer?.openDrawer?.() } export async function initNew( resourceType: string, nDefaultValues?: Record ): Promise { - newResource = true path = undefined resource_type = resourceType defaultValues = nDefaultValues - drawer.openDrawer?.() + drawer?.openDrawer?.() } - let mode: 'edit' | 'new' = newResource ? 'new' : 'edit' - - $: path ? (mode = 'edit') : (mode = 'new') + let mode: 'edit' | 'new' = $derived(!path ? 'new' : 'edit') {#await import('./ResourceEditor.svelte')} {:then Module} diff --git a/frontend/src/lib/components/ResourcePicker.svelte b/frontend/src/lib/components/ResourcePicker.svelte index 062e39804b..fddf9205f5 100644 --- a/frontend/src/lib/components/ResourcePicker.svelte +++ b/frontend/src/lib/components/ResourcePicker.svelte @@ -1,6 +1,6 @@ @@ -317,7 +316,6 @@ class="mt-1" _resourceMetadata={{ resource_type: resourceType }} asset={{ kind: 'resource', path: value }} - {dbManagerDrawer} /> {/if}
diff --git a/frontend/src/lib/components/RunForm.svelte b/frontend/src/lib/components/RunForm.svelte index 969576a06b..da0c87a15f 100644 --- a/frontend/src/lib/components/RunForm.svelte +++ b/frontend/src/lib/components/RunForm.svelte @@ -15,7 +15,7 @@ import Popover from './meltComponents/Popover.svelte' import { Calendar, Check, CornerDownLeft } from 'lucide-svelte' import RunFormAdvancedPopup from './RunFormAdvancedPopup.svelte' - import { page } from '$app/stores' + import { page } from '$app/state' import { replaceState } from '$app/navigation' import JsonInputs from '$lib/components/JsonInputs.svelte' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' @@ -108,7 +108,7 @@ nurl.hash = computeSharableHash(args) try { - replaceState(nurl.toString(), $page.state) + replaceState(nurl.toString(), page.state) } catch (e) { console.error(e) } diff --git a/frontend/src/lib/components/RunFormAdvancedPopup.svelte b/frontend/src/lib/components/RunFormAdvancedPopup.svelte index 7d6ef01bb2..399e55f772 100644 --- a/frontend/src/lib/components/RunFormAdvancedPopup.svelte +++ b/frontend/src/lib/components/RunFormAdvancedPopup.svelte @@ -8,7 +8,9 @@ import { WorkerService } from '$lib/gen' import DateTimeInput from './DateTimeInput.svelte' - export let runnable: + + interface Props { + runnable: | { summary?: string description?: string @@ -21,11 +23,18 @@ created_by?: string extra_perms?: Record } - | undefined + | undefined; + scheduledForStr: string | undefined; + invisible_to_owner: boolean | undefined; + overrideTag: string | undefined; + } - export let scheduledForStr: string | undefined - export let invisible_to_owner: boolean | undefined - export let overrideTag: string | undefined + let { + runnable, + scheduledForStr = $bindable(), + invisible_to_owner = $bindable(), + overrideTag = $bindable() + }: Props = $props(); loadWorkerGroups() async function loadWorkerGroups() { diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index 59b63ab313..6de1461e20 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -96,8 +96,8 @@ let batchRerunOptionsIsOpen = $state(false) // Initialize path filter from route param if provided and not already set via query params - if (initialPath && !filters.val.path) { - filters.val.path = initialPath + if (untrack(() => initialPath) && !filters.val.path) { + filters.val.path = untrack(() => initialPath) } // Apply persistent toggle values from local storage if URL doesn't specify them @@ -148,7 +148,9 @@ (v) => { v.maxTs ? (filters.val.max_ts = new Date(v.maxTs)) : delete filters.val.max_ts v.minTs ? (filters.val.min_ts = new Date(v.minTs)) : delete filters.val.min_ts - v.timeframe ? (filters.val.timeframe = v.timeframe) : delete filters.val.timeframe + v.timeframe && v.timeframe !== 'Latest runs' + ? (filters.val.timeframe = v.timeframe) + : delete filters.val.timeframe } ) let timeframe = $derived(_timeframe.val) @@ -593,7 +595,7 @@ diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index a07ea85d25..55bb25c5ef 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -13,6 +13,7 @@ interface Props { fromWorkspaceSettings?: boolean readOnlyMode: boolean + allowDelete?: boolean initialFileKey?: { s3: string; storage?: string } | undefined selectedFileKey?: { s3: string; storage?: string } | undefined folderOnly?: boolean @@ -24,6 +25,7 @@ let { fromWorkspaceSettings = false, readOnlyMode, + allowDelete = false, initialFileKey = $bindable(undefined), selectedFileKey = $bindable(undefined), folderOnly = false, @@ -94,6 +96,7 @@ }} {fromWorkspaceSettings} {readOnlyMode} + {allowDelete} bind:initialFileKey bind:selectedFileKey bind:workspaceSettingsInitialized diff --git a/frontend/src/lib/components/S3FilePickerInner.svelte b/frontend/src/lib/components/S3FilePickerInner.svelte index 380e3d9a35..74b85bd327 100644 --- a/frontend/src/lib/components/S3FilePickerInner.svelte +++ b/frontend/src/lib/components/S3FilePickerInner.svelte @@ -81,6 +81,7 @@ count: number } > + allowDelete?: boolean replaceUnauthorizedWarning?: Snippet listStoredFilesRequest?: (d: ListStoredFilesData) => CancelablePromise loadFilePreviewRequest?: (d: LoadFilePreviewData) => CancelablePromise @@ -102,11 +103,12 @@ folderOnly = false, regexFilter = undefined, hideS3SpecificDetails = false, - rootPath = '', + rootPath: initialRootPath = '', workspaceSettingsInitialized = $bindable(true), storage = $bindable(undefined), uploadModalOpen = $bindable(false), allFilesByKey = $bindable({}), + allowDelete = false, replaceUnauthorizedWarning, listStoredFilesRequest = HelpersService.listStoredFiles, loadFilePreviewRequest = HelpersService.loadFilePreview, @@ -116,6 +118,7 @@ testConnectionRequest = HelpersService.datasetStorageTestConnection }: Props = $props() + let rootPath = $state(initialRootPath) let rootPathNestingLevel = $derived(1 * (rootPath.split('/').length - 1)) let csvSeparatorChar: string = $state(',') @@ -263,7 +266,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() fileListLoading = false fileInfoLoading = false } @@ -381,7 +384,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() } async function clearAndLoadFiles({ keepFilter }: { keepFilter?: boolean } = {}) { @@ -424,9 +427,16 @@ export async function open(_preSelectedFileKey: S3Object | undefined = undefined) { const preSelectedFileKey = _preSelectedFileKey && parseS3Object(_preSelectedFileKey) storage = preSelectedFileKey?.storage - if (preSelectedFileKey !== undefined) { + if (preSelectedFileKey !== undefined && preSelectedFileKey.s3.endsWith('/')) { + rootPath = preSelectedFileKey.s3 + filter = '' + selectedFileKey = undefined + } else if (preSelectedFileKey !== undefined) { + rootPath = '' initialFileKey = { ...preSelectedFileKey } selectedFileKey = { ...preSelectedFileKey } + } else { + rootPath = '' } reloadContent() } @@ -461,7 +471,7 @@ if (selectedFileKey !== undefined) { if (allFilesByKey[selectedFileKey.s3] === undefined) { selectedFileKey = { s3: '', storage } - } else { + } else if (allFilesByKey[selectedFileKey.s3].type !== 'folder') { loadFileMetadataPlusPreviewAsync(selectedFileKey.s3) } } @@ -518,7 +528,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() } else { selectedFileKey = { s3: item_key, @@ -719,8 +729,10 @@ startIcon={{ icon: MoveRight }} iconOnly={true} /> + {/if} + {#if !readOnlyMode || allowDelete} +
{#if customUi?.settingsPanel?.metadata?.disableScriptKind !== true}
{#snippet header()} @@ -1652,8 +1678,10 @@ /> {:else if script.on_behalf_of_email && !canPreserve} - Currently: {originalOnBehalfOfEmail ?? script.on_behalf_of_email}. - Will be set to {$userStore?.email} on deploy (requires admin or wm_deployers group to override) + Currently: {originalOnBehalfOfEmail ?? script.on_behalf_of_email}. Will be set to {$userStore?.email} on + deploy (requires admin or wm_deployers group to override) {/if} diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 4659abc0ac..f310010c18 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -33,6 +33,8 @@ Bug, Copy, CornerDownLeft, + Disc, + Download, ExternalLink, Github, GitBranch, @@ -86,6 +88,10 @@ import { deepEqual } from 'fast-equals' import { usePreparedAssetSqlQueries } from '$lib/infer.svelte' import { resource, watch } from 'runed' + import { createScriptRecording } from './recording/scriptRecording.svelte' + import { setActiveRecording } from './recording/flowRecording.svelte' + import type { ScriptRecording } from './recording/types' + import DropdownV2 from './DropdownV2.svelte' interface Props { // Exported @@ -94,7 +100,7 @@ path: string | undefined lang: Preview['language'] kind?: string | undefined - template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' + template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox' tag: string | undefined initialArgs?: Record fixedOverflowWidgets?: boolean @@ -117,7 +123,7 @@ lastDeployedCode?: string | undefined disableAi?: boolean assets?: AssetWithAltAccessType[] - editor_bar_right?: import('svelte').Snippet + editorBarRight?: import('svelte').Snippet enablePreprocessorSnippet?: boolean } @@ -149,7 +155,7 @@ lastDeployedCode = undefined, disableAi = false, assets = $bindable(), - editor_bar_right, + editorBarRight, enablePreprocessorSnippet = false }: Props = $props() @@ -235,6 +241,10 @@ let pastPreviews: CompletedJob[] = $state([]) let validCode = $state(true) + // Recording + let scriptRecording = createScriptRecording() + let lastRecording: ScriptRecording | undefined = $state(undefined) + let wsProvider: WebsocketProvider | undefined = $state(undefined) let yContent: Y.Text | undefined = $state(undefined) let peers: { name: string }[] = $state([]) @@ -332,11 +342,18 @@ undefined, { done(_x) { + if (scriptRecording.active) { + lastRecording = scriptRecording.stop() + setActiveRecording(undefined) + } loadPastTests() }, doneError({ error }) { + if (scriptRecording.active) { + lastRecording = scriptRecording.stop() + setActiveRecording(undefined) + } console.error(error) - // sendUserToast('Error running test', true) } } ) @@ -344,6 +361,19 @@ return job } + async function recordAndTest() { + lastRecording = undefined + scriptRecording.start(path ?? '', code, lang ?? '', args ?? {}, schema) + setActiveRecording(scriptRecording) + await runTest() + } + + function downloadRecording() { + if (lastRecording) { + scriptRecording.download(lastRecording) + } + } + async function loadPastTests(): Promise { pastPreviews = await JobService.listCompletedJobs({ workspace: $workspaceStore!, @@ -853,7 +883,7 @@ } } - setContext('disableTooltips', customUi?.disableTooltips === true) + setContext('disableTooltips', untrack(() => customUi)?.disableTooltips === true) let codePanelSize = $state(70) let testPanelSize = $state(30) @@ -1012,7 +1042,7 @@ bind:showHistoryDrawer > {#snippet right()} - {@render editor_bar_right?.()} + {@render editorBarRight?.()} {/snippet} {/if} @@ -1110,40 +1140,72 @@ />
{#if !(debugMode && isDebuggableScript)} -
- {#if testIsLoading} - - {:else} - {@const disableTriggerButton = - customUi?.previewPanel?.disableTriggerButton === true} - - {#if !disableTriggerButton} - +
+
+ {#if testIsLoading} + + {:else} + {@const disableTriggerButton = + customUi?.previewPanel?.disableTriggerButton === true} + + {#if !disableTriggerButton} + + {/if} {/if} +
+ {#if lastRecording} +
{/if} -
+
+ + recordAndTest() + }, + ...(lastRecording + ? [ + { + displayName: 'Download recording', + icon: Download, + action: () => downloadRecording() + } + ] + : []) + ]} + /> +
allowFlow) && options.push(['Flow', 'flow', FlowIcon, '#14b8a6']) const dispatch = createEventDispatcher() async function loadItems(): Promise { diff --git a/frontend/src/lib/components/ScriptWrapper.svelte b/frontend/src/lib/components/ScriptWrapper.svelte index 5e53dc09f3..eec781ce77 100644 --- a/frontend/src/lib/components/ScriptWrapper.svelte +++ b/frontend/src/lib/components/ScriptWrapper.svelte @@ -1,11 +1,12 @@ diff --git a/frontend/src/lib/components/Scrollable.svelte b/frontend/src/lib/components/Scrollable.svelte index 37e3ae2933..7c34b3cbe1 100644 --- a/frontend/src/lib/components/Scrollable.svelte +++ b/frontend/src/lib/components/Scrollable.svelte @@ -2,14 +2,19 @@ import { onMount, onDestroy } from 'svelte' import { twMerge } from 'tailwind-merge' - let isAtBottom: boolean = false - let isScrollable = false + let isAtBottom: boolean = $state(false) + let isScrollable = $state(false) - export let id: string | null | undefined = undefined - export let scrollableClass: string = '' - export let shiftedShadow: boolean = false + interface Props { + id?: string | null | undefined + scrollableClass?: string + shiftedShadow?: boolean + children?: import('svelte').Snippet + } + + let { id = undefined, scrollableClass = '', shiftedShadow = false, children }: Props = $props() let mutationObserver: MutationObserver - let el: HTMLDivElement + let el: HTMLDivElement | undefined = $state() function handleScroll(event) { const scrollableElement = event.target @@ -20,7 +25,8 @@ } function checkIfScrollable(el) { - return el.scrollHeight > el.clientHeight + if (!el) return false + return el?.scrollHeight > el?.clientHeight } function observeScrollability(el) { @@ -33,7 +39,7 @@ } export function scrollIntoView(top: number) { - el.scrollTo({ top, behavior: 'smooth' }) + el?.scrollTo({ top, behavior: 'smooth' }) } onMount(() => { observeScrollability(el) @@ -45,8 +51,8 @@
-
- +
+ {@render children?.()}
{#if !isAtBottom && isScrollable}
opts)) function filterItems() { let trimmed = filter.trim() diff --git a/frontend/src/lib/components/ServiceLogsInner.svelte b/frontend/src/lib/components/ServiceLogsInner.svelte index 0f4f923cd6..df0298f579 100644 --- a/frontend/src/lib/components/ServiceLogsInner.svelte +++ b/frontend/src/lib/components/ServiceLogsInner.svelte @@ -22,7 +22,7 @@ import SplitPanesOrColumnOnMobile from './splitPanes/SplitPanesOrColumnOnMobile.svelte' import Select from './select/Select.svelte' import { goto } from '$lib/navigation' - import { page } from '$app/stores' + import { page } from '$app/state' import { watch } from 'runed' interface Props { @@ -169,13 +169,13 @@ type Selected = { mode: string; workerGroup: string; hostname: string } let initialSelected = - $page.url.searchParams.get('mode') && - $page.url.searchParams.get('workerGroup') && - $page.url.searchParams.get('hostname') + page.url.searchParams.get('mode') && + page.url.searchParams.get('workerGroup') && + page.url.searchParams.get('hostname') ? { - mode: $page.url.searchParams.get('mode')!, - workerGroup: $page.url.searchParams.get('workerGroup')!, - hostname: $page.url.searchParams.get('hostname')! + mode: page.url.searchParams.get('mode')!, + workerGroup: page.url.searchParams.get('workerGroup')!, + hostname: page.url.searchParams.get('hostname')! } : undefined let selected: Selected | undefined = $state(initialSelected) @@ -523,7 +523,13 @@ {#if allLogs == undefined}
{:else if Object.keys(allLogs).length == 0} -
No logs
+
+ No logs + Search only covers a recent time window, configurable in instance settings + under Indexer. +
{:else if minTs && maxTs} {@const minTsN = new Date(minTs).getTime()} {@const maxTsN = new Date(maxTs).getTime()} @@ -663,7 +669,7 @@ { + onClick={() => { let logLineNumber = document.line_number[0] let logFile = document.file_name[0] let host = document.host[0] diff --git a/frontend/src/lib/components/ShareModal.svelte b/frontend/src/lib/components/ShareModal.svelte index e5ad54f88b..10cf5f09e0 100644 --- a/frontend/src/lib/components/ShareModal.svelte +++ b/frontend/src/lib/components/ShareModal.svelte @@ -35,6 +35,7 @@ | 'postgres_trigger' | 'gcp_trigger' | 'email_trigger' + | 'volume' let kind: Kind let path: string = $state('') @@ -53,13 +54,17 @@ let drawer: Drawer | undefined = $state() let own = $state(false) - export async function openDrawer(newPath: string, kind_l: Kind) { + export async function openDrawer(newPath: string, kind_l: Kind, isOwnerOverride?: boolean) { path = newPath kind = kind_l loadAcls() loadGroups() loadUsernames() - loadOwner() + if (isOwnerOverride !== undefined) { + own = isOwnerOverride + } else { + loadOwner() + } drawer?.openDrawer() } @@ -154,12 +159,14 @@ {/if} {#if acls?.length > 0} - - - owner - - - + + {#snippet headerRow()} + + owner + + + + {/snippet} {#snippet body()} {#each acls as [owner, write]} diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 2fdd14e3f9..31da7fd802 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -130,7 +130,7 @@ const dispatch = createEventDispatcher() - const uri = `file:///${hash}.${langToExt(lang)}` + const uri = `file:///${untrack(() => hash)}.${langToExt(untrack(() => lang))}` export function getCode(): string { if (valueAfterDispose != undefined) { diff --git a/frontend/src/lib/components/Slider.svelte b/frontend/src/lib/components/Slider.svelte index c2ca0abbdd..e3269cde09 100644 --- a/frontend/src/lib/components/Slider.svelte +++ b/frontend/src/lib/components/Slider.svelte @@ -5,10 +5,21 @@ import Tooltip from './Tooltip.svelte' import { twMerge } from 'tailwind-merge' - export let text: string - export let tooltip: string | undefined = undefined - export let view = false - export let size: 'xs' | 'sm' | 'md' | 'lg' = 'md' + interface Props { + text: string; + tooltip?: string | undefined; + view?: boolean; + size?: 'xs' | 'sm' | 'md' | 'lg'; + children?: import('svelte').Snippet; + } + + let { + text, + tooltip = undefined, + view = $bindable(false), + size = 'md', + children + }: Props = $props();
- - {#if isCloudHosted()} - The cloud version is updated daily. - {:else} - How to update?
- - docker: docker compose up -d
- - helm - {/if} - + {#snippet text()} + + {#if isCloudHosted()} + The cloud version is updated daily. + {:else} + How to update?
+ - docker: docker compose up -d
+ - helm + {/if} + + {/snippet} {/snippet} {#snippet actions()} diff --git a/frontend/src/lib/components/SuperadminSettingsInner.svelte b/frontend/src/lib/components/SuperadminSettingsInner.svelte index 4d6b0e33f1..1f33e768d8 100644 --- a/frontend/src/lib/components/SuperadminSettingsInner.svelte +++ b/frontend/src/lib/components/SuperadminSettingsInner.svelte @@ -8,7 +8,7 @@ import { sendUserToast } from '$lib/toast' import { base } from '$lib/base' import SearchItems from './SearchItems.svelte' - import { page } from '$app/stores' + import { page } from '$app/state' import { goto as gotoUrl } from '$app/navigation' import Version from './Version.svelte' import Uptodate from './Uptodate.svelte' @@ -50,9 +50,9 @@ } = $props() function removeHash() { - const index = $page.url.href.lastIndexOf('#') + const index = page.url.href.lastIndexOf('#') if (index === -1) return - const hashRemoved = $page.url.href.slice(0, index) + const hashRemoved = page.url.href.slice(0, index) gotoUrl(hashRemoved) } diff --git a/frontend/src/lib/components/TableCustom.svelte b/frontend/src/lib/components/TableCustom.svelte index e40a2c9706..a508d48c8f 100644 --- a/frontend/src/lib/components/TableCustom.svelte +++ b/frontend/src/lib/components/TableCustom.svelte @@ -1,37 +1,53 @@ -
+
- + {@render headerRow?.()} - + {@render body?.()}
{#if paginated}
diff --git a/frontend/src/lib/components/TeamSelector.svelte b/frontend/src/lib/components/TeamSelector.svelte index a03e53bde5..125afaa0bc 100644 --- a/frontend/src/lib/components/TeamSelector.svelte +++ b/frontend/src/lib/components/TeamSelector.svelte @@ -45,8 +45,6 @@ let preSearchNextLink = $state(null) let preSearchTotalCount = $state(0) - let selectedTeamId = $state(selectedTeam?.team_id) - const searchMode = $derived(!teams) // Check if there are more teams to load (based on next_link presence) @@ -62,21 +60,18 @@ return baseTeams }) - $effect(() => { - const newTeam = selectedTeamId - ? displayTeams.find((t) => t.team_id === selectedTeamId) - : undefined - - if (newTeam?.team_id !== selectedTeam?.team_id) { - selectedTeam = newTeam + // Single getter/setter to bridge Select's string value ↔ selectedTeam object. + // This replaces the previous two bidirectional $effect sync blocks. + function setSelectedTeamById(newId: string | undefined) { + if (newId) { + const team = displayTeams.find((t) => t.team_id === newId) + if (team && team.team_id !== selectedTeam?.team_id) { + selectedTeam = team + } + } else if (selectedTeam !== undefined) { + selectedTeam = undefined } - }) - - $effect(() => { - if (selectedTeam?.team_id !== selectedTeamId) { - selectedTeamId = selectedTeam?.team_id - } - }) + } let previousTeamId = $state(undefined) @@ -120,6 +115,7 @@ }) function restorePreSearchState() { + debouncedSearch.clearDebounce() searchRequestId++ // Invalidate any in-flight search if (preSearchTeams !== null) { // Restore the accumulated teams from before the search @@ -262,7 +258,10 @@ disabled={disabled || isFetching} loading={isFetching} bind:filterText={searchFilterText} - bind:value={selectedTeamId} + bind:value={ + () => selectedTeam?.team_id, + (newId) => setSelectedTeamById(newId) + } /> {:else} {:else} {/if} {#if value && isHovered} @@ -65,10 +87,10 @@ class="absolute z-10 top-[9.5px] right-2 rounded-full p-0.5 text-primary bg-surface-secondary hover:bg-surface-hover focus:bg-surface-hover {buttonClass}" aria-label="Clear" - on:click|preventDefault|stopPropagation={clear} + onclick={stopPropagation(preventDefault(clear))} > {/if} - + {@render children?.()}
diff --git a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte index 5fe25dbba2..a04923e156 100644 --- a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte @@ -79,7 +79,7 @@ const Icon = $derived(theme[type].Icon ?? AlertTriangle) - + {#if open}
import ConfirmationModal from './ConfirmationModal.svelte' - import { createEventDispatcher } from 'svelte' + import { createEventDispatcher, untrack } from 'svelte' import type { Trigger } from '$lib/components/triggers/utils' import DataTable from '$lib/components/table/DataTable.svelte' import { twMerge } from 'tailwind-merge' @@ -20,7 +20,7 @@ let { open = $bindable(false), draftTriggers = [], isFlow = false }: Props = $props() - let selectedTriggers: Trigger[] = $state(draftTriggers) + let selectedTriggers: Trigger[] = $state(untrack(() => draftTriggers)) const dispatch = createEventDispatcher<{ canceled: void diff --git a/frontend/src/lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte index 56d965be75..20ebe04c85 100644 --- a/frontend/src/lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte @@ -10,7 +10,7 @@ replaceFalseWithUndefined, type Value } from '$lib/utils' - import { page } from '$app/stores' + import { page } from '$app/state' import type { GetInitialAndModifiedValues } from './unsavedTypes' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' @@ -43,9 +43,9 @@ !bypassBeforeNavigate && getInitialAndModifiedValues && newNavigationState.to && - ((newNavigationState.to.url != $page.url && + ((newNavigationState.to.url != page.url && newNavigationState.to.url.pathname !== newNavigationState.from?.url.pathname) || - (triggerOnSearchParamsChange && newNavigationState.to.url.search != $page.url.search)) + (triggerOnSearchParamsChange && newNavigationState.to.url.search != page.url.search)) ) { goingTo = newNavigationState.to.url diff --git a/frontend/src/lib/components/common/contextmenu/ContextMenu.svelte b/frontend/src/lib/components/common/contextmenu/ContextMenu.svelte index 86a2dc7ccb..ccab6ef3f0 100644 --- a/frontend/src/lib/components/common/contextmenu/ContextMenu.svelte +++ b/frontend/src/lib/components/common/contextmenu/ContextMenu.svelte @@ -8,6 +8,7 @@ getContextMenuContainerClass, CONTEXT_MENU_ITEM_BASE_CLASS, CONTEXT_MENU_ITEM_HOVER_MELT_CLASS, + CONTEXT_MENU_ITEM_DELETE_CLASS, CONTEXT_MENU_ITEM_DISABLED_CLASS, CONTEXT_MENU_DIVIDER_CLASS, CONTEXT_MENU_ANIMATION_CLASSES @@ -20,6 +21,8 @@ disabled?: boolean onClick?: () => void divider?: boolean + type?: 'action' | 'delete' + shortcut?: string } interface Props { @@ -111,18 +114,23 @@ CONTEXT_MENU_ITEM_BASE_CLASS, menuItem.disabled ? CONTEXT_MENU_ITEM_DISABLED_CLASS - : CONTEXT_MENU_ITEM_HOVER_MELT_CLASS + : menuItem.type === 'delete' + ? CONTEXT_MENU_ITEM_DELETE_CLASS + : CONTEXT_MENU_ITEM_HOVER_MELT_CLASS )} use:melt={$item} onclick={() => handleItemClick(menuItem)} > {#if menuItem.icon} - + {/if} {#if menu} {@render menu({ item: menuItem })} {:else} - {menuItem.label} + {menuItem.label} + {/if} + {#if menuItem.shortcut} + {menuItem.shortcut} {/if}
{/if} diff --git a/frontend/src/lib/components/common/contextmenu/contextMenuStyles.ts b/frontend/src/lib/components/common/contextmenu/contextMenuStyles.ts index ecbe6c6f87..16844dc3d9 100644 --- a/frontend/src/lib/components/common/contextmenu/contextMenuStyles.ts +++ b/frontend/src/lib/components/common/contextmenu/contextMenuStyles.ts @@ -27,6 +27,12 @@ export const CONTEXT_MENU_ITEM_HOVER_CLASS = 'hover:bg-surface-hover' */ export const CONTEXT_MENU_ITEM_HOVER_MELT_CLASS = 'data-[highlighted]:bg-surface-hover' +/** + * Delete action styles for context menu items + */ +export const CONTEXT_MENU_ITEM_DELETE_CLASS = + 'text-red-600 dark:text-red-400 data-[highlighted]:bg-red-500/10 dark:data-[highlighted]:bg-red-900/80 dark:data-[highlighted]:text-red-300' + /** * Disabled state styles for context menu items */ diff --git a/frontend/src/lib/components/common/drawer/Disposable.svelte b/frontend/src/lib/components/common/drawer/Disposable.svelte index 05e469cfaf..68f079dac0 100644 --- a/frontend/src/lib/components/common/drawer/Disposable.svelte +++ b/frontend/src/lib/components/common/drawer/Disposable.svelte @@ -26,7 +26,7 @@ onClose }: Props = $props() - let offset = $state(initialOffset) + let offset = $state(untrack(() => initialOffset)) let zIndex = $derived(zIndexes.disposables + offset) export function toggleDrawer() { @@ -87,8 +87,8 @@ } if (open) { - openedDrawers.val.push(id) - offset = initialOffset + openedDrawers.val.length + openedDrawers.val.push(untrack(() => id)) + offset = untrack(() => initialOffset) + openedDrawers.val.length } let wasEverOpen = false diff --git a/frontend/src/lib/components/common/fileInput/FileInput.svelte b/frontend/src/lib/components/common/fileInput/FileInput.svelte index 37a8b3801e..11f7a0caa9 100644 --- a/frontend/src/lib/components/common/fileInput/FileInput.svelte +++ b/frontend/src/lib/components/common/fileInput/FileInput.svelte @@ -8,27 +8,49 @@ type ConvertedFile = string | ArrayBuffer | null - let c = '' - export { c as class } - export let style = '' - export let accept = '*' - export let multiple = false - export let convertTo: ReadFileAs | undefined = undefined - export let hideIcon = false - export let iconSize = 24 - export let returnFileNames = false - export let submittedText: string | undefined = undefined - export let defaultFile: string | string[] | undefined = undefined - export let disabled: boolean | undefined = undefined - export let folderOnly = false - const dispatch = createEventDispatcher() - let input: HTMLInputElement + let input: HTMLInputElement | undefined = $state() type FileWithPath = File & { path?: string } - export let files: FileWithPath[] | undefined = undefined + interface Props { + class?: string + style?: string + accept?: string + multiple?: boolean + convertTo?: ReadFileAs | undefined + hideIcon?: boolean + iconSize?: number + returnFileNames?: boolean + submittedText?: string | undefined + defaultFile?: string | string[] | undefined + disabled?: boolean | undefined + folderOnly?: boolean + files?: FileWithPath[] | undefined + selectedTitle?: import('svelte').Snippet + children?: import('svelte').Snippet + [key: string]: any + } - let pointerStartX = 0 - let pointerStartY = 0 + let { + class: c = '', + style = '', + accept = '*', + multiple = false, + convertTo = undefined, + hideIcon = false, + iconSize = 24, + returnFileNames = false, + submittedText = undefined, + defaultFile = undefined, + disabled = undefined, + folderOnly = false, + files = $bindable(undefined), + selectedTitle, + children, + ...rest + }: Props = $props() + + let pointerStartX = $state(0) + let pointerStartY = $state(0) function handlePointerDown(e: PointerEvent) { pointerStartX = e.clientX @@ -50,7 +72,7 @@ // Needs to be reset so the same file can be selected // multiple times in a row - input.value = '' + if (input) input.value = '' dispatchChange() } @@ -194,10 +216,10 @@ duration-200 px-1 py-8`, c )} - on:dragover={handleDragOver} - on:drop={handleDrop} - on:pointerdown={handlePointerDown} - on:click={(e) => { + ondragover={handleDragOver} + ondrop={handleDrop} + onpointerdown={handlePointerDown} + onclick={(e) => { const deltaX = Math.abs(e.clientX - pointerStartX) const deltaY = Math.abs(e.clientY - pointerStartY) if (deltaX > 5 || deltaY > 5) { @@ -214,11 +236,11 @@ {/if} {#if files}
- + {#if selectedTitle}{@render selectedTitle()}{:else}
{submittedText ? submittedText : `Selected file${files.length > 1 ? 's' : ''}`}:
- + {/if}
    {#each files as { name }, i}
- {:else} - - Drag and drop {folderOnly ? 'a folder' : multiple ? 'files' : 'a file'} - + {:else if children}{@render children()}{:else} + Drag and drop {folderOnly ? 'a folder' : multiple ? 'files' : 'a file'} {/if} 1 ? 's' : ''} chosen` : 'No file chosen'} bind:this={input} - on:change={({ currentTarget }) => { + onchange={({ currentTarget }) => { onChange(currentTarget.files ? Array.from(currentTarget.files) : null) }} {accept} {multiple} - {...$$restProps} + {...rest} /> {#if defaultFile && (!Array.isArray(defaultFile) || defaultFile.length > 0)}
diff --git a/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte b/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte index eaf3adcb20..2a7de694aa 100644 --- a/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte +++ b/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte @@ -6,12 +6,16 @@ import { X } from 'lucide-svelte' import FileUpload from './FileUpload.svelte' - export let title: string - export let open: boolean = false - export let fileKey: string | undefined = undefined + interface Props { + title: string; + open?: boolean; + fileKey?: string | undefined; + } - let s3Folder: string = '' + let { title, open = false, fileKey = $bindable(undefined) }: Props = $props(); + + let s3Folder: string = $state('') const dispatch = createEventDispatcher() function fadeFast(node: HTMLElement) { diff --git a/frontend/src/lib/components/common/kbd/Kbd.svelte b/frontend/src/lib/components/common/kbd/Kbd.svelte index 002f4e8c16..5cfed5353c 100644 --- a/frontend/src/lib/components/common/kbd/Kbd.svelte +++ b/frontend/src/lib/components/common/kbd/Kbd.svelte @@ -1,4 +1,5 @@ @@ -142,4 +147,17 @@ />
{/if} + {#if lang === 'claudesandbox'} +
+ +
+ {/if}
diff --git a/frontend/src/lib/components/common/layout/ListElement.svelte b/frontend/src/lib/components/common/layout/ListElement.svelte index 891079ce42..f865305c9e 100644 --- a/frontend/src/lib/components/common/layout/ListElement.svelte +++ b/frontend/src/lib/components/common/layout/ListElement.svelte @@ -1,3 +1,11 @@ + +
- + {@render children?.()}
diff --git a/frontend/src/lib/components/common/menu/MenuItem.svelte b/frontend/src/lib/components/common/menu/MenuItem.svelte index 4c3d50643c..99d14826e1 100644 --- a/frontend/src/lib/components/common/menu/MenuItem.svelte +++ b/frontend/src/lib/components/common/menu/MenuItem.svelte @@ -1,11 +1,22 @@ - - -
+ + + + +
- + {@render children?.()}
diff --git a/frontend/src/lib/components/common/menu/ResolveOpen.svelte b/frontend/src/lib/components/common/menu/ResolveOpen.svelte index ec32904a3e..5f6d1a0666 100644 --- a/frontend/src/lib/components/common/menu/ResolveOpen.svelte +++ b/frontend/src/lib/components/common/menu/ResolveOpen.svelte @@ -1,10 +1,18 @@ diff --git a/frontend/src/lib/components/common/modal/AlwaysMountedModal.svelte b/frontend/src/lib/components/common/modal/AlwaysMountedModal.svelte index c4049802fc..d80a3a14bb 100644 --- a/frontend/src/lib/components/common/modal/AlwaysMountedModal.svelte +++ b/frontend/src/lib/components/common/modal/AlwaysMountedModal.svelte @@ -1,4 +1,6 @@ - + {#if isOpen} @@ -72,10 +91,7 @@ css?.popup?.class, 'wm-modal-form-popup' )} - use:clickOutside - on:click_outside={() => { - close() - }} + use:clickOutside={{ onClickOutside: () => close() }} >
@@ -84,15 +100,15 @@
- + {@render headerLeft?.()}
- + {@render headerRight?.()}
- - + +
{}} + onclick={stopPropagation(() => {})} > - + {@render children?.()}
diff --git a/frontend/src/lib/components/common/popup/PopupV2.svelte b/frontend/src/lib/components/common/popup/PopupV2.svelte index 55bc4312b5..42257ff53b 100644 --- a/frontend/src/lib/components/common/popup/PopupV2.svelte +++ b/frontend/src/lib/components/common/popup/PopupV2.svelte @@ -1,4 +1,5 @@ {#if href} diff --git a/frontend/src/lib/components/common/table/RowIcon.svelte b/frontend/src/lib/components/common/table/RowIcon.svelte index e5769b9816..7599fa6426 100644 --- a/frontend/src/lib/components/common/table/RowIcon.svelte +++ b/frontend/src/lib/components/common/table/RowIcon.svelte @@ -18,7 +18,10 @@ Unplug } from 'lucide-svelte' - export let kind: + + + interface Props { + kind: | 'script' | 'flow' | 'app' @@ -38,13 +41,15 @@ | 'mqtt' | 'sqs' | 'gcp' - | 'emails' + | 'emails'; + /** For 'trigger' kind, specifies the specific trigger type (routes, schedules, etc.) */ + triggerKind?: string | undefined; + } - /** For 'trigger' kind, specifies the specific trigger type (routes, schedules, etc.) */ - export let triggerKind: string | undefined = undefined + let { kind, triggerKind = undefined }: Props = $props(); // Use triggerKind if kind is 'trigger' and triggerKind is provided - $: effectiveKind = kind === 'trigger' && triggerKind ? triggerKind : kind + let effectiveKind = $derived(kind === 'trigger' && triggerKind ? triggerKind : kind)
diff --git a/frontend/src/lib/components/common/table/ScriptRow.svelte b/frontend/src/lib/components/common/table/ScriptRow.svelte index 6ad3918aff..c98e326067 100644 --- a/frontend/src/lib/components/common/table/ScriptRow.svelte +++ b/frontend/src/lib/components/common/table/ScriptRow.svelte @@ -45,6 +45,7 @@ import { scriptToHubUrl } from '$lib/hub' import { isRuleActive } from '$lib/workspaceProtectionRules.svelte' import { buildForkEditUrl } from '$lib/utils/editInFork' + import { isCloudHosted } from '$lib/cloud' interface Props { script: Script & { canWrite: boolean; use_codebase: boolean } @@ -176,7 +177,7 @@
{/if} {/if} - {#if !isRuleActive('DisableWorkspaceForking') && (!showEditButton || !script.canWrite)} + {#if !isCloudHosted() && !isRuleActive('DisableWorkspaceForking') && (!showEditButton || !script.canWrite)}
+ {#snippet actions()} + + {/snippet} diff --git a/frontend/src/lib/components/copilot/MetadataGen.svelte b/frontend/src/lib/components/copilot/MetadataGen.svelte index 502b38e848..bb18b3dc1e 100644 --- a/frontend/src/lib/components/copilot/MetadataGen.svelte +++ b/frontend/src/lib/components/copilot/MetadataGen.svelte @@ -4,7 +4,7 @@ import { Check, Loader2, Wand2 } from 'lucide-svelte' import { metadataCompletionEnabled } from '$lib/stores' import { copilotInfo } from '$lib/aiStore' - import { onDestroy } from 'svelte' + import { onDestroy, untrack } from 'svelte' import { sendUserToast } from '$lib/toast' import { twMerge } from 'tailwind-merge' import autosize from '$lib/autosize' @@ -143,7 +143,7 @@ Generate a tool name for the script below: let genHeight = $state(0) let focused = $state(false) - let config: PromptConfig = promptConfigs[promptConfigName] + let config: PromptConfig = promptConfigs[untrack(() => promptConfigName)] async function generateContent(automatic = false) { abortController = new AbortController() @@ -187,10 +187,10 @@ Generate a tool name for the script below: if ( $copilotInfo.enabled && $metadataCompletionEnabled && - generateOnAppear && + untrack(() => generateOnAppear) && !content && - code && - !isInitialCode(code) + untrack(() => code) && + !isInitialCode(untrack(() => code) ?? '') ) { setTimeout(() => { el?.focus() diff --git a/frontend/src/lib/components/copilot/RegexGen.svelte b/frontend/src/lib/components/copilot/RegexGen.svelte index faf6406c2a..b716e48385 100644 --- a/frontend/src/lib/components/copilot/RegexGen.svelte +++ b/frontend/src/lib/components/copilot/RegexGen.svelte @@ -1,4 +1,6 @@ + + + {#snippet trigger()} +