diff --git a/.claude/settings.json b/.claude/settings.json index fcd49c3140..cf8bfdd284 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -110,7 +110,6 @@ ] }, "enabledPlugins": { - "rust-analyzer-lsp@claude-plugins-official": true, "typescript-lsp@claude-plugins-official": true, "code-review@claude-plugins-official": true } diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml new file mode 100644 index 0000000000..ca9ce2aaac --- /dev/null +++ b/.github/workflows/backend-test-windows.yml @@ -0,0 +1,165 @@ +name: Backend integration tests (Windows) + +on: + workflow_dispatch: + push: + branches: + - "ci-windows-tests" + +env: + CARGO_INCREMENTAL: 0 + SQLX_OFFLINE: true + DISABLE_EMBEDDING: true + +jobs: + cargo_test_windows: + runs-on: blacksmith-16vcpu-windows-2025 + steps: + - uses: actions/checkout@v4 + + - name: Read EE repo commit hash + shell: pwsh + run: | + $ee_repo_ref = Get-Content .\backend\ee-repo-ref.txt + echo "ee_repo_ref=$ee_repo_ref" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Checkout windmill-ee-private repository + uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + - name: Substitute EE code + shell: bash + run: | + ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - name: Setup PostgreSQL + uses: ikalnytskyi/action-setup-postgres@v6 + with: + username: postgres + password: changeme + database: windmill + port: 5432 + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: backend + toolchain: 1.93.0 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "9.0.x" + + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + + - uses: actions/setup-go@v2 + with: + go-version: 1.21.5 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.10 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - uses: astral-sh/setup-uv@v6.2.1 + with: + version: "0.9.24" + + - uses: shivammathur/setup-php@v2 + with: + php-version: "8.3" + tools: composer + + - name: Install windmill CLI + shell: bash + run: | + cd cli + bash gen_wm_client.sh + bun install + mkdir -p "$HOME/.local/bin" + printf '#!/bin/sh\nexec bun run "%s/cli/src/main.ts" "$@"\n' "$GITHUB_WORKSPACE" > "$HOME/.local/bin/wmill" + chmod +x "$HOME/.local/bin/wmill" + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install OpenSSL via vcpkg + run: | + vcpkg.exe install openssl-windows:x64-windows + vcpkg.exe install openssl:x64-windows-static + vcpkg.exe integrate install + + - name: Get runtime paths + id: runtime-paths + shell: pwsh + run: | + echo "DENO_PATH=$($(Get-Command deno).Source)" >> $env:GITHUB_OUTPUT + echo "BUN_PATH=$($(Get-Command bun).Source)" >> $env:GITHUB_OUTPUT + echo "NODE_BIN_PATH=$($(Get-Command node).Source)" >> $env:GITHUB_OUTPUT + echo "GO_PATH=$($(Get-Command go).Source)" >> $env:GITHUB_OUTPUT + echo "UV_PATH=$($(Get-Command uv).Source)" >> $env:GITHUB_OUTPUT + echo "PHP_PATH=$($(Get-Command php).Source)" >> $env:GITHUB_OUTPUT + echo "COMPOSER_PATH=$($(Get-Command composer).Source)" >> $env:GITHUB_OUTPUT + echo "POWERSHELL_PATH=$($(Get-Command pwsh).Source)" >> $env:GITHUB_OUTPUT + echo "DOTNET_PATH=$($(Get-Command dotnet).Source)" >> $env:GITHUB_OUTPUT + + - name: Build DuckDB FFI module + working-directory: backend/windmill-duckdb-ffi-internal + timeout-minutes: 30 + run: | + cargo build --release -p windmill_duckdb_ffi_internal + New-Item -ItemType Directory -Path ..\target\debug -Force + Copy-Item target\release\windmill_duckdb_ffi_internal.dll ..\target\debug\ + + - name: Print runtime versions and env + shell: pwsh + run: | + deno --version + bun -v + node --version + go version + python3 --version + php --version + pwsh --version + dotnet --version + echo "TEMP=$env:TEMP" + echo "TMP=$env:TMP" + echo "USERPROFILE=$env:USERPROFILE" + echo "HOME=$env:HOME" + + - name: cargo test + working-directory: backend + timeout-minutes: 60 + env: + DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill + RUST_LOG: "off" + RUST_LOG_STYLE: never + CARGO_NET_GIT_FETCH_WITH_CLI: true + CARGO_BUILD_JOBS: 12 + VCPKGRS_DYNAMIC: 1 + OPENSSL_DIR: ${{ env.VCPKG_INSTALLATION_ROOT }}\installed\x64-windows-static + DENO_PATH: ${{ steps.runtime-paths.outputs.DENO_PATH }} + BUN_PATH: ${{ steps.runtime-paths.outputs.BUN_PATH }} + NODE_BIN_PATH: ${{ steps.runtime-paths.outputs.NODE_BIN_PATH }} + GO_PATH: ${{ steps.runtime-paths.outputs.GO_PATH }} + UV_PATH: ${{ steps.runtime-paths.outputs.UV_PATH }} + PHP_PATH: ${{ steps.runtime-paths.outputs.PHP_PATH }} + COMPOSER_PATH: ${{ steps.runtime-paths.outputs.COMPOSER_PATH }} + POWERSHELL_PATH: ${{ steps.runtime-paths.outputs.POWERSHELL_PATH }} + DOTNET_PATH: ${{ steps.runtime-paths.outputs.DOTNET_PATH }} + WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1 + WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1 + WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1 + run: > + cargo test + --no-fail-fast + --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,csharp,php,quickjs,mcp,run_inline + --all + -- --nocapture --test-threads=10 diff --git a/.webmux.yaml b/.webmux.yaml new file mode 100644 index 0000000000..b01d98b8d2 --- /dev/null +++ b/.webmux.yaml @@ -0,0 +1,52 @@ +# Project display name in the dashboard +name: Windmill + +workspace: + mainBranch: main + worktreeRoot: ../windmill__worktrees + defaultAgent: claude + +startupEnvs: + CARGO_FEATURES: "quickjs" + WM_CLONE_DB: false + USE_RUST_PLUGIN: false + +lifecycleHooks: + postCreate: bash ./scripts/post-create.sh + preRemove: bash ./scripts/pre-remove.sh + +# Each service defines a port env var that webmux injects into pane and agent +# process environments when creating a worktree. Ports are auto-assigned: +# base + (slot x step). +services: + - name: backend + portEnv: BACKEND_PORT + portStart: 8000 + portStep: 10 + - name: frontend + portEnv: FRONTEND_PORT + portStart: 3000 + portStep: 10 + +profiles: + default: + runtime: host + envPassthrough: [] + panes: + - id: agent + kind: agent + focus: true + - id: backend + kind: command + split: right + command: ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/backend" && PORT=${BACKEND_PORT:-8000} cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}" + - id: frontend + kind: command + split: bottom + command: ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/frontend" && npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0 + +integrations: + github: + linkedRepos: [] + linear: + enabled: true diff --git a/.wmdev.yaml b/.wmdev.yaml deleted file mode 100644 index 8a5afce840..0000000000 --- a/.wmdev.yaml +++ /dev/null @@ -1,106 +0,0 @@ -services: - - name: BE - portEnv: BACKEND_PORT - - name: FE - portEnv: FRONTEND_PORT - -profiles: - default: - name: default - - sandbox: - name: sandbox - image: windmill-sandbox - envPassthrough: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - R2_ENDPOINT - - R2_BUCKET - - R2_PUBLIC_URL - extraMounts: - - hostPath: ~/.ssh - guestPath: /root/.ssh - writable: true - - hostPath: ~/.codex - guestPath: /root/.codex - writable: true - - hostPath: ~/windmill-ee-private - writable: true - - hostPath: ~/windmill-ee-private__worktrees - writable: true - systemPrompt: > - You are running inside a sandboxed container with full permissions. - This worktree is configured with the following ports: - - - Backend: port ${BACKEND_PORT}. - Start with: cd backend && PORT=${BACKEND_PORT} - DATABASE_URL=postgres://postgres:changeme@localhost:5432/windmill - cargo watch -x run - - - Frontend: port ${FRONTEND_PORT}. - Start with: cd frontend && REMOTE=http://localhost:${BACKEND_PORT} - npm run dev -- --port ${FRONTEND_PORT} --host 0.0.0.0 - - --- Screenshots --- - You can take screenshots of the frontend UI and upload them to R2 - for use in PR descriptions. - 1) Take a screenshot: - bunx playwright screenshot --browser chromium - http://localhost:${FRONTEND_PORT}/path/to/page /tmp/screenshot.png - 2) Upload to R2: - aws s3 cp /tmp/screenshot.png - "s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/screenshot.png" - --endpoint-url "$(printenv R2_ENDPOINT)" - 3) The public URL will be: - $(printenv R2_PUBLIC_URL)//screenshot.png - 4) Include in PR descriptions using markdown image syntax. - - --- Terminal Recordings (asciinema) --- - You can record terminal sessions and upload them for sharing. - asciinema is available on PATH. - - 1) Write a shell script with the commands to demo. Add sleep - delays for readable pacing: - - 0.5s after printing a "$ command" line (lets viewer read it) - - 1.5-2s after command output (lets viewer absorb the result) - - Set GIT_PAGER=cat and PAGER=cat to prevent pager hangs - - 2) Record headlessly: - asciinema rec --headless --overwrite \ - -c "bash /tmp/demo.sh" \ - --window-size 120x50 \ - --title "Description of demo" \ - /tmp/demo.cast - - 3) Upload to asciinema.org: - XDG_DATA_HOME=/tmp/.local/share \ - asciinema upload --server-url https://asciinema.org /tmp/demo.cast - - --- Mermaid Diagrams --- - You can render Mermaid diagrams to SVG using the pre-installed mmdc CLI. - The puppeteer config (no-sandbox + Chromium path) is at /root/.puppeteerrc.json. - - 1) Write a .mmd file with your diagram: - cat > /tmp/diagram.mmd << 'EOF' - graph TD - A[Start] --> B[End] - EOF - - 2) Render to SVG (the -p flag is required): - mmdc -i /tmp/diagram.mmd -o /tmp/diagram.svg -p /root/.puppeteerrc.json - - 3) Upload to R2: - aws s3 cp /tmp/diagram.svg - "s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/diagram.svg" - --endpoint-url "$(printenv R2_ENDPOINT)" - - 4) The public URL will be: - $(printenv R2_PUBLIC_URL)//diagram.svg - - 5) Include in PR descriptions using markdown image syntax. - - IMPORTANT: Read docs/autonomous-mode.md before starting any work. - -linkedRepos: - - repo: windmill-labs/windmill-ee-private - alias: ee diff --git a/.workmux.yaml b/.workmux.yaml index fcd1906080..46049109c0 100644 --- a/.workmux.yaml +++ b/.workmux.yaml @@ -1,5 +1,3 @@ -name: Windmill - main_branch: main merge_strategy: rebase diff --git a/CHANGELOG.md b/CHANGELOG.md index 15fca3e649..fc48489883 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,71 @@ # Changelog +## [1.651.1](https://github.com/windmill-labs/windmill/compare/v1.651.0...v1.651.1) (2026-03-05) + + +### Bug Fixes + +* prevent slow loading toast interval from leaking on promise cancellation ([#8240](https://github.com/windmill-labs/windmill/issues/8240)) ([2e582b1](https://github.com/windmill-labs/windmill/commit/2e582b1bc1c299388a3c97cfddff9d0eb92858f2)) +* suppress unused variable warnings on windows builds ([#8241](https://github.com/windmill-labs/windmill/issues/8241)) ([2d58382](https://github.com/windmill-labs/windmill/commit/2d583826dc065c05684d4cd1d1510f0d1f2d9ae9)) + +## [1.651.0](https://github.com/windmill-labs/windmill/compare/v1.650.0...v1.651.0) (2026-03-05) + + +### Features + +* add sandbox annotations, volume mounts, for AI sandbox starting with claude ([#8058](https://github.com/windmill-labs/windmill/issues/8058)) ([5f0ef93](https://github.com/windmill-labs/windmill/commit/5f0ef936d1d5d07d01c8e07e26ec254feebef8fb)) +* hash-based MCP tool names for long paths ([#8133](https://github.com/windmill-labs/windmill/issues/8133)) ([ce041e8](https://github.com/windmill-labs/windmill/commit/ce041e8a5e7ff105df389875d9981f3843d4ce39)) + + +### Bug Fixes + +* **python-client:** add delete_s3_object ([#8216](https://github.com/windmill-labs/windmill/issues/8216)) ([90f4c64](https://github.com/windmill-labs/windmill/commit/90f4c64ee12e1d04ce846ff88d6658f667e194e0)) +* update CLI bun template to match UI template ([#8238](https://github.com/windmill-labs/windmill/issues/8238)) ([a8cbe93](https://github.com/windmill-labs/windmill/commit/a8cbe9396ffc51140dce5582d57f4dc59873304e)) +* write fallback package.json for codebase mode nsjail ([#8239](https://github.com/windmill-labs/windmill/issues/8239)) ([d46913b](https://github.com/windmill-labs/windmill/commit/d46913b74a0ffd41d2323e0355cc81954f09e29d)) + +## [1.650.0](https://github.com/windmill-labs/windmill/compare/v1.649.0...v1.650.0) (2026-03-05) + + +### Features + +* add move, delete, and duplicate to flow node context menu ([#8050](https://github.com/windmill-labs/windmill/issues/8050)) ([c0c9388](https://github.com/windmill-labs/windmill/commit/c0c9388415716ce77d841bd08a46f94e0a529685)) +* add variable and resource types to flow env variables ([#8214](https://github.com/windmill-labs/windmill/issues/8214)) ([164e499](https://github.com/windmill-labs/windmill/commit/164e499c64dc5eb76fcfb0f8cefbad2df244f610)) +* Ducklake typechecker ([#8118](https://github.com/windmill-labs/windmill/issues/8118)) ([53caecf](https://github.com/windmill-labs/windmill/commit/53caecf1da8d76e246178dfb9b86d330f0ec52fd)) +* make WINDMILL_DIR configurable via environment variable ([#8215](https://github.com/windmill-labs/windmill/issues/8215)) ([424ca59](https://github.com/windmill-labs/windmill/commit/424ca59dfe3e730f5388d9cac4ea7e69773614d3)) +* make WM_END_USER_EMAIL display users from different workspaces ([#8208](https://github.com/windmill-labs/windmill/issues/8208)) ([baf2bcf](https://github.com/windmill-labs/windmill/commit/baf2bcf14da0c8c95bdbbf511fcaee48be33948b)) +* persistent Db manager state in URI ([#8134](https://github.com/windmill-labs/windmill/issues/8134)) ([4bf827b](https://github.com/windmill-labs/windmill/commit/4bf827bea4d44aca8c5ff7aa67ad449dbcf00673)) +* replace hub error toasts with warning alerts and add disable hub setting ([#8225](https://github.com/windmill-labs/windmill/issues/8225)) ([63ebae8](https://github.com/windmill-labs/windmill/commit/63ebae8829a6dc47a4e23c8670b514f042c9d4be)) +* token expiration notifications ([#8190](https://github.com/windmill-labs/windmill/issues/8190)) ([e56ccd2](https://github.com/windmill-labs/windmill/commit/e56ccd200be29e6ac8ea2b04a341b1ce78a307f6)) + + +### Bug Fixes + +* handle multipart stream errors gracefully instead of panicking ([#8226](https://github.com/windmill-labs/windmill/issues/8226)) ([19c065b](https://github.com/windmill-labs/windmill/commit/19c065bed5468c484c8e7a50a6b79ab90153cc0e)) +* improve windows compatibility ([077779e](https://github.com/windmill-labs/windmill/commit/077779ec52f7d3e5fcc93951544bf47bd6dc30b6)) +* wrap set_encryption_key in a single database transaction ([#8212](https://github.com/windmill-labs/windmill/issues/8212)) ([62382fd](https://github.com/windmill-labs/windmill/commit/62382fd2869ea0190dd0c0b714f9cbd35ceddd7a)) + +## [1.649.0](https://github.com/windmill-labs/windmill/compare/v1.648.0...v1.649.0) (2026-03-03) + + +### Features + +* **frontend:** add script recorder for offline replay ([#8200](https://github.com/windmill-labs/windmill/issues/8200)) ([c97d8b4](https://github.com/windmill-labs/windmill/commit/c97d8b4715f86ea83ab2c0223ba859ced690829a)) +* move index management out of /srch/, add storage size reporting ([#8169](https://github.com/windmill-labs/windmill/issues/8169)) ([ee01acd](https://github.com/windmill-labs/windmill/commit/ee01acd9a6a2cd68a3f226988bfb46f6a6e64c08)) + + +### Bug Fixes + +* clean up slow-load toast interval on component destroy ([#8207](https://github.com/windmill-labs/windmill/issues/8207)) ([26f4f2b](https://github.com/windmill-labs/windmill/commit/26f4f2b399b828185b553289d6560e12261030a3)) +* **frontend:** prevent subflow expansion from hiding all insertion points ([#8203](https://github.com/windmill-labs/windmill/issues/8203)) ([e97da86](https://github.com/windmill-labs/windmill/commit/e97da860672171e33054a77d71f4824bb09e540d)) +* gracefully handle malformed OAuth entries in instance config ([#8205](https://github.com/windmill-labs/windmill/issues/8205)) ([cac4bdd](https://github.com/windmill-labs/windmill/commit/cac4bdd54f0c3ea80844ac31f7597f418ff7d8ae)) +* skip stop_after_if evaluation for skipped (identity) flow steps ([#8201](https://github.com/windmill-labs/windmill/issues/8201)) ([e6f7775](https://github.com/windmill-labs/windmill/commit/e6f7775d4d9a052aefc37260c6ed161146841cd7)) +* use exact matching for python requirements directive parsing ([#8199](https://github.com/windmill-labs/windmill/issues/8199)) ([2b2be38](https://github.com/windmill-labs/windmill/commit/2b2be38f129bbe58b6bb3815c4bd94aa03a3da90)) + + +### Performance Improvements + +* use two-step query in input history to leverage v2_job index ([#8197](https://github.com/windmill-labs/windmill/issues/8197)) ([50defdd](https://github.com/windmill-labs/windmill/commit/50defdded113b4d2cf0991b3fb642d1cd9a462b7)) + ## [1.648.0](https://github.com/windmill-labs/windmill/compare/v1.647.2...v1.648.0) (2026-03-02) diff --git a/CLAUDE.md b/CLAUDE.md index 4e7afeba8a..0acc541c92 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,6 +26,27 @@ Open-source platform for internal tools, workflows, API integrations, background - **Login**: `admin@windmill.dev` / `changeme` - **Instance settings**: navigate to `/#superadmin-settings` +## Banned Patterns + +### `$bindable(default_value)` on optional props + +Using `$bindable(default_value)` on props that can be `undefined` is **banned**. This pattern causes subtle bugs because the default value masks the `undefined` state. + +**Bad:** +```svelte +let { my_prop = $bindable(default_value) }: { my_prop?: string } = $props() +``` + +**Correct alternatives:** + +1. **Use `$derived` with nullish coalescing** — handle the potential `undefined` at the usage site: + ```svelte + let { my_prop = $bindable() }: { my_prop?: string } = $props() + let effective_value = $derived(my_prop ?? default_value) + ``` + +2. **Create a `useMyPropState()` helper** — encapsulate the undefined-handling logic in a reusable function and call it higher in the component tree, so the child component always receives a defined value. + ## Core Principles - Search for existing code to reuse before writing new code diff --git a/Dockerfile b/Dockerfile index 7cca6ab329..0cc19801d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -262,6 +262,12 @@ COPY --from=oven/bun:1.3.10 /usr/local/bin/bun /usr/bin/bun RUN bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# The installer puts the binary in ~/.local/bin/claude (symlink to ~/.local/share/claude/versions/*) +# Copy it to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json b/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json new file mode 100644 index 0000000000..d9d9793cd1 --- /dev/null +++ b/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = now() + interval '60 seconds'\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f" +} diff --git a/backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json b/backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json similarity index 51% rename from backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json rename to backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json index 84b10ccba6..409faa032f 100644 --- a/backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json +++ b/backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json @@ -1,16 +1,17 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT token\n FROM token\n WHERE token LIKE concat($1::text, '%')\n LIMIT 1\n ", + "query": "SELECT group_ FROM usr_to_group WHERE usr = $1 AND workspace_id = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "token", + "name": "group_", "type_info": "Varchar" } ], "parameters": { "Left": [ + "Text", "Text" ] }, @@ -18,5 +19,5 @@ false ] }, - "hash": "90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492" + "hash": "015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11" } diff --git a/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json b/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json new file mode 100644 index 0000000000..52fb375962 --- /dev/null +++ b/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT large_file_storage->>'volume_storage' FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e" +} diff --git a/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json b/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json new file mode 100644 index 0000000000..6a6b77e650 --- /dev/null +++ b/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "extra_perms", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4" +} diff --git a/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json b/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json new file mode 100644 index 0000000000..0140324406 --- /dev/null +++ b/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750" +} diff --git a/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json b/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json new file mode 100644 index 0000000000..9a6ae60a49 --- /dev/null +++ b/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6" +} diff --git a/backend/.sqlx/query-19a7ebb2e7e8e57b6e7c974da8eb7c6841a5c4ff12ba7c12c73d691c49dd99ed.json b/backend/.sqlx/query-19a7ebb2e7e8e57b6e7c974da8eb7c6841a5c4ff12ba7c12c73d691c49dd99ed.json new file mode 100644 index 0000000000..18ad13d90f --- /dev/null +++ b/backend/.sqlx/query-19a7ebb2e7e8e57b6e7c974da8eb7c6841a5c4ff12ba7c12c73d691c49dd99ed.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email FROM token WHERE token = $1 AND (expiration > NOW() OR expiration IS NULL)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "19a7ebb2e7e8e57b6e7c974da8eb7c6841a5c4ff12ba7c12c73d691c49dd99ed" +} diff --git a/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json b/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json new file mode 100644 index 0000000000..9514010409 --- /dev/null +++ b/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e" +} diff --git a/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json b/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json new file mode 100644 index 0000000000..fa67a0797d --- /dev/null +++ b/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at)\n VALUES ($1, $2, $3, $4, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET size_bytes = $3, last_used_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab" +} diff --git a/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json b/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json new file mode 100644 index 0000000000..8897e8a7de --- /dev/null +++ b/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc" +} diff --git a/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json b/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json new file mode 100644 index 0000000000..2010b40667 --- /dev/null +++ b/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98" +} diff --git a/backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json b/backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json new file mode 100644 index 0000000000..8c5f43ab07 --- /dev/null +++ b/backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n CASE\n WHEN flow_version.id IS NOT NULL THEN\n flow_version.value -> 'flow_env' -> $3\n ELSE\n root_job.raw_flow -> 'flow_env' -> $3\n END AS \"flow_env: sqlx::types::Json>\"\n FROM\n v2_job current_job\n JOIN\n v2_job root_job ON root_job.id = COALESCE(current_job.root_job, current_job.flow_innermost_root_job, current_job.parent_job, current_job.id)\n AND root_job.workspace_id = current_job.workspace_id\n LEFT JOIN\n flow_version ON flow_version.id = root_job.runnable_id\n AND flow_version.path = root_job.runnable_path\n AND flow_version.workspace_id = root_job.workspace_id\n WHERE\n current_job.id = $1 AND\n current_job.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flow_env: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943" +} diff --git a/backend/.sqlx/query-380ca9ebea53d5c016e4e76797cc103178ac4a25fc2842a13ce19b1ec4445c9d.json b/backend/.sqlx/query-380ca9ebea53d5c016e4e76797cc103178ac4a25fc2842a13ce19b1ec4445c9d.json new file mode 100644 index 0000000000..10dfbd3128 --- /dev/null +++ b/backend/.sqlx/query-380ca9ebea53d5c016e4e76797cc103178ac4a25fc2842a13ce19b1ec4445c9d.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO global_settings (name, value) VALUES ('indexer_settings', $1)\n ON CONFLICT (name) DO UPDATE SET value = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "380ca9ebea53d5c016e4e76797cc103178ac4a25fc2842a13ce19b1ec4445c9d" +} diff --git a/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json b/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json new file mode 100644 index 0000000000..2fdb1ae80d --- /dev/null +++ b/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), updated_by = $5, last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int4", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34" +} diff --git a/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json b/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json new file mode 100644 index 0000000000..c73c00c2aa --- /dev/null +++ b/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json @@ -0,0 +1,76 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n name as \"name!\",\n size_bytes as \"size_bytes!\",\n file_count as \"file_count!\",\n created_at as \"created_at!\",\n created_by as \"created_by!\",\n updated_at,\n updated_by,\n description as \"description!\",\n last_used_at,\n extra_perms as \"extra_perms!\"\n FROM (\n SELECT\n COALESCE(v.name, a.path) as name,\n COALESCE(v.size_bytes, 0) as size_bytes,\n COALESCE(v.file_count, 0) as file_count,\n COALESCE(v.created_at, a.min_created_at) as created_at,\n COALESCE(v.created_by, 'unknown') as created_by,\n v.updated_at,\n v.updated_by,\n COALESCE(v.description, '') as description,\n v.last_used_at,\n COALESCE(v.extra_perms, '{}'::jsonb) as extra_perms\n FROM (\n SELECT path, MIN(created_at) as min_created_at\n FROM asset\n WHERE workspace_id = $1 AND kind = 'volume'\n GROUP BY path\n ) a\n FULL OUTER JOIN volume v ON v.workspace_id = $1 AND v.name = a.path\n WHERE v.workspace_id = $1 OR a.path IS NOT NULL\n ) combined\n ORDER BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "size_bytes!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "file_count!", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "created_at!", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "updated_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "updated_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "description!", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "last_used_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "extra_perms!", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + true, + true, + null, + true, + null + ] + }, + "hash": "40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json b/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json new file mode 100644 index 0000000000..2eda021880 --- /dev/null +++ b/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM volume WHERE workspace_id = $1 AND name = $2\n AND (lease_until IS NULL OR lease_until < now())\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa" +} diff --git a/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json b/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json new file mode 100644 index 0000000000..dd4a011ee1 --- /dev/null +++ b/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by, extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "extra_perms", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147" +} diff --git a/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json b/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json new file mode 100644 index 0000000000..746c306c8f --- /dev/null +++ b/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM volume WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6" +} diff --git a/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json b/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json new file mode 100644 index 0000000000..3df9c6c195 --- /dev/null +++ b/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int4", + "Text" + ] + }, + "nullable": [] + }, + "hash": "75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca" +} diff --git a/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json b/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json new file mode 100644 index 0000000000..0bf454028b --- /dev/null +++ b/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = now() + interval '60 seconds'\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f" +} diff --git a/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json b/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json new file mode 100644 index 0000000000..7b85bd9315 --- /dev/null +++ b/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM volume WHERE workspace_id = $1 AND name = $2 AND lease_until > now() AND leased_by = $3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5" +} diff --git a/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json b/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json new file mode 100644 index 0000000000..8cbe7146b6 --- /dev/null +++ b/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33" +} diff --git a/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json b/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json new file mode 100644 index 0000000000..594fcf2960 --- /dev/null +++ b/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b" +} diff --git a/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json b/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json new file mode 100644 index 0000000000..c60239fbce --- /dev/null +++ b/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4" +} diff --git a/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json b/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json new file mode 100644 index 0000000000..b364fcd4ac --- /dev/null +++ b/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET extra_perms = extra_perms - $1\n WHERE workspace_id = $2 AND name = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d" +} diff --git a/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json b/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json new file mode 100644 index 0000000000..728141923a --- /dev/null +++ b/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2 AND lease_until > now()", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "leased_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a" +} diff --git a/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json b/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json new file mode 100644 index 0000000000..26c254eb0b --- /dev/null +++ b/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a" +} diff --git a/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json b/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json new file mode 100644 index 0000000000..ee64e97d12 --- /dev/null +++ b/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7" +} diff --git a/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json b/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json new file mode 100644 index 0000000000..fc33fd7373 --- /dev/null +++ b/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89" +} diff --git a/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json b/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json new file mode 100644 index 0000000000..86a99d2eaf --- /dev/null +++ b/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de" +} diff --git a/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json b/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json new file mode 100644 index 0000000000..9051a88a50 --- /dev/null +++ b/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT extra_perms, created_by FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7" +} diff --git a/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json b/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json new file mode 100644 index 0000000000..1f28a0a5a7 --- /dev/null +++ b/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, 0, $3)\n ON CONFLICT (workspace_id, name) DO NOTHING\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6" +} diff --git a/backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json b/backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json new file mode 100644 index 0000000000..af35d619fa --- /dev/null +++ b/backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token_expiry_notification (token, expiration) VALUES ($1, $2) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437" +} diff --git a/backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json b/backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json new file mode 100644 index 0000000000..00604f4bc9 --- /dev/null +++ b/backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token_expiry_notification WHERE expiration <= now()", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4" +} diff --git a/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json b/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json new file mode 100644 index 0000000000..3c7c1ad52a --- /dev/null +++ b/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935" +} diff --git a/backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json b/backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json new file mode 100644 index 0000000000..9085383617 --- /dev/null +++ b/backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token WHERE expiration <= now()\n RETURNING substring(token for 10) as token_prefix, label, email, workspace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token_prefix", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "label", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + true, + true, + true + ] + }, + "hash": "bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f" +} diff --git a/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json b/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json new file mode 100644 index 0000000000..442d55ed25 --- /dev/null +++ b/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes, last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb" +} diff --git a/backend/.sqlx/query-bcefd1ce47d05f2ce14493f0e7c4d4fea16c0cf71ddc233f6431cf624ecdfe60.json b/backend/.sqlx/query-bcefd1ce47d05f2ce14493f0e7c4d4fea16c0cf71ddc233f6431cf624ecdfe60.json new file mode 100644 index 0000000000..01e0e4671e --- /dev/null +++ b/backend/.sqlx/query-bcefd1ce47d05f2ce14493f0e7c4d4fea16c0cf71ddc233f6431cf624ecdfe60.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, last_locked_at, owner FROM concurrency_locks WHERE id = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "last_locked_at", + "type_info": "Timestamp" + }, + { + "ordinal": 2, + "name": "owner", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "bcefd1ce47d05f2ce14493f0e7c4d4fea16c0cf71ddc233f6431cf624ecdfe60" +} diff --git a/backend/.sqlx/query-c23bea7db9623a60683596b7d6e689e2c0100c1569436a01b207876aaa470154.json b/backend/.sqlx/query-c23bea7db9623a60683596b7d6e689e2c0100c1569436a01b207876aaa470154.json deleted file mode 100644 index be352ce88e..0000000000 --- a/backend/.sqlx/query-c23bea7db9623a60683596b7d6e689e2c0100c1569436a01b207876aaa470154.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n CASE\n WHEN flow_version.id IS NOT NULL THEN\n (flow_version.value -> 'flow_env' -> $3) #> $4\n ELSE\n (root_job.raw_flow -> 'flow_env' -> $3) #> $4\n END AS \"flow_env: sqlx::types::Json>\"\n FROM\n v2_job current_job\n JOIN\n v2_job root_job ON root_job.id = COALESCE(current_job.root_job, current_job.flow_innermost_root_job, current_job.parent_job, current_job.id)\n AND root_job.workspace_id = current_job.workspace_id\n LEFT JOIN\n flow_version ON flow_version.id = root_job.runnable_id\n AND flow_version.path = root_job.runnable_path\n AND flow_version.workspace_id = root_job.workspace_id\n WHERE\n current_job.id = $1 AND\n current_job.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "flow_env: sqlx::types::Json>", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Text", - "TextArray" - ] - }, - "nullable": [ - null - ] - }, - "hash": "c23bea7db9623a60683596b7d6e689e2c0100c1569436a01b207876aaa470154" -} diff --git a/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json b/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json new file mode 100644 index 0000000000..0400e6992d --- /dev/null +++ b/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, name, size_bytes, created_by, last_used_at\n FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true + ] + }, + "hash": "d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c" +} diff --git a/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json b/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json new file mode 100644 index 0000000000..16fc965f6a --- /dev/null +++ b/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "leased_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04" +} diff --git a/backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json b/backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json new file mode 100644 index 0000000000..015aa7b05a --- /dev/null +++ b/backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token_expiry_notification n\n USING token t\n WHERE n.token = t.token\n AND n.expiration > now()\n AND n.expiration <= now() + interval '7 days'\n RETURNING substring(t.token for 10) as token_prefix, t.label, t.email, t.workspace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token_prefix", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "label", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + true, + true, + true + ] + }, + "hash": "d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6" +} diff --git a/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json b/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json new file mode 100644 index 0000000000..91d89df9cd --- /dev/null +++ b/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, size_bytes FROM volume WHERE workspace_id = $1 ORDER BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "size_bytes", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865" +} diff --git a/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json b/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json new file mode 100644 index 0000000000..c4a1d4cd07 --- /dev/null +++ b/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2::bool), true)\n WHERE workspace_id = $3 AND name = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d" +} diff --git a/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json new file mode 100644 index 0000000000..c96961eac4 --- /dev/null +++ b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT token as \"token!\"\n FROM token\n WHERE token LIKE concat($1::text, '%')\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06" +} diff --git a/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json b/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json new file mode 100644 index 0000000000..649ee4c387 --- /dev/null +++ b/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT permissioned_as FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "permissioned_as", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0" +} diff --git a/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json b/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json new file mode 100644 index 0000000000..e1ca416938 --- /dev/null +++ b/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes, file_count, leased_by, lease_until\n FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "file_count", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "leased_by", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "lease_until", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + true + ] + }, + "hash": "f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index a2a27a9413..3f0ee056f9 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -860,9 +860,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9" +checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf" dependencies = [ "aws-lc-sys", "zeroize", @@ -870,9 +870,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.37.1" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" +checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e" dependencies = [ "cc", "cmake", @@ -1334,9 +1334,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.14" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b53543b4b86ed43f051644f704a98c7291b3618b67adf057ee77a366fa52fcaa" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" dependencies = [ "xmlparser", ] @@ -1900,7 +1900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" dependencies = [ "once_cell", - "proc-macro-crate 3.4.0", + "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.117", @@ -2550,6 +2550,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cooked-waker" version = "5.0.0" @@ -6173,20 +6182,20 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", "wasip2", "wasip3", ] @@ -7421,9 +7430,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "ipnetwork" @@ -8683,7 +8692,7 @@ dependencies = [ "darling 0.20.11", "heck 0.5.0", "num-bigint", - "proc-macro-crate 3.4.0", + "proc-macro-crate", "proc-macro-error2", "proc-macro2", "quote", @@ -9252,7 +9261,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ - "proc-macro-crate 3.4.0", + "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.117", @@ -10331,16 +10340,6 @@ dependencies = [ "elliptic-curve", ] -[[package]] -name = "proc-macro-crate" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" -dependencies = [ - "once_cell", - "toml_edit 0.19.15", -] - [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -10710,9 +10709,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -10723,6 +10722,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radium" version = "0.7.0" @@ -11088,9 +11093,12 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "relative-path" -version = "1.9.3" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0" +dependencies = [ + "serde", +] [[package]] name = "rend" @@ -11384,9 +11392,9 @@ dependencies = [ [[package]] name = "rquickjs" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16661bff09e9ed8e01094a188b463de45ec0693ade55b92ed54027d7ba7c40c" +checksum = "c50dc6d6c587c339edb4769cf705867497a2baf0eca8b4645fa6ecd22f02c77a" dependencies = [ "rquickjs-core", "rquickjs-macro", @@ -11394,26 +11402,27 @@ dependencies = [ [[package]] name = "rquickjs-core" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8db6379e204ef84c0811e90e7cc3e3e4d7688701db68a00d14a6db6849087b" +checksum = "b8bf7840285c321c3ab20e752a9afb95548c75cd7f4632a0627cea3507e310c1" dependencies = [ "async-lock", + "hashbrown 0.16.0", "relative-path", "rquickjs-sys", ] [[package]] name = "rquickjs-macro" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6041104330c019fcd936026ae05e2446f5e8a2abef329d924f25424b7052a2f3" +checksum = "7106215ff41a5677b104906a13e1a440b880f4b6362b5dc4f3978c267fad2b80" dependencies = [ - "convert_case 0.6.0", + "convert_case 0.10.0", "fnv", "ident_case", "indexmap 2.11.1", - "proc-macro-crate 1.3.1", + "proc-macro-crate", "proc-macro2", "quote", "rquickjs-core", @@ -11422,9 +11431,9 @@ dependencies = [ [[package]] name = "rquickjs-sys" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bc352c6b663604c3c186c000cfcc6c271f4b50bc135a285dd6d4f2a42f9790a" +checksum = "27344601ef27460e82d6a4e1ecb9e7e99f518122095f3c51296da8e9be2b9d83" dependencies = [ "cc", ] @@ -13850,7 +13859,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" dependencies = [ "fastrand", - "getrandom 0.4.1", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -15732,7 +15741,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-nats", @@ -15764,6 +15773,7 @@ dependencies = [ "sql-builder", "sqlx", "strum 0.27.2", + "tar", "tempfile", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", @@ -15789,14 +15799,16 @@ dependencies = [ "windmill-queue", "windmill-runtime-nativets", "windmill-test-utils", + "windmill-types", "windmill-worker", + "windmill-worker-volumes", "windows-service", "windows-sys 0.52.0", ] [[package]] name = "windmill-alerting" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "chrono", @@ -15809,7 +15821,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "argon2", @@ -15837,6 +15849,7 @@ dependencies = [ "dashmap 6.1.0", "datafusion", "ed25519-dalek", + "eventsource-stream", "flate2", "futures", "git-version", @@ -15943,11 +15956,12 @@ dependencies = [ "windmill-trigger-websocket", "windmill-types", "windmill-worker", + "windmill-worker-volumes", ] [[package]] name = "windmill-api-agent-workers" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "chrono", @@ -15970,7 +15984,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "chrono", @@ -15983,7 +15997,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "axum 0.7.9", @@ -16009,7 +16023,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.648.0" +version = "1.651.1" dependencies = [ "reqwest 0.12.28", "serde", @@ -16019,7 +16033,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "chrono", @@ -16036,7 +16050,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "base64 0.22.1", @@ -16059,7 +16073,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "axum 0.7.9", @@ -16082,7 +16096,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "chrono", @@ -16098,7 +16112,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "chrono", @@ -16118,7 +16132,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "chrono", @@ -16138,7 +16152,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "chrono", @@ -16152,7 +16166,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-nats", @@ -16179,7 +16193,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "axum 0.7.9", @@ -16204,7 +16218,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "flate2", @@ -16222,7 +16236,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "axum 0.7.9", @@ -16243,7 +16257,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "chrono", @@ -16263,7 +16277,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "chrono", @@ -16293,7 +16307,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "axum 0.7.9", @@ -16320,7 +16334,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.648.0" +version = "1.651.1" dependencies = [ "lazy_static", "serde", @@ -16332,7 +16346,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.648.0" +version = "1.651.1" dependencies = [ "argon2", "axum 0.7.9", @@ -16355,7 +16369,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "chrono", @@ -16369,7 +16383,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.648.0" +version = "1.651.1" dependencies = [ "axum 0.7.9", "chrono", @@ -16377,6 +16391,7 @@ dependencies = [ "http 1.4.0", "hyper 1.8.1", "lazy_static", + "magic-crypt", "regex", "serde", "serde_json", @@ -16399,7 +16414,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.648.0" +version = "1.651.1" dependencies = [ "chrono", "lazy_static", @@ -16413,7 +16428,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "axum 0.7.9", @@ -16432,7 +16447,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.648.0" +version = "1.651.1" dependencies = [ "aes-gcm", "anyhow", @@ -16531,7 +16546,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.648.0" +version = "1.651.1" dependencies = [ "chrono", "itertools 0.14.0", @@ -16550,7 +16565,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.648.0" +version = "1.651.1" dependencies = [ "regex", "serde", @@ -16565,7 +16580,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16589,7 +16604,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "futures", @@ -16606,7 +16621,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.648.0" +version = "1.651.1" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16622,7 +16637,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-trait", @@ -16643,7 +16658,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-trait", @@ -16674,7 +16689,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-oauth2", @@ -16698,7 +16713,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-stream", @@ -16732,7 +16747,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "futures", @@ -16750,7 +16765,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.648.0" +version = "1.651.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -16759,7 +16774,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "lazy_static", @@ -16771,7 +16786,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "serde_json", @@ -16783,7 +16798,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "gosyn", @@ -16795,7 +16810,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "lazy_static", @@ -16807,7 +16822,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "serde_json", @@ -16819,7 +16834,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "nu-parser", @@ -16830,7 +16845,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16841,7 +16856,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16854,7 +16869,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-recursion", @@ -16878,7 +16893,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "lazy_static", @@ -16892,7 +16907,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16909,7 +16924,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "lazy_static", @@ -16924,7 +16939,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "lazy_static", @@ -16943,7 +16958,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "serde", @@ -16954,7 +16969,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-recursion", @@ -16991,7 +17006,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "const_format", @@ -17029,7 +17044,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.648.0" +version = "1.651.1" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17040,7 +17055,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-recursion", @@ -17069,7 +17084,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "axum 0.7.9", @@ -17092,7 +17107,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-trait", @@ -17125,7 +17140,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-trait", @@ -17145,7 +17160,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-trait", @@ -17179,7 +17194,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-trait", @@ -17214,7 +17229,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-trait", @@ -17237,7 +17252,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-trait", @@ -17261,7 +17276,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-nats", @@ -17285,7 +17300,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-trait", @@ -17320,7 +17335,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-trait", @@ -17348,7 +17363,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-trait", @@ -17371,7 +17386,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17389,7 +17404,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.648.0" +version = "1.651.1" dependencies = [ "anyhow", "async-once-cell", @@ -17489,9 +17504,28 @@ dependencies = [ "windmill-queue", "windmill-runtime-nativets", "windmill-types", + "windmill-worker-volumes", "yaml-rust", ] +[[package]] +name = "windmill-worker-volumes" +version = "1.651.1" +dependencies = [ + "bytes", + "futures", + "lazy_static", + "md-5 0.10.6", + "object_store", + "regex", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "windmill-common", +] + [[package]] name = "windows" version = "0.56.0" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 2d964c45b4..7ad583f31e 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.648.0" +version = "1.651.1" authors.workspace = true edition.workspace = true @@ -70,13 +70,14 @@ members = [ "./parsers/windmill-parser-py-imports", "./parsers/windmill-sql-datatype-parser-wasm", "./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu", + "./windmill-worker-volumes", "./windmill-test-utils", "./windmill-api-integration-tests", ] exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.648.0" +version = "1.651.1" authors = ["Ruben Fiszel "] edition = "2021" @@ -250,10 +251,13 @@ reqwest.workspace = true windmill-queue = { workspace = true, features = ["failpoints"] } windmill-dep-map.workspace = true windmill-test-utils.workspace = true +windmill-worker-volumes.workspace = true +windmill-types.workspace = true axum.workspace = true serde.workspace = true windmill-api-client.workspace = true tempfile.workspace = true +tar.workspace = true windmill-parser-ts.workspace = true rumqttc.workspace = true rdkafka.workspace = true @@ -267,6 +271,7 @@ aws-credential-types.workspace = true windmill-api = { path = "./windmill-api", default-features = false } windmill-queue = { path = "./windmill-queue" } windmill-worker = { path = "./windmill-worker" } +windmill-worker-volumes = { path = "./windmill-worker-volumes" } windmill-dep-map = { path = "./windmill-dep-map" } windmill-types = { path = "./windmill-types" } windmill-common = { path = "./windmill-common", default-features = false } @@ -439,6 +444,7 @@ base64 = "^0.22.1" base32 = "^0" hmac = "0.12.1" sha2 = "0.10.6" +md-5 = "0.10.6" sha1 = "0.10.6" sqlx = { version = "0.8.0", features = [ "macros", @@ -512,7 +518,7 @@ nu-parser = { version = "0.101.0", default-features = false } globset = "0.4.16" croner = "2.2.0" rmcp = { version = "=0.15.0", features = ["client", "transport-streamable-http-client", "transport-streamable-http-client-reqwest"] } -rquickjs = { version = "0.8", features = ["futures", "parallel", "macro"] } +rquickjs = { version = "0.11", features = ["futures", "parallel", "macro"] } process-wrap = { version = "8.2.1", features = ["tokio1"] } systemstat = "0.2.4" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 516de61c27..060c5666c7 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -8ffae1f43b31dc8136714fa612d22b6301773e27 +716b350bce1730b302c66ea69df618fa40f2f16b diff --git a/backend/migrations/20260226000000_add_volumes.down.sql b/backend/migrations/20260226000000_add_volumes.down.sql new file mode 100644 index 0000000000..33dc3804be --- /dev/null +++ b/backend/migrations/20260226000000_add_volumes.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS volume; diff --git a/backend/migrations/20260226000000_add_volumes.up.sql b/backend/migrations/20260226000000_add_volumes.up.sql new file mode 100644 index 0000000000..00f40c9768 --- /dev/null +++ b/backend/migrations/20260226000000_add_volumes.up.sql @@ -0,0 +1,22 @@ +-- Add 'volume' to the asset_kind enum +ALTER TYPE asset_kind ADD VALUE IF NOT EXISTS 'volume'; + +-- Volume metadata table +CREATE TABLE volume ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + size_bytes BIGINT NOT NULL DEFAULT 0, + file_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by VARCHAR(255) NOT NULL, + updated_at TIMESTAMPTZ, + updated_by VARCHAR(255), + description TEXT NOT NULL DEFAULT '', + lease_until TIMESTAMPTZ, + leased_by VARCHAR(255), + last_used_at TIMESTAMPTZ, + extra_perms JSONB NOT NULL DEFAULT '{}', + PRIMARY KEY (workspace_id, name) +); + +CREATE INDEX idx_volume_last_used ON volume(workspace_id, last_used_at); diff --git a/backend/migrations/20260302000000_add_token_expiry_notified.down.sql b/backend/migrations/20260302000000_add_token_expiry_notified.down.sql new file mode 100644 index 0000000000..ab827c5de5 --- /dev/null +++ b/backend/migrations/20260302000000_add_token_expiry_notified.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS token_expiry_notification; diff --git a/backend/migrations/20260302000000_add_token_expiry_notified.up.sql b/backend/migrations/20260302000000_add_token_expiry_notified.up.sql new file mode 100644 index 0000000000..61883f070d --- /dev/null +++ b/backend/migrations/20260302000000_add_token_expiry_notified.up.sql @@ -0,0 +1,8 @@ +-- Tracks pending expiry notifications: row exists = not yet notified. +-- Deleted once the notification is sent. Orphaned rows are harmless (filtered out by the join). +CREATE TABLE token_expiry_notification ( + token VARCHAR(255) PRIMARY KEY, + expiration TIMESTAMPTZ NOT NULL +); + +CREATE INDEX idx_token_expiry_notification_expiration ON token_expiry_notification (expiration); diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index cc86ea595c..56cccbabfe 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -238,7 +238,7 @@ lazy_static::lazy_static! { // used for `unsafe` sql interpolation // -- %%name%% (type) = default - static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%\s*([\s\w\/]+)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); + static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%[ \t]*([\w][\w \t\/]*)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); } fn parsed_default(parsed_typ: &Typ, default: String) -> Option { @@ -1547,4 +1547,36 @@ SELECT $1::integer; Ok(()) } + + #[test] + fn test_parse_pgsql_safe_interpolated_args() -> anyhow::Result<()> { + // There was a bug where enum would be "angrycreative"/"bishop"/"test SELECT x" + let code = r#" +-- %%table_name%% angrycreative/bishop/test +SELECT x +"#; + assert_eq!( + parse_pgsql_sig(code)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![Arg { + otyp: Some("__sanitized_enum__".to_string()), + name: "table_name".to_string(), + typ: Typ::Str(Some(vec![ + "angrycreative".to_string(), + "bishop".to_string(), + "test".to_string() + ])), + default: None, + has_default: false, + oidx: None, + },], + no_main_func: None, + has_preprocessor: None + } + ); + + Ok(()) + } } diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 2463995606..44388690e4 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -18,6 +18,7 @@ pub enum AssetKind { Resource, Ducklake, DataTable, + Volume, } #[derive(Serialize, Debug, PartialEq, Clone)] @@ -148,4 +149,5 @@ pub const ASSET_KINDS: &[(&str, AssetKind)] = &[ ("$res:", AssetKind::Resource), ("ducklake://", AssetKind::Ducklake), ("datatable://", AssetKind::DataTable), + ("volume://", AssetKind::Volume), ]; diff --git a/backend/src/main.rs b/backend/src/main.rs index c4440b7f34..c8a1021ced 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -38,11 +38,11 @@ use windmill_common::{ agent_workers::AgentConfig, global_settings::{ APP_WORKSPACED_ROUTE_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, - CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, - CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, + CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, + DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, + ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, @@ -62,7 +62,7 @@ use windmill_common::{ }, worker::{ is_native_mode_from_env, reload_custom_tags_setting, Connection, HUB_CACHE_DIR, - HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_DIR, TMP_LOGS_DIR, WORKER_GROUP, + HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, WINDMILL_DIR, WORKER_GROUP, }, KillpillSender, DEFAULT_HUB_BASE_URL, METRICS_ENABLED, }; @@ -99,10 +99,10 @@ use crate::monitor::{ load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db, reload_app_workspaced_route_setting, reload_base_url_setting, reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting, - reload_critical_error_channels_setting, reload_extra_pip_index_url_setting, - reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_job_default_timeout_setting, - reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, - reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting, + reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting, + reload_extra_pip_index_url_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting, + reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, + reload_license_key, reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config, reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration, }; @@ -238,8 +238,8 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { ) })?; - create_dir_all(HUB_CACHE_DIR)?; - create_dir_all(BUN_BUNDLE_CACHE_DIR)?; + create_dir_all(&*HUB_CACHE_DIR)?; + create_dir_all(&*BUN_BUNDLE_CACHE_DIR)?; for path in paths.values() { tracing::info!("Caching hub script at {path}"); @@ -249,7 +249,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { .as_ref() .is_some_and(|x| x == &ScriptLang::Deno) { - let job_dir = format!("{}/cache_init/{}", TMP_DIR, Uuid::new_v4()); + let job_dir = format!("{}/cache_init/{}", *WINDMILL_DIR, Uuid::new_v4()); create_dir_all(&job_dir)?; let _ = windmill_worker::generate_deno_lock( &Uuid::nil(), @@ -267,7 +267,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { tokio::fs::remove_dir_all(job_dir).await?; } else if res.language.as_ref().is_some_and(|x| x == &ScriptLang::Bun) { let job_id = Uuid::new_v4(); - let job_dir = format!("{}/cache_init/{}", TMP_DIR, job_id); + let job_dir = format!("{}/cache_init/{}", *WINDMILL_DIR, job_id); create_dir_all(&job_dir)?; if let Some(lock) = res.lockfile { let _ = windmill_worker::prepare_job_dir(&lock, &job_dir).await?; @@ -384,9 +384,9 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { println!("Fetched {} resource types from hub", resource_types.len()); - create_dir_all(HUB_RT_CACHE_DIR)?; + create_dir_all(&*HUB_RT_CACHE_DIR)?; - let cache_path = format!("{}/{}", HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); + let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); let content = serde_json::to_string_pretty(&resource_types) .with_context(|| "Failed to serialize resource types")?; @@ -398,7 +398,7 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { } pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyhow::Result<()> { - let cache_path = format!("{}/{}", HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); + let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); if tokio::fs::metadata(&cache_path).await.is_err() { tracing::info!( @@ -969,7 +969,7 @@ Windmill Community Edition {GIT_VERSION} DirBuilder::new() .recursive(true) - .create("/tmp/windmill") + .create(&*WINDMILL_DIR) .expect("could not create initial server dir"); #[cfg(feature = "tantivy")] @@ -1717,6 +1717,11 @@ async fn process_notify_event( tracing::error!(error = %e, "Could not reload critical alert UI setting"); } } + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING => { + if let Err(e) = reload_critical_alerts_on_token_expiry_setting(conn).await { + tracing::error!(error = %e, "Could not reload critical alerts on token expiry setting"); + } + } "workspace_telemetry_enabled" => { // Read the new value from the database and log it let enabled = sqlx::query_scalar!( @@ -1794,27 +1799,27 @@ pub async fn run_workers( let mut handles = Vec::with_capacity(num_workers as usize); for x in [ - TMP_LOGS_DIR, - UV_CACHE_DIR, - DENO_CACHE_DIR, - DENO_CACHE_DIR_DEPS, - DENO_CACHE_DIR_NPM, - BUN_CACHE_DIR, - PY310_CACHE_DIR, - PY311_CACHE_DIR, - PY312_CACHE_DIR, - PY313_CACHE_DIR, - BUN_BUNDLE_CACHE_DIR, - GO_CACHE_DIR, - GO_BIN_CACHE_DIR, - RUST_CACHE_DIR, - CSHARP_CACHE_DIR, - NU_CACHE_DIR, - HUB_CACHE_DIR, - POWERSHELL_CACHE_DIR, - JAVA_CACHE_DIR, - RUBY_CACHE_DIR, - TAR_JAVA_CACHE_DIR, // for related places search: ADD_NEW_LANG + &*TMP_LOGS_DIR, + &*UV_CACHE_DIR, + &*DENO_CACHE_DIR, + &*DENO_CACHE_DIR_DEPS, + &*DENO_CACHE_DIR_NPM, + &*BUN_CACHE_DIR, + &*PY310_CACHE_DIR, + &*PY311_CACHE_DIR, + &*PY312_CACHE_DIR, + &*PY313_CACHE_DIR, + &*BUN_BUNDLE_CACHE_DIR, + &*GO_CACHE_DIR, + &*GO_BIN_CACHE_DIR, + &*RUST_CACHE_DIR, + &*CSHARP_CACHE_DIR, + &*NU_CACHE_DIR, + &*HUB_CACHE_DIR, + &*POWERSHELL_CACHE_DIR, + &*JAVA_CACHE_DIR, + &*RUBY_CACHE_DIR, + &*TAR_JAVA_CACHE_DIR, // for related places search: ADD_NEW_LANG ] { DirBuilder::new() .recursive(true) diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 17e79c3393..37190a5f7a 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -44,19 +44,20 @@ use windmill_common::{ apps::APP_WORKSPACED_ROUTE, auth::create_token_for_owner, ee_oss::CriticalErrorChannel, + email_oss::send_email_if_possible, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, - CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, - DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, - JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, - KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, - NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING, - OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, - POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, + INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, + JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, + MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, + NUGET_CONFIG_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, + POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_INDEX_STRATEGY_SETTING, @@ -73,13 +74,14 @@ use windmill_common::{ load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env, load_worker_config, reload_custom_tags_setting, store_pull_query, store_suspended_pull_query, Connection, WorkerConfig, DEFAULT_TAGS_PER_WORKSPACE, - DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR, + DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, WORKER_GROUP, }, - KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERT_MUTE_UI_ENABLED, - CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, - METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, - OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, + KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, + CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, + HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, + MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, + SERVICE_LOG_RETENTION_SECS, }; use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING}; #[cfg(feature = "parquet")] @@ -207,6 +209,10 @@ pub async fn initial_load( tracing::error!("Error loading critical alert mute ui setting: {e:#}"); } + if let Err(e) = reload_critical_alerts_on_token_expiry_setting(conn).await { + tracing::error!("Error loading critical alerts on token expiry setting: {e:#}"); + } + if let Some(db) = conn.as_sql() { if let Err(e) = load_tag_per_workspace_enabled(db).await { tracing::error!("Error loading default tag per workpsace: {e:#}"); @@ -477,6 +483,21 @@ pub async fn reload_critical_alert_mute_ui_setting(conn: &Connection) -> error:: Ok(()) } +pub async fn reload_critical_alerts_on_token_expiry_setting( + conn: &Connection, +) -> error::Result<()> { + if let Ok(Some(serde_json::Value::Bool(t))) = load_value_from_global_settings_with_conn( + conn, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, + true, + ) + .await + { + CRITICAL_ALERTS_ON_TOKEN_EXPIRY.store(t, Ordering::Relaxed); + } + Ok(()) +} + pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> { let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await; @@ -595,7 +616,7 @@ async fn sleep_until_next_minute_start_plus_one_s() { use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE; async fn find_two_highest_files(hostname: &str) -> (Option, Option) { - let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname); + let log_dir = format!("{}/{}/", *TMP_WINDMILL_LOGS_SERVICE, hostname); let rd_dir = tokio::fs::read_dir(log_dir).await; if let Ok(mut log_files) = rd_dir { let mut highest_file: Option = None; @@ -614,7 +635,8 @@ async fn find_two_highest_files(hostname: &str) -> (Option, Option, + label: Option, + email: Option, + workspace_id: Option, +} + +fn is_user_token(label: Option<&str>) -> bool { + match label { + None => true, + Some(l) => l != "session" && !l.starts_with("ephemeral") && !l.starts_with("Ephemeral"), + } +} + +async fn report_token_expiration(db: &DB, token: &TokenRow, expired: bool) { + if !is_user_token(token.label.as_deref()) { + return; + } + let prefix = token.token_prefix.as_deref().unwrap_or("??????????"); + let email_addr = token.email.as_deref().unwrap_or("unknown"); + let token_desc = match token.label.as_deref() { + Some(l) if !l.is_empty() => format!("'{l}' ({prefix}****)"), + _ => format!("{prefix}****"), + }; + + let (alert_message, email_subject, email_body) = if expired { + ( + format!( + "API token {token_desc} of '{email_addr}' has expired and been deleted" + ), + "Windmill: Your API token has expired", + format!( + "Your API token {token_desc} has expired and been deleted.\n\nPlease create a new token if you still need API access." + ), + ) + } else { + ( + format!("API token {token_desc} of '{email_addr}' is expiring soon"), + "Windmill: Your API token is expiring soon", + format!( + "Your API token {token_desc} is expiring soon.\n\nPlease rotate or renew your token to avoid service disruption." + ), + ) + }; + + tracing::info!("{}", alert_message); + if CRITICAL_ALERTS_ON_TOKEN_EXPIRY.load(Ordering::Relaxed) { + report_critical_error( + alert_message, + db.clone(), + token.workspace_id.as_deref(), + None, + ) + .await; + } + if let Some(email) = &token.email { + send_email_if_possible(email_subject, &email_body, email); + } +} + pub async fn delete_expired_items(db: &DB) -> () { - let tokens_deleted_r: std::result::Result, _> = sqlx::query_scalar( + let expired_tokens_r = sqlx::query_as!( + TokenRow, "DELETE FROM token WHERE expiration <= now() - RETURNING concat(substring(token for 10), '*****')", + RETURNING substring(token for 10) as token_prefix, label, email, workspace_id", ) .fetch_all(db) .await; - match tokens_deleted_r { + match expired_tokens_r { Ok(tokens) => { - if tokens.len() > 0 { - tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens) + if !tokens.is_empty() { + tracing::info!("deleted {} expired tokens", tokens.len()); + for t in &tokens { + report_token_expiration(db, t, true).await; + } } } Err(e) => tracing::error!("Error deleting token: {}", e.to_string()), @@ -935,7 +1021,7 @@ pub async fn delete_expired_items(db: &DB) -> () { .iter() .map(|f| format!("{}/{}", f.hostname, f.file_path)) .collect(); - delete_log_files_from_disk_and_store(paths, TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await; + delete_log_files_from_disk_and_store(paths, &*TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await; } Err(e) => tracing::error!("Error deleting log file: {:?}", e), @@ -1064,6 +1150,41 @@ pub async fn delete_expired_items(db: &DB) -> () { } } +pub async fn check_expiring_tokens(db: &DB) { + // Find tokens expiring within 7 days that still have a pending notification row + let expiring_tokens_r = sqlx::query_as!( + TokenRow, + "DELETE FROM token_expiry_notification n + USING token t + WHERE n.token = t.token + AND n.expiration > now() + AND n.expiration <= now() + interval '7 days' + RETURNING substring(t.token for 10) as token_prefix, t.label, t.email, t.workspace_id", + ) + .fetch_all(db) + .await; + + match expiring_tokens_r { + Ok(tokens) => { + for t in &tokens { + report_token_expiration(db, t, false).await; + } + if !tokens.is_empty() { + tracing::info!("Sent expiration warnings for {} token(s)", tokens.len()); + } + } + Err(e) => tracing::error!("Error checking expiring tokens: {}", e), + } + + // Clean up notification rows whose expiration has passed + if let Err(e) = sqlx::query!("DELETE FROM token_expiry_notification WHERE expiration <= now()") + .execute(db) + .await + { + tracing::error!("Error cleaning up expired token notifications: {}", e); + } +} + /// Delete a batch of expired jobs with LIMIT and SKIP LOCKED for high-scale environments. /// Uses a single transaction per batch to minimize lock duration. /// Returns the number of jobs deleted in this batch. @@ -1140,7 +1261,7 @@ async fn delete_expired_jobs_batch( .filter_map(|opt| opt) .flat_map(|inner_vec| inner_vec.into_iter()) .collect(); - delete_log_files_from_disk_and_store(paths, TMP_DIR, "").await; + delete_log_files_from_disk_and_store(paths, &*WINDMILL_DIR, "").await; } Err(e) => tracing::error!("Error deleting job logs: {:?}", e), } @@ -1367,7 +1488,7 @@ pub async fn reload_maven_settings_xml_setting(conn: &Connection) { let settings_xml = MAVEN_SETTINGS_XML.read().await.clone(); match settings_xml { Some(ref content) if !content.trim().is_empty() => { - let m2_dir = format!("{JAVA_HOME_DIR}/.m2"); + let m2_dir = format!("{}/.m2", *JAVA_HOME_DIR); if let Err(e) = tokio::fs::create_dir_all(&m2_dir).await { tracing::error!("Failed to create .m2 directory: {e:#}"); return; @@ -1378,7 +1499,7 @@ pub async fn reload_maven_settings_xml_setting(conn: &Connection) { } } _ => { - let settings_path = format!("{JAVA_HOME_DIR}/.m2/settings.xml"); + let settings_path = format!("{}/.m2/settings.xml", *JAVA_HOME_DIR); let _ = tokio::fs::remove_file(&settings_path).await; } } @@ -2051,6 +2172,16 @@ pub async fn monitor_db( } }; + // Run every hour (10 iterations * 30s = 5 minutes) + // Check for tokens expiring within 7 days and send alerts + let check_expiring_tokens_f = async { + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(10) { + if let Some(db) = conn.as_sql() { + check_expiring_tokens(&db).await; + } + } + }; + join!( expired_items_f, zombie_jobs_f, @@ -2072,6 +2203,7 @@ pub async fn monitor_db( cleanup_worker_group_stats_f, native_triggers_sync_f, cleanup_notify_events_f, + check_expiring_tokens_f, ); } diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 01db7590f5..3ece1ce7ef 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -151,6 +151,8 @@ sqs_trigger: path(char), queue_url(char), aws_resource_path(char), message_attri FK: (workspace_id) -> workspace(id) token: token(char), label(char), expiration(ts), workspace_id(char), owner(char), email(char), super_admin(bool), created_at(ts), last_used_at(ts), scopes(text[]), job(uuid) FK: (workspace_id) -> workspace(id) +token_expiry_notification: token(char), expiration(ts) + INDEX: idx_token_expiry_notification_expiration (expiration) tutorial_progress: email(char), progress(bit64), skipped_all(bool) unique_ext_jwt_token: jwt_hash(bigint), last_used_at(ts) usage: id(char), is_workspace(bool), month_(int), usage(int) diff --git a/backend/tests/agent_workers.rs b/backend/tests/agent_workers.rs index b6414dc24e..4f12d5f42c 100644 --- a/backend/tests/agent_workers.rs +++ b/backend/tests/agent_workers.rs @@ -1,12 +1,12 @@ #![cfg(all(feature = "private", feature = "agent_worker_server"))] -use windmill_test_utils::*; use serde_json::json; use sqlx::{Pool, Postgres}; use windmill_common::{ jobs::{JobPayload, RawCode}, scripts::ScriptLang, }; +use windmill_test_utils::*; fn bun_code(code: &str) -> RawCode { RawCode { @@ -18,8 +18,8 @@ fn bun_code(code: &str) -> RawCode { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - concurrency_settings: - windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), } } @@ -223,7 +223,10 @@ async fn test_agent_worker_token_and_ping(db: Pool) -> anyhow::Result< .fetch_one(&db) .await?; - assert!(worker_count > 0, "worker ping should be recorded in database"); + assert!( + worker_count > 0, + "worker ping should be recorded in database" + ); // MainLoop ping updates the existing record let resp = http_client @@ -265,3 +268,319 @@ async fn test_agent_worker_multiple_jobs_sequential(db: Pool) -> anyho Ok(()) } + +/// Test the volume HTTP proxy endpoints that agent workers use. +/// +/// Exercises the full volume lifecycle via HTTP: +/// 1. Configure workspace S3 storage (FilesystemStorage) +/// 2. Pre-populate a volume with a file +/// 3. POST /begin — acquire lease, get manifest +/// 4. GET /file/* — download existing file +/// 5. PUT /file/* — upload a new file +/// 6. POST /commit — finalize with stats, release lease +/// 7. Verify DB state and storage +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> { + let (client, _port, _server) = init_client_agent_mode(db.clone()).await; + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file + let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + let base = client.baseurl(); + let http = client.client(); + let vol_base = format!("{base}/w/test-workspace/volumes/test-vol"); + + // 3. POST /begin — acquire lease, get manifest + permissions + let resp = http + .post(format!("{vol_base}/begin")) + .json(&json!({ + "worker_name": "test-worker-1", + "permissioned_as": "u/test-user" + })) + .send() + .await?; + assert!( + resp.status().is_success(), + "begin should succeed, got: {}", + resp.status() + ); + + let begin_body: serde_json::Value = resp.json().await?; + assert!( + begin_body["writable"].as_bool().unwrap(), + "should be writable" + ); + let manifest = begin_body["manifest"].as_object().unwrap(); + assert!( + manifest.contains_key("hello.txt"), + "manifest should contain hello.txt, got: {manifest:?}" + ); + + // 4. GET /file/* — download the existing file + let resp = http + .get(format!("{vol_base}/file/hello.txt")) + .send() + .await?; + assert!( + resp.status().is_success(), + "file download should succeed, got: {}", + resp.status() + ); + let file_bytes = resp.bytes().await?; + assert_eq!( + file_bytes.as_ref(), + b"hello from volume", + "downloaded file content should match" + ); + + // 5. PUT /file/* — upload a new file + let resp = http + .put(format!("{vol_base}/file/output.txt")) + .body(b"written by agent worker".to_vec()) + .send() + .await?; + assert!( + resp.status().is_success(), + "file upload should succeed, got: {}", + resp.status() + ); + + // 6. POST /commit — finalize: report stats, release lease + let resp = http + .post(format!("{vol_base}/commit")) + .json(&json!({ + "worker_name": "test-worker-1", + "deleted_keys": [], + "symlinks": {}, + "file_count": 2, + "size_bytes": 39 + })) + .send() + .await?; + assert!( + resp.status().is_success(), + "commit should succeed, got: {}", + resp.status() + ); + + // 7. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert_eq!(vol_row.file_count, 2, "file_count should be 2"); + assert_eq!(vol_row.size_bytes, 39, "size_bytes should match"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + assert!( + vol_row.lease_until.is_none() || vol_row.lease_until.unwrap() < chrono::Utc::now(), + "lease_until should be cleared or in the past" + ); + + // 8. Verify the uploaded file was persisted in storage + let output_path = vol_dir.join("output.txt"); + assert!(output_path.exists(), "output.txt should be in storage"); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by agent worker"); + + Ok(()) +} + +/// Full E2E test: agent worker in HTTP mode runs a Bun script with a volume mount. +/// +/// The worker pulls the job via HTTP, downloads volume files via the server-side +/// volume proxy endpoints, executes the script, and syncs changes back. +#[cfg(all(feature = "parquet", feature = "enterprise"))] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_http_worker_e2e(db: Pool) -> anyhow::Result<()> { + let (_client, port, _server) = init_client_agent_mode(db.clone()).await; + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file + let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + // 3. Push the job, then run worker with HTTP connection (bun tag) + let code = r#"// volume: test-vol /tmp/data +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("/tmp/data/hello.txt", "utf-8"); + writeFileSync("/tmp/data/output.txt", "written by agent worker"); + return { + read_content: content, + output_exists: existsSync("/tmp/data/output.txt"), + }; +}"#; + + let uuid = RunJob::from(JobPayload::Code(bun_code(code))) + .push(&db) + .await; + let listener = listen_for_completed_jobs(&db).await; + + let conn = testing_http_connection_with_tags( + port, + vec!["bun".into(), "flow".into(), "dependency".into()], + ) + .await; + + in_test_worker(conn, listener.find(&uuid), port).await; + + let result = completed_job(uuid, &db).await; + + assert!(result.success, "job should succeed: {:?}", result.result); + let json = result.json_result().expect("should have JSON result"); + assert_eq!(json["read_content"], json!("hello from volume")); + assert_eq!(json["output_exists"], json!(true)); + + // 4. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert!( + vol_row.file_count >= 2, + "should have at least 2 files (hello.txt + output.txt), got: {}", + vol_row.file_count + ); + assert!(vol_row.size_bytes > 0, "size_bytes should be > 0"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + assert!( + vol_row.lease_until.is_none() || vol_row.lease_until.unwrap() < chrono::Utc::now(), + "lease_until should be cleared or in the past" + ); + + // 5. Verify the new file was written back to the storage + let output_path = vol_dir.join("output.txt"); + assert!( + output_path.exists(), + "output.txt should be synced back to storage" + ); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by agent worker"); + + Ok(()) +} + +/// Test the volume release endpoint (error/cancel path). +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_release(db: Pool) -> anyhow::Result<()> { + let (client, _port, _server) = init_client_agent_mode(db.clone()).await; + + // Set up filesystem storage + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + let base = client.baseurl(); + let http = client.client(); + let vol_base = format!("{base}/w/test-workspace/volumes/test-vol"); + + // Begin (acquire lease) + let resp = http + .post(format!("{vol_base}/begin")) + .json(&json!({ + "worker_name": "test-worker-2", + "permissioned_as": "u/test-user" + })) + .send() + .await?; + assert!(resp.status().is_success(), "begin should succeed"); + + // Verify lease is held + let leased = sqlx::query_scalar!( + "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await? + .flatten(); + assert_eq!(leased.as_deref(), Some("test-worker-2")); + + // Release without commit (simulating error path) + let resp = http + .post(format!("{vol_base}/release")) + .json(&json!({ "worker_name": "test-worker-2" })) + .send() + .await?; + assert!(resp.status().is_success(), "release should succeed"); + + // Verify lease is cleared + let leased = sqlx::query_scalar!( + "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await? + .flatten(); + assert!(leased.is_none(), "lease should be released"); + + Ok(()) +} diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index 15a2b3c27c..1617babb81 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -1,5 +1,6 @@ use sqlx::postgres::Postgres; use sqlx::Pool; +use uuid::Uuid; use windmill_common::jobs::{JobPayload, RawCode}; use windmill_common::scripts::ScriptLang; use windmill_test_utils::*; @@ -1448,3 +1449,240 @@ export function main() { return { a, b }; } ); } } + +// ============================================================================ +// Codebase Mode Tests +// ============================================================================ + +/// Create a TAR archive in memory containing a single `main.js` file. +fn create_codebase_tar(main_js_content: &str) -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + let content = main_js_content.as_bytes(); + let mut header = tar::Header::new_gnu(); + header.set_path("main.js").unwrap(); + header.set_size(content.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append(&header, content).unwrap(); + builder.into_inner().unwrap() +} + +/// Place a TAR codebase at the expected cache path for the given job ID and hash. +fn place_codebase_in_cache(job_id: &Uuid, tar_bytes: &[u8], is_esm: bool) { + let codebase_id = if is_esm { + format!("{}.esm.tar", job_id) + } else { + format!("{}.tar", job_id) + }; + let bundle_path = format!("script_bundle/test-workspace/{}", codebase_id); + let cache_path = format!( + "{}/{}.tar", + *windmill_common::worker::ROOT_CACHE_NOMOUNT_DIR, + bundle_path, + ); + let parent = std::path::Path::new(&cache_path).parent().unwrap(); + std::fs::create_dir_all(parent).unwrap(); + std::fs::write(&cache_path, tar_bytes).unwrap(); +} + +#[sqlx::test(fixtures("base"))] +async fn test_cjs_codebase_tar(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let main_js = r#" +module.exports.main = function() { + return "cjs codebase ok"; +}; +"#; + let inner_content = r#"export function main() { return "cjs codebase ok"; }"#; + + let job_id = Uuid::new_v4(); + let tar_bytes = create_codebase_tar(main_js); + place_codebase_in_cache(&job_id, &tar_bytes, false); + + let job = JobPayload::Code(RawCode { + hash: Some(-43), // PREVIEW_IS_TAR_CODEBASE_HASH + content: inner_content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: Default::default(), + debouncing_settings: Default::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = RunJob::from(job) + .job_id(job_id) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("cjs codebase ok")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_esm_codebase_tar(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let main_js = r#" +export function main() { + return "esm codebase ok"; +} +"#; + let inner_content = r#"export function main() { return "esm codebase ok"; }"#; + + let job_id = Uuid::new_v4(); + let tar_bytes = create_codebase_tar(main_js); + place_codebase_in_cache(&job_id, &tar_bytes, true); + + let job = JobPayload::Code(RawCode { + hash: Some(-45), // PREVIEW_IS_TAR_ESM_CODEBASE_HASH + content: inner_content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: Default::default(), + debouncing_settings: Default::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = RunJob::from(job) + .job_id(job_id) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("esm codebase ok")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_cjs_codebase_tar_nsjail(db: Pool) -> anyhow::Result<()> { + if std::process::Command::new("nsjail") + .arg("--help") + .output() + .is_err() + { + eprintln!("nsjail not found, skipping test"); + return Ok(()); + } + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let main_js = r#" +module.exports.main = function() { + return "cjs nsjail ok"; +}; +"#; + let inner_content = r#"export function main() { return "cjs nsjail ok"; }"#; + + let job_id = Uuid::new_v4(); + let tar_bytes = create_codebase_tar(main_js); + place_codebase_in_cache(&job_id, &tar_bytes, false); + + let job = JobPayload::Code(RawCode { + hash: Some(-43), + content: inner_content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: Default::default(), + debouncing_settings: Default::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + use std::sync::atomic::Ordering; + windmill_worker::JOB_ISOLATION.store( + windmill_worker::JobIsolationLevel::NsjailSandboxing as u8, + Ordering::Relaxed, + ); + + let result = RunJob::from(job) + .job_id(job_id) + .run_until_complete(&db, false, port) + .await; + + windmill_worker::JOB_ISOLATION.store( + windmill_worker::JobIsolationLevel::Undefined as u8, + Ordering::Relaxed, + ); + + let json = result.json_result().unwrap(); + assert_eq!(json, serde_json::json!("cjs nsjail ok")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_esm_codebase_tar_nsjail(db: Pool) -> anyhow::Result<()> { + if std::process::Command::new("nsjail") + .arg("--help") + .output() + .is_err() + { + eprintln!("nsjail not found, skipping test"); + return Ok(()); + } + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let main_js = r#" +export function main() { + return "esm nsjail ok"; +} +"#; + let inner_content = r#"export function main() { return "esm nsjail ok"; }"#; + + let job_id = Uuid::new_v4(); + let tar_bytes = create_codebase_tar(main_js); + place_codebase_in_cache(&job_id, &tar_bytes, true); + + let job = JobPayload::Code(RawCode { + hash: Some(-45), + content: inner_content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: Default::default(), + debouncing_settings: Default::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + use std::sync::atomic::Ordering; + windmill_worker::JOB_ISOLATION.store( + windmill_worker::JobIsolationLevel::NsjailSandboxing as u8, + Ordering::Relaxed, + ); + + let result = RunJob::from(job) + .job_id(job_id) + .run_until_complete(&db, false, port) + .await; + + windmill_worker::JOB_ISOLATION.store( + windmill_worker::JobIsolationLevel::Undefined as u8, + Ordering::Relaxed, + ); + + let json = result.json_result().unwrap(); + assert_eq!(json, serde_json::json!("esm nsjail ok")); + Ok(()) +} diff --git a/backend/tests/end_user_email.rs b/backend/tests/end_user_email.rs new file mode 100644 index 0000000000..1a60f8672c --- /dev/null +++ b/backend/tests/end_user_email.rs @@ -0,0 +1,323 @@ +//! Tests for WM_END_USER_EMAIL environment variable. +//! +//! These tests verify that WM_END_USER_EMAIL is populated with the authenticated +//! user's email when executing app components. +//! +//! TODO: Add tests for scripts and flows once public execution endpoints are identified. +//! Currently only apps support non-workspace-member execution via OptAuthed + token lookup. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::worker::Connection; +use windmill_test_utils::*; + +const SAME_WS_TOKEN: &str = "SECRET_TOKEN"; +const OTHER_WS_TOKEN: &str = "OTHER_WS_TOKEN"; +const NO_WS_TOKEN: &str = "NO_WS_TOKEN"; + +const SAME_WS_EMAIL: &str = "test@windmill.dev"; +const OTHER_WS_EMAIL: &str = "other-ws@windmill.dev"; +const NO_WS_EMAIL: &str = "no-ws@windmill.dev"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +// TODO: Script tests - need to identify public execution endpoints for non-workspace-members +// async fn run_script(port: u16, token: &str) -> anyhow::Result { +// let url = format!( +// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/p/f/test/get_end_user_email", +// port +// ); +// let resp = authed(client().post(&url), token) +// .json(&json!({})) +// .send() +// .await?; +// if !resp.status().is_success() { +// anyhow::bail!("script run failed: {} - {}", resp.status(), resp.text().await?); +// } +// Ok(resp.json::().await? +// .as_str().unwrap_or("").to_string()) +// } + +// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members +// async fn run_flow(port: u16, token: &str) -> anyhow::Result { +// let url = format!( +// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/f/f/test/get_end_user_email_flow", +// port +// ); +// let resp = authed(client().post(&url), token) +// .json(&json!({})) +// .send() +// .await?; +// if !resp.status().is_success() { +// anyhow::bail!("flow run failed: {} - {}", resp.status(), resp.text().await?); +// } +// Ok(resp.json::().await? +// .as_str().unwrap_or("").to_string()) +// } + +/// Create an app with inline script via API +async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<()> { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps/create", + port + ); + let resp = authed(client().post(&url), SAME_WS_TOKEN) + .json(&json!({ + "path": path, + "summary": "Test app for WM_END_USER_EMAIL", + "value": { + "type": "app", + "grid": [], + "subgrids": {}, + "hiddenInlineScripts": [{ + "name": "get_email", + "language": "deno", + "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", + "path": "f/test/email_app/get_email" + }] + }, + "policy": { + "execution_mode": "anonymous", + "on_behalf_of": null, + "on_behalf_of_email": null, + "triggerables_v2": { + "get_email": { + "static_inputs": {}, + "one_of_inputs": {} + }, + // SHA256 hash of raw_code content for anonymous execution + "rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": { + "static_inputs": {}, + "one_of_inputs": {} + } + } + } + })) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!("create app failed: {} - {}", resp.status(), resp.text().await?); + } + Ok(()) +} + +/// Create a raw app with inline script via API (uses regular app endpoint with rawapp type) +async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<()> { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps/create", + port + ); + let resp = authed(client().post(&url), SAME_WS_TOKEN) + .json(&json!({ + "path": path, + "summary": "Test raw app for WM_END_USER_EMAIL", + "value": { + "type": "rawapp", + "css": "", + "inlineScripts": [{ + "name": "get_email", + "language": "deno", + "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }" + }] + }, + "policy": { + "execution_mode": "anonymous", + "on_behalf_of": null, + "on_behalf_of_email": null, + "triggerables_v2": { + "get_email": { + "static_inputs": {}, + "one_of_inputs": {} + }, + // SHA256 hash of raw_code content for anonymous execution + "rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": { + "static_inputs": {}, + "one_of_inputs": {} + } + } + } + })) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!("create raw app failed: {} - {}", resp.status(), resp.text().await?); + } + Ok(()) +} + +async fn run_app_inline_script(port: u16, token: &str, app_path: &str, force_viewer: bool) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps_u/execute_component/{}", + port, app_path + ); + let mut payload = json!({ + "args": {}, + "component": "get_email", + "raw_code": { + "language": "deno", + "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", + "path": format!("{}/get_email", app_path) + } + }); + if force_viewer { + payload["force_viewer_static_fields"] = json!({}); + } + let resp = authed(client().post(&url), token) + .json(&payload) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!("app inline script run failed: {} - {}", resp.status(), resp.text().await?); + } + let job_id = resp.text().await?; + wait_for_job_result(port, token, &job_id).await +} + +async fn run_raw_app_inline_script(port: u16, token: &str, app_path: &str, force_viewer: bool) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps_u/execute_component/{}", + port, app_path + ); + let mut payload = json!({ + "args": {}, + "component": "get_email", + "raw_code": { + "language": "deno", + "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }" + } + }); + if force_viewer { + payload["force_viewer_static_fields"] = json!({}); + } + let resp = authed(client().post(&url), token) + .json(&payload) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!("raw app inline script run failed: {} - {}", resp.status(), resp.text().await?); + } + let job_id = resp.text().await?; + wait_for_job_result(port, token, &job_id).await +} + +async fn wait_for_job_result(port: u16, token: &str, job_id: &str) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/jobs_u/completed/get_result/{}", + port, job_id + ); + for _ in 0..100 { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + let resp = authed(client().get(&url), token).send().await?; + if resp.status().is_success() { + return Ok(resp.json::().await? + .as_str().unwrap_or("").to_string()); + } + } + anyhow::bail!("timeout waiting for job result") +} + +// TODO: Script tests - need to identify public execution endpoints for non-workspace-members +// #[cfg(feature = "deno_core")] +// #[sqlx::test(fixtures("base", "end_user_email"))] +// async fn test_script_wm_end_user_email(db: Pool) -> anyhow::Result<()> { +// initialize_tracing().await; +// set_jwt_secret().await; +// let server = ApiServer::start(db.clone()).await?; +// let port = server.addr.port(); +// +// in_test_worker(Connection::Sql(db.clone()), async move { +// let result = run_script(port, SAME_WS_TOKEN).await?; +// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); +// Ok::<(), anyhow::Error>(()) +// }, port).await?; +// +// Ok(()) +// } + +// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members +// #[cfg(feature = "deno_core")] +// #[sqlx::test(fixtures("base", "end_user_email"))] +// async fn test_flow_wm_end_user_email(db: Pool) -> anyhow::Result<()> { +// initialize_tracing().await; +// set_jwt_secret().await; +// let server = ApiServer::start(db.clone()).await?; +// let port = server.addr.port(); +// +// in_test_worker(Connection::Sql(db.clone()), async move { +// let result = run_flow(port, SAME_WS_TOKEN).await?; +// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); +// Ok::<(), anyhow::Error>(()) +// }, port).await?; +// +// Ok(()) +// } + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_app_wm_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let app_path = "f/test/email_app"; + + in_test_worker(Connection::Sql(db.clone()), async move { + // Create the app with inline script first + create_app_with_inline_script(port, app_path).await?; + + // Same workspace user (force_viewer mode works for workspace members) + let result = run_app_inline_script(port, SAME_WS_TOKEN, app_path, true).await?; + assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); + + // Other workspace user (uses app's anonymous policy + token lookup) + let result = run_app_inline_script(port, OTHER_WS_TOKEN, app_path, false).await?; + assert_eq!(result, OTHER_WS_EMAIL, "other workspace user should get their email"); + + // No workspace user (uses app's anonymous policy + token lookup) + let result = run_app_inline_script(port, NO_WS_TOKEN, app_path, false).await?; + assert_eq!(result, NO_WS_EMAIL, "no workspace user should get their email"); + + Ok::<(), anyhow::Error>(()) + }, port).await?; + + Ok(()) +} + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_raw_app_wm_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let app_path = "f/test/email_raw_app"; + + in_test_worker(Connection::Sql(db.clone()), async move { + // Create the raw app with inline script first + create_raw_app_with_inline_script(port, app_path).await?; + + // Same workspace user (force_viewer mode works for workspace members) + let result = run_raw_app_inline_script(port, SAME_WS_TOKEN, app_path, true).await?; + assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); + + // Other workspace user (uses app's anonymous policy + token lookup) + let result = run_raw_app_inline_script(port, OTHER_WS_TOKEN, app_path, false).await?; + assert_eq!(result, OTHER_WS_EMAIL, "other workspace user should get their email"); + + // No workspace user (uses app's anonymous policy + token lookup) + let result = run_raw_app_inline_script(port, NO_WS_TOKEN, app_path, false).await?; + assert_eq!(result, NO_WS_EMAIL, "no workspace user should get their email"); + + Ok::<(), anyhow::Error>(()) + }, port).await?; + + Ok(()) +} diff --git a/backend/tests/fixtures/end_user_email.sql b/backend/tests/fixtures/end_user_email.sql new file mode 100644 index 0000000000..654ad93680 --- /dev/null +++ b/backend/tests/fixtures/end_user_email.sql @@ -0,0 +1,63 @@ +-- Fixture for WM_END_USER_EMAIL tests +-- Sets up 3 users with different workspace memberships: +-- 1. test@windmill.dev - in test-workspace (from base.sql) +-- 2. other-ws@windmill.dev - in other-workspace only +-- 3. no-ws@windmill.dev - not in any workspace + +-- Second workspace for cross-workspace user +INSERT INTO workspace (id, name, owner) +VALUES ('other-workspace', 'other-workspace', 'other-ws-user'); + +INSERT INTO workspace_key(workspace_id, kind, key) +VALUES ('other-workspace', 'cloud', 'other-key'); + +INSERT INTO workspace_settings (workspace_id) +VALUES ('other-workspace'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) +VALUES ('other-workspace', 'all', 'All users', '{}'); + +-- User in other-workspace only (not in test-workspace) +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) +VALUES ('other-ws@windmill.dev', 'hash', 'password', false, true, 'Other WS User'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) +VALUES ('other-workspace', 'other-ws@windmill.dev', 'other-ws-user', true, 'Admin'); + +INSERT INTO token(token, email, label, super_admin) +VALUES ('OTHER_WS_TOKEN', 'other-ws@windmill.dev', 'other ws token', false); + +-- User not in any workspace +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) +VALUES ('no-ws@windmill.dev', 'hash', 'password', false, true, 'No WS User'); + +INSERT INTO token(token, email, label, super_admin) +VALUES ('NO_WS_TOKEN', 'no-ws@windmill.dev', 'no ws token', false); + +-- Script that returns WM_END_USER_EMAIL (public via extra_perms) +INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, extra_perms) +VALUES ( + 'test-workspace', 'test-user', + 'export function main() { return Deno.env.get("WM_END_USER_EMAIL") || ""; }', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'deno', '', 'script', + '{"g/all": true}' +); + +-- Flow that returns WM_END_USER_EMAIL (public via extra_perms) +INSERT INTO flow (workspace_id, summary, description, path, versions, schema, value, edited_by, extra_perms) +VALUES ( + 'test-workspace', 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email_flow', '{900002}', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + 'test-user', + '{"g/all": true}' +); + +INSERT INTO flow_version (id, workspace_id, path, schema, value, created_by) +VALUES ( + 900002, 'test-workspace', 'f/test/get_end_user_email_flow', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + 'test-user' +); diff --git a/backend/tests/nativets_stress.rs b/backend/tests/nativets_stress.rs index 9717403830..082a43be2c 100644 --- a/backend/tests/nativets_stress.rs +++ b/backend/tests/nativets_stress.rs @@ -206,7 +206,7 @@ fn spawn_workers( std::fs::DirBuilder::new() .recursive(true) - .create(windmill_worker::GO_BIN_CACHE_DIR) + .create(&*windmill_worker::GO_BIN_CACHE_DIR) .expect("could not create initial worker dir"); let (tx, _) = KillpillSender::new(n + 1); diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index dd494de007..22c2313b91 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -1,7 +1,7 @@ -use windmill_test_utils::*; use sqlx::postgres::Postgres; use sqlx::Pool; use windmill_common::scripts::ScriptLang; +use windmill_test_utils::*; #[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] @@ -188,7 +188,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, @@ -207,14 +208,14 @@ def main(): #[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_python_global_site_packages(db: Pool) -> anyhow::Result<()> { - use windmill_common::{cache::concatcp, worker::ROOT_CACHE_DIR}; + use windmill_common::worker::ROOT_CACHE_DIR; initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); // Shared for all 3.12.* - let path = concatcp!(ROOT_CACHE_DIR, "python_3_12/global-site-packages").to_owned(); + let path = format!("{}python_3_12/global-site-packages", *ROOT_CACHE_DIR); std::fs::create_dir_all(&path).unwrap(); std::fs::write(path + "/my_global_site_package_3_12_any.py", "").unwrap(); @@ -237,7 +238,9 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( + ) + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, @@ -271,7 +274,9 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( + ) + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, @@ -310,7 +315,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, @@ -347,7 +353,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, diff --git a/backend/tests/scripts/test_volume_with_claude.ts b/backend/tests/scripts/test_volume_with_claude.ts new file mode 100644 index 0000000000..b6230a19ee --- /dev/null +++ b/backend/tests/scripts/test_volume_with_claude.ts @@ -0,0 +1,102 @@ +// volume: agent-memory .claude +// sandbox + +import Anthropic from "@anthropic-ai/sdk"; +import * as fs from "fs"; +import * as path from "path"; + +type Anthropic = { + api_key: string; + model?: string; +}; + +export async function main(anthropic_resource: Anthropic) { + const claudeDir = ".claude"; + const results: Record = {}; + + // --- Step 1: Verify volume is mounted at the relative path --- + results["volume_exists"] = fs.existsSync(claudeDir); + if (!results["volume_exists"]) { + fs.mkdirSync(claudeDir, { recursive: true }); + } + + const testFile = path.join(claudeDir, "mount-check.txt"); + fs.writeFileSync(testFile, "volume mount verified"); + results["volume_writable"] = fs.readFileSync(testFile, "utf-8") === "volume mount verified"; + + // --- Step 2: Create memory directory structure --- + const memoryDir = path.join(claudeDir, "memory"); + fs.mkdirSync(memoryDir, { recursive: true }); + + const memoryFile = path.join(memoryDir, "MEMORY.md"); + fs.writeFileSync(memoryFile, "# Agent Memory\n\nThis file persists across runs.\n"); + results["memory_file_created"] = fs.existsSync(memoryFile); + + // --- Step 3: Call Claude to generate structured content --- + const client = new Anthropic({ apiKey: anthropic_resource.api_key }); + const model = anthropic_resource.model ?? "claude-sonnet-4-20250514"; + + const response = await client.messages.create({ + model, + max_tokens: 256, + messages: [ + { + role: "user", + content: + 'Return a JSON object with exactly these keys: "greeting" (a short hello), "timestamp" (current ISO date you estimate), "items" (array of 3 random fruit names). Only return the JSON, no markdown.', + }, + ], + }); + + const assistantText = + response.content[0].type === "text" ? response.content[0].text : ""; + results["claude_responded"] = assistantText.length > 0; + results["claude_model"] = response.model; + results["claude_stop_reason"] = response.stop_reason; + + let parsed: Record = {}; + try { + parsed = JSON.parse(assistantText); + results["claude_valid_json"] = true; + results["claude_has_greeting"] = "greeting" in parsed; + results["claude_has_items"] = + Array.isArray(parsed.items) && parsed.items.length === 3; + } catch { + results["claude_valid_json"] = false; + } + + // --- Step 4: Write Claude's response to volume --- + const responsePath = path.join(claudeDir, "claude-response.json"); + fs.writeFileSync(responsePath, JSON.stringify(parsed, null, 2)); + results["response_written"] = fs.existsSync(responsePath); + + // --- Step 5: Read back and verify --- + const readBack = fs.readFileSync(responsePath, "utf-8"); + const readParsed = JSON.parse(readBack); + results["readback_matches"] = + JSON.stringify(readParsed) === JSON.stringify(parsed); + + // --- Step 6: List all volume contents --- + const volumeContents = fs.readdirSync(claudeDir); + results["volume_files"] = volumeContents; + results["volume_file_count"] = volumeContents.length; + + // --- Step 7: Verify memory file persists --- + const memoryContent = fs.readFileSync(memoryFile, "utf-8"); + results["memory_persisted"] = memoryContent.includes("Agent Memory"); + + // --- Summary --- + const allChecks = [ + results["volume_exists"] || true, + results["volume_writable"], + results["claude_responded"], + results["claude_valid_json"], + results["response_written"], + results["readback_matches"], + results["memory_file_created"], + results["memory_persisted"], + ]; + results["all_passed"] = allChecks.every(Boolean); + + return results; +} diff --git a/backend/tests/volume_tests.rs b/backend/tests/volume_tests.rs new file mode 100644 index 0000000000..9df30c5615 --- /dev/null +++ b/backend/tests/volume_tests.rs @@ -0,0 +1,637 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::jobs::{JobPayload, RawCode}; +use windmill_common::scripts::ScriptLang; +use windmill_test_utils::*; + +#[sqlx::test(fixtures("base"))] +async fn test_volume_insert(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "test-volume", + 1024_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT workspace_id, name, size_bytes, created_by, last_used_at + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-volume" + ) + .fetch_one(&db) + .await?; + + assert_eq!(row.workspace_id, "test-workspace"); + assert_eq!(row.name, "test-volume"); + assert_eq!(row.size_bytes, 1024); + assert_eq!(row.created_by, "test-user"); + assert!(row.last_used_at.is_none()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_upsert_size(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET size_bytes = $3, last_used_at = now()", + "test-workspace", + "upsert-vol", + 500_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT size_bytes FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "upsert-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.size_bytes, 500); + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET size_bytes = $3, last_used_at = now()", + "test-workspace", + "upsert-vol", + 2048_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT size_bytes, last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "upsert-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.size_bytes, 2048); + assert!(row.last_used_at.is_some()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_update_last_used(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "used-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .fetch_one(&db) + .await?; + assert!(row.last_used_at.is_none()); + + sqlx::query!( + "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .fetch_one(&db) + .await?; + assert!(row.last_used_at.is_some()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_update_nonexistent_noop(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let result = sqlx::query!( + "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "nonexistent-vol" + ) + .execute(&db) + .await?; + + assert_eq!(result.rows_affected(), 0); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_list_multiple(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + for i in 0..5 { + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + format!("vol-{}", i), + (i * 100) as i64, + "test-user" + ) + .execute(&db) + .await?; + } + + let rows = sqlx::query!( + "SELECT name, size_bytes FROM volume WHERE workspace_id = $1 ORDER BY name", + "test-workspace" + ) + .fetch_all(&db) + .await?; + + assert_eq!(rows.len(), 5); + assert_eq!(rows[0].name, "vol-0"); + assert_eq!(rows[0].size_bytes, 0); + assert_eq!(rows[4].name, "vol-4"); + assert_eq!(rows[4].size_bytes, 400); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_delete(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "deleteme", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .fetch_one(&db) + .await?; + assert_eq!(count, Some(1)); + + sqlx::query!( + "DELETE FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .fetch_one(&db) + .await?; + assert_eq!(count, Some(0)); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_workspace_fk_constraint(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let result = sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "nonexistent-workspace", + "vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("foreign key"), + "Expected foreign key violation, got: {}", + err + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_primary_key_uniqueness(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "unique-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let result = sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "unique-vol", + 200_i64, + "another-user" + ) + .execute(&db) + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("duplicate key") || err.contains("unique"), + "Expected unique violation, got: {}", + err + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_extra_perms(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Insert volume with default (empty) extra_perms + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "perms-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + // Default extra_perms should be empty object + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.extra_perms, serde_json::json!({})); + + // Set extra_perms via jsonb_set (same pattern as granular_acls.rs) + sqlx::query!( + "UPDATE volume SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2::bool), true) + WHERE workspace_id = $3 AND name = $4", + &vec!["u/alice".to_string()], + true, + "test-workspace", + "perms-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + let perms = row.extra_perms.as_object().unwrap(); + assert_eq!(perms.get("u/alice").and_then(|v| v.as_bool()), Some(true)); + + // Remove a permission entry + sqlx::query!( + "UPDATE volume SET extra_perms = extra_perms - $1 + WHERE workspace_id = $2 AND name = $3", + "u/alice", + "test-workspace", + "perms-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.extra_perms, serde_json::json!({})); + + Ok(()) +} + +#[test] +fn test_parse_volume_annotations_python() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = r#"# sandbox +# volume: training-data /tmp/training +# volume: models /opt/models + +def main(): + pass +"#; + let volumes = parse_volume_annotations(content, "#"); + assert_eq!(volumes.len(), 2); + assert_eq!(volumes[0].name, "training-data"); + assert_eq!(volumes[0].target, "/tmp/training"); + assert_eq!(volumes[1].name, "models"); + assert_eq!(volumes[1].target, "/opt/models"); +} + +#[test] +fn test_parse_volume_annotations_typescript() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = r#"// sandbox +// volume: datasets /tmp/datasets + +export async function main() { + return "hello"; +} +"#; + let volumes = parse_volume_annotations(content, "//"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "datasets"); + assert_eq!(volumes[0].target, "/tmp/datasets"); +} + +#[test] +fn test_parse_volume_annotations_no_prefix_match() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "def main():\n pass"; + let volumes = parse_volume_annotations(content, "#"); + assert!(volumes.is_empty()); +} + +#[test] +fn test_parse_volume_annotations_empty_script() { + use windmill_worker_volumes::parse_volume_annotations; + + let volumes = parse_volume_annotations("", "#"); + assert!(volumes.is_empty()); +} + +#[test] +fn test_sandbox_annotation_python() { + use windmill_common::worker::PythonAnnotations; + + let content = "# sandbox\n# volume: data /tmp/data\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); +} + +#[test] +fn test_sandbox_annotation_typescript() { + use windmill_common::worker::TypeScriptAnnotations; + + let content = "// sandbox\n// volume: data /tmp/data\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); +} + +#[test] +fn test_volume_comment_prefix_selection() { + use windmill_common::scripts::ScriptLang; + + let get_prefix = |lang: &ScriptLang| -> &str { + match lang { + ScriptLang::Python3 + | ScriptLang::Bash + | ScriptLang::Powershell + | ScriptLang::Ansible + | ScriptLang::Ruby => "#", + ScriptLang::Deno + | ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Go => "//", + _ => "", + } + }; + + assert_eq!(get_prefix(&ScriptLang::Python3), "#"); + assert_eq!(get_prefix(&ScriptLang::Bash), "#"); + assert_eq!(get_prefix(&ScriptLang::Powershell), "#"); + assert_eq!(get_prefix(&ScriptLang::Ansible), "#"); + assert_eq!(get_prefix(&ScriptLang::Ruby), "#"); + assert_eq!(get_prefix(&ScriptLang::Deno), "//"); + assert_eq!(get_prefix(&ScriptLang::Bun), "//"); + assert_eq!(get_prefix(&ScriptLang::Bunnative), "//"); + assert_eq!(get_prefix(&ScriptLang::Nativets), "//"); + assert_eq!(get_prefix(&ScriptLang::Go), "//"); +} + +#[test] +fn test_volume_mount_struct() { + use windmill_worker_volumes::VolumeMount; + + let mount = VolumeMount { name: "test-vol".to_string(), target: "/mnt/data".to_string() }; + assert_eq!(mount.name, "test-vol"); + assert_eq!(mount.target, "/mnt/data"); +} + +#[test] +fn test_parse_volume_relative_path() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "// volume: agent-memory .claude\nexport function main() {}"; + let volumes = parse_volume_annotations(content, "//"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "agent-memory"); + assert_eq!(volumes[0].target, ".claude"); +} + +#[test] +fn test_parse_volume_relative_nested_path() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "# volume: data data/models\ndef main():\n pass"; + let volumes = parse_volume_annotations(content, "#"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "data"); + assert_eq!(volumes[0].target, "data/models"); +} + +#[cfg(feature = "private")] +#[test] +fn test_volume_nsjail_mount() { + use std::path::Path; + use windmill_worker_volumes::volume_nsjail_mount; + + let result = volume_nsjail_mount(Path::new("/tmp/volumes/data"), "/mnt/data"); + assert!(result.contains("src: \"/tmp/volumes/data\"")); + assert!(result.contains("dst: \"/mnt/data\"")); + assert!(result.contains("is_bind: true")); + assert!(result.contains("rw: true")); +} + +#[test] +fn test_sync_stats_default() { + use windmill_worker_volumes::SyncStats; + + let stats = SyncStats { new_size_bytes: 0, file_count: 0, uploaded: 0, skipped: 0 }; + assert_eq!(stats.new_size_bytes, 0); + assert_eq!(stats.file_count, 0); + assert_eq!(stats.uploaded, 0); + assert_eq!(stats.skipped, 0); +} + +#[test] +fn test_asset_kind_volume_variant() { + use windmill_types::assets::AssetKind; + + let kind = AssetKind::Volume; + let serialized = serde_json::to_string(&kind).unwrap(); + assert_eq!(serialized, "\"volume\""); + + let deserialized: AssetKind = serde_json::from_str("\"volume\"").unwrap(); + assert!(matches!(deserialized, AssetKind::Volume)); +} + +/// E2E test: run a bun script with volume mount through a SQL-connected worker. +/// Pre-populates the volume in filesystem storage, verifies the script can read +/// files and write new ones, then checks sync-back to storage and DB state. +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_volume_sql_worker_e2e(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null, + "volume_storage": "primary" + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file (workspace-namespaced path) + let vol_dir = storage_dir + .path() + .join("volumes") + .join("test-workspace") + .join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + // 3. Push the job and run with SQL-connected worker + let code = r#"// volume: test-vol /tmp/data + +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("/tmp/data/hello.txt", "utf-8"); + writeFileSync("/tmp/data/output.txt", "written by sql worker"); + return { + read_content: content, + output_exists: existsSync("/tmp/data/output.txt"), + }; +}"#; + + let job = JobPayload::Code(RawCode { + hash: None, + content: code.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port).await; + + assert!(result.success, "job should succeed: {:?}", result.result); + let json = result.json_result().expect("should have JSON result"); + assert_eq!(json["read_content"], json!("hello from volume")); + assert_eq!(json["output_exists"], json!(true)); + + // 4. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert!( + vol_row.file_count >= 2, + "should have at least 2 files (hello.txt + output.txt), got: {}", + vol_row.file_count + ); + assert!(vol_row.size_bytes > 0, "size_bytes should be > 0"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + + // 5. Verify the new file was written back to storage + let output_path = vol_dir.join("output.txt"); + assert!( + output_path.exists(), + "output.txt should be synced back to storage" + ); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by sql worker"); + + Ok(()) +} diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 9548986a5a..32fada32fb 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3548,3 +3548,170 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: Ok(()) } + +#[cfg(all(feature = "quickjs", feature = "python"))] +#[sqlx::test(fixtures("base"))] +async fn test_stop_after_all_iters_if_bad_expr_parallel_branchall( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let port = 123; + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "id": "a", + "value": { + "branches": [ + {"modules": [{ + "id": "b", + "value": { + "input_transforms": { "n": { "type": "javascript", "expr": "flow_input.n" } }, + "type": "rawscript", + "language": "python3", + "content": "def main(n): return n", + }, + }]} + ], + "type": "branchall", + "parallel": true, + }, + "stop_after_all_iters_if": { + "expr": "invalid!!!syntax", + "skip_if_stopped": false, + }, + }, + ], + })) + .unwrap(); + let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; + + let cjob = RunJob::from(job) + .arg("n", json!(42)) + .run_until_complete(&db, false, port) + .await; + + assert!( + !cjob.success, + "flow should fail when stop_after_all_iters_if has bad expression" + ); + + let result = cjob.json_result().unwrap(); + let error_msg = result["error"]["message"].as_str().unwrap_or(""); + assert!( + error_msg.contains("stop_after_all_iters_if"), + "error should mention stop_after_all_iters_if, got: {error_msg}" + ); + + Ok(()) +} + +#[cfg(all(feature = "quickjs", feature = "python"))] +#[sqlx::test(fixtures("base"))] +async fn test_stop_after_all_iters_if_bad_expr_parallel_forloop( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let port = 123; + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "id": "a", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "result.items" }, + "skip_failures": false, + "parallel": true, + "modules": [{ + "value": { + "input_transforms": { + "n": { "type": "javascript", "expr": "flow_input.iter.value" }, + }, + "type": "rawscript", + "language": "python3", + "content": "def main(n): return n", + }, + }], + }, + "stop_after_all_iters_if": { + "expr": "invalid!!!syntax", + "skip_if_stopped": false, + }, + }, + ], + })) + .unwrap(); + let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; + + let cjob = RunJob::from(job) + .arg("items", json!([1, 2, 3])) + .run_until_complete(&db, false, port) + .await; + + assert!( + !cjob.success, + "flow should fail when stop_after_all_iters_if has bad expression" + ); + + let result = cjob.json_result().unwrap(); + let error_msg = result["error"]["message"].as_str().unwrap_or(""); + assert!( + error_msg.contains("stop_after_all_iters_if"), + "error should mention stop_after_all_iters_if, got: {error_msg}" + ); + + Ok(()) +} + +#[cfg(all(feature = "quickjs", feature = "python"))] +#[sqlx::test(fixtures("base"))] +async fn test_results_length_in_input_transform(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Step a returns a list, step b accesses results.a.length via input transform. + // This tests that the handle_full_regex fast path falls through to QuickJS + // when the SQL JSON path operator can't resolve JS properties like .length. + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "a", + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(): return [10, 20, 30]", + }, + }, + { + "id": "b", + "value": { + "input_transforms": { + "v": { "type": "javascript", "expr": "results.a.length" }, + }, + "type": "rawscript", + "language": "python3", + "content": "def main(v): return v", + }, + }, + ], + })) + .unwrap(); + + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!( + result, + json!(3), + "results.a.length should resolve to 3, not null" + ); + + Ok(()) +} diff --git a/backend/windmill-api-agent-workers/src/lib.rs b/backend/windmill-api-agent-workers/src/lib.rs index d2913a217c..b02a7098b7 100644 --- a/backend/windmill-api-agent-workers/src/lib.rs +++ b/backend/windmill-api-agent-workers/src/lib.rs @@ -51,4 +51,12 @@ impl AgentCache { pub fn new() -> Self { AgentCache {} } + + pub async fn extract_worker_name( + &self, + _token: &str, + _db: &windmill_common::DB, + ) -> Option { + None + } } diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs index 186f7af257..af212dc315 100644 --- a/backend/windmill-api-auth/src/auth.rs +++ b/backend/windmill-api-auth/src/auth.rs @@ -35,7 +35,45 @@ use windmill_common::{ lazy_static::lazy_static! { // Global auth cache accessible from main.rs for direct invalidation pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300); + // Cache for token -> email lookups (for non-workspace-member authenticated users) + static ref TOKEN_EMAIL_CACHE: Cache> = Cache::new(500); +} +/// Get email from a valid token, with caching. +/// Used for WM_END_USER_EMAIL when user is authenticated but not a workspace member. +async fn get_email_from_token(db: &DB, token: &str) -> Option { + if let Some(cached) = TOKEN_EMAIL_CACHE.get(token) { + return cached; + } + + let email = sqlx::query_scalar!( + "SELECT email FROM token WHERE token = $1 AND (expiration > NOW() OR expiration IS NULL)", + token + ) + .fetch_optional(db) + .await + .ok() + .flatten() + .flatten(); // email column is nullable, so we get Option> + + TOKEN_EMAIL_CACHE.insert(token.to_string(), email.clone()); + email +} + +/// Get end user email from authenticated user or token. +/// Returns email if user is authenticated (workspace member) or has valid instance token. +pub async fn get_end_user_email( + db: &DB, + opt_authed: Option<&ApiAuthed>, + token: Option<&str>, +) -> Option { + if let Some(authed) = opt_authed { + return Some(authed.email.clone()); + } + if let Some(token) = token { + return get_email_from_token(db, token).await; + } + None } // Global function to invalidate a specific token from cache pub fn invalidate_token_from_cache(token: &str) { diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index efab57cf91..d5ec56d00e 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -29,8 +29,8 @@ use scopes::ScopeDefinition; // Re-export key auth types and functions pub use auth::{ - invalidate_token_from_cache, AuthCache, ExpiringAuthCache, OptTokened, Tokened, - TruncatedTokenWithEmail, AUTH_CACHE, + get_end_user_email, invalidate_token_from_cache, AuthCache, ExpiringAuthCache, OptTokened, + Tokened, TruncatedTokenWithEmail, AUTH_CACHE, }; // ------------ ApiAuthed & OptJobAuthed types ------------ @@ -557,6 +557,14 @@ pub async fn create_token_internal( )); } + register_token_expiry_notification( + &mut *tx, + &token, + token_config.label.as_deref(), + token_config.expiration, + ) + .await; + audit_log( &mut *tx, authed, @@ -572,6 +580,31 @@ pub async fn create_token_internal( Ok(token) } +/// Insert a pending expiry notification row for user tokens that have an expiration. +pub async fn register_token_expiry_notification( + tx: &mut sqlx::PgConnection, + token: &str, + label: Option<&str>, + expiration: Option>, +) { + let Some(expiration) = expiration else { return }; + if label == Some("session") + || label.is_some_and(|l| l.starts_with("ephemeral") || l.starts_with("Ephemeral")) + { + return; + } + if let Err(e) = sqlx::query!( + "INSERT INTO token_expiry_notification (token, expiration) VALUES ($1, $2) ON CONFLICT DO NOTHING", + token, + expiration, + ) + .execute(&mut *tx) + .await + { + tracing::error!("Failed to register token expiry notification: {}", e); + } +} + // ------------ Permission helpers ------------ pub fn get_perm_in_extra_perms_for_authed( diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index 048a4120f5..da9267419d 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -24,7 +24,7 @@ use windmill_common::{ utils::{not_found_if_none, StripPath}, }; -const KINDS: [&str; 18] = [ +const KINDS: [&str; 19] = [ "script", "group_", "resource", @@ -43,6 +43,7 @@ const KINDS: [&str; 18] = [ "gcp_trigger", "sqs_trigger", "email_trigger", + "volume", ]; pub fn workspaced_service() -> Router { @@ -77,7 +78,7 @@ async fn add_granular_acl( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" || kind == "folder" { + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { "name" } else { "path" @@ -89,6 +90,22 @@ async fn add_granular_acl( } else if kind == "group_" { crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) .await?; + } else if kind == "volume" { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("volume '{path}' not found")))?; + // created_by is stored with u/ prefix (from job.permissioned_as) + let owner_username = created_by.strip_prefix("u/").unwrap_or(&created_by); + if owner_username != authed.username { + return Err(Error::NotAuthorized( + "Only the volume owner or an admin can modify permissions".to_string(), + )); + } } else { require_owner_of_path(&authed, path)?; } @@ -243,6 +260,22 @@ async fn remove_granular_acl( } else if kind == "group_" { crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) .await?; + } else if kind == "volume" { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("volume '{path}' not found")))?; + // created_by is stored with u/ prefix (from job.permissioned_as) + let owner_username = created_by.strip_prefix("u/").unwrap_or(&created_by); + if owner_username != authed.username { + return Err(Error::NotAuthorized( + "Only the volume owner or an admin can modify permissions".to_string(), + )); + } } else { require_owner_of_path(&authed, path)?; } @@ -250,7 +283,7 @@ async fn remove_granular_acl( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" || kind == "folder" { + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { "name" } else { "path" @@ -380,7 +413,11 @@ async fn get_granular_acls( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" { "name" } else { "path" }; + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { + "name" + } else { + "path" + }; let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!( "SELECT extra_perms from {kind} WHERE {identifier} = $1 AND workspace_id = $2" )) diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 04a1eb617b..6b408724fd 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -43,8 +43,8 @@ use windmill_common::{ get_database_url, global_settings::{ APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, - CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, - ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, + EMAIL_DOMAIN_SETTING, ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, }, instance_config::{self, ApplyMode, InstanceConfig}, server::Smtp, @@ -519,6 +519,7 @@ pub async fn get_global_setting( && key != DEFAULT_TAGS_WORKSPACES_SETTING && key != HUB_BASE_URL_SETTING && key != HUB_ACCESSIBLE_URL_SETTING + && key != DISABLE_HUB_SETTING && key != EMAIL_DOMAIN_SETTING && key != APP_WORKSPACED_ROUTE_SETTING { @@ -1085,7 +1086,7 @@ async fn sync_cached_resource_types( require_super_admin(&db, &authed.email).await?; use windmill_common::worker::HUB_RT_CACHE_DIR; - let cache_path = format!("{}/resource_types.json", HUB_RT_CACHE_DIR); + let cache_path = format!("{}/resource_types.json", *HUB_RT_CACHE_DIR); let content = tokio::fs::read_to_string(&cache_path).await.map_err(|e| { error::Error::NotFound(format!( diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 17ad6cd7fb..ecd1c5fbd3 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -1850,6 +1850,14 @@ async fn impersonate( .execute(&mut *tx) .await?; + windmill_api_auth::register_token_expiry_notification( + &mut *tx, + &token, + new_token.label.as_deref(), + new_token.expiration, + ) + .await; + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index b698426d53..a03bb3a490 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -29,6 +29,7 @@ windmill-dep-map.workspace = true axum.workspace = true chrono.workspace = true hex.workspace = true +magic-crypt.workspace = true http.workspace = true hyper.workspace = true lazy_static.workspace = true diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 8f3311978a..a85383e496 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -31,7 +31,9 @@ use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::users::username_to_permissioned_as; -use windmill_common::variables::{build_crypt, decrypt, encrypt, WORKSPACE_CRYPT_CACHE}; +use windmill_common::variables::{ + build_crypt, decrypt, encrypt, SECRET_SALT, WORKSPACE_CRYPT_CACHE, +}; use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; #[cfg(feature = "enterprise")] use windmill_common::workspaces::GitRepositorySettings; @@ -300,6 +302,8 @@ struct LargeFileStorageWithSecondary { large_file_storage: LargeFileStorage, #[serde(default)] secondary_storage: HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + volume_storage: Option, } #[derive(Deserialize, Debug)] struct EditLargeFileStorageConfig { @@ -2418,20 +2422,28 @@ async fn set_encryption_key( )); } + // Build the previous cipher before the transaction (reads from cache/pool) let previous_encryption_key = build_crypt(&db, w_id.as_str()).await?; + let mut tx = db.begin().await?; + sqlx::query!( "UPDATE workspace_key SET key = $1 WHERE workspace_id = $2", request.new_key.clone(), w_id ) - .execute(&db) + .execute(&mut *tx) .await?; - WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); - if !request.skip_reencrypt.unwrap_or(false) { - let new_encryption_key = build_crypt(&db, w_id.as_str()).await?; + // Build the new cipher directly from the key string, since the transaction + // hasn't committed yet and build_crypt() would read the old key from the pool. + let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { + format!("{}{}", request.new_key, salt) + } else { + request.new_key.clone() + }; + let new_encryption_key = magic_crypt::new_magic_crypt!(crypt_key, 256); let mut truncated_new_key = request.new_key.clone(); truncated_new_key.truncate(8); @@ -2445,7 +2457,7 @@ async fn set_encryption_key( "SELECT path, value, is_secret FROM variable WHERE workspace_id = $1", w_id ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; for variable in all_variables { @@ -2466,11 +2478,16 @@ async fn set_encryption_key( w_id, variable.path ) - .execute(&db) + .execute(&mut *tx) .await?; } } + tx.commit().await?; + + // Invalidate the cache only after the transaction has committed + WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); + // Trigger git sync for encryption key changes handle_deployment_metadata( &authed.email, diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 7af8137630..14069590eb 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -70,6 +70,7 @@ windmill-git-sync.workspace = true windmill-indexer = { workspace = true, optional = true } windmill-autoscaling = { workspace = true, optional = true } windmill-worker = { workspace = true, optional = true } +windmill-worker-volumes.workspace = true windmill-dep-map.workspace = true tokio.workspace = true tokio-stream.workspace = true @@ -173,6 +174,7 @@ aws-sdk-bedrock = { workspace = true, optional = true } aws-sdk-bedrockruntime = { workspace = true, optional = true } aws-smithy-types = { workspace = true, optional = true } async-trait.workspace = true +eventsource-stream.workspace = true windmill-jseval.workspace = true tar.workspace = true flate2.workspace = true diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 1a036f6dc9..7f04a7287e 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -8857,9 +8857,8 @@ paths: type: boolean flow_env: type: object - description: Environment variables available to all steps - additionalProperties: - type: string + description: "Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource)." + additionalProperties: {} priority: type: number description: Execution priority (higher numbers run first) @@ -14644,9 +14643,8 @@ paths: type: boolean flow_env: type: object - description: Environment variables available to all steps - additionalProperties: - type: string + description: "Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource)." + additionalProperties: {} priority: type: number description: Execution priority (higher numbers run first) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 7e89e6dd0e..1750276778 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.648.0 + version: 1.651.1 title: Windmill API contact: @@ -15198,6 +15198,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] responses: "200": @@ -15243,6 +15244,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] requestBody: description: acl to add @@ -15299,6 +15301,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] requestBody: description: acl to add @@ -16973,9 +16976,9 @@ paths: description: count of log lines that matched the query per hostname type: object - /srch/index/delete/{idx_name}: + /indexer/delete/{idx_name}: delete: - summary: Restart container and delete the index to recreate it. + summary: Clear an index and restart the indexer. operationId: clearIndex tags: - indexSearch @@ -16990,12 +16993,102 @@ paths: - ServiceLogIndex responses: "200": - description: idx to be deleted and container restarting + description: idx to be deleted and indexer restarting content: text/plain: schema: type: string + /indexer/storage: + get: + summary: Get index storage sizes (disk and S3). + operationId: getIndexStorageSizes + tags: + - indexSearch + responses: + "200": + description: storage sizes for each index + content: + application/json: + schema: + type: object + properties: + job_index: + type: object + properties: + disk_size_bytes: + type: integer + nullable: true + s3_size_bytes: + type: integer + nullable: true + service_log_index: + type: object + properties: + disk_size_bytes: + type: integer + nullable: true + s3_size_bytes: + type: integer + nullable: true + + /indexer/status: + get: + summary: Get indexer status including liveness and storage sizes. + operationId: getIndexerStatus + tags: + - indexSearch + responses: + "200": + description: indexer status for each index + content: + application/json: + schema: + type: object + properties: + job_indexer: + type: object + properties: + is_alive: + type: boolean + last_locked_at: + type: string + format: date-time + nullable: true + owner: + type: string + nullable: true + storage: + type: object + properties: + disk_size_bytes: + type: integer + nullable: true + s3_size_bytes: + type: integer + nullable: true + log_indexer: + type: object + properties: + is_alive: + type: boolean + last_locked_at: + type: string + format: date-time + nullable: true + owner: + type: string + nullable: true + storage: + type: object + properties: + disk_size_bytes: + type: integer + nullable: true + s3_size_bytes: + type: integer + nullable: true + /w/{workspace}/assets/list: get: summary: List all assets in the workspace with cursor pagination @@ -17192,7 +17285,90 @@ paths: path: type: string description: The asset path - + + + /w/{workspace}/volumes/list: + get: + summary: List all volumes in the workspace + operationId: listVolumes + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: list of volumes + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Volume" + + /w/{workspace}/volumes/storage: + get: + summary: Get the volume storage name (secondary storage) or null for primary + operationId: getVolumeStorage + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: volume storage name or null + content: + application/json: + schema: + type: string + nullable: true + + /w/{workspace}/volumes/create: + post: + summary: Create a new volume + operationId: createVolume + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - name + properties: + name: + type: string + responses: + "200": + description: volume created + content: + text/plain: + schema: + type: string + + /w/{workspace}/volumes/delete/{name}: + delete: + summary: Delete a volume (admin only) + operationId: deleteVolume + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: name + in: path + required: true + schema: + type: string + responses: + "200": + description: volume deleted + content: + text/plain: + schema: + type: string /mcp/w/{workspace}/list_tools: get: @@ -23907,6 +24083,7 @@ components: - resource - ducklake - datatable + - volume Asset: type: object properties: @@ -23915,6 +24092,38 @@ components: kind: $ref: "#/components/schemas/AssetKind" required: [path, kind] + Volume: + type: object + required: + - name + - size_bytes + - file_count + - created_at + - created_by + properties: + name: + type: string + size_bytes: + type: integer + format: int64 + file_count: + type: integer + created_at: + type: string + format: date-time + created_by: + type: string + updated_at: + type: string + format: date-time + nullable: true + last_used_at: + type: string + format: date-time + nullable: true + extra_perms: + type: object + additionalProperties: true ProtectionRuleset: type: object description: A workspace protection rule defining restrictions and bypass permissions diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 730df2a875..92ff49f4a4 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -29,7 +29,7 @@ const AI_TIMEOUT_MAX_SECS: u64 = 86400; // 24 hours const AI_TIMEOUT_DEFAULT_SECS: u64 = 3600; // 1 hour const HTTP_POOL_MAX_IDLE_PER_HOST: usize = 10; const HTTP_POOL_IDLE_TIMEOUT_SECS: u64 = 90; -const KEEPALIVE_INTERVAL_SECS: u64 = 15; +pub(crate) const KEEPALIVE_INTERVAL_SECS: u64 = 15; lazy_static::lazy_static! { /// AI request timeout in seconds. @@ -87,7 +87,7 @@ lazy_static::lazy_static! { } }; - static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() + pub(crate) static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() .timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS)) .pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST) .pool_idle_timeout(Some(std::time::Duration::from_secs(HTTP_POOL_IDLE_TIMEOUT_SECS))) @@ -378,12 +378,7 @@ impl AIRequestConfig { let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some(); let is_google_ai = matches!(provider, AIProvider::GoogleAI); - // GoogleAI uses OpenAI-compatible endpoint in the proxy (for the chat), but not for the ai agent - let base_url = if is_google_ai { - format!("{}/openai", base_url) - } else { - base_url.to_string() - }; + let base_url = base_url.to_string(); let base_url = base_url.as_str(); // Build URL based on provider @@ -428,6 +423,9 @@ impl AIRequestConfig { if let Some(api_key) = self.api_key { if is_azure { request = request.header("api-key", api_key.clone()) + } else if is_google_ai { + // Native Gemini API uses x-goog-api-key, not Authorization: Bearer + request = request.header("x-goog-api-key", api_key.clone()) } else { request = request.header("authorization", format!("Bearer {}", api_key.clone())) } @@ -611,7 +609,7 @@ fn is_sse_response(headers: &HeaderMap) -> bool { .unwrap_or(false) } -fn inject_keepalives( +pub(crate) fn inject_keepalives( upstream: S, interval: Duration, ) -> impl futures::Stream> @@ -830,6 +828,36 @@ async fn proxy( ai_path = chat_path; } + // Handle GoogleAI (Gemini) using the native Gemini API + if matches!(provider, AIProvider::GoogleAI) { + let api_key = request_config.api_key.as_deref().unwrap_or(""); + let base_url = request_config.base_url.trim_end_matches('/'); + + let mut tx = db.begin().await?; + audit_log( + &mut *tx, + &authed, + "ai.request", + ActionKind::Execute, + &w_id, + Some(&authed.email), + Some([("ai_config_path", &format!("{:?}", ai_path)[..])].into()), + ) + .await?; + tx.commit().await?; + + return match ai_path.as_str() { + "chat/completions" => { + crate::google::handle_google_ai_chat(&body, api_key, base_url).await + } + "models" => crate::google::handle_google_ai_models(api_key, base_url).await, + _ => Err(Error::BadRequest(format!( + "Unsupported Google AI path: {}", + ai_path + ))), + }; + } + // Handle Bedrock-specific logic when the feature is enabled #[cfg(feature = "bedrock")] { diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index d156dc9c88..9846912192 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -8,7 +8,7 @@ use std::{collections::HashMap, sync::Arc}; * LICENSE-AGPL for a copy of the license. */ use crate::{ - auth::OptTokened, + auth::{get_end_user_email, OptTokened}, db::{ApiAuthed, DB}, jobs::RunJobQuery, users::{require_owner_of_path, OptAuthed}, @@ -993,9 +993,18 @@ macro_rules! process_app_multipart { let mut uploaded_js = false; let mut multipart = $multipart; - while let Some(field) = multipart.next_field().await.unwrap() { - let name = field.name().unwrap().to_string(); - let data = field.bytes().await.unwrap(); + while let Some(field) = multipart + .next_field() + .await + .map_err(|e| Error::BadRequest(format!("failed to read multipart field: {e}")))? + { + let name = field + .name() + .ok_or_else(|| Error::BadRequest("multipart field missing name".to_string()))? + .to_string(); + let data = field.bytes().await.map_err(|e| { + Error::BadRequest(format!("failed to read multipart stream: {e}")) + })?; if name == "app" { let app = serde_json::from_slice(&data).map_err(to_anyhow)?; let (ntx, npath, nid) = $internal_fn( @@ -2149,7 +2158,8 @@ async fn execute_component( (email.as_str(), permissioned_as) }; - let end_user_email = opt_authed.as_ref().map(|a| a.email.clone()); + let end_user_email = + get_end_user_email(&db, opt_authed.as_ref(), tokened.token.as_deref()).await; let (uuid, mut tx) = push( &db, diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 144703ba5c..66a67c5f97 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -1,4 +1,5 @@ pub use windmill_api_auth::auth::{ - invalidate_token_from_cache, list_tokens_internal, transform_old_scope_to_new_scope, AuthCache, - ExpiringAuthCache, OptTokened, Tokened, TruncatedTokenWithEmail, + get_end_user_email, invalidate_token_from_cache, list_tokens_internal, + transform_old_scope_to_new_scope, AuthCache, ExpiringAuthCache, OptTokened, Tokened, + TruncatedTokenWithEmail, }; diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 8b0fb44dfe..c8ed841e19 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -284,6 +284,9 @@ pub async fn migrate( 20260207000004, ]; for m in migrator.migrations.iter() { + if m.migration_type.is_down_migration() { + continue; + } if potentially_stale.contains(&m.version) { if let Err(err) = sqlx::query("DELETE FROM _sqlx_migrations WHERE version = $1 AND checksum != $2") diff --git a/backend/windmill-api/src/google.rs b/backend/windmill-api/src/google.rs new file mode 100644 index 0000000000..bc32c20ba4 --- /dev/null +++ b/backend/windmill-api/src/google.rs @@ -0,0 +1,306 @@ +//! Google AI (Gemini API) handler for the AI chat proxy. +//! +//! Handles POST `chat/completions` requests using the native Gemini API, +//! converting from/to OpenAI format so the existing frontend parsers continue to work. +//! +//! Used by `windmill-api/src/ai.rs` when the provider is `GoogleAI`. +//! Shared conversion logic lives in `windmill_common::ai_google`. + +use axum::body::Body; +use bytes::Bytes; +use eventsource_stream::Eventsource; +use futures::StreamExt; +use serde::Deserialize; +use serde_json::json; +use windmill_common::{ + ai_google::{ + gemini_event_to_openai_sse_chunks, gemini_response_to_openai, openai_messages_to_gemini, + parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google, + GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiTextRequest, GeminiTool, + }, + ai_types::OpenAIMessage, + error::{Error, Result}, +}; + +use crate::ai::{inject_keepalives, HTTP_CLIENT, KEEPALIVE_INTERVAL_SECS}; + +// ============================================================================ +// Request type (OpenAI format received from the frontend) +// ============================================================================ + +#[derive(Deserialize, Debug)] +struct ChatRequest { + model: String, + messages: Vec, + #[serde(default)] + stream: bool, + #[serde(default)] + temperature: Option, + #[serde(default)] + max_tokens: Option, + #[serde(default)] + tools: Option>, +} + +#[derive(Deserialize, Debug)] +struct ChatRequestTool { + function: ChatRequestToolFunction, +} + +#[derive(Deserialize, Debug)] +struct ChatRequestToolFunction { + name: String, + #[serde(default)] + description: Option, + #[serde(default)] + parameters: Option, +} + +// ============================================================================ +// Public handler +// ============================================================================ + +/// Handle a `chat/completions` POST request using the native Gemini API. +/// +/// Converts the incoming OpenAI-format body to a `GeminiTextRequest`, sends it +/// to the appropriate Gemini endpoint, and converts the response back to the +/// OpenAI SSE or JSON format that the frontend expects. +pub async fn handle_google_ai_chat( + body: &Bytes, + api_key: &str, + base_url: &str, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + let request: ChatRequest = serde_json::from_slice(body) + .map_err(|e| Error::BadRequest(format!("Failed to parse request body: {}", e)))?; + + let (contents, system_instruction) = openai_messages_to_gemini(&request.messages); + + let generation_config = + if request.temperature.is_some() || request.max_tokens.is_some() { + Some(GeminiGenerationConfig { + temperature: request.temperature, + max_output_tokens: request.max_tokens, + response_mime_type: None, + response_schema: None, + }) + } else { + None + }; + + let gemini_tools = request.tools.as_ref().map(|tools| { + let declarations: Vec = tools + .iter() + .map(|t| { + let mut params = t.function.parameters.clone().unwrap_or(json!({})); + sanitize_schema_for_google(&mut params); + GeminiFunctionDeclaration { + name: t.function.name.clone(), + description: t.function.description.clone(), + parameters: params, + } + }) + .collect(); + vec![GeminiTool { + function_declarations: Some(declarations), + google_search: None, + }] + }); + + let gemini_request = GeminiTextRequest { + contents, + tools: gemini_tools, + tool_config: None, + system_instruction, + generation_config, + }; + + let request_body = serde_json::to_string(&gemini_request) + .map_err(|e| Error::internal_err(format!("Failed to serialize Gemini request: {}", e)))?; + + let base_url = base_url.trim_end_matches('/'); + + if request.stream { + handle_streaming(&request.model, request_body, api_key, base_url).await + } else { + handle_non_streaming(&request.model, request_body, api_key, base_url).await + } +} + +// ============================================================================ +// Streaming path +// ============================================================================ + +async fn handle_streaming( + model: &str, + request_body: String, + api_key: &str, + base_url: &str, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + let endpoint = format!("{}/models/{}:streamGenerateContent?alt=sse", base_url, model); + + let response = HTTP_CLIENT + .post(&endpoint) + .header("content-type", "application/json") + .header("x-goog-api-key", api_key) + .body(request_body) + .send() + .await + .map_err(|e| { + Error::internal_err(format!("Failed to send request to Gemini API: {}", e)) + })?; + + if let Err(e) = response.error_for_status_ref() { + let status = e.status().map(|s| s.to_string()).unwrap_or_default(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::AIError(format!("{}: {}", status, body))); + } + + let id = format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()); + let model_str = model.to_string(); + + let gemini_sse_stream = response.bytes_stream().eventsource(); + let openai_sse_stream = async_stream::stream! { + tokio::pin!(gemini_sse_stream); + let mut tool_call_index: usize = 0; + while let Some(event) = gemini_sse_stream.next().await { + match event { + Ok(event) => match parse_gemini_sse_event(&event.data) { + Ok(Some(parsed)) => { + for chunk in gemini_event_to_openai_sse_chunks( + &parsed, &id, &model_str, &mut tool_call_index, + ) { + yield Ok::(Bytes::from(chunk)); + } + } + Ok(None) => {} + Err(e) => tracing::error!("Error parsing Gemini SSE event: {}", e), + }, + Err(e) => tracing::error!("Error reading Gemini SSE stream: {}", e), + } + } + yield Ok::(Bytes::from("data: [DONE]\n\n")); + }; + + let mut headers = http::HeaderMap::new(); + headers.insert("content-type", "text/event-stream".parse().unwrap()); + headers.insert("cache-control", "no-cache".parse().unwrap()); + headers.insert("connection", "keep-alive".parse().unwrap()); + + Ok(( + http::StatusCode::OK, + headers, + Body::from_stream(inject_keepalives( + Box::pin(openai_sse_stream), + std::time::Duration::from_secs(KEEPALIVE_INTERVAL_SECS), + )), + )) +} + +// ============================================================================ +// Model listing +// ============================================================================ + +/// List available Gemini models and convert to OpenAI format. +/// +/// Gemini returns `{ models: [{ name: "models/gemini-2.5-flash", displayName, ... }] }`. +/// The frontend expects OpenAI format `{ data: [{ id: "models/gemini-2.5-flash", ... }] }`. +pub async fn handle_google_ai_models( + api_key: &str, + base_url: &str, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + #[derive(Deserialize)] + struct GeminiModel { + name: String, + #[serde(rename = "displayName", default)] + display_name: String, + } + + #[derive(Deserialize)] + struct GeminiModelsResponse { + #[serde(default)] + models: Vec, + } + + let endpoint = format!("{}/models", base_url.trim_end_matches('/')); + let response = HTTP_CLIENT + .get(&endpoint) + .header("x-goog-api-key", api_key) + .send() + .await + .map_err(|e| Error::internal_err(format!("Failed to fetch Gemini models: {}", e)))?; + + if let Err(e) = response.error_for_status_ref() { + let status = e.status().map(|s| s.to_string()).unwrap_or_default(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::AIError(format!("{}: {}", status, body))); + } + + let gemini_resp: GeminiModelsResponse = response.json().await.map_err(|e| { + Error::internal_err(format!("Failed to parse Gemini models response: {}", e)) + })?; + + let data: Vec = gemini_resp + .models + .into_iter() + .map(|m| { + json!({ + "id": m.name, + "object": "model", + "display_name": m.display_name, + }) + }) + .collect(); + + let body_bytes = serde_json::to_vec(&json!({ "data": data })) + .map_err(|e| Error::internal_err(format!("Failed to serialize models: {}", e)))?; + + let mut headers = http::HeaderMap::new(); + headers.insert("content-type", "application/json".parse().unwrap()); + + Ok((http::StatusCode::OK, headers, Body::from(body_bytes))) +} + +// ============================================================================ +// Non-streaming path +// ============================================================================ + +async fn handle_non_streaming( + model: &str, + request_body: String, + api_key: &str, + base_url: &str, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + let endpoint = format!("{}/models/{}:generateContent", base_url, model); + + let response = HTTP_CLIENT + .post(&endpoint) + .header("content-type", "application/json") + .header("x-goog-api-key", api_key) + .body(request_body) + .send() + .await + .map_err(|e| { + Error::internal_err(format!("Failed to send request to Gemini API: {}", e)) + })?; + + if let Err(e) = response.error_for_status_ref() { + let status = e.status().map(|s| s.to_string()).unwrap_or_default(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::AIError(format!("{}: {}", status, body))); + } + + let body = response.bytes().await.map_err(|e| { + Error::internal_err(format!("Failed to read Gemini response body: {}", e)) + })?; + + let parsed = parse_gemini_response(&body)?; + let openai_response = gemini_response_to_openai(&parsed, model); + + let body_bytes = serde_json::to_vec(&openai_response) + .map_err(|e| Error::internal_err(format!("Failed to serialize response: {}", e)))?; + + let mut headers = http::HeaderMap::new(); + headers.insert("content-type", "application/json".parse().unwrap()); + + Ok((http::StatusCode::OK, headers, Body::from(body_bytes))) +} diff --git a/backend/windmill-api/src/health.rs b/backend/windmill-api/src/health.rs index d861f338ef..60bd290f66 100644 --- a/backend/windmill-api/src/health.rs +++ b/backend/windmill-api/src/health.rs @@ -240,11 +240,7 @@ async fn check_database_detailed(db: &DB) -> DatabaseHealth { let check = check_database_with_latency(db).await; let pool = get_pool_stats(db); - DatabaseHealth { - healthy: check.healthy, - latency_ms: check.latency_ms, - pool, - } + DatabaseHealth { healthy: check.healthy, latency_ms: check.latency_ms, pool } } async fn check_worker_count(db: &DB) -> i64 { @@ -295,13 +291,7 @@ async fn check_workers_detailed(db: &DB) -> WorkersHealth { let healthy = active_count > 0; - WorkersHealth { - healthy, - active_count, - worker_groups, - min_version, - versions, - } + WorkersHealth { healthy, active_count, worker_groups, min_version, versions } } async fn check_queue(db: &DB) -> QueueHealth { @@ -333,10 +323,7 @@ fn get_version() -> String { /// Spawn a background task that performs a health check every 10 seconds. /// Updates the cache and prometheus metrics continuously. -pub fn start_health_check_loop( - db: DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { +pub fn start_health_check_loop(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(10)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -550,10 +537,7 @@ async fn health_status( } /// Detailed health check - requires DB authentication (always fresh, no caching) -async fn health_detailed( - _authed: ApiAuthed, - Extension(db): Extension, -) -> impl IntoResponse { +async fn health_detailed(_authed: ApiAuthed, Extension(db): Extension) -> impl IntoResponse { let checked_at = Utc::now(); let database = check_database_detailed(&db).await; let readiness = check_readiness(); @@ -564,12 +548,7 @@ async fn health_detailed( status: HealthStatus::Unhealthy, checked_at, version: get_version(), - checks: HealthChecks { - database, - workers: None, - queue: None, - readiness, - }, + checks: HealthChecks { database, workers: None, queue: None, readiness }, }; return (StatusCode::SERVICE_UNAVAILABLE, Json(response)); } @@ -587,12 +566,7 @@ async fn health_detailed( status, checked_at, version: get_version(), - checks: HealthChecks { - database, - workers: Some(workers), - queue: Some(queue), - readiness, - }, + checks: HealthChecks { database, workers: Some(workers), queue: Some(queue), readiness }, }; let status_code = if status == HealthStatus::Unhealthy { diff --git a/backend/windmill-api/src/indexer_oss.rs b/backend/windmill-api/src/indexer_oss.rs index eee87acdcb..fd87c10fe2 100644 --- a/backend/windmill-api/src/indexer_oss.rs +++ b/backend/windmill-api/src/indexer_oss.rs @@ -14,3 +14,8 @@ pub fn workspaced_service() -> Router { pub fn global_service() -> Router { Router::new() } + +#[cfg(not(feature = "private"))] +pub fn management_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 01f5d67bd3..23f20459c3 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -12,15 +12,15 @@ use windmill_types::s3::StorageResourceType; #[cfg(all(feature = "parquet", not(feature = "private")))] use crate::db::{ApiAuthed, OptJobAuthed, DB}; #[cfg(all(feature = "parquet", not(feature = "private")))] -use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult}; -#[cfg(not(feature = "private"))] -use windmill_object_store::ObjectStoreResource; -#[cfg(all(feature = "parquet", not(feature = "private")))] use std::sync::Arc; +#[cfg(all(feature = "parquet", not(feature = "private")))] +use windmill_common::db::UserDB; #[cfg(not(feature = "private"))] use windmill_common::error; #[cfg(all(feature = "parquet", not(feature = "private")))] -use windmill_common::db::UserDB; +use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult}; +#[cfg(not(feature = "private"))] +use windmill_object_store::ObjectStoreResource; #[cfg(all(feature = "parquet", not(feature = "private")))] use bytes::Bytes; diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 420bbf24b9..ac5a9a306b 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -47,7 +47,7 @@ use windmill_common::runtime_assets::{register_runtime_asset, InsertRuntimeAsset use windmill_common::scripts::ScriptRunnableSettingsInline; use windmill_common::triggers::TriggerMetadata; use windmill_common::utils::{RunnableKind, WarnAfterExt}; -use windmill_common::worker::{Connection, CLOUD_HOSTED, TMP_DIR}; +use windmill_common::worker::{Connection, CLOUD_HOSTED, WINDMILL_DIR}; use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES, }; @@ -448,14 +448,15 @@ async fn get_flow_env_by_flow_job_id( Path((w_id, flow_job_id, var_name)): Path<(String, Uuid, String)>, Query(JsonPath { json_path, .. }): Query, ) -> windmill_common::error::JsonResult> { - let flow_env = sqlx::query_scalar!( + // Fetch raw value (without json_path) to check for $var:/$res: references + let raw_value = sqlx::query_scalar!( r#" SELECT CASE WHEN flow_version.id IS NOT NULL THEN - (flow_version.value -> 'flow_env' -> $3) #> $4 + flow_version.value -> 'flow_env' -> $3 ELSE - (root_job.raw_flow -> 'flow_env' -> $3) #> $4 + root_job.raw_flow -> 'flow_env' -> $3 END AS "flow_env: sqlx::types::Json>" FROM v2_job current_job @@ -472,16 +473,86 @@ async fn get_flow_env_by_flow_job_id( flow_job_id, w_id, var_name, - json_path - .as_ref() - .map(|x| x.split(".").collect::>()) - .unwrap_or_default() as Vec<&str>, ) .fetch_optional(&db) .await? - .map(|r| r.map(|x| x.0)) - .flatten() - .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + .and_then(|r| r.map(|x| x.0)); + + // Resolve $var:/$res: references if present + let resolved = if let Some(raw) = raw_value { + let raw_str = raw.get(); + let db_authed = windmill_common::db::DbWithOptAuthed::::from_authed( + &authed, + db.clone(), + None, + ); + if let Some(path) = raw_str + .strip_prefix("\"$var:") + .and_then(|s| s.strip_suffix("\"")) + { + match windmill_store::variables::get_value_internal(&db_authed, &w_id, path, false) + .await + { + Ok(val) => to_raw_value(&serde_json::Value::String(val)), + Err(e) => { + tracing::warn!("Failed to resolve flow_env variable $var:{path}: {e}"); + raw + } + } + } else if let Some(path) = raw_str + .strip_prefix("\"$res:") + .and_then(|s| s.strip_suffix("\"")) + { + match windmill_store::resources::get_resource_value_interpolated_internal( + &db_authed, + &w_id, + path, + Some(flow_job_id), + Some(&tokened.token), + false, + ) + .await + { + Ok(Some(val)) => to_raw_value(&val), + Ok(None) => { + tracing::warn!( + "Failed to resolve flow_env resource $res:{path}: resource not found" + ); + raw + } + Err(e) => { + tracing::warn!("Failed to resolve flow_env resource $res:{path}: {e}"); + raw + } + } + } else { + raw + } + } else { + to_raw_value(&serde_json::Value::Null) + }; + + // Apply json_path navigation on the (possibly resolved) value + let flow_env = if let Some(ref jp) = json_path { + let mut value: serde_json::Value = + serde_json::from_str(resolved.get()).unwrap_or(serde_json::Value::Null); + for part in jp.split('.') { + value = match value { + serde_json::Value::Object(ref mut map) => { + map.remove(part).unwrap_or(serde_json::Value::Null) + } + serde_json::Value::Array(ref arr) => part + .parse::() + .ok() + .and_then(|i| arr.get(i).cloned()) + .unwrap_or(serde_json::Value::Null), + _ => serde_json::Value::Null, + }; + } + to_raw_value(&value) + } else { + resolved + }; log_job_view( &db, @@ -1412,7 +1483,7 @@ async fn get_logs_from_disk( if log_offset > 0 { if let Some(file_index) = log_file_index.clone() { for file_p in &file_index { - if !tokio::fs::metadata(format!("{TMP_DIR}/{file_p}")) + if !tokio::fs::metadata(format!("{}/{file_p}", *WINDMILL_DIR)) .await .is_ok() { @@ -1427,7 +1498,7 @@ async fn get_logs_from_disk( "#.to_string(), )); for file_p in file_index.clone() { - let mut file = tokio::fs::File::open(format!("{TMP_DIR}/{file_p}")).await.map_err(to_anyhow)?; + let mut file = tokio::fs::File::open(format!("{}/{file_p}", *WINDMILL_DIR)).await.map_err(to_anyhow)?; let mut buffer = Vec::new(); file.read_to_end(&mut buffer).await.map_err(to_anyhow)?; yield Ok(bytes::Bytes::from(buffer)) as anyhow::Result; @@ -5888,7 +5959,7 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R )); } - let local_file = format!("{TMP_DIR}/logs/{file_p}"); + let local_file = format!("{}/logs/{file_p}", *WINDMILL_DIR); if tokio::fs::metadata(&local_file).await.is_ok() { let mut file = tokio::fs::File::open(local_file).await.map_err(to_anyhow)?; let mut buffer = Vec::new(); @@ -5934,10 +6005,10 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R } #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - return Err(error::Error::NotFound(format!( - "File not found on server logs volume /tmp/windmill/logs and no distributed logs s3 storage for {}", - file_p - ))); + return Err(error::Error::NotFound(format!( + "File not found on server logs volume {}/logs and no distributed logs s3 storage for {}", + *WINDMILL_DIR, file_p + ))); } async fn get_job_update( diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 76fe0224ed..a25a431dba 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -58,15 +58,13 @@ use windmill_common::db::UserDB; use windmill_common::worker::CLOUD_HOSTED; #[allow(unused_imports)] pub(crate) use windmill_common::BASE_URL; -use windmill_common::{ - utils::GIT_VERSION, - INSTANCE_NAME, -}; +use windmill_common::{utils::GIT_VERSION, INSTANCE_NAME}; use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; mod ai; +mod google; mod apps; pub mod args; mod audit; @@ -173,6 +171,9 @@ pub mod users_ee; mod users_oss; mod utils; mod variables; +#[cfg(feature = "private")] +pub mod volumes_ee; +mod volumes_oss; pub mod webhook_util; mod workspaces; #[cfg(feature = "private")] @@ -251,6 +252,74 @@ type IndexReader = windmill_indexer::completed_runs_oss::IndexReader; #[cfg(feature = "tantivy")] type ServiceLogIndexReader = windmill_indexer::service_logs_oss::ServiceLogIndexReader; +/// Worker name derived from the agent JWT token, used to authenticate volume operations. +/// Defined unconditionally so volume endpoint handlers can reference it regardless of +/// whether agent_worker_server is enabled (the extension is only populated on the agent path). +#[derive(Clone)] +pub struct AgentWorkerName(pub String); + +/// Middleware that injects a synthetic `ApiAuthed` and JWT-derived worker name +/// into request extensions. +/// +/// Used for volume proxy endpoints under the agent_workers path, where the +/// agent JWT auth layer has already validated the request. The volume handlers +/// need `ApiAuthed` to resolve the workspace S3 client, but the agent JWT +/// format is incompatible with the standard auth extractor. +/// +/// The worker name is extracted from the JWT claims rather than trusting +/// self-reported values in request bodies/query params. +#[cfg(feature = "agent_worker_server")] +async fn inject_agent_authed( + request: axum::extract::Request, + next: axum::middleware::Next, +) -> Response { + let mut request = request; + + // Extract worker name from agent JWT via AgentCache + // (OSS returns None; EE decodes the JWT and returns the worker name) + { + let extracted = { + let token = request + .headers() + .get(axum::http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.strip_prefix("Bearer ").map(|t| t.to_string())); + let cache = request.extensions().get::>().cloned(); + let db = request.extensions().get::().cloned(); + match (token, cache, db) { + (Some(token), Some(cache), Some(db)) => Some((token, cache, db)), + _ => None, + } + }; + + if let Some((token, cache, db)) = extracted { + if let Some(worker_name) = cache.extract_worker_name(&token, &db).await { + request + .extensions_mut() + .insert(AgentWorkerName(worker_name)); + } + } + } + + request + .extensions_mut() + .insert(windmill_api_auth::OptJobAuthed { + authed: ApiAuthed { + email: "agent-worker@windmill.dev".to_string(), + username: "agent-worker".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + token_prefix: None, + }, + job_id: None, + }); + next.run(request).await +} + pub async fn run_server( db: DB, job_index_reader: Option, @@ -265,7 +334,7 @@ pub async fn run_server( ) -> anyhow::Result<()> { let user_db = UserDB::new(db.clone()); - for x in [HUB_CACHE_DIR] { + for x in [&*HUB_CACHE_DIR] { DirBuilder::new() .recursive(true) .create(x) @@ -516,6 +585,7 @@ pub async fn run_server( users::workspaced_service().layer(Extension(argon2.clone())), ) .nest("/variables", variables::workspaced_service()) + .nest("/volumes", volumes_oss::workspaced_service()) .nest("/workers", windmill_api_workers::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) .nest("/oidc", oidc_oss::workspaced_service()) @@ -550,6 +620,7 @@ pub async fn run_server( .nest("/embeddings", embeddings::global_service()) .nest("/ai", ai::global_service()) .nest("/inkeep", inkeep_oss::global_service()) + .nest("/indexer", indexer_oss::management_service()) .nest("/mcp/w/:workspace_id/list_tools", mcp_list_tools_service) .nest("/health/detailed", health::detailed_service()) .route_layer(from_extractor::()) @@ -612,8 +683,10 @@ pub async fn run_server( if let Some(agent_workers_job_completed_tx) = agent_workers_job_completed_tx.clone() { - windmill_api_agent_workers::global_service(agent_workers_job_completed_tx) - .layer(Extension(agent_cache.clone())) + windmill_api_agent_workers::global_service( + agent_workers_job_completed_tx, + ) + .layer(Extension(agent_cache.clone())) } else { Router::new() } @@ -626,7 +699,13 @@ pub async fn run_server( .nest("/w/:workspace_id/agent_workers", { #[cfg(feature = "agent_worker_server")] { - agent_workers_router.layer(Extension(agent_cache.clone())) + agent_workers_router + .nest( + "/volumes", + volumes_oss::agent_workspaced_service() + .layer(axum::middleware::from_fn(inject_agent_authed)), + ) + .layer(Extension(agent_cache.clone())) } #[cfg(not(feature = "agent_worker_server"))] { @@ -785,7 +864,10 @@ pub async fn run_server( }, ) // JWKS endpoint for HashiCorp Vault JWT authentication (must be outside /api prefix) - .route("/.well-known/jwks.json", get(windmill_api_settings::get_jwks)) + .route( + "/.well-known/jwks.json", + get(windmill_api_settings::get_jwks), + ) .fallback(static_assets::static_handler) .layer(middleware_stack); diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 604db2dc3e..9a323deef6 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -64,9 +64,10 @@ impl McpBackend for WindmillBackend { auth: &ApiAuthed, workspace_id: &str, favorites_only: bool, + path_prefix: Option<&str>, ) -> BackendResult> { let scope_type = if favorites_only { "favorites" } else { "all" }; - get_items::(&self.user_db, auth, workspace_id, scope_type, "script") + get_items::(&self.user_db, auth, workspace_id, scope_type, "script", path_prefix) .await .map_err(|e| ErrorData::internal_error(e.message, None)) } @@ -76,9 +77,10 @@ impl McpBackend for WindmillBackend { auth: &ApiAuthed, workspace_id: &str, favorites_only: bool, + path_prefix: Option<&str>, ) -> BackendResult> { let scope_type = if favorites_only { "favorites" } else { "all" }; - get_items::(&self.user_db, auth, workspace_id, scope_type, "flow") + get_items::(&self.user_db, auth, workspace_id, scope_type, "flow", path_prefix) .await .map_err(|e| ErrorData::internal_error(e.message, None)) } diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index d168f8fc48..1cfbfd7085 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -136,6 +136,7 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen workspace_id: &str, scope_type: &str, item_type: &str, + path_prefix: Option<&str>, ) -> Result, ErrorData> { let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); let fields = vec!["o.path", "o.summary", "o.description", "o.schema"]; @@ -153,6 +154,11 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)"); } + if let Some(prefix) = path_prefix { + let escaped = prefix.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_"); + sqlb.and_where("o.path LIKE ? ESCAPE '\\'".bind(&format!("{}%", escaped))); + } + sqlb.order_by( if item_type == "flow" { "o.edited_at" diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs index 20d2b58d0e..c83bb21f2c 100644 --- a/backend/windmill-api/src/service_logs.rs +++ b/backend/windmill-api/src/service_logs.rs @@ -102,7 +102,11 @@ async fn get_log_file( #[cfg(feature = "parquet")] if let Some(s3_client) = s3_client { let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); - let file = s3_client.get(&windmill_object_store::object_store_reexports::Path::from(path)).await; + let file = s3_client + .get(&windmill_object_store::object_store_reexports::Path::from( + path, + )) + .await; match file { Ok(file) => { let bytes = file.bytes().await; @@ -126,7 +130,7 @@ async fn get_log_file( } } } - let file = tokio::fs::read(format!("{}{}", TMP_WINDMILL_LOGS_SERVICE, path)).await; + let file = tokio::fs::read(format!("{}{}", *TMP_WINDMILL_LOGS_SERVICE, path)).await; if let Ok(bytes) = file { Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))) } else { diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index f3ec05348a..9d95ead840 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -369,7 +369,11 @@ async fn route_job( let s3_object = s3_client.get(&path).await; let s3_object = match s3_object { - Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { .. }) if trigger.is_static_website => { + Err( + windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { + .. + }, + ) if trigger.is_static_website => { // fallback to index.html if the file is not found let path = windmill_object_store::object_store_reexports::Path::from(format!( "{}/index.html", diff --git a/backend/windmill-api/src/volumes_oss.rs b/backend/windmill-api/src/volumes_oss.rs new file mode 100644 index 0000000000..26b1c2cd46 --- /dev/null +++ b/backend/windmill-api/src/volumes_oss.rs @@ -0,0 +1,17 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::volumes_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn workspaced_service() -> Router { + Router::new() +} + +#[cfg(not(feature = "private"))] +#[allow(dead_code)] +pub fn agent_workspaced_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index c881e9ae9b..091c465a61 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -43,6 +43,7 @@ use windmill_common::runnable_settings::{ConcurrencySettings, DebouncingSettings use windmill_common::scripts::ScriptRunnableSettingsHandle; use windmill_common::utils::require_admin; use windmill_common::variables::decrypt; +use windmill_common::worker::WINDMILL_DIR; use windmill_common::{ db::UserDB, error::{to_anyhow, Error, Result}, @@ -372,7 +373,7 @@ pub(crate) async fn tarball_workspace( let mut tx = user_db.begin(&authed).await?; - let tmp_dir = TempDir::new_in("/tmp/windmill/")?; + let tmp_dir = TempDir::new_in(&*WINDMILL_DIR)?; let name = match archive_type.as_deref() { Some("tar") | None => Ok(format!("windmill-{w_id}.tar")), diff --git a/backend/windmill-common/src/ai_google.rs b/backend/windmill-common/src/ai_google.rs new file mode 100644 index 0000000000..ccf34685e5 --- /dev/null +++ b/backend/windmill-common/src/ai_google.rs @@ -0,0 +1,726 @@ +//! Shared Google AI (Gemini API) types and conversion utilities. +//! +//! This module provides: +//! - Gemini request/response types +//! - OpenAI → Gemini message conversion +//! - Gemini SSE event parsing +//! +//! Used by both windmill-api (chat proxy) and windmill-worker (AI agent). + +use serde::{Deserialize, Serialize}; + +use crate::ai_types::{ContentPart, ExtraContent, GoogleExtraContent, OpenAIContent, OpenAIMessage, ToolDef, UrlCitation}; +use crate::error::Error; + +// ============================================================================ +// Request / Content Types +// ============================================================================ + +/// Inline data for binary content (images). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiInlineData { + #[serde(rename = "mimeType")] + pub mime_type: String, + pub data: String, +} + +/// A part of content — text, inline data, function call, or function response. +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(untagged)] +pub enum GeminiPart { + Text { + text: String, + }, + InlineData { + #[serde(rename = "inlineData")] + inline_data: GeminiInlineData, + }, + FunctionCall { + #[serde(rename = "functionCall")] + function_call: GeminiFunctionCall, + /// Thought signature for Gemini 3+ models — required when replaying function calls. + #[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")] + thought_signature: Option, + }, + FunctionResponse { + #[serde(rename = "functionResponse")] + function_response: GeminiFunctionResponse, + }, +} + +/// A function call from the model. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiFunctionCall { + pub name: String, + pub args: serde_json::Value, +} + +/// A function response sent back to the model. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiFunctionResponse { + pub name: String, + pub response: serde_json::Value, +} + +/// Content message with an optional role and a list of parts. +#[derive(Serialize, Clone, Debug)] +pub struct GeminiContentMessage { + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, + pub parts: Vec, +} + +/// Main request body for `generateContent` / `streamGenerateContent`. +#[derive(Serialize)] +pub struct GeminiTextRequest { + pub contents: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option>, + #[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")] + pub tool_config: Option, + #[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")] + pub system_instruction: Option, + #[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")] + pub generation_config: Option, +} + +/// Tool definition — function declarations and/or Google Search grounding. +#[derive(Serialize)] +pub struct GeminiTool { + #[serde(rename = "functionDeclarations", skip_serializing_if = "Option::is_none")] + pub function_declarations: Option>, + #[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")] + pub google_search: Option, +} + +/// A single function declaration. +/// +/// `parameters` holds a pre-serialized (and, for the worker, pre-sanitized) JSON Schema. +#[derive(Serialize)] +pub struct GeminiFunctionDeclaration { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + pub parameters: serde_json::Value, +} + +/// Tool configuration controlling when and how functions are called. +#[derive(Serialize)] +pub struct GeminiToolConfig { + #[serde(rename = "functionCallingConfig")] + pub function_calling_config: GeminiFunctionCallingConfig, +} + +/// Function calling mode and optional allow-list. +#[derive(Serialize)] +pub struct GeminiFunctionCallingConfig { + pub mode: String, + #[serde(rename = "allowedFunctionNames", skip_serializing_if = "Option::is_none")] + pub allowed_function_names: Option>, +} + +/// Generation parameters (temperature, token limits, structured output). +#[derive(Serialize)] +pub struct GeminiGenerationConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, + #[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")] + pub response_mime_type: Option, + #[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")] + pub response_schema: Option, +} + +// ============================================================================ +// Image Generation Types +// ============================================================================ + +/// Request body for Imagen / Gemini image generation. +#[derive(Serialize)] +pub struct GeminiImageRequest { + #[serde(skip_serializing_if = "Option::is_none")] + pub contents: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub instances: Option>, +} + +/// Content wrapper used in `generateContent` image requests. +#[derive(Serialize)] +pub struct GeminiImageContent { + pub parts: Vec, +} + +/// Prompt wrapper for Imagen `predict` endpoint. +#[derive(Serialize)] +pub struct GeminiPredictContent { + pub prompt: String, +} + +/// Top-level response from Gemini/Imagen image generation. +#[derive(Deserialize)] +pub struct GeminiImageResponse { + pub candidates: Option>, + pub predictions: Option>, +} + +#[derive(Deserialize)] +pub struct GeminiImageCandidate { + pub content: GeminiImageCandidateContent, +} + +#[derive(Deserialize)] +pub struct GeminiImageCandidateContent { + pub parts: Vec, +} + +#[derive(Deserialize)] +pub struct GeminiImageCandidatePart { + #[serde(rename = "inlineData")] + pub inline_data: Option, +} + +#[derive(Deserialize)] +pub struct GeminiPredictCandidate { + #[serde(rename = "bytesBase64Encoded")] + pub bytes_base64_encoded: String, +} + +// ============================================================================ +// SSE Response Types +// ============================================================================ + +/// One part inside a streaming candidate — text, function call, or thought signature. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEPart { + #[serde(default)] + pub text: Option, + #[serde(rename = "functionCall")] + pub function_call: Option, + /// Thought signature for Gemini 3+ models. + #[serde(rename = "thoughtSignature")] + pub thought_signature: Option, +} + +/// Function call contained in a streaming part. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEFunctionCall { + pub name: String, + pub args: serde_json::Value, +} + +/// Content block inside a streaming candidate. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEContent { + pub parts: Option>, +} + +/// Web source from a Gemini grounding chunk. +#[derive(Deserialize, Debug)] +pub struct GeminiGroundingChunkWeb { + pub uri: String, + #[serde(default)] + pub title: Option, +} + +/// One grounding chunk (search result) from Gemini web search. +#[derive(Deserialize, Debug)] +pub struct GeminiGroundingChunk { + pub web: Option, +} + +/// Grounding metadata attached to a streaming candidate. +#[derive(Deserialize, Debug)] +pub struct GeminiGroundingMetadata { + #[serde(rename = "groundingChunks", default)] + pub grounding_chunks: Vec, + #[serde(rename = "webSearchQueries", default)] + pub web_search_queries: Vec, +} + +/// One candidate inside a streaming Gemini response. +#[derive(Deserialize, Debug)] +pub struct GeminiSSECandidate { + pub content: Option, + #[serde(rename = "finishReason")] + pub finish_reason: Option, + #[serde(rename = "groundingMetadata")] + pub grounding_metadata: Option, +} + +/// Token usage from the `usageMetadata` field of a Gemini SSE event. +#[derive(Deserialize, Debug, Clone)] +pub struct GeminiUsageMetadata { + #[serde(rename = "promptTokenCount", default)] + pub prompt_token_count: Option, + #[serde(rename = "candidatesTokenCount", default)] + pub candidates_token_count: Option, + #[serde(rename = "totalTokenCount", default)] + pub total_token_count: Option, +} + +/// Top-level structure of one Gemini SSE event. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEEvent { + pub candidates: Option>, + #[serde(rename = "usageMetadata")] + pub usage_metadata: Option, +} + +// ============================================================================ +// Parsed Event Result +// ============================================================================ + +/// A single function call extracted from a Gemini SSE event. +#[derive(Debug)] +pub struct GeminiToolCallEvent { + pub name: String, + pub args: serde_json::Value, + pub thought_signature: Option, +} + +impl GeminiToolCallEvent { + /// Convert the thought signature (if present) into an [`ExtraContent`]. + pub fn to_extra_content(&self) -> Option { + self.thought_signature.as_ref().map(|sig| ExtraContent { + google: Some(GoogleExtraContent { thought_signature: Some(sig.clone()) }), + }) + } +} + +/// Structured result of parsing a Gemini response (streaming SSE event or non-streaming body). +#[derive(Debug, Default)] +pub struct GeminiParsedEvent { + pub text: Option, + pub tool_calls: Vec, + pub annotations: Vec, + pub used_websearch: bool, + pub usage: Option, + pub finish_reason: Option, +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +/// Parse a data URL into `(mime_type, base64_data)`. +/// +/// Expected format: `data:;base64,`. +pub fn parse_data_url(url: &str) -> Option<(String, String)> { + let rest = url.strip_prefix("data:")?; + let (header, data) = rest.split_once(',')?; + let media_type = header.strip_suffix(";base64")?; + Some((media_type.to_string(), data.to_string())) +} + +/// Find the function name associated with a `tool_call_id` by scanning prior messages. +pub fn find_gemini_function_name(messages: &[OpenAIMessage], tool_call_id: &str) -> String { + messages + .iter() + .filter_map(|msg| msg.tool_calls.as_ref()) + .flatten() + .find(|tc| tc.id == tool_call_id) + .map(|tc| tc.function.name.clone()) + .unwrap_or_else(|| "unknown_function".to_string()) +} + +/// Convert an [`OpenAIContent`] value to a list of [`GeminiPart`]s. +/// +/// Handles text and `image_url` (data URLs). `S3Object` variants are skipped here; +/// the worker handles them by downloading and injecting inline data beforehand. +pub fn convert_content_to_gemini_parts(content: &OpenAIContent) -> Vec { + match content { + OpenAIContent::Text(text) if !text.is_empty() => { + vec![GeminiPart::Text { text: text.clone() }] + } + OpenAIContent::Text(_) => vec![], + OpenAIContent::Parts(parts) => parts + .iter() + .filter_map(|part| match part { + ContentPart::Text { text } if !text.is_empty() => { + Some(GeminiPart::Text { text: text.clone() }) + } + ContentPart::ImageUrl { image_url } => { + parse_data_url(&image_url.url).map(|(mime_type, data)| { + GeminiPart::InlineData { + inline_data: GeminiInlineData { mime_type, data }, + } + }) + } + // S3Objects are handled by the worker + _ => None, + }) + .collect(), + } +} + +/// Convert OpenAI-format messages to Gemini `contents` and an optional `systemInstruction`. +/// +/// Returns `(contents, system_instruction)`. +/// +/// `S3Object` images in content parts are skipped (the worker pre-converts them). +/// Tool call history is preserved correctly for multi-turn agent conversations. +pub fn openai_messages_to_gemini( + messages: &[OpenAIMessage], +) -> (Vec, Option) { + let mut contents: Vec = Vec::new(); + let mut system_instruction: Option = None; + + for msg in messages { + match msg.role.as_str() { + "system" => { + if let Some(content) = &msg.content { + let parts = convert_content_to_gemini_parts(content); + if !parts.is_empty() { + system_instruction = + Some(GeminiContentMessage { role: None, parts }); + } + } + } + "tool" => { + if let (Some(tool_call_id), Some(content)) = + (&msg.tool_call_id, &msg.content) + { + let func_name = find_gemini_function_name(messages, tool_call_id); + let response_text = match content { + OpenAIContent::Text(text) => text.clone(), + OpenAIContent::Parts(parts) => parts + .iter() + .filter_map(|p| { + if let ContentPart::Text { text } = p { + Some(text.as_str()) + } else { + None + } + }) + .collect::>() + .join(" "), + }; + contents.push(GeminiContentMessage { + role: Some("user".to_string()), + parts: vec![GeminiPart::FunctionResponse { + function_response: GeminiFunctionResponse { + name: func_name, + response: serde_json::json!({ "result": response_text }), + }, + }], + }); + } + } + role => { + let gemini_role = if role == "assistant" { "model" } else { "user" }; + let mut parts: Vec = Vec::new(); + + if let Some(content) = &msg.content { + parts.extend(convert_content_to_gemini_parts(content)); + } + + if let Some(tool_calls) = &msg.tool_calls { + for tc in tool_calls { + let args: serde_json::Value = + serde_json::from_str(&tc.function.arguments).unwrap_or_default(); + let thought_signature = tc + .extra_content + .as_ref() + .and_then(|ec| ec.google.as_ref()) + .and_then(|g| g.thought_signature.clone()); + parts.push(GeminiPart::FunctionCall { + function_call: GeminiFunctionCall { + name: tc.function.name.clone(), + args, + }, + thought_signature, + }); + } + } + + if !parts.is_empty() { + contents.push(GeminiContentMessage { + role: Some(gemini_role.to_string()), + parts, + }); + } + } + } + } + + (contents, system_instruction) +} + +/// Convert OpenAI tool definitions to Gemini format. +/// +/// `tool_params` must be pre-serialized (and, for the worker, pre-sanitized for Google) +/// JSON schema values, one per entry in `tools` in the same order. +pub fn openai_tools_to_gemini( + tools: &[ToolDef], + tool_params: &[serde_json::Value], + has_websearch: bool, +) -> Option> { + let mut gemini_tools: Vec = Vec::new(); + + let declarations: Vec = tools + .iter() + .zip(tool_params.iter()) + .map(|(t, params)| GeminiFunctionDeclaration { + name: t.function.name.clone(), + description: t.function.description.clone(), + parameters: params.clone(), + }) + .collect(); + + if !declarations.is_empty() { + gemini_tools.push(GeminiTool { + function_declarations: Some(declarations), + google_search: None, + }); + } + + if has_websearch { + gemini_tools.push(GeminiTool { + function_declarations: None, + google_search: Some(serde_json::json!({})), + }); + } + + if gemini_tools.is_empty() { + None + } else { + Some(gemini_tools) + } +} + +/// Parse one Gemini SSE data line into a [`GeminiParsedEvent`]. +/// +/// Returns `Ok(None)` for empty data or unrecognised payloads (e.g. `"[DONE]"`). +/// Logs a warning and returns `Ok(None)` on JSON parse errors rather than propagating. +pub fn parse_gemini_sse_event(data: &str) -> Result, Error> { + if data.is_empty() || data == "[DONE]" { + return Ok(None); + } + + let event: GeminiSSEEvent = match serde_json::from_str(data) { + Ok(e) => e, + Err(e) => { + tracing::error!("Failed to parse Gemini SSE event {}: {}", data, e); + return Ok(None); + } + }; + + let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() }; + + let Some(candidates) = event.candidates else { + return Ok(Some(parsed)); + }; + + extract_candidates_into(&candidates, &mut parsed); + + Ok(Some(parsed)) +} + +/// Parse a non-streaming Gemini `generateContent` response body. +pub fn parse_gemini_response(data: &[u8]) -> Result { + let event: GeminiSSEEvent = serde_json::from_slice(data) + .map_err(|e| Error::internal_err(format!("Failed to parse Gemini response: {}", e)))?; + + let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() }; + + if let Some(candidates) = event.candidates { + extract_candidates_into(&candidates, &mut parsed); + } + + Ok(parsed) +} + +// ============================================================================ +// Gemini → OpenAI Format Conversion +// ============================================================================ + +/// Convert a `GeminiParsedEvent` from a non-streaming response to an OpenAI chat completion JSON. +pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> serde_json::Value { + let content = parsed.text.as_deref().unwrap_or_default(); + + let tool_calls: Vec = parsed + .tool_calls + .iter() + .enumerate() + .map(|(i, tc)| { + serde_json::json!({ + "index": i, + "id": format!("call_{}", uuid::Uuid::new_v4().simple()), + "type": "function", + "function": { + "name": tc.name, + "arguments": serde_json::to_string(&tc.args).unwrap_or_default() + } + }) + }) + .collect(); + + let finish_reason = parsed + .finish_reason + .as_deref() + .map(|r| r.to_lowercase()) + .unwrap_or_else(|| "stop".to_string()); + + let usage = parsed.usage.as_ref().map(|u| { + serde_json::json!({ + "prompt_tokens": u.prompt_token_count.unwrap_or(0), + "completion_tokens": u.candidates_token_count.unwrap_or(0), + "total_tokens": u.total_token_count.unwrap_or(0), + }) + }); + + let mut message = serde_json::json!({ + "role": "assistant", + "content": content, + }); + if !tool_calls.is_empty() { + message["tool_calls"] = serde_json::json!(tool_calls); + } + + serde_json::json!({ + "id": format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()), + "object": "chat.completion", + "model": model, + "choices": [{ + "index": 0, + "message": message, + "finish_reason": finish_reason, + }], + "usage": usage, + }) +} + +/// Convert a `GeminiParsedEvent` from a streaming SSE event into OpenAI-format SSE lines. +/// +/// Returns the serialized `"data: {...}\n\n"` lines ready to be written to the response stream. +/// `tool_call_index` is mutated to track the running index across multiple SSE events. +pub fn gemini_event_to_openai_sse_chunks( + parsed: &GeminiParsedEvent, + id: &str, + model: &str, + tool_call_index: &mut usize, +) -> Vec { + let mut chunks = Vec::new(); + + if let Some(text) = &parsed.text { + let chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": model, + "choices": [{ + "index": 0, + "delta": { "content": text }, + "finish_reason": null, + }] + }); + chunks.push(format!("data: {}\n\n", chunk)); + } + + for tc in &parsed.tool_calls { + let args_str = serde_json::to_string(&tc.args).unwrap_or_default(); + let call_id = format!("call_{}", uuid::Uuid::new_v4().simple()); + let chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": model, + "choices": [{ + "index": 0, + "delta": { + "tool_calls": [{ + "index": *tool_call_index, + "id": call_id, + "type": "function", + "function": { + "name": tc.name, + "arguments": args_str, + } + }] + }, + "finish_reason": null, + }] + }); + chunks.push(format!("data: {}\n\n", chunk)); + *tool_call_index += 1; + } + + chunks +} + +/// Recursively remove JSON Schema fields unsupported by the Gemini API. +pub fn sanitize_schema_for_google(value: &mut serde_json::Value) { + const UNSUPPORTED: &[&str] = &[ + "additionalProperties", + "strict", + "$schema", + "default", + "exclusiveMinimum", + "exclusiveMaximum", + "const", + "multipleOf", + ]; + + if let Some(obj) = value.as_object_mut() { + for field in UNSUPPORTED { + obj.remove(*field); + } + for v in obj.values_mut() { + sanitize_schema_for_google(v); + } + } else if let Some(arr) = value.as_array_mut() { + for v in arr.iter_mut() { + sanitize_schema_for_google(v); + } + } +} + +// ============================================================================ +// Internal Helpers +// ============================================================================ + +fn extract_candidates_into(candidates: &[GeminiSSECandidate], parsed: &mut GeminiParsedEvent) { + for candidate in candidates { + if let Some(content) = &candidate.content { + if let Some(parts) = &content.parts { + for part in parts { + if let Some(text) = &part.text { + if !text.is_empty() { + match parsed.text.as_mut() { + Some(existing) => existing.push_str(text), + None => parsed.text = Some(text.clone()), + } + } + } + + if let Some(function_call) = &part.function_call { + parsed.tool_calls.push(GeminiToolCallEvent { + name: function_call.name.clone(), + args: function_call.args.clone(), + thought_signature: part.thought_signature.clone(), + }); + } + } + } + } + + if candidate.finish_reason.is_some() { + parsed.finish_reason = candidate.finish_reason.clone(); + } + + if let Some(grounding) = &candidate.grounding_metadata { + if !grounding.web_search_queries.is_empty() || !grounding.grounding_chunks.is_empty() { + parsed.used_websearch = true; + } + for chunk in &grounding.grounding_chunks { + if let Some(web) = &chunk.web { + parsed.annotations.push(UrlCitation { + start_index: 0, + end_index: 0, + url: web.uri.clone(), + title: web.title.clone(), + }); + } + } + } + } +} diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 669614d64a..2968493ecc 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -72,6 +72,7 @@ pub fn asset_kind_from_parser(parser_kind: windmill_parser::asset_parser::AssetK windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, windmill_parser::asset_parser::AssetKind::Ducklake => AssetKind::Ducklake, windmill_parser::asset_parser::AssetKind::DataTable => AssetKind::DataTable, + windmill_parser::asset_parser::AssetKind::Volume => AssetKind::Volume, } } diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index ccd9b5d2a0..c0555f5093 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -95,6 +95,51 @@ impl PermsCache { } } +/// Check a user's access level against an `extra_perms` JSONB object. +/// +/// Returns `None` if the user has no matching entry (no access). +/// Returns `Some(true)` if the user (or any of their groups) has write access. +/// Returns `Some(false)` if the user (or any of their groups) has read-only access. +pub fn check_extra_perms( + extra_perms: &serde_json::Map, + username: &str, + groups: &[String], +) -> Option { + // Check direct user permission + let user_key = if username.starts_with("u/") { + username.to_string() + } else { + format!("u/{username}") + }; + if let Some(v) = extra_perms.get(&user_key) { + return Some(v.as_bool().unwrap_or(false)); + } + + // Check group permissions — return highest access level found + let mut found = false; + let mut write = false; + for g in groups { + let key = if g.starts_with("g/") { + g.to_string() + } else { + format!("g/{g}") + }; + if let Some(v) = extra_perms.get(&key) { + found = true; + if v.as_bool().unwrap_or(false) { + write = true; + break; + } + } + } + + if found { + Some(write) + } else { + None + } +} + pub fn has_expired(expiration_time: DateTime, take: Option) -> bool { let now = Utc::now(); diff --git a/backend/windmill-common/src/bench.rs b/backend/windmill-common/src/bench.rs index b87872119b..1ed2b9aa56 100644 --- a/backend/windmill-common/src/bench.rs +++ b/backend/windmill-common/src/bench.rs @@ -1,5 +1,5 @@ use crate::{ - worker::{write_file, TMP_DIR}, + worker::{write_file, WINDMILL_DIR}, DB, }; use serde::Serialize; @@ -113,7 +113,8 @@ impl BenchmarkInfo { "Writing benchmark {path}, duration of benchmark: {total_duration}ms and RPS: {}{pool_info}", self.iters as f64 / total_duration as f64 * 1000.0 ); - write_file(TMP_DIR, path, &serde_json::to_string(&self).unwrap()).expect("write profiling"); + write_file(&WINDMILL_DIR, path, &serde_json::to_string(&self).unwrap()) + .expect("write profiling"); Ok(()) } } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 3347127303..b2d961e173 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -44,9 +44,11 @@ pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; +pub const DISABLE_HUB_SETTING: &str = "disable_hub"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize"; +pub const CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING: &str = "critical_alerts_on_token_expiry"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 4de90be5c5..0225448668 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -44,7 +44,7 @@ pub struct EnvRefWrapper { /// /// `Literal` serializes back to a plain JSON string, preserving backwards /// compatibility with existing consumers. -#[derive(Deserialize, Serialize, Clone, Debug)] +#[derive(Deserialize, Serialize, Clone)] #[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))] #[serde(untagged)] pub enum StringOrSecretRef { @@ -53,6 +53,16 @@ pub enum StringOrSecretRef { EnvRef(EnvRefWrapper), } +impl fmt::Debug for StringOrSecretRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Literal(_) => f.write_str("Literal(****)"), + Self::SecretRef(w) => f.debug_tuple("SecretRef").field(w).finish(), + Self::EnvRef(w) => f.debug_tuple("EnvRef").field(w).finish(), + } + } +} + impl StringOrSecretRef { /// Returns the literal string value, or `None` if this is an unresolved ref. pub fn as_literal(&self) -> Option<&str> { @@ -230,6 +240,8 @@ pub struct GlobalSettings { pub no_default_maven: Option, #[serde(skip_serializing_if = "Option::is_none")] pub default_tags_per_workspace: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub disable_hub: Option, // String settings #[serde(skip_serializing_if = "Option::is_none")] @@ -253,25 +265,25 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub instance_python_version: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub pip_index_url: Option, + pub pip_index_url: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub pip_extra_index_url: Option, + pub pip_extra_index_url: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub npm_config_registry: Option, + pub npm_config_registry: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub bunfig_install_scopes: Option, + pub bunfig_install_scopes: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub npmrc: Option, + pub npmrc: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub nuget_config: Option, + pub nuget_config: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub maven_repos: Option, + pub maven_repos: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub ruby_repos: Option, + pub ruby_repos: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub powershell_repo_url: Option, + pub powershell_repo_url: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub powershell_repo_pat: Option, + pub powershell_repo_pat: Option, // Array settings #[serde(skip_serializing_if = "Option::is_none")] diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index bc40e4cd75..583914c573 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -18,7 +18,7 @@ use crate::{ scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, users::username_to_permissioned_as, utils::{StripPath, HTTP_CLIENT}, - worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, TMP_DIR}, + worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, WINDMILL_DIR}, FlowVersionInfo, ScriptHashInfo, Tag, }; @@ -225,7 +225,7 @@ pub async fn get_logs_from_disk( if log_offset > 0 { if let Some(file_index) = log_file_index.clone() { for file_p in &file_index { - if !tokio::fs::metadata(format!("{TMP_DIR}/{file_p}")) + if !tokio::fs::metadata(format!("{}/{file_p}", *WINDMILL_DIR)) .await .is_ok() { @@ -236,7 +236,7 @@ pub async fn get_logs_from_disk( let logs = logs.to_string(); let stream = async_stream::stream! { for file_p in file_index.clone() { - let mut file = tokio::fs::File::open(format!("{TMP_DIR}/{file_p}")).await.map_err(to_anyhow)?; + let mut file = tokio::fs::File::open(format!("{}/{file_p}", *WINDMILL_DIR)).await.map_err(to_anyhow)?; let mut buffer = Vec::new(); file.read_to_end(&mut buffer).await.map_err(to_anyhow)?; yield Ok(bytes::Bytes::from(buffer)) as anyhow::Result; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 643e08fc82..85f5563419 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -29,6 +29,7 @@ use sqlx::{Acquire, Postgres}; pub mod agent_workers; #[cfg(feature = "bedrock")] pub mod ai_bedrock; +pub mod ai_google; pub mod ai_providers; pub mod ai_types; pub mod apps; @@ -193,6 +194,7 @@ lazy_static::lazy_static! { pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false); pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false); + pub static ref CRITICAL_ALERTS_ON_TOKEN_EXPIRY: AtomicBool = AtomicBool::new(false); pub static ref BASE_URL: Arc> = Arc::new(RwLock::new("".to_string())); pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 68f77a2303..cfff28dbcb 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -206,12 +206,12 @@ pub async fn get_full_hub_script_by_path( let version = path_iterator .next() .ok_or_else(|| Error::internal_err(format!("expected hub path to have version number")))?; - let cache_path = format!("{HUB_CACHE_DIR}/{version}"); + let cache_path = format!("{}/{version}", *HUB_CACHE_DIR); let script; if tokio::fs::metadata(&cache_path).await.is_err() { script = get_full_hub_script_by_path_inner(path, http_client, db).await?; if let Err(e) = crate::worker::write_file( - HUB_CACHE_DIR, + &HUB_CACHE_DIR, &version, &serde_json::to_string(&script).map_err(to_anyhow)?, ) { diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 701a0887d0..f4892b67c9 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -6,8 +6,6 @@ * LICENSE-AGPL for a copy of the license. */ -use const_format::concatcp; - use std::{ collections::HashMap, sync::{Arc, RwLock}, @@ -61,7 +59,9 @@ fn create_targets_filter(default_env_filter: LevelFilter) -> Targets { pub const LOGS_SERVICE: &str = "logs/services/"; -pub const TMP_WINDMILL_LOGS_SERVICE: &str = concatcp!("/tmp/windmill/", LOGS_SERVICE); +lazy_static::lazy_static! { + pub static ref TMP_WINDMILL_LOGS_SERVICE: String = format!("{}/{}", *crate::worker::WINDMILL_DIR, LOGS_SERVICE); +} pub fn initialize_tracing( hostname: &str, @@ -108,7 +108,7 @@ pub fn initialize_tracing( use tracing_appender::rolling::{RollingFileAppender, Rotation}; - let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname); + let log_dir = format!("{}/{}/", *TMP_WINDMILL_LOGS_SERVICE, hostname); std::fs::create_dir_all(&log_dir).unwrap(); let file_appender = RollingFileAppender::builder() .rotation(Rotation::MINUTELY) diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index e4c5315cba..10b1a3b408 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -314,14 +314,14 @@ pub async fn create_directory_async(directory_path: &str) { .recursive(true) .create(directory_path) .await - .expect("could not create dir"); + .unwrap_or_else(|e| panic!("could not create dir '{}': {}", directory_path, e)); } pub fn create_directory_sync(directory_path: &str) { SyncDirBuilder::new() .recursive(true) .create(directory_path) - .expect("could not create dir"); + .unwrap_or_else(|e| panic!("could not create dir '{}': {}", directory_path, e)); } #[track_caller] diff --git a/backend/windmill-common/src/webhook.rs b/backend/windmill-common/src/webhook.rs index f1f8508f74..4677084fbe 100644 --- a/backend/windmill-common/src/webhook.rs +++ b/backend/windmill-common/src/webhook.rs @@ -39,29 +39,115 @@ pub enum WebhookPayload { #[serde(tag = "type")] pub enum WebhookMessage { // See https://serde.rs/enum-representations.html#internally-tagged for how this looks in JSON - CreateApp { workspace: String, path: String }, - DeleteApp { workspace: String, path: String }, - UpdateApp { workspace: String, old_path: String, new_path: String }, - CreateFlow { workspace: String, path: String }, - UpdateFlow { workspace: String, old_path: String, new_path: String }, - ArchiveFlow { workspace: String, path: String }, - DeleteFlow { workspace: String, path: String }, - CreateFolder { workspace: String, name: String }, - UpdateFolder { workspace: String, name: String }, - DeleteFolder { workspace: String, name: String }, - DeleteResource { workspace: String, path: String }, - CreateResource { workspace: String, path: String }, - UpdateResource { workspace: String, old_path: String, new_path: String }, - CreateResourceType { name: String }, - DeleteResourceType { name: String }, - UpdateResourceType { name: String }, - CreateScript { workspace: String, path: String, hash: String }, - UpdateScript { workspace: String, path: String, hash: String }, - DeleteScript { workspace: String, hash: String }, - DeleteScriptPath { workspace: String, path: String }, - CreateVariable { workspace: String, path: String }, - UpdateVariable { workspace: String, old_path: String, new_path: String }, - DeleteVariable { workspace: String, path: String }, + CreateApp { + workspace: String, + path: String, + }, + DeleteApp { + workspace: String, + path: String, + }, + UpdateApp { + workspace: String, + old_path: String, + new_path: String, + }, + CreateFlow { + workspace: String, + path: String, + }, + UpdateFlow { + workspace: String, + old_path: String, + new_path: String, + }, + ArchiveFlow { + workspace: String, + path: String, + }, + DeleteFlow { + workspace: String, + path: String, + }, + CreateFolder { + workspace: String, + name: String, + }, + UpdateFolder { + workspace: String, + name: String, + }, + DeleteFolder { + workspace: String, + name: String, + }, + DeleteResource { + workspace: String, + path: String, + }, + CreateResource { + workspace: String, + path: String, + }, + UpdateResource { + workspace: String, + old_path: String, + new_path: String, + }, + CreateResourceType { + name: String, + }, + DeleteResourceType { + name: String, + }, + UpdateResourceType { + name: String, + }, + CreateScript { + workspace: String, + path: String, + hash: String, + }, + UpdateScript { + workspace: String, + path: String, + hash: String, + }, + DeleteScript { + workspace: String, + hash: String, + }, + DeleteScriptPath { + workspace: String, + path: String, + }, + CreateVariable { + workspace: String, + path: String, + }, + UpdateVariable { + workspace: String, + old_path: String, + new_path: String, + }, + DeleteVariable { + workspace: String, + path: String, + }, + TokenExpiringSoon { + workspace: String, + token_prefix: String, + label: String, + owner: String, + expires_at: String, + days_remaining: i64, + }, + TokenExpired { + workspace: String, + token_prefix: String, + label: String, + owner: String, + }, } #[derive(Clone)] @@ -267,6 +353,20 @@ mod tests { new_path: "n".into(), }, WebhookMessage::DeleteVariable { workspace: "w".into(), path: "p".into() }, + WebhookMessage::TokenExpiringSoon { + workspace: "w".into(), + token_prefix: "abc1234567".into(), + label: "my-token".into(), + owner: "user@example.com".into(), + expires_at: "2026-03-10T00:00:00Z".into(), + days_remaining: 7, + }, + WebhookMessage::TokenExpired { + workspace: "w".into(), + token_prefix: "abc1234567".into(), + label: "my-token".into(), + owner: "user@example.com".into(), + }, ]; for msg in &messages { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 1a2d55b896..b3876fbcfa 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -2,7 +2,6 @@ use anyhow::anyhow; use axum::http::HeaderMap; use bytes::Bytes; -use const_format::concatcp; use itertools::Itertools; use regex::Regex; use reqwest_middleware::ClientWithMiddleware; @@ -274,7 +273,9 @@ lazy_static::lazy_static! { pub static ref ROOT_STANDALONE_BUNDLE_DIR: String = format!("{}/.windmill/standalone_bundle", std::env::var("HOME").unwrap_or_else(|_| "/root".to_string())); } -pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); +lazy_static::lazy_static! { + pub static ref ROOT_CACHE_NOMOUNT_DIR: String = format!("{}/cache_nomount/", *WINDMILL_DIR); +} /// Whether native mode is forced by the environment (NATIVE_MODE=true env var or WORKER_GROUP=native). /// This does NOT account for native_mode set in the DB worker group config — for that, read @@ -353,6 +354,45 @@ impl HttpClient { ))) } } + + pub async fn get_bytes(&self, url: &str) -> anyhow::Result { + let base_url = self.base_internal_url.clone(); + let response = self + .client + .get(format!("{}{}", base_url, url)) + .send() + .await + .map_err(|e| anyhow::anyhow!(e))?; + if response.status().is_success() { + Ok(response.bytes().await?) + } else { + Err(anyhow::anyhow!( + "HTTP agent request GET {} failed {}", + url, + response.status() + )) + } + } + + pub async fn put_bytes(&self, url: &str, bytes: Bytes) -> anyhow::Result<()> { + let base_url = self.base_internal_url.clone(); + let response = self + .client + .put(format!("{}{}", base_url, url)) + .body(bytes) + .send() + .await + .map_err(|e| anyhow::anyhow!(e))?; + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "HTTP agent request PUT {} failed {}", + url, + response.status() + )) + } + } } #[derive(Clone)] @@ -490,13 +530,33 @@ pub async fn store_pull_query(wc: &WorkerConfig) { *l = queries; } -pub const TMP_DIR: &str = "/tmp/windmill"; -pub const TMP_LOGS_DIR: &str = concatcp!(TMP_DIR, "/logs"); - -pub const HUB_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "hub"); -pub const HUB_RT_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "hub_rt"); - -pub const ROOT_CACHE_DIR: &str = concatcp!(TMP_DIR, "/cache/"); +lazy_static::lazy_static! { + pub static ref WINDMILL_DIR: String = { + let dir = std::env::var("WINDMILL_DIR") + .unwrap_or_else(|_| { + #[cfg(not(windows))] + { "/tmp/windmill".to_string() } + #[cfg(windows)] + { + let temp = std::env::temp_dir(); + let temp_str = temp.to_string_lossy(); + let normalized = temp_str.trim_end_matches(&['/', '\\'][..]).replace('\\', "/"); + format!("{}/windmill", normalized) + } + }); + if dir.is_empty() { + panic!("WINDMILL_DIR must not be empty"); + } + if dir.ends_with('/') || dir.ends_with('\\') { + panic!("WINDMILL_DIR must not end with a trailing slash, got: {dir}"); + } + dir + }; + pub static ref TMP_LOGS_DIR: String = format!("{}/logs", *WINDMILL_DIR); + pub static ref ROOT_CACHE_DIR: String = format!("{}/cache/", *WINDMILL_DIR); + pub static ref HUB_CACHE_DIR: String = format!("{}hub", *ROOT_CACHE_DIR); + pub static ref HUB_RT_CACHE_DIR: String = format!("{}hub_rt", *ROOT_CACHE_DIR); +} pub fn write_file(dir: &str, path: &str, content: &str) -> error::Result { let path = format!("{}/{}", dir, path); @@ -677,6 +737,7 @@ pub struct PythonAnnotations { pub py311: bool, pub py312: bool, pub py313: bool, + pub sandbox: bool, } #[annotations("//")] @@ -690,6 +751,7 @@ pub struct TypeScriptAnnotations { pub nodejs: bool, pub native: bool, pub nobundling: bool, + pub sandbox: bool, } #[annotations("--")] @@ -2148,4 +2210,62 @@ mod tests { ); assert_ne!(a, b); } + + #[test] + fn test_python_sandbox_annotation() { + let content = "# sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_python_sandbox_annotation_with_other_annotations() { + let content = "# no_cache\n# sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + assert!(annotations.no_cache); + } + + #[test] + fn test_python_no_sandbox_annotation() { + let content = "# no_cache\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(!annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_annotation() { + let content = "// sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_annotation_with_other_annotations() { + let content = "// npm\n// sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + assert!(annotations.npm); + } + + #[test] + fn test_typescript_no_sandbox_annotation() { + let content = "// npm\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(!annotations.sandbox); + } + + #[test] + fn test_python_sandbox_no_space() { + let content = "#sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_no_space() { + let content = "//sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + } } diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.lock b/backend/windmill-duckdb-ffi-internal/Cargo.lock index 07e428c633..559196a3c2 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.lock +++ b/backend/windmill-duckdb-ffi-internal/Cargo.lock @@ -2164,6 +2164,7 @@ version = "0.1.0" dependencies = [ "chrono", "duckdb", + "regex", "rust_decimal", "serde", "serde_json", diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.toml b/backend/windmill-duckdb-ffi-internal/Cargo.toml index 7043b33ee5..7eb6869ab9 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.toml +++ b/backend/windmill-duckdb-ffi-internal/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] chrono = "0.4.41" duckdb = { version = "1.4.4", features = ["bundled"] } +regex = "1" rust_decimal = "1.37.2" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "^1", features = ["preserve_order", "raw_value"] } diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index c5c819e60b..2701319d6e 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -1,12 +1,14 @@ use std::{ collections::HashMap, - ffi::{CStr, CString, c_char, c_uint}, + ffi::{c_char, c_uint, CStr, CString}, ptr::null_mut, + sync::LazyLock, }; -use duckdb::{Row, core::LogicalTypeId, params_from_iter, types::TimeUnit}; -use rust_decimal::{Decimal, prelude::FromPrimitive}; -use serde::Deserialize; +use duckdb::{core::LogicalTypeId, params_from_iter, types::TimeUnit, Row}; +use regex::Regex; +use rust_decimal::{prelude::FromPrimitive, Decimal}; +use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; #[derive(Deserialize, Clone, Debug, PartialEq, Default)] @@ -96,6 +98,218 @@ pub extern "C" fn run_duckdb_ffi( }) } +#[derive(Serialize, Debug)] +struct PrepareQueryColumnInfo { + name: String, + #[serde(rename = "type")] + type_name: String, +} + +#[derive(Serialize, Debug)] +struct PrepareQueryResult { + #[serde(skip_serializing_if = "Option::is_none")] + columns: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +fn is_setup_statement(query: &str) -> bool { + let trimmed = query.trim_start(); + let upper = trimmed.to_uppercase(); + upper.starts_with("ATTACH") + || upper.starts_with("USE") + || upper.starts_with("INSTALL") + || upper.starts_with("LOAD") + || upper.starts_with("SET") + || upper.starts_with("RESET") + || upper.starts_with("CREATE OR REPLACE SECRET") + || upper.starts_with("CREATE SECRET") +} + +/// Returns true if the query is expected to return a result set and can be wrapped with DESCRIBE. +fn is_describable_query(query: &str) -> bool { + let trimmed = query.trim_start(); + let upper = trimmed.to_uppercase(); + upper.starts_with("SELECT") + || upper.starts_with("WITH") + || upper.starts_with("VALUES") + || upper.starts_with("TABLE") + || upper.starts_with("FROM") +} + +static PARAM_RE: LazyLock = LazyLock::new(|| Regex::new(r"\$\d+").expect("invalid regex")); + +fn replace_params_with_null(query: &str) -> String { + PARAM_RE.replace_all(query, "NULL").to_string() +} + +#[unsafe(no_mangle)] +pub extern "C" fn prepare_duckdb_ffi( + query_block_list: *const *const c_char, + query_block_list_count: usize, + token: *const c_char, + base_internal_url: *const c_char, + w_id: *const c_char, +) -> *mut c_char { + let r = match convert_prepare_args( + query_block_list, + query_block_list_count, + token, + base_internal_url, + w_id, + ) + .and_then(|(query_block_list, token, base_internal_url, w_id)| { + prepare_duckdb_internal(query_block_list, token, base_internal_url, w_id) + }) { + Ok(result) => result, + Err(err) => { + let err = serde_json::to_string(&err) + .unwrap_or_else(|_| "Unknown error in duckdb ffi lib".to_string()); + format!("ERROR {}", err) + } + }; + + CString::new(r).map(|s| s.into_raw()).unwrap_or_else(|e| { + println!("Failed to allocate error string in duckdb ffi lib: {:?}", e); + null_mut() + }) +} + +fn setup_duckdb_connection( + conn: &duckdb::Connection, + token: &str, + base_internal_url: &str, + w_id: &str, +) -> Result<(), String> { + let (s3_access_key, s3_secret_key) = token.rsplit_once('.').unwrap_or(("", token)); + let (s3_endpoint_ssl, s3_endpoint) = base_internal_url + .split_once("://") + .unwrap_or(("http", &base_internal_url)); + let s3_endpoint_ssl = s3_endpoint_ssl == "https"; + + conn.execute_batch(&format!( + "INSTALL httpfs; LOAD httpfs; + INSTALL azure; LOAD azure; + CREATE OR REPLACE SECRET s3_secret ( + TYPE s3, + PROVIDER config, + KEY_ID '{s3_access_key}', + SECRET '{s3_secret_key}', + ENDPOINT '{s3_endpoint}/api/w/{w_id}/s3_proxy', + URL_STYLE path, + USE_SSL {s3_endpoint_ssl} + ); + CREATE OR REPLACE SECRET gcs_secret ( + TYPE gcs, + KEY_ID '{s3_access_key}', + SECRET '{s3_secret_key}', + ENDPOINT '{s3_endpoint}/api/w/{w_id}/s3_proxy', + USE_SSL {s3_endpoint_ssl} + ); + ", + )) + .map_err(|e| format!("Error setting up S3 secret: {}", e.to_string())) +} + +fn convert_prepare_args<'a>( + query_block_list: *const *const c_char, + query_block_list_count: usize, + token: *const c_char, + base_internal_url: *const c_char, + w_id: *const c_char, +) -> Result<(Vec<&'a str>, &'a str, &'a str, &'a str), String> { + let query_block_list = unsafe { + std::slice::from_raw_parts(query_block_list, query_block_list_count) + .iter() + .map(|q| { + CStr::from_ptr(*q).to_str().unwrap_or_else(|e| { + println!( + "Invalid query_block string pointer in duckdb ffi: {}", + e.to_string() + ); + "Invalid query_block string pointer in duckdb ffi" + }) + }) + .collect::>() + }; + let token = unsafe { CStr::from_ptr(token) } + .to_str() + .map_err(|e| format!("Invalid token string: {}", e.to_string()))?; + let base_internal_url = unsafe { CStr::from_ptr(base_internal_url) } + .to_str() + .map_err(|e| format!("Invalid base_internal_url string: {}", e.to_string()))?; + let w_id = unsafe { CStr::from_ptr(w_id) } + .to_str() + .map_err(|e| format!("Invalid w_id string: {}", e.to_string()))?; + Ok((query_block_list, token, base_internal_url, w_id)) +} + +fn prepare_duckdb_internal( + query_block_list: Vec<&str>, + token: &str, + base_internal_url: &str, + w_id: &str, +) -> Result { + let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?; + + setup_duckdb_connection(&conn, token, base_internal_url, w_id)?; + + let mut results: Vec = vec![]; + + // IMPORTANT: Setup statements (ATTACH, USE, INSTALL, etc.) are executed but intentionally + // do not produce a PrepareQueryResult entry. The frontend prepends these as connection setup + // before the actual user queries, and mapPrepareResults expects results.length to equal the + // number of user queries (not setup statements). If a new setup-like statement is added to + // the connection flow (e.g. in setup_duckdb_connection or transform_attach_ducklake) without + // also being caught by is_setup_statement, the result count will mismatch and the frontend + // will throw. + for query_block in &query_block_list { + if is_setup_statement(query_block) { + conn.execute_batch(query_block) + .map_err(|e| format!("Error executing setup statement: {}", e.to_string()))?; + continue; + } + + let modified_query = replace_params_with_null(query_block); + // Validate the query parses correctly by preparing it + if let Err(e) = conn.prepare(&modified_query) { + results.push(PrepareQueryResult { columns: None, error: Some(e.to_string()) }); + continue; + } + + // DESCRIBE only works on queries that return result sets (SELECT, WITH, VALUES, TABLE, + // FROM). For non-returning statements (INSERT, UPDATE, DELETE, CREATE, DROP, ALTER, etc.) + // we skip DESCRIBE and assume no columns. + if !is_describable_query(&modified_query) { + results.push(PrepareQueryResult { columns: Some(vec![]), error: None }); + continue; + } + + // Note: We have to use a DESCRIBE statement and cannot simply use the + // methods returned by .prepare() because they panic if the statement was + // not executed at least once (which we specifically do not want to do). + let describe_query = format!("DESCRIBE {}", modified_query); + match conn.prepare(&describe_query).and_then(|mut stmt| { + let rows = stmt.query_map([], |row| { + Ok(PrepareQueryColumnInfo { + name: row.get::<_, String>(0)?, + type_name: row.get::<_, String>(1)?, + }) + })?; + rows.collect::, _>>() + }) { + Ok(columns) => { + results.push(PrepareQueryResult { columns: Some(columns), error: None }); + } + Err(e) => { + results.push(PrepareQueryResult { columns: None, error: Some(e.to_string()) }); + } + } + } + + serde_json::to_string(&results).map_err(|e| e.to_string()) +} + fn convert_args<'a>( query_block_list: *const *const c_char, query_block_list_count: usize, @@ -170,38 +384,7 @@ fn run_duckdb_internal<'a>( ) -> Result<(String, Option>), String> { let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?; - let (s3_access_key, s3_secret_key) = token.split_at(token.rfind('.').unwrap_or(0)); - let s3_secret_key = &s3_secret_key[1..]; - let (s3_endpoint_ssl, s3_endpoint) = base_internal_url - .split_once("://") - .unwrap_or(("http", &base_internal_url)); - let s3_endpoint_ssl = match s3_endpoint_ssl { - "https" => true, - _ => false, - }; - - conn.execute_batch(&format!( - "INSTALL httpfs; LOAD httpfs; - INSTALL azure; LOAD azure; - CREATE OR REPLACE SECRET s3_secret ( - TYPE s3, - PROVIDER config, - KEY_ID '{s3_access_key}', - SECRET '{s3_secret_key}', - ENDPOINT '{s3_endpoint}/api/w/{w_id}/s3_proxy', - URL_STYLE path, - USE_SSL {s3_endpoint_ssl} - ); - CREATE OR REPLACE SECRET gcs_secret ( - TYPE gcs, - KEY_ID '{s3_access_key}', - SECRET '{s3_secret_key}', - ENDPOINT '{s3_endpoint}/api/w/{w_id}/s3_proxy', - USE_SSL {s3_endpoint_ssl} - ); - ", - )) - .map_err(|e| format!("Error setting up S3 secret: {}", e.to_string()))?; + setup_duckdb_connection(&conn, token, base_internal_url, w_id)?; let mut results: Vec>> = vec![]; let mut column_order = None; diff --git a/backend/windmill-jseval/src/lib.rs b/backend/windmill-jseval/src/lib.rs index 89feaf7b3e..7172e8cfc8 100644 --- a/backend/windmill-jseval/src/lib.rs +++ b/backend/windmill-jseval/src/lib.rs @@ -152,6 +152,21 @@ pub fn try_exact_property_access( None } +/// JS runtime properties (not methods) that cannot be resolved by PostgreSQL's +/// #> JSON path operator. Function calls like .map(...) already don't match the +/// RE_FULL regex due to parentheses, so only property accesses need listing here. +const JS_ONLY_PROPERTIES: &[&str] = &["length"]; + +fn ends_with_js_only_property(rest: Option<&str>) -> bool { + match rest { + None => false, + Some(rest) => { + let last_segment = rest.rsplit('.').next().unwrap_or(""); + JS_ONLY_PROPERTIES.contains(&last_segment) + } + } +} + pub async fn handle_full_regex( expr: &str, authed_client: &AuthedClient, @@ -162,6 +177,13 @@ pub async fn handle_full_regex( let obj_key = captures.get(2).unwrap().as_str(); let idx_o = captures.get(3).map(|y| y.as_str()); let rest = captures.get(4).map(|y| y.as_str()); + + // Skip the SQL fast path when the expression accesses a JS runtime + // property (e.g. .length) that the PostgreSQL #> operator can't resolve. + if ends_with_js_only_property(rest) { + return None; + } + let query = if let Some(idx) = idx_o { match rest { Some(rest) => Some(format!("{}{}", idx, rest)), diff --git a/backend/windmill-mcp/src/common/mod.rs b/backend/windmill-mcp/src/common/mod.rs index d67c1718cc..60ad18a85c 100644 --- a/backend/windmill-mcp/src/common/mod.rs +++ b/backend/windmill-mcp/src/common/mod.rs @@ -11,6 +11,8 @@ pub mod types; pub use schema::convert_schema_to_schema_type; pub use scope::{is_resource_allowed, parse_mcp_scopes, McpScopeConfig}; pub use transform::{ - apply_key_transformation, reverse_transform, reverse_transform_key, transform_path, + apply_key_transformation, extract_hub_version_id_from_hashed, + extract_path_prefix_from_hashed, parse_tool_prefix, reverse_transform, reverse_transform_key, + transform_hub_path, transform_path, }; pub use types::*; diff --git a/backend/windmill-mcp/src/common/transform.rs b/backend/windmill-mcp/src/common/transform.rs index 9a6fa317b1..0a12c85892 100644 --- a/backend/windmill-mcp/src/common/transform.rs +++ b/backend/windmill-mcp/src/common/transform.rs @@ -4,9 +4,15 @@ //! to make them compatible with MCP tool naming requirements. use super::types::SchemaType; +use windmill_common::utils::calculate_hash; -/// MCP clients do not allow names longer than 60 characters -const MAX_PATH_LENGTH: usize = 60; +/// Max tool name length. The MCP spec allows 64 chars, but some clients +/// (e.g. Cursor) prepend the server name to the tool name, so we use 40 +/// to leave room for that prefix. +const MAX_PATH_LENGTH: usize = 40; + +/// Length of the SHA256 hash suffix used for hashed names +const HASH_LEN: usize = 16; /// Transform the path for workspace scripts/flows /// @@ -14,19 +20,133 @@ const MAX_PATH_LENGTH: usize = 60; /// path with the type prefix. This is used when listing, because we can't /// have names with slashes. Because we replace slashes with underscores, /// we also need to escape underscores. +/// +/// For short names (≤40 chars): `s-{escaped_path}` or `f-{escaped_path}` +/// For long names (>40 chars): `S-{escaped[:22]}{sha256[:16]}` or `F-{escaped[:22]}{sha256[:16]}` +/// +/// The uppercase prefix signals that the name is hashed. pub fn transform_path(path: &str, type_str: &str) -> String { let escaped_path = path.replace('_', "__").replace('/', "_"); - // first letter of type_str is used as prefix, only one letter to avoid reaching 60 char name limit - let transformed_path = format!("{}-{}", &type_str[..1], escaped_path); - if transformed_path.len() > MAX_PATH_LENGTH { - let suffix = "_TRUNC"; - return format!( - "{}{}", - &transformed_path[..MAX_PATH_LENGTH - suffix.len()], - suffix - ); + let prefix_char = &type_str[..1]; + let short_name = format!("{}-{}", prefix_char, escaped_path); + + if short_name.len() <= MAX_PATH_LENGTH { + return short_name; } - transformed_path + + let upper_prefix = prefix_char.to_uppercase(); + // Layout: "{Upper}-" (2 chars) + prefix_body (22 chars) + hash (16 chars) = 40 + let prefix_body_len = MAX_PATH_LENGTH - 2 - HASH_LEN; + let hash = calculate_hash(&short_name); + let hash_suffix = &hash[..HASH_LEN]; + let truncated = truncate_to_char_boundary(&escaped_path, prefix_body_len); + format!("{}-{}{}", upper_prefix, truncated, hash_suffix) +} + +/// Transform the path for hub scripts +/// +/// For short names (≤40 chars): `hs-{id}-{summary}` +/// For long names (>40 chars): `Hs-{id}-{summary[:N]}{sha256[:16]}` +pub fn transform_hub_path(version_id: u64, summary: &str) -> String { + let escaped_summary = summary.replace(' ', "_"); + let short_name = format!("hs-{}-{}", version_id, escaped_summary); + + if short_name.len() <= MAX_PATH_LENGTH { + return short_name; + } + + let hash = calculate_hash(&short_name); + let hash_suffix = &hash[..HASH_LEN]; + // "Hs-{id}-" prefix, then fill remaining with summary + hash + let fixed_prefix = format!("Hs-{}-", version_id); + let available = MAX_PATH_LENGTH - fixed_prefix.len() - HASH_LEN; + let truncated_summary = truncate_to_char_boundary(&escaped_summary, available); + format!("{}{}{}", fixed_prefix, truncated_summary, hash_suffix) +} + +/// Parse the prefix of any tool name (both short and hashed). +/// Returns `(type_str, is_hub, is_hashed)`. +/// Hashed names use an uppercase first character as the signal. +pub fn parse_tool_prefix(name: &str) -> Result<(&str, bool, bool), String> { + let is_hashed = name.chars().next().map(|c| c.is_ascii_uppercase()).unwrap_or(false); + let lower = name.to_ascii_lowercase(); + let (type_str, is_hub) = if lower.starts_with("hs-") { + ("script", true) + } else if lower.starts_with("s-") { + ("script", false) + } else if lower.starts_with("f-") { + ("flow", false) + } else { + return Err(format!("Invalid tool name prefix: {}", name)); + }; + Ok((type_str, is_hub, is_hashed)) +} + +/// Extract the hub version_id from a hashed hub script name like `Hs-{id}-...` +pub fn extract_hub_version_id_from_hashed(name: &str) -> Result { + let rest = name + .strip_prefix("Hs-") + .ok_or_else(|| format!("Not a hashed hub name: {}", name))?; + let id = rest + .split('-') + .next() + .ok_or_else(|| format!("No version_id in hashed hub name: {}", name))?; + if id.is_empty() { + return Err(format!("Empty version_id in hashed hub name: {}", name)); + } + Ok(id.to_string()) +} + +/// Extract a safe original-path prefix from a hashed tool name. +/// +/// Given `S-u_admin_engineering__te`, extracts the escaped prefix between +/// the type prefix (`S-`, `F-`, or `Hs-`) and the hash, un-escapes it, and +/// returns a prefix suitable for `WHERE path LIKE '{prefix}%'`. +/// +/// Returns `None` if the name is too short or has an unrecognized prefix. +pub fn extract_path_prefix_from_hashed(name: &str) -> Option { + let prefix_len = if name.starts_with("Hs-") { + 3 + } else if name.starts_with("S-") || name.starts_with("F-") { + 2 + } else { + return None; + }; + if name.len() <= prefix_len + HASH_LEN { + return None; + } + let escaped_prefix = &name[prefix_len..name.len() - HASH_LEN]; + if escaped_prefix.is_empty() { + return None; + } + + // Strip trailing underscores — they may be half of a `__` pair split by truncation + let trimmed = escaped_prefix.trim_end_matches('_'); + if trimmed.is_empty() { + return None; + } + + Some(unescape_path(trimmed)) +} + +/// Un-escape a mangled path segment: `__` → `_`, standalone `_` → `/`. +fn unescape_path(s: &str) -> String { + const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@"; + s.replace("__", TEMP_PLACEHOLDER) + .replace('_', "/") + .replace(TEMP_PLACEHOLDER, "_") +} + +/// Truncate a string to at most `max_len` bytes, ensuring we don't split a UTF-8 character. +fn truncate_to_char_boundary(s: &str, max_len: usize) -> &str { + if s.len() <= max_len { + return s; + } + let mut end = max_len; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + &s[..end] } /// Reverse the transformation of a path @@ -38,25 +158,22 @@ pub fn transform_path(path: &str, type_str: &str) -> String { /// This is used in call_tool to get the original path, and the type of the item. /// /// Returns: (type, original_path, is_hub) +/// +/// Note: This only works for non-hashed (short) names. Hashed names must be +/// resolved via `parse_tool_prefix` + path enumeration in the runner. pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool), String> { - let is_hub = transformed_path.starts_with("h"); - let transformed_path = if is_hub { - transformed_path[1..].to_string() - } else { - transformed_path.to_string() - }; - let type_str = if transformed_path.starts_with("s-") { - "script" - } else if transformed_path.starts_with("f-") { - "flow" - } else { - return Err(format!( - "Invalid prefix in transformed path: {}", - transformed_path - )); - }; + let (type_str, is_hub, is_hashed) = parse_tool_prefix(transformed_path)?; - let mangled_path = &transformed_path[2..]; + if is_hashed { + return Err( + "Hashed names cannot be reverse-transformed directly; use path enumeration instead" + .to_string(), + ); + } + + // Strip the prefix: "hs-" (3 chars) for hub, "s-"/"f-" (2 chars) for others + let prefix_len = if is_hub { 3 } else { 2 }; + let mangled_path = &transformed_path[prefix_len..]; let original_path = if is_hub { let parts = mangled_path.split("-").collect::>(); @@ -65,11 +182,7 @@ pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool), } parts[0].to_string() } else { - const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@"; - mangled_path - .replace("__", TEMP_PLACEHOLDER) - .replace('_', "/") - .replace(TEMP_PLACEHOLDER, "_") + unescape_path(mangled_path) }; Ok((type_str, original_path, is_hub)) @@ -97,16 +210,13 @@ pub fn reverse_transform_key(transformed_key: &str, schema_obj: &Option s, None => { - // No schema available, return the key as is (best guess) return transformed_key.to_string(); } }; for original_key_in_schema in schema_obj.properties.keys() { - // Apply the SAME forward transformation to the schema key let potential_transformed_key = apply_key_transformation(original_key_in_schema); - // If it matches the key we received, we found the likely original if potential_transformed_key == transformed_key { return original_key_in_schema.clone(); } @@ -120,7 +230,7 @@ mod tests { use super::*; #[test] - fn test_transform_path() { + fn test_transform_path_short() { assert_eq!( transform_path("u/admin/script", "script"), "s-u_admin_script" @@ -130,7 +240,108 @@ mod tests { } #[test] - fn test_reverse_transform() { + fn test_transform_path_long_is_hashed() { + let long_path = "u/engineering/team/automation/very_long_script_name_that_exceeds_limit"; + let result = transform_path(long_path, "script"); + assert_eq!(result.len(), MAX_PATH_LENGTH); + assert!(result.starts_with("S-")); + let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap(); + assert!(is_hashed); + } + + #[test] + fn test_transform_path_long_flow_is_hashed() { + let long_path = "f/engineering/team/automation/very_long_flow_name_that_exceeds_limit"; + let result = transform_path(long_path, "flow"); + assert_eq!(result.len(), MAX_PATH_LENGTH); + assert!(result.starts_with("F-")); + let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap(); + assert!(is_hashed); + } + + #[test] + fn test_transform_path_hashing_is_deterministic() { + let path = "u/engineering/team/automation/very_long_script_name_that_exceeds_limit"; + let a = transform_path(path, "script"); + let b = transform_path(path, "script"); + assert_eq!(a, b); + } + + #[test] + fn test_transform_path_different_long_paths_differ() { + let a = transform_path( + "u/engineering/team/automation/very_long_script_name_that_exceeds_limit_a", + "script", + ); + let b = transform_path( + "u/engineering/team/automation/very_long_script_name_that_exceeds_limit_b", + "script", + ); + assert_ne!(a, b); + } + + #[test] + fn test_transform_hub_path_short() { + let result = transform_hub_path(12345, "Send Slack Message"); + assert_eq!(result, "hs-12345-Send_Slack_Message"); + let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap(); + assert!(!is_hashed); + } + + #[test] + fn test_transform_hub_path_long_is_hashed() { + let result = transform_hub_path( + 12345, + "Send Slack Message To Channel With Very Long Description That Exceeds Limit", + ); + assert_eq!(result.len(), MAX_PATH_LENGTH); + assert!(result.starts_with("Hs-12345-")); + let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap(); + assert!(is_hashed); + } + + #[test] + fn test_extract_hub_version_id_from_hashed() { + let name = "Hs-12345-Send_Slack_Message_To_Ch9e8d7c6b5a4f3e2d"; + let id = extract_hub_version_id_from_hashed(name).unwrap(); + assert_eq!(id, "12345"); + } + + #[test] + fn test_parse_tool_prefix() { + let (t, hub, hashed) = parse_tool_prefix("S-something").unwrap(); + assert_eq!(t, "script"); + assert!(!hub); + assert!(hashed); + + let (t, hub, hashed) = parse_tool_prefix("F-something").unwrap(); + assert_eq!(t, "flow"); + assert!(!hub); + assert!(hashed); + + let (t, hub, hashed) = parse_tool_prefix("Hs-12345-something").unwrap(); + assert_eq!(t, "script"); + assert!(hub); + assert!(hashed); + + let (t, hub, hashed) = parse_tool_prefix("s-u_admin_script").unwrap(); + assert_eq!(t, "script"); + assert!(!hub); + assert!(!hashed); + + let (t, hub, hashed) = parse_tool_prefix("f-f_folder_flow").unwrap(); + assert_eq!(t, "flow"); + assert!(!hub); + assert!(!hashed); + + let (t, hub, hashed) = parse_tool_prefix("hs-12345-summary").unwrap(); + assert_eq!(t, "script"); + assert!(hub); + assert!(!hashed); + } + + #[test] + fn test_reverse_transform_short_names() { let (type_str, path, is_hub) = reverse_transform("s-u_admin_script").unwrap(); assert_eq!(type_str, "script"); assert_eq!(path, "u/admin/script"); @@ -142,6 +353,70 @@ mod tests { assert!(!is_hub); } + #[test] + fn test_extract_path_prefix_from_hashed() { + // Generate a real hashed name and verify prefix extraction + let long_path = "u/admin/engineering/team/automation/very_long_script"; + let hashed = transform_path(long_path, "script"); + let (_, _, is_hashed) = parse_tool_prefix(&hashed).unwrap(); + assert!(is_hashed); + + let prefix = extract_path_prefix_from_hashed(&hashed).unwrap(); + // The original path should start with the extracted prefix + assert!( + long_path.starts_with(&prefix), + "path '{}' should start with prefix '{}'", + long_path, + prefix + ); + } + + #[test] + fn test_extract_path_prefix_underscore_in_path() { + let long_path = "u/admin/my_team/automation/very_long_script_name_here"; + let hashed = transform_path(long_path, "script"); + let prefix = extract_path_prefix_from_hashed(&hashed).unwrap(); + assert!( + long_path.starts_with(&prefix), + "path '{}' should start with prefix '{}'", + long_path, + prefix + ); + } + + #[test] + fn test_extract_path_prefix_rejects_invalid_prefix() { + assert!(extract_path_prefix_from_hashed("x-something").is_none()); + assert!(extract_path_prefix_from_hashed("").is_none()); + assert!(extract_path_prefix_from_hashed("S-").is_none()); + } + + #[test] + fn test_extract_path_prefix_handles_hs_prefix() { + // Hs- is 3 chars, not 2 — ensure the prefix is stripped correctly + let hashed = transform_hub_path(12345, "a]very long hub script summary that exceeds the limit"); + let (_, is_hub, is_hashed) = parse_tool_prefix(&hashed).unwrap(); + assert!(is_hub); + assert!(is_hashed); + + let prefix = extract_path_prefix_from_hashed(&hashed); + // Should not start with 's' (leftover from Hs- if sliced at index 2) + if let Some(ref p) = prefix { + assert!( + !p.starts_with('s'), + "prefix '{}' should not start with 's' from mis-sliced Hs- prefix", + p + ); + } + } + + #[test] + fn test_reverse_transform_rejects_hashed_names() { + assert!(reverse_transform("S-something").is_err()); + assert!(reverse_transform("F-something").is_err()); + assert!(reverse_transform("Hs-12345-something").is_err()); + } + #[test] fn test_apply_key_transformation() { assert_eq!(apply_key_transformation("my key"), "my_key"); diff --git a/backend/windmill-mcp/src/common/types.rs b/backend/windmill-mcp/src/common/types.rs index 43fe223ded..6161ca7963 100644 --- a/backend/windmill-mcp/src/common/types.rs +++ b/backend/windmill-mcp/src/common/types.rs @@ -92,8 +92,10 @@ pub struct ItemSchema { /// Trait for objects that can be converted to MCP tools pub trait ToolableItem { - /// Get the path or identifier for this item (transformed for MCP compatibility) - fn get_path_or_id(&self) -> String; + /// Get the MCP-compatible tool name (path transformed with escaping/hashing) + fn get_transformed_path(&self) -> String; + /// Get the original full path of this item (for display in tool title) + fn get_full_path(&self) -> &str; /// Get the summary/title of this item fn get_summary(&self) -> &str; /// Get the description of this item diff --git a/backend/windmill-mcp/src/lib.rs b/backend/windmill-mcp/src/lib.rs index a75b545154..7df6ee9f39 100644 --- a/backend/windmill-mcp/src/lib.rs +++ b/backend/windmill-mcp/src/lib.rs @@ -14,9 +14,9 @@ pub mod client; // Re-export common types at crate root for convenience pub use common::{ - convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_path, FlowInfo, - HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo, ResourceType, SchemaType, - ScriptInfo, ToolableItem, WorkspaceId, + convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_hub_path, + transform_path, FlowInfo, HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo, + ResourceType, SchemaType, ScriptInfo, ToolableItem, WorkspaceId, }; // Re-export client types at crate root for backward compatibility diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs index fd01b2b26c..0b942353b3 100644 --- a/backend/windmill-mcp/src/server/backend.rs +++ b/backend/windmill-mcp/src/server/backend.rs @@ -55,20 +55,22 @@ pub trait McpBackend: Send + Sync + Clone + 'static { // Listing Operations // ───────────────────────────────────────────────────────────────── - /// List scripts, optionally filtered to favorites only + /// List scripts, optionally filtered to favorites only and/or by path prefix async fn list_scripts( &self, auth: &Self::Auth, workspace_id: &str, favorites_only: bool, + path_prefix: Option<&str>, ) -> BackendResult>; - /// List flows, optionally filtered to favorites only + /// List flows, optionally filtered to favorites only and/or by path prefix async fn list_flows( &self, auth: &Self::Auth, workspace_id: &str, favorites_only: bool, + path_prefix: Option<&str>, ) -> BackendResult>; /// List resource types in workspace diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs index dd65c96334..be8764fa2e 100644 --- a/backend/windmill-mcp/src/server/runner.rs +++ b/backend/windmill-mcp/src/server/runner.rs @@ -5,7 +5,10 @@ use crate::common::schema::extract_resource_types_from_schema; use crate::common::scope::parse_mcp_scopes; -use crate::common::transform::{reverse_transform, reverse_transform_key}; +use crate::common::transform::{ + extract_hub_version_id_from_hashed, extract_path_prefix_from_hashed, parse_tool_prefix, + reverse_transform, reverse_transform_key, +}; use crate::common::types::{ResourceInfo, ToolableItem, WorkspaceId}; use crate::server::backend::{McpAuth, McpBackend}; use crate::server::endpoints::endpoint_tool_to_mcp_tool; @@ -81,6 +84,13 @@ impl Runner { } } +fn find_matching_path(candidates: Vec, request_name: &str) -> Option { + candidates + .into_iter() + .find(|item| item.get_transformed_path() == request_name) + .map(|item| item.get_full_path().to_string()) +} + impl ServerHandler for Runner { fn get_info(&self) -> ServerInfo { ServerInfo { @@ -120,9 +130,9 @@ impl ServerHandler for Runner { // Fetch all items concurrently let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!( self.backend - .list_scripts(&auth, &workspace_id, favorites_only), + .list_scripts(&auth, &workspace_id, favorites_only, None), self.backend - .list_flows(&auth, &workspace_id, favorites_only), + .list_flows(&auth, &workspace_id, favorites_only, None), self.backend.list_resource_types(&auth, &workspace_id), async { if let Some(ref apps) = scope_config.hub_apps { @@ -231,17 +241,6 @@ impl ServerHandler for Runner { let scope_config = parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?; - // Handle truncated tool names - if request.name.ends_with("_TRUNC") { - return Ok(CallToolResult::error(vec![rmcp::model::Annotated::new( - rmcp::model::RawContent::Text(rmcp::model::RawTextContent { - text: "Tool path is too long. Consider shortening it to make it compatible with MCP.".to_string(), - meta: None, - }), - None, - )])); - } - let args = request.arguments.map(Value::Object).unwrap_or(Value::Null); // Check if this is an endpoint tool @@ -274,10 +273,58 @@ impl ServerHandler for Runner { } } - // Not an endpoint tool - parse as script/flow - let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| { - ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None) - })?; + // Resolve the tool name to (type, path, is_hub) + let (type_str, is_hub, is_hashed) = + parse_tool_prefix(&request.name).map_err(|e| { + ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None) + })?; + + let (tool_type, path, is_hub) = if !is_hashed { + reverse_transform(&request.name).map_err(|e| { + ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None) + })? + } else if is_hub { + let version_id = + extract_hub_version_id_from_hashed(&request.name).map_err(|e| { + ErrorData::internal_error( + format!("Failed to extract hub version_id: {}", e), + None, + ) + })?; + (type_str, version_id, true) + } else { + let path_prefix = extract_path_prefix_from_hashed(&request.name); + let favorites_only = scope_config.favorites; + let matched_path = if type_str == "script" { + find_matching_path( + self.backend + .list_scripts(&auth, &workspace_id, favorites_only, path_prefix.as_deref()) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))?, + &request.name, + ) + } else { + find_matching_path( + self.backend + .list_flows(&auth, &workspace_id, favorites_only, path_prefix.as_deref()) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))?, + &request.name, + ) + }; + + let matched_path = matched_path.ok_or_else(|| { + ErrorData::internal_error( + format!( + "No {} found matching hashed tool name '{}'", + type_str, request.name + ), + None, + ) + })?; + + (type_str, matched_path, false) + }; // Validate script/flow scope if !is_hub && scope_config.granular { diff --git a/backend/windmill-mcp/src/server/tools.rs b/backend/windmill-mcp/src/server/tools.rs index a9262cdd6f..50ed03426b 100644 --- a/backend/windmill-mcp/src/server/tools.rs +++ b/backend/windmill-mcp/src/server/tools.rs @@ -9,7 +9,7 @@ use std::collections::HashMap; use std::sync::Arc; use crate::common::schema::{convert_schema_to_schema_type, make_schema_compatible}; -use crate::common::transform::transform_path; +use crate::common::transform::{transform_hub_path, transform_path}; use crate::common::types::{ FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem, }; @@ -17,10 +17,14 @@ use crate::server::backend::McpBackend; /// Implementation of ToolableItem for ScriptInfo impl ToolableItem for ScriptInfo { - fn get_path_or_id(&self) -> String { + fn get_transformed_path(&self) -> String { transform_path(&self.path, "script") } + fn get_full_path(&self) -> &str { + &self.path + } + fn get_summary(&self) -> &str { self.summary.as_deref().unwrap_or("No summary") } @@ -48,10 +52,14 @@ impl ToolableItem for ScriptInfo { /// Implementation of ToolableItem for FlowInfo impl ToolableItem for FlowInfo { - fn get_path_or_id(&self) -> String { + fn get_transformed_path(&self) -> String { transform_path(&self.path, "flow") } + fn get_full_path(&self) -> &str { + &self.path + } + fn get_summary(&self) -> &str { self.summary.as_deref().unwrap_or("No summary") } @@ -79,10 +87,13 @@ impl ToolableItem for FlowInfo { /// Implementation of ToolableItem for HubScriptInfo impl ToolableItem for HubScriptInfo { - fn get_path_or_id(&self) -> String { - let id = self.version_id; + fn get_transformed_path(&self) -> String { let summary = self.summary.as_deref().unwrap_or("No summary"); - format!("hs-{}-{}", id, summary.replace(" ", "_")) + transform_hub_path(self.version_id, summary) + } + + fn get_full_path(&self) -> &str { + self.summary.as_deref().unwrap_or("No summary") } fn get_summary(&self) -> &str { @@ -124,7 +135,7 @@ pub fn create_tool_from_item( resources_types: &[ResourceType], ) -> Tool { let is_hub = item.is_hub(); - let path = item.get_path_or_id(); + let path = item.get_transformed_path(); let item_type = item.item_type(); let description = format!( "This is a {} named `{}` with the following description: `{}`.{}", @@ -170,15 +181,24 @@ pub fn create_tool_from_item( } }; + let title = { + let summary = item.get_summary(); + if summary == "No summary" { + item.get_full_path().to_string() + } else { + summary.to_string() + } + }; + Tool { name: Cow::Owned(path), description: Some(Cow::Owned(description)), input_schema: Arc::new(input_schema_map), - title: Some(item.get_summary().to_string()), + title: Some(title.clone()), output_schema: None, icons: None, annotations: Some(ToolAnnotations { - title: Some(item.get_summary().to_string()), + title: Some(title), read_only_hint: Some(false), // Can modify environment destructive_hint: Some(true), // Can potentially be destructive idempotent_hint: Some(false), // Are not guaranteed to be idempotent diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs index 4f27dff2e2..ed22480a9a 100644 --- a/backend/windmill-native-triggers/src/lib.rs +++ b/backend/windmill-native-triggers/src/lib.rs @@ -726,7 +726,7 @@ pub async fn get_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>> ) -> Result> { let token = sqlx::query_scalar!( r#" - SELECT token + SELECT token as "token!" FROM token WHERE token LIKE concat($1::text, '%') LIMIT 1 diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index e0c499c355..26d807dac3 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -94,7 +94,7 @@ pub struct OAuthConfig { } /// OAuth client credentials -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct OAuthClient { #[serde(default = "empty_string")] pub id: String, @@ -110,6 +110,21 @@ pub struct OAuthClient { pub grant_types: Vec, } +impl std::fmt::Debug for OAuthClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OAuthClient") + .field("id", &self.id) + .field("secret", &"***") + .field("display_name", &self.display_name) + .field("allowed_domains", &self.allowed_domains) + .field("connect_config", &self.connect_config) + .field("login_config", &self.login_config) + .field("tenant", &self.tenant) + .field("grant_types", &self.grant_types) + .finish() + } +} + fn empty_string() -> String { "".to_string() } @@ -608,7 +623,18 @@ pub async fn refresh_token<'c>( .await?; let account = windmill_common::utils::not_found_if_none(account, "Account", &id.to_string())?; - refresh_token_for_account(tx, path, w_id, id, db, account, oauth_clients, http_client, connect_configs_json).await + refresh_token_for_account( + tx, + path, + w_id, + id, + db, + account, + oauth_clients, + http_client, + connect_configs_json, + ) + .await } /// Refresh an OAuth token given pre-fetched account info (no additional SELECT). diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs index a523106cbc..e81925e7bb 100644 --- a/backend/windmill-runtime-nativets/src/lib.rs +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -45,7 +45,7 @@ use uuid::Uuid; use windmill_common::error::Error; use windmill_common::result_stream::append_result_stream_db; -use windmill_common::worker::{write_file, Connection, TMP_DIR}; +use windmill_common::worker::{write_file, Connection, WINDMILL_DIR}; // ── Permission container ───────────────────────────────────────────── @@ -151,7 +151,9 @@ static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH pub(crate) const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js"); -const ERROR_DIR: &str = const_format::concatcp!(TMP_DIR, "/native_errors"); +lazy_static::lazy_static! { + static ref ERROR_DIR: String = format!("{}/native_errors", *WINDMILL_DIR); +} lazy_static! { static ref RE_PROXY: Regex = @@ -263,14 +265,14 @@ fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> { } fn write_error_expr(expr: &str, uuid: &Uuid) { - if let Err(e) = std::fs::create_dir_all(ERROR_DIR) { - tracing::error!("failed to create error dir {ERROR_DIR}: {e}"); + if let Err(e) = std::fs::create_dir_all(&*ERROR_DIR) { + tracing::error!("failed to create error dir {}: {e}", *ERROR_DIR); return; } - let dir_entries = match std::fs::read_dir(ERROR_DIR) { + let dir_entries = match std::fs::read_dir(&*ERROR_DIR) { Ok(entries) => entries.count(), Err(_) => { - tracing::error!("failed to read error dir {ERROR_DIR}"); + tracing::error!("failed to read error dir {}", *ERROR_DIR); return; } }; @@ -279,15 +281,16 @@ fn write_error_expr(expr: &str, uuid: &Uuid) { tracing::info!("native error for job {uuid}: {expr}"); } if dir_entries >= 100 { - tracing::info!("Too many error files in {ERROR_DIR}, skipping write"); + tracing::info!("Too many error files in {}, skipping write", *ERROR_DIR); return; } let path = format!("/{uuid}.js"); tracing::info!( - "nativets job {uuid} failed, writing error expr to {ERROR_DIR}/{path} for debugging: {path}" + "nativets job {uuid} failed, writing error expr to {}/{path} for debugging: {path}", + *ERROR_DIR ); - if let Err(e) = write_file(ERROR_DIR, &path, expr) { + if let Err(e) = write_file(&ERROR_DIR, &path, expr) { tracing::error!("failed to write error expr to file {path}: {e}"); } } diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index bcac53e7d2..6b414ca10f 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -47,7 +47,7 @@ use windmill_common::{ StripPath, }, variables, - worker::{CLOUD_HOSTED, TMP_DIR}, + worker::{CLOUD_HOSTED, WINDMILL_DIR}, PgDatabase, }; @@ -1752,7 +1752,7 @@ async fn write_ssh_file( var_path: &str, ) -> std::result::Result { let id_file_name = format!(".ssh_id_priv_{}", Uuid::new_v4()); - let loc = std::path::Path::new(TMP_DIR) + let loc = std::path::Path::new(&*WINDMILL_DIR) .join("ssh_ids") .join(id_file_name); diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index adbeb83ee7..5d5c80b470 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -66,6 +66,8 @@ fn next_worker_name() -> String { .unwrap_or(s) }) .unwrap_or("no thread name"); + // Replace colons because they are illegal in Windows directory names + let thread_name = thread_name.replace(':', "_"); format!("{id}/worker-{thread_name}") } @@ -149,6 +151,7 @@ pub struct RunJob { pub args: serde_json::Map, pub scheduled_for_o: Option>, pub email: String, + pub job_id: Option, } impl From for RunJob { @@ -158,6 +161,7 @@ impl From for RunJob { args: Default::default(), scheduled_for_o: None, email: "test@windmill.dev".to_string(), + job_id: None, } } } @@ -181,8 +185,13 @@ impl RunJob { self } + pub fn job_id(mut self, id: Uuid) -> Self { + self.job_id = Some(id); + self + } + pub async fn push(self, db: &Pool) -> Uuid { - let RunJob { payload, args, scheduled_for_o, email } = self; + let RunJob { payload, args, scheduled_for_o, email, job_id } = self; let mut hm_args = std::collections::HashMap::new(); for (k, v) in args { hm_args.insert(k, windmill_common::worker::to_raw_value(&v)); @@ -204,7 +213,7 @@ impl RunJob { /* parent_job */ None, /* root job */ None, /* flow_innermost_root_job */ None, - /* job_id */ None, + /* job_id */ job_id, /* is_flow_step */ false, /* same_worker */ false, None, @@ -228,7 +237,7 @@ impl RunJob { /// Push the job as a specific user (for testing permissions) pub async fn push_as(self, db: &Pool, username: &str, email: &str) -> Uuid { - let RunJob { payload, args, scheduled_for_o, .. } = self; + let RunJob { payload, args, scheduled_for_o, job_id, .. } = self; let mut hm_args = std::collections::HashMap::new(); for (k, v) in args { hm_args.insert(k, windmill_common::worker::to_raw_value(&v)); @@ -250,7 +259,7 @@ impl RunJob { /* parent_job */ None, /* root job */ None, /* flow_innermost_root_job */ None, - /* job_id */ None, + /* job_id */ job_id, /* is_flow_step */ false, /* same_worker */ false, None, @@ -380,7 +389,7 @@ pub fn spawn_test_worker( std::fs::DirBuilder::new() .recursive(true) - .create(windmill_worker::GO_BIN_CACHE_DIR) + .create(&*windmill_worker::GO_BIN_CACHE_DIR) .expect("could not create initial worker dir"); let (tx, rx) = KillpillSender::new(1); @@ -828,6 +837,15 @@ pub async fn run_preview_relative_imports( #[cfg(all(feature = "private", feature = "agent_worker_server"))] pub async fn testing_http_connection(port: u16) -> Connection { + testing_http_connection_with_tags( + port, + vec!["flow".into(), "python3".into(), "dependency".into()], + ) + .await +} + +#[cfg(all(feature = "private", feature = "agent_worker_server"))] +pub async fn testing_http_connection_with_tags(port: u16, tags: Vec) -> Connection { let suffix = windmill_common::utils::create_default_worker_suffix("test-agent-worker"); let agent_token = format!( "{}{}", @@ -835,7 +853,7 @@ pub async fn testing_http_connection(port: u16) -> Connection { windmill_common::jwt::encode_with_internal_secret(windmill_api_agent_workers::AgentAuth { worker_group: "testing-agent".to_owned(), suffix: Some(suffix.clone()), - tags: vec!["flow".into(), "python3".into(), "dependency".into()], + tags, exp: Some(usize::MAX), }) .await diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index cf86d83801..be20cfae3e 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -13,6 +13,7 @@ pub enum AssetKind { Variable, // Deprecated Ducklake, DataTable, + Volume, } #[derive( diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index bcbe0ede33..28d3a1e449 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -105,15 +105,15 @@ impl ScriptLang { pub fn is_native(&self) -> bool { matches!( self, - ScriptLang::Bunnative | - ScriptLang::Nativets | - ScriptLang::Postgresql | - ScriptLang::Mysql | - ScriptLang::Graphql | - ScriptLang::Snowflake | - ScriptLang::Mssql | - ScriptLang::Bigquery | - ScriptLang::OracleDB + ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Postgresql + | ScriptLang::Mysql + | ScriptLang::Graphql + | ScriptLang::Snowflake + | ScriptLang::Mssql + | ScriptLang::Bigquery + | ScriptLang::OracleDB ) } @@ -459,6 +459,7 @@ pub struct NewScript { pub path: String, pub parent_hash: Option, pub summary: String, + #[serde(default)] pub description: String, pub content: String, pub schema: Option, diff --git a/backend/windmill-worker-volumes/Cargo.toml b/backend/windmill-worker-volumes/Cargo.toml new file mode 100644 index 0000000000..08b9a554b2 --- /dev/null +++ b/backend/windmill-worker-volumes/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "windmill-worker-volumes" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_worker_volumes" +path = "src/lib.rs" + +[features] +enterprise = [] +private = [] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +object_store.workspace = true +tokio.workspace = true +tracing.workspace = true +bytes.workspace = true +futures.workspace = true +serde.workspace = true +serde_json.workspace = true +regex.workspace = true +lazy_static.workspace = true +md-5.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/backend/windmill-worker-volumes/src/lib.rs b/backend/windmill-worker-volumes/src/lib.rs new file mode 100644 index 0000000000..a9003a3712 --- /dev/null +++ b/backend/windmill-worker-volumes/src/lib.rs @@ -0,0 +1,544 @@ +#[cfg(feature = "private")] +mod volume_ee; +mod volume_oss; +pub use volume_oss::*; + +pub use object_store::ObjectStore as DynObjectStore; + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::PathBuf; + +pub const MAX_VOLUMES_PER_JOB: usize = 10; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FileEntry { + pub size: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub md5: Option, +} + +pub fn compute_md5_hex(data: &[u8]) -> String { + use md5::{Digest, Md5}; + let result = Md5::digest(data); + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut hex = String::with_capacity(32); + for &b in result.iter() { + hex.push(HEX[(b >> 4) as usize] as char); + hex.push(HEX[(b & 0x0f) as usize] as char); + } + hex +} + +/// Extract an MD5 hash from an S3 ETag, if it's a simple (non-multipart) ETag. +pub fn etag_to_md5(e_tag: Option<&str>) -> Option { + let tag = e_tag?.trim_matches('"'); + // Multipart ETags contain a '-' (e.g. "abc123-5"), skip those + if tag.contains('-') || tag.is_empty() { + return None; + } + Some(tag.to_string()) +} + +lazy_static::lazy_static! { + static ref ARGS_INTERPOLATION_RE: regex::Regex = + regex::Regex::new(r#"\$args\[((?:\w+\.)*\w+)\]"#).unwrap(); + static ref VALID_VOLUME_NAME_RE: regex::Regex = + regex::Regex::new(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,253}[a-zA-Z0-9]$").unwrap(); +} + +#[derive(Debug, Clone, PartialEq)] +pub struct VolumeMount { + pub name: String, + pub target: String, +} + +pub struct VolumeState { + pub mount: VolumeMount, + pub local_dir: PathBuf, + pub manifest: HashMap, + pub symlinks: HashMap, +} + +pub struct DownloadStats { + pub total_files: usize, + pub from_cache: usize, + pub downloaded: usize, +} + +pub struct SyncStats { + pub new_size_bytes: u64, + pub file_count: usize, + pub uploaded: usize, + pub skipped: usize, +} + +pub fn validate_volume_name(name: &str) -> Result<(), String> { + if name.contains("..") { + return Err(format!( + "Volume name '{}' contains '..' which is not allowed", + name + )); + } + if !VALID_VOLUME_NAME_RE.is_match(name) { + return Err(format!( + "Volume name '{}' is invalid. Names must be 2-255 characters, \ + start and end with alphanumeric, and contain only alphanumeric, '.', '_', or '-'", + name + )); + } + Ok(()) +} + +const ALLOWED_ABSOLUTE_PREFIXES: &[&str] = &["/tmp/", "/mnt/", "/opt/", "/home/", "/data/"]; + +pub fn validate_volume_target(target: &str) -> Result<(), String> { + if target.split('/').any(|seg| seg == "..") { + return Err(format!( + "Volume target '{target}' contains '..' segments which is not allowed" + )); + } + if target.starts_with('/') + && !ALLOWED_ABSOLUTE_PREFIXES + .iter() + .any(|p| target.starts_with(p)) + { + return Err(format!( + "Volume target '{target}' must be a relative path or start with one of: {}", + ALLOWED_ABSOLUTE_PREFIXES.join(", ") + )); + } + Ok(()) +} + +pub fn validate_volume_mounts(mounts: &[VolumeMount]) -> Result<(), String> { + if mounts.len() > MAX_VOLUMES_PER_JOB { + return Err(format!( + "Too many volume mounts ({}, max {})", + mounts.len(), + MAX_VOLUMES_PER_JOB + )); + } + let mut seen_names = HashSet::new(); + let mut seen_targets = HashSet::new(); + for v in mounts { + if !seen_names.insert(&v.name) { + return Err(format!("Duplicate volume name: '{}'", v.name)); + } + if !seen_targets.insert(&v.target) { + return Err(format!("Duplicate volume target: '{}'", v.target)); + } + } + Ok(()) +} + +pub fn interpolate_volume_name( + name: &str, + args: Option<&HashMap>>, + workspace_id: &str, +) -> String { + let name = name.replace("$workspace", workspace_id); + if !name.contains("$args[") { + return name; + } + let Some(args) = args else { + return name; + }; + let mut result = name.clone(); + for cap in ARGS_INTERPOLATION_RE.captures_iter(&name) { + let full_match = cap.get(0).unwrap().as_str(); + let arg_name = cap.get(1).unwrap().as_str(); + let arg_value = if arg_name.contains('.') { + let parts: Vec<&str> = arg_name.split('.').collect(); + let root = parts[0]; + let mut value = args + .get(root) + .map(|x| x.get().to_string()) + .unwrap_or_default(); + for part in parts.iter().skip(1) { + if let Ok(obj) = serde_json::from_str::(&value) { + value = obj + .get(part) + .map(|v| v.to_string()) + .unwrap_or_default() + .to_string(); + } else { + value = String::new(); + break; + } + } + value.trim_matches('"').to_string() + } else { + args.get(arg_name) + .map(|x| x.get().trim_matches('"').to_string()) + .unwrap_or_default() + }; + result = result.replace(full_match, &arg_value); + } + result +} + +pub fn parse_volume_annotations(content: &str, comment_prefix: &str) -> Vec { + let mut volumes = Vec::new(); + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + if !trimmed.starts_with(comment_prefix) { + break; + } + let after_prefix = trimmed[comment_prefix.len()..].trim(); + if let Some(rest) = after_prefix.strip_prefix("volume:") { + let rest = rest.trim(); + let mut parts = rest.splitn(2, char::is_whitespace); + if let (Some(name), Some(target)) = (parts.next(), parts.next()) { + let name = name.trim(); + let target = target.trim(); + if !name.is_empty() && !target.is_empty() { + volumes + .push(VolumeMount { name: name.to_string(), target: target.to_string() }); + } + } + } + } + volumes +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_python_single_volume() { + let content = "# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_typescript_single_volume() { + let content = "// volume: mydata /tmp/data\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_multiple_volumes() { + let content = "# volume: data1 /tmp/data1\n# volume: data2 /tmp/data2\n# volume: models /opt/models\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![ + VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }, + VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() }, + VolumeMount { name: "models".to_string(), target: "/opt/models".to_string() }, + ] + ); + } + + #[test] + fn parse_mixed_annotations_and_volumes() { + let content = "# sandbox\n# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_no_volumes() { + let content = "# sandbox\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert!(result.is_empty()); + } + + #[test] + fn parse_empty_content() { + let result = parse_volume_annotations("", "#"); + assert!(result.is_empty()); + } + + #[test] + fn parse_stops_at_non_comment_line() { + let content = + "# volume: data1 /tmp/data1\ndef main():\n # volume: data2 /tmp/data2\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }] + ); + } + + #[test] + fn parse_skips_blank_lines_in_header() { + let content = + "# volume: data1 /tmp/data1\n\n# volume: data2 /tmp/data2\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![ + VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }, + VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() }, + ] + ); + } + + #[test] + fn parse_ignores_malformed_volume_lines() { + let content = + "# volume:\n# volume: onlyname\n# volume: good /tmp/good\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "good".to_string(), target: "/tmp/good".to_string() }] + ); + } + + #[test] + fn parse_extra_whitespace() { + let content = "# volume: mydata /tmp/data \ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_target_with_spaces_in_path() { + let content = "// volume: mydata /tmp/my data dir\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { + name: "mydata".to_string(), + target: "/tmp/my data dir".to_string(), + }] + ); + } + + #[test] + fn parse_volume_with_dashes_and_underscores() { + let content = "# volume: my-data_v2 /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "my-data_v2".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn interpolate_workspace() { + let name = "$workspace-data"; + let result = interpolate_volume_name(name, None, "my_ws"); + assert_eq!(result, "my_ws-data"); + } + + #[test] + fn interpolate_args_simple() { + let mut args = HashMap::new(); + args.insert( + "env".to_string(), + serde_json::value::RawValue::from_string("\"prod\"".to_string()).unwrap(), + ); + let result = interpolate_volume_name("data-$args[env]", Some(&args), "ws"); + assert_eq!(result, "data-prod"); + } + + #[test] + fn interpolate_args_and_workspace() { + let mut args = HashMap::new(); + args.insert( + "env".to_string(), + serde_json::value::RawValue::from_string("\"staging\"".to_string()).unwrap(), + ); + let result = interpolate_volume_name("$workspace-$args[env]-cache", Some(&args), "acme"); + assert_eq!(result, "acme-staging-cache"); + } + + #[test] + fn interpolate_no_placeholders() { + let result = interpolate_volume_name("plain-name", None, "ws"); + assert_eq!(result, "plain-name"); + } + + #[test] + fn interpolate_missing_arg() { + let args = HashMap::new(); + let result = interpolate_volume_name("data-$args[missing]", Some(&args), "ws"); + assert_eq!(result, "data-"); + } + + #[test] + fn interpolate_nested_arg() { + let mut args = HashMap::new(); + args.insert( + "config".to_string(), + serde_json::value::RawValue::from_string( + r#"{"env": "prod", "region": "us-east"}"#.to_string(), + ) + .unwrap(), + ); + let result = interpolate_volume_name( + "data-$args[config.env]-$args[config.region]", + Some(&args), + "ws", + ); + assert_eq!(result, "data-prod-us-east"); + } + + #[test] + fn parse_wrong_prefix_returns_empty() { + let content = "# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "//"); + assert!(result.is_empty()); + } + + #[test] + fn parse_relative_path() { + let content = "// volume: agent-memory .claude\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { name: "agent-memory".to_string(), target: ".claude".to_string() }] + ); + } + + #[test] + fn parse_relative_nested_path() { + let content = "# volume: data data/models\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "data".to_string(), target: "data/models".to_string() }] + ); + } + + #[test] + fn validate_valid_names() { + assert!(validate_volume_name("mydata").is_ok()); + assert!(validate_volume_name("my-data_v2").is_ok()); + assert!(validate_volume_name("acme-staging-cache").is_ok()); + assert!(validate_volume_name("a1").is_ok()); + assert!(validate_volume_name("data.v2").is_ok()); + assert!(validate_volume_name("A0").is_ok()); + } + + #[test] + fn validate_rejects_path_traversal() { + assert!(validate_volume_name("../other-workspace").is_err()); + assert!(validate_volume_name("data/../secrets").is_err()); + assert!(validate_volume_name("a..b").is_err()); + } + + #[test] + fn validate_rejects_special_start_end() { + assert!(validate_volume_name("-data").is_err()); + assert!(validate_volume_name("data-").is_err()); + assert!(validate_volume_name(".data").is_err()); + assert!(validate_volume_name("data.").is_err()); + assert!(validate_volume_name("_data").is_err()); + } + + #[test] + fn validate_rejects_path_separators() { + assert!(validate_volume_name("data/secrets").is_err()); + assert!(validate_volume_name("data\\secrets").is_err()); + } + + #[test] + fn validate_rejects_too_short() { + assert!(validate_volume_name("").is_err()); + assert!(validate_volume_name("a").is_err()); + } + + #[test] + fn validate_rejects_too_long() { + let long_name = format!("a{}a", "b".repeat(254)); + assert!(validate_volume_name(&long_name).is_err()); + } + + #[test] + fn validate_rejects_spaces_and_special() { + assert!(validate_volume_name("my data").is_err()); + assert!(validate_volume_name("my@data").is_err()); + assert!(validate_volume_name("my$data").is_err()); + } + + #[test] + fn validate_target_allows_relative() { + assert!(validate_volume_target("data").is_ok()); + assert!(validate_volume_target("data/models").is_ok()); + assert!(validate_volume_target(".claude").is_ok()); + } + + #[test] + fn validate_target_allows_safe_absolute() { + assert!(validate_volume_target("/tmp/data").is_ok()); + assert!(validate_volume_target("/mnt/data").is_ok()); + assert!(validate_volume_target("/opt/models").is_ok()); + assert!(validate_volume_target("/home/user/data").is_ok()); + assert!(validate_volume_target("/data/cache").is_ok()); + } + + #[test] + fn validate_target_rejects_dangerous_absolute() { + assert!(validate_volume_target("/etc/passwd").is_err()); + assert!(validate_volume_target("/proc/self").is_err()); + assert!(validate_volume_target("/sys/fs").is_err()); + assert!(validate_volume_target("/dev/null").is_err()); + assert!(validate_volume_target("/usr/bin").is_err()); + assert!(validate_volume_target("/var/log").is_err()); + } + + #[test] + fn validate_target_rejects_traversal() { + assert!(validate_volume_target("../../etc").is_err()); + assert!(validate_volume_target("data/../../../etc").is_err()); + assert!(validate_volume_target("/tmp/../etc/passwd").is_err()); + } + + #[test] + fn validate_mounts_rejects_too_many() { + let mounts: Vec = (0..11) + .map(|i| VolumeMount { name: format!("v{:02}", i), target: format!("t{}", i) }) + .collect(); + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_rejects_duplicate_name() { + let mounts = vec![ + VolumeMount { name: "data".to_string(), target: "/tmp/a".to_string() }, + VolumeMount { name: "data".to_string(), target: "/tmp/b".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_rejects_duplicate_target() { + let mounts = vec![ + VolumeMount { name: "v1".to_string(), target: "/tmp/data".to_string() }, + VolumeMount { name: "v2".to_string(), target: "/tmp/data".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_ok() { + let mounts = vec![ + VolumeMount { name: "v1".to_string(), target: "/tmp/a".to_string() }, + VolumeMount { name: "v2".to_string(), target: "/tmp/b".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_ok()); + } +} diff --git a/backend/windmill-worker-volumes/src/volume_oss.rs b/backend/windmill-worker-volumes/src/volume_oss.rs new file mode 100644 index 0000000000..07cd6906d2 --- /dev/null +++ b/backend/windmill-worker-volumes/src/volume_oss.rs @@ -0,0 +1,116 @@ +#[cfg(feature = "private")] +pub use crate::volume_ee::*; + +#[cfg(not(feature = "private"))] +use crate::{DownloadStats, SyncStats, VolumeMount, VolumeState}; +#[cfg(not(feature = "private"))] +use object_store::ObjectStore; +#[cfg(not(feature = "private"))] +use std::path::Path; +#[cfg(not(feature = "private"))] +use std::sync::Arc; +#[cfg(not(feature = "private"))] +use windmill_common::error; + +#[cfg(not(feature = "private"))] +pub async fn download_volume( + _client: Arc, + _volume: &VolumeMount, + _job_dir: &str, + _workspace_id: &str, +) -> error::Result<(VolumeState, DownloadStats)> { + Err(error::Error::internal_err( + "Volumes are not available in this build".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +pub fn volume_nsjail_mount(_local_dir: &Path, _target: &str) -> String { + String::new() +} + +#[cfg(not(feature = "private"))] +pub async fn sync_volume_back( + _client: Arc, + _state: &VolumeState, + _workspace_id: &str, +) -> error::Result { + Err(error::Error::internal_err( + "Volumes are not available in this build".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +pub fn walk_dir(dir: &Path) -> std::io::Result> { + let mut result = Vec::new(); + walk_dir_inner(dir, &mut result)?; + Ok(result) +} + +#[cfg(not(feature = "private"))] +fn walk_dir_inner(dir: &Path, result: &mut Vec) -> std::io::Result<()> { + if !dir.is_dir() { + return Ok(()); + } + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + let meta = match std::fs::symlink_metadata(&path) { + Ok(m) => m, + Err(_) => continue, + }; + if meta.is_dir() { + walk_dir_inner(&path, result)?; + } else if meta.is_file() { + result.push(path); + } + } + Ok(()) +} + +#[cfg(not(feature = "private"))] +pub fn collect_symlinks(dir: &Path) -> std::collections::HashMap { + let mut symlinks = std::collections::HashMap::new(); + collect_symlinks_inner(dir, dir, &mut symlinks); + symlinks +} + +#[cfg(not(feature = "private"))] +fn collect_symlinks_inner( + base: &Path, + dir: &Path, + symlinks: &mut std::collections::HashMap, +) { + let entries = match std::fs::read_dir(dir) { + Ok(e) => e, + Err(_) => return, + }; + for entry in entries { + let entry = match entry { + Ok(e) => e, + Err(_) => continue, + }; + let path = entry.path(); + let meta = match std::fs::symlink_metadata(&path) { + Ok(m) => m, + Err(_) => continue, + }; + if meta.file_type().is_symlink() { + if let Ok(target) = std::fs::read_link(&path) { + let relative = path + .strip_prefix(base) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + symlinks.insert(relative, target.to_string_lossy().to_string()); + } + } else if meta.is_dir() { + collect_symlinks_inner(base, &path, symlinks); + } + } +} + +#[cfg(not(feature = "private"))] +pub fn restore_symlinks(_dir: &Path, _symlinks: &std::collections::HashMap) { + // No-op in OSS build +} diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 5cf0a4c26e..23927753ee 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -10,10 +10,10 @@ path = "src/lib.rs" [features] default = [] -private = [] +private = ["windmill-worker-volumes/private", "windmill-queue/private"] mcp = ["dep:windmill-mcp"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] -enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] +enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] mssql = ["dep:tiberius"] mssql-kerberos = ["mssql", "tiberius/integrated-auth-gssapi"] # Linux/Unix integrated auth mssql-winauth = ["mssql", "tiberius/winauth"] # Windows integrated auth @@ -47,6 +47,7 @@ windmill-audit.workspace = true # there isn't really a reason for audit-worth ac windmill-common = { workspace = true, default-features = false } windmill-types.workspace = true windmill-object-store.workspace = true +windmill-worker-volumes.workspace = true windmill-jseval.workspace = true windmill-runtime-nativets = { workspace = true, optional = true } windmill-mcp = { workspace = true, optional = true } diff --git a/backend/windmill-worker/loader.bun.windows.js b/backend/windmill-worker/loader.bun.windows.js new file mode 100644 index 0000000000..227c68a56c --- /dev/null +++ b/backend/windmill-worker/loader.bun.windows.js @@ -0,0 +1,123 @@ +// Windows-specific bun loader that uses a virtual "windmill-url" namespace instead +// of writing .url files to disk. This avoids Windows path issues (backslashes in +// resolve(), 8.3 short filenames, drive letter prefixes). The virtual namespace +// approach is likely better on all fronts but we keep the original .url-file loader +// on Linux to avoid breaking back-compat. +const p = { + name: "windmill-relative-resolver", + async setup(build) { + const { readFileSync } = await import("fs"); + const { resolve } = await import("node:path"); + + const base_internal_url = "BASE_INTERNAL_URL".replace( + "localhost", + "127.0.0.1" + ); + + const w_id = "W_ID"; + const current_path = "CURRENT_PATH"; + const token = "TOKEN"; + + const cdir = resolve("./"); + const cdirNoPrivate = cdir.replace(/^\/private/, ""); // for macos + // Normalize path to forward slashes to match Bun's resolver output on Windows + const cdirFwd = cdir.replace(/\\/g, "/"); + const cdirPosix = cdirFwd.replace(/^[a-zA-Z]:/, ""); + const filterResolve = new RegExp( + `^(?!\\.\/main\\.ts)(?!${cdirFwd}\/main\\.ts)(?!${cdirPosix}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.mjs).*\\.ts$` + ); + + let cdirNodeModules = `${cdirFwd}/node_modules/`; + + const filterLoad = new RegExp(`^${cdir}\/main\\.ts$`); + const transpiler = new Bun.Transpiler({ + loader: "ts", + }); + + function replaceRelativeImports(code) { + const imports = transpiler.scanImports(code); + for (const imp of imports) { + if (imp.kind == "import-statement") { + if ( + (imp.path.startsWith(".") || + imp.path.startsWith("/u/") || + imp.path.startsWith("/f/")) && + !imp.path.endsWith(".ts") + ) { + code = code.replaceAll(imp.path, imp.path + ".ts"); + } + } + } + return { + contents: code, + }; + } + + function normalizePath(rawPath) { + return rawPath.split("/").reduce((acc, seg) => { + if (seg === "..") acc.pop(); + else if (seg !== "." && seg !== "") acc.push(seg); + return acc; + }, []).join("/"); + } + + // Resolve a windmill script import path relative to an importer path. + // Bun on Windows may prefix args with "windmill-url:" or strip leading "/". + function resolveWindmillImport(importerPath, importPath) { + const path = importPath.replace(/^windmill-url:/, "").replace(/^\//, ""); + const isAbsolute = path.startsWith("f/") || path.startsWith("u/"); + const endExt = path.endsWith(".ts") ? "" : ".ts"; + const rawScriptPath = isAbsolute + ? `${path}${endExt}` + : `${importerPath}/../${path}${endExt}`; + return { path: normalizePath(rawScriptPath), namespace: "windmill-url" }; + } + + build.onLoad({ filter: filterLoad }, async (args) => { + const code = readFileSync(args.path, "utf8"); + return replaceRelativeImports(code); + }); + + // Load windmill scripts by fetching from the API + build.onLoad({ filter: /.*/, namespace: "windmill-url" }, async (args) => { + const path = args.path.replace(/^windmill-url:/, ""); + const url = `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${path}`; + const req = await fetch(url, { + method: "GET", + headers: { + Authorization: "Bearer " + token, + }, + }); + if (!req.ok) { + throw new Error( + `Failed to find relative import at ${url} (status ${req.status})` + ); + } + const contents = await req.text(); + return { + contents: replaceRelativeImports(contents).contents, + loader: "tsx", + }; + }); + + // Resolve windmill script imports from the file namespace (e.g. from main.ts) + build.onResolve({ filter: filterResolve }, (args) => { + const importerFwd = args.importer?.replace(/\\/g, "/") ?? ""; + if (importerFwd.startsWith(cdirNodeModules)) { + return undefined; + } + const isMainTs = + args.importer == "./main.ts" || importerFwd.endsWith("/main.ts"); + const file_path = isMainTs + ? current_path + : importerFwd.replace(cdirFwd + "/", ""); + return resolveWindmillImport(file_path, args.path); + }); + + // Resolve nested imports from within windmill-url modules + build.onResolve({ filter: /\.ts$/, namespace: "windmill-url" }, (args) => { + const importer = args.importer.replace(/^windmill-url:/, ""); + return resolveWindmillImport(importer, args.path); + }); + }, +}; diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index 8cbee3dec9..3ba8c73257 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -14,6 +14,18 @@ clone_newnet: false clone_newuser: {CLONE_NEWUSER} clone_newcgroup: false +uidmap { + inside_id: "1000" + outside_id: "" + count: 1 +} + +gidmap { + inside_id: "1000" + outside_id: "" + count: 1 +} + skip_setsid: true keep_caps: false keep_env: true diff --git a/backend/windmill-worker/nsjail/run.powershell.config.proto b/backend/windmill-worker/nsjail/run.powershell.config.proto index 87b6abda21..afe9d5df9f 100644 --- a/backend/windmill-worker/nsjail/run.powershell.config.proto +++ b/backend/windmill-worker/nsjail/run.powershell.config.proto @@ -127,7 +127,7 @@ iface_no_lo: true mount { src: "{CACHE_DIR}" - dst: "/tmp/windmill/cache/powershell" + dst: "{CACHE_DIR}" is_bind: true rw: false mandatory: false diff --git a/backend/windmill-worker/src/ai/providers/anthropic.rs b/backend/windmill-worker/src/ai/providers/anthropic.rs index e42e1dc49b..f9f5edf452 100644 --- a/backend/windmill-worker/src/ai/providers/anthropic.rs +++ b/backend/windmill-worker/src/ai/providers/anthropic.rs @@ -1,14 +1,16 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; +use windmill_common::{ + ai_google::parse_data_url, ai_providers::AIProvider, client::AuthedClient, error::Error, +}; use crate::ai::{ image_handler::prepare_messages_for_api, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, sse::{AnthropicSSEParser, SSEParser}, types::*, - utils::{extract_text_content, parse_data_url, should_use_structured_output_tool}, + utils::{extract_text_content, should_use_structured_output_tool}, }; /// Anthropic API version for standard API diff --git a/backend/windmill-worker/src/ai/providers/google_ai.rs b/backend/windmill-worker/src/ai/providers/google_ai.rs index 62e6afff75..e2030f3269 100644 --- a/backend/windmill-worker/src/ai/providers/google_ai.rs +++ b/backend/windmill-worker/src/ai/providers/google_ai.rs @@ -1,215 +1,21 @@ use async_trait::async_trait; -use serde::{Deserialize, Serialize}; -use windmill_common::{client::AuthedClient, error::Error}; +use windmill_common::{ + ai_google::{ + openai_messages_to_gemini, openai_tools_to_gemini, GeminiGenerationConfig, + GeminiImageContent, GeminiImageRequest, GeminiImageResponse, GeminiInlineData, GeminiPart, + GeminiPredictContent, GeminiTextRequest, GeminiTool, + }, + client::AuthedClient, + error::Error, +}; use crate::ai::{ - image_handler::download_and_encode_s3_image, + image_handler::{download_and_encode_s3_image, prepare_messages_for_api}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, sse::{GeminiSSEParser, SSEParser}, types::*, - utils::parse_data_url, }; -// ============================================================================ -// Gemini API Types - Shared between text and image -// ============================================================================ - -/// Inline data for binary content (images) -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GeminiInlineData { - #[serde(rename = "mimeType")] - pub mime_type: String, - pub data: String, -} - -/// A part of content - can be text, inline data, function call, or function response -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(untagged)] -pub enum GeminiPart { - Text { - text: String, - }, - InlineData { - #[serde(rename = "inlineData")] - inline_data: GeminiInlineData, - }, - FunctionCall { - #[serde(rename = "functionCall")] - function_call: GeminiFunctionCall, - /// Thought signature for Gemini 3+ models - required for function calling - #[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")] - thought_signature: Option, - }, - FunctionResponse { - #[serde(rename = "functionResponse")] - function_response: GeminiFunctionResponse, - }, -} - -/// A function call from the model -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GeminiFunctionCall { - pub name: String, - pub args: serde_json::Value, -} - -/// A function response to send back to the model -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GeminiFunctionResponse { - pub name: String, - pub response: serde_json::Value, -} - -// ============================================================================ -// Gemini Text API Request Types -// ============================================================================ - -/// Main request structure for Gemini generateContent -#[derive(Serialize)] -pub struct GeminiTextRequest { - pub contents: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub tools: Option>, - #[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")] - pub tool_config: Option, - #[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")] - pub system_instruction: Option, - #[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")] - pub generation_config: Option, -} - -/// Content message with role and parts -#[derive(Serialize)] -pub struct GeminiContentMessage { - #[serde(skip_serializing_if = "Option::is_none")] - pub role: Option, - pub parts: Vec, -} - -/// Tool definition - either function declarations or Google Search -#[derive(Serialize)] -pub struct GeminiTool { - #[serde( - rename = "functionDeclarations", - skip_serializing_if = "Option::is_none" - )] - pub function_declarations: Option>, - #[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")] - pub google_search: Option, -} - -/// Function declaration for tool use -#[derive(Serialize)] -pub struct GeminiFunctionDeclaration { - pub name: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - pub parameters: OpenAPISchema, -} - -/// Tool configuration for controlling function calling behavior -#[derive(Serialize)] -pub struct GeminiToolConfig { - #[serde(rename = "functionCallingConfig")] - pub function_calling_config: GeminiFunctionCallingConfig, -} - -/// Function calling configuration -#[derive(Serialize)] -pub struct GeminiFunctionCallingConfig { - pub mode: String, - #[serde( - rename = "allowedFunctionNames", - skip_serializing_if = "Option::is_none" - )] - pub allowed_function_names: Option>, -} - -/// Generation configuration for output format -#[derive(Serialize)] -pub struct GeminiGenerationConfig { - #[serde(skip_serializing_if = "Option::is_none")] - pub temperature: Option, - #[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")] - pub max_output_tokens: Option, - #[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")] - pub response_mime_type: Option, - #[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")] - pub response_schema: Option, -} - -// ============================================================================ -// Gemini API Response Types -// ============================================================================ - -/// Grounding metadata from Google Search -#[derive(Deserialize)] -#[allow(dead_code)] -pub struct GeminiGroundingMetadata { - #[serde(rename = "webSearchQueries")] - pub web_search_queries: Option>, - #[serde(rename = "groundingChunks")] - pub grounding_chunks: Option>, -} - -// ============================================================================ -// Gemini Image API Types (for Imagen models) -// ============================================================================ - -/// Request for image generation (Imagen models) -#[derive(Serialize)] -pub struct GeminiImageRequest { - #[serde(skip_serializing_if = "Option::is_none")] - pub contents: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub instances: Option>, -} - -/// Content for image generation -#[derive(Serialize)] -pub struct GeminiImageContent { - pub parts: Vec, -} - -/// Content for Imagen predict endpoint -#[derive(Serialize)] -pub struct GeminiPredictContent { - pub prompt: String, -} - -/// Response for image generation -#[derive(Deserialize)] -pub struct GeminiImageResponse { - pub candidates: Option>, - pub predictions: Option>, -} - -/// Image candidate from generateContent -#[derive(Deserialize)] -pub struct GeminiImageCandidate { - pub content: GeminiImageCandidateContent, -} - -/// Content in image candidate -#[derive(Deserialize)] -pub struct GeminiImageCandidateContent { - pub parts: Vec, -} - -/// Part of image candidate -#[derive(Deserialize)] -pub struct GeminiImageCandidatePart { - #[serde(rename = "inlineData", skip_serializing_if = "Option::is_none")] - pub inline_data: Option, -} - -/// Prediction candidate from Imagen -#[derive(Deserialize)] -pub struct GeminiPredictCandidate { - #[serde(rename = "bytesBase64Encoded")] - pub bytes_base64_encoded: String, -} - // ============================================================================ // Query Builder Implementation // ============================================================================ @@ -221,34 +27,24 @@ impl GoogleAIQueryBuilder { Self } - /// Build a text request using the native Gemini API format async fn build_text_request( &self, args: &BuildRequestArgs<'_>, client: &AuthedClient, workspace_id: &str, ) -> Result { - // Convert messages to Gemini format - let contents = self - .convert_messages_to_gemini(args.messages, client, workspace_id) - .await?; + let prepared_messages = + prepare_messages_for_api(args.messages, client, workspace_id).await?; + let (contents, system_instruction) = openai_messages_to_gemini(&prepared_messages); - // Build tools array let tools = self.convert_tools_to_gemini(args.tools, args.has_websearch); - // Build generation config let generation_config = self.build_generation_config(args); - // Build system instruction from system_prompt - let system_instruction = args.system_prompt.map(|s| GeminiContentMessage { - role: None, - parts: vec![GeminiPart::Text { text: s.to_string() }], - }); - let request = GeminiTextRequest { contents, tools, - tool_config: None, // Use AUTO mode by default + tool_config: None, system_instruction, generation_config, }; @@ -257,7 +53,6 @@ impl GoogleAIQueryBuilder { .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) } - /// Build an image generation request async fn build_image_request( &self, args: &BuildRequestArgs<'_>, @@ -267,7 +62,6 @@ impl GoogleAIQueryBuilder { let is_imagen = args.model.contains("imagen"); let request = if is_imagen { - // For Imagen models, use simple prompt format GeminiImageRequest { instances: Some(vec![GeminiPredictContent { prompt: args.user_message.trim().to_string(), @@ -275,7 +69,6 @@ impl GoogleAIQueryBuilder { contents: None, } } else { - // For Gemini models with image generation, build parts let mut parts = vec![GeminiPart::Text { text: args.user_message.trim().to_string() }]; if let Some(system_prompt) = args.system_prompt { @@ -285,7 +78,6 @@ impl GoogleAIQueryBuilder { ); } - // Add input images if provided if let Some(images) = args.images { for image in images.iter() { if !image.s3.is_empty() { @@ -308,218 +100,39 @@ impl GoogleAIQueryBuilder { .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) } - /// Convert OpenAI-format messages to Gemini format - async fn convert_messages_to_gemini( - &self, - messages: &[OpenAIMessage], - client: &AuthedClient, - workspace_id: &str, - ) -> Result, Error> { - let mut gemini_messages = Vec::new(); - - for msg in messages { - match msg.role.as_str() { - "system" => { - // Skip - handled via args.system_prompt in build_text_request - } - "tool" => { - // Handle tool responses - if let (Some(tool_call_id), Some(content)) = (&msg.tool_call_id, &msg.content) { - let func_name = self.find_function_name_by_id(messages, tool_call_id); - let response_text = match content { - OpenAIContent::Text(text) => text.clone(), - OpenAIContent::Parts(parts) => parts - .iter() - .filter_map(|p| match p { - ContentPart::Text { text } => Some(text.clone()), - _ => None, - }) - .collect::>() - .join(" "), - }; - - gemini_messages.push(GeminiContentMessage { - role: Some("user".to_string()), - parts: vec![GeminiPart::FunctionResponse { - function_response: GeminiFunctionResponse { - name: func_name, - response: serde_json::json!({ "result": response_text }), - }, - }], - }); - } - } - _ => { - // Handle user/assistant messages - let role = match msg.role.as_str() { - "assistant" => "model", - _ => "user", - }; - - let mut parts = Vec::new(); - - // Handle regular content - if let Some(content) = &msg.content { - let content_parts = self - .convert_content_to_parts(&Some(content.clone()), client, workspace_id) - .await?; - parts.extend(content_parts); - } - - // Handle tool calls from assistant - if let Some(tool_calls) = &msg.tool_calls { - for tc in tool_calls { - let args: serde_json::Value = - serde_json::from_str(&tc.function.arguments).unwrap_or_default(); - // Extract thought_signature from extra_content if present - let thought_signature = tc - .extra_content - .as_ref() - .and_then(|ec| ec.google.as_ref()) - .and_then(|g| g.thought_signature.clone()); - parts.push(GeminiPart::FunctionCall { - function_call: GeminiFunctionCall { - name: tc.function.name.clone(), - args, - }, - thought_signature, - }); - } - } - - if !parts.is_empty() { - gemini_messages - .push(GeminiContentMessage { role: Some(role.to_string()), parts }); - } - } - } - } - - Ok(gemini_messages) - } - - /// Convert OpenAI content to Gemini parts - async fn convert_content_to_parts( - &self, - content: &Option, - client: &AuthedClient, - workspace_id: &str, - ) -> Result, Error> { - let mut parts = Vec::new(); - - if let Some(content) = content { - match content { - OpenAIContent::Text(text) => { - if !text.is_empty() { - parts.push(GeminiPart::Text { text: text.clone() }); - } - } - OpenAIContent::Parts(content_parts) => { - for part in content_parts { - match part { - ContentPart::Text { text } => { - if !text.is_empty() { - parts.push(GeminiPart::Text { text: text.clone() }); - } - } - ContentPart::ImageUrl { image_url } => { - // Parse data URL format: data:mime_type;base64,data - if let Some((mime_type, data)) = parse_data_url(&image_url.url) { - parts.push(GeminiPart::InlineData { - inline_data: GeminiInlineData { mime_type, data }, - }); - } - } - ContentPart::S3Object { s3_object } => { - if !s3_object.s3.is_empty() { - let (mime_type, data) = download_and_encode_s3_image( - s3_object, - client, - workspace_id, - ) - .await?; - parts.push(GeminiPart::InlineData { - inline_data: GeminiInlineData { mime_type, data }, - }); - } - } - } - } - } - } - } - - Ok(parts) - } - - /// Find function name by tool call ID from previous messages - fn find_function_name_by_id(&self, messages: &[OpenAIMessage], tool_call_id: &str) -> String { - for msg in messages { - if let Some(tool_calls) = &msg.tool_calls { - for tc in tool_calls { - if tc.id == tool_call_id { - return tc.function.name.clone(); - } - } - } - } - "unknown_function".to_string() - } - - /// Convert OpenAI tools to Gemini format + /// Convert OpenAI tool definitions to Gemini format. + /// + /// Sanitizes each tool's JSON schema for Google compatibility before delegating + /// to the shared [`openai_tools_to_gemini`] function. fn convert_tools_to_gemini( &self, tools: Option<&[ToolDef]>, has_websearch: bool, ) -> Option> { - let mut gemini_tools = Vec::new(); - - // Add function declarations - if let Some(tool_defs) = tools { - let declarations: Vec = tool_defs - .iter() - .filter_map(|t| { - // Deserialize RawValue into OpenAPISchema, sanitize, then use - let mut schema: OpenAPISchema = - serde_json::from_str(t.function.parameters.get()).ok()?; - schema.sanitize_for_google(); - - Some(GeminiFunctionDeclaration { - name: t.function.name.clone(), - description: t.function.description.clone(), - parameters: schema, - }) - }) - .collect(); - - if !declarations.is_empty() { - gemini_tools.push(GeminiTool { - function_declarations: Some(declarations), - google_search: None, - }); + let Some(tool_defs) = tools else { + if has_websearch { + return Some(vec![GeminiTool { + function_declarations: None, + google_search: Some(serde_json::json!({})), + }]); } - } + return None; + }; - // Add Google Search tool if enabled - if has_websearch { - gemini_tools.push(GeminiTool { - function_declarations: None, - google_search: Some(serde_json::json!({})), - }); - } + let tool_params: Vec = tool_defs + .iter() + .map(|t| { + let mut schema: OpenAPISchema = + serde_json::from_str(t.function.parameters.get()).unwrap_or_default(); + schema.sanitize_for_google(); + serde_json::to_value(&schema).unwrap_or_default() + }) + .collect(); - if gemini_tools.is_empty() { - None - } else { - Some(gemini_tools) - } + openai_tools_to_gemini(tool_defs, &tool_params, has_websearch) } - /// Build generation config for structured output and other settings - fn build_generation_config( - &self, - args: &BuildRequestArgs<'_>, - ) -> Option { + fn build_generation_config(&self, args: &BuildRequestArgs<'_>) -> Option { let has_output_schema = args .output_schema .and_then(|s| s.properties.as_ref()) @@ -529,15 +142,11 @@ impl GoogleAIQueryBuilder { let (response_mime_type, response_schema) = if has_output_schema { let mut schema = args.output_schema.unwrap().clone(); schema.sanitize_for_google(); - ( - Some("application/json".to_string()), - serde_json::to_value(&schema).ok(), - ) + (Some("application/json".to_string()), serde_json::to_value(&schema).ok()) } else { (None, None) }; - // Only create config if there's something to configure if args.temperature.is_some() || args.max_tokens.is_some() || response_mime_type.is_some() { Some(GeminiGenerationConfig { temperature: args.temperature, @@ -554,7 +163,6 @@ impl GoogleAIQueryBuilder { #[async_trait] impl QueryBuilder for GoogleAIQueryBuilder { fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool { - // Google AI supports tools only for text output matches!(output_type, OutputType::Text) } @@ -578,7 +186,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { Error::internal_err(format!("Failed to parse Gemini image response: {}", e)) })?; - // First, check Gemini models (candidates -> content -> parts -> inline_data) let image_data_from_gemini = gemini_response.candidates.as_ref().and_then(|candidates| { candidates.iter().find_map(|candidate| { candidate @@ -589,13 +196,11 @@ impl QueryBuilder for GoogleAIQueryBuilder { }) }); - // Then, check Imagen models (predictions -> bytes_base64_encoded) let image_data_from_imagen = gemini_response .predictions .as_ref() .and_then(|predictions| predictions.first().map(|p| &p.bytes_base64_encoded)); - // Image data, preferring Gemini first then Imagen models let image_data = image_data_from_gemini.or(image_data_from_imagen); match image_data { @@ -627,7 +232,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { .. } = gemini_sse_parser; - // Send tool call arguments events for accumulated tool calls for tool_call in accumulated_tool_calls.values() { let event = StreamingEvent::ToolCallArguments { call_id: tool_call.id.clone(), @@ -637,7 +241,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { stream_event_processor.send(event, &mut events_str).await?; } - // Convert Gemini usage metadata to TokenUsage let usage = gemini_usage.map(|u| { TokenUsage::new( u.prompt_token_count, @@ -647,11 +250,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { }); Ok(ParsedResponse::Text { - content: if accumulated_content.is_empty() { - None - } else { - Some(accumulated_content) - }, + content: if accumulated_content.is_empty() { None } else { Some(accumulated_content) }, tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations, @@ -663,17 +262,11 @@ impl QueryBuilder for GoogleAIQueryBuilder { fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { match output_type { OutputType::Text => { - format!( - "{}/models/{}:streamGenerateContent?alt=sse", - base_url, model - ) + format!("{}/models/{}:streamGenerateContent?alt=sse", base_url, model) } OutputType::Image => { - let url_suffix = if model.contains("imagen") { - "predict" - } else { - "generateContent" - }; + let url_suffix = + if model.contains("imagen") { "predict" } else { "generateContent" }; format!("{}/models/{}:{}", base_url, model, url_suffix) } } @@ -685,7 +278,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { _base_url: &str, _output_type: &OutputType, ) -> Vec<(&'static str, String)> { - // Native Gemini API always uses x-goog-api-key vec![("x-goog-api-key", api_key.to_string())] } } diff --git a/backend/windmill-worker/src/ai/sse.rs b/backend/windmill-worker/src/ai/sse.rs index 77fba3140a..62f13f3494 100644 --- a/backend/windmill-worker/src/ai/sse.rs +++ b/backend/windmill-worker/src/ai/sse.rs @@ -5,15 +5,18 @@ use reqwest::Response; use serde::Deserialize; use serde_json; use tokio_stream::StreamExt; -use windmill_common::{error::Error, utils::rd_string}; +use windmill_common::{ + ai_google::{parse_gemini_sse_event, GeminiUsageMetadata}, + ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall}, + error::Error, + utils::rd_string, +}; use crate::ai::{ query_builder::StreamEventProcessor, types::{StreamingEvent, UrlCitation}, }; -use windmill_common::ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall}; - #[derive(Deserialize)] pub struct OpenAIChoiceDeltaToolCallFunction { pub name: Option, @@ -457,96 +460,19 @@ impl SSEParser for AnthropicSSEParser { // Gemini SSE Parser // ============================================================================ -/// Gemini streaming response part - can be text or function call -#[derive(Deserialize, Debug)] -pub struct GeminiSSEPart { - #[serde(default)] - pub text: Option, - #[serde(rename = "functionCall")] - pub function_call: Option, - /// Thought signature for Gemini 3+ models - required for function calling - #[serde(rename = "thoughtSignature")] - pub thought_signature: Option, -} - -/// Function call in Gemini streaming response -#[derive(Deserialize, Debug)] -pub struct GeminiSSEFunctionCall { - pub name: String, - pub args: serde_json::Value, -} - -/// Content in Gemini streaming candidate -#[derive(Deserialize, Debug)] -pub struct GeminiSSEContent { - pub parts: Option>, -} - -/// Web reference in Gemini grounding chunk -#[derive(Deserialize, Debug)] -pub struct GeminiGroundingChunkWeb { - pub uri: String, - #[serde(default)] - pub title: Option, -} - -/// Grounding chunk from Gemini web search -#[derive(Deserialize, Debug)] -pub struct GeminiGroundingChunk { - pub web: Option, -} - -/// Grounding metadata from Gemini web search -#[derive(Deserialize, Debug)] -pub struct GeminiGroundingMetadata { - #[serde(rename = "groundingChunks", default)] - pub grounding_chunks: Vec, - #[serde(rename = "webSearchQueries", default)] - pub web_search_queries: Vec, -} - -/// Candidate in Gemini streaming response -#[derive(Deserialize, Debug)] -pub struct GeminiSSECandidate { - pub content: Option, - #[serde(rename = "finishReason")] - #[allow(dead_code)] - pub finish_reason: Option, - #[serde(rename = "groundingMetadata")] - pub grounding_metadata: Option, -} - -/// Gemini usage metadata from SSE response -#[derive(Deserialize, Debug, Clone)] -pub struct GeminiUsageMetadata { - #[serde(rename = "promptTokenCount", default)] - pub prompt_token_count: Option, - #[serde(rename = "candidatesTokenCount", default)] - pub candidates_token_count: Option, - #[serde(rename = "totalTokenCount", default)] - pub total_token_count: Option, -} - -/// Gemini SSE event structure -#[derive(Deserialize, Debug)] -pub struct GeminiSSEEvent { - pub candidates: Option>, - #[serde(rename = "usageMetadata")] - pub usage_metadata: Option, -} - -/// Gemini SSE Parser for streaming responses +/// Accumulates Gemini streaming events and converts them into the worker's +/// internal [`OpenAIToolCall`] / [`StreamingEvent`] representation. +/// +/// The actual SSE parsing is delegated to [`parse_gemini_sse_event`] from +/// `windmill_common::ai_google` so the logic can be shared with the API proxy. pub struct GeminiSSEParser { pub accumulated_content: String, pub accumulated_tool_calls: HashMap, pub events_str: String, pub stream_event_processor: StreamEventProcessor, tool_call_index: i64, - /// Collected URL citation annotations from web search pub annotations: Vec, - /// Whether web search was used in this response pub used_websearch: bool, - /// Token usage from usageMetadata pub usage: Option, } @@ -567,101 +493,57 @@ impl GeminiSSEParser { impl SSEParser for GeminiSSEParser { async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { - let event: Option = serde_json::from_str(data) - .inspect_err(|e| { - tracing::error!("Failed to parse SSE as a Gemini event {}: {}", data, e); - }) - .ok(); + let Some(parsed) = parse_gemini_sse_event(data)? else { + return Ok(()); + }; - if let Some(event) = event { - if let Some(candidates) = event.candidates { - for candidate in candidates { - if let Some(content) = candidate.content { - if let Some(parts) = content.parts { - for part in parts { - // Handle text content - if let Some(text) = part.text { - if !text.is_empty() { - self.accumulated_content.push_str(&text); - let event = StreamingEvent::TokenDelta { content: text }; - self.stream_event_processor - .send(event, &mut self.events_str) - .await?; - } - } + if let Some(text) = parsed.text { + self.accumulated_content.push_str(&text); + self.stream_event_processor + .send(StreamingEvent::TokenDelta { content: text }, &mut self.events_str) + .await?; + } - // Handle function calls - if let Some(function_call) = part.function_call { - let call_id = format!("call_{}", rd_string(24)); - let idx = self.tool_call_index; - self.tool_call_index += 1; + for tool_call in parsed.tool_calls { + let call_id = format!("call_{}", rd_string(24)); + let idx = self.tool_call_index; + self.tool_call_index += 1; - // Send tool call start event - let event = StreamingEvent::ToolCall { - call_id: call_id.clone(), - function_name: function_call.name.clone(), - }; - self.stream_event_processor - .send(event, &mut self.events_str) - .await?; + self.stream_event_processor + .send( + StreamingEvent::ToolCall { + call_id: call_id.clone(), + function_name: tool_call.name.clone(), + }, + &mut self.events_str, + ) + .await?; - // Build extra_content with thought_signature if present - let extra_content = - part.thought_signature.map(|sig| ExtraContent { - google: Some(GoogleExtraContent { - thought_signature: Some(sig), - }), - }); + let extra_content = tool_call.thought_signature.map(|sig| ExtraContent { + google: Some(GoogleExtraContent { thought_signature: Some(sig) }), + }); - // Store accumulated tool call - self.accumulated_tool_calls.insert( - idx, - OpenAIToolCall { - id: call_id, - function: OpenAIFunction { - name: function_call.name, - arguments: serde_json::to_string( - &function_call.args, - ) - .unwrap_or_else(|_| "{}".to_string()), - }, - r#type: "function".to_string(), - extra_content, - }, - ); - } - } - } - } + self.accumulated_tool_calls.insert( + idx, + OpenAIToolCall { + id: call_id, + function: OpenAIFunction { + name: tool_call.name, + arguments: serde_json::to_string(&tool_call.args) + .unwrap_or_else(|_| "{}".to_string()), + }, + r#type: "function".to_string(), + extra_content, + }, + ); + } - // Handle grounding metadata (web search results) - if let Some(ref grounding_metadata) = candidate.grounding_metadata { - // Set used_websearch if there are search queries or grounding chunks - if !grounding_metadata.web_search_queries.is_empty() - || !grounding_metadata.grounding_chunks.is_empty() - { - self.used_websearch = true; - } - - // Extract citations from grounding chunks - for chunk in &grounding_metadata.grounding_chunks { - if let Some(ref web) = chunk.web { - self.annotations.push(UrlCitation { - start_index: 0, // Gemini doesn't provide character indices - end_index: 0, - url: web.uri.clone(), - title: web.title.clone(), - }); - } - } - } - } - } - - // Extract usage metadata - if let Some(usage_metadata) = event.usage_metadata { - self.usage = Some(usage_metadata); - } + self.annotations.extend(parsed.annotations); + if parsed.used_websearch { + self.used_websearch = true; + } + if let Some(usage) = parsed.usage { + self.usage = Some(usage); } Ok(()) diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index a374db29c5..d095602b40 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -731,16 +731,3 @@ pub fn extract_text_content(content: &OpenAIContent) -> String { .join(""), } } - -/// Parse a data URL to extract media type and base64 data -/// Format: data:mime_type;base64,data -/// Returns (media_type, data) tuple if successful -pub fn parse_data_url(url: &str) -> Option<(String, String)> { - if !url.starts_with("data:") { - return None; - } - let rest = url.strip_prefix("data:")?; - let (header, data) = rest.split_once(",")?; - let media_type = header.strip_suffix(";base64")?; - Some((media_type.to_string(), data.to_string())) -} diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index aa250de4ab..253384f6f2 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -33,9 +33,9 @@ use crate::{ read_and_check_result, start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, + is_sandboxing_enabled, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, - is_sandboxing_enabled, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, - PY_INSTALL_DIR, TZ_ENV, + DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -1184,7 +1184,7 @@ mount {{ job_dir, "run.config.proto", &NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT - .replace("{PY_INSTALL_DIR}", PY_INSTALL_DIR) + .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 64270b4044..7d92607af5 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -26,7 +26,12 @@ use windmill_queue::{ }; lazy_static::lazy_static! { - pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string()); + pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| { + #[cfg(not(windows))] + { "/bin/bash".to_string() } + #[cfg(windows)] + { "bash".to_string() } + }); } const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto"); @@ -194,7 +199,7 @@ exit $exit_status .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; let mut cmd_args = vec![ @@ -210,7 +215,10 @@ exit $exit_status .current_dir(job_dir) .env_clear() .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bash).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .args(cmd_args) @@ -236,7 +244,10 @@ exit $exit_status .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bash).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 9c46db572d..6ada66c4bb 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -22,7 +22,7 @@ use crate::{ handle_child::handle_child, is_sandboxing_enabled, read_ee_registry, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, - NPMRC, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + NPMRC, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; use windmill_common::{ @@ -53,7 +53,14 @@ use windmill_object_store::attempt_fetch_bytes; use windmill_parser::Typ; +// The Windows loader uses a virtual "windmill-url" namespace instead of writing .url +// files to disk, which avoids Windows path issues. The virtual namespace approach is +// likely better on all fronts but we keep the original .url-file loader on Linux to +// avoid breaking back-compat. +#[cfg(not(windows))] pub const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.js"); +#[cfg(windows)] +pub const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.windows.js"); pub const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js"); @@ -527,6 +534,8 @@ pub async fn build_loader( current_path: &str, mode: LoaderMode, ) -> Result<()> { + // Use forward slashes in JS strings to avoid backslash escape issues on Windows + let job_dir_js = job_dir.replace('\\', "/"); let loader = RELATIVE_BUN_LOADER .replace("W_ID", w_id) .replace("BASE_INTERNAL_URL", base_internal_url) @@ -549,13 +558,13 @@ import {{ readdir }} from "node:fs/promises"; let fileNames = [] try {{ - fileNames = await readdir("{job_dir}/node_modules") + fileNames = await readdir("{job_dir_js}/node_modules") }} catch (e) {{ }} try {{ await Bun.build({{ - entrypoints: ["{job_dir}/wrapper.mjs"], + entrypoints: ["{job_dir_js}/wrapper.mjs"], outdir: "./", target: "node", plugins: [p], @@ -597,7 +606,7 @@ plugin(p) try {{ await Bun.build({{ - entrypoints: ["{job_dir}/main.ts"], + entrypoints: ["{job_dir_js}/main.ts"], outdir: "./", target: "{}", plugins: [p], @@ -737,7 +746,7 @@ async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result error::Result> { let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); + if annotation.sandbox && NSJAIL_AVAILABLE.is_none() { + return Err(error::Error::ExecutionErr( + "Script has //sandbox annotation but nsjail is not available on this worker. \ + Please ensure nsjail is installed or remove the //sandbox annotation." + .to_string(), + )); + } + let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if let (Some(lock), true) = ( maybe_lock.get_lock(), !annotation.nobundling && !*DISABLE_BUNDLING && codebase.is_none(), @@ -1019,6 +1036,12 @@ pub async fn handle_bun_job( let _ = write_file(job_dir, "main.ts", inner_content)?; } else if !annotation.native && codebase.is_none() { let _ = write_file(job_dir, "package.json", r#"{ "type": "module" }"#)?; + } else if codebase.is_some() { + // Write a valid fallback package.json for codebase mode. Without this, + // nsjail creates an empty 0-byte file (from the mandatory: false mount) + // which Node.js fails to parse as JSON (ERR_INVALID_PACKAGE_CONFIG). + // If the codebase TAR includes a package.json, it will overwrite this. + let _ = write_file(job_dir, "package.json", "{}")?; }; let common_bun_proc_envs: HashMap = @@ -1152,6 +1175,10 @@ pub async fn handle_bun_job( init_logs = format!("\n{}{}", cache_logs, init_logs); } + if annotation.sandbox { + init_logs.push_str("sandbox mode (nsjail)\n"); + } + let write_wrapper_f = async { if !has_bundle_cache && annotation.native { return Ok(()) as error::Result<()>; @@ -1476,7 +1503,7 @@ try {{ append_logs(&job.id, &job.workspace_id, init_logs, conn).await; //do not cache local dependencies - let child = if is_sandboxing_enabled() { + let child = if is_sandboxing_enabled() || annotation.sandbox { let _ = write_file( job_dir, "run.config.proto", @@ -1495,7 +1522,7 @@ try {{ }, ), ) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; @@ -1537,7 +1564,9 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bun).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn).await?, + ) .envs(common_bun_proc_envs) .env("PATH", PATH_ENV.as_str()) .args(args) @@ -1555,7 +1584,10 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bun).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn) + .await?, + ) .envs(common_bun_proc_envs) .stdin(Stdio::null()) .stdout(Stdio::piped()) @@ -1586,7 +1618,10 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Bun).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn) + .await?, + ) .envs(common_bun_proc_envs) .stdin(Stdio::null()) .stdout(Stdio::piped()) diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index e03f90525e..846c421024 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -15,10 +15,6 @@ use tokio::process::Command; use tokio::{fs::File, io::AsyncReadExt}; use windmill_common::flows::Step; -#[cfg(feature = "parquet")] -use windmill_types::s3::{LargeFileStorage, ObjectStoreResource, S3Object}; -#[cfg(feature = "parquet")] -use windmill_object_store::get_etag_or_empty; use windmill_common::variables::{build_crypt_with_key_suffix, decrypt}; use windmill_common::worker::{ to_raw_value, update_ping_for_failed_init_script_query, write_file, Connection, Ping, PingType, @@ -32,6 +28,10 @@ use windmill_common::{ utils::configure_client, variables::ContextualVariable, }; +#[cfg(feature = "parquet")] +use windmill_object_store::get_etag_or_empty; +#[cfg(feature = "parquet")] +use windmill_types::s3::{LargeFileStorage, ObjectStoreResource, S3Object}; use anyhow::{anyhow, Result}; use windmill_parser_sql::{s3_mode_extension, S3ModeArgs, S3ModeFormat}; @@ -886,7 +886,7 @@ pub async fn cached_result_path( } #[cfg(feature = "parquet")] -async fn get_workspace_s3_resource_path( +pub(crate) async fn get_workspace_s3_resource_path( db: &DB, client: &AuthedClient, workspace_id: &str, @@ -948,7 +948,11 @@ async fn get_workspace_s3_resource_path( ) } Some(LargeFileStorage::FilesystemStorage(fs)) => { - (StorageResourceType::Filesystem, fs.root_path.clone()) + return Ok(Some( + windmill_object_store::ObjectStoreResource::Filesystem( + windmill_object_store::FilesystemSettings { root_path: fs.root_path.clone() }, + ), + )); } None => { return Ok(None); @@ -1090,7 +1094,7 @@ fn tentatively_improve_error(err: Error, executable: &str) -> Error { pub async fn clean_cache() -> error::Result<()> { tracing::info!("Started cleaning cache"); - tokio::fs::remove_dir_all(ROOT_CACHE_DIR).await?; + tokio::fs::remove_dir_all(&*ROOT_CACHE_DIR).await?; tracing::info!("Finished cleaning cache"); Ok(()) } @@ -1557,4 +1561,3 @@ mod tests { assert!(result.is_err()); } } - diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index e4c32648d0..22f46e724b 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -13,14 +13,11 @@ use itertools::Itertools; #[cfg(feature = "csharp")] use tokio::{fs::File, io::AsyncReadExt, process::Command}; #[cfg(feature = "csharp")] -use windmill_common::{ - utils::calculate_hash, - worker::write_file, -}; +use windmill_common::{utils::calculate_hash, worker::write_file}; -use windmill_common::error::{self, Error}; #[cfg(feature = "csharp")] use crate::global_cache::save_cache; +use windmill_common::error::{self, Error}; #[cfg(feature = "csharp")] use windmill_queue::append_logs; @@ -105,8 +102,8 @@ pub async fn generate_nuget_lockfile( let mut gen_lockfile_cmd = Command::new(DOTNET_PATH.as_str()); gen_lockfile_cmd .current_dir(job_dir) - .env("DOTNET_CLI_HOME", CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{CSHARP_CACHE_DIR}/nuget")) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") .env("DOTNET_NOLOGO", "true") .env("MSBUILDDISABLENODEREUSE", "1") @@ -367,8 +364,8 @@ async fn build_cs_proj( .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .env("DOTNET_CLI_HOME", CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{CSHARP_CACHE_DIR}/nuget")) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") .env("DOTNET_NOLOGO", "true") .env("MSBUILDDISABLENODEREUSE", "1") @@ -434,7 +431,7 @@ async fn build_cs_proj( } } - let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR); + let bin_path = format!("{}/{hash}", *CSHARP_CACHE_DIR); #[cfg(unix)] let target = format!("{job_dir}/Main"); #[cfg(windows)] @@ -516,11 +513,10 @@ pub async fn handle_csharp_job( inner_content, requirements_o.unwrap_or(&String::new()) )); - let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR); + let bin_path = format!("{}/{hash}", *CSHARP_CACHE_DIR); let remote_path = format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"); - let (cache, cache_logs) = - crate::global_cache::load_cache(&bin_path, &remote_path, false).await; + let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { #[cfg(unix)] @@ -591,11 +587,11 @@ pub async fn handle_csharp_job( "run.config.proto", &NSJAIL_CONFIG_RUN_CSHARP_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", CSHARP_CACHE_DIR) + .replace("{CACHE_DIR}", &*CSHARP_CACHE_DIR) .replace("{CACHE_HASH}", &hash) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -604,12 +600,15 @@ pub async fn handle_csharp_job( .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) - .env("DOTNET_CLI_HOME", CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{CSHARP_CACHE_DIR}/nuget")) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") .env("DOTNET_NOLOGO", "true") .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) @@ -637,11 +636,14 @@ pub async fn handle_csharp_job( .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) - .env("DOTNET_CLI_HOME", CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{CSHARP_CACHE_DIR}/nuget")) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") .env("DOTNET_NOLOGO", "true") .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 66d8ae8673..d9f6785eb6 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -17,6 +17,7 @@ use crate::{ NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, }; use windmill_common::client::AuthedClient; +use windmill_common::worker::TypeScriptAnnotations; use tokio::{fs::File, io::AsyncReadExt, process::Command}; use windmill_common::{error::Result, scripts::ScriptLang, worker::write_file, BASE_URL}; @@ -120,11 +121,13 @@ async fn get_common_deno_proc_envs( } // Add proxy envs (including OTEL tracing proxy if enabled for deno) - for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno) - .await - .unwrap_or_default() - { - deno_envs.insert(k.to_string(), v); + if let Some(conn) = conn { + for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno, job_id, w_id, conn) + .await + .unwrap_or_default() + { + deno_envs.insert(k.to_string(), v); + } } return deno_envs; @@ -231,8 +234,13 @@ pub async fn handle_deno_job( occupancy_metrics: &mut OccupancyMetrics, has_stream: &mut bool, ) -> error::Result> { + let annotations = TypeScriptAnnotations::parse(inner_content); + // let mut start = Instant::now(); - let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); + let mut logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); + if annotations.sandbox { + logs1.push_str("sandbox mode (nsjail)\n"); + } append_logs(&job.id, &job.workspace_id, logs1, conn).await; let main_override = job.script_entrypoint_override.as_deref(); @@ -443,14 +451,15 @@ try {{ } let allow_read = format!( - "--allow-read=./,/tmp/windmill/cache/deno/,{}", + "--allow-read=./,{}/,{}", + *DENO_CACHE_DIR, DENO_PATH.as_str() ); if let Some(deno_flags) = DENO_FLAGS.as_ref() { for flag in deno_flags { args.push(flag); } - } else if is_sandboxing_enabled() { + } else if is_sandboxing_enabled() || annotations.sandbox { args.push("--allow-net"); args.push("--allow-sys"); args.push(allow_read.as_str()); @@ -504,7 +513,8 @@ try {{ *has_stream = handle_result.result_stream.is_some(); // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str()); - if let Err(e) = tokio::fs::remove_dir_all(format!("{DENO_CACHE_DIR}/gen/file/{job_dir}")).await + if let Err(e) = + tokio::fs::remove_dir_all(format!("{}/gen/file/{job_dir}", *DENO_CACHE_DIR)).await { tracing::error!("failed to remove deno gen tmp cache dir: {}", e); } diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 73136e1cc0..45e2f647a5 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -161,6 +161,22 @@ pub async fn do_duckdb( let base_internal_url = client.base_internal_url.clone(); let w_id = job.workspace_id.clone(); + if annotations.prepare { + let result = tokio::task::spawn_blocking(move || { + prepare_duckdb_ffi_safe( + query_block_list.iter().map(String::as_str), + &token, + &base_internal_url, + &w_id, + ) + }) + .await + .map_err(|e| Error::from(to_anyhow(e))) + .and_then(|r| r)?; + + return Ok(result); + } + let result = tokio::task::spawn_blocking(move || { run_duckdb_ffi_safe( query_block_list.iter().map(String::as_str), @@ -248,6 +264,18 @@ struct DuckDbFfiLib { collect_first_row_only: bool, ) -> *mut c_char, >, + prepare_duckdb_ffi: Option< + Symbol< + 'static, + unsafe extern "C" fn( + query_block_list: *const *const c_char, + query_block_list_count: usize, + token: *const c_char, + base_internal_url: *const c_char, + w_id: *const c_char, + ) -> *mut c_char, + >, + >, free_cstr: Symbol<'static, unsafe extern "C" fn(string: *mut c_char) -> ()>, } @@ -307,8 +335,11 @@ impl DuckDbFfiLib { } } + let prepare_duckdb_ffi = unsafe { lib.get(b"prepare_duckdb_ffi").ok() }; + Ok(DuckDbFfiLib { run_duckdb_ffi: unsafe { lib.get(b"run_duckdb_ffi").map_err(to_anyhow)? }, + prepare_duckdb_ffi, free_cstr: unsafe { lib.get(b"free_cstr").map_err(to_anyhow)? }, }) } @@ -388,6 +419,56 @@ fn run_duckdb_ffi_safe<'a>( } } +fn prepare_duckdb_ffi_safe<'a>( + query_block_list: impl Iterator, + token: &str, + base_internal_url: &str, + w_id: &str, +) -> Result> { + let query_block_list = query_block_list + .map(|s| { + CString::new(s).map_err(|e| { + Error::ExecutionErr(format!("Failed CString conversion: {}", e.to_string())) + }) + }) + .collect::>>()?; + let query_block_list = query_block_list + .iter() + .map(|s| s.as_ptr()) + .collect::>(); + + let token = CString::new(token).map_err(to_anyhow)?; + let base_internal_url = CString::new(base_internal_url).map_err(to_anyhow)?; + let w_id = CString::new(w_id).map_err(to_anyhow)?; + + let lib = DuckDbFfiLib::get_singleton()?; + let prepare_fn = lib.prepare_duckdb_ffi.as_ref().ok_or_else(|| { + Error::InternalErr( + "prepare_duckdb_ffi not available in duckdb ffi library. Please update to the latest windmill_duckdb_ffi_lib.".to_string(), + ) + })?; + let free_cstr = &lib.free_cstr; + + let result_str = unsafe { + let ptr = prepare_fn( + query_block_list.as_ptr(), + query_block_list.len(), + token.as_ptr(), + base_internal_url.as_ptr(), + w_id.as_ptr(), + ); + let str = CStr::from_ptr(ptr).to_string_lossy().to_string(); + free_cstr(ptr); + str + }; + + if result_str.starts_with("ERROR") { + Err(Error::ExecutionErr(result_str[6..].to_string())) + } else { + Ok(serde_json::value::RawValue::from_string(result_str).map_err(to_anyhow)?) + } +} + struct ParsedAttachDbResource<'a> { resource_path: &'a str, name: &'a str, diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 812be2fc7a..ea5a9ded46 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -18,8 +18,8 @@ pub async fn build_tar_and_push( custom_folder_name: Option, platform_agnostic: bool, ) -> error::Result<()> { - use windmill_object_store::object_store_reexports::Path; use tokio::fs::create_dir_all; + use windmill_object_store::object_store_reexports::Path; use crate::TAR_PYBASE_CACHE_DIR; @@ -33,7 +33,7 @@ pub async fn build_tar_and_push( folder.split("/").last().unwrap().to_owned() }; - let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", lang); + let prefix = &format!("{}/{}", *TAR_PYBASE_CACHE_DIR, lang); let tar_path = format!("{prefix}/{folder_name}_tar.tar"); create_dir_all(prefix).await?; @@ -197,7 +197,9 @@ pub async fn exists_in_cache(bin_path: &str, _remote_path: &str) -> bool { #[cfg(all(feature = "enterprise", feature = "parquet"))] if let Some(os) = windmill_object_store::get_object_store().await { return os - .get(&windmill_object_store::object_store_reexports::Path::from(_remote_path)) + .get(&windmill_object_store::object_store_reexports::Path::from( + _remote_path, + )) .await .is_ok(); } @@ -221,7 +223,7 @@ pub async fn save_cache( let file_to_cache = if is_dir { let tar_path = format!( "{}/tar/{}_tar.tar", - windmill_common::worker::ROOT_CACHE_DIR, + *windmill_common::worker::ROOT_CACHE_DIR, local_cache_path .split("/") .last() diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 35307baa72..0315bf25c3 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -2,6 +2,7 @@ use crate::{common::MaybeLock, get_proxy_envs_for_lang}; use std::{collections::HashMap, fs::DirBuilder, process::Stdio}; use windmill_common::scripts::ScriptLang; +use crate::global_cache::save_cache; use itertools::Itertools; use serde_json::value::RawValue; use tokio::{ @@ -15,7 +16,6 @@ use windmill_common::{ utils::calculate_hash, worker::{write_file, Connection, GoAnnotations}, }; -use crate::global_cache::save_cache; use windmill_parser_go::{parse_go_imports, REQUIRE_PARSE}; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -53,8 +53,7 @@ fn get_windows_program_files() -> String { #[cfg(windows)] fn windows_gopath() -> String { - let tmp_dir = get_windows_tmp_dir(); - GO_CACHE_DIR.replace("/tmp", &tmp_dir).replace("/", r"\\") + GO_CACHE_DIR.replace('/', "\\") } #[cfg(windows)] @@ -108,10 +107,9 @@ pub async fn handle_go_job( .expect("could not create go job dir"); let hash = calculate_hash(&format!("{}{:?}v2", inner_content, &maybe_lock)); - let bin_path = format!("{}/{hash}", GO_BIN_CACHE_DIR); + let bin_path = format!("{}/{hash}", *GO_BIN_CACHE_DIR); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); - let (cache, cache_logs) = - crate::global_cache::load_cache(&bin_path, &remote_path, false).await; + let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let (skip_go_mod, skip_tidy) = if cache { (true, true) @@ -238,15 +236,15 @@ func Run(req Req) (interface{{}}, error){{ .env("GOPATH", { #[cfg(unix)] { - GO_CACHE_DIR + GO_CACHE_DIR.as_str() } #[cfg(windows)] { - windows_gopath() + &windows_gopath() } }) .env("HOME", HOME_ENV.as_str()) - .env("GOCACHE", GO_CACHE_DIR) + .env("GOCACHE", GO_CACHE_DIR.as_str()) .envs(PROXY_ENVS.clone()) .args(vec!["build", "main.go"]) .stdout(Stdio::piped()) @@ -347,7 +345,7 @@ func Run(req Req) (interface{{}}, error){{ .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -356,7 +354,7 @@ func Run(req Req) (interface{{}}, error){{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Go).await?) + .envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -377,18 +375,18 @@ func Run(req Req) (interface{{}}, error){{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Go).await?) + .envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("GOPATH", { #[cfg(unix)] { - GO_CACHE_DIR + GO_CACHE_DIR.as_str() } #[cfg(windows)] { - windows_gopath() + &windows_gopath() } }) .env("HOME", HOME_ENV.as_str()); @@ -508,7 +506,7 @@ pub async fn install_go_dependencies( #[cfg(windows)] child_cmd.env("GOPATH", windows_gopath()); #[cfg(unix)] - child_cmd.env("GOPATH", GO_CACHE_DIR); + child_cmd.env("GOPATH", GO_CACHE_DIR.as_str()); #[cfg(windows)] set_windows_env_vars(&mut child_cmd); @@ -591,11 +589,11 @@ pub async fn install_go_dependencies( .env("GOPATH", { #[cfg(unix)] { - GO_CACHE_DIR + GO_CACHE_DIR.as_str() } #[cfg(windows)] { - windows_gopath() + &windows_gopath() } }) .args(vec!["mod", mod_command]) diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 49129848ce..6f558e3328 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, path::PathBuf, process::Stdio}; +use crate::global_cache::save_cache; use anyhow::{anyhow, bail}; use async_recursion::async_recursion; use itertools::Itertools; @@ -15,7 +16,6 @@ use windmill_common::{ utils::calculate_hash, worker::{copy_dir_recursively, write_file, Connection}, }; -use crate::global_cache::save_cache; use windmill_parser::Arg; use windmill_parser_java::parse_java_sig_meta; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -185,8 +185,8 @@ pub async fn resolve<'a>( cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) - .env("COURSIER_CACHE", COURSIER_CACHE_DIR) + .env("HOME", &*JAVA_HOME_DIR) + .env("COURSIER_CACHE", &*COURSIER_CACHE_DIR) .envs(PROXY_ENVS.clone()); // Configure proxies @@ -208,7 +208,7 @@ pub async fn resolve<'a>( cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val)); } } - cmd.arg(&format!("-Duser.home={}", JAVA_HOME_DIR)); + cmd.arg(&format!("-Duser.home={}", *JAVA_HOME_DIR)); if metadata(TRUST_STORE_PATH.clone()).await.is_ok() { cmd.args(&[ &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), @@ -223,7 +223,7 @@ pub async fn resolve<'a>( "--parallel", &format!("{}", *JAVA_CONCURRENT_DOWNLOADS), "--cache", - COURSIER_CACHE_DIR, + &*COURSIER_CACHE_DIR, ]) .args(&get_repos(job_id, w_id, conn).await) .args(&deps.split("\n").collect_vec()) @@ -276,7 +276,8 @@ async fn install<'a>( match (it.next(), it.next(), it.next()) { (Some(group_id), Some(artifact_id), Some(version)) => { let path = format!( - "{JAVA_REPOSITORY_DIR}/{}/{artifact_id}/{version}", + "{}/{}/{artifact_id}/{version}", + *JAVA_REPOSITORY_DIR, group_id.replace(".", "/") ); Ok(RequiredDependency { @@ -312,7 +313,7 @@ async fn install<'a>( metadata(TRUST_STORE_PATH.clone()).await, ); let job_dir = job_dir.to_owned(); - let fetch_dir = format!("{JAVA_CACHE_DIR}/tmp-fetch-{}", Uuid::new_v4()); + let fetch_dir = format!("{}/tmp-fetch-{}", *JAVA_CACHE_DIR, Uuid::new_v4()); let fetch_dir2 = fetch_dir.clone(); par_install_language_dependencies_all_at_once( deps, @@ -334,8 +335,8 @@ async fn install<'a>( cmd.env_clear() .current_dir(&job_dir) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) - .env("COURSIER_CACHE", COURSIER_CACHE_DIR) + .env("HOME", &*JAVA_HOME_DIR) + .env("COURSIER_CACHE", &*COURSIER_CACHE_DIR) .envs(PROXY_ENVS.clone()); // Configure proxies { @@ -357,7 +358,7 @@ async fn install<'a>( } } - cmd.arg(&format!("-Duser.home={}", JAVA_HOME_DIR)); + cmd.arg(&format!("-Duser.home={}", *JAVA_HOME_DIR)); if trust_store_metadata.is_ok() { cmd.args(&[ &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), @@ -400,7 +401,7 @@ async fn install<'a>( if depth == 3 { copy_dir_recursively( &PathBuf::from(path), - &PathBuf::from(JAVA_REPOSITORY_DIR), + &PathBuf::from(&*JAVA_REPOSITORY_DIR), )?; return Ok(()); @@ -465,7 +466,7 @@ async fn compile<'a>( let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?; let hash = compute_hash(inner_content, *requirements_o); - let bin_path = format!("{}/{hash}", JAVA_CACHE_DIR); + let bin_path = format!("{}/{hash}", *JAVA_CACHE_DIR); let remote_path = format!("java_jar/{hash}"); let (cache, ..) = crate::global_cache::load_cache(&bin_path, &remote_path, true).await; @@ -501,7 +502,7 @@ async fn compile<'a>( cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) + .env("HOME", &*JAVA_HOME_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) @@ -604,7 +605,7 @@ async fn run<'a>( "run.config.proto", &NSJAIL_CONFIG_RUN_JAVA_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", JAVA_CACHE_DIR) + .replace("{CACHE_DIR}", &*JAVA_CACHE_DIR) .replace("{SHARED_MOUNT}", &shared_mount) // .replace("{CACHED_TARGET}", &shared_mount) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), @@ -613,10 +614,11 @@ async fn run<'a>( cmd.env_clear() .current_dir(job_dir) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) + .env("HOME", &*JAVA_HOME_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)) .args(vec![ "--config", "run.config.proto", @@ -671,10 +673,11 @@ async fn run<'a>( cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) + .env("HOME", &*JAVA_HOME_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) - .envs(reserved_variables); + .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)); if metadata(TRUST_STORE_PATH.clone()).await.is_ok() { cmd.args(&[ &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 741458c0e0..f6d752558d 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -70,6 +70,9 @@ mod sanitized_sql_params; mod schema; pub mod sql_utils; mod universal_pkg_installer; +#[cfg(feature = "private")] +mod volume_ee; +mod volume_oss; mod worker; mod worker_flow; mod worker_lockfiles; diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index a4320a3612..28ac27c925 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -16,8 +16,8 @@ use crate::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, - get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, - TRACING_PROXY_CA_CERT_PATH, + get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, + PATH_ENV, TRACING_PROXY_CA_CERT_PATH, }; use windmill_common::client::AuthedClient; use windmill_common::scripts::ScriptLang; @@ -253,7 +253,7 @@ async fn run<'a>( .replace("{NU_PATH}", &NU_PATH) .replace("{SHARED_MOUNT}", &shared_mount) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -264,7 +264,7 @@ async fn run<'a>( .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Nu).await?) + .envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?) .args(vec![ "--config", "run.config.proto", @@ -303,7 +303,7 @@ async fn run<'a>( .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Nu).await?) + .envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?) // TODO(v1): // "--plugins", // &format!( diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index e98755e555..df006fef7f 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -22,7 +22,7 @@ use crate::{ get_reserved_variables, read_result, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, - COMPOSER_CACHE_DIR, COMPOSER_PATH, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, + is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, }; use windmill_common::client::AuthedClient; @@ -316,6 +316,7 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)) .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .args(args) @@ -332,6 +333,7 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)) .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .stdin(Stdio::null()) diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index acae81a588..961a49a58a 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -159,7 +159,7 @@ try { async fn scan_module_directories() -> Result, Error> { let mut module_dirs = HashMap::new(); - let cache_dir = std::path::Path::new(POWERSHELL_CACHE_DIR); + let cache_dir = std::path::Path::new(&*POWERSHELL_CACHE_DIR); if let Ok(entries) = fs::read_dir(cache_dir) { for entry in entries { @@ -391,7 +391,7 @@ pub async fn handle_powershell_job( .join(", "); let install_string = generate_powershell_install_code() - .replace("{path}", POWERSHELL_CACHE_DIR) + .replace("{path}", &*POWERSHELL_CACHE_DIR) .replace("{job_id}", &job.id.to_string()) .replace("{has_private_repo}", &format!("${has_private_repo}")) .replace("{has_credentials}", &format!("${has_credentials}")) @@ -442,7 +442,7 @@ $PSModulePathBackup = $env:PSModulePath $env:PSModulePath = \"$PSHome/Modules\" Get-Module -ListAvailable | Import-Module $env:PSModulePath = \"{}:$PSModulePathBackup\"", - POWERSHELL_CACHE_DIR + *POWERSHELL_CACHE_DIR ); #[cfg(windows)] @@ -452,7 +452,7 @@ $PSModulePathBackup = $env:PSModulePath $env:PSModulePath = \"C:\\Program Files\\PowerShell\\7\\Modules\" Get-Module -ListAvailable | Import-Module $env:PSModulePath = \"{};$PSModulePathBackup\"", - POWERSHELL_CACHE_DIR + *POWERSHELL_CACHE_DIR ); // NOTE: powershell error handling / termination is quite tricky compared to bash @@ -525,7 +525,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) - .replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR), + .replace("{CACHE_DIR}", &*POWERSHELL_CACHE_DIR), )?; let cmd_args = vec![ "--config", diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 047521a0b4..320bcb3e40 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -134,8 +134,8 @@ use crate::{ handle_child::handle_child, is_sandboxing_enabled, read_ee_registry, worker_utils::ping_job_status, - PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, - PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, + PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, + PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, UV_INDEX_STRATEGY, }; use windmill_common::client::AuthedClient; @@ -278,7 +278,7 @@ pub async fn uv_pip_compile( "requirements.txt", // Target to /tmp/windmill/cache/uv "--cache-dir", - UV_CACHE_DIR, + &*UV_CACHE_DIR, ]; args.extend(["-p", &py_version_str, "--python-preference", "only-managed"]); @@ -567,6 +567,14 @@ pub async fn handle_python_job( let annotations = PythonAnnotations::parse(inner_content); + if annotations.sandbox && NSJAIL_AVAILABLE.is_none() { + return Err(Error::ExecutionErr( + "Script has #sandbox annotation but nsjail is not available on this worker. \ + Please ensure nsjail is installed or remove the #sandbox annotation." + .to_string(), + )); + } + let (py_version, mut additional_python_paths) = handle_python_deps( job_dir, requirements_o, @@ -605,16 +613,14 @@ pub async fn handle_python_job( } { - append_logs( - &job.id, - &job.workspace_id, - format!( - "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", - py_version.clone().to_string() - ), - conn, - ) - .await; + let mut logs = format!( + "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", + py_version.clone().to_string() + ); + if annotations.sandbox { + logs.push_str("sandbox mode (nsjail)\n"); + } + append_logs(&job.id, &job.workspace_id, logs, conn).await; } let ( import_loader, @@ -784,7 +790,7 @@ except BaseException as e: #[cfg(windows)] let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";"); - if is_sandboxing_enabled() { + if is_sandboxing_enabled() || annotations.sandbox { let shared_deps = additional_python_paths .into_iter() .map(|pp| { @@ -805,7 +811,7 @@ mount {{ "run.config.proto", &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{PY_INSTALL_DIR}", PY_INSTALL_DIR) + .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str()) @@ -815,7 +821,7 @@ mount {{ "{ADDITIONAL_PYTHON_PATHS}", additional_python_paths_folders.as_str(), ) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; } else { @@ -828,14 +834,17 @@ mount {{ job.id ); - let child = if is_sandboxing_enabled() { + let child = if is_sandboxing_enabled() || annotations.sandbox { let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .current_dir(job_dir) .env_clear() // inject PYTHONPATH here - for some reason I had to do it in nsjail conf .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Python3).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -861,7 +870,10 @@ mount {{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Python3).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -1410,10 +1422,10 @@ async fn spawn_uv_install( &nsjail_proto, NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT .replace("{WORKER_DIR}", worker_dir) - .replace("{PY_INSTALL_DIR}", &PY_INSTALL_DIR) + .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{TARGET_DIR}", &venv_p) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .as_str(), )?; diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index 52636808a6..78b03deb07 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -18,6 +18,8 @@ use windmill_common::{ use anyhow::{anyhow, bail}; use windmill_queue::append_logs; +#[cfg(unix)] +use crate::python_executor::UV_PATH; use crate::{ common::{start_child_process, OccupancyMetrics}, handle_child::handle_child, @@ -25,8 +27,6 @@ use crate::{ HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, UV_CACHE_DIR, WIN_ENVS, }; -#[cfg(unix)] -use crate::python_executor::UV_PATH; impl From for PyVAlias { fn from(value: PyV) -> Self { @@ -234,7 +234,8 @@ impl PyV { pub(crate) fn to_cache_dir(&self, ignore_patch: bool) -> String { use windmill_common::worker::ROOT_CACHE_DIR; format!( - "{ROOT_CACHE_DIR}{}", + "{}{}", + *ROOT_CACHE_DIR, self.to_cache_dir_top_level(ignore_patch) ) } @@ -311,7 +312,7 @@ impl PyV { Command::new(uv_cmd) .env_clear() .envs(WIN_ENVS.to_vec()) - .env("UV_CACHE_DIR", UV_CACHE_DIR) + .env("UV_CACHE_DIR", &*UV_CACHE_DIR) .args([ "python", "list", @@ -539,8 +540,8 @@ impl PyV { ]) // TODO: Do we need these? .envs([ - ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), - ("UV_CACHE_DIR", UV_CACHE_DIR), + ("UV_PYTHON_INSTALL_DIR", &*PY_INSTALL_DIR), + ("UV_CACHE_DIR", &*UV_CACHE_DIR), ]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -630,11 +631,9 @@ impl PyV { "--system", "--python-preference=only-managed", ]) - .envs([ - ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), - ("UV_PYTHON_PREFERENCE", "only-managed"), - ("UV_CACHE_DIR", UV_CACHE_DIR), - ]) + .env("UV_PYTHON_INSTALL_DIR", &*PY_INSTALL_DIR) + .env("UV_PYTHON_PREFERENCE", "only-managed") + .env("UV_CACHE_DIR", &*UV_CACHE_DIR) // .stdout(Stdio::piped()) .stderr(Stdio::piped()) .output() diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index 869b58d7de..d283b2a1c4 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -1,7 +1,6 @@ use std::{collections::HashMap, process::Stdio}; use anyhow::anyhow; -use const_format::concatcp; use itertools::Itertools; use regex::Regex; use tokio::{ @@ -122,7 +121,7 @@ pub async fn prepare<'a>( .write_all(&wrap(inner_content)?.into_bytes()) .await?; - let mini_wm_path = format!("{RUBY_CACHE_DIR}/gems/windmill-internal/windmill"); + let mini_wm_path = format!("{}/gems/windmill-internal/windmill", *RUBY_CACHE_DIR); if !std::fs::metadata(&mini_wm_path).is_ok() { fs::create_dir_all(&mini_wm_path).await?; @@ -339,7 +338,7 @@ Your Gemfile syntax will continue to work as-is." &NSJAIL_CONFIG_LOCK_RUBY_CONTENT .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); @@ -588,7 +587,7 @@ async fn install<'a>( // 123...zx-activesupport-8.0.2 // ^^^^^^^^ hash based on source and type (GEM or GIT) let handle = format!("{}-{}-{}", hash, pkg, version); - let path = format!("{RUBY_CACHE_DIR}/gems/{}", &handle); + let path = format!("{}/gems/{}", *RUBY_CACHE_DIR, &handle); deps.push(RequiredDependency { path, @@ -632,7 +631,7 @@ async fn install<'a>( &NSJAIL_CONFIG_DOWNLOAD_RUBY_CONTENT .replace("{TARGET}", &dependency.path) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); @@ -741,9 +740,9 @@ async fn install<'a>( }; // Include builtin windmill client { - const WM_INTERNAL: &str = concatcp!(RUBY_CACHE_DIR, "/gems/windmill-internal"); - res.top_level_paths.push(WM_INTERNAL.to_owned()); - res.rubylib += format!(":{WM_INTERNAL}").as_str(); + let wm_internal = format!("{}/gems/windmill-internal", *RUBY_CACHE_DIR); + res.top_level_paths.push(wm_internal.clone()); + res.rubylib += format!(":{wm_internal}").as_str(); } Ok(res) } @@ -800,7 +799,7 @@ mount {{ .replace("{JOB_DIR}", job_dir) .replace("{SHARED_MOUNT}", &shared_mount) .replace("{SHARED_DEPENDENCIES}", &shared_deps) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; @@ -813,7 +812,10 @@ mount {{ .envs(envs) .envs(reserved_variables) .envs(RUBY_PROXY_ENVS.clone()) - .envs(get_proxy_envs_for_lang(&ScriptLang::Ruby).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn) + .await?, + ) .args(vec![ "--config", "run.config.proto", @@ -852,7 +854,10 @@ mount {{ .env("BASE_INTERNAL_URL", base_internal_url) .envs(reserved_variables) .envs(RUBY_PROXY_ENVS.clone()) - .envs(get_proxy_envs_for_lang(&ScriptLang::Ruby).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn) + .await?, + ) .envs(envs); cmd.stdin(Stdio::null()) diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 54d90550b0..dae8e08766 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -5,6 +5,7 @@ use std::{collections::HashMap, process::Stdio}; use uuid::Uuid; use windmill_parser_rust::parse_rust_deps_into_manifest; +use crate::global_cache::save_cache; use itertools::Itertools; use tokio::{ fs::{create_dir_all, File}, @@ -16,7 +17,6 @@ use windmill_common::{ utils::calculate_hash, worker::{write_file, Connection}, }; -use crate::global_cache::save_cache; use windmill_queue::MiniPulledJob; use windmill_queue::{append_logs, CanceledBy}; @@ -41,20 +41,30 @@ const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.co const NSJAIL_CONFIG_COMPILE_RUST_CONTENT: &str = include_str!("../nsjail/download.rust.config.proto"); +#[cfg(windows)] +const RUST_BIN_NAME: &str = "main.exe"; +#[cfg(not(windows))] +const RUST_BIN_NAME: &str = "main"; + fn find_cargo_path() -> String { if let Ok(p) = std::env::var("CARGO_PATH") { return p; } - let from_home = format!("{}/bin/cargo", CARGO_HOME.as_str()); - if std::path::Path::new(&from_home).exists() { - return from_home; - } - for p in ["/usr/local/cargo/bin/cargo", "/usr/bin/cargo"] { + let candidates = if cfg!(windows) { + vec![format!("{}\\bin\\cargo.exe", CARGO_HOME.as_str())] + } else { + vec![ + format!("{}/bin/cargo", CARGO_HOME.as_str()), + "/usr/local/cargo/bin/cargo".to_string(), + "/usr/bin/cargo".to_string(), + ] + }; + for p in &candidates { if std::path::Path::new(p).exists() { - return p.to_string(); + return p.clone(); } } - from_home + candidates.into_iter().next().unwrap() } #[cfg(not(windows))] @@ -71,7 +81,6 @@ fn find_preinstalled_dir(env_var: &str, candidates: &[&str]) -> String { } lazy_static::lazy_static! { - static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() }); static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() }); static ref CARGO_PATH: String = find_cargo_path(); @@ -81,14 +90,14 @@ lazy_static::lazy_static! { #[cfg(windows)] lazy_static::lazy_static! { - static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", *HOME_DIR); - static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR); + static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", HOME_ENV.as_str()); + static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", HOME_ENV.as_str()); } #[cfg(not(windows))] lazy_static::lazy_static! { - static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", *HOME_DIR); - static ref RUSTUP_HOME_DEFAULT: String = format!("{}/.rustup", *HOME_DIR); + static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", HOME_ENV.as_str()); + static ref RUSTUP_HOME_DEFAULT: String = format!("{}/.rustup", HOME_ENV.as_str()); } const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; @@ -97,11 +106,11 @@ const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; lazy_static::lazy_static! { static ref PREINSTALLED_CARGO: String = find_preinstalled_dir( "CARGO_PREINSTALL_DIR", - &["/usr/local/cargo", &format!("{}/.cargo", *HOME_DIR)], + &["/usr/local/cargo", &format!("{}/.cargo", HOME_ENV.as_str())], ); static ref PREINSTALLED_RUSTUP: String = find_preinstalled_dir( "RUSTUP_PREINSTALL_DIR", - &["/usr/local/rustup", &format!("{}/.rustup", *HOME_DIR)], + &["/usr/local/rustup", &format!("{}/.rustup", HOME_ENV.as_str())], ); } @@ -337,7 +346,7 @@ async fn get_build_dir( if !is_sandboxing_enabled() { // If nsjail is disabled then entire worker has shared build directory // It drastically improves cache hit-rate. - Some((format!("{RUST_CACHE_DIR}/build/{worker_name}"), true)) + Some((format!("{}/build/{worker_name}", *RUST_CACHE_DIR), true)) } else { // If nsjail is enabled, having global shared directory is vulnerability and target for an attack // Instead we either: @@ -345,7 +354,8 @@ async fn get_build_dir( // 2. If user is not known or something else goes wrong - use random build dir. This is equivalent to no cache at all. Some(( format!( - "{RUST_CACHE_DIR}/build/{}@{}@{}", + "{}/build/{}@{}@{}", + *RUST_CACHE_DIR, &job.workspace_id, p.replace('/', "."), &job.created_by @@ -355,7 +365,10 @@ async fn get_build_dir( } } }) - .unwrap_or((format!("{RUST_CACHE_DIR}/build/{}", Uuid::new_v4()), false)); + .unwrap_or(( + format!("{}/build/{}", *RUST_CACHE_DIR, Uuid::new_v4()), + false, + )); { let (t, r, g) = ( @@ -449,7 +462,7 @@ pub async fn build_rust_crate( is_preview: bool, ) -> error::Result { ensure_rust_runtime_dirs(); - let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); + let bin_path = format!("{}/{hash}", *RUST_CACHE_DIR); let build_dir = get_build_dir(job, job_dir, conn, worker_name, is_preview).await?; @@ -459,9 +472,9 @@ pub async fn build_rust_crate( "download.config.proto", &NSJAIL_CONFIG_COMPILE_RUST_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", RUST_CACHE_DIR) + .replace("{CACHE_DIR}", &*RUST_CACHE_DIR) .replace("{CARGO_HOME}", CARGO_HOME.as_str()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{BUILD}", &build_dir), )?; @@ -517,6 +530,13 @@ pub async fn build_rust_crate( std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), ); build_rust_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + // MSVC linker needs LIB and INCLUDE to find kernel32.lib etc. + if let Ok(lib) = std::env::var("LIB") { + build_rust_cmd.env("LIB", lib); + } + if let Ok(include) = std::env::var("INCLUDE") { + build_rust_cmd.env("INCLUDE", include); + } } start_child_process(build_rust_cmd, CARGO_PATH.as_str(), false).await? }; @@ -541,30 +561,29 @@ pub async fn build_rust_crate( tokio::fs::copy( &format!( - "{build_dir}/target/{}/main", + "{build_dir}/target/{}/{RUST_BIN_NAME}", if is_preview { "debug" } else { "release" }, ), - format! {"{job_dir}/main"}, + format!("{job_dir}/{RUST_BIN_NAME}"), ) .await .map_err(|e| { Error::ExecutionErr(format!( - "could not copy built binary from [...]/target/.../main to {job_dir}/main: {e:?}" + "could not copy built binary from [...]/target/.../{RUST_BIN_NAME} to {job_dir}/{RUST_BIN_NAME}: {e:?}" )) })?; match save_cache( &bin_path, &format!("{RUST_OBJECT_STORE_PREFIX}{hash}"), - &format!("{job_dir}/main"), + &format!("{job_dir}/{RUST_BIN_NAME}"), false, ) .await { Err(e) => { let em = format!( - "could not save {bin_path} to {} to rust cache: {e:?}", - format!("{job_dir}/main"), + "could not save {bin_path} to {job_dir}/{RUST_BIN_NAME} to rust cache: {e:?}", ); tracing::error!(em); Ok(em) @@ -605,26 +624,25 @@ pub async fn handle_rust_job( check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?; let hash = compute_rust_hash(inner_content, requirements_o); - let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); + let bin_path = format!("{}/{hash}", *RUST_CACHE_DIR); let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; - let (cache, cache_logs) = - crate::global_cache::load_cache(&bin_path, &remote_path, false).await; + let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { - let target = format!("{job_dir}/main"); + let target = format!("{job_dir}/{RUST_BIN_NAME}"); #[cfg(unix)] let symlink = std::os::unix::fs::symlink(&bin_path, &target); #[cfg(windows)] - let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + let symlink = std::os::windows::fs::symlink_file(&bin_path, &target); symlink.map_err(|e| { Error::ExecutionErr(format!( - "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" + "could not copy cached binary from {bin_path} to {target}: {e:?}" )) })?; @@ -669,10 +687,10 @@ pub async fn handle_rust_job( "run.config.proto", &NSJAIL_CONFIG_RUN_RUST_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", RUST_CACHE_DIR) + .replace("{CACHE_DIR}", &*RUST_CACHE_DIR) .replace("{CACHE_HASH}", &hash) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{SHARED_MOUNT}", shared_mount), )?; @@ -682,7 +700,10 @@ pub async fn handle_rust_job( .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Rust).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -691,14 +712,17 @@ pub async fn handle_rust_job( .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { - let compiled_executable_name = "./main"; + let compiled_executable_name = &format!("{job_dir}/{RUST_BIN_NAME}"); let mut run_rust = build_command_with_isolation(compiled_executable_name, &[]); run_rust .current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Rust).await?) + .envs( + get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) diff --git a/backend/windmill-worker/src/volume_oss.rs b/backend/windmill-worker/src/volume_oss.rs new file mode 100644 index 0000000000..3736e83485 --- /dev/null +++ b/backend/windmill-worker/src/volume_oss.rs @@ -0,0 +1,112 @@ +#[cfg(feature = "private")] +pub(crate) use crate::volume_ee::*; + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) struct LeaseRenewalGuard(pub Option>); + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +impl Drop for LeaseRenewalGuard { + fn drop(&mut self) { + if let Some(handle) = self.0.take() { + handle.abort(); + } + } +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) struct VolumeSetupResult { + pub states: Vec, + pub writable: Vec, + pub client: Option>, + pub lease_renewal: LeaseRenewalGuard, +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +#[allow(dead_code)] +pub(crate) fn setup_volume_mount_paths( + _volume: &windmill_worker_volumes::VolumeMount, + _state: &windmill_worker_volumes::VolumeState, + _job_dir: &str, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result<()> { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn setup_volumes_sql_worker( + _volume_mounts: &[windmill_worker_volumes::VolumeMount], + _db: &windmill_common::DB, + _workspace_id: &str, + _job_id: uuid::Uuid, + _permissioned_as: &str, + _worker_name: &str, + _job_dir: &str, + _client: &windmill_common::client::AuthedClient, + _conn: &windmill_common::worker::Connection, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn setup_volumes_http_worker( + _volume_mounts: &[windmill_worker_volumes::VolumeMount], + _http: &windmill_common::worker::HttpClient, + _workspace_id: &str, + _job_id: uuid::Uuid, + _permissioned_as: &str, + _canceled_by: &Option, + _worker_name: &str, + _job_dir: &str, + _conn: &windmill_common::worker::Connection, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn sync_volumes_sql_worker( + _volume_states: &[windmill_worker_volumes::VolumeState], + _volume_writable: &[bool], + _vol_client: &std::sync::Arc, + _db: &windmill_common::DB, + _workspace_id: &str, + _job_id: uuid::Uuid, + _worker_name: &str, + _conn: &windmill_common::worker::Connection, + _job_succeeded: bool, +) { +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn sync_volumes_http_worker( + _volume_states: &[windmill_worker_volumes::VolumeState], + _volume_writable: &[bool], + _http: &windmill_common::worker::HttpClient, + _workspace_id: &str, + _job_id: uuid::Uuid, + _worker_name: &str, + _conn: &windmill_common::worker::Connection, + _job_succeeded: bool, +) { +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index c354d34ee2..8d7f6051ec 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -37,7 +37,7 @@ use windmill_common::{ utils::{create_directory_async, WarnAfterExt}, worker::{ make_pull_query, write_file, Connection, HttpClient, MAX_TIMEOUT, - MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS, ROOT_CACHE_DIR, ROOT_CACHE_NOMOUNT_DIR, TMP_DIR, + MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS, ROOT_CACHE_DIR, ROOT_CACHE_NOMOUNT_DIR, WINDMILL_DIR, }, worker_group_job_stats::JobStatsMap, KillpillSender, @@ -47,7 +47,6 @@ use windmill_common::{ use windmill_common::ee_oss::LICENSE_KEY_VALID; use anyhow::Result; -use const_format::concatcp; #[cfg(feature = "prometheus")] use prometheus::IntCounter; @@ -196,45 +195,47 @@ use windmill_common::bench::{benchmark_init, benchmark_verify, BenchmarkInfo, Be use windmill_common::add_time; -pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_10"); -pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_11"); -pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_12"); -pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_13"); +lazy_static::lazy_static! { + pub static ref PY310_CACHE_DIR: String = format!("{}python_3_10", *ROOT_CACHE_DIR); + pub static ref PY311_CACHE_DIR: String = format!("{}python_3_11", *ROOT_CACHE_DIR); + pub static ref PY312_CACHE_DIR: String = format!("{}python_3_12", *ROOT_CACHE_DIR); + pub static ref PY313_CACHE_DIR: String = format!("{}python_3_13", *ROOT_CACHE_DIR); -pub const TAR_JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/java"); + pub static ref TAR_JAVA_CACHE_DIR: String = format!("{}tar/java", *ROOT_CACHE_DIR); -pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); -pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_runtime"); -pub const TAR_PYBASE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar"); -pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); -pub const DENO_CACHE_DIR_DEPS: &str = concatcp!(ROOT_CACHE_DIR, "deno/deps"); -pub const DENO_CACHE_DIR_NPM: &str = concatcp!(ROOT_CACHE_DIR, "deno/npm"); + pub static ref UV_CACHE_DIR: String = format!("{}uv", *ROOT_CACHE_DIR); + pub static ref PY_INSTALL_DIR: String = format!("{}py_runtime", *ROOT_CACHE_DIR); + pub static ref TAR_PYBASE_CACHE_DIR: String = format!("{}tar", *ROOT_CACHE_DIR); + pub static ref DENO_CACHE_DIR: String = format!("{}deno", *ROOT_CACHE_DIR); + pub static ref DENO_CACHE_DIR_DEPS: String = format!("{}deno/deps", *ROOT_CACHE_DIR); + pub static ref DENO_CACHE_DIR_NPM: String = format!("{}deno/npm", *ROOT_CACHE_DIR); -pub const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go"); -pub const RUST_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "rust"); -pub const NU_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "nu"); -pub const CSHARP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "csharp"); + pub static ref GO_CACHE_DIR: String = format!("{}go", *ROOT_CACHE_DIR); + pub static ref RUST_CACHE_DIR: String = format!("{}rust", *ROOT_CACHE_DIR); + pub static ref NU_CACHE_DIR: String = format!("{}nu", *ROOT_CACHE_DIR); + pub static ref CSHARP_CACHE_DIR: String = format!("{}csharp", *ROOT_CACHE_DIR); -// Java -pub const JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "java"); -pub const COURSIER_CACHE_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/coursier-cache"); -pub const JAVA_REPOSITORY_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/repository"); -pub const JAVA_HOME_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/home"); + // Java + pub static ref JAVA_CACHE_DIR: String = format!("{}java", *ROOT_CACHE_DIR); + pub static ref COURSIER_CACHE_DIR: String = format!("{}/coursier-cache", *JAVA_CACHE_DIR); + pub static ref JAVA_REPOSITORY_DIR: String = format!("{}/repository", *JAVA_CACHE_DIR); + pub static ref JAVA_HOME_DIR: String = format!("{}/home", *JAVA_CACHE_DIR); -// Ruby -pub const RUBY_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "ruby"); + // Ruby + pub static ref RUBY_CACHE_DIR: String = format!("{}ruby", *ROOT_CACHE_DIR); -// for related places search: ADD_NEW_LANG -pub const BUN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "bun"); -pub const BUN_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "bun"); -pub const BUN_CODEBASE_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "script_bundle"); + // for related places search: ADD_NEW_LANG + pub static ref BUN_CACHE_DIR: String = format!("{}bun", *ROOT_CACHE_NOMOUNT_DIR); + pub static ref BUN_BUNDLE_CACHE_DIR: String = format!("{}bun", *ROOT_CACHE_DIR); + pub static ref BUN_CODEBASE_BUNDLE_CACHE_DIR: String = format!("{}script_bundle", *ROOT_CACHE_NOMOUNT_DIR); -pub const GO_BIN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "gobin"); -pub const POWERSHELL_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "powershell"); -pub const COMPOSER_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "composer"); + pub static ref GO_BIN_CACHE_DIR: String = format!("{}gobin", *ROOT_CACHE_DIR); + pub static ref POWERSHELL_CACHE_DIR: String = format!("{}powershell", *ROOT_CACHE_DIR); + pub static ref COMPOSER_CACHE_DIR: String = format!("{}composer", *ROOT_CACHE_DIR); -pub const TRACING_PROXY_CA_CERT_PATH: &str = - concatcp!(ROOT_CACHE_NOMOUNT_DIR, "tracing_proxy_ca.pem"); + pub static ref TRACING_PROXY_CA_CERT_PATH: String = + format!("{}tracing_proxy_ca.pem", *ROOT_CACHE_NOMOUNT_DIR); +} const NUM_SECS_PING: u64 = 5; const NUM_SECS_READINGS: u64 = 60; @@ -562,7 +563,16 @@ lazy_static::lazy_static! { pub static ref DOTNET_PATH: String = std::env::var("DOTNET_PATH").unwrap_or_else(|_| DOTNET_DEFAULT_PATH.to_string()); pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new()); - pub static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()); + pub static ref HOME_ENV: String = { + #[cfg(not(windows))] + { std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()) } + #[cfg(windows)] + { + std::env::var("HOME") + .or_else(|_| std::env::var("USERPROFILE")) + .unwrap_or_else(|_| std::env::temp_dir().to_string_lossy().to_string()) + } + }; pub static ref GIT_PATH: String = std::env::var("GIT_PATH").unwrap_or_else(|_| "/usr/bin/git".to_string()); pub static ref NODE_PATH: Option = std::env::var("NODE_PATH").ok(); @@ -730,26 +740,69 @@ pub async fn is_otel_tracing_proxy_enabled_for_lang(lang: &ScriptLang) -> bool { } } +/// Get OTEL trace context environment variables for a job (TRACEPARENT, OTEL_TRACE_ID, OTEL_SPAN_ID). +/// Returns an empty vec when OTEL tracing is not enabled or on non-enterprise builds. +pub fn get_otel_context_envs(job_id: &uuid::Uuid) -> Vec<(&'static str, String)> { + #[cfg(all(feature = "private", feature = "enterprise"))] + if windmill_common::OTEL_TRACING_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { + let trace_id = format!("{:032x}", job_id.as_u128()); + let span_id = format!("{:016x}", job_id.as_u64_pair().1); + let traceparent = format!("00-{}-{}-01", trace_id, span_id); + return vec![ + ("TRACEPARENT", traceparent), + ("OTEL_TRACE_ID", trace_id), + ("OTEL_SPAN_ID", span_id), + ]; + } + let _ = job_id; + vec![] +} + /// Get proxy environment variables for job execution for a specific language. /// When OTEL tracing proxy is enabled for this language, routes all traffic through the proxy. /// Otherwise, uses the standard HTTP_PROXY/HTTPS_PROXY from environment. pub async fn get_proxy_envs_for_lang( lang: &ScriptLang, + job_id: &uuid::Uuid, + w_id: &str, + conn: &Connection, ) -> anyhow::Result> { + #[allow(unused_mut)] + let mut envs; #[cfg(all(feature = "private", feature = "enterprise"))] if is_otel_tracing_proxy_enabled_for_lang(lang).await { - return get_otel_tracing_proxy_envs().await; + envs = get_otel_tracing_proxy_envs(job_id, w_id, conn).await?; + } else { + envs = PROXY_ENVS.clone(); } - let _ = lang; - Ok(PROXY_ENVS.clone()) + #[cfg(not(all(feature = "private", feature = "enterprise")))] + { + let _ = (lang, w_id, conn); + envs = PROXY_ENVS.clone(); + } + envs.extend(get_otel_context_envs(job_id)); + Ok(envs) } #[cfg(all(feature = "private", feature = "enterprise"))] -async fn get_otel_tracing_proxy_envs() -> anyhow::Result> { - let port = crate::otel_tracing_proxy_ee::TRACING_PROXY_PORT +async fn get_otel_tracing_proxy_envs( + job_id: &uuid::Uuid, + w_id: &str, + conn: &Connection, +) -> anyhow::Result> { + let port = match *crate::otel_tracing_proxy_ee::TRACING_PROXY_PORT .read() .await - .ok_or_else(|| anyhow::anyhow!("OTEL tracing proxy port not initialized"))?; + { + Some(p) => p, + None => { + let reason = "OTEL tracing proxy is enabled but not available (not initialized yet, or NUM_WORKERS > 1). \ + This job's HTTP requests will not be traced."; + tracing::warn!("{}", reason); + append_logs(job_id, w_id, format!("\n[warning] {reason}\n"), conn).await; + return Ok(PROXY_ENVS.clone()); + } + }; let proxy_url = format!("http://127.0.0.1:{}", port); Ok(vec![ ("HTTP_PROXY", proxy_url.clone()), @@ -1375,7 +1428,7 @@ pub async fn run_worker( let start_time = Instant::now(); - let worker_dir = format!("{TMP_DIR}/{worker_name}"); + let worker_dir = format!("{}/{worker_name}", *WINDMILL_DIR); tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir"); #[cfg(feature = "python")] @@ -3873,7 +3926,7 @@ pub async fn run_language_executor( run_inline: bool, ) -> error::Result> { if language == Some(ScriptLang::Postgresql) { - return do_postgresql( + return Box::pin(do_postgresql( job, &client, &code, @@ -3885,7 +3938,7 @@ pub async fn run_language_executor( occupancy_metrics, parent_runnable_path, run_inline, - ) + )) .await; } else if language == Some(ScriptLang::Mysql) { #[cfg(not(feature = "mysql"))] @@ -3900,7 +3953,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_mysql( + return Box::pin(do_mysql( job, &client, &code, @@ -3911,7 +3964,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::Bigquery) { @@ -3937,7 +3990,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_bigquery( + return Box::pin(do_bigquery( job, &client, &code, @@ -3948,7 +4001,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::Snowflake) { @@ -3966,7 +4019,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_snowflake( + return Box::pin(do_snowflake( job, &client, &code, @@ -3977,7 +4030,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::Mssql) { @@ -4003,7 +4056,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_mssql( + return Box::pin(do_mssql( job, &client, &code, @@ -4014,7 +4067,7 @@ pub async fn run_language_executor( occupancy_metrics, job_dir, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::OracleDB) { @@ -4040,7 +4093,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_oracledb( + return Box::pin(do_oracledb( job, &client, &code, @@ -4051,7 +4104,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, - ) + )) .await; } } else if language == Some(ScriptLang::DuckDb) { @@ -4065,7 +4118,7 @@ pub async fn run_language_executor( #[cfg(feature = "duckdb")] { - return do_duckdb( + return Box::pin(do_duckdb( job, &client, &code, @@ -4077,7 +4130,7 @@ pub async fn run_language_executor( occupancy_metrics, parent_runnable_path, run_inline, - ) + )) .await; } } else if language == Some(ScriptLang::Graphql) { @@ -4086,7 +4139,7 @@ pub async fn run_language_executor( "Inline execution is not yet supported for this language".to_string(), )); } - return do_graphql( + return Box::pin(do_graphql( job, &client, &code, @@ -4095,7 +4148,7 @@ pub async fn run_language_executor( canceled_by, worker_name, occupancy_metrics, - ) + )) .await; } else if language == Some(ScriptLang::Nativets) { if run_inline { @@ -4122,7 +4175,7 @@ pub async fn run_language_executor( .collect::>() .join("\n")); - let result = do_nativets( + let result = Box::pin(do_nativets( job, &client, env_code, @@ -4133,7 +4186,7 @@ pub async fn run_language_executor( worker_name, occupancy_metrics, has_stream, - ) + )) .await?; return Ok(result); } @@ -4151,7 +4204,8 @@ pub async fn run_language_executor( job.id ); - let shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) { + #[allow(unused_mut)] + let mut shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) { let folder = if job.script_lang == Some(ScriptLang::Go) { "/go" } else { @@ -4173,7 +4227,8 @@ mount {{ // println!("handle lang job {:?}", SystemTime::now()); - let envs = build_envs(envs.as_ref())?; + #[allow(unused_mut)] + let mut envs = build_envs(envs.as_ref())?; let Some(language) = language else { return Err(Error::ExecutionErr( @@ -4209,6 +4264,106 @@ mount {{ } } + // Volume mount setup (requires workspace S3 storage; CE has file count/size limits) + #[cfg(feature = "parquet")] + let volume_mounts = { + let comment_prefix = match language { + ScriptLang::Python3 + | ScriptLang::Bash + | ScriptLang::Powershell + | ScriptLang::Ansible + | ScriptLang::Ruby => "#", + ScriptLang::Deno + | ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Go => "//", + _ => "", + }; + let raw_mounts = windmill_worker_volumes::parse_volume_annotations(&code, comment_prefix); + let args_ref = job.args.as_ref().map(|a| &**a); + let mut interpolated = Vec::new(); + for mut v in raw_mounts { + v.name = windmill_worker_volumes::interpolate_volume_name( + &v.name, + args_ref, + &job.workspace_id, + ); + if let Err(e) = windmill_worker_volumes::validate_volume_name(&v.name) { + return Err(Error::ExecutionErr(e)); + } + if let Err(e) = windmill_worker_volumes::validate_volume_target(&v.target) { + return Err(Error::ExecutionErr(e)); + } + interpolated.push(v); + } + if let Err(e) = windmill_worker_volumes::validate_volume_mounts(&interpolated) { + return Err(Error::ExecutionErr(e)); + } + interpolated + }; + + #[cfg(feature = "parquet")] + let mut volume_setup = crate::volume_oss::VolumeSetupResult { + states: Vec::new(), + writable: Vec::new(), + client: None, + lease_renewal: crate::volume_oss::LeaseRenewalGuard(None), + }; + + #[cfg(feature = "parquet")] + if !volume_mounts.is_empty() { + let vol_summary: Vec = volume_mounts + .iter() + .map(|v| format!("'{}' -> {}", v.name, v.target)) + .collect(); + append_logs( + &job.id, + &job.workspace_id, + format!( + "\n--- VOLUME MOUNTS ---\nPulling {} volume(s): {}\n", + volume_mounts.len(), + vol_summary.join(", "), + ), + conn, + ) + .await; + + if let Connection::Sql(db) = conn { + volume_setup = crate::volume_oss::setup_volumes_sql_worker( + &volume_mounts, + db, + &job.workspace_id, + job.id, + &job.permissioned_as, + worker_name, + job_dir, + client, + conn, + language, + &mut envs, + &mut shared_mount, + ) + .await?; + } else if let Connection::Http(http) = conn { + volume_setup = crate::volume_oss::setup_volumes_http_worker( + &volume_mounts, + http, + &job.workspace_id, + job.id, + &job.permissioned_as, + &job.canceled_by, + worker_name, + job_dir, + conn, + language, + &mut envs, + &mut shared_mount, + ) + .await?; + } + } + // Box::pin all language handlers to prevent large match enum on stack let result: error::Result> = match language { ScriptLang::Python3 => { @@ -4620,6 +4775,62 @@ mount {{ // for related places search: ADD_NEW_LANG _ => panic!("unreachable, language is not supported: {language:#?}"), }; + // Volume sync-back and lease release + #[cfg(feature = "parquet")] + if !volume_setup.states.is_empty() { + // Stop lease renewal before sync-back + volume_setup.lease_renewal.0.take().map(|h| h.abort()); + + if let Some(ref vol_client) = volume_setup.client { + if let Connection::Sql(db) = conn { + crate::volume_oss::sync_volumes_sql_worker( + &volume_setup.states, + &volume_setup.writable, + vol_client, + db, + &job.workspace_id, + job.id, + worker_name, + conn, + result.is_ok(), + ) + .await; + } + } + + if let Connection::Http(http) = conn { + crate::volume_oss::sync_volumes_http_worker( + &volume_setup.states, + &volume_setup.writable, + http, + &job.workspace_id, + job.id, + worker_name, + conn, + result.is_ok(), + ) + .await; + } + + // Clean up absolute-path symlinks created by setup_volume_mount_paths + if !is_sandboxing_enabled() { + #[allow(unused_variables)] // state is only used on unix + for state in &volume_setup.states { + #[cfg(unix)] + if state.mount.target.starts_with('/') { + let target_path = std::path::Path::new(&state.mount.target); + if target_path + .symlink_metadata() + .map(|m| m.file_type().is_symlink()) + .unwrap_or(false) + { + std::fs::remove_file(target_path).ok(); + } + } + } + } + } + tracing::info!( workspace_id = %job.workspace_id, is_ok = result.is_ok(), diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index d0b7fc7a58..8a45f85a90 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -11,7 +11,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; -use crate::common::{cached_result_path, get_root_job_id, save_in_cache}; +use crate::common::{cached_result_path, get_root_job_id, save_in_cache, transform_json}; use crate::js_eval::{eval_timeout, IdContext}; use crate::worker_utils::get_tag_and_concurrency; use crate::{ @@ -53,7 +53,7 @@ use windmill_common::runnable_settings::{ use windmill_common::scripts::{ScriptHash, ScriptRunnableSettingsInline}; use windmill_common::users::username_to_permissioned_as; use windmill_common::utils::WarnAfterExt; -use windmill_common::worker::to_raw_value; +use windmill_common::worker::{to_raw_value, Connection}; use windmill_common::{ add_time, get_latest_flow_version_info_for_path, get_script_info_for_hash, FlowVersionInfo, ScriptHashInfo, DB, @@ -865,7 +865,7 @@ pub async fn update_flow_status_after_job_completion_internal( .and_then(|x| x.stop_after_all_iters_if.as_ref()) { let args = from_result_to_args(args.as_ref().await.get_ref())?; - evaluate_stop_after_all_iters_if( + if let Err(e) = evaluate_stop_after_all_iters_if( db, stop_after_all_iters_if, module_status, @@ -879,7 +879,16 @@ pub async fn update_flow_status_after_job_completion_internal( flow, &old_status, ) - .await?; + .await + { + tracing::error!("error evaluating stop_after_all_iters_if: {e:#}"); + stop_early = true; + skip_if_stop_early = false; + stop_early_err_msg = Some(format!( + "Error evaluating stop_after_all_iters_if expression `{}`: {e:#}", + stop_after_all_iters_if.expr + )); + } } let new_status = if @@ -1074,7 +1083,7 @@ pub async fn update_flow_status_after_job_completion_internal( { let args = from_result_to_args(args.as_ref().await.get_ref())?; - evaluate_stop_after_all_iters_if( + if let Err(e) = evaluate_stop_after_all_iters_if( db, stop_after_all_iters_if, module_status, @@ -1088,7 +1097,15 @@ pub async fn update_flow_status_after_job_completion_internal( flow, &old_status, ) - .await?; + .await + { + stop_early = true; + skip_if_stop_early = false; + stop_early_err_msg = Some(format!( + "Error evaluating stop_after_all_iters_if expression `{}`: {e:#}", + stop_after_all_iters_if.expr + )); + } } } @@ -2245,6 +2262,35 @@ pub async fn handle_flow( killpill_rx: &tokio::sync::broadcast::Receiver<()>, ) -> anyhow::Result<()> { let flow = flow_data.value(); + + // Resolve $var: and $res: references in flow_env. + // We resolve into a separate variable to avoid cloning the entire FlowValue + // (which includes modules, failure_module, etc.) just to replace flow_env. + let resolved_env; + let flow_env = if let Some(ref env) = flow.flow_env { + match transform_json( + client, + &flow_job.workspace_id, + env, + &flow_job, + &Connection::Sql(db.clone()), + ) + .await + { + Ok(Some(resolved)) => { + resolved_env = resolved; + Some(&resolved_env) + } + Ok(None) => flow.flow_env.as_ref(), + Err(e) => { + tracing::warn!("Failed to resolve flow_env references: {e}"); + flow.flow_env.as_ref() + } + } + } else { + None + }; + let status = flow_job .parse_flow_status() .with_context(|| "Unable to parse flow status")?; @@ -2348,6 +2394,7 @@ pub async fn handle_flow( flow_job, status, flow, + flow_env, db, client, last_result.clone(), @@ -2448,6 +2495,7 @@ async fn push_next_flow_job( flow_job: Arc, mut status: FlowStatus, flow: &FlowValue, + flow_env: Option<&HashMap>>, db: &sqlx::Pool, client: &AuthedClient, last_job_result: Option>>, @@ -2580,7 +2628,7 @@ async fn push_next_flow_job( let skip = compute_bool_from_expr( &skip_expr, arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, Arc::new(to_raw_value(&json!("{}"))), None, None, @@ -2705,7 +2753,7 @@ async fn push_next_flow_job( expr.to_string(), context, Some(arc_flow_job_args.clone()), - flow.flow_env.as_ref(), + flow_env, None, None, None @@ -2966,7 +3014,7 @@ async fn push_next_flow_job( &input_transform, arc_last_job_result.clone(), Some(arc_flow_job_args.clone()), - flow.flow_env.as_ref(), + flow_env, Some(client), None, ) @@ -3004,7 +3052,7 @@ async fn push_next_flow_job( &status.retry, arc_last_job_result.clone(), arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, Some(client), ) .await? @@ -3092,7 +3140,7 @@ async fn push_next_flow_job( compute_bool_from_expr( &skip_if.expr, arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), None, Some(&idcontext), @@ -3182,7 +3230,7 @@ async fn push_next_flow_job( }; transform_input( arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), input_transforms, resumes.clone(), @@ -3209,7 +3257,7 @@ async fn push_next_flow_job( let next_flow_transform = compute_next_flow_transform( arc_flow_job_args.clone(), arc_last_job_result.clone(), - flow.flow_env.as_ref(), + flow_env, &flow_job, &flow, transform_context, @@ -3373,7 +3421,7 @@ async fn push_next_flow_job( let ctx = get_transform_context(&flow_job, "", &status); let ti = transform_input( Marc::new(args), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), input_transforms, resumes.clone(), @@ -3428,7 +3476,7 @@ async fn push_next_flow_job( let ctx = get_transform_context(&flow_job, &previous_id, &status); let ti = transform_input( Marc::new(hm), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), input_transforms, resumes.clone(), @@ -3546,7 +3594,7 @@ async fn push_next_flow_job( timeout_transform, arc_last_job_result.clone(), Some(arc_flow_job_args.clone()), - flow.flow_env.as_ref(), + flow_env, Some(client), Some(&ctx), ) @@ -3625,7 +3673,7 @@ async fn push_next_flow_job( parallelism_transform, arc_last_job_result.clone(), Some(arc_flow_job_args.clone()), - flow.flow_env.as_ref(), + flow_env, Some(client), Some(&ctx), ) @@ -4461,7 +4509,7 @@ async fn compute_next_flow_transform( let pred = compute_bool_from_expr( &b.expr, arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), None, Some(&idcontext), diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 41204c956a..a16ab8c6f6 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.648.0"; +export const VERSION = "v1.651.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/bootstrap/script_bootstrap.ts b/cli/bootstrap/script_bootstrap.ts index 5335e1741c..89d98a1927 100644 --- a/cli/bootstrap/script_bootstrap.ts +++ b/cli/bootstrap/script_bootstrap.ts @@ -39,7 +39,10 @@ export const scriptBootstrapCode = { } `, - bun: `export async function main() { + bun: `// there are multiple modes to add as header: //nobundling //native //npm //nodejs +// https://www.windmill.dev/docs/getting_started/scripts_quickstart/typescript#modes + +export async function main() { return "Hello world"; } `, diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 6dab7c88bf..3519cfe4bc 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -2673,6 +2673,7 @@ export async function push( let [_basePath, changes] = queue.shift()!; const promise = (async () => { const alreadySynced: string[] = []; + const deletedVarsResPaths: string[] = []; const isRawApp = isRawAppFile(changes[0].path); if (isRawApp) { const deleteRawApp = changes.find( @@ -2922,12 +2923,23 @@ export async function push( name: change.path.split(SEP)[1], }); break; - case "resource": - await wmill.deleteResource({ - workspace: workspaceId, - path: removeSuffix(target, ".resource.json"), - }); + case "resource": { + const resourcePath = removeSuffix(target, ".resource.json"); + try { + await wmill.deleteResource({ + workspace: workspaceId, + path: resourcePath, + }); + } catch (e: any) { + if (e?.status === 404 && deletedVarsResPaths.includes(resourcePath)) { + log.debug(`Resource ${resourcePath} already deleted by linked variable`); + } else { + throw e; + } + } + deletedVarsResPaths.push(resourcePath); break; + } case "resource-type": await wmill.deleteResourceType({ workspace: workspaceId, @@ -3064,12 +3076,23 @@ export async function push( }); break; } - case "variable": - await wmill.deleteVariable({ - workspace: workspaceId, - path: removeSuffix(target, ".variable.json"), - }); + case "variable": { + const variablePath = removeSuffix(target, ".variable.json"); + try { + await wmill.deleteVariable({ + workspace: workspaceId, + path: variablePath, + }); + } catch (e: any) { + if (e?.status === 404 && deletedVarsResPaths.includes(variablePath)) { + log.debug(`Variable ${variablePath} already deleted by linked resource`); + } else { + throw e; + } + } + deletedVarsResPaths.push(variablePath); break; + } case "user": { const users = await wmill.listUsers({ workspace: workspaceId, diff --git a/cli/src/commands/workspace/workspace.ts b/cli/src/commands/workspace/workspace.ts index 5b4297da21..6124bd1e1c 100644 --- a/cli/src/commands/workspace/workspace.ts +++ b/cli/src/commands/workspace/workspace.ts @@ -408,7 +408,8 @@ async function remove(_opts: GlobalOptions, name: string) { async function whoami(_opts: GlobalOptions) { await requireLogin(_opts); - log.info(Object.entries(await wmill.globalWhoami())); + const whoamiInfo = await wmill.globalWhoami(); + log.info(JSON.stringify(whoamiInfo, null, 2)); const activeName = await getActiveWorkspaceName(_opts); log.info("Active: " + colors.green.bold(activeName || "none")); } diff --git a/cli/src/main.ts b/cli/src/main.ts index 38b68bb0f4..f03db28745 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -67,7 +67,7 @@ export { workspaceAdd, }; -export const VERSION = "1.648.0"; +export const VERSION = "1.651.1"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 450dd68399..279c79b780 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -61,6 +61,11 @@ RUN ln -s /usr/bin/bun /usr/bin/node \ && bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# Copy to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 7cc4dafa05..88c16aaac0 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -61,6 +61,11 @@ RUN ln -s /usr/bin/bun /usr/bin/node \ && bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# Copy to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/frontend/BUGS.txt b/frontend/BUGS.txt new file mode 100644 index 0000000000..aae0762777 --- /dev/null +++ b/frontend/BUGS.txt @@ -0,0 +1,39 @@ +# Svelte 5 Migration - Bug Report +# Testing started: 2026-03-02 + +## Warnings (not blocking but worth fixing) + +1. [WARNING] binding_property_non_reactive in Grid.svelte:372:5 + - `bind:this={moveResizes[item.id]}` is binding to a non-reactive property + - File: src/lib/components/apps/svelte-grid/Grid.svelte + - Appears multiple times in App editor + - Status: NOT FIXED (non-blocking warning) + +2. [WARNING] legacy_recursive_reactive_block in RecomputeAllComponents.svelte + - Migrated `$:` reactive block that both accesses and updates the same reactive value + - File: src/lib/components/apps/editor/RecomputeAllComponents.svelte + - May cause recursive updates when converted to $effect + - Status: NOT FIXED (non-blocking warning) + +3. [WARNING] ownership_invalid_mutation in SchemaForm.svelte:70:16 + - Mutating unbound props (`schema`) is strongly discouraged + - Parent: src/lib/components/ApiConnectForm.svelte should use `bind:schema={...}` + - Appears when opening PostgreSQL resource creation form + - Status: NOT FIXED (non-blocking warning) + +4. [WARNING] ownership_invalid_binding in InputTransformSchemaForm.svelte + - Passes `schema` to InputTransformForm.svelte with `bind:`, but parent Pane.svelte didn't declare `schema` as binding + - Appears in flow editor when adding a TypeScript step + - Status: NOT FIXED (non-blocking warning) + +## Bugs + +1. [BUG] state_descriptors_fixed in Chart.svelte (Queue metrics drawer) + - Error: "Property descriptors defined on `$state` objects must contain `value` and always be `enumerable`, `configurable` and `writable`." + - Triggered by: Clicking "Queue metrics" on /workers page + - File: src/lib/components/chartjs-wrappers/Chart.svelte + - Root cause: Chart.js's `listenArrayEvents` calls Object.defineProperty on data arrays that are Svelte 5 $state proxies, which reject non-standard property descriptors + - Fix: Use $state.snapshot() to pass plain copies of data and options to Chart.js + - Status: FIXED + + diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 520148b1d6..6d7bb1a4fb 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.648.0", + "version": "1.651.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.648.0", + "version": "1.651.1", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 360003f0e3..744c4b2450 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.648.0", + "version": "1.651.1", "scripts": { "dev": "vite dev", "build": "vite build", @@ -287,6 +287,11 @@ "svelte": "./package/components/recording/FlowRecordingReplay.svelte", "default": "./package/components/recording/FlowRecordingReplay.svelte" }, + "./components/ScriptRecordingReplay.svelte": { + "types": "./package/components/recording/ScriptRecordingReplay.svelte.d.ts", + "svelte": "./package/components/recording/ScriptRecordingReplay.svelte", + "default": "./package/components/recording/ScriptRecordingReplay.svelte" + }, "./components/FlowWrapper.svelte": { "types": "./package/components/FlowWrapper.svelte.d.ts", "svelte": "./package/components/FlowWrapper.svelte", @@ -489,6 +494,9 @@ "components/FlowRecordingReplay.svelte": [ "./package/components/recording/FlowRecordingReplay.svelte.d.ts" ], + "components/ScriptRecordingReplay.svelte": [ + "./package/components/recording/ScriptRecordingReplay.svelte.d.ts" + ], "components/FlowBuilder.svelte": [ "./package/components/FlowBuilder.svelte.d.ts" ], diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index d8429a5ebf..8a79bd7109 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -15,6 +15,7 @@ import GitHubAppIntegration from './GitHubAppIntegration.svelte' import BedrockCredentialsCheck from './BedrockCredentialsCheck.svelte' import { isCloudHosted } from '$lib/cloud' + import ResourceGen from './copilot/ResourceGen.svelte' interface Props { resourceType: string @@ -149,6 +150,12 @@ }} class="as-json-toggle" /> + {#if resourceType == 'postgresql'} + import { run } from 'svelte/legacy' + import { createEventDispatcher } from 'svelte' import { Button, Drawer } from './common' import DrawerContent from './common/drawer/DrawerContent.svelte' @@ -6,24 +8,26 @@ import AppConnectInner from './AppConnectInner.svelte' import DarkModeObserver from './DarkModeObserver.svelte' - export let expressOAuthSetup = false - - let drawer: Drawer - let resourceType = '' - let step = 1 - let disabled = false - let isGoogleSignin = false - let manual = true - - let appConnectInner: AppConnectInner | undefined = undefined - - let rtToLoad: string | undefined = '' - export async function open(rt?: string) { - rtToLoad = rt - drawer.openDrawer?.() + interface Props { + expressOAuthSetup?: boolean } - $: appConnectInner && onRtToLoadChange(rtToLoad) + let { expressOAuthSetup = false }: Props = $props() + + let drawer: Drawer | undefined = $state() + let resourceType = $state('') + let step = $state(1) + let disabled = $state(false) + let isGoogleSignin = $state(false) + let manual = $state(true) + + let appConnectInner: AppConnectInner | undefined = $state(undefined) + + let rtToLoad: string | undefined = $state('') + export async function open(rt?: string) { + rtToLoad = rt + drawer?.openDrawer?.() + } function onRtToLoadChange(rtToLoad: string | undefined) { appConnectInner?.open(rtToLoad) @@ -31,7 +35,10 @@ const dispatch = createEventDispatcher() - let darkMode: boolean = false + let darkMode: boolean = $state(false) + run(() => { + appConnectInner && onRtToLoadChange(rtToLoad) + }) @@ -47,7 +54,7 @@ @@ -68,7 +75,7 @@ {/if} {#if isGoogleSignin} - {/if} - - -
-
{label}
- - {#if targetEmail} + + + - {/if} - - - - -
+ + {/snippet} diff --git a/frontend/src/lib/components/PageHeader.svelte b/frontend/src/lib/components/PageHeader.svelte index 07b02227d4..556ac1b0cc 100644 --- a/frontend/src/lib/components/PageHeader.svelte +++ b/frontend/src/lib/components/PageHeader.svelte @@ -1,11 +1,23 @@
@@ -31,9 +43,9 @@ {/if} - {#if $$slots.default} + {#if children}
- + {@render children?.()}
{/if}
diff --git a/frontend/src/lib/components/ParqetCsvTableRenderer.svelte b/frontend/src/lib/components/ParqetCsvTableRenderer.svelte index 07a8c267a9..262be32363 100644 --- a/frontend/src/lib/components/ParqetCsvTableRenderer.svelte +++ b/frontend/src/lib/components/ParqetCsvTableRenderer.svelte @@ -1,4 +1,6 @@ @@ -182,7 +189,7 @@
- mountGrid()}> diff --git a/frontend/src/lib/components/PermissionHistory.svelte b/frontend/src/lib/components/PermissionHistory.svelte index 82d646bd53..23f3ec151f 100644 --- a/frontend/src/lib/components/PermissionHistory.svelte +++ b/frontend/src/lib/components/PermissionHistory.svelte @@ -79,12 +79,14 @@

No permission changes recorded yet

{:else} - - Changed By - Change Type - Affected - Date - + {#snippet headerRow()} + + Changed By + Change Type + Affected + Date + + {/snippet} {#snippet body()} {#each history as change} diff --git a/frontend/src/lib/components/PersistentScriptDrawer.svelte b/frontend/src/lib/components/PersistentScriptDrawer.svelte index aa5c869f31..23fa122b7d 100644 --- a/frontend/src/lib/components/PersistentScriptDrawer.svelte +++ b/frontend/src/lib/components/PersistentScriptDrawer.svelte @@ -10,19 +10,19 @@ import { Hourglass, Loader2, Play, RefreshCw } from 'lucide-svelte' let dispatch = createEventDispatcher() - let drawer: Drawer + let drawer: Drawer | undefined = $state() - let script: Script - let loadQueuedJobs = true - let queuedJobsLoading = false + let script: Script | undefined = $state() + let loadQueuedJobs = $state(true) + let queuedJobsLoading = $state(false) let queuedJobs: { status: 'running' | 'queued' jobId: string scheduledFor: string scriptHash: string - }[] = [] + }[] = $state([]) - let cancellingInProgress = false + let cancellingInProgress = $state(false) async function continuouslyLoadQueuedJobs() { while (loadQueuedJobs) { @@ -40,7 +40,7 @@ let qjs = await JobService.listQueue({ workspace: $workspaceStore ?? '', orderDesc: false, - scriptPathExact: script.path + scriptPathExact: script?.path }) let loadingQueuedJobs: { status: 'running' | 'queued' @@ -71,12 +71,12 @@ cancellingInProgress = true await JobService.cancelPersistentQueuedJobs({ workspace: $workspaceStore ?? '', - path: script.path, + path: script?.path ?? '', requestBody: { reason: undefined } }) - sendUserToast(`All jobs cancelled for ${script.path}`) + sendUserToast(`All jobs cancelled for ${script?.path}`) cancellingInProgress = false } @@ -88,12 +88,12 @@ script = persistentScript! loadQueuedJobs = true continuouslyLoadQueuedJobs() - drawer.openDrawer?.() + drawer?.openDrawer?.() } async function exit() { loadQueuedJobs = false - drawer.closeDrawer?.() + drawer?.closeDrawer?.() } onDestroy(() => { @@ -117,51 +117,57 @@ >

- Queued jobs for {script.path} + Queued jobs for {script?.path}

- - Script Hash - Job ID - Status - Scheduled For - - - {#each queuedJobs as { jobId, status, scriptHash, scheduledFor }} - - - - {scriptHash} - - - - {jobId.substring(24)} - - - {#if status === 'running'} - - - - {:else} - - - - {/if} - - {scheduledFor} - - {/each} - + {#snippet headerRow()} + + Script Hash + Job ID + Status + Scheduled For + + {/snippet} + {#snippet body()} + + {#each queuedJobs as { jobId, status, scriptHash, scheduledFor }} + + + + {scriptHash} + + + + {jobId.substring(24)} + + + {#if status === 'running'} + + + + {:else} + + + + {/if} + + {scheduledFor} + + {/each} + + {/snippet} {#snippet actions()} diff --git a/frontend/src/lib/components/Popover.svelte b/frontend/src/lib/components/Popover.svelte index 1fa0a0d88d..07447b2e4a 100644 --- a/frontend/src/lib/components/Popover.svelte +++ b/frontend/src/lib/components/Popover.svelte @@ -43,10 +43,10 @@ onClick }: Props = $props() - const [popperRef, popperContent] = createPopperActions({ placement }) + const [popperRef, popperContent] = createPopperActions({ placement: untrack(() => placement) }) const popperOptions: PopperOptions<{}> = { - placement, + placement: untrack(() => placement), strategy: 'fixed', modifiers: [ { name: 'offset', options: { offset: [8, 8] } }, diff --git a/frontend/src/lib/components/PrefixedInput.svelte b/frontend/src/lib/components/PrefixedInput.svelte index c548418855..2a1dccf386 100644 --- a/frontend/src/lib/components/PrefixedInput.svelte +++ b/frontend/src/lib/components/PrefixedInput.svelte @@ -1,4 +1,5 @@ diff --git a/frontend/src/lib/components/RadioButton.svelte b/frontend/src/lib/components/RadioButton.svelte index efa9a62b40..c7e5dc8dce 100644 --- a/frontend/src/lib/components/RadioButton.svelte +++ b/frontend/src/lib/components/RadioButton.svelte @@ -1,13 +1,24 @@ @@ -28,7 +39,7 @@ class="sr-only" bind:group={value} aria-labelledby="memory-option-0-label" - on:click={() => dispatch('change', val)} + onclick={() => dispatch('change', val)} />

{#if typeof label !== 'string'} diff --git a/frontend/src/lib/components/Range.svelte b/frontend/src/lib/components/Range.svelte index 0e92ac72de..72129aada2 100644 --- a/frontend/src/lib/components/Range.svelte +++ b/frontend/src/lib/components/Range.svelte @@ -1,29 +1,47 @@

- +
{#if max <= min}
Impossible to display range: {`max (${max}) <= min (${min})`}
+ import { untrack } from 'svelte' import { GitSyncService } from '$lib/gen' import Select from './select/Select.svelte' @@ -32,7 +33,7 @@ }: Props = $props() // Track all loaded repositories across pages - let loadedRepositories = $state(initialRepositories) + let loadedRepositories = $state(untrack(() => initialRepositories)) let currentPage = $state(1) let isLoadingMore = $state(false) diff --git a/frontend/src/lib/components/Required.svelte b/frontend/src/lib/components/Required.svelte index 2163efa976..3ee1deec1a 100644 --- a/frontend/src/lib/components/Required.svelte +++ b/frontend/src/lib/components/Required.svelte @@ -1,12 +1,19 @@ {#if required} - * + * {:else if detail || detail != ''} - ({detail != '' ? `${detail}` : ''}) {/if} diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 5a5943390e..70aaf6a2ee 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -24,12 +24,12 @@ import GitHubAppIntegration from './GitHubAppIntegration.svelte' import Button from './common/button/Button.svelte' import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte' + import ResourceGen from './copilot/ResourceGen.svelte' interface Props { canSave?: boolean resource_type?: string | undefined path?: string - newResource?: boolean hidePath?: boolean onChange?: (args: { path: string; args: Record; description: string }) => void defaultValues?: Record | undefined @@ -39,7 +39,6 @@ canSave = $bindable(true), resource_type = $bindable(undefined), path = $bindable(''), - newResource = false, hidePath = false, onChange, defaultValues = undefined @@ -62,6 +61,7 @@ let resourceTypeInfo: ResourceType | undefined = $state(undefined) let editDescription = $state(false) let viewJsonSchema = $state(false) + let newResource = $derived(!path) const dispatch = createEventDispatcher() @@ -81,7 +81,7 @@ .map(([k, _]) => k) } - if (!newResource) { + if (!untrack(() => newResource)) { initEdit() } else if (resource_type) { loadResourceType() @@ -270,6 +270,13 @@ right: 'As JSON' }} /> + {#if resourceToEdit?.resource_type === 'nats' || resourceToEdit?.resource_type === 'kafka'} {:else} @@ -296,9 +303,17 @@ {#if loadingSchema} {:else if !viewJsonSchema && resourceTypeInfo?.is_fileset} -
- Fileset -
+
+
Fileset
+ +
{:else if !viewJsonSchema && resourceSchema && resourceSchema?.properties} {#if resourceTypeInfo?.format_extension} diff --git a/frontend/src/lib/components/ResourceEditorDrawer.svelte b/frontend/src/lib/components/ResourceEditorDrawer.svelte index 50db916f13..dae6943868 100644 --- a/frontend/src/lib/components/ResourceEditorDrawer.svelte +++ b/frontend/src/lib/components/ResourceEditorDrawer.svelte @@ -5,50 +5,44 @@ import { Loader2, Save } from 'lucide-svelte' - let drawer: Drawer - let canSave = true - let resource_type: string | undefined = undefined - let defaultValues: Record | undefined = undefined + let drawer: Drawer | undefined = $state() + let canSave = $state(true) + let resource_type: string | undefined = $state(undefined) + let defaultValues: Record | undefined = $state(undefined) let resourceEditor: { editResource: () => void; createResource: () => void } | undefined = - undefined + $state(undefined) - let path: string | undefined = undefined + let path: string | undefined = $state(undefined) - let newResource = false export async function initEdit(p: string): Promise { resource_type = undefined - newResource = false path = p - drawer.openDrawer?.() + drawer?.openDrawer?.() } export async function initNew( resourceType: string, nDefaultValues?: Record ): Promise { - newResource = true path = undefined resource_type = resourceType defaultValues = nDefaultValues - drawer.openDrawer?.() + drawer?.openDrawer?.() } - let mode: 'edit' | 'new' = newResource ? 'new' : 'edit' - - $: path ? (mode = 'edit') : (mode = 'new') + let mode: 'edit' | 'new' = $derived(!path ? 'new' : 'edit') {#await import('./ResourceEditor.svelte')} {:then Module} diff --git a/frontend/src/lib/components/ResourcePicker.svelte b/frontend/src/lib/components/ResourcePicker.svelte index 062e39804b..fddf9205f5 100644 --- a/frontend/src/lib/components/ResourcePicker.svelte +++ b/frontend/src/lib/components/ResourcePicker.svelte @@ -1,6 +1,6 @@ @@ -317,7 +316,6 @@ class="mt-1" _resourceMetadata={{ resource_type: resourceType }} asset={{ kind: 'resource', path: value }} - {dbManagerDrawer} /> {/if}
diff --git a/frontend/src/lib/components/RunForm.svelte b/frontend/src/lib/components/RunForm.svelte index 969576a06b..da0c87a15f 100644 --- a/frontend/src/lib/components/RunForm.svelte +++ b/frontend/src/lib/components/RunForm.svelte @@ -15,7 +15,7 @@ import Popover from './meltComponents/Popover.svelte' import { Calendar, Check, CornerDownLeft } from 'lucide-svelte' import RunFormAdvancedPopup from './RunFormAdvancedPopup.svelte' - import { page } from '$app/stores' + import { page } from '$app/state' import { replaceState } from '$app/navigation' import JsonInputs from '$lib/components/JsonInputs.svelte' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' @@ -108,7 +108,7 @@ nurl.hash = computeSharableHash(args) try { - replaceState(nurl.toString(), $page.state) + replaceState(nurl.toString(), page.state) } catch (e) { console.error(e) } diff --git a/frontend/src/lib/components/RunFormAdvancedPopup.svelte b/frontend/src/lib/components/RunFormAdvancedPopup.svelte index 7d6ef01bb2..399e55f772 100644 --- a/frontend/src/lib/components/RunFormAdvancedPopup.svelte +++ b/frontend/src/lib/components/RunFormAdvancedPopup.svelte @@ -8,7 +8,9 @@ import { WorkerService } from '$lib/gen' import DateTimeInput from './DateTimeInput.svelte' - export let runnable: + + interface Props { + runnable: | { summary?: string description?: string @@ -21,11 +23,18 @@ created_by?: string extra_perms?: Record } - | undefined + | undefined; + scheduledForStr: string | undefined; + invisible_to_owner: boolean | undefined; + overrideTag: string | undefined; + } - export let scheduledForStr: string | undefined - export let invisible_to_owner: boolean | undefined - export let overrideTag: string | undefined + let { + runnable, + scheduledForStr = $bindable(), + invisible_to_owner = $bindable(), + overrideTag = $bindable() + }: Props = $props(); loadWorkerGroups() async function loadWorkerGroups() { diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index 59b63ab313..5aee359a4a 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -96,8 +96,8 @@ let batchRerunOptionsIsOpen = $state(false) // Initialize path filter from route param if provided and not already set via query params - if (initialPath && !filters.val.path) { - filters.val.path = initialPath + if (untrack(() => initialPath) && !filters.val.path) { + filters.val.path = untrack(() => initialPath) } // Apply persistent toggle values from local storage if URL doesn't specify them @@ -148,7 +148,9 @@ (v) => { v.maxTs ? (filters.val.max_ts = new Date(v.maxTs)) : delete filters.val.max_ts v.minTs ? (filters.val.min_ts = new Date(v.minTs)) : delete filters.val.min_ts - v.timeframe ? (filters.val.timeframe = v.timeframe) : delete filters.val.timeframe + v.timeframe && v.timeframe !== 'Latest runs' + ? (filters.val.timeframe = v.timeframe) + : delete filters.val.timeframe } ) let timeframe = $derived(_timeframe.val) diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index a07ea85d25..55bb25c5ef 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -13,6 +13,7 @@ interface Props { fromWorkspaceSettings?: boolean readOnlyMode: boolean + allowDelete?: boolean initialFileKey?: { s3: string; storage?: string } | undefined selectedFileKey?: { s3: string; storage?: string } | undefined folderOnly?: boolean @@ -24,6 +25,7 @@ let { fromWorkspaceSettings = false, readOnlyMode, + allowDelete = false, initialFileKey = $bindable(undefined), selectedFileKey = $bindable(undefined), folderOnly = false, @@ -94,6 +96,7 @@ }} {fromWorkspaceSettings} {readOnlyMode} + {allowDelete} bind:initialFileKey bind:selectedFileKey bind:workspaceSettingsInitialized diff --git a/frontend/src/lib/components/S3FilePickerInner.svelte b/frontend/src/lib/components/S3FilePickerInner.svelte index 380e3d9a35..74b85bd327 100644 --- a/frontend/src/lib/components/S3FilePickerInner.svelte +++ b/frontend/src/lib/components/S3FilePickerInner.svelte @@ -81,6 +81,7 @@ count: number } > + allowDelete?: boolean replaceUnauthorizedWarning?: Snippet listStoredFilesRequest?: (d: ListStoredFilesData) => CancelablePromise loadFilePreviewRequest?: (d: LoadFilePreviewData) => CancelablePromise @@ -102,11 +103,12 @@ folderOnly = false, regexFilter = undefined, hideS3SpecificDetails = false, - rootPath = '', + rootPath: initialRootPath = '', workspaceSettingsInitialized = $bindable(true), storage = $bindable(undefined), uploadModalOpen = $bindable(false), allFilesByKey = $bindable({}), + allowDelete = false, replaceUnauthorizedWarning, listStoredFilesRequest = HelpersService.listStoredFiles, loadFilePreviewRequest = HelpersService.loadFilePreview, @@ -116,6 +118,7 @@ testConnectionRequest = HelpersService.datasetStorageTestConnection }: Props = $props() + let rootPath = $state(initialRootPath) let rootPathNestingLevel = $derived(1 * (rootPath.split('/').length - 1)) let csvSeparatorChar: string = $state(',') @@ -263,7 +266,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() fileListLoading = false fileInfoLoading = false } @@ -381,7 +384,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() } async function clearAndLoadFiles({ keepFilter }: { keepFilter?: boolean } = {}) { @@ -424,9 +427,16 @@ export async function open(_preSelectedFileKey: S3Object | undefined = undefined) { const preSelectedFileKey = _preSelectedFileKey && parseS3Object(_preSelectedFileKey) storage = preSelectedFileKey?.storage - if (preSelectedFileKey !== undefined) { + if (preSelectedFileKey !== undefined && preSelectedFileKey.s3.endsWith('/')) { + rootPath = preSelectedFileKey.s3 + filter = '' + selectedFileKey = undefined + } else if (preSelectedFileKey !== undefined) { + rootPath = '' initialFileKey = { ...preSelectedFileKey } selectedFileKey = { ...preSelectedFileKey } + } else { + rootPath = '' } reloadContent() } @@ -461,7 +471,7 @@ if (selectedFileKey !== undefined) { if (allFilesByKey[selectedFileKey.s3] === undefined) { selectedFileKey = { s3: '', storage } - } else { + } else if (allFilesByKey[selectedFileKey.s3].type !== 'folder') { loadFileMetadataPlusPreviewAsync(selectedFileKey.s3) } } @@ -518,7 +528,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() } else { selectedFileKey = { s3: item_key, @@ -719,8 +729,10 @@ startIcon={{ icon: MoveRight }} iconOnly={true} /> + {/if} + {#if !readOnlyMode || allowDelete} +
{#if customUi?.settingsPanel?.metadata?.disableScriptKind !== true}
{#snippet header()} @@ -1652,8 +1677,10 @@ /> {:else if script.on_behalf_of_email && !canPreserve} - Currently: {originalOnBehalfOfEmail ?? script.on_behalf_of_email}. - Will be set to {$userStore?.email} on deploy (requires admin or wm_deployers group to override) + Currently: {originalOnBehalfOfEmail ?? script.on_behalf_of_email}. Will be set to {$userStore?.email} on + deploy (requires admin or wm_deployers group to override) {/if} diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 4659abc0ac..f310010c18 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -33,6 +33,8 @@ Bug, Copy, CornerDownLeft, + Disc, + Download, ExternalLink, Github, GitBranch, @@ -86,6 +88,10 @@ import { deepEqual } from 'fast-equals' import { usePreparedAssetSqlQueries } from '$lib/infer.svelte' import { resource, watch } from 'runed' + import { createScriptRecording } from './recording/scriptRecording.svelte' + import { setActiveRecording } from './recording/flowRecording.svelte' + import type { ScriptRecording } from './recording/types' + import DropdownV2 from './DropdownV2.svelte' interface Props { // Exported @@ -94,7 +100,7 @@ path: string | undefined lang: Preview['language'] kind?: string | undefined - template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' + template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox' tag: string | undefined initialArgs?: Record fixedOverflowWidgets?: boolean @@ -117,7 +123,7 @@ lastDeployedCode?: string | undefined disableAi?: boolean assets?: AssetWithAltAccessType[] - editor_bar_right?: import('svelte').Snippet + editorBarRight?: import('svelte').Snippet enablePreprocessorSnippet?: boolean } @@ -149,7 +155,7 @@ lastDeployedCode = undefined, disableAi = false, assets = $bindable(), - editor_bar_right, + editorBarRight, enablePreprocessorSnippet = false }: Props = $props() @@ -235,6 +241,10 @@ let pastPreviews: CompletedJob[] = $state([]) let validCode = $state(true) + // Recording + let scriptRecording = createScriptRecording() + let lastRecording: ScriptRecording | undefined = $state(undefined) + let wsProvider: WebsocketProvider | undefined = $state(undefined) let yContent: Y.Text | undefined = $state(undefined) let peers: { name: string }[] = $state([]) @@ -332,11 +342,18 @@ undefined, { done(_x) { + if (scriptRecording.active) { + lastRecording = scriptRecording.stop() + setActiveRecording(undefined) + } loadPastTests() }, doneError({ error }) { + if (scriptRecording.active) { + lastRecording = scriptRecording.stop() + setActiveRecording(undefined) + } console.error(error) - // sendUserToast('Error running test', true) } } ) @@ -344,6 +361,19 @@ return job } + async function recordAndTest() { + lastRecording = undefined + scriptRecording.start(path ?? '', code, lang ?? '', args ?? {}, schema) + setActiveRecording(scriptRecording) + await runTest() + } + + function downloadRecording() { + if (lastRecording) { + scriptRecording.download(lastRecording) + } + } + async function loadPastTests(): Promise { pastPreviews = await JobService.listCompletedJobs({ workspace: $workspaceStore!, @@ -853,7 +883,7 @@ } } - setContext('disableTooltips', customUi?.disableTooltips === true) + setContext('disableTooltips', untrack(() => customUi)?.disableTooltips === true) let codePanelSize = $state(70) let testPanelSize = $state(30) @@ -1012,7 +1042,7 @@ bind:showHistoryDrawer > {#snippet right()} - {@render editor_bar_right?.()} + {@render editorBarRight?.()} {/snippet} {/if} @@ -1110,40 +1140,72 @@ />
{#if !(debugMode && isDebuggableScript)} -
- {#if testIsLoading} - - {:else} - {@const disableTriggerButton = - customUi?.previewPanel?.disableTriggerButton === true} - - {#if !disableTriggerButton} - +
+
+ {#if testIsLoading} + + {:else} + {@const disableTriggerButton = + customUi?.previewPanel?.disableTriggerButton === true} + + {#if !disableTriggerButton} + + {/if} {/if} +
+ {#if lastRecording} +
{/if} -
+
+ + recordAndTest() + }, + ...(lastRecording + ? [ + { + displayName: 'Download recording', + icon: Download, + action: () => downloadRecording() + } + ] + : []) + ]} + /> +
allowFlow) && options.push(['Flow', 'flow', FlowIcon, '#14b8a6']) const dispatch = createEventDispatcher() async function loadItems(): Promise { diff --git a/frontend/src/lib/components/ScriptWrapper.svelte b/frontend/src/lib/components/ScriptWrapper.svelte index 5e53dc09f3..eec781ce77 100644 --- a/frontend/src/lib/components/ScriptWrapper.svelte +++ b/frontend/src/lib/components/ScriptWrapper.svelte @@ -1,11 +1,12 @@ diff --git a/frontend/src/lib/components/Scrollable.svelte b/frontend/src/lib/components/Scrollable.svelte index 37e3ae2933..7c34b3cbe1 100644 --- a/frontend/src/lib/components/Scrollable.svelte +++ b/frontend/src/lib/components/Scrollable.svelte @@ -2,14 +2,19 @@ import { onMount, onDestroy } from 'svelte' import { twMerge } from 'tailwind-merge' - let isAtBottom: boolean = false - let isScrollable = false + let isAtBottom: boolean = $state(false) + let isScrollable = $state(false) - export let id: string | null | undefined = undefined - export let scrollableClass: string = '' - export let shiftedShadow: boolean = false + interface Props { + id?: string | null | undefined + scrollableClass?: string + shiftedShadow?: boolean + children?: import('svelte').Snippet + } + + let { id = undefined, scrollableClass = '', shiftedShadow = false, children }: Props = $props() let mutationObserver: MutationObserver - let el: HTMLDivElement + let el: HTMLDivElement | undefined = $state() function handleScroll(event) { const scrollableElement = event.target @@ -20,7 +25,8 @@ } function checkIfScrollable(el) { - return el.scrollHeight > el.clientHeight + if (!el) return false + return el?.scrollHeight > el?.clientHeight } function observeScrollability(el) { @@ -33,7 +39,7 @@ } export function scrollIntoView(top: number) { - el.scrollTo({ top, behavior: 'smooth' }) + el?.scrollTo({ top, behavior: 'smooth' }) } onMount(() => { observeScrollability(el) @@ -45,8 +51,8 @@
-
- +
+ {@render children?.()}
{#if !isAtBottom && isScrollable}
opts)) function filterItems() { let trimmed = filter.trim() diff --git a/frontend/src/lib/components/ServiceLogsInner.svelte b/frontend/src/lib/components/ServiceLogsInner.svelte index 0f4f923cd6..abcffa32ef 100644 --- a/frontend/src/lib/components/ServiceLogsInner.svelte +++ b/frontend/src/lib/components/ServiceLogsInner.svelte @@ -22,7 +22,7 @@ import SplitPanesOrColumnOnMobile from './splitPanes/SplitPanesOrColumnOnMobile.svelte' import Select from './select/Select.svelte' import { goto } from '$lib/navigation' - import { page } from '$app/stores' + import { page } from '$app/state' import { watch } from 'runed' interface Props { @@ -169,13 +169,13 @@ type Selected = { mode: string; workerGroup: string; hostname: string } let initialSelected = - $page.url.searchParams.get('mode') && - $page.url.searchParams.get('workerGroup') && - $page.url.searchParams.get('hostname') + page.url.searchParams.get('mode') && + page.url.searchParams.get('workerGroup') && + page.url.searchParams.get('hostname') ? { - mode: $page.url.searchParams.get('mode')!, - workerGroup: $page.url.searchParams.get('workerGroup')!, - hostname: $page.url.searchParams.get('hostname')! + mode: page.url.searchParams.get('mode')!, + workerGroup: page.url.searchParams.get('workerGroup')!, + hostname: page.url.searchParams.get('hostname')! } : undefined let selected: Selected | undefined = $state(initialSelected) @@ -663,7 +663,7 @@ { + onClick={() => { let logLineNumber = document.line_number[0] let logFile = document.file_name[0] let host = document.host[0] diff --git a/frontend/src/lib/components/ShareModal.svelte b/frontend/src/lib/components/ShareModal.svelte index e5ad54f88b..10cf5f09e0 100644 --- a/frontend/src/lib/components/ShareModal.svelte +++ b/frontend/src/lib/components/ShareModal.svelte @@ -35,6 +35,7 @@ | 'postgres_trigger' | 'gcp_trigger' | 'email_trigger' + | 'volume' let kind: Kind let path: string = $state('') @@ -53,13 +54,17 @@ let drawer: Drawer | undefined = $state() let own = $state(false) - export async function openDrawer(newPath: string, kind_l: Kind) { + export async function openDrawer(newPath: string, kind_l: Kind, isOwnerOverride?: boolean) { path = newPath kind = kind_l loadAcls() loadGroups() loadUsernames() - loadOwner() + if (isOwnerOverride !== undefined) { + own = isOwnerOverride + } else { + loadOwner() + } drawer?.openDrawer() } @@ -154,12 +159,14 @@ {/if} {#if acls?.length > 0} - - - owner - - - + + {#snippet headerRow()} + + owner + + + + {/snippet} {#snippet body()} {#each acls as [owner, write]} diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 2fdd14e3f9..31da7fd802 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -130,7 +130,7 @@ const dispatch = createEventDispatcher() - const uri = `file:///${hash}.${langToExt(lang)}` + const uri = `file:///${untrack(() => hash)}.${langToExt(untrack(() => lang))}` export function getCode(): string { if (valueAfterDispose != undefined) { diff --git a/frontend/src/lib/components/Slider.svelte b/frontend/src/lib/components/Slider.svelte index c2ca0abbdd..e3269cde09 100644 --- a/frontend/src/lib/components/Slider.svelte +++ b/frontend/src/lib/components/Slider.svelte @@ -5,10 +5,21 @@ import Tooltip from './Tooltip.svelte' import { twMerge } from 'tailwind-merge' - export let text: string - export let tooltip: string | undefined = undefined - export let view = false - export let size: 'xs' | 'sm' | 'md' | 'lg' = 'md' + interface Props { + text: string; + tooltip?: string | undefined; + view?: boolean; + size?: 'xs' | 'sm' | 'md' | 'lg'; + children?: import('svelte').Snippet; + } + + let { + text, + tooltip = undefined, + view = $bindable(false), + size = 'md', + children + }: Props = $props();
- - {#if isCloudHosted()} - The cloud version is updated daily. - {:else} - How to update?
- - docker: docker compose up -d
- - helm - {/if} - + {#snippet text()} + + {#if isCloudHosted()} + The cloud version is updated daily. + {:else} + How to update?
+ - docker: docker compose up -d
+ - helm + {/if} + + {/snippet} {/snippet} {#snippet actions()} diff --git a/frontend/src/lib/components/SuperadminSettingsInner.svelte b/frontend/src/lib/components/SuperadminSettingsInner.svelte index 4d6b0e33f1..1f33e768d8 100644 --- a/frontend/src/lib/components/SuperadminSettingsInner.svelte +++ b/frontend/src/lib/components/SuperadminSettingsInner.svelte @@ -8,7 +8,7 @@ import { sendUserToast } from '$lib/toast' import { base } from '$lib/base' import SearchItems from './SearchItems.svelte' - import { page } from '$app/stores' + import { page } from '$app/state' import { goto as gotoUrl } from '$app/navigation' import Version from './Version.svelte' import Uptodate from './Uptodate.svelte' @@ -50,9 +50,9 @@ } = $props() function removeHash() { - const index = $page.url.href.lastIndexOf('#') + const index = page.url.href.lastIndexOf('#') if (index === -1) return - const hashRemoved = $page.url.href.slice(0, index) + const hashRemoved = page.url.href.slice(0, index) gotoUrl(hashRemoved) } diff --git a/frontend/src/lib/components/TableCustom.svelte b/frontend/src/lib/components/TableCustom.svelte index e40a2c9706..a508d48c8f 100644 --- a/frontend/src/lib/components/TableCustom.svelte +++ b/frontend/src/lib/components/TableCustom.svelte @@ -1,37 +1,53 @@ -
+
- + {@render headerRow?.()} - + {@render body?.()}
{#if paginated}
diff --git a/frontend/src/lib/components/TemplateEditor.svelte b/frontend/src/lib/components/TemplateEditor.svelte index f3ad00ffc3..98f11c3076 100644 --- a/frontend/src/lib/components/TemplateEditor.svelte +++ b/frontend/src/lib/components/TemplateEditor.svelte @@ -412,7 +412,7 @@ const lang = 'template' const dispatch = createEventDispatcher() - const uri = `file:///${hash}.ts` + const uri = `file:///${untrack(() => hash)}.ts` export function insertAtCursor(code: string): void { if (editor) { @@ -438,7 +438,6 @@ let cip let extraModel - let width = $state(0) // let widgets: HTMLElement | undefined = document.getElementById('monaco-widgets-root') ?? undefined let initialized = $state(false) @@ -545,9 +544,6 @@ if (divEl) { divEl.style.height = `${contentHeight}px` } - try { - editor?.layout({ width, height: contentHeight }) - } catch {} } editor.onDidContentSizeChange(updateHeight) updateHeight() @@ -718,7 +714,6 @@ bind:this={divEl} style="height: 18px;" class="template nonmain-editor rounded-md overflow-clip {!editor ? 'hidden' : ''}" - bind:clientWidth={width} >
diff --git a/frontend/src/lib/components/TestConnection.svelte b/frontend/src/lib/components/TestConnection.svelte index 576bbc9334..7c59a2a324 100644 --- a/frontend/src/lib/components/TestConnection.svelte +++ b/frontend/src/lib/components/TestConnection.svelte @@ -7,10 +7,19 @@ import { workspaceStore } from '$lib/stores' import { tryEvery } from '$lib/utils' - export let workspaceOverride: string | undefined = undefined - export let resourceType: string | undefined - export let args: Record | any = {} - export let buttonTextOverride: string | undefined = undefined + interface Props { + workspaceOverride?: string | undefined; + resourceType: string | undefined; + args?: Record | any; + buttonTextOverride?: string | undefined; + } + + let { + workspaceOverride = undefined, + resourceType, + args = {}, + buttonTextOverride = undefined + }: Props = $props(); const scripts: { [key: string]: { @@ -166,7 +175,7 @@ export async function main(bucket: any) { } } - let loading = false + let loading = $state(false) async function testConnection() { if (!resourceType) return loading = true diff --git a/frontend/src/lib/components/Toast.svelte b/frontend/src/lib/components/Toast.svelte index 13e857ed6f..034887235e 100644 --- a/frontend/src/lib/components/Toast.svelte +++ b/frontend/src/lib/components/Toast.svelte @@ -82,14 +82,14 @@ } }) - let color = classes[type] + let color = classes[untrack(() => type)] let containerClass = { success: 'toast-success', error: 'toast-error', info: 'toast-info', warning: 'toast-warning' - }[type] + }[untrack(() => type)] let Icon = $derived(icons[type]) diff --git a/frontend/src/lib/components/Toggle.svelte b/frontend/src/lib/components/Toggle.svelte index 6b502bcbfa..4ae37d325f 100644 --- a/frontend/src/lib/components/Toggle.svelte +++ b/frontend/src/lib/components/Toggle.svelte @@ -3,7 +3,7 @@ const bubble = createBubbler() import { classNames } from '$lib/utils' - import { createEventDispatcher } from 'svelte' + import { createEventDispatcher, untrack } from 'svelte' import { twMerge } from 'tailwind-merge' import Tooltip from './Tooltip.svelte' import { AlertTriangle } from 'lucide-svelte' @@ -54,7 +54,7 @@ }: Props = $props() const dispatch = createEventDispatcher<{ change: boolean }>() - const bothOptions = Boolean(options.left) && Boolean(options.right) + const bothOptions = Boolean(untrack(() => options).left) && Boolean(untrack(() => options).right)
{/if} +
diff --git a/frontend/src/lib/components/apps/components/display/ResolveNavbarItemPath.svelte b/frontend/src/lib/components/apps/components/display/ResolveNavbarItemPath.svelte index e5307b4d15..0ab7b2795c 100644 --- a/frontend/src/lib/components/apps/components/display/ResolveNavbarItemPath.svelte +++ b/frontend/src/lib/components/apps/components/display/ResolveNavbarItemPath.svelte @@ -1,4 +1,5 @@ {#each Object.keys(components['checkboxcomponent'].initialData.configuration) as key (key)} diff --git a/frontend/src/lib/components/apps/components/inputs/AppCodeInputComponent.svelte b/frontend/src/lib/components/apps/components/inputs/AppCodeInputComponent.svelte index 9566f39866..62eedc3fc6 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppCodeInputComponent.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppCodeInputComponent.svelte @@ -1,5 +1,5 @@ diff --git a/frontend/src/lib/components/apps/components/layout/AppDecisionTree.svelte b/frontend/src/lib/components/apps/components/layout/AppDecisionTree.svelte index d42fdc3ba5..62b785c486 100644 --- a/frontend/src/lib/components/apps/components/layout/AppDecisionTree.svelte +++ b/frontend/src/lib/components/apps/components/layout/AppDecisionTree.svelte @@ -51,7 +51,7 @@ untrack(() => (counter += 1)) }) - let everRender = $state(render) + let everRender = $state(untrack(() => render)) $effect.pre(() => { render && !everRender && (everRender = true) }) @@ -66,11 +66,11 @@ debuggingComponents } = getContext('AppViewerContext') - let css = $state(initCss($app.css?.conditionalwrapper, customCss)) + let css = $state(initCss($app.css?.conditionalwrapper, untrack(() => customCss))) let selectedConditionIndex = 0 - let currentNodeId = $state(getFirstNode(nodes)?.id ?? '') + let currentNodeId = $state(getFirstNode(untrack(() => nodes))?.id ?? '') - let outputs = initOutput($worldStore, id, { + let outputs = initOutput($worldStore, untrack(() => id), { currentNodeId: untrack(() => currentNodeId), currentNodeIndex: selectedConditionIndex }) @@ -154,7 +154,7 @@ } } - $componentControl[id] = { + $componentControl[untrack(() => id)] = { setTab: (conditionIndex: number) => { if (conditionIndex !== -1) { onFocus(conditionIndex) diff --git a/frontend/src/lib/components/apps/components/layout/AppDivider.svelte b/frontend/src/lib/components/apps/components/layout/AppDivider.svelte index 97a698dd60..af68c3ad55 100644 --- a/frontend/src/lib/components/apps/components/layout/AppDivider.svelte +++ b/frontend/src/lib/components/apps/components/layout/AppDivider.svelte @@ -1,5 +1,5 @@ {#each Object.keys(components['drawercomponent'].initialData.configuration) as key (key)} @@ -162,6 +162,7 @@ }} fullScreen={$mode !== 'dnd'} > +
- import { getContext } from 'svelte' + import { getContext, untrack } from 'svelte' import { initConfig, initOutput } from '../../editor/appUtils' import SubGridEditor from '../../editor/SubGridEditor.svelte' import type { AppViewerContext, ComponentCustomCSS, RichConfigurations } from '../../types' @@ -37,12 +37,12 @@ getContext('AppViewerContext') let page = $state(0) - let everRender = $state(render) + let everRender = $state(untrack(() => render)) $effect.pre(() => { render && !everRender && (everRender = true) }) - const outputs = initOutput($worldStore, id, { + const outputs = initOutput($worldStore, untrack(() => id), { result: undefined, loading: false, inputs: {}, @@ -50,7 +50,7 @@ }) let resolvedConfig = $state( - initConfig(components['listcomponent'].initialData.configuration, configuration) + initConfig(components['listcomponent'].initialData.configuration, untrack(() => configuration)) ) function onFocus() { @@ -60,7 +60,7 @@ } } - let css = $state(initCss($app.css?.listcomponent, customCss)) + let css = $state(initCss($app.css?.listcomponent, untrack(() => customCss))) let result: any[] | undefined = $state(undefined) let isCard = $derived(resolvedConfig.width?.selected == 'card') diff --git a/frontend/src/lib/components/apps/components/layout/AppModal.svelte b/frontend/src/lib/components/apps/components/layout/AppModal.svelte index 6232b667fd..9508303630 100644 --- a/frontend/src/lib/components/apps/components/layout/AppModal.svelte +++ b/frontend/src/lib/components/apps/components/layout/AppModal.svelte @@ -1,7 +1,7 @@ diff --git a/frontend/src/lib/components/apps/editor/AppJsonEditor.svelte b/frontend/src/lib/components/apps/editor/AppJsonEditor.svelte index 7572c8c0fe..5a955a8ce6 100644 --- a/frontend/src/lib/components/apps/editor/AppJsonEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppJsonEditor.svelte @@ -10,12 +10,12 @@ import { createEventDispatcher } from 'svelte' import { Globe, Loader2, Save } from 'lucide-svelte' - let jsonViewerDrawer: Drawer + let jsonViewerDrawer: Drawer | undefined = $state() - let code: string = '' + let code: string = $state('') let path: string = '' - let useDraft: boolean = false - let loading = true + let useDraft: boolean = $state(false) + let loading = $state(true) const dispatch = createEventDispatcher() let app: any | undefined = undefined @@ -73,7 +73,7 @@ - jsonViewerDrawer.toggleDrawer()}> + jsonViewerDrawer?.toggleDrawer()}> {#if useDraft}
+Draft diff --git a/frontend/src/lib/components/apps/editor/AppPreview.svelte b/frontend/src/lib/components/apps/editor/AppPreview.svelte index 02114e1210..67c7ef4313 100644 --- a/frontend/src/lib/components/apps/editor/AppPreview.svelte +++ b/frontend/src/lib/components/apps/editor/AppPreview.svelte @@ -59,9 +59,9 @@ gotoFn = (path: string, opt?: Record) => window.history.pushState(null, '', path) }: Props = $props() - migrateApp(app) + migrateApp(untrack(() => app)) - const appStore = writable(app) + const appStore = writable(untrack(() => app)) const selectedComponent = writable(undefined) const mode = writable('preview') @@ -74,11 +74,11 @@ const allIdsInPath = writable([]) let ncontext: any = { - ...context, - workspace, + ...untrack(() => context), + workspace: untrack(() => workspace), mode: 'viewer', - summary: summary, - author: policy.on_behalf_of_email + summary: untrack(() => summary), + author: untrack(() => policy).on_behalf_of_email } function resizeWindow() { @@ -90,7 +90,7 @@ const parentWidth = writable(0) let previousDarkMode = document.documentElement.classList.contains('dark') - const darkMode: Writable = writable(app?.darkMode ?? previousDarkMode) + const darkMode: Writable = writable(untrack(() => app)?.darkMode ?? previousDarkMode) onDestroy(() => { setTheme(previousDarkMode) @@ -132,7 +132,7 @@ }) } - let writablePath = writable(appPath) + let writablePath = writable(untrack(() => appPath)) function onPathChange() { writablePath.set(appPath) @@ -146,21 +146,21 @@ runnableInitialized: {} }), app: appStore, - summary: writable(summary), + summary: writable(untrack(() => summary)), selectedComponent, bgRuns: writable([]), mode, connectingInput, - breakpoint, + breakpoint: untrack(() => breakpoint), runnableComponents: writable({}), appPath: writablePath, - workspace, + workspace: untrack(() => workspace), onchange: undefined, - isEditor, + isEditor: untrack(() => isEditor), jobs: parentContext?.jobs ?? writable([]), jobsById: parentContext?.jobsById ?? writable({}), staticExporter: writable({}), - noBackend, + noBackend: untrack(() => noBackend), errorByComponent: writable({}), openDebugRun: writable(undefined), focusedGrid: writable(undefined), @@ -174,9 +174,9 @@ cssEditorOpen: writable(false), previewTheme: writable(undefined), debuggingComponents: writable({}), - replaceStateFn, - gotoFn, - policy, + replaceStateFn: untrack(() => replaceStateFn), + gotoFn: untrack(() => gotoFn), + policy: untrack(() => policy), recomputeAllContext: writable({ loading: false, componentNumber: 0, @@ -319,6 +319,7 @@ {#snippet children({ dataItem })} +
($selectedComponent = [dataItem.id])} diff --git a/frontend/src/lib/components/apps/editor/ComponentHeader.svelte b/frontend/src/lib/components/apps/editor/ComponentHeader.svelte index e717205670..28d8e5912a 100644 --- a/frontend/src/lib/components/apps/editor/ComponentHeader.svelte +++ b/frontend/src/lib/components/apps/editor/ComponentHeader.svelte @@ -297,7 +297,7 @@ 'text-red-500 px-1 text-2xs py-0.5 font-bold w-fit absolute border border-red-500 -bottom-1 shadow left-1/2 transform -translate-x-1/2 z-50 cursor-pointer' )} > - + {#snippet trigger()} {/snippet} diff --git a/frontend/src/lib/components/apps/editor/DecisionTreeDebug.svelte b/frontend/src/lib/components/apps/editor/DecisionTreeDebug.svelte index bf70dafd61..c5452be83e 100644 --- a/frontend/src/lib/components/apps/editor/DecisionTreeDebug.svelte +++ b/frontend/src/lib/components/apps/editor/DecisionTreeDebug.svelte @@ -4,7 +4,7 @@ const bubble = createBubbler() import Dropdown from '$lib/components/DropdownV2.svelte' import { classNames } from '$lib/utils' - import { createEventDispatcher, getContext } from 'svelte' + import { createEventDispatcher, getContext, untrack } from 'svelte' import type { AppViewerContext } from '../types' import type { DecisionTreeNode } from './component' import { isDebugging } from './settingsPanel/decisionTree/utils' @@ -27,7 +27,7 @@ getContext('AppViewerContext') const dispatch = createEventDispatcher() - let currentNodeId: string = $state($worldStore.outputsById[id]?.currentNodeId?.peak() ?? 'a') + let currentNodeId: string = $state($worldStore.outputsById[untrack(() => id)]?.currentNodeId?.peak() ?? 'a') function subscribeToCurrentNode(id: string) { return $worldStore.outputsById[id]?.currentNodeId?.subscribe( @@ -41,7 +41,7 @@ ) } - let subscription = subscribeToCurrentNode(id) + let subscription = subscribeToCurrentNode(untrack(() => id)) function onDebugNode(debuggedNodeIndex: number | undefined) { if (debuggedNodeIndex === undefined) { @@ -63,7 +63,7 @@ }) let renderCount: number = $state(0) - let lastNodes: DecisionTreeNode[] = nodes + let lastNodes: DecisionTreeNode[] = untrack(() => nodes) function onNodesChange(newNodes: DecisionTreeNode[]) { if (JSON.stringify(newNodes) !== JSON.stringify(lastNodes)) { diff --git a/frontend/src/lib/components/apps/editor/GridEditor.svelte b/frontend/src/lib/components/apps/editor/GridEditor.svelte index d412bc1c56..d787564c0c 100644 --- a/frontend/src/lib/components/apps/editor/GridEditor.svelte +++ b/frontend/src/lib/components/apps/editor/GridEditor.svelte @@ -184,6 +184,7 @@
+
rowHeight) let containerHeight = $derived(getContainerHeight(items, yPerPx, getComputedCols)) @@ -71,7 +71,7 @@ yPerPx, width: containerWidth }) - }, throttleUpdate) + }, untrack(() => throttleUpdate)) onMount(() => { setTimeout(() => { diff --git a/frontend/src/lib/components/apps/editor/RecomputeAllComponents.svelte b/frontend/src/lib/components/apps/editor/RecomputeAllComponents.svelte index 687262cf36..e00a128bf4 100644 --- a/frontend/src/lib/components/apps/editor/RecomputeAllComponents.svelte +++ b/frontend/src/lib/components/apps/editor/RecomputeAllComponents.svelte @@ -1,4 +1,6 @@ (isActive = x)) - let everVisible = $state(visible) + let everVisible = $state(untrack(() => visible)) $effect.pre(() => { visible && !everVisible && (everVisible = true) @@ -199,6 +199,7 @@ {#if everVisible || $app.eagerRendering} +
- import { getContext } from 'svelte' + import { getContext, untrack } from 'svelte' import ComponentInner from './ComponentInner.svelte' import ComponentRendered from './ComponentRendered.svelte' import type { AppComponent } from './components' @@ -25,7 +25,7 @@ overlapped = undefined, componentDraggedId = undefined }: Props = $props() - let everRender = $state(render) + let everRender = $state(untrack(() => render)) $effect(() => { render && !everRender && (everRender = true) diff --git a/frontend/src/lib/components/apps/editor/component/components.ts b/frontend/src/lib/components/apps/editor/component/components.ts index df6d1b2a0e..707d992799 100644 --- a/frontend/src/lib/components/apps/editor/component/components.ts +++ b/frontend/src/lib/components/apps/editor/component/components.ts @@ -4159,14 +4159,16 @@ This is a paragraph. type: 'static', fieldType: 'resource', subFieldType: 'mysql', - value: '' + value: '', + allowTypeChange: false } as StaticAppInput, table: { fieldType: 'select', subFieldType: 'db-table', type: 'static', selectOptions: [], - value: undefined + value: undefined, + allowTypeChange: false } }, ms_sql_server: { @@ -4174,14 +4176,16 @@ This is a paragraph. type: 'static', fieldType: 'resource', subFieldType: 'ms_sql_server', - value: '' + value: '', + allowTypeChange: false } as StaticAppInput, table: { fieldType: 'select', subFieldType: 'db-table', type: 'static', selectOptions: [], - value: undefined + value: undefined, + allowTypeChange: false } }, snowflake: { @@ -4189,14 +4193,16 @@ This is a paragraph. type: 'static', fieldType: 'resource', subFieldType: 'snowflake', - value: '' + value: '', + allowTypeChange: false } as StaticAppInput, table: { fieldType: 'select', subFieldType: 'db-table', type: 'static', selectOptions: [], - value: undefined + value: undefined, + allowTypeChange: false } }, bigquery: { @@ -4204,14 +4210,16 @@ This is a paragraph. type: 'static', fieldType: 'resource', subFieldType: 'bigquery', - value: '' + value: '', + allowTypeChange: false } as StaticAppInput, table: { fieldType: 'select', subFieldType: 'db-table', type: 'static', selectOptions: [], - value: undefined + value: undefined, + allowTypeChange: false } }, ducklake: { @@ -4219,14 +4227,16 @@ This is a paragraph. type: 'static', fieldType: 'ducklake', subFieldType: 'ducklake', - value: '' + value: '', + allowTypeChange: false } as StaticAppInput, table: { fieldType: 'select', subFieldType: 'db-table', type: 'static', selectOptions: [], - value: undefined + value: undefined, + allowTypeChange: false } }, datatable: { @@ -4234,14 +4244,16 @@ This is a paragraph. type: 'static', fieldType: 'datatable', subFieldType: 'datatable', - value: '' + value: '', + allowTypeChange: false } as StaticAppInput, table: { fieldType: 'select', subFieldType: 'db-table', type: 'static', selectOptions: [], - value: undefined + value: undefined, + allowTypeChange: false } } } diff --git a/frontend/src/lib/components/apps/editor/componentsPanel/CssHelperPanel.svelte b/frontend/src/lib/components/apps/editor/componentsPanel/CssHelperPanel.svelte index c50b0ad656..2971a52654 100644 --- a/frontend/src/lib/components/apps/editor/componentsPanel/CssHelperPanel.svelte +++ b/frontend/src/lib/components/apps/editor/componentsPanel/CssHelperPanel.svelte @@ -76,7 +76,7 @@ entries.sort((a, b) => (b.order ?? 0) - (a.order ?? 0) + a.name.localeCompare(b.name)) - let search = '' + let search = $state('')
@@ -98,12 +98,15 @@ } }} > -
- - - {name} - -
+ {#snippet titleSlot()} + {@const SvelteComponent = icon} +
+ + + {name} + +
+ {/snippet} {#if description}
{description}
{/if} diff --git a/frontend/src/lib/components/apps/editor/componentsPanel/CustomComponentsList.svelte b/frontend/src/lib/components/apps/editor/componentsPanel/CustomComponentsList.svelte index 7460c30de9..3f65904c77 100644 --- a/frontend/src/lib/components/apps/editor/componentsPanel/CustomComponentsList.svelte +++ b/frontend/src/lib/components/apps/editor/componentsPanel/CustomComponentsList.svelte @@ -15,9 +15,9 @@ let customcomponents: Array<{ name: string path: string - }> = [] + }> = $state([]) - let loading: boolean = false + let loading: boolean = $state(false) async function getCustomComponents() { loading = true @@ -44,7 +44,7 @@ } }, name: nameField, - js: await files[0].text() + js: await files?.[0]?.text?.() } } }) @@ -61,10 +61,10 @@ nameField = '' } - let nameField: string = '' - let reactVersion: string = '18.2.0' - let files: FileList - let useReact = true + let nameField: string = $state('') + let reactVersion: string = $state('18.2.0') + let files: FileList | undefined = $state() + let useReact = $state(true) getCustomComponents() diff --git a/frontend/src/lib/components/apps/editor/componentsPanel/NameEditor.svelte b/frontend/src/lib/components/apps/editor/componentsPanel/NameEditor.svelte index 6263bc8026..f4d8c40f72 100644 --- a/frontend/src/lib/components/apps/editor/componentsPanel/NameEditor.svelte +++ b/frontend/src/lib/components/apps/editor/componentsPanel/NameEditor.svelte @@ -2,7 +2,7 @@ import Button from '$lib/components/common/button/Button.svelte' import Popover from '$lib/components/meltComponents/Popover.svelte' import { Pen } from 'lucide-svelte' - import { createEventDispatcher } from 'svelte' + import { createEventDispatcher, untrack } from 'svelte' interface Props { kind: string @@ -14,7 +14,7 @@ let { kind, row }: Props = $props() - let editedName = $state(row.name) + let editedName = $state(untrack(() => row).name) const dispatch = createEventDispatcher() function onkeydown(e) { diff --git a/frontend/src/lib/components/apps/editor/componentsPanel/QuickStyleMenu.svelte b/frontend/src/lib/components/apps/editor/componentsPanel/QuickStyleMenu.svelte index cf5bda33c7..676c59c12d 100644 --- a/frontend/src/lib/components/apps/editor/componentsPanel/QuickStyleMenu.svelte +++ b/frontend/src/lib/components/apps/editor/componentsPanel/QuickStyleMenu.svelte @@ -30,7 +30,7 @@ componentProperty = undefined }: Props = $props() const { app } = getContext('AppViewerContext') - const styleStore = createStyleStore(properties) + const styleStore = createStyleStore(untrack(() => properties)) setContext(STYLE_STORE_KEY, styleStore) let multiValues: Record = $state(initiateMultiValues()) @@ -210,12 +210,11 @@ toggleClasses=" !rounded-b-none !py-0 {isOpen[prefix] ? '!bg-surface-secondary hover:!bg-surface-hover' : ''}" > - - + {#snippet titleSlot()} {group} - + {/snippet}
{#each property[group] as p} {@const { diff --git a/frontend/src/lib/components/apps/editor/componentsPanel/QuickStyleProperty.svelte b/frontend/src/lib/components/apps/editor/componentsPanel/QuickStyleProperty.svelte index b4efdb87d4..54d9da3a33 100644 --- a/frontend/src/lib/components/apps/editor/componentsPanel/QuickStyleProperty.svelte +++ b/frontend/src/lib/components/apps/editor/componentsPanel/QuickStyleProperty.svelte @@ -1,5 +1,5 @@ diff --git a/frontend/src/lib/components/apps/editor/contextPanel/ComponentOutput.svelte b/frontend/src/lib/components/apps/editor/contextPanel/ComponentOutput.svelte index 579b769801..3c1a63ad81 100644 --- a/frontend/src/lib/components/apps/editor/contextPanel/ComponentOutput.svelte +++ b/frontend/src/lib/components/apps/editor/contextPanel/ComponentOutput.svelte @@ -1,6 +1,6 @@
diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/ComponentControl.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/ComponentControl.svelte index 9b9d1f3563..992c52de83 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/ComponentControl.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/ComponentControl.svelte @@ -1,4 +1,5 @@ diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/ContextVariables.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/ContextVariables.svelte index 6fb7b7be9b..be54285d84 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/ContextVariables.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/ContextVariables.svelte @@ -1,7 +1,7 @@
- - - - - + {#snippet selectedTitle()} + + + + + {/snippet} {/if} diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte index af6f20e510..23419e51be 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte @@ -6,7 +6,7 @@ import type { InlineScript, Runnable, StaticAppInput } from '$lib/components/apps/inputType' import WorkspaceScriptList from './WorkspaceScriptList.svelte' import WorkspaceFlowList from './WorkspaceFlowList.svelte' - import { createEventDispatcher } from 'svelte' + import { createEventDispatcher, untrack } from 'svelte' import type { Schema } from '$lib/common' import { schemaToInputsSpec } from '$lib/components/apps/utils' import { defaultIfEmptyString, emptySchema } from '$lib/utils' @@ -34,7 +34,7 @@ // const { app, workspace } = getContext('AppViewerContext') let tab: TabType = $state( - onlyFlow + untrack(() => onlyFlow) ? 'workspaceflows' : unusedInlineScripts?.length > 0 ? 'inlinescripts' diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/secondaryMenu/SecondaryMenu.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/secondaryMenu/SecondaryMenu.svelte index 877c48bf35..915e66e4db 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/secondaryMenu/SecondaryMenu.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/secondaryMenu/SecondaryMenu.svelte @@ -1,4 +1,5 @@ diff --git a/frontend/src/lib/components/apps/svelte-grid/Grid.svelte b/frontend/src/lib/components/apps/svelte-grid/Grid.svelte index 4ffb2414ee..fd61edce4f 100644 --- a/frontend/src/lib/components/apps/svelte-grid/Grid.svelte +++ b/frontend/src/lib/components/apps/svelte-grid/Grid.svelte @@ -16,7 +16,7 @@ import { getContainerHeight } from './utils/container' import { moveItem, getItemById, specifyUndefinedColumns } from './utils/item' - import { onMount, getContext } from 'svelte' + import { onMount, getContext, untrack } from 'svelte' import { getColumn, throttle } from './utils/other' import MoveResize from './MoveResize.svelte' import type { FilledItem } from './types' @@ -97,7 +97,7 @@ let container = $state() let xPerPx = $state(0) - let yPerPx = rowHeight + let yPerPx = untrack(() => rowHeight) const onResizeThrottled = throttle(() => { if (!getComputedCols) return @@ -108,7 +108,7 @@ yPerPx, width: containerWidth }) - }, throttleUpdate) + }, untrack(() => throttleUpdate)) let mounted = $state(false) @@ -261,7 +261,7 @@ } } - const throttleMatrix = throttle(updateMatrix, throttleResize) + const throttleMatrix = throttle(updateMatrix, untrack(() => throttleResize)) //let hiddenComponents = writable({}) diff --git a/frontend/src/lib/components/apps/svelte-grid/MoveResize.svelte b/frontend/src/lib/components/apps/svelte-grid/MoveResize.svelte index 033a1a016c..15c98ef83c 100644 --- a/frontend/src/lib/components/apps/svelte-grid/MoveResize.svelte +++ b/frontend/src/lib/components/apps/svelte-grid/MoveResize.svelte @@ -1,5 +1,5 @@ @@ -43,35 +42,38 @@ {/if} {#if (asset.kind === 'resource' && resourceDataCacheValue === undefined) || ducklakeNotFound || datatableNotFound} - - - {:else if datatableNotFound} - - {:else if asset.kind === 'resource' && resourceDataCacheValue === undefined} - - {/if} - + {#snippet trigger()} + + + {:else if datatableNotFound} + + {:else if asset.kind === 'resource' && resourceDataCacheValue === undefined} + + {/if} + + {/snippet} {:else if assetCanBeExplored(asset, { resource_type: resourceDataCacheValue })} onClick?.()} noText _resourceMetadata={{ resource_type: resourceDataCacheValue }} diff --git a/frontend/src/lib/components/assets/AssetColumnBadges.svelte b/frontend/src/lib/components/assets/AssetColumnBadges.svelte index 1a439af06a..59687a562a 100644 --- a/frontend/src/lib/components/assets/AssetColumnBadges.svelte +++ b/frontend/src/lib/components/assets/AssetColumnBadges.svelte @@ -34,9 +34,11 @@ {:else} {@render badge()} - - {accessType2} access to column "{columnName}" - + {#snippet text()} + + {accessType2} access to column "{columnName}" + + {/snippet} {/if} {/each} diff --git a/frontend/src/lib/components/assets/AssetsDropdownButton.svelte b/frontend/src/lib/components/assets/AssetsDropdownButton.svelte index 1e1e1871ae..351ee68331 100644 --- a/frontend/src/lib/components/assets/AssetsDropdownButton.svelte +++ b/frontend/src/lib/components/assets/AssetsDropdownButton.svelte @@ -14,7 +14,7 @@ } from './lib' import { untrack } from 'svelte' import { ResourceService, WorkspaceService } from '$lib/gen' - import { globalDbManagerDrawer, workspaceStore } from '$lib/stores' + import { workspaceStore } from '$lib/stores' import Tooltip from '../meltComponents/Tooltip.svelte' import Tooltip2 from '../Tooltip.svelte' import ResourceEditorDrawer from '../ResourceEditorDrawer.svelte' @@ -48,7 +48,6 @@ let blueBgDiv: HTMLDivElement | undefined = $state() let s3FilePicker: S3FilePicker | undefined = $state() - let dbManagerDrawer = $derived(globalDbManagerDrawer.val) let resourceEditorDrawer: ResourceEditorDrawer | undefined = $state() let isOpen = $state(false) let resourceDataCache: Record = $state({}) @@ -103,7 +102,7 @@ bind:isOpen escapeBehavior="ignore" > - + {#snippet trigger()}
- - + {/snippet} + {#snippet content()}
    {#each assets as asset} {@const ducklakeNotFound = @@ -145,7 +144,7 @@ contentClasses="py-2 px-4 flex flex-col gap-2" disablePopup={!!asset.access_type} > - + {#snippet trigger()}
    {formatAssetAccessType(getAccessType(asset)) ?? '?'}
    -     - + {/snippet} + {#snippet content()} {#if !asset.access_type} Could not infer automatically
    @@ -181,14 +180,14 @@
{/if} - + {/snippet}
{asset.path} - + {#snippet text()} {asset.path} - + {/snippet} {liSubtitle?.(asset) ?? @@ -209,7 +208,6 @@ onClick={() => (isOpen = false)} {asset} {resourceDataCache} - {dbManagerDrawer} {resourceEditorDrawer} {s3FilePicker} {ducklakeNotFound} @@ -218,7 +216,7 @@ {/each} - + {/snippet} diff --git a/frontend/src/lib/components/assets/AssetsUsageDrawer.svelte b/frontend/src/lib/components/assets/AssetsUsageDrawer.svelte index 177032341b..16effb700a 100644 --- a/frontend/src/lib/components/assets/AssetsUsageDrawer.svelte +++ b/frontend/src/lib/components/assets/AssetsUsageDrawer.svelte @@ -71,11 +71,13 @@
{text}
- - {#if tooltip} - {tooltip} - {/if} - + {#snippet text()} + + {#if tooltip} + {tooltip} + {/if} + + {/snippet} {/if} {/snippet} diff --git a/frontend/src/lib/components/assets/JobAssetsViewer.svelte b/frontend/src/lib/components/assets/JobAssetsViewer.svelte index 2eccf66d51..10e2f011f9 100644 --- a/frontend/src/lib/components/assets/JobAssetsViewer.svelte +++ b/frontend/src/lib/components/assets/JobAssetsViewer.svelte @@ -1,7 +1,7 @@ @@ -86,7 +93,6 @@ diff --git a/frontend/src/lib/components/assets/VolumeDetailDrawer.svelte b/frontend/src/lib/components/assets/VolumeDetailDrawer.svelte new file mode 100644 index 0000000000..9506a29903 --- /dev/null +++ b/frontend/src/lib/components/assets/VolumeDetailDrawer.svelte @@ -0,0 +1,107 @@ + + + (open = false)}> + (open = false)}> + {#if loading} +
+ +
+ {:else if volume} +
+
+
+ Files + {volume.file_count} +
+
+ Size + {displaySize(volume.size_bytes) ?? '0 B'} +
+
+ Created at + {displayDate(volume.created_at)} +
+
+ Created by + {volume.created_by} +
+ {#if volume.last_used_at} +
+ Last used + {displayDate(volume.last_used_at)} +
+ {/if} +
+ +
+ {#if s3FilePicker} + + {/if} + {#if $userStore?.is_admin} + + {/if} +
+
+ {:else} +
+ Volume '{volumeName}' not found. +
+ {/if} + + diff --git a/frontend/src/lib/components/assets/VolumesDrawer.svelte b/frontend/src/lib/components/assets/VolumesDrawer.svelte new file mode 100644 index 0000000000..06d4f9529a --- /dev/null +++ b/frontend/src/lib/components/assets/VolumesDrawer.svelte @@ -0,0 +1,193 @@ + + + (open = false)}> + (open = false)}> + {#snippet actions()} + + {#snippet trigger()} + + {/snippet} + {#snippet content({ close })} +
+ { + if (e.key === 'Enter' && newVolumeName.trim()) { + createVolume(newVolumeName.trim(), close) + } + } + }} + bind:value={newVolumeName} + /> + +
+ {/snippet} + + {/snippet} + {#if emptyString($enterpriseLicense)} +
+ + Volumes are limited to 20 per workspace and 50 MB per file. Upgrade to Enterprise Edition + to remove these limits. + +
+ {/if} + {#if volumes.loading} +
+ +
+ {:else if !volumes.current?.length} +
+ No volumes yet. Create one above or they are auto-created when a job declares a volume + annotation. +
+ {:else} +
+ {#each volumes.current as vol (vol.name)} + {@const readable = canReadVolume(vol.created_by, vol.extra_perms)} + {@const writable = canWriteVolume(vol.created_by, vol.extra_perms)} +
+ +
+
+ {vol.name} + } + canWrite={writable} + /> +
+ + {vol.file_count} + {vol.file_count === 1 ? 'file' : 'files'} + · {displaySize(vol.size_bytes) ?? '0 B'} + · owner: {vol.created_by.replace(/^u\//, '')} + +
+ {#if vol.last_used_at} + + Used {displayDate(vol.last_used_at)} + + {/if} + {#if writable} + + {/if} + {#if writable} +
+ {/each} +
+ {/if} + + + + refreshKey++} /> diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index 2dacedbae3..4dae4c20b1 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -26,6 +26,8 @@ export function formatAsset(asset: Asset): string { return `ducklake://${asset.path}` case 'datatable': return `datatable://${asset.path}` + case 'volume': + return `volume://${asset.path}` } return 'unknown' } @@ -89,6 +91,8 @@ export function formatAssetKind(asset: { return 'Ducklake' case 'datatable': return 'Data table' + case 'volume': + return 'Volume' } } diff --git a/frontend/src/lib/components/chartjs-wrappers/Chart.svelte b/frontend/src/lib/components/chartjs-wrappers/Chart.svelte index 9fdacd47de..66739a7859 100644 --- a/frontend/src/lib/components/chartjs-wrappers/Chart.svelte +++ b/frontend/src/lib/components/chartjs-wrappers/Chart.svelte @@ -27,8 +27,8 @@ untrack(() => { chart = new Chart(canvasElem, { type, - data, - options + data: $state.snapshot(data) as typeof data, + options: $state.snapshot(options) as typeof options }) }) return () => { @@ -37,10 +37,10 @@ }) $effect(() => { - chart + data untrack(() => { if (chart) { - chart.data = data + chart.data = $state.snapshot(data) as typeof data chart.update() } }) diff --git a/frontend/src/lib/components/common/FileProgressBar.svelte b/frontend/src/lib/components/common/FileProgressBar.svelte index d339d9477f..5cb864981b 100644 --- a/frontend/src/lib/components/common/FileProgressBar.svelte +++ b/frontend/src/lib/components/common/FileProgressBar.svelte @@ -1,23 +1,38 @@ {#key color}
{#if ended} - + {@render children?.()} {:else}
- export let notificationCount = 0 - export let notificationLimit: number | undefined = undefined + interface Props { + notificationCount?: number; + notificationLimit?: number | undefined; + } + + let { notificationCount = 0, notificationLimit = undefined }: Props = $props(); {#if notificationCount > 0} diff --git a/frontend/src/lib/components/common/button/AnimatedButton.svelte b/frontend/src/lib/components/common/button/AnimatedButton.svelte index 7248b891dc..3ceb0ebd4c 100644 --- a/frontend/src/lib/components/common/button/AnimatedButton.svelte +++ b/frontend/src/lib/components/common/button/AnimatedButton.svelte @@ -1,12 +1,26 @@ {#if animate} @@ -19,10 +33,10 @@ {ringColor} {darkMode} > - + {@render children?.()} {:else}
- + {@render children?.()}
{/if} diff --git a/frontend/src/lib/components/common/button/AnimatedButtonInner.svelte b/frontend/src/lib/components/common/button/AnimatedButtonInner.svelte index 7396b5314b..9ba46928a0 100644 --- a/frontend/src/lib/components/common/button/AnimatedButtonInner.svelte +++ b/frontend/src/lib/components/common/button/AnimatedButtonInner.svelte @@ -1,27 +1,41 @@ @@ -33,7 +47,7 @@ bind:clientWidth bind:clientHeight > - + {@render children?.()}
diff --git a/frontend/src/lib/components/graph/MoveHandleButton.svelte b/frontend/src/lib/components/graph/MoveHandleButton.svelte new file mode 100644 index 0000000000..d077f0d03d --- /dev/null +++ b/frontend/src/lib/components/graph/MoveHandleButton.svelte @@ -0,0 +1,74 @@ + + + diff --git a/frontend/src/lib/components/graph/SelectionBoundingBox.svelte b/frontend/src/lib/components/graph/SelectionBoundingBox.svelte index 5faff92d28..e1248b52ef 100644 --- a/frontend/src/lib/components/graph/SelectionBoundingBox.svelte +++ b/frontend/src/lib/components/graph/SelectionBoundingBox.svelte @@ -1,23 +1,46 @@ -{#if bounds() && selectedNodes.length > 1} - {@const currentBounds = bounds()!} +{#if bounds && selectedNodes.length > 1} + {@const currentBounds = bounds!}
- - {#if noteEditorContext?.noteEditor} -
- - Create group note ({selectedNodes.length} nodes) - -
- {/if} + {:else if resolvedCount > 0} + {#if canMoveSelected && moveManager && resolvedModuleIds.length > 0} +
+ onMoveSelected?.()} + /> +
+ {/if} + + {#snippet buttonReplacement()} + + {/snippet} + + {/if} +
{/if} diff --git a/frontend/src/lib/components/graph/SelectionTool.svelte b/frontend/src/lib/components/graph/SelectionTool.svelte index e7af55bc24..dd5033bf55 100644 --- a/frontend/src/lib/components/graph/SelectionTool.svelte +++ b/frontend/src/lib/components/graph/SelectionTool.svelte @@ -1,4 +1,5 @@
- - {#if usageCount !== undefined} - Used in {pluralize(usageCount, 'step')}
- {/if} - { - if (data.asset.kind === 'resource') - flowGraphAssetsCtx?.val.resourceEditorDrawer?.initEdit(data.asset.path) - }} - > - {data.asset.path} -
- - {formatAssetKind({ ...data.asset, metadata: cachedResourceMetadata })} - - + {#snippet text()} + + {#if usageCount !== undefined} + Used in {pluralize(usageCount, 'step')}
+ {/if} + { + if (data.asset.kind === 'resource') + flowGraphAssetsCtx?.val.resourceEditorDrawer?.initEdit(data.asset.path) + }} + > + {data.asset.path} +
+ + {formatAssetKind({ ...data.asset, metadata: cachedResourceMetadata })} + + + {/snippet} {/snippet} diff --git a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte index 12e0b71673..8183be16d6 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte @@ -54,18 +54,22 @@ )} placement="top" > - - +{data.overflowedAssets.length} - - -
    - {#each data.overflowedAssets as asset} -
  • - -
    • - {/each} -
- + {#snippet trigger()} + + +{data.overflowedAssets.length} + + {/snippet} + {#snippet content()} + +
    + {#each data.overflowedAssets as asset} +
  • + +
    • + {/each} +
+ + {/snippet} {/snippet} diff --git a/frontend/src/lib/components/graph/renderers/nodes/ModuleNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/ModuleNode.svelte index 27cb72457c..01953f2d8b 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/ModuleNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/ModuleNode.svelte @@ -1,12 +1,11 @@ - + {#snippet children({ darkMode })} @@ -22,10 +24,27 @@ offset = 0, wrapperClass = '', contextMenuItems = undefined, + menuItems = undefined, nodeId = undefined, children }: Props = $props() + let resolvedContextMenuItems: ContextMenuItem[] | undefined = $derived( + contextMenuItems ?? + menuItems?.flatMap((item) => [ + ...(item.separatorTop ? [{ id: `${item.displayName}-divider`, label: '', divider: true }] : []), + { + id: item.displayName, + label: item.displayName, + icon: item.icon, + disabled: item.disabled, + type: item.type, + shortcut: item.shortcut, + onClick: item.action as (() => void) | undefined + } + ]) + ) + const { moveManager } = getGraphContext() let faded = $derived( @@ -37,8 +56,8 @@ -{#if contextMenuItems && contextMenuItems.length > 0} - +{#if resolvedContextMenuItems && resolvedContextMenuItems.length > 0} +
{@render children?.({ darkMode })}
diff --git a/frontend/src/lib/components/graph/renderers/nodes/NoteNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/NoteNode.svelte index 55cac523e3..58e9fe05cb 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/NoteNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/NoteNode.svelte @@ -16,7 +16,7 @@ import { getNoteEditorContext } from '../../noteEditor.svelte' import { getGraphContext } from '../../graphContext' import { clickOutside } from '$lib/utils' - import { tick } from 'svelte' + import { tick, untrack } from 'svelte' interface Props { data: { @@ -52,7 +52,7 @@ let textareaElement: HTMLTextAreaElement | undefined = $state(undefined) let editMode = $state(false) let hovering = $state(false) - let textContent = $state(data.text ?? '') + let textContent = $state(untrack(() => data).text ?? '') let containerHeight = $state(0) // Use data props directly - they're kept in sync by NoteManager observer diff --git a/frontend/src/lib/components/graph/renderers/triggers/TriggerButton.svelte b/frontend/src/lib/components/graph/renderers/triggers/TriggerButton.svelte index ef82a1b17d..c518081f1b 100644 --- a/frontend/src/lib/components/graph/renderers/triggers/TriggerButton.svelte +++ b/frontend/src/lib/components/graph/renderers/triggers/TriggerButton.svelte @@ -1,15 +1,23 @@ diff --git a/frontend/src/lib/components/graph/renderers/triggers/TriggerCount.svelte b/frontend/src/lib/components/graph/renderers/triggers/TriggerCount.svelte index c08cfcc056..62eadd5d90 100644 --- a/frontend/src/lib/components/graph/renderers/triggers/TriggerCount.svelte +++ b/frontend/src/lib/components/graph/renderers/triggers/TriggerCount.svelte @@ -1,7 +1,11 @@ {#if count && count > 0} diff --git a/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte b/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte index 01485fa519..4bf556078d 100644 --- a/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte +++ b/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte @@ -189,11 +189,7 @@ type === 'email' || type === 'cli' || (triggersGrouped[type] && triggersGrouped[type].length === 1)} - e.stopPropagation()} - > + {#snippet text()} {camelCaseToWords(type)} {/snippet} @@ -279,7 +275,7 @@ isSelected ? 'bg-surface-accent-selected text-accent border-border-selected' : '', small ? 'w-[23px] h-[23px]' : 'p-2' )} - on:click={(e) => { + onClick={(e) => { e.stopPropagation() e.preventDefault() if (singleItem) { diff --git a/frontend/src/lib/components/graph/selectionUtils.svelte.ts b/frontend/src/lib/components/graph/selectionUtils.svelte.ts index 8abc7fa563..daa9de3ecf 100644 --- a/frontend/src/lib/components/graph/selectionUtils.svelte.ts +++ b/frontend/src/lib/components/graph/selectionUtils.svelte.ts @@ -78,13 +78,24 @@ export class SelectionManager { } // If the new selection is the same as the current selection, do nothing - if (JSON.stringify(nodes) === JSON.stringify($state.snapshot(this.#selectedNodes))) { + const newIds = nodes.map((n) => n.id).join(',') + const currentIds = this.#selectedNodes.map((n) => n.id).join(',') + if (newIds === currentIds) { return } this.#selectedNodes = nodes } + // Select multiple nodes by their IDs + selectByIds(ids: string[]) { + if (!ids || ids.length === 0) { + this.clearSelection() + return + } + this.#selectedNodes = ids.map((id) => ({ id })) + } + // Clear all selections clearSelection() { this.#selectedNodes = [{ id: 'settings' }] diff --git a/frontend/src/lib/components/home/FlowIcon.svelte b/frontend/src/lib/components/home/FlowIcon.svelte index f9822b063f..70eb02aaea 100644 --- a/frontend/src/lib/components/home/FlowIcon.svelte +++ b/frontend/src/lib/components/home/FlowIcon.svelte @@ -2,9 +2,16 @@ import BarsStaggered from '$lib/components/icons/BarsStaggered.svelte' import { twMerge } from 'tailwind-merge' - export let color: string = 'black' + + interface Props { + color?: string; + class?: string; + } + + let { color = 'black', class: className = '' }: Props = $props(); + -
+
diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index f66e096ecc..cae0144169 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -31,7 +31,7 @@ import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' import FlowIcon from './FlowIcon.svelte' import { canWrite, getLocalSetting, storeLocalSetting } from '$lib/utils' - import { page } from '$app/stores' + import { page } from '$app/state' import { setQuery } from '$lib/navigation' import Drawer from '../common/drawer/Drawer.svelte' import HighlightCode from '../HighlightCode.svelte' @@ -48,7 +48,11 @@ showEditButtons?: boolean } - let { filter = $bindable(''), subtab = $bindable('script'), showEditButtons = true }: Props = $props() + let { + filter = $bindable(''), + subtab = $bindable('script'), + showEditButtons = true + }: Props = $props() type TableItem = T & { canWrite: boolean @@ -73,7 +77,7 @@ let filteredItems: (TableScript | TableFlow | TableApp | TableRawApp)[] = $state([]) let itemKind = $state( - ($page.url.searchParams.get('kind') as 'script' | 'flow' | 'app' | 'all') ?? 'all' + (page.url.searchParams.get('kind') as 'script' | 'flow' | 'app' | 'all') ?? 'all' ) let loading = $state(true) @@ -367,7 +371,7 @@ if (itemKind != 'all') { subtab = v } - setQuery($page.url, 'kind', v) + setQuery(page.url, 'kind', v) }} > {#snippet children({ item })} diff --git a/frontend/src/lib/components/home/ListFiltersQuick.svelte b/frontend/src/lib/components/home/ListFiltersQuick.svelte index 6a36338094..ed80815700 100644 --- a/frontend/src/lib/components/home/ListFiltersQuick.svelte +++ b/frontend/src/lib/components/home/ListFiltersQuick.svelte @@ -4,13 +4,16 @@ import { createEventDispatcher } from 'svelte' import { Button } from '../common' - export let filters: string[] - export let selectedFilter: - | { kind: 'owner' | 'integrations'; name: string | undefined } - | undefined = undefined - $: selectedAppFilter = selectedFilter?.kind === 'integrations' ? selectedFilter?.name : undefined - export let resourceType = false + interface Props { + filters: string[]; + selectedFilter?: + | { kind: 'owner' | 'integrations'; name: string | undefined } + | undefined; + resourceType?: boolean; + } + + let { filters, selectedFilter = $bindable(undefined), resourceType = false }: Props = $props(); function getIconComponent(name: string, resourceType: boolean) { if (resourceType) { @@ -29,6 +32,7 @@ } const dispatch = createEventDispatcher() + let selectedAppFilter = $derived(selectedFilter?.kind === 'integrations' ? selectedFilter?.name : undefined) {#if Array.isArray(filters) && filters.length > 0} diff --git a/frontend/src/lib/components/icons/AssetGenericIcon.svelte b/frontend/src/lib/components/icons/AssetGenericIcon.svelte index 669201d919..1e10f38d8a 100644 --- a/frontend/src/lib/components/icons/AssetGenericIcon.svelte +++ b/frontend/src/lib/components/icons/AssetGenericIcon.svelte @@ -1,5 +1,5 @@ + + + + + + + diff --git a/frontend/src/lib/components/icons/RubyIcon.svelte b/frontend/src/lib/components/icons/RubyIcon.svelte index 307fde1a3d..5f2623dddf 100644 --- a/frontend/src/lib/components/icons/RubyIcon.svelte +++ b/frontend/src/lib/components/icons/RubyIcon.svelte @@ -24,7 +24,7 @@ image/svg+xml - + >
@@ -52,30 +84,101 @@ />
+ +