Merge branch 'main' into glm/focus-summary

This commit is contained in:
Guilhem
2026-02-23 09:51:25 +00:00
committed by GitHub
327 changed files with 20206 additions and 7751 deletions
+4
View File
@@ -44,6 +44,10 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
# Bun
COPY --from=oven/bun:1.3.8 /usr/local/bin/bun /usr/bin/bun
# Install windmill CLI
RUN bun install -g windmill-cli \
&& ln -s $(bun pm bin -g)/wmill /usr/bin/wmill
ARG TARGETPLATFORM
# Deno
+33 -2
View File
@@ -19,7 +19,7 @@ defaults:
jobs:
cargo_test:
runs-on: ubicloud-standard-16
runs-on: blacksmith-16vcpu-ubuntu-2404
services:
postgres:
image: postgres
@@ -70,6 +70,16 @@ jobs:
with:
ruby-version: "3.3"
bundler-cache: false
- name: Install windmill CLI from source
run: |
cd $GITHUB_WORKSPACE/cli
bash gen_wm_client.sh
bun install
mkdir -p "$HOME/.local/bin"
printf '#!/bin/sh\nexec bun run "%s/cli/src/main.ts" "$@"\n' "$GITHUB_WORKSPACE" > "$HOME/.local/bin/wmill"
chmod +x "$HOME/.local/bin/wmill"
echo "$HOME/.local/bin" >> $GITHUB_PATH
working-directory: /
- name: Install PowerShell, mold and clang
run: |
sudo apt-get update && sudo apt-get install -y powershell mold clang libcurl4-openssl-dev
@@ -78,6 +88,20 @@ jobs:
with:
cache: false
toolchain: 1.93.0
- name: Cache cargo target directory
uses: useblacksmith/stickydisk@v1
with:
key: cargo-target
path: ./backend/target
- name: Cache cargo registry
uses: useblacksmith/cache@v1
with:
path: |
~/.cargo/registry
~/.cargo/git
key: cargo-registry-${{ hashFiles('backend/Cargo.lock') }}
restore-keys: |
cargo-registry-
- name: Read EE repo commit hash
run: |
echo "ee_repo_ref=$(cat ./ee-repo-ref.txt)" >> "$GITHUB_ENV"
@@ -165,6 +189,12 @@ jobs:
fi
echo "NPM_TOKEN=${NPM_TOKEN}" >> $GITHUB_ENV
{
echo "TEST_NPMRC<<NPMRC_EOF"
echo "@windmill-test:registry=http://localhost:4873/"
echo "//localhost:4873/:_authToken=${NPM_TOKEN}"
echo "NPMRC_EOF"
} >> $GITHUB_ENV
echo "Got NPM token successfully: ${NPM_TOKEN:0:10}..."
# Configure npm globally with the auth token
@@ -199,7 +229,7 @@ jobs:
fi
echo "Verified: Package requires authentication for @windmill-test/private-pkg"
- name: Cache DuckDB FFI module build
uses: actions/cache@v3
uses: useblacksmith/cache@v1
with:
path: ./backend/windmill-duckdb-ffi-internal/target
key: ${{ runner.os }}-duckdb-ffi-${{ hashFiles('./backend/windmill-duckdb-ffi-internal/src/**/*.rs', './backend/windmill-duckdb-ffi-internal/Cargo.toml', './backend/windmill-duckdb-ffi-internal/Cargo.lock') }}
@@ -215,6 +245,7 @@ jobs:
RUST_LOG_STYLE: never
CARGO_NET_GIT_FETCH_WITH_CLI: true
CARGO_BUILD_JOBS: 12
CARGO_INCREMENTAL: 1
WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1
WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1
WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1
+1 -1
View File
@@ -9,7 +9,7 @@ permissions: write-all
jobs:
build_ee:
runs-on: ubicloud
runs-on: ubicloud-standard-4
steps:
- uses: actions/checkout@v4
with:
@@ -9,7 +9,7 @@ permissions: write-all
jobs:
build_ee:
runs-on: ubicloud
runs-on: ubicloud-standard-4
steps:
- uses: actions/checkout@v4
with:
+21 -22
View File
@@ -23,16 +23,16 @@ jobs:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Deno
uses: denoland/setup-deno@v2
with:
deno-version: v2.x
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
- name: Setup Bun
uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Generate Windmill client
working-directory: cli
run: ./gen_wm_client.sh
@@ -69,11 +69,6 @@ jobs:
cache: true
cache-workspaces: backend
- name: Setup Deno
uses: denoland/setup-deno@v2
with:
deno-version: v2.x
- name: Setup Node.js
uses: actions/setup-node@v4
with:
@@ -90,6 +85,10 @@ jobs:
- name: Symlink Node to /usr/bin/node
run: sudo ln -sf $(which node) /usr/bin/node
- name: Install dependencies
working-directory: cli
run: bun install
- name: Generate Windmill clients
working-directory: cli
run: |
@@ -101,12 +100,10 @@ jobs:
env:
DATABASE_URL: postgres://postgres:changeme@localhost:5432
CI_MINIMAL_FEATURES: "true"
run: |
deno test --no-check --allow-all test/ \
--ignore=test/cargo_backend_example.test.ts
run: bun test --timeout 120000 test/
test-windows:
runs-on: windows-latest
runs-on: blacksmith-16vcpu-windows-2025
steps:
- name: Checkout code
@@ -126,11 +123,6 @@ jobs:
cache: true
cache-workspaces: backend
- name: Setup Deno
uses: denoland/setup-deno@v2
with:
deno-version: v2.x
- name: Setup Node.js
uses: actions/setup-node@v4
with:
@@ -150,6 +142,10 @@ jobs:
echo "BUN_PATH=$bunPath" >> $env:GITHUB_OUTPUT
echo "NODE_BIN_PATH=$nodePath" >> $env:GITHUB_OUTPUT
- name: Install dependencies
working-directory: cli
run: bun install
- name: Generate Windmill clients
working-directory: cli
shell: bash
@@ -165,9 +161,12 @@ jobs:
CI_MINIMAL_FEATURES: "true"
BUN_PATH: ${{ steps.runtime-paths.outputs.BUN_PATH }}
NODE_BIN_PATH: ${{ steps.runtime-paths.outputs.NODE_BIN_PATH }}
run: |
deno test --no-check --allow-all test/ `
--ignore=test/cargo_backend_example.test.ts
run: bun test --timeout 120000 test/
- name: Keep runner alive for SSH debug
if: failure()
shell: pwsh
run: Start-Sleep -Seconds 3600
# Combined summary job for branch protection
test-summary:
@@ -6,6 +6,12 @@ on:
- opened
- ready_for_review
- closed
issue_comment:
types:
- created
pull_request_review_comment:
types:
- created
jobs:
notify_discord_when_pr_opened:
@@ -33,3 +39,38 @@ jobs:
PR_NUMBER: ${{ github.event.pull_request.number }}
secrets:
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }}
notify_discord_on_comment:
if: >
github.event_name == 'issue_comment'
&& github.event.issue.pull_request
&& github.event.comment.user.login != 'cloudflare-workers-and-pages[bot]'
&& github.event.comment.user.login != 'ellipsis-dev[bot]'
uses: ./.github/workflows/shareable-discord-notification.yml
with:
PR_STATUS: "comment"
PR_NUMBER: ${{ github.event.issue.number }}
COMMENT_BODY: ${{ github.event.comment.body }}
COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
COMMENT_URL: ${{ github.event.comment.html_url }}
DISCORD_CHANNEL_ID: "1372204995868491786"
DISCORD_GUILD_ID: "930051556043276338"
secrets:
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }}
notify_discord_on_review_comment:
if: >
github.event_name == 'pull_request_review_comment'
&& github.event.comment.user.login != 'cloudflare-workers-and-pages[bot]'
&& github.event.comment.user.login != 'ellipsis-dev[bot]'
uses: ./.github/workflows/shareable-discord-notification.yml
with:
PR_STATUS: "comment"
PR_NUMBER: ${{ github.event.pull_request.number }}
COMMENT_BODY: ${{ github.event.comment.body }}
COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
COMMENT_URL: ${{ github.event.comment.html_url }}
DISCORD_CHANNEL_ID: "1372204995868491786"
DISCORD_GUILD_ID: "930051556043276338"
secrets:
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }}
+2 -2
View File
@@ -25,9 +25,9 @@ jobs:
with:
node-version: "20.x"
registry-url: "https://registry.npmjs.org"
- uses: denoland/setup-deno@v2
- uses: oven-sh/setup-bun@v2
with:
deno-version: v2.x
bun-version: latest
- run: cd cli && ./build.sh && cd npm && npm publish
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
@@ -24,9 +24,22 @@ on:
DISCORD_GUILD_ID:
description: "The Discord guild ID"
type: string
COMMENT_BODY:
description: "The comment body"
type: string
default: ""
COMMENT_AUTHOR:
description: "The comment author"
type: string
default: ""
COMMENT_URL:
description: "The comment URL"
type: string
default: ""
secrets:
DISCORD_WEBHOOK_URL:
description: "Discord Webhook URL"
required: false
DISCORD_BOT_TOKEN:
description: "Discord Bot Token"
@@ -117,3 +130,54 @@ jobs:
curl -X PUT \
-H "Authorization: Bot $BOT_TOKEN" \
"https://discord.com/api/v10/channels/$thread_id/messages/$message_id/reactions/%E2%9C%85/@me"
post_comment:
runs-on: ubuntu-latest
if: ${{ inputs.PR_STATUS == 'comment' }}
steps:
- name: Post comment to Discord thread
env:
BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
CHANNEL_ID: ${{ inputs.DISCORD_CHANNEL_ID }}
GUILD_ID: ${{ inputs.DISCORD_GUILD_ID }}
PR_NUMBER: ${{ inputs.PR_NUMBER }}
COMMENT_BODY: ${{ inputs.COMMENT_BODY }}
COMMENT_AUTHOR: ${{ inputs.COMMENT_AUTHOR }}
COMMENT_URL: ${{ inputs.COMMENT_URL }}
run: |
# 1) Find the thread by PR number
threads=$(curl -s -H "Authorization: Bot $BOT_TOKEN" \
"https://discord.com/api/v10/guilds/${GUILD_ID}/threads/active")
thread_id=$(echo "$threads" | jq -r \
--arg cid "$CHANNEL_ID" \
--arg pref "#${PR_NUMBER}:" \
'.threads[] | select(.parent_id == $cid and (.name | startswith($pref))) | .id')
if [ -z "$thread_id" ]; then
echo "Thread not found for PR #${PR_NUMBER}, skipping"
exit 0
fi
# 2) Truncate comment body to fit Discord's 2000 char limit
# Reserve space for the author line + link (~100 chars)
max_body=1800
if [ ${#COMMENT_BODY} -gt $max_body ]; then
# For bot comments, show the tail (conclusions/code tend to be at the end)
if [[ "$COMMENT_AUTHOR" == *"[bot]"* ]] || [[ "$COMMENT_AUTHOR" == *"-bot"* ]]; then
truncated_body="...${COMMENT_BODY: -$max_body}"
else
truncated_body="${COMMENT_BODY:0:$max_body}..."
fi
else
truncated_body="$COMMENT_BODY"
fi
# 3) Post the comment to the thread
message=$(printf '**%s** [commented](%s):\n%s' "$COMMENT_AUTHOR" "$COMMENT_URL" "$truncated_body")
payload=$(jq -n --arg content "$message" '{content: $content, flags: 4, allowed_mentions: {parse: []}}')
curl -s -X POST \
-H "Authorization: Bot $BOT_TOKEN" \
-H "Content-Type: application/json" \
-d "$payload" \
"https://discord.com/api/v10/channels/${thread_id}/messages"
+4
View File
@@ -14,9 +14,13 @@ backend/.minio-data
!.aiderignore
rust-client/Cargo.toml
# Worktree-generated port isolation
.env.local
# Symlinked cache directories (for git worktrees)
backend/target
frontend/node_modules
typescript-client/node_modules
frontend/.svelte-kit
backend/chrome_profiler.json
.fast-check/
+63
View File
@@ -0,0 +1,63 @@
main_branch: main
merge_strategy: rebase
# worktree_dir: .worktrees
worktree_naming: basename
worktree_prefix: ""
# Default: "wm-"
window_prefix: "wm-"
auto_name:
model: "claude-sonnet-4.6"
system_prompt: |
Generate a concise git branch name based on the task description.
Rules:
- Use kebab-case (lowercase with hyphens)
- Keep it short: 1-3 words, max 4 if necessary
- Focus on the core task/feature, not implementation details
- No prefixes like feat/, fix/, chore/
Examples of good branch names:
- "Add dark mode toggle" → dark-mode
- "Fix the search results not showing" → fix-search
- "Refactor the authentication module" → auth-refactor
- "Add CSV export to reports" → export-csv
- "Shell completion is broken" → shell-completion
Output ONLY the branch name, nothing else.
background: true
# Commands to run in new worktree before tmux window opens.
# These block window creation - use for short tasks only.
# Use "<global>" to inherit from global config.
# Set to empty list to disable: `post_create: []`
# post_create:
# - "<global>"
# - mise use
post_create:
- ./scripts/worktree-env
pre_remove:
- ./scripts/worktree-cleanup
panes:
- command: <agent>
focus: true
- command: 'ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/backend" && PORT=${BACKEND_PORT:-8000} cargo watch -x run'
split: horizontal
- command: 'ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/frontend" && npm install && npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000}'
split: vertical
files:
copy:
- backend/.env
- scripts/
sandbox:
enabled: false
toolchain: off
+66
View File
@@ -1,5 +1,71 @@
# Changelog
## [1.642.0](https://github.com/windmill-labs/windmill/compare/v1.641.0...v1.642.0) (2026-02-22)
### Features
* **cli:** add consistent get/list/new subcommands for all item types ([#8047](https://github.com/windmill-labs/windmill/issues/8047)) ([4fedfdf](https://github.com/windmill-labs/windmill/commit/4fedfdfd11aa8ca7fff6f7aed5ae2b313888f878))
### Bug Fixes
* make WM_FLOW_PATH available in flow step previews ([#8042](https://github.com/windmill-labs/windmill/issues/8042)) ([a91c532](https://github.com/windmill-labs/windmill/commit/a91c532ecadce63cea965c497351fa1a6f39697a))
* preserve debouncing settings for flows with preprocessors ([#8043](https://github.com/windmill-labs/windmill/issues/8043)) ([a00927b](https://github.com/windmill-labs/windmill/commit/a00927b3008a2d953fde1d461723a3c92f375eb4))
## [1.641.0](https://github.com/windmill-labs/windmill/compare/v1.640.0...v1.641.0) (2026-02-21)
### Features
* add .npmrc support for private npm registries ([#8039](https://github.com/windmill-labs/windmill/issues/8039)) ([9eb1531](https://github.com/windmill-labs/windmill/commit/9eb15312f663aa6d700e8ac562d7b5c75c2221f7))
### Bug Fixes
* add created_by ownership check to update/delete saved inputs ([#8038](https://github.com/windmill-labs/windmill/issues/8038)) ([e8a13ed](https://github.com/windmill-labs/windmill/commit/e8a13edde7c0ba2ef80344ab7c7288e7bb2eb6b5))
* run substitute_ee_code.sh after creating EE worktree ([b330f38](https://github.com/windmill-labs/windmill/commit/b330f388894ecd9cc6b64297420ac6f032d32f72))
* tag bunnative dependency jobs as bun instead of nativets ([#8045](https://github.com/windmill-labs/windmill/issues/8045)) ([fd5ebc2](https://github.com/windmill-labs/windmill/commit/fd5ebc2fda589c022074c3bb4dcdb447c7f86cf0))
## [1.640.0](https://github.com/windmill-labs/windmill/compare/v1.639.0...v1.640.0) (2026-02-20)
### Features
* add windmill-ee-private worktree support to workmux ([#8034](https://github.com/windmill-labs/windmill/issues/8034)) ([9f3dd0b](https://github.com/windmill-labs/windmill/commit/9f3dd0bf2b2ba7c622093c54b7b6b5e7ebb26b74))
* **cli:** add --locks-required flag to wmill lint and sync push ([#8026](https://github.com/windmill-labs/windmill/issues/8026)) ([4abe589](https://github.com/windmill-labs/windmill/commit/4abe58939787f375ccfef5b2dbcfbd7e86cff076))
* dedicated nativets ([#8021](https://github.com/windmill-labs/windmill/issues/8021)) ([37c9acb](https://github.com/windmill-labs/windmill/commit/37c9acb232c64c98ecfb64754f5b69b31047c625))
* Support column detection on S3 objects in DuckDB ([#8018](https://github.com/windmill-labs/windmill/issues/8018)) ([87f3de9](https://github.com/windmill-labs/windmill/commit/87f3de9ae5975c88b6748e297f84a539aec4c0ca))
### Bug Fixes
* Fix DuckDB incorrect pg password encoding ([#8028](https://github.com/windmill-labs/windmill/issues/8028)) ([90b1a7a](https://github.com/windmill-labs/windmill/commit/90b1a7a531bce5621ea4de4792a8c9d3d3beec3d))
* **frontend:** use completed_at instead of created_at for job history ([#8022](https://github.com/windmill-labs/windmill/issues/8022)) ([24d7921](https://github.com/windmill-labs/windmill/commit/24d7921bcf23543759719ffd2463959c627b61b8))
### Performance Improvements
* lazy-load JSZip in RawAppEditorHeader ([#8012](https://github.com/windmill-labs/windmill/issues/8012)) ([a1ba10a](https://github.com/windmill-labs/windmill/commit/a1ba10a29e12ab5f553bd9aad74067cc5b3ead9e))
## [1.639.0](https://github.com/windmill-labs/windmill/compare/v1.638.4...v1.639.0) (2026-02-18)
### Features
* improve FolderPicker with edit icon pattern ([#7995](https://github.com/windmill-labs/windmill/issues/7995)) ([db8aa8a](https://github.com/windmill-labs/windmill/commit/db8aa8a0839b5729f0bb847e7a71766c7883ff36))
### Bug Fixes
* default automate_username_creation to true when setting is missing ([#8006](https://github.com/windmill-labs/windmill/issues/8006)) ([d2d08f8](https://github.com/windmill-labs/windmill/commit/d2d08f8817e6e7818eb4b6f092e66ae039f0c756))
* handle raw app folder deletion in sync push without yaml parse error ([#7994](https://github.com/windmill-labs/windmill/issues/7994)) ([f6d99dd](https://github.com/windmill-labs/windmill/commit/f6d99dd18c06a7f5aea93122276dd68c45772b43))
### Performance Improvements
* **cli:** skip relock more accurate ([#7993](https://github.com/windmill-labs/windmill/issues/7993)) ([cd4151a](https://github.com/windmill-labs/windmill/commit/cd4151a84b2c1e0f2e616079091d0429bf469f4e))
## [1.638.4](https://github.com/windmill-labs/windmill/compare/v1.638.3...v1.638.4) (2026-02-17)
+4
View File
@@ -258,6 +258,10 @@ COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.3.8 /usr/local/bin/bun /usr/bin/bun
# Install windmill CLI
RUN bun install -g windmill-cli \
&& ln -s $(bun pm bin -g)/wmill /usr/bin/wmill
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
+177
View File
@@ -0,0 +1,177 @@
# Windmill Development with workmux
This guide covers the workmux-based development setup for Windmill. Each worktree gets its own tmux window with a Claude Code agent, a backend server (with auto-reload), and a frontend dev server — all on isolated ports.
## Prerequisites
- tmux
- Rust toolchain (rustup)
- Node.js + npm
- PostgreSQL running locally (see `backend/.env`)
## Installation
### 1. Install workmux
```bash
cargo install workmux
```
### 2. Install the Claude Code plugin
```bash
workmux claude install
```
This lets workmux manage Claude Code agents in worktree panes.
### 3. Install cargo-watch
Used for auto-recompiling the backend on file changes:
```bash
cargo install cargo-watch
```
### 4. Install llm CLI (required for auto branch naming)
workmux uses the `llm` CLI to automatically generate branch names from prompts. Install it with:
```bash
uv tool install llm
llm install llm-anthropic
```
Then set your Anthropic API key:
```bash
llm keys set anthropic
# paste your API key when prompted
```
### 5. Recommended: shell alias and autocomplete
Set up a `wm` alias for convenience:
```bash
# Add to your ~/.zshrc
alias wm="workmux"
```
Setting up zsh autocomplete is also recommended — see the [workmux docs](https://github.com/rubenfiszel/workmux) for instructions.
## Port Slot System
Each worktree is assigned a **slot** that determines its ports:
| Slot | Backend | Frontend |
|------|---------|----------|
| 0 | 8000 | 3000 |
| 1 | 8010 | 3010 |
| 2 | 8020 | 3020 |
| 3 | 8030 | 3030 |
| ... | ... | ... |
- **Slot 0** is reserved for the main worktree (default `cargo run` / `npm run dev`).
- Without `WM_SLOT`, the script auto-assigns the first available slot (starting from 1) and prints it.
- With `WM_SLOT=N`, it uses that slot and errors if the ports are taken.
## SSH Port Forwarding
If you develop over SSH, add this to `~/.ssh/config` on your **local machine** to pre-configure tunnels for each slot:
```
Host windmill-dev
HostName <remote-ip>
User <username>
# Slot 0 (main worktree)
LocalForward 8000 localhost:8000
LocalForward 3000 localhost:3000
# Slot 1
LocalForward 8010 localhost:8010
LocalForward 3010 localhost:3010
# Slot 2
LocalForward 8020 localhost:8020
LocalForward 3020 localhost:3020
# Slot 3
LocalForward 8030 localhost:8030
LocalForward 3030 localhost:3030
```
Then connect once and all tunnels are active:
```bash
ssh windmill-dev
```
Access the frontend at `http://localhost:<frontend-port>` in your local browser.
## Quickstart
```bash
# Create a new worktree (auto-assigns slot, prints ports)
workmux add my-feature
# Or with an explicit slot
WM_SLOT=2 workmux add my-feature
# Create a worktree and immediately send a prompt to the agent
workmux add -A -p "fix the login bug in auth.rs"
```
The `add` command creates the worktree but does **not** open it. To open the tmux window and start working:
```bash
workmux open my-feature
```
This will open a tmux window with three panes:
- **Claude Code agent** (focused)
- **Backend**: `cargo watch -x run` on the assigned port (auto-reloads on save)
- **Frontend**: `npm run dev` proxying to the backend
When using `-A` with `add`, the worktree is created and opened automatically, and the prompt is sent to the agent right away.
Check which ports were assigned:
```bash
cat <worktree-path>/.env.local
```
### Sending work to the agent
```bash
# Send a prompt to the agent in a worktree
workmux send my-feature "fix the login bug in auth.rs"
# Check agent status
workmux status
```
### Merging and cleaning up
We never merge worktrees directly — always create a PR on GitHub and let it be merged there. Once the PR is merged, clean up the worktree:
```bash
# Close the tmux window but keep the worktree
workmux close my-feature
# After your PR is merged, remove the worktree, branch, and tmux window
workmux rm my-feature
```
> **Note**: Do not use `workmux merge`. Always go through a PR to get your changes into main. You can ask the Claude Code agent in the worktree to create the PR for you.
## Configuration
The setup is defined in `.workmux.yaml` at the repo root. Key sections:
- **`post_create`**: Runs `scripts/worktree-env` to generate `.env.local` with port assignments
- **`panes`**: Defines the tmux layout (agent, backend, frontend)
- **`files.copy`**: Copies `backend/.env` and `scripts/` into each worktree
- **`files.symlink`**: Symlinks `node_modules` and `.svelte-kit` to avoid reinstalling per worktree
## Login
Default credentials: `admin@windmill.dev` / `changeme`
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag)\n SELECT unnest($1::uuid[]), 'test-workspace', now(), 'flow'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": []
},
"hash": "0681b850c033619e1b9498376263681f875a5aba22170ca50ec8b578f7fa478b"
}
@@ -43,7 +43,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH completed AS (\n INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result,\n flow_status, workflow_as_code_status, status, worker)\n SELECT\n q.workspace_id, q.id, q.started_at,\n (EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000,\n CASE WHEN q.running\n THEN $3::text::jsonb\n ELSE $4::text::jsonb\n END,\n s.flow_status,\n s.workflow_as_code_status,\n 'skipped'::job_status,\n q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_status s ON s.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = EXCLUDED.result\n RETURNING 1 AS x\n ), _deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n ), _logged AS (\n INSERT INTO job_logs (logs, job_id, workspace_id)\n VALUES ($5, $1, $2)\n ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs)\n )\n SELECT x FROM completed\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "x",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Uuid",
"Varchar",
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT result::text FROM v2_job_completed WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "result",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "18b6262a60400f2b58ab26615466c23b4c1a7805c66b70b0fcfb7d33b122a7bf"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag)\n VALUES ($1, $2, now(), 'flow')",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Varchar"
]
},
"nullable": []
},
"hash": "1af6885dbc5055281acb82b3e57f7dba2e4b04d9535058fab695660a14bf8890"
}
@@ -1,65 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT client, refresh_token, grant_type, cc_client_id, cc_client_secret, cc_token_url, mcp_server_url, is_workspace_integration FROM account WHERE workspace_id = $1 AND id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "client",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "refresh_token",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "grant_type",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "cc_client_id",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "cc_client_secret",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "cc_token_url",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "mcp_server_url",
"type_info": "Text"
},
{
"ordinal": 7,
"name": "is_workspace_integration",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Int4"
]
},
"nullable": [
false,
false,
false,
true,
true,
true,
true,
false
]
},
"hash": "1ba2e23d4ba816048ec1e88af9e342867fc0443cabea16d111afa2b91d3fe03b"
}
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job (id, kind, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, runnable_path)\n VALUES ($1, 'flow', 'flow', 'test-user', 'u/test-user', 'test@windmill.dev', $2, $3)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Varchar",
"Varchar"
]
},
"nullable": []
},
"hash": "27f70ebe788cca2e88732d8bf978883037bebca4cf75ba459858e4fb197f940b"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "debounce_batch",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
false
]
},
"hash": "2a95f18e80c55a7e8178a4bd2b781d41fa47efd4da5bb9bc2d72b9aa1e33617f"
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job (id, kind, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, runnable_path)\n VALUES ($1, 'flow', 'flow', 'test-user', 'u/test-user', 'test@windmill.dev', 'ws2', 'f/test/flow')",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "4010328a9f1611064f497726b69c08625a55a4dab25c3d9b5ece07e44d14915b"
}
@@ -1,35 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO debounce_key (job_id, key)\n VALUES ($1, $2)\n ON CONFLICT (key)\n DO UPDATE SET\n previous_job_id = debounce_key.job_id,\n job_id = EXCLUDED.job_id, -- replace current job with new one \n debounced_times = debounce_key.debounced_times + 1 -- evaluated only if conflict,\n -- conflict means there is already existing value,\n -- which means overriding it will also imply adding new entry to v2_job_debounce_batch and thus debouncing the job\n -- so the counter should be incremented\n RETURNING\n debounced_times,\n first_started_at,\n previous_job_id AS job_id_to_debounce\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "debounced_times",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "first_started_at",
"type_info": "Timestamptz"
},
{
"ordinal": 2,
"name": "job_id_to_debounce",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Uuid",
"Varchar"
]
},
"nullable": [
false,
false,
true
]
},
"hash": "454ace9ce391725ef4f4c129cd66e4c12a5c40f512b70551958178c8b4d6c183"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "WITH ids AS (\n SELECT id as job_id FROM v2_job_debounce_batch WHERE debounce_batch = (\n SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1\n )\n ) SELECT args->>'items' FROM ids LEFT JOIN v2_job ON v2_job.id = ids.job_id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "48536968f4173715d4ef8293683c2a3eb4bd22fbe18c34890a3dc4e96e4e6133"
}
@@ -42,7 +42,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -38,7 +38,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT v2_job_queue.id, v2_job.tag, v2_job_queue.scheduled_for, v2_job_queue.workspace_id FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id WHERE running = false AND scheduled_for < now() - ($1 || ' minutes')::interval",
"query": "SELECT v2_job_queue.id, v2_job.tag, v2_job_queue.scheduled_for, v2_job_queue.workspace_id FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id WHERE running = false AND scheduled_for < now() - ($1 || ' minutes')::interval AND v2_job.trigger_kind IS DISTINCT FROM 'schedule'::job_trigger_kind",
"describe": {
"columns": [
{
@@ -36,5 +36,5 @@
false
]
},
"hash": "b45e17ad532a23b394226c9a5d7ab5a21e20202dbbf9c67831cc62eb067cd2ba"
"hash": "53648c069749df45c0459d733b3e429af20c69c841fb0c3bceafe3ea6c3f5329"
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag)\n SELECT unnest($1::uuid[]), 'test-workspace', now(), 'deno'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": []
},
"hash": "539d661500254e2e346490710f5772cb88a1ab6bbddd97a77e06644ac0f61762"
}
@@ -1,18 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH job_result AS (\n SELECT result\n FROM v2_job_completed\n WHERE id = $1\n ),\n updated_queue AS (\n UPDATE v2_job_queue\n SET running = false,\n tag = COALESCE($3, tag)\n WHERE id = $2\n )\n UPDATE v2_job\n SET\n tag = COALESCE($3, tag),\n concurrent_limit = COALESCE($4, concurrent_limit),\n concurrency_time_window_s = COALESCE($5, concurrency_time_window_s),\n args = COALESCE(\n CASE\n WHEN job_result.result IS NULL THEN NULL\n WHEN jsonb_typeof(job_result.result) = 'object'\n THEN job_result.result\n WHEN jsonb_typeof(job_result.result) = 'null'\n THEN NULL\n ELSE jsonb_build_object('value', job_result.result)\n END,\n '{}'::jsonb\n ),\n preprocessed = TRUE\n FROM job_result\n WHERE v2_job.id = $2;\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid",
"Varchar",
"Int4",
"Int4"
]
},
"nullable": []
},
"hash": "5b8c1803f0ccead11517fbc8a9bdc0227dc3922217fa18f0b71ff0484d65838c"
}
@@ -77,7 +77,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE debounce_key SET first_started_at = now() - interval '20 seconds' WHERE key = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "66342c32f7ae0238803cb1896d9f23a74b64573f77dd32189a25b6e8369f147b"
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job (id, kind, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, runnable_path)\n SELECT unnest($1::uuid[]), 'flow', 'flow', 'test-user', 'u/test-user', 'test@windmill.dev', 'test-workspace', 'f/test/flow'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": []
},
"hash": "66faba2137791e0cb1353545c06f9f7c23a1559e7a761db7c2195736b8b30709"
}
@@ -0,0 +1,19 @@
{
"db_name": "PostgreSQL",
"query": "WITH job_result AS (\n SELECT result\n FROM v2_job_completed\n WHERE id = $1\n ),\n updated_queue AS (\n UPDATE v2_job_queue\n SET running = false,\n tag = COALESCE($3, tag),\n scheduled_for = COALESCE($6, scheduled_for)\n WHERE id = $2\n )\n UPDATE v2_job\n SET\n tag = COALESCE($3, tag),\n concurrent_limit = COALESCE($4, concurrent_limit),\n concurrency_time_window_s = COALESCE($5, concurrency_time_window_s),\n args = COALESCE(\n CASE\n WHEN job_result.result IS NULL THEN NULL\n WHEN jsonb_typeof(job_result.result) = 'object'\n THEN job_result.result\n WHEN jsonb_typeof(job_result.result) = 'null'\n THEN NULL\n ELSE jsonb_build_object('value', job_result.result)\n END,\n '{}'::jsonb\n ),\n preprocessed = TRUE\n FROM job_result\n WHERE v2_job.id = $2;\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid",
"Varchar",
"Int4",
"Int4",
"Timestamptz"
]
},
"nullable": []
},
"hash": "79b437ad31ddab94310989b8fb6a1c130b9be1ab4b6a100fffffd687677b9c92"
}
@@ -44,7 +44,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT logs as \"logs!\" FROM job_logs WHERE job_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "logs!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
true
]
},
"hash": "7ca599330c9913c7e66b27e2ffcfa18d53cbdd16e179749f0aea7980a901b23c"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag)\n VALUES ($1, $2, now(), 'deno')",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Varchar"
]
},
"nullable": []
},
"hash": "7ca7dabfe360845a5b57552b0d02267d5dbbc488bc7ab990c0bda1594bf5ef3a"
}
@@ -42,7 +42,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_debounce_batch (id, debounce_batch)\n -- if it the first one, nextval will be evaluated, otherwise take from the job we will debounce\n SELECT\n $2,\n COALESCE(\n (\n SELECT debounce_batch\n FROM v2_job_debounce_batch\n WHERE id = $1\n LIMIT 1\n ), -- maybe use current batch\n nextval('debounce_batch_seq')\n )\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid"
]
},
"nullable": []
},
"hash": "8360ab72d60f07dde6ecae599e6531b5b86862029ab51fdbdd44ec16239108e2"
}
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO workspace_settings (workspace_id) VALUES ('ws2')",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "8cf5af21cde4e4de45f995efa2a9b56ce20c26869ca78d7e17b3504b92ae85b1"
}
@@ -102,7 +102,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -0,0 +1,34 @@
{
"db_name": "PostgreSQL",
"query": "SELECT job_id, previous_job_id, debounced_times FROM debounce_key WHERE key = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "job_id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "previous_job_id",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "debounced_times",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
true,
false
]
},
"hash": "8f442110817244aa9533b014aa3d74a6582937dfe4759932b63b8e531984008e"
}
@@ -32,7 +32,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -0,0 +1,35 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH dk AS (\n INSERT INTO debounce_key (job_id, key)\n VALUES ($1, $2)\n ON CONFLICT (key)\n DO UPDATE SET\n previous_job_id = debounce_key.job_id,\n job_id = EXCLUDED.job_id,\n debounced_times = debounce_key.debounced_times + 1\n RETURNING\n debounced_times,\n first_started_at,\n previous_job_id AS job_id_to_debounce\n ), _batch AS (\n INSERT INTO v2_job_debounce_batch (id, debounce_batch)\n SELECT\n $1,\n COALESCE(\n (SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = dk.job_id_to_debounce LIMIT 1),\n nextval('debounce_batch_seq')\n )\n FROM dk\n )\n SELECT debounced_times, first_started_at, job_id_to_debounce FROM dk\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "debounced_times",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "first_started_at",
"type_info": "Timestamptz"
},
{
"ordinal": 2,
"name": "job_id_to_debounce",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Uuid",
"Varchar"
]
},
"nullable": [
false,
false,
true
]
},
"hash": "98033aae3182bde22d5b2ff08ef6e8a4f8f3a9bf04238b33e9caf46836df73d9"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job (id, kind, tag, created_by, permissioned_as, permissioned_as_email, workspace_id)\n VALUES ($1, 'noop', 'deno', 'test-user', 'u/test-user', 'test@windmill.dev', $2)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Varchar"
]
},
"nullable": []
},
"hash": "9f50ec7681a1fcd11cb452c7aba7e8897e49a4b6affa4e9976680336b6bd3115"
}
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO workspace (id, name, owner) VALUES ('ws2', 'Workspace 2', 'test-user')",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "a057ff9f5998a162ae6de05f6127b7eefc826af7bf1bb89fd758f7c03c881033"
}
@@ -72,7 +72,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -77,7 +77,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_runtime (id) VALUES ($1)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "a68754521bf751450602f04dd4243199a18885e1739a5a0e7f6100eab6f3c803"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue SET runnable_settings_handle = $1 WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Uuid"
]
},
"nullable": []
},
"hash": "abb56f78aa39c6b6ae8b0ccb7b724c1f80d717e7c9d76b287da63cb5ee8e8b25"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT 1 as x FROM v2_job_completed WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "x",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "adb98040c8039e5cc27fe0579941f723b38d1784bd2f1b16d8724f1f1612dcbf"
}
@@ -102,7 +102,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -72,7 +72,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_runtime (id) SELECT unnest($1::uuid[])",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": []
},
"hash": "b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT 1 as x FROM v2_job_queue WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "x",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "b795dc228f93c8b9bedb4a3e7467d941819a7202214e0634580bdc2ec30f0b70"
}
@@ -41,7 +41,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag) VALUES ($1, 'ws2', now(), 'flow')",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "c1a1ae759ebb84fde3e6d2727991b2a1fcc73e520fee79653bb960dc00c3e2db"
}
@@ -0,0 +1,35 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH dk AS (\n INSERT INTO debounce_key (job_id, key)\n VALUES ($1, $2)\n ON CONFLICT (key)\n DO UPDATE SET\n previous_job_id = debounce_key.job_id,\n job_id = EXCLUDED.job_id,\n debounced_times = debounce_key.debounced_times + 1\n RETURNING\n debounced_times,\n first_started_at,\n previous_job_id AS job_id_to_debounce\n ), _batch AS (\n INSERT INTO v2_job_debounce_batch (id, debounce_batch)\n SELECT\n $1,\n COALESCE(\n (SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = dk.job_id_to_debounce LIMIT 1),\n nextval('debounce_batch_seq')\n )\n FROM dk\n )\n SELECT debounced_times, first_started_at, job_id_to_debounce FROM dk\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "debounced_times",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "first_started_at",
"type_info": "Timestamptz"
},
{
"ordinal": 2,
"name": "job_id_to_debounce",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Uuid",
"Varchar"
]
},
"nullable": [
false,
false,
true
]
},
"hash": "c2347460b73ae9d3167031c263032e97ebefb46be9e58bd3da9067748075311b"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) as \"count!\" FROM v2_job_queue WHERE id = ANY($1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": [
null
]
},
"hash": "c63a1949247f1618f6b6acee9bf6b4d3081dfed2e6ff533dbff0bdfd52687cbb"
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job (id, kind, tag, created_by, permissioned_as, permissioned_as_email, workspace_id)\n SELECT unnest($1::uuid[]), 'noop', 'deno', 'test-user', 'u/test-user', 'test@windmill.dev', 'test-workspace'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": []
},
"hash": "c6d963e5cefeea728414892df9f28a89f435fc0fb7e55f243b021200f33d2151"
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH _ AS (\n UPDATE debounce_key\n SET debounced_times = 0,\n first_started_at = now(),\n previous_job_id = NULL\n WHERE job_id = $1\n )\n UPDATE v2_job_debounce_batch\n SET debounce_batch = nextval('debounce_batch_seq')\n WHERE id = $1\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "c9530931f670eab1208c4a284a55afdc3fcbb0eb5f98fd63e2ec89442becbfaa"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) as \"count!\" FROM v2_job_completed WHERE id = ANY($1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": [
null
]
},
"hash": "cc309de42a3b630bb83d1b2437633ef0b98ce5e5fba1f5c1dda3ddd874ff3a39"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = ANY($1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "debounce_batch",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": [
false
]
},
"hash": "ccfed494a8d89eb2c88d72738c341a4dd87701b0636eb9fa001cd1d9cbcd663b"
}
@@ -41,7 +41,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = ANY($1) ORDER BY debounce_batch",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "debounce_batch",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": [
false
]
},
"hash": "d9400849888dd021b0504b93004ab5e76296ed437c210942363ba06845f9f963"
}
@@ -31,7 +31,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -37,7 +37,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -77,7 +77,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job (id, kind, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, runnable_path, args)\n VALUES ($1, 'flow', 'flow', 'test-user', 'u/test-user', 'test@windmill.dev', $2, $3, $4)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Varchar",
"Varchar",
"Jsonb"
]
},
"nullable": []
},
"hash": "f8cec94b94098e752f7c71cbe5e9996410a07bacebc37ed21e0bedf1a33a8fdc"
}
@@ -32,7 +32,8 @@
"aiagent",
"unassigned_script",
"unassigned_flow",
"unassigned_singlestepflow"
"unassigned_singlestepflow",
"snapshotbuild"
]
}
}
+8
View File
@@ -0,0 +1,8 @@
panes:
# Pane 1: Install dependencies, then start dev server
- command: cargo run
# Pane 2: AI agent
- command: <agent>
split: horizontal
focus: true
+12 -1
View File
@@ -44,11 +44,22 @@ Windmill uses a workspace-based architecture with multiple crates:
## Enterprise Features
- Enterprise files use the `*_ee.rs` suffix
- Enterprise source is in `windmill-ee-private` folder (sibling directory at `../../windmill-ee-private`), symlinked into each crate's `src/`
- Enterprise source is in `windmill-ee-private` folder (sibling directory at `../../windmill-ee-private` or `~/windmill-ee-private`), symlinked into each crate's `src/`
- The `_ee.rs` files are gitignored in the main repo — they are tracked only in the `windmill-ee-private` repo
- You can and should modify `windmill-ee-private` directly when needed (e.g., when creating new crates that need EE code, mirror the package structure there)
- Use feature flags: `#[cfg(feature = "enterprise")]`
- Isolate enterprise code in separate modules
### EE PR Workflow (MUST DO when modifying `*_ee.rs` files)
When you modify any `*_ee.rs` file and create a PR on the windmill repo, you **MUST** also:
1. **Create a matching branch** in the `windmill-ee-private` repo (use the same branch name). If using worktrees, the EE worktree is at `~/windmill-ee-private__worktrees/<branch-name>/`
2. **Commit and push** the `_ee.rs` changes in that branch
3. **Create a PR** on `windmill-ee-private` with a link to the companion windmill PR
4. **Update `ee-repo-ref.txt`**: Run `bash write_latest_ee_ref.sh` from `backend/` to write the latest EE commit hash. **Important**: the script may fall back to `~/windmill-ee-private` (main branch) instead of the worktree — verify it wrote the correct commit hash from your branch, not from main. If wrong, manually write the correct hash.
5. **Commit `ee-repo-ref.txt`** in the windmill repo so CI picks up the correct EE ref
## Code Validation (MUST DO)
After making backend changes, you MUST run `cargo check` and fix all errors and warnings before considering the work done.
+202 -200
View File
File diff suppressed because it is too large Load Diff
+5 -4
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.638.4"
version = "1.642.0"
authors.workspace = true
edition.workspace = true
@@ -76,7 +76,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal"]
[workspace.package]
version = "1.638.4"
version = "1.642.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -103,7 +103,7 @@ enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmi
local_reports = ["windmill-common/local_reports"]
enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"]
stripe = ["windmill-api/stripe"]
benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"]
benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark", "windmill-api-agent-workers?/benchmark"]
embedding = ["windmill-api/embedding"]
parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker/parquet"]
prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus", "dep:prometheus"]
@@ -254,6 +254,7 @@ axum.workspace = true
serde.workspace = true
windmill-api-client.workspace = true
tempfile.workspace = true
windmill-parser-ts.workspace = true
rumqttc.workspace = true
rdkafka.workspace = true
async-nats.workspace = true
@@ -479,7 +480,7 @@ bit-vec = "=0.6.3"
mappable-rc = "^0"
mysql_async = { version = "*", default-features = false, features = ["minimal", "default", "native-tls-tls", "rust_decimal"]}
postgres-native-tls = "^0"
native-tls = "^0"
native-tls = ">=0.2, <0.2.17"
# samael will break compilation on MacOS. Use this fork instead to make it work
# samael = { git="https://github.com/njaremko/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = ["xmlsec"] }
libxml = { version = "=0.3.3" }
+1 -1
View File
@@ -1 +1 @@
0eccae6a9a9ecde09816cd4d88ca9ab305659e4c
0fede4b1086bc1456be9cc55b203228c979c5c5e
@@ -2,8 +2,8 @@ use std::collections::BTreeMap;
use sqlparser::{
ast::{
CopyTarget, Expr, ObjectName, ObjectNamePart, SelectItem, TableFactor, TableObject, Value,
ValueWithSpan, Visit, Visitor,
CopyTarget, Expr, FunctionArg, FunctionArgExpr, ObjectName, ObjectNamePart, SelectItem,
TableFactor, TableObject, Value, ValueWithSpan, Visit, Visitor,
},
dialect::DuckDbDialect,
parser::Parser,
@@ -125,6 +125,72 @@ impl AssetCollector {
Some(ParseAssetsResult { kind: *kind, access_type, path, columns: None })
}
/// If `table_factor` is a string literal used directly as a table name (e.g. FROM 's3:///file.parquet'),
/// return a `ParseAssetsResult` for it.
fn get_s3_asset_from_str_literal_table(
&self,
table_factor: &TableFactor,
) -> Option<ParseAssetsResult> {
let name = match table_factor {
TableFactor::Table { name, args: None, .. } => name,
_ => return None,
};
let s3_str = get_str_lit_from_obj_name(name)?;
let (kind, path) = parse_asset_syntax(s3_str, false)?;
if kind != AssetKind::S3Object {
return None;
}
Some(ParseAssetsResult {
kind,
path: path.to_string(),
access_type: Some(R),
columns: None,
})
}
/// If `table_factor` is a read function (read_parquet/read_csv/read_json) whose first
/// positional argument is an S3 string literal, return a `ParseAssetsResult` for it.
fn get_s3_asset_from_table_function(
&self,
table_factor: &TableFactor,
) -> Option<ParseAssetsResult> {
let (name, args) = match table_factor {
TableFactor::Table { name, args: Some(args), .. } => (name, args),
_ => return None,
};
let fname = get_trivial_obj_name(name)?;
if !is_read_fn(fname) {
return None;
}
let s3_str = args.args.first().and_then(|arg| match arg {
FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(ValueWithSpan {
value: Value::SingleQuotedString(s),
..
})))
| FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(ValueWithSpan {
value: Value::DoubleQuotedString(s),
..
}))) => Some(s.as_str()),
_ => None,
})?;
let (kind, path) = parse_asset_syntax(s3_str, false)?;
if kind != AssetKind::S3Object {
return None;
}
Some(ParseAssetsResult {
kind,
path: path.to_string(),
access_type: Some(R),
columns: None,
})
}
fn handle_string_literal(&mut self, s: &str) {
// Check if the string matches our asset syntax patterns
if let Some((kind, path)) = parse_asset_syntax(s, false) {
@@ -190,13 +256,19 @@ impl AssetCollector {
projection: &[SelectItem],
from_tables: &[sqlparser::ast::TableWithJoins],
) {
// Check if this is a single-table SELECT (to avoid ambiguity)
// Check if this is a single-table SELECT (to avoid ambiguity).
// For S3 table functions (read_parquet/read_csv/read_json), detect the asset even
// though args are present, since we know the file path from the string literal arg.
let single_table = if from_tables.len() == 1 {
if let TableFactor::Table { name, args, .. } = &from_tables[0].relation {
if args.is_some() && args.as_ref().map_or(0, |a| a.args.len()) > 0 {
return; // Skip table functions
let relation = &from_tables[0].relation;
if let TableFactor::Table { name, args, .. } = relation {
let has_args = args.as_ref().map_or(false, |a| !a.args.is_empty());
if has_args {
self.get_s3_asset_from_table_function(relation)
} else {
self.get_associated_asset_from_obj_name(name, Some(R))
.or_else(|| self.get_s3_asset_from_str_literal_table(relation))
}
self.get_associated_asset_from_obj_name(name, Some(R))
} else {
None
}
@@ -204,26 +276,48 @@ impl AssetCollector {
None
};
// Build a map of table aliases/names to assets for multi-table queries
// Build a map of table aliases/names to assets for multi-table queries.
// For S3 table functions, only aliased references are unambiguous
// (e.g. SELECT t.col1 FROM read_parquet('s3://...') AS t).
let mut table_to_asset: BTreeMap<String, ParseAssetsResult> = BTreeMap::new();
for table_with_joins in from_tables {
if let TableFactor::Table { name, alias, args, .. } = &table_with_joins.relation {
if args.is_some() && args.as_ref().map_or(0, |a| a.args.len()) > 0 {
continue; // Skip table functions
}
if let Some(asset) = self.get_associated_asset_from_obj_name(name, Some(R)) {
// Use alias if present, otherwise use the table name
let table_key = if let Some(alias) = alias {
alias.name.value.clone()
let has_args = args.as_ref().map_or(false, |a| !a.args.is_empty());
if has_args {
// For table functions, only add to the alias map when an alias is present
if let Some(alias) = alias {
if let Some(asset) =
self.get_s3_asset_from_table_function(&table_with_joins.relation)
{
table_to_asset.insert(alias.name.value.clone(), asset);
}
}
} else if let Some(asset) = self
.get_associated_asset_from_obj_name(name, Some(R))
.or_else(|| {
self.get_s3_asset_from_str_literal_table(&table_with_joins.relation)
})
{
// For string literal S3 tables (e.g. FROM 's3:///file.parquet'), only add to
// the alias map when an alias is present (to avoid false positives).
// For regular named tables, use alias or table name as key.
let is_str_literal = get_str_lit_from_obj_name(name).is_some();
if is_str_literal {
if let Some(alias) = alias {
table_to_asset.insert(alias.name.value.clone(), asset);
}
} else {
// For qualified names like "dl.table1", use just the last part
name.0
.last()
.and_then(|id| id.as_ident())
.map(|id| id.value.clone())
.unwrap_or_default()
};
table_to_asset.insert(table_key, asset);
let table_key = if let Some(alias) = alias {
alias.name.value.clone()
} else {
name.0
.last()
.and_then(|id| id.as_ident())
.map(|id| id.value.clone())
.unwrap_or_default()
};
table_to_asset.insert(table_key, asset);
}
}
}
}
@@ -1271,4 +1365,163 @@ mod tests {
assert_eq!(columns.get("age"), Some(&W)); // Only written
assert_eq!(columns.get("id"), Some(&R)); // Only read
}
#[test]
fn test_sql_asset_parser_s3_single_table_column_detection() {
let input = r#"
SELECT col1, col2 FROM read_parquet('s3:///example_file.parquet');
"#;
let result = parse_assets(input).unwrap().assets;
assert_eq!(result.len(), 1);
assert_eq!(result[0].kind, AssetKind::S3Object);
assert_eq!(result[0].path, "/example_file.parquet");
assert_eq!(result[0].access_type, Some(R));
let columns = result[0].columns.as_ref().expect("Should have columns");
assert_eq!(columns.len(), 2);
assert_eq!(columns.get("col1"), Some(&R));
assert_eq!(columns.get("col2"), Some(&R));
}
#[test]
fn test_sql_asset_parser_s3_single_table_column_with_alias() {
let input = r#"
SELECT col1 AS c1, col2 AS c2 FROM read_parquet('s3:///example_file.parquet');
"#;
let result = parse_assets(input).unwrap().assets;
assert_eq!(result.len(), 1);
let columns = result[0].columns.as_ref().expect("Should have columns");
assert_eq!(columns.get("col1"), Some(&R));
assert_eq!(columns.get("col2"), Some(&R));
}
#[test]
fn test_sql_asset_parser_s3_wildcard_no_columns() {
let input = r#"
SELECT * FROM read_parquet('s3:///example_file.parquet');
"#;
let result = parse_assets(input).unwrap().assets;
assert_eq!(result.len(), 1);
assert_eq!(result[0].kind, AssetKind::S3Object);
assert_eq!(result[0].path, "/example_file.parquet");
assert!(result[0].columns.is_none());
}
#[test]
fn test_sql_asset_parser_s3_table_alias_qualified_columns() {
let input = r#"
SELECT t.col1, t.col2 FROM read_parquet('s3:///example_file.parquet') AS t;
"#;
let result = parse_assets(input).unwrap().assets;
assert_eq!(result.len(), 1);
assert_eq!(result[0].kind, AssetKind::S3Object);
assert_eq!(result[0].path, "/example_file.parquet");
let columns = result[0].columns.as_ref().expect("Should have columns");
assert_eq!(columns.get("col1"), Some(&R));
assert_eq!(columns.get("col2"), Some(&R));
}
#[test]
fn test_sql_asset_parser_s3_multi_table_aliased_columns() {
let input = r#"
SELECT t1.col1, t2.col2
FROM read_parquet('s3:///file1.parquet') AS t1,
read_csv('s3://bucket/file2.csv') AS t2;
"#;
let result = parse_assets(input).unwrap().assets;
assert_eq!(result.len(), 2);
assert!(result.iter().any(|a| {
a.path == "/file1.parquet"
&& a.columns.as_ref().map_or(false, |c| c.contains_key("col1"))
}));
assert!(result.iter().any(|a| {
a.path == "bucket/file2.csv"
&& a.columns.as_ref().map_or(false, |c| c.contains_key("col2"))
}));
}
#[test]
fn test_sql_asset_parser_s3_multi_table_no_alias_no_columns() {
// Without aliases, unqualified columns in a multi-table query are ambiguous
let input = r#"
SELECT col1, col2
FROM read_parquet('s3:///file1.parquet'),
read_parquet('s3:///file2.parquet');
"#;
let result = parse_assets(input).unwrap().assets;
// Table-level assets should still be detected, but no columns
assert_eq!(result.iter().filter(|a| a.columns.is_some()).count(), 0);
}
#[test]
fn test_sql_asset_parser_s3_str_literal_table_column_detection() {
// FROM 's3:///file.parquet' (string literal as table, no read_parquet wrapper)
let input = r#"
SELECT a,b,c FROM 's3:///test.parquet';
"#;
let result = parse_assets(input).unwrap().assets;
assert_eq!(result.len(), 1);
assert_eq!(result[0].kind, AssetKind::S3Object);
assert_eq!(result[0].path, "/test.parquet");
assert_eq!(result[0].access_type, Some(R));
let columns = result[0].columns.as_ref().expect("Should have columns");
assert_eq!(columns.len(), 3);
assert_eq!(columns.get("a"), Some(&R));
assert_eq!(columns.get("b"), Some(&R));
assert_eq!(columns.get("c"), Some(&R));
}
#[test]
fn test_sql_asset_parser_s3_str_literal_table_with_alias_columns() {
let input = r#"
SELECT t.col1, t.col2 FROM 's3://bucket/file.parquet' AS t;
"#;
let result = parse_assets(input).unwrap().assets;
assert_eq!(result.len(), 1);
assert_eq!(result[0].kind, AssetKind::S3Object);
assert_eq!(result[0].path, "bucket/file.parquet");
let columns = result[0].columns.as_ref().expect("Should have columns");
assert_eq!(columns.get("col1"), Some(&R));
assert_eq!(columns.get("col2"), Some(&R));
}
#[test]
fn test_sql_asset_parser_s3_str_literal_wildcard_no_columns() {
let input = r#"
SELECT * FROM 's3:///test.parquet';
"#;
let result = parse_assets(input).unwrap().assets;
assert_eq!(result.len(), 1);
assert_eq!(result[0].kind, AssetKind::S3Object);
assert!(result[0].columns.is_none());
}
#[test]
fn test_sql_asset_parser_s3_read_csv_columns() {
let input = r#"
SELECT name, age FROM read_csv('s3://my-bucket/data.csv');
"#;
let result = parse_assets(input).unwrap().assets;
assert_eq!(result.len(), 1);
assert_eq!(result[0].kind, AssetKind::S3Object);
assert_eq!(result[0].path, "my-bucket/data.csv");
let columns = result[0].columns.as_ref().expect("Should have columns");
assert_eq!(columns.get("name"), Some(&R));
assert_eq!(columns.get("age"), Some(&R));
}
}
@@ -59,3 +59,6 @@ wasm-bindgen.workspace = true
serde_json.workspace = true
getrandom = { workspace = true, features = ["js"] }
# getrandom 0.3 is pulled in transitively by rand 0.9 (via windmill-types).
# It requires the "wasm_js" feature to work on wasm32-unknown-unknown.
getrandom3 = { package = "getrandom", version = "0.3", features = ["wasm_js"] }
+16 -8
View File
@@ -39,8 +39,6 @@ use windmill_common::ee_oss::{jobs_waiting_alerts, worker_groups_alerts};
#[cfg(feature = "oauth2")]
use windmill_common::global_settings::OAUTH_SETTING;
#[cfg(feature = "parquet")]
use windmill_object_store::reload_object_store_setting;
use windmill_common::{
agent_workers::DECODED_AGENT_TOKEN,
apps::APP_WORKSPACED_ROUTE,
@@ -56,7 +54,7 @@ use windmill_common::{
HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING,
KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING,
NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING,
NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING,
OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
@@ -84,14 +82,16 @@ use windmill_common::{
OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS,
};
use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING};
#[cfg(feature = "parquet")]
use windmill_object_store::reload_object_store_setting;
use windmill_queue::{cancel_job, get_queued_job_v2, SameWorkerPayload};
use windmill_worker::{
result_processor::handle_job_error, JobCompletedSender, JobIsolationLevel,
OtelTracingProxySettings, SameWorkerSender, BUNFIG_INSTALL_SCOPES, CARGO_REGISTRIES,
INSTANCE_PYTHON_VERSION, JAVA_HOME_DIR, JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR,
MAVEN_REPOS, MAVEN_SETTINGS_XML, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE,
NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL,
POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UV_INDEX_STRATEGY,
MAVEN_REPOS, MAVEN_SETTINGS_XML, NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY,
NSJAIL_AVAILABLE, NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL,
PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UV_INDEX_STRATEGY,
};
#[cfg(feature = "parquet")]
@@ -330,6 +330,7 @@ pub async fn initial_load(
reload_uv_index_strategy_setting(&conn).await;
reload_npm_config_registry_setting(&conn).await;
reload_bunfig_install_scopes_setting(&conn).await;
reload_npmrc_setting(&conn).await;
reload_instance_python_version_setting(&conn).await;
reload_nuget_config_setting(&conn).await;
reload_powershell_repo_url_setting(&conn).await;
@@ -1204,7 +1205,10 @@ async fn delete_log_files_from_disk_and_store(
#[cfg(feature = "parquet")]
if _should_del_from_store {
if let Some(os) = _os2 {
let p = windmill_object_store::object_store_reexports::Path::from(format!("{}{}", _s3_prefix, path));
let p = windmill_object_store::object_store_reexports::Path::from(format!(
"{}{}",
_s3_prefix, path
));
if let Err(e) = os.delete(&p).await {
tracing::error!("Failed to delete from object store {}: {e}", p.to_string())
} else {
@@ -1303,6 +1307,10 @@ pub async fn reload_bunfig_install_scopes_setting(conn: &Connection) {
.await;
}
pub async fn reload_npmrc_setting(conn: &Connection) {
reload_option_setting_with_tracing(conn, NPMRC_SETTING, "NPMRC", NPMRC.clone()).await;
}
pub async fn reload_nuget_config_setting(conn: &Connection) {
reload_option_setting_with_tracing(
conn,
@@ -2341,7 +2349,7 @@ pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> {
async fn stale_job_cancellation(db: &Pool<Postgres>) {
if let Some(threshold) = *STALE_JOB_THRESHOLD_MINUTES {
let stale_jobs = sqlx::query!(
"SELECT v2_job_queue.id, v2_job.tag, v2_job_queue.scheduled_for, v2_job_queue.workspace_id FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id WHERE running = false AND scheduled_for < now() - ($1 || ' minutes')::interval",
"SELECT v2_job_queue.id, v2_job.tag, v2_job_queue.scheduled_for, v2_job_queue.workspace_id FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id WHERE running = false AND scheduled_for < now() - ($1 || ' minutes')::interval AND v2_job.trigger_kind IS DISTINCT FROM 'schedule'::job_trigger_kind",
threshold.to_string()
)
.fetch_all(db)
+15 -1
View File
@@ -7,6 +7,7 @@ REVERT="NO"
COPY="NO"
MOVE_NEW_FILES="NO"
EE_CODE_DIR="../windmill-ee-private/"
DIR_EXPLICIT="NO"
while [[ $# -gt 0 ]]; do
case $1 in
@@ -34,6 +35,7 @@ while [[ $# -gt 0 ]]; do
# Path to the local directory of the windmill-ee-private repository. By defaults, it
# assumes it is cloned next to the Windmill OSS repo.
EE_CODE_DIR="$2"
DIR_EXPLICIT="YES"
shift # past argument
shift # past value
;;
@@ -55,10 +57,22 @@ else
fi
# Fallback to ~/windmill-ee-private if the default location doesn't exist
if [ ! -d "${EE_CODE_DIR}" ] && [ "${EE_CODE_DIR}" == "${root_dirpath}/../windmill-ee-private/" ]; then
if [ ! -d "${EE_CODE_DIR}" ]; then
EE_CODE_DIR="${HOME}/windmill-ee-private"
fi
# Unless --dir was explicitly set, try to find an EE worktree on the same branch
if [ "$DIR_EXPLICIT" == "NO" ] && [ -d "${HOME}/windmill-ee-private" ]; then
current_branch=$(git -C "${root_dirpath}" branch --show-current 2>/dev/null || true)
if [ -n "$current_branch" ]; then
ee_worktree=$(git -C "${HOME}/windmill-ee-private" worktree list 2>/dev/null \
| awk -v branch="[${current_branch}]" '$NF == branch {print $1; exit}')
if [ -n "$ee_worktree" ] && [ -d "$ee_worktree" ]; then
EE_CODE_DIR="$ee_worktree"
fi
fi
fi
echo "EE code directory = ${EE_CODE_DIR} | Revert = ${REVERT}"
if [ ! -d "${EE_CODE_DIR}" ]; then
+474
View File
@@ -0,0 +1,474 @@
#!/usr/bin/env bash
# End-to-end debounce tests against the running backend API
# Usage: BACKEND_PORT=8030 ./test_debounce_e2e.sh
set -uo pipefail
BASE="http://localhost:${BACKEND_PORT:-8030}/api"
W="admins"
EMAIL="admin@windmill.dev"
PASSWORD="changeme"
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
pass=0
fail=0
log_pass() { echo -e "${GREEN}PASS${NC}: $1"; ((pass++)) || true; }
log_fail() { echo -e "${RED}FAIL${NC}: $1$2"; ((fail++)) || true; }
log_info() { echo -e "${YELLOW}INFO${NC}: $1"; }
# Unique suffix for idempotent re-runs
TS=$(date +%s)
# --- Auth ---
log_info "Logging in..."
TOKEN=$(curl -s "$BASE/auth/login" \
-H 'Content-Type: application/json' \
-d "{\"email\":\"$EMAIL\",\"password\":\"$PASSWORD\"}")
if [ -z "$TOKEN" ]; then
echo "Failed to login"; exit 1
fi
AUTH="Authorization: Bearer $TOKEN"
log_info "Logged in"
# --- Helpers ---
api() {
# Usage: api METHOD path [data]
local method="$1" path="$2" data="${3:-}"
if [ -n "$data" ]; then
curl -s "$BASE/w/$W/$path" -X "$method" -H "$AUTH" -H 'Content-Type: application/json' -d "$data"
else
curl -s "$BASE/w/$W/$path" -X "$method" -H "$AUTH"
fi
}
wait_job() {
local job_id="$1" max_wait="${2:-30}"
for _ in $(seq 1 "$max_wait"); do
local r
r=$(api GET "jobs/completed/get_result_maybe/$job_id")
if echo "$r" | jq -e '.completed == true' > /dev/null 2>&1; then
echo "$r"; return 0
fi
sleep 1
done
echo '{"completed":false,"error":"timeout"}'; return 1
}
BUN_EMPTY_LOCK=$'{"dependencies": {}}\n//bun.lock\n'
create_script() {
# Usage: create_script path language content [extra_json_fields]
# Note: lock must be non-empty; empty string ("") is treated as None by the backend
# (scripts.rs:798-800), which triggers dependency resolution instead of direct deployment.
# For bun scripts, the lock must contain "//bun.lock" as a split pattern.
local path="$1" lang="$2" content="$3" extra="${4:-}"
local json
json=$(jq -n \
--arg path "$path" \
--arg lang "$lang" \
--arg content "$content" \
--arg summary "test" \
--arg desc "test" \
--arg lock "$BUN_EMPTY_LOCK" \
'{path: $path, language: $lang, content: $content, summary: $summary, description: $desc, lock: $lock}')
if [ -n "$extra" ]; then
json=$(echo "$json" | jq ". + $extra")
fi
local hash
hash=$(api POST "scripts/create" "$json")
# Small delay for DB visibility after tx commit
sleep 0.2
echo "$hash"
}
run_script() {
# Usage: run_script path args_json
api POST "jobs/run/p/$1" "$2"
}
###############################################################################
# TEST 1: Deploy a script and run it 5 times in close succession
###############################################################################
echo ""
log_info "=== TEST 1: Deploy & run script 5 times rapidly ==="
P1="u/admin/e2e_simple_$TS"
H1=$(create_script "$P1" "bun" 'export function main(x: number = 0) { return { result: x * 2 }; }')
if echo "$H1" | grep -qE '^[0-9a-f]{16}$'; then
log_pass "Script created: $H1"
else
log_fail "Script creation" "$H1"
fi
log_info "Running 5 times rapidly..."
JOB_IDS=()
for i in $(seq 1 5); do
JID=$(run_script "$P1" "{\"x\": $i}")
JOB_IDS+=("$JID")
done
log_info "Jobs: ${JOB_IDS[*]}"
log_info "Waiting for completion..."
all_ok=true
for i in "${!JOB_IDS[@]}"; do
JID="${JOB_IDS[$i]}"
R=$(wait_job "$JID" 30)
success=$(echo "$R" | jq -r '.success // false')
value=$(echo "$R" | jq -r '.result.result // "null"')
expected=$(( (i + 1) * 2 ))
if [ "$success" = "true" ] && [ "$value" = "$expected" ]; then
log_pass "Job $((i+1)): x=$((i+1))$value (correct)"
else
log_fail "Job $((i+1))" "success=$success value=$value expected=$expected"
all_ok=false
fi
done
if [ "$all_ok" = "true" ]; then
log_pass "All 5 runs completed correctly (no debounce — different args)"
fi
###############################################################################
# TEST 2: Redeploy script WITHOUT lock in close succession
###############################################################################
echo ""
log_info "=== TEST 2: Redeploy without lock in rapid succession ==="
P2="u/admin/e2e_nolock_$TS"
# Deploy 5 versions of the same script without lock → triggers dependency jobs
DEPLOY_HASHES=()
for i in $(seq 1 5); do
content="export function main(x: number = 0) { return { result: x * $i, version: $i }; }"
parent_extra=""
if [ "${#DEPLOY_HASHES[@]}" -gt 0 ]; then
last_hash="${DEPLOY_HASHES[-1]}"
parent_extra="{\"parent_hash\": \"$last_hash\"}"
fi
# Deploy without lock (omit lock field entirely)
json=$(jq -n \
--arg path "$P2" \
--arg content "$content" \
--arg summary "v$i" \
--arg desc "test" \
'{path: $path, language: "bun", content: $content, summary: $summary, description: $desc}')
if [ -n "$parent_extra" ]; then
json=$(echo "$json" | jq ". + $parent_extra")
fi
hash=$(api POST "scripts/create" "$json")
if echo "$hash" | grep -qE '^[0-9a-f]{16}$'; then
DEPLOY_HASHES+=("$hash")
log_info "Deploy $i: $hash"
else
log_fail "Deploy $i" "$hash"
# If path conflict, the script already exists from a previous version
break
fi
sleep 0.1
done
# Wait for dependency resolution
log_info "Waiting 15s for dependency jobs..."
sleep 15
# Check the latest script — should have lock resolved
SCRIPT_INFO=$(api GET "scripts/get/p/$P2")
LOCK=$(echo "$SCRIPT_INFO" | jq -r '.lock // "null"')
if [ "$LOCK" != "null" ] && [ -n "$LOCK" ]; then
log_pass "Latest version has lock resolved"
else
log_info "Lock not yet resolved: $LOCK"
fi
# Run the latest version to verify it works
sleep 0.5
JID2=$(run_script "$P2" '{"x": 10}')
if echo "$JID2" | grep -qE '^[0-9a-f-]{36}$'; then
R2=$(wait_job "$JID2" 30)
success=$(echo "$R2" | jq -r '.success // false')
if [ "$success" = "true" ]; then
version=$(echo "$R2" | jq -r '.result.version // "?"')
log_pass "Latest version runs: version=$version"
else
err=$(echo "$R2" | jq -r '.result.error.message // "unknown"' 2>/dev/null)
log_fail "Run latest version" "success=false err=$err"
fi
else
log_fail "Run latest version" "bad job id: $JID2"
fi
###############################################################################
# TEST 3: Script with debounce_delay_s — rapid runs with SAME args
###############################################################################
echo ""
log_info "=== TEST 3: Debounce with same args (should debounce) ==="
P3="u/admin/e2e_debounce_$TS"
H3=$(create_script "$P3" "bun" \
'export function main(x: number = 0) { return { result: x }; }' \
'{"debounce_delay_s": 3}')
if echo "$H3" | grep -qE '^[0-9a-f]{16}$'; then
log_pass "Debounce script created: $H3"
else
log_fail "Debounce script creation" "$H3"
fi
log_info "Running 5 times with same args {x: 42}..."
DEB_IDS=()
for i in $(seq 1 5); do
JID=$(run_script "$P3" '{"x": 42}')
DEB_IDS+=("$JID")
log_info " Run $i: $JID"
done
log_info "Waiting 10s for debounce delay (3s) + execution..."
sleep 10
executed=0
skipped=0
for JID in "${DEB_IDS[@]}"; do
if ! echo "$JID" | grep -qE '^[0-9a-f-]{36}$'; then
log_info " Invalid job id: $JID"
continue
fi
R=$(wait_job "$JID" 5 2>/dev/null || echo '{"completed":false}')
completed=$(echo "$R" | jq -r '.completed // false')
success=$(echo "$R" | jq -r '.success // false')
if [ "$completed" = "true" ] && [ "$success" = "true" ]; then
((executed++)) || true
elif [ "$completed" = "true" ]; then
((skipped++)) || true
fi
done
log_info "Results: $executed executed, $skipped skipped out of ${#DEB_IDS[@]}"
if [ "$executed" -eq 1 ] && [ "$skipped" -ge 3 ]; then
log_pass "Debouncing perfect: 1 executed, $skipped skipped"
elif [ "$executed" -le 2 ] && [ "$skipped" -ge 2 ]; then
log_pass "Debouncing working: $executed executed, $skipped skipped"
else
log_fail "Debounce same args" "executed=$executed skipped=$skipped (want ~1 exec, ~4 skip)"
fi
###############################################################################
# TEST 3b: Different args should NOT debounce against each other
###############################################################################
echo ""
log_info "=== TEST 3b: Debounce with different args (should NOT debounce) ==="
DIFF_IDS=()
for i in $(seq 1 3); do
JID=$(run_script "$P3" "{\"x\": $((i * 100))}")
DIFF_IDS+=("$JID")
done
log_info "Waiting 8s..."
sleep 8
diff_exec=0
for JID in "${DIFF_IDS[@]}"; do
if ! echo "$JID" | grep -qE '^[0-9a-f-]{36}$'; then continue; fi
R=$(wait_job "$JID" 5 2>/dev/null || echo '{"completed":false}')
success=$(echo "$R" | jq -r '.success // false')
if [ "$success" = "true" ]; then ((diff_exec++)) || true; fi
done
if [ "$diff_exec" -eq 3 ]; then
log_pass "Different args: all 3 executed independently"
else
log_fail "Different args" "only $diff_exec/3 executed"
fi
###############################################################################
# TEST 4: Custom debounce_key with $args interpolation
###############################################################################
echo ""
log_info "=== TEST 4: Custom debounce key ==="
P4="u/admin/e2e_custom_key_$TS"
H4=$(create_script "$P4" "bun" \
'export function main(event_id: string = "", data: string = "") { return { event_id, data }; }' \
'{"debounce_delay_s": 3, "debounce_key": "event#$args.event_id"}')
if echo "$H4" | grep -qE '^[0-9a-f]{16}$'; then
log_pass "Custom key script created: $H4"
else
log_fail "Custom key script creation" "$H4"
fi
# Same event_id → should debounce
log_info "3 runs with same event_id..."
SAME_IDS=()
for i in $(seq 1 3); do
JID=$(run_script "$P4" "{\"event_id\": \"evt_001\", \"data\": \"payload_$i\"}")
SAME_IDS+=("$JID")
done
# Different event_id → should NOT debounce
JID_DIFF=$(run_script "$P4" '{"event_id": "evt_002", "data": "different"}')
log_info "Waiting 8s..."
sleep 8
same_exec=0
same_skip=0
for JID in "${SAME_IDS[@]}"; do
if ! echo "$JID" | grep -qE '^[0-9a-f-]{36}$'; then continue; fi
R=$(wait_job "$JID" 5 2>/dev/null || echo '{"completed":false}')
completed=$(echo "$R" | jq -r '.completed // false')
success=$(echo "$R" | jq -r '.success // false')
if [ "$completed" = "true" ] && [ "$success" = "true" ]; then
data=$(echo "$R" | jq -r '.result.data // "?"')
((same_exec++)) || true
log_info " Executed: data=$data"
elif [ "$completed" = "true" ]; then
((same_skip++)) || true
fi
done
log_info "Same event_id: $same_exec executed, $same_skip skipped"
if [ "$same_exec" -eq 1 ] && [ "$same_skip" -ge 1 ]; then
log_pass "Custom key debounce: same event_id debounced correctly"
elif [ "$same_exec" -le 2 ]; then
log_pass "Custom key debounce working: $same_exec executed, $same_skip skipped"
else
log_fail "Custom key debounce" "exec=$same_exec skip=$same_skip"
fi
# Check different event_id ran independently
if echo "$JID_DIFF" | grep -qE '^[0-9a-f-]{36}$'; then
R_DIFF=$(wait_job "$JID_DIFF" 10 2>/dev/null || echo '{"completed":false}')
diff_success=$(echo "$R_DIFF" | jq -r '.success // false')
if [ "$diff_success" = "true" ]; then
log_pass "Different event_id: executed independently"
else
log_info "Different event_id: success=$diff_success"
fi
fi
###############################################################################
# TEST 5: Git sync with bad target — debounced deployment callbacks
###############################################################################
echo ""
log_info "=== TEST 5: Git sync debounce + aggregation ==="
# Create git repo resource
api POST "resources/create?update_if_exists=true" '{
"path": "u/admin/e2e_bad_git_repo",
"description": "Bad git repo for testing",
"resource_type": "git_repository",
"value": {"url": "https://github.com/nonexistent/nope.git", "branch": "main", "token": "bad"}
}' > /dev/null 2>&1
log_info "Created git repo resource"
# Create a sync script at a folder path where the 2nd segment is a number >= 28103.
# is_script_meets_min_version parses split("/").skip(1).next() as the version number.
# This enables debounce_delay_s=5 and debounce_args_to_accumulate=["items"].
api POST "folders/create" '{"name": "28103"}' > /dev/null 2>&1
P5="f/28103/e2e_sync_$TS"
H5=$(create_script "$P5" "bun" \
'export function main(repo_url_resource_path: string = "", workspace_id: string = "", items: any[] = [], use_individual_branch: boolean = false, group_by_folder: boolean = false, parent_workspace_id: string = "") { return { synced: items.length, items }; }')
if echo "$H5" | grep -qE '^[0-9a-f]{16}$'; then
log_pass "Sync script created: $H5"
else
log_fail "Sync script creation" "$H5"
fi
# Configure git sync with include_path to match deployed scripts.
# Without include_path, path_matches_filters returns false and no DeploymentCallback is created.
api POST "workspaces/edit_git_sync_config" "{
\"git_sync_settings\": {
\"include_type\": [\"script\"],
\"include_path\": [\"**\"],
\"repositories\": [{
\"script_path\": \"$P5\",
\"git_repo_resource_path\": \"\$res:u/admin/e2e_bad_git_repo\",
\"use_individual_branch\": false,
\"group_by_folder\": false
}]
}
}" > /dev/null 2>&1
log_pass "Git sync configured with include_path and versioned folder script path"
# Deploy 5 scripts rapidly to trigger git sync.
# Scripts are created with lock="" (via create_script), so handle_deployment_metadata
# fires immediately after tx commit (not after dependency resolution).
log_info "Deploying 5 scripts to trigger git sync..."
for i in $(seq 1 5); do
dp="u/admin/e2e_gitsync_${TS}_$i"
create_script "$dp" "bun" "export function main() { return { v: $i }; }" > /dev/null
log_info " Deployed $dp"
done
# Wait for debounce delay (5s) + execution
log_info "Waiting 15s for debounce (5s) + execution..."
sleep 15
# Check deployment callback jobs for our sync script.
# Debounced jobs have is_skipped=true (but success=true), so we use is_skipped to distinguish.
SYNC_JOBS=$(api GET "jobs/completed/list?script_path_exact=$P5&job_kinds=deploymentcallback")
SYNC_TOTAL=$(echo "$SYNC_JOBS" | jq 'length')
SYNC_EXECUTED=$(echo "$SYNC_JOBS" | jq '[.[] | select(.is_skipped != true)] | length')
SYNC_SKIPPED=$(echo "$SYNC_JOBS" | jq '[.[] | select(.is_skipped == true)] | length')
log_info "Sync jobs: total=$SYNC_TOTAL executed=$SYNC_EXECUTED skipped=$SYNC_SKIPPED"
if [ "$SYNC_TOTAL" -gt 0 ]; then
# With debouncing (5s delay), rapid deploys should be consolidated.
# All 5 jobs are created but most should be skipped (debounced).
if [ "$SYNC_SKIPPED" -gt 0 ]; then
log_pass "Git sync debouncing: $SYNC_EXECUTED executed, $SYNC_SKIPPED debounced out of $SYNC_TOTAL"
else
log_fail "Git sync debouncing" "No jobs were debounced ($SYNC_TOTAL all executed independently)"
fi
# Check if items were aggregated in the executed (non-skipped) job(s)
for idx in $(seq 0 $((SYNC_TOTAL - 1))); do
is_skipped=$(echo "$SYNC_JOBS" | jq -r ".[$idx].is_skipped")
[ "$is_skipped" = "true" ] && continue
jid=$(echo "$SYNC_JOBS" | jq -r ".[$idx].id")
r=$(api GET "jobs/completed/get_result/$jid")
items_count=$(echo "$r" | jq '.items | length // 0')
log_info " Executed sync job $jid: items=$items_count"
if [ "$items_count" -gt 1 ]; then
log_pass "Items aggregated: $items_count items in single sync job"
fi
done
else
# Check queued — jobs may still be pending debounce delay
Q=$(api GET "jobs/queue/list?script_path_exact=$P5&job_kinds=deploymentcallback")
QC=$(echo "$Q" | jq 'length')
log_info "No completed sync jobs. $QC queued."
if [ "$QC" -gt 0 ] && [ "$QC" -lt 5 ]; then
log_pass "Git sync debouncing (queued): $QC jobs for 5 deploys"
elif [ "$QC" -eq 0 ]; then
log_fail "Git sync" "No deployment callback jobs found (completed or queued)"
fi
fi
# Cleanup git sync
api POST "workspaces/edit_git_sync_config" '{"git_sync_settings": null}' > /dev/null 2>&1
log_info "Git sync config cleared"
###############################################################################
# Summary
###############################################################################
echo ""
echo "========================================="
echo -e "Results: ${GREEN}$pass passed${NC}, ${RED}$fail failed${NC}"
echo "========================================="
if [ "$fail" -gt 0 ]; then
exit 1
fi
+155 -62
View File
@@ -1,8 +1,8 @@
use windmill_test_utils::*;
use sqlx::postgres::Postgres;
use sqlx::Pool;
use windmill_common::jobs::{JobPayload, RawCode};
use windmill_common::scripts::ScriptLang;
use windmill_test_utils::*;
// ============================================================================
// Basic Execution Tests
@@ -27,8 +27,8 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -63,8 +63,8 @@ export function main(name: string, count: number) {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -104,8 +104,9 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
)
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -135,8 +136,9 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
)
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -167,8 +169,9 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
)
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -207,8 +210,8 @@ export async function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -245,8 +248,9 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
)
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -276,8 +280,9 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
)
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -318,8 +323,8 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -358,8 +363,8 @@ export function notMain() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -398,8 +403,8 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -437,8 +442,8 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -474,8 +479,8 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -516,8 +521,8 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -613,8 +618,9 @@ export function main() {
path: Some("f/nested/test_deep".to_string()),
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
)
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -647,8 +653,9 @@ export function main() {
path: Some("f/nested/test_deep_relative".to_string()),
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
)
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -693,8 +700,8 @@ export function main() {
path: Some("f/circular/test_both".to_string()),
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -741,8 +748,8 @@ export function main(x: number) {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -791,8 +798,8 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -836,8 +843,8 @@ export function main() {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -859,11 +866,11 @@ export function main() {
// ============================================================================
mod dedicated_worker_protocol {
use windmill_test_utils::{parse_dedicated_worker_line, DedicatedWorkerResult};
use std::io::{BufRead, BufReader, Write};
use std::process::{Command, Stdio};
use windmill_test_utils::{parse_dedicated_worker_line, DedicatedWorkerResult};
use windmill_worker::{
build_loader, generate_dedicated_worker_wrapper, BUN_DEDICATED_WORKER_ARGS, LoaderMode,
build_loader, generate_dedicated_worker_wrapper, LoaderMode, BUN_DEDICATED_WORKER_ARGS,
BUN_PATH, NODE_BIN_PATH,
};
@@ -934,12 +941,8 @@ mod dedicated_worker_protocol {
let temp_dir = tempfile::tempdir().unwrap();
// Create files and get the wrapper path (bundled for node, raw for bun)
let wrapper_path = create_test_worker_files(
temp_dir.path(),
script,
arg_names,
runtime == "node",
);
let wrapper_path =
create_test_worker_files(temp_dir.path(), script, arg_names, runtime == "node");
let wrapper_str = wrapper_path.to_str().unwrap();
// Build args matching production behavior
@@ -992,7 +995,10 @@ mod dedicated_worker_protocol {
match parse_dedicated_worker_line(response.trim()) {
DedicatedWorkerResult::Success(value) => results.push(Ok(value)),
DedicatedWorkerResult::Error(err) => {
let msg = err["message"].as_str().unwrap_or("Unknown error").to_string();
let msg = err["message"]
.as_str()
.unwrap_or("Unknown error")
.to_string();
results.push(Err(msg));
}
other => panic!("Unexpected response: {:?}", other),
@@ -1162,8 +1168,8 @@ export function main(name: string) {
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
@@ -1190,6 +1196,68 @@ export function main(name: string) {
Ok(())
}
/// Test that full .npmrc content works for bun jobs with private registries.
/// Requires:
/// - `TEST_NPMRC` environment variable set to the full .npmrc content
#[cfg(feature = "private_registry_test")]
#[sqlx::test(fixtures("base"))]
async fn test_bun_job_private_npmrc(db: Pool<Postgres>) -> anyhow::Result<()> {
use windmill_worker::NPMRC;
let npmrc_content = std::env::var("TEST_NPMRC")
.expect("TEST_NPMRC must be set when running private_registry_test");
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
{
let mut npmrc = NPMRC.write().await;
*npmrc = Some(npmrc_content.clone());
}
let content = r#"
import { greet } from "@windmill-test/private-pkg";
export function main(name: string) {
return greet(name);
}
"#
.to_owned();
let job = JobPayload::Code(RawCode {
hash: None,
content,
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
});
let result = RunJob::from(job)
.arg("name", serde_json::json!("World"))
.run_until_complete(&db, false, port)
.await
.json_result()
.unwrap();
{
let mut npmrc = NPMRC.write().await;
*npmrc = None;
}
assert_eq!(
result,
serde_json::json!("Hello from private package, World!")
);
Ok(())
}
/// Tests for RELATIVE_BUN_BUILDER (loader_builder.bun.js)
/// These tests verify Bun's behavior for import scanning and package.json generation.
/// Purpose: Catch regressions when upgrading Bun versions.
@@ -1241,8 +1309,8 @@ mod bun_builder_tests {
}
// Read generated package.json
let package_json = std::fs::read_to_string(dir.join("package.json"))
.expect("package.json not generated");
let package_json =
std::fs::read_to_string(dir.join("package.json")).expect("package.json not generated");
serde_json::from_str(&package_json).expect("Invalid JSON in package.json")
}
@@ -1257,7 +1325,10 @@ export function main() { return lodash; }
let pkg = run_builder(main_ts);
let deps = pkg["dependencies"].as_object().unwrap();
assert!(deps.contains_key("lodash"), "lodash should be in dependencies");
assert!(
deps.contains_key("lodash"),
"lodash should be in dependencies"
);
assert_eq!(deps["lodash"], "latest");
}
@@ -1271,7 +1342,10 @@ export function main() { return _; }
let pkg = run_builder(main_ts);
let deps = pkg["dependencies"].as_object().unwrap();
assert!(deps.contains_key("lodash"), "lodash should be in dependencies");
assert!(
deps.contains_key("lodash"),
"lodash should be in dependencies"
);
assert_eq!(deps["lodash"], "4.17.21");
}
@@ -1304,9 +1378,18 @@ export function main() { return { lodash, axios, dayjs }; }
let pkg = run_builder(main_ts);
let deps = pkg["dependencies"].as_object().unwrap();
assert!(deps.contains_key("lodash"), "lodash should be in dependencies");
assert!(deps.contains_key("axios"), "axios should be in dependencies");
assert!(deps.contains_key("dayjs"), "dayjs should be in dependencies");
assert!(
deps.contains_key("lodash"),
"lodash should be in dependencies"
);
assert!(
deps.contains_key("axios"),
"axios should be in dependencies"
);
assert!(
deps.contains_key("dayjs"),
"dayjs should be in dependencies"
);
assert_eq!(deps.len(), 3, "Should have exactly 3 dependencies");
}
@@ -1330,8 +1413,15 @@ export function main() { return { fs, path, lodash }; }
!deps.contains_key("path"),
"path (builtin) should NOT be in dependencies"
);
assert!(deps.contains_key("lodash"), "lodash should be in dependencies");
assert_eq!(deps.len(), 1, "Should have exactly 1 dependency (lodash only)");
assert!(
deps.contains_key("lodash"),
"lodash should be in dependencies"
);
assert_eq!(
deps.len(),
1,
"Should have exactly 1 dependency (lodash only)"
);
}
/// Test: semver.order() resolves version conflicts (picks lowest version)
@@ -1347,7 +1437,10 @@ export function main() { return { a, b }; }
let pkg = run_builder(main_ts);
let deps = pkg["dependencies"].as_object().unwrap();
assert!(deps.contains_key("lodash"), "lodash should be in dependencies");
assert!(
deps.contains_key("lodash"),
"lodash should be in dependencies"
);
// The builder sorts by semver and picks the first (lowest) version
assert_eq!(
deps["lodash"], "4.17.10",
+10
View File
@@ -0,0 +1,10 @@
-- Fixture for testing wmill CLI variable/resource get from bash scripts
INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms)
VALUES ('test-workspace', 'u/test-user/test_var', 'hello from variable', false, 'A test variable', '{"u/test-user": true}');
INSERT INTO resource_type (workspace_id, name, schema, description, created_by)
VALUES ('test-workspace', 'test_object', '{}', 'Test object type', 'test-user');
INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by)
VALUES ('test-workspace', 'u/test-user/test_res', '{"host": "localhost", "port": 5432}', 'A test resource', 'test_object', '{"u/test-user": true}', 'test-user');
+275
View File
@@ -0,0 +1,275 @@
/*
* Tests for the PrewarmedIsolate used by nativets dedicated workers.
*
* Run with:
* cargo test -p windmill --features "deno_core" --test nativets_dedicated -- --nocapture
*/
#[cfg(feature = "deno_core")]
mod prewarmed_isolate_tests {
use std::process::Command;
use windmill_runtime_nativets::{NativeAnnotation, PrewarmedIsolate};
use windmill_worker::{build_loader, LoaderMode, BUN_PATH};
fn default_annotation() -> NativeAnnotation {
NativeAnnotation { useragent: None, proxy: None }
}
/// Bundle a TypeScript script into JS suitable for `PrewarmedIsolate`.
///
/// Returns `(ts_source, js_bundle, arg_names)`.
async fn bundle_script(script: &str) -> (String, String, Vec<String>) {
let temp_dir = tempfile::tempdir().unwrap();
let dir = temp_dir.path();
let dir_str = dir.to_str().unwrap();
std::fs::write(dir.join("main.ts"), script).unwrap();
build_loader(
dir_str,
"http://localhost:8000",
"test_token",
"test-workspace",
"f/test/script",
LoaderMode::BrowserBundle,
)
.await
.expect("build_loader failed");
let output = Command::new(BUN_PATH.as_str())
.args(["run", dir.join("node_builder.ts").to_str().unwrap()])
.current_dir(dir)
.output()
.expect("Failed to run bun build");
if !output.status.success() {
panic!(
"Bun build failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
let ts = std::fs::read_to_string(dir.join("main.ts")).unwrap();
let js = std::fs::read_to_string(dir.join("main.js")).unwrap();
let parsed = windmill_parser_ts::parse_deno_signature(&ts, true, false, None)
.expect("failed to parse signature");
let arg_names: Vec<String> = parsed.args.into_iter().map(|a| a.name).collect();
(ts, js, arg_names)
}
const TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
async fn run_prewarmed_test(
script: &str,
jobs: Vec<serde_json::Value>,
) -> Vec<Result<serde_json::Value, String>> {
tokio::time::timeout(TEST_TIMEOUT, run_prewarmed_test_inner(script, jobs))
.await
.expect("test timed out after 30s")
}
async fn run_prewarmed_test_inner(
script: &str,
jobs: Vec<serde_json::Value>,
) -> Vec<Result<serde_json::Value, String>> {
windmill_runtime_nativets::setup_deno_runtime().expect("V8 init failed");
let (_ts, js, arg_names) = bundle_script(script).await;
let ann = default_annotation();
let mut results = Vec::new();
for job_args in &jobs {
let mut isolate =
PrewarmedIsolate::spawn("".to_string(), js.clone(), ann.clone(), arg_names.clone());
isolate.wait_ready().await.expect("isolate failed to warm");
let args = serde_json::to_string(job_args).unwrap();
let executing = isolate.start_execution(args);
let prewarmed_result = executing.wait().await.expect("isolate execution failed");
match prewarmed_result.result {
Ok(raw) => {
let value: serde_json::Value =
serde_json::from_str(raw.get()).unwrap_or(serde_json::Value::Null);
results.push(Ok(value));
}
Err(e) => {
results.push(Err(e));
}
}
}
results
}
#[tokio::test(flavor = "multi_thread")]
async fn test_prewarmed_simple() {
let script = r#"
export function main(x: number, y: number): number {
return x + y;
}
"#;
let results = run_prewarmed_test(script, vec![serde_json::json!({"x": 2, "y": 3})]).await;
assert_eq!(results.len(), 1);
assert_eq!(results[0], Ok(serde_json::json!(5)));
}
#[tokio::test(flavor = "multi_thread")]
async fn test_prewarmed_isolation() {
let script = r#"
let counter = 0;
export function main(): number {
counter++;
return counter;
}
"#;
let results =
run_prewarmed_test(script, vec![serde_json::json!({}), serde_json::json!({})]).await;
assert_eq!(results.len(), 2);
// Each job gets a fresh isolate, so counter should be 1 both times
assert_eq!(results[0], Ok(serde_json::json!(1)));
assert_eq!(results[1], Ok(serde_json::json!(1)));
}
#[tokio::test(flavor = "multi_thread")]
async fn test_prewarmed_error() {
let script = r#"
export function main(msg: string): never {
throw new Error(msg);
}
"#;
let results =
run_prewarmed_test(script, vec![serde_json::json!({"msg": "test error"})]).await;
assert_eq!(results.len(), 1);
assert!(results[0].is_err());
assert!(
results[0].as_ref().unwrap_err().contains("test error"),
"Error should contain 'test error', got: {}",
results[0].as_ref().unwrap_err()
);
}
#[tokio::test(flavor = "multi_thread")]
async fn test_prewarmed_async() {
let script = r#"
export async function main(x: number): Promise<number> {
const val = await Promise.resolve(x * 10);
return val + 1;
}
"#;
let results = run_prewarmed_test(script, vec![serde_json::json!({"x": 7})]).await;
assert_eq!(results.len(), 1);
assert_eq!(results[0], Ok(serde_json::json!(71)));
}
#[tokio::test(flavor = "multi_thread")]
async fn test_prewarmed_pipeline() {
windmill_runtime_nativets::setup_deno_runtime().expect("V8 init failed");
let script = r#"
export function main(n: number): number {
return n * 2;
}
"#;
let (_ts, js, arg_names) = bundle_script(script).await;
let ann = default_annotation();
// Pre-warm first isolate
let mut warm =
PrewarmedIsolate::spawn("".to_string(), js.clone(), ann.clone(), arg_names.clone());
warm.wait_ready()
.await
.expect("first isolate failed to warm");
let mut results = Vec::new();
for i in 1..=3 {
let args = serde_json::to_string(&serde_json::json!({"n": i})).unwrap();
let executing = warm.start_execution(args);
// Pipeline: start pre-warming next isolate while current one runs
warm =
PrewarmedIsolate::spawn("".to_string(), js.clone(), ann.clone(), arg_names.clone());
let prewarmed_result = executing.wait().await.expect("isolate execution failed");
match prewarmed_result.result {
Ok(raw) => {
let value: serde_json::Value =
serde_json::from_str(raw.get()).unwrap_or(serde_json::Value::Null);
results.push(value);
}
Err(e) => panic!("unexpected error: {e}"),
}
warm.wait_ready()
.await
.expect("next isolate failed to warm");
}
assert_eq!(
results,
vec![
serde_json::json!(2),
serde_json::json!(4),
serde_json::json!(6),
]
);
}
#[tokio::test(flavor = "multi_thread")]
async fn test_prewarmed_complex_return() {
let script = r#"
export function main(name: string, items: number[]): any {
return {
greeting: `hello ${name}`,
sum: items.reduce((a, b) => a + b, 0),
items: items.map(x => x * 2),
};
}
"#;
let results = run_prewarmed_test(
script,
vec![serde_json::json!({"name": "world", "items": [1, 2, 3]})],
)
.await;
assert_eq!(results.len(), 1);
assert_eq!(
results[0],
Ok(serde_json::json!({
"greeting": "hello world",
"sum": 6,
"items": [2, 4, 6],
}))
);
}
#[tokio::test(flavor = "multi_thread")]
async fn test_prewarmed_null_undefined() {
let script = r#"
export function main(returnNull: boolean): any {
if (returnNull) {
return null;
}
return undefined;
}
"#;
let results = run_prewarmed_test(
script,
vec![
serde_json::json!({"returnNull": true}),
serde_json::json!({"returnNull": false}),
],
)
.await;
assert_eq!(results.len(), 2);
assert_eq!(results[0], Ok(serde_json::Value::Null));
assert_eq!(results[1], Ok(serde_json::Value::Null));
}
}
+134
View File
@@ -993,6 +993,80 @@ echo "hello $msg"
Ok(())
}
#[sqlx::test(fixtures("base", "wmill_cli_test"))]
async fn test_bash_wmill_variable_get(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
// The bash script uses wmill CLI to get the variable value.
// The worker sets WM_TOKEN, WM_WORKSPACE, and BASE_INTERNAL_URL as env vars,
// and the CLI auto-configures from them when no workspace is explicitly set.
// We point WMILL_CONFIG_DIR to a clean temp dir so no local active workspace interferes.
let content = r#"
export WMILL_CONFIG_DIR=$(mktemp -d)
result=$(wmill variable get "u/test-user/test_var" --json | jq -r .value)
echo "$result"
"#
.to_owned();
let job = RunJob::from(JobPayload::Code(RawCode {
hash: None,
content,
path: None,
lock: None,
language: ScriptLang::Bash,
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
}))
.run_until_complete(&db, false, port)
.await;
assert_eq!(job.json_result(), Some(json!("hello from variable")));
Ok(())
}
#[sqlx::test(fixtures("base", "wmill_cli_test"))]
async fn test_bash_wmill_resource_get(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
// The bash script uses wmill CLI to get the resource value.
// We point WMILL_CONFIG_DIR to a clean temp dir so no local active workspace interferes.
let content = r#"
export WMILL_CONFIG_DIR=$(mktemp -d)
result=$(wmill resource get "u/test-user/test_res" --json | jq -c .value)
echo "$result"
"#
.to_owned();
let job = RunJob::from(JobPayload::Code(RawCode {
hash: None,
content,
path: None,
lock: None,
language: ScriptLang::Bash,
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
}))
.run_until_complete(&db, false, port)
.await;
// Bash echo outputs are returned as strings, so the JSON is a string value
assert_eq!(
job.json_result(),
Some(json!("{\"host\":\"localhost\",\"port\":5432}"))
);
Ok(())
}
#[cfg(feature = "nu")]
#[sqlx::test(fixtures("base"))]
async fn test_nu_job(db: Pool<Postgres>) -> anyhow::Result<()> {
@@ -1592,6 +1666,66 @@ export async function main(a: Date) {
Ok(())
}
/// Test that full .npmrc content works for deno jobs with private registries.
/// Requires:
/// - `TEST_NPMRC` environment variable set to the full .npmrc content
#[cfg(feature = "private_registry_test")]
#[sqlx::test(fixtures("base"))]
async fn test_deno_job_private_npmrc(db: Pool<Postgres>) -> anyhow::Result<()> {
use windmill_worker::NPMRC;
let npmrc_content = std::env::var("TEST_NPMRC")
.expect("TEST_NPMRC must be set when running private_registry_test");
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
{
let mut npmrc = NPMRC.write().await;
*npmrc = Some(npmrc_content.clone());
}
let content = r#"
import { greet } from "npm:@windmill-test/private-pkg";
export function main(name: string) {
return greet(name);
}
"#
.to_owned();
let result = RunJob::from(JobPayload::Code(RawCode {
hash: None,
content,
path: None,
language: ScriptLang::Deno,
lock: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
}))
.arg("name", json!("World"))
.run_until_complete(&db, false, port)
.await
.json_result()
.unwrap();
{
let mut npmrc = NPMRC.write().await;
*npmrc = None;
}
assert_eq!(
result,
serde_json::json!("Hello from private package, World!")
);
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base"))]
async fn test_python_job_datetime_and_bytes(db: Pool<Postgres>) -> anyhow::Result<()> {
@@ -13,7 +13,7 @@ default = []
enterprise = ["windmill-common/enterprise", "windmill-queue/enterprise"]
private = ["windmill-common/private", "windmill-queue/private"]
python = ["dep:windmill-parser-py-imports"]
benchmark = []
benchmark = ["windmill-queue/benchmark"]
[dependencies]
windmill-api-auth.workspace = true
+10 -8
View File
@@ -6,7 +6,6 @@
* LICENSE-AGPL for a copy of the license.
*/
use windmill_api_auth::ApiAuthed;
use axum::{
extract::{Path, Query},
routing::{get, post},
@@ -20,6 +19,7 @@ use std::{
fmt::{Display, Formatter},
vec,
};
use windmill_api_auth::ApiAuthed;
use windmill_common::{
db::UserDB,
error::JsonResult,
@@ -109,7 +109,7 @@ pub struct Input {
#[derive(Debug, Serialize, Deserialize, FromRow)]
pub struct CompletedJobMini {
id: Uuid,
created_at: chrono::DateTime<chrono::Utc>,
completed_at: chrono::DateTime<chrono::Utc>,
args: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
created_by: String,
success: bool,
@@ -147,9 +147,9 @@ async fn get_input_history(
};
let sql = &format!(
"select id, v2_job.created_at, created_by, 'null'::jsonb as args, status = 'success' as success from v2_job JOIN v2_job_completed USING (id) \
"select id, v2_job_completed.completed_at, created_by, 'null'::jsonb as args, status = 'success' as success from v2_job JOIN v2_job_completed USING (id) \
where v2_job.workspace_id = $3 and {} = $1 and kind = any($2) {args_query} AND v2_job_completed.status != 'skipped' {include_non_root} \
order by v2_job.created_at desc limit $4 offset $5",
order by v2_job_completed.completed_at desc limit $4 offset $5",
r.runnable_type.column_name(),
);
@@ -189,10 +189,10 @@ async fn get_input_history(
id: row.id,
name: format!(
"{} {}",
row.created_at.format("%H:%M %-d/%-m"),
row.completed_at.format("%H:%M %-d/%-m"),
row.created_by
),
created_at: row.created_at,
created_at: row.completed_at,
args: sqlx::types::Json(
serde_json::value::RawValue::from_string("null".to_string()).unwrap(),
),
@@ -352,11 +352,12 @@ async fn update_input(
) -> JsonResult<String> {
let mut tx = user_db.begin(&authed).await?;
sqlx::query("UPDATE input SET name = $1, is_public = $2 WHERE id = $3 and workspace_id = $4")
sqlx::query("UPDATE input SET name = $1, is_public = $2 WHERE id = $3 and workspace_id = $4 AND created_by = $5")
.bind(&input.name)
.bind(&input.is_public)
.bind(&input.id)
.bind(&w_id)
.bind(&authed.username)
.execute(&mut *tx)
.await?;
@@ -372,9 +373,10 @@ async fn delete_input(
) -> JsonResult<String> {
let mut tx = user_db.begin(&authed).await?;
sqlx::query("DELETE FROM input WHERE id = $1 and workspace_id = $2")
sqlx::query("DELETE FROM input WHERE id = $1 and workspace_id = $2 AND created_by = $3")
.bind(&i_id)
.bind(&w_id)
.bind(&authed.username)
.execute(&mut *tx)
.await?;
@@ -13,6 +13,7 @@ windmill-api-auth.workspace = true
windmill-common = { workspace = true, default-features = false }
axum.workspace = true
flate2.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
sqlx.workspace = true
+58 -58
View File
@@ -14,8 +14,10 @@ use std::collections::HashMap;
use tower_http::cors::{Any, CorsLayer};
use windmill_common::{
error::{Error, JsonResult, Result},
global_settings::{load_value_from_global_settings, NPM_CONFIG_REGISTRY_SETTING},
utils::StripPath,
global_settings::{
load_value_from_global_settings, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING,
},
utils::{parse_npmrc_registry, StripPath},
};
use windmill_api_auth::ApiAuthed;
@@ -129,6 +131,14 @@ pub fn workspaced_service() -> Router {
)
}
fn build_registry_request(url: &str, auth_token: &Option<String>) -> reqwest::RequestBuilder {
let mut req = HTTP_CLIENT.get(url);
if let Some(token) = auth_token {
req = req.bearer_auth(token);
}
req
}
/// Get package metadata (versions and tags) from the private registry
async fn get_package_metadata(
_authed: ApiAuthed,
@@ -136,21 +146,14 @@ async fn get_package_metadata(
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
) -> JsonResult<PackageVersions> {
let package = parse_package_name(package_path.to_path());
let npm_registry = get_npm_registry(&db).await?;
if npm_registry.is_none() {
return Err(Error::BadRequest(
"No private npm registry configured".to_string(),
));
}
let registry_url = npm_registry.unwrap();
let (registry_url, auth_token) = get_npm_registry(&db)
.await?
.ok_or_else(|| Error::BadRequest("No private npm registry configured".to_string()))?;
let package_url = format_registry_url(&registry_url, &package, None, None);
tracing::info!("Fetching package metadata from: {}", package_url);
let response = HTTP_CLIENT
.get(&package_url)
let response = build_registry_request(&package_url, &auth_token)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
@@ -167,7 +170,6 @@ async fn get_package_metadata(
.await
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
// Extract versions and dist-tags from the package metadata
let mut versions = Vec::new();
let mut tags = HashMap::new();
@@ -194,22 +196,15 @@ async fn resolve_package_version(
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
) -> JsonResult<PackageVersion> {
let package = parse_package_name(package_path.to_path());
let npm_registry = get_npm_registry(&db).await?;
if npm_registry.is_none() {
return Err(Error::BadRequest(
"No private npm registry configured".to_string(),
));
}
let registry_url = npm_registry.unwrap();
let (registry_url, auth_token) = get_npm_registry(&db)
.await?
.ok_or_else(|| Error::BadRequest("No private npm registry configured".to_string()))?;
let reference = query.tag.unwrap_or_else(|| "latest".to_string());
let package_url = format_registry_url(&registry_url, &package, None, None);
tracing::info!("Resolving package version from: {}", package_url);
let response = HTTP_CLIENT
.get(&package_url)
let response = build_registry_request(&package_url, &auth_token)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
@@ -256,21 +251,14 @@ async fn get_package_filetree(
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
) -> JsonResult<PackageFiletree> {
let (package, version) = parse_package_and_version(package_version_path.to_path())?;
let npm_registry = get_npm_registry(&db).await?;
if npm_registry.is_none() {
return Err(Error::BadRequest(
"No private npm registry configured".to_string(),
));
}
let registry_url = npm_registry.unwrap();
let (registry_url, auth_token) = get_npm_registry(&db)
.await?
.ok_or_else(|| Error::BadRequest("No private npm registry configured".to_string()))?;
let package_url = format_registry_url(&registry_url, &package, None, None);
tracing::info!("Fetching package filetree from: {}", package_url);
let response = HTTP_CLIENT
.get(&package_url)
let response = build_registry_request(&package_url, &auth_token)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
@@ -287,7 +275,6 @@ async fn get_package_filetree(
.await
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
// Get the tarball URL for this version
let tarball_url = package_json
.get("versions")
.and_then(|v| v.get(&version))
@@ -296,9 +283,7 @@ async fn get_package_filetree(
.and_then(|t| t.as_str())
.ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?;
// Download and extract tarball to get file list
let tarball_response = HTTP_CLIENT
.get(tarball_url)
let tarball_response = build_registry_request(tarball_url, &auth_token)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?;
@@ -337,21 +322,14 @@ async fn get_package_file(
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
) -> Result<String> {
let (package, version, filepath) = parse_package_version_and_file(full_path.to_path())?;
let npm_registry = get_npm_registry(&db).await?;
if npm_registry.is_none() {
return Err(Error::BadRequest(
"No private npm registry configured".to_string(),
));
}
let registry_url = npm_registry.unwrap();
let (registry_url, auth_token) = get_npm_registry(&db)
.await?
.ok_or_else(|| Error::BadRequest("No private npm registry configured".to_string()))?;
let package_url = format_registry_url(&registry_url, &package, None, None);
tracing::info!("Fetching package file from: {}", package_url);
let response = HTTP_CLIENT
.get(&package_url)
let response = build_registry_request(&package_url, &auth_token)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
@@ -368,7 +346,6 @@ async fn get_package_file(
.await
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
// Get the tarball URL for this version
let tarball_url = package_json
.get("versions")
.and_then(|v| v.get(&version))
@@ -377,9 +354,7 @@ async fn get_package_file(
.and_then(|t| t.as_str())
.ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?;
// Download tarball
let tarball_response = HTTP_CLIENT
.get(tarball_url)
let tarball_response = build_registry_request(tarball_url, &auth_token)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?;
@@ -402,13 +377,38 @@ async fn get_package_file(
Ok(file_content)
}
/// Get the npm registry URL from global settings
async fn get_npm_registry(db: &sqlx::Pool<sqlx::Postgres>) -> Result<Option<String>> {
/// Get the npm registry URL and optional auth token from global settings.
/// Checks the `npmrc` setting first, then falls back to `npm_config_registry`.
async fn get_npm_registry(
db: &sqlx::Pool<sqlx::Postgres>,
) -> Result<Option<(String, Option<String>)>> {
let npmrc = load_value_from_global_settings(db, NPMRC_SETTING)
.await?
.and_then(|v| v.as_str().map(|s| s.to_string()));
if let Some(ref npmrc_content) = npmrc {
if let Some(parsed) = parse_npmrc_registry(npmrc_content) {
return Ok(Some(parsed));
}
}
let registry = load_value_from_global_settings(db, NPM_CONFIG_REGISTRY_SETTING)
.await?
.and_then(|v| v.as_str().map(|s| s.to_string()));
Ok(registry)
if let Some(ref s) = registry {
let (url, token) = if s.contains(":_authToken=") {
let parts: Vec<&str> = s.split(":_authToken=").collect();
let url = parts[0].to_string();
let token = parts.get(1).map(|t| t.to_string());
(url, token)
} else {
(s.clone(), None)
};
return Ok(Some((url, token)));
}
Ok(None)
}
/// Format a registry URL for a package
+4 -2
View File
@@ -43,8 +43,9 @@ use windmill_common::{
get_database_url,
global_settings::{
APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
EMAIL_DOMAIN_SETTING, ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING,
HUB_BASE_URL_SETTING,
},
instance_config::{self, ApplyMode, InstanceConfig},
server::Smtp,
@@ -512,6 +513,7 @@ pub async fn get_global_setting(
&& !key.starts_with("default_recovery_handler_")
&& !key.starts_with("default_success_handler_")
&& key != AUTOMATE_USERNAME_CREATION_SETTING
&& key != DEFAULT_TAGS_WORKSPACES_SETTING
&& key != HUB_BASE_URL_SETTING
&& key != HUB_ACCESSIBLE_URL_SETTING
&& key != EMAIL_DOMAIN_SETTING
+3 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.638.4
version: 1.642.0
title: Windmill API
contact:
@@ -19671,6 +19671,8 @@ components:
type: boolean
lock:
type: string
flow_path:
type: string
required:
- args
+17 -5
View File
@@ -42,8 +42,6 @@ use windmill_common::runnable_settings::{
};
#[cfg(feature = "inline_preview")]
use windmill_common::runtime_assets::{register_runtime_asset, InsertRuntimeAssetParams};
use windmill_types::s3::BundleFormat;
use windmill_object_store::upload_artifact_to_store;
use windmill_common::scripts::ScriptRunnableSettingsInline;
use windmill_common::triggers::TriggerMetadata;
use windmill_common::utils::{RunnableKind, WarnAfterExt};
@@ -54,8 +52,10 @@ use windmill_common::workspace_dependencies::{
use windmill_common::DYNAMIC_INPUT_CACHE;
#[cfg(all(feature = "enterprise", feature = "smtp"))]
use windmill_common::{email_oss::send_email_html, server::load_smtp_config};
use windmill_object_store::upload_artifact_to_store;
#[cfg(feature = "inline_preview")]
use windmill_parser::asset_parser::AssetKind;
use windmill_types::s3::BundleFormat;
#[cfg(feature = "inline_preview")]
use windmill_worker::get_worker_internal_server_inline_utils;
@@ -1386,7 +1386,8 @@ async fn get_logs_from_store(
log_file_index: &Option<Vec<String>>,
) -> Option<error::Result<Body>> {
use futures::StreamExt;
let stream = windmill_object_store::get_logs_from_store(log_offset, logs, log_file_index).await?;
let stream =
windmill_object_store::get_logs_from_store(log_offset, logs, log_file_index).await?;
let header = bytes::Bytes::from(
r#"to remove ansi colors, use: | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g'
"#
@@ -2849,6 +2850,7 @@ struct Preview {
dedicated_worker: Option<bool>,
lock: Option<String>,
format: Option<String>,
flow_path: Option<String>,
}
#[cfg(feature = "inline_preview")]
@@ -4509,6 +4511,14 @@ async fn run_preview_script(
check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?;
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into());
let preview_args = preview.args.unwrap_or_default();
let flow_path_extra = preview.flow_path.map(|fp| {
let mut extra = HashMap::new();
extra.insert("_FLOW_PATH".to_string(), to_raw_value(&fp));
extra
});
let push_args = PushArgs { extra: flow_path_extra, args: &preview_args };
let (uuid, tx) = push(
&db,
tx,
@@ -4532,7 +4542,7 @@ async fn run_preview_script(
dedicated_worker: preview.dedicated_worker,
}),
},
PushArgs::from(&preview.args.unwrap_or_default()),
push_args,
authed.display_username(),
&authed.email,
username_to_permissioned_as(&authed.username),
@@ -5772,7 +5782,9 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(os) = windmill_object_store::get_object_store().await {
let file = os
.get(&windmill_object_store::object_store_reexports::Path::from(format!("logs/{file_p}")))
.get(&windmill_object_store::object_store_reexports::Path::from(
format!("logs/{file_p}"),
))
.await;
if let Ok(file) = file {
if let Ok(bytes) = file.bytes().await {
@@ -10,6 +10,7 @@ pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb";
pub const LICENSE_KEY_SETTING: &str = "license_key";
pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry";
pub const BUNFIG_INSTALL_SCOPES_SETTING: &str = "bunfig_install_scopes";
pub const NPMRC_SETTING: &str = "npmrc";
pub const NUGET_CONFIG_SETTING: &str = "nuget_config";
pub const POWERSHELL_REPO_URL_SETTING: &str = "powershell_repo_url";
pub const POWERSHELL_REPO_PAT_SETTING: &str = "powershell_repo_pat";
+17 -3
View File
@@ -261,6 +261,8 @@ pub struct GlobalSettings {
#[serde(skip_serializing_if = "Option::is_none")]
pub bunfig_install_scopes: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub npmrc: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub nuget_config: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub maven_repos: Option<String>,
@@ -774,7 +776,11 @@ pub const PROTECTED_SETTINGS: &[&str] = &[
/// Note: jwt_secret is intentionally NOT hidden — it is included in YAML exports so that
/// operators can set it via ConfigMap. It is protected from deletion (PROTECTED_SETTINGS)
/// and from being set to empty/null, and its value is partially redacted in log output.
pub const HIDDEN_SETTINGS: &[&str] = &["uid", "min_keep_alive_version", "automate_username_creation"];
pub const HIDDEN_SETTINGS: &[&str] = &[
"uid",
"min_keep_alive_version",
"automate_username_creation",
];
/// Top-level settings whose entire value is sensitive and must be fully redacted in logs.
const SENSITIVE_SETTINGS: &[&str] = &[
@@ -788,6 +794,7 @@ const SENSITIVE_SETTINGS: &[&str] = &[
"pip_extra_index_url",
"npm_config_registry",
"bunfig_install_scopes",
"npmrc",
"maven_repos",
"ruby_repos",
"powershell_repo_pat",
@@ -798,7 +805,10 @@ const SENSITIVE_SETTINGS: &[&str] = &[
const NESTED_SENSITIVE_FIELDS: &[(&str, &[&str])] = &[
("smtp_settings", &["smtp_password"]),
("secret_backend", &["token"]),
("object_store_cache_config", &["secret_key", "serviceAccountKey"]),
(
"object_store_cache_config",
&["secret_key", "serviceAccountKey"],
),
];
fn redact_json_value(value: &serde_json::Value) -> serde_json::Value {
@@ -2353,7 +2363,11 @@ mod tests {
);
let diff = diff_global_settings(&current, &desired, ApplyMode::Merge);
assert_eq!(diff.upserts.len(), 1, "Same client with newer expiry should update even with different signature");
assert_eq!(
diff.upserts.len(),
1,
"Same client with newer expiry should update even with different signature"
);
}
#[test]
-24
View File
@@ -404,30 +404,6 @@ impl PgDatabase {
)
}
pub fn to_conn_str(&self) -> String {
format!(
"dbname={dbname} {user} host={host} {password} {port} {sslmode}",
dbname = self.dbname,
user = self
.user
.as_ref()
.map(|u| format!("user={}", urlencoding::encode(u)))
.unwrap_or_default(),
host = self.host,
password = self
.password
.as_ref()
.map(|p| format!("password={}", urlencoding::encode(p)))
.unwrap_or_default(),
port = self.port.map(|p| format!("port={}", p)).unwrap_or_default(),
sslmode = self
.sslmode
.as_ref()
.map(|s| format!("sslmode={}", s.clone()))
.unwrap_or_default(),
)
}
pub async fn connect(
&self,
) -> Result<(tokio_postgres::Client, TokioPgConnection), error::Error> {
+93
View File
@@ -1281,3 +1281,96 @@ mod tests {
assert_eq!(parsed, serde_json::json!([[1], [2], [3], [4], [5]]));
}
}
/// Parse .npmrc content to extract the default registry URL and its auth token.
/// Returns `Some((registry_url, Option<auth_token>))` if a default registry is found.
pub fn parse_npmrc_registry(npmrc_content: &str) -> Option<(String, Option<String>)> {
let mut registry_url: Option<String> = None;
let mut auth_tokens: Vec<(String, String)> = Vec::new();
for line in npmrc_content.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') || line.starts_with(';') {
continue;
}
if let Some(url) = line.strip_prefix("registry=") {
registry_url = Some(url.trim().to_string());
}
if line.starts_with("//") {
if let Some((prefix, token)) = line.split_once(":_authToken=") {
auth_tokens.push((prefix.to_string(), token.to_string()));
}
}
}
let url = registry_url?;
let url_without_protocol = url.trim_start_matches("https:").trim_start_matches("http:");
let url_prefix = url_without_protocol.trim_end_matches('/');
let token = auth_tokens
.iter()
.find(|(prefix, _)| {
let p = prefix.trim_end_matches('/');
p == url_prefix
})
.map(|(_, token)| token.clone());
Some((url, token))
}
#[cfg(test)]
mod npmrc_tests {
use super::parse_npmrc_registry;
#[test]
fn test_parse_simple_registry() {
let npmrc = "registry=https://registry.mycompany.com/\n//registry.mycompany.com/:_authToken=secret123\n";
let result = parse_npmrc_registry(npmrc);
assert_eq!(
result,
Some((
"https://registry.mycompany.com/".to_string(),
Some("secret123".to_string())
))
);
}
#[test]
fn test_parse_registry_without_auth() {
let npmrc = "registry=https://registry.npmjs.org/\n";
let result = parse_npmrc_registry(npmrc);
assert_eq!(
result,
Some(("https://registry.npmjs.org/".to_string(), None))
);
}
#[test]
fn test_parse_scoped_only_no_default() {
let npmrc =
"@myorg:registry=https://registry.myorg.com/\n//registry.myorg.com/:_authToken=tok\n";
let result = parse_npmrc_registry(npmrc);
assert_eq!(result, None);
}
#[test]
fn test_parse_with_comments() {
let npmrc = "# My registry\nregistry=https://r.example.com/\n; auth\n//r.example.com/:_authToken=tok\n";
let result = parse_npmrc_registry(npmrc);
assert_eq!(
result,
Some((
"https://r.example.com/".to_string(),
Some("tok".to_string())
))
);
}
#[test]
fn test_parse_empty_npmrc() {
assert_eq!(parse_npmrc_registry(""), None);
assert_eq!(parse_npmrc_registry("# just a comment"), None);
}
}
+1 -1
View File
@@ -427,7 +427,7 @@ fn format_pull_query(peek: String) -> String {
j.same_worker, j.pre_run_error, j.visible_to_owner,
j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job,
j.timeout, j.flow_step_id, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle, j.priority, j.raw_code, j.raw_lock, j.raw_flow,
j.script_entrypoint_override, j.preprocessed, pj.runnable_path as parent_runnable_path,
j.script_entrypoint_override, j.preprocessed, COALESCE(pj.runnable_path, j.args->>'_FLOW_PATH') as parent_runnable_path,
COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin,
p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email
FROM q, j
+30 -2
View File
@@ -576,7 +576,18 @@ pub async fn exchange_token(
Ok(token)
}
/// Refresh an OAuth token and update the database
/// Pre-fetched account fields needed for token refresh.
pub struct OAuthAccountInfo {
pub client: String,
pub refresh_token: String,
pub grant_type: String,
pub cc_client_id: Option<String>,
pub cc_client_secret: Option<String>,
pub cc_token_url: Option<String>,
}
/// Refresh an OAuth token and update the database.
/// Fetches the account from DB, then delegates to `refresh_token_for_account`.
pub async fn refresh_token<'c>(
mut tx: Transaction<'c, Postgres>,
path: &str,
@@ -587,7 +598,8 @@ pub async fn refresh_token<'c>(
http_client: &reqwest::Client,
connect_configs_json: &str,
) -> error::Result<String> {
let account = sqlx::query!(
let account = sqlx::query_as!(
OAuthAccountInfo,
"SELECT client, refresh_token, grant_type, cc_client_id, cc_client_secret, cc_token_url FROM account WHERE workspace_id = $1 AND id = $2",
w_id,
id,
@@ -595,6 +607,22 @@ pub async fn refresh_token<'c>(
.fetch_optional(&mut *tx)
.await?;
let account = windmill_common::utils::not_found_if_none(account, "Account", &id.to_string())?;
refresh_token_for_account(tx, path, w_id, id, db, account, oauth_clients, http_client, connect_configs_json).await
}
/// Refresh an OAuth token given pre-fetched account info (no additional SELECT).
pub async fn refresh_token_for_account<'c>(
mut tx: Transaction<'c, Postgres>,
path: &str,
w_id: &str,
id: i32,
db: &DB,
account: OAuthAccountInfo,
oauth_clients: &AllClients,
http_client: &reqwest::Client,
connect_configs_json: &str,
) -> error::Result<String> {
let oauth_client_info = oauth_clients
.connects
.get(&account.client)
+6 -3
View File
@@ -5321,7 +5321,11 @@ async fn push_inner<'c, 'd>(
.as_ref()
.map(|x| {
let tag_lang = if x == &ScriptLang::Bunnative {
ScriptLang::Nativets.as_str()
if job_kind == JobKind::Dependencies {
ScriptLang::Bun.as_str()
} else {
ScriptLang::Nativets.as_str()
}
} else {
x.as_str()
};
@@ -5364,7 +5368,6 @@ async fn push_inner<'c, 'd>(
job_id,
&args,
&mut tx,
_db,
)
.await?
}
@@ -6204,7 +6207,7 @@ pub async fn get_same_worker_job(
v2_job.raw_code,
v2_job.raw_lock,
v2_job.raw_flow,
pj.runnable_path as parent_runnable_path,
COALESCE(pj.runnable_path, v2_job.args->>'_FLOW_PATH') as parent_runnable_path,
p.email as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin,
p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email
FROM v2_job_queue
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,155 @@
use std::collections::HashMap;
use serde_json::value::RawValue;
use crate::{
create_nativets_runtime, execute_main, load_client_module, load_user_module, CreatedRuntime,
ExecuteError, MainArgs, NativeAnnotation,
};
pub struct PrewarmedResult {
pub result: Result<Box<RawValue>, String>,
pub logs: String,
}
pub struct ExecutingIsolate {
result_rx: tokio::sync::oneshot::Receiver<PrewarmedResult>,
handle: tokio::task::JoinHandle<anyhow::Result<()>>,
}
impl ExecutingIsolate {
pub async fn wait(self) -> anyhow::Result<PrewarmedResult> {
let result = self
.result_rx
.await
.map_err(|_| anyhow::anyhow!("isolate result channel closed"))?;
self.handle
.await
.map_err(|e| anyhow::anyhow!("isolate thread panicked: {e}"))??;
Ok(result)
}
}
pub struct PrewarmedIsolate {
args_tx: Option<tokio::sync::oneshot::Sender<String>>,
result_rx: Option<tokio::sync::oneshot::Receiver<PrewarmedResult>>,
ready_rx: Option<tokio::sync::oneshot::Receiver<()>>,
handle: Option<tokio::task::JoinHandle<anyhow::Result<()>>>,
}
/// Parse a JSON args object and reorder into positional args matching `arg_names`.
fn args_to_positional(args_json: &str, arg_names: &[String]) -> Vec<Option<Box<RawValue>>> {
let map: HashMap<String, Box<RawValue>> = serde_json::from_str(args_json).unwrap_or_default();
arg_names
.iter()
.map(|name| map.get(name).cloned())
.collect()
}
impl PrewarmedIsolate {
/// Spawn a new isolate on a blocking thread.
///
/// The isolate loads `env_code` + WINDMILL_CLIENT as `windmill.ts`,
/// then loads `js_code` as `eval.ts`, and waits for args to execute.
pub fn spawn(
env_code: String,
js_code: String,
ann: NativeAnnotation,
arg_names: Vec<String>,
) -> Self {
let (args_tx, args_rx) = tokio::sync::oneshot::channel::<String>();
let (result_tx, result_rx) = tokio::sync::oneshot::channel::<PrewarmedResult>();
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel::<()>();
let handle = tokio::task::spawn_blocking(move || {
let CreatedRuntime { mut js_runtime, log_receiver, mut memory_limit_rx } =
create_nativets_runtime(ann, vec![])?;
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
runtime.block_on(async {
load_client_module(&mut js_runtime, &env_code).await?;
load_user_module(&mut js_runtime, format!("{env_code}\n{js_code}")).await?;
let _ = ready_tx.send(());
let args_json = match args_rx.await {
Ok(a) => a,
Err(_) => return Ok(()),
};
let positional = args_to_positional(&args_json, &arg_names);
{
let op_state = js_runtime.op_state();
let mut op_state = op_state.borrow_mut();
op_state.put(MainArgs { args: positional });
}
let log_handle = tokio::spawn(async move {
let mut log_receiver = log_receiver;
let mut logs = String::new();
while let Some(log) = log_receiver.recv().await {
logs.push_str(&log);
logs.push('\n');
}
logs
});
let exec_result = tokio::select! {
r = execute_main(&mut js_runtime, None, false, None) => r,
_ = memory_limit_rx.recv() => {
Err(ExecuteError::Script("Memory limit reached, killing isolate".to_string()))
}
};
let result = match exec_result {
Ok(raw) => Ok(raw),
Err(ExecuteError::Script(msg)) => Err(msg),
Err(ExecuteError::Js { message, stack, .. }) => {
let msg = message.unwrap_or_default();
let err = match stack {
Some(s) => format!("{msg}\n{s}"),
None => msg,
};
Err(err)
}
};
drop(js_runtime);
let logs = log_handle.await.unwrap_or_default();
let _ = result_tx.send(PrewarmedResult { result, logs });
Ok(())
})
});
PrewarmedIsolate {
args_tx: Some(args_tx),
result_rx: Some(result_rx),
ready_rx: Some(ready_rx),
handle: Some(handle),
}
}
/// Wait for the isolate to finish loading modules.
pub async fn wait_ready(&mut self) -> anyhow::Result<()> {
if let Some(rx) = self.ready_rx.take() {
rx.await
.map_err(|_| anyhow::anyhow!("isolate failed during pre-warm"))?;
}
Ok(())
}
/// Send args and start execution. Returns an `ExecutingIsolate` that
/// can be awaited independently, allowing the caller to pre-warm
/// the next isolate in parallel.
pub fn start_execution(mut self, args: String) -> ExecutingIsolate {
let args_tx = self.args_tx.take().expect("start_execution called twice");
let _ = args_tx.send(args);
ExecutingIsolate {
result_rx: self.result_rx.take().expect("result_rx missing"),
handle: self.handle.take().expect("handle missing"),
}
}
}
+239 -150
View File
@@ -12,6 +12,9 @@
//! TypeScript scripts via the nativets runtime. By isolating this here,
//! deno_core compilation no longer blocks windmill-worker or windmill-api.
mod dedicated;
pub use dedicated::{ExecutingIsolate, PrewarmedIsolate, PrewarmedResult};
use std::{
borrow::Cow,
cell::RefCell,
@@ -112,14 +115,15 @@ impl NetPermissions for PermissionsContainer {
// ── Types ────────────────────────────────────────────────────────────
struct MainArgs {
args: Vec<Option<Box<RawValue>>>,
pub(crate) struct MainArgs {
pub(crate) args: Vec<Option<Box<RawValue>>>,
}
struct LogString {
pub s: mpsc::UnboundedSender<String>,
}
#[derive(Clone)]
pub struct NativeAnnotation {
pub useragent: Option<String>,
pub proxy: Option<(String, Option<(String, String)>)>,
@@ -145,7 +149,7 @@ impl Drop for IsolateDropGuard {
static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH_SNAPSHOT.bin"));
const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js");
pub(crate) const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js");
const ERROR_DIR: &str = const_format::concatcp!(TMP_DIR, "/native_errors");
@@ -177,7 +181,10 @@ pub fn setup_deno_runtime() -> anyhow::Result<()> {
.collect::<Vec<_>>();
if !unrecognized_v8_flags.is_empty() {
init_err = Some(format!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags));
init_err = Some(format!(
"Unrecognized V8 flags: {:?}",
unrecognized_v8_flags
));
}
// Use an unprotected platform that doesn't enforce thread-isolated allocations
@@ -349,6 +356,140 @@ fn op_log(op_state: Rc<RefCell<OpState>>, #[string] log: &str) {
}
}
// ── Shared V8 runtime creation ───────────────────────────────────────
pub(crate) struct CreatedRuntime {
pub(crate) js_runtime: JsRuntime,
pub(crate) log_receiver: mpsc::UnboundedReceiver<String>,
pub(crate) memory_limit_rx: mpsc::UnboundedReceiver<()>,
}
/// Create a JsRuntime with the standard nativets extensions, heap limit
/// callback, and log channel. Must be called on a blocking thread (not
/// on the async tokio runtime) because V8 isolate creation is
/// synchronous and potentially heavy.
pub(crate) fn create_nativets_runtime(
ann: NativeAnnotation,
initial_args: Vec<Option<Box<RawValue>>>,
) -> anyhow::Result<CreatedRuntime> {
let ops = vec![op_get_static_args(), op_log()];
let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() };
let fetch_options = deno_fetch::Options {
root_cert_store_provider: None,
user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()),
proxy: ann.proxy.map(|x| deno_tls::Proxy {
url: x.0,
basic_auth: x
.1
.map(|(username, password)| deno_tls::BasicAuth { username, password }),
}),
..Default::default()
};
let exts: Vec<Extension> = vec![
deno_telemetry::deno_telemetry::init_ops(),
deno_webidl::deno_webidl::init_ops(),
deno_url::deno_url::init_ops(),
deno_console::deno_console::init_ops(),
deno_web::deno_web::init_ops::<PermissionsContainer>(Arc::new(BlobStore::default()), None),
deno_fetch::deno_fetch::init_ops::<PermissionsContainer>(fetch_options),
deno_net::deno_net::init_ops::<PermissionsContainer>(None, None),
ext,
];
let options = RuntimeOptions {
is_main: true,
extensions: exts,
create_params: Some(
deno_core::v8::CreateParams::default().heap_limits(0, 1024 * 1024 * 128),
),
startup_snapshot: Some(RUNTIME_SNAPSHOT),
module_loader: Some(Rc::new(deno_core::FsModuleLoader)),
extension_transpiler: None,
..Default::default()
};
let (memory_limit_tx, memory_limit_rx) = mpsc::unbounded_channel::<()>();
setup_deno_runtime().expect("V8 platform init failed");
let mut js_runtime = {
let _v8_lock = V8_ISOLATE_CREATE_LOCK
.lock()
.unwrap_or_else(|e| e.into_inner());
JsRuntime::new(options)
};
js_runtime.add_near_heap_limit_callback(move |x, y| {
tracing::error!("heap limit reached: {x} {y}");
if memory_limit_tx.send(()).is_err() {
tracing::warn!(
"memory limit notification channel closed - isolate may already be terminating"
);
}
y * 2
});
let (log_sender, log_receiver) = mpsc::unbounded_channel::<String>();
{
let op_state = js_runtime.op_state();
let mut op_state = op_state.borrow_mut();
op_state.put(PermissionsContainer {});
op_state.put(MainArgs { args: initial_args });
op_state.put(LogString { s: log_sender });
}
Ok(CreatedRuntime { js_runtime, log_receiver, memory_limit_rx })
}
// ── Shared module-loading helpers ────────────────────────────────────
pub(crate) async fn load_client_module(
js_runtime: &mut JsRuntime,
env_code: &str,
) -> anyhow::Result<()> {
js_runtime
.load_side_es_module_from_code(
&deno_core::resolve_url("file:///windmill.ts")
.map_err(windmill_common::error::to_anyhow)?,
format!("{env_code}\n{WINDMILL_CLIENT}"),
)
.await
.map_err(windmill_common::error::to_anyhow)?;
Ok(())
}
pub(crate) async fn load_user_module(
js_runtime: &mut JsRuntime,
source: String,
) -> anyhow::Result<()> {
use anyhow::Context;
js_runtime
.load_side_es_module_from_code(
&deno_core::resolve_url("file:///eval.ts")
.map_err(windmill_common::error::to_anyhow)?,
source,
)
.await
.context("failed to load module")?;
Ok(())
}
/// Extract a string result from a resolved V8 global and convert to `Box<RawValue>`.
pub(crate) fn extract_global_string(
js_runtime: &mut JsRuntime,
global: v8::Global<v8::Value>,
) -> Result<Box<RawValue>, String> {
let scope = &mut js_runtime.handle_scope();
let local = v8::Local::new(scope, global);
match serde_v8::from_v8::<Option<String>>(scope, local) {
Ok(s) => Ok(unsafe_raw(s.unwrap_or_else(|| "null".to_string()))),
Err(e) => Err(format!("failed to deserialize result: {e}")),
}
}
// ── eval_fetch_timeout ───────────────────────────────────────────────
/// Execute a NativeTS script using deno_core/V8.
@@ -436,63 +577,9 @@ pub async fn eval_fetch_timeout(
}
let result_f = tokio::task::spawn_blocking(move || {
let ops = vec![op_get_static_args(), op_log()];
let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() };
let CreatedRuntime { mut js_runtime, mut log_receiver, mut memory_limit_rx } =
create_nativets_runtime(ann, spread)?;
let fetch_options = deno_fetch::Options {
root_cert_store_provider: None,
user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()),
proxy: ann.proxy.map(|x| deno_tls::Proxy {
url: x.0,
basic_auth: x
.1
.map(|(username, password)| deno_tls::BasicAuth { username, password }),
}),
..Default::default()
};
let exts: Vec<Extension> = vec![
deno_telemetry::deno_telemetry::init_ops(),
deno_webidl::deno_webidl::init_ops(),
deno_url::deno_url::init_ops(),
deno_console::deno_console::init_ops(),
deno_web::deno_web::init_ops::<PermissionsContainer>(
Arc::new(BlobStore::default()),
None,
),
deno_fetch::deno_fetch::init_ops::<PermissionsContainer>(fetch_options),
deno_net::deno_net::init_ops::<PermissionsContainer>(None, None),
ext,
];
let options = RuntimeOptions {
is_main: true,
extensions: exts,
create_params: Some(
deno_core::v8::CreateParams::default().heap_limits(0, 1024 * 1024 * 128),
),
startup_snapshot: Some(RUNTIME_SNAPSHOT),
module_loader: Some(Rc::new(deno_core::FsModuleLoader)),
extension_transpiler: None,
..Default::default()
};
let (memory_limit_tx, mut memory_limit_rx) = mpsc::unbounded_channel::<()>();
// Ensure V8 platform is initialized (idempotent, no-op if already done).
setup_deno_runtime().expect("V8 platform init failed");
// Serialize isolate creation as extra safety net against concurrent V8
// isolate creation races. The main fix is the unprotected platform in
// setup_deno_runtime(), but this provides defense in depth.
let mut js_runtime = {
let _v8_lock = V8_ISOLATE_CREATE_LOCK
.lock()
.unwrap_or_else(|e| e.into_inner());
JsRuntime::new(options)
};
// Bootstrap OpenTelemetry for fetch auto-instrumentation if OTEL was initialized.
if otel_initialized {
if let Err(e) =
js_runtime.execute_script("<otel_bootstrap>", "globalThis.__bootstrapOtel()")
@@ -501,24 +588,6 @@ pub async fn eval_fetch_timeout(
}
}
js_runtime.add_near_heap_limit_callback(move |x, y| {
tracing::error!("heap limit reached: {x} {y}");
if memory_limit_tx.send(()).is_err() {
tracing::error!("failed to send memory limit reached notification - isolate may already be terminating");
};
y * 2
});
let (log_sender, mut log_receiver) = mpsc::unbounded_channel::<String>();
{
let op_state = js_runtime.op_state();
let mut op_state = op_state.borrow_mut();
op_state.put(PermissionsContainer {});
op_state.put(MainArgs { args: spread });
op_state.put(LogString { s: log_sender });
}
*isolate_handle.lock().unwrap_or_else(|e| e.into_inner()) =
Some(js_runtime.v8_isolate().thread_safe_handle());
@@ -601,42 +670,97 @@ async fn eval_fetch(
script_entrypoint_override: Option<String>,
load_client: bool,
job_id: &Uuid,
_otel_initialized: bool,
otel_initialized: bool,
) -> windmill_common::error::Result<Box<RawValue>> {
if load_client {
if let Some(env_code) = env_code.as_ref() {
let _ = js_runtime
.load_side_es_module_from_code(
&deno_core::resolve_url("file:///windmill.ts")
.map_err(windmill_common::error::to_anyhow)?,
format!("{env_code}\n{}", WINDMILL_CLIENT.to_string()),
)
.await
.map_err(windmill_common::error::to_anyhow)?;
load_client_module(js_runtime, env_code).await?;
}
}
use anyhow::Context;
use deno_core::error::CoreError;
use windmill_common::worker::to_raw_value;
let source = format!("{}\n{expr}", env_code.unwrap_or_default());
let _ = js_runtime
.load_side_es_module_from_code(
&deno_core::resolve_url("file:///eval.ts")
.map_err(windmill_common::error::to_anyhow)?,
source.to_string(),
)
.await
.map_err(|e| {
write_error_expr(expr, &job_id);
e
})
.context("failed to load module")?;
if let Err(e) = load_user_module(js_runtime, source.clone()).await {
write_error_expr(expr, job_id);
return Err(e.into());
}
let main_override = script_entrypoint_override.unwrap_or("main".to_string());
let result = execute_main(
js_runtime,
script_entrypoint_override.as_deref(),
otel_initialized,
Some(job_id),
)
.await;
match result {
Ok(raw) => Ok(raw),
Err(ExecuteError::Script(msg)) => {
write_error_expr(expr, job_id);
Err(Error::ExecutionErr(msg))
}
Err(ExecuteError::Js { message, stack, name, source: eval_source }) => {
write_error_expr(expr, job_id);
use windmill_common::worker::to_raw_value;
let stack_head = eval_source.and_then(|(file, line_no)| {
if file == "file:///eval.ts" {
source
.lines()
.nth(line_no.saturating_sub(1))
.map(|l| format!("{l}\n"))
} else {
None
}
});
let stack_s = format!(
"{}{}",
stack_head.unwrap_or_default(),
stack.as_deref().unwrap_or_default()
);
let stack = if stack_s.is_empty() {
None
} else {
Some(stack_s)
};
Err(Error::ExecutionRawError(to_raw_value(&serde_json::json!({
"message": message,
"stack": stack,
"name": name,
}))))
}
}
}
// ── Shared execution engine ──────────────────────────────────────────
pub(crate) enum ExecuteError {
/// Non-JS error (V8 internal, init failure, deserialization)
Script(String),
/// JS exception with structured error info
Js {
message: Option<String>,
stack: Option<String>,
name: Option<String>,
/// (file_name, line_number) from the first stack frame, if in user code
source: Option<(String, usize)>,
},
}
/// Execute the `main` function from the already-loaded `eval.ts` module.
///
/// Args must already be set in `MainArgs` in the runtime's OpState.
/// Modules (`windmill.ts` and `eval.ts`) must already be loaded.
pub(crate) async fn execute_main(
js_runtime: &mut JsRuntime,
entrypoint: Option<&str>,
_otel_initialized: bool,
_job_id: Option<&Uuid>,
) -> Result<Box<RawValue>, ExecuteError> {
let main_fn = entrypoint.unwrap_or("main");
#[cfg(all(feature = "private", feature = "enterprise"))]
let otel_context_inject = if _otel_initialized {
let trace_id = job_id.as_simple().to_string();
let trace_id = _job_id
.map(|id| id.as_simple().to_string())
.unwrap_or_default();
format!(
r#"globalThis.__enterSpan?.({{
isRecording: () => true,
@@ -687,7 +811,7 @@ function processStreamIterative(res) {{
{otel_context_inject}
let args = Deno.core.ops.op_get_static_args().map(JSON.parse)
import("file:///eval.ts").then((module) => module.{main_override}(...args))
import("file:///eval.ts").then((module) => module.{main_fn}(...args))
.then(res => {{
if (isAsyncIterable(res)) {{
return processStreamIterative(res)
@@ -698,60 +822,25 @@ import("file:///eval.ts").then((module) => module.{main_override}(...args))
"#
),
)
.map_err(|e| {
write_error_expr(expr, &job_id);
e
})
.context("native script initialization")?;
.map_err(|e| ExecuteError::Script(format!("native script initialization: {e}")))?;
let fut = js_runtime.resolve(script);
let global = js_runtime
.with_event_loop_promise(fut, PollEventLoopOptions::default())
.await
.map_err(|e| {
write_error_expr(expr, &job_id);
e
});
.await;
match global {
Ok(global) => {
let scope = &mut js_runtime.handle_scope();
let local = v8::Local::new(scope, global);
let r = serde_v8::from_v8::<Option<String>>(scope, local)
.map_err(windmill_common::error::to_anyhow)?;
Ok(unsafe_raw(r.unwrap_or_else(|| "null".to_string())))
extract_global_string(js_runtime, global).map_err(|e| ExecuteError::Script(e))
}
Err(CoreError::Js(e)) => {
let stack_head = e.frames.first().and_then(|f| {
if f.file_name.as_ref().is_some_and(|x| x == "file:///eval.ts") {
Some(format!(
"{}\n",
source
.lines()
.nth((f.line_number.unwrap_or(1)) as usize - 1)
.unwrap_or("")
.to_string()
))
} else {
None
}
Err(deno_core::error::CoreError::Js(e)) => {
let source = e.frames.first().and_then(|f| {
f.file_name
.as_ref()
.map(|name| (name.clone(), f.line_number.unwrap_or(1) as usize))
});
let stack_s = format!(
"{}{}",
stack_head.unwrap_or("".to_string()),
e.stack.unwrap_or("".to_string())
);
let stack = if stack_s.is_empty() {
None
} else {
Some(stack_s)
};
Err(Error::ExecutionRawError(to_raw_value(&serde_json::json!({
"message": e.message,
"stack": stack,
"name": e.name,
}))))
Err(ExecuteError::Js { message: e.message, stack: e.stack, name: e.name, source })
}
Err(e) => Err(Error::ExecutionErr(e.print_with_cause())),
Err(e) => Err(ExecuteError::Script(e.print_with_cause())),
}
}

Some files were not shown because too many files have changed in this diff Show More