mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-09-06 00:02:13 +00:00
Merge branch 'main' into glm/focus-summary
This commit is contained in:
@@ -44,6 +44,10 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
|
||||
# Bun
|
||||
COPY --from=oven/bun:1.3.8 /usr/local/bin/bun /usr/bin/bun
|
||||
|
||||
# Install windmill CLI
|
||||
RUN bun install -g windmill-cli \
|
||||
&& ln -s $(bun pm bin -g)/wmill /usr/bin/wmill
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
|
||||
# Deno
|
||||
|
||||
@@ -19,7 +19,7 @@ defaults:
|
||||
|
||||
jobs:
|
||||
cargo_test:
|
||||
runs-on: ubicloud-standard-16
|
||||
runs-on: blacksmith-16vcpu-ubuntu-2404
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
@@ -70,6 +70,16 @@ jobs:
|
||||
with:
|
||||
ruby-version: "3.3"
|
||||
bundler-cache: false
|
||||
- name: Install windmill CLI from source
|
||||
run: |
|
||||
cd $GITHUB_WORKSPACE/cli
|
||||
bash gen_wm_client.sh
|
||||
bun install
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
printf '#!/bin/sh\nexec bun run "%s/cli/src/main.ts" "$@"\n' "$GITHUB_WORKSPACE" > "$HOME/.local/bin/wmill"
|
||||
chmod +x "$HOME/.local/bin/wmill"
|
||||
echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
working-directory: /
|
||||
- name: Install PowerShell, mold and clang
|
||||
run: |
|
||||
sudo apt-get update && sudo apt-get install -y powershell mold clang libcurl4-openssl-dev
|
||||
@@ -78,6 +88,20 @@ jobs:
|
||||
with:
|
||||
cache: false
|
||||
toolchain: 1.93.0
|
||||
- name: Cache cargo target directory
|
||||
uses: useblacksmith/stickydisk@v1
|
||||
with:
|
||||
key: cargo-target
|
||||
path: ./backend/target
|
||||
- name: Cache cargo registry
|
||||
uses: useblacksmith/cache@v1
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
key: cargo-registry-${{ hashFiles('backend/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
cargo-registry-
|
||||
- name: Read EE repo commit hash
|
||||
run: |
|
||||
echo "ee_repo_ref=$(cat ./ee-repo-ref.txt)" >> "$GITHUB_ENV"
|
||||
@@ -165,6 +189,12 @@ jobs:
|
||||
fi
|
||||
|
||||
echo "NPM_TOKEN=${NPM_TOKEN}" >> $GITHUB_ENV
|
||||
{
|
||||
echo "TEST_NPMRC<<NPMRC_EOF"
|
||||
echo "@windmill-test:registry=http://localhost:4873/"
|
||||
echo "//localhost:4873/:_authToken=${NPM_TOKEN}"
|
||||
echo "NPMRC_EOF"
|
||||
} >> $GITHUB_ENV
|
||||
echo "Got NPM token successfully: ${NPM_TOKEN:0:10}..."
|
||||
|
||||
# Configure npm globally with the auth token
|
||||
@@ -199,7 +229,7 @@ jobs:
|
||||
fi
|
||||
echo "Verified: Package requires authentication for @windmill-test/private-pkg"
|
||||
- name: Cache DuckDB FFI module build
|
||||
uses: actions/cache@v3
|
||||
uses: useblacksmith/cache@v1
|
||||
with:
|
||||
path: ./backend/windmill-duckdb-ffi-internal/target
|
||||
key: ${{ runner.os }}-duckdb-ffi-${{ hashFiles('./backend/windmill-duckdb-ffi-internal/src/**/*.rs', './backend/windmill-duckdb-ffi-internal/Cargo.toml', './backend/windmill-duckdb-ffi-internal/Cargo.lock') }}
|
||||
@@ -215,6 +245,7 @@ jobs:
|
||||
RUST_LOG_STYLE: never
|
||||
CARGO_NET_GIT_FETCH_WITH_CLI: true
|
||||
CARGO_BUILD_JOBS: 12
|
||||
CARGO_INCREMENTAL: 1
|
||||
WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1
|
||||
WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1
|
||||
WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1
|
||||
|
||||
@@ -9,7 +9,7 @@ permissions: write-all
|
||||
|
||||
jobs:
|
||||
build_ee:
|
||||
runs-on: ubicloud
|
||||
runs-on: ubicloud-standard-4
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
|
||||
@@ -9,7 +9,7 @@ permissions: write-all
|
||||
|
||||
jobs:
|
||||
build_ee:
|
||||
runs-on: ubicloud
|
||||
runs-on: ubicloud-standard-4
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
|
||||
@@ -23,16 +23,16 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Deno
|
||||
uses: denoland/setup-deno@v2
|
||||
with:
|
||||
deno-version: v2.x
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: latest
|
||||
|
||||
- name: Generate Windmill client
|
||||
working-directory: cli
|
||||
run: ./gen_wm_client.sh
|
||||
@@ -69,11 +69,6 @@ jobs:
|
||||
cache: true
|
||||
cache-workspaces: backend
|
||||
|
||||
- name: Setup Deno
|
||||
uses: denoland/setup-deno@v2
|
||||
with:
|
||||
deno-version: v2.x
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
@@ -90,6 +85,10 @@ jobs:
|
||||
- name: Symlink Node to /usr/bin/node
|
||||
run: sudo ln -sf $(which node) /usr/bin/node
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: cli
|
||||
run: bun install
|
||||
|
||||
- name: Generate Windmill clients
|
||||
working-directory: cli
|
||||
run: |
|
||||
@@ -101,12 +100,10 @@ jobs:
|
||||
env:
|
||||
DATABASE_URL: postgres://postgres:changeme@localhost:5432
|
||||
CI_MINIMAL_FEATURES: "true"
|
||||
run: |
|
||||
deno test --no-check --allow-all test/ \
|
||||
--ignore=test/cargo_backend_example.test.ts
|
||||
run: bun test --timeout 120000 test/
|
||||
|
||||
test-windows:
|
||||
runs-on: windows-latest
|
||||
runs-on: blacksmith-16vcpu-windows-2025
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -126,11 +123,6 @@ jobs:
|
||||
cache: true
|
||||
cache-workspaces: backend
|
||||
|
||||
- name: Setup Deno
|
||||
uses: denoland/setup-deno@v2
|
||||
with:
|
||||
deno-version: v2.x
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
@@ -150,6 +142,10 @@ jobs:
|
||||
echo "BUN_PATH=$bunPath" >> $env:GITHUB_OUTPUT
|
||||
echo "NODE_BIN_PATH=$nodePath" >> $env:GITHUB_OUTPUT
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: cli
|
||||
run: bun install
|
||||
|
||||
- name: Generate Windmill clients
|
||||
working-directory: cli
|
||||
shell: bash
|
||||
@@ -165,9 +161,12 @@ jobs:
|
||||
CI_MINIMAL_FEATURES: "true"
|
||||
BUN_PATH: ${{ steps.runtime-paths.outputs.BUN_PATH }}
|
||||
NODE_BIN_PATH: ${{ steps.runtime-paths.outputs.NODE_BIN_PATH }}
|
||||
run: |
|
||||
deno test --no-check --allow-all test/ `
|
||||
--ignore=test/cargo_backend_example.test.ts
|
||||
run: bun test --timeout 120000 test/
|
||||
|
||||
- name: Keep runner alive for SSH debug
|
||||
if: failure()
|
||||
shell: pwsh
|
||||
run: Start-Sleep -Seconds 3600
|
||||
|
||||
# Combined summary job for branch protection
|
||||
test-summary:
|
||||
|
||||
@@ -6,6 +6,12 @@ on:
|
||||
- opened
|
||||
- ready_for_review
|
||||
- closed
|
||||
issue_comment:
|
||||
types:
|
||||
- created
|
||||
pull_request_review_comment:
|
||||
types:
|
||||
- created
|
||||
|
||||
jobs:
|
||||
notify_discord_when_pr_opened:
|
||||
@@ -33,3 +39,38 @@ jobs:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
secrets:
|
||||
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }}
|
||||
|
||||
notify_discord_on_comment:
|
||||
if: >
|
||||
github.event_name == 'issue_comment'
|
||||
&& github.event.issue.pull_request
|
||||
&& github.event.comment.user.login != 'cloudflare-workers-and-pages[bot]'
|
||||
&& github.event.comment.user.login != 'ellipsis-dev[bot]'
|
||||
uses: ./.github/workflows/shareable-discord-notification.yml
|
||||
with:
|
||||
PR_STATUS: "comment"
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
COMMENT_BODY: ${{ github.event.comment.body }}
|
||||
COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
|
||||
COMMENT_URL: ${{ github.event.comment.html_url }}
|
||||
DISCORD_CHANNEL_ID: "1372204995868491786"
|
||||
DISCORD_GUILD_ID: "930051556043276338"
|
||||
secrets:
|
||||
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }}
|
||||
|
||||
notify_discord_on_review_comment:
|
||||
if: >
|
||||
github.event_name == 'pull_request_review_comment'
|
||||
&& github.event.comment.user.login != 'cloudflare-workers-and-pages[bot]'
|
||||
&& github.event.comment.user.login != 'ellipsis-dev[bot]'
|
||||
uses: ./.github/workflows/shareable-discord-notification.yml
|
||||
with:
|
||||
PR_STATUS: "comment"
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
COMMENT_BODY: ${{ github.event.comment.body }}
|
||||
COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
|
||||
COMMENT_URL: ${{ github.event.comment.html_url }}
|
||||
DISCORD_CHANNEL_ID: "1372204995868491786"
|
||||
DISCORD_GUILD_ID: "930051556043276338"
|
||||
secrets:
|
||||
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }}
|
||||
|
||||
@@ -25,9 +25,9 @@ jobs:
|
||||
with:
|
||||
node-version: "20.x"
|
||||
registry-url: "https://registry.npmjs.org"
|
||||
- uses: denoland/setup-deno@v2
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
deno-version: v2.x
|
||||
bun-version: latest
|
||||
- run: cd cli && ./build.sh && cd npm && npm publish
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
|
||||
@@ -24,9 +24,22 @@ on:
|
||||
DISCORD_GUILD_ID:
|
||||
description: "The Discord guild ID"
|
||||
type: string
|
||||
COMMENT_BODY:
|
||||
description: "The comment body"
|
||||
type: string
|
||||
default: ""
|
||||
COMMENT_AUTHOR:
|
||||
description: "The comment author"
|
||||
type: string
|
||||
default: ""
|
||||
COMMENT_URL:
|
||||
description: "The comment URL"
|
||||
type: string
|
||||
default: ""
|
||||
secrets:
|
||||
DISCORD_WEBHOOK_URL:
|
||||
description: "Discord Webhook URL"
|
||||
required: false
|
||||
DISCORD_BOT_TOKEN:
|
||||
description: "Discord Bot Token"
|
||||
|
||||
@@ -117,3 +130,54 @@ jobs:
|
||||
curl -X PUT \
|
||||
-H "Authorization: Bot $BOT_TOKEN" \
|
||||
"https://discord.com/api/v10/channels/$thread_id/messages/$message_id/reactions/%E2%9C%85/@me"
|
||||
|
||||
post_comment:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ inputs.PR_STATUS == 'comment' }}
|
||||
steps:
|
||||
- name: Post comment to Discord thread
|
||||
env:
|
||||
BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
|
||||
CHANNEL_ID: ${{ inputs.DISCORD_CHANNEL_ID }}
|
||||
GUILD_ID: ${{ inputs.DISCORD_GUILD_ID }}
|
||||
PR_NUMBER: ${{ inputs.PR_NUMBER }}
|
||||
COMMENT_BODY: ${{ inputs.COMMENT_BODY }}
|
||||
COMMENT_AUTHOR: ${{ inputs.COMMENT_AUTHOR }}
|
||||
COMMENT_URL: ${{ inputs.COMMENT_URL }}
|
||||
run: |
|
||||
# 1) Find the thread by PR number
|
||||
threads=$(curl -s -H "Authorization: Bot $BOT_TOKEN" \
|
||||
"https://discord.com/api/v10/guilds/${GUILD_ID}/threads/active")
|
||||
thread_id=$(echo "$threads" | jq -r \
|
||||
--arg cid "$CHANNEL_ID" \
|
||||
--arg pref "#${PR_NUMBER}:" \
|
||||
'.threads[] | select(.parent_id == $cid and (.name | startswith($pref))) | .id')
|
||||
|
||||
if [ -z "$thread_id" ]; then
|
||||
echo "Thread not found for PR #${PR_NUMBER}, skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 2) Truncate comment body to fit Discord's 2000 char limit
|
||||
# Reserve space for the author line + link (~100 chars)
|
||||
max_body=1800
|
||||
if [ ${#COMMENT_BODY} -gt $max_body ]; then
|
||||
# For bot comments, show the tail (conclusions/code tend to be at the end)
|
||||
if [[ "$COMMENT_AUTHOR" == *"[bot]"* ]] || [[ "$COMMENT_AUTHOR" == *"-bot"* ]]; then
|
||||
truncated_body="...${COMMENT_BODY: -$max_body}"
|
||||
else
|
||||
truncated_body="${COMMENT_BODY:0:$max_body}..."
|
||||
fi
|
||||
else
|
||||
truncated_body="$COMMENT_BODY"
|
||||
fi
|
||||
|
||||
# 3) Post the comment to the thread
|
||||
message=$(printf '**%s** [commented](%s):\n%s' "$COMMENT_AUTHOR" "$COMMENT_URL" "$truncated_body")
|
||||
payload=$(jq -n --arg content "$message" '{content: $content, flags: 4, allowed_mentions: {parse: []}}')
|
||||
|
||||
curl -s -X POST \
|
||||
-H "Authorization: Bot $BOT_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$payload" \
|
||||
"https://discord.com/api/v10/channels/${thread_id}/messages"
|
||||
|
||||
@@ -14,9 +14,13 @@ backend/.minio-data
|
||||
!.aiderignore
|
||||
rust-client/Cargo.toml
|
||||
|
||||
# Worktree-generated port isolation
|
||||
.env.local
|
||||
|
||||
# Symlinked cache directories (for git worktrees)
|
||||
backend/target
|
||||
frontend/node_modules
|
||||
typescript-client/node_modules
|
||||
frontend/.svelte-kit
|
||||
backend/chrome_profiler.json
|
||||
.fast-check/
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
main_branch: main
|
||||
|
||||
merge_strategy: rebase
|
||||
# worktree_dir: .worktrees
|
||||
|
||||
worktree_naming: basename
|
||||
|
||||
worktree_prefix: ""
|
||||
|
||||
# Default: "wm-"
|
||||
window_prefix: "wm-"
|
||||
|
||||
auto_name:
|
||||
model: "claude-sonnet-4.6"
|
||||
system_prompt: |
|
||||
Generate a concise git branch name based on the task description.
|
||||
|
||||
Rules:
|
||||
- Use kebab-case (lowercase with hyphens)
|
||||
- Keep it short: 1-3 words, max 4 if necessary
|
||||
- Focus on the core task/feature, not implementation details
|
||||
- No prefixes like feat/, fix/, chore/
|
||||
|
||||
Examples of good branch names:
|
||||
- "Add dark mode toggle" → dark-mode
|
||||
- "Fix the search results not showing" → fix-search
|
||||
- "Refactor the authentication module" → auth-refactor
|
||||
- "Add CSV export to reports" → export-csv
|
||||
- "Shell completion is broken" → shell-completion
|
||||
|
||||
Output ONLY the branch name, nothing else.
|
||||
background: true
|
||||
|
||||
|
||||
# Commands to run in new worktree before tmux window opens.
|
||||
# These block window creation - use for short tasks only.
|
||||
# Use "<global>" to inherit from global config.
|
||||
# Set to empty list to disable: `post_create: []`
|
||||
# post_create:
|
||||
# - "<global>"
|
||||
# - mise use
|
||||
post_create:
|
||||
- ./scripts/worktree-env
|
||||
|
||||
pre_remove:
|
||||
- ./scripts/worktree-cleanup
|
||||
|
||||
panes:
|
||||
- command: <agent>
|
||||
focus: true
|
||||
- command: 'ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/backend" && PORT=${BACKEND_PORT:-8000} cargo watch -x run'
|
||||
split: horizontal
|
||||
- command: 'ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/frontend" && npm install && npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000}'
|
||||
split: vertical
|
||||
|
||||
files:
|
||||
copy:
|
||||
- backend/.env
|
||||
- scripts/
|
||||
|
||||
sandbox:
|
||||
enabled: false
|
||||
toolchain: off
|
||||
@@ -1,5 +1,71 @@
|
||||
# Changelog
|
||||
|
||||
## [1.642.0](https://github.com/windmill-labs/windmill/compare/v1.641.0...v1.642.0) (2026-02-22)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **cli:** add consistent get/list/new subcommands for all item types ([#8047](https://github.com/windmill-labs/windmill/issues/8047)) ([4fedfdf](https://github.com/windmill-labs/windmill/commit/4fedfdfd11aa8ca7fff6f7aed5ae2b313888f878))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* make WM_FLOW_PATH available in flow step previews ([#8042](https://github.com/windmill-labs/windmill/issues/8042)) ([a91c532](https://github.com/windmill-labs/windmill/commit/a91c532ecadce63cea965c497351fa1a6f39697a))
|
||||
* preserve debouncing settings for flows with preprocessors ([#8043](https://github.com/windmill-labs/windmill/issues/8043)) ([a00927b](https://github.com/windmill-labs/windmill/commit/a00927b3008a2d953fde1d461723a3c92f375eb4))
|
||||
|
||||
## [1.641.0](https://github.com/windmill-labs/windmill/compare/v1.640.0...v1.641.0) (2026-02-21)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add .npmrc support for private npm registries ([#8039](https://github.com/windmill-labs/windmill/issues/8039)) ([9eb1531](https://github.com/windmill-labs/windmill/commit/9eb15312f663aa6d700e8ac562d7b5c75c2221f7))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* add created_by ownership check to update/delete saved inputs ([#8038](https://github.com/windmill-labs/windmill/issues/8038)) ([e8a13ed](https://github.com/windmill-labs/windmill/commit/e8a13edde7c0ba2ef80344ab7c7288e7bb2eb6b5))
|
||||
* run substitute_ee_code.sh after creating EE worktree ([b330f38](https://github.com/windmill-labs/windmill/commit/b330f388894ecd9cc6b64297420ac6f032d32f72))
|
||||
* tag bunnative dependency jobs as bun instead of nativets ([#8045](https://github.com/windmill-labs/windmill/issues/8045)) ([fd5ebc2](https://github.com/windmill-labs/windmill/commit/fd5ebc2fda589c022074c3bb4dcdb447c7f86cf0))
|
||||
|
||||
## [1.640.0](https://github.com/windmill-labs/windmill/compare/v1.639.0...v1.640.0) (2026-02-20)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add windmill-ee-private worktree support to workmux ([#8034](https://github.com/windmill-labs/windmill/issues/8034)) ([9f3dd0b](https://github.com/windmill-labs/windmill/commit/9f3dd0bf2b2ba7c622093c54b7b6b5e7ebb26b74))
|
||||
* **cli:** add --locks-required flag to wmill lint and sync push ([#8026](https://github.com/windmill-labs/windmill/issues/8026)) ([4abe589](https://github.com/windmill-labs/windmill/commit/4abe58939787f375ccfef5b2dbcfbd7e86cff076))
|
||||
* dedicated nativets ([#8021](https://github.com/windmill-labs/windmill/issues/8021)) ([37c9acb](https://github.com/windmill-labs/windmill/commit/37c9acb232c64c98ecfb64754f5b69b31047c625))
|
||||
* Support column detection on S3 objects in DuckDB ([#8018](https://github.com/windmill-labs/windmill/issues/8018)) ([87f3de9](https://github.com/windmill-labs/windmill/commit/87f3de9ae5975c88b6748e297f84a539aec4c0ca))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fix DuckDB incorrect pg password encoding ([#8028](https://github.com/windmill-labs/windmill/issues/8028)) ([90b1a7a](https://github.com/windmill-labs/windmill/commit/90b1a7a531bce5621ea4de4792a8c9d3d3beec3d))
|
||||
* **frontend:** use completed_at instead of created_at for job history ([#8022](https://github.com/windmill-labs/windmill/issues/8022)) ([24d7921](https://github.com/windmill-labs/windmill/commit/24d7921bcf23543759719ffd2463959c627b61b8))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* lazy-load JSZip in RawAppEditorHeader ([#8012](https://github.com/windmill-labs/windmill/issues/8012)) ([a1ba10a](https://github.com/windmill-labs/windmill/commit/a1ba10a29e12ab5f553bd9aad74067cc5b3ead9e))
|
||||
|
||||
## [1.639.0](https://github.com/windmill-labs/windmill/compare/v1.638.4...v1.639.0) (2026-02-18)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* improve FolderPicker with edit icon pattern ([#7995](https://github.com/windmill-labs/windmill/issues/7995)) ([db8aa8a](https://github.com/windmill-labs/windmill/commit/db8aa8a0839b5729f0bb847e7a71766c7883ff36))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* default automate_username_creation to true when setting is missing ([#8006](https://github.com/windmill-labs/windmill/issues/8006)) ([d2d08f8](https://github.com/windmill-labs/windmill/commit/d2d08f8817e6e7818eb4b6f092e66ae039f0c756))
|
||||
* handle raw app folder deletion in sync push without yaml parse error ([#7994](https://github.com/windmill-labs/windmill/issues/7994)) ([f6d99dd](https://github.com/windmill-labs/windmill/commit/f6d99dd18c06a7f5aea93122276dd68c45772b43))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* **cli:** skip relock more accurate ([#7993](https://github.com/windmill-labs/windmill/issues/7993)) ([cd4151a](https://github.com/windmill-labs/windmill/commit/cd4151a84b2c1e0f2e616079091d0429bf469f4e))
|
||||
|
||||
## [1.638.4](https://github.com/windmill-labs/windmill/compare/v1.638.3...v1.638.4) (2026-02-17)
|
||||
|
||||
|
||||
|
||||
@@ -258,6 +258,10 @@ COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno
|
||||
|
||||
COPY --from=oven/bun:1.3.8 /usr/local/bin/bun /usr/bin/bun
|
||||
|
||||
# Install windmill CLI
|
||||
RUN bun install -g windmill-cli \
|
||||
&& ln -s $(bun pm bin -g)/wmill /usr/bin/wmill
|
||||
|
||||
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
|
||||
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
|
||||
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
# Windmill Development with workmux
|
||||
|
||||
This guide covers the workmux-based development setup for Windmill. Each worktree gets its own tmux window with a Claude Code agent, a backend server (with auto-reload), and a frontend dev server — all on isolated ports.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- tmux
|
||||
- Rust toolchain (rustup)
|
||||
- Node.js + npm
|
||||
- PostgreSQL running locally (see `backend/.env`)
|
||||
|
||||
## Installation
|
||||
|
||||
### 1. Install workmux
|
||||
|
||||
```bash
|
||||
cargo install workmux
|
||||
```
|
||||
|
||||
### 2. Install the Claude Code plugin
|
||||
|
||||
```bash
|
||||
workmux claude install
|
||||
```
|
||||
|
||||
This lets workmux manage Claude Code agents in worktree panes.
|
||||
|
||||
### 3. Install cargo-watch
|
||||
|
||||
Used for auto-recompiling the backend on file changes:
|
||||
|
||||
```bash
|
||||
cargo install cargo-watch
|
||||
```
|
||||
|
||||
### 4. Install llm CLI (required for auto branch naming)
|
||||
|
||||
workmux uses the `llm` CLI to automatically generate branch names from prompts. Install it with:
|
||||
|
||||
```bash
|
||||
uv tool install llm
|
||||
llm install llm-anthropic
|
||||
```
|
||||
|
||||
Then set your Anthropic API key:
|
||||
|
||||
```bash
|
||||
llm keys set anthropic
|
||||
# paste your API key when prompted
|
||||
```
|
||||
|
||||
### 5. Recommended: shell alias and autocomplete
|
||||
|
||||
Set up a `wm` alias for convenience:
|
||||
|
||||
```bash
|
||||
# Add to your ~/.zshrc
|
||||
alias wm="workmux"
|
||||
```
|
||||
|
||||
Setting up zsh autocomplete is also recommended — see the [workmux docs](https://github.com/rubenfiszel/workmux) for instructions.
|
||||
|
||||
## Port Slot System
|
||||
|
||||
Each worktree is assigned a **slot** that determines its ports:
|
||||
|
||||
| Slot | Backend | Frontend |
|
||||
|------|---------|----------|
|
||||
| 0 | 8000 | 3000 |
|
||||
| 1 | 8010 | 3010 |
|
||||
| 2 | 8020 | 3020 |
|
||||
| 3 | 8030 | 3030 |
|
||||
| ... | ... | ... |
|
||||
|
||||
- **Slot 0** is reserved for the main worktree (default `cargo run` / `npm run dev`).
|
||||
- Without `WM_SLOT`, the script auto-assigns the first available slot (starting from 1) and prints it.
|
||||
- With `WM_SLOT=N`, it uses that slot and errors if the ports are taken.
|
||||
|
||||
## SSH Port Forwarding
|
||||
|
||||
If you develop over SSH, add this to `~/.ssh/config` on your **local machine** to pre-configure tunnels for each slot:
|
||||
|
||||
```
|
||||
Host windmill-dev
|
||||
HostName <remote-ip>
|
||||
User <username>
|
||||
# Slot 0 (main worktree)
|
||||
LocalForward 8000 localhost:8000
|
||||
LocalForward 3000 localhost:3000
|
||||
# Slot 1
|
||||
LocalForward 8010 localhost:8010
|
||||
LocalForward 3010 localhost:3010
|
||||
# Slot 2
|
||||
LocalForward 8020 localhost:8020
|
||||
LocalForward 3020 localhost:3020
|
||||
# Slot 3
|
||||
LocalForward 8030 localhost:8030
|
||||
LocalForward 3030 localhost:3030
|
||||
```
|
||||
|
||||
Then connect once and all tunnels are active:
|
||||
|
||||
```bash
|
||||
ssh windmill-dev
|
||||
```
|
||||
|
||||
Access the frontend at `http://localhost:<frontend-port>` in your local browser.
|
||||
|
||||
## Quickstart
|
||||
|
||||
```bash
|
||||
# Create a new worktree (auto-assigns slot, prints ports)
|
||||
workmux add my-feature
|
||||
|
||||
# Or with an explicit slot
|
||||
WM_SLOT=2 workmux add my-feature
|
||||
|
||||
# Create a worktree and immediately send a prompt to the agent
|
||||
workmux add -A -p "fix the login bug in auth.rs"
|
||||
```
|
||||
|
||||
The `add` command creates the worktree but does **not** open it. To open the tmux window and start working:
|
||||
|
||||
```bash
|
||||
workmux open my-feature
|
||||
```
|
||||
|
||||
This will open a tmux window with three panes:
|
||||
|
||||
- **Claude Code agent** (focused)
|
||||
- **Backend**: `cargo watch -x run` on the assigned port (auto-reloads on save)
|
||||
- **Frontend**: `npm run dev` proxying to the backend
|
||||
|
||||
When using `-A` with `add`, the worktree is created and opened automatically, and the prompt is sent to the agent right away.
|
||||
|
||||
Check which ports were assigned:
|
||||
|
||||
```bash
|
||||
cat <worktree-path>/.env.local
|
||||
```
|
||||
|
||||
### Sending work to the agent
|
||||
|
||||
```bash
|
||||
# Send a prompt to the agent in a worktree
|
||||
workmux send my-feature "fix the login bug in auth.rs"
|
||||
|
||||
# Check agent status
|
||||
workmux status
|
||||
```
|
||||
|
||||
### Merging and cleaning up
|
||||
|
||||
We never merge worktrees directly — always create a PR on GitHub and let it be merged there. Once the PR is merged, clean up the worktree:
|
||||
|
||||
```bash
|
||||
# Close the tmux window but keep the worktree
|
||||
workmux close my-feature
|
||||
|
||||
# After your PR is merged, remove the worktree, branch, and tmux window
|
||||
workmux rm my-feature
|
||||
```
|
||||
|
||||
> **Note**: Do not use `workmux merge`. Always go through a PR to get your changes into main. You can ask the Claude Code agent in the worktree to create the PR for you.
|
||||
|
||||
## Configuration
|
||||
|
||||
The setup is defined in `.workmux.yaml` at the repo root. Key sections:
|
||||
|
||||
- **`post_create`**: Runs `scripts/worktree-env` to generate `.env.local` with port assignments
|
||||
- **`panes`**: Defines the tmux layout (agent, backend, frontend)
|
||||
- **`files.copy`**: Copies `backend/.env` and `scripts/` into each worktree
|
||||
- **`files.symlink`**: Symlinks `node_modules` and `.svelte-kit` to avoid reinstalling per worktree
|
||||
|
||||
## Login
|
||||
|
||||
Default credentials: `admin@windmill.dev` / `changeme`
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag)\n SELECT unnest($1::uuid[]), 'test-workspace', now(), 'flow'",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "0681b850c033619e1b9498376263681f875a5aba22170ca50ec8b578f7fa478b"
|
||||
}
|
||||
+2
-1
@@ -43,7 +43,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n WITH completed AS (\n INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result,\n flow_status, workflow_as_code_status, status, worker)\n SELECT\n q.workspace_id, q.id, q.started_at,\n (EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000,\n CASE WHEN q.running\n THEN $3::text::jsonb\n ELSE $4::text::jsonb\n END,\n s.flow_status,\n s.workflow_as_code_status,\n 'skipped'::job_status,\n q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_status s ON s.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = EXCLUDED.result\n RETURNING 1 AS x\n ), _deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n ), _logged AS (\n INSERT INTO job_logs (logs, job_id, workspace_id)\n VALUES ($5, $1, $2)\n ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs)\n )\n SELECT x FROM completed\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "x",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Varchar",
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346"
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT result::text FROM v2_job_completed WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "result",
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "18b6262a60400f2b58ab26615466c23b4c1a7805c66b70b0fcfb7d33b122a7bf"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag)\n VALUES ($1, $2, now(), 'flow')",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "1af6885dbc5055281acb82b3e57f7dba2e4b04d9535058fab695660a14bf8890"
|
||||
}
|
||||
-65
@@ -1,65 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT client, refresh_token, grant_type, cc_client_id, cc_client_secret, cc_token_url, mcp_server_url, is_workspace_integration FROM account WHERE workspace_id = $1 AND id = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "client",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "refresh_token",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "grant_type",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "cc_client_id",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "cc_client_secret",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "cc_token_url",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "mcp_server_url",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "is_workspace_integration",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int4"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "1ba2e23d4ba816048ec1e88af9e342867fc0443cabea16d111afa2b91d3fe03b"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job (id, kind, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, runnable_path)\n VALUES ($1, 'flow', 'flow', 'test-user', 'u/test-user', 'test@windmill.dev', $2, $3)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Varchar",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "27f70ebe788cca2e88732d8bf978883037bebca4cf75ba459858e4fb197f940b"
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "debounce_batch",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "2a95f18e80c55a7e8178a4bd2b781d41fa47efd4da5bb9bc2d72b9aa1e33617f"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job (id, kind, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, runnable_path)\n VALUES ($1, 'flow', 'flow', 'test-user', 'u/test-user', 'test@windmill.dev', 'ws2', 'f/test/flow')",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "4010328a9f1611064f497726b69c08625a55a4dab25c3d9b5ece07e44d14915b"
|
||||
}
|
||||
-35
@@ -1,35 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n INSERT INTO debounce_key (job_id, key)\n VALUES ($1, $2)\n ON CONFLICT (key)\n DO UPDATE SET\n previous_job_id = debounce_key.job_id,\n job_id = EXCLUDED.job_id, -- replace current job with new one \n debounced_times = debounce_key.debounced_times + 1 -- evaluated only if conflict,\n -- conflict means there is already existing value,\n -- which means overriding it will also imply adding new entry to v2_job_debounce_batch and thus debouncing the job\n -- so the counter should be incremented\n RETURNING\n debounced_times,\n first_started_at,\n previous_job_id AS job_id_to_debounce\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "debounced_times",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "first_started_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "job_id_to_debounce",
|
||||
"type_info": "Uuid"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "454ace9ce391725ef4f4c129cd66e4c12a5c40f512b70551958178c8b4d6c183"
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH ids AS (\n SELECT id as job_id FROM v2_job_debounce_batch WHERE debounce_batch = (\n SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1\n )\n ) SELECT args->>'items' FROM ids LEFT JOIN v2_job ON v2_job.id = ids.job_id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "?column?",
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "48536968f4173715d4ef8293683c2a3eb4bd22fbe18c34890a3dc4e96e4e6133"
|
||||
}
|
||||
+2
-1
@@ -42,7 +42,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -38,7 +38,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT v2_job_queue.id, v2_job.tag, v2_job_queue.scheduled_for, v2_job_queue.workspace_id FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id WHERE running = false AND scheduled_for < now() - ($1 || ' minutes')::interval",
|
||||
"query": "SELECT v2_job_queue.id, v2_job.tag, v2_job_queue.scheduled_for, v2_job_queue.workspace_id FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id WHERE running = false AND scheduled_for < now() - ($1 || ' minutes')::interval AND v2_job.trigger_kind IS DISTINCT FROM 'schedule'::job_trigger_kind",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -36,5 +36,5 @@
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "b45e17ad532a23b394226c9a5d7ab5a21e20202dbbf9c67831cc62eb067cd2ba"
|
||||
"hash": "53648c069749df45c0459d733b3e429af20c69c841fb0c3bceafe3ea6c3f5329"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag)\n SELECT unnest($1::uuid[]), 'test-workspace', now(), 'deno'",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "539d661500254e2e346490710f5772cb88a1ab6bbddd97a77e06644ac0f61762"
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH job_result AS (\n SELECT result\n FROM v2_job_completed\n WHERE id = $1\n ),\n updated_queue AS (\n UPDATE v2_job_queue\n SET running = false,\n tag = COALESCE($3, tag)\n WHERE id = $2\n )\n UPDATE v2_job\n SET\n tag = COALESCE($3, tag),\n concurrent_limit = COALESCE($4, concurrent_limit),\n concurrency_time_window_s = COALESCE($5, concurrency_time_window_s),\n args = COALESCE(\n CASE\n WHEN job_result.result IS NULL THEN NULL\n WHEN jsonb_typeof(job_result.result) = 'object'\n THEN job_result.result\n WHEN jsonb_typeof(job_result.result) = 'null'\n THEN NULL\n ELSE jsonb_build_object('value', job_result.result)\n END,\n '{}'::jsonb\n ),\n preprocessed = TRUE\n FROM job_result\n WHERE v2_job.id = $2;\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Uuid",
|
||||
"Varchar",
|
||||
"Int4",
|
||||
"Int4"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "5b8c1803f0ccead11517fbc8a9bdc0227dc3922217fa18f0b71ff0484d65838c"
|
||||
}
|
||||
+2
-1
@@ -77,7 +77,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE debounce_key SET first_started_at = now() - interval '20 seconds' WHERE key = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "66342c32f7ae0238803cb1896d9f23a74b64573f77dd32189a25b6e8369f147b"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job (id, kind, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, runnable_path)\n SELECT unnest($1::uuid[]), 'flow', 'flow', 'test-user', 'u/test-user', 'test@windmill.dev', 'test-workspace', 'f/test/flow'",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "66faba2137791e0cb1353545c06f9f7c23a1559e7a761db7c2195736b8b30709"
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH job_result AS (\n SELECT result\n FROM v2_job_completed\n WHERE id = $1\n ),\n updated_queue AS (\n UPDATE v2_job_queue\n SET running = false,\n tag = COALESCE($3, tag),\n scheduled_for = COALESCE($6, scheduled_for)\n WHERE id = $2\n )\n UPDATE v2_job\n SET\n tag = COALESCE($3, tag),\n concurrent_limit = COALESCE($4, concurrent_limit),\n concurrency_time_window_s = COALESCE($5, concurrency_time_window_s),\n args = COALESCE(\n CASE\n WHEN job_result.result IS NULL THEN NULL\n WHEN jsonb_typeof(job_result.result) = 'object'\n THEN job_result.result\n WHEN jsonb_typeof(job_result.result) = 'null'\n THEN NULL\n ELSE jsonb_build_object('value', job_result.result)\n END,\n '{}'::jsonb\n ),\n preprocessed = TRUE\n FROM job_result\n WHERE v2_job.id = $2;\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Uuid",
|
||||
"Varchar",
|
||||
"Int4",
|
||||
"Int4",
|
||||
"Timestamptz"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "79b437ad31ddab94310989b8fb6a1c130b9be1ab4b6a100fffffd687677b9c92"
|
||||
}
|
||||
+2
-1
@@ -44,7 +44,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT logs as \"logs!\" FROM job_logs WHERE job_id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "logs!",
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "7ca599330c9913c7e66b27e2ffcfa18d53cbdd16e179749f0aea7980a901b23c"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag)\n VALUES ($1, $2, now(), 'deno')",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "7ca7dabfe360845a5b57552b0d02267d5dbbc488bc7ab990c0bda1594bf5ef3a"
|
||||
}
|
||||
+2
-1
@@ -42,7 +42,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job_debounce_batch (id, debounce_batch)\n -- if it the first one, nextval will be evaluated, otherwise take from the job we will debounce\n SELECT\n $2,\n COALESCE(\n (\n SELECT debounce_batch\n FROM v2_job_debounce_batch\n WHERE id = $1\n LIMIT 1\n ), -- maybe use current batch\n nextval('debounce_batch_seq')\n )\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "8360ab72d60f07dde6ecae599e6531b5b86862029ab51fdbdd44ec16239108e2"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO workspace_settings (workspace_id) VALUES ('ws2')",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "8cf5af21cde4e4de45f995efa2a9b56ce20c26869ca78d7e17b3504b92ae85b1"
|
||||
}
|
||||
+2
-1
@@ -102,7 +102,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT job_id, previous_job_id, debounced_times FROM debounce_key WHERE key = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "job_id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "previous_job_id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "debounced_times",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
true,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "8f442110817244aa9533b014aa3d74a6582937dfe4759932b63b8e531984008e"
|
||||
}
|
||||
+2
-1
@@ -32,7 +32,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n WITH dk AS (\n INSERT INTO debounce_key (job_id, key)\n VALUES ($1, $2)\n ON CONFLICT (key)\n DO UPDATE SET\n previous_job_id = debounce_key.job_id,\n job_id = EXCLUDED.job_id,\n debounced_times = debounce_key.debounced_times + 1\n RETURNING\n debounced_times,\n first_started_at,\n previous_job_id AS job_id_to_debounce\n ), _batch AS (\n INSERT INTO v2_job_debounce_batch (id, debounce_batch)\n SELECT\n $1,\n COALESCE(\n (SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = dk.job_id_to_debounce LIMIT 1),\n nextval('debounce_batch_seq')\n )\n FROM dk\n )\n SELECT debounced_times, first_started_at, job_id_to_debounce FROM dk\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "debounced_times",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "first_started_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "job_id_to_debounce",
|
||||
"type_info": "Uuid"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "98033aae3182bde22d5b2ff08ef6e8a4f8f3a9bf04238b33e9caf46836df73d9"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job (id, kind, tag, created_by, permissioned_as, permissioned_as_email, workspace_id)\n VALUES ($1, 'noop', 'deno', 'test-user', 'u/test-user', 'test@windmill.dev', $2)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "9f50ec7681a1fcd11cb452c7aba7e8897e49a4b6affa4e9976680336b6bd3115"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO workspace (id, name, owner) VALUES ('ws2', 'Workspace 2', 'test-user')",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "a057ff9f5998a162ae6de05f6127b7eefc826af7bf1bb89fd758f7c03c881033"
|
||||
}
|
||||
+2
-1
@@ -72,7 +72,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -77,7 +77,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job_runtime (id) VALUES ($1)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "a68754521bf751450602f04dd4243199a18885e1739a5a0e7f6100eab6f3c803"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE v2_job_queue SET runnable_settings_handle = $1 WHERE id = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int8",
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "abb56f78aa39c6b6ae8b0ccb7b724c1f80d717e7c9d76b287da63cb5ee8e8b25"
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT 1 as x FROM v2_job_completed WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "x",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "adb98040c8039e5cc27fe0579941f723b38d1784bd2f1b16d8724f1f1612dcbf"
|
||||
}
|
||||
+2
-1
@@ -102,7 +102,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -72,7 +72,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job_runtime (id) SELECT unnest($1::uuid[])",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46"
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT 1 as x FROM v2_job_queue WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "x",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "b795dc228f93c8b9bedb4a3e7467d941819a7202214e0634580bdc2ec30f0b70"
|
||||
}
|
||||
+2
-1
@@ -41,7 +41,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag) VALUES ($1, 'ws2', now(), 'flow')",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "c1a1ae759ebb84fde3e6d2727991b2a1fcc73e520fee79653bb960dc00c3e2db"
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n WITH dk AS (\n INSERT INTO debounce_key (job_id, key)\n VALUES ($1, $2)\n ON CONFLICT (key)\n DO UPDATE SET\n previous_job_id = debounce_key.job_id,\n job_id = EXCLUDED.job_id,\n debounced_times = debounce_key.debounced_times + 1\n RETURNING\n debounced_times,\n first_started_at,\n previous_job_id AS job_id_to_debounce\n ), _batch AS (\n INSERT INTO v2_job_debounce_batch (id, debounce_batch)\n SELECT\n $1,\n COALESCE(\n (SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = dk.job_id_to_debounce LIMIT 1),\n nextval('debounce_batch_seq')\n )\n FROM dk\n )\n SELECT debounced_times, first_started_at, job_id_to_debounce FROM dk\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "debounced_times",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "first_started_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "job_id_to_debounce",
|
||||
"type_info": "Uuid"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "c2347460b73ae9d3167031c263032e97ebefb46be9e58bd3da9067748075311b"
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT COUNT(*) as \"count!\" FROM v2_job_queue WHERE id = ANY($1)",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "count!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "c63a1949247f1618f6b6acee9bf6b4d3081dfed2e6ff533dbff0bdfd52687cbb"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job (id, kind, tag, created_by, permissioned_as, permissioned_as_email, workspace_id)\n SELECT unnest($1::uuid[]), 'noop', 'deno', 'test-user', 'u/test-user', 'test@windmill.dev', 'test-workspace'",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "c6d963e5cefeea728414892df9f28a89f435fc0fb7e55f243b021200f33d2151"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n WITH _ AS (\n UPDATE debounce_key\n SET debounced_times = 0,\n first_started_at = now(),\n previous_job_id = NULL\n WHERE job_id = $1\n )\n UPDATE v2_job_debounce_batch\n SET debounce_batch = nextval('debounce_batch_seq')\n WHERE id = $1\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "c9530931f670eab1208c4a284a55afdc3fcbb0eb5f98fd63e2ec89442becbfaa"
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT COUNT(*) as \"count!\" FROM v2_job_completed WHERE id = ANY($1)",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "count!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "cc309de42a3b630bb83d1b2437633ef0b98ce5e5fba1f5c1dda3ddd874ff3a39"
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = ANY($1)",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "debounce_batch",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "ccfed494a8d89eb2c88d72738c341a4dd87701b0636eb9fa001cd1d9cbcd663b"
|
||||
}
|
||||
+2
-1
@@ -41,7 +41,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = ANY($1) ORDER BY debounce_batch",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "debounce_batch",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "d9400849888dd021b0504b93004ab5e76296ed437c210942363ba06845f9f963"
|
||||
}
|
||||
+2
-1
@@ -31,7 +31,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -37,7 +37,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -77,7 +77,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job (id, kind, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, runnable_path, args)\n VALUES ($1, 'flow', 'flow', 'test-user', 'u/test-user', 'test@windmill.dev', $2, $3, $4)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Jsonb"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "f8cec94b94098e752f7c71cbe5e9996410a07bacebc37ed21e0bedf1a33a8fdc"
|
||||
}
|
||||
+2
-1
@@ -32,7 +32,8 @@
|
||||
"aiagent",
|
||||
"unassigned_script",
|
||||
"unassigned_flow",
|
||||
"unassigned_singlestepflow"
|
||||
"unassigned_singlestepflow",
|
||||
"snapshotbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
panes:
|
||||
# Pane 1: Install dependencies, then start dev server
|
||||
- command: cargo run
|
||||
|
||||
# Pane 2: AI agent
|
||||
- command: <agent>
|
||||
split: horizontal
|
||||
focus: true
|
||||
+12
-1
@@ -44,11 +44,22 @@ Windmill uses a workspace-based architecture with multiple crates:
|
||||
## Enterprise Features
|
||||
|
||||
- Enterprise files use the `*_ee.rs` suffix
|
||||
- Enterprise source is in `windmill-ee-private` folder (sibling directory at `../../windmill-ee-private`), symlinked into each crate's `src/`
|
||||
- Enterprise source is in `windmill-ee-private` folder (sibling directory at `../../windmill-ee-private` or `~/windmill-ee-private`), symlinked into each crate's `src/`
|
||||
- The `_ee.rs` files are gitignored in the main repo — they are tracked only in the `windmill-ee-private` repo
|
||||
- You can and should modify `windmill-ee-private` directly when needed (e.g., when creating new crates that need EE code, mirror the package structure there)
|
||||
- Use feature flags: `#[cfg(feature = "enterprise")]`
|
||||
- Isolate enterprise code in separate modules
|
||||
|
||||
### EE PR Workflow (MUST DO when modifying `*_ee.rs` files)
|
||||
|
||||
When you modify any `*_ee.rs` file and create a PR on the windmill repo, you **MUST** also:
|
||||
|
||||
1. **Create a matching branch** in the `windmill-ee-private` repo (use the same branch name). If using worktrees, the EE worktree is at `~/windmill-ee-private__worktrees/<branch-name>/`
|
||||
2. **Commit and push** the `_ee.rs` changes in that branch
|
||||
3. **Create a PR** on `windmill-ee-private` with a link to the companion windmill PR
|
||||
4. **Update `ee-repo-ref.txt`**: Run `bash write_latest_ee_ref.sh` from `backend/` to write the latest EE commit hash. **Important**: the script may fall back to `~/windmill-ee-private` (main branch) instead of the worktree — verify it wrote the correct commit hash from your branch, not from main. If wrong, manually write the correct hash.
|
||||
5. **Commit `ee-repo-ref.txt`** in the windmill repo so CI picks up the correct EE ref
|
||||
|
||||
## Code Validation (MUST DO)
|
||||
|
||||
After making backend changes, you MUST run `cargo check` and fix all errors and warnings before considering the work done.
|
||||
|
||||
Generated
+202
-200
File diff suppressed because it is too large
Load Diff
+5
-4
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "windmill"
|
||||
version = "1.638.4"
|
||||
version = "1.642.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
@@ -76,7 +76,7 @@ members = [
|
||||
exclude = ["./windmill-duckdb-ffi-internal"]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.638.4"
|
||||
version = "1.642.0"
|
||||
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
|
||||
edition = "2021"
|
||||
|
||||
@@ -103,7 +103,7 @@ enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmi
|
||||
local_reports = ["windmill-common/local_reports"]
|
||||
enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"]
|
||||
stripe = ["windmill-api/stripe"]
|
||||
benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"]
|
||||
benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark", "windmill-api-agent-workers?/benchmark"]
|
||||
embedding = ["windmill-api/embedding"]
|
||||
parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker/parquet"]
|
||||
prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus", "dep:prometheus"]
|
||||
@@ -254,6 +254,7 @@ axum.workspace = true
|
||||
serde.workspace = true
|
||||
windmill-api-client.workspace = true
|
||||
tempfile.workspace = true
|
||||
windmill-parser-ts.workspace = true
|
||||
rumqttc.workspace = true
|
||||
rdkafka.workspace = true
|
||||
async-nats.workspace = true
|
||||
@@ -479,7 +480,7 @@ bit-vec = "=0.6.3"
|
||||
mappable-rc = "^0"
|
||||
mysql_async = { version = "*", default-features = false, features = ["minimal", "default", "native-tls-tls", "rust_decimal"]}
|
||||
postgres-native-tls = "^0"
|
||||
native-tls = "^0"
|
||||
native-tls = ">=0.2, <0.2.17"
|
||||
# samael will break compilation on MacOS. Use this fork instead to make it work
|
||||
# samael = { git="https://github.com/njaremko/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = ["xmlsec"] }
|
||||
libxml = { version = "=0.3.3" }
|
||||
|
||||
@@ -1 +1 @@
|
||||
0eccae6a9a9ecde09816cd4d88ca9ab305659e4c
|
||||
0fede4b1086bc1456be9cc55b203228c979c5c5e
|
||||
|
||||
@@ -2,8 +2,8 @@ use std::collections::BTreeMap;
|
||||
|
||||
use sqlparser::{
|
||||
ast::{
|
||||
CopyTarget, Expr, ObjectName, ObjectNamePart, SelectItem, TableFactor, TableObject, Value,
|
||||
ValueWithSpan, Visit, Visitor,
|
||||
CopyTarget, Expr, FunctionArg, FunctionArgExpr, ObjectName, ObjectNamePart, SelectItem,
|
||||
TableFactor, TableObject, Value, ValueWithSpan, Visit, Visitor,
|
||||
},
|
||||
dialect::DuckDbDialect,
|
||||
parser::Parser,
|
||||
@@ -125,6 +125,72 @@ impl AssetCollector {
|
||||
Some(ParseAssetsResult { kind: *kind, access_type, path, columns: None })
|
||||
}
|
||||
|
||||
/// If `table_factor` is a string literal used directly as a table name (e.g. FROM 's3:///file.parquet'),
|
||||
/// return a `ParseAssetsResult` for it.
|
||||
fn get_s3_asset_from_str_literal_table(
|
||||
&self,
|
||||
table_factor: &TableFactor,
|
||||
) -> Option<ParseAssetsResult> {
|
||||
let name = match table_factor {
|
||||
TableFactor::Table { name, args: None, .. } => name,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let s3_str = get_str_lit_from_obj_name(name)?;
|
||||
let (kind, path) = parse_asset_syntax(s3_str, false)?;
|
||||
if kind != AssetKind::S3Object {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(ParseAssetsResult {
|
||||
kind,
|
||||
path: path.to_string(),
|
||||
access_type: Some(R),
|
||||
columns: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// If `table_factor` is a read function (read_parquet/read_csv/read_json) whose first
|
||||
/// positional argument is an S3 string literal, return a `ParseAssetsResult` for it.
|
||||
fn get_s3_asset_from_table_function(
|
||||
&self,
|
||||
table_factor: &TableFactor,
|
||||
) -> Option<ParseAssetsResult> {
|
||||
let (name, args) = match table_factor {
|
||||
TableFactor::Table { name, args: Some(args), .. } => (name, args),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let fname = get_trivial_obj_name(name)?;
|
||||
if !is_read_fn(fname) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let s3_str = args.args.first().and_then(|arg| match arg {
|
||||
FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(ValueWithSpan {
|
||||
value: Value::SingleQuotedString(s),
|
||||
..
|
||||
})))
|
||||
| FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(ValueWithSpan {
|
||||
value: Value::DoubleQuotedString(s),
|
||||
..
|
||||
}))) => Some(s.as_str()),
|
||||
_ => None,
|
||||
})?;
|
||||
|
||||
let (kind, path) = parse_asset_syntax(s3_str, false)?;
|
||||
if kind != AssetKind::S3Object {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(ParseAssetsResult {
|
||||
kind,
|
||||
path: path.to_string(),
|
||||
access_type: Some(R),
|
||||
columns: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn handle_string_literal(&mut self, s: &str) {
|
||||
// Check if the string matches our asset syntax patterns
|
||||
if let Some((kind, path)) = parse_asset_syntax(s, false) {
|
||||
@@ -190,13 +256,19 @@ impl AssetCollector {
|
||||
projection: &[SelectItem],
|
||||
from_tables: &[sqlparser::ast::TableWithJoins],
|
||||
) {
|
||||
// Check if this is a single-table SELECT (to avoid ambiguity)
|
||||
// Check if this is a single-table SELECT (to avoid ambiguity).
|
||||
// For S3 table functions (read_parquet/read_csv/read_json), detect the asset even
|
||||
// though args are present, since we know the file path from the string literal arg.
|
||||
let single_table = if from_tables.len() == 1 {
|
||||
if let TableFactor::Table { name, args, .. } = &from_tables[0].relation {
|
||||
if args.is_some() && args.as_ref().map_or(0, |a| a.args.len()) > 0 {
|
||||
return; // Skip table functions
|
||||
let relation = &from_tables[0].relation;
|
||||
if let TableFactor::Table { name, args, .. } = relation {
|
||||
let has_args = args.as_ref().map_or(false, |a| !a.args.is_empty());
|
||||
if has_args {
|
||||
self.get_s3_asset_from_table_function(relation)
|
||||
} else {
|
||||
self.get_associated_asset_from_obj_name(name, Some(R))
|
||||
.or_else(|| self.get_s3_asset_from_str_literal_table(relation))
|
||||
}
|
||||
self.get_associated_asset_from_obj_name(name, Some(R))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -204,26 +276,48 @@ impl AssetCollector {
|
||||
None
|
||||
};
|
||||
|
||||
// Build a map of table aliases/names to assets for multi-table queries
|
||||
// Build a map of table aliases/names to assets for multi-table queries.
|
||||
// For S3 table functions, only aliased references are unambiguous
|
||||
// (e.g. SELECT t.col1 FROM read_parquet('s3://...') AS t).
|
||||
let mut table_to_asset: BTreeMap<String, ParseAssetsResult> = BTreeMap::new();
|
||||
for table_with_joins in from_tables {
|
||||
if let TableFactor::Table { name, alias, args, .. } = &table_with_joins.relation {
|
||||
if args.is_some() && args.as_ref().map_or(0, |a| a.args.len()) > 0 {
|
||||
continue; // Skip table functions
|
||||
}
|
||||
if let Some(asset) = self.get_associated_asset_from_obj_name(name, Some(R)) {
|
||||
// Use alias if present, otherwise use the table name
|
||||
let table_key = if let Some(alias) = alias {
|
||||
alias.name.value.clone()
|
||||
let has_args = args.as_ref().map_or(false, |a| !a.args.is_empty());
|
||||
if has_args {
|
||||
// For table functions, only add to the alias map when an alias is present
|
||||
if let Some(alias) = alias {
|
||||
if let Some(asset) =
|
||||
self.get_s3_asset_from_table_function(&table_with_joins.relation)
|
||||
{
|
||||
table_to_asset.insert(alias.name.value.clone(), asset);
|
||||
}
|
||||
}
|
||||
} else if let Some(asset) = self
|
||||
.get_associated_asset_from_obj_name(name, Some(R))
|
||||
.or_else(|| {
|
||||
self.get_s3_asset_from_str_literal_table(&table_with_joins.relation)
|
||||
})
|
||||
{
|
||||
// For string literal S3 tables (e.g. FROM 's3:///file.parquet'), only add to
|
||||
// the alias map when an alias is present (to avoid false positives).
|
||||
// For regular named tables, use alias or table name as key.
|
||||
let is_str_literal = get_str_lit_from_obj_name(name).is_some();
|
||||
if is_str_literal {
|
||||
if let Some(alias) = alias {
|
||||
table_to_asset.insert(alias.name.value.clone(), asset);
|
||||
}
|
||||
} else {
|
||||
// For qualified names like "dl.table1", use just the last part
|
||||
name.0
|
||||
.last()
|
||||
.and_then(|id| id.as_ident())
|
||||
.map(|id| id.value.clone())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
table_to_asset.insert(table_key, asset);
|
||||
let table_key = if let Some(alias) = alias {
|
||||
alias.name.value.clone()
|
||||
} else {
|
||||
name.0
|
||||
.last()
|
||||
.and_then(|id| id.as_ident())
|
||||
.map(|id| id.value.clone())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
table_to_asset.insert(table_key, asset);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1271,4 +1365,163 @@ mod tests {
|
||||
assert_eq!(columns.get("age"), Some(&W)); // Only written
|
||||
assert_eq!(columns.get("id"), Some(&R)); // Only read
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sql_asset_parser_s3_single_table_column_detection() {
|
||||
let input = r#"
|
||||
SELECT col1, col2 FROM read_parquet('s3:///example_file.parquet');
|
||||
"#;
|
||||
let result = parse_assets(input).unwrap().assets;
|
||||
|
||||
assert_eq!(result.len(), 1);
|
||||
assert_eq!(result[0].kind, AssetKind::S3Object);
|
||||
assert_eq!(result[0].path, "/example_file.parquet");
|
||||
assert_eq!(result[0].access_type, Some(R));
|
||||
|
||||
let columns = result[0].columns.as_ref().expect("Should have columns");
|
||||
assert_eq!(columns.len(), 2);
|
||||
assert_eq!(columns.get("col1"), Some(&R));
|
||||
assert_eq!(columns.get("col2"), Some(&R));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sql_asset_parser_s3_single_table_column_with_alias() {
|
||||
let input = r#"
|
||||
SELECT col1 AS c1, col2 AS c2 FROM read_parquet('s3:///example_file.parquet');
|
||||
"#;
|
||||
let result = parse_assets(input).unwrap().assets;
|
||||
|
||||
assert_eq!(result.len(), 1);
|
||||
let columns = result[0].columns.as_ref().expect("Should have columns");
|
||||
assert_eq!(columns.get("col1"), Some(&R));
|
||||
assert_eq!(columns.get("col2"), Some(&R));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sql_asset_parser_s3_wildcard_no_columns() {
|
||||
let input = r#"
|
||||
SELECT * FROM read_parquet('s3:///example_file.parquet');
|
||||
"#;
|
||||
let result = parse_assets(input).unwrap().assets;
|
||||
|
||||
assert_eq!(result.len(), 1);
|
||||
assert_eq!(result[0].kind, AssetKind::S3Object);
|
||||
assert_eq!(result[0].path, "/example_file.parquet");
|
||||
assert!(result[0].columns.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sql_asset_parser_s3_table_alias_qualified_columns() {
|
||||
let input = r#"
|
||||
SELECT t.col1, t.col2 FROM read_parquet('s3:///example_file.parquet') AS t;
|
||||
"#;
|
||||
let result = parse_assets(input).unwrap().assets;
|
||||
|
||||
assert_eq!(result.len(), 1);
|
||||
assert_eq!(result[0].kind, AssetKind::S3Object);
|
||||
assert_eq!(result[0].path, "/example_file.parquet");
|
||||
|
||||
let columns = result[0].columns.as_ref().expect("Should have columns");
|
||||
assert_eq!(columns.get("col1"), Some(&R));
|
||||
assert_eq!(columns.get("col2"), Some(&R));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sql_asset_parser_s3_multi_table_aliased_columns() {
|
||||
let input = r#"
|
||||
SELECT t1.col1, t2.col2
|
||||
FROM read_parquet('s3:///file1.parquet') AS t1,
|
||||
read_csv('s3://bucket/file2.csv') AS t2;
|
||||
"#;
|
||||
let result = parse_assets(input).unwrap().assets;
|
||||
|
||||
assert_eq!(result.len(), 2);
|
||||
|
||||
assert!(result.iter().any(|a| {
|
||||
a.path == "/file1.parquet"
|
||||
&& a.columns.as_ref().map_or(false, |c| c.contains_key("col1"))
|
||||
}));
|
||||
assert!(result.iter().any(|a| {
|
||||
a.path == "bucket/file2.csv"
|
||||
&& a.columns.as_ref().map_or(false, |c| c.contains_key("col2"))
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sql_asset_parser_s3_multi_table_no_alias_no_columns() {
|
||||
// Without aliases, unqualified columns in a multi-table query are ambiguous
|
||||
let input = r#"
|
||||
SELECT col1, col2
|
||||
FROM read_parquet('s3:///file1.parquet'),
|
||||
read_parquet('s3:///file2.parquet');
|
||||
"#;
|
||||
let result = parse_assets(input).unwrap().assets;
|
||||
|
||||
// Table-level assets should still be detected, but no columns
|
||||
assert_eq!(result.iter().filter(|a| a.columns.is_some()).count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sql_asset_parser_s3_str_literal_table_column_detection() {
|
||||
// FROM 's3:///file.parquet' (string literal as table, no read_parquet wrapper)
|
||||
let input = r#"
|
||||
SELECT a,b,c FROM 's3:///test.parquet';
|
||||
"#;
|
||||
let result = parse_assets(input).unwrap().assets;
|
||||
|
||||
assert_eq!(result.len(), 1);
|
||||
assert_eq!(result[0].kind, AssetKind::S3Object);
|
||||
assert_eq!(result[0].path, "/test.parquet");
|
||||
assert_eq!(result[0].access_type, Some(R));
|
||||
|
||||
let columns = result[0].columns.as_ref().expect("Should have columns");
|
||||
assert_eq!(columns.len(), 3);
|
||||
assert_eq!(columns.get("a"), Some(&R));
|
||||
assert_eq!(columns.get("b"), Some(&R));
|
||||
assert_eq!(columns.get("c"), Some(&R));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sql_asset_parser_s3_str_literal_table_with_alias_columns() {
|
||||
let input = r#"
|
||||
SELECT t.col1, t.col2 FROM 's3://bucket/file.parquet' AS t;
|
||||
"#;
|
||||
let result = parse_assets(input).unwrap().assets;
|
||||
|
||||
assert_eq!(result.len(), 1);
|
||||
assert_eq!(result[0].kind, AssetKind::S3Object);
|
||||
assert_eq!(result[0].path, "bucket/file.parquet");
|
||||
|
||||
let columns = result[0].columns.as_ref().expect("Should have columns");
|
||||
assert_eq!(columns.get("col1"), Some(&R));
|
||||
assert_eq!(columns.get("col2"), Some(&R));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sql_asset_parser_s3_str_literal_wildcard_no_columns() {
|
||||
let input = r#"
|
||||
SELECT * FROM 's3:///test.parquet';
|
||||
"#;
|
||||
let result = parse_assets(input).unwrap().assets;
|
||||
|
||||
assert_eq!(result.len(), 1);
|
||||
assert_eq!(result[0].kind, AssetKind::S3Object);
|
||||
assert!(result[0].columns.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sql_asset_parser_s3_read_csv_columns() {
|
||||
let input = r#"
|
||||
SELECT name, age FROM read_csv('s3://my-bucket/data.csv');
|
||||
"#;
|
||||
let result = parse_assets(input).unwrap().assets;
|
||||
|
||||
assert_eq!(result.len(), 1);
|
||||
assert_eq!(result[0].kind, AssetKind::S3Object);
|
||||
assert_eq!(result[0].path, "my-bucket/data.csv");
|
||||
|
||||
let columns = result[0].columns.as_ref().expect("Should have columns");
|
||||
assert_eq!(columns.get("name"), Some(&R));
|
||||
assert_eq!(columns.get("age"), Some(&R));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,3 +59,6 @@ wasm-bindgen.workspace = true
|
||||
|
||||
serde_json.workspace = true
|
||||
getrandom = { workspace = true, features = ["js"] }
|
||||
# getrandom 0.3 is pulled in transitively by rand 0.9 (via windmill-types).
|
||||
# It requires the "wasm_js" feature to work on wasm32-unknown-unknown.
|
||||
getrandom3 = { package = "getrandom", version = "0.3", features = ["wasm_js"] }
|
||||
|
||||
+16
-8
@@ -39,8 +39,6 @@ use windmill_common::ee_oss::{jobs_waiting_alerts, worker_groups_alerts};
|
||||
|
||||
#[cfg(feature = "oauth2")]
|
||||
use windmill_common::global_settings::OAUTH_SETTING;
|
||||
#[cfg(feature = "parquet")]
|
||||
use windmill_object_store::reload_object_store_setting;
|
||||
use windmill_common::{
|
||||
agent_workers::DECODED_AGENT_TOKEN,
|
||||
apps::APP_WORKSPACED_ROUTE,
|
||||
@@ -56,7 +54,7 @@ use windmill_common::{
|
||||
HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
|
||||
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING,
|
||||
KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING,
|
||||
NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING,
|
||||
NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING,
|
||||
OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
|
||||
POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
|
||||
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
|
||||
@@ -84,14 +82,16 @@ use windmill_common::{
|
||||
OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS,
|
||||
};
|
||||
use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING};
|
||||
#[cfg(feature = "parquet")]
|
||||
use windmill_object_store::reload_object_store_setting;
|
||||
use windmill_queue::{cancel_job, get_queued_job_v2, SameWorkerPayload};
|
||||
use windmill_worker::{
|
||||
result_processor::handle_job_error, JobCompletedSender, JobIsolationLevel,
|
||||
OtelTracingProxySettings, SameWorkerSender, BUNFIG_INSTALL_SCOPES, CARGO_REGISTRIES,
|
||||
INSTANCE_PYTHON_VERSION, JAVA_HOME_DIR, JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR,
|
||||
MAVEN_REPOS, MAVEN_SETTINGS_XML, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE,
|
||||
NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL,
|
||||
POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UV_INDEX_STRATEGY,
|
||||
MAVEN_REPOS, MAVEN_SETTINGS_XML, NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY,
|
||||
NSJAIL_AVAILABLE, NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL,
|
||||
PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UV_INDEX_STRATEGY,
|
||||
};
|
||||
|
||||
#[cfg(feature = "parquet")]
|
||||
@@ -330,6 +330,7 @@ pub async fn initial_load(
|
||||
reload_uv_index_strategy_setting(&conn).await;
|
||||
reload_npm_config_registry_setting(&conn).await;
|
||||
reload_bunfig_install_scopes_setting(&conn).await;
|
||||
reload_npmrc_setting(&conn).await;
|
||||
reload_instance_python_version_setting(&conn).await;
|
||||
reload_nuget_config_setting(&conn).await;
|
||||
reload_powershell_repo_url_setting(&conn).await;
|
||||
@@ -1204,7 +1205,10 @@ async fn delete_log_files_from_disk_and_store(
|
||||
#[cfg(feature = "parquet")]
|
||||
if _should_del_from_store {
|
||||
if let Some(os) = _os2 {
|
||||
let p = windmill_object_store::object_store_reexports::Path::from(format!("{}{}", _s3_prefix, path));
|
||||
let p = windmill_object_store::object_store_reexports::Path::from(format!(
|
||||
"{}{}",
|
||||
_s3_prefix, path
|
||||
));
|
||||
if let Err(e) = os.delete(&p).await {
|
||||
tracing::error!("Failed to delete from object store {}: {e}", p.to_string())
|
||||
} else {
|
||||
@@ -1303,6 +1307,10 @@ pub async fn reload_bunfig_install_scopes_setting(conn: &Connection) {
|
||||
.await;
|
||||
}
|
||||
|
||||
pub async fn reload_npmrc_setting(conn: &Connection) {
|
||||
reload_option_setting_with_tracing(conn, NPMRC_SETTING, "NPMRC", NPMRC.clone()).await;
|
||||
}
|
||||
|
||||
pub async fn reload_nuget_config_setting(conn: &Connection) {
|
||||
reload_option_setting_with_tracing(
|
||||
conn,
|
||||
@@ -2341,7 +2349,7 @@ pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> {
|
||||
async fn stale_job_cancellation(db: &Pool<Postgres>) {
|
||||
if let Some(threshold) = *STALE_JOB_THRESHOLD_MINUTES {
|
||||
let stale_jobs = sqlx::query!(
|
||||
"SELECT v2_job_queue.id, v2_job.tag, v2_job_queue.scheduled_for, v2_job_queue.workspace_id FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id WHERE running = false AND scheduled_for < now() - ($1 || ' minutes')::interval",
|
||||
"SELECT v2_job_queue.id, v2_job.tag, v2_job_queue.scheduled_for, v2_job_queue.workspace_id FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id WHERE running = false AND scheduled_for < now() - ($1 || ' minutes')::interval AND v2_job.trigger_kind IS DISTINCT FROM 'schedule'::job_trigger_kind",
|
||||
threshold.to_string()
|
||||
)
|
||||
.fetch_all(db)
|
||||
|
||||
@@ -7,6 +7,7 @@ REVERT="NO"
|
||||
COPY="NO"
|
||||
MOVE_NEW_FILES="NO"
|
||||
EE_CODE_DIR="../windmill-ee-private/"
|
||||
DIR_EXPLICIT="NO"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
@@ -34,6 +35,7 @@ while [[ $# -gt 0 ]]; do
|
||||
# Path to the local directory of the windmill-ee-private repository. By defaults, it
|
||||
# assumes it is cloned next to the Windmill OSS repo.
|
||||
EE_CODE_DIR="$2"
|
||||
DIR_EXPLICIT="YES"
|
||||
shift # past argument
|
||||
shift # past value
|
||||
;;
|
||||
@@ -55,10 +57,22 @@ else
|
||||
fi
|
||||
|
||||
# Fallback to ~/windmill-ee-private if the default location doesn't exist
|
||||
if [ ! -d "${EE_CODE_DIR}" ] && [ "${EE_CODE_DIR}" == "${root_dirpath}/../windmill-ee-private/" ]; then
|
||||
if [ ! -d "${EE_CODE_DIR}" ]; then
|
||||
EE_CODE_DIR="${HOME}/windmill-ee-private"
|
||||
fi
|
||||
|
||||
# Unless --dir was explicitly set, try to find an EE worktree on the same branch
|
||||
if [ "$DIR_EXPLICIT" == "NO" ] && [ -d "${HOME}/windmill-ee-private" ]; then
|
||||
current_branch=$(git -C "${root_dirpath}" branch --show-current 2>/dev/null || true)
|
||||
if [ -n "$current_branch" ]; then
|
||||
ee_worktree=$(git -C "${HOME}/windmill-ee-private" worktree list 2>/dev/null \
|
||||
| awk -v branch="[${current_branch}]" '$NF == branch {print $1; exit}')
|
||||
if [ -n "$ee_worktree" ] && [ -d "$ee_worktree" ]; then
|
||||
EE_CODE_DIR="$ee_worktree"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "EE code directory = ${EE_CODE_DIR} | Revert = ${REVERT}"
|
||||
|
||||
if [ ! -d "${EE_CODE_DIR}" ]; then
|
||||
|
||||
Executable
+474
@@ -0,0 +1,474 @@
|
||||
#!/usr/bin/env bash
|
||||
# End-to-end debounce tests against the running backend API
|
||||
# Usage: BACKEND_PORT=8030 ./test_debounce_e2e.sh
|
||||
set -uo pipefail
|
||||
|
||||
BASE="http://localhost:${BACKEND_PORT:-8030}/api"
|
||||
W="admins"
|
||||
EMAIL="admin@windmill.dev"
|
||||
PASSWORD="changeme"
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
pass=0
|
||||
fail=0
|
||||
|
||||
log_pass() { echo -e "${GREEN}PASS${NC}: $1"; ((pass++)) || true; }
|
||||
log_fail() { echo -e "${RED}FAIL${NC}: $1 — $2"; ((fail++)) || true; }
|
||||
log_info() { echo -e "${YELLOW}INFO${NC}: $1"; }
|
||||
|
||||
# Unique suffix for idempotent re-runs
|
||||
TS=$(date +%s)
|
||||
|
||||
# --- Auth ---
|
||||
log_info "Logging in..."
|
||||
TOKEN=$(curl -s "$BASE/auth/login" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"email\":\"$EMAIL\",\"password\":\"$PASSWORD\"}")
|
||||
|
||||
if [ -z "$TOKEN" ]; then
|
||||
echo "Failed to login"; exit 1
|
||||
fi
|
||||
|
||||
AUTH="Authorization: Bearer $TOKEN"
|
||||
log_info "Logged in"
|
||||
|
||||
# --- Helpers ---
|
||||
api() {
|
||||
# Usage: api METHOD path [data]
|
||||
local method="$1" path="$2" data="${3:-}"
|
||||
if [ -n "$data" ]; then
|
||||
curl -s "$BASE/w/$W/$path" -X "$method" -H "$AUTH" -H 'Content-Type: application/json' -d "$data"
|
||||
else
|
||||
curl -s "$BASE/w/$W/$path" -X "$method" -H "$AUTH"
|
||||
fi
|
||||
}
|
||||
|
||||
wait_job() {
|
||||
local job_id="$1" max_wait="${2:-30}"
|
||||
for _ in $(seq 1 "$max_wait"); do
|
||||
local r
|
||||
r=$(api GET "jobs/completed/get_result_maybe/$job_id")
|
||||
if echo "$r" | jq -e '.completed == true' > /dev/null 2>&1; then
|
||||
echo "$r"; return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo '{"completed":false,"error":"timeout"}'; return 1
|
||||
}
|
||||
|
||||
BUN_EMPTY_LOCK=$'{"dependencies": {}}\n//bun.lock\n'
|
||||
|
||||
create_script() {
|
||||
# Usage: create_script path language content [extra_json_fields]
|
||||
# Note: lock must be non-empty; empty string ("") is treated as None by the backend
|
||||
# (scripts.rs:798-800), which triggers dependency resolution instead of direct deployment.
|
||||
# For bun scripts, the lock must contain "//bun.lock" as a split pattern.
|
||||
local path="$1" lang="$2" content="$3" extra="${4:-}"
|
||||
local json
|
||||
json=$(jq -n \
|
||||
--arg path "$path" \
|
||||
--arg lang "$lang" \
|
||||
--arg content "$content" \
|
||||
--arg summary "test" \
|
||||
--arg desc "test" \
|
||||
--arg lock "$BUN_EMPTY_LOCK" \
|
||||
'{path: $path, language: $lang, content: $content, summary: $summary, description: $desc, lock: $lock}')
|
||||
if [ -n "$extra" ]; then
|
||||
json=$(echo "$json" | jq ". + $extra")
|
||||
fi
|
||||
local hash
|
||||
hash=$(api POST "scripts/create" "$json")
|
||||
# Small delay for DB visibility after tx commit
|
||||
sleep 0.2
|
||||
echo "$hash"
|
||||
}
|
||||
|
||||
run_script() {
|
||||
# Usage: run_script path args_json
|
||||
api POST "jobs/run/p/$1" "$2"
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# TEST 1: Deploy a script and run it 5 times in close succession
|
||||
###############################################################################
|
||||
echo ""
|
||||
log_info "=== TEST 1: Deploy & run script 5 times rapidly ==="
|
||||
|
||||
P1="u/admin/e2e_simple_$TS"
|
||||
H1=$(create_script "$P1" "bun" 'export function main(x: number = 0) { return { result: x * 2 }; }')
|
||||
|
||||
if echo "$H1" | grep -qE '^[0-9a-f]{16}$'; then
|
||||
log_pass "Script created: $H1"
|
||||
else
|
||||
log_fail "Script creation" "$H1"
|
||||
fi
|
||||
|
||||
log_info "Running 5 times rapidly..."
|
||||
JOB_IDS=()
|
||||
for i in $(seq 1 5); do
|
||||
JID=$(run_script "$P1" "{\"x\": $i}")
|
||||
JOB_IDS+=("$JID")
|
||||
done
|
||||
log_info "Jobs: ${JOB_IDS[*]}"
|
||||
|
||||
log_info "Waiting for completion..."
|
||||
all_ok=true
|
||||
for i in "${!JOB_IDS[@]}"; do
|
||||
JID="${JOB_IDS[$i]}"
|
||||
R=$(wait_job "$JID" 30)
|
||||
success=$(echo "$R" | jq -r '.success // false')
|
||||
value=$(echo "$R" | jq -r '.result.result // "null"')
|
||||
expected=$(( (i + 1) * 2 ))
|
||||
if [ "$success" = "true" ] && [ "$value" = "$expected" ]; then
|
||||
log_pass "Job $((i+1)): x=$((i+1)) → $value (correct)"
|
||||
else
|
||||
log_fail "Job $((i+1))" "success=$success value=$value expected=$expected"
|
||||
all_ok=false
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$all_ok" = "true" ]; then
|
||||
log_pass "All 5 runs completed correctly (no debounce — different args)"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# TEST 2: Redeploy script WITHOUT lock in close succession
|
||||
###############################################################################
|
||||
echo ""
|
||||
log_info "=== TEST 2: Redeploy without lock in rapid succession ==="
|
||||
|
||||
P2="u/admin/e2e_nolock_$TS"
|
||||
|
||||
# Deploy 5 versions of the same script without lock → triggers dependency jobs
|
||||
DEPLOY_HASHES=()
|
||||
for i in $(seq 1 5); do
|
||||
content="export function main(x: number = 0) { return { result: x * $i, version: $i }; }"
|
||||
parent_extra=""
|
||||
if [ "${#DEPLOY_HASHES[@]}" -gt 0 ]; then
|
||||
last_hash="${DEPLOY_HASHES[-1]}"
|
||||
parent_extra="{\"parent_hash\": \"$last_hash\"}"
|
||||
fi
|
||||
|
||||
# Deploy without lock (omit lock field entirely)
|
||||
json=$(jq -n \
|
||||
--arg path "$P2" \
|
||||
--arg content "$content" \
|
||||
--arg summary "v$i" \
|
||||
--arg desc "test" \
|
||||
'{path: $path, language: "bun", content: $content, summary: $summary, description: $desc}')
|
||||
if [ -n "$parent_extra" ]; then
|
||||
json=$(echo "$json" | jq ". + $parent_extra")
|
||||
fi
|
||||
|
||||
hash=$(api POST "scripts/create" "$json")
|
||||
if echo "$hash" | grep -qE '^[0-9a-f]{16}$'; then
|
||||
DEPLOY_HASHES+=("$hash")
|
||||
log_info "Deploy $i: $hash"
|
||||
else
|
||||
log_fail "Deploy $i" "$hash"
|
||||
# If path conflict, the script already exists from a previous version
|
||||
break
|
||||
fi
|
||||
sleep 0.1
|
||||
done
|
||||
|
||||
# Wait for dependency resolution
|
||||
log_info "Waiting 15s for dependency jobs..."
|
||||
sleep 15
|
||||
|
||||
# Check the latest script — should have lock resolved
|
||||
SCRIPT_INFO=$(api GET "scripts/get/p/$P2")
|
||||
LOCK=$(echo "$SCRIPT_INFO" | jq -r '.lock // "null"')
|
||||
if [ "$LOCK" != "null" ] && [ -n "$LOCK" ]; then
|
||||
log_pass "Latest version has lock resolved"
|
||||
else
|
||||
log_info "Lock not yet resolved: $LOCK"
|
||||
fi
|
||||
|
||||
# Run the latest version to verify it works
|
||||
sleep 0.5
|
||||
JID2=$(run_script "$P2" '{"x": 10}')
|
||||
if echo "$JID2" | grep -qE '^[0-9a-f-]{36}$'; then
|
||||
R2=$(wait_job "$JID2" 30)
|
||||
success=$(echo "$R2" | jq -r '.success // false')
|
||||
if [ "$success" = "true" ]; then
|
||||
version=$(echo "$R2" | jq -r '.result.version // "?"')
|
||||
log_pass "Latest version runs: version=$version"
|
||||
else
|
||||
err=$(echo "$R2" | jq -r '.result.error.message // "unknown"' 2>/dev/null)
|
||||
log_fail "Run latest version" "success=false err=$err"
|
||||
fi
|
||||
else
|
||||
log_fail "Run latest version" "bad job id: $JID2"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# TEST 3: Script with debounce_delay_s — rapid runs with SAME args
|
||||
###############################################################################
|
||||
echo ""
|
||||
log_info "=== TEST 3: Debounce with same args (should debounce) ==="
|
||||
|
||||
P3="u/admin/e2e_debounce_$TS"
|
||||
H3=$(create_script "$P3" "bun" \
|
||||
'export function main(x: number = 0) { return { result: x }; }' \
|
||||
'{"debounce_delay_s": 3}')
|
||||
|
||||
if echo "$H3" | grep -qE '^[0-9a-f]{16}$'; then
|
||||
log_pass "Debounce script created: $H3"
|
||||
else
|
||||
log_fail "Debounce script creation" "$H3"
|
||||
fi
|
||||
|
||||
log_info "Running 5 times with same args {x: 42}..."
|
||||
DEB_IDS=()
|
||||
for i in $(seq 1 5); do
|
||||
JID=$(run_script "$P3" '{"x": 42}')
|
||||
DEB_IDS+=("$JID")
|
||||
log_info " Run $i: $JID"
|
||||
done
|
||||
|
||||
log_info "Waiting 10s for debounce delay (3s) + execution..."
|
||||
sleep 10
|
||||
|
||||
executed=0
|
||||
skipped=0
|
||||
for JID in "${DEB_IDS[@]}"; do
|
||||
if ! echo "$JID" | grep -qE '^[0-9a-f-]{36}$'; then
|
||||
log_info " Invalid job id: $JID"
|
||||
continue
|
||||
fi
|
||||
R=$(wait_job "$JID" 5 2>/dev/null || echo '{"completed":false}')
|
||||
completed=$(echo "$R" | jq -r '.completed // false')
|
||||
success=$(echo "$R" | jq -r '.success // false')
|
||||
if [ "$completed" = "true" ] && [ "$success" = "true" ]; then
|
||||
((executed++)) || true
|
||||
elif [ "$completed" = "true" ]; then
|
||||
((skipped++)) || true
|
||||
fi
|
||||
done
|
||||
|
||||
log_info "Results: $executed executed, $skipped skipped out of ${#DEB_IDS[@]}"
|
||||
if [ "$executed" -eq 1 ] && [ "$skipped" -ge 3 ]; then
|
||||
log_pass "Debouncing perfect: 1 executed, $skipped skipped"
|
||||
elif [ "$executed" -le 2 ] && [ "$skipped" -ge 2 ]; then
|
||||
log_pass "Debouncing working: $executed executed, $skipped skipped"
|
||||
else
|
||||
log_fail "Debounce same args" "executed=$executed skipped=$skipped (want ~1 exec, ~4 skip)"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# TEST 3b: Different args should NOT debounce against each other
|
||||
###############################################################################
|
||||
echo ""
|
||||
log_info "=== TEST 3b: Debounce with different args (should NOT debounce) ==="
|
||||
|
||||
DIFF_IDS=()
|
||||
for i in $(seq 1 3); do
|
||||
JID=$(run_script "$P3" "{\"x\": $((i * 100))}")
|
||||
DIFF_IDS+=("$JID")
|
||||
done
|
||||
|
||||
log_info "Waiting 8s..."
|
||||
sleep 8
|
||||
|
||||
diff_exec=0
|
||||
for JID in "${DIFF_IDS[@]}"; do
|
||||
if ! echo "$JID" | grep -qE '^[0-9a-f-]{36}$'; then continue; fi
|
||||
R=$(wait_job "$JID" 5 2>/dev/null || echo '{"completed":false}')
|
||||
success=$(echo "$R" | jq -r '.success // false')
|
||||
if [ "$success" = "true" ]; then ((diff_exec++)) || true; fi
|
||||
done
|
||||
|
||||
if [ "$diff_exec" -eq 3 ]; then
|
||||
log_pass "Different args: all 3 executed independently"
|
||||
else
|
||||
log_fail "Different args" "only $diff_exec/3 executed"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# TEST 4: Custom debounce_key with $args interpolation
|
||||
###############################################################################
|
||||
echo ""
|
||||
log_info "=== TEST 4: Custom debounce key ==="
|
||||
|
||||
P4="u/admin/e2e_custom_key_$TS"
|
||||
H4=$(create_script "$P4" "bun" \
|
||||
'export function main(event_id: string = "", data: string = "") { return { event_id, data }; }' \
|
||||
'{"debounce_delay_s": 3, "debounce_key": "event#$args.event_id"}')
|
||||
|
||||
if echo "$H4" | grep -qE '^[0-9a-f]{16}$'; then
|
||||
log_pass "Custom key script created: $H4"
|
||||
else
|
||||
log_fail "Custom key script creation" "$H4"
|
||||
fi
|
||||
|
||||
# Same event_id → should debounce
|
||||
log_info "3 runs with same event_id..."
|
||||
SAME_IDS=()
|
||||
for i in $(seq 1 3); do
|
||||
JID=$(run_script "$P4" "{\"event_id\": \"evt_001\", \"data\": \"payload_$i\"}")
|
||||
SAME_IDS+=("$JID")
|
||||
done
|
||||
|
||||
# Different event_id → should NOT debounce
|
||||
JID_DIFF=$(run_script "$P4" '{"event_id": "evt_002", "data": "different"}')
|
||||
|
||||
log_info "Waiting 8s..."
|
||||
sleep 8
|
||||
|
||||
same_exec=0
|
||||
same_skip=0
|
||||
for JID in "${SAME_IDS[@]}"; do
|
||||
if ! echo "$JID" | grep -qE '^[0-9a-f-]{36}$'; then continue; fi
|
||||
R=$(wait_job "$JID" 5 2>/dev/null || echo '{"completed":false}')
|
||||
completed=$(echo "$R" | jq -r '.completed // false')
|
||||
success=$(echo "$R" | jq -r '.success // false')
|
||||
if [ "$completed" = "true" ] && [ "$success" = "true" ]; then
|
||||
data=$(echo "$R" | jq -r '.result.data // "?"')
|
||||
((same_exec++)) || true
|
||||
log_info " Executed: data=$data"
|
||||
elif [ "$completed" = "true" ]; then
|
||||
((same_skip++)) || true
|
||||
fi
|
||||
done
|
||||
|
||||
log_info "Same event_id: $same_exec executed, $same_skip skipped"
|
||||
if [ "$same_exec" -eq 1 ] && [ "$same_skip" -ge 1 ]; then
|
||||
log_pass "Custom key debounce: same event_id debounced correctly"
|
||||
elif [ "$same_exec" -le 2 ]; then
|
||||
log_pass "Custom key debounce working: $same_exec executed, $same_skip skipped"
|
||||
else
|
||||
log_fail "Custom key debounce" "exec=$same_exec skip=$same_skip"
|
||||
fi
|
||||
|
||||
# Check different event_id ran independently
|
||||
if echo "$JID_DIFF" | grep -qE '^[0-9a-f-]{36}$'; then
|
||||
R_DIFF=$(wait_job "$JID_DIFF" 10 2>/dev/null || echo '{"completed":false}')
|
||||
diff_success=$(echo "$R_DIFF" | jq -r '.success // false')
|
||||
if [ "$diff_success" = "true" ]; then
|
||||
log_pass "Different event_id: executed independently"
|
||||
else
|
||||
log_info "Different event_id: success=$diff_success"
|
||||
fi
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# TEST 5: Git sync with bad target — debounced deployment callbacks
|
||||
###############################################################################
|
||||
echo ""
|
||||
log_info "=== TEST 5: Git sync debounce + aggregation ==="
|
||||
|
||||
# Create git repo resource
|
||||
api POST "resources/create?update_if_exists=true" '{
|
||||
"path": "u/admin/e2e_bad_git_repo",
|
||||
"description": "Bad git repo for testing",
|
||||
"resource_type": "git_repository",
|
||||
"value": {"url": "https://github.com/nonexistent/nope.git", "branch": "main", "token": "bad"}
|
||||
}' > /dev/null 2>&1
|
||||
log_info "Created git repo resource"
|
||||
|
||||
# Create a sync script at a folder path where the 2nd segment is a number >= 28103.
|
||||
# is_script_meets_min_version parses split("/").skip(1).next() as the version number.
|
||||
# This enables debounce_delay_s=5 and debounce_args_to_accumulate=["items"].
|
||||
api POST "folders/create" '{"name": "28103"}' > /dev/null 2>&1
|
||||
P5="f/28103/e2e_sync_$TS"
|
||||
H5=$(create_script "$P5" "bun" \
|
||||
'export function main(repo_url_resource_path: string = "", workspace_id: string = "", items: any[] = [], use_individual_branch: boolean = false, group_by_folder: boolean = false, parent_workspace_id: string = "") { return { synced: items.length, items }; }')
|
||||
|
||||
if echo "$H5" | grep -qE '^[0-9a-f]{16}$'; then
|
||||
log_pass "Sync script created: $H5"
|
||||
else
|
||||
log_fail "Sync script creation" "$H5"
|
||||
fi
|
||||
|
||||
# Configure git sync with include_path to match deployed scripts.
|
||||
# Without include_path, path_matches_filters returns false and no DeploymentCallback is created.
|
||||
api POST "workspaces/edit_git_sync_config" "{
|
||||
\"git_sync_settings\": {
|
||||
\"include_type\": [\"script\"],
|
||||
\"include_path\": [\"**\"],
|
||||
\"repositories\": [{
|
||||
\"script_path\": \"$P5\",
|
||||
\"git_repo_resource_path\": \"\$res:u/admin/e2e_bad_git_repo\",
|
||||
\"use_individual_branch\": false,
|
||||
\"group_by_folder\": false
|
||||
}]
|
||||
}
|
||||
}" > /dev/null 2>&1
|
||||
log_pass "Git sync configured with include_path and versioned folder script path"
|
||||
|
||||
# Deploy 5 scripts rapidly to trigger git sync.
|
||||
# Scripts are created with lock="" (via create_script), so handle_deployment_metadata
|
||||
# fires immediately after tx commit (not after dependency resolution).
|
||||
log_info "Deploying 5 scripts to trigger git sync..."
|
||||
for i in $(seq 1 5); do
|
||||
dp="u/admin/e2e_gitsync_${TS}_$i"
|
||||
create_script "$dp" "bun" "export function main() { return { v: $i }; }" > /dev/null
|
||||
log_info " Deployed $dp"
|
||||
done
|
||||
|
||||
# Wait for debounce delay (5s) + execution
|
||||
log_info "Waiting 15s for debounce (5s) + execution..."
|
||||
sleep 15
|
||||
|
||||
# Check deployment callback jobs for our sync script.
|
||||
# Debounced jobs have is_skipped=true (but success=true), so we use is_skipped to distinguish.
|
||||
SYNC_JOBS=$(api GET "jobs/completed/list?script_path_exact=$P5&job_kinds=deploymentcallback")
|
||||
SYNC_TOTAL=$(echo "$SYNC_JOBS" | jq 'length')
|
||||
SYNC_EXECUTED=$(echo "$SYNC_JOBS" | jq '[.[] | select(.is_skipped != true)] | length')
|
||||
SYNC_SKIPPED=$(echo "$SYNC_JOBS" | jq '[.[] | select(.is_skipped == true)] | length')
|
||||
|
||||
log_info "Sync jobs: total=$SYNC_TOTAL executed=$SYNC_EXECUTED skipped=$SYNC_SKIPPED"
|
||||
|
||||
if [ "$SYNC_TOTAL" -gt 0 ]; then
|
||||
# With debouncing (5s delay), rapid deploys should be consolidated.
|
||||
# All 5 jobs are created but most should be skipped (debounced).
|
||||
if [ "$SYNC_SKIPPED" -gt 0 ]; then
|
||||
log_pass "Git sync debouncing: $SYNC_EXECUTED executed, $SYNC_SKIPPED debounced out of $SYNC_TOTAL"
|
||||
else
|
||||
log_fail "Git sync debouncing" "No jobs were debounced ($SYNC_TOTAL all executed independently)"
|
||||
fi
|
||||
|
||||
# Check if items were aggregated in the executed (non-skipped) job(s)
|
||||
for idx in $(seq 0 $((SYNC_TOTAL - 1))); do
|
||||
is_skipped=$(echo "$SYNC_JOBS" | jq -r ".[$idx].is_skipped")
|
||||
[ "$is_skipped" = "true" ] && continue
|
||||
jid=$(echo "$SYNC_JOBS" | jq -r ".[$idx].id")
|
||||
r=$(api GET "jobs/completed/get_result/$jid")
|
||||
items_count=$(echo "$r" | jq '.items | length // 0')
|
||||
log_info " Executed sync job $jid: items=$items_count"
|
||||
if [ "$items_count" -gt 1 ]; then
|
||||
log_pass "Items aggregated: $items_count items in single sync job"
|
||||
fi
|
||||
done
|
||||
else
|
||||
# Check queued — jobs may still be pending debounce delay
|
||||
Q=$(api GET "jobs/queue/list?script_path_exact=$P5&job_kinds=deploymentcallback")
|
||||
QC=$(echo "$Q" | jq 'length')
|
||||
log_info "No completed sync jobs. $QC queued."
|
||||
if [ "$QC" -gt 0 ] && [ "$QC" -lt 5 ]; then
|
||||
log_pass "Git sync debouncing (queued): $QC jobs for 5 deploys"
|
||||
elif [ "$QC" -eq 0 ]; then
|
||||
log_fail "Git sync" "No deployment callback jobs found (completed or queued)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Cleanup git sync
|
||||
api POST "workspaces/edit_git_sync_config" '{"git_sync_settings": null}' > /dev/null 2>&1
|
||||
log_info "Git sync config cleared"
|
||||
|
||||
###############################################################################
|
||||
# Summary
|
||||
###############################################################################
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo -e "Results: ${GREEN}$pass passed${NC}, ${RED}$fail failed${NC}"
|
||||
echo "========================================="
|
||||
|
||||
if [ "$fail" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
+155
-62
@@ -1,8 +1,8 @@
|
||||
use windmill_test_utils::*;
|
||||
use sqlx::postgres::Postgres;
|
||||
use sqlx::Pool;
|
||||
use windmill_common::jobs::{JobPayload, RawCode};
|
||||
use windmill_common::scripts::ScriptLang;
|
||||
use windmill_test_utils::*;
|
||||
|
||||
// ============================================================================
|
||||
// Basic Execution Tests
|
||||
@@ -27,8 +27,8 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -63,8 +63,8 @@ export function main(name: string, count: number) {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -104,8 +104,9 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
|
||||
)
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -135,8 +136,9 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
|
||||
)
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -167,8 +169,9 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
|
||||
)
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -207,8 +210,8 @@ export async function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -245,8 +248,9 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
|
||||
)
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -276,8 +280,9 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
|
||||
)
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -318,8 +323,8 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -358,8 +363,8 @@ export function notMain() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -398,8 +403,8 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -437,8 +442,8 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -474,8 +479,8 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -516,8 +521,8 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -613,8 +618,9 @@ export function main() {
|
||||
path: Some("f/nested/test_deep".to_string()),
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
|
||||
)
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -647,8 +653,9 @@ export function main() {
|
||||
path: Some("f/nested/test_deep_relative".to_string()),
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(
|
||||
)
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -693,8 +700,8 @@ export function main() {
|
||||
path: Some("f/circular/test_both".to_string()),
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -741,8 +748,8 @@ export function main(x: number) {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -791,8 +798,8 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -836,8 +843,8 @@ export function main() {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -859,11 +866,11 @@ export function main() {
|
||||
// ============================================================================
|
||||
|
||||
mod dedicated_worker_protocol {
|
||||
use windmill_test_utils::{parse_dedicated_worker_line, DedicatedWorkerResult};
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::process::{Command, Stdio};
|
||||
use windmill_test_utils::{parse_dedicated_worker_line, DedicatedWorkerResult};
|
||||
use windmill_worker::{
|
||||
build_loader, generate_dedicated_worker_wrapper, BUN_DEDICATED_WORKER_ARGS, LoaderMode,
|
||||
build_loader, generate_dedicated_worker_wrapper, LoaderMode, BUN_DEDICATED_WORKER_ARGS,
|
||||
BUN_PATH, NODE_BIN_PATH,
|
||||
};
|
||||
|
||||
@@ -934,12 +941,8 @@ mod dedicated_worker_protocol {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
|
||||
// Create files and get the wrapper path (bundled for node, raw for bun)
|
||||
let wrapper_path = create_test_worker_files(
|
||||
temp_dir.path(),
|
||||
script,
|
||||
arg_names,
|
||||
runtime == "node",
|
||||
);
|
||||
let wrapper_path =
|
||||
create_test_worker_files(temp_dir.path(), script, arg_names, runtime == "node");
|
||||
let wrapper_str = wrapper_path.to_str().unwrap();
|
||||
|
||||
// Build args matching production behavior
|
||||
@@ -992,7 +995,10 @@ mod dedicated_worker_protocol {
|
||||
match parse_dedicated_worker_line(response.trim()) {
|
||||
DedicatedWorkerResult::Success(value) => results.push(Ok(value)),
|
||||
DedicatedWorkerResult::Error(err) => {
|
||||
let msg = err["message"].as_str().unwrap_or("Unknown error").to_string();
|
||||
let msg = err["message"]
|
||||
.as_str()
|
||||
.unwrap_or("Unknown error")
|
||||
.to_string();
|
||||
results.push(Err(msg));
|
||||
}
|
||||
other => panic!("Unexpected response: {:?}", other),
|
||||
@@ -1162,8 +1168,8 @@ export function main(name: string) {
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
@@ -1190,6 +1196,68 @@ export function main(name: string) {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test that full .npmrc content works for bun jobs with private registries.
|
||||
/// Requires:
|
||||
/// - `TEST_NPMRC` environment variable set to the full .npmrc content
|
||||
#[cfg(feature = "private_registry_test")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_bun_job_private_npmrc(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
use windmill_worker::NPMRC;
|
||||
|
||||
let npmrc_content = std::env::var("TEST_NPMRC")
|
||||
.expect("TEST_NPMRC must be set when running private_registry_test");
|
||||
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
{
|
||||
let mut npmrc = NPMRC.write().await;
|
||||
*npmrc = Some(npmrc_content.clone());
|
||||
}
|
||||
|
||||
let content = r#"
|
||||
import { greet } from "@windmill-test/private-pkg";
|
||||
|
||||
export function main(name: string) {
|
||||
return greet(name);
|
||||
}
|
||||
"#
|
||||
.to_owned();
|
||||
|
||||
let job = JobPayload::Code(RawCode {
|
||||
hash: None,
|
||||
content,
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
});
|
||||
|
||||
let result = RunJob::from(job)
|
||||
.arg("name", serde_json::json!("World"))
|
||||
.run_until_complete(&db, false, port)
|
||||
.await
|
||||
.json_result()
|
||||
.unwrap();
|
||||
|
||||
{
|
||||
let mut npmrc = NPMRC.write().await;
|
||||
*npmrc = None;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
result,
|
||||
serde_json::json!("Hello from private package, World!")
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tests for RELATIVE_BUN_BUILDER (loader_builder.bun.js)
|
||||
/// These tests verify Bun's behavior for import scanning and package.json generation.
|
||||
/// Purpose: Catch regressions when upgrading Bun versions.
|
||||
@@ -1241,8 +1309,8 @@ mod bun_builder_tests {
|
||||
}
|
||||
|
||||
// Read generated package.json
|
||||
let package_json = std::fs::read_to_string(dir.join("package.json"))
|
||||
.expect("package.json not generated");
|
||||
let package_json =
|
||||
std::fs::read_to_string(dir.join("package.json")).expect("package.json not generated");
|
||||
|
||||
serde_json::from_str(&package_json).expect("Invalid JSON in package.json")
|
||||
}
|
||||
@@ -1257,7 +1325,10 @@ export function main() { return lodash; }
|
||||
let pkg = run_builder(main_ts);
|
||||
let deps = pkg["dependencies"].as_object().unwrap();
|
||||
|
||||
assert!(deps.contains_key("lodash"), "lodash should be in dependencies");
|
||||
assert!(
|
||||
deps.contains_key("lodash"),
|
||||
"lodash should be in dependencies"
|
||||
);
|
||||
assert_eq!(deps["lodash"], "latest");
|
||||
}
|
||||
|
||||
@@ -1271,7 +1342,10 @@ export function main() { return _; }
|
||||
let pkg = run_builder(main_ts);
|
||||
let deps = pkg["dependencies"].as_object().unwrap();
|
||||
|
||||
assert!(deps.contains_key("lodash"), "lodash should be in dependencies");
|
||||
assert!(
|
||||
deps.contains_key("lodash"),
|
||||
"lodash should be in dependencies"
|
||||
);
|
||||
assert_eq!(deps["lodash"], "4.17.21");
|
||||
}
|
||||
|
||||
@@ -1304,9 +1378,18 @@ export function main() { return { lodash, axios, dayjs }; }
|
||||
let pkg = run_builder(main_ts);
|
||||
let deps = pkg["dependencies"].as_object().unwrap();
|
||||
|
||||
assert!(deps.contains_key("lodash"), "lodash should be in dependencies");
|
||||
assert!(deps.contains_key("axios"), "axios should be in dependencies");
|
||||
assert!(deps.contains_key("dayjs"), "dayjs should be in dependencies");
|
||||
assert!(
|
||||
deps.contains_key("lodash"),
|
||||
"lodash should be in dependencies"
|
||||
);
|
||||
assert!(
|
||||
deps.contains_key("axios"),
|
||||
"axios should be in dependencies"
|
||||
);
|
||||
assert!(
|
||||
deps.contains_key("dayjs"),
|
||||
"dayjs should be in dependencies"
|
||||
);
|
||||
assert_eq!(deps.len(), 3, "Should have exactly 3 dependencies");
|
||||
}
|
||||
|
||||
@@ -1330,8 +1413,15 @@ export function main() { return { fs, path, lodash }; }
|
||||
!deps.contains_key("path"),
|
||||
"path (builtin) should NOT be in dependencies"
|
||||
);
|
||||
assert!(deps.contains_key("lodash"), "lodash should be in dependencies");
|
||||
assert_eq!(deps.len(), 1, "Should have exactly 1 dependency (lodash only)");
|
||||
assert!(
|
||||
deps.contains_key("lodash"),
|
||||
"lodash should be in dependencies"
|
||||
);
|
||||
assert_eq!(
|
||||
deps.len(),
|
||||
1,
|
||||
"Should have exactly 1 dependency (lodash only)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test: semver.order() resolves version conflicts (picks lowest version)
|
||||
@@ -1347,7 +1437,10 @@ export function main() { return { a, b }; }
|
||||
let pkg = run_builder(main_ts);
|
||||
let deps = pkg["dependencies"].as_object().unwrap();
|
||||
|
||||
assert!(deps.contains_key("lodash"), "lodash should be in dependencies");
|
||||
assert!(
|
||||
deps.contains_key("lodash"),
|
||||
"lodash should be in dependencies"
|
||||
);
|
||||
// The builder sorts by semver and picks the first (lowest) version
|
||||
assert_eq!(
|
||||
deps["lodash"], "4.17.10",
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
-- Fixture for testing wmill CLI variable/resource get from bash scripts
|
||||
|
||||
INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms)
|
||||
VALUES ('test-workspace', 'u/test-user/test_var', 'hello from variable', false, 'A test variable', '{"u/test-user": true}');
|
||||
|
||||
INSERT INTO resource_type (workspace_id, name, schema, description, created_by)
|
||||
VALUES ('test-workspace', 'test_object', '{}', 'Test object type', 'test-user');
|
||||
|
||||
INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by)
|
||||
VALUES ('test-workspace', 'u/test-user/test_res', '{"host": "localhost", "port": 5432}', 'A test resource', 'test_object', '{"u/test-user": true}', 'test-user');
|
||||
@@ -0,0 +1,275 @@
|
||||
/*
|
||||
* Tests for the PrewarmedIsolate used by nativets dedicated workers.
|
||||
*
|
||||
* Run with:
|
||||
* cargo test -p windmill --features "deno_core" --test nativets_dedicated -- --nocapture
|
||||
*/
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
mod prewarmed_isolate_tests {
|
||||
use std::process::Command;
|
||||
use windmill_runtime_nativets::{NativeAnnotation, PrewarmedIsolate};
|
||||
use windmill_worker::{build_loader, LoaderMode, BUN_PATH};
|
||||
|
||||
fn default_annotation() -> NativeAnnotation {
|
||||
NativeAnnotation { useragent: None, proxy: None }
|
||||
}
|
||||
|
||||
/// Bundle a TypeScript script into JS suitable for `PrewarmedIsolate`.
|
||||
///
|
||||
/// Returns `(ts_source, js_bundle, arg_names)`.
|
||||
async fn bundle_script(script: &str) -> (String, String, Vec<String>) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let dir = temp_dir.path();
|
||||
let dir_str = dir.to_str().unwrap();
|
||||
|
||||
std::fs::write(dir.join("main.ts"), script).unwrap();
|
||||
|
||||
build_loader(
|
||||
dir_str,
|
||||
"http://localhost:8000",
|
||||
"test_token",
|
||||
"test-workspace",
|
||||
"f/test/script",
|
||||
LoaderMode::BrowserBundle,
|
||||
)
|
||||
.await
|
||||
.expect("build_loader failed");
|
||||
|
||||
let output = Command::new(BUN_PATH.as_str())
|
||||
.args(["run", dir.join("node_builder.ts").to_str().unwrap()])
|
||||
.current_dir(dir)
|
||||
.output()
|
||||
.expect("Failed to run bun build");
|
||||
|
||||
if !output.status.success() {
|
||||
panic!(
|
||||
"Bun build failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
let ts = std::fs::read_to_string(dir.join("main.ts")).unwrap();
|
||||
let js = std::fs::read_to_string(dir.join("main.js")).unwrap();
|
||||
let parsed = windmill_parser_ts::parse_deno_signature(&ts, true, false, None)
|
||||
.expect("failed to parse signature");
|
||||
let arg_names: Vec<String> = parsed.args.into_iter().map(|a| a.name).collect();
|
||||
(ts, js, arg_names)
|
||||
}
|
||||
|
||||
const TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
|
||||
async fn run_prewarmed_test(
|
||||
script: &str,
|
||||
jobs: Vec<serde_json::Value>,
|
||||
) -> Vec<Result<serde_json::Value, String>> {
|
||||
tokio::time::timeout(TEST_TIMEOUT, run_prewarmed_test_inner(script, jobs))
|
||||
.await
|
||||
.expect("test timed out after 30s")
|
||||
}
|
||||
|
||||
async fn run_prewarmed_test_inner(
|
||||
script: &str,
|
||||
jobs: Vec<serde_json::Value>,
|
||||
) -> Vec<Result<serde_json::Value, String>> {
|
||||
windmill_runtime_nativets::setup_deno_runtime().expect("V8 init failed");
|
||||
|
||||
let (_ts, js, arg_names) = bundle_script(script).await;
|
||||
let ann = default_annotation();
|
||||
|
||||
let mut results = Vec::new();
|
||||
|
||||
for job_args in &jobs {
|
||||
let mut isolate =
|
||||
PrewarmedIsolate::spawn("".to_string(), js.clone(), ann.clone(), arg_names.clone());
|
||||
isolate.wait_ready().await.expect("isolate failed to warm");
|
||||
|
||||
let args = serde_json::to_string(job_args).unwrap();
|
||||
let executing = isolate.start_execution(args);
|
||||
let prewarmed_result = executing.wait().await.expect("isolate execution failed");
|
||||
|
||||
match prewarmed_result.result {
|
||||
Ok(raw) => {
|
||||
let value: serde_json::Value =
|
||||
serde_json::from_str(raw.get()).unwrap_or(serde_json::Value::Null);
|
||||
results.push(Ok(value));
|
||||
}
|
||||
Err(e) => {
|
||||
results.push(Err(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_prewarmed_simple() {
|
||||
let script = r#"
|
||||
export function main(x: number, y: number): number {
|
||||
return x + y;
|
||||
}
|
||||
"#;
|
||||
let results = run_prewarmed_test(script, vec![serde_json::json!({"x": 2, "y": 3})]).await;
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0], Ok(serde_json::json!(5)));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_prewarmed_isolation() {
|
||||
let script = r#"
|
||||
let counter = 0;
|
||||
export function main(): number {
|
||||
counter++;
|
||||
return counter;
|
||||
}
|
||||
"#;
|
||||
let results =
|
||||
run_prewarmed_test(script, vec![serde_json::json!({}), serde_json::json!({})]).await;
|
||||
|
||||
assert_eq!(results.len(), 2);
|
||||
// Each job gets a fresh isolate, so counter should be 1 both times
|
||||
assert_eq!(results[0], Ok(serde_json::json!(1)));
|
||||
assert_eq!(results[1], Ok(serde_json::json!(1)));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_prewarmed_error() {
|
||||
let script = r#"
|
||||
export function main(msg: string): never {
|
||||
throw new Error(msg);
|
||||
}
|
||||
"#;
|
||||
let results =
|
||||
run_prewarmed_test(script, vec![serde_json::json!({"msg": "test error"})]).await;
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
assert!(results[0].is_err());
|
||||
assert!(
|
||||
results[0].as_ref().unwrap_err().contains("test error"),
|
||||
"Error should contain 'test error', got: {}",
|
||||
results[0].as_ref().unwrap_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_prewarmed_async() {
|
||||
let script = r#"
|
||||
export async function main(x: number): Promise<number> {
|
||||
const val = await Promise.resolve(x * 10);
|
||||
return val + 1;
|
||||
}
|
||||
"#;
|
||||
let results = run_prewarmed_test(script, vec![serde_json::json!({"x": 7})]).await;
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0], Ok(serde_json::json!(71)));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_prewarmed_pipeline() {
|
||||
windmill_runtime_nativets::setup_deno_runtime().expect("V8 init failed");
|
||||
|
||||
let script = r#"
|
||||
export function main(n: number): number {
|
||||
return n * 2;
|
||||
}
|
||||
"#;
|
||||
let (_ts, js, arg_names) = bundle_script(script).await;
|
||||
let ann = default_annotation();
|
||||
|
||||
// Pre-warm first isolate
|
||||
let mut warm =
|
||||
PrewarmedIsolate::spawn("".to_string(), js.clone(), ann.clone(), arg_names.clone());
|
||||
warm.wait_ready()
|
||||
.await
|
||||
.expect("first isolate failed to warm");
|
||||
|
||||
let mut results = Vec::new();
|
||||
|
||||
for i in 1..=3 {
|
||||
let args = serde_json::to_string(&serde_json::json!({"n": i})).unwrap();
|
||||
let executing = warm.start_execution(args);
|
||||
|
||||
// Pipeline: start pre-warming next isolate while current one runs
|
||||
warm =
|
||||
PrewarmedIsolate::spawn("".to_string(), js.clone(), ann.clone(), arg_names.clone());
|
||||
|
||||
let prewarmed_result = executing.wait().await.expect("isolate execution failed");
|
||||
match prewarmed_result.result {
|
||||
Ok(raw) => {
|
||||
let value: serde_json::Value =
|
||||
serde_json::from_str(raw.get()).unwrap_or(serde_json::Value::Null);
|
||||
results.push(value);
|
||||
}
|
||||
Err(e) => panic!("unexpected error: {e}"),
|
||||
}
|
||||
|
||||
warm.wait_ready()
|
||||
.await
|
||||
.expect("next isolate failed to warm");
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
results,
|
||||
vec![
|
||||
serde_json::json!(2),
|
||||
serde_json::json!(4),
|
||||
serde_json::json!(6),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_prewarmed_complex_return() {
|
||||
let script = r#"
|
||||
export function main(name: string, items: number[]): any {
|
||||
return {
|
||||
greeting: `hello ${name}`,
|
||||
sum: items.reduce((a, b) => a + b, 0),
|
||||
items: items.map(x => x * 2),
|
||||
};
|
||||
}
|
||||
"#;
|
||||
let results = run_prewarmed_test(
|
||||
script,
|
||||
vec![serde_json::json!({"name": "world", "items": [1, 2, 3]})],
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(
|
||||
results[0],
|
||||
Ok(serde_json::json!({
|
||||
"greeting": "hello world",
|
||||
"sum": 6,
|
||||
"items": [2, 4, 6],
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_prewarmed_null_undefined() {
|
||||
let script = r#"
|
||||
export function main(returnNull: boolean): any {
|
||||
if (returnNull) {
|
||||
return null;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
"#;
|
||||
let results = run_prewarmed_test(
|
||||
script,
|
||||
vec![
|
||||
serde_json::json!({"returnNull": true}),
|
||||
serde_json::json!({"returnNull": false}),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(results.len(), 2);
|
||||
assert_eq!(results[0], Ok(serde_json::Value::Null));
|
||||
assert_eq!(results[1], Ok(serde_json::Value::Null));
|
||||
}
|
||||
}
|
||||
@@ -993,6 +993,80 @@ echo "hello $msg"
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base", "wmill_cli_test"))]
|
||||
async fn test_bash_wmill_variable_get(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
// The bash script uses wmill CLI to get the variable value.
|
||||
// The worker sets WM_TOKEN, WM_WORKSPACE, and BASE_INTERNAL_URL as env vars,
|
||||
// and the CLI auto-configures from them when no workspace is explicitly set.
|
||||
// We point WMILL_CONFIG_DIR to a clean temp dir so no local active workspace interferes.
|
||||
let content = r#"
|
||||
export WMILL_CONFIG_DIR=$(mktemp -d)
|
||||
result=$(wmill variable get "u/test-user/test_var" --json | jq -r .value)
|
||||
echo "$result"
|
||||
"#
|
||||
.to_owned();
|
||||
|
||||
let job = RunJob::from(JobPayload::Code(RawCode {
|
||||
hash: None,
|
||||
content,
|
||||
path: None,
|
||||
lock: None,
|
||||
language: ScriptLang::Bash,
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
}))
|
||||
.run_until_complete(&db, false, port)
|
||||
.await;
|
||||
assert_eq!(job.json_result(), Some(json!("hello from variable")));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base", "wmill_cli_test"))]
|
||||
async fn test_bash_wmill_resource_get(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
// The bash script uses wmill CLI to get the resource value.
|
||||
// We point WMILL_CONFIG_DIR to a clean temp dir so no local active workspace interferes.
|
||||
let content = r#"
|
||||
export WMILL_CONFIG_DIR=$(mktemp -d)
|
||||
result=$(wmill resource get "u/test-user/test_res" --json | jq -c .value)
|
||||
echo "$result"
|
||||
"#
|
||||
.to_owned();
|
||||
|
||||
let job = RunJob::from(JobPayload::Code(RawCode {
|
||||
hash: None,
|
||||
content,
|
||||
path: None,
|
||||
lock: None,
|
||||
language: ScriptLang::Bash,
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
}))
|
||||
.run_until_complete(&db, false, port)
|
||||
.await;
|
||||
// Bash echo outputs are returned as strings, so the JSON is a string value
|
||||
assert_eq!(
|
||||
job.json_result(),
|
||||
Some(json!("{\"host\":\"localhost\",\"port\":5432}"))
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "nu")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_nu_job(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
@@ -1592,6 +1666,66 @@ export async function main(a: Date) {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test that full .npmrc content works for deno jobs with private registries.
|
||||
/// Requires:
|
||||
/// - `TEST_NPMRC` environment variable set to the full .npmrc content
|
||||
#[cfg(feature = "private_registry_test")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_deno_job_private_npmrc(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
use windmill_worker::NPMRC;
|
||||
|
||||
let npmrc_content = std::env::var("TEST_NPMRC")
|
||||
.expect("TEST_NPMRC must be set when running private_registry_test");
|
||||
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
{
|
||||
let mut npmrc = NPMRC.write().await;
|
||||
*npmrc = Some(npmrc_content.clone());
|
||||
}
|
||||
|
||||
let content = r#"
|
||||
import { greet } from "npm:@windmill-test/private-pkg";
|
||||
|
||||
export function main(name: string) {
|
||||
return greet(name);
|
||||
}
|
||||
"#
|
||||
.to_owned();
|
||||
|
||||
let result = RunJob::from(JobPayload::Code(RawCode {
|
||||
hash: None,
|
||||
content,
|
||||
path: None,
|
||||
language: ScriptLang::Deno,
|
||||
lock: None,
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
}))
|
||||
.arg("name", json!("World"))
|
||||
.run_until_complete(&db, false, port)
|
||||
.await
|
||||
.json_result()
|
||||
.unwrap();
|
||||
|
||||
{
|
||||
let mut npmrc = NPMRC.write().await;
|
||||
*npmrc = None;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
result,
|
||||
serde_json::json!("Hello from private package, World!")
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "python")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_python_job_datetime_and_bytes(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
|
||||
@@ -13,7 +13,7 @@ default = []
|
||||
enterprise = ["windmill-common/enterprise", "windmill-queue/enterprise"]
|
||||
private = ["windmill-common/private", "windmill-queue/private"]
|
||||
python = ["dep:windmill-parser-py-imports"]
|
||||
benchmark = []
|
||||
benchmark = ["windmill-queue/benchmark"]
|
||||
|
||||
[dependencies]
|
||||
windmill-api-auth.workspace = true
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use windmill_api_auth::ApiAuthed;
|
||||
use axum::{
|
||||
extract::{Path, Query},
|
||||
routing::{get, post},
|
||||
@@ -20,6 +19,7 @@ use std::{
|
||||
fmt::{Display, Formatter},
|
||||
vec,
|
||||
};
|
||||
use windmill_api_auth::ApiAuthed;
|
||||
use windmill_common::{
|
||||
db::UserDB,
|
||||
error::JsonResult,
|
||||
@@ -109,7 +109,7 @@ pub struct Input {
|
||||
#[derive(Debug, Serialize, Deserialize, FromRow)]
|
||||
pub struct CompletedJobMini {
|
||||
id: Uuid,
|
||||
created_at: chrono::DateTime<chrono::Utc>,
|
||||
completed_at: chrono::DateTime<chrono::Utc>,
|
||||
args: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
|
||||
created_by: String,
|
||||
success: bool,
|
||||
@@ -147,9 +147,9 @@ async fn get_input_history(
|
||||
};
|
||||
|
||||
let sql = &format!(
|
||||
"select id, v2_job.created_at, created_by, 'null'::jsonb as args, status = 'success' as success from v2_job JOIN v2_job_completed USING (id) \
|
||||
"select id, v2_job_completed.completed_at, created_by, 'null'::jsonb as args, status = 'success' as success from v2_job JOIN v2_job_completed USING (id) \
|
||||
where v2_job.workspace_id = $3 and {} = $1 and kind = any($2) {args_query} AND v2_job_completed.status != 'skipped' {include_non_root} \
|
||||
order by v2_job.created_at desc limit $4 offset $5",
|
||||
order by v2_job_completed.completed_at desc limit $4 offset $5",
|
||||
r.runnable_type.column_name(),
|
||||
|
||||
);
|
||||
@@ -189,10 +189,10 @@ async fn get_input_history(
|
||||
id: row.id,
|
||||
name: format!(
|
||||
"{} {}",
|
||||
row.created_at.format("%H:%M %-d/%-m"),
|
||||
row.completed_at.format("%H:%M %-d/%-m"),
|
||||
row.created_by
|
||||
),
|
||||
created_at: row.created_at,
|
||||
created_at: row.completed_at,
|
||||
args: sqlx::types::Json(
|
||||
serde_json::value::RawValue::from_string("null".to_string()).unwrap(),
|
||||
),
|
||||
@@ -352,11 +352,12 @@ async fn update_input(
|
||||
) -> JsonResult<String> {
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
sqlx::query("UPDATE input SET name = $1, is_public = $2 WHERE id = $3 and workspace_id = $4")
|
||||
sqlx::query("UPDATE input SET name = $1, is_public = $2 WHERE id = $3 and workspace_id = $4 AND created_by = $5")
|
||||
.bind(&input.name)
|
||||
.bind(&input.is_public)
|
||||
.bind(&input.id)
|
||||
.bind(&w_id)
|
||||
.bind(&authed.username)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
@@ -372,9 +373,10 @@ async fn delete_input(
|
||||
) -> JsonResult<String> {
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
sqlx::query("DELETE FROM input WHERE id = $1 and workspace_id = $2")
|
||||
sqlx::query("DELETE FROM input WHERE id = $1 and workspace_id = $2 AND created_by = $3")
|
||||
.bind(&i_id)
|
||||
.bind(&w_id)
|
||||
.bind(&authed.username)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ windmill-api-auth.workspace = true
|
||||
windmill-common = { workspace = true, default-features = false }
|
||||
axum.workspace = true
|
||||
flate2.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sqlx.workspace = true
|
||||
|
||||
@@ -14,8 +14,10 @@ use std::collections::HashMap;
|
||||
use tower_http::cors::{Any, CorsLayer};
|
||||
use windmill_common::{
|
||||
error::{Error, JsonResult, Result},
|
||||
global_settings::{load_value_from_global_settings, NPM_CONFIG_REGISTRY_SETTING},
|
||||
utils::StripPath,
|
||||
global_settings::{
|
||||
load_value_from_global_settings, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING,
|
||||
},
|
||||
utils::{parse_npmrc_registry, StripPath},
|
||||
};
|
||||
|
||||
use windmill_api_auth::ApiAuthed;
|
||||
@@ -129,6 +131,14 @@ pub fn workspaced_service() -> Router {
|
||||
)
|
||||
}
|
||||
|
||||
fn build_registry_request(url: &str, auth_token: &Option<String>) -> reqwest::RequestBuilder {
|
||||
let mut req = HTTP_CLIENT.get(url);
|
||||
if let Some(token) = auth_token {
|
||||
req = req.bearer_auth(token);
|
||||
}
|
||||
req
|
||||
}
|
||||
|
||||
/// Get package metadata (versions and tags) from the private registry
|
||||
async fn get_package_metadata(
|
||||
_authed: ApiAuthed,
|
||||
@@ -136,21 +146,14 @@ async fn get_package_metadata(
|
||||
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
|
||||
) -> JsonResult<PackageVersions> {
|
||||
let package = parse_package_name(package_path.to_path());
|
||||
let npm_registry = get_npm_registry(&db).await?;
|
||||
|
||||
if npm_registry.is_none() {
|
||||
return Err(Error::BadRequest(
|
||||
"No private npm registry configured".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let registry_url = npm_registry.unwrap();
|
||||
let (registry_url, auth_token) = get_npm_registry(&db)
|
||||
.await?
|
||||
.ok_or_else(|| Error::BadRequest("No private npm registry configured".to_string()))?;
|
||||
let package_url = format_registry_url(®istry_url, &package, None, None);
|
||||
|
||||
tracing::info!("Fetching package metadata from: {}", package_url);
|
||||
|
||||
let response = HTTP_CLIENT
|
||||
.get(&package_url)
|
||||
let response = build_registry_request(&package_url, &auth_token)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
|
||||
@@ -167,7 +170,6 @@ async fn get_package_metadata(
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
|
||||
|
||||
// Extract versions and dist-tags from the package metadata
|
||||
let mut versions = Vec::new();
|
||||
let mut tags = HashMap::new();
|
||||
|
||||
@@ -194,22 +196,15 @@ async fn resolve_package_version(
|
||||
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
|
||||
) -> JsonResult<PackageVersion> {
|
||||
let package = parse_package_name(package_path.to_path());
|
||||
let npm_registry = get_npm_registry(&db).await?;
|
||||
|
||||
if npm_registry.is_none() {
|
||||
return Err(Error::BadRequest(
|
||||
"No private npm registry configured".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let registry_url = npm_registry.unwrap();
|
||||
let (registry_url, auth_token) = get_npm_registry(&db)
|
||||
.await?
|
||||
.ok_or_else(|| Error::BadRequest("No private npm registry configured".to_string()))?;
|
||||
let reference = query.tag.unwrap_or_else(|| "latest".to_string());
|
||||
let package_url = format_registry_url(®istry_url, &package, None, None);
|
||||
|
||||
tracing::info!("Resolving package version from: {}", package_url);
|
||||
|
||||
let response = HTTP_CLIENT
|
||||
.get(&package_url)
|
||||
let response = build_registry_request(&package_url, &auth_token)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
|
||||
@@ -256,21 +251,14 @@ async fn get_package_filetree(
|
||||
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
|
||||
) -> JsonResult<PackageFiletree> {
|
||||
let (package, version) = parse_package_and_version(package_version_path.to_path())?;
|
||||
let npm_registry = get_npm_registry(&db).await?;
|
||||
|
||||
if npm_registry.is_none() {
|
||||
return Err(Error::BadRequest(
|
||||
"No private npm registry configured".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let registry_url = npm_registry.unwrap();
|
||||
let (registry_url, auth_token) = get_npm_registry(&db)
|
||||
.await?
|
||||
.ok_or_else(|| Error::BadRequest("No private npm registry configured".to_string()))?;
|
||||
let package_url = format_registry_url(®istry_url, &package, None, None);
|
||||
|
||||
tracing::info!("Fetching package filetree from: {}", package_url);
|
||||
|
||||
let response = HTTP_CLIENT
|
||||
.get(&package_url)
|
||||
let response = build_registry_request(&package_url, &auth_token)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
|
||||
@@ -287,7 +275,6 @@ async fn get_package_filetree(
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
|
||||
|
||||
// Get the tarball URL for this version
|
||||
let tarball_url = package_json
|
||||
.get("versions")
|
||||
.and_then(|v| v.get(&version))
|
||||
@@ -296,9 +283,7 @@ async fn get_package_filetree(
|
||||
.and_then(|t| t.as_str())
|
||||
.ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?;
|
||||
|
||||
// Download and extract tarball to get file list
|
||||
let tarball_response = HTTP_CLIENT
|
||||
.get(tarball_url)
|
||||
let tarball_response = build_registry_request(tarball_url, &auth_token)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?;
|
||||
@@ -337,21 +322,14 @@ async fn get_package_file(
|
||||
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
|
||||
) -> Result<String> {
|
||||
let (package, version, filepath) = parse_package_version_and_file(full_path.to_path())?;
|
||||
let npm_registry = get_npm_registry(&db).await?;
|
||||
|
||||
if npm_registry.is_none() {
|
||||
return Err(Error::BadRequest(
|
||||
"No private npm registry configured".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let registry_url = npm_registry.unwrap();
|
||||
let (registry_url, auth_token) = get_npm_registry(&db)
|
||||
.await?
|
||||
.ok_or_else(|| Error::BadRequest("No private npm registry configured".to_string()))?;
|
||||
let package_url = format_registry_url(®istry_url, &package, None, None);
|
||||
|
||||
tracing::info!("Fetching package file from: {}", package_url);
|
||||
|
||||
let response = HTTP_CLIENT
|
||||
.get(&package_url)
|
||||
let response = build_registry_request(&package_url, &auth_token)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
|
||||
@@ -368,7 +346,6 @@ async fn get_package_file(
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
|
||||
|
||||
// Get the tarball URL for this version
|
||||
let tarball_url = package_json
|
||||
.get("versions")
|
||||
.and_then(|v| v.get(&version))
|
||||
@@ -377,9 +354,7 @@ async fn get_package_file(
|
||||
.and_then(|t| t.as_str())
|
||||
.ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?;
|
||||
|
||||
// Download tarball
|
||||
let tarball_response = HTTP_CLIENT
|
||||
.get(tarball_url)
|
||||
let tarball_response = build_registry_request(tarball_url, &auth_token)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?;
|
||||
@@ -402,13 +377,38 @@ async fn get_package_file(
|
||||
Ok(file_content)
|
||||
}
|
||||
|
||||
/// Get the npm registry URL from global settings
|
||||
async fn get_npm_registry(db: &sqlx::Pool<sqlx::Postgres>) -> Result<Option<String>> {
|
||||
/// Get the npm registry URL and optional auth token from global settings.
|
||||
/// Checks the `npmrc` setting first, then falls back to `npm_config_registry`.
|
||||
async fn get_npm_registry(
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
) -> Result<Option<(String, Option<String>)>> {
|
||||
let npmrc = load_value_from_global_settings(db, NPMRC_SETTING)
|
||||
.await?
|
||||
.and_then(|v| v.as_str().map(|s| s.to_string()));
|
||||
|
||||
if let Some(ref npmrc_content) = npmrc {
|
||||
if let Some(parsed) = parse_npmrc_registry(npmrc_content) {
|
||||
return Ok(Some(parsed));
|
||||
}
|
||||
}
|
||||
|
||||
let registry = load_value_from_global_settings(db, NPM_CONFIG_REGISTRY_SETTING)
|
||||
.await?
|
||||
.and_then(|v| v.as_str().map(|s| s.to_string()));
|
||||
|
||||
Ok(registry)
|
||||
if let Some(ref s) = registry {
|
||||
let (url, token) = if s.contains(":_authToken=") {
|
||||
let parts: Vec<&str> = s.split(":_authToken=").collect();
|
||||
let url = parts[0].to_string();
|
||||
let token = parts.get(1).map(|t| t.to_string());
|
||||
(url, token)
|
||||
} else {
|
||||
(s.clone(), None)
|
||||
};
|
||||
return Ok(Some((url, token)));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Format a registry URL for a package
|
||||
|
||||
@@ -43,8 +43,9 @@ use windmill_common::{
|
||||
get_database_url,
|
||||
global_settings::{
|
||||
APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING,
|
||||
CRITICAL_ALERT_MUTE_UI_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
|
||||
HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING,
|
||||
CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
|
||||
EMAIL_DOMAIN_SETTING, ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING,
|
||||
HUB_BASE_URL_SETTING,
|
||||
},
|
||||
instance_config::{self, ApplyMode, InstanceConfig},
|
||||
server::Smtp,
|
||||
@@ -512,6 +513,7 @@ pub async fn get_global_setting(
|
||||
&& !key.starts_with("default_recovery_handler_")
|
||||
&& !key.starts_with("default_success_handler_")
|
||||
&& key != AUTOMATE_USERNAME_CREATION_SETTING
|
||||
&& key != DEFAULT_TAGS_WORKSPACES_SETTING
|
||||
&& key != HUB_BASE_URL_SETTING
|
||||
&& key != HUB_ACCESSIBLE_URL_SETTING
|
||||
&& key != EMAIL_DOMAIN_SETTING
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: "3.0.3"
|
||||
|
||||
info:
|
||||
version: 1.638.4
|
||||
version: 1.642.0
|
||||
title: Windmill API
|
||||
|
||||
contact:
|
||||
@@ -19671,6 +19671,8 @@ components:
|
||||
type: boolean
|
||||
lock:
|
||||
type: string
|
||||
flow_path:
|
||||
type: string
|
||||
required:
|
||||
- args
|
||||
|
||||
|
||||
@@ -42,8 +42,6 @@ use windmill_common::runnable_settings::{
|
||||
};
|
||||
#[cfg(feature = "inline_preview")]
|
||||
use windmill_common::runtime_assets::{register_runtime_asset, InsertRuntimeAssetParams};
|
||||
use windmill_types::s3::BundleFormat;
|
||||
use windmill_object_store::upload_artifact_to_store;
|
||||
use windmill_common::scripts::ScriptRunnableSettingsInline;
|
||||
use windmill_common::triggers::TriggerMetadata;
|
||||
use windmill_common::utils::{RunnableKind, WarnAfterExt};
|
||||
@@ -54,8 +52,10 @@ use windmill_common::workspace_dependencies::{
|
||||
use windmill_common::DYNAMIC_INPUT_CACHE;
|
||||
#[cfg(all(feature = "enterprise", feature = "smtp"))]
|
||||
use windmill_common::{email_oss::send_email_html, server::load_smtp_config};
|
||||
use windmill_object_store::upload_artifact_to_store;
|
||||
#[cfg(feature = "inline_preview")]
|
||||
use windmill_parser::asset_parser::AssetKind;
|
||||
use windmill_types::s3::BundleFormat;
|
||||
#[cfg(feature = "inline_preview")]
|
||||
use windmill_worker::get_worker_internal_server_inline_utils;
|
||||
|
||||
@@ -1386,7 +1386,8 @@ async fn get_logs_from_store(
|
||||
log_file_index: &Option<Vec<String>>,
|
||||
) -> Option<error::Result<Body>> {
|
||||
use futures::StreamExt;
|
||||
let stream = windmill_object_store::get_logs_from_store(log_offset, logs, log_file_index).await?;
|
||||
let stream =
|
||||
windmill_object_store::get_logs_from_store(log_offset, logs, log_file_index).await?;
|
||||
let header = bytes::Bytes::from(
|
||||
r#"to remove ansi colors, use: | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g'
|
||||
"#
|
||||
@@ -2849,6 +2850,7 @@ struct Preview {
|
||||
dedicated_worker: Option<bool>,
|
||||
lock: Option<String>,
|
||||
format: Option<String>,
|
||||
flow_path: Option<String>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "inline_preview")]
|
||||
@@ -4509,6 +4511,14 @@ async fn run_preview_script(
|
||||
check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into());
|
||||
|
||||
let preview_args = preview.args.unwrap_or_default();
|
||||
let flow_path_extra = preview.flow_path.map(|fp| {
|
||||
let mut extra = HashMap::new();
|
||||
extra.insert("_FLOW_PATH".to_string(), to_raw_value(&fp));
|
||||
extra
|
||||
});
|
||||
let push_args = PushArgs { extra: flow_path_extra, args: &preview_args };
|
||||
|
||||
let (uuid, tx) = push(
|
||||
&db,
|
||||
tx,
|
||||
@@ -4532,7 +4542,7 @@ async fn run_preview_script(
|
||||
dedicated_worker: preview.dedicated_worker,
|
||||
}),
|
||||
},
|
||||
PushArgs::from(&preview.args.unwrap_or_default()),
|
||||
push_args,
|
||||
authed.display_username(),
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
@@ -5772,7 +5782,9 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
if let Some(os) = windmill_object_store::get_object_store().await {
|
||||
let file = os
|
||||
.get(&windmill_object_store::object_store_reexports::Path::from(format!("logs/{file_p}")))
|
||||
.get(&windmill_object_store::object_store_reexports::Path::from(
|
||||
format!("logs/{file_p}"),
|
||||
))
|
||||
.await;
|
||||
if let Ok(file) = file {
|
||||
if let Ok(bytes) = file.bytes().await {
|
||||
|
||||
@@ -10,6 +10,7 @@ pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb";
|
||||
pub const LICENSE_KEY_SETTING: &str = "license_key";
|
||||
pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry";
|
||||
pub const BUNFIG_INSTALL_SCOPES_SETTING: &str = "bunfig_install_scopes";
|
||||
pub const NPMRC_SETTING: &str = "npmrc";
|
||||
pub const NUGET_CONFIG_SETTING: &str = "nuget_config";
|
||||
pub const POWERSHELL_REPO_URL_SETTING: &str = "powershell_repo_url";
|
||||
pub const POWERSHELL_REPO_PAT_SETTING: &str = "powershell_repo_pat";
|
||||
|
||||
@@ -261,6 +261,8 @@ pub struct GlobalSettings {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bunfig_install_scopes: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub npmrc: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub nuget_config: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub maven_repos: Option<String>,
|
||||
@@ -774,7 +776,11 @@ pub const PROTECTED_SETTINGS: &[&str] = &[
|
||||
/// Note: jwt_secret is intentionally NOT hidden — it is included in YAML exports so that
|
||||
/// operators can set it via ConfigMap. It is protected from deletion (PROTECTED_SETTINGS)
|
||||
/// and from being set to empty/null, and its value is partially redacted in log output.
|
||||
pub const HIDDEN_SETTINGS: &[&str] = &["uid", "min_keep_alive_version", "automate_username_creation"];
|
||||
pub const HIDDEN_SETTINGS: &[&str] = &[
|
||||
"uid",
|
||||
"min_keep_alive_version",
|
||||
"automate_username_creation",
|
||||
];
|
||||
|
||||
/// Top-level settings whose entire value is sensitive and must be fully redacted in logs.
|
||||
const SENSITIVE_SETTINGS: &[&str] = &[
|
||||
@@ -788,6 +794,7 @@ const SENSITIVE_SETTINGS: &[&str] = &[
|
||||
"pip_extra_index_url",
|
||||
"npm_config_registry",
|
||||
"bunfig_install_scopes",
|
||||
"npmrc",
|
||||
"maven_repos",
|
||||
"ruby_repos",
|
||||
"powershell_repo_pat",
|
||||
@@ -798,7 +805,10 @@ const SENSITIVE_SETTINGS: &[&str] = &[
|
||||
const NESTED_SENSITIVE_FIELDS: &[(&str, &[&str])] = &[
|
||||
("smtp_settings", &["smtp_password"]),
|
||||
("secret_backend", &["token"]),
|
||||
("object_store_cache_config", &["secret_key", "serviceAccountKey"]),
|
||||
(
|
||||
"object_store_cache_config",
|
||||
&["secret_key", "serviceAccountKey"],
|
||||
),
|
||||
];
|
||||
|
||||
fn redact_json_value(value: &serde_json::Value) -> serde_json::Value {
|
||||
@@ -2353,7 +2363,11 @@ mod tests {
|
||||
);
|
||||
|
||||
let diff = diff_global_settings(¤t, &desired, ApplyMode::Merge);
|
||||
assert_eq!(diff.upserts.len(), 1, "Same client with newer expiry should update even with different signature");
|
||||
assert_eq!(
|
||||
diff.upserts.len(),
|
||||
1,
|
||||
"Same client with newer expiry should update even with different signature"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -404,30 +404,6 @@ impl PgDatabase {
|
||||
)
|
||||
}
|
||||
|
||||
pub fn to_conn_str(&self) -> String {
|
||||
format!(
|
||||
"dbname={dbname} {user} host={host} {password} {port} {sslmode}",
|
||||
dbname = self.dbname,
|
||||
user = self
|
||||
.user
|
||||
.as_ref()
|
||||
.map(|u| format!("user={}", urlencoding::encode(u)))
|
||||
.unwrap_or_default(),
|
||||
host = self.host,
|
||||
password = self
|
||||
.password
|
||||
.as_ref()
|
||||
.map(|p| format!("password={}", urlencoding::encode(p)))
|
||||
.unwrap_or_default(),
|
||||
port = self.port.map(|p| format!("port={}", p)).unwrap_or_default(),
|
||||
sslmode = self
|
||||
.sslmode
|
||||
.as_ref()
|
||||
.map(|s| format!("sslmode={}", s.clone()))
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn connect(
|
||||
&self,
|
||||
) -> Result<(tokio_postgres::Client, TokioPgConnection), error::Error> {
|
||||
|
||||
@@ -1281,3 +1281,96 @@ mod tests {
|
||||
assert_eq!(parsed, serde_json::json!([[1], [2], [3], [4], [5]]));
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse .npmrc content to extract the default registry URL and its auth token.
|
||||
/// Returns `Some((registry_url, Option<auth_token>))` if a default registry is found.
|
||||
pub fn parse_npmrc_registry(npmrc_content: &str) -> Option<(String, Option<String>)> {
|
||||
let mut registry_url: Option<String> = None;
|
||||
let mut auth_tokens: Vec<(String, String)> = Vec::new();
|
||||
|
||||
for line in npmrc_content.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') || line.starts_with(';') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(url) = line.strip_prefix("registry=") {
|
||||
registry_url = Some(url.trim().to_string());
|
||||
}
|
||||
|
||||
if line.starts_with("//") {
|
||||
if let Some((prefix, token)) = line.split_once(":_authToken=") {
|
||||
auth_tokens.push((prefix.to_string(), token.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let url = registry_url?;
|
||||
let url_without_protocol = url.trim_start_matches("https:").trim_start_matches("http:");
|
||||
let url_prefix = url_without_protocol.trim_end_matches('/');
|
||||
|
||||
let token = auth_tokens
|
||||
.iter()
|
||||
.find(|(prefix, _)| {
|
||||
let p = prefix.trim_end_matches('/');
|
||||
p == url_prefix
|
||||
})
|
||||
.map(|(_, token)| token.clone());
|
||||
|
||||
Some((url, token))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod npmrc_tests {
|
||||
use super::parse_npmrc_registry;
|
||||
|
||||
#[test]
|
||||
fn test_parse_simple_registry() {
|
||||
let npmrc = "registry=https://registry.mycompany.com/\n//registry.mycompany.com/:_authToken=secret123\n";
|
||||
let result = parse_npmrc_registry(npmrc);
|
||||
assert_eq!(
|
||||
result,
|
||||
Some((
|
||||
"https://registry.mycompany.com/".to_string(),
|
||||
Some("secret123".to_string())
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_registry_without_auth() {
|
||||
let npmrc = "registry=https://registry.npmjs.org/\n";
|
||||
let result = parse_npmrc_registry(npmrc);
|
||||
assert_eq!(
|
||||
result,
|
||||
Some(("https://registry.npmjs.org/".to_string(), None))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_scoped_only_no_default() {
|
||||
let npmrc =
|
||||
"@myorg:registry=https://registry.myorg.com/\n//registry.myorg.com/:_authToken=tok\n";
|
||||
let result = parse_npmrc_registry(npmrc);
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_with_comments() {
|
||||
let npmrc = "# My registry\nregistry=https://r.example.com/\n; auth\n//r.example.com/:_authToken=tok\n";
|
||||
let result = parse_npmrc_registry(npmrc);
|
||||
assert_eq!(
|
||||
result,
|
||||
Some((
|
||||
"https://r.example.com/".to_string(),
|
||||
Some("tok".to_string())
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_empty_npmrc() {
|
||||
assert_eq!(parse_npmrc_registry(""), None);
|
||||
assert_eq!(parse_npmrc_registry("# just a comment"), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -427,7 +427,7 @@ fn format_pull_query(peek: String) -> String {
|
||||
j.same_worker, j.pre_run_error, j.visible_to_owner,
|
||||
j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job,
|
||||
j.timeout, j.flow_step_id, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle, j.priority, j.raw_code, j.raw_lock, j.raw_flow,
|
||||
j.script_entrypoint_override, j.preprocessed, pj.runnable_path as parent_runnable_path,
|
||||
j.script_entrypoint_override, j.preprocessed, COALESCE(pj.runnable_path, j.args->>'_FLOW_PATH') as parent_runnable_path,
|
||||
COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin,
|
||||
p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email
|
||||
FROM q, j
|
||||
|
||||
@@ -576,7 +576,18 @@ pub async fn exchange_token(
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// Refresh an OAuth token and update the database
|
||||
/// Pre-fetched account fields needed for token refresh.
|
||||
pub struct OAuthAccountInfo {
|
||||
pub client: String,
|
||||
pub refresh_token: String,
|
||||
pub grant_type: String,
|
||||
pub cc_client_id: Option<String>,
|
||||
pub cc_client_secret: Option<String>,
|
||||
pub cc_token_url: Option<String>,
|
||||
}
|
||||
|
||||
/// Refresh an OAuth token and update the database.
|
||||
/// Fetches the account from DB, then delegates to `refresh_token_for_account`.
|
||||
pub async fn refresh_token<'c>(
|
||||
mut tx: Transaction<'c, Postgres>,
|
||||
path: &str,
|
||||
@@ -587,7 +598,8 @@ pub async fn refresh_token<'c>(
|
||||
http_client: &reqwest::Client,
|
||||
connect_configs_json: &str,
|
||||
) -> error::Result<String> {
|
||||
let account = sqlx::query!(
|
||||
let account = sqlx::query_as!(
|
||||
OAuthAccountInfo,
|
||||
"SELECT client, refresh_token, grant_type, cc_client_id, cc_client_secret, cc_token_url FROM account WHERE workspace_id = $1 AND id = $2",
|
||||
w_id,
|
||||
id,
|
||||
@@ -595,6 +607,22 @@ pub async fn refresh_token<'c>(
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
let account = windmill_common::utils::not_found_if_none(account, "Account", &id.to_string())?;
|
||||
|
||||
refresh_token_for_account(tx, path, w_id, id, db, account, oauth_clients, http_client, connect_configs_json).await
|
||||
}
|
||||
|
||||
/// Refresh an OAuth token given pre-fetched account info (no additional SELECT).
|
||||
pub async fn refresh_token_for_account<'c>(
|
||||
mut tx: Transaction<'c, Postgres>,
|
||||
path: &str,
|
||||
w_id: &str,
|
||||
id: i32,
|
||||
db: &DB,
|
||||
account: OAuthAccountInfo,
|
||||
oauth_clients: &AllClients,
|
||||
http_client: &reqwest::Client,
|
||||
connect_configs_json: &str,
|
||||
) -> error::Result<String> {
|
||||
let oauth_client_info = oauth_clients
|
||||
.connects
|
||||
.get(&account.client)
|
||||
|
||||
@@ -5321,7 +5321,11 @@ async fn push_inner<'c, 'd>(
|
||||
.as_ref()
|
||||
.map(|x| {
|
||||
let tag_lang = if x == &ScriptLang::Bunnative {
|
||||
ScriptLang::Nativets.as_str()
|
||||
if job_kind == JobKind::Dependencies {
|
||||
ScriptLang::Bun.as_str()
|
||||
} else {
|
||||
ScriptLang::Nativets.as_str()
|
||||
}
|
||||
} else {
|
||||
x.as_str()
|
||||
};
|
||||
@@ -5364,7 +5368,6 @@ async fn push_inner<'c, 'd>(
|
||||
job_id,
|
||||
&args,
|
||||
&mut tx,
|
||||
_db,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
@@ -6204,7 +6207,7 @@ pub async fn get_same_worker_job(
|
||||
v2_job.raw_code,
|
||||
v2_job.raw_lock,
|
||||
v2_job.raw_flow,
|
||||
pj.runnable_path as parent_runnable_path,
|
||||
COALESCE(pj.runnable_path, v2_job.args->>'_FLOW_PATH') as parent_runnable_path,
|
||||
p.email as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin,
|
||||
p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email
|
||||
FROM v2_job_queue
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,155 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::value::RawValue;
|
||||
|
||||
use crate::{
|
||||
create_nativets_runtime, execute_main, load_client_module, load_user_module, CreatedRuntime,
|
||||
ExecuteError, MainArgs, NativeAnnotation,
|
||||
};
|
||||
|
||||
pub struct PrewarmedResult {
|
||||
pub result: Result<Box<RawValue>, String>,
|
||||
pub logs: String,
|
||||
}
|
||||
|
||||
pub struct ExecutingIsolate {
|
||||
result_rx: tokio::sync::oneshot::Receiver<PrewarmedResult>,
|
||||
handle: tokio::task::JoinHandle<anyhow::Result<()>>,
|
||||
}
|
||||
|
||||
impl ExecutingIsolate {
|
||||
pub async fn wait(self) -> anyhow::Result<PrewarmedResult> {
|
||||
let result = self
|
||||
.result_rx
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("isolate result channel closed"))?;
|
||||
self.handle
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("isolate thread panicked: {e}"))??;
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct PrewarmedIsolate {
|
||||
args_tx: Option<tokio::sync::oneshot::Sender<String>>,
|
||||
result_rx: Option<tokio::sync::oneshot::Receiver<PrewarmedResult>>,
|
||||
ready_rx: Option<tokio::sync::oneshot::Receiver<()>>,
|
||||
handle: Option<tokio::task::JoinHandle<anyhow::Result<()>>>,
|
||||
}
|
||||
|
||||
/// Parse a JSON args object and reorder into positional args matching `arg_names`.
|
||||
fn args_to_positional(args_json: &str, arg_names: &[String]) -> Vec<Option<Box<RawValue>>> {
|
||||
let map: HashMap<String, Box<RawValue>> = serde_json::from_str(args_json).unwrap_or_default();
|
||||
arg_names
|
||||
.iter()
|
||||
.map(|name| map.get(name).cloned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl PrewarmedIsolate {
|
||||
/// Spawn a new isolate on a blocking thread.
|
||||
///
|
||||
/// The isolate loads `env_code` + WINDMILL_CLIENT as `windmill.ts`,
|
||||
/// then loads `js_code` as `eval.ts`, and waits for args to execute.
|
||||
pub fn spawn(
|
||||
env_code: String,
|
||||
js_code: String,
|
||||
ann: NativeAnnotation,
|
||||
arg_names: Vec<String>,
|
||||
) -> Self {
|
||||
let (args_tx, args_rx) = tokio::sync::oneshot::channel::<String>();
|
||||
let (result_tx, result_rx) = tokio::sync::oneshot::channel::<PrewarmedResult>();
|
||||
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
|
||||
let handle = tokio::task::spawn_blocking(move || {
|
||||
let CreatedRuntime { mut js_runtime, log_receiver, mut memory_limit_rx } =
|
||||
create_nativets_runtime(ann, vec![])?;
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
|
||||
runtime.block_on(async {
|
||||
load_client_module(&mut js_runtime, &env_code).await?;
|
||||
load_user_module(&mut js_runtime, format!("{env_code}\n{js_code}")).await?;
|
||||
|
||||
let _ = ready_tx.send(());
|
||||
|
||||
let args_json = match args_rx.await {
|
||||
Ok(a) => a,
|
||||
Err(_) => return Ok(()),
|
||||
};
|
||||
|
||||
let positional = args_to_positional(&args_json, &arg_names);
|
||||
{
|
||||
let op_state = js_runtime.op_state();
|
||||
let mut op_state = op_state.borrow_mut();
|
||||
op_state.put(MainArgs { args: positional });
|
||||
}
|
||||
|
||||
let log_handle = tokio::spawn(async move {
|
||||
let mut log_receiver = log_receiver;
|
||||
let mut logs = String::new();
|
||||
while let Some(log) = log_receiver.recv().await {
|
||||
logs.push_str(&log);
|
||||
logs.push('\n');
|
||||
}
|
||||
logs
|
||||
});
|
||||
|
||||
let exec_result = tokio::select! {
|
||||
r = execute_main(&mut js_runtime, None, false, None) => r,
|
||||
_ = memory_limit_rx.recv() => {
|
||||
Err(ExecuteError::Script("Memory limit reached, killing isolate".to_string()))
|
||||
}
|
||||
};
|
||||
|
||||
let result = match exec_result {
|
||||
Ok(raw) => Ok(raw),
|
||||
Err(ExecuteError::Script(msg)) => Err(msg),
|
||||
Err(ExecuteError::Js { message, stack, .. }) => {
|
||||
let msg = message.unwrap_or_default();
|
||||
let err = match stack {
|
||||
Some(s) => format!("{msg}\n{s}"),
|
||||
None => msg,
|
||||
};
|
||||
Err(err)
|
||||
}
|
||||
};
|
||||
|
||||
drop(js_runtime);
|
||||
let logs = log_handle.await.unwrap_or_default();
|
||||
let _ = result_tx.send(PrewarmedResult { result, logs });
|
||||
Ok(())
|
||||
})
|
||||
});
|
||||
|
||||
PrewarmedIsolate {
|
||||
args_tx: Some(args_tx),
|
||||
result_rx: Some(result_rx),
|
||||
ready_rx: Some(ready_rx),
|
||||
handle: Some(handle),
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for the isolate to finish loading modules.
|
||||
pub async fn wait_ready(&mut self) -> anyhow::Result<()> {
|
||||
if let Some(rx) = self.ready_rx.take() {
|
||||
rx.await
|
||||
.map_err(|_| anyhow::anyhow!("isolate failed during pre-warm"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Send args and start execution. Returns an `ExecutingIsolate` that
|
||||
/// can be awaited independently, allowing the caller to pre-warm
|
||||
/// the next isolate in parallel.
|
||||
pub fn start_execution(mut self, args: String) -> ExecutingIsolate {
|
||||
let args_tx = self.args_tx.take().expect("start_execution called twice");
|
||||
let _ = args_tx.send(args);
|
||||
ExecutingIsolate {
|
||||
result_rx: self.result_rx.take().expect("result_rx missing"),
|
||||
handle: self.handle.take().expect("handle missing"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,9 @@
|
||||
//! TypeScript scripts via the nativets runtime. By isolating this here,
|
||||
//! deno_core compilation no longer blocks windmill-worker or windmill-api.
|
||||
|
||||
mod dedicated;
|
||||
pub use dedicated::{ExecutingIsolate, PrewarmedIsolate, PrewarmedResult};
|
||||
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
cell::RefCell,
|
||||
@@ -112,14 +115,15 @@ impl NetPermissions for PermissionsContainer {
|
||||
|
||||
// ── Types ────────────────────────────────────────────────────────────
|
||||
|
||||
struct MainArgs {
|
||||
args: Vec<Option<Box<RawValue>>>,
|
||||
pub(crate) struct MainArgs {
|
||||
pub(crate) args: Vec<Option<Box<RawValue>>>,
|
||||
}
|
||||
|
||||
struct LogString {
|
||||
pub s: mpsc::UnboundedSender<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NativeAnnotation {
|
||||
pub useragent: Option<String>,
|
||||
pub proxy: Option<(String, Option<(String, String)>)>,
|
||||
@@ -145,7 +149,7 @@ impl Drop for IsolateDropGuard {
|
||||
|
||||
static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH_SNAPSHOT.bin"));
|
||||
|
||||
const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js");
|
||||
pub(crate) const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js");
|
||||
|
||||
const ERROR_DIR: &str = const_format::concatcp!(TMP_DIR, "/native_errors");
|
||||
|
||||
@@ -177,7 +181,10 @@ pub fn setup_deno_runtime() -> anyhow::Result<()> {
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if !unrecognized_v8_flags.is_empty() {
|
||||
init_err = Some(format!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags));
|
||||
init_err = Some(format!(
|
||||
"Unrecognized V8 flags: {:?}",
|
||||
unrecognized_v8_flags
|
||||
));
|
||||
}
|
||||
|
||||
// Use an unprotected platform that doesn't enforce thread-isolated allocations
|
||||
@@ -349,6 +356,140 @@ fn op_log(op_state: Rc<RefCell<OpState>>, #[string] log: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Shared V8 runtime creation ───────────────────────────────────────
|
||||
|
||||
pub(crate) struct CreatedRuntime {
|
||||
pub(crate) js_runtime: JsRuntime,
|
||||
pub(crate) log_receiver: mpsc::UnboundedReceiver<String>,
|
||||
pub(crate) memory_limit_rx: mpsc::UnboundedReceiver<()>,
|
||||
}
|
||||
|
||||
/// Create a JsRuntime with the standard nativets extensions, heap limit
|
||||
/// callback, and log channel. Must be called on a blocking thread (not
|
||||
/// on the async tokio runtime) because V8 isolate creation is
|
||||
/// synchronous and potentially heavy.
|
||||
pub(crate) fn create_nativets_runtime(
|
||||
ann: NativeAnnotation,
|
||||
initial_args: Vec<Option<Box<RawValue>>>,
|
||||
) -> anyhow::Result<CreatedRuntime> {
|
||||
let ops = vec![op_get_static_args(), op_log()];
|
||||
let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() };
|
||||
|
||||
let fetch_options = deno_fetch::Options {
|
||||
root_cert_store_provider: None,
|
||||
user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()),
|
||||
proxy: ann.proxy.map(|x| deno_tls::Proxy {
|
||||
url: x.0,
|
||||
basic_auth: x
|
||||
.1
|
||||
.map(|(username, password)| deno_tls::BasicAuth { username, password }),
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let exts: Vec<Extension> = vec![
|
||||
deno_telemetry::deno_telemetry::init_ops(),
|
||||
deno_webidl::deno_webidl::init_ops(),
|
||||
deno_url::deno_url::init_ops(),
|
||||
deno_console::deno_console::init_ops(),
|
||||
deno_web::deno_web::init_ops::<PermissionsContainer>(Arc::new(BlobStore::default()), None),
|
||||
deno_fetch::deno_fetch::init_ops::<PermissionsContainer>(fetch_options),
|
||||
deno_net::deno_net::init_ops::<PermissionsContainer>(None, None),
|
||||
ext,
|
||||
];
|
||||
|
||||
let options = RuntimeOptions {
|
||||
is_main: true,
|
||||
extensions: exts,
|
||||
create_params: Some(
|
||||
deno_core::v8::CreateParams::default().heap_limits(0, 1024 * 1024 * 128),
|
||||
),
|
||||
startup_snapshot: Some(RUNTIME_SNAPSHOT),
|
||||
module_loader: Some(Rc::new(deno_core::FsModuleLoader)),
|
||||
extension_transpiler: None,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (memory_limit_tx, memory_limit_rx) = mpsc::unbounded_channel::<()>();
|
||||
|
||||
setup_deno_runtime().expect("V8 platform init failed");
|
||||
|
||||
let mut js_runtime = {
|
||||
let _v8_lock = V8_ISOLATE_CREATE_LOCK
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
JsRuntime::new(options)
|
||||
};
|
||||
|
||||
js_runtime.add_near_heap_limit_callback(move |x, y| {
|
||||
tracing::error!("heap limit reached: {x} {y}");
|
||||
if memory_limit_tx.send(()).is_err() {
|
||||
tracing::warn!(
|
||||
"memory limit notification channel closed - isolate may already be terminating"
|
||||
);
|
||||
}
|
||||
y * 2
|
||||
});
|
||||
|
||||
let (log_sender, log_receiver) = mpsc::unbounded_channel::<String>();
|
||||
|
||||
{
|
||||
let op_state = js_runtime.op_state();
|
||||
let mut op_state = op_state.borrow_mut();
|
||||
op_state.put(PermissionsContainer {});
|
||||
op_state.put(MainArgs { args: initial_args });
|
||||
op_state.put(LogString { s: log_sender });
|
||||
}
|
||||
|
||||
Ok(CreatedRuntime { js_runtime, log_receiver, memory_limit_rx })
|
||||
}
|
||||
|
||||
// ── Shared module-loading helpers ────────────────────────────────────
|
||||
|
||||
pub(crate) async fn load_client_module(
|
||||
js_runtime: &mut JsRuntime,
|
||||
env_code: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
js_runtime
|
||||
.load_side_es_module_from_code(
|
||||
&deno_core::resolve_url("file:///windmill.ts")
|
||||
.map_err(windmill_common::error::to_anyhow)?,
|
||||
format!("{env_code}\n{WINDMILL_CLIENT}"),
|
||||
)
|
||||
.await
|
||||
.map_err(windmill_common::error::to_anyhow)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn load_user_module(
|
||||
js_runtime: &mut JsRuntime,
|
||||
source: String,
|
||||
) -> anyhow::Result<()> {
|
||||
use anyhow::Context;
|
||||
js_runtime
|
||||
.load_side_es_module_from_code(
|
||||
&deno_core::resolve_url("file:///eval.ts")
|
||||
.map_err(windmill_common::error::to_anyhow)?,
|
||||
source,
|
||||
)
|
||||
.await
|
||||
.context("failed to load module")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extract a string result from a resolved V8 global and convert to `Box<RawValue>`.
|
||||
pub(crate) fn extract_global_string(
|
||||
js_runtime: &mut JsRuntime,
|
||||
global: v8::Global<v8::Value>,
|
||||
) -> Result<Box<RawValue>, String> {
|
||||
let scope = &mut js_runtime.handle_scope();
|
||||
let local = v8::Local::new(scope, global);
|
||||
match serde_v8::from_v8::<Option<String>>(scope, local) {
|
||||
Ok(s) => Ok(unsafe_raw(s.unwrap_or_else(|| "null".to_string()))),
|
||||
Err(e) => Err(format!("failed to deserialize result: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
// ── eval_fetch_timeout ───────────────────────────────────────────────
|
||||
|
||||
/// Execute a NativeTS script using deno_core/V8.
|
||||
@@ -436,63 +577,9 @@ pub async fn eval_fetch_timeout(
|
||||
}
|
||||
|
||||
let result_f = tokio::task::spawn_blocking(move || {
|
||||
let ops = vec![op_get_static_args(), op_log()];
|
||||
let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() };
|
||||
let CreatedRuntime { mut js_runtime, mut log_receiver, mut memory_limit_rx } =
|
||||
create_nativets_runtime(ann, spread)?;
|
||||
|
||||
let fetch_options = deno_fetch::Options {
|
||||
root_cert_store_provider: None,
|
||||
user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()),
|
||||
proxy: ann.proxy.map(|x| deno_tls::Proxy {
|
||||
url: x.0,
|
||||
basic_auth: x
|
||||
.1
|
||||
.map(|(username, password)| deno_tls::BasicAuth { username, password }),
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let exts: Vec<Extension> = vec![
|
||||
deno_telemetry::deno_telemetry::init_ops(),
|
||||
deno_webidl::deno_webidl::init_ops(),
|
||||
deno_url::deno_url::init_ops(),
|
||||
deno_console::deno_console::init_ops(),
|
||||
deno_web::deno_web::init_ops::<PermissionsContainer>(
|
||||
Arc::new(BlobStore::default()),
|
||||
None,
|
||||
),
|
||||
deno_fetch::deno_fetch::init_ops::<PermissionsContainer>(fetch_options),
|
||||
deno_net::deno_net::init_ops::<PermissionsContainer>(None, None),
|
||||
ext,
|
||||
];
|
||||
|
||||
let options = RuntimeOptions {
|
||||
is_main: true,
|
||||
extensions: exts,
|
||||
create_params: Some(
|
||||
deno_core::v8::CreateParams::default().heap_limits(0, 1024 * 1024 * 128),
|
||||
),
|
||||
startup_snapshot: Some(RUNTIME_SNAPSHOT),
|
||||
module_loader: Some(Rc::new(deno_core::FsModuleLoader)),
|
||||
extension_transpiler: None,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (memory_limit_tx, mut memory_limit_rx) = mpsc::unbounded_channel::<()>();
|
||||
|
||||
// Ensure V8 platform is initialized (idempotent, no-op if already done).
|
||||
setup_deno_runtime().expect("V8 platform init failed");
|
||||
|
||||
// Serialize isolate creation as extra safety net against concurrent V8
|
||||
// isolate creation races. The main fix is the unprotected platform in
|
||||
// setup_deno_runtime(), but this provides defense in depth.
|
||||
let mut js_runtime = {
|
||||
let _v8_lock = V8_ISOLATE_CREATE_LOCK
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
JsRuntime::new(options)
|
||||
};
|
||||
|
||||
// Bootstrap OpenTelemetry for fetch auto-instrumentation if OTEL was initialized.
|
||||
if otel_initialized {
|
||||
if let Err(e) =
|
||||
js_runtime.execute_script("<otel_bootstrap>", "globalThis.__bootstrapOtel()")
|
||||
@@ -501,24 +588,6 @@ pub async fn eval_fetch_timeout(
|
||||
}
|
||||
}
|
||||
|
||||
js_runtime.add_near_heap_limit_callback(move |x, y| {
|
||||
tracing::error!("heap limit reached: {x} {y}");
|
||||
if memory_limit_tx.send(()).is_err() {
|
||||
tracing::error!("failed to send memory limit reached notification - isolate may already be terminating");
|
||||
};
|
||||
y * 2
|
||||
});
|
||||
|
||||
let (log_sender, mut log_receiver) = mpsc::unbounded_channel::<String>();
|
||||
|
||||
{
|
||||
let op_state = js_runtime.op_state();
|
||||
let mut op_state = op_state.borrow_mut();
|
||||
op_state.put(PermissionsContainer {});
|
||||
op_state.put(MainArgs { args: spread });
|
||||
op_state.put(LogString { s: log_sender });
|
||||
}
|
||||
|
||||
*isolate_handle.lock().unwrap_or_else(|e| e.into_inner()) =
|
||||
Some(js_runtime.v8_isolate().thread_safe_handle());
|
||||
|
||||
@@ -601,42 +670,97 @@ async fn eval_fetch(
|
||||
script_entrypoint_override: Option<String>,
|
||||
load_client: bool,
|
||||
job_id: &Uuid,
|
||||
_otel_initialized: bool,
|
||||
otel_initialized: bool,
|
||||
) -> windmill_common::error::Result<Box<RawValue>> {
|
||||
if load_client {
|
||||
if let Some(env_code) = env_code.as_ref() {
|
||||
let _ = js_runtime
|
||||
.load_side_es_module_from_code(
|
||||
&deno_core::resolve_url("file:///windmill.ts")
|
||||
.map_err(windmill_common::error::to_anyhow)?,
|
||||
format!("{env_code}\n{}", WINDMILL_CLIENT.to_string()),
|
||||
)
|
||||
.await
|
||||
.map_err(windmill_common::error::to_anyhow)?;
|
||||
load_client_module(js_runtime, env_code).await?;
|
||||
}
|
||||
}
|
||||
use anyhow::Context;
|
||||
use deno_core::error::CoreError;
|
||||
use windmill_common::worker::to_raw_value;
|
||||
let source = format!("{}\n{expr}", env_code.unwrap_or_default());
|
||||
let _ = js_runtime
|
||||
.load_side_es_module_from_code(
|
||||
&deno_core::resolve_url("file:///eval.ts")
|
||||
.map_err(windmill_common::error::to_anyhow)?,
|
||||
source.to_string(),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
write_error_expr(expr, &job_id);
|
||||
e
|
||||
})
|
||||
.context("failed to load module")?;
|
||||
if let Err(e) = load_user_module(js_runtime, source.clone()).await {
|
||||
write_error_expr(expr, job_id);
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
let main_override = script_entrypoint_override.unwrap_or("main".to_string());
|
||||
let result = execute_main(
|
||||
js_runtime,
|
||||
script_entrypoint_override.as_deref(),
|
||||
otel_initialized,
|
||||
Some(job_id),
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(raw) => Ok(raw),
|
||||
Err(ExecuteError::Script(msg)) => {
|
||||
write_error_expr(expr, job_id);
|
||||
Err(Error::ExecutionErr(msg))
|
||||
}
|
||||
Err(ExecuteError::Js { message, stack, name, source: eval_source }) => {
|
||||
write_error_expr(expr, job_id);
|
||||
use windmill_common::worker::to_raw_value;
|
||||
let stack_head = eval_source.and_then(|(file, line_no)| {
|
||||
if file == "file:///eval.ts" {
|
||||
source
|
||||
.lines()
|
||||
.nth(line_no.saturating_sub(1))
|
||||
.map(|l| format!("{l}\n"))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
let stack_s = format!(
|
||||
"{}{}",
|
||||
stack_head.unwrap_or_default(),
|
||||
stack.as_deref().unwrap_or_default()
|
||||
);
|
||||
let stack = if stack_s.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(stack_s)
|
||||
};
|
||||
Err(Error::ExecutionRawError(to_raw_value(&serde_json::json!({
|
||||
"message": message,
|
||||
"stack": stack,
|
||||
"name": name,
|
||||
}))))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Shared execution engine ──────────────────────────────────────────
|
||||
|
||||
pub(crate) enum ExecuteError {
|
||||
/// Non-JS error (V8 internal, init failure, deserialization)
|
||||
Script(String),
|
||||
/// JS exception with structured error info
|
||||
Js {
|
||||
message: Option<String>,
|
||||
stack: Option<String>,
|
||||
name: Option<String>,
|
||||
/// (file_name, line_number) from the first stack frame, if in user code
|
||||
source: Option<(String, usize)>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Execute the `main` function from the already-loaded `eval.ts` module.
|
||||
///
|
||||
/// Args must already be set in `MainArgs` in the runtime's OpState.
|
||||
/// Modules (`windmill.ts` and `eval.ts`) must already be loaded.
|
||||
pub(crate) async fn execute_main(
|
||||
js_runtime: &mut JsRuntime,
|
||||
entrypoint: Option<&str>,
|
||||
_otel_initialized: bool,
|
||||
_job_id: Option<&Uuid>,
|
||||
) -> Result<Box<RawValue>, ExecuteError> {
|
||||
let main_fn = entrypoint.unwrap_or("main");
|
||||
|
||||
#[cfg(all(feature = "private", feature = "enterprise"))]
|
||||
let otel_context_inject = if _otel_initialized {
|
||||
let trace_id = job_id.as_simple().to_string();
|
||||
let trace_id = _job_id
|
||||
.map(|id| id.as_simple().to_string())
|
||||
.unwrap_or_default();
|
||||
format!(
|
||||
r#"globalThis.__enterSpan?.({{
|
||||
isRecording: () => true,
|
||||
@@ -687,7 +811,7 @@ function processStreamIterative(res) {{
|
||||
{otel_context_inject}
|
||||
|
||||
let args = Deno.core.ops.op_get_static_args().map(JSON.parse)
|
||||
import("file:///eval.ts").then((module) => module.{main_override}(...args))
|
||||
import("file:///eval.ts").then((module) => module.{main_fn}(...args))
|
||||
.then(res => {{
|
||||
if (isAsyncIterable(res)) {{
|
||||
return processStreamIterative(res)
|
||||
@@ -698,60 +822,25 @@ import("file:///eval.ts").then((module) => module.{main_override}(...args))
|
||||
"#
|
||||
),
|
||||
)
|
||||
.map_err(|e| {
|
||||
write_error_expr(expr, &job_id);
|
||||
e
|
||||
})
|
||||
.context("native script initialization")?;
|
||||
.map_err(|e| ExecuteError::Script(format!("native script initialization: {e}")))?;
|
||||
|
||||
let fut = js_runtime.resolve(script);
|
||||
let global = js_runtime
|
||||
.with_event_loop_promise(fut, PollEventLoopOptions::default())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
write_error_expr(expr, &job_id);
|
||||
e
|
||||
});
|
||||
.await;
|
||||
|
||||
match global {
|
||||
Ok(global) => {
|
||||
let scope = &mut js_runtime.handle_scope();
|
||||
let local = v8::Local::new(scope, global);
|
||||
let r = serde_v8::from_v8::<Option<String>>(scope, local)
|
||||
.map_err(windmill_common::error::to_anyhow)?;
|
||||
Ok(unsafe_raw(r.unwrap_or_else(|| "null".to_string())))
|
||||
extract_global_string(js_runtime, global).map_err(|e| ExecuteError::Script(e))
|
||||
}
|
||||
Err(CoreError::Js(e)) => {
|
||||
let stack_head = e.frames.first().and_then(|f| {
|
||||
if f.file_name.as_ref().is_some_and(|x| x == "file:///eval.ts") {
|
||||
Some(format!(
|
||||
"{}\n",
|
||||
source
|
||||
.lines()
|
||||
.nth((f.line_number.unwrap_or(1)) as usize - 1)
|
||||
.unwrap_or("")
|
||||
.to_string()
|
||||
))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
Err(deno_core::error::CoreError::Js(e)) => {
|
||||
let source = e.frames.first().and_then(|f| {
|
||||
f.file_name
|
||||
.as_ref()
|
||||
.map(|name| (name.clone(), f.line_number.unwrap_or(1) as usize))
|
||||
});
|
||||
let stack_s = format!(
|
||||
"{}{}",
|
||||
stack_head.unwrap_or("".to_string()),
|
||||
e.stack.unwrap_or("".to_string())
|
||||
);
|
||||
let stack = if stack_s.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(stack_s)
|
||||
};
|
||||
Err(Error::ExecutionRawError(to_raw_value(&serde_json::json!({
|
||||
"message": e.message,
|
||||
"stack": stack,
|
||||
"name": e.name,
|
||||
}))))
|
||||
Err(ExecuteError::Js { message: e.message, stack: e.stack, name: e.name, source })
|
||||
}
|
||||
Err(e) => Err(Error::ExecutionErr(e.print_with_cause())),
|
||||
Err(e) => Err(ExecuteError::Script(e.print_with_cause())),
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user