starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-23 00:00:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into hc/hub-raw-apps

Browse Source
This commit is contained in:
Ruben Fiszel
2026-02-04 16:32:35 +00:00
committed by GitHub
parent ea965a06f2 0caa533fbd
commit 1d06c65c55
121 changed files with 7258 additions and 1710 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.claude/hooks/format-backend.sh
Executable
+20
View File
@@ -0,0 +1,20 @@
#!/bin/bash
# Format backend Rust files with rustfmt after Claude edits them
# Get the file path from the tool result (passed via stdin as JSON)
INPUT=$(cat)
FILE_PATH=$(echo "$INPUT" | jq -r '.tool_input.file_path // empty')
# Exit if no file path
if [ -z "$FILE_PATH" ]; then
exit 0
fi
# Check if the file is in the backend directory and is a Rust file
if [[ "$FILE_PATH" == *"/backend/"* ]] && [[ "$FILE_PATH" =~ \.rs$ ]]; then
cd "$CLAUDE_PROJECT_DIR/backend" || exit 0
# Run rustfmt with config from rustfmt.toml (edition=2021)
rustfmt --config-path rustfmt.toml "$FILE_PATH" 2>/dev/null || true
fi
exit 0
.claude/hooks/format-frontend.sh
Executable
+23
View File
@@ -0,0 +1,23 @@
#!/bin/bash
# Format frontend files with prettier after Claude edits them
# Get the file path from the tool result (passed via stdin as JSON)
INPUT=$(cat)
FILE_PATH=$(echo "$INPUT" | jq -r '.tool_input.file_path // empty')
# Exit if no file path
if [ -z "$FILE_PATH" ]; then
exit 0
fi
# Check if the file is in the frontend directory
if [[ "$FILE_PATH" == *"/frontend/"* ]]; then
# Check if it's a formattable file type
if [[ "$FILE_PATH" =~ \.(ts|js|svelte|json|css|html|md)$ ]]; then
cd "$CLAUDE_PROJECT_DIR/frontend" || exit 0
# Run prettier silently, don't fail the hook if prettier fails
npx prettier --write "$FILE_PATH" 2>/dev/null || true
fi
fi
exit 0
.claude/hooks/notify-user.sh
Executable
+25
View File
@@ -0,0 +1,25 @@
#!/bin/bash
# Notify user when Claude requires input (works on macOS and Linux)
# Check if we're in an SSH session
if [[ -n "$SSH_CLIENT" || -n "$SSH_TTY" || -n "$SSH_CONNECTION" ]]; then
# SSH session - use terminal bell
# If using VSCode, enable audible terminal bell for SSH sessions:
# Add the following to .vscode/settings.json:
# "accessibility.signals.terminalBell": {
# "sound": "on"
# },
# "terminal.integrated.enableVisualBell": true
printf '\a'
else
# Local session - use native notifications
if [[ "$OSTYPE" == "darwin"* ]]; then
osascript -e 'display notification "Claude is waiting for your input" with title "Claude Code" sound name "Glass"' 2>/dev/null || printf '\a'
elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
notify-send "Claude Code" "Claude is waiting for your input" 2>/dev/null || printf '\a'
else
printf '\a'
fi
fi
exit 0
.claude/settings.json
+33 -1
View File
@@ -23,7 +23,11 @@
"Bash(git log:*)",
"Bash(git branch:*)",
"Bash(git show:*)",
"Bash(git blame:*)"
"Bash(git blame:*)",
"Bash(cargo check:*)",
"mcp__ide__getDiagnostics",
"Bash(npm run generate-backend-client:*)",
"Bash(npm run check:*)"
],
"deny": [
"Read(.env)",
@@ -91,6 +95,34 @@
}
]
}
],
"PostToolUse": [
{
"matcher": "Edit|Write",
"hooks": [
{
"type": "command",
"command": "\"$CLAUDE_PROJECT_DIR\"/.claude/hooks/format-frontend.sh",
"timeout": 30
},
{
"type": "command",
"command": "\"$CLAUDE_PROJECT_DIR\"/.claude/hooks/format-backend.sh",
"timeout": 30
}
]
}
],
"Notification": [
{
"hooks": [
{
"type": "command",
"command": "\"$CLAUDE_PROJECT_DIR\"/.claude/hooks/notify-user.sh",
"timeout": 10
}
]
}
]
},
"enabledPlugins": {
.github/DockerfileBackendTests
+1 -1
View File
@@ -42,7 +42,7 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER
RUN /usr/local/bin/python3 -m pip install pip-tools
# Bun
COPY --from=oven/bun:1.2.23 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.3.8 /usr/local/bin/bun /usr/bin/bun
ARG TARGETPLATFORM
.github/workflows/backend-test.yml
+113 -4
View File
@@ -44,7 +44,10 @@ jobs:
go-version: 1.21.5
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.1.43
bun-version: 1.3.8
- uses: actions/setup-node@v4
with:
node-version: '20'
- uses: astral-sh/setup-uv@v6.2.1
with:
version: "0.9.24"
@@ -67,6 +70,111 @@ jobs:
- name: Substitute EE code (EE logic is behind feature flag)
run: |
./substitute_ee_code.sh --copy --dir ./windmill-ee-private
- name: Setup private npm registry with test package
working-directory: /tmp
run: |
set -e
# Install Verdaccio globally
npm install -g verdaccio
# Create Verdaccio config that requires authentication for @windmill-test packages
mkdir -p /tmp/verdaccio/storage
cat > /tmp/verdaccio/config.yaml << 'VERDACCIO_CONFIG'
storage: /tmp/verdaccio/storage
auth:
htpasswd:
file: /tmp/verdaccio/htpasswd
max_users: 100
uplinks:
npmjs:
url: https://registry.npmjs.org/
packages:
'@windmill-test/*':
access: $authenticated
publish: $authenticated
'@*/*':
access: $all
publish: $authenticated
proxy: npmjs
'**':
access: $all
publish: $authenticated
proxy: npmjs
server:
keepAliveTimeout: 60
middlewares:
audit:
enabled: true
log: { type: stdout, format: pretty, level: warn }
VERDACCIO_CONFIG
# Create empty htpasswd file (users will be created via API)
touch /tmp/verdaccio/htpasswd
# Start Verdaccio in background
verdaccio --config /tmp/verdaccio/config.yaml &
VERDACCIO_PID=$!
# Wait for Verdaccio to be ready
echo "Waiting for Verdaccio to start..."
for i in {1..30}; do
if curl -s http://localhost:4873/-/ping > /dev/null 2>&1; then
echo "Verdaccio is ready"
break
fi
sleep 1
done
# Login to get a token
echo "Getting auth token..."
RESPONSE=$(curl -s -X PUT \
-H "Content-Type: application/json" \
-d '{"name":"testuser","password":"testpass123"}' \
http://localhost:4873/-/user/org.couchdb.user:testuser)
echo "Auth response: $RESPONSE"
NPM_TOKEN=$(echo "$RESPONSE" | jq -r '.token')
if [ -z "$NPM_TOKEN" ] || [ "$NPM_TOKEN" = "null" ]; then
echo "Failed to get NPM token from response"
exit 1
fi
echo "NPM_TOKEN=${NPM_TOKEN}" >> $GITHUB_ENV
echo "Got NPM token successfully: ${NPM_TOKEN:0:10}..."
# Configure npm globally with the auth token
echo "//localhost:4873/:_authToken=${NPM_TOKEN}" > ~/.npmrc
echo "Configured ~/.npmrc with auth token"
# Create a simple test package
mkdir -p /tmp/windmill-test-private-pkg
cat > /tmp/windmill-test-private-pkg/package.json << 'PKG_JSON'
{
"name": "@windmill-test/private-pkg",
"version": "1.0.0",
"main": "index.js"
}
PKG_JSON
cat > /tmp/windmill-test-private-pkg/index.js << 'PKG_JS'
module.exports.greet = (name) => `Hello from private package, ${name}!`;
PKG_JS
# Publish to Verdaccio with auth
cd /tmp/windmill-test-private-pkg
echo "Publishing package..."
npm publish --registry http://localhost:4873
echo "Package published successfully"
# Verify the package requires auth by trying anonymous access (should fail)
rm -f ~/.npmrc
echo "Testing anonymous access (should fail)..."
if npm view @windmill-test/private-pkg --registry http://localhost:4873 2>/dev/null; then
echo "ERROR: Package should require authentication but anonymous access worked"
exit 1
fi
echo "Verified: Package requires authentication for @windmill-test/private-pkg"
- name: Cache DuckDB FFI module build
uses: actions/cache@v3
with:
@@ -84,9 +192,10 @@ jobs:
RUST_LOG_STYLE: never
CARGO_NET_GIT_FETCH_WITH_CLI: true
WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1
WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1
WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1
WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1
TEST_NPM_REGISTRY: "http://localhost:4873/:_authToken=${{ env.NPM_TOKEN }}"
run: |
deno --version && bun -v && go version && python3 --version
deno --version && bun -v && node --version && go version && python3 --version
cd windmill-duckdb-ffi-internal && ./build_dev.sh && cd ..
DENO_PATH=$(which deno) BUN_PATH=$(which bun) GO_PATH=$(which go) UV_PATH=$(which uv) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private --all -- --nocapture
DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test --all -- --nocapture
.gitignore
+6
View File
@@ -13,3 +13,9 @@ backend/.minio-data
.aider*
!.aiderignore
rust-client/Cargo.toml
# Symlinked cache directories (for git worktrees)
backend/target
frontend/node_modules
typescript-client/node_modules
frontend/.svelte-kit
CHANGELOG.md
+17
View File
@@ -1,5 +1,22 @@
# Changelog
## [1.624.0](https://github.com/windmill-labs/windmill/compare/v1.623.1...v1.624.0) (2026-02-03)
### Features
* default to quickjs on ce for flow eval ([#7756](https://github.com/windmill-labs/windmill/issues/7756)) ([bdf9447](https://github.com/windmill-labs/windmill/commit/bdf9447e821c6d02198534198a5878849cac23e5))
* runtime assets ([#7656](https://github.com/windmill-labs/windmill/issues/7656)) ([635a24f](https://github.com/windmill-labs/windmill/commit/635a24f82cae8e85b584efca115968872723889f))
### Bug Fixes
* **cli:** prevent branch-specific items from being marked for deletion on pull ([#7781](https://github.com/windmill-labs/windmill/issues/7781)) ([701eb4b](https://github.com/windmill-labs/windmill/commit/701eb4bae47a809e6da34c62b8e250ac6379db53))
* Fix app multiselect not refreshing result when creating element ([#7766](https://github.com/windmill-labs/windmill/issues/7766)) ([3a719ce](https://github.com/windmill-labs/windmill/commit/3a719cea6b7b099f32054957eb04148c592786ad))
* **frontend:** improve runs detail page ([#7694](https://github.com/windmill-labs/windmill/issues/7694)) ([3b5c165](https://github.com/windmill-labs/windmill/commit/3b5c1657c7d41178283d02017914543461565a3a))
* Prettier and less invasive toasts ([#7758](https://github.com/windmill-labs/windmill/issues/7758)) ([df51f96](https://github.com/windmill-labs/windmill/commit/df51f9690520db80db2133e2e61002f399c0dfaf))
* remove $schema field from Google AI output schema requests ([#7765](https://github.com/windmill-labs/windmill/issues/7765)) ([18d85f1](https://github.com/windmill-labs/windmill/commit/18d85f14127e50673ccb460bfa9ebe80730df68e))
## [1.623.1](https://github.com/windmill-labs/windmill/compare/v1.623.0...v1.623.1) (2026-02-01)
Dockerfile
+1 -1
View File
@@ -234,7 +234,7 @@ COPY --from=windmill_duckdb_ffi_internal_builder /windmill-duckdb-ffi-internal/t
COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.2.23 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.3.8 /usr/local/bin/bun /usr/bin/bun
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
README.md
+73 -168
View File
@@ -3,10 +3,10 @@
</p>
<p align=center>
Open-source developer infrastructure for internal tools (APIs, background jobs, workflows and UIs). Self-hostable alternative to Retool, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs and custom UIs to trigger workflows and scripts as internal apps.
Open-source developer platform for internal code: APIs, background jobs, workflows and UIs. Self-hostable alternative to Retool, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs and custom UIs to trigger workflows and scripts as internal apps.
<p align=center>
Scripts are turned into sharable UIs automatically, and can be composed together into flows or used into richer apps built with low-code. Supported script languages supported are: Python, TypeScript, Go, Bash, SQL, and GraphQL.
Scripts are turned into sharable UIs automatically, and can be composed together into flows or used into richer apps built with low-code. Supported languages: Python, TypeScript, Go, Bash, SQL, GraphQL, PowerShell, Rust, and more.
</p>
<p align="center">
@@ -36,75 +36,58 @@ Scripts are turned into sharable UIs automatically, and can be composed together
# Windmill - Developer platform for APIs, background jobs, workflows and UIs
Windmill is <b>fully open-sourced (AGPLv3)</b> and Windmill Labs offers
dedicated instance and commercial support and licenses.
Windmill is fully open-sourced (AGPLv3) and Windmill Labs offers dedicated instances and commercial support and licenses.
![Windmill Diagram](./imgs/stacks.svg)
https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-822f-0c7ee7104252
https://github.com/user-attachments/assets/d80de1d9-64de-4d89-aacd-6df23fa81fc4
- [Windmill - Developer platform for APIs, background jobs, workflows and UIs](#windmill---developer-platform-for-apis-background-jobs-workflows-and-uis)
- [Main Concepts](#main-concepts)
- [Show me some actual script code](#show-me-some-actual-script-code)
- [CLI](#cli)
- [Running scripts locally](#running-scripts-locally)
- [Local Development](#local-development)
- [Stack](#stack)
- [Fastest Self-Hostable Workflow Engine](#fastest-self-hostable-workflow-engine)
- [Security](#security)
- [Sandboxing](#sandboxing)
- [Secrets, credentials and sensitive values](#secrets-credentials-and-sensitive-values)
- [Performance](#performance)
- [Architecture](#architecture)
- [How to self-host](#how-to-self-host)
- [Docker compose](#docker-compose)
- [Kubernetes (k8s) and Helm charts](#kubernetes-k8s-and-helm-charts)
- [Run from binaries](#run-from-binaries)
- [Kubernetes (Helm charts)](#kubernetes-helm-charts)
- [Cloud providers](#cloud-providers)
- [OAuth, SSO \& SMTP](#oauth-sso--smtp)
- [Commercial license](#commercial-license)
- [License](#license)
- [Integrations](#integrations)
- [Environment Variables](#environment-variables)
- [Run a local dev setup](#run-a-local-dev-setup)
- [only Frontend](#only-frontend)
- [Frontend only](#frontend-only)
- [Backend + Frontend](#backend--frontend)
- [Contributors](#contributors)
- [Copyright](#copyright)
## Main Concepts
1. Define a minimal and generic script in Python, TypeScript, Go or Bash that
solves a specific task. The code can be defined in the
[provided Web IDE](https://www.windmill.dev/docs/code_editor) or
[synchronized with your own GitHub repo](https://www.windmill.dev/docs/advanced/cli/sync)
(e.g. through
[VS Code](https://www.windmill.dev/docs/cli_local_dev/vscode-extension)
extension):
1. Define a minimal and generic script in Python, TypeScript, Go or Bash that solves a specific task. The code can be defined in the provided Web IDE or synchronized with your own GitHub repo (e.g. through VS Code extension): [provided Web IDE](https://www.windmill.dev/docs/code_editor) or [synchronized with your own GitHub repo](https://www.windmill.dev/docs/advanced/cli/sync) (e.g. through [VS Code](https://www.windmill.dev/docs/cli_local_dev/vscode-extension) extension):
![Step 1](./imgs/windmill-editor.png)
![Step 1](./imgs/windmill-editor.png)
2. Your scripts parameters are automatically parsed and
[generate a frontend](https://www.windmill.dev/docs/core_concepts/auto_generated_uis).
2. Your scripts parameters are automatically parsed and [generate a frontend](https://www.windmill.dev/docs/core_concepts/auto_generated_uis).
![Step 2](./imgs/windmill-run.png)
![Step 3](./imgs/windmill-result.png)
3. Make it [flow](https://www.windmill.dev/docs/flows/flow_editor)! You can
chain your scripts or scripts made by the community shared on
[WindmillHub](https://hub.windmill.dev).
3. Make it [flow](https://www.windmill.dev/docs/flows/flow_editor)! You can chain your scripts or scripts made by the community shared on [WindmillHub](https://hub.windmill.dev).
![Step 3](./imgs/windmill-flow.png)
![Step 3](./imgs/windmill-flow.png)
4. Build [complex UIs](https://www.windmill.dev/docs/apps/app_editor) on top of
your scripts and flows.
4. Build [complex UIs](https://www.windmill.dev/docs/apps/app_editor) on top of your scripts and flows.
![Step 4](./imgs/windmill-builder.png)
![Step 4](./imgs/windmill-builder.png)
Scripts and flows can also be triggered by a
[cron schedule](https://www.windmill.dev/docs/core_concepts/scheduling) (e.g.
'_/5 _ \* \* \*') or through
[webhooks](https://www.windmill.dev/docs/core_concepts/webhooks).
Scripts and flows can be triggered by [schedules](https://www.windmill.dev/docs/core_concepts/scheduling), [webhooks](https://www.windmill.dev/docs/core_concepts/webhooks), [HTTP routes](https://www.windmill.dev/docs/core_concepts/http_routing), [Kafka](https://www.windmill.dev/docs/core_concepts/kafka_triggers), [WebSockets](https://www.windmill.dev/docs/core_concepts/websocket_triggers), [emails](https://www.windmill.dev/docs/core_concepts/email_triggers), and more.
You can build your entire infra on top of Windmill!
Build your entire infra on top of Windmill!
## Show me some actual script code
@@ -144,43 +127,31 @@ export async function main(
}
```
## CLI
## Local Development
We have a powerful CLI to interact with the windmill platform and sync your
scripts from local files, GitHub repos and to run scripts and flows on the
instance from local commands. See
[more details](https://www.windmill.dev/docs/advanced/cli).
Windmill supports multiple ways to develop locally and sync with your instance:
![CLI Screencast](./cli/vhs/output/setup.gif)
| Tool | Description |
|------|-------------|
| **[CLI](https://www.windmill.dev/docs/advanced/cli)** | Sync scripts from local files or GitHub, run scripts/flows from the command line |
| **[VS Code Extension](https://www.windmill.dev/docs/cli_local_dev/vscode-extension)** | Edit and test scripts & flows directly from VS Code / Cursor with full IDE support |
| **[Git Sync](https://www.windmill.dev/docs/advanced/git_sync)** | Two-way sync between Windmill and your Git repository |
| **[Claude Code](https://www.windmill.dev/docs/core_concepts/ai_generation)** | AI-assisted development with Claude for scripts, flows, and apps |
### Running scripts locally
https://github.com/user-attachments/assets/c541c326-e9ae-4602-a09a-1989aaded1e9
You can run your script locally easily, you simply need to pass the right
environment variables for the `wmill` client library to fetch resources and
variables from your instance if necessary. See more:
<https://www.windmill.dev/docs/advanced/local_development>.
To develop & test locally scripts & flows, we recommend using the Windmill VS
Code extension: <https://www.windmill.dev/docs/cli_local_dev/vscode-extension>.
You can run scripts locally by passing the right environment variables for the `wmill` client library to fetch resources and variables from your instance. See [local development docs](https://www.windmill.dev/docs/advanced/local_development).
## Stack
- Postgres as the database.
- Backend in Rust with the following highly-available and horizontally scalable.
Architecture:
- Stateless API backend.
- Workers that pull jobs from a queue in Postgres (and later, Kafka or Redis.
Upvote [#173](#https://github.com/windmill-labs/windmill/issues/173) if
interested).
- Frontend in Svelte.
- Scripts executions are sandboxed using Google's
[nsjail](https://github.com/google/nsjail).
- Javascript runtime is the
[deno_core rust library](https://denolib.gitbook.io/guide/) (which itself uses
the [rusty_v8](https://github.com/denoland/rusty_v8) and hence V8 underneath).
- TypeScript runtime is Bun and deno.
- Python runtime is python3.
- Golang runtime is 1.19.1.
- **Database**: Postgres (compatible with Aurora, Cloud SQL, Neon, Azure PostgreSQL)
- **Backend**: Rust - stateless API servers and workers pulling jobs from a Postgres queue
- **Frontend**: Svelte 5
- **Sandboxing**: [nsjail](https://github.com/google/nsjail) and PID namespace isolation
- **Runtimes**:
- TypeScript/JavaScript: Bun (default) and Deno
- Python: python3 with uv for dependency management
- Go, Bash, PowerShell, PHP, Rust, C#, Java, Ansible
## Fastest Self-Hostable Workflow Engine
@@ -197,19 +168,10 @@ page.
## Security
### Sandboxing
- **Sandboxing**: [nsjail](https://github.com/google/nsjail) for filesystem/resource isolation, and PID namespace isolation (enabled by default) to prevent jobs from accessing worker process memory
- **Secrets**: One encryption key per workspace for credentials stored in Windmill's K/V store. We recommend encrypting the Postgres database as well.
Windmill can use [nsjail](https://github.com/google/nsjail). It is production
multi-tenant grade secure. Do not take our word for it, take
[fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/).
### Secrets, credentials and sensitive values
There is one encryption key per workspace to encrypt the credentials and secrets
stored in Windmill's K/V store.
In addition, we strongly recommend that you encrypt the whole Postgres database.
That is what we do at <https://app.windmill.dev>.
See [Security documentation](https://www.windmill.dev/docs/advanced/security_isolation) for details.
## Performance
@@ -229,19 +191,13 @@ back to the database is ~50ms. A typical lightweight deno job will take around
## How to self-host
We only provide docker-compose setup here. For more advanced setups, like
compiling from source or using without a postgres super user, see
[Self-Host documentation](https://www.windmill.dev/docs/advanced/self_host).
For detailed setup options, see [Self-Host documentation](https://www.windmill.dev/docs/advanced/self_host).
### Docker compose
Windmill can be deployed using 3 files:
([docker-compose.yml](./docker-compose.yml), [Caddyfile](./Caddyfile) and a
[.env](./.env)) in a single command.
Deploy Windmill with 3 files ([docker-compose.yml](./docker-compose.yml), [Caddyfile](./Caddyfile), [.env](./.env)):
Make sure Docker is started, and run:
```
```bash
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env
@@ -249,86 +205,45 @@ curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env
docker compose up -d
```
Go to http://localhost et voilà :)
Go to http://localhost - default credentials: `admin@windmill.dev` / `changeme`
The default super-admin user is: admin@windmill.dev / changeme.
**Using an external database**: Set `DATABASE_URL` in `.env` to point to your managed Postgres (AWS RDS, GCP Cloud SQL, Azure, Neon, etc.) and set db replicas to 0.
From there, you can follow the setup app and create other users.
More details in
[Self-Host Documention](https://www.windmill.dev/docs/advanced/self_host#docker).
### Kubernetes (k8s) and Helm charts
We publish helm charts at:
<https://github.com/windmill-labs/windmill-helm-charts>.
### Run from binaries
Each release includes the corresponding binaries for x86_64. You can simply
download the latest `windmill` binary using the following set of bash commands.
### Kubernetes (Helm charts)
```bash
BINARY_NAME='windmill-amd64' # or windmill-ee-amd64 for the enterprise edition
LATEST_RELEASE=$(curl -L -s -H 'Accept: application/json' https://github.com/windmill-labs/windmill/releases/latest)
LATEST_VERSION=$(echo $LATEST_RELEASE | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
ARTIFACT_URL="https://github.com/windmill-labs/windmill/releases/download/$LATEST_VERSION/$BINARY_NAME"
wget "$ARTIFACT_URL" -O windmill
helm repo add windmill https://windmill-labs.github.io/windmill-helm-charts/
helm install windmill-chart windmill/windmill --namespace=windmill --create-namespace
```
See [windmill-helm-charts](https://github.com/windmill-labs/windmill-helm-charts) for configuration options.
### Cloud providers
Windmill works on AWS (EKS/ECS), GCP, Azure, Ubicloud, Fly.io, Render.com, Hetzner, Digital Ocean, and others. Rule of thumb: 1 worker per 1vCPU and 1-2 GB RAM.
### OAuth, SSO & SMTP
Windmill Community Edition allows to configure the OAuth, SSO (including Google
Workspace SSO, Microsoft/Azure and Okta) directly from the UI in the superadmin
settings. Do note that there is a limit of 10 SSO users on the community
edition.
Configure OAuth and SSO (Google Workspace, Microsoft/Azure, Okta) directly from the superadmin UI. [See documentation](https://www.windmill.dev/docs/misc/setup_oauth).
[See documentation](https://www.windmill.dev/docs/misc/setup_oauth).
### License
### Commercial license
The Community Edition is free to use internally. For commercial redistribution or managed services, contact <sales@windmill.dev>. See [LICENSE](./LICENSE) and [Pricing](https://www.windmill.dev/pricing) for details.
See the [LICENSE](https://github.com/windmill-labs/windmill/blob/main/LICENSE)
file for the full license text.
The "Community Edition" of Windmill available in the docker images hosted under ghcr.io/windmill-labs/windmill and the github binary releases contains the files under the AGPLv3 and Apache 2 sources but also includes proprietary and non-public code and features which are not open source and under the following terms: Windmill Labs, Inc. grants a right to use all the features of the "Community Edition" for free without restrictions other than the limits and quotas set in the software and a right to distribute the community edition as is but not to sell, resell, serve Windmill as a managed service, modify or wrap under any form without an explicit agreement.
The "Community Edition" of Windmill available in the docker images hosted under
ghcr.io/windmill-labs/windmill and the github binary releases contains the files
under the AGPLv3 and Apache 2 sources but also includes proprietary and
non-public code and features which are not open source and under the following
terms: Windmill Labs, Inc. grants a right to use all the features of the
"Community Edition" for free without restrictions other than the limits and
quotas set in the software and a right to distribute the community edition as is
but not to sell, resell, serve Windmill as a managed service, modify or wrap
under any form without an explicit agreement.
The binary compilable from source code in this repository without the "enterprise" feature flag is open-source under the [LICENSE-AGPLv3](https://github.com/windmill-labs/windmill/blob/main/LICENSE-AGPL) License terms and conditions.
The binary compilable from source code in this repository without the
"enterprise" feature flag is open-source under the
[LICENSE-AGPLv3](https://github.com/windmill-labs/windmill/blob/main/LICENSE-AGPL)
License terms and conditions.
To [re-expose directly any Windmill parts to your users](https://www.windmill.dev/docs/misc/white_labelling) as a feature of your product, with the exception of iframed public Windmill "apps", or to build a feature on top of "Windmill Community Edition" that you sell commercially or embed in a distributable product or binary, you must get a commercial license. Contact us at <sales@windmill.dev> if you have any questions. To do the same from the binary compiled from the source code in this repository without the "enterprise" feature flag, you must comply with the AGPLv3 license terms and conditions or get a commercial license from Windmill Labs, Inc.
To
[re-expose directly any Windmill parts to your users](https://www.windmill.dev/docs/misc/white_labelling)
as a feature of your product, with the exception of iframed public Windmill
"apps", or to build a feature on top of "Windmill Community Edition" that you
sell commercially or embed in a distributable product or binary, you must get a
commercial license. Contact us at <sales@windmill.dev> if you have any
questions. To do the same from the binary compiled from the source code in this
repository without the "enterprise" feature flag, you must comply with the
AGPLv3 license terms and conditions or get a commercial license from Windmill
Labs, Inc.
To use Windmill "Community Edition" as is internally in your organization, or to
use its APIs as is, you do NOT need a commercial license.
To use Windmill "Community Edition" as is internally in your organization, or to use its APIs as is, you do NOT need a commercial license.
### Integrations
In Windmill, integrations are referred to as
[resources and resource types](https://www.windmill.dev/docs/core_concepts/resources_and_types).
Each Resource has a Resource Type that defines the schema that the resource
In Windmill, integrations are referred to as [resources and resource types](https://www.windmill.dev/docs/core_concepts/resources_and_types). Each Resource has a Resource Type that defines the schema that the resource
needs to implement.
On self-hosted instances, you might want to import all the approved resource
types from [WindmillHub](https://hub.windmill.dev). A setup script will prompt
you to have it being synced automatically everyday.
On self-hosted instances, you might want to import all the approved resource types from [WindmillHub](https://hub.windmill.dev). A setup script will prompt you to have it being synced automatically everyday.
## Environment Variables
@@ -369,30 +284,20 @@ you to have it being synced automatically everyday.
## Run a local dev setup
Using [Nix](./frontend/README_DEV.md#nix) (Recommended).
We recommend using [Nix](./frontend/README_DEV.md#nix). See [./frontend/README_DEV.md](./frontend/README_DEV.md) for all options.
See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all
running options.
### Frontend only
### only Frontend
Uses the backend of <https://app.windmill.dev> with local frontend (hot-reload):
This will use the backend of <https://app.windmill.dev> but your own frontend
with hot-code reloading. Note that you will need to use a username / password
login due to CSRF checks using a different auth provider.
In the `frontend/` directory:
1. install the dependencies with `npm install` (or `pnpm install` or `yarn`)
2. generate the windmill client:
```
npm run generate-backend-client
## on mac use
npm run generate-backend-client-mac
```bash
cd frontend
npm install
npm run generate-backend-client # or generate-backend-client-mac on Mac
npm run dev
```
3. Run your dev server with `npm run dev`
4. Et voilà, windmill should be available at `http://localhost/`
Windmill available at `http://localhost/`
### Backend + Frontend
@@ -419,7 +324,7 @@ running options.
6. Go to `backend/`:
1. `env DATABASE_URL=<YOUR_DATABASE_URL> RUST_LOG=info cargo run`
2. You can specify any feature flag you want to enable, for example `cargo run --features python` to enable the python executor.
7. Et voilà, windmill should be available at `http://localhost:3000`
7. Windmill should be available at `http://localhost:3000`
## Contributors
@@ -429,4 +334,4 @@ running options.
## Copyright
Windmill Labs, Inc 2023
© 2023-2026 Windmill Labs, Inc.
backend/.sqlx/query-3b6bd7b41f130ce6df62fdecb351a3e01be0726d02d3f863e0ea5c476a8e785e.json
Generated
-26
View File
@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE kafka_trigger \n SET \n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $11,\n error_handler_args = $12,\n retry = $13\n WHERE \n workspace_id = $9 AND path = $10\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"VarcharArray",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "3b6bd7b41f130ce6df62fdecb351a3e01be0726d02d3f863e0ea5c476a8e785e"
}
backend/.sqlx/query-46f6a3665d11ef573e27ab14be8a80c78d7eb735bc9615ebf2668a5715d83349.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT (large_file_storage IS NOT NULL\n AND large_file_storage != 'null'::jsonb\n AND jsonb_typeof(large_file_storage) = 'object') AS \"has_primary!\"\n FROM workspace_settings WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "has_primary!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "46f6a3665d11ef573e27ab14be8a80c78d7eb735bc9615ebf2668a5715d83349"
}
backend/.sqlx/query-a750630d79166f5b6d5be0e049c36135212666ce74dbf290d701b6402b800f13.json
Generated
+35
View File
@@ -0,0 +1,35 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n jsonb_strip_nulls(jsonb_build_object(\n 'path', path,\n 'kind', kind,\n 'access_type', usage_access_type,\n 'columns', columns\n )) as \"list!: _\"\n FROM asset\n WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3\n ORDER BY path, kind",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "list!: _",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "asset_usage_kind",
"kind": {
"Enum": [
"script",
"flow",
"job"
]
}
}
}
]
},
"nullable": [
null
]
},
"hash": "a750630d79166f5b6d5be0e049c36135212666ce74dbf290d701b6402b800f13"
}
backend/.sqlx/query-a9e29764b5b9d94269e2b8aa755c71b61774c8ff8ae218d7a8d6ed0ac0169366.json
Generated
+55
View File
@@ -0,0 +1,55 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns)\n VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
{
"Custom": {
"name": "asset_kind",
"kind": {
"Enum": [
"s3object",
"resource",
"variable",
"ducklake",
"datatable"
]
}
}
},
{
"Custom": {
"name": "asset_access_type",
"kind": {
"Enum": [
"r",
"w",
"rw"
]
}
}
},
"Varchar",
{
"Custom": {
"name": "asset_usage_kind",
"kind": {
"Enum": [
"script",
"flow",
"job"
]
}
}
},
"Jsonb"
]
},
"nullable": []
},
"hash": "a9e29764b5b9d94269e2b8aa755c71b61774c8ff8ae218d7a8d6ed0ac0169366"
}
backend/.sqlx/query-1de3e078c108a8a0136fccdf9187cc3500260bac39c7f1dfa05a93628569465b.json → backend/.sqlx/query-aed5439aa6dad950e505f9f8f6914fa5ca21319c501b2822c9bc751ddfc9a0a4.json
Generated
+3 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n script_path,\n is_flow,\n mode,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now(), $11, $12, $13\n )\n ",
"query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n script_path,\n is_flow,\n mode,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), $12, $13, $14\n )\n ",
"describe": {
"columns": [],
"parameters": {
@@ -10,6 +10,7 @@
"Varchar",
"Varchar",
"VarcharArray",
"JsonbArray",
"Varchar",
"Bool",
{
@@ -33,5 +34,5 @@
},
"nullable": []
},
"hash": "1de3e078c108a8a0136fccdf9187cc3500260bac39c7f1dfa05a93628569465b"
"hash": "aed5439aa6dad950e505f9f8f6914fa5ca21319c501b2822c9bc751ddfc9a0a4"
}
backend/.sqlx/query-e2921e44c70cf6c76c55177f2b56985e84c59ecb3e1a13fcf27d5f7ae5f8d84c.json
Generated
+27
View File
@@ -0,0 +1,27 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE\n workspace_id = $10 AND path = $11\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"VarcharArray",
"JsonbArray",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "e2921e44c70cf6c76c55177f2b56985e84c59ecb3e1a13fcf27d5f7ae5f8d84c"
}
backend/Cargo.lock
Generated
+69 -69
View File
@@ -234,9 +234,9 @@ dependencies = [
[[package]]
name = "arc-swap"
version = "1.8.0"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d03449bb8ca2cc2ef70869af31463d1ae5ccc8fa3e334b307203fbf815207e"
checksum = "9ded5f9a03ac8f24d1b8a25101ee812cd32cdc8c50a4c50237de2c4915850e73"
dependencies = [
"rustversion",
]
@@ -490,7 +490,7 @@ dependencies = [
"memchr",
"num",
"regex",
"regex-syntax 0.8.8",
"regex-syntax 0.8.9",
]
[[package]]
@@ -2024,9 +2024,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
version = "1.11.0"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3"
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
dependencies = [
"serde",
]
@@ -2287,9 +2287,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.56"
version = "4.5.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75ca66430e33a14957acc24c5077b503e7d374151b2b4b3a10c83b4ceb4be0e"
checksum = "6899ea499e3fb9305a65d5ebf6e3d2248c5fab291f300ad0a704fbe142eae31a"
dependencies = [
"clap_builder",
"clap_derive",
@@ -2297,9 +2297,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.56"
version = "4.5.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "793207c7fa6300a0608d1080b858e5fdbe713cdc1c8db9fb17777d8a13e63df0"
checksum = "7b12c8b680195a62a8364d16b8447b01b6c2c8f9aaf68bee653be34d4245e238"
dependencies = [
"anstream",
"anstyle",
@@ -3471,7 +3471,7 @@ dependencies = [
"log",
"recursive",
"regex",
"regex-syntax 0.8.8",
"regex-syntax 0.8.9",
]
[[package]]
@@ -5478,7 +5478,7 @@ checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"
dependencies = [
"bit-set 0.8.0",
"regex-automata",
"regex-syntax 0.8.8",
"regex-syntax 0.8.9",
]
[[package]]
@@ -5489,7 +5489,7 @@ checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
dependencies = [
"bit-set 0.8.0",
"regex-automata",
"regex-syntax 0.8.8",
"regex-syntax 0.8.9",
]
[[package]]
@@ -5610,9 +5610,9 @@ dependencies = [
[[package]]
name = "flate2"
version = "1.1.8"
version = "1.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369"
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
dependencies = [
"crc32fast",
"libz-sys",
@@ -5631,9 +5631,9 @@ dependencies = [
[[package]]
name = "float8"
version = "0.6.0"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f463a8a37ede13dac13316d1a1eeafa992300906a0c4c7fa1177f366d10bcbf"
checksum = "719a903cc23e4a89e87962c2a80fdb45cdaad0983a89bd150bb57b4c8571a7d5"
dependencies = [
"half",
"num-traits",
@@ -6183,7 +6183,7 @@ dependencies = [
"bstr",
"log",
"regex-automata",
"regex-syntax 0.8.8",
"regex-syntax 0.8.9",
]
[[package]]
@@ -7008,14 +7008,13 @@ dependencies = [
[[package]]
name = "hyper-util"
version = "0.1.19"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f"
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-channel",
"futures-core",
"futures-util",
"http 1.4.0",
"http-body 1.0.1",
@@ -10888,32 +10887,32 @@ dependencies = [
[[package]]
name = "regex"
version = "1.12.2"
version = "1.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4"
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax 0.8.8",
"regex-syntax 0.8.9",
]
[[package]]
name = "regex-automata"
version = "0.4.13"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c"
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax 0.8.8",
"regex-syntax 0.8.9",
]
[[package]]
name = "regex-lite"
version = "0.1.8"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da"
checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
[[package]]
name = "regex-syntax"
@@ -10923,9 +10922,9 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
[[package]]
name = "regex-syntax"
version = "0.8.8"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c"
[[package]]
name = "relative-path"
@@ -13468,9 +13467,9 @@ dependencies = [
[[package]]
name = "system-configuration"
version = "0.6.1"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
dependencies = [
"bitflags 2.9.4",
"core-foundation 0.9.4",
@@ -13600,7 +13599,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
dependencies = [
"byteorder",
"regex-syntax 0.8.8",
"regex-syntax 0.8.9",
"utf8-ranges",
]
@@ -14533,7 +14532,7 @@ checksum = "0203df02a3b6dd63575cc1d6e609edc2181c9a11867a271b25cfd2abff3ec5ca"
dependencies = [
"cc",
"regex",
"regex-syntax 0.8.8",
"regex-syntax 0.8.9",
"tree-sitter-language",
]
@@ -15488,7 +15487,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"aws-sdk-config",
@@ -15528,6 +15527,7 @@ dependencies = [
"sqlx",
"strum 0.27.2",
"systemstat",
"tempfile",
"tikv-jemalloc-ctl",
"tikv-jemalloc-sys",
"tikv-jemallocator",
@@ -15551,7 +15551,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"argon2",
@@ -15683,7 +15683,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -15693,7 +15693,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"chrono",
"lazy_static",
@@ -15707,7 +15707,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -15726,7 +15726,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15822,7 +15822,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"regex",
"serde",
@@ -15837,7 +15837,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -15861,7 +15861,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -15877,7 +15877,7 @@ dependencies = [
[[package]]
name = "windmill-mcp"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15897,7 +15897,7 @@ dependencies = [
[[package]]
name = "windmill-oauth"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"async-oauth2",
@@ -15921,7 +15921,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -15930,7 +15930,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15942,7 +15942,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15954,7 +15954,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"gosyn",
@@ -15966,7 +15966,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15978,7 +15978,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15990,7 +15990,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -16001,7 +16001,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16012,7 +16012,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16025,7 +16025,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16049,7 +16049,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16063,7 +16063,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -16080,7 +16080,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16094,7 +16094,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16113,7 +16113,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"serde",
@@ -16124,7 +16124,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16161,7 +16161,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"wasm-bindgen",
"wasm-bindgen-test",
@@ -16171,7 +16171,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.623.1"
version = "1.624.0"
dependencies = [
"anyhow",
"async-once-cell",
@@ -17072,18 +17072,18 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.37"
version = "0.8.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7456cf00f0685ad319c5b1693f291a650eaf345e941d082fc4e03df8a03996ac"
checksum = "57cf3aa6855b23711ee9852dfc97dfaa51c45feaba5b645d0c777414d494a961"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.37"
version = "0.8.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1328722bbf2115db7e19d69ebcc15e795719e2d66b60827c6a69a117365e37a0"
checksum = "8a616990af1a287837c4fe6596ad77ef57948f787e46ce28e166facc0cc1cb75"
dependencies = [
"proc-macro2",
"quote",
@@ -17178,9 +17178,9 @@ dependencies = [
[[package]]
name = "zlib-rs"
version = "0.5.5"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3"
checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c"
[[package]]
name = "zstd"
backend/Cargo.toml
+4 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.623.1"
version = "1.624.0"
authors.workspace = true
edition.workspace = true
@@ -35,7 +35,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal"]
[workspace.package]
version = "1.623.1"
version = "1.624.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -90,6 +90,7 @@ zip = ["windmill-api/zip"]
static_frontend = ["windmill-api/static_frontend"]
scoped_cache = ["windmill-common/scoped_cache"]
test_job_debouncing = []
private_registry_test = []
# Languages
python = ["windmill-worker/python", "windmill-api/python"]
rust = ["windmill-worker/rust"]
@@ -182,6 +183,7 @@ axum.workspace = true
serde.workspace = true
windmill-api-client.workspace = true
deno_core = { workspace = true, features = ["include_js_files_for_snapshotting", "unsafe_use_unprotected_platform"] }
tempfile.workspace = true
[workspace.dependencies]
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
138a4f5f868f3bded5bb7cb77b222b532c07e4af
88e49a7c9746080a8a95e30828655d5783a616d6
backend/migrations/20260130150931_kafka_trigger_filters.down.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE kafka_trigger DROP COLUMN filters;
backend/migrations/20260130150931_kafka_trigger_filters.up.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE kafka_trigger ADD COLUMN filters JSONB[] NOT NULL DEFAULT '{}';
backend/migrations/20260203122047_asset_columns.down.sql
+2
View File
@@ -0,0 +1,2 @@
-- Remove columns field from asset table
ALTER TABLE asset DROP COLUMN columns;
backend/migrations/20260203122047_asset_columns.up.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add columns field to asset table to store column-level access information
-- This is a JSONB map of column name to access type (r, w, or rw)
ALTER TABLE asset ADD COLUMN columns JSONB;
backend/migrations/20260203172950_polling_based_events.down.sql
+121
View File
@@ -0,0 +1,121 @@
-- Revert to pg_notify based event system
-- Restore notify_config_change function
CREATE OR REPLACE FUNCTION notify_config_change()
RETURNS TRIGGER AS $$
BEGIN
PERFORM pg_notify('notify_config_change', NEW.name::text);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Restore notify_global_setting_change function
CREATE OR REPLACE FUNCTION notify_global_setting_change()
RETURNS TRIGGER AS $$
BEGIN
PERFORM pg_notify('notify_global_setting_change', NEW.name::text);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Restore notify_global_setting_delete function
CREATE OR REPLACE FUNCTION notify_global_setting_delete()
RETURNS TRIGGER AS $$
BEGIN
PERFORM pg_notify('notify_global_setting_change', OLD.name::text);
RETURN OLD;
END;
$$ LANGUAGE plpgsql;
-- Restore notify_webhook_change function
CREATE OR REPLACE FUNCTION notify_webhook_change()
RETURNS TRIGGER AS $$
BEGIN
PERFORM pg_notify('notify_webhook_change', NEW.workspace_id);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Restore notify_workspace_envs_change function
CREATE OR REPLACE FUNCTION notify_workspace_envs_change()
RETURNS TRIGGER AS $$
BEGIN
PERFORM pg_notify('notify_workspace_envs_change', NEW.workspace_id);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Restore notify_workspace_premium_change function
CREATE OR REPLACE FUNCTION notify_workspace_premium_change()
RETURNS TRIGGER AS $$
BEGIN
PERFORM pg_notify('notify_workspace_premium_change', NEW.id);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Restore notify_team_plan_status_change function
CREATE OR REPLACE FUNCTION notify_team_plan_status_change()
RETURNS TRIGGER AS $$
BEGIN
PERFORM pg_notify('notify_workspace_premium_change', NEW.workspace_id);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Restore notify_runnable_version_change function
CREATE OR REPLACE FUNCTION notify_runnable_version_change()
RETURNS TRIGGER AS $$
DECLARE
source_type TEXT;
kind TEXT;
BEGIN
source_type := TG_ARGV[0];
IF source_type = 'script' THEN
kind := NEW.kind;
ELSE
kind := 'flow';
END IF;
PERFORM pg_notify('notify_runnable_version_change', NEW.workspace_id || ':' || source_type || ':' || NEW.path || ':' || kind);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Restore notify_http_trigger_change function
CREATE OR REPLACE FUNCTION notify_http_trigger_change()
RETURNS TRIGGER AS $$
BEGIN
PERFORM pg_notify('notify_http_trigger_change', NEW.workspace_id || ':' || NEW.path);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Restore notify_token_invalidation function
CREATE OR REPLACE FUNCTION notify_token_invalidation()
RETURNS TRIGGER AS $$
BEGIN
IF OLD.label = 'session' AND OLD.email IS NOT NULL THEN
PERFORM pg_notify('notify_token_invalidation', OLD.token);
END IF;
RETURN OLD;
END;
$$ LANGUAGE plpgsql;
-- Restore notify_workspace_key_change function
CREATE OR REPLACE FUNCTION notify_workspace_key_change()
RETURNS TRIGGER AS $$
BEGIN
IF TG_OP = 'DELETE' THEN
PERFORM pg_notify('notify_workspace_key_change', OLD.workspace_id);
RETURN OLD;
ELSE
PERFORM pg_notify('notify_workspace_key_change', NEW.workspace_id);
RETURN NEW;
END IF;
END;
$$ LANGUAGE plpgsql;
-- Drop the notify_event table
DROP TABLE IF EXISTS notify_event;
backend/migrations/20260203172950_polling_based_events.up.sql
+135
View File
@@ -0,0 +1,135 @@
-- Create notify_event table for polling-based event system
-- This replaces PostgreSQL LISTEN/NOTIFY with a table-based approach
CREATE TABLE IF NOT EXISTS notify_event (
id BIGSERIAL PRIMARY KEY,
channel TEXT NOT NULL,
payload TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS notify_event_created_at_idx ON notify_event (created_at);
-- Drop redundant index if it exists (id is already the PRIMARY KEY)
DROP INDEX IF EXISTS notify_event_id_idx;
-- Update notify_config_change function
CREATE OR REPLACE FUNCTION notify_config_change()
RETURNS TRIGGER AS $$
BEGIN
INSERT INTO notify_event (channel, payload) VALUES ('notify_config_change', NEW.name::text);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Update notify_global_setting_change function
CREATE OR REPLACE FUNCTION notify_global_setting_change()
RETURNS TRIGGER AS $$
BEGIN
INSERT INTO notify_event (channel, payload) VALUES ('notify_global_setting_change', NEW.name::text);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Update notify_global_setting_delete function
CREATE OR REPLACE FUNCTION notify_global_setting_delete()
RETURNS TRIGGER AS $$
BEGIN
INSERT INTO notify_event (channel, payload) VALUES ('notify_global_setting_change', OLD.name::text);
RETURN OLD;
END;
$$ LANGUAGE plpgsql;
-- Update notify_webhook_change function
CREATE OR REPLACE FUNCTION notify_webhook_change()
RETURNS TRIGGER AS $$
BEGIN
INSERT INTO notify_event (channel, payload) VALUES ('notify_webhook_change', NEW.workspace_id);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Update notify_workspace_envs_change function
CREATE OR REPLACE FUNCTION notify_workspace_envs_change()
RETURNS TRIGGER AS $$
BEGIN
INSERT INTO notify_event (channel, payload) VALUES ('notify_workspace_envs_change', COALESCE(NEW.workspace_id, OLD.workspace_id));
RETURN COALESCE(NEW, OLD);
END;
$$ LANGUAGE plpgsql;
-- Update notify_workspace_premium_change function
CREATE OR REPLACE FUNCTION notify_workspace_premium_change()
RETURNS TRIGGER AS $$
BEGIN
INSERT INTO notify_event (channel, payload) VALUES ('notify_workspace_premium_change', NEW.id);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Update notify_team_plan_status_change function
CREATE OR REPLACE FUNCTION notify_team_plan_status_change()
RETURNS TRIGGER AS $$
BEGIN
INSERT INTO notify_event (channel, payload) VALUES ('notify_workspace_premium_change', NEW.workspace_id);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Update notify_runnable_version_change function
CREATE OR REPLACE FUNCTION notify_runnable_version_change()
RETURNS TRIGGER AS $$
DECLARE
source_type TEXT;
kind TEXT;
BEGIN
source_type := TG_ARGV[0];
IF source_type = 'script' THEN
kind := NEW.kind;
ELSE
kind := 'flow';
END IF;
INSERT INTO notify_event (channel, payload) VALUES ('notify_runnable_version_change', NEW.workspace_id || ':' || source_type || ':' || NEW.path || ':' || kind);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Update notify_http_trigger_change function
CREATE OR REPLACE FUNCTION notify_http_trigger_change()
RETURNS TRIGGER AS $$
BEGIN
INSERT INTO notify_event (channel, payload) VALUES ('notify_http_trigger_change', COALESCE(NEW.workspace_id, OLD.workspace_id) || ':' || COALESCE(NEW.path, OLD.path));
RETURN COALESCE(NEW, OLD);
END;
$$ LANGUAGE plpgsql;
-- Update notify_token_invalidation function
CREATE OR REPLACE FUNCTION notify_token_invalidation()
RETURNS TRIGGER AS $$
BEGIN
IF OLD.label = 'session' AND OLD.email IS NOT NULL THEN
INSERT INTO notify_event (channel, payload) VALUES ('notify_token_invalidation', OLD.token);
END IF;
RETURN OLD;
END;
$$ LANGUAGE plpgsql;
-- Update notify_workspace_key_change function
CREATE OR REPLACE FUNCTION notify_workspace_key_change()
RETURNS TRIGGER AS $$
BEGIN
IF TG_OP = 'DELETE' THEN
INSERT INTO notify_event (channel, payload) VALUES ('notify_workspace_key_change', OLD.workspace_id);
RETURN OLD;
ELSE
INSERT INTO notify_event (channel, payload) VALUES ('notify_workspace_key_change', NEW.workspace_id);
RETURN NEW;
END IF;
END;
$$ LANGUAGE plpgsql;
-- NOTE: var_cache_invalidation / resource_cache_invalidation triggers were
-- intentionally dropped in migration 20250902085504. We do NOT re-create them
-- here to keep this migration scoped to the LISTEN/NOTIFY → polling swap only.
backend/parsers/windmill-parser-py/src/asset_parser.rs
+48 -21
View File
@@ -19,9 +19,12 @@ pub fn parse_assets(input: &str) -> anyhow::Result<ParseAssetsOutput> {
// if a db = wmill.datatable() was never used (e.g db.query(...)),
// we still want to register the asset as unknown access type
if asset_was_used(&assets_finder.assets, (kind, &path)) == false {
assets_finder
.assets
.push(ParseAssetsResult { kind, access_type: None, path });
assets_finder.assets.push(ParseAssetsResult {
kind,
path,
access_type: None,
columns: None,
});
}
}
@@ -48,8 +51,12 @@ impl Visitor for AssetsFinder {
match removed {
Some((kind, path, _)) => {
if !asset_was_used(&self.assets, (kind, &path)) {
self.assets
.push(ParseAssetsResult { kind, access_type: None, path });
self.assets.push(ParseAssetsResult {
kind,
path,
access_type: None,
columns: None,
});
}
}
None => {}
@@ -76,6 +83,7 @@ impl Visitor for AssetsFinder {
kind,
path: path.to_string(),
access_type: None,
columns: None,
});
}
}
@@ -97,6 +105,7 @@ impl Visitor for AssetsFinder {
kind,
path: path.to_string(),
access_type: None,
columns: None,
});
}
}
@@ -252,8 +261,12 @@ impl AssetsFinder {
let path = parse_asset_syntax(&value, false)
.map(|(_, p)| p)
.unwrap_or(&value);
self.assets
.push(ParseAssetsResult { kind, path: path.to_string(), access_type });
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
access_type,
columns: None,
});
}
_ => return Err(()),
};
@@ -281,7 +294,8 @@ def main():
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "/test.csv".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},])
);
}
@@ -299,7 +313,8 @@ def main():
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "main".to_string(),
access_type: None
access_type: None,
columns: None,
},])
);
}
@@ -318,7 +333,8 @@ def main(x: int):
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/friends".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},])
);
}
@@ -340,12 +356,14 @@ def main(x: int):
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/analytics".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/friends".to_string(),
access_type: Some(RW)
access_type: Some(RW),
columns: None,
},
])
);
@@ -372,17 +390,20 @@ def g():
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "another1/customers".to_string(),
access_type: Some(W)
access_type: Some(W),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "another2".to_string(),
access_type: None
access_type: None,
columns: None,
},
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "main/friends".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},
])
);
@@ -404,12 +425,14 @@ def g():
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "another1".to_string(),
access_type: None
access_type: None,
columns: None,
},
ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main".to_string(),
access_type: None
access_type: None,
columns: None,
},
])
);
@@ -429,7 +452,8 @@ def main(x: int):
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/public.friends".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},])
);
}
@@ -448,7 +472,8 @@ def main():
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "lake1/analytics.metrics".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},])
);
}
@@ -468,7 +493,8 @@ def main(x: int):
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/public.users".to_string(),
access_type: Some(RW)
access_type: Some(RW),
columns: None,
},])
);
}
@@ -486,7 +512,8 @@ def main():
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt".to_string(),
access_type: None
access_type: None,
columns: None,
},])
);
}
backend/parsers/windmill-parser-sql/src/asset_parser.rs
+585 -70
View File
@@ -1,9 +1,9 @@
use std::collections::HashMap;
use std::collections::BTreeMap;
use sqlparser::{
ast::{
CopyTarget, Expr, ObjectName, TableFactor, TableObject, Value, ValueWithSpan, Visit,
Visitor,
CopyTarget, Expr, ObjectName, ObjectNamePart, SelectItem, TableFactor, TableObject, Value,
ValueWithSpan, Visit, Visitor,
},
dialect::DuckDbDialect,
parser::Parser,
@@ -24,9 +24,12 @@ pub fn parse_assets(input: &str) -> anyhow::Result<ParseAssetsOutput> {
for (_, (kind, path)) in collector.var_identifiers {
if !asset_was_used(&collector.assets, (kind, &path)) {
collector
.assets
.push(ParseAssetsResult { kind, access_type: None, path: path });
collector.assets.push(ParseAssetsResult {
kind,
access_type: None,
path: path,
columns: None,
});
}
}
@@ -39,7 +42,7 @@ struct AssetCollector {
// e.g set to Read when we are inside a SELECT ... FROM ... statement
current_access_type_stack: Vec<AssetUsageAccessType>,
// e.g ATTACH 'ducklake://a' AS dl; => { "dl": (Ducklake, "a") }
var_identifiers: HashMap<String, (AssetKind, String)>,
var_identifiers: BTreeMap<String, (AssetKind, String)>,
// e.g USE dl;
currently_used_asset: Option<(AssetKind, String)>,
}
@@ -49,15 +52,19 @@ impl AssetCollector {
Self {
assets: Vec::new(),
current_access_type_stack: Vec::with_capacity(8),
var_identifiers: HashMap::new(),
var_identifiers: BTreeMap::new(),
currently_used_asset: None,
}
}
// Detect when we do 'a.b' and 'a' is associated with an asset in var_identifiers
// Or when we access 'b' and we did USE a;
fn get_associated_asset_from_obj_name(&self, name: &ObjectName) -> Option<ParseAssetsResult> {
let access_type = self.current_access_type_stack.last().copied();
fn get_associated_asset_from_obj_name(
&self,
name: &ObjectName,
access_type: Option<AssetUsageAccessType>,
) -> Option<ParseAssetsResult> {
let access_type = access_type.or_else(|| self.current_access_type_stack.last().copied());
if let Some((kind, path)) = &self.currently_used_asset {
// We don't want to infer that any simple identifier refers to an asset if
// we are not in a known R/W context
@@ -81,7 +88,7 @@ impl AssetCollector {
.collect::<Option<Vec<String>>>()?
.join(".");
let path = format!("{}/{}", path, specific_table);
return Some(ParseAssetsResult { kind: *kind, access_type, path });
return Some(ParseAssetsResult { kind: *kind, access_type, path, columns: None });
}
}
@@ -101,7 +108,7 @@ impl AssetCollector {
} else {
path.clone()
};
Some(ParseAssetsResult { kind: *kind, access_type, path })
Some(ParseAssetsResult { kind: *kind, access_type, path, columns: None })
}
fn handle_string_literal(&mut self, s: &str) {
@@ -112,6 +119,7 @@ impl AssetCollector {
kind,
path: path.to_string(),
access_type: self.current_access_type_stack.last().copied(),
columns: None,
});
}
}
@@ -126,13 +134,6 @@ impl AssetCollector {
if let Some(str_lit) = get_str_lit_from_obj_name(name) {
self.handle_string_literal(str_lit);
}
// Writes to tables should be handled directly when visiting the statement
if self.current_access_type_stack.last() == Some(&R) {
if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
self.assets.push(asset);
}
}
}
fn handle_obj_name_post(&mut self, name: &ObjectName) {
@@ -146,20 +147,144 @@ impl AssetCollector {
}
}
fn handle_table_with_joins(&mut self, table_with_joins: &sqlparser::ast::TableWithJoins) {
if let TableFactor::Table { name, .. } = &table_with_joins.relation {
if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
fn handle_table_with_joins(
&mut self,
table_with_joins: &sqlparser::ast::TableWithJoins,
access_type: Option<AssetUsageAccessType>,
) {
if let TableFactor::Table { name, args, .. } = &table_with_joins.relation {
if args.is_some() && args.as_ref().map_or(0, |a| a.args.len()) > 0 {
return;
}
if let Some(asset) = self.get_associated_asset_from_obj_name(name, access_type) {
self.assets.push(asset);
}
}
for join in &table_with_joins.joins {
if let TableFactor::Table { name, .. } = &join.relation {
if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
if let Some(asset) = self.get_associated_asset_from_obj_name(name, access_type) {
self.assets.push(asset);
}
}
}
}
// Extract columns from SELECT items and create individual asset results for each column
// Only processes columns that reference known assets to avoid false positives
fn extract_column_assets(
&mut self,
projection: &[SelectItem],
from_tables: &[sqlparser::ast::TableWithJoins],
) {
// Check if this is a single-table SELECT (to avoid ambiguity)
let single_table = if from_tables.len() == 1 {
if let TableFactor::Table { name, args, .. } = &from_tables[0].relation {
if args.is_some() && args.as_ref().map_or(0, |a| a.args.len()) > 0 {
return; // Skip table functions
}
self.get_associated_asset_from_obj_name(name, Some(R))
} else {
None
}
} else {
None
};
// Build a map of table aliases/names to assets for multi-table queries
let mut table_to_asset: BTreeMap<String, ParseAssetsResult> = BTreeMap::new();
for table_with_joins in from_tables {
if let TableFactor::Table { name, alias, args, .. } = &table_with_joins.relation {
if args.is_some() && args.as_ref().map_or(0, |a| a.args.len()) > 0 {
continue; // Skip table functions
}
if let Some(asset) = self.get_associated_asset_from_obj_name(name, Some(R)) {
// Use alias if present, otherwise use the table name
let table_key = if let Some(alias) = alias {
alias.name.value.clone()
} else {
// For qualified names like "dl.table1", use just the last part
name.0
.last()
.and_then(|id| id.as_ident())
.map(|id| id.value.clone())
.unwrap_or_default()
};
table_to_asset.insert(table_key, asset);
}
}
}
// Process each SELECT item
for item in projection {
match item {
SelectItem::UnnamedExpr(Expr::Identifier(ident))
| SelectItem::ExprWithAlias { expr: Expr::Identifier(ident), .. } => {
// Simple column: SELECT a
// Only add if we have a single table (unambiguous)
if let Some(asset) = &single_table {
let mut columns = BTreeMap::new();
columns.insert(ident.value.clone(), R);
self.assets.push(ParseAssetsResult {
kind: asset.kind,
path: asset.path.clone(),
access_type: Some(R),
columns: Some(columns),
});
}
}
SelectItem::UnnamedExpr(Expr::CompoundIdentifier(parts))
| SelectItem::ExprWithAlias { expr: Expr::CompoundIdentifier(parts), .. } => {
// Qualified column: SELECT table1.a or SELECT x.table1.a
if parts.len() >= 2 {
let column_name = parts.last().map(|id| id.value.clone());
if let Some(column_name) = column_name {
// Check if the prefix matches a known table
let table_prefix = parts.first().map(|id| id.value.clone());
if let Some(table_prefix) = table_prefix {
if let Some(asset) = table_to_asset.get(&table_prefix) {
// Found a matching table, add column asset
let mut columns = BTreeMap::new();
columns.insert(column_name.clone(), R);
self.assets.push(ParseAssetsResult {
kind: asset.kind,
path: asset.path.clone(),
access_type: Some(R),
columns: Some(columns),
});
} else if parts.len() >= 3 {
// Could be x.table1.column format or db.schema.table.column
// Convert Idents to ObjectNameParts
let obj_parts: Vec<ObjectNamePart> = parts[..parts.len() - 1]
.iter()
.cloned()
.map(|ident| ObjectNamePart::Identifier(ident))
.collect();
let obj_name = ObjectName(obj_parts);
if let Some(asset) =
self.get_associated_asset_from_obj_name(&obj_name, Some(R))
{
let mut columns = BTreeMap::new();
columns.insert(column_name.clone(), R);
self.assets.push(ParseAssetsResult {
kind: asset.kind,
path: asset.path.clone(),
access_type: Some(R),
columns: Some(columns),
});
}
}
}
}
}
}
_ => {
// Ignore wildcards, expressions, etc.
}
}
}
}
}
impl Visitor for AssetCollector {
@@ -218,51 +343,154 @@ impl Visitor for AssetCollector {
statement: &sqlparser::ast::Statement,
) -> std::ops::ControlFlow<Self::Break> {
match statement {
sqlparser::ast::Statement::Query(_) => {
// don't forget pop() in post_visit_statement
self.current_access_type_stack.push(R);
sqlparser::ast::Statement::Query(q) => {
if let Some(select) = q.body.as_select() {
// First, handle table references (adds table-level assets)
for t in &select.from {
self.handle_table_with_joins(t, Some(R));
}
// Then, extract column-level assets
self.extract_column_assets(&select.projection, &select.from);
}
}
sqlparser::ast::Statement::Insert(insert) => {
let access_type = if insert.returning.is_some() { RW } else { W };
self.current_access_type_stack.push(access_type);
match insert.table {
TableObject::TableName(ref name) => {
if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
self.assets.push(asset);
if let Some(asset) =
self.get_associated_asset_from_obj_name(name, Some(access_type))
{
// Add table-level asset
self.assets.push(ParseAssetsResult {
kind: asset.kind,
path: asset.path.clone(),
access_type: asset.access_type,
columns: None,
});
// Extract column information for INSERT with explicit columns (Write access)
if !insert.columns.is_empty() {
for col in &insert.columns {
let columns = BTreeMap::from([(col.value.clone(), W)]);
self.assets.push(ParseAssetsResult {
kind: asset.kind,
path: asset.path.clone(),
access_type: Some(W),
columns: Some(columns),
});
}
}
// Extract column information from RETURNING clause (Read access)
if let Some(returning) = &insert.returning {
for item in returning {
match item {
SelectItem::UnnamedExpr(Expr::Identifier(ident))
| SelectItem::ExprWithAlias {
expr: Expr::Identifier(ident),
..
} => {
let mut col_map = BTreeMap::new();
col_map.insert(ident.value.clone(), R);
self.assets.push(ParseAssetsResult {
kind: asset.kind,
path: asset.path.clone(),
access_type: Some(R),
columns: Some(col_map),
});
}
_ => {
// Ignore wildcards and complex expressions
}
}
}
}
}
}
_ => {}
}
self.current_access_type_stack.pop();
}
sqlparser::ast::Statement::Update { returning, table, from, .. } => {
sqlparser::ast::Statement::Update { returning, table, from, assignments, .. } => {
if let Some(from_tables) = from {
let from_tables = match from_tables {
sqlparser::ast::UpdateTableFromKind::AfterSet(tables) => tables,
sqlparser::ast::UpdateTableFromKind::BeforeSet(tables) => tables,
};
self.current_access_type_stack.push(R);
for table_with_joins in from_tables {
self.handle_table_with_joins(table_with_joins);
self.handle_table_with_joins(table_with_joins, Some(R));
}
self.current_access_type_stack.pop();
}
let access_type = if returning.is_some() { RW } else { W };
self.current_access_type_stack.push(access_type);
self.handle_table_with_joins(table, Some(access_type));
self.handle_table_with_joins(table);
// Extract column information from UPDATE SET clauses (Write access)
// Only process if it's a single table update
if let TableFactor::Table { name, .. } = &table.relation {
if let Some(asset) =
self.get_associated_asset_from_obj_name(name, Some(access_type))
{
// Process each assignment to extract column names
for assignment in assignments {
// assignment.target is an AssignmentTarget enum
// We only handle simple column names (ColumnName variant)
if let sqlparser::ast::AssignmentTarget::ColumnName(col_name) =
&assignment.target
{
// For simple column updates, this is typically a single ident
if col_name.0.len() == 1 {
if let Some(col_ident) =
col_name.0.first().and_then(|p| p.as_ident())
{
let mut col_map = BTreeMap::new();
col_map.insert(col_ident.value.clone(), W);
self.assets.push(ParseAssetsResult {
kind: asset.kind,
path: asset.path.clone(),
access_type: Some(W),
columns: Some(col_map),
});
}
}
}
}
self.current_access_type_stack.pop();
// Extract column information from RETURNING clause (Read access)
if let Some(returning_items) = returning {
for item in returning_items {
match item {
SelectItem::UnnamedExpr(Expr::Identifier(ident))
| SelectItem::ExprWithAlias {
expr: Expr::Identifier(ident),
..
} => {
let mut col_map = BTreeMap::new();
col_map.insert(ident.value.clone(), R);
self.assets.push(ParseAssetsResult {
kind: asset.kind,
path: asset.path.clone(),
access_type: Some(R),
columns: Some(col_map),
});
}
_ => {
// Ignore wildcards and complex expressions
}
}
}
}
}
}
}
sqlparser::ast::Statement::Delete(delete) => {
let access_type = if delete.returning.is_some() { RW } else { W };
self.current_access_type_stack.push(access_type);
for name in &delete.tables {
if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
if let Some(asset) =
self.get_associated_asset_from_obj_name(name, Some(access_type))
{
self.assets.push(asset);
}
}
@@ -271,25 +499,22 @@ impl Visitor for AssetCollector {
sqlparser::ast::FromTable::WithoutKeyword(tables) => tables,
};
for table_with_joins in tables {
self.handle_table_with_joins(table_with_joins);
self.handle_table_with_joins(table_with_joins, Some(access_type));
}
self.current_access_type_stack.pop();
}
sqlparser::ast::Statement::CreateTable(create_table) => {
self.current_access_type_stack.push(W);
if let Some(asset) = self.get_associated_asset_from_obj_name(&create_table.name) {
if let Some(asset) =
self.get_associated_asset_from_obj_name(&create_table.name, Some(W))
{
self.assets.push(asset);
}
self.current_access_type_stack.pop();
}
sqlparser::ast::Statement::CreateView { name, .. } => {
self.current_access_type_stack.push(W);
if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
if let Some(asset) = self.get_associated_asset_from_obj_name(name, Some(W)) {
self.assets.push(asset);
}
self.current_access_type_stack.pop();
}
sqlparser::ast::Statement::Copy { target: CopyTarget::File { filename }, .. } => {
@@ -339,14 +564,8 @@ impl Visitor for AssetCollector {
fn post_visit_statement(
&mut self,
statement: &sqlparser::ast::Statement,
_statement: &sqlparser::ast::Statement,
) -> std::ops::ControlFlow<Self::Break> {
match statement {
sqlparser::ast::Statement::Query(_) => {
self.current_access_type_stack.pop();
}
_ => {}
}
std::ops::ControlFlow::Continue(())
}
@@ -409,17 +628,20 @@ mod tests {
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "/a.parquet".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "/c.parquet".to_string(),
access_type: Some(W)
access_type: Some(W),
columns: None
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "snd/b.parquet".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None
},
])
);
@@ -438,7 +660,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "my_dl".to_string(),
access_type: None
access_type: None,
columns: None
},])
);
}
@@ -455,7 +678,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "my_dl/table1".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None
},])
);
}
@@ -473,7 +697,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "my_dt/table1".to_string(),
access_type: Some(W)
access_type: Some(W),
columns: None
},])
);
}
@@ -504,7 +729,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "my_dl/table1".to_string(),
access_type: Some(W)
access_type: Some(W),
columns: None
},])
);
}
@@ -521,7 +747,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "main/table1".to_string(),
access_type: Some(W)
access_type: Some(W),
columns: None
},])
);
}
@@ -543,7 +770,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main/friends".to_string(),
access_type: Some(RW)
access_type: Some(RW),
columns: None
},])
);
}
@@ -561,7 +789,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main".to_string(),
access_type: None
access_type: None,
columns: None
},])
);
}
@@ -579,7 +808,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main/table1".to_string(),
access_type: Some(W)
access_type: Some(W),
columns: None
},])
);
}
@@ -597,7 +827,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main/table1".to_string(),
access_type: Some(W)
access_type: Some(W),
columns: Some(BTreeMap::from([("id".to_string(), W)])),
},])
);
}
@@ -615,7 +846,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::Resource,
path: "u/user/pg_resource/table1".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None
},])
);
}
@@ -632,7 +864,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main/table1".to_string(),
access_type: Some(W)
access_type: Some(W),
columns: Some(BTreeMap::from([("id".to_string(), W)])),
},])
);
}
@@ -650,7 +883,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main/sch.table1".to_string(),
access_type: Some(RW)
access_type: Some(RW),
columns: Some(BTreeMap::from([("id".to_string(), W)])),
},])
);
}
@@ -669,8 +903,289 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main/sch.table1".to_string(),
access_type: Some(RW)
access_type: Some(RW),
columns: Some(BTreeMap::from([("id".to_string(), W)])),
},])
);
}
#[test]
fn test_sql_asset_parser_single_table_column_detection() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
SELECT a, b FROM dl.table1;
"#;
let s = parse_assets(input).map(|s| s.assets);
let result = s.unwrap();
// Should have one asset with merged columns
assert_eq!(result.len(), 1);
assert_eq!(result[0].path, "my_dl/table1");
assert_eq!(result[0].access_type, Some(R));
// Check that both columns are present in the merged asset
let columns = result[0].columns.as_ref().expect("Should have columns");
assert_eq!(columns.len(), 2);
assert_eq!(columns.get("a"), Some(&R));
assert_eq!(columns.get("b"), Some(&R));
}
#[test]
fn test_sql_asset_parser_explicit_table_prefix_columns() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
SELECT dl.table1.a, dl.table1.b FROM dl.table1;
"#;
let s = parse_assets(input).map(|s| s.assets);
// Should detect columns with explicit table prefix
let result = s.unwrap();
// Check we have the table asset
// Check we have column assets
assert!(result.iter().any(|a| {
a.path == "my_dl/table1"
&& a.columns
.as_ref()
.map_or(false, |cols| cols.contains_key("a"))
}));
assert!(result.iter().any(|a| {
a.path == "my_dl/table1"
&& a.columns
.as_ref()
.map_or(false, |cols| cols.contains_key("b"))
}));
}
#[test]
fn test_sql_asset_parser_multi_table_no_simple_columns() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
SELECT a, b FROM dl.table1, dl.table2;
"#;
let s = parse_assets(input).map(|s| s.assets);
// Simple columns (a, b) should NOT be detected with multiple tables
// Only table-level assets should be present
let result = s.unwrap();
// Should have 2 table assets
assert_eq!(result.iter().filter(|a| a.columns.is_none()).count(), 2);
// Should have NO column assets (ambiguous which table they belong to)
assert_eq!(result.iter().filter(|a| a.columns.is_some()).count(), 0);
}
#[test]
fn test_sql_asset_parser_multi_table_with_qualified_columns() {
let input = r#"
ATTACH 'ducklake://my_dl1' AS dl1;
ATTACH 'ducklake://my_dl2' AS dl2;
SELECT table1.a, table2.b FROM dl1.table1, dl2.table2;
"#;
let s = parse_assets(input).map(|s| s.assets);
// Qualified columns should be detected even with multiple tables
let result = s.unwrap();
// Check we have column assets for both tables
assert!(result.iter().any(|a| {
a.path == "my_dl1/table1"
&& a.columns
.as_ref()
.map_or(false, |cols| cols.contains_key("a"))
}));
assert!(result.iter().any(|a| {
a.path == "my_dl2/table2"
&& a.columns
.as_ref()
.map_or(false, |cols| cols.contains_key("b"))
}));
}
#[test]
fn test_sql_asset_parser_use_with_simple_columns() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
USE dl;
SELECT a, b, c FROM table1;
"#;
let s = parse_assets(input).map(|s| s.assets);
let result = s.unwrap();
// Should detect columns since it's a single table
assert!(result.iter().any(|a| {
a.path == "my_dl/table1"
&& a.columns
.as_ref()
.map_or(false, |cols| cols.contains_key("a"))
}));
assert!(result.iter().any(|a| {
a.path == "my_dl/table1"
&& a.columns
.as_ref()
.map_or(false, |cols| cols.contains_key("b"))
}));
assert!(result.iter().any(|a| {
a.path == "my_dl/table1"
&& a.columns
.as_ref()
.map_or(false, |cols| cols.contains_key("c"))
}));
}
#[test]
fn test_sql_asset_parser_wildcard_no_columns() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
SELECT * FROM dl.table1;
"#;
let s = parse_assets(input).map(|s| s.assets);
let result = s.unwrap();
// Wildcard should NOT create column assets, only table asset
assert_eq!(result.len(), 1);
assert!(result[0].columns.is_none());
}
#[test]
fn test_sql_asset_parser_columns_with_alias() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
SELECT a AS column_a, b AS column_b FROM dl.table1;
"#;
let s = parse_assets(input).map(|s| s.assets);
let result = s.unwrap();
// Should detect columns even when aliased
assert!(result.iter().any(|a| {
a.path == "my_dl/table1"
&& a.columns
.as_ref()
.map_or(false, |cols| cols.contains_key("a"))
}));
assert!(result.iter().any(|a| {
a.path == "my_dl/table1"
&& a.columns
.as_ref()
.map_or(false, |cols| cols.contains_key("b"))
}));
}
#[test]
fn test_sql_asset_parser_columns_with_table_alias() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
SELECT t.a, t.b FROM dl.table1 AS t;
"#;
let s = parse_assets(input).map(|s| s.assets);
let result = s.unwrap();
// Should detect columns using the table alias
assert!(result.iter().any(|a| {
a.path == "my_dl/table1"
&& a.columns
.as_ref()
.map_or(false, |cols| cols.contains_key("a"))
}));
assert!(result.iter().any(|a| {
a.path == "my_dl/table1"
&& a.columns
.as_ref()
.map_or(false, |cols| cols.contains_key("b"))
}));
}
#[test]
fn test_sql_asset_parser_insert_with_columns() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
INSERT INTO dl.table1 (name, age, email) VALUES ('John', 30, 'john@example.com');
"#;
let s = parse_assets(input).map(|s| s.assets);
let result = s.unwrap();
// Should have one asset with merged columns
assert_eq!(result.len(), 1);
assert_eq!(result[0].path, "my_dl/table1");
assert_eq!(result[0].access_type, Some(W));
// Check that all columns are present in the merged asset
let columns = result[0].columns.as_ref().expect("Should have columns");
assert_eq!(columns.len(), 3);
assert_eq!(columns.get("name"), Some(&W));
assert_eq!(columns.get("age"), Some(&W));
assert_eq!(columns.get("email"), Some(&W));
}
#[test]
fn test_sql_asset_parser_insert_without_columns() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
INSERT INTO dl.table1 VALUES ('John', 30);
"#;
let s = parse_assets(input).map(|s| s.assets);
let result = s.unwrap();
// Should have one asset without column information
assert_eq!(result.len(), 1);
assert_eq!(result[0].path, "my_dl/table1");
assert_eq!(result[0].access_type, Some(W));
assert!(result[0].columns.is_none());
}
#[test]
fn test_sql_asset_parser_update_multiple_columns() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
UPDATE dl.table1 SET name = 'Jane', age = 25, active = true;
"#;
let s = parse_assets(input).map(|s| s.assets);
let result = s.unwrap();
// Should have one asset with merged columns
assert_eq!(result.len(), 1);
assert_eq!(result[0].path, "my_dl/table1");
assert_eq!(result[0].access_type, Some(W));
// Check that all columns are present
let columns = result[0].columns.as_ref().expect("Should have columns");
assert_eq!(columns.len(), 3);
assert_eq!(columns.get("name"), Some(&W));
assert_eq!(columns.get("age"), Some(&W));
assert_eq!(columns.get("active"), Some(&W));
}
#[test]
fn test_sql_asset_parser_update_returning() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
UPDATE dl.table1 SET name = 'Jane', age = 26 RETURNING id, name;
"#;
let s = parse_assets(input).map(|s| s.assets);
let result = s.unwrap();
// Should have RW access type when RETURNING is used
assert_eq!(result.len(), 1);
assert_eq!(result[0].path, "my_dl/table1");
assert_eq!(result[0].access_type, Some(RW));
// Check that columns are present with correct access types
// name and age are written (W), id and name are read (R)
// name should be RW (both written and read)
let columns = result[0].columns.as_ref().expect("Should have columns");
assert_eq!(columns.len(), 3);
assert_eq!(columns.get("name"), Some(&RW)); // Written in SET, read in RETURNING
assert_eq!(columns.get("age"), Some(&W)); // Only written
assert_eq!(columns.get("id"), Some(&R)); // Only read
}
}
backend/parsers/windmill-parser-ts/src/asset_parser.rs
+45 -20
View File
@@ -117,6 +117,7 @@ impl Visit for AssetsFinder {
kind,
path: path.to_string(),
access_type: None,
columns: None,
});
}
}
@@ -177,8 +178,12 @@ impl Visit for AssetsFinder {
if asset_was_used(&self.assets, (kind, path)) {
continue;
}
self.assets
.push(ParseAssetsResult { kind, access_type: None, path: path.clone() });
self.assets.push(ParseAssetsResult {
kind,
access_type: None,
path: path.clone(),
columns: None,
});
}
// Restore state - identifiers declared in this block go out of scope
@@ -294,8 +299,12 @@ impl AssetsFinder {
let path = parse_asset_syntax(&value, false)
.map(|(_, p)| p)
.unwrap_or(&value);
self.assets
.push(ParseAssetsResult { kind, path: path.to_string(), access_type });
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
access_type,
columns: None,
});
}
_ => return Err(()),
}
@@ -321,7 +330,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "/test.csv".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},])
);
}
@@ -340,7 +350,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt".to_string(),
access_type: None
access_type: None,
columns: None,
},])
);
}
@@ -360,7 +371,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/friends".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},])
);
}
@@ -383,12 +395,14 @@ mod tests {
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/analytics".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/friends".to_string(),
access_type: Some(RW)
access_type: Some(RW),
columns: None,
},
])
);
@@ -418,17 +432,20 @@ mod tests {
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "another1/customers".to_string(),
access_type: Some(W)
access_type: Some(W),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "another2".to_string(),
access_type: None
access_type: None,
columns: None,
},
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "main/friends".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},
])
);
@@ -452,12 +469,14 @@ mod tests {
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "another1".to_string(),
access_type: None
access_type: None,
columns: None,
},
ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main".to_string(),
access_type: None
access_type: None,
columns: None,
},
])
);
@@ -478,7 +497,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "main/myschema.friends".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},])
);
}
@@ -499,7 +519,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/public.users".to_string(),
access_type: Some(RW)
access_type: Some(RW),
columns: None,
},])
);
}
@@ -518,7 +539,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt".to_string(),
access_type: None
access_type: None,
columns: None,
},])
);
}
@@ -539,7 +561,8 @@ mod tests {
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/users".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},])
);
}
@@ -562,12 +585,14 @@ mod tests {
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/private.users".to_string(),
access_type: Some(R)
access_type: Some(R),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/test".to_string(),
access_type: Some(W)
access_type: Some(W),
columns: None,
},
])
);
backend/parsers/windmill-parser-yaml/src/asset_parser.rs
+3
View File
@@ -12,6 +12,7 @@ pub fn parse_assets(input: &str) -> anyhow::Result<ParseAssetsOutput> {
kind: AssetKind::Resource,
path: delegate_to_git_repo_details.resource,
access_type: Some(AssetUsageAccessType::R),
columns: None,
})
}
@@ -21,6 +22,7 @@ pub fn parse_assets(input: &str) -> anyhow::Result<ParseAssetsOutput> {
kind: AssetKind::Resource,
path: pinned_res,
access_type: Some(AssetUsageAccessType::R),
columns: None,
})
}
}
@@ -31,6 +33,7 @@ pub fn parse_assets(input: &str) -> anyhow::Result<ParseAssetsOutput> {
kind: AssetKind::Resource,
path: resource,
access_type: Some(AssetUsageAccessType::R),
columns: None,
})
}
}
backend/parsers/windmill-parser/src/asset_parser.rs
+36 -1
View File
@@ -1,4 +1,5 @@
use serde::Serialize;
use std::collections::BTreeMap;
#[derive(Serialize, PartialEq, Clone, Copy, Debug)]
#[serde(rename_all(serialize = "lowercase"))]
@@ -19,12 +20,14 @@ pub enum AssetKind {
DataTable,
}
#[derive(Serialize, Debug, PartialEq)]
#[derive(Serialize, Debug, PartialEq, Clone)]
pub struct ParseAssetsResult {
pub kind: AssetKind,
pub path: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub access_type: Option<AssetUsageAccessType>, // None in case of ambiguity
#[serde(skip_serializing_if = "Option::is_none")]
pub columns: Option<BTreeMap<String, AssetUsageAccessType>>, // Map column name to access type, "*" represents wildcard
}
#[derive(Serialize, Debug, PartialEq)]
@@ -66,6 +69,8 @@ pub fn merge_assets(assets: Vec<ParseAssetsResult>) -> Vec<ParseAssetsResult> {
(Some(R), Some(R)) => Some(R),
(Some(W), Some(W)) => Some(W),
};
// merge columns: union the column sets and merge access types per column
existing.columns = merge_column_maps(existing.columns.take(), asset.columns);
} else {
arr.push(asset);
}
@@ -74,6 +79,36 @@ pub fn merge_assets(assets: Vec<ParseAssetsResult>) -> Vec<ParseAssetsResult> {
arr
}
fn merge_column_maps(
existing: Option<BTreeMap<String, AssetUsageAccessType>>,
new: Option<BTreeMap<String, AssetUsageAccessType>>,
) -> Option<BTreeMap<String, AssetUsageAccessType>> {
match (existing, new) {
(None, None) => None,
(Some(map), None) | (None, Some(map)) => Some(map),
(Some(mut existing_map), Some(new_map)) => {
for (col_name, new_access) in new_map {
existing_map
.entry(col_name)
.and_modify(|existing_access| {
*existing_access = merge_access_types(*existing_access, new_access);
})
.or_insert(new_access);
}
Some(existing_map)
}
}
}
fn merge_access_types(a: AssetUsageAccessType, b: AssetUsageAccessType) -> AssetUsageAccessType {
match (a, b) {
(R, W) | (W, R) => RW,
(RW, _) | (_, RW) => RW,
(R, R) => R,
(W, W) => W,
}
}
// Will return false if the user assigned an asset to a variable like:
// let sql = wmill.datatable('main')
// But never used it. In that case we don't know which table is being used,
backend/rustfmt.toml
+1
View File
@@ -1,3 +1,4 @@
edition = "2021"
max_width = 100
use_small_heuristics = "Default"
match_arm_leading_pipes="Preserve"
backend/src/main.rs
+334 -388
View File
@@ -16,7 +16,7 @@ use monitor::{
send_current_log_file_to_object_store, send_logs_to_object_store, WORKERS_NAMES,
};
use rand::Rng;
use sqlx::{postgres::PgListener, Pool, Postgres};
use sqlx::{Pool, Postgres};
use std::{
collections::HashMap,
fs::{create_dir_all, DirBuilder},
@@ -212,11 +212,17 @@ where
}
lazy_static::lazy_static! {
static ref PG_LISTENER_REFRESH_PERIOD_SECS: u64 = std::env::var("PG_LISTENER_REFRESH_PERIOD_SECS")
// Period in seconds between full settings reload (12 hours by default)
static ref SETTINGS_RELOAD_PERIOD_SECS: u64 = std::env::var("SETTINGS_RELOAD_PERIOD_SECS")
.ok()
.and_then(|x| x.parse::<u64>().ok())
.unwrap_or(3600 * 12);
// Period in seconds between polling for notify events (10s by default)
static ref LISTEN_NEW_EVENTS_INTERVAL_SEC: u64 = std::env::var("LISTEN_NEW_EVENTS_INTERVAL_SEC")
.ok()
.and_then(|x| x.parse::<u64>().ok())
.unwrap_or(10);
}
pub fn main() -> anyhow::Result<()> {
@@ -1138,8 +1144,18 @@ Windmill Community Edition {GIT_VERSION}
let base_internal_url = base_internal_url.to_string();
let db = db.clone();
let h = tokio::spawn(async move {
let mut listener = retry_listen_pg(&db).await;
let mut last_listener_refresh = Instant::now();
// Initialize last_event_id to current max to avoid processing old events on startup
let mut last_event_id: i64 = match windmill_common::notify_events::get_latest_event_id(&db).await {
Ok(id) => {
tracing::info!("Initialized notify event polling with last_event_id: {}", id);
id
}
Err(e) => {
tracing::warn!("Could not get latest event id, starting from 0: {e:#}");
0
}
};
let mut last_settings_reload = Instant::now();
let mut monitor_iteration: u64 = 0;
let rd_shift: u8 = rand::rng().random_range(0..200);
loop {
@@ -1158,349 +1174,36 @@ Windmill Community Edition {GIT_VERSION}
tracing::info!("received killpill for monitor job");
break;
},
notification = listener.try_recv() => {
match notification {
Ok(n) => {
if n.is_none() {
tracing::error!("Could not receive notification, attempting to reconnect to pg listener");
continue;
}
let n = n.unwrap();
tracing::info!("Received new pg notification: {n:?}");
match n.channel() {
"notify_config_change" => {
match n.payload() {
"server" if server_mode => {
tracing::error!("Server config change detected but server config is obsolete: {}", n.payload());
},
a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => {
tracing::info!("Worker config change detected: {}", n.payload());
reload_worker_config(&db, tx.clone(), true).await;
},
_ => {
tracing::debug!("config changed but did not target this server/worker");
}
}
},
"notify_webhook_change" => {
let workspace_id = n.payload();
tracing::info!("Webhook change detected, invalidating webhook cache: {}", workspace_id);
windmill_api::webhook_util::WEBHOOK_CACHE.remove(workspace_id);
},
"notify_workspace_envs_change" => {
let workspace_id = n.payload();
tracing::info!("Workspace envs change detected, invalidating workspace envs cache: {}", workspace_id);
windmill_common::variables::CUSTOM_ENVS_CACHE.remove(workspace_id);
},
"notify_workspace_key_change" => {
let workspace_id = n.payload();
tracing::info!("Workspace key change detected, invalidating workspace key cache: {}", workspace_id);
windmill_common::variables::WORKSPACE_CRYPT_CACHE.remove(workspace_id);
},
"notify_workspace_premium_change" => {
let workspace_id = n.payload();
tracing::info!("Workspace premium change detected, invalidating workspace premium cache: {}", workspace_id);
windmill_common::workspaces::TEAM_PLAN_CACHE.remove(workspace_id);
},
"notify_runnable_version_change" => {
let payload = n.payload();
tracing::info!("Runnable version change detected: {}", payload);
match payload.split(':').collect::<Vec<&str>>().as_slice() {
[workspace_id, source_type, path, kind] => {
let key = (workspace_id.to_string(), path.to_string());
match source_type {
&"script" => {
windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key);
match kind {
&"preprocessor" => {
match sqlx::query_scalar!(
"SELECT fv.id
FROM flow f
INNER JOIN flow_version fv ON fv.id = f.versions[array_upper(f.versions, 1)]
WHERE fv.value->'preprocessor_module'->'value'->>'path' = $1 AND f.workspace_id = $2",
path,
workspace_id
).fetch_all(&db).await {
Ok(flow_versions) => {
tracing::debug!("Workspace preprocessor {} changed, removing runnable format version cache for flow versions {:?}", path, flow_versions);
for version in flow_versions {
for trigger_kind in TriggerKind::iter() {
let key = (windmill_common::triggers::HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), version, trigger_kind);
windmill_common::triggers::RUNNABLE_FORMAT_VERSION_CACHE.remove(&key);
}
}
}
Err(e) => {
tracing::error!("Error fetching flow paths: {e:#}");
}
}
},
_ => {}
}
}
&"flow" => {
let dynamic_input_key = windmill_common::jobs::generate_dynamic_input_key(workspace_id, path);
windmill_common::DYNAMIC_INPUT_CACHE.remove(&dynamic_input_key);
windmill_common::FLOW_VERSION_CACHE.remove(&key);
},
_ => {
tracing::warn!("Unknown runnable version change payload: {}", payload);
}
}
},
_ => {
tracing::warn!("Unknown runnable version change payload: {}", payload);
}
}
},
#[cfg(feature = "http_trigger")]
"notify_http_trigger_change" => {
tracing::info!("HTTP trigger change detected: {}", n.payload());
match windmill_api::triggers::http::refresh_routers(&db).await {
Ok((true, _)) => {
tracing::info!("Refreshed HTTP routers (trigger change)");
},
Ok((false, _)) => {
tracing::warn!("Should have refreshed HTTP routers (trigger change) but did not");
},
Err(err) => {
tracing::error!("Error refreshing HTTP routers (trigger change): {err:#}");
}
};
},
"notify_token_invalidation" => {
let token = n.payload();
tracing::info!("Token invalidation detected for token: {}...", &token[..token.len().min(8)]);
windmill_api::auth::invalidate_token_from_cache(token);
},
"var_cache_invalidation" => {
if let Ok(payload) = serde_json::from_str::<serde_json::Value>(n.payload()) {
if let (Some(workspace_id), Some(path)) =
(payload.get("workspace_id").and_then(|v| v.as_str()),
payload.get("path").and_then(|v| v.as_str())) {
tracing::info!("Variable cache invalidation detected: {}:{}", workspace_id, path);
windmill_api::var_resource_cache::invalidate_variable_cache(&workspace_id, &path);
}
}
},
"resource_cache_invalidation" => {
if let Ok(payload) = serde_json::from_str::<serde_json::Value>(n.payload()) {
if let (Some(workspace_id), Some(path)) =
(payload.get("workspace_id").and_then(|v| v.as_str()),
payload.get("path").and_then(|v| v.as_str())) {
tracing::info!("Resource cache invalidation detected: {}:{}", workspace_id, path);
windmill_api::var_resource_cache::invalidate_resource_cache(&workspace_id, &path);
}
}
},
"notify_global_setting_change" => {
tracing::info!("Global setting change detected: {}", n.payload());
match n.payload() {
BASE_URL_SETTING => {
if let Err(e) = reload_base_url_setting(&conn).await {
tracing::error!(error = %e, "Could not reload base url setting");
}
},
OAUTH_SETTING => {
if let Err(e) = reload_base_url_setting(&conn).await {
tracing::error!(error = %e, "Could not reload oauth setting");
}
},
CUSTOM_TAGS_SETTING => {
if let Err(e) = reload_custom_tags_setting(&db).await {
tracing::error!(error = %e, "Could not reload custom tags setting");
}
},
LICENSE_KEY_SETTING => {
if let Err(e) = reload_license_key(&db.into()).await {
tracing::error!("Failed to reload license key: {e:#}");
}
},
DEFAULT_TAGS_PER_WORKSPACE_SETTING => {
if let Err(e) = load_tag_per_workspace_enabled(&db).await {
tracing::error!("Error loading default tag per workspace: {e:#}");
}
},
DEFAULT_TAGS_WORKSPACES_SETTING => {
if let Err(e) = load_tag_per_workspace_workspaces(&db).await {
tracing::error!("Error loading default tag per workspace workspaces: {e:#}");
}
},
SMTP_SETTING => {
reload_smtp_config(&db).await;
},
TEAMS_SETTING => {
tracing::info!("Teams setting changed.");
},
INDEXER_SETTING => {
reload_indexer_config(&db).await;
},
TIMEOUT_WAIT_RESULT_SETTING => {
reload_timeout_wait_result_setting(&conn).await
},
RETENTION_PERIOD_SECS_SETTING => {
reload_retention_period_setting(&conn).await
},
MONITOR_LOGS_ON_OBJECT_STORE_SETTING => {
reload_delete_logs_periodically_setting(&conn).await
},
JOB_DEFAULT_TIMEOUT_SECS_SETTING => {
reload_job_default_timeout_setting(&conn).await
},
#[cfg(feature = "parquet")]
OBJECT_STORE_CONFIG_SETTING => {
if !disable_s3_store {
reload_object_store_setting(&db).await;
}
},
SCIM_TOKEN_SETTING => {
reload_scim_token_setting(&conn).await
},
EXTRA_PIP_INDEX_URL_SETTING => {
reload_extra_pip_index_url_setting(&conn).await
},
PIP_INDEX_URL_SETTING => {
reload_pip_index_url_setting(&conn).await
},
INSTANCE_PYTHON_VERSION_SETTING => {
reload_instance_python_version_setting(&conn).await
},
NPM_CONFIG_REGISTRY_SETTING => {
reload_npm_config_registry_setting(&conn).await
},
BUNFIG_INSTALL_SCOPES_SETTING => {
reload_bunfig_install_scopes_setting(&conn).await
},
NUGET_CONFIG_SETTING => {
reload_nuget_config_setting(&conn).await
},
POWERSHELL_REPO_URL_SETTING => {
reload_powershell_repo_url_setting(&conn).await
},
POWERSHELL_REPO_PAT_SETTING => {
reload_powershell_repo_pat_setting(&conn).await
},
MAVEN_REPOS_SETTING => {
reload_maven_repos_setting(&conn).await
},
NO_DEFAULT_MAVEN_SETTING => {
reload_no_default_maven_setting(&conn).await
},
RUBY_REPOS_SETTING => {
reload_ruby_repos_setting(&conn).await
},
HUB_API_SECRET_SETTING => {
reload_hub_api_secret_setting(&conn).await
},
KEEP_JOB_DIR_SETTING => {
load_keep_job_dir(&conn).await;
},
OTEL_TRACING_PROXY_SETTING => {
reload_otel_tracing_proxy_setting(&conn).await;
if worker_mode {
tracing::info!("OTEL tracing proxy setting changed, restarting worker");
send_delayed_killpill(&tx, 4, "OTEL tracing proxy setting change").await;
}
},
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => {
load_require_preexisting_user(&db).await;
},
EXPOSE_METRICS_SETTING => {
tracing::info!("Metrics setting changed, restarting");
send_delayed_killpill(&tx, 40, "metrics setting change").await;
},
EMAIL_DOMAIN_SETTING => {
tracing::info!("Email domain setting changed");
if server_mode {
send_delayed_killpill(&tx, 4, "email domain setting change").await;
}
},
EXPOSE_DEBUG_METRICS_SETTING => {
if let Err(e) = load_metrics_debug_enabled(&conn).await {
tracing::error!(error = %e, "Could not reload debug metrics setting");
}
},
APP_WORKSPACED_ROUTE_SETTING => {
if let Err(e) = reload_app_workspaced_route_setting(&db).await {
tracing::error!(error = %e, "Could not reload app workspaced route setting");
}
},
OTEL_SETTING => {
tracing::info!("OTEL setting changed, restarting");
send_delayed_killpill(&tx, 4, "OTEL setting change").await;
},
REQUEST_SIZE_LIMIT_SETTING => {
if server_mode {
tracing::info!("Request limit size change detected, killing server expecting to be restarted");
send_delayed_killpill(&tx, 4, "request size limit change").await;
}
},
SAML_METADATA_SETTING => {
tracing::info!("SAML metadata change detected, killing server expecting to be restarted");
send_delayed_killpill(&tx, 0, "SAML metadata change").await;
},
HUB_BASE_URL_SETTING => {
if let Err(e) = reload_hub_base_url_setting(&conn, server_mode).await {
tracing::error!(error = %e, "Could not reload hub base url setting");
}
},
CRITICAL_ERROR_CHANNELS_SETTING => {
if let Err(e) = reload_critical_error_channels_setting(&db).await {
tracing::error!(error = %e, "Could not reload critical error emails setting");
}
},
CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING => {
if let Err(e) = reload_critical_alerts_on_db_oversize(&db).await {
tracing::error!(error = %e, "Could not reload critical alerts on db oversize setting");
}
},
JWT_SECRET_SETTING => {
if let Err(e) = reload_jwt_secret_setting(&db).await {
tracing::error!(error = %e, "Could not reload jwt secret setting");
}
},
CRITICAL_ALERT_MUTE_UI_SETTING => {
tracing::info!("Critical alert UI setting changed");
if let Err(e) = reload_critical_alert_mute_ui_setting(&conn).await {
tracing::error!(error = %e, "Could not reload critical alert UI setting");
}
},
a @_ => {
tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a);
}
}
},
_ => {
tracing::warn!("Unknown notification received");
continue;
_ = tokio::time::sleep(Duration::from_secs(*LISTEN_NEW_EVENTS_INTERVAL_SEC)) => {
// Poll for new events from notify_event table
match windmill_common::notify_events::poll_notify_events(&db, last_event_id).await {
Ok(events) => {
for event in events {
if !*windmill_common::QUIET_LOGS {
tracing::info!("Processing notify event: channel={}, payload={}", event.channel, event.payload);
}
process_notify_event(
&event.channel,
&event.payload,
&db,
&conn,
&tx,
server_mode,
worker_mode,
#[cfg(feature = "parquet")]
disable_s3_store,
).await;
last_event_id = last_event_id.max(event.id);
}
},
}
Err(e) => {
tracing::error!(error = %e, "Could not receive notification, attempting to reconnect listener");
let db = db.clone();
tokio::select! {
biased;
_ = monitor_killpill_rx.recv() => {
tracing::info!("received killpill for monitor job");
break;
},
new_listener = async move { retry_listen_pg(&db).await } => {
listener = new_listener;
continue;
}
}
tracing::error!("Error polling notify events: {e:#}");
}
};
},
_ = tokio::time::sleep(Duration::from_secs(30)) => {
if last_listener_refresh.elapsed() > Duration::from_secs(*PG_LISTENER_REFRESH_PERIOD_SECS) {
tracing::info!("Refreshing pg listeners, settings and license key after {}s", Duration::from_secs(*PG_LISTENER_REFRESH_PERIOD_SECS).as_secs());
if let Err(e) = listener.unlisten_all().await {
tracing::error!(error = %e, "Could not unlisten to database");
}
listener = retry_listen_pg(&db).await;
}
// Periodic full settings reload
if last_settings_reload.elapsed() > Duration::from_secs(*SETTINGS_RELOAD_PERIOD_SECS) {
tracing::info!("Reloading settings and license key after {}s", Duration::from_secs(*SETTINGS_RELOAD_PERIOD_SECS).as_secs());
initial_load(
&conn,
tx.clone(),
@@ -1514,7 +1217,7 @@ Windmill Community Edition {GIT_VERSION}
if let Err(err) = reload_license_key(&conn).await {
tracing::error!("Failed to reload license key: {err:#}");
}
last_listener_refresh = Instant::now();
last_settings_reload = Instant::now();
}
if server_mode {
@@ -1668,50 +1371,293 @@ Windmill Community Edition {GIT_VERSION}
std::process::exit(0);
}
async fn listen_pg(db: &Pool<Postgres>) -> Option<PgListener> {
let mut listener = match PgListener::connect_with(db).await {
Ok(l) => l,
Err(e) => {
tracing::error!(error = %e, "Could not connect to database");
return None;
}
};
#[allow(unused_mut)]
let mut channels = vec![
"notify_config_change",
"notify_global_setting_change",
"notify_webhook_change",
"notify_workspace_envs_change",
"notify_workspace_key_change",
"notify_runnable_version_change",
"notify_token_invalidation",
];
#[cfg(feature = "http_trigger")]
channels.push("notify_http_trigger_change");
#[cfg(feature = "cloud")]
channels.push("notify_workspace_premium_change");
if let Err(e) = listener.listen_all(channels).await {
tracing::error!(error = %e, "Could not listen to database");
return None;
}
return Some(listener);
}
async fn retry_listen_pg(db: &Pool<Postgres>) -> PgListener {
let mut listener = listen_pg(db).await;
loop {
if listener.is_none() {
tracing::info!("Retrying listening to pg listen in 5 seconds");
tokio::time::sleep(Duration::from_secs(5)).await;
listener = listen_pg(db).await;
} else {
tracing::info!("Successfully connected to pg listen");
return listener.unwrap();
/// Process a single notify event from the polling-based event system.
/// This replaces the old PgListener notification handling.
#[allow(unused_variables)]
async fn process_notify_event(
channel: &str,
payload: &str,
db: &Pool<Postgres>,
conn: &Connection,
tx: &KillpillSender,
server_mode: bool,
worker_mode: bool,
#[cfg(feature = "parquet")]
disable_s3_store: bool,
) {
match channel {
"notify_config_change" => {
if payload == "server" && server_mode {
tracing::error!("Server config change detected but server config is obsolete: {}", payload);
} else if worker_mode && payload == format!("worker__{}", *WORKER_GROUP) {
tracing::info!("Worker config change detected: {}", payload);
reload_worker_config(db, tx.clone(), true).await;
} else {
tracing::debug!("config changed but did not target this server/worker");
}
},
"notify_webhook_change" => {
tracing::info!("Webhook change detected, invalidating webhook cache: {}", payload);
windmill_api::webhook_util::WEBHOOK_CACHE.remove(payload);
},
"notify_workspace_envs_change" => {
tracing::info!("Workspace envs change detected, invalidating workspace envs cache: {}", payload);
windmill_common::variables::CUSTOM_ENVS_CACHE.remove(payload);
},
"notify_workspace_key_change" => {
tracing::info!("Workspace key change detected, invalidating workspace key cache: {}", payload);
windmill_common::variables::WORKSPACE_CRYPT_CACHE.remove(payload);
},
"notify_workspace_premium_change" => {
tracing::info!("Workspace premium change detected, invalidating workspace premium cache: {}", payload);
windmill_common::workspaces::TEAM_PLAN_CACHE.remove(payload);
},
"notify_runnable_version_change" => {
tracing::info!("Runnable version change detected: {}", payload);
match payload.split(':').collect::<Vec<&str>>().as_slice() {
[workspace_id, source_type, path, kind] => {
let key = (workspace_id.to_string(), path.to_string());
match *source_type {
"script" => {
windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key);
if *kind == "preprocessor" {
match sqlx::query_scalar::<_, i64>(
"SELECT fv.id
FROM flow f
INNER JOIN flow_version fv ON fv.id = f.versions[array_upper(f.versions, 1)]
WHERE fv.value->'preprocessor_module'->'value'->>'path' = $1 AND f.workspace_id = $2",
)
.bind(*path)
.bind(*workspace_id)
.fetch_all(db).await {
Ok(flow_versions) => {
tracing::debug!("Workspace preprocessor {} changed, removing runnable format version cache for flow versions {:?}", path, flow_versions);
for version in flow_versions {
for trigger_kind in TriggerKind::iter() {
let key = (windmill_common::triggers::HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), version, trigger_kind);
windmill_common::triggers::RUNNABLE_FORMAT_VERSION_CACHE.remove(&key);
}
}
}
Err(e) => {
tracing::error!("Error fetching flow paths: {e:#}");
}
}
}
}
"flow" => {
let dynamic_input_key = windmill_common::jobs::generate_dynamic_input_key(workspace_id, path);
windmill_common::DYNAMIC_INPUT_CACHE.remove(&dynamic_input_key);
windmill_common::FLOW_VERSION_CACHE.remove(&key);
},
_ => {
tracing::warn!("Unknown runnable version change payload: {}", payload);
}
}
},
_ => {
tracing::warn!("Unknown runnable version change payload: {}", payload);
}
}
},
#[cfg(feature = "http_trigger")]
"notify_http_trigger_change" => {
tracing::info!("HTTP trigger change detected: {}", payload);
match windmill_api::triggers::http::refresh_routers(db).await {
Ok((true, _)) => {
tracing::info!("Refreshed HTTP routers (trigger change)");
},
Ok((false, _)) => {
tracing::warn!("Should have refreshed HTTP routers (trigger change) but did not");
},
Err(err) => {
tracing::error!("Error refreshing HTTP routers (trigger change): {err:#}");
}
};
},
"notify_token_invalidation" => {
tracing::info!("Token invalidation detected for token: {}...", payload.get(..8).unwrap_or(payload));
windmill_api::auth::invalidate_token_from_cache(payload);
},
"notify_global_setting_change" => {
tracing::info!("Global setting change detected: {}", payload);
match payload {
BASE_URL_SETTING => {
if let Err(e) = reload_base_url_setting(conn).await {
tracing::error!(error = %e, "Could not reload base url setting");
}
},
OAUTH_SETTING => {
if let Err(e) = reload_base_url_setting(conn).await {
tracing::error!(error = %e, "Could not reload oauth setting");
}
},
CUSTOM_TAGS_SETTING => {
if let Err(e) = reload_custom_tags_setting(db).await {
tracing::error!(error = %e, "Could not reload custom tags setting");
}
},
LICENSE_KEY_SETTING => {
if let Err(e) = reload_license_key(&db.into()).await {
tracing::error!("Failed to reload license key: {e:#}");
}
},
DEFAULT_TAGS_PER_WORKSPACE_SETTING => {
if let Err(e) = load_tag_per_workspace_enabled(db).await {
tracing::error!("Error loading default tag per workspace: {e:#}");
}
},
DEFAULT_TAGS_WORKSPACES_SETTING => {
if let Err(e) = load_tag_per_workspace_workspaces(db).await {
tracing::error!("Error loading default tag per workspace workspaces: {e:#}");
}
},
SMTP_SETTING => {
reload_smtp_config(db).await;
},
TEAMS_SETTING => {
tracing::info!("Teams setting changed.");
},
INDEXER_SETTING => {
reload_indexer_config(db).await;
},
TIMEOUT_WAIT_RESULT_SETTING => {
reload_timeout_wait_result_setting(conn).await
},
RETENTION_PERIOD_SECS_SETTING => {
reload_retention_period_setting(conn).await
},
MONITOR_LOGS_ON_OBJECT_STORE_SETTING => {
reload_delete_logs_periodically_setting(conn).await
},
JOB_DEFAULT_TIMEOUT_SECS_SETTING => {
reload_job_default_timeout_setting(conn).await
},
#[cfg(feature = "parquet")]
OBJECT_STORE_CONFIG_SETTING => {
if !disable_s3_store {
reload_object_store_setting(db).await;
}
},
SCIM_TOKEN_SETTING => {
reload_scim_token_setting(conn).await
},
EXTRA_PIP_INDEX_URL_SETTING => {
reload_extra_pip_index_url_setting(conn).await
},
PIP_INDEX_URL_SETTING => {
reload_pip_index_url_setting(conn).await
},
INSTANCE_PYTHON_VERSION_SETTING => {
reload_instance_python_version_setting(conn).await
},
NPM_CONFIG_REGISTRY_SETTING => {
reload_npm_config_registry_setting(conn).await
},
BUNFIG_INSTALL_SCOPES_SETTING => {
reload_bunfig_install_scopes_setting(conn).await
},
NUGET_CONFIG_SETTING => {
reload_nuget_config_setting(conn).await
},
POWERSHELL_REPO_URL_SETTING => {
reload_powershell_repo_url_setting(conn).await
},
POWERSHELL_REPO_PAT_SETTING => {
reload_powershell_repo_pat_setting(conn).await
},
MAVEN_REPOS_SETTING => {
reload_maven_repos_setting(conn).await
},
NO_DEFAULT_MAVEN_SETTING => {
reload_no_default_maven_setting(conn).await
},
RUBY_REPOS_SETTING => {
reload_ruby_repos_setting(conn).await
},
HUB_API_SECRET_SETTING => {
reload_hub_api_secret_setting(conn).await
},
KEEP_JOB_DIR_SETTING => {
load_keep_job_dir(conn).await;
},
OTEL_TRACING_PROXY_SETTING => {
reload_otel_tracing_proxy_setting(conn).await;
if worker_mode {
tracing::info!("OTEL tracing proxy setting changed, restarting worker");
send_delayed_killpill(tx, 4, "OTEL tracing proxy setting change").await;
}
},
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => {
load_require_preexisting_user(db).await;
},
EXPOSE_METRICS_SETTING => {
tracing::info!("Metrics setting changed, restarting");
send_delayed_killpill(tx, 40, "metrics setting change").await;
},
EMAIL_DOMAIN_SETTING => {
tracing::info!("Email domain setting changed");
if server_mode {
send_delayed_killpill(tx, 4, "email domain setting change").await;
}
},
EXPOSE_DEBUG_METRICS_SETTING => {
if let Err(e) = load_metrics_debug_enabled(conn).await {
tracing::error!(error = %e, "Could not reload debug metrics setting");
}
},
APP_WORKSPACED_ROUTE_SETTING => {
if let Err(e) = reload_app_workspaced_route_setting(db).await {
tracing::error!(error = %e, "Could not reload app workspaced route setting");
}
},
OTEL_SETTING => {
tracing::info!("OTEL setting changed, restarting");
send_delayed_killpill(tx, 4, "OTEL setting change").await;
},
REQUEST_SIZE_LIMIT_SETTING => {
if server_mode {
tracing::info!("Request limit size change detected, killing server expecting to be restarted");
send_delayed_killpill(tx, 4, "request size limit change").await;
}
},
SAML_METADATA_SETTING => {
tracing::info!("SAML metadata change detected, killing server expecting to be restarted");
send_delayed_killpill(tx, 0, "SAML metadata change").await;
},
HUB_BASE_URL_SETTING => {
if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await {
tracing::error!(error = %e, "Could not reload hub base url setting");
}
},
CRITICAL_ERROR_CHANNELS_SETTING => {
if let Err(e) = reload_critical_error_channels_setting(db).await {
tracing::error!(error = %e, "Could not reload critical error emails setting");
}
},
CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING => {
if let Err(e) = reload_critical_alerts_on_db_oversize(db).await {
tracing::error!(error = %e, "Could not reload critical alerts on db oversize setting");
}
},
JWT_SECRET_SETTING => {
if let Err(e) = reload_jwt_secret_setting(db).await {
tracing::error!(error = %e, "Could not reload jwt secret setting");
}
},
CRITICAL_ALERT_MUTE_UI_SETTING => {
tracing::info!("Critical alert UI setting changed");
if let Err(e) = reload_critical_alert_mute_ui_setting(conn).await {
tracing::error!(error = %e, "Could not reload critical alert UI setting");
}
},
_ => {
tracing::info!("Unrecognized Global Setting Change Payload: {:?}", payload);
}
}
},
_ => {
tracing::warn!("Unknown notification channel: {}", channel);
}
}
}
backend/src/monitor.rs
+19
View File
@@ -1921,6 +1921,24 @@ pub async fn monitor_db(
}
};
// Run every 5 minutes (10 iterations * 30s = 5 minutes)
// Cleanup old notify events (older than 10 minutes)
let cleanup_notify_events_f = async {
if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(10) {
if let Some(db) = conn.as_sql() {
match windmill_common::notify_events::cleanup_old_events(db, 10).await {
Ok(count) if count > 0 => {
tracing::debug!("Cleaned up {} old notify events", count);
}
Err(e) => {
tracing::error!("Error cleaning up notify events: {:?}", e);
}
_ => {}
}
}
}
};
join!(
expired_items_f,
zombie_jobs_f,
@@ -1940,6 +1958,7 @@ pub async fn monitor_db(
cleanup_flow_iterator_data_f,
cleanup_worker_group_stats_f,
native_triggers_sync_f,
cleanup_notify_events_f,
);
}
backend/tests/bun_jobs.rs
+1358
View File
File diff suppressed because it is too large Load Diff
backend/tests/common/mod.rs
+44
View File
@@ -803,3 +803,47 @@ pub async fn rebuild_dmap(client: &windmill_api_client::Client) -> bool {
.status()
.is_success()
}
// ============================================================================
// Dedicated Worker Protocol Helpers
// ============================================================================
/// Result from parsing a dedicated worker stdout line
#[derive(Debug, Clone, PartialEq)]
pub enum DedicatedWorkerResult {
/// Worker printed "start" indicating it's ready
Start,
/// Worker returned a successful result
Success(serde_json::Value),
/// Worker returned an error result
Error(serde_json::Value),
/// Line is not a protocol message (e.g., logs)
Other(String),
}
/// Parse a line from dedicated worker stdout according to the protocol:
/// - "start" -> Ready signal
/// - "wm_res[success]:JSON" -> Success with result
/// - "wm_res[error]:JSON" -> Error with details
/// - anything else -> Other (logs)
pub fn parse_dedicated_worker_line(line: &str) -> DedicatedWorkerResult {
if line == "start" {
return DedicatedWorkerResult::Start;
}
if let Some(json_str) = line.strip_prefix("wm_res[success]:") {
match serde_json::from_str(json_str) {
Ok(value) => return DedicatedWorkerResult::Success(value),
Err(_) => return DedicatedWorkerResult::Other(line.to_string()),
}
}
if let Some(json_str) = line.strip_prefix("wm_res[error]:") {
match serde_json::from_str(json_str) {
Ok(value) => return DedicatedWorkerResult::Error(value),
Err(_) => return DedicatedWorkerResult::Other(line.to_string()),
}
}
DedicatedWorkerResult::Other(line.to_string())
}
backend/tests/fixtures/bun_edge_cases.sql
Vendored
+152
View File
@@ -0,0 +1,152 @@
-- Fixture for Bun edge case tests
-- Tests deeply nested imports (level1 -> level2 -> level3)
-- Level 3: Base script (deepest level)
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
export function main() {
return "level3";
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/nested/level3', 20001, 'bun', '');
-- Level 2: Imports level3 using RELATIVE path (./level3.ts)
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
import { main as level3 } from "./level3.ts";
export function main() {
return "level2 -> " + level3();
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/nested/level2', 20002, 'bun', '');
-- Level 1: Imports level2 using RELATIVE path (./level2.ts)
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
import { main as level2 } from "./level2.ts";
export function main() {
return "level1 -> " + level2();
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/nested/level1', 20003, 'bun', '');
-- Script with preprocessor function
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, has_preprocessor) VALUES (
'test-workspace',
'test-user',
'
export function preprocessor(value: number) {
return { value: value * 2 };
}
export function main(value: number) {
return value + 100;
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"value":{"type":"number"}},"required":["value"],"type":"object"}',
'Script with preprocessor',
'',
'f/edge_cases/with_preprocessor', 20004, 'bun', '', true);
-- Script with nodejs annotation
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'//nodejs
export function main() {
return process.version;
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'NodeJS mode script',
'',
'f/edge_cases/nodejs_mode', 20005, 'bun', '');
-- Script with nobundling annotation
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'//nobundling
export function main() {
return "no bundle";
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'No bundling mode script',
'',
'f/edge_cases/nobundling_mode', 20006, 'bun', '');
-- Script that uses circular-ish import pattern (A imports B, B imports C, test imports A and C)
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
export const SHARED_VALUE = "shared";
export function main() {
return SHARED_VALUE;
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/circular/shared', 20007, 'bun', '');
-- module_a uses RELATIVE path import (./shared.ts)
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
import { SHARED_VALUE } from "./shared.ts";
export function getValue() {
return "from_a_" + SHARED_VALUE;
}
export function main() {
return getValue();
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/circular/module_a', 20008, 'bun', '');
-- module_b uses ABSOLUTE path import (/f/circular/shared.ts)
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
import { SHARED_VALUE } from "/f/circular/shared.ts";
export function getValue() {
return "from_b_" + SHARED_VALUE;
}
export function main() {
return getValue();
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/circular/module_b', 20009, 'bun', '');
backend/tests/notify_events.rs
+835
View File
@@ -0,0 +1,835 @@
/*!
* Tests for the polling-based notify_event system that replaces PostgreSQL LISTEN/NOTIFY.
*
* These tests verify:
* 1. Database triggers correctly insert events into notify_event table
* 2. Polling functions retrieve events correctly
* 3. Cleanup functions delete old events
* 4. All notification channels work as expected
*/
use sqlx::{Pool, Postgres};
use windmill_common::notify_events::{cleanup_old_events, get_latest_event_id, poll_notify_events};
mod common;
/// Helper to insert a test event directly
async fn insert_test_event(db: &Pool<Postgres>, channel: &str, payload: &str) -> i64 {
sqlx::query_scalar::<_, i64>(
"INSERT INTO notify_event (channel, payload) VALUES ($1, $2) RETURNING id",
)
.bind(channel)
.bind(payload)
.fetch_one(db)
.await
.expect("Failed to insert test event")
}
/// Helper to count events for a channel
async fn count_events_for_channel(db: &Pool<Postgres>, channel: &str) -> i64 {
let result: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM notify_event WHERE channel = $1")
.bind(channel)
.fetch_one(db)
.await
.expect("Failed to count events");
result.0
}
// ============================================================================
// Basic Functionality Tests
// ============================================================================
#[sqlx::test(fixtures("base"))]
async fn test_get_latest_event_id_returns_valid_id(db: Pool<Postgres>) {
// Get current latest id
let latest_id = get_latest_event_id(&db).await.expect("Should get latest event id");
assert!(latest_id >= 0, "Latest id should be non-negative");
// Insert a new event and verify latest_id increases
let new_id = insert_test_event(&db, "test_latest_id", "payload").await;
let new_latest_id = get_latest_event_id(&db).await.expect("Should get latest event id");
assert!(new_latest_id >= new_id, "Latest id should be >= new event id");
}
#[sqlx::test(fixtures("base"))]
async fn test_get_latest_event_id_with_events(db: Pool<Postgres>) {
let _id1 = insert_test_event(&db, "test_channel_1", "payload1").await;
let _id2 = insert_test_event(&db, "test_channel_2", "payload2").await;
let id3 = insert_test_event(&db, "test_channel_3", "payload3").await;
let latest_id = get_latest_event_id(&db).await.expect("Should get latest event id");
assert!(latest_id >= id3, "Latest id should be >= last inserted id");
}
#[sqlx::test(fixtures("base"))]
async fn test_poll_notify_events_no_new_events(db: Pool<Postgres>) {
// Get latest id first
let latest_id = get_latest_event_id(&db).await.unwrap();
// Poll from the latest id - should return empty since no new events
let events = poll_notify_events(&db, latest_id).await.expect("Should poll events");
assert!(events.is_empty(), "Should return empty vec when polling from latest id");
}
#[sqlx::test(fixtures("base"))]
async fn test_poll_notify_events_returns_new_events(db: Pool<Postgres>) {
let before_id = get_latest_event_id(&db).await.unwrap();
let _id1 = insert_test_event(&db, "test_poll_channel", "payload1").await;
let _id2 = insert_test_event(&db, "test_poll_channel", "payload2").await;
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
assert!(events.len() >= 2, "Should return at least 2 new events");
// Verify the events we inserted are present
let our_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "test_poll_channel")
.collect();
assert_eq!(our_events.len(), 2, "Should have exactly our 2 test events");
// Verify ordering (ascending by id)
assert!(our_events[0].id < our_events[1].id, "Events should be ordered by id ascending");
}
#[sqlx::test(fixtures("base"))]
async fn test_poll_notify_events_respects_last_event_id(db: Pool<Postgres>) {
let id1 = insert_test_event(&db, "test_respect_id", "payload1").await;
let _id2 = insert_test_event(&db, "test_respect_id", "payload2").await;
let _id3 = insert_test_event(&db, "test_respect_id", "payload3").await;
// Poll from id1 should only return id2 and id3
let events = poll_notify_events(&db, id1).await.expect("Should poll events");
let our_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "test_respect_id")
.collect();
assert_eq!(our_events.len(), 2, "Should only return events after id1");
assert!(our_events.iter().all(|e| e.id > id1), "All events should have id > id1");
}
#[sqlx::test(fixtures("base"))]
async fn test_cleanup_old_events(db: Pool<Postgres>) {
// Use unique channel names to avoid interference from other tests
let old_channel = format!("test_cleanup_old_{}", uuid::Uuid::new_v4());
let recent_channel = format!("test_cleanup_recent_{}", uuid::Uuid::new_v4());
// Insert an event with old timestamp
sqlx::query(
"INSERT INTO notify_event (channel, payload, created_at) VALUES ($1, $2, now() - interval '15 minutes')",
)
.bind(&old_channel)
.bind("old_payload")
.execute(&db)
.await
.expect("Failed to insert old event");
// Insert a recent event
sqlx::query(
"INSERT INTO notify_event (channel, payload) VALUES ($1, $2)",
)
.bind(&recent_channel)
.bind("recent_payload")
.execute(&db)
.await
.expect("Failed to insert recent event");
// Count before cleanup
let old_count_before = count_events_for_channel(&db, &old_channel).await;
assert_eq!(old_count_before, 1, "Should have 1 old event before cleanup");
// Cleanup events older than 10 minutes
let deleted = cleanup_old_events(&db, 10).await.expect("Should cleanup events");
assert!(deleted >= 1, "Should delete at least 1 old event");
// Verify old event is gone
let old_count = count_events_for_channel(&db, &old_channel).await;
assert_eq!(old_count, 0, "Old event should be deleted");
// Verify recent event is still there
let recent_count = count_events_for_channel(&db, &recent_channel).await;
assert_eq!(recent_count, 1, "Recent event should still exist");
}
// ============================================================================
// Database Trigger Tests - Verify triggers insert events correctly
// ============================================================================
#[sqlx::test(fixtures("base"))]
async fn test_trigger_notify_config_change(db: Pool<Postgres>) {
let before_id = get_latest_event_id(&db).await.unwrap();
// Insert or update a config entry
sqlx::query(
"INSERT INTO config (name, config) VALUES ('test_config_trigger', '{}'::jsonb)
ON CONFLICT (name) DO UPDATE SET config = '{}'::jsonb",
)
.execute(&db)
.await
.expect("Failed to insert config");
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let config_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "notify_config_change" && e.payload == "test_config_trigger")
.collect();
assert!(!config_events.is_empty(), "Should have notify_config_change event");
}
#[sqlx::test(fixtures("base"))]
async fn test_trigger_notify_global_setting_change_insert(db: Pool<Postgres>) {
let before_id = get_latest_event_id(&db).await.unwrap();
// Use a unique setting name for testing
let setting_name = format!("test_setting_{}", uuid::Uuid::new_v4());
// Insert a global setting
sqlx::query("INSERT INTO global_settings (name, value) VALUES ($1, '{}'::jsonb)")
.bind(&setting_name)
.execute(&db)
.await
.expect("Failed to insert global setting");
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let setting_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "notify_global_setting_change" && e.payload == setting_name)
.collect();
assert!(!setting_events.is_empty(), "Should have notify_global_setting_change event on insert");
}
#[sqlx::test(fixtures("base"))]
async fn test_trigger_notify_global_setting_change_update(db: Pool<Postgres>) {
// Use a unique setting name for testing
let setting_name = format!("test_setting_update_{}", uuid::Uuid::new_v4());
// First insert
sqlx::query("INSERT INTO global_settings (name, value) VALUES ($1, '{}'::jsonb)")
.bind(&setting_name)
.execute(&db)
.await
.expect("Failed to insert global setting");
let before_id = get_latest_event_id(&db).await.unwrap();
// Update the setting
sqlx::query("UPDATE global_settings SET value = '{\"updated\": true}'::jsonb WHERE name = $1")
.bind(&setting_name)
.execute(&db)
.await
.expect("Failed to update global setting");
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let setting_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "notify_global_setting_change" && e.payload == setting_name)
.collect();
assert!(!setting_events.is_empty(), "Should have notify_global_setting_change event on update");
}
#[sqlx::test(fixtures("base"))]
async fn test_trigger_notify_global_setting_change_delete(db: Pool<Postgres>) {
// Use a unique setting name for testing
let setting_name = format!("test_setting_delete_{}", uuid::Uuid::new_v4());
// First insert
sqlx::query("INSERT INTO global_settings (name, value) VALUES ($1, '{}'::jsonb)")
.bind(&setting_name)
.execute(&db)
.await
.expect("Failed to insert global setting");
let before_id = get_latest_event_id(&db).await.unwrap();
// Delete the setting
sqlx::query("DELETE FROM global_settings WHERE name = $1")
.bind(&setting_name)
.execute(&db)
.await
.expect("Failed to delete global setting");
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let setting_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "notify_global_setting_change" && e.payload == setting_name)
.collect();
assert!(!setting_events.is_empty(), "Should have notify_global_setting_change event on delete");
}
#[sqlx::test(fixtures("base"))]
async fn test_trigger_notify_workspace_envs_change(db: Pool<Postgres>) {
let before_id = get_latest_event_id(&db).await.unwrap();
// Insert a workspace env (test-workspace exists from fixture)
sqlx::query(
"INSERT INTO workspace_env (workspace_id, name, value) VALUES ('test-workspace', 'TEST_ENV_VAR', 'test_value')
ON CONFLICT (workspace_id, name) DO UPDATE SET value = 'test_value_updated'",
)
.execute(&db)
.await
.expect("Failed to insert workspace env");
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let env_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "notify_workspace_envs_change" && e.payload == "test-workspace")
.collect();
assert!(!env_events.is_empty(), "Should have notify_workspace_envs_change event");
}
#[sqlx::test(fixtures("base"))]
async fn test_trigger_notify_workspace_key_change(db: Pool<Postgres>) {
let before_id = get_latest_event_id(&db).await.unwrap();
// Insert a workspace key (base fixture already has one, so this will conflict and update)
sqlx::query(
"INSERT INTO workspace_key (workspace_id, kind, key) VALUES ('test-workspace', 'cloud', 'test_key_value')
ON CONFLICT (workspace_id, kind) DO UPDATE SET key = 'test_key_value_updated'",
)
.execute(&db)
.await
.expect("Failed to insert workspace key");
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let key_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "notify_workspace_key_change" && e.payload == "test-workspace")
.collect();
assert!(!key_events.is_empty(), "Should have notify_workspace_key_change event");
}
#[sqlx::test(fixtures("base"))]
async fn test_trigger_notify_token_invalidation(db: Pool<Postgres>) {
// First insert a session token
let token = format!("test_token_{}", uuid::Uuid::new_v4());
sqlx::query(
"INSERT INTO token (token, label, email, workspace_id, owner, expiration)
VALUES ($1, 'session', 'test@test.com', 'test-workspace', 'test-user', now() + interval '1 hour')",
)
.bind(&token)
.execute(&db)
.await
.expect("Failed to insert token");
let before_id = get_latest_event_id(&db).await.unwrap();
// Delete the token (should trigger notification)
sqlx::query("DELETE FROM token WHERE token = $1")
.bind(&token)
.execute(&db)
.await
.expect("Failed to delete token");
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let token_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "notify_token_invalidation" && e.payload == token)
.collect();
assert!(!token_events.is_empty(), "Should have notify_token_invalidation event");
}
#[sqlx::test(fixtures("base"))]
async fn test_trigger_notify_webhook_change(db: Pool<Postgres>) {
let before_id = get_latest_event_id(&db).await.unwrap();
// Update webhook setting (workspace_settings exists from base fixture)
sqlx::query("UPDATE workspace_settings SET webhook = 'https://test.webhook.com' WHERE workspace_id = 'test-workspace'")
.execute(&db)
.await
.expect("Failed to update webhook");
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let webhook_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "notify_webhook_change" && e.payload == "test-workspace")
.collect();
assert!(!webhook_events.is_empty(), "Should have notify_webhook_change event");
}
#[sqlx::test(fixtures("base"))]
async fn test_trigger_notify_workspace_premium_change(db: Pool<Postgres>) {
let before_id = get_latest_event_id(&db).await.unwrap();
// Toggle premium status
sqlx::query("UPDATE workspace SET premium = NOT premium WHERE id = 'test-workspace'")
.execute(&db)
.await
.expect("Failed to update workspace premium");
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let premium_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "notify_workspace_premium_change" && e.payload == "test-workspace")
.collect();
assert!(!premium_events.is_empty(), "Should have notify_workspace_premium_change event");
}
// ============================================================================
// HTTP Trigger Tests
// ============================================================================
#[sqlx::test(fixtures("base"))]
async fn test_trigger_notify_http_trigger_change(db: Pool<Postgres>) {
let before_id = get_latest_event_id(&db).await.unwrap();
let trigger_path = format!("test_http_trigger_{}", uuid::Uuid::new_v4());
// Insert an HTTP trigger
sqlx::query(
"INSERT INTO http_trigger (path, route_path, route_path_key, script_path, is_flow, workspace_id, edited_by, email, http_method, authentication_method)
VALUES ($1, '/test/route', '/test/route', 'test/script', false, 'test-workspace', 'test-user', 'test@test.com', 'get', 'none')",
)
.bind(&trigger_path)
.execute(&db)
.await
.expect("Failed to insert HTTP trigger");
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let http_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "notify_http_trigger_change")
.filter(|e| e.payload.contains("test-workspace") && e.payload.contains(&trigger_path))
.collect();
assert!(!http_events.is_empty(), "Should have notify_http_trigger_change event");
}
// ============================================================================
// Script/Flow Version Change Tests
// ============================================================================
#[sqlx::test(fixtures("base"))]
async fn test_trigger_notify_runnable_version_change_script(db: Pool<Postgres>) {
// First create a script without lock
let script_path = format!("f/test/script_{}", uuid::Uuid::new_v4());
let script_hash: i64 = rand::random::<i64>().abs();
sqlx::query(
"INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, language, kind)
VALUES ('test-workspace', $1, $2, 'test', 'test', 'def main(): pass', 'test-user', 'python3', 'script')",
)
.bind(script_hash)
.bind(&script_path)
.execute(&db)
.await
.expect("Failed to insert script");
let before_id = get_latest_event_id(&db).await.unwrap();
// Update the lock field (this should trigger the notification)
sqlx::query("UPDATE script SET lock = 'test_lock_content' WHERE hash = $1")
.bind(script_hash)
.execute(&db)
.await
.expect("Failed to update script lock");
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let script_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "notify_runnable_version_change")
.filter(|e| e.payload.contains("test-workspace") && e.payload.contains("script"))
.collect();
assert!(!script_events.is_empty(), "Should have notify_runnable_version_change event for script");
// Verify payload format: workspace_id:source_type:path:kind
let parts: Vec<&str> = script_events[0].payload.split(':').collect();
assert!(parts.len() >= 4, "Payload should have at least 4 parts");
assert_eq!(parts[0], "test-workspace", "First part should be workspace_id");
assert_eq!(parts[1], "script", "Second part should be 'script'");
}
#[sqlx::test(fixtures("base"))]
async fn test_trigger_notify_runnable_version_change_flow(db: Pool<Postgres>) {
// First create a flow with empty versions array
let flow_path = format!("f/test/flow_{}", uuid::Uuid::new_v4());
sqlx::query(
"INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, schema, versions)
VALUES ('test-workspace', $1, 'test', 'test', '{}'::jsonb, 'test-user', '{}'::json, ARRAY[]::bigint[])",
)
.bind(&flow_path)
.execute(&db)
.await
.expect("Failed to insert flow");
let before_id = get_latest_event_id(&db).await.unwrap();
// Update the flow's versions array (this triggers flow_versions_append_trigger)
sqlx::query(
"UPDATE flow SET versions = array_append(versions, 1::bigint) WHERE workspace_id = 'test-workspace' AND path = $1",
)
.bind(&flow_path)
.execute(&db)
.await
.expect("Failed to update flow versions");
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let flow_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "notify_runnable_version_change")
.filter(|e| e.payload.contains("test-workspace") && e.payload.contains("flow"))
.collect();
assert!(!flow_events.is_empty(), "Should have notify_runnable_version_change event for flow");
// Verify payload format
let parts: Vec<&str> = flow_events[0].payload.split(':').collect();
assert!(parts.len() >= 4, "Payload should have at least 4 parts");
assert_eq!(parts[0], "test-workspace", "First part should be workspace_id");
assert_eq!(parts[1], "flow", "Second part should be 'flow'");
}
// ============================================================================
// Concurrent Access Tests
// ============================================================================
#[sqlx::test(fixtures("base"))]
async fn test_concurrent_event_insertion(db: Pool<Postgres>) {
// Use a unique channel name for this test run
let channel = format!("test_concurrent_{}", uuid::Uuid::new_v4());
let before_id = get_latest_event_id(&db).await.unwrap();
// Insert multiple events concurrently
let handles: Vec<_> = (0..10)
.map(|i| {
let db = db.clone();
let ch = channel.clone();
tokio::spawn(async move {
sqlx::query_scalar::<_, i64>(
"INSERT INTO notify_event (channel, payload) VALUES ($1, $2) RETURNING id",
)
.bind(&ch)
.bind(format!("payload_{}", i))
.fetch_one(&db)
.await
.expect("Failed to insert event")
})
})
.collect();
// Wait for all insertions
for handle in handles {
handle.await.expect("Task should complete");
}
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let concurrent_events: Vec<_> = events
.iter()
.filter(|e| e.channel == channel)
.collect();
assert_eq!(concurrent_events.len(), 10, "Should have all 10 concurrent events");
// Verify all events have unique IDs
let ids: std::collections::HashSet<i64> = concurrent_events.iter().map(|e| e.id).collect();
assert_eq!(ids.len(), 10, "All events should have unique IDs");
}
#[sqlx::test(fixtures("base"))]
async fn test_polling_isolation(db: Pool<Postgres>) {
// Use a unique channel name for this test
let channel = format!("test_isolation_{}", uuid::Uuid::new_v4());
// Get baseline before inserting
let baseline_id = get_latest_event_id(&db).await.unwrap();
// Insert some events
let id1 = sqlx::query_scalar::<_, i64>(
"INSERT INTO notify_event (channel, payload) VALUES ($1, $2) RETURNING id",
)
.bind(&channel)
.bind("payload1")
.fetch_one(&db)
.await
.expect("Failed to insert event");
let id2 = sqlx::query_scalar::<_, i64>(
"INSERT INTO notify_event (channel, payload) VALUES ($1, $2) RETURNING id",
)
.bind(&channel)
.bind("payload2")
.fetch_one(&db)
.await
.expect("Failed to insert event");
let _id3 = sqlx::query_scalar::<_, i64>(
"INSERT INTO notify_event (channel, payload) VALUES ($1, $2) RETURNING id",
)
.bind(&channel)
.bind("payload3")
.fetch_one(&db)
.await
.expect("Failed to insert event");
// Two different "consumers" polling from different points
let events_from_baseline = poll_notify_events(&db, baseline_id).await.expect("Should poll events");
let events_from_id1 = poll_notify_events(&db, id1).await.expect("Should poll events");
let events_from_id2 = poll_notify_events(&db, id2).await.expect("Should poll events");
// Filter to our test events
let from_baseline: Vec<_> = events_from_baseline.iter().filter(|e| e.channel == channel).collect();
let from_id1: Vec<_> = events_from_id1.iter().filter(|e| e.channel == channel).collect();
let from_id2: Vec<_> = events_from_id2.iter().filter(|e| e.channel == channel).collect();
assert_eq!(from_baseline.len(), 3, "Polling from baseline should include all 3 events");
assert_eq!(from_id1.len(), 2, "Polling from id1 should include id2 and id3");
assert_eq!(from_id2.len(), 1, "Polling from id2 should include only id3");
}
// ============================================================================
// Edge Case Tests
// ============================================================================
#[sqlx::test(fixtures("base"))]
async fn test_empty_payload(db: Pool<Postgres>) {
let before_id = get_latest_event_id(&db).await.unwrap();
insert_test_event(&db, "test_empty_payload", "").await;
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let empty_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "test_empty_payload")
.collect();
assert_eq!(empty_events.len(), 1, "Should have event with empty payload");
assert_eq!(empty_events[0].payload, "", "Payload should be empty string");
}
#[sqlx::test(fixtures("base"))]
async fn test_large_payload(db: Pool<Postgres>) {
let before_id = get_latest_event_id(&db).await.unwrap();
// Create a large payload (1KB)
let large_payload = "x".repeat(1024);
insert_test_event(&db, "test_large_payload", &large_payload).await;
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let large_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "test_large_payload")
.collect();
assert_eq!(large_events.len(), 1, "Should have event with large payload");
assert_eq!(large_events[0].payload.len(), 1024, "Payload should be preserved");
}
#[sqlx::test(fixtures("base"))]
async fn test_special_characters_in_payload(db: Pool<Postgres>) {
let before_id = get_latest_event_id(&db).await.unwrap();
let special_payload = r#"{"key": "value with \"quotes\" and 'apostrophes'", "unicode": "日本語", "newline": "line1\nline2"}"#;
insert_test_event(&db, "test_special_chars", special_payload).await;
let events = poll_notify_events(&db, before_id).await.expect("Should poll events");
let special_events: Vec<_> = events
.iter()
.filter(|e| e.channel == "test_special_chars")
.collect();
assert_eq!(special_events.len(), 1, "Should have event with special characters");
assert_eq!(special_events[0].payload, special_payload, "Special characters should be preserved");
}
#[sqlx::test(fixtures("base"))]
async fn test_cleanup_with_no_old_events(db: Pool<Postgres>) {
// Use a unique channel name for this test
let channel = format!("test_no_old_{}", uuid::Uuid::new_v4());
// Insert only recent events
sqlx::query("INSERT INTO notify_event (channel, payload) VALUES ($1, $2)")
.bind(&channel)
.bind("recent1")
.execute(&db)
.await
.expect("Failed to insert event");
sqlx::query("INSERT INTO notify_event (channel, payload) VALUES ($1, $2)")
.bind(&channel)
.bind("recent2")
.execute(&db)
.await
.expect("Failed to insert event");
let before_count = count_events_for_channel(&db, &channel).await;
assert_eq!(before_count, 2, "Should have 2 recent events");
// Cleanup old events (none of our events should be deleted since they're recent)
let _deleted = cleanup_old_events(&db, 10).await.expect("Should cleanup events");
let after_count = count_events_for_channel(&db, &channel).await;
assert_eq!(after_count, 2, "Recent events should not be deleted");
}
// ============================================================================
// Multi-Server Integration Tests
// ============================================================================
// These tests start two actual windmill server processes on different ports
// with LISTEN_NEW_EVENTS_INTERVAL_SEC=1, trigger DB changes, and verify
// both servers process the events via their log output.
use std::io::{BufRead, BufReader};
use std::process::{Child, Command, Stdio};
use std::sync::{Arc, Mutex};
struct ServerProcess {
child: Child,
log_lines: Arc<Mutex<Vec<String>>>,
_stdout_handle: std::thread::JoinHandle<()>,
_stderr_handle: std::thread::JoinHandle<()>,
}
impl ServerProcess {
fn start(port: u16, db_url: &str) -> Self {
let binary = std::env::var("WINDMILL_BINARY")
.unwrap_or_else(|_| format!("{}/target/debug/windmill", env!("CARGO_MANIFEST_DIR")));
let mut child = Command::new(&binary)
.env("DATABASE_URL", db_url)
.env("MODE", "server")
.env("PORT", port.to_string())
.env("LISTEN_NEW_EVENTS_INTERVAL_SEC", "1")
.env("RUST_LOG", "info")
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap_or_else(|e| panic!("Failed to start windmill on port {port}: {e}"));
let stdout = child.stdout.take().expect("Failed to capture stdout");
let stderr = child.stderr.take().expect("Failed to capture stderr");
let log_lines = Arc::new(Mutex::new(Vec::new()));
let log_lines_stdout = log_lines.clone();
let log_lines_stderr = log_lines.clone();
// Read both stdout and stderr into the same log buffer
let _reader_handle = std::thread::spawn(move || {
let reader = BufReader::new(stdout);
for line in reader.lines() {
if let Ok(line) = line {
log_lines_stdout.lock().unwrap().push(line);
}
}
});
let _stderr_handle = std::thread::spawn(move || {
let reader = BufReader::new(stderr);
for line in reader.lines() {
if let Ok(line) = line {
log_lines_stderr.lock().unwrap().push(line);
}
}
});
ServerProcess { child, log_lines, _stdout_handle: _reader_handle, _stderr_handle }
}
fn logs_contain(&self, needle: &str) -> bool {
self.log_lines.lock().unwrap().iter().any(|l| l.contains(needle))
}
fn dump_logs(&self) -> String {
self.log_lines.lock().unwrap().join("\n")
}
}
impl Drop for ServerProcess {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
/// Wait for server to be ready by polling its HTTP endpoint.
async fn wait_for_server(port: u16, timeout_secs: u64) -> bool {
let client = reqwest::Client::new();
let url = format!("http://127.0.0.1:{}/api/version", port);
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(timeout_secs);
while tokio::time::Instant::now() < deadline {
if client.get(&url).send().await.is_ok() {
return true;
}
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
}
false
}
/// Helper to get a database connection (only used by the e2e multi-server test)
async fn get_db() -> Pool<Postgres> {
let database_url = std::env::var("DATABASE_URL")
.unwrap_or_else(|_| "postgres://postgres:changeme@localhost:5432/windmill".to_string());
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&database_url)
.await
.expect("Failed to connect to database")
}
#[tokio::test]
#[ignore = "slow - starts two server processes with 1s poll interval"]
async fn test_two_server_processes_both_receive_event() {
let db_url = std::env::var("DATABASE_URL")
.unwrap_or_else(|_| "postgres://postgres:changeme@localhost:5432/windmill".to_string());
// Start two server processes on different ports with 1s poll interval
let mut server_a = ServerProcess::start(19100, &db_url);
let mut server_b = ServerProcess::start(19200, &db_url);
// Wait for both servers to be ready
let (ready_a, ready_b) = tokio::join!(
wait_for_server(19100, 30),
wait_for_server(19200, 30),
);
assert!(ready_a, "Server A (port 19100) failed to start. Logs:\n{}", server_a.dump_logs());
assert!(ready_b, "Server B (port 19200) failed to start. Logs:\n{}", server_b.dump_logs());
// Give servers a moment to complete their first poll cycle
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
// Trigger a global setting change via direct DB insert
let db = get_db().await;
let setting_name = format!("test_e2e_{}", uuid::Uuid::new_v4());
sqlx::query(
"INSERT INTO global_settings (name, value) VALUES ($1, '\"e2e_test\"'::jsonb)
ON CONFLICT (name) DO UPDATE SET value = '\"e2e_test\"'::jsonb",
)
.bind(&setting_name)
.execute(&db)
.await
.expect("Failed to insert global setting");
// Wait for at least 2 poll cycles (interval is 1s)
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
let needle = format!("Global setting change detected: {}", setting_name);
assert!(
server_a.logs_contain(&needle),
"Server A should have processed the global setting event.\nSearching for: {}\nServer A logs:\n{}",
needle, server_a.dump_logs()
);
assert!(
server_b.logs_contain(&needle),
"Server B should have processed the global setting event.\nSearching for: {}\nServer B logs:\n{}",
needle, server_b.dump_logs()
);
// Cleanup
sqlx::query("DELETE FROM global_settings WHERE name = $1")
.bind(&setting_name)
.execute(&db)
.await
.ok();
// Explicitly kill before drop to avoid port conflicts with other tests
let _ = server_a.child.kill();
let _ = server_b.child.kill();
}
backend/windmill-api/openapi.yaml
+49 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.623.1
version: 1.624.0
title: Windmill API
contact:
@@ -4021,6 +4021,13 @@ paths:
- setting
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: include_default
in: query
description: If true, include "_default_" in the list if primary workspace storage is set
required: false
schema:
type: boolean
default: false
responses:
"200":
description: status
@@ -16579,6 +16586,11 @@ paths:
$ref: "#/components/schemas/AssetUsageKind"
access_type:
$ref: "#/components/schemas/AssetUsageAccessType"
columns:
type: object
description: The columns used (for tables)
additionalProperties:
$ref: "#/components/schemas/AssetUsageAccessType"
created_at:
type: string
format: date-time
@@ -20763,6 +20775,17 @@ components:
type: array
items:
type: string
filters:
type: array
items:
type: object
properties:
key:
type: string
value: {}
required:
- key
- value
server_id:
type: string
last_server_ping:
@@ -20781,6 +20804,7 @@ components:
- kafka_resource_path
- group_id
- topics
- filters
NewKafkaTrigger:
type: object
@@ -20799,6 +20823,17 @@ components:
type: array
items:
type: string
filters:
type: array
items:
type: object
properties:
key:
type: string
value: {}
required:
- key
- value
mode:
$ref: "#/components/schemas/TriggerMode"
error_handler_path:
@@ -20815,6 +20850,7 @@ components:
- kafka_resource_path
- group_id
- topics
- filters
EditKafkaTrigger:
type: object
@@ -20827,6 +20863,17 @@ components:
type: array
items:
type: string
filters:
type: array
items:
type: object
properties:
key:
type: string
value: {}
required:
- key
- value
path:
type: string
script_path:
@@ -20846,6 +20893,7 @@ components:
- kafka_resource_path
- group_id
- topics
- filters
- is_flow
NatsTrigger:
backend/windmill-api/src/assets.rs
+5 -3
View File
@@ -158,6 +158,7 @@ async fn list_assets(
'path', asset.usage_path,
'kind', asset.usage_kind,
'access_type', asset.usage_access_type,
'columns', asset.columns,
'created_at', asset.created_at,
'metadata', (CASE
WHEN asset.usage_kind = 'job' THEN
@@ -266,11 +267,12 @@ async fn list_assets_by_usages(
for usage in body.usages {
let assets = sqlx::query_scalar!(
r#"SELECT
jsonb_build_object(
jsonb_strip_nulls(jsonb_build_object(
'path', path,
'kind', kind,
'access_type', usage_access_type
) as "list!: _"
'access_type', usage_access_type,
'columns', columns
)) as "list!: _"
FROM asset
WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3
ORDER BY path, kind"#,
backend/windmill-api/src/jobs.rs
+8
View File
@@ -6378,10 +6378,18 @@ fn register_potential_assets_on_inline_execution(
match assets {
Some(Ok(assets)) => {
for asset in assets {
let columns = asset.columns.as_ref().map(|cols| {
cols.iter()
.map(|(col_name, col_access_type)| {
(col_name.clone(), (*col_access_type).into())
})
.collect()
});
register_runtime_asset(InsertRuntimeAssetParams {
access_type: asset.access_type.map(|a| a.into()),
asset_kind: asset.kind.into(),
asset_path: asset.path,
columns,
job_id,
workspace_id: w_id.to_string(),
created_at: None,
backend/windmill-api/src/triggers/filter.rs
+144
View File
@@ -0,0 +1,144 @@
use serde::{
de::{self, MapAccess, Visitor},
Deserialize, Deserializer,
};
use serde_json::Value;
use std::fmt;
#[derive(Deserialize)]
pub struct JsonFilter {
pub key: String,
pub value: Value,
}
#[derive(Deserialize)]
#[serde(untagged)]
pub enum Filter {
JsonFilter(JsonFilter),
}
struct SupersetVisitor<'a> {
key: &'a str,
value_to_check: &'a Value,
}
impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> {
type Value = bool;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("a JSON object with a specific key at the top level")
}
fn visit_map<V>(self, mut map: V) -> std::result::Result<Self::Value, V::Error>
where
V: MapAccess<'de>,
{
let mut result = false;
let mut found = false;
// Must consume entire map to satisfy deserializer contract
while let Some(key) = map.next_key::<String>()? {
if !found && key == self.key {
let json_value: Value = map.next_value()?;
result = is_superset(&json_value, self.value_to_check);
found = true;
} else {
// Skip values we don't need (cheaper than full deserialization)
let _ = map.next_value::<de::IgnoredAny>()?;
}
}
Ok(result)
}
}
pub fn is_superset(json_value: &Value, value_to_check: &Value) -> bool {
match (json_value, value_to_check) {
(Value::Object(json_map), Value::Object(check_map)) => {
check_map.iter().all(|(k, v)| {
json_map
.get(k)
.map_or(false, |json_val| is_superset(json_val, v))
})
}
(Value::Array(json_array), Value::Array(check_array)) => {
check_array.iter().all(|check_item| {
json_array
.iter()
.any(|json_item| is_superset(json_item, check_item))
})
}
_ => json_value == value_to_check,
}
}
pub fn is_value_superset<'a, 'de, D>(
deserializer: D,
key: &'a str,
value_to_check: &'a Value,
) -> std::result::Result<bool, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_map(SupersetVisitor { key, value_to_check })
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn test_filter_with_other_top_level_keys() {
let payload = r#"{"event_type": "test", "other": "data"}"#;
let key = "event_type";
let value = json!("test");
let mut deserializer = serde_json::Deserializer::from_str(payload);
let result = is_value_superset(&mut deserializer, key, &value).unwrap();
assert!(result, "Should match when key exists with correct value");
}
#[test]
fn test_filter_with_key_not_first() {
let payload = r#"{"other": "data", "event_type": "test"}"#;
let key = "event_type";
let value = json!("test");
let mut deserializer = serde_json::Deserializer::from_str(payload);
let result = is_value_superset(&mut deserializer, key, &value).unwrap();
assert!(result, "Should match even when key is not first");
}
#[test]
fn test_filter_with_nested_object() {
let payload = r#"{"data": {"status": "active", "count": 5}, "other": "value"}"#;
let key = "data";
let value = json!({"status": "active"});
let mut deserializer = serde_json::Deserializer::from_str(payload);
let result = is_value_superset(&mut deserializer, key, &value).unwrap();
assert!(result, "Should match when nested object is superset");
}
#[test]
fn test_filter_no_match() {
let payload = r#"{"event_type": "other", "data": "value"}"#;
let key = "event_type";
let value = json!("test");
let mut deserializer = serde_json::Deserializer::from_str(payload);
let result = is_value_superset(&mut deserializer, key, &value).unwrap();
assert!(!result, "Should not match when value differs");
}
#[test]
fn test_filter_key_not_found() {
let payload = r#"{"other": "data"}"#;
let key = "event_type";
let value = json!("test");
let mut deserializer = serde_json::Deserializer::from_str(payload);
let result = is_value_superset(&mut deserializer, key, &value).unwrap();
assert!(!result, "Should not match when key doesn't exist");
}
}
backend/windmill-api/src/triggers/http/handler.rs
+7 -1
View File
@@ -963,7 +963,13 @@ async fn route_job(
let s3_object = s3_object.map_err(|err| {
tracing::warn!("Error retrieving file from S3: {:?}", err);
Error::internal_err(format!("Error retrieving file: {}", err.to_string()))
let mut msg = format!("Error retrieving file: {err}");
let mut source = std::error::Error::source(&err);
while let Some(e) = source {
msg.push_str(&format!("\n caused by: {e}"));
source = e.source();
}
Error::internal_err(msg)
})?;
let mut response_headers = http::HeaderMap::new();
backend/windmill-api/src/triggers/mod.rs
+1
View File
@@ -30,6 +30,7 @@ pub mod sqs;
#[cfg(feature = "websocket")]
pub mod websocket;
pub mod filter;
pub mod global_handler;
mod handler;
mod listener;
backend/windmill-api/src/triggers/websocket/listener.rs
+4 -83
View File
@@ -1,5 +1,6 @@
use super::WebsocketTrigger;
use crate::triggers::{
filter::{is_value_superset, Filter, JsonFilter},
listener::ListeningTrigger,
trigger_helpers::{
trigger_runnable, trigger_runnable_and_wait_for_raw_result,
@@ -13,12 +14,9 @@ use async_trait::async_trait;
use futures::{stream::SplitSink, SinkExt, StreamExt};
use http::Response;
use itertools::Itertools;
use serde::{
de::{self, MapAccess, Visitor},
Deserialize, Deserializer,
};
use serde_json::{value::RawValue, Value};
use std::{borrow::Cow, collections::HashMap, fmt, sync::Arc};
use serde::Deserialize;
use serde_json::value::RawValue;
use std::{borrow::Cow, collections::HashMap, sync::Arc};
use tokio::{net::TcpStream, sync::RwLock};
use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream};
use windmill_common::{
@@ -443,18 +441,6 @@ impl Listener for WebsocketTrigger {
}
}
#[derive(Deserialize)]
pub struct JsonFilter {
key: String,
value: Value,
}
#[derive(Deserialize)]
#[serde(untagged)]
pub enum Filter {
JsonFilter(JsonFilter),
}
pub struct ReturnMessageChannels {
send_message_tx: tokio::sync::mpsc::Sender<String>,
killpill_rx: tokio::sync::broadcast::Receiver<()>,
@@ -477,71 +463,6 @@ enum InitialMessage {
RunnableResult { path: String, args: Box<RawValue>, is_flow: bool },
}
struct SupersetVisitor<'a> {
key: &'a str,
value_to_check: &'a Value,
}
impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> {
type Value = bool;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("a JSON object with a specific key at the top level")
}
fn visit_map<V>(self, mut map: V) -> std::result::Result<Self::Value, V::Error>
where
V: MapAccess<'de>,
{
while let Some(key) = map.next_key::<String>()? {
if key == self.key {
// Deserialize the value for the key and check if it's a superset
let json_value: Value = map.next_value()?;
return Ok(is_superset(&json_value, self.value_to_check));
} else {
// Skip the value if it's not the one we're interested in
let _ = map.next_value::<de::IgnoredAny>()?;
}
}
// If the key was not found, return false
Ok(false)
}
}
fn is_superset(json_value: &Value, value_to_check: &Value) -> bool {
match (json_value, value_to_check) {
(Value::Object(json_map), Value::Object(check_map)) => {
// Check that all keys and values in check_map exist and match in json_map
check_map.iter().all(|(k, v)| {
json_map
.get(k)
.map_or(false, |json_val| is_superset(json_val, v))
})
}
(Value::Array(json_array), Value::Array(check_array)) => {
// Check that all elements in check_array exist in json_array
check_array.iter().all(|check_item| {
json_array
.iter()
.any(|json_item| is_superset(json_item, check_item))
})
}
_ => json_value == value_to_check,
}
}
// A function to deserialize and check if the value at the given key is a superset of a passed value
fn is_value_superset<'a, 'de, D>(
deserializer: D,
key: &'a str,
value_to_check: &'a Value,
) -> std::result::Result<bool, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_map(SupersetVisitor { key, value_to_check })
}
fn raw_value_to_args_hashmap(
args: Option<&Box<RawValue>>,
) -> Result<HashMap<String, Box<RawValue>>> {
backend/windmill-api/src/workspaces.rs
+26 -1
View File
@@ -895,18 +895,43 @@ async fn delete_slack_oauth_config(
))
}
#[derive(Deserialize)]
struct GetSecondaryStorageNamesQuery {
#[serde(default)]
include_default: bool,
}
async fn get_secondary_storage_names(
_authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Query(query): Query<GetSecondaryStorageNamesQuery>,
) -> JsonResult<Vec<String>> {
let result: Vec<String> = sqlx::query_scalar!(
let mut result: Vec<String> = sqlx::query_scalar!(
"SELECT jsonb_object_keys(large_file_storage->'secondary_storage') AS \"secondary_storage_name!: _\"
FROM workspace_settings WHERE workspace_id = $1",
&w_id
)
.fetch_all(&db)
.await?;
// If include_default is true, check if primary storage is set and add "_default_"
if query.include_default {
let has_primary_storage: Option<bool> = sqlx::query_scalar!(
"SELECT (large_file_storage IS NOT NULL
AND large_file_storage != 'null'::jsonb
AND jsonb_typeof(large_file_storage) = 'object') AS \"has_primary!\"
FROM workspace_settings WHERE workspace_id = $1",
&w_id
)
.fetch_optional(&db)
.await?;
if has_primary_storage.unwrap_or(false) {
result.insert(0, "_default_".to_string());
}
}
Ok(Json(result))
}
backend/windmill-common/src/assets.rs
+37 -15
View File
@@ -1,5 +1,6 @@
use serde::{Deserialize, Serialize};
use sqlx::PgExecutor;
use std::collections::BTreeMap;
use crate::{error, scripts::ScriptHash};
@@ -37,23 +38,15 @@ pub enum AssetUsageAccessType {
RW,
}
pub struct Asset {
pub path: String,
pub kind: AssetKind,
}
pub struct AssetUsage {
pub path: String,
pub kind: AssetUsageKind,
pub access_type: AssetUsageAccessType,
}
#[derive(Serialize, Deserialize, Debug, Clone, Hash, sqlx::Type)]
#[derive(Serialize, Deserialize, Debug, Clone, Hash)]
pub struct AssetWithAltAccessType {
pub path: String,
pub kind: AssetKind,
pub access_type: Option<AssetUsageAccessType>,
pub alt_access_type: Option<AssetUsageAccessType>,
/// Map of column name to access type for column-level access tracking
#[serde(skip_serializing_if = "Option::is_none")]
pub columns: Option<BTreeMap<String, AssetUsageAccessType>>,
}
pub async fn insert_static_asset_usage<'e>(
@@ -63,15 +56,22 @@ pub async fn insert_static_asset_usage<'e>(
usage_path: &str,
usage_kind: AssetUsageKind,
) -> error::Result<()> {
// Convert columns BTreeMap to JSONB format
let columns_json = asset
.columns
.as_ref()
.map(|cols| serde_json::to_value(cols).unwrap_or(serde_json::Value::Null));
sqlx::query!(
r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)
VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING"#,
r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns)
VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING"#,
workspace_id,
asset.path,
asset.kind as AssetKind,
(asset.access_type.or(asset.alt_access_type)) as Option<AssetUsageAccessType>,
usage_path,
usage_kind as AssetUsageKind
usage_kind as AssetUsageKind,
columns_json as Option<serde_json::Value>
)
.execute(executor)
.await?;
@@ -125,6 +125,28 @@ pub fn merge_asset_usage_access_types(
}
}
pub fn merge_asset_columns(
a: &Option<BTreeMap<String, AssetUsageAccessType>>,
b: &Option<BTreeMap<String, AssetUsageAccessType>>,
) -> Option<BTreeMap<String, AssetUsageAccessType>> {
match (a, b) {
(None, None) => None,
(Some(cols), None) | (None, Some(cols)) => Some(cols.clone()),
(Some(cols_a), Some(cols_b)) => {
let mut merged = cols_a.clone();
for (col, access_b) in cols_b {
let access_a = merged.get(col);
let merged_access =
merge_asset_usage_access_types(access_a.cloned(), Some(*access_b));
if let Some(access) = merged_access {
merged.insert(col.clone(), access);
}
}
Some(merged)
}
}
}
impl From<windmill_parser::asset_parser::AssetKind> for AssetKind {
fn from(parser_kind: windmill_parser::asset_parser::AssetKind) -> Self {
match parser_kind {
backend/windmill-common/src/lib.rs
+1
View File
@@ -60,6 +60,7 @@ pub mod job_metrics;
#[cfg(all(feature = "parquet", feature = "private"))]
pub mod job_s3_helpers_ee;
pub mod min_version;
pub mod notify_events;
#[cfg(feature = "parquet")]
pub mod job_s3_helpers_oss;
pub mod workspace_dependencies;
backend/windmill-common/src/notify_events.rs
+63
View File
@@ -0,0 +1,63 @@
/*
* Author: Windmill Labs
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Polling-based event notification system.
//!
//! This module provides a table-based alternative to PostgreSQL LISTEN/NOTIFY
//! for propagating cache invalidation and setting change events across
//! workers and servers.
use sqlx::{FromRow, Pool, Postgres};
use crate::error::Error;
#[derive(Debug, Clone, FromRow)]
pub struct NotifyEvent {
pub id: i64,
pub channel: String,
pub payload: String,
}
/// Fetch all events with id greater than `last_event_id`.
/// Returns events ordered by id ascending.
pub async fn poll_notify_events(
db: &Pool<Postgres>,
last_event_id: i64,
) -> Result<Vec<NotifyEvent>, Error> {
let events = sqlx::query_as::<_, NotifyEvent>(
"SELECT id, channel, payload FROM notify_event WHERE id > $1 ORDER BY id LIMIT 1000",
)
.bind(last_event_id)
.fetch_all(db)
.await?;
Ok(events)
}
/// Get the current maximum event id.
/// Used to initialize last_event_id on startup to avoid processing old events.
pub async fn get_latest_event_id(db: &Pool<Postgres>) -> Result<i64, Error> {
let result: (i64,) = sqlx::query_as("SELECT COALESCE(MAX(id), 0) FROM notify_event")
.fetch_one(db)
.await?;
Ok(result.0)
}
/// Delete events older than the specified number of minutes.
/// Returns the number of deleted rows.
pub async fn cleanup_old_events(db: &Pool<Postgres>, older_than_minutes: i32) -> Result<u64, Error> {
let result = sqlx::query(
"DELETE FROM notify_event WHERE created_at < now() - make_interval(mins => $1)",
)
.bind(older_than_minutes)
.execute(db)
.await?;
Ok(result.rows_affected())
}
backend/windmill-common/src/runtime_assets.rs
+13 -4
View File
@@ -1,13 +1,19 @@
use std::{collections::HashMap, sync::OnceLock};
use std::{
collections::{BTreeMap, HashMap},
sync::OnceLock,
};
use itertools::Itertools;
use serde_json::value::RawValue;
use sqlx::{Pool, Postgres, QueryBuilder};
use sqlx::{types::Json, Pool, Postgres, QueryBuilder};
use tokio::sync::mpsc;
use windmill_parser::asset_parser::parse_asset_syntax;
use crate::{
assets::{merge_asset_usage_access_types, AssetKind, AssetUsageAccessType, AssetUsageKind},
assets::{
merge_asset_columns, merge_asset_usage_access_types, AssetKind, AssetUsageAccessType,
AssetUsageKind,
},
error,
};
@@ -59,6 +65,7 @@ pub struct InsertRuntimeAssetParams {
pub job_id: uuid::Uuid,
pub access_type: Option<AssetUsageAccessType>,
pub created_at: Option<chrono::DateTime<chrono::Utc>>,
pub columns: Option<BTreeMap<String, AssetUsageAccessType>>,
}
async fn insert_runtime_assets(
@@ -66,13 +73,14 @@ async fn insert_runtime_assets(
assets: &[InsertRuntimeAssetParams],
) -> error::Result<()> {
for chunk in assets.chunks(1000) {
let mut query_builder = QueryBuilder::new("INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, created_at) ");
let mut query_builder = QueryBuilder::new("INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, columns, usage_kind, created_at) ");
query_builder.push_values(chunk, |mut b, asset| {
b.push_bind(&asset.workspace_id)
.push_bind(&asset.asset_path)
.push_bind(&asset.asset_kind)
.push_bind(&asset.access_type)
.push_bind(asset.job_id.to_string())
.push_bind(Json(&asset.columns))
.push_bind(&AssetUsageKind::Job)
.push_bind(&asset.created_at);
});
@@ -104,6 +112,7 @@ async fn prune_runtime_assets(
// Same job used the same asset multiple times
last_same_job.access_type =
merge_asset_usage_access_types(last_same_job.access_type, asset.access_type);
last_same_job.columns = merge_asset_columns(&last_same_job.columns, &asset.columns);
} else if v.len() < max_n {
v.push(asset);
}
backend/windmill-common/src/s3_helpers.rs
+9 -9
View File
@@ -793,8 +793,8 @@ pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result<Arc<
let store = store_builder.build().map_err(|err| {
tracing::error!("Error building object store client: {:?}", err);
error::Error::internal_err(format!(
"Error building object store client: {}",
err.to_string()
"Error building object store client: {:?}",
err
))
})?;
@@ -860,8 +860,8 @@ fn build_azure_blob_client(
let store = store_builder.build().map_err(|err| {
tracing::error!("Error building object store client: {:?}", err);
error::Error::internal_err(format!(
"Error building object store client: {}",
err.to_string()
"Error building object store client: {:?}",
err
))
})?;
@@ -900,8 +900,8 @@ async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result<Arc<d
.map_err(|err| {
tracing::error!("Error building GCS object store client: {:?}", err);
error::Error::internal_err(format!(
"Error building GCS object store client: {}",
err.to_string()
"Error building GCS object store client: {:?}",
err
))
})?;
@@ -1257,21 +1257,21 @@ pub fn lfs_to_object_store_resource(
match lfs {
LargeFileStorage::S3Storage(_) | LargeFileStorage::S3AwsOidc(_) => {
let s3_resource: S3Resource = serde_json::from_value(resource_value).map_err(|e| {
error::Error::internal_err(format!("Error parsing S3 resource: {}", e))
error::Error::internal_err(format!("Error parsing S3 resource: {e:?}"))
})?;
Ok(ObjectStoreResource::S3(s3_resource))
}
LargeFileStorage::AzureBlobStorage(_) | LargeFileStorage::AzureWorkloadIdentity(_) => {
let azure_blob_resource: AzureBlobResource = serde_json::from_value(resource_value)
.map_err(|e| {
error::Error::internal_err(format!("Error parsing Azure Blob resource: {}", e))
error::Error::internal_err(format!("Error parsing Azure Blob resource: {e:?}"))
})?;
Ok(ObjectStoreResource::Azure(azure_blob_resource))
}
LargeFileStorage::GoogleCloudStorage(_) => {
let gcs_resource: GcsResource =
serde_json::from_value(resource_value).map_err(|e| {
error::Error::internal_err(format!("Error parsing GCS resource: {}", e))
error::Error::internal_err(format!("Error parsing GCS resource: {e:?}"))
})?;
Ok(ObjectStoreResource::Gcs(gcs_resource))
}
backend/windmill-worker/src/bun_executor.rs
+196 -68
View File
@@ -16,13 +16,13 @@ use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
parse_npm_config, read_file, read_file_content, read_result, start_child_process,
write_file_binary, MaybeLock, OccupancyMetrics, StreamNotifier,
DEV_CONF_NSJAIL,
write_file_binary, MaybeLock, OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
handle_child::handle_child,
BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH,
DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, NPM_CONFIG_REGISTRY,
NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, get_proxy_envs_for_lang,
NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV,
};
use windmill_common::{
client::AuthedClient,
@@ -51,9 +51,9 @@ use windmill_common::s3_helpers::attempt_fetch_bytes;
use windmill_parser::Typ;
const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.js");
pub const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.js");
const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js");
pub const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js");
const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.config.proto");
@@ -64,6 +64,62 @@ pub const BUN_LOCKB_SPLIT_WINDOWS: &str = "\r\n//bun.lockb\r\n";
pub const EMPTY_FILE: &str = "<empty>";
/// Bun args for dedicated worker (without the script path)
pub const BUN_DEDICATED_WORKER_ARGS: &[&str] = &["run", "-i", "--prefer-offline"];
/// Generate the dedicated worker wrapper content.
/// - `arg_names`: The argument names for the main function (e.g., ["x", "y"])
/// - `main_import`: The import path for the main module (e.g., "./main.ts")
/// - `date_conversions`: Optional date conversion statements for Datetime args
pub fn generate_dedicated_worker_wrapper(
arg_names: &[&str],
main_import: &str,
date_conversions: Option<&str>,
) -> String {
let spread = arg_names.join(",");
let dates = date_conversions.unwrap_or("");
let is_debug = std::env::var("RUST_LOG").is_ok_and(|x| x == "windmill=debug");
let print_lines = if is_debug {
r#"console.log(line);"#
} else {
""
};
format!(
r#"
import * as Main from "{main_import}";
import * as Readline from "node:readline"
BigInt.prototype.toJSON = function () {{
return this.toString();
}};
console.log('start');
function getArgs(line) {{
let {{ {spread} }} = JSON.parse(line)
{dates}
return [ {spread} ];
}}
for await (const line of Readline.createInterface({{ input: process.stdin }})) {{
{print_lines}
if (line === "end") {{
process.exit(0);
}}
try {{
const args = getArgs(line);
const res = await Main.main(...args);
console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value));
}} catch (e) {{
console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: line }}));
}}
}}
"#
)
}
/// Returns (package.json, bun.lock(b), is_empty, is_binary)
fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool, bool) {
if let Some(index) = lockfile.find(BUN_LOCK_SPLIT) {
@@ -115,24 +171,25 @@ pub async fn gen_bun_lockfile(
gen_bunfig(job_dir).await?;
write_file(job_dir, "package.json", package_json_content.as_str())?;
} else {
let loader = RELATIVE_BUN_LOADER
.replace("W_ID", w_id)
.replace("BASE_INTERNAL_URL", base_internal_url)
.replace("TOKEN", token)
.replace(
"CURRENT_PATH",
&crate::common::use_flow_root_path(script_path),
)
.replace("RAW_GET_ENDPOINT", "raw");
write_file(
&job_dir,
"build.js",
&format!(
r#"
{}
{loader}
{RELATIVE_BUN_BUILDER}
"#,
RELATIVE_BUN_LOADER
.replace("W_ID", w_id)
.replace("BASE_INTERNAL_URL", base_internal_url)
.replace("TOKEN", token)
.replace(
"CURRENT_PATH",
&crate::common::use_flow_root_path(script_path)
)
.replace("RAW_GET_ENDPOINT", "raw")
"#
),
)?;
@@ -382,14 +439,14 @@ pub async fn install_bun_lockfile(
}
#[derive(PartialEq)]
enum LoaderMode {
pub enum LoaderMode {
Node,
Bun,
BunBundle,
NodeBundle,
BrowserBundle,
}
async fn build_loader(
pub async fn build_loader(
job_dir: &str,
base_internal_url: &str,
token: &str,
@@ -406,13 +463,14 @@ async fn build_loader(
&crate::common::use_flow_root_path(current_path),
)
.replace("RAW_GET_ENDPOINT", "raw_unpinned");
if mode == LoaderMode::Node {
write_file(
&job_dir,
"node_builder.ts",
&format!(
r#"
{}
{loader}
import {{ readdir }} from "node:fs/promises";
@@ -420,7 +478,6 @@ let fileNames = []
try {{
fileNames = await readdir("{job_dir}/node_modules")
}} catch (e) {{
}}
try {{
@@ -437,8 +494,7 @@ try {{
console.log("Failed to build node bundle");
process.exit(1);
}}
"#,
loader
"#
),
)?;
} else if mode == LoaderMode::Bun {
@@ -449,11 +505,10 @@ try {{
r#"
import {{ plugin }} from "bun";
{}
{loader}
plugin(p)
"#,
loader
"#
),
)?;
} else if mode == LoaderMode::BunBundle
@@ -465,7 +520,7 @@ plugin(p)
"node_builder.ts",
&format!(
r#"
{}
{loader}
try {{
await Bun.build({{
@@ -486,7 +541,6 @@ try {{
process.exit(1);
}}
"#,
loader,
if mode == LoaderMode::BunBundle {
"bun"
} else if mode == LoaderMode::NodeBundle {
@@ -1726,55 +1780,21 @@ pub async fn start_worker(
.map(|x| return format!("{x} = {x} ? new Date({x}) : undefined"))
.join("\n");
let spread = args.into_iter().map(|x| x.name).join(",");
let arg_names: Vec<&str> = args.iter().map(|x| x.name.as_str()).collect();
// logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str());
// we cannot use Bun.read and Bun.write because it results in an EBADF error on cloud
let is_debug = std::env::var("RUST_LOG").is_ok_and(|x| x == "windmill=debug");
let print_lines = if is_debug {
r#"console.log(line);"#
} else {
""
};
let main_import = if codebase.is_some() {
"./main.js"
} else {
"./main.ts"
};
let wrapper_content: String = format!(
r#"
import * as Main from "{main_import}";
import * as Readline from "node:readline"
BigInt.prototype.toJSON = function () {{
return this.toString();
}};
console.log('start');
function getArgs(line) {{
let {{ {spread} }} = JSON.parse(line)
{dates}
return [ {spread} ];
}}
for await (const line of Readline.createInterface({{ input: process.stdin }})) {{
{print_lines}
if (line === "end") {{
process.exit(0);
}}
try {{
const args = getArgs(line);
const res = await Main.main(...args);
console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value));
}} catch (e) {{
console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: line }}));
}}
}}
"#,
);
let dates_opt = if dates.is_empty() {
None
} else {
Some(dates.as_str())
};
let wrapper_content = generate_dedicated_worker_wrapper(&arg_names, main_import, dates_opt);
write_file(job_dir, "wrapper.mjs", &wrapper_content)?;
}
@@ -1865,3 +1885,111 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) {
.await
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_split_lockfile_text_unix() {
let lockfile = r#"{"dependencies":{"lodash":"^4.17.21"}}
//bun.lock
lockfile-content-here"#;
let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile);
assert_eq!(pkg, r#"{"dependencies":{"lodash":"^4.17.21"}}"#);
assert_eq!(lock, Some("lockfile-content-here"));
assert!(!is_empty);
assert!(!is_binary);
}
#[test]
fn test_split_lockfile_text_windows() {
let lockfile = "{\"dependencies\":{}}\r\n//bun.lock\r\nlockfile-content";
let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile);
assert_eq!(pkg, "{\"dependencies\":{}}");
assert_eq!(lock, Some("lockfile-content"));
assert!(!is_empty);
assert!(!is_binary);
}
#[test]
fn test_split_lockfile_binary_unix() {
let lockfile = r#"{"dependencies":{}}
//bun.lockb
YmluYXJ5LWNvbnRlbnQ="#; // base64 encoded "binary-content"
let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile);
assert_eq!(pkg, r#"{"dependencies":{}}"#);
assert_eq!(lock, Some("YmluYXJ5LWNvbnRlbnQ="));
assert!(!is_empty);
assert!(is_binary);
}
#[test]
fn test_split_lockfile_binary_windows() {
let lockfile = "{\"dependencies\":{}}\r\n//bun.lockb\r\nYmluYXJ5LWNvbnRlbnQ=";
let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile);
assert_eq!(pkg, "{\"dependencies\":{}}");
assert_eq!(lock, Some("YmluYXJ5LWNvbnRlbnQ="));
assert!(!is_empty);
assert!(is_binary);
}
#[test]
fn test_split_lockfile_empty() {
let lockfile = r#"{"dependencies":{}}
//bun.lock
<empty>"#;
let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile);
assert_eq!(pkg, r#"{"dependencies":{}}"#);
assert_eq!(lock, Some(EMPTY_FILE));
assert!(is_empty);
assert!(!is_binary);
}
#[test]
fn test_split_lockfile_no_lock() {
let lockfile = r#"{"dependencies":{"lodash":"^4.17.21"}}"#;
let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile);
assert_eq!(pkg, r#"{"dependencies":{"lodash":"^4.17.21"}}"#);
assert!(lock.is_none());
assert!(!is_empty);
assert!(!is_binary);
}
#[test]
fn test_split_lockfile_multiline_package_json() {
let lockfile = r#"{
"dependencies": {
"lodash": "^4.17.21"
}
}
//bun.lock
lockfile-content"#;
let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile);
assert_eq!(
pkg,
r#"{
"dependencies": {
"lodash": "^4.17.21"
}
}"#
);
assert_eq!(lock, Some("lockfile-content"));
assert!(!is_empty);
assert!(!is_binary);
}
}
backend/windmill-worker/src/js_eval_parity_tests.rs
+67
View File
@@ -486,6 +486,73 @@ mod parity_tests {
Ok(())
}
#[tokio::test]
async fn parity_date_serialization() -> anyhow::Result<()> {
let env = HashMap::new();
// Test direct Date object serialization (the key issue that was fixed)
// Both engines should serialize Date to ISO string via toJSON
test_parity("new Date('2024-01-15T12:30:00.000Z')", env.clone(), None).await?;
// Date within an object
test_parity(
"({ date: new Date('2024-01-15T00:00:00.000Z'), name: 'test' })",
env.clone(),
None,
)
.await?;
// Date within an array
test_parity(
"[new Date('2024-01-15T00:00:00.000Z'), new Date('2024-01-16T00:00:00.000Z')]",
env.clone(),
None,
)
.await?;
// Deeply nested Date
test_parity(
"({ level1: { level2: { date: new Date('2024-01-15T00:00:00.000Z') } } })",
env.clone(),
None,
)
.await?;
// Custom object with toJSON (arrow function style)
test_parity(
"({ value: 42, toJSON: () => ({ converted: 84 }) })",
env.clone(),
None,
)
.await?;
// toJSON that returns a Date (should be further serialized)
test_parity(
"({ toJSON: () => new Date('2024-01-15T00:00:00.000Z') })",
env.clone(),
None,
)
.await?;
Ok(())
}
#[tokio::test]
async fn parity_special_object_serialization() -> anyhow::Result<()> {
let env = HashMap::new();
// RegExp serialization (both should return {})
test_parity("/test/gi", env.clone(), None).await?;
// Map serialization (both should return {})
test_parity("new Map([['key', 'value']])", env.clone(), None).await?;
// Set serialization (both should return {})
test_parity("new Set([1, 2, 3])", env.clone(), None).await?;
Ok(())
}
#[tokio::test]
async fn parity_array_advanced() -> anyhow::Result<()> {
let mut env = HashMap::new();
backend/windmill-worker/src/js_eval_quickjs.rs
+194 -69
View File
@@ -266,24 +266,25 @@ async fn eval_quickjs_inner(
setup_results_proxy(&ctx, &globals, by_id, op_state_clone.clone())?;
}
// Determine if we need to add return statement
// Determine if we need to add return statement.
// Wrap with .then((x) => JSON.stringify(x ?? null)) to serialize the result
// using the standard JSON.stringify, matching deno_core's behavior exactly.
let code = if should_add_return_quickjs(&transformed_expr) {
format!("(async function() {{ return {}; }})()", transformed_expr)
format!("(async function() {{ return {}; }})().then((x) => JSON.stringify(x ?? null))", transformed_expr)
} else {
format!("(async function() {{ {} }})()", transformed_expr)
format!("(async function() {{ {} }})().then((x) => JSON.stringify(x ?? null))", transformed_expr)
};
// Evaluate the expression (returns a Promise)
// Evaluate the expression (returns a Promise that resolves to a JSON string)
let promise: rquickjs::Promise = ctx.eval(code).catch(&ctx).map_err(quickjs_error_to_anyhow)?;
// Await the promise
// Await the promise — result is already a JSON string from JSON.stringify
let result: Value = promise.into_future().await.catch(&ctx).map_err(quickjs_error_to_anyhow)?;
// Convert result to JSON
let json_result = js_to_json(&ctx, &result)?;
let json_str = serde_json::to_string(&json_result)?;
let json_str = String::from_js(&ctx, result)
.unwrap_or_else(|_| "null".to_string());
Ok(windmill_common::worker::to_raw_value(&serde_json::from_str::<serde_json::Value>(&json_str)?))
Ok(crate::common::unsafe_raw(json_str))
})
.await
}
@@ -557,66 +558,11 @@ fn json_to_js<'js>(
}
/// Convert a QuickJS Value to a serde_json::Value
fn js_to_json<'js>(
ctx: &rquickjs::Ctx<'js>,
val: &Value<'js>,
) -> anyhow::Result<serde_json::Value> {
if val.is_null() || val.is_undefined() {
return Ok(serde_json::Value::Null);
}
if let Some(b) = val.as_bool() {
return Ok(serde_json::Value::Bool(b));
}
if let Some(i) = val.as_int() {
return Ok(serde_json::Value::Number(i.into()));
}
if let Some(f) = val.as_float() {
// Check if this float represents an exact integer
// This preserves integer formatting for values like timestamps
if f.fract() == 0.0 && f.abs() <= (i64::MAX as f64) {
let i = f as i64;
// Verify the conversion is exact (for very large numbers)
if (i as f64) == f {
return Ok(serde_json::Value::Number(i.into()));
}
}
if let Some(n) = serde_json::Number::from_f64(f) {
return Ok(serde_json::Value::Number(n));
} else {
return Ok(serde_json::Value::Null);
}
}
if let Ok(s) = String::from_js(ctx, val.clone()) {
return Ok(serde_json::Value::String(s));
}
if let Ok(arr) = rquickjs::Array::from_js(ctx, val.clone()) {
let mut json_arr = Vec::new();
for i in 0..arr.len() {
if let Ok(item) = arr.get::<Value>(i) {
json_arr.push(js_to_json(ctx, &item)?);
}
}
return Ok(serde_json::Value::Array(json_arr));
}
if let Ok(obj) = Object::from_js(ctx, val.clone()) {
let mut json_obj = serde_json::Map::new();
for res in obj.props::<String, Value>() {
if let Ok((k, v)) = res {
json_obj.insert(k, js_to_json(ctx, &v)?);
}
}
return Ok(serde_json::Value::Object(json_obj));
}
// Fallback
Ok(serde_json::Value::String("[object]".to_string()))
}
///
/// This mimics JavaScript's JSON.stringify behavior:
/// - For objects with a `toJSON` method (like Date), call it and use the result
/// - Arrays are recursively serialized
/// - Plain objects enumerate their own properties
/// Determines if we should prepend "return" to the expression
fn should_add_return_quickjs(expr: &str) -> bool {
@@ -794,4 +740,183 @@ mod tests {
assert!(!should_add_return_quickjs("let x = 5; x + 1"));
}
#[tokio::test]
async fn test_eval_quickjs_date_serialization() -> anyhow::Result<()> {
// Test that Date objects serialize to ISO strings, matching JSON.stringify behavior
let result = eval_timeout_quickjs(
"new Date('2024-01-15T12:30:00.000Z')".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
// Should be an ISO string, not an empty object
assert_eq!(result.get(), "\"2024-01-15T12:30:00.000Z\"");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_date_in_object() -> anyhow::Result<()> {
// Test that Date objects within other objects serialize correctly
let result = eval_timeout_quickjs(
"({ date: new Date('2024-01-15T12:30:00.000Z'), name: 'test' })".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
let value: serde_json::Value = serde_json::from_str(result.get())?;
assert_eq!(value["date"], "2024-01-15T12:30:00.000Z");
assert_eq!(value["name"], "test");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_date_in_array() -> anyhow::Result<()> {
// Test that Date objects in arrays serialize correctly
let result = eval_timeout_quickjs(
"[new Date('2024-01-15T00:00:00.000Z'), new Date('2024-01-16T00:00:00.000Z')]"
.to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
let value: serde_json::Value = serde_json::from_str(result.get())?;
assert_eq!(value[0], "2024-01-15T00:00:00.000Z");
assert_eq!(value[1], "2024-01-16T00:00:00.000Z");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_custom_tojson() -> anyhow::Result<()> {
// Test that JSON.stringify handles custom toJSON when returning objects
let result = eval_timeout_quickjs(
r#"({ a: 1, toJSON: () => ({ converted: true }) })"#.to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
// JSON.stringify should call toJSON and use that result
let value: serde_json::Value = serde_json::from_str(result.get())?;
assert_eq!(value["converted"], true);
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_deeply_nested_date() -> anyhow::Result<()> {
// Test that Date objects deep in nested structures are handled
let result = eval_timeout_quickjs(
"({ level1: { level2: { date: new Date('2024-01-15T00:00:00.000Z') } } })".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
let value: serde_json::Value = serde_json::from_str(result.get())?;
assert_eq!(
value["level1"]["level2"]["date"],
"2024-01-15T00:00:00.000Z"
);
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_regexp_serialization() -> anyhow::Result<()> {
// RegExp objects serialize to empty objects in JSON (same as JSON.stringify behavior)
let result = eval_timeout_quickjs(
"/test/gi".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
// RegExp doesn't have toJSON, so it serializes to an empty object (same as Deno)
assert_eq!(result.get(), "{}");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_map_serialization() -> anyhow::Result<()> {
// Map objects serialize to empty objects in JSON (same as JSON.stringify behavior)
let result = eval_timeout_quickjs(
"new Map([['key', 'value']])".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
// Map doesn't have toJSON, serializes to empty object (same as Deno)
assert_eq!(result.get(), "{}");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_set_serialization() -> anyhow::Result<()> {
// Set objects serialize to empty objects in JSON (same as JSON.stringify behavior)
let result = eval_timeout_quickjs(
"new Set([1, 2, 3])".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
// Set doesn't have toJSON, serializes to empty object (same as Deno)
assert_eq!(result.get(), "{}");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_tojson_returning_date() -> anyhow::Result<()> {
// When toJSON returns a Date object, JSON.stringify does NOT call toJSON again
// on the returned value (per spec). Date has no own enumerable properties,
// so it serializes to {}.
let result = eval_timeout_quickjs(
"({ toJSON: () => new Date('2024-01-15T00:00:00.000Z') })".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
assert_eq!(result.get(), "{}");
Ok(())
}
}
backend/windmill-worker/src/lib.rs
+3 -2
View File
@@ -95,8 +95,9 @@ pub use otel_tracing_proxy_ee::{load_internal_otel_exporter, DENO_OTEL_INITIALIZ
pub use result_processor::handle_job_error;
pub use bun_executor::{
compute_bundle_local_and_remote_path, get_common_bun_proc_envs, install_bun_lockfile,
prebundle_bun_script, prepare_job_dir,
build_loader, compute_bundle_local_and_remote_path, generate_dedicated_worker_wrapper,
get_common_bun_proc_envs, install_bun_lockfile, prebundle_bun_script, prepare_job_dir,
BUN_DEDICATED_WORKER_ARGS, LoaderMode, RELATIVE_BUN_BUILDER, RELATIVE_BUN_LOADER,
};
pub use deno_executor::generate_deno_lock;
pub use prepare_deps::run_prepare_deps_cli;
backend/windmill-worker/src/worker.rs
+9 -2
View File
@@ -1533,7 +1533,11 @@ pub async fn run_worker(
let is_dedicated_worker: bool = {
let config = WORKER_CONFIG.read().await;
config.dedicated_worker.is_some() || config.dedicated_workers.as_ref().is_some_and(|dws| !dws.is_empty())
config.dedicated_worker.is_some()
|| config
.dedicated_workers
.as_ref()
.is_some_and(|dws| !dws.is_empty())
};
#[cfg(feature = "benchmark")]
@@ -2045,7 +2049,9 @@ pub async fn run_worker(
dedicated_workers.get(&key)
})
} else {
job.runnable_path.as_ref().and_then(|path| dedicated_workers.get(path))
job.runnable_path
.as_ref()
.and_then(|path| dedicated_workers.get(path))
};
if let Some(dedicated_worker_tx) = dedicated_worker_tx {
let dedicated_job = DedicatedWorkerJob {
@@ -2773,6 +2779,7 @@ async fn detect_and_store_runtime_assets_from_job_args(
asset_kind: asset.kind,
access_type: None,
created_at: None,
columns: None,
};
register_runtime_asset(asset);
}
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.623.1";
export const VERSION = "v1.624.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/deno.lock
Generated
+2
View File
@@ -1553,6 +1553,8 @@
"https://deno.land/std@0.224.0/assert/unimplemented.ts": "8c55a5793e9147b4f1ef68cd66496b7d5ba7a9e7ca30c6da070c1a58da723d73",
"https://deno.land/std@0.224.0/assert/unreachable.ts": "5ae3dbf63ef988615b93eb08d395dda771c96546565f9e521ed86f6510c29e19",
"https://deno.land/std@0.224.0/cli/parse_args.ts": "5250832fb7c544d9111e8a41ad272c016f5a53f975ef84d5a9fe5fcb70566ece",
"https://deno.land/std@0.224.0/encoding/_util.ts": "beacef316c1255da9bc8e95afb1fa56ed69baef919c88dc06ae6cb7a6103d376",
"https://deno.land/std@0.224.0/encoding/hex.ts": "6270f25e5d85f99fcf315278670ba012b04b7c94b67715b53f30d03249687c07",
"https://deno.land/std@0.224.0/fmt/colors.ts": "508563c0659dd7198ba4bbf87e97f654af3c34eb56ba790260f252ad8012e1c5",
"https://deno.land/std@0.224.0/fs/_create_walk_entry.ts": "5d9d2aaec05bcf09a06748b1684224d33eba7a4de24cf4cf5599991ca6b5b412",
"https://deno.land/std@0.224.0/fs/_get_file_info_type.ts": "da7bec18a7661dba360a1db475b826b18977582ce6fc9b25f3d4ee0403fe8cbd",
cli/src/commands/sync/sync.ts
+9 -4
View File
@@ -1280,6 +1280,7 @@ export async function elementsToMap(
skips: Skips,
specificItems?: SpecificItemsConfig,
branchOverride?: string,
isRemote?: boolean,
): Promise<{ [key: string]: string }> {
const map: { [key: string]: string } = {};
const processedBasePaths = new Set<string>();
@@ -1446,7 +1447,8 @@ export async function elementsToMap(
continue;
}
// Skip base file if it's configured as branch-specific (expect branch version)
if (isSpecificItem(path, specificItems)) {
// Only for LOCAL files - remote workspace only has base paths
if (!isRemote && isSpecificItem(path, specificItems)) {
continue;
}
map[path] = content;
@@ -1486,13 +1488,14 @@ async function compareDynFSElement(
ignoreCodebaseChanges: boolean,
specificItems?: SpecificItemsConfig,
branchOverride?: string,
isEls1Remote?: boolean,
): Promise<Change[]> {
const [m1, m2] = els2
? await Promise.all([
elementsToMap(els1, ignore, json, skips, specificItems, branchOverride),
elementsToMap(els2, ignore, json, skips, specificItems, branchOverride),
elementsToMap(els1, ignore, json, skips, specificItems, branchOverride, isEls1Remote),
elementsToMap(els2, ignore, json, skips, specificItems, branchOverride, !isEls1Remote),
])
: [await elementsToMap(els1, ignore, json, skips, specificItems, branchOverride), {}];
: [await elementsToMap(els1, ignore, json, skips, specificItems, branchOverride, isEls1Remote), {}];
const changes: Change[] = [];
@@ -1995,6 +1998,7 @@ export async function pull(
true,
specificItems,
opts.branch,
true, // els1 (remote) is the remote source
);
log.info(
@@ -2486,6 +2490,7 @@ export async function push(
false,
specificItems,
opts.branch,
false, // els1 (local) is not the remote source
);
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
cli/src/main.ts
+1 -1
View File
@@ -77,7 +77,7 @@ export {
// }
// });
export const VERSION = "1.623.1";
export const VERSION = "1.624.0";
// Re-exported from constants.ts to maintain backwards compatibility
export { WM_FORK_PREFIX } from "./core/constants.ts";
cli/src/utils/metadata.ts
+192 -60
View File
@@ -211,6 +211,180 @@ export async function updateScriptSchema(
}
}
// ---------------------------------------------------------------------------
// Annotation parser — mirrors backend's WorkspaceDependenciesAnnotatedRefs::parse
// (windmill-common/src/workspace_dependencies.rs) so the cache key captures
// exactly the parts of scriptContent that affect lockfile generation.
// ---------------------------------------------------------------------------
type AnnotationMode = "manual" | "extra";
interface WorkspaceDepsAnnotation {
mode: AnnotationMode;
external: string[];
inline: string | null;
}
const LANG_ANNOTATION_CONFIG: Partial<
Record<ScriptLanguage, { comment: string; keyword: string; validityRe?: RegExp }>
> = {
python3: { comment: "#", keyword: "requirements", validityRe: /^#\s?(\S+)\s*$/ },
bun: { comment: "//", keyword: "package_json" },
nativets: { comment: "//", keyword: "package_json" },
go: { comment: "//", keyword: "go_mod" },
php: { comment: "//", keyword: "composer_json" },
};
export function extractWorkspaceDepsAnnotation(
scriptContent: string,
language: ScriptLanguage,
): WorkspaceDepsAnnotation | null {
const config = LANG_ANNOTATION_CONFIG[language];
if (!config) return null;
const { comment, keyword, validityRe } = config;
const extraMarker = `extra_${keyword}:`;
const manualMarker = `${keyword}:`;
const lines = scriptContent.split("\n");
// Find first annotation line (mirrors Rust find_position)
let pos = -1;
for (let i = 0; i < lines.length; i++) {
const l = lines[i];
if (l.startsWith(comment) && (l.includes(extraMarker) || l.includes(manualMarker))) {
pos = i;
break;
}
}
if (pos === -1) return null;
const annotationLine = lines[pos];
const mode: AnnotationMode = annotationLine.includes(extraMarker) ? "extra" : "manual";
// Parse external references from the annotation line
const marker = mode === "extra" ? extraMarker : manualMarker;
const unparsed = annotationLine.replaceAll(marker, "").replaceAll(comment, "");
const external = unparsed
.split(",")
.map((s) => s.trim())
.filter((s) => s.length > 0);
// Parse inline deps from subsequent lines
const inlineParts: string[] = [];
for (let i = pos + 1; i < lines.length; i++) {
const l = lines[i];
if (validityRe) {
const match = validityRe.exec(l);
if (match && match[1]) {
inlineParts.push(match[1]);
} else {
break;
}
} else {
if (!l.startsWith(comment)) {
break;
}
inlineParts.push(l.substring(comment.length));
}
}
const inlineStr = inlineParts.join("\n");
const inline = inlineStr.trim().length > 0 ? inlineStr : null;
return { mode, external, inline };
}
export async function computeLockCacheKey(
scriptContent: string,
language: ScriptLanguage,
rawWorkspaceDependencies: Record<string, string>,
): Promise<string> {
const annotation = extractWorkspaceDepsAnnotation(scriptContent, language);
const annotationStr = annotation
? `${annotation.mode}|${annotation.external.join(",")}|${annotation.inline ?? ""}`
: "none";
const sortedDepsKeys = Object.keys(rawWorkspaceDependencies).sort();
const depsStr = sortedDepsKeys.map((k) => `${k}=${rawWorkspaceDependencies[k]}`).join(";");
return await generateHash(`${language}|${annotationStr}|${depsStr}`);
}
const lockCache = new Map<string, string>();
export function clearLockCache(): void {
lockCache.clear();
}
async function fetchScriptLock(
workspace: Workspace,
scriptContent: string,
language: ScriptLanguage,
remotePath: string,
rawWorkspaceDependencies: Record<string, string>,
): Promise<string> {
const hasRawDeps = Object.keys(rawWorkspaceDependencies).length > 0;
const cacheKey = hasRawDeps
? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies)
: undefined;
if (cacheKey && lockCache.has(cacheKey)) {
log.info(`Using cached lockfile for ${remotePath}`);
return lockCache.get(cacheKey)!;
}
const extraHeaders = getHeaders();
const rawResponse = await fetch(
`${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/dependencies`,
{
method: "POST",
headers: {
Cookie: `token=${workspace.token}`,
"Content-Type": "application/json",
...extraHeaders,
},
body: JSON.stringify({
raw_scripts: [
{
raw_code: scriptContent,
language: language,
script_path: remotePath,
},
],
raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0
? rawWorkspaceDependencies : null,
entrypoint: remotePath,
}),
}
);
let responseText = "reading response failed";
try {
responseText = await rawResponse.text();
const response = JSON.parse(responseText);
const lock = response.lock;
if (lock === undefined) {
if (response?.["error"]?.["message"]) {
throw new LockfileGenerationError(
`Failed to generate lockfile: ${response?.["error"]?.["message"]}`
);
}
throw new LockfileGenerationError(
`Failed to generate lockfile: ${JSON.stringify(response, null, 2)}`
);
}
if (cacheKey) {
lockCache.set(cacheKey, lock);
}
return lock;
} catch (e) {
if (e instanceof LockfileGenerationError) {
throw e;
}
throw new LockfileGenerationError(
`Failed to generate lockfile:${rawResponse.statusText}, ${responseText}, ${e}`
);
}
}
async function updateScriptLock(
workspace: Workspace,
scriptContent: string,
@@ -235,70 +409,28 @@ async function updateScriptLock(
const dependencyPaths = Object.keys(rawWorkspaceDependencies).join(', ');
log.info(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`);
}
// generate the script lock running a dependency job in Windmill and update it inplace
// TODO: update this once the client is released
const extraHeaders = getHeaders();
const rawResponse = await fetch(
`${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/dependencies`,
{
method: "POST",
headers: {
Cookie: `token=${workspace.token}`,
"Content-Type": "application/json",
...extraHeaders,
},
body: JSON.stringify({
raw_scripts: [
{
raw_code: scriptContent,
language: language,
script_path: remotePath,
},
],
raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0
? rawWorkspaceDependencies : null,
entrypoint: remotePath,
}),
}
const lock = await fetchScriptLock(
workspace,
scriptContent,
language,
remotePath,
rawWorkspaceDependencies,
);
let responseText = "reading response failed";
try {
responseText = await rawResponse.text();
const response = JSON.parse(responseText);
const lock = response.lock;
if (lock === undefined) {
if (response?.["error"]?.["message"]) {
throw new LockfileGenerationError(
`Failed to generate lockfile: ${response?.["error"]?.["message"]}`
);
const lockPath = remotePath + ".script.lock";
if (lock != "") {
await Deno.writeTextFile(lockPath, lock);
metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/");
} else {
try {
if (await Deno.stat(lockPath)) {
await Deno.remove(lockPath);
}
throw new LockfileGenerationError(
`Failed to generate lockfile: ${JSON.stringify(response, null, 2)}`
);
} catch (e) {
log.info(colors.yellow(`Error removing lock file ${lockPath}: ${e}`));
}
const lockPath = remotePath + ".script.lock";
if (lock != "") {
await Deno.writeTextFile(lockPath, lock);
metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/");
} else {
try {
if (await Deno.stat(lockPath)) {
await Deno.remove(lockPath);
}
} catch (e) {
log.info(colors.yellow(`Error removing lock file ${lockPath}: ${e}`));
}
metadataContent.lock = "";
}
} catch (e) {
if (e instanceof LockfileGenerationError) {
throw e;
}
throw new LockfileGenerationError(
`Failed to generate lockfile:${rawResponse.statusText}, ${responseText}, ${e}`
);
metadataContent.lock = "";
}
}
cli/test/elements_to_map_branch_specific.test.ts
+476
View File
@@ -0,0 +1,476 @@
import { assertEquals } from "https://deno.land/std@0.224.0/assert/mod.ts";
// Import the function we need to test
import { elementsToMap } from "../src/commands/sync/sync.ts";
import type { SpecificItemsConfig } from "../src/core/specific_items.ts";
// =============================================================================
// elementsToMap TESTS FOR BRANCH-SPECIFIC ITEMS
// Tests for the regression where remote base files were incorrectly skipped
// when configured as branch-specific, causing them to be marked for deletion
// on pull operations.
//
// Regression: PR #7643 (commit 287b7e7d9, Jan 21, 2026)
// =============================================================================
/**
* Mock DynFSElement implementation for testing
*/
interface MockFile {
path: string;
content: string;
isDirectory?: boolean;
}
function createMockDynFSElement(files: MockFile[]): {
isDirectory: boolean;
path: string;
getContentText(): Promise<string>;
getChildren(): AsyncIterable<{
isDirectory: boolean;
path: string;
getContentText(): Promise<string>;
getChildren(): AsyncIterable<unknown>;
}>;
} {
return {
isDirectory: true,
path: "",
async getContentText() {
return "";
},
async *getChildren() {
for (const file of files) {
yield {
isDirectory: file.isDirectory ?? false,
path: file.path,
async getContentText() {
return file.content;
},
async *getChildren() {
// No children for files
},
};
}
},
};
}
const noIgnore = () => false;
const defaultSkips = {};
// =============================================================================
// REGRESSION TEST: Remote base files should NOT be skipped
// =============================================================================
Deno.test("elementsToMap: remote base file is NOT skipped when configured as branch-specific (isRemote=true)", async () => {
// This is the key regression test.
// When pulling from remote, the workspace only has base paths (e.g., TestVar.variable.yaml)
// These should NOT be skipped even if configured as branch-specific, because the remote
// workspace doesn't have branch-specific file naming.
const config: SpecificItemsConfig = {
variables: ["f/Shared/Variable/**"],
};
const remoteFiles: MockFile[] = [
{
path: "f/Shared/Variable/TestVar.variable.yaml",
content: "value: test\nis_secret: false",
},
];
const mockElement = createMockDynFSElement(remoteFiles);
// When isRemote=true, base files should be included even if they match branch-specific config
const result = await elementsToMap(
mockElement,
noIgnore,
false,
defaultSkips,
config,
"staging", // branchOverride
true, // isRemote = true
);
// The base file should be in the map
assertEquals(
Object.keys(result).includes("f/Shared/Variable/TestVar.variable.yaml"),
true,
"Remote base file should NOT be skipped when isRemote=true"
);
});
Deno.test("elementsToMap: local base file IS skipped when configured as branch-specific (isRemote=false)", async () => {
// When processing local files, if a base file is configured as branch-specific,
// it should be skipped because we expect the branch-specific version to be used instead.
const config: SpecificItemsConfig = {
variables: ["f/Shared/Variable/**"],
};
const localFiles: MockFile[] = [
{
path: "f/Shared/Variable/TestVar.variable.yaml",
content: "value: test\nis_secret: false",
},
];
const mockElement = createMockDynFSElement(localFiles);
// When isRemote=false, base files should be skipped if configured as branch-specific
const result = await elementsToMap(
mockElement,
noIgnore,
false,
defaultSkips,
config,
"staging", // branchOverride
false, // isRemote = false
);
// The base file should NOT be in the map (skipped because branch-specific expected)
assertEquals(
Object.keys(result).includes("f/Shared/Variable/TestVar.variable.yaml"),
false,
"Local base file SHOULD be skipped when isRemote=false and configured as branch-specific"
);
});
Deno.test("elementsToMap: local branch-specific file is mapped to base path (isRemote=false)", async () => {
// When processing local files with branch-specific naming, they should be mapped to base paths
const config: SpecificItemsConfig = {
variables: ["f/Shared/Variable/**"],
};
const localFiles: MockFile[] = [
{
path: "f/Shared/Variable/TestVar.staging.variable.yaml",
content: "value: staging-test\nis_secret: false",
},
];
const mockElement = createMockDynFSElement(localFiles);
const result = await elementsToMap(
mockElement,
noIgnore,
false,
defaultSkips,
config,
"staging", // branchOverride
false, // isRemote = false
);
// The branch-specific file should be mapped to the base path
assertEquals(
Object.keys(result).includes("f/Shared/Variable/TestVar.variable.yaml"),
true,
"Branch-specific file should be mapped to base path"
);
assertEquals(
result["f/Shared/Variable/TestVar.variable.yaml"],
"value: staging-test\nis_secret: false",
);
});
// =============================================================================
// PULL SCENARIO: Remote (base path) vs Local (branch-specific path)
// This simulates the actual pull scenario where:
// - Remote has: f/Shared/Variable/TestVar.variable.yaml
// - Local has: f/Shared/Variable/TestVar.staging.variable.yaml
// - Expected: No deletion, the files should match
// =============================================================================
Deno.test("elementsToMap: pull scenario - remote and local maps should align correctly", async () => {
const config: SpecificItemsConfig = {
variables: ["f/Shared/Variable/**"],
};
// Remote workspace has base path
const remoteFiles: MockFile[] = [
{
path: "f/Shared/Variable/TestVar.variable.yaml",
content: "value: test\nis_secret: false",
},
];
// Local has branch-specific path
const localFiles: MockFile[] = [
{
path: "f/Shared/Variable/TestVar.staging.variable.yaml",
content: "value: staging-test\nis_secret: false",
},
];
const remoteElement = createMockDynFSElement(remoteFiles);
const localElement = createMockDynFSElement(localFiles);
// Process remote (isRemote=true)
const remoteMap = await elementsToMap(
remoteElement,
noIgnore,
false,
defaultSkips,
config,
"staging",
true, // isRemote
);
// Process local (isRemote=false)
const localMap = await elementsToMap(
localElement,
noIgnore,
false,
defaultSkips,
config,
"staging",
false, // isRemote
);
// Both maps should have the same base path key
const remoteKeys = Object.keys(remoteMap);
const localKeys = Object.keys(localMap);
assertEquals(
remoteKeys.includes("f/Shared/Variable/TestVar.variable.yaml"),
true,
"Remote map should include base path"
);
assertEquals(
localKeys.includes("f/Shared/Variable/TestVar.variable.yaml"),
true,
"Local map should include base path (mapped from branch-specific)"
);
});
// =============================================================================
// NON-CONFIGURED ITEMS: Should work the same regardless of isRemote
// =============================================================================
Deno.test("elementsToMap: non-configured items included regardless of isRemote", async () => {
const config: SpecificItemsConfig = {
variables: ["f/Other/**"], // Only "Other" folder is branch-specific
};
const files: MockFile[] = [
{
path: "f/Shared/Variable/TestVar.variable.yaml",
content: "value: test\nis_secret: false",
},
];
const mockElement = createMockDynFSElement(files);
// Test with isRemote=true
const remoteResult = await elementsToMap(
mockElement,
noIgnore,
false,
defaultSkips,
config,
"staging",
true,
);
// Test with isRemote=false
const localResult = await elementsToMap(
createMockDynFSElement(files),
noIgnore,
false,
defaultSkips,
config,
"staging",
false,
);
// Both should include the file since it's not in the branch-specific config
assertEquals(
Object.keys(remoteResult).includes("f/Shared/Variable/TestVar.variable.yaml"),
true,
"Non-configured item should be included when isRemote=true"
);
assertEquals(
Object.keys(localResult).includes("f/Shared/Variable/TestVar.variable.yaml"),
true,
"Non-configured item should be included when isRemote=false"
);
});
// =============================================================================
// RESOURCE TYPE TESTS
// =============================================================================
Deno.test("elementsToMap: remote resource base file not skipped when configured", async () => {
const config: SpecificItemsConfig = {
resources: ["f/db/**"],
};
const remoteFiles: MockFile[] = [
{
path: "f/db/connection.resource.yaml",
content: "value: { host: localhost }",
},
];
const mockElement = createMockDynFSElement(remoteFiles);
const result = await elementsToMap(
mockElement,
noIgnore,
false,
defaultSkips,
config,
"staging",
true, // isRemote
);
assertEquals(
Object.keys(result).includes("f/db/connection.resource.yaml"),
true,
"Remote resource base file should NOT be skipped"
);
});
// =============================================================================
// TRIGGER TYPE TESTS
// =============================================================================
Deno.test("elementsToMap: remote trigger base file not skipped when configured", async () => {
const config: SpecificItemsConfig = {
triggers: ["f/webhooks/**"],
};
const remoteFiles: MockFile[] = [
{
path: "f/webhooks/handler.http_trigger.yaml",
content: "path: /webhook",
},
];
const mockElement = createMockDynFSElement(remoteFiles);
const result = await elementsToMap(
mockElement,
noIgnore,
false,
{ includeTriggers: true }, // Must include triggers explicitly
config,
"staging",
true, // isRemote
);
assertEquals(
Object.keys(result).includes("f/webhooks/handler.http_trigger.yaml"),
true,
"Remote trigger base file should NOT be skipped"
);
});
// =============================================================================
// SETTINGS TYPE TESTS
// =============================================================================
Deno.test("elementsToMap: remote settings.yaml not skipped when configured", async () => {
const config: SpecificItemsConfig = {
settings: true,
};
const remoteFiles: MockFile[] = [
{
path: "settings.yaml",
content: "openai_resource_path: null",
},
];
const mockElement = createMockDynFSElement(remoteFiles);
const result = await elementsToMap(
mockElement,
noIgnore,
false,
{ includeSettings: true },
config,
"staging",
true, // isRemote
);
assertEquals(
Object.keys(result).includes("settings.yaml"),
true,
"Remote settings.yaml should NOT be skipped"
);
});
// =============================================================================
// FOLDER TYPE TESTS
// =============================================================================
Deno.test("elementsToMap: remote folder meta not skipped when configured", async () => {
const config: SpecificItemsConfig = {
folders: ["f/env_*"],
};
const remoteFiles: MockFile[] = [
{
path: "f/env_staging/folder.meta.yaml",
content: "display_name: Staging Environment",
},
];
const mockElement = createMockDynFSElement(remoteFiles);
const result = await elementsToMap(
mockElement,
noIgnore,
false,
defaultSkips,
config,
"staging",
true, // isRemote
);
assertEquals(
Object.keys(result).includes("f/env_staging/folder.meta.yaml"),
true,
"Remote folder meta should NOT be skipped"
);
});
// =============================================================================
// BACKWARD COMPATIBILITY: isRemote undefined behaves like local (false)
// =============================================================================
Deno.test("elementsToMap: isRemote undefined behaves like local (backward compatible)", async () => {
const config: SpecificItemsConfig = {
variables: ["f/**"],
};
const files: MockFile[] = [
{
path: "f/test.variable.yaml",
content: "value: test\nis_secret: false",
},
];
const mockElement = createMockDynFSElement(files);
// When isRemote is undefined (backward compatibility), it should behave like local
const result = await elementsToMap(
mockElement,
noIgnore,
false,
defaultSkips,
config,
"staging",
// isRemote omitted
);
// Base file should be skipped (same behavior as isRemote=false)
assertEquals(
Object.keys(result).includes("f/test.variable.yaml"),
false,
"isRemote undefined should behave like isRemote=false (skip base file)"
);
});
cli/test/lock_cache.test.ts
+572
View File
@@ -0,0 +1,572 @@
/**
* Lock Cache Tests
*
* Tests the in-memory lock cache used when fetching lockfiles for scripts with
* raw_workspace_dependencies.
*
* Part 1: Unit tests for annotation parsing (mirrors backend).
* Part 2: Unit tests for cache key computation.
* Part 3: Behavioral tests comparing old logic (no cache, always fetches)
* vs new logic (caches by key, skips duplicate fetches).
*/
import {
assertEquals,
assertNotEquals,
} from "https://deno.land/std@0.224.0/assert/mod.ts";
import { encodeHex } from "https://deno.land/std@0.224.0/encoding/hex.ts";
// ---------------------------------------------------------------------------
// Mirrors extractWorkspaceDepsAnnotation + computeLockCacheKey from
// src/utils/metadata.ts so we can test the algorithm without pulling in the
// full (unresolvable-in-tests) module graph.
// ---------------------------------------------------------------------------
type AnnotationMode = "manual" | "extra";
interface WorkspaceDepsAnnotation {
mode: AnnotationMode;
external: string[];
inline: string | null;
}
const LANG_ANNOTATION_CONFIG: Record<
string,
{ comment: string; keyword: string; validityRe?: RegExp } | undefined
> = {
python3: { comment: "#", keyword: "requirements", validityRe: /^#\s?(\S+)\s*$/ },
bun: { comment: "//", keyword: "package_json" },
nativets: { comment: "//", keyword: "package_json" },
go: { comment: "//", keyword: "go_mod" },
php: { comment: "//", keyword: "composer_json" },
};
function extractWorkspaceDepsAnnotation(
scriptContent: string,
language: string,
): WorkspaceDepsAnnotation | null {
const config = LANG_ANNOTATION_CONFIG[language];
if (!config) return null;
const { comment, keyword, validityRe } = config;
const extraMarker = `extra_${keyword}:`;
const manualMarker = `${keyword}:`;
const lines = scriptContent.split("\n");
let pos = -1;
for (let i = 0; i < lines.length; i++) {
const l = lines[i];
if (l.startsWith(comment) && (l.includes(extraMarker) || l.includes(manualMarker))) {
pos = i;
break;
}
}
if (pos === -1) return null;
const annotationLine = lines[pos];
const mode: AnnotationMode = annotationLine.includes(extraMarker) ? "extra" : "manual";
const marker = mode === "extra" ? extraMarker : manualMarker;
const unparsed = annotationLine.replaceAll(marker, "").replaceAll(comment, "");
const external = unparsed
.split(",")
.map((s) => s.trim())
.filter((s) => s.length > 0);
const inlineParts: string[] = [];
for (let i = pos + 1; i < lines.length; i++) {
const l = lines[i];
if (validityRe) {
const match = validityRe.exec(l);
if (match && match[1]) {
inlineParts.push(match[1]);
} else {
break;
}
} else {
if (!l.startsWith(comment)) {
break;
}
inlineParts.push(l.substring(comment.length));
}
}
const inlineStr = inlineParts.join("\n");
const inline = inlineStr.trim().length > 0 ? inlineStr : null;
return { mode, external, inline };
}
async function computeLockCacheKey(
scriptContent: string,
language: string,
rawWorkspaceDependencies: Record<string, string>,
): Promise<string> {
const annotation = extractWorkspaceDepsAnnotation(scriptContent, language);
const annotationStr = annotation
? `${annotation.mode}|${annotation.external.join(",")}|${annotation.inline ?? ""}`
: "none";
const sortedDepsKeys = Object.keys(rawWorkspaceDependencies).sort();
const depsStr = sortedDepsKeys
.map((k) => `${k}=${rawWorkspaceDependencies[k]}`)
.join(";");
const content = `${language}|${annotationStr}|${depsStr}`;
const buf = new TextEncoder().encode(content);
return encodeHex(await crypto.subtle.digest("SHA-256", buf));
}
// ---------------------------------------------------------------------------
// Helpers that mirror the two fetch strategies (old / new).
// ---------------------------------------------------------------------------
interface ScriptInput {
scriptContent: string;
language: string;
remotePath: string;
rawWorkspaceDependencies: Record<string, string>;
}
/** Old logic: always calls the remote for every script. */
async function fetchScriptLockOld(
input: ScriptInput,
remoteFn: (input: ScriptInput) => Promise<string>,
): Promise<string> {
return await remoteFn(input);
}
/** New logic: only caches when raw_workspace_dependencies are non-empty. */
async function fetchScriptLockNew(
input: ScriptInput,
remoteFn: (input: ScriptInput) => Promise<string>,
cache: Map<string, string>,
): Promise<string> {
const hasRawDeps = Object.keys(input.rawWorkspaceDependencies).length > 0;
const cacheKey = hasRawDeps
? await computeLockCacheKey(
input.scriptContent,
input.language,
input.rawWorkspaceDependencies,
)
: undefined;
if (cacheKey && cache.has(cacheKey)) {
return cache.get(cacheKey)!;
}
const lock = await remoteFn(input);
if (cacheKey) {
cache.set(cacheKey, lock);
}
return lock;
}
// =============================================================================
// Part 1 — Annotation parsing
// =============================================================================
Deno.test("python: manual requirements with external refs + inline deps", () => {
const code = `# requirements: default, base
#requests==2.31.0
#pandas>=1.5.0
def main():
pass`;
const r = extractWorkspaceDepsAnnotation(code, "python3")!;
assertEquals(r.mode, "manual");
assertEquals(r.external, ["default", "base"]);
assertEquals(r.inline, "requests==2.31.0\npandas>=1.5.0");
});
Deno.test("python: extra_requirements mode", () => {
const code = `# extra_requirements: utils
#numpy>=1.24.0
def main():
pass`;
const r = extractWorkspaceDepsAnnotation(code, "python3")!;
assertEquals(r.mode, "extra");
assertEquals(r.external, ["utils"]);
assertEquals(r.inline, "numpy>=1.24.0");
});
Deno.test("python: empty requirements (opt-out)", () => {
const code = `# requirements:
def main():
pass`;
const r = extractWorkspaceDepsAnnotation(code, "python3")!;
assertEquals(r.mode, "manual");
assertEquals(r.external, []);
assertEquals(r.inline, null);
});
Deno.test("python: no annotation → null", () => {
const code = `def main():
print("hello")`;
assertEquals(extractWorkspaceDepsAnnotation(code, "python3"), null);
});
Deno.test("bun: package_json annotation with inline", () => {
const code = `// package_json: utils, base
//{
// "dependencies": {
// "axios": "^1.6.0"
// }
//}
export function main() {}`;
const r = extractWorkspaceDepsAnnotation(code, "bun")!;
assertEquals(r.mode, "manual");
assertEquals(r.external, ["utils", "base"]);
assertEquals(r.inline, `{
"dependencies": {
"axios": "^1.6.0"
}
}`);
});
Deno.test("go: go_mod annotation", () => {
const code = `// go_mod: base,
//github.com/gin-gonic/gin v1.9.1
package main
func main() {}`;
const r = extractWorkspaceDepsAnnotation(code, "go")!;
assertEquals(r.mode, "manual");
assertEquals(r.external, ["base"]);
assertEquals(r.inline, "github.com/gin-gonic/gin v1.9.1");
});
Deno.test("unsupported language → null", () => {
assertEquals(extractWorkspaceDepsAnnotation("print(1)", "deno"), null);
assertEquals(extractWorkspaceDepsAnnotation("print(1)", "bash"), null);
});
// =============================================================================
// Part 2 — Cache key computation
// =============================================================================
Deno.test("same annotation + language + deps → same key", async () => {
const code = `# requirements: default
#requests==2.31.0
print("hello")`;
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
const a = await computeLockCacheKey(code, "python3", deps);
const b = await computeLockCacheKey(code, "python3", deps);
assertEquals(a, b);
});
Deno.test("different code, same annotation → same key", async () => {
const codeA = `# requirements: default
#requests==2.31.0
print("hello")`;
const codeB = `# requirements: default
#requests==2.31.0
print("world")`;
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
assertEquals(
await computeLockCacheKey(codeA, "python3", deps),
await computeLockCacheKey(codeB, "python3", deps),
);
});
Deno.test("different annotation inline → different key", async () => {
const codeA = `# requirements: default
#requests==2.31.0
print("hello")`;
const codeB = `# requirements: default
#flask==3.0.0
print("hello")`;
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
assertNotEquals(
await computeLockCacheKey(codeA, "python3", deps),
await computeLockCacheKey(codeB, "python3", deps),
);
});
Deno.test("different annotation external refs → different key", async () => {
const codeA = `# requirements: default
print("hello")`;
const codeB = `# requirements: base
print("hello")`;
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
assertNotEquals(
await computeLockCacheKey(codeA, "python3", deps),
await computeLockCacheKey(codeB, "python3", deps),
);
});
Deno.test("manual vs extra mode → different key", async () => {
const codeA = `# requirements: default
print("hello")`;
const codeB = `# extra_requirements: default
print("hello")`;
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
assertNotEquals(
await computeLockCacheKey(codeA, "python3", deps),
await computeLockCacheKey(codeB, "python3", deps),
);
});
Deno.test("no annotation, same code → same key", async () => {
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
assertEquals(
await computeLockCacheKey("print('a')", "python3", deps),
await computeLockCacheKey("print('b')", "python3", deps),
);
});
Deno.test("different deps → different key", async () => {
const code = `# requirements: default
print("hello")`;
assertNotEquals(
await computeLockCacheKey(code, "python3", { d: "a" }),
await computeLockCacheKey(code, "python3", { d: "b" }),
);
});
Deno.test("different language → different key", async () => {
const deps = { d: "v" };
assertNotEquals(
await computeLockCacheKey("x", "bun", deps),
await computeLockCacheKey("x", "python3", deps),
);
});
Deno.test("dep key order does not matter", async () => {
const code = "print('hello')";
assertEquals(
await computeLockCacheKey(code, "python3", { a: "1", b: "2" }),
await computeLockCacheKey(code, "python3", { b: "2", a: "1" }),
);
});
// =============================================================================
// Part 3 — Multi-script fetch behavior: old logic vs new logic
// =============================================================================
function makeRemoteFn(): {
remoteFn: (input: ScriptInput) => Promise<string>;
callCount: () => number;
} {
const calls: ScriptInput[] = [];
return {
remoteFn: async (input: ScriptInput) => {
calls.push(input);
const depsStr = Object.entries(input.rawWorkspaceDependencies).sort().map(([k,v]) => `${k}=${v}`).join(",");
return `lock for ${input.language}:${input.scriptContent}:${depsStr}`;
},
callCount: () => calls.length,
};
}
// -- Two scripts, same annotation + language + deps -------------------------
Deno.test("old logic: two scripts same annotation → 2 remote calls", async () => {
const { remoteFn, callCount } = makeRemoteFn();
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
const scripts: ScriptInput[] = [
{ scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps },
{ scriptContent: "# requirements: default\nprint(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps },
];
for (const s of scripts) await fetchScriptLockOld(s, remoteFn);
assertEquals(callCount(), 2);
});
Deno.test("new logic: two scripts same annotation → 1 remote call (cache shared)", async () => {
const { remoteFn, callCount } = makeRemoteFn();
const cache = new Map<string, string>();
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
const scripts: ScriptInput[] = [
{ scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps },
{ scriptContent: "# requirements: default\nprint(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps },
];
const results: string[] = [];
for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache));
assertEquals(callCount(), 1);
assertEquals(results[0], results[1]);
});
// -- Two scripts, different annotations + same deps -------------------------
Deno.test("new logic: different annotations same deps → 2 remote calls", async () => {
const { remoteFn, callCount } = makeRemoteFn();
const cache = new Map<string, string>();
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
const scripts: ScriptInput[] = [
{ scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps },
{ scriptContent: "# requirements: base\nprint(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps },
];
const results: string[] = [];
for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache));
assertEquals(callCount(), 2);
assertNotEquals(results[0], results[1]);
});
// -- Two scripts, same annotation + different deps --------------------------
Deno.test("new logic: same annotation different deps → 2 remote calls", async () => {
const { remoteFn, callCount } = makeRemoteFn();
const cache = new Map<string, string>();
const scripts: ScriptInput[] = [
{ scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a",
rawWorkspaceDependencies: { "dependencies/requirements.in": "requests==2.31.0" } },
{ scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "b",
rawWorkspaceDependencies: { "dependencies/requirements.in": "requests==2.32.0" } },
];
const results: string[] = [];
for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache));
assertEquals(callCount(), 2);
assertNotEquals(results[0], results[1]);
});
// -- Many scripts, same annotation + deps -----------------------------------
Deno.test("old logic: 5 scripts same annotation+deps → 5 remote calls", async () => {
const { remoteFn, callCount } = makeRemoteFn();
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
const ann = "# requirements: default\n";
const scripts: ScriptInput[] = [
{ scriptContent: ann + "print(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps },
{ scriptContent: ann + "print(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps },
{ scriptContent: ann + "print(3)", language: "python3", remotePath: "c", rawWorkspaceDependencies: deps },
{ scriptContent: ann + "print(1)", language: "python3", remotePath: "d", rawWorkspaceDependencies: deps },
{ scriptContent: ann + "print(2)", language: "python3", remotePath: "e", rawWorkspaceDependencies: deps },
];
for (const s of scripts) await fetchScriptLockOld(s, remoteFn);
assertEquals(callCount(), 5);
});
Deno.test("new logic: 5 scripts same annotation+deps → 1 remote call", async () => {
const { remoteFn, callCount } = makeRemoteFn();
const cache = new Map<string, string>();
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
const ann = "# requirements: default\n";
const scripts: ScriptInput[] = [
{ scriptContent: ann + "print(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps },
{ scriptContent: ann + "print(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps },
{ scriptContent: ann + "print(3)", language: "python3", remotePath: "c", rawWorkspaceDependencies: deps },
{ scriptContent: ann + "print(1)", language: "python3", remotePath: "d", rawWorkspaceDependencies: deps },
{ scriptContent: ann + "print(2)", language: "python3", remotePath: "e", rawWorkspaceDependencies: deps },
];
const results: string[] = [];
for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache));
assertEquals(callCount(), 1);
for (let i = 1; i < results.length; i++) {
assertEquals(results[0], results[i]);
}
});
// -- Many scripts, 2 annotation groups + same deps -------------------------
Deno.test("new logic: 4 scripts with 2 annotation groups → 2 remote calls", async () => {
const { remoteFn, callCount } = makeRemoteFn();
const cache = new Map<string, string>();
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
const scripts: ScriptInput[] = [
{ scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps },
{ scriptContent: "# requirements: base\nprint(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps },
{ scriptContent: "# requirements: default\nprint(3)", language: "python3", remotePath: "c", rawWorkspaceDependencies: deps },
{ scriptContent: "# requirements: base\nprint(4)", language: "python3", remotePath: "d", rawWorkspaceDependencies: deps },
];
const results: string[] = [];
for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache));
assertEquals(callCount(), 2);
assertEquals(results[0], results[2]); // same annotation "default"
assertEquals(results[1], results[3]); // same annotation "base"
assertNotEquals(results[0], results[1]);
});
// -- Scripts with no workspace deps (empty) ---------------------------------
Deno.test("new logic: empty deps → no caching", async () => {
const { remoteFn, callCount } = makeRemoteFn();
const cache = new Map<string, string>();
const scripts: ScriptInput[] = [
{ scriptContent: "print(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: {} },
{ scriptContent: "print(1)", language: "python3", remotePath: "b", rawWorkspaceDependencies: {} },
];
for (const s of scripts) await fetchScriptLockNew(s, remoteFn, cache);
assertEquals(callCount(), 2);
assertEquals(cache.size, 0);
});
// -- No annotation scripts with raw deps → share cache ---------------------
Deno.test("new logic: no annotation + same deps → 1 remote call", async () => {
const { remoteFn, callCount } = makeRemoteFn();
const cache = new Map<string, string>();
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
const scripts: ScriptInput[] = [
{ scriptContent: "print(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps },
{ scriptContent: "print(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps },
];
const results: string[] = [];
for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache));
assertEquals(callCount(), 1);
assertEquals(results[0], results[1]);
});
// -- Mix of annotated and non-annotated scripts -----------------------------
Deno.test("new logic: mix of annotated and non-annotated → separate cache groups", async () => {
const { remoteFn, callCount } = makeRemoteFn();
const cache = new Map<string, string>();
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
const scripts: ScriptInput[] = [
{ scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps },
{ scriptContent: "print(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps },
{ scriptContent: "# requirements: default\nprint(3)", language: "python3", remotePath: "c", rawWorkspaceDependencies: deps },
{ scriptContent: "print(4)", language: "python3", remotePath: "d", rawWorkspaceDependencies: deps },
];
const results: string[] = [];
for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache));
assertEquals(callCount(), 2); // one for annotated group, one for no-annotation group
assertEquals(results[0], results[2]); // both annotated "default"
assertEquals(results[1], results[3]); // both no annotation
assertNotEquals(results[0], results[1]); // annotated ≠ non-annotated
});
// -- Cache returns correct lock value ---------------------------------------
Deno.test("new logic: cached value matches original remote response", async () => {
const cache = new Map<string, string>();
const deps = { "dependencies/requirements.in": "requests==2.31.0" };
let callIdx = 0;
const remoteFn = async (_input: ScriptInput) => {
callIdx++;
return "resolved-lock-content-abc123";
};
const r1 = await fetchScriptLockNew(
{ scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps },
remoteFn, cache,
);
const r2 = await fetchScriptLockNew(
{ scriptContent: "# requirements: default\nprint(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps },
remoteFn, cache,
);
assertEquals(callIdx, 1);
assertEquals(r1, "resolved-lock-content-abc123");
assertEquals(r2, "resolved-lock-content-abc123");
});
cli/windmill-utils-internal/package.json
+14 -4
View File
@@ -1,12 +1,22 @@
{
"name": "windmill-utils-internal",
"version": "1.3.2",
"version": "1.3.3",
"description": "Internal utility functions for Windmill",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"main": "dist/cjs/index.js",
"module": "dist/esm/index.js",
"types": "dist/esm/index.d.ts",
"exports": {
".": {
"require": "./dist/cjs/index.js",
"import": "./dist/esm/index.js",
"types": "./dist/esm/index.d.ts"
}
},
"scripts": {
"dev": "./gen_wm_client.sh && ./remove-ts-ext.sh",
"build": "./gen_wm_client.sh && ./remove-ts-ext.sh && tsc",
"build:cjs": "tsc -p tsconfig.cjs.json",
"build:esm": "tsc -p tsconfig.esm.json",
"build": "./gen_wm_client.sh && ./remove-ts-ext.sh && npm run build:cjs && npm run build:esm",
"prepublishOnly": "npm run build"
},
"keywords": [
cli/windmill-utils-internal/src/inline-scripts/extractor.ts
+56 -13
View File
@@ -1,5 +1,5 @@
import { newPathAssigner, PathAssigner } from "../path-utils/path-assigner.ts";
import { FlowModule } from "../gen/types.gen.ts";
import { FlowModule, RawScript } from "../gen/types.gen.ts";
/**
* Represents an inline script extracted from a flow module
@@ -11,6 +11,28 @@ interface InlineScript {
content: string;
}
function extractRawscriptInline(
id: string,
summary: string | undefined,
rawscript: RawScript,
mapping: Record<string, string>,
separator: string,
assigner: PathAssigner
): InlineScript[] {
const [basePath, ext] = assigner.assignPath(summary ?? id, rawscript.language);
const path = mapping[id] ?? basePath + ext;
const content = rawscript.content;
const r = [{ path: path, content: content }];
rawscript.content = "!inline " + path.replaceAll(separator, "/");
const lock = rawscript.lock;
if (lock && lock != "") {
const lockPath = basePath + "lock";
rawscript.lock = "!inline " + lockPath.replaceAll(separator, "/");
r.push({ path: lockPath, content: lock });
}
return r;
}
/**
* Options for extractInlineScripts function
*/
@@ -44,18 +66,14 @@ export function extractInlineScripts(
return modules.flatMap((m) => {
if (m.value.type == "rawscript") {
const [basePath, ext] = assigner.assignPath(m.summary, m.value.language);
const path = mapping[m.id] ?? basePath + ext;
const content = m.value.content;
const r = [{ path: path, content: content }];
m.value.content = "!inline " + path.replaceAll(separator, "/");
const lock = m.value.lock;
if (lock && lock != "") {
const lockPath = basePath + "lock";
m.value.lock = "!inline " + lockPath.replaceAll(separator, "/");
r.push({ path: lockPath, content: lock });
}
return r;
return extractRawscriptInline(
m.id,
m.summary,
m.value,
mapping,
separator,
assigner
);
} else if (m.value.type == "forloopflow") {
return extractInlineScripts(
m.value.modules,
@@ -95,6 +113,23 @@ export function extractInlineScripts(
assigner
),
];
} else if (m.value.type == "aiagent") {
return (m.value.tools ?? []).flatMap((tool) => {
const toolValue = tool.value;
// Only process flowmodule tools with rawscript type
if (!toolValue || toolValue.tool_type !== 'flowmodule' || toolValue.type !== 'rawscript') {
return [];
}
return extractRawscriptInline(
tool.id,
tool.summary,
toolValue,
mapping,
separator,
assigner
);
});
} else {
return [];
}
@@ -140,6 +175,14 @@ export function extractCurrentMapping(
extractCurrentMapping(b.modules, mapping)
);
extractCurrentMapping(m.value.default, mapping);
} else if (m.value.type === "aiagent") {
(m.value.tools ?? []).forEach((tool) => {
const toolValue = tool.value;
if (!toolValue || toolValue.tool_type !== 'flowmodule' || toolValue.type !== 'rawscript' || !toolValue.content || !toolValue.content.startsWith("!inline")) {
return;
}
mapping[tool.id] = toolValue.content.trim().split(" ")[1];
});
}
});
cli/windmill-utils-internal/src/inline-scripts/replacer.ts
+74 -61
View File
@@ -1,4 +1,49 @@
import { FlowModule } from "../gen/types.gen.ts";
import { FlowModule, RawScript } from "../gen/types.gen.ts";
async function replaceRawscriptInline(
id: string,
rawscript: RawScript,
fileReader: (path: string) => Promise<string>,
logger: { info: (message: string) => void; error: (message: string) => void },
separator: string,
removeLocks?: string[]
): Promise<void> {
if (!rawscript.content || !rawscript.content.startsWith("!inline")) {
return;
}
const path = rawscript.content.split(" ")[1];
const pathSuffix = path.split(".").slice(1).join(".");
const newPath = id + "." + pathSuffix;
try {
rawscript.content = await fileReader(path);
} catch {
logger.error(`Script file ${path} not found`);
try {
rawscript.content = await fileReader(newPath);
} catch {
logger.error(`Script file ${newPath} not found`);
}
}
const lock = rawscript.lock;
if (removeLocks && removeLocks.includes(path)) {
rawscript.lock = undefined;
} else if (
lock &&
typeof lock === "string" &&
lock.trimStart().startsWith("!inline ")
) {
const lockPath = lock.split(" ")[1];
try {
rawscript.lock = await fileReader(lockPath.replaceAll("/", separator));
} catch {
logger.error(`Lock file ${lockPath} not found, treating as empty`);
rawscript.lock = "";
}
}
}
/**
* Replaces inline script references with actual file content from the filesystem.
@@ -32,66 +77,15 @@ export async function replaceInlineScripts(
throw new Error(`Module value is undefined for module ${module.id}`);
}
if (module.value.type === "rawscript" && module.value.content && module.value.content.startsWith("!inline")) {
const path = module.value.content.split(" ")[1];
// const pathPrefix = path.split(".")[0];
const pathSuffix = path.split(".").slice(1).join(".");
// new path is the module id with the same suffix
const newPath = module.id + "." + pathSuffix;
try {
module.value.content = await fileReader(path);
} catch {
logger.error(`Script file ${path} not found`);
// try new path
try {
module.value.content = await fileReader(newPath);
} catch {
logger.error(`Script file ${newPath} not found`);
}
}
// rename the file if the prefix is different from the module id (fix old naming)
// if (pathPrefix != module.id && renamer) {
// logger.info(`Renaming ${path} to ${module.id}.${pathSuffix}`);
// try {
// renamer(localPath + path, localPath + module.id + "." + pathSuffix);
// } catch {
// logger.info(`Failed to rename ${path} to ${module.id}.${pathSuffix}`);
// }
// }
const lock = module.value.lock;
if (removeLocks && removeLocks.includes(path)) {
module.value.lock = undefined;
// delete the file if the prefix is different from the module id (fix old naming)
// if (lock && lock != "") {
// const path = lock.split(" ")[1];
// const pathPrefix = path.split(".")[0];
// if (pathPrefix != module.id && deleter) {
// logger.info(`Deleting ${path}`);
// try {
// deleter(localPath + path);
// } catch {
// logger.error(`Failed to delete ${path}`);
// }
// }
// }
} else if (
lock &&
typeof lock == "string" &&
lock.trimStart().startsWith("!inline ")
) {
const path = lock.split(" ")[1];
try {
module.value.lock = await fileReader(path.replaceAll("/", separator));
} catch {
logger.error(`Lock file ${path} not found, treating as empty`);
module.value.lock = "";
}
}
if (module.value.type === "rawscript") {
await replaceRawscriptInline(
module.id,
module.value,
fileReader,
logger,
separator,
removeLocks
);
} else if (module.value.type === "forloopflow" || module.value.type === "whileloopflow") {
await replaceInlineScripts(module.value.modules, fileReader, logger, localPath, separator, removeLocks);
} else if (module.value.type === "branchall") {
@@ -103,6 +97,25 @@ export async function replaceInlineScripts(
await replaceInlineScripts(branch.modules, fileReader, logger, localPath, separator, removeLocks);
}));
await replaceInlineScripts(module.value.default, fileReader, logger, localPath, separator, removeLocks);
} else if (module.value.type === "aiagent") {
await Promise.all((module.value.tools ?? []).map(async (tool) => {
const toolValue = tool.value;
if (
!toolValue ||
toolValue.tool_type !== "flowmodule" ||
toolValue.type !== "rawscript"
) {
return;
}
await replaceRawscriptInline(
tool.id,
toolValue,
fileReader,
logger,
separator,
removeLocks
);
}));
}
}));
}
cli/windmill-utils-internal/tsconfig.cjs.json
+7
View File
@@ -0,0 +1,7 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"module": "CommonJS",
"outDir": "./dist/cjs"
}
}
cli/windmill-utils-internal/tsconfig.esm.json
+7
View File
@@ -0,0 +1,7 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"module": "ES2022",
"outDir": "./dist/esm"
}
}
cli/windmill-utils-internal/tsconfig.json
+13 -6
View File
@@ -1,8 +1,10 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ES2022",
"lib": ["ES2022"],
"module": "commonjs",
"lib": [
"ES2022"
],
"declaration": true,
"outDir": "./dist",
"rootDir": "./src",
@@ -11,7 +13,7 @@
"noUnusedParameters": false,
"noImplicitReturns": true,
"noFallthroughCasesInSwitch": true,
"moduleResolution": "bundler",
"moduleResolution": "node",
"baseUrl": "./",
"esModuleInterop": true,
"experimentalDecorators": true,
@@ -19,6 +21,11 @@
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
}
"include": [
"src/**/*"
],
"exclude": [
"node_modules",
"dist"
]
}
docker/DockerfileSlim
+1 -1
View File
@@ -34,7 +34,7 @@ RUN mkdir -p /tmp/windmill/cache && \
rm -rf /tmp/build_cache && \
mkdir -p -m 777 /tmp/windmill/cache/uv
COPY --from=oven/bun:1.2.23 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.3.8 /usr/local/bin/bun /usr/bin/bun
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
docker/DockerfileSlimEe
+1 -1
View File
@@ -34,7 +34,7 @@ RUN mkdir -p /tmp/windmill/cache && \
rm -rf /tmp/build_cache && \
mkdir -p -m 777 /tmp/windmill/cache/uv
COPY --from=oven/bun:1.2.23 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.3.8 /usr/local/bin/bun /usr/bin/bun
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
frontend/package-lock.json
Generated
+55 -143
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-components",
"version": "1.623.1",
"version": "1.624.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.623.1",
"version": "1.624.0",
"hasInstallScript": true,
"license": "AGPL-3.0",
"dependencies": {
@@ -79,11 +79,11 @@
"windmill-parser-wasm-java": "1.510.1",
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.574.1",
"windmill-parser-wasm-py": "1.601.1",
"windmill-parser-wasm-regex": "1.593.0",
"windmill-parser-wasm-py": "1.623.1",
"windmill-parser-wasm-regex": "1.623.1",
"windmill-parser-wasm-ruby": "1.526.1",
"windmill-parser-wasm-rust": "1.558.1",
"windmill-parser-wasm-ts": "1.593.0",
"windmill-parser-wasm-ts": "1.623.1",
"windmill-parser-wasm-yaml": "1.593.0",
"windmill-sql-datatype-parser-wasm": "1.512.0",
"windmill-utils-internal": "^1.3.2",
@@ -259,7 +259,6 @@
"integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@babel/helper-validator-identifier": "^7.27.1",
"js-tokens": "^4.0.0",
@@ -275,7 +274,6 @@
"integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=6.9.0"
}
@@ -821,7 +819,6 @@
}
],
"license": "MIT",
"peer": true,
"engines": {
"node": "^14 || ^16 || >=18"
},
@@ -1331,6 +1328,7 @@
"integrity": "sha512-Jer+M7DgIwT5IHfTayb4Iw/fkkxWNmC/mqn/nMh9JrbPbkxmyabfLQnhJ+JDn5HK77f84j34lubO3iqFtYAfMg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@floating-ui/core": "^1.3.1",
"@floating-ui/dom": "^1.4.5",
@@ -2106,6 +2104,7 @@
"integrity": "sha512-Vp3zX/qlwerQmHMP6x0Ry1oY7eKKRcOWGc2P59srOp4zcqyn+etJyQpELgOi4+ZSUgteX8Y387NuwruLgGXLUQ==",
"devOptional": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@standard-schema/spec": "^1.0.0",
"@sveltejs/acorn-typescript": "^1.0.5",
@@ -2183,6 +2182,7 @@
"integrity": "sha512-YZs/OSKOQAQCnJvM/P+F1URotNnYNeU3P2s4oIpzm1uFaqUEqRxUB0g5ejMjEb5Gjb9/PiBI5Ktrq4rUUF8UVQ==",
"devOptional": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@sveltejs/vite-plugin-svelte-inspector": "^5.0.0",
"debug": "^4.4.1",
@@ -2709,8 +2709,7 @@
"resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/@types/ms": {
"version": "2.1.0",
@@ -2723,8 +2722,7 @@
"resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz",
"integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/@types/semver": {
"version": "7.7.1",
@@ -2794,6 +2792,7 @@
"integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==",
"dev": true,
"license": "BSD-2-Clause",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "5.62.0",
"@typescript-eslint/types": "5.62.0",
@@ -2962,7 +2961,6 @@
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"dependencies": {
"@vitest/mocker": "4.0.15",
"@vitest/utils": "4.0.15",
@@ -2987,7 +2985,6 @@
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"dependencies": {
"@vitest/browser": "4.0.15",
"@vitest/mocker": "4.0.15",
@@ -3013,7 +3010,6 @@
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"dependencies": {
"@vitest/spy": "4.0.15",
"estree-walker": "^3.0.3",
@@ -3042,7 +3038,6 @@
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"dependencies": {
"@vitest/spy": "4.0.15",
"estree-walker": "^3.0.3",
@@ -3274,6 +3269,7 @@
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
"license": "MIT",
"peer": true,
"bin": {
"acorn": "bin/acorn"
},
@@ -3327,6 +3323,7 @@
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz",
"integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==",
"license": "MIT",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@@ -3472,7 +3469,6 @@
"integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -3500,7 +3496,6 @@
"integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=8"
}
@@ -3593,8 +3588,7 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-2.0.0.tgz",
"integrity": "sha512-1ugUSr8BHXRnK23KfuYS+gVMC3LB8QGH9W1iGtDPsNWoQbgtXSExkBu2aDR4epiGWZOjZsj6lDl/N/AqqTC3UA==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/base64-js": {
"version": "1.5.1",
@@ -3721,6 +3715,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"baseline-browser-mapping": "^2.8.9",
"caniuse-lite": "^1.0.30001746",
@@ -3918,7 +3913,6 @@
"integrity": "sha512-Rjs1H+A9R+Ig+4E/9oyB66UC5Mj9Xq3N//vcLf2WzgdTi/3gUu3Z9KoqmlrEG4VuuLK8wJHofxzdQXz/knhiYg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"camelcase": "^6.3.0",
"map-obj": "^4.1.0",
@@ -3938,7 +3932,6 @@
"integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -3952,7 +3945,6 @@
"integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -3966,7 +3958,6 @@
"integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==",
"dev": true,
"license": "(MIT OR CC0-1.0)",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -4075,6 +4066,7 @@
"resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz",
"integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==",
"license": "MIT",
"peer": true,
"dependencies": {
"@kurkle/color": "^0.3.0"
},
@@ -4321,7 +4313,6 @@
"integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"import-fresh": "^3.3.0",
"js-yaml": "^4.1.0",
@@ -4386,7 +4377,6 @@
"integrity": "sha512-IQOkD3hbR5KrN93MtcYuad6YPuTSUhntLHDuLEbFWE+ff2/XSZNdZG+LcbbIW5AXKg/WFIfYItIzVoHngHXZzA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12 || >=16"
}
@@ -4652,6 +4642,7 @@
"resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz",
"integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==",
"license": "ISC",
"peer": true,
"engines": {
"node": ">=12"
}
@@ -4705,6 +4696,7 @@
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz",
"integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==",
"license": "MIT",
"peer": true,
"dependencies": {
"@babel/runtime": "^7.21.0"
},
@@ -4739,7 +4731,6 @@
"integrity": "sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -4753,7 +4744,6 @@
"integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"decamelize": "^1.1.0",
"map-obj": "^1.0.0"
@@ -4771,7 +4761,6 @@
"integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -4782,7 +4771,6 @@
"integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -5268,7 +5256,6 @@
"integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"is-arrayish": "^0.2.1"
}
@@ -5356,6 +5343,7 @@
"deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.6.1",
@@ -5902,7 +5890,6 @@
"integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 4.9.1"
}
@@ -6388,7 +6375,6 @@
"integrity": "sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"global-prefix": "^3.0.0"
},
@@ -6402,7 +6388,6 @@
"integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"ini": "^1.3.5",
"kind-of": "^6.0.2",
@@ -6418,7 +6403,6 @@
"integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
"dev": true,
"license": "ISC",
"peer": true,
"dependencies": {
"isexe": "^2.0.0"
},
@@ -6468,8 +6452,7 @@
"resolved": "https://registry.npmjs.org/globjoin/-/globjoin-0.1.4.tgz",
"integrity": "sha512-xYfnw62CKG8nLkZBfWbhWwDw02CHty86jfPcc2cr3ZfeuK9ysoVPPEUxf21bAD/rWAgk52SuBrLJlefNy8mvFg==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/gopd": {
"version": "1.2.0",
@@ -6546,7 +6529,6 @@
"integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=6"
}
@@ -6768,7 +6750,6 @@
"integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==",
"dev": true,
"license": "ISC",
"peer": true,
"dependencies": {
"lru-cache": "^6.0.0"
},
@@ -6782,7 +6763,6 @@
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"dev": true,
"license": "ISC",
"peer": true,
"dependencies": {
"yallist": "^4.0.0"
},
@@ -6795,8 +6775,7 @@
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
"dev": true,
"license": "ISC",
"peer": true
"license": "ISC"
},
"node_modules/html-tags": {
"version": "3.3.1",
@@ -6804,7 +6783,6 @@
"integrity": "sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=8"
},
@@ -6888,7 +6866,6 @@
"integrity": "sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=8"
}
@@ -6909,7 +6886,6 @@
"integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12"
},
@@ -6980,8 +6956,7 @@
"resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
"integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/is-binary-path": {
"version": "2.1.0",
@@ -7086,7 +7061,6 @@
"integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -7097,7 +7071,6 @@
"integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -7202,8 +7175,7 @@
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/js-yaml": {
"version": "4.1.0",
@@ -7239,8 +7211,7 @@
"resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
"integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/json-refs": {
"version": "3.0.15",
@@ -7407,7 +7378,6 @@
"integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -8001,8 +7971,7 @@
"resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz",
"integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/lodash.uniq": {
"version": "4.5.0",
@@ -8070,7 +8039,6 @@
"integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=8"
},
@@ -8121,7 +8089,6 @@
"integrity": "sha512-APMBEanjybaPzUrfqU0IMU5I0AswKMH7k8OTLs0vvV4KZpExkTkY87nR/zpbuTPj+gARop7aGUbl11pnDfW6xg==",
"dev": true,
"license": "MIT",
"peer": true,
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
@@ -8362,7 +8329,6 @@
"integrity": "sha512-/d+PQ4GKmGvM9Bee/DPa8z3mXs/pkvJE2KEThngVNOqtmljC6K7NMPxtc2JeZYTmpWb9k/TmxjeL18ez3h7vCw==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/minimist": "^1.2.2",
"camelcase-keys": "^7.0.0",
@@ -8390,7 +8356,6 @@
"integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==",
"dev": true,
"license": "(MIT OR CC0-1.0)",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -9084,7 +9049,6 @@
"integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"arrify": "^1.0.1",
"is-plain-obj": "^1.1.0",
@@ -9163,6 +9127,7 @@
"resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-api/-/monaco-vscode-editor-api-25.0.0.tgz",
"integrity": "sha512-uiY06RTWFo2WZdh6OybkLlDhuG+8LlkjUDpr9/wW55uucqHo4X8fx4XKEtD98cscC+6FKQkbG2yyUiOJ/npHOw==",
"license": "MIT",
"peer": true,
"dependencies": {
"@codingame/monaco-vscode-api": "25.0.0"
}
@@ -9399,7 +9364,6 @@
"integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==",
"dev": true,
"license": "BSD-2-Clause",
"peer": true,
"dependencies": {
"hosted-git-info": "^4.0.1",
"is-core-module": "^2.5.0",
@@ -9745,7 +9709,6 @@
"integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@babel/code-frame": "^7.0.0",
"error-ex": "^1.3.1",
@@ -9957,7 +9920,6 @@
"dev": true,
"license": "ISC",
"optional": true,
"peer": true,
"dependencies": {
"pngjs": "^7.0.0"
},
@@ -10047,7 +10009,6 @@
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"engines": {
"node": ">=14.19.0"
}
@@ -10072,6 +10033,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"nanoid": "^3.3.11",
"picocolors": "^1.1.1",
@@ -10260,6 +10222,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"lilconfig": "^3.0.0",
"yaml": "^2.3.4"
@@ -10649,8 +10612,7 @@
"resolved": "https://registry.npmjs.org/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.6.tgz",
"integrity": "sha512-0sglIs9Wmkzbr8lQwEyIzlDOOC9bGmfVKcJTaxv3vMmd3uo4o4DerC3En0bnmgceeql9BfC8hRkp7cg0fjdVqw==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/postcss-safe-parser": {
"version": "6.0.0",
@@ -10826,6 +10788,7 @@
"integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==",
"dev": true,
"license": "MIT",
"peer": true,
"bin": {
"prettier": "bin/prettier.cjs"
},
@@ -11136,7 +11099,6 @@
"integrity": "sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/normalize-package-data": "^2.4.0",
"normalize-package-data": "^3.0.2",
@@ -11156,7 +11118,6 @@
"integrity": "sha512-snVCqPczksT0HS2EC+SxUndvSzn6LRCwpfSvLrIfR5BKDQQZMaI6jPRC9dYvYFDRAuFEAnkwww8kBBNE/3VvzQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"find-up": "^5.0.0",
"read-pkg": "^6.0.0",
@@ -11175,7 +11136,6 @@
"integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==",
"dev": true,
"license": "(MIT OR CC0-1.0)",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -11189,7 +11149,6 @@
"integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==",
"dev": true,
"license": "(MIT OR CC0-1.0)",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -11232,7 +11191,6 @@
"integrity": "sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"indent-string": "^5.0.0",
"strip-indent": "^4.0.0"
@@ -11875,7 +11833,6 @@
"integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"ansi-styles": "^4.0.0",
"astral-regex": "^2.0.0",
@@ -11952,7 +11909,6 @@
"integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==",
"dev": true,
"license": "Apache-2.0",
"peer": true,
"dependencies": {
"spdx-expression-parse": "^3.0.0",
"spdx-license-ids": "^3.0.0"
@@ -11963,8 +11919,7 @@
"resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz",
"integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==",
"dev": true,
"license": "CC-BY-3.0",
"peer": true
"license": "CC-BY-3.0"
},
"node_modules/spdx-expression-parse": {
"version": "3.0.1",
@@ -11972,7 +11927,6 @@
"integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"spdx-exceptions": "^2.1.0",
"spdx-license-ids": "^3.0.0"
@@ -11983,8 +11937,7 @@
"resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.22.tgz",
"integrity": "sha512-4PRT4nh1EImPbt2jASOKHX7PB7I+e4IWNLvkKFDxNhJlfjbYlleYQh285Z/3mPTHSAK/AvdMmw5BNNuYH8ShgQ==",
"dev": true,
"license": "CC0-1.0",
"peer": true
"license": "CC0-1.0"
},
"node_modules/sprintf-js": {
"version": "1.0.3",
@@ -12078,7 +12031,6 @@
"integrity": "sha512-SlyRoSkdh1dYP0PzclLE7r0M9sgbFKKMFXpFRUMNuKhQSbC6VQIGzq3E0qsfvGJaUFJPGv6Ws1NZ/haTAjfbMA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12"
},
@@ -12104,8 +12056,7 @@
"resolved": "https://registry.npmjs.org/style-search/-/style-search-0.1.0.tgz",
"integrity": "sha512-Dj1Okke1C3uKKwQcetra4jSuk0DqbzbYtXipzFlFMZtowbF1x7BKJwB9AayVMyFARvU8EDrZdcax4At/452cAg==",
"dev": true,
"license": "ISC",
"peer": true
"license": "ISC"
},
"node_modules/style-to-object": {
"version": "0.4.4",
@@ -12154,7 +12105,6 @@
"integrity": "sha512-78O4c6IswZ9TzpcIiQJIN49K3qNoXTM8zEJzhaTE/xRTCZswaovSEVIa/uwbOltZrk16X4jAxjaOhzz/hTm1Kw==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@csstools/css-parser-algorithms": "^2.3.1",
"@csstools/css-tokenizer": "^2.2.0",
@@ -12237,7 +12187,6 @@
}
],
"license": "MIT-0",
"peer": true,
"engines": {
"node": "^14 || ^16 || >=18"
},
@@ -12251,7 +12200,6 @@
"integrity": "sha512-TfW7/1iI4Cy7Y8L6iqNdZQVvdXn0f8B4QcIXmkIbtTIe/Okm/nSlHb4IwGzRVOd3WfSieCgvf5cMzEfySAIl0g==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"flat-cache": "^3.2.0"
},
@@ -12264,8 +12212,7 @@
"resolved": "https://registry.npmjs.org/known-css-properties/-/known-css-properties-0.29.0.tgz",
"integrity": "sha512-Ne7wqW7/9Cz54PDt4I3tcV+hAyat8ypyOGzYRJQfdxnnjeWsTxt1cy8pjvvKeI5kfXuyvULyeeAvwvvtAX3ayQ==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/stylelint/node_modules/postcss-selector-parser": {
"version": "6.1.2",
@@ -12288,7 +12235,6 @@
"integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=8"
}
@@ -12423,7 +12369,6 @@
"integrity": "sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"has-flag": "^4.0.0",
"supports-color": "^7.0.0"
@@ -12453,6 +12398,7 @@
"resolved": "https://registry.npmjs.org/svelte/-/svelte-5.39.12.tgz",
"integrity": "sha512-CEzwxFuEycokU8K8CE/OuwVbmei+ivu2HvBGYIdASfMa1hCRSNr4RRkzNSvbAvu6h+BOig2CsZTAEY+WKvwZpA==",
"license": "MIT",
"peer": true,
"dependencies": {
"@jridgewell/remapping": "^2.3.4",
"@jridgewell/sourcemap-codec": "^1.5.0",
@@ -12548,21 +12494,6 @@
}
}
},
"node_modules/svelte-check/node_modules/picomatch": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/svelte-eslint-parser": {
"version": "0.43.0",
"resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz",
@@ -12750,8 +12681,7 @@
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/svg-tags/-/svg-tags-1.0.0.tgz",
"integrity": "sha512-ovssysQTa+luh7A5Weu3Rta6FJlFBBbInjOh722LIt6klpU2/HtdUbszju/G4devcvk8PGt7FCLv5wftu3THUA==",
"dev": true,
"peer": true
"dev": true
},
"node_modules/svgo": {
"version": "3.3.2",
@@ -12802,7 +12732,6 @@
"integrity": "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==",
"dev": true,
"license": "BSD-3-Clause",
"peer": true,
"dependencies": {
"ajv": "^8.0.1",
"lodash.truncate": "^4.4.2",
@@ -12830,6 +12759,7 @@
"integrity": "sha512-6A2rnmW5xZMdw11LYjhcI5846rt9pbLSabY5XPxo+XWdxwZaFEn47Go4NzFiHu9sNNmr/kXivP1vStfvMaK1GQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@alloc/quick-lru": "^5.2.0",
"arg": "^5.0.2",
@@ -13072,6 +13002,7 @@
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
"devOptional": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12"
},
@@ -13134,7 +13065,6 @@
"integrity": "sha512-jRKj0n0jXWo6kh62nA5TEh3+4igKDXLvzBJcPpiizP7oOolUrYIxmVBG9TOtHYFHoddUk6YvAkGeGoSVTXfQXQ==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12"
},
@@ -13233,6 +13163,7 @@
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"license": "Apache-2.0",
"peer": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@@ -13446,7 +13377,6 @@
"integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==",
"dev": true,
"license": "Apache-2.0",
"peer": true,
"dependencies": {
"spdx-correct": "^3.0.0",
"spdx-expression-parse": "^3.0.0"
@@ -13501,6 +13431,7 @@
"integrity": "sha512-5hI5NCJwKBGtzWtdKB3c2fOEpI77Iaa0z4mSzZPU1cJ/OqrGbFafm90edVCd7T9Snz+Sh09TMAv4EQqyVLzuEg==",
"devOptional": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@oxc-project/runtime": "0.101.0",
"fdir": "^6.5.0",
@@ -13613,6 +13544,7 @@
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
"devOptional": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12"
},
@@ -13626,6 +13558,7 @@
"integrity": "sha512-n1RxDp8UJm6N0IbJLQo+yzLZ2sQCDyl1o0LeugbPWf8+8Fttp29GghsQBjYJVmWq3gBFfe9Hs1spR44vovn2wA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@vitest/expect": "4.0.15",
"@vitest/mocker": "4.0.15",
@@ -14254,6 +14187,7 @@
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12"
},
@@ -14277,6 +14211,7 @@
"integrity": "sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"esbuild": "^0.27.0",
"fdir": "^6.5.0",
@@ -14603,14 +14538,14 @@
"integrity": "sha512-COyid6B1RYs+bpzUCInsA4HY/WZkpDLfkQ90+AqU/TVTpzYSbAC2JCbIwy0cRElBvlhI4bQ+9Wg6hSQKMpEkpA=="
},
"node_modules/windmill-parser-wasm-py": {
"version": "1.601.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.601.1.tgz",
"integrity": "sha512-xcNZE/8B29yfl6UuQDPSXMD+83/W2Hzt2uhn+WrNvy0+qzk6nLh/vJGrf2srLBngYX1TxhUI5Jgseg0PK9yvNw=="
"version": "1.623.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.623.1.tgz",
"integrity": "sha512-lFBlZg6hvhHzsU5oPJq0478UyMTZ9UKVd8Hc8ggmxPIHZaJBeJ+56NR75hmGwg0VJcRff8ed+zEm1PgmjVhD+w=="
},
"node_modules/windmill-parser-wasm-regex": {
"version": "1.593.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.593.0.tgz",
"integrity": "sha512-m8BvTGJc2710YODmKKDXiASfssoJ/YFJGfYRhRnQvltENvaRee2NZuf4XoUkOVJESDdCSEX6U6fVkFvF9rXp2Q=="
"version": "1.623.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.623.1.tgz",
"integrity": "sha512-rW3pl4ysIXmVAmwxSTKhR2sUfScXKfTqxrUebDg0ZNcUzY97S8mNRdQGZfaQ1NWhWPRQNXfL/z/yaUOQh5ConQ=="
},
"node_modules/windmill-parser-wasm-ruby": {
"version": "1.526.1",
@@ -14623,9 +14558,9 @@
"integrity": "sha512-21S7lm1KF8zO1187rbq14hzPHII2RdM2+D44MoAh1F6VoaScj+Puq0z5B1O/hwn/95R/a9jBlL2D8jbkXtlD1A=="
},
"node_modules/windmill-parser-wasm-ts": {
"version": "1.593.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.593.0.tgz",
"integrity": "sha512-NFY9gaEIpJOwGZJeGYDS3+/16QiPYdxFgmb1bKhXyEAdWjYMQ+otrTezf+K09lmvPzB+len37GlNCU72OgyQ6A=="
"version": "1.623.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.623.1.tgz",
"integrity": "sha512-FBwi/zXxjhZcCvi04oFdNivazru1ynIqSbafHSArfaaBWesBO3nye9UO/WXUlWZm5a7BExbU+3R/eVJrGaornw=="
},
"node_modules/windmill-parser-wasm-yaml": {
"version": "1.593.0",
@@ -14776,7 +14711,6 @@
"integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==",
"dev": true,
"license": "ISC",
"peer": true,
"dependencies": {
"imurmurhash": "^0.1.4",
"signal-exit": "^4.0.1"
@@ -14785,29 +14719,6 @@
"node": "^14.17.0 || ^16.13.0 || >=18.0.0"
}
},
"node_modules/ws": {
"version": "8.19.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz",
"integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==",
"license": "MIT",
"optional": true,
"peer": true,
"engines": {
"node": ">=10.0.0"
},
"peerDependencies": {
"bufferutil": "^4.0.1",
"utf-8-validate": ">=5.0.2"
},
"peerDependenciesMeta": {
"bufferutil": {
"optional": true
},
"utf-8-validate": {
"optional": true
}
}
},
"node_modules/xml-utils": {
"version": "1.10.2",
"resolved": "https://registry.npmjs.org/xml-utils/-/xml-utils-1.10.2.tgz",
@@ -14995,7 +14906,6 @@
"integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==",
"dev": true,
"license": "ISC",
"peer": true,
"engines": {
"node": ">=10"
}
@@ -15015,6 +14925,7 @@
"resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.27.tgz",
"integrity": "sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==",
"license": "MIT",
"peer": true,
"dependencies": {
"lib0": "^0.2.99"
},
@@ -15050,6 +14961,7 @@
"resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz",
"integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==",
"license": "MIT",
"peer": true,
"funding": {
"url": "https://github.com/sponsors/colinhacks"
}
frontend/package.json
+4 -4
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.623.1",
"version": "1.624.0",
"scripts": {
"dev": "vite dev",
"build": "vite build",
@@ -149,11 +149,11 @@
"windmill-parser-wasm-java": "1.510.1",
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.574.1",
"windmill-parser-wasm-py": "1.601.1",
"windmill-parser-wasm-regex": "1.593.0",
"windmill-parser-wasm-py": "1.623.1",
"windmill-parser-wasm-regex": "1.623.1",
"windmill-parser-wasm-ruby": "1.526.1",
"windmill-parser-wasm-rust": "1.558.1",
"windmill-parser-wasm-ts": "1.593.0",
"windmill-parser-wasm-ts": "1.623.1",
"windmill-parser-wasm-yaml": "1.593.0",
"windmill-sql-datatype-parser-wasm": "1.512.0",
"windmill-utils-internal": "^1.3.2",
frontend/src/lib/components/FlowGraphViewerStep.svelte
+1 -1
View File
@@ -245,7 +245,7 @@
<p class="font-medium text-secondary pb-2"> Iterator expression: </p>
{#if stepDetail.value.iterator.type == 'static'}
<ObjectViewer json={stepDetail.value.iterator.value} />
{:else}
{:else if stepDetail.value.iterator.type == 'javascript'}
<span class="text-xs">
<Highlight language={typescript} code={cleanExpr(stepDetail.value.iterator.expr)} />
</span>
frontend/src/lib/components/InputTransformForm.svelte
+12 -9
View File
@@ -42,9 +42,6 @@
import { inputBorderClass } from './text_input/TextInput.svelte'
import FakeMonacoPlaceHolder from './FakeMonacoPlaceHolder.svelte'
// We add 'ai' for ai agent tools. 'ai' means the field will be filled by the AI agent dynamically.
type PropertyType = InputTransform['type'] | 'ai'
interface Props {
schema: Schema | { properties?: Record<string, any>; required?: string[] }
arg: InputTransform | any
@@ -162,7 +159,7 @@
})
}
function getPropertyType(arg: InputTransform | any): PropertyType {
function getPropertyType(arg: InputTransform | any): InputTransform['type'] {
// For agent tools, if static with undefined/empty value, treat as 'ai', meaning the field will be filled by the AI agent dynamically.
if (
isAgentTool &&
@@ -174,7 +171,7 @@
return 'ai'
}
let type: PropertyType = arg?.type ?? 'static'
let type: InputTransform['type'] = arg?.type ?? 'static'
if (
type == 'javascript' &&
@@ -408,7 +405,7 @@
function updateStaticInput(
inputCat: InputCat,
propertyType: PropertyType,
propertyType: InputTransform['type'],
arg: InputTransform | any
) {
if (!isStaticTemplate(inputCat)) {
@@ -821,7 +818,11 @@
otherArgs={Object.fromEntries(
Object.entries(otherArgs).map(([key, transform]) => [
key,
transform?.type === 'static' ? transform.value : transform?.expr
transform?.type === 'static'
? transform.value
: transform?.type === 'javascript'
? transform.expr
: undefined
])
)}
>
@@ -830,12 +831,14 @@
<S3ArrayHelperButton
{connecting}
onClick={() =>
switchToJsAndConnect((path) => appendPathToArrayExpr(arg.expr, path))}
switchToJsAndConnect((path) =>
appendPathToArrayExpr(arg?.type === 'javascript' ? arg.expr : '', path)
)}
/>
{/if}
{/snippet}
</ArgInput>
{:else if arg.expr != undefined}
{:else if arg?.type === 'javascript' && arg.expr != undefined}
<div
class={`bg-surface-input rounded-md flex flex-col pl-2 overflow-auto ${inputBorderClass({ forceFocus: focused })}`}
>
frontend/src/lib/components/InputTransformsViewer.svelte
+3 -1
View File
@@ -33,10 +33,12 @@
{val.value}
</span>
{/if}
{:else}
{:else if val.type == 'javascript'}
<span class="text-xs text-primary whitespace-pre-wrap font-mono">
{cleanExpr(val.expr)}
</span>
{:else if val.type == 'ai'}
<span class="text-xs text-primary whitespace-pre-wrap font-mono">Filled by AI</span>
{/if}
</Cell>
</Row>
frontend/src/lib/components/PageHeader.svelte
+1 -1
View File
@@ -8,7 +8,7 @@
export let childrenWrapperDivClasses: string = ''
</script>
<div class="flex flex-row flex-wrap justify-between pb-2 my-4 mr-2">
<div class="flex flex-row flex-wrap justify-between items-center pb-2 my-4 mr-2 min-h-16">
{#if primary}
<span class="flex items-center gap-2">
<h1 class="text-2xl font-semibold text-emphasis whitespace-nowrap leading-6 tracking-tight"
frontend/src/lib/components/Toast.svelte
+74 -50
View File
@@ -36,21 +36,23 @@
})
}
}
export type ToastType = AlertType
</script>
<script lang="ts">
import { toast } from '@zerodevx/svelte-toast'
import { CheckCircle2, XCircleIcon } from 'lucide-svelte'
import Button from './common/button/Button.svelte'
import { type ToastAction } from '$lib/toast'
import { processMessage } from './toast'
import { onDestroy, untrack } from 'svelte'
import { twMerge } from 'tailwind-merge'
import { classes, icons, type AlertType } from '$lib/components/common/alert/model'
interface Props {
message: string
toastId: string
error?: boolean
type?: ToastType
actions?: ToastAction[]
errorMessage?: string | undefined
duration?: number
@@ -59,7 +61,7 @@
let {
message,
toastId,
error = false,
type = 'success',
actions = [],
errorMessage = undefined,
duration = 5000
@@ -75,81 +77,103 @@
onDestroy(() => {
delete toastStates[toastId]
})
let state = $derived.by(() => toastStates[toastId] as ToastState | undefined)
let _state = $derived.by(() => toastStates[toastId] as ToastState | undefined)
$effect(() => {
if (!state) {
if (!_state) {
toast.pop(toastId)
}
})
let color = error
? { text: 'text-red-400', bg: 'bg-red-400' }
: { text: 'text-green-400', bg: 'bg-green-300' }
let color = classes[type]
let hover = $derived(Object.values(toastStates).some((state) => state.hover))
let containerClass = {
success: 'toast-success',
error: 'toast-error',
info: 'toast-info',
warning: 'toast-warning'
}[type]
let Icon = $derived(icons[type])
let showMore = $state(false)
const MAX_MSG_LEN = 160
let isLongMessage = $derived(message.length > MAX_MSG_LEN)
let displayMessage = $derived(
isLongMessage && !showMore ? message.slice(0, MAX_MSG_LEN) + '... ' : message
)
// let hover = $derived(Object.values(toastStates).some((state) => state.hover))
</script>
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div
class={twMerge(
'pointer-events-auto w-full max-w-sm overflow-hidden bg-surface-tertiary drop-shadow-base shadow-lg ring-1 ring-black ring-opacity-5 border rounded-md',
error ? 'toast-error' : 'toast-success'
'pointer-events-auto w-full overflow-hidden rounded-md relative flex items-center bg-surface',
containerClass
)}
onmouseenter={() => state && (state.hover = true)}
onmouseleave={() => state && (state.hover = false)}
onmouseenter={() => _state && (_state.hover = true)}
onmouseleave={() => _state && (_state.hover = false)}
>
<div class="p-2 min-h-[60px] flex flex-col">
<div class="flex items-start w-full">
<div class="flex-shrink-0 mt-0.5">
{#if error}
<XCircleIcon class="h-4 w-4 {color.text}" />
{:else}
<CheckCircle2 class="h-4 w-4 {color.text}" />
{/if}
</div>
<div class="ml-3 flex-1 w-0">
<p class="text-sm text-primary break-words">{@html processMessage(message)}</p>
{#if errorMessage}
<p class="text-xs {color.text} w-full overflow-auto mt-2">
{errorMessage}
</p>
{/if}
</div>
<div class="ml-4 flex flex-shrink-0">
<button
type="button"
onclick={handleClose}
class="inline-flex rounded-md text-gray-400 hover:text-primary focus:outline-none"
>
<span class="sr-only">Close</span>
<svg class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor" aria-hidden="true">
<path
d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
/>
</svg>
</button>
</div>
<div
class="flex items-center h-full w-full min-h-10 rounded-md px-2 py-1 {color.descriptionClass} {color.bgClass}"
>
<div class="flex-shrink-0 mt-0.5">
<Icon class="h-4 w-4 {color.iconClass}" />
</div>
<div class="mt-2 flex flex-col gap-2 w-full items-center">
<div class="ml-3 flex-1 w-0">
<p class="text-xs break-words">
{@html processMessage(displayMessage)}
{#if isLongMessage && !showMore}
<button
type="button"
class="ml-1 {color.descriptionClass} font-medium hover:underline focus:outline-none"
onclick={() => (showMore = true)}
>
Show more
</button>
{/if}
</p>
{#if errorMessage}
<p class="text-xs {color.descriptionClass} w-full overflow-auto mt-2">
{errorMessage}
</p>
{/if}
</div>
<div class="flex justify-center ml-2">
{#each actions as action, index (index)}
<Button
variant={action.buttonType ?? 'default'}
variant={action.buttonType ?? 'subtle'}
unifiedSize="sm"
onClick={() => {
action.callback()
toast.pop(toastId)
}}
wrapperClasses="w-full"
btnClasses="{color.descriptionClass} font-medium"
>
{action.label}
</Button>
{/each}
</div>
<div class="ml-4 flex flex-shrink-0">
<button
type="button"
onclick={handleClose}
class="inline-flex rounded-md hover:text-primary focus:outline-none"
>
<span class="sr-only">Close</span>
<svg class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor" aria-hidden="true">
<path
d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
/>
</svg>
</button>
</div>
</div>
<!-- Duration indicator -->
<div
class="h-0.5 transition-colors {hover ? 'bg-gray-300' : color.bg}"
style="width: {Math.max(0, 1 - (state?.elapsed ?? duration) / duration) * 100}%"
class="h-[1px] absolute bottom-0 transition-colors bg-current {color.iconClass} opacity-60"
style="width: {Math.max(0, 1 - (_state?.elapsed ?? duration) / duration) * 100}%"
>
</div>
</div>
frontend/src/lib/components/TooltipInner.svelte
+7 -1
View File
@@ -2,13 +2,19 @@
import Markdown from 'svelte-exmarkdown'
import { ExternalLink } from 'lucide-svelte'
import { gfmPlugin } from 'svelte-exmarkdown/gfm'
import { twMerge } from 'tailwind-merge'
export let documentationLink: string | undefined = undefined
export let markdownTooltip: string | undefined = undefined
export let customBgClass: string | undefined = undefined
const plugins = [gfmPlugin()]
</script>
<div
class="shadow-lg max-w-sm break-words py-2 px-3 rounded-md text-xs font-normal text-primary bg-surface-secondary whitespace-normal text-left dark:border max-h-64 overflow-y-auto"
class={twMerge(
'shadow-lg max-w-sm break-words py-2 px-3 rounded-md text-xs font-normal text-primary whitespace-normal text-left dark:border max-h-64 overflow-y-auto',
customBgClass || 'bg-surface-secondary'
)}
>
{#if markdownTooltip}
<div class="prose-sm">
frontend/src/lib/components/assets/AssetColumnBadges.svelte
+44
View File
@@ -0,0 +1,44 @@
<script lang="ts">
import { type AssetUsageAccessType } from 'windmill-utils-internal/dist/gen/types.gen'
import { formatAssetAccessType } from './lib'
import Tooltip from '../meltComponents/Tooltip.svelte'
import { twMerge } from 'tailwind-merge'
type Props = {
columns: Record<string, AssetUsageAccessType> | undefined
disableTooltip?: boolean
badgeClasses?: string
disableWrap?: boolean
}
let { columns, disableTooltip, badgeClasses, disableWrap }: Props = $props()
let entries = $derived(columns && Object.entries(columns))
</script>
{#if entries?.length}
<div class={twMerge('flex gap-1', disableWrap ? '' : 'flex-wrap')}>
{#each entries as [columnName, accessType]}
{@const accessType2 = formatAssetAccessType(accessType)}
{#snippet badge()}
<div
class={twMerge(
'text-xs text-secondary border rounded-md px-1 bg-surface-tertiary dark:bg-surface-secondary',
badgeClasses
)}
>
{columnName}
</div>
{/snippet}
{#if disableTooltip}
{@render badge()}
{:else}
<Tooltip>
{@render badge()}
<svelte:fragment slot="text">
{accessType2} access to column "{columnName}"
</svelte:fragment>
</Tooltip>
{/if}
{/each}
</div>
{/if}
frontend/src/lib/components/assets/AssetsUsageDrawer.svelte
+16 -14
View File
@@ -6,8 +6,9 @@
import Tooltip from '../meltComponents/Tooltip.svelte'
import Tooltip2 from '../Tooltip.svelte'
import { twMerge } from 'tailwind-merge'
import { displayDate } from '$lib/utils'
import { capitalize, displayDate } from '$lib/utils'
import Alert from '../common/alert/Alert.svelte'
import AssetColumnBadges from './AssetColumnBadges.svelte'
let usagesDrawerData:
| {
@@ -64,14 +65,10 @@
</DrawerContent>
</Drawer>
{#snippet badge(text: string | undefined, tooltip?: string)}
{#snippet rightBadge(text: string | undefined, tooltip?: string)}
{#if text}
<Tooltip disablePopup={!tooltip}>
<div
class={twMerge(
'text-xs bg-surface font-normal border text-primary min-w-12 p-1 text-center rounded-md'
)}
>
<div class={twMerge('text-xs font-normal text-primary min-w-12 p-1 text-center rounded-md')}>
{text}
</div>
<svelte:fragment slot="text">
@@ -92,7 +89,7 @@
<a
href={getAssetUsagePageUri(u)}
aria-label={`${u.kind}/${u.path}`}
class="text-xs text-primary font-normal flex items-center py-3 px-4 gap-2 hover:bg-surface-hover cursor-pointer"
class="text-xs min-h-14 text-primary font-normal flex items-center py-2 px-4 gap-2 hover:bg-surface-hover cursor-pointer"
>
<RowIcon
kind={!u.metadata?.job_kind
@@ -110,14 +107,19 @@
} as const
)[u.metadata.job_kind] ?? 'script')}
/>
<div class="flex flex-col justify-center flex-1">
<span class="font-semibold text-emphasis">
{u.kind == 'job' ? (u.metadata?.runnable_path ?? 'Unknown job') : u.path}
<div class="flex flex-col justify-center flex-1 ml-2">
<span>
<span class="font-semibold text-emphasis">
{u.kind == 'job' ? (u.metadata?.runnable_path ?? 'Unknown job') : u.path}
</span>
</span>
<span class="text-2xs text-secondary">{u.kind == 'job' ? u.path : u.kind}</span>
<span class="text-2xs text-secondary">
{u.kind == 'job' ? u.path : capitalize(u.kind)}
</span>
<AssetColumnBadges columns={u.columns} badgeClasses="mt-0.5" />
</div>
{@render badge(displayDate(u.created_at), 'Asset detection time')}
{@render badge(accessType)}
{@render rightBadge(displayDate(u.created_at), 'Asset detection time')}
{@render rightBadge(accessType)}
</a>
</li>
{/each}
frontend/src/lib/components/assets/lib.ts
+1
View File
@@ -12,6 +12,7 @@ export type AssetKind = _AssetKind
export type AssetWithAccessType = Asset & { access_type?: AssetUsageAccessType }
export type AssetWithAltAccessType = AssetWithAccessType & {
alt_access_type?: AssetUsageAccessType
columns?: Record<string, AssetUsageAccessType>
}
export type AssetUsage = ListAssetsResponse['assets'][number]['usages'][number]
frontend/src/lib/components/common/alert/Alert.svelte
+2 -16
View File
@@ -1,14 +1,7 @@
<script lang="ts">
import { type AlertType, classes } from './model'
import { type AlertType, classes, icons } from './model'
import Tooltip from '$lib/components/Tooltip.svelte'
import {
AlertCircle,
AlertTriangle,
CheckCircle2,
Info,
ChevronDown,
ChevronUp
} from 'lucide-svelte'
import { ChevronDown, ChevronUp } from 'lucide-svelte'
import { slide } from 'svelte/transition'
import { twMerge } from 'tailwind-merge'
@@ -54,13 +47,6 @@
children
}: Props = $props()
const icons: Record<AlertType, any> = {
info: Info,
warning: AlertCircle,
error: AlertTriangle,
success: CheckCircle2
}
function toggleCollapse() {
if (collapsible) {
isCollapsed = !isCollapsed
frontend/src/lib/components/common/alert/model.ts
+9
View File
@@ -1,3 +1,5 @@
import { AlertCircle, AlertTriangle, CheckCircle2, Info } from 'lucide-svelte'
export type AlertType = 'success' | 'error' | 'warning' | 'info'
export const classes: Record<AlertType, Record<string, string>> = {
@@ -27,3 +29,10 @@ export const classes: Record<AlertType, Record<string, string>> = {
descriptionClass: 'text-green-700 dark:text-green-100/90'
}
}
export const icons: Record<AlertType, any> = {
info: Info,
warning: AlertCircle,
error: AlertTriangle,
success: CheckCircle2
}
frontend/src/lib/components/copilot/chat/flow/openFlow.json
+1 -1
View File
File diff suppressed because one or more lines are too long
frontend/src/lib/components/copilot/chat/flow/openFlowZod.ts
+3 -3
View File
File diff suppressed because one or more lines are too long
frontend/src/lib/components/flows/FlowProgressBar.svelte
+9
View File
@@ -1,6 +1,7 @@
<script lang="ts">
import { type Job } from '$lib/gen'
import ProgressBar from '../progressBar/ProgressBar.svelte'
import { forLater } from '$lib/forLater'
interface Props {
job?: Job | undefined
@@ -31,10 +32,16 @@
let currentStepId: string | undefined = $state(undefined)
let isWaitingForEvents = $state(false)
let isCanceled = $state(false)
let isScheduled = $state(false)
let progressBar = $state<ProgressBar | undefined>(undefined)
function updateJobProgress(job: Job) {
// Check if job is scheduled for later
const isJobScheduled = Boolean('running' in job && 'scheduled_for' in job &&
job.scheduled_for && forLater(job.scheduled_for))
isScheduled = isJobScheduled
const modules = job?.flow_status?.modules
if (!modules?.length) {
return
@@ -118,6 +125,7 @@
currentStepId = undefined
isWaitingForEvents = false
isCanceled = false
isScheduled = false
}
$effect(() => {
job && updateJobProgress(job)
@@ -140,4 +148,5 @@
{showStepId}
{isWaitingForEvents}
{isCanceled}
{isScheduled}
/>
frontend/src/lib/components/flows/missingInputWarnings.ts
+8
View File
@@ -7,6 +7,8 @@ import type { StateStore } from '$lib/utils'
import type { FlowState } from './flowState'
import { dfs } from './dfs'
const isAiTransform = (transform: InputTransform | undefined) => transform?.type === 'ai'
function isInputFilled(
inputTransforms: Record<string, InputTransform>,
key: string,
@@ -20,6 +22,9 @@ function isInputFilled(
if (inputTransforms.hasOwnProperty(key)) {
const transform = inputTransforms[key]
if (isAiTransform(transform)) {
return true
}
if (
transform?.type === 'static' &&
(transform?.value === undefined || transform?.value === '' || transform?.value === null)
@@ -41,6 +46,9 @@ async function isConnectedToMissingModule(
input_transform: InputTransform,
moduleIds: string[]
): Promise<string | undefined> {
if (isAiTransform(input_transform)) {
return undefined
}
const val: string =
input_transform.type === 'static' ? String(input_transform.value) : input_transform.expr
frontend/src/lib/components/flows/utils.svelte.ts
+3 -1
View File
@@ -49,7 +49,7 @@ export function evalValue(
if (t.type == 'static') {
v = t.value
} else {
} else if (t.type == 'javascript') {
try {
let context = {
flow_input: pickableProperties?.flow_input,
@@ -62,6 +62,8 @@ export function evalValue(
}
v = undefined
}
} else {
v = undefined
}
if (v === NEVER_TESTED_THIS_FAR) {
v = undefined
frontend/src/lib/components/graph/FlowGraphV2.svelte
+1 -1
View File
@@ -948,7 +948,7 @@
{height}
{width}
minZoom={0.2}
maxZoom={1.2}
maxZoom={1.6}
connectionLineType={ConnectionLineType.SmoothStep}
defaultEdgeOptions={{ type: 'smoothstep' }}
preventScrolling={scroll}
frontend/src/lib/components/graph/graphBuilder.svelte.ts
+2
View File
@@ -297,6 +297,7 @@ export type AssetN = {
type: 'asset'
data: {
asset: AssetWithAltAccessType
displayedAccessType: 'r' | 'w'
}
}
@@ -304,6 +305,7 @@ export type AssetsOverflowedN = {
type: 'assetsOverflowed'
data: {
overflowedAssets: AssetWithAltAccessType[]
displayedAccessType: 'r' | 'w'
}
}
frontend/src/lib/components/graph/renderers/nodes/AssetNode.svelte
+29 -10
View File
@@ -57,7 +57,7 @@
// All asset nodes displayed on top
const inputAssetNodes: (Node & AssetN)[] = displayedInputAssets.map((asset, i) => {
let inputAssetXGap = 12
let inputAssetWidth = 150
let inputAssetWidth = 165
const targetRowW =
MAX_ASSET_ROW_WIDTH -
@@ -73,7 +73,7 @@
return {
type: 'asset' as const,
parentId: node.id,
data: { asset },
data: { asset, displayedAccessType: 'r' },
id: `${node.id}-asset-in-${asset.kind}-${asset.path}`,
width: inputAssetWidth,
position: {
@@ -94,7 +94,7 @@
// All asset nodes displayed on the bottom
const outputAssetNodes: (Node & AssetN)[] = displayedOutputAssets.map((asset, i) => {
let outputAssetXGap = 12
let outputAssetWidth = 150
let outputAssetWidth = 165
const targetRowW =
MAX_ASSET_ROW_WIDTH -
@@ -110,7 +110,7 @@
return {
type: 'asset' as const,
parentId: node.id,
data: { asset },
data: { asset, displayedAccessType: 'w' },
id: `${node.id}-asset-out-${asset.kind}-${asset.path}`,
width: outputAssetWidth,
position: {
@@ -150,7 +150,7 @@
if (overflowedInputAssets.length)
allAssetNodes.push({
type: 'assetsOverflowed',
data: { overflowedAssets: overflowedInputAssets },
data: { overflowedAssets: overflowedInputAssets, displayedAccessType: 'r' },
id: `${node.id}-assets-overflowed-in`,
parentId: node.id,
width: ASSETS_OVERFLOWED_NODE_WIDTH,
@@ -169,7 +169,7 @@
if (overflowedOutputAssets.length)
allAssetNodes.push({
type: 'assetsOverflowed',
data: { overflowedAssets: overflowedOutputAssets },
data: { overflowedAssets: overflowedOutputAssets, displayedAccessType: 'w' },
id: `${node.id}-assets-overflowed-out`,
parentId: node.id,
width: ASSETS_OVERFLOWED_NODE_WIDTH,
@@ -237,6 +237,7 @@
import { userStore } from '$lib/stores'
import { deepEqual } from 'fast-equals'
import { slide } from 'svelte/transition'
import AssetColumnBadges from '$lib/components/assets/AssetColumnBadges.svelte'
interface Props {
data: AssetN['data']
@@ -254,15 +255,24 @@
})
const usageCount = $derived(flowGraphAssetsCtx?.val.computeAssetsCount?.(data.asset))
const colors = $derived(getNodeColorClasses(undefined, isSelected))
let assetColumns = $derived(
data.asset.columns &&
Object.fromEntries(
Object.entries(data.asset.columns).filter(
([_, accessType]) => accessType && accessType === data.displayedAccessType
)
)
)
</script>
<NodeWrapper wrapperClass="bg-surface-secondary rounded-md">
{#snippet children({ darkMode })}
<!-- svelte-ignore a11y_no_static_element_interactions -->
<Tooltip>
<Tooltip customBgClass="bg-surface-tertiary">
<div
class={twMerge(
'h-6 flex items-center gap-1.5 rounded-md drop-shadow-base overflow-clip transition-colors',
'h-6 flex items-center rounded-md drop-shadow-base overflow-clip transition-colors',
colors.outline,
colors.text,
colors.bg
@@ -274,11 +284,19 @@
>
<AssetGenericIcon
assetKind={data.asset.kind}
class="shrink-0 ml-1 {isSelected ? 'text-accent' : 'text-tertiary'}"
class="shrink-0 ml-1 mr-1.5 {isSelected ? 'text-accent' : 'text-tertiary'}"
size="16px"
/>
<span class="text-3xs truncate flex-1">
<span
class="text-3xs truncate flex-1 flex items-center gap-1 [mask-image:linear-gradient(to_right,black_85%,transparent)] mr-0.5"
>
{formatShortAssetPath(data.asset)}
<AssetColumnBadges
columns={assetColumns}
disableTooltip
disableWrap
badgeClasses="text-3xs transition-opacity opacity-50 {isSelected ? 'opacity-0' : ''}"
/>
</span>
{#if data.asset.kind === 'resource' && cachedResourceMetadata === undefined}
<Tooltip class={'pr-1 flex items-center justify-center'}>
@@ -319,6 +337,7 @@
<span class="text-hint text-xs">
{formatAssetKind({ ...data.asset, metadata: cachedResourceMetadata })}</span
>
<AssetColumnBadges columns={assetColumns} disableTooltip />
</svelte:fragment>
</Tooltip>
{/snippet}
frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte
+1 -1
View File
@@ -60,7 +60,7 @@
<ul>
{#each data.overflowedAssets as asset}
<li class="w-48">
<AssetNode data={{ asset }} />
<AssetNode data={{ asset, displayedAccessType: data.displayedAccessType }} />
</li>
{/each}
</ul>
frontend/src/lib/components/instanceSettings.ts
+6 -6
View File
@@ -314,7 +314,7 @@ export const settings: Record<string, Setting[]> = {
label: 'UV index url',
description: 'Add private Pip registry',
key: 'pip_index_url',
fieldType: 'text',
fieldType: 'password',
placeholder: 'https://username:password@pypi.company.com/simple',
storage: 'setting',
ee_only: ''
@@ -323,7 +323,7 @@ export const settings: Record<string, Setting[]> = {
label: 'UV extra index url',
description: 'Add private extra Pip registry',
key: 'pip_extra_index_url',
fieldType: 'text',
fieldType: 'password',
placeholder: 'https://username:password@pypi.company.com/simple',
storage: 'setting',
ee_only: ''
@@ -332,7 +332,7 @@ export const settings: Record<string, Setting[]> = {
label: 'Npm config registry',
description: 'Add private npm registry',
key: 'npm_config_registry',
fieldType: 'text',
fieldType: 'password',
placeholder: 'https://registry.npmjs.org/:_authToken=npm_FOOBAR',
storage: 'setting',
ee_only: ''
@@ -342,7 +342,7 @@ export const settings: Record<string, Setting[]> = {
description:
'Add private scoped registries for Bun, See: https://bun.sh/docs/install/registries',
key: 'bunfig_install_scopes',
fieldType: 'text',
fieldType: 'password',
placeholder: '"@myorg3" = { token = "mytoken", url = "https://registry.myorg.com/" }',
storage: 'setting',
ee_only: ''
@@ -360,7 +360,7 @@ export const settings: Record<string, Setting[]> = {
label: 'Maven/Ivy repositories',
description: 'Add private Maven/Ivy repositories',
key: 'maven_repos',
fieldType: 'text',
fieldType: 'password',
placeholder: 'https://user:password@artifacts.foo.com/maven',
storage: 'setting',
ee_only: ''
@@ -377,7 +377,7 @@ export const settings: Record<string, Setting[]> = {
label: 'Ruby Gems repositories',
description: 'Add private Ruby repositories with credentials. Should end with /',
key: 'ruby_repos',
fieldType: 'text',
fieldType: 'password',
placeholder: 'https://user:password@gems.foo.com/',
storage: 'setting',
ee_only: ''
frontend/src/lib/components/meltComponents/Tooltip.svelte
+2 -1
View File
@@ -16,6 +16,7 @@
export let openDelay: number = 300
export let closeDelay: number = 0
export let portal: string | undefined | null = 'body'
export let customBgClass: string | undefined = undefined
const {
elements: { trigger, content },
@@ -47,7 +48,7 @@
{#if $open && !disablePopup}
<div use:melt={$content} transition:fade={{ duration: 100 }} style="z-index: {zIndexes.tooltip}">
<TooltipInner {documentationLink} {markdownTooltip}>
<TooltipInner {documentationLink} {markdownTooltip} {customBgClass}>
<slot name="text" />
</TooltipInner>
</div>
frontend/src/lib/components/progressBar/ProgressBar.svelte
+10 -5
View File
@@ -32,6 +32,8 @@
isWaitingForEvents?: boolean
// Whether the job was canceled
isCanceled?: boolean
// Whether the job is scheduled for later
isScheduled?: boolean
}
let {
@@ -50,7 +52,8 @@
stepId,
showStepId = false,
isWaitingForEvents = false,
isCanceled = false
isCanceled = false,
isScheduled = false
}: Props = $props()
let duration = 200
@@ -146,10 +149,10 @@
: 'text-blue-700 dark:text-blue-200'}"
>
<div class={twMerge(slim ? 'text-xs' : 'text-sm', 'flex items-center gap-1')}>
{#if status == 'running' && !isCanceled}
{#if status == 'running' && !isCanceled && !isScheduled}
<Loader2 class="animate-spin" size={14} />
{/if}
{#key status + isWaitingForEvents + stepId + isCanceled}
{#key status + isWaitingForEvents + stepId + isCanceled + isScheduled}
<span in:fade={{ duration: 150 }}>
{#if status == 'error'}
Error occurred
@@ -159,6 +162,8 @@
Done
{:else if isWaitingForEvents}
Waiting to be resumed
{:else if isScheduled}
(not started)
{:else if showStepId}
{stepId
? `${isCanceled ? 'Canceled at' : 'Running'} step ${stepId}`
@@ -166,9 +171,9 @@
{:else if hideStepTitle}
{isCanceled ? 'Canceled' : 'Running'}
{:else if subIndexIsPercent}
{`${isCanceled ? 'Canceled at' : ''} Step ${index + 1} (${subIndex !== undefined ? `${subIndex}%)` : ''}`}
{(isCanceled ? 'Canceled at ' : '') + `Step ${index + 1} (${subIndex !== undefined ? subIndex + '%' : ''})`}
{:else}
{`${isCanceled ? 'Canceled at' : ''} Step ${index + 1}${subIndex !== undefined ? `.${subIndex + 1}` : ''}`}
{(isCanceled ? 'Canceled at ' : '') + `Step ${index + 1}${subIndex !== undefined ? `.${subIndex + 1}` : ''}`}
{/if}
</span>
{/key}
frontend/src/lib/components/triggers/TriggerFilters.svelte
+78
View File
@@ -0,0 +1,78 @@
<script lang="ts">
import { Button } from '$lib/components/common'
import Section from '$lib/components/Section.svelte'
import { Plus, X } from 'lucide-svelte'
import { fade } from 'svelte/transition'
import JsonEditor from '$lib/components/JsonEditor.svelte'
interface Props {
filters: { key: string; value: any }[]
disabled?: boolean
}
let { filters = $bindable([]), disabled = false }: Props = $props()
</script>
<Section label="Filters">
<p class="text-xs mb-1 text-primary">
Filters will limit the execution of the trigger to only messages that match all criteria.<br />
The JSON filter checks if the value at the key is equal or a superset of the filter value.
</p>
<div class="flex flex-col gap-4 mt-1">
{#each filters as v, i (i)}
<div class="flex w-full gap-2 items-center">
<div class="w-full flex flex-col gap-2 border p-2 rounded-md">
<label class="flex flex-col w-full">
<div class="text-secondary text-sm mb-2">Key</div>
<input type="text" bind:value={v.key} {disabled} />
</label>
<!-- svelte-ignore a11y_label_has_associated_control -->
<label class="flex flex-col w-full">
<div class="text-secondary text-sm mb-2">Value</div>
<JsonEditor bind:value={v.value} code={JSON.stringify(v.value)} {disabled} />
</label>
{#if v.key}
{@const isObject = v.value !== null && typeof v.value === 'object'}
<div class="text-xs text-tertiary font-mono mt-2 p-2 bg-surface-secondary rounded">
payload.{v.key}
{isObject ? '⊇' : '=='}
{JSON.stringify(v.value)}
</div>
{/if}
</div>
<button
transition:fade|local={{ duration: 100 }}
class="rounded-full p-1 bg-surface-secondary duration-200 hover:bg-surface-hover"
aria-label="Clear"
onclick={() => {
filters = filters.filter((_, index) => index !== i)
}}
{disabled}
>
<X size={14} />
</button>
</div>
{/each}
<div class="flex items-baseline">
<Button
variant="default"
size="xs"
btnClasses="mt-1"
onclick={() => {
if (filters == undefined || !Array.isArray(filters)) {
filters = []
}
filters = filters.concat({
key: '',
value: ''
})
}}
{disabled}
startIcon={{ icon: Plus }}
>
Add filter
</Button>
</div>
</div>
</Section>
frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte
+7
View File
@@ -22,6 +22,7 @@
import { deepEqual } from 'fast-equals'
import TriggerSuspendedJobsAlert from '../TriggerSuspendedJobsAlert.svelte'
import TriggerSuspendedJobsModal from '../TriggerSuspendedJobsModal.svelte'
import TriggerFilters from '../TriggerFilters.svelte'
interface Props {
useDrawer?: boolean
@@ -85,6 +86,7 @@
let error_handler_path: string | undefined = $state()
let error_handler_args: Record<string, any> = $state({})
let retry: Retry | undefined = $state()
let filters: { key: string; value: any }[] = $state([])
let suspendedJobsModal = $state<TriggerSuspendedJobsModal | null>(null)
let originalConfig = $state<Record<string, any> | undefined>(undefined)
@@ -173,6 +175,7 @@
error_handler_path = nDefaultValues?.error_handler_path ?? undefined
error_handler_args = nDefaultValues?.error_handler_args ?? {}
retry = nDefaultValues?.retry ?? undefined
filters = nDefaultValues?.filters ?? []
errorHandlerSelected = getHandlerType(error_handler_path ?? '')
mode = nDefaultValues?.mode ?? 'enabled'
originalConfig = undefined
@@ -199,6 +202,7 @@
error_handler_path = cfg?.error_handler_path
error_handler_args = cfg?.error_handler_args ?? {}
retry = cfg?.retry
filters = cfg?.filters ?? []
errorHandlerSelected = getHandlerType(error_handler_path ?? '')
}
@@ -223,6 +227,7 @@
kafka_resource_path: kafkaResourcePath,
group_id: kafkaCfg.group_id,
topics: kafkaCfg.topics,
filters,
mode,
extra_perms: extra_perms,
error_handler_path,
@@ -439,6 +444,8 @@
showTestingBadge={isEditor}
/>
<TriggerFilters bind:filters disabled={!can_write} />
<Section label="Advanced" collapsable>
<div class="flex flex-col gap-4">
<div class="min-h-96">
frontend/src/lib/components/triggers/kafka/utils.ts
+1
View File
@@ -23,6 +23,7 @@ export async function saveKafkaTriggerFromCfg(
kafka_resource_path: cfg.kafka_resource_path,
group_id: cfg.group_id,
topics: cfg.topics,
filters: cfg.filters ?? [],
...errorHandlerAndRetries
}
try {
frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte
+2 -80
View File
@@ -25,6 +25,7 @@
import { fade } from 'svelte/transition'
import type { Schema } from '$lib/common'
import JsonEditor from '$lib/components/JsonEditor.svelte'
import TriggerFilters from '../TriggerFilters.svelte'
import Toggle from '$lib/components/Toggle.svelte'
import WebsocketEditorConfigSection from './WebsocketEditorConfigSection.svelte'
import { untrack, type Snippet } from 'svelte'
@@ -684,86 +685,7 @@
</div>
</Section>
<Section label="Filters">
<p class="text-xs mb-1 text-primary">
Filters will limit the execution of the trigger to only messages that match all criteria.<br
/>
The JSON filter checks if the value at the key is equal or a superset of the filter value.
</p>
<div class="flex flex-col gap-4 mt-1">
{#each filters as v, i}
<div class="flex w-full gap-2 items-center">
<div class="w-full flex flex-col gap-2 border p-2 rounded-md">
<div class="flex flex-row gap-2 w-full">
<label class="flex flex-col w-full">
<div class="text-secondary text-sm mb-2">Type</div>
<select
class="w-20"
onchange={(e) => {
if (e.target?.['value']) {
filters[i] = {
key: '',
value: ''
}
}
}}
value={'json'}
disabled={!can_write}
>
<option value="json">JSON</option>
</select>
</label>
</div>
<label class="flex flex-col w-full">
<div class="text-secondary text-sm mb-2">Key</div>
<input type="text" bind:value={v.key} disabled={!can_write} />
</label>
<!-- svelte-ignore a11y_label_has_associated_control -->
<label class="flex flex-col w-full">
<div class="text-secondary text-sm mb-2">Value</div>
<JsonEditor
bind:value={v.value}
code={JSON.stringify(v.value)}
disabled={!can_write}
/>
</label>
</div>
<button
transition:fade|local={{ duration: 100 }}
class="rounded-full p-1 bg-surface-secondary duration-200 hover:bg-surface-hover"
aria-label="Clear"
onclick={() => {
filters = filters.filter((_, index) => index !== i)
}}
disabled={!can_write}
>
<X size={14} />
</button>
</div>
{/each}
<div class="flex items-baseline">
<Button
variant="default"
size="xs"
btnClasses="mt-1"
on:click={() => {
if (filters == undefined || !Array.isArray(filters)) {
filters = []
}
filters = filters.concat({
key: '',
value: ''
})
}}
disabled={!can_write}
startIcon={{ icon: Plus }}
>
Add item
</Button>
</div>
</div>
</Section>
<TriggerFilters bind:filters disabled={!can_write} />
<Section label="Advanced" collapsable>
<div class="flex flex-col gap-4">
frontend/src/lib/components/workspaceSettings/StorageSettings.svelte
+28 -7
View File
@@ -203,7 +203,7 @@
{#if emptyString(tableRow[1].resourcePath) || isDirty(tableRow[0])}
<Popover
openOnHover
contentClasses="p-2 text-sm text-secondary italic"
contentClasses="p-2 text-xs text-secondary"
class="cursor-not-allowed"
>
<svelte:fragment slot="trigger">
@@ -242,11 +242,12 @@
{/each}
<Row class="!border-0">
<Cell colspan={tableHeadNames.length} class="pt-0 pb-2">
<div class="flex justify-center">
{#snippet addSecondaryStorageBtn()}
<Button
size="sm"
btnClasses="max-w-fit"
variant="default"
disabled={!s3ResourceSettings.resourcePath}
on:click={() => {
if (s3ResourceSettings.secondaryStorage === undefined) {
s3ResourceSettings.secondaryStorage = []
@@ -264,12 +265,32 @@
}}
>
<Plus /> Add secondary storage
<Tooltip>
Secondary storage is a feature that allows you to read and write from storage that
isn't your main storage by specifying it in the s3 object as "secondary_storage"
with the name of it
</Tooltip>
{#if s3ResourceSettings.resourcePath}
<Tooltip>
Secondary storage is a feature that allows you to read and write from storage that
isn't your main storage by specifying it in the s3 object as "secondary_storage"
with the name of it
</Tooltip>
{/if}
</Button>
{/snippet}
<div class="flex justify-center w-full">
{#if !s3ResourceSettings.resourcePath}
<Popover
class="cursor-not-allowed"
openOnHover
contentClasses="p-2 text-xs text-secondary"
>
{#snippet trigger()}
{@render addSecondaryStorageBtn()}
{/snippet}
{#snippet content()}
Setup a primary storage to use secondary storages
{/snippet}
</Popover>
{:else}
{@render addSecondaryStorageBtn()}
{/if}
</div>
</Cell>
</Row>

Some files were not shown because too many files have changed in this diff Show More