diff --git a/.claude/hooks/format-backend.sh b/.claude/hooks/format-backend.sh new file mode 100755 index 0000000000..d6077d7482 --- /dev/null +++ b/.claude/hooks/format-backend.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Format backend Rust files with rustfmt after Claude edits them + +# Get the file path from the tool result (passed via stdin as JSON) +INPUT=$(cat) +FILE_PATH=$(echo "$INPUT" | jq -r '.tool_input.file_path // empty') + +# Exit if no file path +if [ -z "$FILE_PATH" ]; then + exit 0 +fi + +# Check if the file is in the backend directory and is a Rust file +if [[ "$FILE_PATH" == *"/backend/"* ]] && [[ "$FILE_PATH" =~ \.rs$ ]]; then + cd "$CLAUDE_PROJECT_DIR/backend" || exit 0 + # Run rustfmt with config from rustfmt.toml (edition=2021) + rustfmt --config-path rustfmt.toml "$FILE_PATH" 2>/dev/null || true +fi + +exit 0 diff --git a/.claude/hooks/format-frontend.sh b/.claude/hooks/format-frontend.sh new file mode 100755 index 0000000000..d0b4f0559b --- /dev/null +++ b/.claude/hooks/format-frontend.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Format frontend files with prettier after Claude edits them + +# Get the file path from the tool result (passed via stdin as JSON) +INPUT=$(cat) +FILE_PATH=$(echo "$INPUT" | jq -r '.tool_input.file_path // empty') + +# Exit if no file path +if [ -z "$FILE_PATH" ]; then + exit 0 +fi + +# Check if the file is in the frontend directory +if [[ "$FILE_PATH" == *"/frontend/"* ]]; then + # Check if it's a formattable file type + if [[ "$FILE_PATH" =~ \.(ts|js|svelte|json|css|html|md)$ ]]; then + cd "$CLAUDE_PROJECT_DIR/frontend" || exit 0 + # Run prettier silently, don't fail the hook if prettier fails + npx prettier --write "$FILE_PATH" 2>/dev/null || true + fi +fi + +exit 0 diff --git a/.claude/hooks/notify-user.sh b/.claude/hooks/notify-user.sh new file mode 100755 index 0000000000..086b55c6c2 --- /dev/null +++ b/.claude/hooks/notify-user.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Notify user when Claude requires input (works on macOS and Linux) + +# Check if we're in an SSH session +if [[ -n "$SSH_CLIENT" || -n "$SSH_TTY" || -n "$SSH_CONNECTION" ]]; then + # SSH session - use terminal bell + # If using VSCode, enable audible terminal bell for SSH sessions: + # Add the following to .vscode/settings.json: + # "accessibility.signals.terminalBell": { + # "sound": "on" + # }, + # "terminal.integrated.enableVisualBell": true + printf '\a' +else + # Local session - use native notifications + if [[ "$OSTYPE" == "darwin"* ]]; then + osascript -e 'display notification "Claude is waiting for your input" with title "Claude Code" sound name "Glass"' 2>/dev/null || printf '\a' + elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + notify-send "Claude Code" "Claude is waiting for your input" 2>/dev/null || printf '\a' + else + printf '\a' + fi +fi + +exit 0 diff --git a/.claude/settings.json b/.claude/settings.json index 5fd636bdf1..eac00f2227 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -23,7 +23,11 @@ "Bash(git log:*)", "Bash(git branch:*)", "Bash(git show:*)", - "Bash(git blame:*)" + "Bash(git blame:*)", + "Bash(cargo check:*)", + "mcp__ide__getDiagnostics", + "Bash(npm run generate-backend-client:*)", + "Bash(npm run check:*)" ], "deny": [ "Read(.env)", @@ -91,6 +95,34 @@ } ] } + ], + "PostToolUse": [ + { + "matcher": "Edit|Write", + "hooks": [ + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR\"/.claude/hooks/format-frontend.sh", + "timeout": 30 + }, + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR\"/.claude/hooks/format-backend.sh", + "timeout": 30 + } + ] + } + ], + "Notification": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR\"/.claude/hooks/notify-user.sh", + "timeout": 10 + } + ] + } ] }, "enabledPlugins": { diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 8acc762451..8783bb241c 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -42,7 +42,7 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER RUN /usr/local/bin/python3 -m pip install pip-tools # Bun -COPY --from=oven/bun:1.2.23 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.3.8 /usr/local/bin/bun /usr/bin/bun ARG TARGETPLATFORM diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 0de231e9bb..1f51ee2fa4 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -44,7 +44,10 @@ jobs: go-version: 1.21.5 - uses: oven-sh/setup-bun@v2 with: - bun-version: 1.1.43 + bun-version: 1.3.8 + - uses: actions/setup-node@v4 + with: + node-version: '20' - uses: astral-sh/setup-uv@v6.2.1 with: version: "0.9.24" @@ -67,6 +70,111 @@ jobs: - name: Substitute EE code (EE logic is behind feature flag) run: | ./substitute_ee_code.sh --copy --dir ./windmill-ee-private + - name: Setup private npm registry with test package + working-directory: /tmp + run: | + set -e + + # Install Verdaccio globally + npm install -g verdaccio + + # Create Verdaccio config that requires authentication for @windmill-test packages + mkdir -p /tmp/verdaccio/storage + cat > /tmp/verdaccio/config.yaml << 'VERDACCIO_CONFIG' + storage: /tmp/verdaccio/storage + auth: + htpasswd: + file: /tmp/verdaccio/htpasswd + max_users: 100 + uplinks: + npmjs: + url: https://registry.npmjs.org/ + packages: + '@windmill-test/*': + access: $authenticated + publish: $authenticated + '@*/*': + access: $all + publish: $authenticated + proxy: npmjs + '**': + access: $all + publish: $authenticated + proxy: npmjs + server: + keepAliveTimeout: 60 + middlewares: + audit: + enabled: true + log: { type: stdout, format: pretty, level: warn } + VERDACCIO_CONFIG + + # Create empty htpasswd file (users will be created via API) + touch /tmp/verdaccio/htpasswd + + # Start Verdaccio in background + verdaccio --config /tmp/verdaccio/config.yaml & + VERDACCIO_PID=$! + + # Wait for Verdaccio to be ready + echo "Waiting for Verdaccio to start..." + for i in {1..30}; do + if curl -s http://localhost:4873/-/ping > /dev/null 2>&1; then + echo "Verdaccio is ready" + break + fi + sleep 1 + done + + # Login to get a token + echo "Getting auth token..." + RESPONSE=$(curl -s -X PUT \ + -H "Content-Type: application/json" \ + -d '{"name":"testuser","password":"testpass123"}' \ + http://localhost:4873/-/user/org.couchdb.user:testuser) + + echo "Auth response: $RESPONSE" + NPM_TOKEN=$(echo "$RESPONSE" | jq -r '.token') + + if [ -z "$NPM_TOKEN" ] || [ "$NPM_TOKEN" = "null" ]; then + echo "Failed to get NPM token from response" + exit 1 + fi + + echo "NPM_TOKEN=${NPM_TOKEN}" >> $GITHUB_ENV + echo "Got NPM token successfully: ${NPM_TOKEN:0:10}..." + + # Configure npm globally with the auth token + echo "//localhost:4873/:_authToken=${NPM_TOKEN}" > ~/.npmrc + echo "Configured ~/.npmrc with auth token" + + # Create a simple test package + mkdir -p /tmp/windmill-test-private-pkg + cat > /tmp/windmill-test-private-pkg/package.json << 'PKG_JSON' + { + "name": "@windmill-test/private-pkg", + "version": "1.0.0", + "main": "index.js" + } + PKG_JSON + cat > /tmp/windmill-test-private-pkg/index.js << 'PKG_JS' + module.exports.greet = (name) => `Hello from private package, ${name}!`; + PKG_JS + + # Publish to Verdaccio with auth + cd /tmp/windmill-test-private-pkg + echo "Publishing package..." + npm publish --registry http://localhost:4873 + echo "Package published successfully" + + # Verify the package requires auth by trying anonymous access (should fail) + rm -f ~/.npmrc + echo "Testing anonymous access (should fail)..." + if npm view @windmill-test/private-pkg --registry http://localhost:4873 2>/dev/null; then + echo "ERROR: Package should require authentication but anonymous access worked" + exit 1 + fi + echo "Verified: Package requires authentication for @windmill-test/private-pkg" - name: Cache DuckDB FFI module build uses: actions/cache@v3 with: @@ -84,9 +192,10 @@ jobs: RUST_LOG_STYLE: never CARGO_NET_GIT_FETCH_WITH_CLI: true WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1 - WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1 + WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1 WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1 + TEST_NPM_REGISTRY: "http://localhost:4873/:_authToken=${{ env.NPM_TOKEN }}" run: | - deno --version && bun -v && go version && python3 --version + deno --version && bun -v && node --version && go version && python3 --version cd windmill-duckdb-ffi-internal && ./build_dev.sh && cd .. - DENO_PATH=$(which deno) BUN_PATH=$(which bun) GO_PATH=$(which go) UV_PATH=$(which uv) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private --all -- --nocapture + DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test --all -- --nocapture diff --git a/.gitignore b/.gitignore index 7b93b49807..471fe31b59 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,9 @@ backend/.minio-data .aider* !.aiderignore rust-client/Cargo.toml + +# Symlinked cache directories (for git worktrees) +backend/target +frontend/node_modules +typescript-client/node_modules +frontend/.svelte-kit diff --git a/CHANGELOG.md b/CHANGELOG.md index b0555f1102..f4244c0869 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## [1.624.0](https://github.com/windmill-labs/windmill/compare/v1.623.1...v1.624.0) (2026-02-03) + + +### Features + +* default to quickjs on ce for flow eval ([#7756](https://github.com/windmill-labs/windmill/issues/7756)) ([bdf9447](https://github.com/windmill-labs/windmill/commit/bdf9447e821c6d02198534198a5878849cac23e5)) +* runtime assets ([#7656](https://github.com/windmill-labs/windmill/issues/7656)) ([635a24f](https://github.com/windmill-labs/windmill/commit/635a24f82cae8e85b584efca115968872723889f)) + + +### Bug Fixes + +* **cli:** prevent branch-specific items from being marked for deletion on pull ([#7781](https://github.com/windmill-labs/windmill/issues/7781)) ([701eb4b](https://github.com/windmill-labs/windmill/commit/701eb4bae47a809e6da34c62b8e250ac6379db53)) +* Fix app multiselect not refreshing result when creating element ([#7766](https://github.com/windmill-labs/windmill/issues/7766)) ([3a719ce](https://github.com/windmill-labs/windmill/commit/3a719cea6b7b099f32054957eb04148c592786ad)) +* **frontend:** improve runs detail page ([#7694](https://github.com/windmill-labs/windmill/issues/7694)) ([3b5c165](https://github.com/windmill-labs/windmill/commit/3b5c1657c7d41178283d02017914543461565a3a)) +* Prettier and less invasive toasts ([#7758](https://github.com/windmill-labs/windmill/issues/7758)) ([df51f96](https://github.com/windmill-labs/windmill/commit/df51f9690520db80db2133e2e61002f399c0dfaf)) +* remove $schema field from Google AI output schema requests ([#7765](https://github.com/windmill-labs/windmill/issues/7765)) ([18d85f1](https://github.com/windmill-labs/windmill/commit/18d85f14127e50673ccb460bfa9ebe80730df68e)) + ## [1.623.1](https://github.com/windmill-labs/windmill/compare/v1.623.0...v1.623.1) (2026-02-01) diff --git a/Dockerfile b/Dockerfile index e1e30876c4..8b15e5e79e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -234,7 +234,7 @@ COPY --from=windmill_duckdb_ffi_internal_builder /windmill-duckdb-ffi-internal/t COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno -COPY --from=oven/bun:1.2.23 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.3.8 /usr/local/bin/bun /usr/bin/bun COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/README.md b/README.md index 5bfad3e7bf..51fef5b58b 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@

-Open-source developer infrastructure for internal tools (APIs, background jobs, workflows and UIs). Self-hostable alternative to Retool, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs and custom UIs to trigger workflows and scripts as internal apps. +Open-source developer platform for internal code: APIs, background jobs, workflows and UIs. Self-hostable alternative to Retool, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs and custom UIs to trigger workflows and scripts as internal apps.

-Scripts are turned into sharable UIs automatically, and can be composed together into flows or used into richer apps built with low-code. Supported script languages supported are: Python, TypeScript, Go, Bash, SQL, and GraphQL. +Scripts are turned into sharable UIs automatically, and can be composed together into flows or used into richer apps built with low-code. Supported languages: Python, TypeScript, Go, Bash, SQL, GraphQL, PowerShell, Rust, and more.

@@ -36,75 +36,58 @@ Scripts are turned into sharable UIs automatically, and can be composed together # Windmill - Developer platform for APIs, background jobs, workflows and UIs -Windmill is fully open-sourced (AGPLv3) and Windmill Labs offers -dedicated instance and commercial support and licenses. +Windmill is fully open-sourced (AGPLv3) and Windmill Labs offers dedicated instances and commercial support and licenses. ![Windmill Diagram](./imgs/stacks.svg) -https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-822f-0c7ee7104252 +https://github.com/user-attachments/assets/d80de1d9-64de-4d89-aacd-6df23fa81fc4 - [Windmill - Developer platform for APIs, background jobs, workflows and UIs](#windmill---developer-platform-for-apis-background-jobs-workflows-and-uis) - [Main Concepts](#main-concepts) - [Show me some actual script code](#show-me-some-actual-script-code) - - [CLI](#cli) - - [Running scripts locally](#running-scripts-locally) + - [Local Development](#local-development) - [Stack](#stack) - [Fastest Self-Hostable Workflow Engine](#fastest-self-hostable-workflow-engine) - [Security](#security) - - [Sandboxing](#sandboxing) - - [Secrets, credentials and sensitive values](#secrets-credentials-and-sensitive-values) - [Performance](#performance) - [Architecture](#architecture) - [How to self-host](#how-to-self-host) - [Docker compose](#docker-compose) - - [Kubernetes (k8s) and Helm charts](#kubernetes-k8s-and-helm-charts) - - [Run from binaries](#run-from-binaries) + - [Kubernetes (Helm charts)](#kubernetes-helm-charts) + - [Cloud providers](#cloud-providers) - [OAuth, SSO \& SMTP](#oauth-sso--smtp) - - [Commercial license](#commercial-license) + - [License](#license) - [Integrations](#integrations) - [Environment Variables](#environment-variables) - [Run a local dev setup](#run-a-local-dev-setup) - - [only Frontend](#only-frontend) + - [Frontend only](#frontend-only) - [Backend + Frontend](#backend--frontend) - [Contributors](#contributors) - [Copyright](#copyright) ## Main Concepts -1. Define a minimal and generic script in Python, TypeScript, Go or Bash that - solves a specific task. The code can be defined in the - [provided Web IDE](https://www.windmill.dev/docs/code_editor) or - [synchronized with your own GitHub repo](https://www.windmill.dev/docs/advanced/cli/sync) - (e.g. through - [VS Code](https://www.windmill.dev/docs/cli_local_dev/vscode-extension) - extension): +1. Define a minimal and generic script in Python, TypeScript, Go or Bash that solves a specific task. The code can be defined in the provided Web IDE or synchronized with your own GitHub repo (e.g. through VS Code extension): [provided Web IDE](https://www.windmill.dev/docs/code_editor) or [synchronized with your own GitHub repo](https://www.windmill.dev/docs/advanced/cli/sync) (e.g. through [VS Code](https://www.windmill.dev/docs/cli_local_dev/vscode-extension) extension): - ![Step 1](./imgs/windmill-editor.png) +![Step 1](./imgs/windmill-editor.png) -2. Your scripts parameters are automatically parsed and - [generate a frontend](https://www.windmill.dev/docs/core_concepts/auto_generated_uis). +2. Your scripts parameters are automatically parsed and [generate a frontend](https://www.windmill.dev/docs/core_concepts/auto_generated_uis). ![Step 2](./imgs/windmill-run.png) ![Step 3](./imgs/windmill-result.png) -3. Make it [flow](https://www.windmill.dev/docs/flows/flow_editor)! You can - chain your scripts or scripts made by the community shared on - [WindmillHub](https://hub.windmill.dev). +3. Make it [flow](https://www.windmill.dev/docs/flows/flow_editor)! You can chain your scripts or scripts made by the community shared on [WindmillHub](https://hub.windmill.dev). - ![Step 3](./imgs/windmill-flow.png) +![Step 3](./imgs/windmill-flow.png) -4. Build [complex UIs](https://www.windmill.dev/docs/apps/app_editor) on top of - your scripts and flows. +4. Build [complex UIs](https://www.windmill.dev/docs/apps/app_editor) on top of your scripts and flows. - ![Step 4](./imgs/windmill-builder.png) +![Step 4](./imgs/windmill-builder.png) -Scripts and flows can also be triggered by a -[cron schedule](https://www.windmill.dev/docs/core_concepts/scheduling) (e.g. -'_/5 _ \* \* \*') or through -[webhooks](https://www.windmill.dev/docs/core_concepts/webhooks). +Scripts and flows can be triggered by [schedules](https://www.windmill.dev/docs/core_concepts/scheduling), [webhooks](https://www.windmill.dev/docs/core_concepts/webhooks), [HTTP routes](https://www.windmill.dev/docs/core_concepts/http_routing), [Kafka](https://www.windmill.dev/docs/core_concepts/kafka_triggers), [WebSockets](https://www.windmill.dev/docs/core_concepts/websocket_triggers), [emails](https://www.windmill.dev/docs/core_concepts/email_triggers), and more. -You can build your entire infra on top of Windmill! +Build your entire infra on top of Windmill! ## Show me some actual script code @@ -144,43 +127,31 @@ export async function main( } ``` -## CLI +## Local Development -We have a powerful CLI to interact with the windmill platform and sync your -scripts from local files, GitHub repos and to run scripts and flows on the -instance from local commands. See -[more details](https://www.windmill.dev/docs/advanced/cli). +Windmill supports multiple ways to develop locally and sync with your instance: -![CLI Screencast](./cli/vhs/output/setup.gif) +| Tool | Description | +|------|-------------| +| **[CLI](https://www.windmill.dev/docs/advanced/cli)** | Sync scripts from local files or GitHub, run scripts/flows from the command line | +| **[VS Code Extension](https://www.windmill.dev/docs/cli_local_dev/vscode-extension)** | Edit and test scripts & flows directly from VS Code / Cursor with full IDE support | +| **[Git Sync](https://www.windmill.dev/docs/advanced/git_sync)** | Two-way sync between Windmill and your Git repository | +| **[Claude Code](https://www.windmill.dev/docs/core_concepts/ai_generation)** | AI-assisted development with Claude for scripts, flows, and apps | -### Running scripts locally +https://github.com/user-attachments/assets/c541c326-e9ae-4602-a09a-1989aaded1e9 -You can run your script locally easily, you simply need to pass the right -environment variables for the `wmill` client library to fetch resources and -variables from your instance if necessary. See more: -. - -To develop & test locally scripts & flows, we recommend using the Windmill VS -Code extension: . +You can run scripts locally by passing the right environment variables for the `wmill` client library to fetch resources and variables from your instance. See [local development docs](https://www.windmill.dev/docs/advanced/local_development). ## Stack -- Postgres as the database. -- Backend in Rust with the following highly-available and horizontally scalable. - Architecture: - - Stateless API backend. - - Workers that pull jobs from a queue in Postgres (and later, Kafka or Redis. - Upvote [#173](#https://github.com/windmill-labs/windmill/issues/173) if - interested). -- Frontend in Svelte. -- Scripts executions are sandboxed using Google's - [nsjail](https://github.com/google/nsjail). -- Javascript runtime is the - [deno_core rust library](https://denolib.gitbook.io/guide/) (which itself uses - the [rusty_v8](https://github.com/denoland/rusty_v8) and hence V8 underneath). -- TypeScript runtime is Bun and deno. -- Python runtime is python3. -- Golang runtime is 1.19.1. +- **Database**: Postgres (compatible with Aurora, Cloud SQL, Neon, Azure PostgreSQL) +- **Backend**: Rust - stateless API servers and workers pulling jobs from a Postgres queue +- **Frontend**: Svelte 5 +- **Sandboxing**: [nsjail](https://github.com/google/nsjail) and PID namespace isolation +- **Runtimes**: + - TypeScript/JavaScript: Bun (default) and Deno + - Python: python3 with uv for dependency management + - Go, Bash, PowerShell, PHP, Rust, C#, Java, Ansible ## Fastest Self-Hostable Workflow Engine @@ -197,19 +168,10 @@ page. ## Security -### Sandboxing +- **Sandboxing**: [nsjail](https://github.com/google/nsjail) for filesystem/resource isolation, and PID namespace isolation (enabled by default) to prevent jobs from accessing worker process memory +- **Secrets**: One encryption key per workspace for credentials stored in Windmill's K/V store. We recommend encrypting the Postgres database as well. -Windmill can use [nsjail](https://github.com/google/nsjail). It is production -multi-tenant grade secure. Do not take our word for it, take -[fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/). - -### Secrets, credentials and sensitive values - -There is one encryption key per workspace to encrypt the credentials and secrets -stored in Windmill's K/V store. - -In addition, we strongly recommend that you encrypt the whole Postgres database. -That is what we do at . +See [Security documentation](https://www.windmill.dev/docs/advanced/security_isolation) for details. ## Performance @@ -229,19 +191,13 @@ back to the database is ~50ms. A typical lightweight deno job will take around ## How to self-host -We only provide docker-compose setup here. For more advanced setups, like -compiling from source or using without a postgres super user, see -[Self-Host documentation](https://www.windmill.dev/docs/advanced/self_host). +For detailed setup options, see [Self-Host documentation](https://www.windmill.dev/docs/advanced/self_host). ### Docker compose -Windmill can be deployed using 3 files: -([docker-compose.yml](./docker-compose.yml), [Caddyfile](./Caddyfile) and a -[.env](./.env)) in a single command. +Deploy Windmill with 3 files ([docker-compose.yml](./docker-compose.yml), [Caddyfile](./Caddyfile), [.env](./.env)): -Make sure Docker is started, and run: - -``` +```bash curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env @@ -249,86 +205,45 @@ curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env docker compose up -d ``` -Go to http://localhost et voilà :) +Go to http://localhost - default credentials: `admin@windmill.dev` / `changeme` -The default super-admin user is: admin@windmill.dev / changeme. +**Using an external database**: Set `DATABASE_URL` in `.env` to point to your managed Postgres (AWS RDS, GCP Cloud SQL, Azure, Neon, etc.) and set db replicas to 0. -From there, you can follow the setup app and create other users. - -More details in -[Self-Host Documention](https://www.windmill.dev/docs/advanced/self_host#docker). - -### Kubernetes (k8s) and Helm charts - -We publish helm charts at: -. - -### Run from binaries - -Each release includes the corresponding binaries for x86_64. You can simply -download the latest `windmill` binary using the following set of bash commands. +### Kubernetes (Helm charts) ```bash -BINARY_NAME='windmill-amd64' # or windmill-ee-amd64 for the enterprise edition -LATEST_RELEASE=$(curl -L -s -H 'Accept: application/json' https://github.com/windmill-labs/windmill/releases/latest) -LATEST_VERSION=$(echo $LATEST_RELEASE | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/') -ARTIFACT_URL="https://github.com/windmill-labs/windmill/releases/download/$LATEST_VERSION/$BINARY_NAME" -wget "$ARTIFACT_URL" -O windmill +helm repo add windmill https://windmill-labs.github.io/windmill-helm-charts/ +helm install windmill-chart windmill/windmill --namespace=windmill --create-namespace ``` +See [windmill-helm-charts](https://github.com/windmill-labs/windmill-helm-charts) for configuration options. + +### Cloud providers + +Windmill works on AWS (EKS/ECS), GCP, Azure, Ubicloud, Fly.io, Render.com, Hetzner, Digital Ocean, and others. Rule of thumb: 1 worker per 1vCPU and 1-2 GB RAM. + ### OAuth, SSO & SMTP -Windmill Community Edition allows to configure the OAuth, SSO (including Google -Workspace SSO, Microsoft/Azure and Okta) directly from the UI in the superadmin -settings. Do note that there is a limit of 10 SSO users on the community -edition. +Configure OAuth and SSO (Google Workspace, Microsoft/Azure, Okta) directly from the superadmin UI. [See documentation](https://www.windmill.dev/docs/misc/setup_oauth). -[See documentation](https://www.windmill.dev/docs/misc/setup_oauth). +### License -### Commercial license +The Community Edition is free to use internally. For commercial redistribution or managed services, contact . See [LICENSE](./LICENSE) and [Pricing](https://www.windmill.dev/pricing) for details. -See the [LICENSE](https://github.com/windmill-labs/windmill/blob/main/LICENSE) -file for the full license text. +The "Community Edition" of Windmill available in the docker images hosted under ghcr.io/windmill-labs/windmill and the github binary releases contains the files under the AGPLv3 and Apache 2 sources but also includes proprietary and non-public code and features which are not open source and under the following terms: Windmill Labs, Inc. grants a right to use all the features of the "Community Edition" for free without restrictions other than the limits and quotas set in the software and a right to distribute the community edition as is but not to sell, resell, serve Windmill as a managed service, modify or wrap under any form without an explicit agreement. -The "Community Edition" of Windmill available in the docker images hosted under -ghcr.io/windmill-labs/windmill and the github binary releases contains the files -under the AGPLv3 and Apache 2 sources but also includes proprietary and -non-public code and features which are not open source and under the following -terms: Windmill Labs, Inc. grants a right to use all the features of the -"Community Edition" for free without restrictions other than the limits and -quotas set in the software and a right to distribute the community edition as is -but not to sell, resell, serve Windmill as a managed service, modify or wrap -under any form without an explicit agreement. +The binary compilable from source code in this repository without the "enterprise" feature flag is open-source under the [LICENSE-AGPLv3](https://github.com/windmill-labs/windmill/blob/main/LICENSE-AGPL) License terms and conditions. -The binary compilable from source code in this repository without the -"enterprise" feature flag is open-source under the -[LICENSE-AGPLv3](https://github.com/windmill-labs/windmill/blob/main/LICENSE-AGPL) -License terms and conditions. +To [re-expose directly any Windmill parts to your users](https://www.windmill.dev/docs/misc/white_labelling) as a feature of your product, with the exception of iframed public Windmill "apps", or to build a feature on top of "Windmill Community Edition" that you sell commercially or embed in a distributable product or binary, you must get a commercial license. Contact us at if you have any questions. To do the same from the binary compiled from the source code in this repository without the "enterprise" feature flag, you must comply with the AGPLv3 license terms and conditions or get a commercial license from Windmill Labs, Inc. -To -[re-expose directly any Windmill parts to your users](https://www.windmill.dev/docs/misc/white_labelling) -as a feature of your product, with the exception of iframed public Windmill -"apps", or to build a feature on top of "Windmill Community Edition" that you -sell commercially or embed in a distributable product or binary, you must get a -commercial license. Contact us at if you have any -questions. To do the same from the binary compiled from the source code in this -repository without the "enterprise" feature flag, you must comply with the -AGPLv3 license terms and conditions or get a commercial license from Windmill -Labs, Inc. - -To use Windmill "Community Edition" as is internally in your organization, or to -use its APIs as is, you do NOT need a commercial license. +To use Windmill "Community Edition" as is internally in your organization, or to use its APIs as is, you do NOT need a commercial license. ### Integrations -In Windmill, integrations are referred to as -[resources and resource types](https://www.windmill.dev/docs/core_concepts/resources_and_types). -Each Resource has a Resource Type that defines the schema that the resource +In Windmill, integrations are referred to as [resources and resource types](https://www.windmill.dev/docs/core_concepts/resources_and_types). Each Resource has a Resource Type that defines the schema that the resource needs to implement. -On self-hosted instances, you might want to import all the approved resource -types from [WindmillHub](https://hub.windmill.dev). A setup script will prompt -you to have it being synced automatically everyday. +On self-hosted instances, you might want to import all the approved resource types from [WindmillHub](https://hub.windmill.dev). A setup script will prompt you to have it being synced automatically everyday. ## Environment Variables @@ -369,30 +284,20 @@ you to have it being synced automatically everyday. ## Run a local dev setup -Using [Nix](./frontend/README_DEV.md#nix) (Recommended). +We recommend using [Nix](./frontend/README_DEV.md#nix). See [./frontend/README_DEV.md](./frontend/README_DEV.md) for all options. -See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all -running options. +### Frontend only -### only Frontend +Uses the backend of with local frontend (hot-reload): -This will use the backend of but your own frontend -with hot-code reloading. Note that you will need to use a username / password -login due to CSRF checks using a different auth provider. - -In the `frontend/` directory: - -1. install the dependencies with `npm install` (or `pnpm install` or `yarn`) -2. generate the windmill client: - -``` -npm run generate-backend-client -## on mac use -npm run generate-backend-client-mac +```bash +cd frontend +npm install +npm run generate-backend-client # or generate-backend-client-mac on Mac +npm run dev ``` -3. Run your dev server with `npm run dev` -4. Et voilà, windmill should be available at `http://localhost/` +Windmill available at `http://localhost/` ### Backend + Frontend @@ -419,7 +324,7 @@ running options. 6. Go to `backend/`: 1. `env DATABASE_URL= RUST_LOG=info cargo run` 2. You can specify any feature flag you want to enable, for example `cargo run --features python` to enable the python executor. -7. Et voilà, windmill should be available at `http://localhost:3000` +7. Windmill should be available at `http://localhost:3000` ## Contributors @@ -429,4 +334,4 @@ running options. ## Copyright -Windmill Labs, Inc 2023 +© 2023-2026 Windmill Labs, Inc. diff --git a/backend/.sqlx/query-3b6bd7b41f130ce6df62fdecb351a3e01be0726d02d3f863e0ea5c476a8e785e.json b/backend/.sqlx/query-3b6bd7b41f130ce6df62fdecb351a3e01be0726d02d3f863e0ea5c476a8e785e.json deleted file mode 100644 index fa18a36f44..0000000000 --- a/backend/.sqlx/query-3b6bd7b41f130ce6df62fdecb351a3e01be0726d02d3f863e0ea5c476a8e785e.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE kafka_trigger \n SET \n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $11,\n error_handler_args = $12,\n retry = $13\n WHERE \n workspace_id = $9 AND path = $10\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "VarcharArray", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "3b6bd7b41f130ce6df62fdecb351a3e01be0726d02d3f863e0ea5c476a8e785e" -} diff --git a/backend/.sqlx/query-46f6a3665d11ef573e27ab14be8a80c78d7eb735bc9615ebf2668a5715d83349.json b/backend/.sqlx/query-46f6a3665d11ef573e27ab14be8a80c78d7eb735bc9615ebf2668a5715d83349.json new file mode 100644 index 0000000000..c0dfdfac70 --- /dev/null +++ b/backend/.sqlx/query-46f6a3665d11ef573e27ab14be8a80c78d7eb735bc9615ebf2668a5715d83349.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (large_file_storage IS NOT NULL\n AND large_file_storage != 'null'::jsonb\n AND jsonb_typeof(large_file_storage) = 'object') AS \"has_primary!\"\n FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_primary!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "46f6a3665d11ef573e27ab14be8a80c78d7eb735bc9615ebf2668a5715d83349" +} diff --git a/backend/.sqlx/query-a750630d79166f5b6d5be0e049c36135212666ce74dbf290d701b6402b800f13.json b/backend/.sqlx/query-a750630d79166f5b6d5be0e049c36135212666ce74dbf290d701b6402b800f13.json new file mode 100644 index 0000000000..ebf92515e1 --- /dev/null +++ b/backend/.sqlx/query-a750630d79166f5b6d5be0e049c36135212666ce74dbf290d701b6402b800f13.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n jsonb_strip_nulls(jsonb_build_object(\n 'path', path,\n 'kind', kind,\n 'access_type', usage_access_type,\n 'columns', columns\n )) as \"list!: _\"\n FROM asset\n WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3\n ORDER BY path, kind", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "list!: _", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + } + ] + }, + "nullable": [ + null + ] + }, + "hash": "a750630d79166f5b6d5be0e049c36135212666ce74dbf290d701b6402b800f13" +} diff --git a/backend/.sqlx/query-a9e29764b5b9d94269e2b8aa755c71b61774c8ff8ae218d7a8d6ed0ac0169366.json b/backend/.sqlx/query-a9e29764b5b9d94269e2b8aa755c71b61774c8ff8ae218d7a8d6ed0ac0169366.json new file mode 100644 index 0000000000..b39c1b5b31 --- /dev/null +++ b/backend/.sqlx/query-a9e29764b5b9d94269e2b8aa755c71b61774c8ff8ae218d7a8d6ed0ac0169366.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns)\n VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable" + ] + } + } + }, + { + "Custom": { + "name": "asset_access_type", + "kind": { + "Enum": [ + "r", + "w", + "rw" + ] + } + } + }, + "Varchar", + { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + }, + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "a9e29764b5b9d94269e2b8aa755c71b61774c8ff8ae218d7a8d6ed0ac0169366" +} diff --git a/backend/.sqlx/query-1de3e078c108a8a0136fccdf9187cc3500260bac39c7f1dfa05a93628569465b.json b/backend/.sqlx/query-aed5439aa6dad950e505f9f8f6914fa5ca21319c501b2822c9bc751ddfc9a0a4.json similarity index 60% rename from backend/.sqlx/query-1de3e078c108a8a0136fccdf9187cc3500260bac39c7f1dfa05a93628569465b.json rename to backend/.sqlx/query-aed5439aa6dad950e505f9f8f6914fa5ca21319c501b2822c9bc751ddfc9a0a4.json index 18b3bc3a0e..1d165a0d13 100644 --- a/backend/.sqlx/query-1de3e078c108a8a0136fccdf9187cc3500260bac39c7f1dfa05a93628569465b.json +++ b/backend/.sqlx/query-aed5439aa6dad950e505f9f8f6914fa5ca21319c501b2822c9bc751ddfc9a0a4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n script_path,\n is_flow,\n mode,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now(), $11, $12, $13\n )\n ", + "query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n script_path,\n is_flow,\n mode,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), $12, $13, $14\n )\n ", "describe": { "columns": [], "parameters": { @@ -10,6 +10,7 @@ "Varchar", "Varchar", "VarcharArray", + "JsonbArray", "Varchar", "Bool", { @@ -33,5 +34,5 @@ }, "nullable": [] }, - "hash": "1de3e078c108a8a0136fccdf9187cc3500260bac39c7f1dfa05a93628569465b" + "hash": "aed5439aa6dad950e505f9f8f6914fa5ca21319c501b2822c9bc751ddfc9a0a4" } diff --git a/backend/.sqlx/query-e2921e44c70cf6c76c55177f2b56985e84c59ecb3e1a13fcf27d5f7ae5f8d84c.json b/backend/.sqlx/query-e2921e44c70cf6c76c55177f2b56985e84c59ecb3e1a13fcf27d5f7ae5f8d84c.json new file mode 100644 index 0000000000..ccf11a0ac1 --- /dev/null +++ b/backend/.sqlx/query-e2921e44c70cf6c76c55177f2b56985e84c59ecb3e1a13fcf27d5f7ae5f8d84c.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE\n workspace_id = $10 AND path = $11\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "VarcharArray", + "JsonbArray", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "e2921e44c70cf6c76c55177f2b56985e84c59ecb3e1a13fcf27d5f7ae5f8d84c" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b917f87599..aba370eb85 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -234,9 +234,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d03449bb8ca2cc2ef70869af31463d1ae5ccc8fa3e334b307203fbf815207e" +checksum = "9ded5f9a03ac8f24d1b8a25101ee812cd32cdc8c50a4c50237de2c4915850e73" dependencies = [ "rustversion", ] @@ -490,7 +490,7 @@ dependencies = [ "memchr", "num", "regex", - "regex-syntax 0.8.8", + "regex-syntax 0.8.9", ] [[package]] @@ -2024,9 +2024,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" dependencies = [ "serde", ] @@ -2287,9 +2287,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.56" +version = "4.5.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75ca66430e33a14957acc24c5077b503e7d374151b2b4b3a10c83b4ceb4be0e" +checksum = "6899ea499e3fb9305a65d5ebf6e3d2248c5fab291f300ad0a704fbe142eae31a" dependencies = [ "clap_builder", "clap_derive", @@ -2297,9 +2297,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.56" +version = "4.5.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793207c7fa6300a0608d1080b858e5fdbe713cdc1c8db9fb17777d8a13e63df0" +checksum = "7b12c8b680195a62a8364d16b8447b01b6c2c8f9aaf68bee653be34d4245e238" dependencies = [ "anstream", "anstyle", @@ -3471,7 +3471,7 @@ dependencies = [ "log", "recursive", "regex", - "regex-syntax 0.8.8", + "regex-syntax 0.8.9", ] [[package]] @@ -5478,7 +5478,7 @@ checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" dependencies = [ "bit-set 0.8.0", "regex-automata", - "regex-syntax 0.8.8", + "regex-syntax 0.8.9", ] [[package]] @@ -5489,7 +5489,7 @@ checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" dependencies = [ "bit-set 0.8.0", "regex-automata", - "regex-syntax 0.8.8", + "regex-syntax 0.8.9", ] [[package]] @@ -5610,9 +5610,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "libz-sys", @@ -5631,9 +5631,9 @@ dependencies = [ [[package]] name = "float8" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f463a8a37ede13dac13316d1a1eeafa992300906a0c4c7fa1177f366d10bcbf" +checksum = "719a903cc23e4a89e87962c2a80fdb45cdaad0983a89bd150bb57b4c8571a7d5" dependencies = [ "half", "num-traits", @@ -6183,7 +6183,7 @@ dependencies = [ "bstr", "log", "regex-automata", - "regex-syntax 0.8.8", + "regex-syntax 0.8.9", ] [[package]] @@ -7008,14 +7008,13 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", - "futures-core", "futures-util", "http 1.4.0", "http-body 1.0.1", @@ -10888,32 +10887,32 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax 0.8.8", + "regex-syntax 0.8.9", ] [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.8", + "regex-syntax 0.8.9", ] [[package]] name = "regex-lite" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" [[package]] name = "regex-syntax" @@ -10923,9 +10922,9 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" [[package]] name = "relative-path" @@ -13468,9 +13467,9 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ "bitflags 2.9.4", "core-foundation 0.9.4", @@ -13600,7 +13599,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" dependencies = [ "byteorder", - "regex-syntax 0.8.8", + "regex-syntax 0.8.9", "utf8-ranges", ] @@ -14533,7 +14532,7 @@ checksum = "0203df02a3b6dd63575cc1d6e609edc2181c9a11867a271b25cfd2abff3ec5ca" dependencies = [ "cc", "regex", - "regex-syntax 0.8.8", + "regex-syntax 0.8.9", "tree-sitter-language", ] @@ -15488,7 +15487,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "aws-sdk-config", @@ -15528,6 +15527,7 @@ dependencies = [ "sqlx", "strum 0.27.2", "systemstat", + "tempfile", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "tikv-jemallocator", @@ -15551,7 +15551,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "argon2", @@ -15683,7 +15683,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.623.1" +version = "1.624.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15693,7 +15693,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.623.1" +version = "1.624.0" dependencies = [ "chrono", "lazy_static", @@ -15707,7 +15707,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -15726,7 +15726,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "async-recursion", @@ -15822,7 +15822,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.623.1" +version = "1.624.0" dependencies = [ "regex", "serde", @@ -15837,7 +15837,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15861,7 +15861,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.623.1" +version = "1.624.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15877,7 +15877,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "async-trait", @@ -15897,7 +15897,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "async-oauth2", @@ -15921,7 +15921,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.623.1" +version = "1.624.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15930,7 +15930,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "lazy_static", @@ -15942,7 +15942,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "serde_json", @@ -15954,7 +15954,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "gosyn", @@ -15966,7 +15966,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "lazy_static", @@ -15978,7 +15978,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "serde_json", @@ -15990,7 +15990,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "nu-parser", @@ -16001,7 +16001,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16012,7 +16012,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16025,7 +16025,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "async-recursion", @@ -16049,7 +16049,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "lazy_static", @@ -16063,7 +16063,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16080,7 +16080,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "lazy_static", @@ -16094,7 +16094,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "lazy_static", @@ -16113,7 +16113,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "serde", @@ -16124,7 +16124,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "async-recursion", @@ -16161,7 +16161,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.623.1" +version = "1.624.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -16171,7 +16171,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.623.1" +version = "1.624.0" dependencies = [ "anyhow", "async-once-cell", @@ -17072,18 +17072,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.37" +version = "0.8.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7456cf00f0685ad319c5b1693f291a650eaf345e941d082fc4e03df8a03996ac" +checksum = "57cf3aa6855b23711ee9852dfc97dfaa51c45feaba5b645d0c777414d494a961" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.37" +version = "0.8.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1328722bbf2115db7e19d69ebcc15e795719e2d66b60827c6a69a117365e37a0" +checksum = "8a616990af1a287837c4fe6596ad77ef57948f787e46ce28e166facc0cc1cb75" dependencies = [ "proc-macro2", "quote", @@ -17178,9 +17178,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.5" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" +checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c" [[package]] name = "zstd" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f934bb3c59..c9f11920b0 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.623.1" +version = "1.624.0" authors.workspace = true edition.workspace = true @@ -35,7 +35,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.623.1" +version = "1.624.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -90,6 +90,7 @@ zip = ["windmill-api/zip"] static_frontend = ["windmill-api/static_frontend"] scoped_cache = ["windmill-common/scoped_cache"] test_job_debouncing = [] +private_registry_test = [] # Languages python = ["windmill-worker/python", "windmill-api/python"] rust = ["windmill-worker/rust"] @@ -182,6 +183,7 @@ axum.workspace = true serde.workspace = true windmill-api-client.workspace = true deno_core = { workspace = true, features = ["include_js_files_for_snapshotting", "unsafe_use_unprotected_platform"] } +tempfile.workspace = true [workspace.dependencies] diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 6eb01f8fe8..153181e90c 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -138a4f5f868f3bded5bb7cb77b222b532c07e4af +88e49a7c9746080a8a95e30828655d5783a616d6 diff --git a/backend/migrations/20260130150931_kafka_trigger_filters.down.sql b/backend/migrations/20260130150931_kafka_trigger_filters.down.sql new file mode 100644 index 0000000000..b416e770a1 --- /dev/null +++ b/backend/migrations/20260130150931_kafka_trigger_filters.down.sql @@ -0,0 +1 @@ +ALTER TABLE kafka_trigger DROP COLUMN filters; diff --git a/backend/migrations/20260130150931_kafka_trigger_filters.up.sql b/backend/migrations/20260130150931_kafka_trigger_filters.up.sql new file mode 100644 index 0000000000..04fe70af7e --- /dev/null +++ b/backend/migrations/20260130150931_kafka_trigger_filters.up.sql @@ -0,0 +1 @@ +ALTER TABLE kafka_trigger ADD COLUMN filters JSONB[] NOT NULL DEFAULT '{}'; diff --git a/backend/migrations/20260203122047_asset_columns.down.sql b/backend/migrations/20260203122047_asset_columns.down.sql new file mode 100644 index 0000000000..a97fc56c19 --- /dev/null +++ b/backend/migrations/20260203122047_asset_columns.down.sql @@ -0,0 +1,2 @@ +-- Remove columns field from asset table +ALTER TABLE asset DROP COLUMN columns; diff --git a/backend/migrations/20260203122047_asset_columns.up.sql b/backend/migrations/20260203122047_asset_columns.up.sql new file mode 100644 index 0000000000..2133b8da7d --- /dev/null +++ b/backend/migrations/20260203122047_asset_columns.up.sql @@ -0,0 +1,3 @@ +-- Add columns field to asset table to store column-level access information +-- This is a JSONB map of column name to access type (r, w, or rw) +ALTER TABLE asset ADD COLUMN columns JSONB; diff --git a/backend/migrations/20260203172950_polling_based_events.down.sql b/backend/migrations/20260203172950_polling_based_events.down.sql new file mode 100644 index 0000000000..ef95991b98 --- /dev/null +++ b/backend/migrations/20260203172950_polling_based_events.down.sql @@ -0,0 +1,121 @@ +-- Revert to pg_notify based event system + +-- Restore notify_config_change function +CREATE OR REPLACE FUNCTION notify_config_change() +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('notify_config_change', NEW.name::text); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Restore notify_global_setting_change function +CREATE OR REPLACE FUNCTION notify_global_setting_change() +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('notify_global_setting_change', NEW.name::text); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Restore notify_global_setting_delete function +CREATE OR REPLACE FUNCTION notify_global_setting_delete() +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('notify_global_setting_change', OLD.name::text); + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +-- Restore notify_webhook_change function +CREATE OR REPLACE FUNCTION notify_webhook_change() +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('notify_webhook_change', NEW.workspace_id); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Restore notify_workspace_envs_change function +CREATE OR REPLACE FUNCTION notify_workspace_envs_change() +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('notify_workspace_envs_change', NEW.workspace_id); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Restore notify_workspace_premium_change function +CREATE OR REPLACE FUNCTION notify_workspace_premium_change() +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('notify_workspace_premium_change', NEW.id); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Restore notify_team_plan_status_change function +CREATE OR REPLACE FUNCTION notify_team_plan_status_change() +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('notify_workspace_premium_change', NEW.workspace_id); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Restore notify_runnable_version_change function +CREATE OR REPLACE FUNCTION notify_runnable_version_change() +RETURNS TRIGGER AS $$ +DECLARE + source_type TEXT; + kind TEXT; +BEGIN + source_type := TG_ARGV[0]; + + IF source_type = 'script' THEN + kind := NEW.kind; + ELSE + kind := 'flow'; + END IF; + + PERFORM pg_notify('notify_runnable_version_change', NEW.workspace_id || ':' || source_type || ':' || NEW.path || ':' || kind); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Restore notify_http_trigger_change function +CREATE OR REPLACE FUNCTION notify_http_trigger_change() +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('notify_http_trigger_change', NEW.workspace_id || ':' || NEW.path); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Restore notify_token_invalidation function +CREATE OR REPLACE FUNCTION notify_token_invalidation() +RETURNS TRIGGER AS $$ +BEGIN + IF OLD.label = 'session' AND OLD.email IS NOT NULL THEN + PERFORM pg_notify('notify_token_invalidation', OLD.token); + END IF; + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +-- Restore notify_workspace_key_change function +CREATE OR REPLACE FUNCTION notify_workspace_key_change() +RETURNS TRIGGER AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + PERFORM pg_notify('notify_workspace_key_change', OLD.workspace_id); + RETURN OLD; + ELSE + PERFORM pg_notify('notify_workspace_key_change', NEW.workspace_id); + RETURN NEW; + END IF; +END; +$$ LANGUAGE plpgsql; + +-- Drop the notify_event table +DROP TABLE IF EXISTS notify_event; diff --git a/backend/migrations/20260203172950_polling_based_events.up.sql b/backend/migrations/20260203172950_polling_based_events.up.sql new file mode 100644 index 0000000000..e85597a689 --- /dev/null +++ b/backend/migrations/20260203172950_polling_based_events.up.sql @@ -0,0 +1,135 @@ +-- Create notify_event table for polling-based event system +-- This replaces PostgreSQL LISTEN/NOTIFY with a table-based approach + +CREATE TABLE IF NOT EXISTS notify_event ( + id BIGSERIAL PRIMARY KEY, + channel TEXT NOT NULL, + payload TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS notify_event_created_at_idx ON notify_event (created_at); + +-- Drop redundant index if it exists (id is already the PRIMARY KEY) +DROP INDEX IF EXISTS notify_event_id_idx; + +-- Update notify_config_change function +CREATE OR REPLACE FUNCTION notify_config_change() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO notify_event (channel, payload) VALUES ('notify_config_change', NEW.name::text); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Update notify_global_setting_change function +CREATE OR REPLACE FUNCTION notify_global_setting_change() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO notify_event (channel, payload) VALUES ('notify_global_setting_change', NEW.name::text); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Update notify_global_setting_delete function +CREATE OR REPLACE FUNCTION notify_global_setting_delete() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO notify_event (channel, payload) VALUES ('notify_global_setting_change', OLD.name::text); + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +-- Update notify_webhook_change function +CREATE OR REPLACE FUNCTION notify_webhook_change() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO notify_event (channel, payload) VALUES ('notify_webhook_change', NEW.workspace_id); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Update notify_workspace_envs_change function +CREATE OR REPLACE FUNCTION notify_workspace_envs_change() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO notify_event (channel, payload) VALUES ('notify_workspace_envs_change', COALESCE(NEW.workspace_id, OLD.workspace_id)); + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql; + +-- Update notify_workspace_premium_change function +CREATE OR REPLACE FUNCTION notify_workspace_premium_change() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO notify_event (channel, payload) VALUES ('notify_workspace_premium_change', NEW.id); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Update notify_team_plan_status_change function +CREATE OR REPLACE FUNCTION notify_team_plan_status_change() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO notify_event (channel, payload) VALUES ('notify_workspace_premium_change', NEW.workspace_id); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Update notify_runnable_version_change function +CREATE OR REPLACE FUNCTION notify_runnable_version_change() +RETURNS TRIGGER AS $$ +DECLARE + source_type TEXT; + kind TEXT; +BEGIN + source_type := TG_ARGV[0]; + + IF source_type = 'script' THEN + kind := NEW.kind; + ELSE + kind := 'flow'; + END IF; + + INSERT INTO notify_event (channel, payload) VALUES ('notify_runnable_version_change', NEW.workspace_id || ':' || source_type || ':' || NEW.path || ':' || kind); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Update notify_http_trigger_change function +CREATE OR REPLACE FUNCTION notify_http_trigger_change() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO notify_event (channel, payload) VALUES ('notify_http_trigger_change', COALESCE(NEW.workspace_id, OLD.workspace_id) || ':' || COALESCE(NEW.path, OLD.path)); + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql; + +-- Update notify_token_invalidation function +CREATE OR REPLACE FUNCTION notify_token_invalidation() +RETURNS TRIGGER AS $$ +BEGIN + IF OLD.label = 'session' AND OLD.email IS NOT NULL THEN + INSERT INTO notify_event (channel, payload) VALUES ('notify_token_invalidation', OLD.token); + END IF; + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +-- Update notify_workspace_key_change function +CREATE OR REPLACE FUNCTION notify_workspace_key_change() +RETURNS TRIGGER AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + INSERT INTO notify_event (channel, payload) VALUES ('notify_workspace_key_change', OLD.workspace_id); + RETURN OLD; + ELSE + INSERT INTO notify_event (channel, payload) VALUES ('notify_workspace_key_change', NEW.workspace_id); + RETURN NEW; + END IF; +END; +$$ LANGUAGE plpgsql; + +-- NOTE: var_cache_invalidation / resource_cache_invalidation triggers were +-- intentionally dropped in migration 20250902085504. We do NOT re-create them +-- here to keep this migration scoped to the LISTEN/NOTIFY → polling swap only. diff --git a/backend/parsers/windmill-parser-py/src/asset_parser.rs b/backend/parsers/windmill-parser-py/src/asset_parser.rs index f94c09ac7f..3818eb2a1b 100644 --- a/backend/parsers/windmill-parser-py/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-py/src/asset_parser.rs @@ -19,9 +19,12 @@ pub fn parse_assets(input: &str) -> anyhow::Result { // if a db = wmill.datatable() was never used (e.g db.query(...)), // we still want to register the asset as unknown access type if asset_was_used(&assets_finder.assets, (kind, &path)) == false { - assets_finder - .assets - .push(ParseAssetsResult { kind, access_type: None, path }); + assets_finder.assets.push(ParseAssetsResult { + kind, + path, + access_type: None, + columns: None, + }); } } @@ -48,8 +51,12 @@ impl Visitor for AssetsFinder { match removed { Some((kind, path, _)) => { if !asset_was_used(&self.assets, (kind, &path)) { - self.assets - .push(ParseAssetsResult { kind, access_type: None, path }); + self.assets.push(ParseAssetsResult { + kind, + path, + access_type: None, + columns: None, + }); } } None => {} @@ -76,6 +83,7 @@ impl Visitor for AssetsFinder { kind, path: path.to_string(), access_type: None, + columns: None, }); } } @@ -97,6 +105,7 @@ impl Visitor for AssetsFinder { kind, path: path.to_string(), access_type: None, + columns: None, }); } } @@ -252,8 +261,12 @@ impl AssetsFinder { let path = parse_asset_syntax(&value, false) .map(|(_, p)| p) .unwrap_or(&value); - self.assets - .push(ParseAssetsResult { kind, path: path.to_string(), access_type }); + self.assets.push(ParseAssetsResult { + kind, + path: path.to_string(), + access_type, + columns: None, + }); } _ => return Err(()), }; @@ -281,7 +294,8 @@ def main(): Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, path: "/test.csv".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, },]) ); } @@ -299,7 +313,8 @@ def main(): Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "main".to_string(), - access_type: None + access_type: None, + columns: None, },]) ); } @@ -318,7 +333,8 @@ def main(x: int): Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/friends".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, },]) ); } @@ -340,12 +356,14 @@ def main(x: int): ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/analytics".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, }, ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/friends".to_string(), - access_type: Some(RW) + access_type: Some(RW), + columns: None, }, ]) ); @@ -372,17 +390,20 @@ def g(): ParseAssetsResult { kind: AssetKind::DataTable, path: "another1/customers".to_string(), - access_type: Some(W) + access_type: Some(W), + columns: None, }, ParseAssetsResult { kind: AssetKind::Ducklake, path: "another2".to_string(), - access_type: None + access_type: None, + columns: None, }, ParseAssetsResult { kind: AssetKind::DataTable, path: "main/friends".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, }, ]) ); @@ -404,12 +425,14 @@ def g(): ParseAssetsResult { kind: AssetKind::DataTable, path: "another1".to_string(), - access_type: None + access_type: None, + columns: None, }, ParseAssetsResult { kind: AssetKind::Ducklake, path: "main".to_string(), - access_type: None + access_type: None, + columns: None, }, ]) ); @@ -429,7 +452,8 @@ def main(x: int): Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/public.friends".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, },]) ); } @@ -448,7 +472,8 @@ def main(): Ok(vec![ParseAssetsResult { kind: AssetKind::Ducklake, path: "lake1/analytics.metrics".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, },]) ); } @@ -468,7 +493,8 @@ def main(x: int): Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/public.users".to_string(), - access_type: Some(RW) + access_type: Some(RW), + columns: None, },]) ); } @@ -486,7 +512,8 @@ def main(): Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt".to_string(), - access_type: None + access_type: None, + columns: None, },]) ); } diff --git a/backend/parsers/windmill-parser-sql/src/asset_parser.rs b/backend/parsers/windmill-parser-sql/src/asset_parser.rs index 489f9c661f..51b90b379a 100644 --- a/backend/parsers/windmill-parser-sql/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql/src/asset_parser.rs @@ -1,9 +1,9 @@ -use std::collections::HashMap; +use std::collections::BTreeMap; use sqlparser::{ ast::{ - CopyTarget, Expr, ObjectName, TableFactor, TableObject, Value, ValueWithSpan, Visit, - Visitor, + CopyTarget, Expr, ObjectName, ObjectNamePart, SelectItem, TableFactor, TableObject, Value, + ValueWithSpan, Visit, Visitor, }, dialect::DuckDbDialect, parser::Parser, @@ -24,9 +24,12 @@ pub fn parse_assets(input: &str) -> anyhow::Result { for (_, (kind, path)) in collector.var_identifiers { if !asset_was_used(&collector.assets, (kind, &path)) { - collector - .assets - .push(ParseAssetsResult { kind, access_type: None, path: path }); + collector.assets.push(ParseAssetsResult { + kind, + access_type: None, + path: path, + columns: None, + }); } } @@ -39,7 +42,7 @@ struct AssetCollector { // e.g set to Read when we are inside a SELECT ... FROM ... statement current_access_type_stack: Vec, // e.g ATTACH 'ducklake://a' AS dl; => { "dl": (Ducklake, "a") } - var_identifiers: HashMap, + var_identifiers: BTreeMap, // e.g USE dl; currently_used_asset: Option<(AssetKind, String)>, } @@ -49,15 +52,19 @@ impl AssetCollector { Self { assets: Vec::new(), current_access_type_stack: Vec::with_capacity(8), - var_identifiers: HashMap::new(), + var_identifiers: BTreeMap::new(), currently_used_asset: None, } } // Detect when we do 'a.b' and 'a' is associated with an asset in var_identifiers // Or when we access 'b' and we did USE a; - fn get_associated_asset_from_obj_name(&self, name: &ObjectName) -> Option { - let access_type = self.current_access_type_stack.last().copied(); + fn get_associated_asset_from_obj_name( + &self, + name: &ObjectName, + access_type: Option, + ) -> Option { + let access_type = access_type.or_else(|| self.current_access_type_stack.last().copied()); if let Some((kind, path)) = &self.currently_used_asset { // We don't want to infer that any simple identifier refers to an asset if // we are not in a known R/W context @@ -81,7 +88,7 @@ impl AssetCollector { .collect::>>()? .join("."); let path = format!("{}/{}", path, specific_table); - return Some(ParseAssetsResult { kind: *kind, access_type, path }); + return Some(ParseAssetsResult { kind: *kind, access_type, path, columns: None }); } } @@ -101,7 +108,7 @@ impl AssetCollector { } else { path.clone() }; - Some(ParseAssetsResult { kind: *kind, access_type, path }) + Some(ParseAssetsResult { kind: *kind, access_type, path, columns: None }) } fn handle_string_literal(&mut self, s: &str) { @@ -112,6 +119,7 @@ impl AssetCollector { kind, path: path.to_string(), access_type: self.current_access_type_stack.last().copied(), + columns: None, }); } } @@ -126,13 +134,6 @@ impl AssetCollector { if let Some(str_lit) = get_str_lit_from_obj_name(name) { self.handle_string_literal(str_lit); } - - // Writes to tables should be handled directly when visiting the statement - if self.current_access_type_stack.last() == Some(&R) { - if let Some(asset) = self.get_associated_asset_from_obj_name(name) { - self.assets.push(asset); - } - } } fn handle_obj_name_post(&mut self, name: &ObjectName) { @@ -146,20 +147,144 @@ impl AssetCollector { } } - fn handle_table_with_joins(&mut self, table_with_joins: &sqlparser::ast::TableWithJoins) { - if let TableFactor::Table { name, .. } = &table_with_joins.relation { - if let Some(asset) = self.get_associated_asset_from_obj_name(name) { + fn handle_table_with_joins( + &mut self, + table_with_joins: &sqlparser::ast::TableWithJoins, + access_type: Option, + ) { + if let TableFactor::Table { name, args, .. } = &table_with_joins.relation { + if args.is_some() && args.as_ref().map_or(0, |a| a.args.len()) > 0 { + return; + } + if let Some(asset) = self.get_associated_asset_from_obj_name(name, access_type) { self.assets.push(asset); } } for join in &table_with_joins.joins { if let TableFactor::Table { name, .. } = &join.relation { - if let Some(asset) = self.get_associated_asset_from_obj_name(name) { + if let Some(asset) = self.get_associated_asset_from_obj_name(name, access_type) { self.assets.push(asset); } } } } + + // Extract columns from SELECT items and create individual asset results for each column + // Only processes columns that reference known assets to avoid false positives + fn extract_column_assets( + &mut self, + projection: &[SelectItem], + from_tables: &[sqlparser::ast::TableWithJoins], + ) { + // Check if this is a single-table SELECT (to avoid ambiguity) + let single_table = if from_tables.len() == 1 { + if let TableFactor::Table { name, args, .. } = &from_tables[0].relation { + if args.is_some() && args.as_ref().map_or(0, |a| a.args.len()) > 0 { + return; // Skip table functions + } + self.get_associated_asset_from_obj_name(name, Some(R)) + } else { + None + } + } else { + None + }; + + // Build a map of table aliases/names to assets for multi-table queries + let mut table_to_asset: BTreeMap = BTreeMap::new(); + for table_with_joins in from_tables { + if let TableFactor::Table { name, alias, args, .. } = &table_with_joins.relation { + if args.is_some() && args.as_ref().map_or(0, |a| a.args.len()) > 0 { + continue; // Skip table functions + } + if let Some(asset) = self.get_associated_asset_from_obj_name(name, Some(R)) { + // Use alias if present, otherwise use the table name + let table_key = if let Some(alias) = alias { + alias.name.value.clone() + } else { + // For qualified names like "dl.table1", use just the last part + name.0 + .last() + .and_then(|id| id.as_ident()) + .map(|id| id.value.clone()) + .unwrap_or_default() + }; + table_to_asset.insert(table_key, asset); + } + } + } + + // Process each SELECT item + for item in projection { + match item { + SelectItem::UnnamedExpr(Expr::Identifier(ident)) + | SelectItem::ExprWithAlias { expr: Expr::Identifier(ident), .. } => { + // Simple column: SELECT a + // Only add if we have a single table (unambiguous) + if let Some(asset) = &single_table { + let mut columns = BTreeMap::new(); + columns.insert(ident.value.clone(), R); + self.assets.push(ParseAssetsResult { + kind: asset.kind, + path: asset.path.clone(), + access_type: Some(R), + columns: Some(columns), + }); + } + } + SelectItem::UnnamedExpr(Expr::CompoundIdentifier(parts)) + | SelectItem::ExprWithAlias { expr: Expr::CompoundIdentifier(parts), .. } => { + // Qualified column: SELECT table1.a or SELECT x.table1.a + if parts.len() >= 2 { + let column_name = parts.last().map(|id| id.value.clone()); + + if let Some(column_name) = column_name { + // Check if the prefix matches a known table + let table_prefix = parts.first().map(|id| id.value.clone()); + + if let Some(table_prefix) = table_prefix { + if let Some(asset) = table_to_asset.get(&table_prefix) { + // Found a matching table, add column asset + let mut columns = BTreeMap::new(); + columns.insert(column_name.clone(), R); + self.assets.push(ParseAssetsResult { + kind: asset.kind, + path: asset.path.clone(), + access_type: Some(R), + columns: Some(columns), + }); + } else if parts.len() >= 3 { + // Could be x.table1.column format or db.schema.table.column + // Convert Idents to ObjectNameParts + let obj_parts: Vec = parts[..parts.len() - 1] + .iter() + .cloned() + .map(|ident| ObjectNamePart::Identifier(ident)) + .collect(); + let obj_name = ObjectName(obj_parts); + if let Some(asset) = + self.get_associated_asset_from_obj_name(&obj_name, Some(R)) + { + let mut columns = BTreeMap::new(); + columns.insert(column_name.clone(), R); + self.assets.push(ParseAssetsResult { + kind: asset.kind, + path: asset.path.clone(), + access_type: Some(R), + columns: Some(columns), + }); + } + } + } + } + } + } + _ => { + // Ignore wildcards, expressions, etc. + } + } + } + } } impl Visitor for AssetCollector { @@ -218,51 +343,154 @@ impl Visitor for AssetCollector { statement: &sqlparser::ast::Statement, ) -> std::ops::ControlFlow { match statement { - sqlparser::ast::Statement::Query(_) => { - // don't forget pop() in post_visit_statement - self.current_access_type_stack.push(R); + sqlparser::ast::Statement::Query(q) => { + if let Some(select) = q.body.as_select() { + // First, handle table references (adds table-level assets) + for t in &select.from { + self.handle_table_with_joins(t, Some(R)); + } + // Then, extract column-level assets + self.extract_column_assets(&select.projection, &select.from); + } } sqlparser::ast::Statement::Insert(insert) => { let access_type = if insert.returning.is_some() { RW } else { W }; - self.current_access_type_stack.push(access_type); match insert.table { TableObject::TableName(ref name) => { - if let Some(asset) = self.get_associated_asset_from_obj_name(name) { - self.assets.push(asset); + if let Some(asset) = + self.get_associated_asset_from_obj_name(name, Some(access_type)) + { + // Add table-level asset + self.assets.push(ParseAssetsResult { + kind: asset.kind, + path: asset.path.clone(), + access_type: asset.access_type, + columns: None, + }); + + // Extract column information for INSERT with explicit columns (Write access) + if !insert.columns.is_empty() { + for col in &insert.columns { + let columns = BTreeMap::from([(col.value.clone(), W)]); + self.assets.push(ParseAssetsResult { + kind: asset.kind, + path: asset.path.clone(), + access_type: Some(W), + columns: Some(columns), + }); + } + } + + // Extract column information from RETURNING clause (Read access) + if let Some(returning) = &insert.returning { + for item in returning { + match item { + SelectItem::UnnamedExpr(Expr::Identifier(ident)) + | SelectItem::ExprWithAlias { + expr: Expr::Identifier(ident), + .. + } => { + let mut col_map = BTreeMap::new(); + col_map.insert(ident.value.clone(), R); + self.assets.push(ParseAssetsResult { + kind: asset.kind, + path: asset.path.clone(), + access_type: Some(R), + columns: Some(col_map), + }); + } + _ => { + // Ignore wildcards and complex expressions + } + } + } + } } } _ => {} } - self.current_access_type_stack.pop(); } - sqlparser::ast::Statement::Update { returning, table, from, .. } => { + sqlparser::ast::Statement::Update { returning, table, from, assignments, .. } => { if let Some(from_tables) = from { let from_tables = match from_tables { sqlparser::ast::UpdateTableFromKind::AfterSet(tables) => tables, sqlparser::ast::UpdateTableFromKind::BeforeSet(tables) => tables, }; - self.current_access_type_stack.push(R); for table_with_joins in from_tables { - self.handle_table_with_joins(table_with_joins); + self.handle_table_with_joins(table_with_joins, Some(R)); } - self.current_access_type_stack.pop(); } let access_type = if returning.is_some() { RW } else { W }; - self.current_access_type_stack.push(access_type); + self.handle_table_with_joins(table, Some(access_type)); - self.handle_table_with_joins(table); + // Extract column information from UPDATE SET clauses (Write access) + // Only process if it's a single table update + if let TableFactor::Table { name, .. } = &table.relation { + if let Some(asset) = + self.get_associated_asset_from_obj_name(name, Some(access_type)) + { + // Process each assignment to extract column names + for assignment in assignments { + // assignment.target is an AssignmentTarget enum + // We only handle simple column names (ColumnName variant) + if let sqlparser::ast::AssignmentTarget::ColumnName(col_name) = + &assignment.target + { + // For simple column updates, this is typically a single ident + if col_name.0.len() == 1 { + if let Some(col_ident) = + col_name.0.first().and_then(|p| p.as_ident()) + { + let mut col_map = BTreeMap::new(); + col_map.insert(col_ident.value.clone(), W); + self.assets.push(ParseAssetsResult { + kind: asset.kind, + path: asset.path.clone(), + access_type: Some(W), + columns: Some(col_map), + }); + } + } + } + } - self.current_access_type_stack.pop(); + // Extract column information from RETURNING clause (Read access) + if let Some(returning_items) = returning { + for item in returning_items { + match item { + SelectItem::UnnamedExpr(Expr::Identifier(ident)) + | SelectItem::ExprWithAlias { + expr: Expr::Identifier(ident), + .. + } => { + let mut col_map = BTreeMap::new(); + col_map.insert(ident.value.clone(), R); + self.assets.push(ParseAssetsResult { + kind: asset.kind, + path: asset.path.clone(), + access_type: Some(R), + columns: Some(col_map), + }); + } + _ => { + // Ignore wildcards and complex expressions + } + } + } + } + } + } } sqlparser::ast::Statement::Delete(delete) => { let access_type = if delete.returning.is_some() { RW } else { W }; - self.current_access_type_stack.push(access_type); for name in &delete.tables { - if let Some(asset) = self.get_associated_asset_from_obj_name(name) { + if let Some(asset) = + self.get_associated_asset_from_obj_name(name, Some(access_type)) + { self.assets.push(asset); } } @@ -271,25 +499,22 @@ impl Visitor for AssetCollector { sqlparser::ast::FromTable::WithoutKeyword(tables) => tables, }; for table_with_joins in tables { - self.handle_table_with_joins(table_with_joins); + self.handle_table_with_joins(table_with_joins, Some(access_type)); } - self.current_access_type_stack.pop(); } sqlparser::ast::Statement::CreateTable(create_table) => { - self.current_access_type_stack.push(W); - if let Some(asset) = self.get_associated_asset_from_obj_name(&create_table.name) { + if let Some(asset) = + self.get_associated_asset_from_obj_name(&create_table.name, Some(W)) + { self.assets.push(asset); } - self.current_access_type_stack.pop(); } sqlparser::ast::Statement::CreateView { name, .. } => { - self.current_access_type_stack.push(W); - if let Some(asset) = self.get_associated_asset_from_obj_name(name) { + if let Some(asset) = self.get_associated_asset_from_obj_name(name, Some(W)) { self.assets.push(asset); } - self.current_access_type_stack.pop(); } sqlparser::ast::Statement::Copy { target: CopyTarget::File { filename }, .. } => { @@ -339,14 +564,8 @@ impl Visitor for AssetCollector { fn post_visit_statement( &mut self, - statement: &sqlparser::ast::Statement, + _statement: &sqlparser::ast::Statement, ) -> std::ops::ControlFlow { - match statement { - sqlparser::ast::Statement::Query(_) => { - self.current_access_type_stack.pop(); - } - _ => {} - } std::ops::ControlFlow::Continue(()) } @@ -409,17 +628,20 @@ mod tests { ParseAssetsResult { kind: AssetKind::S3Object, path: "/a.parquet".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None }, ParseAssetsResult { kind: AssetKind::S3Object, path: "/c.parquet".to_string(), - access_type: Some(W) + access_type: Some(W), + columns: None }, ParseAssetsResult { kind: AssetKind::S3Object, path: "snd/b.parquet".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None }, ]) ); @@ -438,7 +660,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::Ducklake, path: "my_dl".to_string(), - access_type: None + access_type: None, + columns: None },]) ); } @@ -455,7 +678,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::Ducklake, path: "my_dl/table1".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None },]) ); } @@ -473,7 +697,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "my_dt/table1".to_string(), - access_type: Some(W) + access_type: Some(W), + columns: None },]) ); } @@ -504,7 +729,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::Ducklake, path: "my_dl/table1".to_string(), - access_type: Some(W) + access_type: Some(W), + columns: None },]) ); } @@ -521,7 +747,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "main/table1".to_string(), - access_type: Some(W) + access_type: Some(W), + columns: None },]) ); } @@ -543,7 +770,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::Ducklake, path: "main/friends".to_string(), - access_type: Some(RW) + access_type: Some(RW), + columns: None },]) ); } @@ -561,7 +789,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::Ducklake, path: "main".to_string(), - access_type: None + access_type: None, + columns: None },]) ); } @@ -579,7 +808,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::Ducklake, path: "main/table1".to_string(), - access_type: Some(W) + access_type: Some(W), + columns: None },]) ); } @@ -597,7 +827,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::Ducklake, path: "main/table1".to_string(), - access_type: Some(W) + access_type: Some(W), + columns: Some(BTreeMap::from([("id".to_string(), W)])), },]) ); } @@ -615,7 +846,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::Resource, path: "u/user/pg_resource/table1".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None },]) ); } @@ -632,7 +864,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::Ducklake, path: "main/table1".to_string(), - access_type: Some(W) + access_type: Some(W), + columns: Some(BTreeMap::from([("id".to_string(), W)])), },]) ); } @@ -650,7 +883,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::Ducklake, path: "main/sch.table1".to_string(), - access_type: Some(RW) + access_type: Some(RW), + columns: Some(BTreeMap::from([("id".to_string(), W)])), },]) ); } @@ -669,8 +903,289 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::Ducklake, path: "main/sch.table1".to_string(), - access_type: Some(RW) + access_type: Some(RW), + columns: Some(BTreeMap::from([("id".to_string(), W)])), },]) ); } + + #[test] + fn test_sql_asset_parser_single_table_column_detection() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + SELECT a, b FROM dl.table1; + "#; + let s = parse_assets(input).map(|s| s.assets); + + let result = s.unwrap(); + + // Should have one asset with merged columns + assert_eq!(result.len(), 1); + assert_eq!(result[0].path, "my_dl/table1"); + assert_eq!(result[0].access_type, Some(R)); + + // Check that both columns are present in the merged asset + let columns = result[0].columns.as_ref().expect("Should have columns"); + assert_eq!(columns.len(), 2); + assert_eq!(columns.get("a"), Some(&R)); + assert_eq!(columns.get("b"), Some(&R)); + } + + #[test] + fn test_sql_asset_parser_explicit_table_prefix_columns() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + SELECT dl.table1.a, dl.table1.b FROM dl.table1; + "#; + let s = parse_assets(input).map(|s| s.assets); + + // Should detect columns with explicit table prefix + let result = s.unwrap(); + // Check we have the table asset + + // Check we have column assets + assert!(result.iter().any(|a| { + a.path == "my_dl/table1" + && a.columns + .as_ref() + .map_or(false, |cols| cols.contains_key("a")) + })); + assert!(result.iter().any(|a| { + a.path == "my_dl/table1" + && a.columns + .as_ref() + .map_or(false, |cols| cols.contains_key("b")) + })); + } + + #[test] + fn test_sql_asset_parser_multi_table_no_simple_columns() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + SELECT a, b FROM dl.table1, dl.table2; + "#; + let s = parse_assets(input).map(|s| s.assets); + + // Simple columns (a, b) should NOT be detected with multiple tables + // Only table-level assets should be present + let result = s.unwrap(); + + // Should have 2 table assets + assert_eq!(result.iter().filter(|a| a.columns.is_none()).count(), 2); + + // Should have NO column assets (ambiguous which table they belong to) + assert_eq!(result.iter().filter(|a| a.columns.is_some()).count(), 0); + } + + #[test] + fn test_sql_asset_parser_multi_table_with_qualified_columns() { + let input = r#" + ATTACH 'ducklake://my_dl1' AS dl1; + ATTACH 'ducklake://my_dl2' AS dl2; + SELECT table1.a, table2.b FROM dl1.table1, dl2.table2; + "#; + let s = parse_assets(input).map(|s| s.assets); + + // Qualified columns should be detected even with multiple tables + let result = s.unwrap(); + + // Check we have column assets for both tables + assert!(result.iter().any(|a| { + a.path == "my_dl1/table1" + && a.columns + .as_ref() + .map_or(false, |cols| cols.contains_key("a")) + })); + assert!(result.iter().any(|a| { + a.path == "my_dl2/table2" + && a.columns + .as_ref() + .map_or(false, |cols| cols.contains_key("b")) + })); + } + + #[test] + fn test_sql_asset_parser_use_with_simple_columns() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + USE dl; + SELECT a, b, c FROM table1; + "#; + let s = parse_assets(input).map(|s| s.assets); + + let result = s.unwrap(); + + // Should detect columns since it's a single table + assert!(result.iter().any(|a| { + a.path == "my_dl/table1" + && a.columns + .as_ref() + .map_or(false, |cols| cols.contains_key("a")) + })); + assert!(result.iter().any(|a| { + a.path == "my_dl/table1" + && a.columns + .as_ref() + .map_or(false, |cols| cols.contains_key("b")) + })); + assert!(result.iter().any(|a| { + a.path == "my_dl/table1" + && a.columns + .as_ref() + .map_or(false, |cols| cols.contains_key("c")) + })); + } + + #[test] + fn test_sql_asset_parser_wildcard_no_columns() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + SELECT * FROM dl.table1; + "#; + let s = parse_assets(input).map(|s| s.assets); + + let result = s.unwrap(); + + // Wildcard should NOT create column assets, only table asset + assert_eq!(result.len(), 1); + assert!(result[0].columns.is_none()); + } + + #[test] + fn test_sql_asset_parser_columns_with_alias() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + SELECT a AS column_a, b AS column_b FROM dl.table1; + "#; + let s = parse_assets(input).map(|s| s.assets); + + let result = s.unwrap(); + + // Should detect columns even when aliased + assert!(result.iter().any(|a| { + a.path == "my_dl/table1" + && a.columns + .as_ref() + .map_or(false, |cols| cols.contains_key("a")) + })); + assert!(result.iter().any(|a| { + a.path == "my_dl/table1" + && a.columns + .as_ref() + .map_or(false, |cols| cols.contains_key("b")) + })); + } + + #[test] + fn test_sql_asset_parser_columns_with_table_alias() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + SELECT t.a, t.b FROM dl.table1 AS t; + "#; + let s = parse_assets(input).map(|s| s.assets); + + let result = s.unwrap(); + + // Should detect columns using the table alias + assert!(result.iter().any(|a| { + a.path == "my_dl/table1" + && a.columns + .as_ref() + .map_or(false, |cols| cols.contains_key("a")) + })); + assert!(result.iter().any(|a| { + a.path == "my_dl/table1" + && a.columns + .as_ref() + .map_or(false, |cols| cols.contains_key("b")) + })); + } + + #[test] + fn test_sql_asset_parser_insert_with_columns() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + INSERT INTO dl.table1 (name, age, email) VALUES ('John', 30, 'john@example.com'); + "#; + let s = parse_assets(input).map(|s| s.assets); + + let result = s.unwrap(); + + // Should have one asset with merged columns + assert_eq!(result.len(), 1); + assert_eq!(result[0].path, "my_dl/table1"); + assert_eq!(result[0].access_type, Some(W)); + + // Check that all columns are present in the merged asset + let columns = result[0].columns.as_ref().expect("Should have columns"); + assert_eq!(columns.len(), 3); + assert_eq!(columns.get("name"), Some(&W)); + assert_eq!(columns.get("age"), Some(&W)); + assert_eq!(columns.get("email"), Some(&W)); + } + + #[test] + fn test_sql_asset_parser_insert_without_columns() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + INSERT INTO dl.table1 VALUES ('John', 30); + "#; + let s = parse_assets(input).map(|s| s.assets); + + let result = s.unwrap(); + + // Should have one asset without column information + assert_eq!(result.len(), 1); + assert_eq!(result[0].path, "my_dl/table1"); + assert_eq!(result[0].access_type, Some(W)); + assert!(result[0].columns.is_none()); + } + + #[test] + fn test_sql_asset_parser_update_multiple_columns() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + UPDATE dl.table1 SET name = 'Jane', age = 25, active = true; + "#; + let s = parse_assets(input).map(|s| s.assets); + + let result = s.unwrap(); + + // Should have one asset with merged columns + assert_eq!(result.len(), 1); + assert_eq!(result[0].path, "my_dl/table1"); + assert_eq!(result[0].access_type, Some(W)); + + // Check that all columns are present + let columns = result[0].columns.as_ref().expect("Should have columns"); + assert_eq!(columns.len(), 3); + assert_eq!(columns.get("name"), Some(&W)); + assert_eq!(columns.get("age"), Some(&W)); + assert_eq!(columns.get("active"), Some(&W)); + } + + #[test] + fn test_sql_asset_parser_update_returning() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + UPDATE dl.table1 SET name = 'Jane', age = 26 RETURNING id, name; + "#; + let s = parse_assets(input).map(|s| s.assets); + + let result = s.unwrap(); + + // Should have RW access type when RETURNING is used + assert_eq!(result.len(), 1); + assert_eq!(result[0].path, "my_dl/table1"); + assert_eq!(result[0].access_type, Some(RW)); + + // Check that columns are present with correct access types + // name and age are written (W), id and name are read (R) + // name should be RW (both written and read) + let columns = result[0].columns.as_ref().expect("Should have columns"); + assert_eq!(columns.len(), 3); + assert_eq!(columns.get("name"), Some(&RW)); // Written in SET, read in RETURNING + assert_eq!(columns.get("age"), Some(&W)); // Only written + assert_eq!(columns.get("id"), Some(&R)); // Only read + } } diff --git a/backend/parsers/windmill-parser-ts/src/asset_parser.rs b/backend/parsers/windmill-parser-ts/src/asset_parser.rs index 2b85070584..66bda658fe 100644 --- a/backend/parsers/windmill-parser-ts/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-ts/src/asset_parser.rs @@ -117,6 +117,7 @@ impl Visit for AssetsFinder { kind, path: path.to_string(), access_type: None, + columns: None, }); } } @@ -177,8 +178,12 @@ impl Visit for AssetsFinder { if asset_was_used(&self.assets, (kind, path)) { continue; } - self.assets - .push(ParseAssetsResult { kind, access_type: None, path: path.clone() }); + self.assets.push(ParseAssetsResult { + kind, + access_type: None, + path: path.clone(), + columns: None, + }); } // Restore state - identifiers declared in this block go out of scope @@ -294,8 +299,12 @@ impl AssetsFinder { let path = parse_asset_syntax(&value, false) .map(|(_, p)| p) .unwrap_or(&value); - self.assets - .push(ParseAssetsResult { kind, path: path.to_string(), access_type }); + self.assets.push(ParseAssetsResult { + kind, + path: path.to_string(), + access_type, + columns: None, + }); } _ => return Err(()), } @@ -321,7 +330,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, path: "/test.csv".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, },]) ); } @@ -340,7 +350,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt".to_string(), - access_type: None + access_type: None, + columns: None, },]) ); } @@ -360,7 +371,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/friends".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, },]) ); } @@ -383,12 +395,14 @@ mod tests { ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/analytics".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, }, ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/friends".to_string(), - access_type: Some(RW) + access_type: Some(RW), + columns: None, }, ]) ); @@ -418,17 +432,20 @@ mod tests { ParseAssetsResult { kind: AssetKind::DataTable, path: "another1/customers".to_string(), - access_type: Some(W) + access_type: Some(W), + columns: None, }, ParseAssetsResult { kind: AssetKind::Ducklake, path: "another2".to_string(), - access_type: None + access_type: None, + columns: None, }, ParseAssetsResult { kind: AssetKind::DataTable, path: "main/friends".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, }, ]) ); @@ -452,12 +469,14 @@ mod tests { ParseAssetsResult { kind: AssetKind::DataTable, path: "another1".to_string(), - access_type: None + access_type: None, + columns: None, }, ParseAssetsResult { kind: AssetKind::Ducklake, path: "main".to_string(), - access_type: None + access_type: None, + columns: None, }, ]) ); @@ -478,7 +497,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "main/myschema.friends".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, },]) ); } @@ -499,7 +519,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/public.users".to_string(), - access_type: Some(RW) + access_type: Some(RW), + columns: None, },]) ); } @@ -518,7 +539,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt".to_string(), - access_type: None + access_type: None, + columns: None, },]) ); } @@ -539,7 +561,8 @@ mod tests { Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/users".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, },]) ); } @@ -562,12 +585,14 @@ mod tests { ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/private.users".to_string(), - access_type: Some(R) + access_type: Some(R), + columns: None, }, ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/test".to_string(), - access_type: Some(W) + access_type: Some(W), + columns: None, }, ]) ); diff --git a/backend/parsers/windmill-parser-yaml/src/asset_parser.rs b/backend/parsers/windmill-parser-yaml/src/asset_parser.rs index 9e4cedf08d..7e67d563ff 100644 --- a/backend/parsers/windmill-parser-yaml/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-yaml/src/asset_parser.rs @@ -12,6 +12,7 @@ pub fn parse_assets(input: &str) -> anyhow::Result { kind: AssetKind::Resource, path: delegate_to_git_repo_details.resource, access_type: Some(AssetUsageAccessType::R), + columns: None, }) } @@ -21,6 +22,7 @@ pub fn parse_assets(input: &str) -> anyhow::Result { kind: AssetKind::Resource, path: pinned_res, access_type: Some(AssetUsageAccessType::R), + columns: None, }) } } @@ -31,6 +33,7 @@ pub fn parse_assets(input: &str) -> anyhow::Result { kind: AssetKind::Resource, path: resource, access_type: Some(AssetUsageAccessType::R), + columns: None, }) } } diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 4c287974ed..4f88fe17c1 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -1,4 +1,5 @@ use serde::Serialize; +use std::collections::BTreeMap; #[derive(Serialize, PartialEq, Clone, Copy, Debug)] #[serde(rename_all(serialize = "lowercase"))] @@ -19,12 +20,14 @@ pub enum AssetKind { DataTable, } -#[derive(Serialize, Debug, PartialEq)] +#[derive(Serialize, Debug, PartialEq, Clone)] pub struct ParseAssetsResult { pub kind: AssetKind, pub path: String, #[serde(skip_serializing_if = "Option::is_none")] pub access_type: Option, // None in case of ambiguity + #[serde(skip_serializing_if = "Option::is_none")] + pub columns: Option>, // Map column name to access type, "*" represents wildcard } #[derive(Serialize, Debug, PartialEq)] @@ -66,6 +69,8 @@ pub fn merge_assets(assets: Vec) -> Vec { (Some(R), Some(R)) => Some(R), (Some(W), Some(W)) => Some(W), }; + // merge columns: union the column sets and merge access types per column + existing.columns = merge_column_maps(existing.columns.take(), asset.columns); } else { arr.push(asset); } @@ -74,6 +79,36 @@ pub fn merge_assets(assets: Vec) -> Vec { arr } +fn merge_column_maps( + existing: Option>, + new: Option>, +) -> Option> { + match (existing, new) { + (None, None) => None, + (Some(map), None) | (None, Some(map)) => Some(map), + (Some(mut existing_map), Some(new_map)) => { + for (col_name, new_access) in new_map { + existing_map + .entry(col_name) + .and_modify(|existing_access| { + *existing_access = merge_access_types(*existing_access, new_access); + }) + .or_insert(new_access); + } + Some(existing_map) + } + } +} + +fn merge_access_types(a: AssetUsageAccessType, b: AssetUsageAccessType) -> AssetUsageAccessType { + match (a, b) { + (R, W) | (W, R) => RW, + (RW, _) | (_, RW) => RW, + (R, R) => R, + (W, W) => W, + } +} + // Will return false if the user assigned an asset to a variable like: // let sql = wmill.datatable('main') // But never used it. In that case we don't know which table is being used, diff --git a/backend/rustfmt.toml b/backend/rustfmt.toml index 4aa3caaffa..912ee9d237 100644 --- a/backend/rustfmt.toml +++ b/backend/rustfmt.toml @@ -1,3 +1,4 @@ +edition = "2021" max_width = 100 use_small_heuristics = "Default" match_arm_leading_pipes="Preserve" diff --git a/backend/src/main.rs b/backend/src/main.rs index 855262c5d5..fc9a412ab3 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -16,7 +16,7 @@ use monitor::{ send_current_log_file_to_object_store, send_logs_to_object_store, WORKERS_NAMES, }; use rand::Rng; -use sqlx::{postgres::PgListener, Pool, Postgres}; +use sqlx::{Pool, Postgres}; use std::{ collections::HashMap, fs::{create_dir_all, DirBuilder}, @@ -212,11 +212,17 @@ where } lazy_static::lazy_static! { - static ref PG_LISTENER_REFRESH_PERIOD_SECS: u64 = std::env::var("PG_LISTENER_REFRESH_PERIOD_SECS") + // Period in seconds between full settings reload (12 hours by default) + static ref SETTINGS_RELOAD_PERIOD_SECS: u64 = std::env::var("SETTINGS_RELOAD_PERIOD_SECS") .ok() .and_then(|x| x.parse::().ok()) .unwrap_or(3600 * 12); + // Period in seconds between polling for notify events (10s by default) + static ref LISTEN_NEW_EVENTS_INTERVAL_SEC: u64 = std::env::var("LISTEN_NEW_EVENTS_INTERVAL_SEC") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(10); } pub fn main() -> anyhow::Result<()> { @@ -1138,8 +1144,18 @@ Windmill Community Edition {GIT_VERSION} let base_internal_url = base_internal_url.to_string(); let db = db.clone(); let h = tokio::spawn(async move { - let mut listener = retry_listen_pg(&db).await; - let mut last_listener_refresh = Instant::now(); + // Initialize last_event_id to current max to avoid processing old events on startup + let mut last_event_id: i64 = match windmill_common::notify_events::get_latest_event_id(&db).await { + Ok(id) => { + tracing::info!("Initialized notify event polling with last_event_id: {}", id); + id + } + Err(e) => { + tracing::warn!("Could not get latest event id, starting from 0: {e:#}"); + 0 + } + }; + let mut last_settings_reload = Instant::now(); let mut monitor_iteration: u64 = 0; let rd_shift: u8 = rand::rng().random_range(0..200); loop { @@ -1158,349 +1174,36 @@ Windmill Community Edition {GIT_VERSION} tracing::info!("received killpill for monitor job"); break; }, - notification = listener.try_recv() => { - match notification { - Ok(n) => { - if n.is_none() { - tracing::error!("Could not receive notification, attempting to reconnect to pg listener"); - continue; - } - let n = n.unwrap(); - tracing::info!("Received new pg notification: {n:?}"); - match n.channel() { - "notify_config_change" => { - match n.payload() { - "server" if server_mode => { - tracing::error!("Server config change detected but server config is obsolete: {}", n.payload()); - }, - a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => { - tracing::info!("Worker config change detected: {}", n.payload()); - reload_worker_config(&db, tx.clone(), true).await; - }, - _ => { - tracing::debug!("config changed but did not target this server/worker"); - } - } - }, - "notify_webhook_change" => { - let workspace_id = n.payload(); - tracing::info!("Webhook change detected, invalidating webhook cache: {}", workspace_id); - windmill_api::webhook_util::WEBHOOK_CACHE.remove(workspace_id); - }, - "notify_workspace_envs_change" => { - let workspace_id = n.payload(); - tracing::info!("Workspace envs change detected, invalidating workspace envs cache: {}", workspace_id); - windmill_common::variables::CUSTOM_ENVS_CACHE.remove(workspace_id); - }, - "notify_workspace_key_change" => { - let workspace_id = n.payload(); - tracing::info!("Workspace key change detected, invalidating workspace key cache: {}", workspace_id); - windmill_common::variables::WORKSPACE_CRYPT_CACHE.remove(workspace_id); - }, - "notify_workspace_premium_change" => { - let workspace_id = n.payload(); - tracing::info!("Workspace premium change detected, invalidating workspace premium cache: {}", workspace_id); - windmill_common::workspaces::TEAM_PLAN_CACHE.remove(workspace_id); - }, - "notify_runnable_version_change" => { - let payload = n.payload(); - tracing::info!("Runnable version change detected: {}", payload); - match payload.split(':').collect::>().as_slice() { - [workspace_id, source_type, path, kind] => { - let key = (workspace_id.to_string(), path.to_string()); - match source_type { - &"script" => { - windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key); - match kind { - &"preprocessor" => { - match sqlx::query_scalar!( - "SELECT fv.id - FROM flow f - INNER JOIN flow_version fv ON fv.id = f.versions[array_upper(f.versions, 1)] - WHERE fv.value->'preprocessor_module'->'value'->>'path' = $1 AND f.workspace_id = $2", - path, - workspace_id - ).fetch_all(&db).await { - Ok(flow_versions) => { - tracing::debug!("Workspace preprocessor {} changed, removing runnable format version cache for flow versions {:?}", path, flow_versions); - for version in flow_versions { - for trigger_kind in TriggerKind::iter() { - let key = (windmill_common::triggers::HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), version, trigger_kind); - windmill_common::triggers::RUNNABLE_FORMAT_VERSION_CACHE.remove(&key); - } - } - } - Err(e) => { - tracing::error!("Error fetching flow paths: {e:#}"); - } - } - }, - _ => {} - } - } - &"flow" => { - let dynamic_input_key = windmill_common::jobs::generate_dynamic_input_key(workspace_id, path); - windmill_common::DYNAMIC_INPUT_CACHE.remove(&dynamic_input_key); - windmill_common::FLOW_VERSION_CACHE.remove(&key); - }, - _ => { - tracing::warn!("Unknown runnable version change payload: {}", payload); - } - } - }, - _ => { - tracing::warn!("Unknown runnable version change payload: {}", payload); - } - } - }, - #[cfg(feature = "http_trigger")] - "notify_http_trigger_change" => { - tracing::info!("HTTP trigger change detected: {}", n.payload()); - match windmill_api::triggers::http::refresh_routers(&db).await { - Ok((true, _)) => { - tracing::info!("Refreshed HTTP routers (trigger change)"); - }, - Ok((false, _)) => { - tracing::warn!("Should have refreshed HTTP routers (trigger change) but did not"); - }, - Err(err) => { - tracing::error!("Error refreshing HTTP routers (trigger change): {err:#}"); - } - }; - }, - "notify_token_invalidation" => { - let token = n.payload(); - tracing::info!("Token invalidation detected for token: {}...", &token[..token.len().min(8)]); - windmill_api::auth::invalidate_token_from_cache(token); - }, - "var_cache_invalidation" => { - if let Ok(payload) = serde_json::from_str::(n.payload()) { - if let (Some(workspace_id), Some(path)) = - (payload.get("workspace_id").and_then(|v| v.as_str()), - payload.get("path").and_then(|v| v.as_str())) { - tracing::info!("Variable cache invalidation detected: {}:{}", workspace_id, path); - windmill_api::var_resource_cache::invalidate_variable_cache(&workspace_id, &path); - } - } - }, - "resource_cache_invalidation" => { - if let Ok(payload) = serde_json::from_str::(n.payload()) { - if let (Some(workspace_id), Some(path)) = - (payload.get("workspace_id").and_then(|v| v.as_str()), - payload.get("path").and_then(|v| v.as_str())) { - tracing::info!("Resource cache invalidation detected: {}:{}", workspace_id, path); - windmill_api::var_resource_cache::invalidate_resource_cache(&workspace_id, &path); - } - } - }, - "notify_global_setting_change" => { - tracing::info!("Global setting change detected: {}", n.payload()); - match n.payload() { - BASE_URL_SETTING => { - if let Err(e) = reload_base_url_setting(&conn).await { - tracing::error!(error = %e, "Could not reload base url setting"); - } - }, - OAUTH_SETTING => { - if let Err(e) = reload_base_url_setting(&conn).await { - tracing::error!(error = %e, "Could not reload oauth setting"); - } - }, - CUSTOM_TAGS_SETTING => { - if let Err(e) = reload_custom_tags_setting(&db).await { - tracing::error!(error = %e, "Could not reload custom tags setting"); - } - }, - LICENSE_KEY_SETTING => { - if let Err(e) = reload_license_key(&db.into()).await { - tracing::error!("Failed to reload license key: {e:#}"); - } - }, - DEFAULT_TAGS_PER_WORKSPACE_SETTING => { - if let Err(e) = load_tag_per_workspace_enabled(&db).await { - tracing::error!("Error loading default tag per workspace: {e:#}"); - } - }, - DEFAULT_TAGS_WORKSPACES_SETTING => { - if let Err(e) = load_tag_per_workspace_workspaces(&db).await { - tracing::error!("Error loading default tag per workspace workspaces: {e:#}"); - } - }, - SMTP_SETTING => { - reload_smtp_config(&db).await; - }, - TEAMS_SETTING => { - tracing::info!("Teams setting changed."); - }, - INDEXER_SETTING => { - reload_indexer_config(&db).await; - }, - TIMEOUT_WAIT_RESULT_SETTING => { - reload_timeout_wait_result_setting(&conn).await - }, - RETENTION_PERIOD_SECS_SETTING => { - reload_retention_period_setting(&conn).await - }, - MONITOR_LOGS_ON_OBJECT_STORE_SETTING => { - reload_delete_logs_periodically_setting(&conn).await - }, - JOB_DEFAULT_TIMEOUT_SECS_SETTING => { - reload_job_default_timeout_setting(&conn).await - }, - #[cfg(feature = "parquet")] - OBJECT_STORE_CONFIG_SETTING => { - if !disable_s3_store { - reload_object_store_setting(&db).await; - } - }, - SCIM_TOKEN_SETTING => { - reload_scim_token_setting(&conn).await - }, - EXTRA_PIP_INDEX_URL_SETTING => { - reload_extra_pip_index_url_setting(&conn).await - }, - PIP_INDEX_URL_SETTING => { - reload_pip_index_url_setting(&conn).await - }, - INSTANCE_PYTHON_VERSION_SETTING => { - reload_instance_python_version_setting(&conn).await - }, - NPM_CONFIG_REGISTRY_SETTING => { - reload_npm_config_registry_setting(&conn).await - }, - BUNFIG_INSTALL_SCOPES_SETTING => { - reload_bunfig_install_scopes_setting(&conn).await - }, - NUGET_CONFIG_SETTING => { - reload_nuget_config_setting(&conn).await - }, - POWERSHELL_REPO_URL_SETTING => { - reload_powershell_repo_url_setting(&conn).await - }, - POWERSHELL_REPO_PAT_SETTING => { - reload_powershell_repo_pat_setting(&conn).await - }, - MAVEN_REPOS_SETTING => { - reload_maven_repos_setting(&conn).await - }, - NO_DEFAULT_MAVEN_SETTING => { - reload_no_default_maven_setting(&conn).await - }, - RUBY_REPOS_SETTING => { - reload_ruby_repos_setting(&conn).await - }, - HUB_API_SECRET_SETTING => { - reload_hub_api_secret_setting(&conn).await - }, - KEEP_JOB_DIR_SETTING => { - load_keep_job_dir(&conn).await; - }, - OTEL_TRACING_PROXY_SETTING => { - reload_otel_tracing_proxy_setting(&conn).await; - if worker_mode { - tracing::info!("OTEL tracing proxy setting changed, restarting worker"); - send_delayed_killpill(&tx, 4, "OTEL tracing proxy setting change").await; - } - }, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => { - load_require_preexisting_user(&db).await; - }, - EXPOSE_METRICS_SETTING => { - tracing::info!("Metrics setting changed, restarting"); - send_delayed_killpill(&tx, 40, "metrics setting change").await; - }, - EMAIL_DOMAIN_SETTING => { - tracing::info!("Email domain setting changed"); - if server_mode { - send_delayed_killpill(&tx, 4, "email domain setting change").await; - } - }, - EXPOSE_DEBUG_METRICS_SETTING => { - if let Err(e) = load_metrics_debug_enabled(&conn).await { - tracing::error!(error = %e, "Could not reload debug metrics setting"); - } - }, - APP_WORKSPACED_ROUTE_SETTING => { - if let Err(e) = reload_app_workspaced_route_setting(&db).await { - tracing::error!(error = %e, "Could not reload app workspaced route setting"); - } - }, - OTEL_SETTING => { - tracing::info!("OTEL setting changed, restarting"); - send_delayed_killpill(&tx, 4, "OTEL setting change").await; - }, - REQUEST_SIZE_LIMIT_SETTING => { - if server_mode { - tracing::info!("Request limit size change detected, killing server expecting to be restarted"); - send_delayed_killpill(&tx, 4, "request size limit change").await; - } - }, - SAML_METADATA_SETTING => { - tracing::info!("SAML metadata change detected, killing server expecting to be restarted"); - send_delayed_killpill(&tx, 0, "SAML metadata change").await; - }, - HUB_BASE_URL_SETTING => { - if let Err(e) = reload_hub_base_url_setting(&conn, server_mode).await { - tracing::error!(error = %e, "Could not reload hub base url setting"); - } - }, - CRITICAL_ERROR_CHANNELS_SETTING => { - if let Err(e) = reload_critical_error_channels_setting(&db).await { - tracing::error!(error = %e, "Could not reload critical error emails setting"); - } - }, - CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING => { - if let Err(e) = reload_critical_alerts_on_db_oversize(&db).await { - tracing::error!(error = %e, "Could not reload critical alerts on db oversize setting"); - } - - }, - JWT_SECRET_SETTING => { - if let Err(e) = reload_jwt_secret_setting(&db).await { - tracing::error!(error = %e, "Could not reload jwt secret setting"); - } - }, - CRITICAL_ALERT_MUTE_UI_SETTING => { - tracing::info!("Critical alert UI setting changed"); - if let Err(e) = reload_critical_alert_mute_ui_setting(&conn).await { - tracing::error!(error = %e, "Could not reload critical alert UI setting"); - } - }, - a @_ => { - tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a); - } - } - }, - _ => { - tracing::warn!("Unknown notification received"); - continue; + _ = tokio::time::sleep(Duration::from_secs(*LISTEN_NEW_EVENTS_INTERVAL_SEC)) => { + // Poll for new events from notify_event table + match windmill_common::notify_events::poll_notify_events(&db, last_event_id).await { + Ok(events) => { + for event in events { + if !*windmill_common::QUIET_LOGS { + tracing::info!("Processing notify event: channel={}, payload={}", event.channel, event.payload); } + process_notify_event( + &event.channel, + &event.payload, + &db, + &conn, + &tx, + server_mode, + worker_mode, + #[cfg(feature = "parquet")] + disable_s3_store, + ).await; + last_event_id = last_event_id.max(event.id); } - }, + } Err(e) => { - tracing::error!(error = %e, "Could not receive notification, attempting to reconnect listener"); - let db = db.clone(); - tokio::select! { - biased; - _ = monitor_killpill_rx.recv() => { - tracing::info!("received killpill for monitor job"); - break; - }, - new_listener = async move { retry_listen_pg(&db).await } => { - listener = new_listener; - continue; - } - } + tracing::error!("Error polling notify events: {e:#}"); } - }; - }, - _ = tokio::time::sleep(Duration::from_secs(30)) => { - if last_listener_refresh.elapsed() > Duration::from_secs(*PG_LISTENER_REFRESH_PERIOD_SECS) { - tracing::info!("Refreshing pg listeners, settings and license key after {}s", Duration::from_secs(*PG_LISTENER_REFRESH_PERIOD_SECS).as_secs()); - if let Err(e) = listener.unlisten_all().await { - tracing::error!(error = %e, "Could not unlisten to database"); - } - listener = retry_listen_pg(&db).await; + } + + // Periodic full settings reload + if last_settings_reload.elapsed() > Duration::from_secs(*SETTINGS_RELOAD_PERIOD_SECS) { + tracing::info!("Reloading settings and license key after {}s", Duration::from_secs(*SETTINGS_RELOAD_PERIOD_SECS).as_secs()); initial_load( &conn, tx.clone(), @@ -1514,7 +1217,7 @@ Windmill Community Edition {GIT_VERSION} if let Err(err) = reload_license_key(&conn).await { tracing::error!("Failed to reload license key: {err:#}"); } - last_listener_refresh = Instant::now(); + last_settings_reload = Instant::now(); } if server_mode { @@ -1668,50 +1371,293 @@ Windmill Community Edition {GIT_VERSION} std::process::exit(0); } -async fn listen_pg(db: &Pool) -> Option { - let mut listener = match PgListener::connect_with(db).await { - Ok(l) => l, - Err(e) => { - tracing::error!(error = %e, "Could not connect to database"); - return None; - } - }; - - #[allow(unused_mut)] - let mut channels = vec![ - "notify_config_change", - "notify_global_setting_change", - "notify_webhook_change", - "notify_workspace_envs_change", - "notify_workspace_key_change", - "notify_runnable_version_change", - "notify_token_invalidation", - ]; - - #[cfg(feature = "http_trigger")] - channels.push("notify_http_trigger_change"); - - #[cfg(feature = "cloud")] - channels.push("notify_workspace_premium_change"); - - if let Err(e) = listener.listen_all(channels).await { - tracing::error!(error = %e, "Could not listen to database"); - return None; - } - - return Some(listener); -} - -async fn retry_listen_pg(db: &Pool) -> PgListener { - let mut listener = listen_pg(db).await; - loop { - if listener.is_none() { - tracing::info!("Retrying listening to pg listen in 5 seconds"); - tokio::time::sleep(Duration::from_secs(5)).await; - listener = listen_pg(db).await; - } else { - tracing::info!("Successfully connected to pg listen"); - return listener.unwrap(); +/// Process a single notify event from the polling-based event system. +/// This replaces the old PgListener notification handling. +#[allow(unused_variables)] +async fn process_notify_event( + channel: &str, + payload: &str, + db: &Pool, + conn: &Connection, + tx: &KillpillSender, + server_mode: bool, + worker_mode: bool, + #[cfg(feature = "parquet")] + disable_s3_store: bool, +) { + match channel { + "notify_config_change" => { + if payload == "server" && server_mode { + tracing::error!("Server config change detected but server config is obsolete: {}", payload); + } else if worker_mode && payload == format!("worker__{}", *WORKER_GROUP) { + tracing::info!("Worker config change detected: {}", payload); + reload_worker_config(db, tx.clone(), true).await; + } else { + tracing::debug!("config changed but did not target this server/worker"); + } + }, + "notify_webhook_change" => { + tracing::info!("Webhook change detected, invalidating webhook cache: {}", payload); + windmill_api::webhook_util::WEBHOOK_CACHE.remove(payload); + }, + "notify_workspace_envs_change" => { + tracing::info!("Workspace envs change detected, invalidating workspace envs cache: {}", payload); + windmill_common::variables::CUSTOM_ENVS_CACHE.remove(payload); + }, + "notify_workspace_key_change" => { + tracing::info!("Workspace key change detected, invalidating workspace key cache: {}", payload); + windmill_common::variables::WORKSPACE_CRYPT_CACHE.remove(payload); + }, + "notify_workspace_premium_change" => { + tracing::info!("Workspace premium change detected, invalidating workspace premium cache: {}", payload); + windmill_common::workspaces::TEAM_PLAN_CACHE.remove(payload); + }, + "notify_runnable_version_change" => { + tracing::info!("Runnable version change detected: {}", payload); + match payload.split(':').collect::>().as_slice() { + [workspace_id, source_type, path, kind] => { + let key = (workspace_id.to_string(), path.to_string()); + match *source_type { + "script" => { + windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key); + if *kind == "preprocessor" { + match sqlx::query_scalar::<_, i64>( + "SELECT fv.id + FROM flow f + INNER JOIN flow_version fv ON fv.id = f.versions[array_upper(f.versions, 1)] + WHERE fv.value->'preprocessor_module'->'value'->>'path' = $1 AND f.workspace_id = $2", + ) + .bind(*path) + .bind(*workspace_id) + .fetch_all(db).await { + Ok(flow_versions) => { + tracing::debug!("Workspace preprocessor {} changed, removing runnable format version cache for flow versions {:?}", path, flow_versions); + for version in flow_versions { + for trigger_kind in TriggerKind::iter() { + let key = (windmill_common::triggers::HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), version, trigger_kind); + windmill_common::triggers::RUNNABLE_FORMAT_VERSION_CACHE.remove(&key); + } + } + } + Err(e) => { + tracing::error!("Error fetching flow paths: {e:#}"); + } + } + } + } + "flow" => { + let dynamic_input_key = windmill_common::jobs::generate_dynamic_input_key(workspace_id, path); + windmill_common::DYNAMIC_INPUT_CACHE.remove(&dynamic_input_key); + windmill_common::FLOW_VERSION_CACHE.remove(&key); + }, + _ => { + tracing::warn!("Unknown runnable version change payload: {}", payload); + } + } + }, + _ => { + tracing::warn!("Unknown runnable version change payload: {}", payload); + } + } + }, + #[cfg(feature = "http_trigger")] + "notify_http_trigger_change" => { + tracing::info!("HTTP trigger change detected: {}", payload); + match windmill_api::triggers::http::refresh_routers(db).await { + Ok((true, _)) => { + tracing::info!("Refreshed HTTP routers (trigger change)"); + }, + Ok((false, _)) => { + tracing::warn!("Should have refreshed HTTP routers (trigger change) but did not"); + }, + Err(err) => { + tracing::error!("Error refreshing HTTP routers (trigger change): {err:#}"); + } + }; + }, + "notify_token_invalidation" => { + tracing::info!("Token invalidation detected for token: {}...", payload.get(..8).unwrap_or(payload)); + windmill_api::auth::invalidate_token_from_cache(payload); + }, + "notify_global_setting_change" => { + tracing::info!("Global setting change detected: {}", payload); + match payload { + BASE_URL_SETTING => { + if let Err(e) = reload_base_url_setting(conn).await { + tracing::error!(error = %e, "Could not reload base url setting"); + } + }, + OAUTH_SETTING => { + if let Err(e) = reload_base_url_setting(conn).await { + tracing::error!(error = %e, "Could not reload oauth setting"); + } + }, + CUSTOM_TAGS_SETTING => { + if let Err(e) = reload_custom_tags_setting(db).await { + tracing::error!(error = %e, "Could not reload custom tags setting"); + } + }, + LICENSE_KEY_SETTING => { + if let Err(e) = reload_license_key(&db.into()).await { + tracing::error!("Failed to reload license key: {e:#}"); + } + }, + DEFAULT_TAGS_PER_WORKSPACE_SETTING => { + if let Err(e) = load_tag_per_workspace_enabled(db).await { + tracing::error!("Error loading default tag per workspace: {e:#}"); + } + }, + DEFAULT_TAGS_WORKSPACES_SETTING => { + if let Err(e) = load_tag_per_workspace_workspaces(db).await { + tracing::error!("Error loading default tag per workspace workspaces: {e:#}"); + } + }, + SMTP_SETTING => { + reload_smtp_config(db).await; + }, + TEAMS_SETTING => { + tracing::info!("Teams setting changed."); + }, + INDEXER_SETTING => { + reload_indexer_config(db).await; + }, + TIMEOUT_WAIT_RESULT_SETTING => { + reload_timeout_wait_result_setting(conn).await + }, + RETENTION_PERIOD_SECS_SETTING => { + reload_retention_period_setting(conn).await + }, + MONITOR_LOGS_ON_OBJECT_STORE_SETTING => { + reload_delete_logs_periodically_setting(conn).await + }, + JOB_DEFAULT_TIMEOUT_SECS_SETTING => { + reload_job_default_timeout_setting(conn).await + }, + #[cfg(feature = "parquet")] + OBJECT_STORE_CONFIG_SETTING => { + if !disable_s3_store { + reload_object_store_setting(db).await; + } + }, + SCIM_TOKEN_SETTING => { + reload_scim_token_setting(conn).await + }, + EXTRA_PIP_INDEX_URL_SETTING => { + reload_extra_pip_index_url_setting(conn).await + }, + PIP_INDEX_URL_SETTING => { + reload_pip_index_url_setting(conn).await + }, + INSTANCE_PYTHON_VERSION_SETTING => { + reload_instance_python_version_setting(conn).await + }, + NPM_CONFIG_REGISTRY_SETTING => { + reload_npm_config_registry_setting(conn).await + }, + BUNFIG_INSTALL_SCOPES_SETTING => { + reload_bunfig_install_scopes_setting(conn).await + }, + NUGET_CONFIG_SETTING => { + reload_nuget_config_setting(conn).await + }, + POWERSHELL_REPO_URL_SETTING => { + reload_powershell_repo_url_setting(conn).await + }, + POWERSHELL_REPO_PAT_SETTING => { + reload_powershell_repo_pat_setting(conn).await + }, + MAVEN_REPOS_SETTING => { + reload_maven_repos_setting(conn).await + }, + NO_DEFAULT_MAVEN_SETTING => { + reload_no_default_maven_setting(conn).await + }, + RUBY_REPOS_SETTING => { + reload_ruby_repos_setting(conn).await + }, + HUB_API_SECRET_SETTING => { + reload_hub_api_secret_setting(conn).await + }, + KEEP_JOB_DIR_SETTING => { + load_keep_job_dir(conn).await; + }, + OTEL_TRACING_PROXY_SETTING => { + reload_otel_tracing_proxy_setting(conn).await; + if worker_mode { + tracing::info!("OTEL tracing proxy setting changed, restarting worker"); + send_delayed_killpill(tx, 4, "OTEL tracing proxy setting change").await; + } + }, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => { + load_require_preexisting_user(db).await; + }, + EXPOSE_METRICS_SETTING => { + tracing::info!("Metrics setting changed, restarting"); + send_delayed_killpill(tx, 40, "metrics setting change").await; + }, + EMAIL_DOMAIN_SETTING => { + tracing::info!("Email domain setting changed"); + if server_mode { + send_delayed_killpill(tx, 4, "email domain setting change").await; + } + }, + EXPOSE_DEBUG_METRICS_SETTING => { + if let Err(e) = load_metrics_debug_enabled(conn).await { + tracing::error!(error = %e, "Could not reload debug metrics setting"); + } + }, + APP_WORKSPACED_ROUTE_SETTING => { + if let Err(e) = reload_app_workspaced_route_setting(db).await { + tracing::error!(error = %e, "Could not reload app workspaced route setting"); + } + }, + OTEL_SETTING => { + tracing::info!("OTEL setting changed, restarting"); + send_delayed_killpill(tx, 4, "OTEL setting change").await; + }, + REQUEST_SIZE_LIMIT_SETTING => { + if server_mode { + tracing::info!("Request limit size change detected, killing server expecting to be restarted"); + send_delayed_killpill(tx, 4, "request size limit change").await; + } + }, + SAML_METADATA_SETTING => { + tracing::info!("SAML metadata change detected, killing server expecting to be restarted"); + send_delayed_killpill(tx, 0, "SAML metadata change").await; + }, + HUB_BASE_URL_SETTING => { + if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await { + tracing::error!(error = %e, "Could not reload hub base url setting"); + } + }, + CRITICAL_ERROR_CHANNELS_SETTING => { + if let Err(e) = reload_critical_error_channels_setting(db).await { + tracing::error!(error = %e, "Could not reload critical error emails setting"); + } + }, + CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING => { + if let Err(e) = reload_critical_alerts_on_db_oversize(db).await { + tracing::error!(error = %e, "Could not reload critical alerts on db oversize setting"); + } + }, + JWT_SECRET_SETTING => { + if let Err(e) = reload_jwt_secret_setting(db).await { + tracing::error!(error = %e, "Could not reload jwt secret setting"); + } + }, + CRITICAL_ALERT_MUTE_UI_SETTING => { + tracing::info!("Critical alert UI setting changed"); + if let Err(e) = reload_critical_alert_mute_ui_setting(conn).await { + tracing::error!(error = %e, "Could not reload critical alert UI setting"); + } + }, + _ => { + tracing::info!("Unrecognized Global Setting Change Payload: {:?}", payload); + } + } + }, + _ => { + tracing::warn!("Unknown notification channel: {}", channel); } } } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 3e4e7dce92..07a4c5069b 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1921,6 +1921,24 @@ pub async fn monitor_db( } }; + // Run every 5 minutes (10 iterations * 30s = 5 minutes) + // Cleanup old notify events (older than 10 minutes) + let cleanup_notify_events_f = async { + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(10) { + if let Some(db) = conn.as_sql() { + match windmill_common::notify_events::cleanup_old_events(db, 10).await { + Ok(count) if count > 0 => { + tracing::debug!("Cleaned up {} old notify events", count); + } + Err(e) => { + tracing::error!("Error cleaning up notify events: {:?}", e); + } + _ => {} + } + } + } + }; + join!( expired_items_f, zombie_jobs_f, @@ -1940,6 +1958,7 @@ pub async fn monitor_db( cleanup_flow_iterator_data_f, cleanup_worker_group_stats_f, native_triggers_sync_f, + cleanup_notify_events_f, ); } diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs new file mode 100644 index 0000000000..cf3ce4d350 --- /dev/null +++ b/backend/tests/bun_jobs.rs @@ -0,0 +1,1358 @@ +mod common; +use crate::common::*; +use sqlx::postgres::Postgres; +use sqlx::Pool; +use windmill_common::jobs::{JobPayload, RawCode}; +use windmill_common::scripts::ScriptLang; + +// ============================================================================ +// Basic Execution Tests +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_bun_job_simple(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +export function main() { + return "hello world"; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("hello world")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_bun_job_with_args(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +export function main(name: string, count: number) { + return `Hello ${name}, count: ${count}`; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = RunJob::from(job) + .arg("name", serde_json::json!("World")) + .arg("count", serde_json::json!(42)) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("Hello World, count: 42")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_bun_job_return_types(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Test object return + { + let content = r#" +export function main() { + return { name: "test", value: 123 }; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!({"name": "test", "value": 123})); + } + + // Test array return + { + let content = r#" +export function main() { + return [1, 2, 3, "four"]; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!([1, 2, 3, "four"])); + } + + // Test BigInt serialization + { + let content = r#" +export function main() { + // Use BigInt literal notation to avoid JavaScript number precision loss + return 9007199254740993n; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + // BigInt should be serialized as string + assert_eq!(result, serde_json::json!("9007199254740993")); + } + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_bun_job_async(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +export async function main() { + await new Promise(resolve => setTimeout(resolve, 100)); + return "async completed"; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("async completed")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_bun_job_null_undefined_handling(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Test null return + { + let content = r#" +export function main() { + return null; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!(null)); + } + + // Test undefined return (should be serialized as null) + { + let content = r#" +export function main() { + return undefined; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!(null)); + } + + Ok(()) +} + +// ============================================================================ +// Error Handling Tests +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_bun_job_runtime_error(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +export function main() { + throw new Error("intentional error"); +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let completed = run_job_in_new_worker_until_complete(&db, false, job, port).await; + + assert!(!completed.success); + let result = completed.json_result().unwrap(); + // Error is wrapped: {"error": {"message": "...", "name": "...", "stack": "..."}} + let error = &result["error"]; + assert!(error["message"] + .as_str() + .unwrap() + .contains("intentional error")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_bun_job_missing_main_function(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +export function notMain() { + return "hello"; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let completed = run_job_in_new_worker_until_complete(&db, false, job, port).await; + + assert!(!completed.success); + let result = completed.json_result().unwrap(); + // Error is wrapped: {"error": {"message": "...", "name": "...", "stack": "..."}} + let error = &result["error"]; + assert!(error["message"] + .as_str() + .unwrap() + .contains("main function is missing")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_bun_job_syntax_error(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +export function main() { + return "unclosed string +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let completed = run_job_in_new_worker_until_complete(&db, false, job, port).await; + + assert!(!completed.success); + Ok(()) +} + +// ============================================================================ +// Annotation Mode Tests +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_bun_nodejs_mode(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#"//nodejs + +export function main() { + // Node.js specific API + return process.version.startsWith("v"); +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!(true)); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_bun_nobundling_mode(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#"//nobundling + +export function main() { + return "nobundling works"; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("nobundling works")); + Ok(()) +} + +// ============================================================================ +// Native Mode Tests (requires deno_core feature) +// ============================================================================ + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_bun_native_mode(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#"//native + +export function main() { + return "native execution"; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("native execution")); + Ok(()) +} + +// ============================================================================ +// Relative Import Tests +// ============================================================================ + +#[sqlx::test(fixtures("base", "relative_bun"))] +async fn test_bun_relative_imports(db: Pool) -> anyhow::Result<()> { + let content = r#" +import { main as test1 } from "/f/system/same_folder_script.ts"; +import { main as test2 } from "./same_folder_script.ts"; +import { main as test3 } from "/f/system_relative/different_folder_script.ts"; +import { main as test4 } from "../system_relative/different_folder_script.ts"; + +export function main() { + return [test1(), test2(), test3(), test4()]; +} +"# + .to_string(); + + run_deployed_relative_imports(&db, content.clone(), ScriptLang::Bun).await?; + run_preview_relative_imports(&db, content, ScriptLang::Bun).await?; + Ok(()) +} + +#[sqlx::test(fixtures("base", "relative_bun"))] +async fn test_bun_nested_imports(db: Pool) -> anyhow::Result<()> { + // Test with absolute path (/f/...) + let content_absolute = r#" +import { main as test } from "/f/system_relative/nested_script.ts"; + +export function main() { + return test(); +} +"# + .to_string(); + + run_deployed_relative_imports(&db, content_absolute.clone(), ScriptLang::Bun).await?; + run_preview_relative_imports(&db, content_absolute, ScriptLang::Bun).await?; + + // Test with relative path (../...) + let content_relative = r#" +import { main as test } from "../system_relative/nested_script.ts"; + +export function main() { + return test(); +} +"# + .to_string(); + + run_preview_relative_imports(&db, content_relative, ScriptLang::Bun).await?; + Ok(()) +} + +// ============================================================================ +// Deeply Nested Import Tests +// ============================================================================ + +#[sqlx::test(fixtures("base", "bun_edge_cases"))] +async fn test_bun_deeply_nested_imports(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Test with absolute path import (/f/nested/level1.ts) + // Note: The fixture uses relative imports internally (level1 -> ./level2 -> ./level3) + { + let content = r#" +import { main as level1 } from "/f/nested/level1.ts"; + +export function main() { + return level1(); +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: Some("f/nested/test_deep".to_string()), + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + // level1 -> level2 -> level3, each adds to the chain + assert_eq!(result, serde_json::json!("level1 -> level2 -> level3")); + } + + // Test with relative path import (./level1.ts) + { + let content = r#" +import { main as level1 } from "./level1.ts"; + +export function main() { + return level1(); +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: Some("f/nested/test_deep_relative".to_string()), + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + // Same result: level1 -> level2 -> level3 + assert_eq!(result, serde_json::json!("level1 -> level2 -> level3")); + } + + Ok(()) +} + +#[sqlx::test(fixtures("base", "bun_edge_cases"))] +async fn test_bun_shared_imports_both_styles(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Test importing modules that use different import styles internally: + // - module_a uses relative import: ./shared.ts + // - module_b uses absolute import: /f/circular/shared.ts + // Both should work correctly + let content = r#" +import { getValue as getA } from "/f/circular/module_a.ts"; +import { getValue as getB } from "./module_b.ts"; + +export function main() { + return [getA(), getB()]; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: Some("f/circular/test_both".to_string()), + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + // Both modules should correctly import SHARED_VALUE + assert_eq!( + result, + serde_json::json!(["from_a_shared", "from_b_shared"]) + ); + Ok(()) +} + +// ============================================================================ +// Preprocessor Tests +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_bun_preprocessor_execution(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Test that the main function works correctly + // Note: Preprocessor execution requires specific job configuration + // (flow_step_id != "preprocessor" and preprocessed == Some(false)) + // which is not set by default in RawCode jobs + let content = r#" +export function main(x: number) { + return x + 10; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + // x=5, main adds 10 = 15 + let result = RunJob::from(job) + .arg("x", serde_json::json!(5)) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!(15)); + Ok(()) +} + +// ============================================================================ +// Wmill SDK Tests +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_bun_job_with_wmill_env_vars(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Test accessing Windmill environment variables (which the SDK uses internally) + // This validates that the execution environment is properly configured + let content = r#" +export function main() { + // WM_WORKSPACE and WM_TOKEN are injected by Windmill worker + return { + workspace: process.env.WM_WORKSPACE, + hasToken: process.env.WM_TOKEN !== undefined && process.env.WM_TOKEN.length > 0, + baseUrl: process.env.BASE_URL !== undefined + }; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result["workspace"], serde_json::json!("test-workspace")); + assert_eq!(result["hasToken"], serde_json::json!(true)); + assert_eq!(result["baseUrl"], serde_json::json!(true)); + Ok(()) +} + +// ============================================================================ +// Environment Variable Tests +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_bun_job_env_vars(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +export function main() { + return { + workspace: process.env.WM_WORKSPACE, + hasToken: !!process.env.WM_TOKEN, + }; +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result["workspace"], serde_json::json!("test-workspace")); + assert_eq!(result["hasToken"], serde_json::json!(true)); + Ok(()) +} + +// ============================================================================ +// Dedicated Worker Protocol Tests +// ============================================================================ + +mod dedicated_worker_protocol { + use crate::common::{parse_dedicated_worker_line, DedicatedWorkerResult}; + use std::io::{BufRead, BufReader, Write}; + use std::process::{Command, Stdio}; + use windmill_worker::{ + build_loader, generate_dedicated_worker_wrapper, BUN_DEDICATED_WORKER_ARGS, LoaderMode, + BUN_PATH, NODE_BIN_PATH, + }; + + /// Creates test worker files and optionally bundles for Node.js (like production) + /// Returns the path to the wrapper file to execute + fn create_test_worker_files( + dir: &std::path::Path, + script: &str, + arg_names: &[&str], + bundle_for_node: bool, + ) -> std::path::PathBuf { + let dir_str = dir.to_str().unwrap(); + std::fs::write(dir.join("main.ts"), script).unwrap(); + + if bundle_for_node { + // For Node.js: bundle to JavaScript first (like production's build_loader with LoaderMode::Node) + let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.js", None); + std::fs::write(dir.join("wrapper.mjs"), wrapper).unwrap(); + + // Use the exact same build_loader function as production + tokio::runtime::Runtime::new() + .unwrap() + .block_on(build_loader( + dir_str, + "http://localhost:8000", + "test_token", + "test-workspace", + "f/test/script", + LoaderMode::Node, + )) + .expect("build_loader failed"); + + // Run the bundler with bun (build_loader creates node_builder.ts) + let output = Command::new(BUN_PATH.as_str()) + .args(["run", dir.join("node_builder.ts").to_str().unwrap()]) + .current_dir(dir) + .output() + .expect("Failed to run bun build"); + + if !output.status.success() { + panic!( + "Bun build failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + // Bun outputs to wrapper.js, rename to wrapper.mjs for ES module + let bundled_path = dir.join("wrapper.js"); + let output_path = dir.join("wrapper_bundled.mjs"); + std::fs::rename(&bundled_path, &output_path).unwrap(); + output_path + } else { + // For Bun: use TypeScript directly (like production) + let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.ts", None); + let wrapper_path = dir.join("wrapper.mjs"); + std::fs::write(&wrapper_path, wrapper).unwrap(); + wrapper_path + } + } + + /// Helper to run a dedicated worker test with given runtime + fn run_worker_test( + runtime: &str, + script: &str, + arg_names: &[&str], + jobs: Vec, + ) -> Vec> { + let temp_dir = tempfile::tempdir().unwrap(); + + // Create files and get the wrapper path (bundled for node, raw for bun) + let wrapper_path = create_test_worker_files( + temp_dir.path(), + script, + arg_names, + runtime == "node", + ); + let wrapper_str = wrapper_path.to_str().unwrap(); + + // Build args matching production behavior + let (cmd, args): (&str, Vec<&str>) = match runtime { + "bun" => { + // Production: bun run -i --prefer-offline wrapper.mjs + let mut args: Vec<&str> = BUN_DEDICATED_WORKER_ARGS.to_vec(); + args.push(wrapper_str); + (BUN_PATH.as_str(), args) + } + "node" => { + // Production: node wrapper.mjs (after bundling to JS) + (NODE_BIN_PATH.as_str(), vec![wrapper_str]) + } + _ => panic!("Unknown runtime: {}", runtime), + }; + + let mut child = Command::new(cmd) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .current_dir(temp_dir.path()) + .spawn() + .expect("Failed to spawn worker process"); + + let mut stdin = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let mut reader = BufReader::new(stdout); + + // Wait for "start" signal + let mut start_line = String::new(); + reader.read_line(&mut start_line).unwrap(); + assert_eq!( + parse_dedicated_worker_line(start_line.trim()), + DedicatedWorkerResult::Start, + "Expected 'start', got: {}", + start_line.trim() + ); + + let mut results = Vec::new(); + + for job_args in jobs { + writeln!(stdin, "{}", job_args.to_string()).unwrap(); + stdin.flush().unwrap(); + + let mut response = String::new(); + reader.read_line(&mut response).unwrap(); + + match parse_dedicated_worker_line(response.trim()) { + DedicatedWorkerResult::Success(value) => results.push(Ok(value)), + DedicatedWorkerResult::Error(err) => { + let msg = err["message"].as_str().unwrap_or("Unknown error").to_string(); + results.push(Err(msg)); + } + other => panic!("Unexpected response: {:?}", other), + } + } + + writeln!(stdin, "end").unwrap(); + stdin.flush().unwrap(); + let _ = child.wait().expect("Worker process failed to exit"); + + results + } + + // ==================== Node.js Runtime Tests ==================== + + #[test] + fn test_dedicated_worker_nodejs_simple() { + let script = r#" +export function main(x: number, y: number): number { + return x + y; +} +"#; + let results = run_worker_test( + "node", + script, + &["x", "y"], + vec![serde_json::json!({"x": 5, "y": 3})], + ); + + assert_eq!(results.len(), 1); + assert_eq!(results[0], Ok(serde_json::json!(8))); + } + + #[test] + fn test_dedicated_worker_nodejs_multiple_jobs() { + let script = r#" +export function main(n: number): number { + return n * 2; +} +"#; + let jobs: Vec = (1..=5).map(|i| serde_json::json!({"n": i})).collect(); + let results = run_worker_test("node", script, &["n"], jobs); + + assert_eq!(results.len(), 5); + for (i, result) in results.iter().enumerate() { + let expected = ((i + 1) * 2) as i64; + assert_eq!(*result, Ok(serde_json::json!(expected))); + } + } + + #[test] + fn test_dedicated_worker_nodejs_error() { + let script = r#" +export function main(msg: string): never { + throw new Error(msg); +} +"#; + let results = run_worker_test( + "node", + script, + &["msg"], + vec![serde_json::json!({"msg": "test error"})], + ); + + assert_eq!(results.len(), 1); + assert!(results[0].is_err()); + assert_eq!(results[0], Err("test error".to_string())); + } + + // ==================== Bun Runtime Tests ==================== + + #[test] + fn test_dedicated_worker_bun_simple() { + let script = r#" +export function main(x: number, y: number): number { + return x + y; +} +"#; + let results = run_worker_test( + "bun", + script, + &["x", "y"], + vec![serde_json::json!({"x": 5, "y": 3})], + ); + + assert_eq!(results.len(), 1); + assert_eq!(results[0], Ok(serde_json::json!(8))); + } + + #[test] + fn test_dedicated_worker_bun_multiple_jobs() { + let script = r#" +export function main(n: number): number { + return n * 2; +} +"#; + let jobs: Vec = (1..=5).map(|i| serde_json::json!({"n": i})).collect(); + let results = run_worker_test("bun", script, &["n"], jobs); + + assert_eq!(results.len(), 5); + for (i, result) in results.iter().enumerate() { + let expected = ((i + 1) * 2) as i64; + assert_eq!(*result, Ok(serde_json::json!(expected))); + } + } + + #[test] + fn test_dedicated_worker_bun_error() { + let script = r#" +export function main(msg: string): never { + throw new Error(msg); +} +"#; + let results = run_worker_test( + "bun", + script, + &["msg"], + vec![serde_json::json!({"msg": "test error"})], + ); + + assert_eq!(results.len(), 1); + assert!(results[0].is_err()); + assert_eq!(results[0], Err("test error".to_string())); + } +} + +// ============================================================================ +// Private Registry Tests +// ============================================================================ + +/// Test that bun can install packages from a private npm registry with authentication. +/// The registry requires auth tokens for accessing @windmill-test/* packages. +/// Requires: +/// - `private_registry_test` feature enabled +/// - `TEST_NPM_REGISTRY` environment variable set to registry URL with auth token +/// Format: `http://registry-url/:_authToken=TOKEN` +#[cfg(feature = "private_registry_test")] +#[sqlx::test(fixtures("base"))] +async fn test_bun_job_private_npm_registry(db: Pool) -> anyhow::Result<()> { + use windmill_worker::NPM_CONFIG_REGISTRY; + + let registry_url = std::env::var("TEST_NPM_REGISTRY") + .expect("TEST_NPM_REGISTRY must be set when running private_registry_test"); + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Set the private registry configuration + { + let mut registry = NPM_CONFIG_REGISTRY.write().await; + *registry = Some(registry_url.clone()); + } + + let content = r#" +import { greet } from "@windmill-test/private-pkg"; + +export function main(name: string) { + return greet(name); +} +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let result = RunJob::from(job) + .arg("name", serde_json::json!("World")) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + // Clean up + { + let mut registry = NPM_CONFIG_REGISTRY.write().await; + *registry = None; + } + + assert_eq!( + result, + serde_json::json!("Hello from private package, World!") + ); + Ok(()) +} + +/// Tests for RELATIVE_BUN_BUILDER (loader_builder.bun.js) +/// These tests verify Bun's behavior for import scanning and package.json generation. +/// Purpose: Catch regressions when upgrading Bun versions. +mod bun_builder_tests { + use std::process::{Command, Stdio}; + use windmill_worker::{BUN_PATH, RELATIVE_BUN_BUILDER, RELATIVE_BUN_LOADER}; + + /// Run the builder and return the generated package.json content + fn run_builder(main_ts_content: &str) -> serde_json::Value { + let temp_dir = tempfile::tempdir().unwrap(); + let dir = temp_dir.path(); + + // Write main.ts + std::fs::write(dir.join("main.ts"), main_ts_content).unwrap(); + + // Write build.js using the loader and builder constants directly + // Parameters are dummy values since tests don't use Windmill relative imports + let loader = RELATIVE_BUN_LOADER + .replace("W_ID", "test-workspace") + .replace("BASE_INTERNAL_URL", "http://localhost:8000") + .replace("TOKEN", "test-token") + .replace("CURRENT_PATH", "f/test/script") + .replace("RAW_GET_ENDPOINT", "raw"); + + let build_script = format!( + r#" +{loader} + +{RELATIVE_BUN_BUILDER} +"# + ); + std::fs::write(dir.join("build.js"), build_script).unwrap(); + + // Run bun build.js + let output = Command::new(BUN_PATH.as_str()) + .args(["run", "build.js"]) + .current_dir(dir) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .expect("Failed to run bun"); + + if !output.status.success() { + panic!( + "Builder failed:\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + // Read generated package.json + let package_json = std::fs::read_to_string(dir.join("package.json")) + .expect("package.json not generated"); + + serde_json::from_str(&package_json).expect("Invalid JSON in package.json") + } + + /// Test: scanImports() detects basic imports + #[test] + fn test_builder_simple_import() { + let main_ts = r#" +import lodash from "lodash"; +export function main() { return lodash; } +"#; + let pkg = run_builder(main_ts); + let deps = pkg["dependencies"].as_object().unwrap(); + + assert!(deps.contains_key("lodash"), "lodash should be in dependencies"); + assert_eq!(deps["lodash"], "latest"); + } + + /// Test: scanImports() preserves version info from versioned imports + #[test] + fn test_builder_versioned_import() { + let main_ts = r#" +import _ from "lodash@4.17.21"; +export function main() { return _; } +"#; + let pkg = run_builder(main_ts); + let deps = pkg["dependencies"].as_object().unwrap(); + + assert!(deps.contains_key("lodash"), "lodash should be in dependencies"); + assert_eq!(deps["lodash"], "4.17.21"); + } + + /// Test: scanImports() handles @scope/package correctly + #[test] + fn test_builder_scoped_package() { + let main_ts = r#" +import babel from "@babel/core"; +export function main() { return babel; } +"#; + let pkg = run_builder(main_ts); + let deps = pkg["dependencies"].as_object().unwrap(); + + assert!( + deps.contains_key("@babel/core"), + "@babel/core should be in dependencies" + ); + assert_eq!(deps["@babel/core"], "latest"); + } + + /// Test: scanImports() handles multiple imports + #[test] + fn test_builder_multiple_packages() { + let main_ts = r#" +import lodash from "lodash"; +import axios from "axios"; +import dayjs from "dayjs"; +export function main() { return { lodash, axios, dayjs }; } +"#; + let pkg = run_builder(main_ts); + let deps = pkg["dependencies"].as_object().unwrap(); + + assert!(deps.contains_key("lodash"), "lodash should be in dependencies"); + assert!(deps.contains_key("axios"), "axios should be in dependencies"); + assert!(deps.contains_key("dayjs"), "dayjs should be in dependencies"); + assert_eq!(deps.len(), 3, "Should have exactly 3 dependencies"); + } + + /// Test: isBuiltin() filters out Node.js builtins + #[test] + fn test_builder_builtin_skipped() { + let main_ts = r#" +import fs from "fs"; +import path from "path"; +import lodash from "lodash"; +export function main() { return { fs, path, lodash }; } +"#; + let pkg = run_builder(main_ts); + let deps = pkg["dependencies"].as_object().unwrap(); + + assert!( + !deps.contains_key("fs"), + "fs (builtin) should NOT be in dependencies" + ); + assert!( + !deps.contains_key("path"), + "path (builtin) should NOT be in dependencies" + ); + assert!(deps.contains_key("lodash"), "lodash should be in dependencies"); + assert_eq!(deps.len(), 1, "Should have exactly 1 dependency (lodash only)"); + } + + /// Test: semver.order() resolves version conflicts (picks lowest version) + #[test] + fn test_builder_version_conflict() { + // This test simulates what happens when the same package is imported with different versions + // The builder should use semver.order() to pick the lowest version + let main_ts = r#" +import a from "lodash@4.17.21"; +import b from "lodash@4.17.10"; +export function main() { return { a, b }; } +"#; + let pkg = run_builder(main_ts); + let deps = pkg["dependencies"].as_object().unwrap(); + + assert!(deps.contains_key("lodash"), "lodash should be in dependencies"); + // The builder sorts by semver and picks the first (lowest) version + assert_eq!( + deps["lodash"], "4.17.10", + "Should resolve to lower version 4.17.10" + ); + } +} diff --git a/backend/tests/common/mod.rs b/backend/tests/common/mod.rs index 2279b2c53c..ba89de3019 100644 --- a/backend/tests/common/mod.rs +++ b/backend/tests/common/mod.rs @@ -803,3 +803,47 @@ pub async fn rebuild_dmap(client: &windmill_api_client::Client) -> bool { .status() .is_success() } + +// ============================================================================ +// Dedicated Worker Protocol Helpers +// ============================================================================ + +/// Result from parsing a dedicated worker stdout line +#[derive(Debug, Clone, PartialEq)] +pub enum DedicatedWorkerResult { + /// Worker printed "start" indicating it's ready + Start, + /// Worker returned a successful result + Success(serde_json::Value), + /// Worker returned an error result + Error(serde_json::Value), + /// Line is not a protocol message (e.g., logs) + Other(String), +} + +/// Parse a line from dedicated worker stdout according to the protocol: +/// - "start" -> Ready signal +/// - "wm_res[success]:JSON" -> Success with result +/// - "wm_res[error]:JSON" -> Error with details +/// - anything else -> Other (logs) +pub fn parse_dedicated_worker_line(line: &str) -> DedicatedWorkerResult { + if line == "start" { + return DedicatedWorkerResult::Start; + } + + if let Some(json_str) = line.strip_prefix("wm_res[success]:") { + match serde_json::from_str(json_str) { + Ok(value) => return DedicatedWorkerResult::Success(value), + Err(_) => return DedicatedWorkerResult::Other(line.to_string()), + } + } + + if let Some(json_str) = line.strip_prefix("wm_res[error]:") { + match serde_json::from_str(json_str) { + Ok(value) => return DedicatedWorkerResult::Error(value), + Err(_) => return DedicatedWorkerResult::Other(line.to_string()), + } + } + + DedicatedWorkerResult::Other(line.to_string()) +} diff --git a/backend/tests/fixtures/bun_edge_cases.sql b/backend/tests/fixtures/bun_edge_cases.sql new file mode 100644 index 0000000000..46829ec9b3 --- /dev/null +++ b/backend/tests/fixtures/bun_edge_cases.sql @@ -0,0 +1,152 @@ +-- Fixture for Bun edge case tests +-- Tests deeply nested imports (level1 -> level2 -> level3) + +-- Level 3: Base script (deepest level) +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +export function main() { + return "level3"; +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/nested/level3', 20001, 'bun', ''); + +-- Level 2: Imports level3 using RELATIVE path (./level3.ts) +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +import { main as level3 } from "./level3.ts"; + +export function main() { + return "level2 -> " + level3(); +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/nested/level2', 20002, 'bun', ''); + +-- Level 1: Imports level2 using RELATIVE path (./level2.ts) +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +import { main as level2 } from "./level2.ts"; + +export function main() { + return "level1 -> " + level2(); +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/nested/level1', 20003, 'bun', ''); + +-- Script with preprocessor function +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, has_preprocessor) VALUES ( +'test-workspace', +'test-user', +' +export function preprocessor(value: number) { + return { value: value * 2 }; +} + +export function main(value: number) { + return value + 100; +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"value":{"type":"number"}},"required":["value"],"type":"object"}', +'Script with preprocessor', +'', +'f/edge_cases/with_preprocessor', 20004, 'bun', '', true); + +-- Script with nodejs annotation +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +'//nodejs + +export function main() { + return process.version; +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'NodeJS mode script', +'', +'f/edge_cases/nodejs_mode', 20005, 'bun', ''); + +-- Script with nobundling annotation +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +'//nobundling + +export function main() { + return "no bundle"; +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'No bundling mode script', +'', +'f/edge_cases/nobundling_mode', 20006, 'bun', ''); + +-- Script that uses circular-ish import pattern (A imports B, B imports C, test imports A and C) +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +export const SHARED_VALUE = "shared"; + +export function main() { + return SHARED_VALUE; +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/circular/shared', 20007, 'bun', ''); + +-- module_a uses RELATIVE path import (./shared.ts) +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +import { SHARED_VALUE } from "./shared.ts"; + +export function getValue() { + return "from_a_" + SHARED_VALUE; +} + +export function main() { + return getValue(); +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/circular/module_a', 20008, 'bun', ''); + +-- module_b uses ABSOLUTE path import (/f/circular/shared.ts) +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +import { SHARED_VALUE } from "/f/circular/shared.ts"; + +export function getValue() { + return "from_b_" + SHARED_VALUE; +} + +export function main() { + return getValue(); +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/circular/module_b', 20009, 'bun', ''); diff --git a/backend/tests/notify_events.rs b/backend/tests/notify_events.rs new file mode 100644 index 0000000000..a85a2f1c5a --- /dev/null +++ b/backend/tests/notify_events.rs @@ -0,0 +1,835 @@ +/*! + * Tests for the polling-based notify_event system that replaces PostgreSQL LISTEN/NOTIFY. + * + * These tests verify: + * 1. Database triggers correctly insert events into notify_event table + * 2. Polling functions retrieve events correctly + * 3. Cleanup functions delete old events + * 4. All notification channels work as expected + */ + +use sqlx::{Pool, Postgres}; +use windmill_common::notify_events::{cleanup_old_events, get_latest_event_id, poll_notify_events}; + +mod common; + +/// Helper to insert a test event directly +async fn insert_test_event(db: &Pool, channel: &str, payload: &str) -> i64 { + sqlx::query_scalar::<_, i64>( + "INSERT INTO notify_event (channel, payload) VALUES ($1, $2) RETURNING id", + ) + .bind(channel) + .bind(payload) + .fetch_one(db) + .await + .expect("Failed to insert test event") +} + +/// Helper to count events for a channel +async fn count_events_for_channel(db: &Pool, channel: &str) -> i64 { + let result: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM notify_event WHERE channel = $1") + .bind(channel) + .fetch_one(db) + .await + .expect("Failed to count events"); + result.0 +} + +// ============================================================================ +// Basic Functionality Tests +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_get_latest_event_id_returns_valid_id(db: Pool) { + // Get current latest id + let latest_id = get_latest_event_id(&db).await.expect("Should get latest event id"); + assert!(latest_id >= 0, "Latest id should be non-negative"); + + // Insert a new event and verify latest_id increases + let new_id = insert_test_event(&db, "test_latest_id", "payload").await; + let new_latest_id = get_latest_event_id(&db).await.expect("Should get latest event id"); + assert!(new_latest_id >= new_id, "Latest id should be >= new event id"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_get_latest_event_id_with_events(db: Pool) { + let _id1 = insert_test_event(&db, "test_channel_1", "payload1").await; + let _id2 = insert_test_event(&db, "test_channel_2", "payload2").await; + let id3 = insert_test_event(&db, "test_channel_3", "payload3").await; + + let latest_id = get_latest_event_id(&db).await.expect("Should get latest event id"); + assert!(latest_id >= id3, "Latest id should be >= last inserted id"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_poll_notify_events_no_new_events(db: Pool) { + // Get latest id first + let latest_id = get_latest_event_id(&db).await.unwrap(); + + // Poll from the latest id - should return empty since no new events + let events = poll_notify_events(&db, latest_id).await.expect("Should poll events"); + assert!(events.is_empty(), "Should return empty vec when polling from latest id"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_poll_notify_events_returns_new_events(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + let _id1 = insert_test_event(&db, "test_poll_channel", "payload1").await; + let _id2 = insert_test_event(&db, "test_poll_channel", "payload2").await; + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + assert!(events.len() >= 2, "Should return at least 2 new events"); + + // Verify the events we inserted are present + let our_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "test_poll_channel") + .collect(); + assert_eq!(our_events.len(), 2, "Should have exactly our 2 test events"); + + // Verify ordering (ascending by id) + assert!(our_events[0].id < our_events[1].id, "Events should be ordered by id ascending"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_poll_notify_events_respects_last_event_id(db: Pool) { + let id1 = insert_test_event(&db, "test_respect_id", "payload1").await; + let _id2 = insert_test_event(&db, "test_respect_id", "payload2").await; + let _id3 = insert_test_event(&db, "test_respect_id", "payload3").await; + + // Poll from id1 should only return id2 and id3 + let events = poll_notify_events(&db, id1).await.expect("Should poll events"); + let our_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "test_respect_id") + .collect(); + + assert_eq!(our_events.len(), 2, "Should only return events after id1"); + assert!(our_events.iter().all(|e| e.id > id1), "All events should have id > id1"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_cleanup_old_events(db: Pool) { + // Use unique channel names to avoid interference from other tests + let old_channel = format!("test_cleanup_old_{}", uuid::Uuid::new_v4()); + let recent_channel = format!("test_cleanup_recent_{}", uuid::Uuid::new_v4()); + + // Insert an event with old timestamp + sqlx::query( + "INSERT INTO notify_event (channel, payload, created_at) VALUES ($1, $2, now() - interval '15 minutes')", + ) + .bind(&old_channel) + .bind("old_payload") + .execute(&db) + .await + .expect("Failed to insert old event"); + + // Insert a recent event + sqlx::query( + "INSERT INTO notify_event (channel, payload) VALUES ($1, $2)", + ) + .bind(&recent_channel) + .bind("recent_payload") + .execute(&db) + .await + .expect("Failed to insert recent event"); + + // Count before cleanup + let old_count_before = count_events_for_channel(&db, &old_channel).await; + assert_eq!(old_count_before, 1, "Should have 1 old event before cleanup"); + + // Cleanup events older than 10 minutes + let deleted = cleanup_old_events(&db, 10).await.expect("Should cleanup events"); + assert!(deleted >= 1, "Should delete at least 1 old event"); + + // Verify old event is gone + let old_count = count_events_for_channel(&db, &old_channel).await; + assert_eq!(old_count, 0, "Old event should be deleted"); + + // Verify recent event is still there + let recent_count = count_events_for_channel(&db, &recent_channel).await; + assert_eq!(recent_count, 1, "Recent event should still exist"); +} + +// ============================================================================ +// Database Trigger Tests - Verify triggers insert events correctly +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_notify_config_change(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Insert or update a config entry + sqlx::query( + "INSERT INTO config (name, config) VALUES ('test_config_trigger', '{}'::jsonb) + ON CONFLICT (name) DO UPDATE SET config = '{}'::jsonb", + ) + .execute(&db) + .await + .expect("Failed to insert config"); + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let config_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "notify_config_change" && e.payload == "test_config_trigger") + .collect(); + + assert!(!config_events.is_empty(), "Should have notify_config_change event"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_notify_global_setting_change_insert(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Use a unique setting name for testing + let setting_name = format!("test_setting_{}", uuid::Uuid::new_v4()); + + // Insert a global setting + sqlx::query("INSERT INTO global_settings (name, value) VALUES ($1, '{}'::jsonb)") + .bind(&setting_name) + .execute(&db) + .await + .expect("Failed to insert global setting"); + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let setting_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "notify_global_setting_change" && e.payload == setting_name) + .collect(); + + assert!(!setting_events.is_empty(), "Should have notify_global_setting_change event on insert"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_notify_global_setting_change_update(db: Pool) { + // Use a unique setting name for testing + let setting_name = format!("test_setting_update_{}", uuid::Uuid::new_v4()); + + // First insert + sqlx::query("INSERT INTO global_settings (name, value) VALUES ($1, '{}'::jsonb)") + .bind(&setting_name) + .execute(&db) + .await + .expect("Failed to insert global setting"); + + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Update the setting + sqlx::query("UPDATE global_settings SET value = '{\"updated\": true}'::jsonb WHERE name = $1") + .bind(&setting_name) + .execute(&db) + .await + .expect("Failed to update global setting"); + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let setting_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "notify_global_setting_change" && e.payload == setting_name) + .collect(); + + assert!(!setting_events.is_empty(), "Should have notify_global_setting_change event on update"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_notify_global_setting_change_delete(db: Pool) { + // Use a unique setting name for testing + let setting_name = format!("test_setting_delete_{}", uuid::Uuid::new_v4()); + + // First insert + sqlx::query("INSERT INTO global_settings (name, value) VALUES ($1, '{}'::jsonb)") + .bind(&setting_name) + .execute(&db) + .await + .expect("Failed to insert global setting"); + + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Delete the setting + sqlx::query("DELETE FROM global_settings WHERE name = $1") + .bind(&setting_name) + .execute(&db) + .await + .expect("Failed to delete global setting"); + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let setting_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "notify_global_setting_change" && e.payload == setting_name) + .collect(); + + assert!(!setting_events.is_empty(), "Should have notify_global_setting_change event on delete"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_notify_workspace_envs_change(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Insert a workspace env (test-workspace exists from fixture) + sqlx::query( + "INSERT INTO workspace_env (workspace_id, name, value) VALUES ('test-workspace', 'TEST_ENV_VAR', 'test_value') + ON CONFLICT (workspace_id, name) DO UPDATE SET value = 'test_value_updated'", + ) + .execute(&db) + .await + .expect("Failed to insert workspace env"); + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let env_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "notify_workspace_envs_change" && e.payload == "test-workspace") + .collect(); + + assert!(!env_events.is_empty(), "Should have notify_workspace_envs_change event"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_notify_workspace_key_change(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Insert a workspace key (base fixture already has one, so this will conflict and update) + sqlx::query( + "INSERT INTO workspace_key (workspace_id, kind, key) VALUES ('test-workspace', 'cloud', 'test_key_value') + ON CONFLICT (workspace_id, kind) DO UPDATE SET key = 'test_key_value_updated'", + ) + .execute(&db) + .await + .expect("Failed to insert workspace key"); + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let key_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "notify_workspace_key_change" && e.payload == "test-workspace") + .collect(); + + assert!(!key_events.is_empty(), "Should have notify_workspace_key_change event"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_notify_token_invalidation(db: Pool) { + // First insert a session token + let token = format!("test_token_{}", uuid::Uuid::new_v4()); + sqlx::query( + "INSERT INTO token (token, label, email, workspace_id, owner, expiration) + VALUES ($1, 'session', 'test@test.com', 'test-workspace', 'test-user', now() + interval '1 hour')", + ) + .bind(&token) + .execute(&db) + .await + .expect("Failed to insert token"); + + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Delete the token (should trigger notification) + sqlx::query("DELETE FROM token WHERE token = $1") + .bind(&token) + .execute(&db) + .await + .expect("Failed to delete token"); + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let token_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "notify_token_invalidation" && e.payload == token) + .collect(); + + assert!(!token_events.is_empty(), "Should have notify_token_invalidation event"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_notify_webhook_change(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Update webhook setting (workspace_settings exists from base fixture) + sqlx::query("UPDATE workspace_settings SET webhook = 'https://test.webhook.com' WHERE workspace_id = 'test-workspace'") + .execute(&db) + .await + .expect("Failed to update webhook"); + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let webhook_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "notify_webhook_change" && e.payload == "test-workspace") + .collect(); + + assert!(!webhook_events.is_empty(), "Should have notify_webhook_change event"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_notify_workspace_premium_change(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Toggle premium status + sqlx::query("UPDATE workspace SET premium = NOT premium WHERE id = 'test-workspace'") + .execute(&db) + .await + .expect("Failed to update workspace premium"); + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let premium_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "notify_workspace_premium_change" && e.payload == "test-workspace") + .collect(); + + assert!(!premium_events.is_empty(), "Should have notify_workspace_premium_change event"); +} + +// ============================================================================ +// HTTP Trigger Tests +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_notify_http_trigger_change(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + let trigger_path = format!("test_http_trigger_{}", uuid::Uuid::new_v4()); + + // Insert an HTTP trigger + sqlx::query( + "INSERT INTO http_trigger (path, route_path, route_path_key, script_path, is_flow, workspace_id, edited_by, email, http_method, authentication_method) + VALUES ($1, '/test/route', '/test/route', 'test/script', false, 'test-workspace', 'test-user', 'test@test.com', 'get', 'none')", + ) + .bind(&trigger_path) + .execute(&db) + .await + .expect("Failed to insert HTTP trigger"); + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let http_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "notify_http_trigger_change") + .filter(|e| e.payload.contains("test-workspace") && e.payload.contains(&trigger_path)) + .collect(); + + assert!(!http_events.is_empty(), "Should have notify_http_trigger_change event"); +} + +// ============================================================================ +// Script/Flow Version Change Tests +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_notify_runnable_version_change_script(db: Pool) { + // First create a script without lock + let script_path = format!("f/test/script_{}", uuid::Uuid::new_v4()); + let script_hash: i64 = rand::random::().abs(); + + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, language, kind) + VALUES ('test-workspace', $1, $2, 'test', 'test', 'def main(): pass', 'test-user', 'python3', 'script')", + ) + .bind(script_hash) + .bind(&script_path) + .execute(&db) + .await + .expect("Failed to insert script"); + + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Update the lock field (this should trigger the notification) + sqlx::query("UPDATE script SET lock = 'test_lock_content' WHERE hash = $1") + .bind(script_hash) + .execute(&db) + .await + .expect("Failed to update script lock"); + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let script_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "notify_runnable_version_change") + .filter(|e| e.payload.contains("test-workspace") && e.payload.contains("script")) + .collect(); + + assert!(!script_events.is_empty(), "Should have notify_runnable_version_change event for script"); + + // Verify payload format: workspace_id:source_type:path:kind + let parts: Vec<&str> = script_events[0].payload.split(':').collect(); + assert!(parts.len() >= 4, "Payload should have at least 4 parts"); + assert_eq!(parts[0], "test-workspace", "First part should be workspace_id"); + assert_eq!(parts[1], "script", "Second part should be 'script'"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_notify_runnable_version_change_flow(db: Pool) { + // First create a flow with empty versions array + let flow_path = format!("f/test/flow_{}", uuid::Uuid::new_v4()); + + sqlx::query( + "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, schema, versions) + VALUES ('test-workspace', $1, 'test', 'test', '{}'::jsonb, 'test-user', '{}'::json, ARRAY[]::bigint[])", + ) + .bind(&flow_path) + .execute(&db) + .await + .expect("Failed to insert flow"); + + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Update the flow's versions array (this triggers flow_versions_append_trigger) + sqlx::query( + "UPDATE flow SET versions = array_append(versions, 1::bigint) WHERE workspace_id = 'test-workspace' AND path = $1", + ) + .bind(&flow_path) + .execute(&db) + .await + .expect("Failed to update flow versions"); + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let flow_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "notify_runnable_version_change") + .filter(|e| e.payload.contains("test-workspace") && e.payload.contains("flow")) + .collect(); + + assert!(!flow_events.is_empty(), "Should have notify_runnable_version_change event for flow"); + + // Verify payload format + let parts: Vec<&str> = flow_events[0].payload.split(':').collect(); + assert!(parts.len() >= 4, "Payload should have at least 4 parts"); + assert_eq!(parts[0], "test-workspace", "First part should be workspace_id"); + assert_eq!(parts[1], "flow", "Second part should be 'flow'"); +} + +// ============================================================================ +// Concurrent Access Tests +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_concurrent_event_insertion(db: Pool) { + // Use a unique channel name for this test run + let channel = format!("test_concurrent_{}", uuid::Uuid::new_v4()); + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Insert multiple events concurrently + let handles: Vec<_> = (0..10) + .map(|i| { + let db = db.clone(); + let ch = channel.clone(); + tokio::spawn(async move { + sqlx::query_scalar::<_, i64>( + "INSERT INTO notify_event (channel, payload) VALUES ($1, $2) RETURNING id", + ) + .bind(&ch) + .bind(format!("payload_{}", i)) + .fetch_one(&db) + .await + .expect("Failed to insert event") + }) + }) + .collect(); + + // Wait for all insertions + for handle in handles { + handle.await.expect("Task should complete"); + } + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let concurrent_events: Vec<_> = events + .iter() + .filter(|e| e.channel == channel) + .collect(); + + assert_eq!(concurrent_events.len(), 10, "Should have all 10 concurrent events"); + + // Verify all events have unique IDs + let ids: std::collections::HashSet = concurrent_events.iter().map(|e| e.id).collect(); + assert_eq!(ids.len(), 10, "All events should have unique IDs"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_polling_isolation(db: Pool) { + // Use a unique channel name for this test + let channel = format!("test_isolation_{}", uuid::Uuid::new_v4()); + + // Get baseline before inserting + let baseline_id = get_latest_event_id(&db).await.unwrap(); + + // Insert some events + let id1 = sqlx::query_scalar::<_, i64>( + "INSERT INTO notify_event (channel, payload) VALUES ($1, $2) RETURNING id", + ) + .bind(&channel) + .bind("payload1") + .fetch_one(&db) + .await + .expect("Failed to insert event"); + + let id2 = sqlx::query_scalar::<_, i64>( + "INSERT INTO notify_event (channel, payload) VALUES ($1, $2) RETURNING id", + ) + .bind(&channel) + .bind("payload2") + .fetch_one(&db) + .await + .expect("Failed to insert event"); + + let _id3 = sqlx::query_scalar::<_, i64>( + "INSERT INTO notify_event (channel, payload) VALUES ($1, $2) RETURNING id", + ) + .bind(&channel) + .bind("payload3") + .fetch_one(&db) + .await + .expect("Failed to insert event"); + + // Two different "consumers" polling from different points + let events_from_baseline = poll_notify_events(&db, baseline_id).await.expect("Should poll events"); + let events_from_id1 = poll_notify_events(&db, id1).await.expect("Should poll events"); + let events_from_id2 = poll_notify_events(&db, id2).await.expect("Should poll events"); + + // Filter to our test events + let from_baseline: Vec<_> = events_from_baseline.iter().filter(|e| e.channel == channel).collect(); + let from_id1: Vec<_> = events_from_id1.iter().filter(|e| e.channel == channel).collect(); + let from_id2: Vec<_> = events_from_id2.iter().filter(|e| e.channel == channel).collect(); + + assert_eq!(from_baseline.len(), 3, "Polling from baseline should include all 3 events"); + assert_eq!(from_id1.len(), 2, "Polling from id1 should include id2 and id3"); + assert_eq!(from_id2.len(), 1, "Polling from id2 should include only id3"); +} + +// ============================================================================ +// Edge Case Tests +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_empty_payload(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + insert_test_event(&db, "test_empty_payload", "").await; + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let empty_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "test_empty_payload") + .collect(); + + assert_eq!(empty_events.len(), 1, "Should have event with empty payload"); + assert_eq!(empty_events[0].payload, "", "Payload should be empty string"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_large_payload(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + // Create a large payload (1KB) + let large_payload = "x".repeat(1024); + insert_test_event(&db, "test_large_payload", &large_payload).await; + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let large_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "test_large_payload") + .collect(); + + assert_eq!(large_events.len(), 1, "Should have event with large payload"); + assert_eq!(large_events[0].payload.len(), 1024, "Payload should be preserved"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_special_characters_in_payload(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + let special_payload = r#"{"key": "value with \"quotes\" and 'apostrophes'", "unicode": "日本語", "newline": "line1\nline2"}"#; + insert_test_event(&db, "test_special_chars", special_payload).await; + + let events = poll_notify_events(&db, before_id).await.expect("Should poll events"); + let special_events: Vec<_> = events + .iter() + .filter(|e| e.channel == "test_special_chars") + .collect(); + + assert_eq!(special_events.len(), 1, "Should have event with special characters"); + assert_eq!(special_events[0].payload, special_payload, "Special characters should be preserved"); +} + +#[sqlx::test(fixtures("base"))] +async fn test_cleanup_with_no_old_events(db: Pool) { + // Use a unique channel name for this test + let channel = format!("test_no_old_{}", uuid::Uuid::new_v4()); + + // Insert only recent events + sqlx::query("INSERT INTO notify_event (channel, payload) VALUES ($1, $2)") + .bind(&channel) + .bind("recent1") + .execute(&db) + .await + .expect("Failed to insert event"); + + sqlx::query("INSERT INTO notify_event (channel, payload) VALUES ($1, $2)") + .bind(&channel) + .bind("recent2") + .execute(&db) + .await + .expect("Failed to insert event"); + + let before_count = count_events_for_channel(&db, &channel).await; + assert_eq!(before_count, 2, "Should have 2 recent events"); + + // Cleanup old events (none of our events should be deleted since they're recent) + let _deleted = cleanup_old_events(&db, 10).await.expect("Should cleanup events"); + + let after_count = count_events_for_channel(&db, &channel).await; + assert_eq!(after_count, 2, "Recent events should not be deleted"); +} + +// ============================================================================ +// Multi-Server Integration Tests +// ============================================================================ +// These tests start two actual windmill server processes on different ports +// with LISTEN_NEW_EVENTS_INTERVAL_SEC=1, trigger DB changes, and verify +// both servers process the events via their log output. + +use std::io::{BufRead, BufReader}; +use std::process::{Child, Command, Stdio}; +use std::sync::{Arc, Mutex}; + +struct ServerProcess { + child: Child, + log_lines: Arc>>, + _stdout_handle: std::thread::JoinHandle<()>, + _stderr_handle: std::thread::JoinHandle<()>, +} + +impl ServerProcess { + fn start(port: u16, db_url: &str) -> Self { + let binary = std::env::var("WINDMILL_BINARY") + .unwrap_or_else(|_| format!("{}/target/debug/windmill", env!("CARGO_MANIFEST_DIR"))); + + let mut child = Command::new(&binary) + .env("DATABASE_URL", db_url) + .env("MODE", "server") + .env("PORT", port.to_string()) + .env("LISTEN_NEW_EVENTS_INTERVAL_SEC", "1") + .env("RUST_LOG", "info") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap_or_else(|e| panic!("Failed to start windmill on port {port}: {e}")); + + let stdout = child.stdout.take().expect("Failed to capture stdout"); + let stderr = child.stderr.take().expect("Failed to capture stderr"); + let log_lines = Arc::new(Mutex::new(Vec::new())); + let log_lines_stdout = log_lines.clone(); + let log_lines_stderr = log_lines.clone(); + + // Read both stdout and stderr into the same log buffer + let _reader_handle = std::thread::spawn(move || { + let reader = BufReader::new(stdout); + for line in reader.lines() { + if let Ok(line) = line { + log_lines_stdout.lock().unwrap().push(line); + } + } + }); + let _stderr_handle = std::thread::spawn(move || { + let reader = BufReader::new(stderr); + for line in reader.lines() { + if let Ok(line) = line { + log_lines_stderr.lock().unwrap().push(line); + } + } + }); + + ServerProcess { child, log_lines, _stdout_handle: _reader_handle, _stderr_handle } + } + + fn logs_contain(&self, needle: &str) -> bool { + self.log_lines.lock().unwrap().iter().any(|l| l.contains(needle)) + } + + fn dump_logs(&self) -> String { + self.log_lines.lock().unwrap().join("\n") + } +} + +impl Drop for ServerProcess { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// Wait for server to be ready by polling its HTTP endpoint. +async fn wait_for_server(port: u16, timeout_secs: u64) -> bool { + let client = reqwest::Client::new(); + let url = format!("http://127.0.0.1:{}/api/version", port); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(timeout_secs); + while tokio::time::Instant::now() < deadline { + if client.get(&url).send().await.is_ok() { + return true; + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + false +} + +/// Helper to get a database connection (only used by the e2e multi-server test) +async fn get_db() -> Pool { + let database_url = std::env::var("DATABASE_URL") + .unwrap_or_else(|_| "postgres://postgres:changeme@localhost:5432/windmill".to_string()); + sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .connect(&database_url) + .await + .expect("Failed to connect to database") +} + +#[tokio::test] +#[ignore = "slow - starts two server processes with 1s poll interval"] +async fn test_two_server_processes_both_receive_event() { + let db_url = std::env::var("DATABASE_URL") + .unwrap_or_else(|_| "postgres://postgres:changeme@localhost:5432/windmill".to_string()); + + // Start two server processes on different ports with 1s poll interval + let mut server_a = ServerProcess::start(19100, &db_url); + let mut server_b = ServerProcess::start(19200, &db_url); + + // Wait for both servers to be ready + let (ready_a, ready_b) = tokio::join!( + wait_for_server(19100, 30), + wait_for_server(19200, 30), + ); + assert!(ready_a, "Server A (port 19100) failed to start. Logs:\n{}", server_a.dump_logs()); + assert!(ready_b, "Server B (port 19200) failed to start. Logs:\n{}", server_b.dump_logs()); + + // Give servers a moment to complete their first poll cycle + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + + // Trigger a global setting change via direct DB insert + let db = get_db().await; + let setting_name = format!("test_e2e_{}", uuid::Uuid::new_v4()); + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, '\"e2e_test\"'::jsonb) + ON CONFLICT (name) DO UPDATE SET value = '\"e2e_test\"'::jsonb", + ) + .bind(&setting_name) + .execute(&db) + .await + .expect("Failed to insert global setting"); + + // Wait for at least 2 poll cycles (interval is 1s) + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + + let needle = format!("Global setting change detected: {}", setting_name); + assert!( + server_a.logs_contain(&needle), + "Server A should have processed the global setting event.\nSearching for: {}\nServer A logs:\n{}", + needle, server_a.dump_logs() + ); + assert!( + server_b.logs_contain(&needle), + "Server B should have processed the global setting event.\nSearching for: {}\nServer B logs:\n{}", + needle, server_b.dump_logs() + ); + + // Cleanup + sqlx::query("DELETE FROM global_settings WHERE name = $1") + .bind(&setting_name) + .execute(&db) + .await + .ok(); + + // Explicitly kill before drop to avoid port conflicts with other tests + let _ = server_a.child.kill(); + let _ = server_b.child.kill(); +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 6ba9e8935f..4ca0fc21f4 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.623.1 + version: 1.624.0 title: Windmill API contact: @@ -4021,6 +4021,13 @@ paths: - setting parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: include_default + in: query + description: If true, include "_default_" in the list if primary workspace storage is set + required: false + schema: + type: boolean + default: false responses: "200": description: status @@ -16579,6 +16586,11 @@ paths: $ref: "#/components/schemas/AssetUsageKind" access_type: $ref: "#/components/schemas/AssetUsageAccessType" + columns: + type: object + description: The columns used (for tables) + additionalProperties: + $ref: "#/components/schemas/AssetUsageAccessType" created_at: type: string format: date-time @@ -20763,6 +20775,17 @@ components: type: array items: type: string + filters: + type: array + items: + type: object + properties: + key: + type: string + value: {} + required: + - key + - value server_id: type: string last_server_ping: @@ -20781,6 +20804,7 @@ components: - kafka_resource_path - group_id - topics + - filters NewKafkaTrigger: type: object @@ -20799,6 +20823,17 @@ components: type: array items: type: string + filters: + type: array + items: + type: object + properties: + key: + type: string + value: {} + required: + - key + - value mode: $ref: "#/components/schemas/TriggerMode" error_handler_path: @@ -20815,6 +20850,7 @@ components: - kafka_resource_path - group_id - topics + - filters EditKafkaTrigger: type: object @@ -20827,6 +20863,17 @@ components: type: array items: type: string + filters: + type: array + items: + type: object + properties: + key: + type: string + value: {} + required: + - key + - value path: type: string script_path: @@ -20846,6 +20893,7 @@ components: - kafka_resource_path - group_id - topics + - filters - is_flow NatsTrigger: diff --git a/backend/windmill-api/src/assets.rs b/backend/windmill-api/src/assets.rs index 94d4b3877e..7afea4038f 100644 --- a/backend/windmill-api/src/assets.rs +++ b/backend/windmill-api/src/assets.rs @@ -158,6 +158,7 @@ async fn list_assets( 'path', asset.usage_path, 'kind', asset.usage_kind, 'access_type', asset.usage_access_type, + 'columns', asset.columns, 'created_at', asset.created_at, 'metadata', (CASE WHEN asset.usage_kind = 'job' THEN @@ -266,11 +267,12 @@ async fn list_assets_by_usages( for usage in body.usages { let assets = sqlx::query_scalar!( r#"SELECT - jsonb_build_object( + jsonb_strip_nulls(jsonb_build_object( 'path', path, 'kind', kind, - 'access_type', usage_access_type - ) as "list!: _" + 'access_type', usage_access_type, + 'columns', columns + )) as "list!: _" FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3 ORDER BY path, kind"#, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 48cfb80182..15adf46caa 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -6378,10 +6378,18 @@ fn register_potential_assets_on_inline_execution( match assets { Some(Ok(assets)) => { for asset in assets { + let columns = asset.columns.as_ref().map(|cols| { + cols.iter() + .map(|(col_name, col_access_type)| { + (col_name.clone(), (*col_access_type).into()) + }) + .collect() + }); register_runtime_asset(InsertRuntimeAssetParams { access_type: asset.access_type.map(|a| a.into()), asset_kind: asset.kind.into(), asset_path: asset.path, + columns, job_id, workspace_id: w_id.to_string(), created_at: None, diff --git a/backend/windmill-api/src/triggers/filter.rs b/backend/windmill-api/src/triggers/filter.rs new file mode 100644 index 0000000000..553e0a70f5 --- /dev/null +++ b/backend/windmill-api/src/triggers/filter.rs @@ -0,0 +1,144 @@ +use serde::{ + de::{self, MapAccess, Visitor}, + Deserialize, Deserializer, +}; +use serde_json::Value; +use std::fmt; + +#[derive(Deserialize)] +pub struct JsonFilter { + pub key: String, + pub value: Value, +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum Filter { + JsonFilter(JsonFilter), +} + +struct SupersetVisitor<'a> { + key: &'a str, + value_to_check: &'a Value, +} + +impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> { + type Value = bool; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JSON object with a specific key at the top level") + } + + fn visit_map(self, mut map: V) -> std::result::Result + where + V: MapAccess<'de>, + { + let mut result = false; + let mut found = false; + + // Must consume entire map to satisfy deserializer contract + while let Some(key) = map.next_key::()? { + if !found && key == self.key { + let json_value: Value = map.next_value()?; + result = is_superset(&json_value, self.value_to_check); + found = true; + } else { + // Skip values we don't need (cheaper than full deserialization) + let _ = map.next_value::()?; + } + } + Ok(result) + } +} + +pub fn is_superset(json_value: &Value, value_to_check: &Value) -> bool { + match (json_value, value_to_check) { + (Value::Object(json_map), Value::Object(check_map)) => { + check_map.iter().all(|(k, v)| { + json_map + .get(k) + .map_or(false, |json_val| is_superset(json_val, v)) + }) + } + (Value::Array(json_array), Value::Array(check_array)) => { + check_array.iter().all(|check_item| { + json_array + .iter() + .any(|json_item| is_superset(json_item, check_item)) + }) + } + _ => json_value == value_to_check, + } +} + +pub fn is_value_superset<'a, 'de, D>( + deserializer: D, + key: &'a str, + value_to_check: &'a Value, +) -> std::result::Result +where + D: Deserializer<'de>, +{ + deserializer.deserialize_map(SupersetVisitor { key, value_to_check }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_filter_with_other_top_level_keys() { + let payload = r#"{"event_type": "test", "other": "data"}"#; + let key = "event_type"; + let value = json!("test"); + + let mut deserializer = serde_json::Deserializer::from_str(payload); + let result = is_value_superset(&mut deserializer, key, &value).unwrap(); + assert!(result, "Should match when key exists with correct value"); + } + + #[test] + fn test_filter_with_key_not_first() { + let payload = r#"{"other": "data", "event_type": "test"}"#; + let key = "event_type"; + let value = json!("test"); + + let mut deserializer = serde_json::Deserializer::from_str(payload); + let result = is_value_superset(&mut deserializer, key, &value).unwrap(); + assert!(result, "Should match even when key is not first"); + } + + #[test] + fn test_filter_with_nested_object() { + let payload = r#"{"data": {"status": "active", "count": 5}, "other": "value"}"#; + let key = "data"; + let value = json!({"status": "active"}); + + let mut deserializer = serde_json::Deserializer::from_str(payload); + let result = is_value_superset(&mut deserializer, key, &value).unwrap(); + assert!(result, "Should match when nested object is superset"); + } + + #[test] + fn test_filter_no_match() { + let payload = r#"{"event_type": "other", "data": "value"}"#; + let key = "event_type"; + let value = json!("test"); + + let mut deserializer = serde_json::Deserializer::from_str(payload); + let result = is_value_superset(&mut deserializer, key, &value).unwrap(); + assert!(!result, "Should not match when value differs"); + } + + #[test] + fn test_filter_key_not_found() { + let payload = r#"{"other": "data"}"#; + let key = "event_type"; + let value = json!("test"); + + let mut deserializer = serde_json::Deserializer::from_str(payload); + let result = is_value_superset(&mut deserializer, key, &value).unwrap(); + assert!(!result, "Should not match when key doesn't exist"); + } +} diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index 07153f9573..a38532c46d 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -963,7 +963,13 @@ async fn route_job( let s3_object = s3_object.map_err(|err| { tracing::warn!("Error retrieving file from S3: {:?}", err); - Error::internal_err(format!("Error retrieving file: {}", err.to_string())) + let mut msg = format!("Error retrieving file: {err}"); + let mut source = std::error::Error::source(&err); + while let Some(e) = source { + msg.push_str(&format!("\n caused by: {e}")); + source = e.source(); + } + Error::internal_err(msg) })?; let mut response_headers = http::HeaderMap::new(); diff --git a/backend/windmill-api/src/triggers/mod.rs b/backend/windmill-api/src/triggers/mod.rs index b73f27ed33..ce8538de2d 100644 --- a/backend/windmill-api/src/triggers/mod.rs +++ b/backend/windmill-api/src/triggers/mod.rs @@ -30,6 +30,7 @@ pub mod sqs; #[cfg(feature = "websocket")] pub mod websocket; +pub mod filter; pub mod global_handler; mod handler; mod listener; diff --git a/backend/windmill-api/src/triggers/websocket/listener.rs b/backend/windmill-api/src/triggers/websocket/listener.rs index 9d4c2209e2..0f22dffdc3 100644 --- a/backend/windmill-api/src/triggers/websocket/listener.rs +++ b/backend/windmill-api/src/triggers/websocket/listener.rs @@ -1,5 +1,6 @@ use super::WebsocketTrigger; use crate::triggers::{ + filter::{is_value_superset, Filter, JsonFilter}, listener::ListeningTrigger, trigger_helpers::{ trigger_runnable, trigger_runnable_and_wait_for_raw_result, @@ -13,12 +14,9 @@ use async_trait::async_trait; use futures::{stream::SplitSink, SinkExt, StreamExt}; use http::Response; use itertools::Itertools; -use serde::{ - de::{self, MapAccess, Visitor}, - Deserialize, Deserializer, -}; -use serde_json::{value::RawValue, Value}; -use std::{borrow::Cow, collections::HashMap, fmt, sync::Arc}; +use serde::Deserialize; +use serde_json::value::RawValue; +use std::{borrow::Cow, collections::HashMap, sync::Arc}; use tokio::{net::TcpStream, sync::RwLock}; use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; use windmill_common::{ @@ -443,18 +441,6 @@ impl Listener for WebsocketTrigger { } } -#[derive(Deserialize)] -pub struct JsonFilter { - key: String, - value: Value, -} - -#[derive(Deserialize)] -#[serde(untagged)] -pub enum Filter { - JsonFilter(JsonFilter), -} - pub struct ReturnMessageChannels { send_message_tx: tokio::sync::mpsc::Sender, killpill_rx: tokio::sync::broadcast::Receiver<()>, @@ -477,71 +463,6 @@ enum InitialMessage { RunnableResult { path: String, args: Box, is_flow: bool }, } -struct SupersetVisitor<'a> { - key: &'a str, - value_to_check: &'a Value, -} - -impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> { - type Value = bool; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a JSON object with a specific key at the top level") - } - - fn visit_map(self, mut map: V) -> std::result::Result - where - V: MapAccess<'de>, - { - while let Some(key) = map.next_key::()? { - if key == self.key { - // Deserialize the value for the key and check if it's a superset - let json_value: Value = map.next_value()?; - return Ok(is_superset(&json_value, self.value_to_check)); - } else { - // Skip the value if it's not the one we're interested in - let _ = map.next_value::()?; - } - } - // If the key was not found, return false - Ok(false) - } -} - -fn is_superset(json_value: &Value, value_to_check: &Value) -> bool { - match (json_value, value_to_check) { - (Value::Object(json_map), Value::Object(check_map)) => { - // Check that all keys and values in check_map exist and match in json_map - check_map.iter().all(|(k, v)| { - json_map - .get(k) - .map_or(false, |json_val| is_superset(json_val, v)) - }) - } - (Value::Array(json_array), Value::Array(check_array)) => { - // Check that all elements in check_array exist in json_array - check_array.iter().all(|check_item| { - json_array - .iter() - .any(|json_item| is_superset(json_item, check_item)) - }) - } - _ => json_value == value_to_check, - } -} - -// A function to deserialize and check if the value at the given key is a superset of a passed value -fn is_value_superset<'a, 'de, D>( - deserializer: D, - key: &'a str, - value_to_check: &'a Value, -) -> std::result::Result -where - D: Deserializer<'de>, -{ - deserializer.deserialize_map(SupersetVisitor { key, value_to_check }) -} - fn raw_value_to_args_hashmap( args: Option<&Box>, ) -> Result>> { diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index da56b0dcd2..537785887b 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -895,18 +895,43 @@ async fn delete_slack_oauth_config( )) } +#[derive(Deserialize)] +struct GetSecondaryStorageNamesQuery { + #[serde(default)] + include_default: bool, +} + async fn get_secondary_storage_names( _authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, + Query(query): Query, ) -> JsonResult> { - let result: Vec = sqlx::query_scalar!( + let mut result: Vec = sqlx::query_scalar!( "SELECT jsonb_object_keys(large_file_storage->'secondary_storage') AS \"secondary_storage_name!: _\" FROM workspace_settings WHERE workspace_id = $1", &w_id ) .fetch_all(&db) .await?; + + // If include_default is true, check if primary storage is set and add "_default_" + if query.include_default { + let has_primary_storage: Option = sqlx::query_scalar!( + "SELECT (large_file_storage IS NOT NULL + AND large_file_storage != 'null'::jsonb + AND jsonb_typeof(large_file_storage) = 'object') AS \"has_primary!\" + FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&db) + .await?; + + if has_primary_storage.unwrap_or(false) { + result.insert(0, "_default_".to_string()); + } + } + Ok(Json(result)) } diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index a27070feee..eb4c482c67 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; use sqlx::PgExecutor; +use std::collections::BTreeMap; use crate::{error, scripts::ScriptHash}; @@ -37,23 +38,15 @@ pub enum AssetUsageAccessType { RW, } -pub struct Asset { - pub path: String, - pub kind: AssetKind, -} - -pub struct AssetUsage { - pub path: String, - pub kind: AssetUsageKind, - pub access_type: AssetUsageAccessType, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Hash, sqlx::Type)] +#[derive(Serialize, Deserialize, Debug, Clone, Hash)] pub struct AssetWithAltAccessType { pub path: String, pub kind: AssetKind, pub access_type: Option, pub alt_access_type: Option, + /// Map of column name to access type for column-level access tracking + #[serde(skip_serializing_if = "Option::is_none")] + pub columns: Option>, } pub async fn insert_static_asset_usage<'e>( @@ -63,15 +56,22 @@ pub async fn insert_static_asset_usage<'e>( usage_path: &str, usage_kind: AssetUsageKind, ) -> error::Result<()> { + // Convert columns BTreeMap to JSONB format + let columns_json = asset + .columns + .as_ref() + .map(|cols| serde_json::to_value(cols).unwrap_or(serde_json::Value::Null)); + sqlx::query!( - r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) - VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING"#, + r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns) + VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING"#, workspace_id, asset.path, asset.kind as AssetKind, (asset.access_type.or(asset.alt_access_type)) as Option, usage_path, - usage_kind as AssetUsageKind + usage_kind as AssetUsageKind, + columns_json as Option ) .execute(executor) .await?; @@ -125,6 +125,28 @@ pub fn merge_asset_usage_access_types( } } +pub fn merge_asset_columns( + a: &Option>, + b: &Option>, +) -> Option> { + match (a, b) { + (None, None) => None, + (Some(cols), None) | (None, Some(cols)) => Some(cols.clone()), + (Some(cols_a), Some(cols_b)) => { + let mut merged = cols_a.clone(); + for (col, access_b) in cols_b { + let access_a = merged.get(col); + let merged_access = + merge_asset_usage_access_types(access_a.cloned(), Some(*access_b)); + if let Some(access) = merged_access { + merged.insert(col.clone(), access); + } + } + Some(merged) + } + } +} + impl From for AssetKind { fn from(parser_kind: windmill_parser::asset_parser::AssetKind) -> Self { match parser_kind { diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 83e7a2312f..3fd4de438a 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -60,6 +60,7 @@ pub mod job_metrics; #[cfg(all(feature = "parquet", feature = "private"))] pub mod job_s3_helpers_ee; pub mod min_version; +pub mod notify_events; #[cfg(feature = "parquet")] pub mod job_s3_helpers_oss; pub mod workspace_dependencies; diff --git a/backend/windmill-common/src/notify_events.rs b/backend/windmill-common/src/notify_events.rs new file mode 100644 index 0000000000..e00b6d7d49 --- /dev/null +++ b/backend/windmill-common/src/notify_events.rs @@ -0,0 +1,63 @@ +/* + * Author: Windmill Labs + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Polling-based event notification system. +//! +//! This module provides a table-based alternative to PostgreSQL LISTEN/NOTIFY +//! for propagating cache invalidation and setting change events across +//! workers and servers. + +use sqlx::{FromRow, Pool, Postgres}; + +use crate::error::Error; + +#[derive(Debug, Clone, FromRow)] +pub struct NotifyEvent { + pub id: i64, + pub channel: String, + pub payload: String, +} + +/// Fetch all events with id greater than `last_event_id`. +/// Returns events ordered by id ascending. +pub async fn poll_notify_events( + db: &Pool, + last_event_id: i64, +) -> Result, Error> { + let events = sqlx::query_as::<_, NotifyEvent>( + "SELECT id, channel, payload FROM notify_event WHERE id > $1 ORDER BY id LIMIT 1000", + ) + .bind(last_event_id) + .fetch_all(db) + .await?; + + Ok(events) +} + +/// Get the current maximum event id. +/// Used to initialize last_event_id on startup to avoid processing old events. +pub async fn get_latest_event_id(db: &Pool) -> Result { + let result: (i64,) = sqlx::query_as("SELECT COALESCE(MAX(id), 0) FROM notify_event") + .fetch_one(db) + .await?; + + Ok(result.0) +} + +/// Delete events older than the specified number of minutes. +/// Returns the number of deleted rows. +pub async fn cleanup_old_events(db: &Pool, older_than_minutes: i32) -> Result { + let result = sqlx::query( + "DELETE FROM notify_event WHERE created_at < now() - make_interval(mins => $1)", + ) + .bind(older_than_minutes) + .execute(db) + .await?; + + Ok(result.rows_affected()) +} diff --git a/backend/windmill-common/src/runtime_assets.rs b/backend/windmill-common/src/runtime_assets.rs index 0060495371..811e62e9d8 100644 --- a/backend/windmill-common/src/runtime_assets.rs +++ b/backend/windmill-common/src/runtime_assets.rs @@ -1,13 +1,19 @@ -use std::{collections::HashMap, sync::OnceLock}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::OnceLock, +}; use itertools::Itertools; use serde_json::value::RawValue; -use sqlx::{Pool, Postgres, QueryBuilder}; +use sqlx::{types::Json, Pool, Postgres, QueryBuilder}; use tokio::sync::mpsc; use windmill_parser::asset_parser::parse_asset_syntax; use crate::{ - assets::{merge_asset_usage_access_types, AssetKind, AssetUsageAccessType, AssetUsageKind}, + assets::{ + merge_asset_columns, merge_asset_usage_access_types, AssetKind, AssetUsageAccessType, + AssetUsageKind, + }, error, }; @@ -59,6 +65,7 @@ pub struct InsertRuntimeAssetParams { pub job_id: uuid::Uuid, pub access_type: Option, pub created_at: Option>, + pub columns: Option>, } async fn insert_runtime_assets( @@ -66,13 +73,14 @@ async fn insert_runtime_assets( assets: &[InsertRuntimeAssetParams], ) -> error::Result<()> { for chunk in assets.chunks(1000) { - let mut query_builder = QueryBuilder::new("INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, created_at) "); + let mut query_builder = QueryBuilder::new("INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, columns, usage_kind, created_at) "); query_builder.push_values(chunk, |mut b, asset| { b.push_bind(&asset.workspace_id) .push_bind(&asset.asset_path) .push_bind(&asset.asset_kind) .push_bind(&asset.access_type) .push_bind(asset.job_id.to_string()) + .push_bind(Json(&asset.columns)) .push_bind(&AssetUsageKind::Job) .push_bind(&asset.created_at); }); @@ -104,6 +112,7 @@ async fn prune_runtime_assets( // Same job used the same asset multiple times last_same_job.access_type = merge_asset_usage_access_types(last_same_job.access_type, asset.access_type); + last_same_job.columns = merge_asset_columns(&last_same_job.columns, &asset.columns); } else if v.len() < max_n { v.push(asset); } diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 0f36bb72cf..b3980aa596 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -793,8 +793,8 @@ pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result error::Result { let s3_resource: S3Resource = serde_json::from_value(resource_value).map_err(|e| { - error::Error::internal_err(format!("Error parsing S3 resource: {}", e)) + error::Error::internal_err(format!("Error parsing S3 resource: {e:?}")) })?; Ok(ObjectStoreResource::S3(s3_resource)) } LargeFileStorage::AzureBlobStorage(_) | LargeFileStorage::AzureWorkloadIdentity(_) => { let azure_blob_resource: AzureBlobResource = serde_json::from_value(resource_value) .map_err(|e| { - error::Error::internal_err(format!("Error parsing Azure Blob resource: {}", e)) + error::Error::internal_err(format!("Error parsing Azure Blob resource: {e:?}")) })?; Ok(ObjectStoreResource::Azure(azure_blob_resource)) } LargeFileStorage::GoogleCloudStorage(_) => { let gcs_resource: GcsResource = serde_json::from_value(resource_value).map_err(|e| { - error::Error::internal_err(format!("Error parsing GCS resource: {}", e)) + error::Error::internal_err(format!("Error parsing GCS resource: {e:?}")) })?; Ok(ObjectStoreResource::Gcs(gcs_resource)) } diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index aff0b6d650..e15aa09b46 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -16,13 +16,13 @@ use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_file_content, read_result, start_child_process, - write_file_binary, MaybeLock, OccupancyMetrics, StreamNotifier, - DEV_CONF_NSJAIL, + write_file_binary, MaybeLock, OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL, }, + get_proxy_envs_for_lang, handle_child::handle_child, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, NPM_CONFIG_REGISTRY, - NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, get_proxy_envs_for_lang, + NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; use windmill_common::{ client::AuthedClient, @@ -51,9 +51,9 @@ use windmill_common::s3_helpers::attempt_fetch_bytes; use windmill_parser::Typ; -const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.js"); +pub const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.js"); -const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js"); +pub const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js"); const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.config.proto"); @@ -64,6 +64,62 @@ pub const BUN_LOCKB_SPLIT_WINDOWS: &str = "\r\n//bun.lockb\r\n"; pub const EMPTY_FILE: &str = ""; +/// Bun args for dedicated worker (without the script path) +pub const BUN_DEDICATED_WORKER_ARGS: &[&str] = &["run", "-i", "--prefer-offline"]; + +/// Generate the dedicated worker wrapper content. +/// - `arg_names`: The argument names for the main function (e.g., ["x", "y"]) +/// - `main_import`: The import path for the main module (e.g., "./main.ts") +/// - `date_conversions`: Optional date conversion statements for Datetime args +pub fn generate_dedicated_worker_wrapper( + arg_names: &[&str], + main_import: &str, + date_conversions: Option<&str>, +) -> String { + let spread = arg_names.join(","); + let dates = date_conversions.unwrap_or(""); + let is_debug = std::env::var("RUST_LOG").is_ok_and(|x| x == "windmill=debug"); + let print_lines = if is_debug { + r#"console.log(line);"# + } else { + "" + }; + + format!( + r#" +import * as Main from "{main_import}"; +import * as Readline from "node:readline" + +BigInt.prototype.toJSON = function () {{ + return this.toString(); +}}; + +console.log('start'); + +function getArgs(line) {{ + let {{ {spread} }} = JSON.parse(line) + {dates} + return [ {spread} ]; +}} + +for await (const line of Readline.createInterface({{ input: process.stdin }})) {{ + {print_lines} + + if (line === "end") {{ + process.exit(0); + }} + try {{ + const args = getArgs(line); + const res = await Main.main(...args); + console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value)); + }} catch (e) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: line }})); + }} +}} +"# + ) +} + /// Returns (package.json, bun.lock(b), is_empty, is_binary) fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool, bool) { if let Some(index) = lockfile.find(BUN_LOCK_SPLIT) { @@ -115,24 +171,25 @@ pub async fn gen_bun_lockfile( gen_bunfig(job_dir).await?; write_file(job_dir, "package.json", package_json_content.as_str())?; } else { + let loader = RELATIVE_BUN_LOADER + .replace("W_ID", w_id) + .replace("BASE_INTERNAL_URL", base_internal_url) + .replace("TOKEN", token) + .replace( + "CURRENT_PATH", + &crate::common::use_flow_root_path(script_path), + ) + .replace("RAW_GET_ENDPOINT", "raw"); + write_file( &job_dir, "build.js", &format!( r#" -{} +{loader} {RELATIVE_BUN_BUILDER} -"#, - RELATIVE_BUN_LOADER - .replace("W_ID", w_id) - .replace("BASE_INTERNAL_URL", base_internal_url) - .replace("TOKEN", token) - .replace( - "CURRENT_PATH", - &crate::common::use_flow_root_path(script_path) - ) - .replace("RAW_GET_ENDPOINT", "raw") +"# ), )?; @@ -382,14 +439,14 @@ pub async fn install_bun_lockfile( } #[derive(PartialEq)] -enum LoaderMode { +pub enum LoaderMode { Node, Bun, BunBundle, NodeBundle, BrowserBundle, } -async fn build_loader( +pub async fn build_loader( job_dir: &str, base_internal_url: &str, token: &str, @@ -406,13 +463,14 @@ async fn build_loader( &crate::common::use_flow_root_path(current_path), ) .replace("RAW_GET_ENDPOINT", "raw_unpinned"); + if mode == LoaderMode::Node { write_file( &job_dir, "node_builder.ts", &format!( r#" -{} +{loader} import {{ readdir }} from "node:fs/promises"; @@ -420,7 +478,6 @@ let fileNames = [] try {{ fileNames = await readdir("{job_dir}/node_modules") }} catch (e) {{ - }} try {{ @@ -437,8 +494,7 @@ try {{ console.log("Failed to build node bundle"); process.exit(1); }} -"#, - loader +"# ), )?; } else if mode == LoaderMode::Bun { @@ -449,11 +505,10 @@ try {{ r#" import {{ plugin }} from "bun"; -{} +{loader} plugin(p) -"#, - loader +"# ), )?; } else if mode == LoaderMode::BunBundle @@ -465,7 +520,7 @@ plugin(p) "node_builder.ts", &format!( r#" -{} +{loader} try {{ await Bun.build({{ @@ -486,7 +541,6 @@ try {{ process.exit(1); }} "#, - loader, if mode == LoaderMode::BunBundle { "bun" } else if mode == LoaderMode::NodeBundle { @@ -1726,55 +1780,21 @@ pub async fn start_worker( .map(|x| return format!("{x} = {x} ? new Date({x}) : undefined")) .join("\n"); - let spread = args.into_iter().map(|x| x.name).join(","); + let arg_names: Vec<&str> = args.iter().map(|x| x.name.as_str()).collect(); // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str()); // we cannot use Bun.read and Bun.write because it results in an EBADF error on cloud - let is_debug = std::env::var("RUST_LOG").is_ok_and(|x| x == "windmill=debug"); - let print_lines = if is_debug { - r#"console.log(line);"# - } else { - "" - }; - let main_import = if codebase.is_some() { "./main.js" } else { "./main.ts" }; - let wrapper_content: String = format!( - r#" -import * as Main from "{main_import}"; -import * as Readline from "node:readline" - -BigInt.prototype.toJSON = function () {{ - return this.toString(); -}}; - -console.log('start'); - -function getArgs(line) {{ - let {{ {spread} }} = JSON.parse(line) - {dates} - return [ {spread} ]; -}} - -for await (const line of Readline.createInterface({{ input: process.stdin }})) {{ - {print_lines} - - if (line === "end") {{ - process.exit(0); - }} - try {{ - const args = getArgs(line); - const res = await Main.main(...args); - console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value)); - }} catch (e) {{ - console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: line }})); - }} -}} -"#, - ); + let dates_opt = if dates.is_empty() { + None + } else { + Some(dates.as_str()) + }; + let wrapper_content = generate_dedicated_worker_wrapper(&arg_names, main_import, dates_opt); write_file(job_dir, "wrapper.mjs", &wrapper_content)?; } @@ -1865,3 +1885,111 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) { .await } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_split_lockfile_text_unix() { + let lockfile = r#"{"dependencies":{"lodash":"^4.17.21"}} +//bun.lock +lockfile-content-here"#; + + let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile); + + assert_eq!(pkg, r#"{"dependencies":{"lodash":"^4.17.21"}}"#); + assert_eq!(lock, Some("lockfile-content-here")); + assert!(!is_empty); + assert!(!is_binary); + } + + #[test] + fn test_split_lockfile_text_windows() { + let lockfile = "{\"dependencies\":{}}\r\n//bun.lock\r\nlockfile-content"; + + let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile); + + assert_eq!(pkg, "{\"dependencies\":{}}"); + assert_eq!(lock, Some("lockfile-content")); + assert!(!is_empty); + assert!(!is_binary); + } + + #[test] + fn test_split_lockfile_binary_unix() { + let lockfile = r#"{"dependencies":{}} +//bun.lockb +YmluYXJ5LWNvbnRlbnQ="#; // base64 encoded "binary-content" + + let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile); + + assert_eq!(pkg, r#"{"dependencies":{}}"#); + assert_eq!(lock, Some("YmluYXJ5LWNvbnRlbnQ=")); + assert!(!is_empty); + assert!(is_binary); + } + + #[test] + fn test_split_lockfile_binary_windows() { + let lockfile = "{\"dependencies\":{}}\r\n//bun.lockb\r\nYmluYXJ5LWNvbnRlbnQ="; + + let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile); + + assert_eq!(pkg, "{\"dependencies\":{}}"); + assert_eq!(lock, Some("YmluYXJ5LWNvbnRlbnQ=")); + assert!(!is_empty); + assert!(is_binary); + } + + #[test] + fn test_split_lockfile_empty() { + let lockfile = r#"{"dependencies":{}} +//bun.lock +"#; + + let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile); + + assert_eq!(pkg, r#"{"dependencies":{}}"#); + assert_eq!(lock, Some(EMPTY_FILE)); + assert!(is_empty); + assert!(!is_binary); + } + + #[test] + fn test_split_lockfile_no_lock() { + let lockfile = r#"{"dependencies":{"lodash":"^4.17.21"}}"#; + + let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile); + + assert_eq!(pkg, r#"{"dependencies":{"lodash":"^4.17.21"}}"#); + assert!(lock.is_none()); + assert!(!is_empty); + assert!(!is_binary); + } + + #[test] + fn test_split_lockfile_multiline_package_json() { + let lockfile = r#"{ + "dependencies": { + "lodash": "^4.17.21" + } +} +//bun.lock +lockfile-content"#; + + let (pkg, lock, is_empty, is_binary) = split_lockfile(lockfile); + + assert_eq!( + pkg, + r#"{ + "dependencies": { + "lodash": "^4.17.21" + } +}"# + ); + assert_eq!(lock, Some("lockfile-content")); + assert!(!is_empty); + assert!(!is_binary); + } +} diff --git a/backend/windmill-worker/src/js_eval_parity_tests.rs b/backend/windmill-worker/src/js_eval_parity_tests.rs index 742f2bf665..7b403e8ac9 100644 --- a/backend/windmill-worker/src/js_eval_parity_tests.rs +++ b/backend/windmill-worker/src/js_eval_parity_tests.rs @@ -486,6 +486,73 @@ mod parity_tests { Ok(()) } + #[tokio::test] + async fn parity_date_serialization() -> anyhow::Result<()> { + let env = HashMap::new(); + + // Test direct Date object serialization (the key issue that was fixed) + // Both engines should serialize Date to ISO string via toJSON + test_parity("new Date('2024-01-15T12:30:00.000Z')", env.clone(), None).await?; + + // Date within an object + test_parity( + "({ date: new Date('2024-01-15T00:00:00.000Z'), name: 'test' })", + env.clone(), + None, + ) + .await?; + + // Date within an array + test_parity( + "[new Date('2024-01-15T00:00:00.000Z'), new Date('2024-01-16T00:00:00.000Z')]", + env.clone(), + None, + ) + .await?; + + // Deeply nested Date + test_parity( + "({ level1: { level2: { date: new Date('2024-01-15T00:00:00.000Z') } } })", + env.clone(), + None, + ) + .await?; + + // Custom object with toJSON (arrow function style) + test_parity( + "({ value: 42, toJSON: () => ({ converted: 84 }) })", + env.clone(), + None, + ) + .await?; + + // toJSON that returns a Date (should be further serialized) + test_parity( + "({ toJSON: () => new Date('2024-01-15T00:00:00.000Z') })", + env.clone(), + None, + ) + .await?; + + Ok(()) + } + + #[tokio::test] + async fn parity_special_object_serialization() -> anyhow::Result<()> { + let env = HashMap::new(); + + // RegExp serialization (both should return {}) + test_parity("/test/gi", env.clone(), None).await?; + + // Map serialization (both should return {}) + test_parity("new Map([['key', 'value']])", env.clone(), None).await?; + + // Set serialization (both should return {}) + test_parity("new Set([1, 2, 3])", env.clone(), None).await?; + + Ok(()) + } + #[tokio::test] async fn parity_array_advanced() -> anyhow::Result<()> { let mut env = HashMap::new(); diff --git a/backend/windmill-worker/src/js_eval_quickjs.rs b/backend/windmill-worker/src/js_eval_quickjs.rs index 2402bcc82f..2dc48c753a 100644 --- a/backend/windmill-worker/src/js_eval_quickjs.rs +++ b/backend/windmill-worker/src/js_eval_quickjs.rs @@ -266,24 +266,25 @@ async fn eval_quickjs_inner( setup_results_proxy(&ctx, &globals, by_id, op_state_clone.clone())?; } - // Determine if we need to add return statement + // Determine if we need to add return statement. + // Wrap with .then((x) => JSON.stringify(x ?? null)) to serialize the result + // using the standard JSON.stringify, matching deno_core's behavior exactly. let code = if should_add_return_quickjs(&transformed_expr) { - format!("(async function() {{ return {}; }})()", transformed_expr) + format!("(async function() {{ return {}; }})().then((x) => JSON.stringify(x ?? null))", transformed_expr) } else { - format!("(async function() {{ {} }})()", transformed_expr) + format!("(async function() {{ {} }})().then((x) => JSON.stringify(x ?? null))", transformed_expr) }; - // Evaluate the expression (returns a Promise) + // Evaluate the expression (returns a Promise that resolves to a JSON string) let promise: rquickjs::Promise = ctx.eval(code).catch(&ctx).map_err(quickjs_error_to_anyhow)?; - // Await the promise + // Await the promise — result is already a JSON string from JSON.stringify let result: Value = promise.into_future().await.catch(&ctx).map_err(quickjs_error_to_anyhow)?; - // Convert result to JSON - let json_result = js_to_json(&ctx, &result)?; - let json_str = serde_json::to_string(&json_result)?; + let json_str = String::from_js(&ctx, result) + .unwrap_or_else(|_| "null".to_string()); - Ok(windmill_common::worker::to_raw_value(&serde_json::from_str::(&json_str)?)) + Ok(crate::common::unsafe_raw(json_str)) }) .await } @@ -557,66 +558,11 @@ fn json_to_js<'js>( } /// Convert a QuickJS Value to a serde_json::Value -fn js_to_json<'js>( - ctx: &rquickjs::Ctx<'js>, - val: &Value<'js>, -) -> anyhow::Result { - if val.is_null() || val.is_undefined() { - return Ok(serde_json::Value::Null); - } - - if let Some(b) = val.as_bool() { - return Ok(serde_json::Value::Bool(b)); - } - - if let Some(i) = val.as_int() { - return Ok(serde_json::Value::Number(i.into())); - } - - if let Some(f) = val.as_float() { - // Check if this float represents an exact integer - // This preserves integer formatting for values like timestamps - if f.fract() == 0.0 && f.abs() <= (i64::MAX as f64) { - let i = f as i64; - // Verify the conversion is exact (for very large numbers) - if (i as f64) == f { - return Ok(serde_json::Value::Number(i.into())); - } - } - if let Some(n) = serde_json::Number::from_f64(f) { - return Ok(serde_json::Value::Number(n)); - } else { - return Ok(serde_json::Value::Null); - } - } - - if let Ok(s) = String::from_js(ctx, val.clone()) { - return Ok(serde_json::Value::String(s)); - } - - if let Ok(arr) = rquickjs::Array::from_js(ctx, val.clone()) { - let mut json_arr = Vec::new(); - for i in 0..arr.len() { - if let Ok(item) = arr.get::(i) { - json_arr.push(js_to_json(ctx, &item)?); - } - } - return Ok(serde_json::Value::Array(json_arr)); - } - - if let Ok(obj) = Object::from_js(ctx, val.clone()) { - let mut json_obj = serde_json::Map::new(); - for res in obj.props::() { - if let Ok((k, v)) = res { - json_obj.insert(k, js_to_json(ctx, &v)?); - } - } - return Ok(serde_json::Value::Object(json_obj)); - } - - // Fallback - Ok(serde_json::Value::String("[object]".to_string())) -} +/// +/// This mimics JavaScript's JSON.stringify behavior: +/// - For objects with a `toJSON` method (like Date), call it and use the result +/// - Arrays are recursively serialized +/// - Plain objects enumerate their own properties /// Determines if we should prepend "return" to the expression fn should_add_return_quickjs(expr: &str) -> bool { @@ -794,4 +740,183 @@ mod tests { assert!(!should_add_return_quickjs("let x = 5; x + 1")); } + + #[tokio::test] + async fn test_eval_quickjs_date_serialization() -> anyhow::Result<()> { + // Test that Date objects serialize to ISO strings, matching JSON.stringify behavior + let result = eval_timeout_quickjs( + "new Date('2024-01-15T12:30:00.000Z')".to_string(), + HashMap::new(), + None, + None, + None, + None, + None, + ) + .await?; + + // Should be an ISO string, not an empty object + assert_eq!(result.get(), "\"2024-01-15T12:30:00.000Z\""); + Ok(()) + } + + #[tokio::test] + async fn test_eval_quickjs_date_in_object() -> anyhow::Result<()> { + // Test that Date objects within other objects serialize correctly + let result = eval_timeout_quickjs( + "({ date: new Date('2024-01-15T12:30:00.000Z'), name: 'test' })".to_string(), + HashMap::new(), + None, + None, + None, + None, + None, + ) + .await?; + + let value: serde_json::Value = serde_json::from_str(result.get())?; + assert_eq!(value["date"], "2024-01-15T12:30:00.000Z"); + assert_eq!(value["name"], "test"); + Ok(()) + } + + #[tokio::test] + async fn test_eval_quickjs_date_in_array() -> anyhow::Result<()> { + // Test that Date objects in arrays serialize correctly + let result = eval_timeout_quickjs( + "[new Date('2024-01-15T00:00:00.000Z'), new Date('2024-01-16T00:00:00.000Z')]" + .to_string(), + HashMap::new(), + None, + None, + None, + None, + None, + ) + .await?; + + let value: serde_json::Value = serde_json::from_str(result.get())?; + assert_eq!(value[0], "2024-01-15T00:00:00.000Z"); + assert_eq!(value[1], "2024-01-16T00:00:00.000Z"); + Ok(()) + } + + #[tokio::test] + async fn test_eval_quickjs_custom_tojson() -> anyhow::Result<()> { + // Test that JSON.stringify handles custom toJSON when returning objects + let result = eval_timeout_quickjs( + r#"({ a: 1, toJSON: () => ({ converted: true }) })"#.to_string(), + HashMap::new(), + None, + None, + None, + None, + None, + ) + .await?; + + // JSON.stringify should call toJSON and use that result + let value: serde_json::Value = serde_json::from_str(result.get())?; + assert_eq!(value["converted"], true); + Ok(()) + } + + #[tokio::test] + async fn test_eval_quickjs_deeply_nested_date() -> anyhow::Result<()> { + // Test that Date objects deep in nested structures are handled + let result = eval_timeout_quickjs( + "({ level1: { level2: { date: new Date('2024-01-15T00:00:00.000Z') } } })".to_string(), + HashMap::new(), + None, + None, + None, + None, + None, + ) + .await?; + + let value: serde_json::Value = serde_json::from_str(result.get())?; + assert_eq!( + value["level1"]["level2"]["date"], + "2024-01-15T00:00:00.000Z" + ); + Ok(()) + } + + #[tokio::test] + async fn test_eval_quickjs_regexp_serialization() -> anyhow::Result<()> { + // RegExp objects serialize to empty objects in JSON (same as JSON.stringify behavior) + let result = eval_timeout_quickjs( + "/test/gi".to_string(), + HashMap::new(), + None, + None, + None, + None, + None, + ) + .await?; + + // RegExp doesn't have toJSON, so it serializes to an empty object (same as Deno) + assert_eq!(result.get(), "{}"); + Ok(()) + } + + #[tokio::test] + async fn test_eval_quickjs_map_serialization() -> anyhow::Result<()> { + // Map objects serialize to empty objects in JSON (same as JSON.stringify behavior) + let result = eval_timeout_quickjs( + "new Map([['key', 'value']])".to_string(), + HashMap::new(), + None, + None, + None, + None, + None, + ) + .await?; + + // Map doesn't have toJSON, serializes to empty object (same as Deno) + assert_eq!(result.get(), "{}"); + Ok(()) + } + + #[tokio::test] + async fn test_eval_quickjs_set_serialization() -> anyhow::Result<()> { + // Set objects serialize to empty objects in JSON (same as JSON.stringify behavior) + let result = eval_timeout_quickjs( + "new Set([1, 2, 3])".to_string(), + HashMap::new(), + None, + None, + None, + None, + None, + ) + .await?; + + // Set doesn't have toJSON, serializes to empty object (same as Deno) + assert_eq!(result.get(), "{}"); + Ok(()) + } + + #[tokio::test] + async fn test_eval_quickjs_tojson_returning_date() -> anyhow::Result<()> { + // When toJSON returns a Date object, JSON.stringify does NOT call toJSON again + // on the returned value (per spec). Date has no own enumerable properties, + // so it serializes to {}. + let result = eval_timeout_quickjs( + "({ toJSON: () => new Date('2024-01-15T00:00:00.000Z') })".to_string(), + HashMap::new(), + None, + None, + None, + None, + None, + ) + .await?; + + assert_eq!(result.get(), "{}"); + Ok(()) + } } diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index fe6ca82b09..fed4e8d31b 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -95,8 +95,9 @@ pub use otel_tracing_proxy_ee::{load_internal_otel_exporter, DENO_OTEL_INITIALIZ pub use result_processor::handle_job_error; pub use bun_executor::{ - compute_bundle_local_and_remote_path, get_common_bun_proc_envs, install_bun_lockfile, - prebundle_bun_script, prepare_job_dir, + build_loader, compute_bundle_local_and_remote_path, generate_dedicated_worker_wrapper, + get_common_bun_proc_envs, install_bun_lockfile, prebundle_bun_script, prepare_job_dir, + BUN_DEDICATED_WORKER_ARGS, LoaderMode, RELATIVE_BUN_BUILDER, RELATIVE_BUN_LOADER, }; pub use deno_executor::generate_deno_lock; pub use prepare_deps::run_prepare_deps_cli; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index ec7e67ae40..1bc84df1cb 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -1533,7 +1533,11 @@ pub async fn run_worker( let is_dedicated_worker: bool = { let config = WORKER_CONFIG.read().await; - config.dedicated_worker.is_some() || config.dedicated_workers.as_ref().is_some_and(|dws| !dws.is_empty()) + config.dedicated_worker.is_some() + || config + .dedicated_workers + .as_ref() + .is_some_and(|dws| !dws.is_empty()) }; #[cfg(feature = "benchmark")] @@ -2045,7 +2049,9 @@ pub async fn run_worker( dedicated_workers.get(&key) }) } else { - job.runnable_path.as_ref().and_then(|path| dedicated_workers.get(path)) + job.runnable_path + .as_ref() + .and_then(|path| dedicated_workers.get(path)) }; if let Some(dedicated_worker_tx) = dedicated_worker_tx { let dedicated_job = DedicatedWorkerJob { @@ -2773,6 +2779,7 @@ async fn detect_and_store_runtime_assets_from_job_args( asset_kind: asset.kind, access_type: None, created_at: None, + columns: None, }; register_runtime_asset(asset); } diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index a2257ea98b..f5faea3ba0 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.623.1"; +export const VERSION = "v1.624.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/deno.lock b/cli/deno.lock index 59a1002364..536c8960c8 100644 --- a/cli/deno.lock +++ b/cli/deno.lock @@ -1553,6 +1553,8 @@ "https://deno.land/std@0.224.0/assert/unimplemented.ts": "8c55a5793e9147b4f1ef68cd66496b7d5ba7a9e7ca30c6da070c1a58da723d73", "https://deno.land/std@0.224.0/assert/unreachable.ts": "5ae3dbf63ef988615b93eb08d395dda771c96546565f9e521ed86f6510c29e19", "https://deno.land/std@0.224.0/cli/parse_args.ts": "5250832fb7c544d9111e8a41ad272c016f5a53f975ef84d5a9fe5fcb70566ece", + "https://deno.land/std@0.224.0/encoding/_util.ts": "beacef316c1255da9bc8e95afb1fa56ed69baef919c88dc06ae6cb7a6103d376", + "https://deno.land/std@0.224.0/encoding/hex.ts": "6270f25e5d85f99fcf315278670ba012b04b7c94b67715b53f30d03249687c07", "https://deno.land/std@0.224.0/fmt/colors.ts": "508563c0659dd7198ba4bbf87e97f654af3c34eb56ba790260f252ad8012e1c5", "https://deno.land/std@0.224.0/fs/_create_walk_entry.ts": "5d9d2aaec05bcf09a06748b1684224d33eba7a4de24cf4cf5599991ca6b5b412", "https://deno.land/std@0.224.0/fs/_get_file_info_type.ts": "da7bec18a7661dba360a1db475b826b18977582ce6fc9b25f3d4ee0403fe8cbd", diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 55543e7b4d..251ef4148b 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -1280,6 +1280,7 @@ export async function elementsToMap( skips: Skips, specificItems?: SpecificItemsConfig, branchOverride?: string, + isRemote?: boolean, ): Promise<{ [key: string]: string }> { const map: { [key: string]: string } = {}; const processedBasePaths = new Set(); @@ -1446,7 +1447,8 @@ export async function elementsToMap( continue; } // Skip base file if it's configured as branch-specific (expect branch version) - if (isSpecificItem(path, specificItems)) { + // Only for LOCAL files - remote workspace only has base paths + if (!isRemote && isSpecificItem(path, specificItems)) { continue; } map[path] = content; @@ -1486,13 +1488,14 @@ async function compareDynFSElement( ignoreCodebaseChanges: boolean, specificItems?: SpecificItemsConfig, branchOverride?: string, + isEls1Remote?: boolean, ): Promise { const [m1, m2] = els2 ? await Promise.all([ - elementsToMap(els1, ignore, json, skips, specificItems, branchOverride), - elementsToMap(els2, ignore, json, skips, specificItems, branchOverride), + elementsToMap(els1, ignore, json, skips, specificItems, branchOverride, isEls1Remote), + elementsToMap(els2, ignore, json, skips, specificItems, branchOverride, !isEls1Remote), ]) - : [await elementsToMap(els1, ignore, json, skips, specificItems, branchOverride), {}]; + : [await elementsToMap(els1, ignore, json, skips, specificItems, branchOverride, isEls1Remote), {}]; const changes: Change[] = []; @@ -1995,6 +1998,7 @@ export async function pull( true, specificItems, opts.branch, + true, // els1 (remote) is the remote source ); log.info( @@ -2486,6 +2490,7 @@ export async function push( false, specificItems, opts.branch, + false, // els1 (local) is not the remote source ); const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); diff --git a/cli/src/main.ts b/cli/src/main.ts index 19219a1e95..89cf72a4c9 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -77,7 +77,7 @@ export { // } // }); -export const VERSION = "1.623.1"; +export const VERSION = "1.624.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index 6a598e6d73..5ed715cc9d 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -211,6 +211,180 @@ export async function updateScriptSchema( } } +// --------------------------------------------------------------------------- +// Annotation parser — mirrors backend's WorkspaceDependenciesAnnotatedRefs::parse +// (windmill-common/src/workspace_dependencies.rs) so the cache key captures +// exactly the parts of scriptContent that affect lockfile generation. +// --------------------------------------------------------------------------- + +type AnnotationMode = "manual" | "extra"; + +interface WorkspaceDepsAnnotation { + mode: AnnotationMode; + external: string[]; + inline: string | null; +} + +const LANG_ANNOTATION_CONFIG: Partial< + Record +> = { + python3: { comment: "#", keyword: "requirements", validityRe: /^#\s?(\S+)\s*$/ }, + bun: { comment: "//", keyword: "package_json" }, + nativets: { comment: "//", keyword: "package_json" }, + go: { comment: "//", keyword: "go_mod" }, + php: { comment: "//", keyword: "composer_json" }, +}; + +export function extractWorkspaceDepsAnnotation( + scriptContent: string, + language: ScriptLanguage, +): WorkspaceDepsAnnotation | null { + const config = LANG_ANNOTATION_CONFIG[language]; + if (!config) return null; + + const { comment, keyword, validityRe } = config; + const extraMarker = `extra_${keyword}:`; + const manualMarker = `${keyword}:`; + + const lines = scriptContent.split("\n"); + + // Find first annotation line (mirrors Rust find_position) + let pos = -1; + for (let i = 0; i < lines.length; i++) { + const l = lines[i]; + if (l.startsWith(comment) && (l.includes(extraMarker) || l.includes(manualMarker))) { + pos = i; + break; + } + } + if (pos === -1) return null; + + const annotationLine = lines[pos]; + const mode: AnnotationMode = annotationLine.includes(extraMarker) ? "extra" : "manual"; + + // Parse external references from the annotation line + const marker = mode === "extra" ? extraMarker : manualMarker; + const unparsed = annotationLine.replaceAll(marker, "").replaceAll(comment, ""); + const external = unparsed + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + + // Parse inline deps from subsequent lines + const inlineParts: string[] = []; + for (let i = pos + 1; i < lines.length; i++) { + const l = lines[i]; + if (validityRe) { + const match = validityRe.exec(l); + if (match && match[1]) { + inlineParts.push(match[1]); + } else { + break; + } + } else { + if (!l.startsWith(comment)) { + break; + } + inlineParts.push(l.substring(comment.length)); + } + } + + const inlineStr = inlineParts.join("\n"); + const inline = inlineStr.trim().length > 0 ? inlineStr : null; + + return { mode, external, inline }; +} + +export async function computeLockCacheKey( + scriptContent: string, + language: ScriptLanguage, + rawWorkspaceDependencies: Record, +): Promise { + const annotation = extractWorkspaceDepsAnnotation(scriptContent, language); + const annotationStr = annotation + ? `${annotation.mode}|${annotation.external.join(",")}|${annotation.inline ?? ""}` + : "none"; + const sortedDepsKeys = Object.keys(rawWorkspaceDependencies).sort(); + const depsStr = sortedDepsKeys.map((k) => `${k}=${rawWorkspaceDependencies[k]}`).join(";"); + return await generateHash(`${language}|${annotationStr}|${depsStr}`); +} + +const lockCache = new Map(); + +export function clearLockCache(): void { + lockCache.clear(); +} + +async function fetchScriptLock( + workspace: Workspace, + scriptContent: string, + language: ScriptLanguage, + remotePath: string, + rawWorkspaceDependencies: Record, +): Promise { + const hasRawDeps = Object.keys(rawWorkspaceDependencies).length > 0; + const cacheKey = hasRawDeps + ? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies) + : undefined; + if (cacheKey && lockCache.has(cacheKey)) { + log.info(`Using cached lockfile for ${remotePath}`); + return lockCache.get(cacheKey)!; + } + + const extraHeaders = getHeaders(); + const rawResponse = await fetch( + `${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/dependencies`, + { + method: "POST", + headers: { + Cookie: `token=${workspace.token}`, + "Content-Type": "application/json", + ...extraHeaders, + }, + body: JSON.stringify({ + raw_scripts: [ + { + raw_code: scriptContent, + language: language, + script_path: remotePath, + }, + ], + raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0 + ? rawWorkspaceDependencies : null, + entrypoint: remotePath, + }), + } + ); + + let responseText = "reading response failed"; + try { + responseText = await rawResponse.text(); + const response = JSON.parse(responseText); + const lock = response.lock; + if (lock === undefined) { + if (response?.["error"]?.["message"]) { + throw new LockfileGenerationError( + `Failed to generate lockfile: ${response?.["error"]?.["message"]}` + ); + } + throw new LockfileGenerationError( + `Failed to generate lockfile: ${JSON.stringify(response, null, 2)}` + ); + } + if (cacheKey) { + lockCache.set(cacheKey, lock); + } + return lock; + } catch (e) { + if (e instanceof LockfileGenerationError) { + throw e; + } + throw new LockfileGenerationError( + `Failed to generate lockfile:${rawResponse.statusText}, ${responseText}, ${e}` + ); + } +} + async function updateScriptLock( workspace: Workspace, scriptContent: string, @@ -235,70 +409,28 @@ async function updateScriptLock( const dependencyPaths = Object.keys(rawWorkspaceDependencies).join(', '); log.info(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`); } - - // generate the script lock running a dependency job in Windmill and update it inplace - // TODO: update this once the client is released - const extraHeaders = getHeaders(); - const rawResponse = await fetch( - `${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/dependencies`, - { - method: "POST", - headers: { - Cookie: `token=${workspace.token}`, - "Content-Type": "application/json", - ...extraHeaders, - }, - body: JSON.stringify({ - raw_scripts: [ - { - raw_code: scriptContent, - language: language, - script_path: remotePath, - }, - ], - raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0 - ? rawWorkspaceDependencies : null, - entrypoint: remotePath, - }), - } + + const lock = await fetchScriptLock( + workspace, + scriptContent, + language, + remotePath, + rawWorkspaceDependencies, ); - let responseText = "reading response failed"; - try { - responseText = await rawResponse.text(); - const response = JSON.parse(responseText); - const lock = response.lock; - if (lock === undefined) { - if (response?.["error"]?.["message"]) { - throw new LockfileGenerationError( - `Failed to generate lockfile: ${response?.["error"]?.["message"]}` - ); + const lockPath = remotePath + ".script.lock"; + if (lock != "") { + await Deno.writeTextFile(lockPath, lock); + metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/"); + } else { + try { + if (await Deno.stat(lockPath)) { + await Deno.remove(lockPath); } - throw new LockfileGenerationError( - `Failed to generate lockfile: ${JSON.stringify(response, null, 2)}` - ); + } catch (e) { + log.info(colors.yellow(`Error removing lock file ${lockPath}: ${e}`)); } - const lockPath = remotePath + ".script.lock"; - if (lock != "") { - await Deno.writeTextFile(lockPath, lock); - metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/"); - } else { - try { - if (await Deno.stat(lockPath)) { - await Deno.remove(lockPath); - } - } catch (e) { - log.info(colors.yellow(`Error removing lock file ${lockPath}: ${e}`)); - } - metadataContent.lock = ""; - } - } catch (e) { - if (e instanceof LockfileGenerationError) { - throw e; - } - throw new LockfileGenerationError( - `Failed to generate lockfile:${rawResponse.statusText}, ${responseText}, ${e}` - ); + metadataContent.lock = ""; } } diff --git a/cli/test/elements_to_map_branch_specific.test.ts b/cli/test/elements_to_map_branch_specific.test.ts new file mode 100644 index 0000000000..03af7d52ba --- /dev/null +++ b/cli/test/elements_to_map_branch_specific.test.ts @@ -0,0 +1,476 @@ +import { assertEquals } from "https://deno.land/std@0.224.0/assert/mod.ts"; + +// Import the function we need to test +import { elementsToMap } from "../src/commands/sync/sync.ts"; +import type { SpecificItemsConfig } from "../src/core/specific_items.ts"; + +// ============================================================================= +// elementsToMap TESTS FOR BRANCH-SPECIFIC ITEMS +// Tests for the regression where remote base files were incorrectly skipped +// when configured as branch-specific, causing them to be marked for deletion +// on pull operations. +// +// Regression: PR #7643 (commit 287b7e7d9, Jan 21, 2026) +// ============================================================================= + +/** + * Mock DynFSElement implementation for testing + */ +interface MockFile { + path: string; + content: string; + isDirectory?: boolean; +} + +function createMockDynFSElement(files: MockFile[]): { + isDirectory: boolean; + path: string; + getContentText(): Promise; + getChildren(): AsyncIterable<{ + isDirectory: boolean; + path: string; + getContentText(): Promise; + getChildren(): AsyncIterable; + }>; +} { + return { + isDirectory: true, + path: "", + async getContentText() { + return ""; + }, + async *getChildren() { + for (const file of files) { + yield { + isDirectory: file.isDirectory ?? false, + path: file.path, + async getContentText() { + return file.content; + }, + async *getChildren() { + // No children for files + }, + }; + } + }, + }; +} + +const noIgnore = () => false; +const defaultSkips = {}; + +// ============================================================================= +// REGRESSION TEST: Remote base files should NOT be skipped +// ============================================================================= + +Deno.test("elementsToMap: remote base file is NOT skipped when configured as branch-specific (isRemote=true)", async () => { + // This is the key regression test. + // When pulling from remote, the workspace only has base paths (e.g., TestVar.variable.yaml) + // These should NOT be skipped even if configured as branch-specific, because the remote + // workspace doesn't have branch-specific file naming. + + const config: SpecificItemsConfig = { + variables: ["f/Shared/Variable/**"], + }; + + const remoteFiles: MockFile[] = [ + { + path: "f/Shared/Variable/TestVar.variable.yaml", + content: "value: test\nis_secret: false", + }, + ]; + + const mockElement = createMockDynFSElement(remoteFiles); + + // When isRemote=true, base files should be included even if they match branch-specific config + const result = await elementsToMap( + mockElement, + noIgnore, + false, + defaultSkips, + config, + "staging", // branchOverride + true, // isRemote = true + ); + + // The base file should be in the map + assertEquals( + Object.keys(result).includes("f/Shared/Variable/TestVar.variable.yaml"), + true, + "Remote base file should NOT be skipped when isRemote=true" + ); +}); + +Deno.test("elementsToMap: local base file IS skipped when configured as branch-specific (isRemote=false)", async () => { + // When processing local files, if a base file is configured as branch-specific, + // it should be skipped because we expect the branch-specific version to be used instead. + + const config: SpecificItemsConfig = { + variables: ["f/Shared/Variable/**"], + }; + + const localFiles: MockFile[] = [ + { + path: "f/Shared/Variable/TestVar.variable.yaml", + content: "value: test\nis_secret: false", + }, + ]; + + const mockElement = createMockDynFSElement(localFiles); + + // When isRemote=false, base files should be skipped if configured as branch-specific + const result = await elementsToMap( + mockElement, + noIgnore, + false, + defaultSkips, + config, + "staging", // branchOverride + false, // isRemote = false + ); + + // The base file should NOT be in the map (skipped because branch-specific expected) + assertEquals( + Object.keys(result).includes("f/Shared/Variable/TestVar.variable.yaml"), + false, + "Local base file SHOULD be skipped when isRemote=false and configured as branch-specific" + ); +}); + +Deno.test("elementsToMap: local branch-specific file is mapped to base path (isRemote=false)", async () => { + // When processing local files with branch-specific naming, they should be mapped to base paths + + const config: SpecificItemsConfig = { + variables: ["f/Shared/Variable/**"], + }; + + const localFiles: MockFile[] = [ + { + path: "f/Shared/Variable/TestVar.staging.variable.yaml", + content: "value: staging-test\nis_secret: false", + }, + ]; + + const mockElement = createMockDynFSElement(localFiles); + + const result = await elementsToMap( + mockElement, + noIgnore, + false, + defaultSkips, + config, + "staging", // branchOverride + false, // isRemote = false + ); + + // The branch-specific file should be mapped to the base path + assertEquals( + Object.keys(result).includes("f/Shared/Variable/TestVar.variable.yaml"), + true, + "Branch-specific file should be mapped to base path" + ); + assertEquals( + result["f/Shared/Variable/TestVar.variable.yaml"], + "value: staging-test\nis_secret: false", + ); +}); + +// ============================================================================= +// PULL SCENARIO: Remote (base path) vs Local (branch-specific path) +// This simulates the actual pull scenario where: +// - Remote has: f/Shared/Variable/TestVar.variable.yaml +// - Local has: f/Shared/Variable/TestVar.staging.variable.yaml +// - Expected: No deletion, the files should match +// ============================================================================= + +Deno.test("elementsToMap: pull scenario - remote and local maps should align correctly", async () => { + const config: SpecificItemsConfig = { + variables: ["f/Shared/Variable/**"], + }; + + // Remote workspace has base path + const remoteFiles: MockFile[] = [ + { + path: "f/Shared/Variable/TestVar.variable.yaml", + content: "value: test\nis_secret: false", + }, + ]; + + // Local has branch-specific path + const localFiles: MockFile[] = [ + { + path: "f/Shared/Variable/TestVar.staging.variable.yaml", + content: "value: staging-test\nis_secret: false", + }, + ]; + + const remoteElement = createMockDynFSElement(remoteFiles); + const localElement = createMockDynFSElement(localFiles); + + // Process remote (isRemote=true) + const remoteMap = await elementsToMap( + remoteElement, + noIgnore, + false, + defaultSkips, + config, + "staging", + true, // isRemote + ); + + // Process local (isRemote=false) + const localMap = await elementsToMap( + localElement, + noIgnore, + false, + defaultSkips, + config, + "staging", + false, // isRemote + ); + + // Both maps should have the same base path key + const remoteKeys = Object.keys(remoteMap); + const localKeys = Object.keys(localMap); + + assertEquals( + remoteKeys.includes("f/Shared/Variable/TestVar.variable.yaml"), + true, + "Remote map should include base path" + ); + assertEquals( + localKeys.includes("f/Shared/Variable/TestVar.variable.yaml"), + true, + "Local map should include base path (mapped from branch-specific)" + ); +}); + +// ============================================================================= +// NON-CONFIGURED ITEMS: Should work the same regardless of isRemote +// ============================================================================= + +Deno.test("elementsToMap: non-configured items included regardless of isRemote", async () => { + const config: SpecificItemsConfig = { + variables: ["f/Other/**"], // Only "Other" folder is branch-specific + }; + + const files: MockFile[] = [ + { + path: "f/Shared/Variable/TestVar.variable.yaml", + content: "value: test\nis_secret: false", + }, + ]; + + const mockElement = createMockDynFSElement(files); + + // Test with isRemote=true + const remoteResult = await elementsToMap( + mockElement, + noIgnore, + false, + defaultSkips, + config, + "staging", + true, + ); + + // Test with isRemote=false + const localResult = await elementsToMap( + createMockDynFSElement(files), + noIgnore, + false, + defaultSkips, + config, + "staging", + false, + ); + + // Both should include the file since it's not in the branch-specific config + assertEquals( + Object.keys(remoteResult).includes("f/Shared/Variable/TestVar.variable.yaml"), + true, + "Non-configured item should be included when isRemote=true" + ); + assertEquals( + Object.keys(localResult).includes("f/Shared/Variable/TestVar.variable.yaml"), + true, + "Non-configured item should be included when isRemote=false" + ); +}); + +// ============================================================================= +// RESOURCE TYPE TESTS +// ============================================================================= + +Deno.test("elementsToMap: remote resource base file not skipped when configured", async () => { + const config: SpecificItemsConfig = { + resources: ["f/db/**"], + }; + + const remoteFiles: MockFile[] = [ + { + path: "f/db/connection.resource.yaml", + content: "value: { host: localhost }", + }, + ]; + + const mockElement = createMockDynFSElement(remoteFiles); + + const result = await elementsToMap( + mockElement, + noIgnore, + false, + defaultSkips, + config, + "staging", + true, // isRemote + ); + + assertEquals( + Object.keys(result).includes("f/db/connection.resource.yaml"), + true, + "Remote resource base file should NOT be skipped" + ); +}); + +// ============================================================================= +// TRIGGER TYPE TESTS +// ============================================================================= + +Deno.test("elementsToMap: remote trigger base file not skipped when configured", async () => { + const config: SpecificItemsConfig = { + triggers: ["f/webhooks/**"], + }; + + const remoteFiles: MockFile[] = [ + { + path: "f/webhooks/handler.http_trigger.yaml", + content: "path: /webhook", + }, + ]; + + const mockElement = createMockDynFSElement(remoteFiles); + + const result = await elementsToMap( + mockElement, + noIgnore, + false, + { includeTriggers: true }, // Must include triggers explicitly + config, + "staging", + true, // isRemote + ); + + assertEquals( + Object.keys(result).includes("f/webhooks/handler.http_trigger.yaml"), + true, + "Remote trigger base file should NOT be skipped" + ); +}); + +// ============================================================================= +// SETTINGS TYPE TESTS +// ============================================================================= + +Deno.test("elementsToMap: remote settings.yaml not skipped when configured", async () => { + const config: SpecificItemsConfig = { + settings: true, + }; + + const remoteFiles: MockFile[] = [ + { + path: "settings.yaml", + content: "openai_resource_path: null", + }, + ]; + + const mockElement = createMockDynFSElement(remoteFiles); + + const result = await elementsToMap( + mockElement, + noIgnore, + false, + { includeSettings: true }, + config, + "staging", + true, // isRemote + ); + + assertEquals( + Object.keys(result).includes("settings.yaml"), + true, + "Remote settings.yaml should NOT be skipped" + ); +}); + +// ============================================================================= +// FOLDER TYPE TESTS +// ============================================================================= + +Deno.test("elementsToMap: remote folder meta not skipped when configured", async () => { + const config: SpecificItemsConfig = { + folders: ["f/env_*"], + }; + + const remoteFiles: MockFile[] = [ + { + path: "f/env_staging/folder.meta.yaml", + content: "display_name: Staging Environment", + }, + ]; + + const mockElement = createMockDynFSElement(remoteFiles); + + const result = await elementsToMap( + mockElement, + noIgnore, + false, + defaultSkips, + config, + "staging", + true, // isRemote + ); + + assertEquals( + Object.keys(result).includes("f/env_staging/folder.meta.yaml"), + true, + "Remote folder meta should NOT be skipped" + ); +}); + +// ============================================================================= +// BACKWARD COMPATIBILITY: isRemote undefined behaves like local (false) +// ============================================================================= + +Deno.test("elementsToMap: isRemote undefined behaves like local (backward compatible)", async () => { + const config: SpecificItemsConfig = { + variables: ["f/**"], + }; + + const files: MockFile[] = [ + { + path: "f/test.variable.yaml", + content: "value: test\nis_secret: false", + }, + ]; + + const mockElement = createMockDynFSElement(files); + + // When isRemote is undefined (backward compatibility), it should behave like local + const result = await elementsToMap( + mockElement, + noIgnore, + false, + defaultSkips, + config, + "staging", + // isRemote omitted + ); + + // Base file should be skipped (same behavior as isRemote=false) + assertEquals( + Object.keys(result).includes("f/test.variable.yaml"), + false, + "isRemote undefined should behave like isRemote=false (skip base file)" + ); +}); diff --git a/cli/test/lock_cache.test.ts b/cli/test/lock_cache.test.ts new file mode 100644 index 0000000000..c217749c6c --- /dev/null +++ b/cli/test/lock_cache.test.ts @@ -0,0 +1,572 @@ +/** + * Lock Cache Tests + * + * Tests the in-memory lock cache used when fetching lockfiles for scripts with + * raw_workspace_dependencies. + * + * Part 1: Unit tests for annotation parsing (mirrors backend). + * Part 2: Unit tests for cache key computation. + * Part 3: Behavioral tests comparing old logic (no cache, always fetches) + * vs new logic (caches by key, skips duplicate fetches). + */ + +import { + assertEquals, + assertNotEquals, +} from "https://deno.land/std@0.224.0/assert/mod.ts"; +import { encodeHex } from "https://deno.land/std@0.224.0/encoding/hex.ts"; + +// --------------------------------------------------------------------------- +// Mirrors extractWorkspaceDepsAnnotation + computeLockCacheKey from +// src/utils/metadata.ts so we can test the algorithm without pulling in the +// full (unresolvable-in-tests) module graph. +// --------------------------------------------------------------------------- + +type AnnotationMode = "manual" | "extra"; + +interface WorkspaceDepsAnnotation { + mode: AnnotationMode; + external: string[]; + inline: string | null; +} + +const LANG_ANNOTATION_CONFIG: Record< + string, + { comment: string; keyword: string; validityRe?: RegExp } | undefined +> = { + python3: { comment: "#", keyword: "requirements", validityRe: /^#\s?(\S+)\s*$/ }, + bun: { comment: "//", keyword: "package_json" }, + nativets: { comment: "//", keyword: "package_json" }, + go: { comment: "//", keyword: "go_mod" }, + php: { comment: "//", keyword: "composer_json" }, +}; + +function extractWorkspaceDepsAnnotation( + scriptContent: string, + language: string, +): WorkspaceDepsAnnotation | null { + const config = LANG_ANNOTATION_CONFIG[language]; + if (!config) return null; + + const { comment, keyword, validityRe } = config; + const extraMarker = `extra_${keyword}:`; + const manualMarker = `${keyword}:`; + + const lines = scriptContent.split("\n"); + + let pos = -1; + for (let i = 0; i < lines.length; i++) { + const l = lines[i]; + if (l.startsWith(comment) && (l.includes(extraMarker) || l.includes(manualMarker))) { + pos = i; + break; + } + } + if (pos === -1) return null; + + const annotationLine = lines[pos]; + const mode: AnnotationMode = annotationLine.includes(extraMarker) ? "extra" : "manual"; + + const marker = mode === "extra" ? extraMarker : manualMarker; + const unparsed = annotationLine.replaceAll(marker, "").replaceAll(comment, ""); + const external = unparsed + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + + const inlineParts: string[] = []; + for (let i = pos + 1; i < lines.length; i++) { + const l = lines[i]; + if (validityRe) { + const match = validityRe.exec(l); + if (match && match[1]) { + inlineParts.push(match[1]); + } else { + break; + } + } else { + if (!l.startsWith(comment)) { + break; + } + inlineParts.push(l.substring(comment.length)); + } + } + + const inlineStr = inlineParts.join("\n"); + const inline = inlineStr.trim().length > 0 ? inlineStr : null; + + return { mode, external, inline }; +} + +async function computeLockCacheKey( + scriptContent: string, + language: string, + rawWorkspaceDependencies: Record, +): Promise { + const annotation = extractWorkspaceDepsAnnotation(scriptContent, language); + const annotationStr = annotation + ? `${annotation.mode}|${annotation.external.join(",")}|${annotation.inline ?? ""}` + : "none"; + const sortedDepsKeys = Object.keys(rawWorkspaceDependencies).sort(); + const depsStr = sortedDepsKeys + .map((k) => `${k}=${rawWorkspaceDependencies[k]}`) + .join(";"); + const content = `${language}|${annotationStr}|${depsStr}`; + const buf = new TextEncoder().encode(content); + return encodeHex(await crypto.subtle.digest("SHA-256", buf)); +} + +// --------------------------------------------------------------------------- +// Helpers that mirror the two fetch strategies (old / new). +// --------------------------------------------------------------------------- + +interface ScriptInput { + scriptContent: string; + language: string; + remotePath: string; + rawWorkspaceDependencies: Record; +} + +/** Old logic: always calls the remote for every script. */ +async function fetchScriptLockOld( + input: ScriptInput, + remoteFn: (input: ScriptInput) => Promise, +): Promise { + return await remoteFn(input); +} + +/** New logic: only caches when raw_workspace_dependencies are non-empty. */ +async function fetchScriptLockNew( + input: ScriptInput, + remoteFn: (input: ScriptInput) => Promise, + cache: Map, +): Promise { + const hasRawDeps = Object.keys(input.rawWorkspaceDependencies).length > 0; + const cacheKey = hasRawDeps + ? await computeLockCacheKey( + input.scriptContent, + input.language, + input.rawWorkspaceDependencies, + ) + : undefined; + if (cacheKey && cache.has(cacheKey)) { + return cache.get(cacheKey)!; + } + const lock = await remoteFn(input); + if (cacheKey) { + cache.set(cacheKey, lock); + } + return lock; +} + +// ============================================================================= +// Part 1 — Annotation parsing +// ============================================================================= + +Deno.test("python: manual requirements with external refs + inline deps", () => { + const code = `# requirements: default, base +#requests==2.31.0 +#pandas>=1.5.0 + +def main(): + pass`; + const r = extractWorkspaceDepsAnnotation(code, "python3")!; + assertEquals(r.mode, "manual"); + assertEquals(r.external, ["default", "base"]); + assertEquals(r.inline, "requests==2.31.0\npandas>=1.5.0"); +}); + +Deno.test("python: extra_requirements mode", () => { + const code = `# extra_requirements: utils +#numpy>=1.24.0 + +def main(): + pass`; + const r = extractWorkspaceDepsAnnotation(code, "python3")!; + assertEquals(r.mode, "extra"); + assertEquals(r.external, ["utils"]); + assertEquals(r.inline, "numpy>=1.24.0"); +}); + +Deno.test("python: empty requirements (opt-out)", () => { + const code = `# requirements: +def main(): + pass`; + const r = extractWorkspaceDepsAnnotation(code, "python3")!; + assertEquals(r.mode, "manual"); + assertEquals(r.external, []); + assertEquals(r.inline, null); +}); + +Deno.test("python: no annotation → null", () => { + const code = `def main(): + print("hello")`; + assertEquals(extractWorkspaceDepsAnnotation(code, "python3"), null); +}); + +Deno.test("bun: package_json annotation with inline", () => { + const code = `// package_json: utils, base +//{ +// "dependencies": { +// "axios": "^1.6.0" +// } +//} + +export function main() {}`; + const r = extractWorkspaceDepsAnnotation(code, "bun")!; + assertEquals(r.mode, "manual"); + assertEquals(r.external, ["utils", "base"]); + assertEquals(r.inline, `{ + "dependencies": { + "axios": "^1.6.0" + } +}`); +}); + +Deno.test("go: go_mod annotation", () => { + const code = `// go_mod: base, +//github.com/gin-gonic/gin v1.9.1 + +package main +func main() {}`; + const r = extractWorkspaceDepsAnnotation(code, "go")!; + assertEquals(r.mode, "manual"); + assertEquals(r.external, ["base"]); + assertEquals(r.inline, "github.com/gin-gonic/gin v1.9.1"); +}); + +Deno.test("unsupported language → null", () => { + assertEquals(extractWorkspaceDepsAnnotation("print(1)", "deno"), null); + assertEquals(extractWorkspaceDepsAnnotation("print(1)", "bash"), null); +}); + +// ============================================================================= +// Part 2 — Cache key computation +// ============================================================================= + +Deno.test("same annotation + language + deps → same key", async () => { + const code = `# requirements: default +#requests==2.31.0 +print("hello")`; + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + const a = await computeLockCacheKey(code, "python3", deps); + const b = await computeLockCacheKey(code, "python3", deps); + assertEquals(a, b); +}); + +Deno.test("different code, same annotation → same key", async () => { + const codeA = `# requirements: default +#requests==2.31.0 +print("hello")`; + const codeB = `# requirements: default +#requests==2.31.0 +print("world")`; + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + assertEquals( + await computeLockCacheKey(codeA, "python3", deps), + await computeLockCacheKey(codeB, "python3", deps), + ); +}); + +Deno.test("different annotation inline → different key", async () => { + const codeA = `# requirements: default +#requests==2.31.0 +print("hello")`; + const codeB = `# requirements: default +#flask==3.0.0 +print("hello")`; + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + assertNotEquals( + await computeLockCacheKey(codeA, "python3", deps), + await computeLockCacheKey(codeB, "python3", deps), + ); +}); + +Deno.test("different annotation external refs → different key", async () => { + const codeA = `# requirements: default +print("hello")`; + const codeB = `# requirements: base +print("hello")`; + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + assertNotEquals( + await computeLockCacheKey(codeA, "python3", deps), + await computeLockCacheKey(codeB, "python3", deps), + ); +}); + +Deno.test("manual vs extra mode → different key", async () => { + const codeA = `# requirements: default +print("hello")`; + const codeB = `# extra_requirements: default +print("hello")`; + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + assertNotEquals( + await computeLockCacheKey(codeA, "python3", deps), + await computeLockCacheKey(codeB, "python3", deps), + ); +}); + +Deno.test("no annotation, same code → same key", async () => { + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + assertEquals( + await computeLockCacheKey("print('a')", "python3", deps), + await computeLockCacheKey("print('b')", "python3", deps), + ); +}); + +Deno.test("different deps → different key", async () => { + const code = `# requirements: default +print("hello")`; + assertNotEquals( + await computeLockCacheKey(code, "python3", { d: "a" }), + await computeLockCacheKey(code, "python3", { d: "b" }), + ); +}); + +Deno.test("different language → different key", async () => { + const deps = { d: "v" }; + assertNotEquals( + await computeLockCacheKey("x", "bun", deps), + await computeLockCacheKey("x", "python3", deps), + ); +}); + +Deno.test("dep key order does not matter", async () => { + const code = "print('hello')"; + assertEquals( + await computeLockCacheKey(code, "python3", { a: "1", b: "2" }), + await computeLockCacheKey(code, "python3", { b: "2", a: "1" }), + ); +}); + +// ============================================================================= +// Part 3 — Multi-script fetch behavior: old logic vs new logic +// ============================================================================= + +function makeRemoteFn(): { + remoteFn: (input: ScriptInput) => Promise; + callCount: () => number; +} { + const calls: ScriptInput[] = []; + return { + remoteFn: async (input: ScriptInput) => { + calls.push(input); + const depsStr = Object.entries(input.rawWorkspaceDependencies).sort().map(([k,v]) => `${k}=${v}`).join(","); + return `lock for ${input.language}:${input.scriptContent}:${depsStr}`; + }, + callCount: () => calls.length, + }; +} + +// -- Two scripts, same annotation + language + deps ------------------------- + +Deno.test("old logic: two scripts same annotation → 2 remote calls", async () => { + const { remoteFn, callCount } = makeRemoteFn(); + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + + const scripts: ScriptInput[] = [ + { scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps }, + { scriptContent: "# requirements: default\nprint(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps }, + ]; + + for (const s of scripts) await fetchScriptLockOld(s, remoteFn); + assertEquals(callCount(), 2); +}); + +Deno.test("new logic: two scripts same annotation → 1 remote call (cache shared)", async () => { + const { remoteFn, callCount } = makeRemoteFn(); + const cache = new Map(); + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + + const scripts: ScriptInput[] = [ + { scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps }, + { scriptContent: "# requirements: default\nprint(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps }, + ]; + + const results: string[] = []; + for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache)); + assertEquals(callCount(), 1); + assertEquals(results[0], results[1]); +}); + +// -- Two scripts, different annotations + same deps ------------------------- + +Deno.test("new logic: different annotations same deps → 2 remote calls", async () => { + const { remoteFn, callCount } = makeRemoteFn(); + const cache = new Map(); + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + + const scripts: ScriptInput[] = [ + { scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps }, + { scriptContent: "# requirements: base\nprint(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps }, + ]; + + const results: string[] = []; + for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache)); + assertEquals(callCount(), 2); + assertNotEquals(results[0], results[1]); +}); + +// -- Two scripts, same annotation + different deps -------------------------- + +Deno.test("new logic: same annotation different deps → 2 remote calls", async () => { + const { remoteFn, callCount } = makeRemoteFn(); + const cache = new Map(); + + const scripts: ScriptInput[] = [ + { scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", + rawWorkspaceDependencies: { "dependencies/requirements.in": "requests==2.31.0" } }, + { scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "b", + rawWorkspaceDependencies: { "dependencies/requirements.in": "requests==2.32.0" } }, + ]; + + const results: string[] = []; + for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache)); + assertEquals(callCount(), 2); + assertNotEquals(results[0], results[1]); +}); + +// -- Many scripts, same annotation + deps ----------------------------------- + +Deno.test("old logic: 5 scripts same annotation+deps → 5 remote calls", async () => { + const { remoteFn, callCount } = makeRemoteFn(); + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + const ann = "# requirements: default\n"; + + const scripts: ScriptInput[] = [ + { scriptContent: ann + "print(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps }, + { scriptContent: ann + "print(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps }, + { scriptContent: ann + "print(3)", language: "python3", remotePath: "c", rawWorkspaceDependencies: deps }, + { scriptContent: ann + "print(1)", language: "python3", remotePath: "d", rawWorkspaceDependencies: deps }, + { scriptContent: ann + "print(2)", language: "python3", remotePath: "e", rawWorkspaceDependencies: deps }, + ]; + + for (const s of scripts) await fetchScriptLockOld(s, remoteFn); + assertEquals(callCount(), 5); +}); + +Deno.test("new logic: 5 scripts same annotation+deps → 1 remote call", async () => { + const { remoteFn, callCount } = makeRemoteFn(); + const cache = new Map(); + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + const ann = "# requirements: default\n"; + + const scripts: ScriptInput[] = [ + { scriptContent: ann + "print(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps }, + { scriptContent: ann + "print(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps }, + { scriptContent: ann + "print(3)", language: "python3", remotePath: "c", rawWorkspaceDependencies: deps }, + { scriptContent: ann + "print(1)", language: "python3", remotePath: "d", rawWorkspaceDependencies: deps }, + { scriptContent: ann + "print(2)", language: "python3", remotePath: "e", rawWorkspaceDependencies: deps }, + ]; + + const results: string[] = []; + for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache)); + assertEquals(callCount(), 1); + for (let i = 1; i < results.length; i++) { + assertEquals(results[0], results[i]); + } +}); + +// -- Many scripts, 2 annotation groups + same deps ------------------------- + +Deno.test("new logic: 4 scripts with 2 annotation groups → 2 remote calls", async () => { + const { remoteFn, callCount } = makeRemoteFn(); + const cache = new Map(); + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + + const scripts: ScriptInput[] = [ + { scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps }, + { scriptContent: "# requirements: base\nprint(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps }, + { scriptContent: "# requirements: default\nprint(3)", language: "python3", remotePath: "c", rawWorkspaceDependencies: deps }, + { scriptContent: "# requirements: base\nprint(4)", language: "python3", remotePath: "d", rawWorkspaceDependencies: deps }, + ]; + + const results: string[] = []; + for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache)); + assertEquals(callCount(), 2); + assertEquals(results[0], results[2]); // same annotation "default" + assertEquals(results[1], results[3]); // same annotation "base" + assertNotEquals(results[0], results[1]); +}); + +// -- Scripts with no workspace deps (empty) --------------------------------- + +Deno.test("new logic: empty deps → no caching", async () => { + const { remoteFn, callCount } = makeRemoteFn(); + const cache = new Map(); + + const scripts: ScriptInput[] = [ + { scriptContent: "print(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: {} }, + { scriptContent: "print(1)", language: "python3", remotePath: "b", rawWorkspaceDependencies: {} }, + ]; + + for (const s of scripts) await fetchScriptLockNew(s, remoteFn, cache); + assertEquals(callCount(), 2); + assertEquals(cache.size, 0); +}); + +// -- No annotation scripts with raw deps → share cache --------------------- + +Deno.test("new logic: no annotation + same deps → 1 remote call", async () => { + const { remoteFn, callCount } = makeRemoteFn(); + const cache = new Map(); + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + + const scripts: ScriptInput[] = [ + { scriptContent: "print(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps }, + { scriptContent: "print(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps }, + ]; + + const results: string[] = []; + for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache)); + assertEquals(callCount(), 1); + assertEquals(results[0], results[1]); +}); + +// -- Mix of annotated and non-annotated scripts ----------------------------- + +Deno.test("new logic: mix of annotated and non-annotated → separate cache groups", async () => { + const { remoteFn, callCount } = makeRemoteFn(); + const cache = new Map(); + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + + const scripts: ScriptInput[] = [ + { scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps }, + { scriptContent: "print(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps }, + { scriptContent: "# requirements: default\nprint(3)", language: "python3", remotePath: "c", rawWorkspaceDependencies: deps }, + { scriptContent: "print(4)", language: "python3", remotePath: "d", rawWorkspaceDependencies: deps }, + ]; + + const results: string[] = []; + for (const s of scripts) results.push(await fetchScriptLockNew(s, remoteFn, cache)); + assertEquals(callCount(), 2); // one for annotated group, one for no-annotation group + assertEquals(results[0], results[2]); // both annotated "default" + assertEquals(results[1], results[3]); // both no annotation + assertNotEquals(results[0], results[1]); // annotated ≠ non-annotated +}); + +// -- Cache returns correct lock value --------------------------------------- + +Deno.test("new logic: cached value matches original remote response", async () => { + const cache = new Map(); + const deps = { "dependencies/requirements.in": "requests==2.31.0" }; + + let callIdx = 0; + const remoteFn = async (_input: ScriptInput) => { + callIdx++; + return "resolved-lock-content-abc123"; + }; + + const r1 = await fetchScriptLockNew( + { scriptContent: "# requirements: default\nprint(1)", language: "python3", remotePath: "a", rawWorkspaceDependencies: deps }, + remoteFn, cache, + ); + const r2 = await fetchScriptLockNew( + { scriptContent: "# requirements: default\nprint(2)", language: "python3", remotePath: "b", rawWorkspaceDependencies: deps }, + remoteFn, cache, + ); + + assertEquals(callIdx, 1); + assertEquals(r1, "resolved-lock-content-abc123"); + assertEquals(r2, "resolved-lock-content-abc123"); +}); diff --git a/cli/windmill-utils-internal/package.json b/cli/windmill-utils-internal/package.json index 20f20a4c9a..e6463c8e58 100644 --- a/cli/windmill-utils-internal/package.json +++ b/cli/windmill-utils-internal/package.json @@ -1,12 +1,22 @@ { "name": "windmill-utils-internal", - "version": "1.3.2", + "version": "1.3.3", "description": "Internal utility functions for Windmill", - "main": "dist/index.js", - "types": "dist/index.d.ts", + "main": "dist/cjs/index.js", + "module": "dist/esm/index.js", + "types": "dist/esm/index.d.ts", + "exports": { + ".": { + "require": "./dist/cjs/index.js", + "import": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts" + } + }, "scripts": { "dev": "./gen_wm_client.sh && ./remove-ts-ext.sh", - "build": "./gen_wm_client.sh && ./remove-ts-ext.sh && tsc", + "build:cjs": "tsc -p tsconfig.cjs.json", + "build:esm": "tsc -p tsconfig.esm.json", + "build": "./gen_wm_client.sh && ./remove-ts-ext.sh && npm run build:cjs && npm run build:esm", "prepublishOnly": "npm run build" }, "keywords": [ diff --git a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts index 62cc81ba1f..d7be5338da 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts @@ -1,5 +1,5 @@ import { newPathAssigner, PathAssigner } from "../path-utils/path-assigner.ts"; -import { FlowModule } from "../gen/types.gen.ts"; +import { FlowModule, RawScript } from "../gen/types.gen.ts"; /** * Represents an inline script extracted from a flow module @@ -11,6 +11,28 @@ interface InlineScript { content: string; } +function extractRawscriptInline( + id: string, + summary: string | undefined, + rawscript: RawScript, + mapping: Record, + separator: string, + assigner: PathAssigner +): InlineScript[] { + const [basePath, ext] = assigner.assignPath(summary ?? id, rawscript.language); + const path = mapping[id] ?? basePath + ext; + const content = rawscript.content; + const r = [{ path: path, content: content }]; + rawscript.content = "!inline " + path.replaceAll(separator, "/"); + const lock = rawscript.lock; + if (lock && lock != "") { + const lockPath = basePath + "lock"; + rawscript.lock = "!inline " + lockPath.replaceAll(separator, "/"); + r.push({ path: lockPath, content: lock }); + } + return r; +} + /** * Options for extractInlineScripts function */ @@ -44,18 +66,14 @@ export function extractInlineScripts( return modules.flatMap((m) => { if (m.value.type == "rawscript") { - const [basePath, ext] = assigner.assignPath(m.summary, m.value.language); - const path = mapping[m.id] ?? basePath + ext; - const content = m.value.content; - const r = [{ path: path, content: content }]; - m.value.content = "!inline " + path.replaceAll(separator, "/"); - const lock = m.value.lock; - if (lock && lock != "") { - const lockPath = basePath + "lock"; - m.value.lock = "!inline " + lockPath.replaceAll(separator, "/"); - r.push({ path: lockPath, content: lock }); - } - return r; + return extractRawscriptInline( + m.id, + m.summary, + m.value, + mapping, + separator, + assigner + ); } else if (m.value.type == "forloopflow") { return extractInlineScripts( m.value.modules, @@ -95,6 +113,23 @@ export function extractInlineScripts( assigner ), ]; + } else if (m.value.type == "aiagent") { + return (m.value.tools ?? []).flatMap((tool) => { + const toolValue = tool.value; + // Only process flowmodule tools with rawscript type + if (!toolValue || toolValue.tool_type !== 'flowmodule' || toolValue.type !== 'rawscript') { + return []; + } + + return extractRawscriptInline( + tool.id, + tool.summary, + toolValue, + mapping, + separator, + assigner + ); + }); } else { return []; } @@ -140,6 +175,14 @@ export function extractCurrentMapping( extractCurrentMapping(b.modules, mapping) ); extractCurrentMapping(m.value.default, mapping); + } else if (m.value.type === "aiagent") { + (m.value.tools ?? []).forEach((tool) => { + const toolValue = tool.value; + if (!toolValue || toolValue.tool_type !== 'flowmodule' || toolValue.type !== 'rawscript' || !toolValue.content || !toolValue.content.startsWith("!inline")) { + return; + } + mapping[tool.id] = toolValue.content.trim().split(" ")[1]; + }); } }); diff --git a/cli/windmill-utils-internal/src/inline-scripts/replacer.ts b/cli/windmill-utils-internal/src/inline-scripts/replacer.ts index cad496a269..03a159ee83 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/replacer.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/replacer.ts @@ -1,4 +1,49 @@ -import { FlowModule } from "../gen/types.gen.ts"; +import { FlowModule, RawScript } from "../gen/types.gen.ts"; + +async function replaceRawscriptInline( + id: string, + rawscript: RawScript, + fileReader: (path: string) => Promise, + logger: { info: (message: string) => void; error: (message: string) => void }, + separator: string, + removeLocks?: string[] +): Promise { + if (!rawscript.content || !rawscript.content.startsWith("!inline")) { + return; + } + + const path = rawscript.content.split(" ")[1]; + const pathSuffix = path.split(".").slice(1).join("."); + const newPath = id + "." + pathSuffix; + + try { + rawscript.content = await fileReader(path); + } catch { + logger.error(`Script file ${path} not found`); + try { + rawscript.content = await fileReader(newPath); + } catch { + logger.error(`Script file ${newPath} not found`); + } + } + + const lock = rawscript.lock; + if (removeLocks && removeLocks.includes(path)) { + rawscript.lock = undefined; + } else if ( + lock && + typeof lock === "string" && + lock.trimStart().startsWith("!inline ") + ) { + const lockPath = lock.split(" ")[1]; + try { + rawscript.lock = await fileReader(lockPath.replaceAll("/", separator)); + } catch { + logger.error(`Lock file ${lockPath} not found, treating as empty`); + rawscript.lock = ""; + } + } +} /** * Replaces inline script references with actual file content from the filesystem. @@ -32,66 +77,15 @@ export async function replaceInlineScripts( throw new Error(`Module value is undefined for module ${module.id}`); } - if (module.value.type === "rawscript" && module.value.content && module.value.content.startsWith("!inline")) { - const path = module.value.content.split(" ")[1]; - // const pathPrefix = path.split(".")[0]; - const pathSuffix = path.split(".").slice(1).join("."); - // new path is the module id with the same suffix - const newPath = module.id + "." + pathSuffix; - - try { - module.value.content = await fileReader(path); - } catch { - logger.error(`Script file ${path} not found`); - // try new path - try { - module.value.content = await fileReader(newPath); - } catch { - logger.error(`Script file ${newPath} not found`); - } - } - - // rename the file if the prefix is different from the module id (fix old naming) - // if (pathPrefix != module.id && renamer) { - // logger.info(`Renaming ${path} to ${module.id}.${pathSuffix}`); - // try { - // renamer(localPath + path, localPath + module.id + "." + pathSuffix); - // } catch { - // logger.info(`Failed to rename ${path} to ${module.id}.${pathSuffix}`); - // } - // } - - const lock = module.value.lock; - if (removeLocks && removeLocks.includes(path)) { - module.value.lock = undefined; - - // delete the file if the prefix is different from the module id (fix old naming) - // if (lock && lock != "") { - // const path = lock.split(" ")[1]; - // const pathPrefix = path.split(".")[0]; - // if (pathPrefix != module.id && deleter) { - // logger.info(`Deleting ${path}`); - // try { - // deleter(localPath + path); - // } catch { - // logger.error(`Failed to delete ${path}`); - // } - // } - // } - - } else if ( - lock && - typeof lock == "string" && - lock.trimStart().startsWith("!inline ") - ) { - const path = lock.split(" ")[1]; - try { - module.value.lock = await fileReader(path.replaceAll("/", separator)); - } catch { - logger.error(`Lock file ${path} not found, treating as empty`); - module.value.lock = ""; - } - } + if (module.value.type === "rawscript") { + await replaceRawscriptInline( + module.id, + module.value, + fileReader, + logger, + separator, + removeLocks + ); } else if (module.value.type === "forloopflow" || module.value.type === "whileloopflow") { await replaceInlineScripts(module.value.modules, fileReader, logger, localPath, separator, removeLocks); } else if (module.value.type === "branchall") { @@ -103,6 +97,25 @@ export async function replaceInlineScripts( await replaceInlineScripts(branch.modules, fileReader, logger, localPath, separator, removeLocks); })); await replaceInlineScripts(module.value.default, fileReader, logger, localPath, separator, removeLocks); + } else if (module.value.type === "aiagent") { + await Promise.all((module.value.tools ?? []).map(async (tool) => { + const toolValue = tool.value; + if ( + !toolValue || + toolValue.tool_type !== "flowmodule" || + toolValue.type !== "rawscript" + ) { + return; + } + await replaceRawscriptInline( + tool.id, + toolValue, + fileReader, + logger, + separator, + removeLocks + ); + })); } })); } \ No newline at end of file diff --git a/cli/windmill-utils-internal/tsconfig.cjs.json b/cli/windmill-utils-internal/tsconfig.cjs.json new file mode 100644 index 0000000000..8ab4792db4 --- /dev/null +++ b/cli/windmill-utils-internal/tsconfig.cjs.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "CommonJS", + "outDir": "./dist/cjs" + } +} diff --git a/cli/windmill-utils-internal/tsconfig.esm.json b/cli/windmill-utils-internal/tsconfig.esm.json new file mode 100644 index 0000000000..03da56818b --- /dev/null +++ b/cli/windmill-utils-internal/tsconfig.esm.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "ES2022", + "outDir": "./dist/esm" + } +} diff --git a/cli/windmill-utils-internal/tsconfig.json b/cli/windmill-utils-internal/tsconfig.json index 35814c1f7a..19b88b57d4 100644 --- a/cli/windmill-utils-internal/tsconfig.json +++ b/cli/windmill-utils-internal/tsconfig.json @@ -1,8 +1,10 @@ { "compilerOptions": { "target": "ES2022", - "module": "ES2022", - "lib": ["ES2022"], + "module": "commonjs", + "lib": [ + "ES2022" + ], "declaration": true, "outDir": "./dist", "rootDir": "./src", @@ -11,7 +13,7 @@ "noUnusedParameters": false, "noImplicitReturns": true, "noFallthroughCasesInSwitch": true, - "moduleResolution": "bundler", + "moduleResolution": "node", "baseUrl": "./", "esModuleInterop": true, "experimentalDecorators": true, @@ -19,6 +21,11 @@ "skipLibCheck": true, "forceConsistentCasingInFileNames": true }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} + "include": [ + "src/**/*" + ], + "exclude": [ + "node_modules", + "dist" + ] +} \ No newline at end of file diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index bd07238b4c..84420b8be2 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -34,7 +34,7 @@ RUN mkdir -p /tmp/windmill/cache && \ rm -rf /tmp/build_cache && \ mkdir -p -m 777 /tmp/windmill/cache/uv -COPY --from=oven/bun:1.2.23 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.3.8 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index b127d1b782..80de5fd0c9 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -34,7 +34,7 @@ RUN mkdir -p /tmp/windmill/cache && \ rm -rf /tmp/build_cache && \ mkdir -p -m 777 /tmp/windmill/cache/uv -COPY --from=oven/bun:1.2.23 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.3.8 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 54fa07dbdb..995257f2f3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.623.1", + "version": "1.624.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.623.1", + "version": "1.624.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -79,11 +79,11 @@ "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.574.1", - "windmill-parser-wasm-py": "1.601.1", - "windmill-parser-wasm-regex": "1.593.0", + "windmill-parser-wasm-py": "1.623.1", + "windmill-parser-wasm-regex": "1.623.1", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", - "windmill-parser-wasm-ts": "1.593.0", + "windmill-parser-wasm-ts": "1.623.1", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "^1.3.2", @@ -259,7 +259,6 @@ "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", @@ -275,7 +274,6 @@ "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -821,7 +819,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": "^14 || ^16 || >=18" }, @@ -1331,6 +1328,7 @@ "integrity": "sha512-Jer+M7DgIwT5IHfTayb4Iw/fkkxWNmC/mqn/nMh9JrbPbkxmyabfLQnhJ+JDn5HK77f84j34lubO3iqFtYAfMg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@floating-ui/core": "^1.3.1", "@floating-ui/dom": "^1.4.5", @@ -2106,6 +2104,7 @@ "integrity": "sha512-Vp3zX/qlwerQmHMP6x0Ry1oY7eKKRcOWGc2P59srOp4zcqyn+etJyQpELgOi4+ZSUgteX8Y387NuwruLgGXLUQ==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@standard-schema/spec": "^1.0.0", "@sveltejs/acorn-typescript": "^1.0.5", @@ -2183,6 +2182,7 @@ "integrity": "sha512-YZs/OSKOQAQCnJvM/P+F1URotNnYNeU3P2s4oIpzm1uFaqUEqRxUB0g5ejMjEb5Gjb9/PiBI5Ktrq4rUUF8UVQ==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@sveltejs/vite-plugin-svelte-inspector": "^5.0.0", "debug": "^4.4.1", @@ -2709,8 +2709,7 @@ "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/ms": { "version": "2.1.0", @@ -2723,8 +2722,7 @@ "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/semver": { "version": "7.7.1", @@ -2794,6 +2792,7 @@ "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -2962,7 +2961,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "dependencies": { "@vitest/mocker": "4.0.15", "@vitest/utils": "4.0.15", @@ -2987,7 +2985,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "dependencies": { "@vitest/browser": "4.0.15", "@vitest/mocker": "4.0.15", @@ -3013,7 +3010,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "dependencies": { "@vitest/spy": "4.0.15", "estree-walker": "^3.0.3", @@ -3042,7 +3038,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "dependencies": { "@vitest/spy": "4.0.15", "estree-walker": "^3.0.3", @@ -3274,6 +3269,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3327,6 +3323,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -3472,7 +3469,6 @@ "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -3500,7 +3496,6 @@ "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -3593,8 +3588,7 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-2.0.0.tgz", "integrity": "sha512-1ugUSr8BHXRnK23KfuYS+gVMC3LB8QGH9W1iGtDPsNWoQbgtXSExkBu2aDR4epiGWZOjZsj6lDl/N/AqqTC3UA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/base64-js": { "version": "1.5.1", @@ -3721,6 +3715,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.9", "caniuse-lite": "^1.0.30001746", @@ -3918,7 +3913,6 @@ "integrity": "sha512-Rjs1H+A9R+Ig+4E/9oyB66UC5Mj9Xq3N//vcLf2WzgdTi/3gUu3Z9KoqmlrEG4VuuLK8wJHofxzdQXz/knhiYg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "camelcase": "^6.3.0", "map-obj": "^4.1.0", @@ -3938,7 +3932,6 @@ "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -3952,7 +3945,6 @@ "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -3966,7 +3958,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -4075,6 +4066,7 @@ "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", "license": "MIT", + "peer": true, "dependencies": { "@kurkle/color": "^0.3.0" }, @@ -4321,7 +4313,6 @@ "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "import-fresh": "^3.3.0", "js-yaml": "^4.1.0", @@ -4386,7 +4377,6 @@ "integrity": "sha512-IQOkD3hbR5KrN93MtcYuad6YPuTSUhntLHDuLEbFWE+ff2/XSZNdZG+LcbbIW5AXKg/WFIfYItIzVoHngHXZzA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12 || >=16" } @@ -4652,6 +4642,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -4705,6 +4696,7 @@ "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.21.0" }, @@ -4739,7 +4731,6 @@ "integrity": "sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -4753,7 +4744,6 @@ "integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "decamelize": "^1.1.0", "map-obj": "^1.0.0" @@ -4771,7 +4761,6 @@ "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -4782,7 +4771,6 @@ "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -5268,7 +5256,6 @@ "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "is-arrayish": "^0.2.1" } @@ -5356,6 +5343,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -5902,7 +5890,6 @@ "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 4.9.1" } @@ -6388,7 +6375,6 @@ "integrity": "sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "global-prefix": "^3.0.0" }, @@ -6402,7 +6388,6 @@ "integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ini": "^1.3.5", "kind-of": "^6.0.2", @@ -6418,7 +6403,6 @@ "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "isexe": "^2.0.0" }, @@ -6468,8 +6452,7 @@ "resolved": "https://registry.npmjs.org/globjoin/-/globjoin-0.1.4.tgz", "integrity": "sha512-xYfnw62CKG8nLkZBfWbhWwDw02CHty86jfPcc2cr3ZfeuK9ysoVPPEUxf21bAD/rWAgk52SuBrLJlefNy8mvFg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/gopd": { "version": "1.2.0", @@ -6546,7 +6529,6 @@ "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6" } @@ -6768,7 +6750,6 @@ "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -6782,7 +6763,6 @@ "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "yallist": "^4.0.0" }, @@ -6795,8 +6775,7 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/html-tags": { "version": "3.3.1", @@ -6804,7 +6783,6 @@ "integrity": "sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" }, @@ -6888,7 +6866,6 @@ "integrity": "sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -6909,7 +6886,6 @@ "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -6980,8 +6956,7 @@ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/is-binary-path": { "version": "2.1.0", @@ -7086,7 +7061,6 @@ "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7097,7 +7071,6 @@ "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7202,8 +7175,7 @@ "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/js-yaml": { "version": "4.1.0", @@ -7239,8 +7211,7 @@ "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/json-refs": { "version": "3.0.15", @@ -7407,7 +7378,6 @@ "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -8001,8 +7971,7 @@ "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/lodash.uniq": { "version": "4.5.0", @@ -8070,7 +8039,6 @@ "integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" }, @@ -8121,7 +8089,6 @@ "integrity": "sha512-APMBEanjybaPzUrfqU0IMU5I0AswKMH7k8OTLs0vvV4KZpExkTkY87nR/zpbuTPj+gARop7aGUbl11pnDfW6xg==", "dev": true, "license": "MIT", - "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -8362,7 +8329,6 @@ "integrity": "sha512-/d+PQ4GKmGvM9Bee/DPa8z3mXs/pkvJE2KEThngVNOqtmljC6K7NMPxtc2JeZYTmpWb9k/TmxjeL18ez3h7vCw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/minimist": "^1.2.2", "camelcase-keys": "^7.0.0", @@ -8390,7 +8356,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -9084,7 +9049,6 @@ "integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "arrify": "^1.0.1", "is-plain-obj": "^1.1.0", @@ -9163,6 +9127,7 @@ "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-api/-/monaco-vscode-editor-api-25.0.0.tgz", "integrity": "sha512-uiY06RTWFo2WZdh6OybkLlDhuG+8LlkjUDpr9/wW55uucqHo4X8fx4XKEtD98cscC+6FKQkbG2yyUiOJ/npHOw==", "license": "MIT", + "peer": true, "dependencies": { "@codingame/monaco-vscode-api": "25.0.0" } @@ -9399,7 +9364,6 @@ "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", "dev": true, "license": "BSD-2-Clause", - "peer": true, "dependencies": { "hosted-git-info": "^4.0.1", "is-core-module": "^2.5.0", @@ -9745,7 +9709,6 @@ "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", @@ -9957,7 +9920,6 @@ "dev": true, "license": "ISC", "optional": true, - "peer": true, "dependencies": { "pngjs": "^7.0.0" }, @@ -10047,7 +10009,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "engines": { "node": ">=14.19.0" } @@ -10072,6 +10033,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -10260,6 +10222,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "lilconfig": "^3.0.0", "yaml": "^2.3.4" @@ -10649,8 +10612,7 @@ "resolved": "https://registry.npmjs.org/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.6.tgz", "integrity": "sha512-0sglIs9Wmkzbr8lQwEyIzlDOOC9bGmfVKcJTaxv3vMmd3uo4o4DerC3En0bnmgceeql9BfC8hRkp7cg0fjdVqw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/postcss-safe-parser": { "version": "6.0.0", @@ -10826,6 +10788,7 @@ "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -11136,7 +11099,6 @@ "integrity": "sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/normalize-package-data": "^2.4.0", "normalize-package-data": "^3.0.2", @@ -11156,7 +11118,6 @@ "integrity": "sha512-snVCqPczksT0HS2EC+SxUndvSzn6LRCwpfSvLrIfR5BKDQQZMaI6jPRC9dYvYFDRAuFEAnkwww8kBBNE/3VvzQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "find-up": "^5.0.0", "read-pkg": "^6.0.0", @@ -11175,7 +11136,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -11189,7 +11149,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -11232,7 +11191,6 @@ "integrity": "sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "indent-string": "^5.0.0", "strip-indent": "^4.0.0" @@ -11875,7 +11833,6 @@ "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ansi-styles": "^4.0.0", "astral-regex": "^2.0.0", @@ -11952,7 +11909,6 @@ "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "spdx-expression-parse": "^3.0.0", "spdx-license-ids": "^3.0.0" @@ -11963,8 +11919,7 @@ "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", "dev": true, - "license": "CC-BY-3.0", - "peer": true + "license": "CC-BY-3.0" }, "node_modules/spdx-expression-parse": { "version": "3.0.1", @@ -11972,7 +11927,6 @@ "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "spdx-exceptions": "^2.1.0", "spdx-license-ids": "^3.0.0" @@ -11983,8 +11937,7 @@ "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.22.tgz", "integrity": "sha512-4PRT4nh1EImPbt2jASOKHX7PB7I+e4IWNLvkKFDxNhJlfjbYlleYQh285Z/3mPTHSAK/AvdMmw5BNNuYH8ShgQ==", "dev": true, - "license": "CC0-1.0", - "peer": true + "license": "CC0-1.0" }, "node_modules/sprintf-js": { "version": "1.0.3", @@ -12078,7 +12031,6 @@ "integrity": "sha512-SlyRoSkdh1dYP0PzclLE7r0M9sgbFKKMFXpFRUMNuKhQSbC6VQIGzq3E0qsfvGJaUFJPGv6Ws1NZ/haTAjfbMA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -12104,8 +12056,7 @@ "resolved": "https://registry.npmjs.org/style-search/-/style-search-0.1.0.tgz", "integrity": "sha512-Dj1Okke1C3uKKwQcetra4jSuk0DqbzbYtXipzFlFMZtowbF1x7BKJwB9AayVMyFARvU8EDrZdcax4At/452cAg==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/style-to-object": { "version": "0.4.4", @@ -12154,7 +12105,6 @@ "integrity": "sha512-78O4c6IswZ9TzpcIiQJIN49K3qNoXTM8zEJzhaTE/xRTCZswaovSEVIa/uwbOltZrk16X4jAxjaOhzz/hTm1Kw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@csstools/css-parser-algorithms": "^2.3.1", "@csstools/css-tokenizer": "^2.2.0", @@ -12237,7 +12187,6 @@ } ], "license": "MIT-0", - "peer": true, "engines": { "node": "^14 || ^16 || >=18" }, @@ -12251,7 +12200,6 @@ "integrity": "sha512-TfW7/1iI4Cy7Y8L6iqNdZQVvdXn0f8B4QcIXmkIbtTIe/Okm/nSlHb4IwGzRVOd3WfSieCgvf5cMzEfySAIl0g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "flat-cache": "^3.2.0" }, @@ -12264,8 +12212,7 @@ "resolved": "https://registry.npmjs.org/known-css-properties/-/known-css-properties-0.29.0.tgz", "integrity": "sha512-Ne7wqW7/9Cz54PDt4I3tcV+hAyat8ypyOGzYRJQfdxnnjeWsTxt1cy8pjvvKeI5kfXuyvULyeeAvwvvtAX3ayQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/stylelint/node_modules/postcss-selector-parser": { "version": "6.1.2", @@ -12288,7 +12235,6 @@ "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -12423,7 +12369,6 @@ "integrity": "sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "has-flag": "^4.0.0", "supports-color": "^7.0.0" @@ -12453,6 +12398,7 @@ "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.39.12.tgz", "integrity": "sha512-CEzwxFuEycokU8K8CE/OuwVbmei+ivu2HvBGYIdASfMa1hCRSNr4RRkzNSvbAvu6h+BOig2CsZTAEY+WKvwZpA==", "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", @@ -12548,21 +12494,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -12750,8 +12681,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/svg-tags/-/svg-tags-1.0.0.tgz", "integrity": "sha512-ovssysQTa+luh7A5Weu3Rta6FJlFBBbInjOh722LIt6klpU2/HtdUbszju/G4devcvk8PGt7FCLv5wftu3THUA==", - "dev": true, - "peer": true + "dev": true }, "node_modules/svgo": { "version": "3.3.2", @@ -12802,7 +12732,6 @@ "integrity": "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "ajv": "^8.0.1", "lodash.truncate": "^4.4.2", @@ -12830,6 +12759,7 @@ "integrity": "sha512-6A2rnmW5xZMdw11LYjhcI5846rt9pbLSabY5XPxo+XWdxwZaFEn47Go4NzFiHu9sNNmr/kXivP1vStfvMaK1GQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -13072,6 +13002,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "devOptional": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -13134,7 +13065,6 @@ "integrity": "sha512-jRKj0n0jXWo6kh62nA5TEh3+4igKDXLvzBJcPpiizP7oOolUrYIxmVBG9TOtHYFHoddUk6YvAkGeGoSVTXfQXQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -13233,6 +13163,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -13446,7 +13377,6 @@ "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "spdx-correct": "^3.0.0", "spdx-expression-parse": "^3.0.0" @@ -13501,6 +13431,7 @@ "integrity": "sha512-5hI5NCJwKBGtzWtdKB3c2fOEpI77Iaa0z4mSzZPU1cJ/OqrGbFafm90edVCd7T9Snz+Sh09TMAv4EQqyVLzuEg==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@oxc-project/runtime": "0.101.0", "fdir": "^6.5.0", @@ -13613,6 +13544,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "devOptional": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -13626,6 +13558,7 @@ "integrity": "sha512-n1RxDp8UJm6N0IbJLQo+yzLZ2sQCDyl1o0LeugbPWf8+8Fttp29GghsQBjYJVmWq3gBFfe9Hs1spR44vovn2wA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/expect": "4.0.15", "@vitest/mocker": "4.0.15", @@ -14254,6 +14187,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -14277,6 +14211,7 @@ "integrity": "sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -14603,14 +14538,14 @@ "integrity": "sha512-COyid6B1RYs+bpzUCInsA4HY/WZkpDLfkQ90+AqU/TVTpzYSbAC2JCbIwy0cRElBvlhI4bQ+9Wg6hSQKMpEkpA==" }, "node_modules/windmill-parser-wasm-py": { - "version": "1.601.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.601.1.tgz", - "integrity": "sha512-xcNZE/8B29yfl6UuQDPSXMD+83/W2Hzt2uhn+WrNvy0+qzk6nLh/vJGrf2srLBngYX1TxhUI5Jgseg0PK9yvNw==" + "version": "1.623.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.623.1.tgz", + "integrity": "sha512-lFBlZg6hvhHzsU5oPJq0478UyMTZ9UKVd8Hc8ggmxPIHZaJBeJ+56NR75hmGwg0VJcRff8ed+zEm1PgmjVhD+w==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.593.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.593.0.tgz", - "integrity": "sha512-m8BvTGJc2710YODmKKDXiASfssoJ/YFJGfYRhRnQvltENvaRee2NZuf4XoUkOVJESDdCSEX6U6fVkFvF9rXp2Q==" + "version": "1.623.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.623.1.tgz", + "integrity": "sha512-rW3pl4ysIXmVAmwxSTKhR2sUfScXKfTqxrUebDg0ZNcUzY97S8mNRdQGZfaQ1NWhWPRQNXfL/z/yaUOQh5ConQ==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", @@ -14623,9 +14558,9 @@ "integrity": "sha512-21S7lm1KF8zO1187rbq14hzPHII2RdM2+D44MoAh1F6VoaScj+Puq0z5B1O/hwn/95R/a9jBlL2D8jbkXtlD1A==" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.593.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.593.0.tgz", - "integrity": "sha512-NFY9gaEIpJOwGZJeGYDS3+/16QiPYdxFgmb1bKhXyEAdWjYMQ+otrTezf+K09lmvPzB+len37GlNCU72OgyQ6A==" + "version": "1.623.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.623.1.tgz", + "integrity": "sha512-FBwi/zXxjhZcCvi04oFdNivazru1ynIqSbafHSArfaaBWesBO3nye9UO/WXUlWZm5a7BExbU+3R/eVJrGaornw==" }, "node_modules/windmill-parser-wasm-yaml": { "version": "1.593.0", @@ -14776,7 +14711,6 @@ "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "imurmurhash": "^0.1.4", "signal-exit": "^4.0.1" @@ -14785,29 +14719,6 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, - "node_modules/ws": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", - "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/xml-utils": { "version": "1.10.2", "resolved": "https://registry.npmjs.org/xml-utils/-/xml-utils-1.10.2.tgz", @@ -14995,7 +14906,6 @@ "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "dev": true, "license": "ISC", - "peer": true, "engines": { "node": ">=10" } @@ -15015,6 +14925,7 @@ "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.27.tgz", "integrity": "sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==", "license": "MIT", + "peer": true, "dependencies": { "lib0": "^0.2.99" }, @@ -15050,6 +14961,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/frontend/package.json b/frontend/package.json index cc63f84d8f..4c6875d1ee 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.623.1", + "version": "1.624.0", "scripts": { "dev": "vite dev", "build": "vite build", @@ -149,11 +149,11 @@ "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.574.1", - "windmill-parser-wasm-py": "1.601.1", - "windmill-parser-wasm-regex": "1.593.0", + "windmill-parser-wasm-py": "1.623.1", + "windmill-parser-wasm-regex": "1.623.1", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", - "windmill-parser-wasm-ts": "1.593.0", + "windmill-parser-wasm-ts": "1.623.1", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "^1.3.2", diff --git a/frontend/src/lib/components/FlowGraphViewerStep.svelte b/frontend/src/lib/components/FlowGraphViewerStep.svelte index 0904373416..81d15843d8 100644 --- a/frontend/src/lib/components/FlowGraphViewerStep.svelte +++ b/frontend/src/lib/components/FlowGraphViewerStep.svelte @@ -245,7 +245,7 @@

Iterator expression:

{#if stepDetail.value.iterator.type == 'static'} - {:else} + {:else if stepDetail.value.iterator.type == 'javascript'} diff --git a/frontend/src/lib/components/InputTransformForm.svelte b/frontend/src/lib/components/InputTransformForm.svelte index 9cfeecf9da..3ea8560261 100644 --- a/frontend/src/lib/components/InputTransformForm.svelte +++ b/frontend/src/lib/components/InputTransformForm.svelte @@ -42,9 +42,6 @@ import { inputBorderClass } from './text_input/TextInput.svelte' import FakeMonacoPlaceHolder from './FakeMonacoPlaceHolder.svelte' - // We add 'ai' for ai agent tools. 'ai' means the field will be filled by the AI agent dynamically. - type PropertyType = InputTransform['type'] | 'ai' - interface Props { schema: Schema | { properties?: Record; required?: string[] } arg: InputTransform | any @@ -162,7 +159,7 @@ }) } - function getPropertyType(arg: InputTransform | any): PropertyType { + function getPropertyType(arg: InputTransform | any): InputTransform['type'] { // For agent tools, if static with undefined/empty value, treat as 'ai', meaning the field will be filled by the AI agent dynamically. if ( isAgentTool && @@ -174,7 +171,7 @@ return 'ai' } - let type: PropertyType = arg?.type ?? 'static' + let type: InputTransform['type'] = arg?.type ?? 'static' if ( type == 'javascript' && @@ -408,7 +405,7 @@ function updateStaticInput( inputCat: InputCat, - propertyType: PropertyType, + propertyType: InputTransform['type'], arg: InputTransform | any ) { if (!isStaticTemplate(inputCat)) { @@ -821,7 +818,11 @@ otherArgs={Object.fromEntries( Object.entries(otherArgs).map(([key, transform]) => [ key, - transform?.type === 'static' ? transform.value : transform?.expr + transform?.type === 'static' + ? transform.value + : transform?.type === 'javascript' + ? transform.expr + : undefined ]) )} > @@ -830,12 +831,14 @@ - switchToJsAndConnect((path) => appendPathToArrayExpr(arg.expr, path))} + switchToJsAndConnect((path) => + appendPathToArrayExpr(arg?.type === 'javascript' ? arg.expr : '', path) + )} /> {/if} {/snippet} - {:else if arg.expr != undefined} + {:else if arg?.type === 'javascript' && arg.expr != undefined}
diff --git a/frontend/src/lib/components/InputTransformsViewer.svelte b/frontend/src/lib/components/InputTransformsViewer.svelte index f42201c427..15318ec31c 100644 --- a/frontend/src/lib/components/InputTransformsViewer.svelte +++ b/frontend/src/lib/components/InputTransformsViewer.svelte @@ -33,10 +33,12 @@ {val.value} {/if} - {:else} + {:else if val.type == 'javascript'} {cleanExpr(val.expr)} + {:else if val.type == 'ai'} + Filled by AI {/if} diff --git a/frontend/src/lib/components/PageHeader.svelte b/frontend/src/lib/components/PageHeader.svelte index 2085214bf4..07b02227d4 100644 --- a/frontend/src/lib/components/PageHeader.svelte +++ b/frontend/src/lib/components/PageHeader.svelte @@ -8,7 +8,7 @@ export let childrenWrapperDivClasses: string = '' -
+
{#if primary}

state && (state.hover = true)} - onmouseleave={() => state && (state.hover = false)} + onmouseenter={() => _state && (_state.hover = true)} + onmouseleave={() => _state && (_state.hover = false)} > -
-
-
- {#if error} - - {:else} - - {/if} -
-
-

{@html processMessage(message)}

- {#if errorMessage} -

- {errorMessage} -

- {/if} -
- -
- -
+
+
+
-
+
+

+ {@html processMessage(displayMessage)} + {#if isLongMessage && !showMore} + + {/if} +

+ {#if errorMessage} +

+ {errorMessage} +

+ {/if} +
+ +
{#each actions as action, index (index)} {/each}
+ +
+ +
diff --git a/frontend/src/lib/components/TooltipInner.svelte b/frontend/src/lib/components/TooltipInner.svelte index 28dab0595f..437fbc1c02 100644 --- a/frontend/src/lib/components/TooltipInner.svelte +++ b/frontend/src/lib/components/TooltipInner.svelte @@ -2,13 +2,19 @@ import Markdown from 'svelte-exmarkdown' import { ExternalLink } from 'lucide-svelte' import { gfmPlugin } from 'svelte-exmarkdown/gfm' + import { twMerge } from 'tailwind-merge' + export let documentationLink: string | undefined = undefined export let markdownTooltip: string | undefined = undefined + export let customBgClass: string | undefined = undefined const plugins = [gfmPlugin()]
{#if markdownTooltip}
diff --git a/frontend/src/lib/components/assets/AssetColumnBadges.svelte b/frontend/src/lib/components/assets/AssetColumnBadges.svelte new file mode 100644 index 0000000000..8bb02bd456 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetColumnBadges.svelte @@ -0,0 +1,44 @@ + + +{#if entries?.length} +
+ {#each entries as [columnName, accessType]} + {@const accessType2 = formatAssetAccessType(accessType)} + {#snippet badge()} +
+ {columnName} +
+ {/snippet} + {#if disableTooltip} + {@render badge()} + {:else} + + {@render badge()} + + {accessType2} access to column "{columnName}" + + + {/if} + {/each} +
+{/if} diff --git a/frontend/src/lib/components/assets/AssetsUsageDrawer.svelte b/frontend/src/lib/components/assets/AssetsUsageDrawer.svelte index e6e6f78fc9..177032341b 100644 --- a/frontend/src/lib/components/assets/AssetsUsageDrawer.svelte +++ b/frontend/src/lib/components/assets/AssetsUsageDrawer.svelte @@ -6,8 +6,9 @@ import Tooltip from '../meltComponents/Tooltip.svelte' import Tooltip2 from '../Tooltip.svelte' import { twMerge } from 'tailwind-merge' - import { displayDate } from '$lib/utils' + import { capitalize, displayDate } from '$lib/utils' import Alert from '../common/alert/Alert.svelte' + import AssetColumnBadges from './AssetColumnBadges.svelte' let usagesDrawerData: | { @@ -64,14 +65,10 @@ -{#snippet badge(text: string | undefined, tooltip?: string)} +{#snippet rightBadge(text: string | undefined, tooltip?: string)} {#if text} -
+
{text}
@@ -92,7 +89,7 @@ -
- - {u.kind == 'job' ? (u.metadata?.runnable_path ?? 'Unknown job') : u.path} +
+ + + {u.kind == 'job' ? (u.metadata?.runnable_path ?? 'Unknown job') : u.path} + - {u.kind == 'job' ? u.path : u.kind} + + {u.kind == 'job' ? u.path : capitalize(u.kind)} + +
- {@render badge(displayDate(u.created_at), 'Asset detection time')} - {@render badge(accessType)} + {@render rightBadge(displayDate(u.created_at), 'Asset detection time')} + {@render rightBadge(accessType)} {/each} diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index 228a2fabe5..2475e2b126 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -12,6 +12,7 @@ export type AssetKind = _AssetKind export type AssetWithAccessType = Asset & { access_type?: AssetUsageAccessType } export type AssetWithAltAccessType = AssetWithAccessType & { alt_access_type?: AssetUsageAccessType + columns?: Record } export type AssetUsage = ListAssetsResponse['assets'][number]['usages'][number] diff --git a/frontend/src/lib/components/common/alert/Alert.svelte b/frontend/src/lib/components/common/alert/Alert.svelte index 3a1361b2ff..7c03de5dca 100644 --- a/frontend/src/lib/components/common/alert/Alert.svelte +++ b/frontend/src/lib/components/common/alert/Alert.svelte @@ -1,14 +1,7 @@ {#snippet children({ darkMode })} - +
- + {formatShortAssetPath(data.asset)} + {#if data.asset.kind === 'resource' && cachedResourceMetadata === undefined} @@ -319,6 +337,7 @@ {formatAssetKind({ ...data.asset, metadata: cachedResourceMetadata })} + {/snippet} diff --git a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte index 2353d37aa8..623a725b01 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte @@ -60,7 +60,7 @@
    {#each data.overflowedAssets as asset}
  • - +
    • {/each}
diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index a0c0ee7b69..4b6543d2d3 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -314,7 +314,7 @@ export const settings: Record = { label: 'UV index url', description: 'Add private Pip registry', key: 'pip_index_url', - fieldType: 'text', + fieldType: 'password', placeholder: 'https://username:password@pypi.company.com/simple', storage: 'setting', ee_only: '' @@ -323,7 +323,7 @@ export const settings: Record = { label: 'UV extra index url', description: 'Add private extra Pip registry', key: 'pip_extra_index_url', - fieldType: 'text', + fieldType: 'password', placeholder: 'https://username:password@pypi.company.com/simple', storage: 'setting', ee_only: '' @@ -332,7 +332,7 @@ export const settings: Record = { label: 'Npm config registry', description: 'Add private npm registry', key: 'npm_config_registry', - fieldType: 'text', + fieldType: 'password', placeholder: 'https://registry.npmjs.org/:_authToken=npm_FOOBAR', storage: 'setting', ee_only: '' @@ -342,7 +342,7 @@ export const settings: Record = { description: 'Add private scoped registries for Bun, See: https://bun.sh/docs/install/registries', key: 'bunfig_install_scopes', - fieldType: 'text', + fieldType: 'password', placeholder: '"@myorg3" = { token = "mytoken", url = "https://registry.myorg.com/" }', storage: 'setting', ee_only: '' @@ -360,7 +360,7 @@ export const settings: Record = { label: 'Maven/Ivy repositories', description: 'Add private Maven/Ivy repositories', key: 'maven_repos', - fieldType: 'text', + fieldType: 'password', placeholder: 'https://user:password@artifacts.foo.com/maven', storage: 'setting', ee_only: '' @@ -377,7 +377,7 @@ export const settings: Record = { label: 'Ruby Gems repositories', description: 'Add private Ruby repositories with credentials. Should end with /', key: 'ruby_repos', - fieldType: 'text', + fieldType: 'password', placeholder: 'https://user:password@gems.foo.com/', storage: 'setting', ee_only: '' diff --git a/frontend/src/lib/components/meltComponents/Tooltip.svelte b/frontend/src/lib/components/meltComponents/Tooltip.svelte index 297709b672..ea464fa123 100644 --- a/frontend/src/lib/components/meltComponents/Tooltip.svelte +++ b/frontend/src/lib/components/meltComponents/Tooltip.svelte @@ -16,6 +16,7 @@ export let openDelay: number = 300 export let closeDelay: number = 0 export let portal: string | undefined | null = 'body' + export let customBgClass: string | undefined = undefined const { elements: { trigger, content }, @@ -47,7 +48,7 @@ {#if $open && !disablePopup}
- +
diff --git a/frontend/src/lib/components/progressBar/ProgressBar.svelte b/frontend/src/lib/components/progressBar/ProgressBar.svelte index 0144fdb0fe..fe6b7c0a77 100644 --- a/frontend/src/lib/components/progressBar/ProgressBar.svelte +++ b/frontend/src/lib/components/progressBar/ProgressBar.svelte @@ -32,6 +32,8 @@ isWaitingForEvents?: boolean // Whether the job was canceled isCanceled?: boolean + // Whether the job is scheduled for later + isScheduled?: boolean } let { @@ -50,7 +52,8 @@ stepId, showStepId = false, isWaitingForEvents = false, - isCanceled = false + isCanceled = false, + isScheduled = false }: Props = $props() let duration = 200 @@ -146,10 +149,10 @@ : 'text-blue-700 dark:text-blue-200'}" >
- {#if status == 'running' && !isCanceled} + {#if status == 'running' && !isCanceled && !isScheduled} {/if} - {#key status + isWaitingForEvents + stepId + isCanceled} + {#key status + isWaitingForEvents + stepId + isCanceled + isScheduled} {#if status == 'error'} Error occurred @@ -159,6 +162,8 @@ Done {:else if isWaitingForEvents} Waiting to be resumed + {:else if isScheduled} + (not started) {:else if showStepId} {stepId ? `${isCanceled ? 'Canceled at' : 'Running'} step ${stepId}` @@ -166,9 +171,9 @@ {:else if hideStepTitle} {isCanceled ? 'Canceled' : 'Running'} {:else if subIndexIsPercent} - {`${isCanceled ? 'Canceled at' : ''} Step ${index + 1} (${subIndex !== undefined ? `${subIndex}%)` : ''}`} + {(isCanceled ? 'Canceled at ' : '') + `Step ${index + 1} (${subIndex !== undefined ? subIndex + '%' : ''})`} {:else} - {`${isCanceled ? 'Canceled at' : ''} Step ${index + 1}${subIndex !== undefined ? `.${subIndex + 1}` : ''}`} + {(isCanceled ? 'Canceled at ' : '') + `Step ${index + 1}${subIndex !== undefined ? `.${subIndex + 1}` : ''}`} {/if} {/key} diff --git a/frontend/src/lib/components/triggers/TriggerFilters.svelte b/frontend/src/lib/components/triggers/TriggerFilters.svelte new file mode 100644 index 0000000000..e2657f6cbd --- /dev/null +++ b/frontend/src/lib/components/triggers/TriggerFilters.svelte @@ -0,0 +1,78 @@ + + +
+

+ Filters will limit the execution of the trigger to only messages that match all criteria.
+ The JSON filter checks if the value at the key is equal or a superset of the filter value. +

+
+ {#each filters as v, i (i)} +
+
+ + + + {#if v.key} + {@const isObject = v.value !== null && typeof v.value === 'object'} +
+ payload.{v.key} + {isObject ? '⊇' : '=='} + {JSON.stringify(v.value)} +
+ {/if} +
+ +
+ {/each} + +
+ +
+
+
diff --git a/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte index f36af9e0f0..e77846672d 100644 --- a/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte @@ -22,6 +22,7 @@ import { deepEqual } from 'fast-equals' import TriggerSuspendedJobsAlert from '../TriggerSuspendedJobsAlert.svelte' import TriggerSuspendedJobsModal from '../TriggerSuspendedJobsModal.svelte' + import TriggerFilters from '../TriggerFilters.svelte' interface Props { useDrawer?: boolean @@ -85,6 +86,7 @@ let error_handler_path: string | undefined = $state() let error_handler_args: Record = $state({}) let retry: Retry | undefined = $state() + let filters: { key: string; value: any }[] = $state([]) let suspendedJobsModal = $state(null) let originalConfig = $state | undefined>(undefined) @@ -173,6 +175,7 @@ error_handler_path = nDefaultValues?.error_handler_path ?? undefined error_handler_args = nDefaultValues?.error_handler_args ?? {} retry = nDefaultValues?.retry ?? undefined + filters = nDefaultValues?.filters ?? [] errorHandlerSelected = getHandlerType(error_handler_path ?? '') mode = nDefaultValues?.mode ?? 'enabled' originalConfig = undefined @@ -199,6 +202,7 @@ error_handler_path = cfg?.error_handler_path error_handler_args = cfg?.error_handler_args ?? {} retry = cfg?.retry + filters = cfg?.filters ?? [] errorHandlerSelected = getHandlerType(error_handler_path ?? '') } @@ -223,6 +227,7 @@ kafka_resource_path: kafkaResourcePath, group_id: kafkaCfg.group_id, topics: kafkaCfg.topics, + filters, mode, extra_perms: extra_perms, error_handler_path, @@ -439,6 +444,8 @@ showTestingBadge={isEditor} /> + +
diff --git a/frontend/src/lib/components/triggers/kafka/utils.ts b/frontend/src/lib/components/triggers/kafka/utils.ts index d83d345b3c..419b4225d1 100644 --- a/frontend/src/lib/components/triggers/kafka/utils.ts +++ b/frontend/src/lib/components/triggers/kafka/utils.ts @@ -23,6 +23,7 @@ export async function saveKafkaTriggerFromCfg( kafka_resource_path: cfg.kafka_resource_path, group_id: cfg.group_id, topics: cfg.topics, + filters: cfg.filters ?? [], ...errorHandlerAndRetries } try { diff --git a/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte index 36e7481b33..fedcbf3ee9 100644 --- a/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte @@ -25,6 +25,7 @@ import { fade } from 'svelte/transition' import type { Schema } from '$lib/common' import JsonEditor from '$lib/components/JsonEditor.svelte' + import TriggerFilters from '../TriggerFilters.svelte' import Toggle from '$lib/components/Toggle.svelte' import WebsocketEditorConfigSection from './WebsocketEditorConfigSection.svelte' import { untrack, type Snippet } from 'svelte' @@ -684,86 +685,7 @@
-
-

- Filters will limit the execution of the trigger to only messages that match all criteria.
- The JSON filter checks if the value at the key is equal or a superset of the filter value. -

-
- {#each filters as v, i} -
-
-
- -
- - - -
- -
- {/each} - -
- -
-
-
+
diff --git a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte index 01ac8173b9..570fe8cf37 100644 --- a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte @@ -203,7 +203,7 @@ {#if emptyString(tableRow[1].resourcePath) || isDirty(tableRow[0])} @@ -242,11 +242,12 @@ {/each} -
+ {#snippet addSecondaryStorageBtn()} + {/snippet} +
+ {#if !s3ResourceSettings.resourcePath} + + {#snippet trigger()} + {@render addSecondaryStorageBtn()} + {/snippet} + {#snippet content()} + Setup a primary storage to use secondary storages + {/snippet} + + {:else} + {@render addSecondaryStorageBtn()} + {/if}
diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index b12ca1e671..4360e912c5 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -1,4 +1,10 @@ -import { ScriptService, type MainArgSignature, FlowService, type Script } from '$lib/gen' +import { + ScriptService, + type MainArgSignature, + FlowService, + type Script, + type AssetUsageAccessType +} from '$lib/gen' import { get, writable } from 'svelte/store' import type { Schema, SupportedLanguage } from './common.js' import { emptySchema, sortObject } from './utils.js' @@ -90,8 +96,18 @@ async function initWasmRuby() { } type InferAssetsResult = - | { status: 'ok'; assets: AssetWithAccessType[]; sql_queries?: InferAssetsSqlQueryDetails[] } - | { status: 'error'; error: string; assets?: undefined; sql_queries?: undefined } + | { + status: 'ok' + assets: AssetWithAccessType[] + sql_queries?: InferAssetsSqlQueryDetails[] + columns?: Record + } + | { + status: 'error' + error: string + assets?: undefined + sql_queries?: undefined + } export type InferAssetsSqlQueryDetails = { query_string: string // SQL query with $1 placeholders for interpolations diff --git a/frontend/src/lib/toast.ts b/frontend/src/lib/toast.ts index f598874ca9..244794a5cc 100644 --- a/frontend/src/lib/toast.ts +++ b/frontend/src/lib/toast.ts @@ -1,4 +1,4 @@ -import Toast from '$lib/components/Toast.svelte' +import Toast, { type ToastType } from '$lib/components/Toast.svelte' import { toast } from '@zerodevx/svelte-toast' import type { ComponentProps } from 'svelte' import type { Button } from './components/common' @@ -11,15 +11,18 @@ export type ToastAction = { export function sendUserToast( message: string, - error: boolean = false, + _type: boolean | ToastType = 'success', actions: ToastAction[] = [], errorMessage: string | undefined = undefined, duration: number = 5000 ): void { + const type = typeof _type === 'boolean' ? (_type ? 'error' : 'success') : _type + const error = type === 'error' if (globalThis.windmillToast) { globalThis.windmillToast({ message, error, + type, actions, errorMessage, duration @@ -34,7 +37,7 @@ export function sendUserToast( src: Toast, props: { message, - error, + type, actions, errorMessage, duration @@ -47,7 +50,10 @@ export function sendUserToast( '--toastPadding': '0', '--toastMsgPadding': '0', '--toastBackground': '#00000000', - '--toastBorderRadius': '0.4rem' + '--toastBorderRadius': '0.4rem', + '--toastWidth': '34rem', + '--toastMinHeight': '1rem', + '--toastBoxShadow': 'none' } }) } diff --git a/frontend/src/routes/(root)/(logged)/assets/+page.svelte b/frontend/src/routes/(root)/(logged)/assets/+page.svelte index a62b18c431..252507d2f9 100644 --- a/frontend/src/routes/(root)/(logged)/assets/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/assets/+page.svelte @@ -6,7 +6,13 @@ import S3FilePicker from '$lib/components/S3FilePicker.svelte' import { Cell, DataTable } from '$lib/components/table' import Head from '$lib/components/table/Head.svelte' - import { AssetService, type AssetKind, type ListAssetsResponse } from '$lib/gen' + import { + AssetService, + SettingService, + WorkspaceService, + type AssetKind, + type ListAssetsResponse + } from '$lib/gen' import { userStore, workspaceStore, userWorkspaces } from '$lib/stores' import { pluralize, truncate } from '$lib/utils' import ExploreAssetButton, { @@ -15,13 +21,15 @@ import AssetsUsageDrawer from '$lib/components/assets/AssetsUsageDrawer.svelte' import AssetGenericIcon from '$lib/components/icons/AssetGenericIcon.svelte' import { Tooltip } from '$lib/components/meltComponents' - import { AlertTriangle, Loader2 } from 'lucide-svelte' + import { AlertTriangle, Loader2, SettingsIcon } from 'lucide-svelte' import { StaleWhileLoading, useInfiniteQuery, useScrollToBottom } from '$lib/svelte5Utils.svelte' - import { Debounced, watch } from 'runed' + import { Debounced, resource, watch, type ResourceReturn } from 'runed' import Label from '$lib/components/Label.svelte' import MultiSelect from '$lib/components/select/MultiSelect.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import RefreshButton from '$lib/components/common/button/RefreshButton.svelte' + import Section from '$lib/components/Section.svelte' + import Button from '$lib/components/common/button/Button.svelte' interface AssetCursor { created_at?: string @@ -76,6 +84,33 @@ let s3FilePicker: S3FilePicker | undefined = $state() let dbManagerDrawer: DbManagerDrawer | undefined = $state() let assetsUsageDropdown: AssetsUsageDrawer | undefined = $state() + + let allS3Storages = resource( + () => $workspaceStore, + () => + SettingService.getSecondaryStorageNames({ + workspace: $workspaceStore!, + includeDefault: true + }).then((s) => + s.map((s) => + s == '_default_' ? { label: 'Primary storage', value: '/' } : { label: s, value: `${s}/` } + ) + ) + ) + let allDucklakes = resource( + () => $workspaceStore, + () => + WorkspaceService.listDucklakes({ workspace: $workspaceStore! }).then((d) => + d.map((d) => ({ label: d == 'main' ? 'Primary ducklake' : d, value: d })) + ) + ) + let allDataTables = resource( + () => $workspaceStore, + () => + WorkspaceService.listDataTables({ workspace: $workspaceStore! }).then((d) => + d.map((d) => ({ label: d == 'main' ? 'Primary data table' : d, value: d })) + ) + ) {#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.assets} @@ -90,101 +125,179 @@ tooltip="Assets show up here whenever you use them in Windmill." documentationLink="https://www.windmill.dev/docs/core_concepts/assets" /> -
-
- - - -
- assetsQuery.reset()} loading={assetsQuery.isLoading} /> -
- - - - - Asset name - - - - - - {#if assets != undefined && assets.length === 0} - - - No assets found - - {/if} - {#each assets as asset} - {@const assetUri = formatAsset(asset)} - - - - - {formatAssetKind(asset)} - - - - {truncate(asset.path, 92)} - {formatAssetKind(asset)} - - - assetsUsageDropdown?.open(asset)}> - {pluralize(asset.usages.length, 'usage')} - - - - {#if assetCanBeExplored(asset, asset.metadata) && !$userStore?.operator} - - {/if} - {#if asset.kind === 'resource' && asset.metadata === undefined} - - - Could not find resource - - {/if} - - - {/each} - {#if assets == undefined && assetsQuery.isLoading} - - -
- - Loading assets... + +
+
+ {#snippet card(props: { + title: string + assetKind: AssetKind + data: ResourceReturn<{ label: string; value: string }[]> + settingsHref: string + })} +
+
+

{props.title}

+ Settings +
+ {#if props.data.current?.length} +
+ {#each props.data.current ?? [] as item} +
+ {item.label} + +
+ {/each}
- - - {/if} - - - {#if assetsQuery.isFetchingNextPage} - - {:else if assets?.length && !assetsQuery.hasNextPage} -
No more assets to load
- {/if} + {/if} + {#if props.data.loading} +
+ +
+ {:else if props.data.error} +
+ Error loading {props.title.toLowerCase()} +
+ {:else if props.data.current?.length === 0} +
+ No {props.title.toLowerCase()} yet +
+ + {/if} +
+ {/snippet} + {@render card({ + title: 'Data tables', + data: allDataTables, + assetKind: 'datatable', + settingsHref: '/workspace_settings?tab=windmill_data_tables' + })} + {@render card({ + title: 'Ducklakes', + data: allDucklakes, + assetKind: 'ducklake', + settingsHref: '/workspace_settings?tab=windmill_lfs' + })} + {@render card({ + title: 'Workspace object storages', + data: allS3Storages, + assetKind: 's3object', + settingsHref: '/workspace_settings?tab=windmill_lfs' + })} +
+
+
+
+
+ + + +
+ assetsQuery.reset()} loading={assetsQuery.isLoading} /> +
+ {@render table()} + {#if assetsQuery.isFetchingNextPage} + + {:else if assets?.length && !assetsQuery.hasNextPage} +
No more assets to load
+ {/if} +
{/if} + +{#snippet table()} + + + + + Asset name + + + + + + {#if assets != undefined && assets.length === 0} + + + No assets found + + {/if} + {#each assets as asset} + {@const assetUri = formatAsset(asset)} + + + + + {formatAssetKind(asset)} + + + + {truncate(asset.path, 92)} + {formatAssetKind(asset)} + + + assetsUsageDropdown?.open(asset)}> + {pluralize(asset.usages.length, 'usage')} + + + + {#if assetCanBeExplored(asset, asset.metadata) && !$userStore?.operator} + + {/if} + {#if asset.kind === 'resource' && asset.metadata === undefined} + + + Could not find resource + + {/if} + + + {/each} + {#if assets == undefined && assetsQuery.isLoading} + + +
+ + Loading assets... +
+ + + {/if} + + +{/snippet} diff --git a/frontend/src/routes/(root)/(logged)/tutorials/+page.svelte b/frontend/src/routes/(root)/(logged)/tutorials/+page.svelte index 93ebdd4b20..db41264b5a 100644 --- a/frontend/src/routes/(root)/(logged)/tutorials/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/tutorials/+page.svelte @@ -1,7 +1,6 @@ -
-
- -

Tutorials

+ {#if activeTabs.length > 0} +
+ - -
- {/if} -
+ Mark all as completed + + +
+ {/if} + +
{#if $userStore?.is_admin}
diff --git a/frontend/src/routes/+layout.svelte b/frontend/src/routes/+layout.svelte index a54c75d5b8..feb5211bfb 100644 --- a/frontend/src/routes/+layout.svelte +++ b/frontend/src/routes/+layout.svelte @@ -16,8 +16,8 @@ next: 0, // next progress value pausable: true, // pause progress bar tween on mouse hover dismissable: true, // allow dismiss with close button - reversed: false, // insert new toast to bottom of stack - intro: { x: 256 }, // toast intro fly animation settings + reversed: true, // insert new toast to bottom of stack + intro: { y: -32 }, // toast intro fly animation settings theme: {} // css var overrides } @@ -43,11 +43,22 @@ diff --git a/imgs/windmill-banner.png b/imgs/windmill-banner.png index 6b50789f5a..907d7565bc 100644 Binary files a/imgs/windmill-banner.png and b/imgs/windmill-banner.png differ diff --git a/imgs/windmill-builder.png b/imgs/windmill-builder.png index 93d42b55ef..7bd98fd79c 100644 Binary files a/imgs/windmill-builder.png and b/imgs/windmill-builder.png differ diff --git a/imgs/windmill-editor.png b/imgs/windmill-editor.png index 16b0dcfdc9..63b4202880 100644 Binary files a/imgs/windmill-editor.png and b/imgs/windmill-editor.png differ diff --git a/imgs/windmill-flow.png b/imgs/windmill-flow.png index bbe2ec981a..cdc613eefe 100644 Binary files a/imgs/windmill-flow.png and b/imgs/windmill-flow.png differ diff --git a/imgs/windmill-result.png b/imgs/windmill-result.png index fd34ad63a8..5e8c1606bd 100644 Binary files a/imgs/windmill-result.png and b/imgs/windmill-result.png differ diff --git a/imgs/windmill-run.png b/imgs/windmill-run.png index d2e760f912..f5749bdc3b 100644 Binary files a/imgs/windmill-run.png and b/imgs/windmill-run.png differ diff --git a/integration_tests/docker-compose.yml b/integration_tests/docker-compose.yml index 8d3d3fadca..70f0c7ab26 100644 --- a/integration_tests/docker-compose.yml +++ b/integration_tests/docker-compose.yml @@ -36,6 +36,7 @@ services: environment: - DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable - MODE=server + - LISTEN_NEW_EVENTS_INTERVAL_SEC=1 healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8000/api/version || exit 1"] interval: 10s @@ -60,6 +61,7 @@ services: - DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable - MODE=worker - WORKER_GROUP=default + - LISTEN_NEW_EVENTS_INTERVAL_SEC=1 healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8000/api/version || exit 1"] interval: 10s diff --git a/lsp/Pipfile b/lsp/Pipfile index 05037ca4f7..24fffdf87e 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.623.1" -wmill_pg = ">=1.623.1" +wmill = ">=1.624.0" +wmill_pg = ">=1.624.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 2467803b53..af3840b286 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.623.1 + version: 1.624.0 title: OpenFlow Spec contact: name: Ruben Fiszel @@ -333,11 +333,13 @@ components: oneOf: - $ref: '#/components/schemas/StaticTransform' - $ref: '#/components/schemas/JavascriptTransform' + - $ref: '#/components/schemas/AiTransform' discriminator: propertyName: type mapping: static: '#/components/schemas/StaticTransform' javascript: '#/components/schemas/JavascriptTransform' + ai: '#/components/schemas/AiTransform' StaticTransform: type: object @@ -367,6 +369,17 @@ components: - expr - type + AiTransform: + type: object + description: Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter. + properties: + type: + type: string + enum: + - ai + required: + - type + FlowModuleValue: description: The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type oneOf: diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index f51cc2f571..1bd9323bcb 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.623.1' + ModuleVersion = '1.624.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index eeeda177e5..ef9f7b2df4 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.623.1" +version = "1.624.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 02e06d5264..3ed1b8d1f7 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.623.1" +version = "1.624.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 6ddcd56f64..eeb7306056 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.623.1", + "version": "1.624.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index fcfeacbe25..0b8d3e185d 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.623.1", + "version": "1.624.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "sideEffects": false, diff --git a/version.txt b/version.txt index 30984353b9..544107d452 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.623.1 +1.624.0 diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index a1994aecb1..4b4d1c1fc1 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.0.1", + "version": "1.0.2", "description": "YAML validator for Windmill", "main": "dist/index.js", "types": "dist/index.d.ts",