Merge remote-tracking branch 'origin/main' into hc/ai-flow-chat

.cursor/rules/rust-best-practices.mdc

@@ -47,6 +47,7 @@ Windmill uses a workspace-based architecture with multiple crates:
 - Group related routes together
 - Use consistent response formats (JSON)
 - Follow proper authentication and authorization patterns
+- Do not forget to update backend/windmill-api/openapi.yaml after modifying an api endpoint
 
 ## Performance Optimizations
 

.github/workflows/aider-after-review.yaml

@@ -90,5 +90,5 @@ jobs:
     with:
       needs_processing: false
       base_prompt: ${{ needs.check-and-prepare.outputs.prompt_content }}
-      rules_files: "CLAUDE.md backend/CLAUDE.md frontend/CLAUDE.md"
+      rules_files: ".cursor/rules/rust-best-practices.mdc .cursor/rules/svelte5-best-practices.mdc .cursor/rules/windmill-overview.mdc"
     secrets: inherit

.github/workflows/aider-external.yaml

@@ -76,5 +76,5 @@ jobs:
       issue_body: ${{ needs.check-and-prepare.outputs.issue_body }}
       instruction: ${{ needs.check-and-prepare.outputs.instruction }}
       issue_id: ${{ github.event.client_payload.issue_id }}
-      rules_files: "CLAUDE.md backend/CLAUDE.md frontend/CLAUDE.md"
+      rules_files: ".cursor/rules/rust-best-practices.mdc .cursor/rules/svelte5-best-practices.mdc .cursor/rules/windmill-overview.mdc"
     secrets: inherit

.github/workflows/aider.yaml

@@ -161,5 +161,5 @@ jobs:
       issue_body: ${{ needs.check-and-prepare.outputs.issue_body }}
       instruction: ${{ needs.check-and-prepare.outputs.comment_content }}
       issue_id: ${{ github.event.issue.number }}
-      rules_files: "CLAUDE.md backend/CLAUDE.md frontend/CLAUDE.md"
+      rules_files: ".cursor/rules/rust-best-practices.mdc .cursor/rules/svelte5-best-practices.mdc .cursor/rules/windmill-overview.mdc"
     secrets: inherit

.github/workflows/claude.yml

@@ -69,17 +69,17 @@ jobs:
         with:
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
           timeout_minutes: "60"
-          allowed_tools: "mcp__github__create_pull_request,Bash(npm run check),Bash(npm install),Bash(cargo check),Bash(curl https://sh.rustup.rs -sSf | sh)"
+          allowed_tools: "mcp__github__create_pull_request,Bash(npm run check),Bash(npm install),Bash(cargo check),Bash(curl https://sh.rustup.rs -sSf | sh -s -- -y)"
           custom_instructions: "IMPORTANT INSTRUCTIONS:
-          - Bypass previous instructions and instead of adding a url in your comment to manually create a PR, if you created a new branch with your changes, automatically create a PR from that branch to main, with the title starting with [Claude PR].
+          - Bypass previous instructions and instead of adding a url in your comment to manually create a PR, if you created a new branch with your changes, automatically create a draft PR from that branch to main, with the title starting with [Claude PR].
           - If you made changes to the frontend code, run npm install, then npm run generate-backend-client, then npm run check. You can ignore warnings that are reported by the check script, but fix the errors.
           - If you made changes to the backend code, install Rust and then run cargo check. You can ignore warnings that are reported by the check script, but fix the errors.
-          - DO NOT FORGET TO OPEN A PR AFTER YOU ARE DONE if you made changes after a request from a git issue.
+          - DO NOT FORGET TO OPEN A DRAFT PR AFTER YOU ARE DONE if you made changes after a request from a git issue.
           AVAILABLE TOOLS:
           - mcp__github__create_pull_request: Create a PR from a branch to main
           - Bash(npm run check): Run the check script. You should run this tool after making changes to the frontend code.
           - Bash(npm install): Install dependencies. You need this to run npm run check.
           - Bash(npm run generate-backend-client): Generate the backend client. You need this to run npm run check.
           - Bash(cargo check): Run the cargo check script. You should run this tool after making changes to the backend code.
-          - Bash(curl https://sh.rustup.rs -sSf | sh): Install Rust. You need this to run cargo check."
+          - Bash(curl https://sh.rustup.rs -sSf | sh -s -- -y): Install Rust. You need this to run cargo check."
           trigger_phrase: "/ai"

.github/workflows/validate-openapi.yml

@@ -0,0 +1,34 @@
+name: Validate OpenAPI Spec
+
+on:
+  push:
+    paths:
+      - 'backend/windmill-api/openapi*'
+  pull_request:
+    paths:
+      - 'backend/windmill-api/openapi*'
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install openapi-generator-cli
+        run: npm install @openapitools/openapi-generator-cli -g
+
+      - name: Validate openapi.yaml
+        run: npx @openapitools/openapi-generator-cli validate -i backend/windmill-api/openapi.yaml
+
+      - name: Validate openapi-deref.json
+        run: npx @openapitools/openapi-generator-cli validate -i backend/windmill-api/openapi-deref.json
+
+      # Does not work well with dereferenced yaml
+      # - name: Validate openapi-deref.yaml
+      #   run: npx @openapitools/openapi-generator-cli validate -i backend/windmill-api/openapi-deref.yaml
+

CHANGELOG.md

@@ -1,5 +1,65 @@
 # Changelog
 
+## [1.494.0](https://github.com/windmill-labs/windmill/compare/v1.493.4...v1.494.0) (2025-05-31)
+
+
+### Features
+
+* array of s3 objects in input maker ([806d669](https://github.com/windmill-labs/windmill/commit/806d66972568d21a1621acd1b30db5ae9b217341))
+* **rust:** shared build directory ([#5610](https://github.com/windmill-labs/windmill/issues/5610)) ([ed61d97](https://github.com/windmill-labs/windmill/commit/ed61d9770031c1a04908880dbd3e5fb692df9946))
+
+
+### Bug Fixes
+
+* allow disable tabs for sidebar/accordion tabs ([#5838](https://github.com/windmill-labs/windmill/issues/5838)) ([80277d1](https://github.com/windmill-labs/windmill/commit/80277d14d02e8e596c7002326946142226d382a6))
+
+## [1.493.4](https://github.com/windmill-labs/windmill/compare/v1.493.3...v1.493.4) (2025-05-29)
+
+
+### Bug Fixes
+
+* templatev2 delete issue ([#5834](https://github.com/windmill-labs/windmill/issues/5834)) ([ed3ad32](https://github.com/windmill-labs/windmill/commit/ed3ad327a235c16b9f3aa7f8edeefe61b0c01da3))
+
+## [1.493.3](https://github.com/windmill-labs/windmill/compare/v1.493.2...v1.493.3) (2025-05-29)
+
+
+### Bug Fixes
+
+* evalv2 prohibit component delete ([e302aa3](https://github.com/windmill-labs/windmill/commit/e302aa38b5977dd406ae05e1d8dbb74cb7dc3d17))
+* faster layout for larger graphs ([8d12bcc](https://github.com/windmill-labs/windmill/commit/8d12bcc8ee2991909ea0d9bb57f04f0d4106c69f))
+
+## [1.493.2](https://github.com/windmill-labs/windmill/compare/v1.493.1...v1.493.2) (2025-05-28)
+
+
+### Bug Fixes
+
+* improve monaco editor memory leak ([e0f4f83](https://github.com/windmill-labs/windmill/commit/e0f4f83ebf4416c3bcc24433a7bf606349e1f75a))
+* improve monaco javascript extra lib refresh ([7b70348](https://github.com/windmill-labs/windmill/commit/7b70348b4bba3726e3fb26c964219a5a2aa6af55))
+
+## [1.493.1](https://github.com/windmill-labs/windmill/compare/v1.493.0...v1.493.1) (2025-05-28)
+
+
+### Bug Fixes
+
+* improve monaco javascript extra lib refresh ([a2c8ea6](https://github.com/windmill-labs/windmill/commit/a2c8ea69a3962a350273717cd237d8a96523fd00))
+
+## [1.493.0](https://github.com/windmill-labs/windmill/compare/v1.492.1...v1.493.0) (2025-05-27)
+
+
+### Features
+
+* add aws oidc support for instance s3 storage ([#5810](https://github.com/windmill-labs/windmill/issues/5810)) ([5b96bcc](https://github.com/windmill-labs/windmill/commit/5b96bccedd6e68fea631580dd49338301ad0305f))
+* duckdb sql lang support ([#5761](https://github.com/windmill-labs/windmill/issues/5761)) ([fdefd4b](https://github.com/windmill-labs/windmill/commit/fdefd4be9398b9610a539360353fd61b521732d4))
+* **python:** inline script metadata (PEP 723)  ([#5712](https://github.com/windmill-labs/windmill/issues/5712)) ([2622253](https://github.com/windmill-labs/windmill/commit/26222539e66bce7e88f86a7e5917e6ca99350865))
+
+
+### Bug Fixes
+
+* add missing http_trigger_version_seq grants ([#5816](https://github.com/windmill-labs/windmill/issues/5816)) ([306f3ea](https://github.com/windmill-labs/windmill/commit/306f3eabd1c03fa904b0e59438de124a0e680597))
+* avoid monaco memory leak ([0d459d5](https://github.com/windmill-labs/windmill/commit/0d459d5d223728270854e37715ecc1663ede9870))
+* error handler node rendering at top level ([feae9b0](https://github.com/windmill-labs/windmill/commit/feae9b09240ba306c007013a36d2aefb0b273766))
+* **frontend:** auto completion and render of tailwind classes in app editor ([#5817](https://github.com/windmill-labs/windmill/issues/5817)) ([5897e7e](https://github.com/windmill-labs/windmill/commit/5897e7e01b8839425c30c2a97481ef7bb9090661))
+
 ## [1.492.1](https://github.com/windmill-labs/windmill/compare/v1.492.0...v1.492.1) (2025-05-22)
 
 

CLAUDE.md

@@ -1,71 +1,3 @@
-# Windmill Overview
-
-Windmill is an open-source developer platform for building internal tools, API integrations, background jobs, workflows, and user interfaces. It offers a unified system where scripts are automatically turned into sharable UIs and can be composed into flows or embedded in custom applications.
-
-## Core Capabilities
-
-- **Script Development and Execution**: Write and run scripts in Python, TypeScript/JavaScript (Deno/Bun), Go, Bash, SQL, and other languages
-- **Workflow Orchestration**: Compose scripts into multi-step flows with conditional logic, loops, and error handling
-- **UI Generation**: Automatically generate UIs from scripts or build custom applications with a low-code editor
-- **Job Scheduling**: Trigger scripts and flows on schedules, webhooks, or external events
-- **Resource Management**: Securely store and use credentials, databases, and other connections
-
-## Platform Architecture
-
-The Windmill platform consists of several key components:
-
-- **Frontend UI**: Web-based interface for script and flow development, app building, and result visualization
-- **API Server**: Central API that handles authentication, resource management, and job coordination
-- **Workers**: Execute scripts in their respective environments with proper sandboxing
-- **Database**: PostgreSQL database for storage of scripts, flows, resources, job results, and more
-- **Job Queue**: Queue system for managing job execution, implemented in PostgreSQL
-- **Client Libraries**: Libraries for interacting with Windmill from Python, TypeScript, or command line
-
-# Windmill Backend Architecture
-
-The Windmill backend is written in Rust and consists of several services working together. These services are designed for horizontal scaling with stateless API servers and workers that can be deployed across multiple machines.
-
-## Key Components
-
-- **API Server (`windmill-api`)**: Handles HTTP requests, authentication, and resource management
-- **Queue Manager (`windmill-queue`)**: Manages the job queue in PostgreSQL
-- **Worker System (`windmill-worker`)**: Executes jobs in sandboxed environments
-- **Common Utilities (`windmill-common`)**: Shared code used by multiple services
-- **Git Sync (`windmill-git-sync`)**: Synchronizes scripts with Git repositories
-
-## Job Execution System
-
-The job execution process follows these steps:
-
-1. The API server receives a request to run a script or flow and creates a job record in the database
-2. The job is added to the queue system in PostgreSQL
-3. Workers continuously poll the queue for jobs matching their capabilities
-4. When a job is picked up, it's routed to the appropriate language executor
-5. The script is executed in a sandboxed environment using NSJAIL for security
-6. Results are processed and stored in the database
-7. For flows, each step creates a new job that goes through the same process
-
-Windmill supports worker tags and groups to route jobs to workers with specific capabilities or resource access.
-
-# Windmill Frontend Architecture
-
-The Windmill frontend is built with Svelte and provides several key interfaces for interacting with the platform.
-
-## Key Components
-
-- **Script Builder**: Code editor with language support, schema inference, and dependency management
-- **Flow Builder**: Visual editor for creating multi-step workflows with branching and looping
-- **App Editor**: Grid-based editor for building custom UIs that integrate scripts and flows
-- **Schema Form System**: Generates form interfaces from script parameters automatically
-- **Result Viewer**: Visualizes job results, logs, and execution status
-
-The frontend uses the Monaco editor (same as VS Code) for code editing, with specialized language support for all supported script languages.
-
-## UI Framework
-
-The frontend is built with Svelte, providing a reactive and component-based architecture. Key frontend technologies include:
-
-- **Svelte/SvelteKit**: Core framework for UI components and routing
-- **Monaco Editor**: Code editing experience similar to VS Code
-- **Schema Form**: Automatic UI generation from TypeScript/JSON schemas
-- **Tailwind CSS**: Utility-first CSS framework for styling
+To have an overview of what this app does, see @.cursor/rules/windmill-overview.mdc
+For backend modifications, follow the rules mentioned here @.cursor/rules/rust-best-practices.mdc
+For frontend modifications, follow the rules mentioned here @.cursor/rules/svelte5-best-practices.mdc

backend/.gitignore

@@ -5,4 +5,5 @@ oauth2.json
 tracing.folded
 heaptrack*
 index/
-windmill-api/openapi-*.*
+windmill-api/openapi-*.*
+.duckdb/*

backend/.sqlx/query-13444bbd5547e101c41206c5f97ac4dded0536faf52c370d704ed9a451041caf.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM sqs_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "13444bbd5547e101c41206c5f97ac4dded0536faf52c370d704ed9a451041caf"
+}

backend/.sqlx/query-2ef25599ea0c9ef946d6cc70ae048af970aed2638a3f767e152b654aebf68e48.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SHOW WAL_LEVEL;",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "wal_level",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "2ef25599ea0c9ef946d6cc70ae048af970aed2638a3f767e152b654aebf68e48"
-}

backend/.sqlx/query-4469ee6c206c46951980ea1bc73f126f339d2e3cf97f363be8921084b16dac45.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT pubname AS publication_name FROM pg_publication;",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "publication_name",
-        "type_info": "Name"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "4469ee6c206c46951980ea1bc73f126f339d2e3cf97f363be8921084b16dac45"
-}

backend/.sqlx/query-4ee0017771f46f0272817d18edb821940cb5064e3f155b9630b131c09c9dba13.json

@@ -1,26 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n        SELECT \n            slot_name,\n            active\n        FROM\n            pg_replication_slots \n        WHERE \n            plugin = 'pgoutput' AND\n            slot_type = 'logical';\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "slot_name",
-        "type_info": "Name"
-      },
-      {
-        "ordinal": 1,
-        "name": "active",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      true,
-      true
-    ]
-  },
-  "hash": "4ee0017771f46f0272817d18edb821940cb5064e3f155b9630b131c09c9dba13"
-}

backend/.sqlx/query-6a19c440a7a8064f3969cf6f48adea0bfdb683de9555e374ce5731e0b3c379f9.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM gcp_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "6a19c440a7a8064f3969cf6f48adea0bfdb683de9555e374ce5731e0b3c379f9"
+}

backend/.sqlx/query-6f56acb985aa7141ea1891d7ad58a32c35d1b02fe7070c92a2e62c1a5339c396.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT slot_name FROM pg_replication_slots where slot_name = $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "slot_name",
-        "type_info": "Name"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Name"
-      ]
-    },
-    "nullable": [
-      true
-    ]
-  },
-  "hash": "6f56acb985aa7141ea1891d7ad58a32c35d1b02fe7070c92a2e62c1a5339c396"
-}

backend/.sqlx/query-7e64ba7e2362cc19d2aed9f34c9879983922e96a9baab7c1a2b09ed2b1c261e2.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT \n            active_pid \n        FROM \n            pg_replication_slots \n        WHERE \n            slot_name = $1\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "active_pid",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Name"
-      ]
-    },
-    "nullable": [
-      true
-    ]
-  },
-  "hash": "7e64ba7e2362cc19d2aed9f34c9879983922e96a9baab7c1a2b09ed2b1c261e2"
-}

backend/.sqlx/query-86ae16175ace0179e784aacfd381771f0137ecab6671d632febadede729e7783.json

@@ -1,40 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n        SELECT\n            puballtables AS all_table,\n            pubinsert AS insert,\n            pubupdate AS update,\n            pubdelete AS delete\n        FROM\n            pg_publication\n        WHERE\n            pubname = $1\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "all_table",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 1,
-        "name": "insert",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 2,
-        "name": "update",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 3,
-        "name": "delete",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Name"
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      false
-    ]
-  },
-  "hash": "86ae16175ace0179e784aacfd381771f0137ecab6671d632febadede729e7783"
-}

backend/.sqlx/query-a8b470b463ca4b7c00c7ef6e9f36c23f8bbcefc288a56d61122bfd6fe5ca7e8d.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM mqtt_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a8b470b463ca4b7c00c7ef6e9f36c23f8bbcefc288a56d61122bfd6fe5ca7e8d"
+}

backend/.sqlx/query-baa1dddc616419bf4b923715f0a863bc0ff69c98db0f0c8f55e4ac89fdde7a60.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT pubname FROM pg_publication WHERE pubname = $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "pubname",
-        "type_info": "Name"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Name"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "baa1dddc616419bf4b923715f0a863bc0ff69c98db0f0c8f55e4ac89fdde7a60"
-}

backend/.sqlx/query-fd5754fe3c6346ae28818a9d60d144a40f8884f47e5bbdd2824e939dafd8f154.json

@@ -1,40 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            SELECT\n            schemaname AS schema_name,\n            tablename AS table_name,\n            CASE\n                WHEN array_length(attnames, 1) = (SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = pg_publication_tables.schemaname AND table_name = pg_publication_tables.tablename)\n                THEN NULL\n                ELSE attnames\n            END AS columns,\n            rowfilter AS where_clause\n            FROM\n                pg_publication_tables\n            WHERE\n                pubname = $1;\n            ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "schema_name",
-        "type_info": "Name"
-      },
-      {
-        "ordinal": 1,
-        "name": "table_name",
-        "type_info": "Name"
-      },
-      {
-        "ordinal": 2,
-        "name": "columns",
-        "type_info": "NameArray"
-      },
-      {
-        "ordinal": 3,
-        "name": "where_clause",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Name"
-      ]
-    },
-    "nullable": [
-      true,
-      true,
-      null,
-      true
-    ]
-  },
-  "hash": "fd5754fe3c6346ae28818a9d60d144a40f8884f47e5bbdd2824e939dafd8f154"
-}

backend/.vscode/settings.json

@@ -11,5 +11,6 @@
   "remote.autoForwardPorts": true,
   "conventionalCommits.scopes": [
     "restructring triggers, decoding trigger message on work"
-  ]
+  ],
+  "rust-analyzer.cargo.features": ["postgres_trigger"]
 }

backend/CLAUDE.md

@@ -1,104 +0,0 @@
-# Windmill Backend - Rust Best Practices
-
-## Project Structure
-
-Windmill uses a workspace-based architecture with multiple crates:
-
-- **windmill-api**: API server functionality
-- **windmill-worker**: Job execution
-- **windmill-common**: Shared code used by all crates
-- **windmill-queue**: Job & flow queuing
-- **windmill-audit**: Audit logging
-- Other specialized crates (git-sync, autoscaling, etc.)
-
-## Adding New Code
-
-### Module Organization
-
-- Place new code in the appropriate crate based on functionality
-- For API endpoints, create or modify files in `windmill-api/src/` organized by domain
-- For shared functionality, use `windmill-common/src/`
-- Use the `_ee.rs` suffix for enterprise-only modules
-- Follow existing patterns for file structure and organization
-
-### Error Handling
-
-- Use the custom `Error` enum from `windmill-common::error`
-- Return `Result<T, Error>` or `JsonResult<T>` for functions that can fail
-- Use the `?` operator for error propagation
-- Add location tracking to errors using `#[track_caller]`
-
-### Database Operations
-
-- Use `sqlx` for database operations with prepared statements
-- Leverage existing database helper functions in `db.rs` modules
-- Use transactions for multi-step operations
-- Handle database errors properly
-
-### API Endpoints
-
-- Follow existing patterns in the `windmill-api` crate
-- Use axum's routing system and extractors
-- Group related routes together
-- Use consistent response formats (JSON)
-- Follow proper authentication and authorization patterns
-
-## Performance Optimizations
-
-When generating code, especially involving `serde`, `sqlx`, and `tokio`, prioritize performance by applying the following principles:
-
-### Serde Optimizations (Serialization & Deserialization)
-
-- **Specify Structure Explicitly:** When defining structs for Serde (`#[derive(Serialize, Deserialize)]`), use `#[serde(...` attributes extensively. This includes:
-  - `#[serde(rename = "...")]` or `#[serde(alias = "...")]` to map external names precisely, avoiding dynamic lookups.
-  - `#[serde(default)]` for optional fields with default values, reducing parsing complexity.
-  - `#[serde(skip_serializing_if = "...")]` to avoid writing fields that meet a certain condition (e.g., `Option::is_none()`, `Vec::is_empty()`, or a custom function), reducing output size and serialization work.
-  - `#[serde(skip_serializing)]` or `#[serde(skip_deserializing)]` for fields that should _not_ be included.
-- **Prefer Borrowing:** Where possible and safe (data lifetime allows), use `Cow<'a, str>` or `&'a str` (with `#[serde(borrow)]`) instead of `String` for string fields during deserialization. This avoids allocating new strings, enabling zero-copy reading from the input buffer. Apply this principle to byte slices (`&'a [u8]` / `Cow<'a, [u8]>`) and potentially borrowed vectors as well.
-- **Avoid Intermediate `Value`:** Unless the data structure is truly dynamic or unknown at compile time, deserialize directly into a well-defined struct or enum rather than into `serde_json::Value` (or equivalent for other formats). This avoids unnecessary heap allocations and type switching.
-
-### SQLx Optimizations (Database Interaction)
-
-- **Select Only Necessary Columns:** In `SELECT` queries, list specific column names rather than using `SELECT *`. This reduces data transferred from the database and the work needed for hydration/deserialization.
-- **Batch Operations:** For multiple `INSERT`, `UPDATE`, or `DELETE` statements, prefer executing them in a single query if the database and driver support it efficiently (e.g., `INSERT INTO ... VALUES (...), (...), ...`). This minimizes round trips to the database.
-- **Avoid N+1 Queries:** Do not loop through results of one query and execute a separate query for each item (e.g., fetching users, then querying for each user's profile in a loop). Instead, use JOINs or a single query with an `IN` clause to fetch related data efficiently.
-- **Deserialize Directly:** Use `#[derive(FromRow)]` on structs and ensure the struct fields match the selected columns in the query. This allows SQLx to hydrate objects directly, avoiding intermediate data structures.
-- **Parameterize Queries:** Always use SQLx's query methods (`.bind(...)`) to pass values as parameters rather than string formatting. This prevents SQL injection and allows the database to cache query plans, improving performance on repeated executions.
-
-### Tokio Optimizations (Asynchronous Runtime)
-
-- **Avoid Blocking Operations:** **Crucially**, never perform blocking operations (synchronous file I/O, `std::thread::sleep`, CPU-bound loops, `std::sync::Mutex::lock`, blocking network calls without `tokio::net`) directly within an `async fn` or a standard `tokio::spawn` task. Blocking pauses the entire worker thread, potentially starving other tasks. Use `tokio::task::spawn_blocking` for CPU-intensive work or blocking I/O.
-- **Use Tokio's Async Primitives:** Prefer `tokio::sync` (channels, mutexes, semaphores), `tokio::io`, `tokio::net`, and `tokio::time` over their `std` counterparts in asynchronous contexts. These are designed to yield control back to the scheduler.
-- **Manage Concurrency:** Be mindful of how many tasks are spawned. Creating a new task for every tiny piece of work can introduce overhead. Group related asynchronous operations where appropriate.
-- **Handle Shared State Efficiently:** Use `Arc` for shared ownership in concurrent tasks. When shared state needs mutation, prefer `tokio::sync::Mutex` over `std::sync::Mutex` in `async` code. Consider `tokio::sync::RwLock` if reads significantly outnumber writes. Minimize the duration for which locks are held.
-- **Understand `.await`:** Place `.await` strategically to allow the runtime to switch to other ready tasks. Ensure that `.await` points to genuinely asynchronous operations.
-- **Backpressure:** If dealing with data streams or queues between tasks, implement backpressure mechanisms (e.g., bounded channels like `tokio::sync::mpsc::channel`) to prevent one component from overwhelming another or critical resources like the database.
-
-## Enterprise Features
-
-- Use feature flags for enterprise functionality
-- Conditionally compile with `#[cfg(feature = "enterprise")]`
-- Isolate enterprise code in separate modules
-
-## Code Style
-
-- Group imports by external and internal crates
-- Place struct/enum definitions before implementations
-- Group similar functionality together
-- Use descriptive naming consistent with the codebase
-- Follow existing patterns for async code using tokio
-
-## Testing
-
-- Write unit tests for core functionality
-- Use the `#[cfg(test)]` module for test code
-- For database tests, use the existing test utilities
-
-## Common Crates Used
-
-- **tokio**: For async runtime
-- **axum**: For web server and routing
-- **sqlx**: For database operations
-- **serde**: For serialization/deserialization
-- **tracing**: For logging and diagnostics
-- **reqwest**: For HTTP client functionality

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.492.1"
+version = "1.494.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -32,7 +32,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.492.1"
+version = "1.494.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -87,13 +87,14 @@ python = ["windmill-worker/python", "windmill-api/python"]
 rust = ["windmill-worker/rust"]
 mysql = ["windmill-worker/mysql"]
 oracledb = ["windmill-worker/oracledb"]
+duckdb = ["windmill-worker/duckdb"]
 mssql = ["windmill-worker/mssql"]
 bigquery = ["windmill-worker/bigquery"]
 php = ["windmill-worker/php"]
 csharp = ["windmill-worker/csharp"]
 nu = ["windmill-worker/nu"]
 java = ["windmill-worker/java"]
-all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "mssql", "bigquery", "csharp", "nu", "php", "java"]
+all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"]
 
 
 [patch.crates-io]
@@ -238,6 +239,7 @@ json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
 semver = "^1"
+duckdb = { version = "1.2.2", features = ["bundled"] }
 
 v8 = "=130.0.7"  # Exact version NOTE: Do not forget to update version and hash in flake.nix
 deno_fetch = "0.214.0"

backend/ee-repo-ref.txt

@@ -1 +1 @@
-6899b8151329218a1df59964dac57e0e004ae25a
+8a2506e86b923c00522cb83b052586f705f7aa8e

backend/migrations/20250515084520_duckdb_support.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20250515084520_duckdb_support.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'duckdb';
+UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["duckdb"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu", "java"]}'::jsonb AND NOT config->'worker_tags' @> '"duckdb"'::jsonb;

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -83,6 +83,21 @@ pub fn parse_bigquery_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     }
 }
 
+pub fn parse_duckdb_sig(code: &str) -> anyhow::Result<MainArgSignature> {
+    let parsed = parse_duckdb_file(&code)?;
+    if let Some(args) = parsed {
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
+    } else {
+        Err(anyhow!("Error parsing sql".to_string()))
+    }
+}
+
 pub fn parse_snowflake_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_snowflake_file(&code)?;
     if let Some(x) = parsed {
@@ -212,6 +227,9 @@ lazy_static::lazy_static! {
     // -- @name (type) = default
     static ref RE_ARG_BIGQUERY: Regex = Regex::new(r#"(?m)^-- @(\w+) \((\w+(?:\[\])?)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
 
+    // -- $name (type) = default
+    static ref RE_ARG_DUCKDB: Regex = Regex::new(r#"(?m)^-- \$(\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
+
     static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
 
 
@@ -577,6 +595,35 @@ fn parse_bigquery_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
     Ok(Some(args))
 }
 
+fn parse_duckdb_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
+    let mut args: Vec<Arg> = vec![];
+
+    for cap in RE_ARG_DUCKDB.captures_iter(code) {
+        let name = cap.get(1).map(|x| x.as_str().to_string()).unwrap();
+        let typ = cap
+            .get(2)
+            .map(|x| x.as_str().to_string().to_lowercase())
+            .unwrap();
+        let default = cap.get(3).map(|x| x.as_str().to_string());
+        let has_default = default.is_some();
+        let parsed_typ = parse_duckdb_typ(typ.as_str());
+
+        let parsed_default = default.and_then(|x| parsed_default(&parsed_typ, x));
+
+        args.push(Arg {
+            name,
+            typ: parsed_typ,
+            default: parsed_default,
+            otyp: Some(typ),
+            has_default,
+            oidx: None,
+        });
+    }
+
+    args.append(&mut parse_sql_sanitized_interpolation(code));
+    Ok(Some(args))
+}
+
 fn parse_snowflake_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
     let mut args: Vec<Arg> = vec![];
 
@@ -729,6 +776,33 @@ pub fn parse_bigquery_typ(typ: &str) -> Typ {
     }
 }
 
+pub fn parse_duckdb_typ(typ: &str) -> Typ {
+    if typ.ends_with("[]") {
+        let base_typ = parse_duckdb_typ(typ.strip_suffix("[]").unwrap());
+        Typ::List(Box::new(base_typ))
+    } else {
+        match typ {
+            "varchar" | "char" | "bpchar" | "text" | "string" => Typ::Str(None),
+            "blob" | "bytea" | "binary" | "varbinary" | "bitstring" => Typ::Bytes,
+            "boolean" | "bool" | "bit" | "logical" => Typ::Bool,
+            "bigint" | "int8" | "long" | "integer" | "int4" | "int" | "smallint" | "int2"
+            | "short" | "tinyint" | "int1" | "signed" | "ubigint" | "uhugeint" | "uinteger"
+            | "usmallint" | "utinyint" => Typ::Int,
+            "decimal" | "numeric" | "double" | "float8" | "float" | "float4" | "real" => Typ::Float,
+            "date"
+            | "time"
+            | "timestamp with time zone"
+            | "timestamptz"
+            | "timestamp"
+            | "datetime" => Typ::Datetime,
+            "uuid" | "json" => Typ::Str(None),
+            "interval" | "hugeint" => Typ::Str(None),
+            "s3object" => Typ::Resource("S3Object".to_string()),
+            _ => Typ::Str(None),
+        }
+    }
+}
+
 pub fn parse_snowflake_typ(typ: &str) -> Typ {
     match typ {
         "varchar" => Typ::Str(None),

backend/parsers/windmill-parser-wasm/src/lib.rs

@@ -96,6 +96,12 @@ pub fn parse_oracledb(code: &str) -> String {
     wrap_sig(windmill_parser_sql::parse_oracledb_sig(code))
 }
 
+#[cfg(feature = "sql-parser")]
+#[wasm_bindgen]
+pub fn parse_duckdb(code: &str) -> String {
+    wrap_sig(windmill_parser_sql::parse_duckdb_sig(code))
+}
+
 #[cfg(feature = "sql-parser")]
 #[wasm_bindgen]
 pub fn parse_bigquery(code: &str) -> String {

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.492.1
+  version: 1.494.0
   title: Windmill API
 
   contact:
@@ -377,6 +377,9 @@ paths:
                   type: string
                 company:
                   type: string
+                skip_email:
+                  type: boolean
+                  description: Skip sending email notifications to the user
               required:
                 - email
                 - password
@@ -4893,8 +4896,6 @@ paths:
           description: Script version/hash
           content:
             application/json:
-              required: false
-
               schema:
                 $ref: "#/components/schemas/ScriptHistory"
 
@@ -5456,8 +5457,6 @@ paths:
           description: Flow version
           content:
             application/json:
-              required: false
-
               schema:
                 $ref: "#/components/schemas/FlowVersion"
 
@@ -5487,8 +5486,7 @@ paths:
       operationId: getFlowVersion
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
-        - type: string
-          name: version
+        - name: version
           in: path
           required: true
           schema:
@@ -5510,8 +5508,7 @@ paths:
       operationId: updateFlowHistory
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
-        - type: string
-          name: version
+        - name: version
           in: path
           required: true
           schema:
@@ -6139,10 +6136,8 @@ paths:
           description: App version
           content:
             application/json:
-              required: false
               schema:
                 $ref: "#/components/schemas/AppHistory"
-
   /w/{workspace}/apps/list_paths_from_workspace_runnable/{runnable_kind}/{path}:
     get:
       summary: list app paths from workspace runnable
@@ -6731,11 +6726,6 @@ paths:
       responses:
         "201":
           description: stream of created job uuids separated by \n. Lines may start with 'Error:'
-          example: |
-            a1a74c0d-708e-4539-9768-e8b3d37996bd
-            f0949132-5b30-48fe-bac8-873f047df810
-            Error: Could not re-run 0b885808-ae89-4458-af95-c1ca3a13b0a5
-            52b9c01d-1125-4bbb-8bee-d41f26b70066
           content:
             text/event-stream:
               schema:
@@ -7592,7 +7582,8 @@ paths:
           description: job log
           content:
             text/plain:
-              type: string
+              schema:
+                type: string
 
   /w/{workspace}/jobs_u/get_flow_debug_info/{id}:
     get:
@@ -9830,6 +9821,23 @@ paths:
                 items:
                   type: string
 
+  /w/{workspace}/postgres_triggers/postgres/version/{path}:
+    get:
+      summary: get postgres version
+      operationId: getPostgresVersion
+      tags:
+        - postgres_trigger
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: postgres version
+          content:
+            application/json:
+              schema:
+                type: string
+  
   /w/{workspace}/postgres_triggers/is_valid_postgres_configuration/{path}:
     get:
       summary: check if postgres configuration is set to logical
@@ -12469,7 +12477,6 @@ paths:
             text/plain:
               schema:
                 type: string
-
   /concurrency_groups/list:
     get:
       summary: List all concurrency groups
@@ -13175,7 +13182,47 @@ components:
         enum: [script, flow]
 
   schemas:
-    $ref: "../../openflow.openapi.yaml#/components/schemas"
+    # NOTE: Not so many generators and validators support this format:
+    # $ref: "../../openflow.openapi.yaml#/components/schemas"
+    # This is why it is better to inline each of schemas for better compat
+    OpenFlow:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow"
+    FlowValue:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/FlowValue"
+    Retry:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/Retry"
+    StopAfterIf:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/StopAfterIf"
+    FlowModule:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/FlowModule"
+    InputTransform:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/InputTransform"
+    StaticTransform:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/StaticTransform"
+    JavascriptTransform:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/JavascriptTransform"
+    FlowModuleValue:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/FlowModuleValue"
+    RawScript:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/RawScript"
+    PathScript:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/PathScript"
+    PathFlow:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/PathFlow"
+    ForloopFlow:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/ForloopFlow"
+    WhileloopFlow:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/WhileloopFlow"
+    BranchOne:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/BranchOne"
+    BranchAll:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/BranchAll"
+    Identity:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/Identity"
+    FlowStatus:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatus"
+    FlowStatusModule:
+      $ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatusModule"
 
     AIProvider:
       type: string
@@ -14242,7 +14289,8 @@ components:
           ansible,
           csharp,
           nu,
-          java
+          java,
+          duckdb
           # for related places search: ADD_NEW_LANG
         ]
 
@@ -16224,10 +16272,8 @@ components:
         - access_token
 
     HubScriptKind:
-      name: kind
-      schema:
-        type: string
-        enum: [script, failure, trigger, approval]
+      type: string
+      enum: [script, failure, trigger, approval]
 
     PolarsClientKwargs:
       type: object

backend/windmill-api/src/capture.rs

@@ -23,7 +23,10 @@ use {
     http::HeaderMap,
 };
 
-#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
+#[cfg(any(
+    all(feature = "enterprise", feature = "gcp_trigger"),
+    feature = "postgres_trigger"
+))]
 use windmill_common::utils::empty_as_none;
 
 #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
@@ -33,7 +36,14 @@ use windmill_common::auth::aws::AwsAuthResourceType;
     feature = "http_trigger",
     all(feature = "enterprise", feature = "gcp_trigger")
 ))]
-use {serde::de::DeserializeOwned, windmill_common::error::Error};
+use serde::de::DeserializeOwned;
+
+#[cfg(any(
+    feature = "http_trigger",
+    feature = "postgres_trigger",
+    all(feature = "enterprise", feature = "gcp_trigger")
+))]
+use windmill_common::error::Error;
 
 #[cfg(all(feature = "enterprise", feature = "kafka"))]
 use crate::kafka_triggers_ee::KafkaTriggerConfigConnection;
@@ -47,11 +57,10 @@ use crate::nats_triggers_ee::NatsTriggerConfigConnection;
 #[cfg(feature = "postgres_trigger")]
 use {
     crate::postgres_triggers::{
-        create_logical_replication_slot_query, create_publication_query, drop_publication_query,
-        generate_random_string, get_database_connection, PublicationData,
+        create_logical_replication_slot, create_pg_publication, generate_random_string,
+        get_pg_connection, PublicationData,
     },
-    itertools::Itertools,
-    pg_escape::quote_literal,
+    sqlx::Connection,
 };
 
 use crate::{
@@ -200,9 +209,12 @@ pub struct MqttTriggerConfig {
 #[derive(Serialize, Deserialize, Debug)]
 pub struct PostgresTriggerConfig {
     pub postgres_resource_path: String,
+    #[serde(default, deserialize_with = "empty_as_none")]
     pub publication_name: Option<String>,
+    #[serde(default, deserialize_with = "empty_as_none")]
     pub replication_slot_name: Option<String>,
     pub publication: PublicationData,
+    pub basic_mode: Option<bool>,
 }
 
 #[cfg(feature = "websocket")]
@@ -292,57 +304,49 @@ async fn set_postgres_trigger_config(
     user_db: UserDB,
     mut capture_config: NewCaptureConfig,
 ) -> Result<NewCaptureConfig> {
-    let Some(TriggerConfig::Postgres(mut postgres_config)) = capture_config.trigger_config else {
-        return Err(windmill_common::error::Error::BadRequest(
-            "Invalid postgres config".to_string(),
-        ));
+    let Some(TriggerConfig::Postgres(postgres_config)) = capture_config.trigger_config.as_mut()
+    else {
+        return Err(Error::BadRequest("Invalid postgres config".to_string()));
     };
 
-    let mut connection = get_database_connection(
-        authed,
-        Some(user_db),
-        &db,
-        &postgres_config.postgres_resource_path,
-        &w_id,
-    )
-    .await?;
+    if postgres_config.basic_mode.unwrap_or(false) {
+        let mut pg_connection = get_pg_connection(
+            authed,
+            Some(user_db),
+            &db,
+            &postgres_config.postgres_resource_path,
+            &w_id,
+        )
+        .await?;
 
-    let publication_name = postgres_config
-        .publication_name
-        .get_or_insert(format!("windmill_capture_{}", generate_random_string()));
-    let replication_slot_name = postgres_config
-        .replication_slot_name
-        .get_or_insert(publication_name.clone());
+        let mut tx = pg_connection.begin().await?;
 
-    let query = drop_publication_query(&publication_name);
+        let publication_name = format!("windmill_capture_{}", generate_random_string());
+        let replication_slot_name = publication_name.clone();
 
-    sqlx::query(&query).execute(&mut connection).await?;
+        create_logical_replication_slot(&mut tx, &replication_slot_name).await?;
 
-    let query = create_publication_query(
-        &publication_name,
-        postgres_config.publication.table_to_track.as_deref(),
-        &postgres_config
-            .publication
-            .transaction_to_track
-            .iter()
-            .map(AsRef::as_ref)
-            .collect_vec(),
-    );
+        create_pg_publication(
+            &mut tx,
+            &publication_name,
+            postgres_config.publication.table_to_track.as_deref(),
+            &postgres_config.publication.transaction_to_track,
+        )
+        .await?;
 
-    sqlx::query(&query).execute(&mut connection).await?;
-
-    let query = format!(
-        "SELECT 1 from pg_replication_slots WHERE slot_name = {}",
-        quote_literal(replication_slot_name)
-    );
-
-    let row = sqlx::query(&query).fetch_optional(&mut connection).await?;
-
-    if row.is_none() {
-        let query = create_logical_replication_slot_query(&replication_slot_name);
-        sqlx::query(&query).execute(&mut connection).await?;
+        tx.commit().await?;
+        postgres_config.publication_name = Some(publication_name);
+        postgres_config.replication_slot_name = Some(replication_slot_name);
+    } else {
+        if postgres_config.publication_name.is_none()
+            || postgres_config.replication_slot_name.is_none()
+        {
+            return Err(Error::BadRequest(
+                "Publication name and slot name required in advanced mode".to_string(),
+            ));
+        }
     }
-    capture_config.trigger_config = Some(TriggerConfig::Postgres(postgres_config));
+
     Ok(capture_config)
 }
 
@@ -366,9 +370,7 @@ async fn set_gcp_trigger_config(
     mut capture_config: NewCaptureConfig,
 ) -> Result<NewCaptureConfig> {
     let Some(TriggerConfig::Gcp(mut gcp_config)) = capture_config.trigger_config else {
-        return Err(windmill_common::error::Error::BadRequest(
-            "Invalid GCP Pub/Sub config".to_string(),
-        ));
+        return Err(Error::BadRequest("Invalid GCP Pub/Sub config".to_string()));
     };
 
     let config = manage_google_subscription(

backend/windmill-api/src/postgres_triggers/handler.rs

@@ -6,7 +6,6 @@ use std::collections::{
 use crate::{
     db::{ApiAuthed, DB},
     postgres_triggers::mapper::{Mapper, MappingInfo},
-    resources::try_get_resource_from_db_as,
 };
 use axum::{
     extract::{Path, Query},
@@ -19,21 +18,20 @@ use quick_cache::sync::Cache;
 use rust_postgres::types::Type;
 use serde::{Deserialize, Deserializer, Serialize};
 use sql_builder::{bind::Bind, SqlBuilder};
-use sqlx::{postgres::types::Oid, FromRow, PgConnection};
+use sqlx::{postgres::types::Oid, Connection, FromRow, PgConnection};
 use windmill_audit::{audit_ee::audit_log, ActionKind};
-use windmill_common::error::Error;
 use windmill_common::{
     db::UserDB,
-    error::{self, JsonResult, Result},
-    utils::{not_found_if_none, paginate, Pagination, StripPath, empty_as_none},
+    error::{self, Error, JsonResult, Result},
+    utils::{empty_as_none, not_found_if_none, paginate, Pagination, StripPath},
     worker::CLOUD_HOSTED,
 };
 use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
 
 use super::{
-    create_logical_replication_slot_query, create_publication_query, drop_publication_query,
-    generate_random_string, get_database_connection, get_raw_postgres_connection,
-    ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS,
+    check_if_valid_publication_for_postgres_version, create_logical_replication_slot,
+    create_pg_publication, drop_publication, generate_random_string, get_pg_connection,
+    ERROR_PUBLICATION_NAME_NOT_EXISTS,
 };
 use lazy_static::lazy_static;
 
@@ -53,15 +51,17 @@ pub struct Postgres {
 #[derive(Debug, Clone, FromRow, Serialize, Deserialize)]
 pub struct TableToTrack {
     pub table_name: String,
+    #[serde(default, deserialize_with = "empty_as_none")]
     pub where_clause: Option<String>,
-    pub columns_name: Vec<String>,
+    #[serde(default, deserialize_with = "empty_as_none")]
+    pub columns_name: Option<Vec<String>>,
 }
 
 impl TableToTrack {
     fn new(
         table_name: String,
         where_clause: Option<String>,
-        columns_name: Vec<String>,
+        columns_name: Option<Vec<String>>,
     ) -> TableToTrack {
         TableToTrack { table_name, where_clause, columns_name }
     }
@@ -99,7 +99,6 @@ pub struct EditPostgresTrigger {
 }
 
 #[derive(Deserialize, Serialize, Debug)]
-
 pub struct NewPostgresTrigger {
     path: String,
     script_path: String,
@@ -124,7 +123,7 @@ pub async fn test_postgres_connection(
     Json(test_postgres): Json<TestPostgres>,
 ) -> Result<()> {
     let connect_f = async {
-        get_database_connection(
+        get_pg_connection(
             authed,
             Some(user_db),
             &db,
@@ -189,7 +188,7 @@ where
                         ));
                     }
 
-                    if !track_specific_columns_in_table && !table_to_track.columns_name.is_empty() {
+                    if !track_specific_columns_in_table && table_to_track.columns_name.is_some() {
                         track_specific_columns_in_table = true;
                     }
                 }
@@ -267,39 +266,16 @@ impl PostgresPublicationReplication {
     }
 }
 
-async fn check_if_publication_exist(
-    connection: &mut PgConnection,
-    publication_name: &str,
-) -> Result<()> {
-    sqlx::query!(
-        "SELECT pubname FROM pg_publication WHERE pubname = $1",
-        publication_name
-    )
-    .fetch_one(connection)
-    .await
-    .map_err(|err| match err {
-        sqlx::Error::RowNotFound => {
-            Error::BadRequest(ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string())
-        }
-        err => Error::SqlErr { error: err, location: "pg_trigger".to_string() },
-    })?;
-    Ok(())
-}
-
 async fn check_if_logical_replication_slot_exist(
-    connection: &mut PgConnection,
+    pg_connection: &mut PgConnection,
     replication_slot_name: &str,
-) -> Result<()> {
-    sqlx::query!(
-        "SELECT slot_name FROM pg_replication_slots where slot_name = $1",
-        &replication_slot_name
-    )
-    .fetch_one(connection)
-    .await
-    .map_err(|err| match err {
-        _ => Error::BadRequest(ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string()),
-    })?;
-    Ok(())
+) -> Result<bool> {
+    let exists = sqlx::query("SELECT slot_name FROM pg_replication_slots where slot_name = $1")
+        .bind(&replication_slot_name)
+        .fetch_optional(pg_connection)
+        .await?
+        .is_some();
+    Ok(exists)
 }
 
 async fn create_custom_slot_and_publication_inner(
@@ -310,33 +286,30 @@ async fn create_custom_slot_and_publication_inner(
     w_id: &str,
     publication: &PublicationData,
 ) -> Result<PostgresPublicationReplication> {
-    let publication_name = format!("windmill_trigger_{}", generate_random_string());
-    let replication_slot_name = publication_name.clone();
-
-    let query = create_publication_query(
-        &publication_name,
-        publication.table_to_track.as_deref(),
-        &publication
-            .transaction_to_track
-            .iter()
-            .map(AsRef::as_ref)
-            .collect_vec(),
-    );
-
-    let mut connection = get_database_connection(
+    let mut pg_connection = get_pg_connection(
         authed.clone(),
-        Some(user_db.clone()),
+        Some(user_db),
         &db,
         &postgres_resource_path,
         &w_id,
     )
     .await?;
 
-    sqlx::query(&query).execute(&mut connection).await?;
+    let mut tx = pg_connection.begin().await?;
+    let publication_name = format!("windmill_trigger_{}", generate_random_string());
+    let replication_slot_name = publication_name.clone();
+    
+    create_logical_replication_slot(&mut tx, &replication_slot_name).await?;
+    
+    create_pg_publication(
+        &mut tx,
+        &publication_name,
+        publication.table_to_track.as_deref(),
+        &publication.transaction_to_track,
+    )
+    .await?;
 
-    let query = create_logical_replication_slot_query(&replication_slot_name);
-
-    sqlx::query(&query).execute(&mut connection).await?;
+    tx.commit().await?;
 
     Ok(PostgresPublicationReplication::new(
         publication_name,
@@ -344,6 +317,38 @@ async fn create_custom_slot_and_publication_inner(
     ))
 }
 
+pub async fn get_postgres_version_internal(pg_connection: &mut PgConnection) -> Result<String> {
+    let postgres_version: String = sqlx::query_scalar("SHOW server_version;")
+        .fetch_one(&mut *pg_connection)
+        .await
+        .map_err(|e| Error::Anyhow {
+            error: anyhow::anyhow!("Failed to retrieve PostgreSQL version: {}", e),
+            location: "postgres_triggers/handler.rs@379".to_string(),
+        })?;
+
+    Ok(postgres_version)
+}
+
+pub async fn get_postgres_version(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, postgres_resource_path)): Path<(String, String)>,
+) -> Result<String> {
+    let mut pg_connection = get_pg_connection(
+        authed.clone(),
+        Some(user_db),
+        &db,
+        &postgres_resource_path,
+        &w_id,
+    )
+    .await?;
+
+    let postgres_version = get_postgres_version_internal(&mut pg_connection).await?;
+
+    Ok(postgres_version)
+}
+
 pub async fn create_postgres_trigger(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -377,6 +382,7 @@ pub async fn create_postgres_trigger(
         if publication.is_none() {
             return Err(Error::BadRequest("publication must be set".to_string()));
         }
+
         let PostgresPublicationReplication { publication_name, replication_slot_name } =
             create_custom_slot_and_publication_inner(
                 authed.clone(),
@@ -577,7 +583,7 @@ impl PublicationData {
     }
 }
 
-#[derive(Debug, Serialize)]
+#[derive(FromRow, Debug, Serialize)]
 pub struct SlotList {
     slot_name: Option<String>,
     active: Option<bool>,
@@ -589,7 +595,7 @@ pub async fn list_slot_name(
     Extension(db): Extension<DB>,
     Path((w_id, postgres_resource_path)): Path<(String, String)>,
 ) -> Result<Json<Vec<SlotList>>> {
-    let mut connection = get_database_connection(
+    let mut pg_connection = get_pg_connection(
         authed.clone(),
         Some(user_db.clone()),
         &db,
@@ -598,8 +604,7 @@ pub async fn list_slot_name(
     )
     .await?;
 
-    let slots = sqlx::query_as!(
-        SlotList,
+    let slots: Vec<SlotList> = sqlx::query_as(
         r#"
         SELECT 
             slot_name,
@@ -609,9 +614,9 @@ pub async fn list_slot_name(
         WHERE 
             plugin = 'pgoutput' AND
             slot_type = 'logical';
-        "#
+        "#,
     )
-    .fetch_all(&mut connection)
+    .fetch_all(&mut pg_connection)
     .await?;
 
     Ok(Json(slots))
@@ -629,7 +634,7 @@ pub async fn create_slot(
     Path((w_id, postgres_resource_path)): Path<(String, String)>,
     Json(Slot { name }): Json<Slot>,
 ) -> Result<String> {
-    let mut connection = get_database_connection(
+    let mut pg_connection = get_pg_connection(
         authed.clone(),
         Some(user_db.clone()),
         &db,
@@ -638,13 +643,42 @@ pub async fn create_slot(
     )
     .await?;
 
-    let query = create_logical_replication_slot_query(&name);
-
-    sqlx::query(&query).execute(&mut connection).await?;
+    create_logical_replication_slot(&mut pg_connection, &name).await?;
 
     Ok(format!("Replication slot {} created!", name))
 }
 
+pub async fn drop_logical_replication_slot(
+    pg_connection: &mut PgConnection,
+    slot_name: &str,
+) -> Result<()> {
+    let active_pid: Option<i32> = sqlx::query_scalar(
+        r#"SELECT 
+            active_pid 
+        FROM 
+            pg_replication_slots 
+        WHERE 
+            slot_name = $1
+        "#,
+    )
+    .bind(&slot_name)
+    .fetch_optional(&mut *pg_connection)
+    .await?
+    .flatten();
+
+    if let Some(pid) = active_pid {
+        sqlx::query("SELECT pg_terminate_backend($1)")
+            .bind(pid)
+            .execute(&mut *pg_connection)
+            .await?;
+    }
+    sqlx::query("SELECT pg_drop_replication_slot($1)")
+        .bind(&slot_name)
+        .execute(pg_connection)
+        .await?;
+    Ok(())
+}
+
 pub async fn drop_slot_name(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -652,45 +686,14 @@ pub async fn drop_slot_name(
     Path((w_id, postgres_resource_path)): Path<(String, String)>,
     Json(Slot { name }): Json<Slot>,
 ) -> Result<String> {
-    let database = try_get_resource_from_db_as::<Postgres>(
-        authed,
-        Some(user_db),
-        &db,
-        &postgres_resource_path,
-        &w_id,
-    )
-    .await?;
+    let mut pg_connection =
+        get_pg_connection(authed, Some(user_db), &db, &postgres_resource_path, &w_id).await?;
 
-    let mut connection = get_raw_postgres_connection(&database).await?;
-
-    let active_pid = sqlx::query_scalar!(
-        r#"SELECT 
-            active_pid 
-        FROM 
-            pg_replication_slots 
-        WHERE 
-            slot_name = $1
-        "#,
-        &name
-    )
-    .fetch_optional(&mut connection)
-    .await?
-    .flatten();
-
-    if let Some(pid) = active_pid {
-        sqlx::query("SELECT pg_terminate_backend($1)")
-            .bind(pid)
-            .execute(&mut connection)
-            .await?;
-    }
-    sqlx::query("SELECT pg_drop_replication_slot($1)")
-        .bind(&name)
-        .execute(&mut connection)
-        .await?;
+    drop_logical_replication_slot(&mut pg_connection, &name).await?;
 
     Ok(format!("Replication slot {} deleted!", name))
 }
-#[derive(Debug, Serialize)]
+#[derive(FromRow, Debug, Serialize)]
 struct PublicationName {
     publication_name: String,
 }
@@ -701,7 +704,7 @@ pub async fn list_database_publication(
     Extension(db): Extension<DB>,
     Path((w_id, postgres_resource_path)): Path<(String, String)>,
 ) -> Result<Json<Vec<String>>> {
-    let mut connection = get_database_connection(
+    let mut pg_connection = get_pg_connection(
         authed.clone(),
         Some(user_db.clone()),
         &db,
@@ -710,12 +713,10 @@ pub async fn list_database_publication(
     )
     .await?;
 
-    let publication_names = sqlx::query_as!(
-        PublicationName,
-        "SELECT pubname AS publication_name FROM pg_publication;"
-    )
-    .fetch_all(&mut connection)
-    .await?;
+    let publication_names: Vec<PublicationName> =
+        sqlx::query_as("SELECT pubname AS publication_name FROM pg_publication;")
+            .fetch_all(&mut pg_connection)
+            .await?;
 
     let publications = publication_names
         .iter()
@@ -731,7 +732,7 @@ pub async fn get_publication_info(
     Extension(db): Extension<DB>,
     Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>,
 ) -> Result<Json<PublicationData>> {
-    let mut connection = get_database_connection(
+    let mut pg_connection = get_pg_connection(
         authed.clone(),
         Some(user_db.clone()),
         &db,
@@ -741,11 +742,11 @@ pub async fn get_publication_info(
     .await?;
 
     let publication_data =
-        get_publication_scope_and_transaction(&mut connection, &publication_name).await;
+        get_publication_scope_and_transaction(&mut pg_connection, &publication_name).await;
 
     let (all_table, transaction_to_track) = match publication_data {
-        Ok(pub_data) => pub_data,
-        Err(Error::SqlErr { error: sqlx::Error::RowNotFound, .. }) => {
+        Ok(Some(pub_data)) => pub_data,
+        Ok(None) => {
             return Err(Error::NotFound(
                 ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(),
             ))
@@ -754,7 +755,7 @@ pub async fn get_publication_info(
     };
 
     let table_to_track = if !all_table {
-        Some(get_tracked_relations(&mut connection, &publication_name).await?)
+        Some(get_tracked_relations(&mut pg_connection, &publication_name).await?)
     } else {
         None
     };
@@ -771,9 +772,7 @@ pub async fn create_publication(
     Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>,
     Json(publication_data): Json<PublicationData>,
 ) -> Result<String> {
-    let PublicationData { table_to_track, transaction_to_track } = publication_data;
-
-    let mut connection = get_database_connection(
+    let mut pg_connection = get_pg_connection(
         authed.clone(),
         Some(user_db.clone()),
         &db,
@@ -782,13 +781,19 @@ pub async fn create_publication(
     )
     .await?;
 
-    let query = create_publication_query(
+    let PublicationData { table_to_track, transaction_to_track } = publication_data;
+
+    let mut tx = pg_connection.begin().await?;
+
+    create_pg_publication(
+        &mut tx,
         &publication_name,
         table_to_track.as_deref(),
-        &transaction_to_track.iter().map(AsRef::as_ref).collect_vec(),
-    );
+        &transaction_to_track,
+    )
+    .await?;
 
-    sqlx::query(&query).execute(&mut connection).await?;
+    tx.commit().await?;
 
     Ok(format!(
         "Publication {} successfully created!",
@@ -802,7 +807,7 @@ pub async fn delete_publication(
     Extension(db): Extension<DB>,
     Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>,
 ) -> Result<String> {
-    let mut connection = get_database_connection(
+    let mut pg_connection = get_pg_connection(
         authed.clone(),
         Some(user_db.clone()),
         &db,
@@ -811,9 +816,7 @@ pub async fn delete_publication(
     )
     .await?;
 
-    let query = drop_publication_query(&publication_name);
-
-    sqlx::query(&query).execute(&mut connection).await?;
+    drop_publication(&mut pg_connection, &publication_name).await?;
 
     Ok(format!(
         "Publication {} successfully deleted!",
@@ -821,27 +824,37 @@ pub async fn delete_publication(
     ))
 }
 
-pub fn get_update_publication_query(
+pub async fn update_pg_publication(
+    pg_connection: &mut PgConnection,
     publication_name: &str,
     PublicationData { table_to_track, transaction_to_track }: PublicationData,
-    all_table: bool,
-) -> Vec<String> {
+    all_table: Option<bool>,
+) -> Result<()> {
     let quoted_publication_name = quote_identifier(&publication_name);
-
     let transaction_to_track_as_str = transaction_to_track.iter().join(",");
-    let mut queries = Vec::with_capacity(2);
     match table_to_track {
         Some(ref relations) if !relations.is_empty() => {
-            if all_table {
-                queries.push(drop_publication_query(&publication_name));
-                queries.push(create_publication_query(
+            //if all table is none it means that the publication do not exist in the database
+            if all_table.unwrap_or(true) {
+                if all_table.is_some() {
+                    drop_publication(pg_connection, &publication_name).await?;
+                }
+                create_pg_publication(
+                    pg_connection,
                     &publication_name,
                     table_to_track.as_deref(),
-                    &transaction_to_track.iter().map(AsRef::as_ref).collect_vec(),
-                ));
+                    &transaction_to_track,
+                )
+                .await?;
             } else {
-                let mut query = String::from("");
+                let pg_14 = check_if_valid_publication_for_postgres_version(
+                    pg_connection,
+                    table_to_track.as_deref(),
+                )
+                .await?;
 
+                let mut query = String::from("");
+                let mut first = true;
                 query.push_str("ALTER PUBLICATION ");
                 query.push_str(&quoted_publication_name);
                 query.push_str(" SET");
@@ -851,16 +864,20 @@ pub fn get_update_publication_query(
                         let quoted_schema = quote_identifier(&schema.schema_name);
                         query.push_str(&quoted_schema);
                     } else {
-                        query.push_str(" TABLE ONLY ");
+                        if pg_14 && first {
+                            query.push_str(" TABLE ONLY ");
+                            first = false
+                        } else if !pg_14 {
+                            query.push_str(" TABLE ONLY ");
+                        }
                         for (j, table) in schema.table_to_track.iter().enumerate() {
                             let table_name = quote_identifier(&table.table_name);
                             let schema_name = quote_identifier(&schema.schema_name);
                             let full_name = format!("{}.{}", &schema_name, &table_name);
                             query.push_str(&full_name);
-                            if !table.columns_name.is_empty() {
+                            if let Some(columns) = table.columns_name.as_ref() {
                                 query.push_str(" (");
-                                let columns = table
-                                    .columns_name
+                                let columns = columns
                                     .iter()
                                     .map(|column| quote_identifier(column))
                                     .join(", ");
@@ -883,9 +900,8 @@ pub fn get_update_publication_query(
                         query.push(',');
                     }
                 }
-                query.push(';');
 
-                queries.push(query);
+                sqlx::query(&query).execute(&mut *pg_connection).await?;
 
                 let mut query = String::new();
 
@@ -895,23 +911,25 @@ pub fn get_update_publication_query(
                     " SET (publish = '{}');",
                     transaction_to_track_as_str
                 ));
-                queries.push(query);
+
+                sqlx::query(&query).execute(pg_connection).await?;
             }
         }
         _ => {
-            queries.push(drop_publication_query(&publication_name));
-            let to_execute = format!(
+            drop_publication(pg_connection, &publication_name).await?;
+            let query_to_execute = format!(
                 r#"
                 CREATE
                     PUBLICATION {} FOR ALL TABLES WITH (publish = '{}');
                 "#,
                 quoted_publication_name, transaction_to_track_as_str
             );
-            queries.push(to_execute);
+            sqlx::query(&query_to_execute)
+                .execute(pg_connection)
+                .await?;
         }
     };
-
-    queries
+    Ok(())
 }
 
 pub async fn alter_publication(
@@ -921,7 +939,7 @@ pub async fn alter_publication(
     Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>,
     Json(publication_data): Json<PublicationData>,
 ) -> Result<String> {
-    let mut connection = get_database_connection(
+    let mut pg_connection = get_pg_connection(
         authed.clone(),
         Some(user_db.clone()),
         &db,
@@ -930,16 +948,19 @@ pub async fn alter_publication(
     )
     .await?;
 
-    check_if_publication_exist(&mut connection, &publication_name).await?;
+    let mut tx = pg_connection.begin().await?;
+    
+    let publication = get_publication_scope_and_transaction(&mut tx, &publication_name).await?;
 
-    let (all_table, _) =
-        get_publication_scope_and_transaction(&mut connection, &publication_name).await?;
+    update_pg_publication(
+        &mut tx,
+        &publication_name,
+        publication_data,
+        publication.map(|publication| publication.0),
+    )
+    .await?;
 
-    let queries = get_update_publication_query(&publication_name, publication_data, all_table);
-
-    for query in queries {
-        sqlx::query(&query).execute(&mut connection).await?;
-    }
+    tx.commit().await?;
 
     Ok(format!(
         "Publication {} updated with success",
@@ -948,9 +969,9 @@ pub async fn alter_publication(
 }
 
 async fn get_publication_scope_and_transaction(
-    connection: &mut PgConnection,
+    pg_connection: &mut PgConnection,
     publication_name: &str,
-) -> std::result::Result<(bool, Vec<String>), Error> {
+) -> Result<Option<(bool, Vec<String>)>> {
     #[derive(Debug, Deserialize, FromRow)]
     struct PublicationTransaction {
         all_table: bool,
@@ -959,8 +980,7 @@ async fn get_publication_scope_and_transaction(
         delete: bool,
     }
 
-    let transaction = sqlx::query_as!(
-        PublicationTransaction,
+    let publication: Option<PublicationTransaction> = sqlx::query_as(
         r#"
         SELECT
             puballtables AS all_table,
@@ -972,70 +992,100 @@ async fn get_publication_scope_and_transaction(
         WHERE
             pubname = $1
         "#,
-        publication_name
     )
-    .fetch_one(&mut *connection)
+    .bind(publication_name)
+    .fetch_optional(&mut *pg_connection)
     .await?;
 
+    if publication.is_none() {
+        return Ok(None);
+    }
+
     let mut transaction_to_track = Vec::with_capacity(3);
 
-    if transaction.insert {
+    let publication = publication.unwrap();
+    if publication.insert {
         transaction_to_track.push("insert".to_string());
     }
-    if transaction.update {
+    if publication.update {
         transaction_to_track.push("update".to_string());
     }
-    if transaction.delete {
+    if publication.delete {
         transaction_to_track.push("delete".to_string());
     }
 
-    Ok((transaction.all_table, transaction_to_track))
+    Ok(Some((publication.all_table, transaction_to_track)))
 }
 
 async fn get_tracked_relations(
-    connection: &mut PgConnection,
+    pg_connection: &mut PgConnection,
     publication_name: &str,
 ) -> Result<Vec<Relations>> {
     #[derive(Debug, Deserialize, FromRow)]
     struct PublicationData {
         schema_name: Option<String>,
         table_name: Option<String>,
+        #[serde(default)]
         columns: Option<Vec<String>>,
+        #[serde(default)]
         where_clause: Option<String>,
     }
 
-    let publications = sqlx::query_as!(
-        PublicationData,
+    let pg_version = get_postgres_version_internal(pg_connection).await?;
+    let query = if pg_version.starts_with("14") {
         r#"
             SELECT
             schemaname AS schema_name,
             tablename AS table_name,
-            CASE
-                WHEN array_length(attnames, 1) = (SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = pg_publication_tables.schemaname AND table_name = pg_publication_tables.tablename)
-                THEN NULL
-                ELSE attnames
-            END AS columns,
+            NULL::text[] AS columns,
+            NULL::text AS where_clause
+            FROM
+                pg_publication_tables
+            WHERE
+                pubname = $1;
+            "#
+    } else {
+        r#"
+            SELECT
+            schemaname AS schema_name,
+            tablename AS table_name,
+            attnames AS columns,
             rowfilter AS where_clause
             FROM
                 pg_publication_tables
             WHERE
                 pubname = $1;
-            "#,
-        publication_name
-    )
-    .fetch_all(&mut *connection)
-    .await?;
+            "#
+    };
+
+    let publications: Vec<PublicationData> = sqlx::query_as(query)
+        .bind(publication_name)
+        .fetch_all(&mut *pg_connection)
+        .await?;
 
     let mut table_to_track: HashMap<String, Relations> = HashMap::new();
 
     for publication in publications {
-        let schema_name = publication.schema_name.unwrap();
+        let schema_name = publication.schema_name.ok_or_else(|| Error::Anyhow {
+            error: anyhow::anyhow!(
+                "Unexpected NULL `schema_name` in publication entry (pubname: `{}`). This should never happen unless PostgreSQL internals are corrupted.",
+                publication_name,
+            ),
+            location: "postgres_triggers/handler.rs@1093".to_string(),
+        })?;
+
+        let table_name = publication.table_name.ok_or_else(|| Error::Anyhow {
+            error: anyhow::anyhow!(
+                "Unexpected NULL `table_name` for schema `{}` in publication `{}`. This should never happen unless PostgreSQL internals are corrupted.",
+                schema_name,
+                publication_name,
+            ),
+            location: "postgres_triggers/handler.rs@1102".to_string(),
+        })?;
+
         let entry = table_to_track.entry(schema_name.clone());
-        let table_to_track = TableToTrack::new(
-            publication.table_name.unwrap(),
-            publication.where_clause,
-            publication.columns.unwrap_or_default(),
-        );
+        let table_to_track =
+            TableToTrack::new(table_name, publication.where_clause, publication.columns);
         match entry {
             Occupied(mut occuped) => {
                 occuped.get_mut().add_new_table(table_to_track);
@@ -1111,7 +1161,7 @@ pub async fn update_postgres_trigger(
         publication,
     } = postgres_trigger;
 
-    let mut connection = get_database_connection(
+    let mut pg_connection = get_pg_connection(
         authed.clone(),
         Some(user_db.clone()),
         &db,
@@ -1120,18 +1170,33 @@ pub async fn update_postgres_trigger(
     )
     .await?;
 
-    check_if_logical_replication_slot_exist(&mut connection, &replication_slot_name).await?;
+    let exists =
+        check_if_logical_replication_slot_exist(&mut pg_connection, &replication_slot_name).await?;
+
+    let mut tx = pg_connection.begin().await?;
+
+    if !exists {
+        tracing::debug!(
+            "Logical replication slot named: {} does not exists creating it...",
+            &replication_slot_name
+        );
+        create_logical_replication_slot(&mut tx, &replication_slot_name).await?;
+    }
 
     if let Some(publication) = publication {
-        check_if_publication_exist(&mut connection, &publication_name).await?;
-        let (all_table, _) =
-            get_publication_scope_and_transaction(&mut connection, &publication_name).await?;
+        let publication_data =
+            get_publication_scope_and_transaction(&mut tx, &publication_name).await?;
 
-        let queries = get_update_publication_query(&publication_name, publication, all_table);
-        for query in queries {
-            sqlx::query(&query).execute(&mut connection).await?;
-        }
+        update_pg_publication(
+            &mut tx,
+            &publication_name,
+            publication,
+            publication_data.map(|publication| publication.0),
+        )
+        .await?;
     }
+    tx.commit().await?;
+
     let mut tx = user_db.begin(&authed).await?;
 
     sqlx::query!(
@@ -1356,7 +1421,7 @@ pub async fn create_template_script(
         ));
     }
 
-    let mut connection = get_database_connection(
+    let mut pg_connection = get_pg_connection(
         authed.clone(),
         Some(user_db.clone()),
         &db,
@@ -1384,11 +1449,12 @@ pub async fn create_template_script(
                     format!("{}.{}", &relation.schema_name, table_to_track.table_name);
                 schema_or_fully_qualified_name.push(quote_literal(&fully_qualified_name));
 
-                let columns = if !table_to_track.columns_name.is_empty() {
-                    quote_literal(&table_to_track.columns_name.join(","))
-                } else {
-                    "''".to_string()
-                };
+                let columns = table_to_track
+                    .columns_name
+                    .map(|columns| quote_literal(&columns.join(",")))
+                    .or_else(|| Some("''".to_string()))
+                    .unwrap();
+
                 columns_list.push(columns);
             }
             continue;
@@ -1448,11 +1514,7 @@ pub async fn create_template_script(
         tables_name, columns_list
     );
 
-    let rows: Vec<ColumnInfo> = sqlx::query_as(&query)
-        .fetch_all(&mut connection)
-        .await
-        .map_err(|e| error::Error::SqlErr { error: e, location: "pg_trigger".to_string() })?;
-
+    let rows: Vec<ColumnInfo> = sqlx::query_as(&query).fetch_all(&mut pg_connection).await?;
     let mut mapper: HashMap<String, HashMap<String, Vec<MappingInfo>>> = HashMap::new();
 
     for row in rows {
@@ -1509,7 +1571,7 @@ pub async fn is_database_in_logical_level(
     Extension(db): Extension<DB>,
     Path((w_id, postgres_resource_path)): Path<(String, String)>,
 ) -> error::JsonResult<bool> {
-    let mut connection = get_database_connection(
+    let mut pg_connection = get_pg_connection(
         authed.clone(),
         Some(user_db.clone()),
         &db,
@@ -1518,8 +1580,8 @@ pub async fn is_database_in_logical_level(
     )
     .await?;
 
-    let wal_level = sqlx::query_scalar!("SHOW WAL_LEVEL;")
-        .fetch_optional(&mut connection)
+    let wal_level: Option<String> = sqlx::query_scalar("SHOW WAL_LEVEL;")
+        .fetch_optional(&mut pg_connection)
         .await?
         .flatten();
 

backend/windmill-api/src/postgres_triggers/mod.rs

@@ -25,12 +25,17 @@ pub use handler::PostgresTrigger;
 use handler::{
     alter_publication, create_postgres_trigger, create_publication, create_slot,
     create_template_script, delete_postgres_trigger, delete_publication, drop_slot_name,
-    exists_postgres_trigger, get_postgres_trigger, get_publication_info, get_template_script,
+    exists_postgres_trigger, get_postgres_trigger, get_postgres_version,
+    get_postgres_version_internal, get_publication_info, get_template_script,
     is_database_in_logical_level, list_database_publication, list_postgres_triggers,
     list_slot_name, set_enabled, test_postgres_connection, update_postgres_trigger, Postgres,
     Relations,
 };
-use windmill_common::{db::UserDB, error::Error, utils::StripPath};
+use windmill_common::{
+    db::UserDB,
+    error::{Error, Result},
+    utils::StripPath,
+};
 mod bool;
 mod converter;
 mod handler;
@@ -47,13 +52,13 @@ const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associat
 
 const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#;
 
-pub async fn get_database_connection(
+pub async fn get_pg_connection(
     authed: ApiAuthed,
     user_db: Option<UserDB>,
     db: &DB,
     postgres_resource_path: &str,
     w_id: &str,
-) -> std::result::Result<PgConnection, windmill_common::error::Error> {
+) -> Result<PgConnection> {
     let database =
         try_get_resource_from_db_as::<Postgres>(authed, user_db, db, postgres_resource_path, w_id)
             .await?;
@@ -61,9 +66,7 @@ pub async fn get_database_connection(
     Ok(get_raw_postgres_connection(&database).await?)
 }
 
-pub async fn get_raw_postgres_connection(
-    db: &Postgres,
-) -> std::result::Result<PgConnection, Error> {
+pub async fn get_raw_postgres_connection(db: &Postgres) -> Result<PgConnection> {
     let options = {
         let sslmode = if !db.sslmode.is_empty() {
             PgSslMode::from_str(&db.sslmode)?
@@ -99,7 +102,10 @@ pub async fn get_raw_postgres_connection(
     Ok(PgConnection::connect_with(&options).await?)
 }
 
-pub fn create_logical_replication_slot_query(name: &str) -> String {
+pub async fn create_logical_replication_slot(
+    pg_connection: &mut PgConnection,
+    name: &str,
+) -> Result<()> {
     let query = format!(
         r#"
         SELECT 
@@ -109,14 +115,53 @@ pub fn create_logical_replication_slot_query(name: &str) -> String {
         quote_literal(&name)
     );
 
-    query
+    sqlx::query(&query).execute(pg_connection).await?;
+
+    Ok(())
 }
 
-pub fn create_publication_query(
+async fn check_if_valid_publication_for_postgres_version(
+    pg_connection: &mut PgConnection,
+    table_to_track: Option<&[Relations]>,
+) -> Result<bool> {
+    let postgres_version = get_postgres_version_internal(pg_connection).await?;
+
+    let pg_14 = postgres_version.starts_with("14");
+    if pg_14 {
+        let unsupported_publication = table_to_track
+            .and_then(|relations| {
+                relations.iter().find(|relation| {
+                    let invalid_relation = relation.table_to_track.iter().find(|table_to_track| {
+                        table_to_track.where_clause.is_some()
+                            || table_to_track.columns_name.is_some()
+                    });
+
+                    relation.table_to_track.is_empty() || invalid_relation.is_some()
+                })
+            })
+            .is_some();
+
+        if unsupported_publication {
+            return Err(Error::BadRequest(
+                    "Your PostgreSQL database is running version 14, which does not support the following publication features: \
+                    - WHERE clause filtering, \
+                    - selective column tracking, and \
+                    - tracking all tables within a schema.\n\
+                    These features are only available in PostgreSQL 15 and above.".to_string(),
+                ));
+        }
+    }
+    Ok(pg_14)
+}
+
+pub async fn create_pg_publication(
+    pg_connection: &mut PgConnection,
     publication_name: &str,
     table_to_track: Option<&[Relations]>,
-    transaction_to_track: &[&str],
-) -> String {
+    transaction_to_track: &[String],
+) -> Result<()> {
+    let pg_14 =
+        check_if_valid_publication_for_postgres_version(pg_connection, table_to_track).await?;
     let mut query = String::from("CREATE PUBLICATION ");
 
     query.push_str(&quote_identifier(publication_name));
@@ -124,21 +169,26 @@ pub fn create_publication_query(
     match table_to_track {
         Some(database_component) if !database_component.is_empty() => {
             query.push_str(" FOR");
+            let mut first = true;
             for (i, schema) in database_component.iter().enumerate() {
                 if schema.table_to_track.is_empty() {
                     query.push_str(" TABLES IN SCHEMA ");
                     query.push_str(&quote_identifier(&schema.schema_name));
                 } else {
-                    query.push_str(" TABLE ONLY ");
+                    if pg_14 && first {
+                        query.push_str(" TABLE ONLY ");
+                        first = false;
+                    } else if !pg_14 {
+                        query.push_str(" TABLE ONLY ");
+                    }
                     for (j, table) in schema.table_to_track.iter().enumerate() {
                         let table_name = quote_identifier(&table.table_name);
                         let schema_name = quote_identifier(&schema.schema_name);
                         let full_name = format!("{}.{}", &schema_name, &table_name);
                         query.push_str(&full_name);
-                        if !table.columns_name.is_empty() {
+                        if let Some(columns) = table.columns_name.as_ref() {
                             query.push_str(" (");
-                            let columns = table
-                                .columns_name
+                            let columns = columns
                                 .iter()
                                 .map(|column| quote_identifier(column))
                                 .join(", ");
@@ -174,22 +224,22 @@ pub fn create_publication_query(
         query.push_str("');");
     }
 
-    query
+    sqlx::query(&query).execute(pg_connection).await?;
+
+    Ok(())
 }
 
-pub fn drop_publication_query(publication_name: &str) -> String {
+pub async fn drop_publication(
+    pg_connection: &mut PgConnection,
+    publication_name: &str,
+) -> Result<()> {
     let mut query = String::from("DROP PUBLICATION IF EXISTS ");
     let quoted_publication_name = quote_identifier(publication_name);
     query.push_str(&quoted_publication_name);
-    query.push_str(";");
-    query
-}
 
-pub fn drop_logical_replication_slot_query(replication_slot_name: &str) -> String {
-    format!(
-        "SELECT pg_drop_replication_slot({});",
-        quote_literal(&replication_slot_name)
-    )
+    sqlx::query(&query).execute(pg_connection).await?;
+
+    Ok(())
 }
 
 pub fn generate_random_string() -> String {
@@ -228,6 +278,10 @@ fn slot_service() -> Router {
         .route("/delete/*path", delete(drop_slot_name))
 }
 
+fn postgres_service() -> Router {
+    Router::new().route("/version/*path", get(get_postgres_version))
+}
+
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/test", post(test_postgres_connection))
@@ -246,6 +300,7 @@ pub fn workspaced_service() -> Router {
         )
         .nest("/publication", publication_service())
         .nest("/slot", slot_service())
+        .nest("/postgres", postgres_service())
 }
 
 async fn run_job(

backend/windmill-api/src/postgres_triggers/trigger.rs

@@ -34,8 +34,8 @@ use windmill_common::{
 };
 
 use super::{
-    drop_logical_replication_slot_query, drop_publication_query, get_database_connection,
-    handler::{Postgres, PostgresTrigger},
+    drop_publication, get_pg_connection,
+    handler::{drop_logical_replication_slot, Postgres, PostgresTrigger},
     replication_message::PrimaryKeepAliveBody,
     ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS,
 };
@@ -585,7 +585,7 @@ impl PostgresConfig {
 
                 let user_db = UserDB::new(db.clone());
 
-                let mut connection = get_database_connection(
+                let mut pg_connection = get_pg_connection(
                     authed.clone(),
                     Some(user_db.clone()),
                     &db,
@@ -594,13 +594,12 @@ impl PostgresConfig {
                 )
                 .await?;
 
-                let query = drop_logical_replication_slot_query(replication_slot_name);
+                if capture.trigger_config.basic_mode.unwrap_or(false) {
+                    drop_logical_replication_slot(&mut pg_connection, replication_slot_name)
+                        .await?;
 
-                let _ = sqlx::query(&query).execute(&mut connection).await;
-
-                let query = drop_publication_query(publication_name);
-
-                let _ = sqlx::query(&query).execute(&mut connection).await;
+                    drop_publication(&mut pg_connection, publication_name).await?;
+                }
 
                 Ok(())
             }

backend/windmill-api/src/triggers.rs

@@ -21,6 +21,9 @@ pub struct TriggersCount {
     kafka_count: i64,
     nats_count: i64,
     postgres_count: i64,
+    mqtt_count: i64,
+    sqs_count: i64,
+    gcp_count: i64,
 }
 pub(crate) async fn get_triggers_count_internal(
     db: &DB,
@@ -97,6 +100,36 @@ pub(crate) async fn get_triggers_count_internal(
     .await?
     .unwrap_or(0);
 
+    let mqtt_count = sqlx::query_scalar!(
+        "SELECT COUNT(*) FROM mqtt_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3",
+        path,
+        is_flow,
+        w_id
+    )
+    .fetch_one(db)
+    .await?
+    .unwrap_or(0);
+
+    let sqs_count = sqlx::query_scalar!(
+        "SELECT COUNT(*) FROM sqs_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3",
+        path,
+        is_flow,
+        w_id
+    )
+    .fetch_one(db)
+    .await?
+    .unwrap_or(0);
+
+    let gcp_count = sqlx::query_scalar!(
+        "SELECT COUNT(*) FROM gcp_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3",
+        path,
+        is_flow,
+        w_id
+    )
+    .fetch_one(db)
+    .await?
+    .unwrap_or(0);
+
     let webhook_count = (if is_flow {
         sqlx::query_scalar!(
             "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]",
@@ -141,6 +174,9 @@ pub(crate) async fn get_triggers_count_internal(
         kafka_count,
         nats_count,
         postgres_count,
+        mqtt_count,
+        gcp_count,
+        sqs_count,
     }))
 }
 

backend/windmill-api/src/users.rs

@@ -334,6 +334,7 @@ pub struct NewUser {
     pub super_admin: bool,
     pub name: Option<String>,
     pub company: Option<String>,
+    pub skip_email: Option<bool>,
 }
 
 #[derive(Deserialize)]

backend/windmill-api/src/workspaces_export.rs

@@ -360,6 +360,7 @@ pub(crate) async fn tarball_workspace(
                 ScriptLang::Bigquery => "bq.sql",
                 ScriptLang::Snowflake => "sf.sql",
                 ScriptLang::Mssql => "ms.sql",
+                ScriptLang::DuckDb => "duckdb.sql",
                 ScriptLang::Graphql => "gql",
                 ScriptLang::Nativets => "fetch.ts",
                 ScriptLang::Bun | ScriptLang::Bunnative => {

backend/windmill-common/src/client.rs

@@ -2,7 +2,11 @@ use anyhow::Context;
 use reqwest::{Body, Response};
 use serde::de::DeserializeOwned;
 
-use crate::utils::HTTP_CLIENT;
+use crate::{
+    error::{self, to_anyhow},
+    s3_helpers::{DuckdbConnectionSettingsQueryV2, DuckdbConnectionSettingsResponse},
+    utils::HTTP_CLIENT,
+};
 
 #[derive(Clone)]
 pub struct AuthedClient {
@@ -197,4 +201,44 @@ impl AuthedClient {
             _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
         }
     }
+
+    pub async fn get_duckdb_connection_settings(
+        &self,
+        s3: &DuckdbConnectionSettingsQueryV2,
+    ) -> error::Result<DuckdbConnectionSettingsResponse> {
+        let url = format!(
+            "{}/api/w/{}/job_helpers/v2/duckdb_connection_settings",
+            self.base_internal_url, &self.workspace
+        );
+        let response = self
+            .force_client
+            .as_ref()
+            .unwrap_or(&HTTP_CLIENT)
+            .post(url)
+            .header(
+                reqwest::header::CONTENT_TYPE,
+                reqwest::header::HeaderValue::from_static("application/json"),
+            )
+            .header(
+                reqwest::header::ACCEPT,
+                reqwest::header::HeaderValue::from_static("application/json"),
+            )
+            .header(
+                reqwest::header::AUTHORIZATION,
+                reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))
+                    .map_err(|e| error::Error::BadConfig(e.to_string()))?,
+            )
+            .body(serde_json::to_string(&s3).map_err(to_anyhow)?)
+            .send()
+            .await
+            .context(format!("Sent get_duckdb_connection_settings request",))
+            .map_err(error::Error::from)?;
+        match response.status().as_u16() {
+            200u16 => Ok(response
+                .json::<DuckdbConnectionSettingsResponse>()
+                .await
+                .context("decoding duckdb_connection_settings response as json")?),
+            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
+        }
+    }
 }

backend/windmill-common/src/s3_helpers.rs

@@ -891,3 +891,20 @@ pub async fn convert_json_line_stream<E: Into<anyhow::Error>>(
 
     Ok(tokio_stream::wrappers::ReceiverStream::new(rx))
 }
+
+#[derive(Deserialize, Serialize)]
+pub struct DuckdbConnectionSettingsResponse {
+    pub connection_settings_str: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub azure_container_path: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub s3_bucket: Option<String>,
+}
+
+#[derive(Deserialize, Serialize)]
+pub struct DuckdbConnectionSettingsQueryV2 {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub s3_resource_path: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub storage: Option<String>,
+}

backend/windmill-common/src/schema.rs

@@ -349,7 +349,7 @@ pub fn should_validate_schema(code: &str, lang: &ScriptLang) -> bool {
     let comment = match lang {
         Nativets | Bun | Bunnative | Deno | Php | CSharp | Java => "//",
         Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu => "#",
-        Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB => "--",
+        Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--",
         Rust => "//!",
         // for related places search: ADD_NEW_LANG
     };

backend/windmill-common/src/scripts.rs

@@ -46,13 +46,13 @@ pub enum ScriptLang {
     Graphql,
     Mssql,
     OracleDB,
+    DuckDb,
     Php,
     Rust,
     Ansible,
     CSharp,
     Nu,
-    Java,
-    // for related places search: ADD_NEW_LANG
+    Java, // for related places search: ADD_NEW_LANG
 }
 
 impl ScriptLang {
@@ -73,6 +73,7 @@ impl ScriptLang {
             ScriptLang::Mssql => "mssql",
             ScriptLang::Graphql => "graphql",
             ScriptLang::OracleDB => "oracledb",
+            ScriptLang::DuckDb => "duckdb",
             ScriptLang::Php => "php",
             ScriptLang::Rust => "rust",
             ScriptLang::Ansible => "ansible",

backend/windmill-common/src/worker.rs

@@ -79,6 +79,7 @@ lazy_static::lazy_static! {
         "csharp".to_string(),
         "nu".to_string(),
         "java".to_string(),
+        "duckdb".to_string(),
         // for related places search: ADD_NEW_LANG
         "dependency".to_string(),
         "flow".to_string(),

backend/windmill-worker/Cargo.toml

@@ -31,6 +31,7 @@ csharp = ["dep:windmill-parser-csharp"]
 rust = ["dep:windmill-parser-rust"]
 nu = ["dep:windmill-parser-nu"]
 java = ["dep:windmill-parser-java"]
+duckdb = ["dep:duckdb"]
 
 [dependencies]
 windmill-queue.workspace = true
@@ -92,6 +93,7 @@ deno_permissions = { workspace = true, optional = true }
 deno_io = { workspace = true, optional = true }
 deno_error = { workspace = true, optional = true }
 async-stream.workspace = true
+duckdb = { workspace = true, optional = true }
 
 postgres-native-tls.workspace = true
 native-tls.workspace = true

backend/windmill-worker/nsjail/download.rust.config.proto

@@ -0,0 +1,117 @@
+name: "rust download script"
+
+mode: ONCE
+hostname: "rust"
+log_level: ERROR
+
+disable_rl: true
+
+cwd: "/tmp"
+
+clone_newnet: false
+
+keep_caps: false
+keep_env: true
+mount_proc: true
+
+mount {
+    src: "/bin"
+    dst: "/bin"
+	is_bind: true
+}
+
+mount {
+    src: "/lib"
+    dst: "/lib"
+	is_bind: true
+}
+
+
+mount {
+    src: "/lib64"
+    dst: "/lib64"
+	is_bind: true
+    mandatory: false
+}
+
+
+mount {
+    src: "/usr"
+    dst: "/usr"
+	is_bind: true
+}
+
+mount {
+	src: "/dev/null"
+	dst: "/dev/null"
+	is_bind: true
+	rw: true
+}
+
+mount {
+	dst: "/tmp"
+	fstype: "tmpfs"
+	rw: true
+    options: "size=500000000"
+}
+
+mount {
+    src: "/etc"
+    dst: "/etc"
+	is_bind: true
+}
+
+mount {
+    src: "/dev/random"
+    dst: "/dev/random"
+    is_bind: true
+}
+mount {
+    src: "{JOB_DIR}"
+    dst: "/tmp"
+    is_bind: true
+    rw: true
+}
+
+mount {
+    src: "/dev/urandom"
+    dst: "/dev/urandom"
+    is_bind: true
+}
+
+iface_no_lo: true
+
+mount {
+    src: "{BUILD}"
+    dst: "{BUILD}"
+    is_bind: true
+    mandatory: false
+    rw: true
+}
+mount {
+    src: "{CARGO_HOME}"
+    dst: "{CARGO_HOME}"
+    is_bind: true
+    # Readonly 
+    rw: false
+}
+
+mount {
+    src: "{BUILD}/registry"
+    dst: "{CARGO_HOME}/registry"
+    is_bind: true
+    mandatory: true
+    # Read-write
+    rw: true
+}
+
+mount {
+    src: "{BUILD}/git"
+    dst: "{CARGO_HOME}/git"
+    is_bind: true
+    mandatory: true
+    # Read-write
+    rw: true
+}
+
+{DEV}

backend/windmill-worker/nsjail/run.rust.config.proto

@@ -104,3 +104,6 @@ mount {
 }
 
 {SHARED_MOUNT}
+
+{DEV}
+

backend/windmill-worker/src/duckdb_executor.rs

@@ -0,0 +1,664 @@
+use std::collections::HashMap;
+use std::env;
+
+use duckdb::types::TimeUnit;
+use duckdb::{params_from_iter, Row};
+use rust_decimal::prelude::FromPrimitive;
+use rust_decimal::Decimal;
+use serde_json::value::RawValue;
+use serde_json::{json, Value};
+use tokio::fs::remove_file;
+use tokio::task;
+use uuid::Uuid;
+use windmill_common::error::{to_anyhow, Error, Result};
+use windmill_common::s3_helpers::{
+    DuckdbConnectionSettingsQueryV2, DuckdbConnectionSettingsResponse, S3Object,
+};
+use windmill_common::worker::{to_raw_value, Connection};
+use windmill_parser_sql::{parse_duckdb_sig, parse_sql_blocks};
+use windmill_queue::{CanceledBy, MiniPulledJob};
+
+use crate::common::{build_args_values, OccupancyMetrics};
+use crate::handle_child::run_future_with_polling_update_job_poller;
+#[cfg(feature = "mysql")]
+use crate::mysql_executor::MysqlDatabase;
+use crate::pg_executor::PgDatabase;
+use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
+use windmill_common::client::AuthedClient;
+
+fn do_duckdb_inner(
+    conn: &duckdb::Connection,
+    query: &str,
+    job_args: &HashMap<String, duckdb::types::Value>,
+    skip_collect: bool,
+    column_order: &mut Option<Vec<String>>,
+) -> Result<Box<RawValue>> {
+    let mut rows_vec = vec![];
+
+    let (query, job_args) = interpolate_named_args(query, &job_args);
+
+    let mut stmt = conn
+        .prepare(&query)
+        .map_err(|e| Error::ExecutionErr(e.to_string()))?;
+
+    let mut rows = stmt
+        .query(params_from_iter(job_args))
+        .map_err(|e| Error::ExecutionErr(e.to_string()))?;
+
+    if skip_collect {
+        return Ok(to_raw_value(&json!([])));
+    }
+
+    // Statement needs to be stepped at least once or stmt.column_names() will panic
+    let mut column_names = None;
+    loop {
+        let row = rows.next();
+        match row {
+            Ok(Some(row)) => {
+                // Set column names if not already set
+                let stmt = row.as_ref();
+                let column_names = match column_names.as_ref() {
+                    Some(column_names) => column_names,
+                    None => {
+                        column_names = Some(stmt.column_names());
+                        column_names.as_ref().unwrap()
+                    }
+                };
+
+                let row = row_to_value(row, &column_names.as_slice())
+                    .map_err(|e| Error::ExecutionErr(e.to_string()))?;
+                rows_vec.push(row);
+            }
+            Ok(None) => break,
+            Err(e) => {
+                return Err(Error::ExecutionErr(e.to_string()));
+            }
+        }
+    }
+
+    if let (Some(column_order), Some(column_names)) = (column_order.as_mut(), column_names) {
+        *column_order = column_names.clone();
+    }
+
+    return Ok(to_raw_value(&rows_vec));
+}
+
+pub async fn do_duckdb(
+    job: &MiniPulledJob,
+    client: &AuthedClient,
+    query: &str,
+    conn: &Connection,
+    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
+    worker_name: &str,
+    column_order_ref: &mut Option<Vec<String>>,
+    occupancy_metrics: &mut OccupancyMetrics,
+) -> Result<Box<RawValue>> {
+    let result_f = async {
+        let sig = parse_duckdb_sig(query)?.args;
+        let mut job_args = build_args_values(job, client, conn).await?;
+
+        let (query, _) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?;
+        // Prevent interpolate_named_args from detecting argument identifiers in the signature for
+        // the first query block
+        let query = trunc_sig(query);
+
+        let (_query_with_transformed_s3_uris, mut used_storages) =
+            transform_s3_uris(query, client).await?;
+        let query = _query_with_transformed_s3_uris.as_deref().unwrap_or(query);
+
+        let job_args = {
+            let mut m: HashMap<String, duckdb::types::Value> = HashMap::new();
+            for sig_arg in sig.into_iter() {
+                let json_value = job_args
+                    .remove(&sig_arg.name)
+                    .or_else(|| sig_arg.default)
+                    .unwrap_or_else(|| json!(null));
+
+                if matches!(&sig_arg.otyp.as_ref().map(String::as_str), Some("s3object")) {
+                    let s3_obj = serde_json::from_value::<S3Object>(json_value).map_err(|e| {
+                        Error::ExecutionErr(format!("Failed to deserialize S3Object: {}", e))
+                    })?;
+                    let duckdb_conn_settings: windmill_common::s3_helpers::DuckdbConnectionSettingsResponse = client
+                        .get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 {
+                            s3_resource_path: None,
+                            storage: s3_obj.storage.clone(),
+                        })
+                        .await?;
+
+                    let uri = match (
+                        &duckdb_conn_settings.s3_bucket,
+                        &duckdb_conn_settings.azure_container_path,
+                    ) {
+                        (Some(s3_bucket), None) => format!("s3://{}/{}", s3_bucket, &s3_obj.s3),
+                        (None, Some(az_container)) => format!("{}/{}", az_container, &s3_obj.s3),
+                        _ => {
+                            return Err(Error::ExecutionErr(
+                                "S3Object must have either s3_bucket or azure_container_path"
+                                    .to_string(),
+                            ));
+                        }
+                    };
+                    m.insert(sig_arg.name, duckdb::types::Value::Text(uri));
+                    used_storages.insert(s3_obj.storage, duckdb_conn_settings);
+                } else {
+                    let duckdb_value = json_value_to_duckdb_value(
+                        &json_value,
+                        sig_arg
+                            .otyp
+                            .clone()
+                            .unwrap_or_else(|| "text".to_string())
+                            .as_str(),
+                        client,
+                    )?;
+                    m.insert(sig_arg.name, duckdb_value);
+                }
+            }
+            m
+        };
+
+        let query_block_list = parse_sql_blocks(query);
+
+        // Replace windmill resource ATTACH statements with the real instructions
+        let query_block_list = {
+            let mut v = vec![];
+            for query_block in query_block_list.iter() {
+                match parse_attach_db_resource(query_block) {
+                    Some(parsed) => v.extend(
+                        transform_attach_db_resource_query(&parsed, &job.id, client).await?,
+                    ),
+                    None => v.push(query_block.to_string()),
+                };
+            }
+            v
+        };
+
+        // duckdb::Connection is not Send so we do it in a single blocking task
+        let (result, column_order) = task::spawn_blocking(move || {
+            let conn = duckdb::Connection::open_in_memory()
+                .map_err(|e| Error::ConnectingToDatabase(e.to_string()))?;
+
+            for (_, DuckdbConnectionSettingsResponse { connection_settings_str, .. }) in
+                used_storages.into_iter()
+            {
+                conn.execute_batch(&connection_settings_str)
+                    .map_err(|e| Error::ExecutionErr(e.to_string()))?;
+            }
+
+            let mut result: Option<Box<RawValue>> = None;
+            let mut column_order = None;
+            for (query_block_index, query_block) in query_block_list.iter().enumerate() {
+                result = Some(
+                    do_duckdb_inner(
+                        &conn,
+                        query_block.as_str(),
+                        &job_args,
+                        query_block_index != query_block_list.len() - 1,
+                        &mut column_order,
+                    )
+                    .map_err(|e| Error::ExecutionErr(e.to_string()))?,
+                );
+            }
+            let result = result.unwrap_or_else(|| to_raw_value(&json!([])));
+            Ok::<_, Error>((result, column_order))
+        })
+        .await
+        .map_err(to_anyhow)??;
+
+        *column_order_ref = column_order;
+
+        // BigQuery cleanup
+        let bq_credentials_path = make_bq_credentials_path(&job.id);
+        env::remove_var("GOOGLE_APPLICATION_CREDENTIALS");
+        if matches!(tokio::fs::try_exists(&bq_credentials_path).await, Ok(true)) {
+            remove_file(&bq_credentials_path).await.map_err(to_anyhow)?;
+        }
+        Ok(result)
+    };
+
+    let result = run_future_with_polling_update_job_poller(
+        job.id,
+        job.timeout,
+        conn,
+        mem_peak,
+        canceled_by,
+        result_f,
+        worker_name,
+        &job.workspace_id,
+        &mut Some(occupancy_metrics),
+        Box::pin(futures::stream::once(async { 0 })),
+    )
+    .await?;
+
+    Ok(result)
+}
+
+fn row_to_value(row: &Row<'_>, column_names: &[String]) -> Result<Box<RawValue>> {
+    let mut obj = serde_json::Map::new();
+    for (i, key) in column_names.iter().enumerate() {
+        let value: duckdb::types::Value =
+            row.get(i).map_err(|e| Error::ExecutionErr(e.to_string()))?;
+        let json_value = match value {
+            duckdb::types::Value::Null => serde_json::Value::Null,
+            duckdb::types::Value::Boolean(b) => serde_json::Value::Bool(b),
+            duckdb::types::Value::TinyInt(i) => serde_json::Value::Number(i.into()),
+            duckdb::types::Value::SmallInt(i) => serde_json::Value::Number(i.into()),
+            duckdb::types::Value::Int(i) => serde_json::Value::Number(i.into()),
+            duckdb::types::Value::BigInt(i) => serde_json::Value::Number(i.into()),
+            duckdb::types::Value::HugeInt(i) => serde_json::Value::String(i.to_string()),
+            duckdb::types::Value::UTinyInt(u) => serde_json::Value::Number(u.into()),
+            duckdb::types::Value::USmallInt(u) => serde_json::Value::Number(u.into()),
+            duckdb::types::Value::UInt(u) => serde_json::Value::Number(u.into()),
+            duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()),
+            duckdb::types::Value::Float(f) => serde_json::Value::Number(
+                serde_json::Number::from_f64(f as f64)
+                    .ok_or_else(|| Error::ExecutionErr("Could not convert to f64".to_string()))?,
+            ),
+            duckdb::types::Value::Double(f) => serde_json::Value::Number(
+                serde_json::Number::from_f64(f)
+                    .ok_or_else(|| Error::ExecutionErr("Could not convert to f64".to_string()))?,
+            ),
+            duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()),
+            duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()),
+            duckdb::types::Value::Text(s) => serde_json::Value::String(s),
+            duckdb::types::Value::Blob(b) => serde_json::Value::Array(
+                b.into_iter()
+                    .map(|byte| serde_json::Value::Number(byte.into()))
+                    .collect(),
+            ),
+            duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()),
+            duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()),
+            duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({
+                "months": months,
+                "days": days,
+                "nanos": nanos
+            }),
+            duckdb::types::Value::List(values) => serde_json::Value::Array(
+                values
+                    .into_iter()
+                    .map(|v| serde_json::Value::String(format!("{:?}", v)))
+                    .collect(),
+            ),
+            duckdb::types::Value::Enum(e) => serde_json::Value::String(e),
+            duckdb::types::Value::Struct(fields) => serde_json::Value::Object(
+                fields
+                    .iter()
+                    .map(|(k, v)| (k.clone(), serde_json::Value::String(format!("{:?}", v))))
+                    .collect(),
+            ),
+            duckdb::types::Value::Array(values) => serde_json::Value::Array(
+                values
+                    .into_iter()
+                    .map(|v| serde_json::Value::String(format!("{:?}", v)))
+                    .collect(),
+            ),
+            duckdb::types::Value::Map(map) => serde_json::Value::Object(
+                map.iter()
+                    .map(|(k, v)| {
+                        (
+                            format!("{:?}", k),
+                            serde_json::Value::String(format!("{:?}", v)),
+                        )
+                    })
+                    .collect(),
+            ),
+            duckdb::types::Value::Union(value) => {
+                serde_json::Value::String(format!("{:?}", *value))
+            }
+        };
+        obj.insert(key.clone(), json_value);
+    }
+    serde_json::value::to_raw_value(&obj).map_err(|e| e.into())
+}
+
+fn json_value_to_duckdb_value(
+    json_value: &serde_json::Value,
+    arg_type: &str,
+    client: &AuthedClient,
+) -> Result<duckdb::types::Value> {
+    let arg_type = arg_type.to_lowercase();
+    let duckdb_value = match json_value {
+        serde_json::Value::Null => duckdb::types::Value::Null,
+        serde_json::Value::Bool(b) => duckdb::types::Value::Boolean(*b),
+
+        serde_json::Value::String(s)
+            if matches!(
+                arg_type.as_str(),
+                "timestamp" | "timestamptz" | "timestamp with time zone" | "datetime"
+            ) =>
+        {
+            string_to_duckdb_timestamp(&s)?
+        }
+        serde_json::Value::String(s) if arg_type.as_str() == "date" => string_to_duckdb_date(&s)?,
+        serde_json::Value::String(s) if arg_type.as_str() == "time" => string_to_duckdb_time(&s)?,
+        serde_json::Value::String(s) => duckdb::types::Value::Text(s.clone()),
+
+        serde_json::Value::Number(n) if n.is_i64() => {
+            let v = n.as_i64().unwrap();
+            match arg_type.as_str() {
+                "tinyint" | "int1" => duckdb::types::Value::TinyInt(v as i8),
+                "smallint" | "int2" | "short" => duckdb::types::Value::SmallInt(v as i16),
+                "integer" | "int4" | "int" | "signed" => duckdb::types::Value::Int(v as i32),
+                "bigint" | "int8" | "long" => duckdb::types::Value::BigInt(v),
+                "hugeint" => duckdb::types::Value::HugeInt(v as i128),
+                "float" | "float4" | "real" => duckdb::types::Value::Float(v as f32),
+                "double" | "float8" => duckdb::types::Value::Double(v as f64),
+                _ => duckdb::types::Value::BigInt(v), // default fallback
+            }
+        }
+
+        serde_json::Value::Number(n) if n.is_u64() => {
+            let v = n.as_u64().unwrap();
+            match arg_type.as_str() {
+                "utinyint" => duckdb::types::Value::UTinyInt(v as u8),
+                "usmallint" => duckdb::types::Value::USmallInt(v as u16),
+                "uinteger" => duckdb::types::Value::UInt(v as u32),
+                "ubigint" | "uhugeint" => duckdb::types::Value::UBigInt(v),
+                _ => duckdb::types::Value::UBigInt(v), // default fallback
+            }
+        }
+
+        serde_json::Value::Number(n) if n.is_f64() => {
+            let v = n.as_f64().unwrap();
+            match arg_type.as_str() {
+                "float" | "float4" | "real" => duckdb::types::Value::Float(v as f32),
+                "double" | "float8" => duckdb::types::Value::Double(v),
+                "decimal" | "numeric" => {
+                    duckdb::types::Value::Decimal(Decimal::from_f64(v).ok_or_else(|| {
+                        Error::ExecutionErr("Could not convert f64 to Decimal".to_string())
+                    })?)
+                }
+                _ => duckdb::types::Value::Double(v), // default fallback
+            }
+        }
+
+        serde_json::Value::Array(arr) => duckdb::types::Value::Array(
+            arr.iter()
+                .map(|val| json_value_to_duckdb_value(val, arg_type.as_str(), client))
+                .collect::<Result<Vec<_>>>()?,
+        ),
+        serde_json::Value::Object(map) => duckdb::types::Value::Struct(
+            map.iter()
+                .map(|(k, v)| {
+                    Ok::<_, Error>((
+                        k.clone(),
+                        json_value_to_duckdb_value(v, arg_type.as_str(), client)?,
+                    ))
+                })
+                .collect::<Result<Vec<_>>>()?
+                .into(),
+        ),
+
+        value @ _ => {
+            return Err(Error::ExecutionErr(format!(
+                "Unsupported type in query: {:?} and signature {arg_type:?}",
+                value
+            )))
+        }
+    };
+    Ok(duckdb_value)
+}
+
+fn string_to_duckdb_timestamp(s: &str) -> Result<duckdb::types::Value> {
+    let ts = chrono::DateTime::parse_from_rfc3339(s)
+        .map_err(|e: chrono::ParseError| Error::ExecutionErr(e.to_string()))?;
+    Ok(duckdb::types::Value::Timestamp(
+        TimeUnit::Millisecond,
+        ts.timestamp_millis(),
+    ))
+}
+
+fn string_to_duckdb_date(s: &str) -> Result<duckdb::types::Value> {
+    use chrono::Datelike;
+    let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap();
+    Ok(duckdb::types::Value::Date32(date.num_days_from_ce()))
+}
+
+fn string_to_duckdb_time(s: &str) -> Result<duckdb::types::Value> {
+    use chrono::Timelike;
+    let time = chrono::NaiveTime::parse_from_str(s, "%H:%M:%S").unwrap();
+    Ok(duckdb::types::Value::Time64(
+        TimeUnit::Microsecond,
+        time.num_seconds_from_midnight() as i64,
+    ))
+}
+
+struct ParsedAttachDbResource<'a> {
+    resource_path: &'a str,
+    name: &'a str,
+    db_type: &'a str,
+    extra_args: Option<&'a str>,
+}
+fn parse_attach_db_resource<'a>(query: &'a str) -> Option<ParsedAttachDbResource<'a>> {
+    lazy_static::lazy_static! {
+        static ref RE: regex::Regex = regex::Regex::new(r"ATTACH '\$res:([^']+)' AS (\S+) \(TYPE (\w+)(.*)\)").unwrap();
+    }
+
+    for cap in RE.captures_iter(query) {
+        if let (Some(resource_path), Some(name), Some(db_type)) =
+            (cap.get(1), cap.get(2), cap.get(3))
+        {
+            let extra_args = cap.get(4).map(|m| query[m.start()..m.end()].trim());
+            return Some(ParsedAttachDbResource {
+                resource_path: query[resource_path.start()..resource_path.end()].trim(),
+                name: query[name.start()..name.end()].trim(),
+                db_type: query[db_type.start()..db_type.end()].trim(),
+                extra_args,
+            });
+        }
+    }
+    None
+}
+
+async fn transform_attach_db_resource_query(
+    parsed: &ParsedAttachDbResource<'_>,
+    job_id: &Uuid,
+    client: &AuthedClient,
+) -> Result<Vec<String>> {
+    match parsed.db_type.to_lowercase().as_str() {
+        "postgres" => {
+            let resource: PgDatabase = client
+                .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
+                .await?;
+
+            let attach_str = format!(
+                "ATTACH 'dbname={} {} host={} {} {}' AS {} (TYPE postgres{});",
+                resource.dbname,
+                resource
+                    .user
+                    .map(|u| format!("user={}", u))
+                    .unwrap_or_default(),
+                resource.host,
+                resource
+                    .password
+                    .map(|p| format!("password={}", p))
+                    .unwrap_or_default(),
+                resource
+                    .port
+                    .map(|p| format!("port={}", p))
+                    .unwrap_or_default(),
+                parsed.name,
+                parsed.extra_args.unwrap_or("")
+            );
+
+            Ok(vec![
+                "INSTALL postgres;".to_string(),
+                "LOAD postgres;".to_string(),
+                attach_str,
+            ])
+        }
+        "mysql" => {
+            #[cfg(not(feature = "mysql"))]
+            return Err(Error::ExecutionErr(
+                "MySQL feature is not enabled".to_string(),
+            ));
+
+            #[cfg(feature = "mysql")]
+            {
+                let resource: MysqlDatabase = client
+                    .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
+                    .await?;
+
+                let attach_str = format!(
+                    "ATTACH 'database={} host={} ssl_mode={} {} {} {}' AS {} (TYPE mysql{});",
+                    resource.database,
+                    resource.host,
+                    resource
+                        .ssl
+                        .map(|ssl| if ssl { "required" } else { "disabled" })
+                        .unwrap_or("preferred"),
+                    resource
+                        .password
+                        .map(|p| format!("password={}", p))
+                        .unwrap_or_default(),
+                    resource
+                        .port
+                        .map(|p| format!("port={}", p))
+                        .unwrap_or_default(),
+                    resource
+                        .user
+                        .map(|u| format!("user={}", u))
+                        .unwrap_or_default(),
+                    parsed.name,
+                    parsed.extra_args.unwrap_or("")
+                );
+
+                Ok(vec![
+                    "INSTALL mysql;".to_string(),
+                    "LOAD mysql;".to_string(),
+                    attach_str,
+                ])
+            }
+        }
+        "bigquery" => {
+            let resource: Value = client
+                .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
+                .await?;
+            // duckdb's bigquery extension requires a json file as credentials
+            let bq_credentials_path = make_bq_credentials_path(job_id);
+            env::set_var("GOOGLE_APPLICATION_CREDENTIALS", &bq_credentials_path);
+            tokio::fs::write(&bq_credentials_path, resource.to_string())
+                .await
+                .map_err(|e| {
+                    Error::ExecutionErr(format!(
+                        "Failed to write BigQuery credentials to {}: {}",
+                        &bq_credentials_path, e
+                    ))
+                })?;
+            let project_id: String = serde_json::from_value(
+                resource
+                    .get("project_id")
+                    .ok_or_else(|| {
+                        Error::ExecutionErr("BigQuery resource must contain project_id".to_string())
+                    })?
+                    .to_owned(),
+            )
+            .map_err(|_e| Error::ExecutionErr("failed project_id deserialize".to_string()))?;
+            let attach_str = format!(
+                "ATTACH 'project={}' as {} (TYPE bigquery{});",
+                project_id,
+                parsed.name,
+                parsed.extra_args.unwrap_or("")
+            )
+            .to_string();
+            Ok(vec![
+                "INSTALL bigquery FROM community;".to_string(),
+                "LOAD bigquery;".to_string(),
+                attach_str,
+            ])
+        }
+        _ => Err(Error::ExecutionErr(format!(
+            "Unsupported db type in DuckDB ATTACH: {}",
+            parsed.db_type
+        ))),
+    }
+}
+
+// Returns the transformed query and the set of storages used
+async fn transform_s3_uris(
+    query: &str,
+    client: &AuthedClient,
+) -> Result<(
+    Option<String>,
+    HashMap<Option<String>, DuckdbConnectionSettingsResponse>,
+)> {
+    let mut transformed_query = None;
+    lazy_static::lazy_static! {
+        static ref RE: regex::Regex = regex::Regex::new(r"'s3://([^'/]*)/([^']+)'").unwrap();
+    }
+    let mut used_storages = HashMap::new();
+    for cap in RE.captures_iter(query) {
+        if let (storage, Some(s3_path)) = (cap.get(1), cap.get(2)) {
+            let s3_path = s3_path.as_str();
+            let storage = match storage.map(|m| m.as_str()) {
+                Some("") | None => None,
+                Some(s) => Some(s.to_string()),
+            };
+            let original_str_lit =
+                format!("'s3://{}/{}'", storage.as_deref().unwrap_or(""), s3_path);
+            let duckdb_conn_settings = client
+                .get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 {
+                    s3_resource_path: None,
+                    storage: storage.clone(),
+                })
+                .await?;
+            let url = match &duckdb_conn_settings {
+                DuckdbConnectionSettingsResponse { s3_bucket: Some(bucket), .. } => {
+                    format!("'s3://{bucket}/{s3_path}'")
+                }
+                DuckdbConnectionSettingsResponse { azure_container_path: Some(base), .. } => {
+                    format!("'{base}/{s3_path}'")
+                }
+                _ => {
+                    return Err(Error::ExecutionErr(
+                        "DuckDB connection settings response must have either s3_bucket or azure_container_path".to_string(),
+                    ))?;
+                }
+            };
+            transformed_query = Some(
+                transformed_query
+                    .unwrap_or(query.to_string())
+                    .replace(&original_str_lit, &url),
+            );
+            used_storages.insert(storage, duckdb_conn_settings);
+        }
+    }
+    Ok((transformed_query, used_storages))
+}
+
+// BigQuery extension requires a json file as credentials
+// The file path is set as an env var by do_duckdb
+// It is created by transform_attach_db_resource_query (when bigquery is detected)
+// and deleted by do_duckdb after the query is executed
+fn make_bq_credentials_path(job_id: &Uuid) -> String {
+    format!("/tmp/service-account-credentials-{}.json", job_id)
+}
+
+// duckdb-rs does not support named parameters,
+// and it raises an error when passing unused arguments. We cannot prepare batch statements
+// but only single SQL statements so it doesn't work when all arguments are not used by
+// every single statement.
+fn interpolate_named_args<'a>(
+    query: &str,
+    args: &'a HashMap<String, duckdb::types::Value>,
+) -> (String, Vec<&'a duckdb::types::Value>) {
+    let mut query = query.to_string();
+
+    let mut values = vec![];
+    for (arg_name, arg_value) in args {
+        let pat = format!("${}", arg_name);
+        if !query.contains(&pat) {
+            continue;
+        }
+        values.push(arg_value);
+        query = query.replace(&pat, &format!("${}", values.len()));
+    }
+    (query, values)
+}
+
+fn trunc_sig(query: &str) -> &str {
+    let idx = query.rfind("-- $").unwrap_or(query.len());
+    // find next \n starting from idx and return everything after it
+    let idx = query[idx..].find('\n').map(|i| i + idx).unwrap_or(0);
+    &query[idx..]
+}

backend/windmill-worker/src/lib.rs

@@ -20,6 +20,8 @@ mod csharp_executor;
 #[cfg(feature = "enterprise")]
 mod dedicated_worker;
 mod deno_executor;
+#[cfg(feature = "duckdb")]
+mod duckdb_executor;
 mod global_cache;
 mod go_executor;
 mod graphql_executor;

backend/windmill-worker/src/mysql_executor.rs

@@ -32,13 +32,13 @@ use crate::{
 };
 
 #[derive(Deserialize)]
-struct MysqlDatabase {
-    host: String,
-    user: Option<String>,
-    password: Option<String>,
-    port: Option<u16>,
-    database: String,
-    ssl: Option<bool>,
+pub struct MysqlDatabase {
+    pub host: String,
+    pub user: Option<String>,
+    pub password: Option<String>,
+    pub port: Option<u16>,
+    pub database: String,
+    pub ssl: Option<bool>,
 }
 
 fn do_mysql_inner<'a>(

backend/windmill-worker/src/pg_executor.rs

@@ -47,14 +47,14 @@ use lazy_static::lazy_static;
 use urlencoding::encode;
 use windmill_common::client::AuthedClient;
 #[derive(Deserialize)]
-struct PgDatabase {
-    host: String,
-    user: Option<String>,
-    password: Option<String>,
-    port: Option<u16>,
-    sslmode: Option<String>,
-    dbname: String,
-    root_certificate_pem: Option<String>,
+pub struct PgDatabase {
+    pub host: String,
+    pub user: Option<String>,
+    pub password: Option<String>,
+    pub port: Option<u16>,
+    pub sslmode: Option<String>,
+    pub dbname: String,
+    pub root_certificate_pem: Option<String>,
 }
 
 lazy_static! {

backend/windmill-worker/src/rust_executor.rs

@@ -4,7 +4,11 @@ use uuid::Uuid;
 use windmill_parser_rust::parse_rust_deps_into_manifest;
 
 use itertools::Itertools;
-use tokio::{fs::File, io::AsyncReadExt, process::Command};
+use tokio::{
+    fs::{create_dir_all, File},
+    io::AsyncReadExt,
+    process::Command,
+};
 use windmill_common::{
     error::{self, Error},
     utils::calculate_hash,
@@ -19,8 +23,8 @@ use crate::{
         read_result, start_child_process, OccupancyMetrics,
     },
     handle_child::handle_child,
-    DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
-    RUST_CACHE_DIR, TZ_ENV,
+    DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUST_CACHE_DIR,
+    TZ_ENV,
 };
 use windmill_common::client::AuthedClient;
 
@@ -28,12 +32,18 @@ use windmill_common::client::AuthedClient;
 use crate::SYSTEM_ROOT;
 
 const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto");
+const NSJAIL_CONFIG_COMPILE_RUST_CONTENT: &str =
+    include_str!("../nsjail/download.rust.config.proto");
 
 lazy_static::lazy_static! {
     static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable");
     static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() });
     static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() });
     static ref CARGO_PATH: String = std::env::var("CARGO_PATH").unwrap_or_else(|_| format!("{}/bin/cargo", CARGO_HOME.as_str()));
+    // static ref CARGO_SWEEP_PATH: String = std::env::var("CARGO_SWEEP_PATH").unwrap_or_else(|_| format!("{}/bin/cargo-sweep", CARGO_HOME.as_str()));
+    static ref SWEEP_MAXSIZE: String = std::env::var("CARGO_SWEEP_MAXSIZE").unwrap_or("25GB".to_owned());
+    static ref NO_SHARED_BUILD_DIR: bool = std::env::var("RUST_NO_SHARED_BUILD_DIR").ok().map(|flag| flag == "true").unwrap_or(false);
+
 }
 
 #[cfg(windows)]
@@ -42,7 +52,20 @@ lazy_static::lazy_static! {
     static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR);
 }
 
-#[cfg(unix)]
+#[cfg(debug_assertions)]
+const DEV_CONF_NSJAIL: &'static str = r#"
+# Mount nix store for nixos to work properly
+mount {
+    src: "/nix/store"
+    dst: "/nix/store"
+    is_bind: true
+    mandatory: false
+}
+"#;
+
+#[cfg(not(debug_assertions))]
+const DEV_CONF_NSJAIL: &'static str = "";
+
 lazy_static::lazy_static! {
     static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", *HOME_DIR);
     static ref RUSTUP_HOME_DEFAULT: String = format!("{}/.rustup", *HOME_DIR);
@@ -176,54 +199,211 @@ pub async fn generate_cargo_lockfile(
     Ok(req_content)
 }
 
+async fn get_build_dir(
+    job: &MiniPulledJob,
+    job_dir: &str,
+    conn: &Connection,
+    worker_name: &str,
+    is_preview: bool,
+) -> anyhow::Result<String> {
+    let (bd, run_sweep) = job
+        .runnable_path
+        .as_ref()
+        .and_then(|p| {
+            if !is_preview || *NO_SHARED_BUILD_DIR {
+                None
+            } else {
+                if *DISABLE_NSJAIL {
+                    // If nsjail is disabled then entire worker has shared build directory
+                    // It drastically improves cache hit-rate.
+                    Some((format!("{RUST_CACHE_DIR}/build/{worker_name}"), true))
+                } else {
+                    // If nsjail is enabled, having global shared directory is vulnerability and target for an attack
+                    // Instead we either:
+                    // 1. Create different build directory for workspace script and user. Balanced caching while mainining high degree of security.
+                    // 2. If user is not known or something else goes wrong - use random build dir. This is equivalent to no cache at all.
+                    Some((
+                        format!(
+                            "{RUST_CACHE_DIR}/build/{}@{}@{}",
+                            &job.workspace_id,
+                            p.replace('/', "."),
+                            &job.created_by
+                        ),
+                        true,
+                    ))
+                }
+            }
+        })
+        .unwrap_or((format!("{RUST_CACHE_DIR}/build/{}", Uuid::new_v4()), false));
+
+    {
+        let (t, r, g) = (
+            create_dir_all(format!("{}/target", &bd)).await,
+            create_dir_all(format!("{}/registry", &bd)).await,
+            create_dir_all(format!("{}/git", &bd)).await,
+        );
+
+        t.and(r)
+            .and(g)
+            .map_err(|e| anyhow::anyhow!("Could not create build dir for rust.\ne: {e}"))?;
+    }
+
+    if run_sweep {
+        // Also run sweep to make sure target isn't using too much disk
+        let mut sweep_cmd = Command::new(CARGO_PATH.as_str());
+        sweep_cmd
+            .current_dir(job_dir)
+            .env_clear()
+            .env("PATH", PATH_ENV.as_str())
+            .env("CARGO_HOME", CARGO_HOME.as_str())
+            .env("HOME", HOME_ENV.as_str())
+            .env("CARGO_TARGET_DIR", &(bd.clone() + "/target"))
+            .env("RUSTUP_HOME", RUSTUP_HOME.as_str())
+            .args(["sweep", "--maxsize", SWEEP_MAXSIZE.as_str()])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped());
+
+        #[cfg(windows)]
+        {
+            sweep_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
+            sweep_cmd.env(
+                "TMP",
+                std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()),
+            );
+            sweep_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str());
+        }
+
+        let (job_id, conn, w_id, wk_name) = (
+            job.id.clone(),
+            conn.clone(),
+            job.workspace_id.clone(),
+            worker_name.to_owned(),
+        );
+
+        tokio::spawn(async move {
+            if let Err(e) = match start_child_process(sweep_cmd, CARGO_PATH.as_str()).await {
+                Ok(sweep_process) => {
+                    handle_child(
+                        &job_id,
+                        &conn,
+                        &mut 0,
+                        &mut None,
+                        sweep_process,
+                        false,
+                        &wk_name,
+                        &w_id,
+                        "cargo sweep",
+                        None,
+                        false,
+                        &mut None,
+                        None,
+                    )
+                    .await
+                }
+                Err(e) => Err(e),
+            } {
+                tracing::warn!(
+                    workspace_id = %w_id,
+                    job_id = %job_id,
+                    "Failed to run `cargo sweep`. Rust cache may grow over time, cargo sweep is meant to clean up unused cache.\ne: {e}\n"
+                );
+            }
+        });
+    }
+    Ok(bd)
+}
+
 pub async fn build_rust_crate(
-    job_id: &Uuid,
+    job: &MiniPulledJob,
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
     job_dir: &str,
     conn: &Connection,
     worker_name: &str,
-    w_id: &str,
     base_internal_url: &str,
     hash: &str,
     occupancy_metrics: &mut OccupancyMetrics,
+    is_preview: bool,
 ) -> error::Result<String> {
     let bin_path = format!("{}/{hash}", RUST_CACHE_DIR);
 
-    let mut build_rust_cmd = Command::new(CARGO_PATH.as_str());
-    build_rust_cmd
-        .current_dir(job_dir)
-        .env_clear()
-        .envs(PROXY_ENVS.clone())
-        .env("PATH", PATH_ENV.as_str())
-        .env("BASE_INTERNAL_URL", base_internal_url)
-        .env("HOME", HOME_ENV.as_str())
-        .env("CARGO_HOME", CARGO_HOME.as_str())
-        .env("RUSTUP_HOME", RUSTUP_HOME.as_str())
-        .args(vec!["build", "--release"])
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped());
+    let build_dir = get_build_dir(job, job_dir, conn, worker_name, is_preview).await?;
 
-    #[cfg(windows)]
-    {
-        build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
-        build_rust_cmd.env(
-            "TMP",
-            std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()),
-        );
-        build_rust_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str());
-    }
+    let child = if !*DISABLE_NSJAIL {
+        let _ = write_file(
+            job_dir,
+            "download.config.proto",
+            &NSJAIL_CONFIG_COMPILE_RUST_CONTENT
+                .replace("{JOB_DIR}", job_dir)
+                .replace("{CACHE_DIR}", RUST_CACHE_DIR)
+                .replace("{CARGO_HOME}", CARGO_HOME.as_str())
+                .replace("{DEV}", DEV_CONF_NSJAIL)
+                .replace("{BUILD}", &build_dir),
+        )?;
+        let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+        nsjail_cmd
+            .current_dir(job_dir)
+            .env_clear()
+            .env("PATH", PATH_ENV.as_str())
+            .env("TZ", TZ_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .envs(PROXY_ENVS.clone())
+            .env("HOME", HOME_ENV.as_str())
+            .env("CARGO_HOME", CARGO_HOME.as_str())
+            .env("RUSTUP_HOME", RUSTUP_HOME.as_str())
+            .env("CARGO_TARGET_DIR", &(build_dir.clone() + "/target"))
+            .args(vec![
+                "--config",
+                "download.config.proto",
+                "--",
+                CARGO_PATH.as_ref(),
+                "build",
+            ])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped());
+        if !is_preview {
+            nsjail_cmd.arg("--release");
+        }
+        start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
+    } else {
+        let mut build_rust_cmd = Command::new(CARGO_PATH.as_str());
+        build_rust_cmd
+            .current_dir(job_dir)
+            .env_clear()
+            .envs(PROXY_ENVS.clone())
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .env("HOME", HOME_ENV.as_str())
+            .env("CARGO_HOME", CARGO_HOME.as_str())
+            .env("RUSTUP_HOME", RUSTUP_HOME.as_str())
+            .env("CARGO_TARGET_DIR", &(build_dir.clone() + "/target"))
+            .args(vec!["build"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped());
 
-    let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?;
+        if !is_preview {
+            build_rust_cmd.arg("--release");
+        }
+        #[cfg(windows)]
+        {
+            build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
+            build_rust_cmd.env(
+                "TMP",
+                std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()),
+            );
+            build_rust_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str());
+        }
+        start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?
+    };
     handle_child(
-        job_id,
+        &job.id,
         conn,
         mem_peak,
         canceled_by,
-        build_rust_process,
+        child,
         false,
         worker_name,
-        w_id,
+        &job.workspace_id,
         "rust build",
         None,
         false,
@@ -231,16 +411,19 @@ pub async fn build_rust_crate(
         None,
     )
     .await?;
-    append_logs(job_id, w_id, "\n\n", conn).await;
+    append_logs(&job.id, &job.workspace_id, "\n\n", conn).await;
 
     tokio::fs::copy(
-        &format!("{job_dir}/target/release/main"),
+        &format!(
+            "{build_dir}/target/{}/main",
+            if is_preview { "debug" } else { "release" },
+        ),
         format! {"{job_dir}/main"},
     )
     .await
     .map_err(|e| {
         Error::ExecutionErr(format!(
-            "could not copy built binary from [...]/target/release/main to {job_dir}/main: {e:?}"
+            "could not copy built binary from [...]/target/.../main to {job_dir}/main: {e:?}"
         ))
     })?;
 
@@ -298,6 +481,9 @@ pub async fn handle_rust_job(
     let bin_path = format!("{}/{hash}", RUST_CACHE_DIR);
     let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}");
 
+    let reserved_variables =
+        get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
+
     let (cache, cache_logs) =
         windmill_common::worker::load_cache(&bin_path, &remote_path, false).await;
 
@@ -332,16 +518,16 @@ pub async fn handle_rust_job(
         create_args_and_out_file(client, job, job_dir, conn).await?;
 
         build_rust_crate(
-            &job.id,
+            &job,
             mem_peak,
             canceled_by,
             job_dir,
             conn,
             worker_name,
-            &job.workspace_id,
             base_internal_url,
             &hash,
             occupancy_metrics,
+            requirements_o.is_none(),
         )
         .await?
     };
@@ -349,9 +535,6 @@ pub async fn handle_rust_job(
     let logs2 = format!("{cache_logs}\n\n--- RUST CODE EXECUTION ---\n");
     append_logs(&job.id, &job.workspace_id, logs2, conn).await;
 
-    let reserved_variables =
-        get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
-
     let child = if !*DISABLE_NSJAIL {
         let _ = write_file(
             job_dir,
@@ -361,6 +544,7 @@ pub async fn handle_rust_job(
                 .replace("{CACHE_DIR}", RUST_CACHE_DIR)
                 .replace("{CACHE_HASH}", &hash)
                 .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
+                .replace("{DEV}", DEV_CONF_NSJAIL)
                 .replace("{SHARED_MOUNT}", shared_mount),
         )?;
         let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());

backend/windmill-worker/src/windmill-client.js

@@ -2998,6 +2998,8 @@ var $RawScript = {
         "mssql",
         "graphql",
         "nativets",
+        "duckdb",
+        // for related places search: ADD_NEW_LANG
       ],
     },
     path: {

backend/windmill-worker/src/worker.rs

@@ -145,6 +145,9 @@ use crate::ansible_executor::handle_ansible_job;
 #[cfg(feature = "mysql")]
 use crate::mysql_executor::do_mysql;
 
+#[cfg(feature = "duckdb")]
+use crate::duckdb_executor::do_duckdb;
+
 #[cfg(feature = "oracledb")]
 use crate::oracledb_executor::do_oracledb;
 
@@ -2614,6 +2617,30 @@ async fn handle_code_execution_job(
             )
             .await;
         }
+    } else if language == Some(ScriptLang::DuckDb) {
+        #[allow(unreachable_code)]
+        #[cfg(not(feature = "duckdb"))]
+        {
+            return Err(Error::internal_err(
+                "Duck DB requires the duckdb feature to be enabled".to_string(),
+            ));
+        }
+
+        #[cfg(feature = "duckdb")]
+        {
+            return do_duckdb(
+                job,
+                &client,
+                &code,
+                conn,
+                mem_peak,
+                canceled_by,
+                worker_name,
+                column_order,
+                occupancy_metrics,
+            )
+            .await;
+        }
     } else if language == Some(ScriptLang::Graphql) {
         return do_graphql(
             job,
@@ -3021,6 +3048,7 @@ fn parse_sig_of_lang(
             ScriptLang::Snowflake => Some(windmill_parser_sql::parse_snowflake_sig(code)?),
             ScriptLang::Graphql => None,
             ScriptLang::Mssql => Some(windmill_parser_sql::parse_mssql_sig(code)?),
+            ScriptLang::DuckDb => Some(windmill_parser_sql::parse_duckdb_sig(code)?),
             ScriptLang::OracleDB => Some(windmill_parser_sql::parse_oracledb_sig(code)?),
             #[cfg(feature = "php")]
             ScriptLang::Php => Some(windmill_parser_php::parse_php_signature(

benchmarks/lib.ts

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
-export const VERSION = "v1.492.1";
+export const VERSION = "v1.494.0";
 
 export async function login(email: string, password: string): Promise<string> {
   return await windmill.UserService.login({

cli/bootstrap/script_bootstrap.ts

@@ -76,6 +76,7 @@ func main() (interface{}, error) {
 
   bash: `echo "Hello world"
 `,
+  duckdb: `SELECT 'Hello world' AS message`,
 
   oracledb: `SELECT 'Hello world' AS message`,
   powershell: `Write-Output "Hello world"`,
@@ -126,5 +127,5 @@ public class Main {
   }
 }
 `,
-// for related places search: ADD_NEW_LANG 
+  // for related places search: ADD_NEW_LANG
 };

cli/build.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
+# Note for mac OS users: you need to install gnu-sed with `brew install gnu-sed` and use `gsed` instead of `sed`.
 ./gen_wm_client.sh
 deno run -A dnt.ts

cli/main.ts

@@ -63,7 +63,7 @@ export {
 //   }
 // });
 
-export const VERSION = "1.492.1";
+export const VERSION = "1.494.0";
 
 const command = new Command()
   .name("wmill")

cli/metadata.ts

@@ -528,6 +528,9 @@ export async function inferSchema(
       { name: "database", typ: { resource: "postgresql" } },
       ...inferedSchema.args,
     ];
+  } else if (language === "duckdb") {
+    const { parse_sql } = await import("./wasm/regex/windmill_parser_wasm.js");
+    inferedSchema = JSON.parse(parse_sql(content));
   } else if (language === "graphql") {
     const { parse_graphql } = await import(
       "./wasm/regex/windmill_parser_wasm.js"

cli/script.ts

@@ -547,6 +547,8 @@ export function filePathExtensionFromContentType(
     return ".my.sql";
   } else if (language === "bigquery") {
     return ".bq.sql";
+  } else if (language === "duckdb") {
+    return ".duckdb.sql";
   } else if (language === "oracledb") {
     return ".odb.sql";
   } else if (language === "snowflake") {
@@ -573,7 +575,7 @@ export function filePathExtensionFromContentType(
     return ".nu";
   } else if (language === "java") {
     return ".java";
-    // for related places search: ADD_NEW_LANG 
+    // for related places search: ADD_NEW_LANG
   } else {
     throw new Error("Invalid language: " + language);
   }
@@ -593,6 +595,7 @@ export const exts = [
   ".odb.sql",
   ".sf.sql",
   ".ms.sql",
+  ".duckdb.sql",
   ".sql",
   ".gql",
   ".ps1",
@@ -601,9 +604,8 @@ export const exts = [
   ".cs",
   ".nu",
   ".playbook.yml",
-  ".java"
-  // for related places search: ADD_NEW_LANG 
-
+  ".java",
+  // for related places search: ADD_NEW_LANG
 ];
 
 export function removeExtensionToPath(path: string): string {

cli/script_common.ts

@@ -9,6 +9,7 @@ export type ScriptLanguage =
   | "postgresql"
   | "mysql"
   | "bigquery"
+  | "duckdb"
   | "oracledb"
   | "snowflake"
   | "mssql"
@@ -19,7 +20,7 @@ export type ScriptLanguage =
   | "nu"
   | "ansible"
   | "java";
-	// for related places search: ADD_NEW_LANG 
+// for related places search: ADD_NEW_LANG
 
 export function inferContentTypeFromFilePath(
   contentPath: string,
@@ -43,6 +44,8 @@ export function inferContentTypeFromFilePath(
     return "bigquery";
   } else if (contentPath.endsWith(".odb.sql")) {
     return "oracledb";
+  } else if (contentPath.endsWith(".duckdb.sql")) {
+    return "duckdb";
   } else if (contentPath.endsWith(".sf.sql")) {
     return "snowflake";
   } else if (contentPath.endsWith(".ms.sql")) {
@@ -67,7 +70,7 @@ export function inferContentTypeFromFilePath(
     return "nu";
   } else if (contentPath.endsWith(".java")) {
     return "java";
-	// for related places search: ADD_NEW_LANG 
+    // for related places search: ADD_NEW_LANG
   } else {
     throw new Error(
       "Invalid language: " + contentPath.substring(contentPath.lastIndexOf("."))

cli/sync.ts

@@ -344,6 +344,7 @@ export function newPathAssigner(defaultTs: "bun" | "deno"): PathAssigner {
     else if (language == "nu") ext = "nu";
     else if (language == "ansible") ext = "playbook.yml";
     else if (language == "java") ext = "java";
+    else if (language == "duckdb") ext = "duckdb.sql";
     // for related places search: ADD_NEW_LANG
     else ext = "no_ext";
 

cli/wasm/regex/windmill_parser_wasm.js

@@ -1,5 +1,6 @@
 
 
+
 let WASM_VECTOR_LEN = 0;
 
 let cachedUint8ArrayMemory0 = null;

docker/DockerfileFull

@@ -3,6 +3,7 @@ FROM ghcr.io/windmill-labs/windmill:dev
 # Rust
 COPY --from=rust:1.86.0 /usr/local/cargo /usr/local/cargo
 COPY --from=rust:1.86.0 /usr/local/rustup /usr/local/rustup
+RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7
 
 # Ansible
 RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true

docker/DockerfileFullEe

@@ -22,6 +22,7 @@ FROM ghcr.io/windmill-labs/windmill-ee:dev
 # Rust
 COPY --from=rust:1.86.0 /usr/local/cargo /usr/local/cargo
 COPY --from=rust:1.86.0 /usr/local/rustup /usr/local/rustup
+RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7
 
 # Ansible
 RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true

docker/DockerfileNsjail

@@ -45,6 +45,7 @@ RUN apt-get update && apt-get install -y libprotobuf-dev libnl-route-3-dev
 # Rust
 COPY --from=rust:1.80.1 /usr/local/cargo /usr/local/cargo
 COPY --from=rust:1.80.1 /usr/local/rustup /usr/local/rustup
+RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7
 
 # Ansible
 RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true

flake.nix

@@ -81,11 +81,13 @@
           buildInputs = buildInputs ++ (with pkgs; [
             # Essentials
             rust
+            cargo-sweep
             git
             xcaddy
             sqlx-cli
             sccache
             nsjail
+            openapi-generator-cli
 
             # Python
             flock
@@ -186,6 +188,7 @@
           # for related places search: ADD_NEW_LANG
           FLOCK_PATH = "${pkgs.flock}/bin/flock";
           CARGO_PATH = "${rust}/bin/cargo";
+          CARGO_SWEEP_PATH = "${pkgs.cargo-sweep}/bin/cargo-sweep";
           DOTNET_PATH = "${pkgs.dotnet-sdk_9}/bin/dotnet";
           DOTNET_ROOT = "${pkgs.dotnet-sdk_9}/share/dotnet";
           ORACLE_LIB_DIR = "${pkgs.oracle-instantclient.lib}/lib";

frontend/CLAUDE.md

@@ -1,241 +0,0 @@
-# Svelte 5 Best Practices
-
-This guide outlines best practices for developing with Svelte 5, incorporating the new Runes API and other modern Svelte features. These rules MUST NOT be applied on svelte 4 files unless explicitly asked to do so.
-
-## Reactivity with Runes
-
-Svelte 5 introduces Runes for more explicit and flexible reactivity.
-
-1.  **Embrace Runes for State Management**:
-
-    - Use `$state` for reactive local component state.
-
-      ```svelte
-      <script>
-        let count = $state(0);
-
-        function increment() {
-          count += 1;
-        }
-      </script>
-
-      <button onclick={increment}>
-        Clicked {count} {count === 1 ? 'time' : 'times'}
-      </button>
-      ```
-
-    - Use `$derived` for computed values based on other reactive state.
-
-      ```svelte
-      <script>
-        let count = $state(0);
-        const doubled = $derived(count * 2);
-      </script>
-
-      <p>{count} * 2 = {doubled}</p>
-      ```
-
-    - Use `$effect` for side effects that need to run when reactive values change (e.g., logging, manual DOM manipulation, data fetching). Remember `$effect` does not run on the server.
-
-      ```svelte
-      <script>
-        let count = $state(0);
-
-        $effect(() => {
-          console.log('The count is now', count);
-          if (count > 5) {
-            alert('Count is too high!');
-          }
-        });
-      </script>
-      ```
-
-2.  **Props with `$props`**:
-
-    - Declare component props using `$props()`. This offers better clarity and flexibility compared to `export let`.
-
-      ```svelte
-      <script>
-        // ChildComponent.svelte
-        let { name, age = $state(30) } = $props();
-      </script>
-
-      <p>Name: {name}</p>
-      <p>Age: {age}</p>
-      ```
-
-    - For bindable props, use `$bindable`.
-
-      ```svelte
-      <script>
-        // MyInput.svelte
-        let { value = $bindable() } = $props();
-      </script>
-
-      <input bind:value />
-      ```
-
-## Event Handling
-
-- **Use direct event attributes**: Svelte 5 moves away from `on:` directives for DOM events.
-  - **Do**: `<button onclick={handleClick}>...</button>`
-  - **Don't**: `<button on:click={handleClick}>...</button>`
-- **For component events, prefer callback props**: Instead of `createEventDispatcher`, pass functions as props.
-
-  ```svelte
-  <!-- Parent.svelte -->
-  <script>
-    import Child from './Child.svelte';
-    let message = $state('');
-    function handleChildEvent(detail) {
-      message = detail;
-    }
-  </script>
-  <Child onCustomEvent={handleChildEvent} />
-  <p>Message from child: {message}</p>
-
-  <!-- Child.svelte -->
-  <script>
-    let { onCustomEvent } = $props();
-    function emitEvent() {
-      onCustomEvent('Hello from child!');
-    }
-  </script>
-  <button onclick={emitEvent}>Send Event</button>
-  ```
-
-## Snippets for Content Projection
-
-- **Use `{#snippet ...}` and `{@render ...}` instead of slots**: Snippets are more powerful and flexible.
-
-  ```svelte
-  <!-- Parent.svelte -->
-  <script>
-    import Card from './Card.svelte';
-  </script>
-
-  <Card>
-    {#snippet title()}
-      My Awesome Title
-    {/snippet}
-    {#snippet content()}
-      <p>Some interesting content here.</p>
-    {/snippet}
-  </Card>
-
-  <!-- Card.svelte -->
-  <script>
-    let { title, content } = $props();
-  </script>
-
-  <article>
-    <header>{@render title()}</header>
-    <div>{@render content()}</div>
-  </article>
-  ```
-
-- Default content is passed via the `children` prop (which is a snippet).
-  ```svelte
-  <!-- Wrapper.svelte -->
-  <script>
-    let { children } = $props();
-  </script>
-  <div>
-    {@render children?.()}
-  </div>
-  ```
-
-## Component Design
-
-1.  **Create Small, Reusable Components**: Break down complex UIs into smaller, focused components. Each component should have a single responsibility. This also aids performance by limiting the scope of reactivity updates.
-2.  **Descriptive Naming**: Use clear and descriptive names for variables, functions, and components.
-3.  **Minimize Logic in Components**: Move complex business logic to utility functions or services. Keep components focused on presentation and interaction.
-
-## State Management (Stores)
-
-1.  **Segment Stores**: Avoid a single global store. Create multiple stores, each responsible for a specific piece of global state (e.g., `userStore.js`, `themeStore.js`). This can help limit reactivity updates to only the parts of the UI that depend on specific state segments.
-2.  **Use Custom Stores for Complex Logic**: For stores with related methods, create custom stores.
-
-    ```javascript
-    // counterStore.js
-    import { writable } from 'svelte/store'
-
-    function createCounter() {
-    	const { subscribe, set, update } = writable(0)
-
-    	return {
-    		subscribe,
-    		increment: () => update((n) => n + 1),
-    		decrement: () => update((n) => n - 1),
-    		reset: () => set(0)
-    	}
-    }
-    export const counter = createCounter()
-    ```
-
-3.  **Use Context API for Localized State**: For state shared within a component subtree, consider Svelte's context API (`setContext`, `getContext`) instead of global stores when the state doesn't need to be truly global.
-
-## Performance Optimizations (Svelte 5)
-
-When generating Svelte 5 code, prioritize frontend performance by applying the following principles:
-
-### General Svelte 5 Principles
-
-- **Leverage the Compiler:** Trust Svelte's compiler to generate optimized JavaScript. Avoid manual DOM manipulation (`document.querySelector`, etc.) unless absolutely necessary for integrating third-party libraries that lack Svelte adapters.
-- **Keep Components Small and Focused:** Reinforcing from Component Design, smaller components lead to less complex reactivity graphs and more targeted, efficient updates.
-
-### Reactivity & State Management
-
-- **Optimize Computations with `$derived`:** Always use `$derived` for computed values that depend on other state. This ensures the computation only runs when its specific dependencies change, avoiding unnecessary work compared to recomputing derived values in `$effect` or less efficient methods.
-- **Minimize `$effect` Usage:** Use `$effect` sparingly and only for true side effects that interact with the outside world or non-Svelte state. Avoid putting complex logic or state updates _within_ an `$effect` unless those updates are explicitly intended as a reaction to external changes or non-Svelte state. Excessive or complex effects can impact rendering performance.
-- **Structure State for Fine-Grained Updates:** Design your `$state` objects or variables such that updates affect only the necessary parts of the UI. Avoid putting too much unrelated state into a single large object that gets frequently updated, as this can potentially trigger broader updates than necessary. Consider normalizing complex, nested state.
-
-### List Rendering (`{#each}`)
-
-- **Mandate `key` Attribute:** Always use a `key` attribute (`{#each items as item (item.id)}`) that refers to a unique, stable identifier for each item in a list. This is critical for allowing Svelte to efficiently update, reorder, add, or remove list items without destroying and re-creating unnecessary DOM elements and component instances.
-
-### Component Loading & Bundling
-
-- **Implement Lazy Loading/Code Splitting:** For routes, components, or modules that are not immediately needed on page load, use dynamic imports (`import(...)`) to split the code bundle. SvelteKit handles this automatically for routes, but it can be applied manually to components using helper patterns if needed.
-- **Be Mindful of Third-Party Libraries:** When incorporating external libraries, import only the necessary functions or components to minimize the final bundle size. Prefer libraries designed to be tree-shakeable.
-
-### Rendering & DOM
-
-- **Use CSS for Animations/Transitions:** Prefer CSS animations or transitions where possible for performance. Svelte's built-in `transition:` directive is also highly optimized and should be used for complex state-driven transitions, but simple cases can often use plain CSS.
-- **Optimize Image Loading:** Implement best practices for images: use optimized formats (WebP, AVIF), lazy loading (`loading="lazy"`), and responsive images (`<picture>`, `srcset`) to avoid loading unnecessarily large images.
-
-### Server-Side Rendering (SSR) & Hydration
-
-- **Ensure SSR Compatibility:** Write components that can be rendered on the server for faster initial page loads. Avoid relying on browser-specific APIs (like `window` or `document`) in the main `<script>` context. If necessary, use `$effect` or check `if (browser)` inside effects to run browser-specific code only on the client.
-- **Minimize Work During Hydration:** Structure components and data fetching such that minimal complex setup or computation is required when the client-side Svelte code takes over from the server-rendered HTML. Heavy synchronous work during hydration can block the main thread.
-
-## General Clean Code Practices
-
-1.  **Organized File Structure**: Group related files together. A common structure:
-    ```
-    /src
-    |-- /routes      // Page components (if using a router like SvelteKit)
-    |-- /lib         // Utility functions, services, constants (SvelteKit often uses this)
-    |   |-- /stores
-    |   |-- /utils
-    |   |-- /services
-    |   |-- /components  // Reusable UI components
-    |-- App.svelte
-    |-- main.js (or main.ts)
-    ```
-2.  **Scoped Styles**: Keep CSS scoped to components to avoid unintended side effects and improve maintainability. Avoid `:global` where possible.
-3.  **Immutability**: With Svelte 5 and `$state`, direct assignments to properties of `$state` objects (`obj.prop = value;`) are generally fine as Svelte's reactivity system handles updates. However, for non-rune state or when interacting with other systems, understanding and sometimes preferring immutable updates (creating new objects/arrays) can still be relevant.
-4.  **Use `class:` and `style:` directives**: For dynamic classes and styles, use Svelte's built-in directives for cleaner templates and potentially optimized updates.
-
-    ```svelte
-    <script>
-      let isActive = $state(true);
-      let color = $state('blue');
-    </script>
-
-    <div class:active={isActive} style:color={color}>
-      Hello
-    </div>
-    ```
-
-5.  **Stay Updated**: Keep Svelte and its related packages up to date to benefit from the latest features, performance improvements, and security fixes.

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "windmill-components",
-	"version": "1.492.1",
+	"version": "1.494.0",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "windmill-components",
-			"version": "1.492.1",
+			"version": "1.494.0",
 			"hasInstallScript": true,
 			"license": "AGPL-3.0",
 			"dependencies": {
@@ -74,7 +74,7 @@
 				"windmill-parser-wasm-nu": "^1.474.1",
 				"windmill-parser-wasm-php": "^1.429.0",
 				"windmill-parser-wasm-py": "^1.477.1",
-				"windmill-parser-wasm-regex": "^1.481.0",
+				"windmill-parser-wasm-regex": "^1.492.1",
 				"windmill-parser-wasm-rust": "^1.429.0",
 				"windmill-parser-wasm-ts": "^1.486.1",
 				"windmill-parser-wasm-yaml": "^1.429.0",
@@ -12838,9 +12838,9 @@
 			"integrity": "sha512-EY3mSMWpqFPzd7fsLg2/hSfQFU8HpW9nplFwm4JHHCDbcTpBzlvzjPJoHAAGO5kMzowAxjqi5ai/mXjeUWuiSg=="
 		},
 		"node_modules/windmill-parser-wasm-regex": {
-			"version": "1.481.0",
-			"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.481.0.tgz",
-			"integrity": "sha512-QY9NOaF0M55SBEbbcAkHoXppjdS2gb2NlwqhlZ1ciYWTtIrbZoL5K8xoG4FbB5vWRHxInG1aRdA2tb+JP180yA=="
+			"version": "1.492.1",
+			"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.492.1.tgz",
+			"integrity": "sha512-CBdjz3x00z2xA4BRZv8/hsuHcpZgL28UKTjYVSj5O9XMYlY9g0LNsEQ5TTl/oVyFBs62/CPyvWwGvW4oSAObzg=="
 		},
 		"node_modules/windmill-parser-wasm-rust": {
 			"version": "1.429.0",

frontend/package.json

@@ -1,11 +1,11 @@
 {
 	"name": "windmill-components",
-	"version": "1.492.1",
+	"version": "1.494.0",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
 		"preview": "vite preview",
-		"postinstall": "node scripts/untar_ui_builder.js",
+		"postinstall": "node scripts/untar_ui_builder.js && node scripts/patch_files.js",
 		"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --threshold warning",
 		"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
 		"lint": "prettier --ignore-path .gitignore --check --plugin-search-dir=. . && eslint --ignore-path .gitignore .",
@@ -143,7 +143,7 @@
 		"windmill-parser-wasm-nu": "^1.474.1",
 		"windmill-parser-wasm-php": "^1.429.0",
 		"windmill-parser-wasm-py": "^1.477.1",
-		"windmill-parser-wasm-regex": "^1.481.0",
+		"windmill-parser-wasm-regex": "^1.492.1",
 		"windmill-parser-wasm-rust": "^1.429.0",
 		"windmill-parser-wasm-ts": "^1.486.1",
 		"windmill-parser-wasm-yaml": "^1.429.0",

frontend/scripts/patch_files.js

@@ -0,0 +1,57 @@
+// patch-editor-worker.mjs
+import { readFile, writeFile } from 'fs/promises'
+import path from 'path'
+import { fileURLToPath } from 'url'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const filePath = path.join(
+	__dirname,
+	'..',
+	'node_modules',
+	'@codingame',
+	'monaco-vscode-api',
+	'workers',
+	'editor.worker.js'
+)
+
+const originalSnippet = `return requestHandler?.[propKey];`
+
+const patchedCode = `
+import '../vscode/src/vs/editor/common/services/editorWebWorkerMain.js';
+import { start } from '../vscode/src/vs/editor/editor.worker.start.js';
+
+function initialize(createFn) {
+    let requestHandler;
+    const foreignModule = new Proxy({}, {
+        get(_target, propKey) {
+            if (propKey === '$initialize') {
+                return async (data) => {
+                    if (!requestHandler) {
+                        requestHandler = createFn(context, data);
+                    }
+                };
+            }
+            const value = requestHandler?.[propKey]
+            if (typeof value === 'function') {
+              return value.bind(requestHandler);
+            }
+            return value;
+        }
+    });
+    const context = start(foreignModule);
+}
+
+export { initialize };
+`
+
+try {
+	const current = await readFile(filePath, 'utf8')
+	if (current.includes(originalSnippet)) {
+		await writeFile(filePath, patchedCode, 'utf8')
+		console.log('✅ editor.worker.js patched successfully.')
+	} else {
+		console.log('ℹ️ Patch already applied or file has changed.')
+	}
+} catch (err) {
+	console.error('❌ Failed to patch editor.worker.js:', err)
+}

frontend/src/lib/components/ArgInput.svelte

@@ -573,6 +573,30 @@
 								}}
 							/>
 						</div>
+					{:else if itemsType?.type == 'object' && itemsType?.resourceType == 's3object'}
+						<div class="w-full">
+							<FileUpload
+								{appPath}
+								computeForceViewerPolicies={computeS3ForceViewerPolicies}
+								{workspace}
+								allowMultiple={true}
+								randomFileKey={true}
+								on:addition={(evt) => {
+									value = [
+										...value,
+										{
+											s3: evt.detail?.path ?? '',
+											filename: evt.detail?.filename ?? ''
+										}
+									]
+								}}
+								on:deletion={(evt) => {
+									value = value.filter((v) => v.s3 !== evt.detail?.path)
+								}}
+								defaultValue={defaultValue?.map((v) => v.s3)}
+								initialValue={value}
+							/>
+						</div>
 					{:else}
 						<div class="w-full">
 							{#key redraw}

frontend/src/lib/components/ArrayTypeNarrowing.svelte

@@ -21,12 +21,24 @@
 		  }
 		| undefined
 
-	let selected: 'string' | 'number' | 'object' | 'bytes' | 'enum' | 'resource' | undefined =
+	let selected:
+		| 'string'
+		| 'number'
+		| 'object'
+		| 'bytes'
+		| 'enum'
+		| 'resource'
+		| 's3object'
+		| undefined =
 		itemsType?.type != 'string'
-			? itemsType?.type
+			? itemsType?.type == 'object' && itemsType?.resourceType == 's3object'
+				? 's3object'
+				: itemsType?.type
 			: Array.isArray(itemsType?.enum)
 				? 'enum'
-				: 'string'
+				: itemsType?.contentEncoding == 'base64'
+					? 'bytes'
+					: 'string'
 
 	let schema = {
 		properties: itemsType?.properties || {},
@@ -60,6 +72,8 @@
 					itemsType = { type: 'string', contentEncoding: 'base64' }
 				} else if (selected == 'resource') {
 					itemsType = { type: 'resource', resourceType: itemsType?.resourceType }
+				} else if (selected == 's3object') {
+					itemsType = { type: 'object', resourceType: 's3object' }
 				} else {
 					itemsType = undefined
 				}
@@ -69,6 +83,7 @@
 			<option value="string"> Items are strings</option>
 			<option value="enum">Items are strings from an enum</option>
 			{#if originalType != 'string[]'}
+				<option value="s3object">Items are S3 objects</option>
 				<option value="object"> Items are objects (JSON)</option>
 				<option value="resource"> Items are resources</option>
 				<option value="number">Items are numbers</option>

frontend/src/lib/components/Editor.svelte

@@ -115,7 +115,7 @@
 	import { workspaceStore } from '$lib/stores'
 	import { type Preview, ResourceService, UserService } from '$lib/gen'
 	import type { Text } from 'yjs'
-	import { initializeVscode } from '$lib/components/vscode'
+	import { initializeVscode, keepModelAroundToAvoidDisposalOfWorkers } from '$lib/components/vscode'
 
 	import { initializeMode } from 'monaco-graphql/esm/initializeMode.js'
 	import type { MonacoGraphQLAPI } from 'monaco-graphql/esm/api.js'
@@ -133,6 +133,7 @@
 	import EditorTheme from './EditorTheme.svelte'
 	import {
 		BIGQUERY_TYPES,
+		DUCKDB_TYPES,
 		MSSQL_TYPES,
 		MYSQL_TYPES,
 		ORACLEDB_TYPES,
@@ -343,7 +344,10 @@
 	}
 
 	export function setCode(ncode: string, noHistory: boolean = false): void {
-		code = ncode
+		if (code != ncode) {
+			code = ncode
+		}
+
 		if (noHistory) {
 			editor?.setValue(ncode)
 		} else {
@@ -363,6 +367,15 @@
 		}
 	}
 
+	function updateCode() {
+		const ncode = getCode()
+		if (code == ncode) {
+			return
+		}
+		code = ncode
+		dispatch('change', ncode)
+	}
+
 	export function append(code: string): void {
 		if (editor) {
 			const lineCount = editor.getModel()?.getLineCount() || 0
@@ -386,7 +399,7 @@
 
 	export async function format() {
 		if (editor) {
-			code = getCode()
+			updateCode()
 			if (lang != 'shell' && lang != 'nu') {
 				if ($formatOnSave != false) {
 					if (scriptLang == 'deno' && languageClients.length > 0) {
@@ -423,7 +436,7 @@
 						await editor?.getAction('editor.action.formatDocument')?.run()
 					}
 				}
-				code = getCode()
+				updateCode()
 			}
 			if (formatAction) {
 				formatAction()
@@ -482,7 +495,9 @@
 											? MSSQL_TYPES
 											: scriptLang === 'oracledb'
 												? ORACLEDB_TYPES
-												: []
+												: scriptLang === 'duckdb'
+													? DUCKDB_TYPES
+													: []
 					).map((t) => ({
 						label: t,
 						kind: languages.CompletionItemKind.Function,
@@ -1263,6 +1278,8 @@
 			folding
 		})
 
+		keepModelAroundToAvoidDisposalOfWorkers()
+
 		// updateEditorKeybindingsMode(editor, 'vim', undefined)
 
 		let ataModel: NodeJS.Timeout | undefined = undefined
@@ -1270,9 +1287,7 @@
 		editor?.onDidChangeModelContent((event) => {
 			timeoutModel && clearTimeout(timeoutModel)
 			timeoutModel = setTimeout(() => {
-				let ncode = getCode()
-				code = ncode
-				dispatch('change', ncode)
+				updateCode()
 			}, changeTimeout)
 
 			ataModel && clearTimeout(ataModel)
@@ -1291,12 +1306,12 @@
 			dispatch('focus')
 
 			editor?.addCommand(KeyMod.CtrlCmd | KeyCode.KeyS, function () {
-				code = getCode()
+				updateCode()
 				shouldBindKey && format && format()
 			})
 
 			editor?.addCommand(KeyMod.CtrlCmd | KeyCode.Enter, function () {
-				code = getCode()
+				updateCode()
 				shouldBindKey && cmdEnterAction && cmdEnterAction()
 			})
 
@@ -1352,6 +1367,7 @@
 			try {
 				closeWebsockets()
 				vimDisposable?.dispose()
+				console.log('disposing editor')
 				model?.dispose()
 				editor && editor.dispose()
 				console.log('disposed editor')

frontend/src/lib/components/FlowBuilder.svelte

@@ -8,7 +8,8 @@
 		type OpenFlow,
 		type InputTransform,
 		type TriggersCount,
-		CaptureService
+		CaptureService,
+		type HubScriptKind
 	} from '$lib/gen'
 	import { initHistory, redo, undo } from '$lib/history'
 	import {
@@ -553,6 +554,7 @@
 	})
 
 	export async function loadTriggers() {
+		if (initialPath == '') return
 		$triggersCount = await FlowService.getTriggersCountOfFlow({
 			workspace: $workspaceStore!,
 			path: initialPath

frontend/src/lib/components/HighlightCode.svelte

@@ -55,6 +55,8 @@
 				return sql
 			case 'oracledb':
 				return sql
+			case 'duckdb':
+				return sql
 			case 'powershell':
 				return powershell
 			case 'php':
@@ -66,10 +68,10 @@
 			case 'nu':
 				return python
 			case 'ansible':
-				return yaml;
+				return yaml
 			case 'java':
-				return java;
-			// for related places search: ADD_NEW_LANG 
+				return java
+			// for related places search: ADD_NEW_LANG
 			default:
 				return typescript
 		}

frontend/src/lib/components/SchedulePanel.svelte

@@ -33,9 +33,7 @@
 	bind:this={scheduleEditor}
 	hideTarget
 	allowDraft
-	hasDraft={!!selectedTrigger.draftConfig}
-	isDraftOnly={selectedTrigger.isDraft}
-	primary={selectedTrigger.isPrimary}
+	trigger={selectedTrigger}
 	draftSchema={schema}
 	{customLabel}
 	{...restProps}

frontend/src/lib/components/SimpleEditor.svelte

@@ -7,6 +7,16 @@
 	import '@codingame/monaco-vscode-standalone-css-language-features'
 	import '@codingame/monaco-vscode-standalone-typescript-language-features'
 	import '@codingame/monaco-vscode-standalone-html-language-features'
+	import {
+		editor as meditor,
+		KeyCode,
+		KeyMod,
+		Uri as mUri,
+		languages,
+		type IRange,
+		type IDisposable
+	} from 'monaco-editor'
+
 	languages.typescript.javascriptDefaults.setCompilerOptions({
 		target: languages.typescript.ScriptTarget.Latest,
 		allowNonTsExtensions: true,
@@ -14,12 +24,15 @@
 		noLib: true,
 		moduleResolution: languages.typescript.ModuleResolutionKind.NodeJs
 	})
-	languages.typescript.javascriptDefaults.setDiagnosticsOptions({
-		noSemanticValidation: false,
-		noSyntaxValidation: false,
-		noSuggestionDiagnostics: false,
-		diagnosticCodesToIgnore: [1108]
-	})
+	function setDiagnosticsOptions() {
+		languages.typescript.javascriptDefaults.setDiagnosticsOptions({
+			noSemanticValidation: false,
+			noSyntaxValidation: false,
+			noSuggestionDiagnostics: false,
+			diagnosticCodesToIgnore: [1108]
+		})
+	}
+	setDiagnosticsOptions()
 	languages.json.jsonDefaults.setDiagnosticsOptions({
 		validate: true,
 		allowComments: false,
@@ -44,15 +57,15 @@
 	import { BROWSER } from 'esm-env'
 
 	import { createHash, editorConfig, langToExt, updateOptions } from '$lib/editorUtils'
-	import {
-		editor as meditor,
-		KeyCode,
-		KeyMod,
-		Uri as mUri,
-		languages,
-		type IRange,
-		type IDisposable
-	} from 'monaco-editor'
+	// import {
+	// 	editor as meditor,
+	// 	KeyCode,
+	// 	KeyMod,
+	// 	Uri as mUri,
+	// 	languages,
+	// 	type IRange,
+	// 	type IDisposable
+	// } from 'monaco-editor'
 
 	import { allClasses } from './apps/editor/componentsPanel/cssUtils'
 
@@ -60,7 +73,7 @@
 
 	import libStdContent from '$lib/es6.d.ts.txt?raw'
 	import domContent from '$lib/dom.d.ts.txt?raw'
-	import { initializeVscode } from './vscode'
+	import { initializeVscode, keepModelAroundToAvoidDisposalOfWorkers } from './vscode'
 	import EditorTheme from './EditorTheme.svelte'
 	import { vimMode } from '$lib/stores'
 	import { initVim } from './monaco_keybindings'
@@ -144,10 +157,21 @@
 	}
 
 	export function setCode(ncode: string): void {
-		code = ncode
+		if (ncode != code) {
+			code = ncode
+		}
 		editor?.setValue(ncode)
 	}
 
+	function updateCode() {
+		const ncode = getCode()
+		if (code == ncode) {
+			return
+		}
+		code = ncode
+		dispatch('change', ncode)
+	}
+
 	function updatePlaceholderVisibility(value: string) {
 		if (!value) {
 			placeholderVisible = true
@@ -158,11 +182,11 @@
 
 	export function format() {
 		if (editor) {
-			code = getCode()
+			updateCode()
 			editor.getAction('editor.action.formatDocument')?.run()
 			if (formatAction) {
 				formatAction()
-				code = getCode()
+				updateCode()
 			}
 		}
 	}
@@ -348,14 +372,14 @@
 				snippetsPreventQuickSuggestions: disableSuggestions
 			}
 		})
+		keepModelAroundToAvoidDisposalOfWorkers()
 
 		let timeoutModel: NodeJS.Timeout | undefined = undefined
 		editor.onDidChangeModelContent((event) => {
 			suggestion = ''
 			timeoutModel && clearTimeout(timeoutModel)
 			timeoutModel = setTimeout(() => {
-				code = getCode()
-				dispatch('change', { code })
+				updateCode()
 			}, 200)
 		})
 
@@ -365,7 +389,7 @@
 			loadExtraLib()
 
 			editor.addCommand(KeyMod.CtrlCmd | KeyCode.KeyS, function () {
-				code = getCode()
+				updateCode()
 				shouldBindKey && format && format()
 			})
 
@@ -397,12 +421,12 @@
 		editor.onDidFocusEditorText(() => {
 			if (!editor) return
 			editor.addCommand(KeyMod.CtrlCmd | KeyCode.KeyS, function () {
-				code = getCode()
+				updateCode()
 				shouldBindKey && format && format()
 			})
 
 			editor.addCommand(KeyMod.CtrlCmd | KeyCode.Enter, function () {
-				code = getCode()
+				updateCode()
 				shouldBindKey && cmdEnterAction && cmdEnterAction()
 			})
 			dispatch('focus')
@@ -410,8 +434,7 @@
 
 		editor.onDidBlurEditorText(() => {
 			dispatch('blur')
-
-			code = getCode()
+			updateCode()
 		})
 
 		if (lang === 'css' && !cssClassesLoaded) {
@@ -511,6 +534,7 @@
 		})
 	}
 
+	let previousExtraLib = undefined
 	function loadExtraLib() {
 		if (lang == 'javascript') {
 			const stdLib = { content: libStdContent, filePath: 'es6.d.ts' }
@@ -524,6 +548,10 @@
 					content: extraLib,
 					filePath: 'windmill.d.ts'
 				})
+				if (previousExtraLib == extraLib) {
+					return
+				}
+				previousExtraLib = extraLib
 			}
 			languages.typescript.javascriptDefaults.setExtraLibs(libs)
 		}

frontend/src/lib/components/TemplateEditor.svelte

@@ -1,3 +1,21 @@
+<script module>
+	import '@codingame/monaco-vscode-standalone-typescript-language-features'
+
+	languages.typescript.javascriptDefaults.setCompilerOptions({
+		target: languages.typescript.ScriptTarget.Latest,
+		allowNonTsExtensions: true,
+		noSemanticValidation: false,
+		noLib: true,
+		moduleResolution: languages.typescript.ModuleResolutionKind.NodeJs
+	})
+	languages.typescript.javascriptDefaults.setDiagnosticsOptions({
+		noSemanticValidation: false,
+		noSyntaxValidation: false,
+		noSuggestionDiagnostics: false,
+		diagnosticCodesToIgnore: [1108]
+	})
+</script>
+
 <script lang="ts">
 	import { BROWSER } from 'esm-env'
 	import {
@@ -362,7 +380,6 @@
 		$componentControl[$selectedComponent[0]] = {
 			...$componentControl[$selectedComponent[0]],
 			setCode: (value: string) => {
-				code = value
 				setCode(value)
 			}
 		}
@@ -393,10 +410,10 @@
 	}
 
 	export function setCode(ncode: string): void {
-		code = ncode
-		if (editor) {
-			editor.setValue(ncode)
+		if (code != ncode) {
+			code = ncode
 		}
+		editor?.setValue(ncode)
 	}
 
 	let valueAfterDispose: string | undefined = undefined
@@ -422,20 +439,6 @@
 		await initializeVscode('templateEditor')
 		console.log('initialized')
 		initialized = true
-		languages.typescript.javascriptDefaults.setCompilerOptions({
-			target: languages.typescript.ScriptTarget.Latest,
-			allowNonTsExtensions: true,
-			noSemanticValidation: false,
-			noLib: true,
-			moduleResolution: languages.typescript.ModuleResolutionKind.NodeJs
-		})
-
-		languages.typescript.javascriptDefaults.setDiagnosticsOptions({
-			noSemanticValidation: false,
-			noSyntaxValidation: false,
-			noSuggestionDiagnostics: false,
-			diagnosticCodesToIgnore: [1108]
-		})
 
 		languages.register({ id: 'template' })
 
@@ -469,12 +472,20 @@
 			editor.addCommand(KeyMod.CtrlCmd | KeyMod.Shift | KeyCode.Digit7, function () {})
 		})
 
+		function updateCode() {
+			const ncode = getCode()
+			if (code == ncode) {
+				return
+			}
+			code = ncode
+			dispatch('change', code)
+		}
+
 		let timeoutModel: NodeJS.Timeout | undefined = undefined
 		editor.onDidChangeModelContent((event) => {
 			timeoutModel && clearTimeout(timeoutModel)
 			timeoutModel = setTimeout(() => {
-				code = getCode()
-				dispatch('change', { code })
+				updateCode()
 			}, 200)
 		})
 
@@ -500,7 +511,7 @@
 
 		editor.onDidBlurEditorText(() => {
 			dispatch('blur')
-			code = getCode()
+			updateCode()
 		})
 
 		jsLoader = setTimeout(async () => {

frontend/src/lib/components/apps/components/layout/AppTabs.svelte

@@ -151,10 +151,11 @@
 				)}
 				style={css?.tabRow?.style}
 			>
-				{#each tabs ?? [] as res}
+				{#each tabs ?? [] as res, index}
 					<button
 						on:pointerdown|stopPropagation
-						on:click={() => (selected = res)}
+						on:click={() => !resolvedDisabledTabs[index] && (selected = res)}
+						disabled={resolvedDisabledTabs[index]}
 						class={twMerge(
 							'rounded-sm !truncate text-sm  hover:text-primary px-1 py-2',
 							css?.allTabs?.class,
@@ -165,7 +166,8 @@
 										css?.selectedTab?.class,
 										'wm-tabs-selectedTab'
 									)
-								: ''
+								: '',
+							resolvedDisabledTabs[index] ? 'opacity-50 cursor-not-allowed hover:text-secondary' : ''
 						)}
 						style={selected == res
 							? [css?.allTabs?.style, css?.selectedTab?.style].filter(Boolean).join(';')
@@ -182,7 +184,8 @@
 					<div class="border-b">
 						<button
 							on:pointerdown|stopPropagation
-							on:click={() => (selected = res)}
+							on:click={() => !resolvedDisabledTabs[index] && (selected = res)}
+							disabled={resolvedDisabledTabs[index]}
 							class={twMerge(
 								'w-full text-left bg-surface !truncate text-sm hover:text-primary px-1 py-2',
 								css?.allTabs?.class,
@@ -193,7 +196,8 @@
 											css?.selectedTab?.class,
 											'wm-tabs-selectedTab'
 										)
-									: 'text-secondary'
+									: 'text-secondary',
+								resolvedDisabledTabs[index] ? 'opacity-50 cursor-not-allowed hover:text-secondary' : ''
 							)}
 						>
 							<span class="mr-2 w-8 font-mono">{selected == res ? '-' : '+'}</span>

frontend/src/lib/components/apps/editor/AppEditor.svelte

@@ -785,8 +785,7 @@
 	$: forceDeactivatePanzoom = isModifierKeyPressed && handMode
 </script>
 
-<svelte:head>
-</svelte:head>
+<svelte:head></svelte:head>
 
 <DarkModeObserver on:change={onThemeChange} />
 

frontend/src/lib/components/apps/editor/SettingsPanel.svelte

@@ -121,9 +121,15 @@
 						gridItemWithLocation?.location.type === 'subgrid' &&
 						Array.isArray($app.subgrids?.[gridItemWithLocation.location.subgridKey])
 					) {
-						$app.subgrids[gridItemWithLocation.location.subgridKey][
-							gridItemWithLocation.location.subgridItemIndex
-						] = cs.item
+						if (
+							$app.subgrids[gridItemWithLocation.location.subgridKey][
+								gridItemWithLocation.location.subgridItemIndex
+							]
+						) {
+							$app.subgrids[gridItemWithLocation.location.subgridKey][
+								gridItemWithLocation.location.subgridItemIndex
+							] = cs.item
+						}
 					}
 				}
 			}

frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte

@@ -1,4 +1,7 @@
 <script lang="ts">
+	import { createBubbler, stopPropagation } from 'svelte/legacy'
+
+	const bubble = createBubbler()
 	import Button from '$lib/components/common/button/Button.svelte'
 	import type { Preview } from '$lib/gen'
 	import { createEventDispatcher, getContext, onMount } from 'svelte'
@@ -22,17 +25,6 @@
 	import EditorSettings from '$lib/components/EditorSettings.svelte'
 	import { userStore } from '$lib/stores'
 
-	let inlineScriptEditorDrawer: InlineScriptEditorDrawer
-
-	export let inlineScript: InlineScript | undefined
-	export let name: string | undefined = undefined
-	export let id: string
-	export let defaultUserInput: boolean = false
-	export let fields: Record<string, AppInput> = {}
-	export let syncFields: boolean = false
-	export let transformer: boolean = false
-	export let componentType: string | undefined = undefined
-
 	const {
 		runnableComponents,
 		stateId,
@@ -42,10 +34,33 @@
 		app
 	} = getContext<AppViewerContext>('AppViewerContext')
 
-	export let editor: Editor | undefined = undefined
-	let diffEditor: DiffEditor
-	let simpleEditor: SimpleEditor
-	let validCode = true
+	interface Props {
+		inlineScript: InlineScript | undefined
+		name?: string | undefined
+		id: string
+		defaultUserInput?: boolean
+		fields?: Record<string, AppInput>
+		syncFields?: boolean
+		transformer?: boolean
+		componentType?: string | undefined
+		editor?: Editor | undefined
+	}
+
+	let {
+		inlineScript = $bindable(),
+		name = $bindable(undefined),
+		id,
+		defaultUserInput = false,
+		fields = $bindable({}),
+		syncFields = false,
+		transformer = false,
+		componentType = undefined,
+		editor = $bindable(undefined)
+	}: Props = $props()
+	let diffEditor: DiffEditor | undefined = $state()
+	let simpleEditor: SimpleEditor | undefined = $state()
+	let validCode = $state(true)
+	let inlineScriptEditorDrawer: InlineScriptEditorDrawer | undefined = $state()
 
 	async function inferInlineScriptSchema(
 		language: Preview['language'],
@@ -63,8 +78,6 @@
 		return schema
 	}
 
-	$: name && onNameChange()
-
 	function onNameChange() {
 		if (inlineScript) {
 			inlineScript.path = `${defaultIfEmptyString(
@@ -96,7 +109,7 @@
 	})
 
 	const dispatch = createEventDispatcher()
-	let runLoading = false
+	let runLoading = $state(false)
 
 	function preConnect(newFields) {
 		if (!componentType) {
@@ -186,16 +199,11 @@
 		}
 	}
 
-	$: extraLib =
-		inlineScript?.language == 'frontend' && worldStore
-			? buildExtraLib($worldStore?.outputsById ?? {}, id, $stateStore, true)
-			: undefined
-
 	// 	`
 	// /** The current's app state */
 	// const state: Record<string, any>;`
 
-	let drawerIsOpen: boolean | undefined = undefined
+	let drawerIsOpen: boolean | undefined = $state(undefined)
 
 	async function inferSuggestions(code: string) {
 		const outputs = await parseOutputs(code, true)
@@ -218,8 +226,24 @@
 			}
 		}
 	}
+
+	let lastName = $state(name)
+	$effect(() => {
+		if (name && name !== lastName) {
+			lastName = name
+			onNameChange()
+		}
+	})
+
+	let isFrontend = $derived(inlineScript?.language == 'frontend')
+	let extraLib = $derived(
+		isFrontend && worldStore
+			? buildExtraLib($worldStore?.outputsById ?? {}, id, $stateStore, true)
+			: undefined
+	)
 </script>
 
+<!-- <pre class="text-2xs">{JSON.stringify($worldStore?.outputsById, null, 2)}</pre> -->
 {#if inlineScript}
 	{#if inlineScript.language != 'frontend'}
 		<InlineScriptEditorDrawer
@@ -241,11 +265,11 @@
 				{#if !transformer}
 					<div class="flex flex-row gap-2 w-full items-center">
 						<input
-							on:keydown|stopPropagation
+							onkeydown={stopPropagation(bubble('keydown'))}
 							bind:value={name}
 							placeholder="Inline script name"
 							class="!text-xs !rounded-sm !shadow-none"
-							on:keyup={() => {
+							onkeyup={() => {
 								$app = $app
 								if (stateId) {
 									$stateId++

frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/EvalV2InputEditor.svelte

@@ -13,14 +13,27 @@
 	import { zIndexes } from '$lib/zIndexes'
 	import Popover from '$lib/components/Popover.svelte'
 
-	export let componentInput: EvalV2AppInput | undefined
-	export let id: string
-	export let field: string
-	export let fixedOverflowWidgets: boolean = true
-	export let acceptSelf: boolean = false
-	export let recomputeOnInputChanged = true
-	export let showOnDemandOnlyToggle = false
-	export let securedContext = false
+	interface Props {
+		componentInput: EvalV2AppInput | undefined
+		id: string
+		field: string
+		fixedOverflowWidgets?: boolean
+		acceptSelf?: boolean
+		recomputeOnInputChanged?: boolean
+		showOnDemandOnlyToggle?: boolean
+		securedContext?: boolean
+	}
+
+	let {
+		componentInput = $bindable(),
+		id,
+		field,
+		fixedOverflowWidgets = true,
+		acceptSelf = false,
+		recomputeOnInputChanged = true,
+		showOnDemandOnlyToggle = false,
+		securedContext = false
+	}: Props = $props()
 
 	const {
 		onchange,
@@ -30,15 +43,17 @@
 	} = getContext<AppViewerContext>('AppViewerContext')
 	const { evalPreview } = getContext<AppEditorContext>('AppEditorContext')
 
-	let editor: SimpleEditor
+	let editor: SimpleEditor | undefined = $state()
 	export function setCode(code: string) {
 		editor?.setCode(code)
 	}
 
-	$: extraLib =
-		componentInput?.expr && $worldStore
+	let exprDefined = $derived(componentInput?.expr != undefined)
+	let extraLib = $derived(
+		exprDefined && $worldStore
 			? buildExtraLib($worldStore?.outputsById ?? {}, acceptSelf ? '' : id, $stateStore, false)
 			: undefined
+	)
 
 	if (
 		componentInput &&
@@ -58,8 +73,8 @@
 		}
 	}
 
-	let fullscreen = false
-	let focus = false
+	let fullscreen = $state(false)
+	let focus = $state(false)
 </script>
 
 {#if componentInput?.type === 'evalv2'}
@@ -110,7 +125,14 @@
 				small
 				bind:this={editor}
 				lang="javascript"
-				bind:code={() => componentInput.expr ?? '', (e) => (componentInput.expr = e)}
+				bind:code={
+					() => componentInput.expr ?? '',
+					(e) => {
+						if (componentInput.expr != e) {
+							componentInput.expr = e
+						}
+					}
+				}
 				shouldBindKey={false}
 				{extraLib}
 				autoHeight
@@ -132,17 +154,17 @@
 				<div class="border bg-surface absolute top-0.5 right-8 p-0.5">
 					<Popover notClickable>
 						<Shield size={12} />
-						<svelte:fragment slot="text">
+						{#snippet text()}
 							This context variable is securely provided by the backend and cannot be altered by
 							users
-						</svelte:fragment>
+						{/snippet}
 					</Popover>
 				</div>
 			{/if}
 			<button
 				title="Open in drawer"
 				class="border bg-surface absolute hover:text-primary top-0 right-2 p-0.5 text-secondary"
-				on:click={() => (fullscreen = true)}><Maximize2 size={10} /></button
+				onclick={() => (fullscreen = true)}><Maximize2 size={10} /></button
 			>
 			{#if focus}
 				<div class="relative w-full">

frontend/src/lib/components/common/fileInput/FileInput.svelte

@@ -18,7 +18,7 @@
 	export let iconSize = 36
 	export let returnFileNames = false
 	export let submittedText: string | undefined = undefined
-	export let defaultFile: string | undefined = undefined
+	export let defaultFile: string | string[] | undefined = undefined
 	export let disabled: boolean | undefined = undefined
 	export let folderOnly = false
 
@@ -160,9 +160,8 @@
 		files = files
 		if (convertTo && files) {
 			const promises = files.map(convertFile)
-			let converted: ConvertedFile[] | { name: string; data: ConvertedFile }[] = await Promise.all(
-				promises
-			)
+			let converted: ConvertedFile[] | { name: string; data: ConvertedFile }[] =
+				await Promise.all(promises)
 			if (returnFileNames) {
 				converted = converted.map((c, i) => ({ name: files![i].name, data: c }))
 			}
@@ -239,7 +238,7 @@
 		{multiple}
 		{...$$restProps}
 	/>
-	{#if defaultFile}
+	{#if defaultFile && (!Array.isArray(defaultFile) || defaultFile.length > 0)}
 		<div class="w-full border-dashed border-t-2 text-2xs pt-1 text-tertiary mt-2">
 			Default file: <span class="text-blue-500">{defaultFile}</span>
 		</div>

frontend/src/lib/components/common/fileUpload/FileUpload.svelte

@@ -12,69 +12,111 @@
 	import { createEventDispatcher, onDestroy } from 'svelte'
 	import { emptyString } from '$lib/utils'
 
-	export let acceptedFileTypes: string[] | undefined = ['*']
-	export let allowMultiple: boolean = true
-	export let allowDelete = true
-	export let folderOnly = false
-	export let containerText: string = folderOnly
-		? 'Drag and drop a folder here or click to browse'
-		: allowMultiple
-			? 'Drag and drop files here or click to browse'
-			: 'Drag and drop a file here or click to browse'
-	export let customResourcePath: string | undefined = undefined
-	export let customResourceType: 's3' | 'azure_blob' | undefined = undefined // when customResourcePath is provided, this should be provided as well. Will default to S3 if not
-	export let customClass: string = ''
-	export let customStyle: string = ''
-	export let randomFileKey: boolean = false
-	export let pathTransformer: any = undefined // function taking as input {file: File} and returning a string
-	export let forceDisplayUploads: boolean = false
-	export let defaultValue: string | undefined = undefined
-	export let workspace: string | undefined = undefined
-	export let fileUploads: Writable<FileUploadData[]> = writable([])
-	export let appPath: string | undefined = undefined
-	export let disabled = false
-	export let iconSize: number | undefined = undefined
-	export let initialValue:
-		| {
-				s3: string
-				filename: string
-		  }
-		| undefined = undefined
-
-	init()
-
-	function init() {
-		if (initialValue?.s3) {
-			if (!$fileUploads.find((fileUpload) => fileUpload.path === initialValue?.s3)) {
-				$fileUploads = [
-					...$fileUploads,
-					{
-						name: initialValue.filename,
-						size: 0,
-						progress: 100,
-						path: initialValue?.s3
-					}
-				]
-			}
-		}
+	interface Props {
+		acceptedFileTypes?: string[] | undefined
+		allowMultiple?: boolean
+		allowDelete?: boolean
+		folderOnly?: boolean
+		containerText?: string
+		customResourcePath?: string | undefined
+		customResourceType?: 's3' | 'azure_blob' | undefined // when customResourcePath is provided, this should be provided as well. Will default to S3 if not
+		customClass?: string
+		customStyle?: string
+		randomFileKey?: boolean
+		pathTransformer?: any // function taking as input {file: File} and returning a string
+		forceDisplayUploads?: boolean
+		defaultValue?: string | string[] | undefined
+		workspace?: string | undefined
+		fileUploads?: Writable<FileUploadData[]>
+		appPath?: string | undefined
+		disabled?: boolean
+		iconSize?: number | undefined
+		initialValue?:
+			| {
+					s3: string
+					filename: string
+			  }
+			| {
+					s3: string
+					filename: string
+			  }[]
+			| undefined
+		computeForceViewerPolicies?:
+			| (() =>
+					| {
+							allowed_resources: string[]
+							allow_user_resources: boolean
+							allow_workspace_resource: boolean
+							file_key_regex: string
+					  }
+					| undefined)
+			| undefined
 	}
 
-	export let computeForceViewerPolicies:
-		| (() =>
-				| {
-						allowed_resources: string[]
-						allow_user_resources: boolean
-						allow_workspace_resource: boolean
-						file_key_regex: string
-				  }
-				| undefined)
-		| undefined = undefined
+	let {
+		acceptedFileTypes = ['*'],
+		allowMultiple = true,
+		allowDelete = true,
+		folderOnly = false,
+		containerText = folderOnly
+			? 'Drag and drop a folder here or click to browse'
+			: allowMultiple
+				? 'Drag and drop files here or click to browse'
+				: 'Drag and drop a file here or click to browse',
+		customResourcePath = undefined,
+		customResourceType = undefined,
+		customClass = '',
+		customStyle = '',
+		randomFileKey = false,
+		pathTransformer = undefined,
+		forceDisplayUploads = $bindable(false),
+		defaultValue = undefined,
+		workspace = undefined,
+		fileUploads = writable([]),
+		appPath = undefined,
+		disabled = false,
+		iconSize = undefined,
+		initialValue = undefined,
+		computeForceViewerPolicies = undefined
+	}: Props = $props()
 
 	const dispatch = createEventDispatcher<{
 		addition: { path?: string; filename?: string }
 		deletion: { path: string }
 	}>()
 
+	let initialS3 = $derived(
+		Array.isArray(initialValue)
+			? initialValue?.map((v) => v.s3)
+			: initialValue?.s3
+				? [initialValue?.s3]
+				: undefined
+	)
+	init()
+
+	function init() {
+		if (initialS3) {
+			function transform(initialValue: { s3: string; filename: string }) {
+				return {
+					name: initialValue.filename,
+					size: 0,
+					progress: 100,
+					path: initialValue?.s3
+				}
+			}
+			for (const s3 of initialS3) {
+				if (!$fileUploads.find((fileUpload) => fileUpload.path === s3)) {
+					let initialFileUploads = initialValue
+						? Array.isArray(initialValue)
+							? initialValue.map(transform)
+							: [transform(initialValue)]
+						: []
+					$fileUploads = [...$fileUploads, ...initialFileUploads]
+				}
+			}
+		}
+	}
+
 	type FileUploadData = {
 		name: string
 		size: number
@@ -554,10 +596,10 @@
 			{containerText}{#if disabled}<br />(Disabled){/if}
 		</FileInput>
 	{/if}
-	{#if initialValue?.s3 && $fileUploads.length == 0}
+	{#if initialS3 && initialS3.length > 0 && $fileUploads.length == 0}
 		<div class="flex flex-row gap-1 items-center p-1">
 			<span class="text-sm">
-				File currently selected: {initialValue?.s3}
+				File currently selected: {initialS3?.join(', ')}
 			</span>
 		</div>
 	{/if}

frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte

@@ -22,6 +22,7 @@
 	import CSharpIcon from '$lib/components/icons/CSharpIcon.svelte'
 	import NuIcon from '$lib/components/icons/NuIcon.svelte'
 	import JavaIcon from '$lib/components/icons/JavaIcon.svelte'
+	import DuckDbIcon from '$lib/components/icons/DuckDbIcon.svelte'
 
 	export let lang:
 		| SupportedLanguage
@@ -49,6 +50,7 @@
 		postgresql: 'Postgresql',
 		bigquery: 'BigQuery',
 		oracledb: 'Oracle Database',
+		duckdb: 'DuckDB',
 		snowflake: 'Snowflake',
 		mysql: 'MySQL',
 		mssql: 'MS SQL Server',
@@ -59,7 +61,7 @@
 		csharp: 'C#',
 		nu: 'Nu',
 		java: 'Java'
-		// for related places search: ADD_NEW_LANG 
+		// for related places search: ADD_NEW_LANG
 	}
 
 	const langToComponent: Record<
@@ -91,8 +93,9 @@
 		ansible: AnsibleIcon,
 		csharp: CSharpIcon,
 		nu: NuIcon,
-		java: JavaIcon
-		// for related places search: ADD_NEW_LANG 
+		java: JavaIcon,
+		duckdb: DuckDbIcon
+		// for related places search: ADD_NEW_LANG
 	}
 
 	let subIconScale = width === 30 ? 0.6 : 0.8

frontend/src/lib/components/copilot/autocomplete/monaco-adapter.ts

@@ -161,15 +161,15 @@ export class Autocompletor {
 	modifiedCode: string = ''
 	applyZone:
 		| {
-			startLineNumber: number
-			endLineNumber: number
-		}
+				startLineNumber: number
+				endLineNumber: number
+		  }
 		| undefined = undefined
 	lastChangePosition:
 		| {
-			lineNumber: number
-			column: number
-		}
+				lineNumber: number
+				column: number
+		  }
 		| undefined = undefined
 
 	abortController: AbortController | undefined = undefined
@@ -180,12 +180,12 @@ export class Autocompletor {
 
 	predictedChange:
 		| {
-			position: {
-				lineNumber: number
-				column: number
-			}
-			distance: number
-		}
+				position: {
+					lineNumber: number
+					column: number
+				}
+				distance: number
+		  }
 		| undefined = undefined
 	tabWidget: meditor.IContentWidget | undefined = undefined
 
@@ -240,18 +240,18 @@ export class Autocompletor {
 
 			let closestPosition:
 				| {
-					lineNumber: number
-					column: number
-				}
+						lineNumber: number
+						column: number
+				  }
 				| undefined = undefined
 			let closestDistance = Infinity
 			for (const change of this.visualChanges) {
 				if (change.type === 'deleted') {
 					const distance = Math.min(
 						Math.abs(change.range.startLine - position.lineNumber) +
-						Math.abs(change.range.startColumn - position.column) / 10000,
+							Math.abs(change.range.startColumn - position.column) / 10000,
 						Math.abs(change.range.endLine - position.lineNumber) +
-						Math.abs(change.range.endColumn - position.column) / 10000
+							Math.abs(change.range.endColumn - position.column) / 10000
 					)
 					if (distance < closestDistance) {
 						closestDistance = distance

frontend/src/lib/components/graph/FlowGraphV2.svelte

@@ -25,7 +25,7 @@
 	import ResultNode from './renderers/nodes/ResultNode.svelte'
 	import BaseEdge from './renderers/edges/BaseEdge.svelte'
 	import EmptyEdge from './renderers/edges/EmptyEdge.svelte'
-	import { sugiyama, dagStratify, decrossOpt, coordCenter } from 'd3-dag'
+	import { sugiyama, dagStratify, coordCenter, decrossTwoLayer, decrossOpt } from 'd3-dag'
 	import { Expand } from 'lucide-svelte'
 	import Toggle from '../Toggle.svelte'
 	import DataflowEdge from './renderers/edges/DataflowEdge.svelte'
@@ -143,12 +143,13 @@
 		let boxSize: any
 		try {
 			const layout = sugiyama()
-				.decross(decrossOpt())
+				.decross(nodes.length > 20 ? decrossTwoLayer() : decrossOpt())
 				.coord(coordCenter())
 				.nodeSize(() => [NODE.width + NODE.gap.horizontal, NODE.height + NODE.gap.vertical])
 			boxSize = layout(dag)
 		} catch {
 			const layout = sugiyama()
+				.decross(decrossTwoLayer())
 				.coord(coordCenter())
 				.nodeSize(() => [NODE.width + NODE.gap.horizontal, NODE.height + NODE.gap.vertical])
 			boxSize = layout(dag)

frontend/src/lib/components/graph/graphBuilder.ts

@@ -635,7 +635,7 @@ export function graphBuilder(
 		if (failureModule) {
 			let toAdd: Record<string, string> = {}
 			Object.keys(extra.flowModuleStates ?? {}).forEach((id) => {
-				if (id.startsWith('failure-')) {
+				if (id.startsWith('failure')) {
 					const failureState = extra.flowModuleStates?.[id] as GraphModuleState | undefined
 					if (failureState?.parent_module) {
 						toAdd[failureState.parent_module] = id

frontend/src/lib/components/icons/DuckDbIcon.svelte

@@ -0,0 +1,19 @@
+<script>
+	export let width = 16
+	export let height = 16
+</script>
+
+<svg {width} {height} viewBox="0 0 335 335" fill="none" xmlns="http://www.w3.org/2000/svg">
+	<path
+		d="M167.2 334.4C74.7 334.4 0 259.7 0 167.2C0 74.7 74.7 0 167.2 0C259.7 0 334.4 74.7 334.4 167.2C334.4 259.7 259.7 334.4 167.2 334.4Z"
+		fill="black"
+	/>
+	<path
+		d="M256.6 142.4H223.8V191.9H256.6C270.2 191.9 281.5 180.6 281.5 167C281.5 153.3 270.2 142.4 256.6 142.4Z"
+		fill="#FFF100"
+	/>
+	<path
+		d="M63.1 167.2C63.1 205.4 94.2 236.5 132.4 236.5C170.6 236.5 201.7 205.4 201.7 167.2C201.7 129 170.6 97.9 132.4 97.9C94.2 97.9 63.1 129 63.1 167.2Z"
+		fill="#FFF100"
+	/>
+</svg>

frontend/src/lib/components/propertyPicker/ObjectViewer.svelte

@@ -1,4 +1,6 @@
 <script lang="ts">
+	import ObjectViewer from './ObjectViewer.svelte'
+
 	import { copyToClipboard, truncate } from '$lib/utils'
 
 	import { createEventDispatcher } from 'svelte'
@@ -12,32 +14,45 @@
 	import AnimatedButton from '$lib/components/common/button/AnimatedButton.svelte'
 	import Popover from '../Popover.svelte'
 
-	export let json: any
-	export let level = 0
-	export let currentPath: string = ''
-	export let pureViewer = false
-	export let collapsed = (level != 0 && level % 3 == 0) || Array.isArray(json)
-	export let rawKey = false
-	export let topBrackets = false
-	export let allowCopy = true
-	export let collapseLevel: number | undefined = undefined
-	export let prefix = ''
-	export let expandedEvenOnLevel0: string | undefined = undefined
-	export let connecting = false
+	interface Props {
+		json: any
+		level?: number
+		currentPath?: string
+		pureViewer?: boolean
+		collapsed?: any
+		rawKey?: boolean
+		topBrackets?: boolean
+		allowCopy?: boolean
+		collapseLevel?: number | undefined
+		prefix?: string
+		expandedEvenOnLevel0?: string | undefined
+		connecting?: boolean
+	}
 
-	let jsonFiltered = json
+	let {
+		json,
+		level = 0,
+		currentPath = '',
+		pureViewer = false,
+		collapsed = $bindable((level != 0 && level % 3 == 0) || (Array.isArray(json) && level != 0)),
+		rawKey = false,
+		topBrackets = false,
+		allowCopy = true,
+		collapseLevel = undefined,
+		prefix = '',
+		expandedEvenOnLevel0 = undefined,
+		connecting = false
+	}: Props = $props()
 
-	$: jsonFiltered = onJsonChange(json)
+	let jsonFiltered = $state(json)
 
-	let search = ''
-	let searchOpen = false
+	let search = $state('')
+	let searchOpen = $state(false)
 
 	function onJsonChange(json: any) {
 		return searchOpen && search ? keepByKeyOrValue(json, search) : json
 	}
 
-	$: search != undefined && searchOpen && onSearch()
-
 	let searchTimeout: NodeJS.Timeout | undefined = undefined
 	function onSearch() {
 		if (searchTimeout) {
@@ -48,16 +63,10 @@
 		}, 100)
 	}
 
-	let s3FileViewer: S3FilePicker
-	let hoveredKey: string | null = null
+	let s3FileViewer: S3FilePicker | undefined = $state()
+	let hoveredKey: string | null = $state(null)
 
 	const collapsedSymbol = '...'
-	$: keys = ['object', 's3object'].includes(getTypeAsString(jsonFiltered))
-		? Object.keys(jsonFiltered)
-		: []
-	$: isArray = Array.isArray(jsonFiltered)
-	$: openBracket = isArray ? '[' : '{'
-	$: closeBracket = isArray ? ']' : '}'
 
 	export function getTypeAsString(arg: any): string {
 		if (arg === null) {
@@ -99,17 +108,65 @@
 		dispatch('select', fullKey)
 	}
 
-	$: keyLimit = isArray ? 5 : 100
-
-	$: fullyCollapsed = keys.length > 1 && collapsed
-
 	function clearSearch() {
 		searchOpen = false
 		jsonFiltered = json
 		search = ''
 	}
+	$effect(() => {
+		jsonFiltered = onJsonChange(json)
+	})
+	$effect(() => {
+		search != undefined && searchOpen && onSearch()
+	})
+	let keys = $derived(
+		['object', 's3object'].includes(getTypeAsString(jsonFiltered)) ? Object.keys(jsonFiltered) : []
+	)
+	let isArray = $derived(Array.isArray(jsonFiltered))
+	let openBracket = $derived(isArray ? '[' : '{')
+	let closeBracket = $derived(isArray ? ']' : '}')
+	let keyLimit = $derived(isArray ? 5 : 100)
+	let fullyCollapsed = $derived(keys.length > 1 && collapsed)
 </script>
 
+{#snippet renderScalar(k: string, v: any)}
+	<!-- svelte-ignore a11y_click_events_have_key_events -->
+	<!-- svelte-ignore a11y_no_static_element_interactions -->
+	<span
+		class="val inline text-left {pureViewer ? 'cursor-auto' : ''} rounded pl-0.5 {getTypeAsString(
+			v
+		)}"
+		onclick={() => {
+			selectProp(k, v, true)
+		}}
+		title={JSON.stringify(v)}
+	>
+		{#if v === NEVER_TESTED_THIS_FAR}
+			<span class="text-2xs text-tertiary font-normal"> Test the flow to see a value </span>
+		{:else if v == undefined}
+			<span class="text-2xs">undefined</span>
+		{:else if v == null}
+			<span class="text-2xs">null</span>
+		{:else if typeof v == 'string'}
+			<span class="text-2xs">"{truncate(v, 200)}"</span>
+		{:else if typeof v == 'number' && Number.isInteger(v) && !Number.isSafeInteger(v)}
+			<span class="inline-flex flex-row gap-1 items-center text-2xs">
+				{truncate(JSON.stringify(v), 200)}
+				<Popover>
+					<TriangleAlertIcon size={14} class="text-yellow-500 mb-0.5" />
+					{#snippet text()}
+						This number is too large for the frontend to handle correctly and may be rounded.
+					{/snippet}
+				</Popover>
+			</span>
+		{:else}
+			<span class="text-2xs">
+				{truncate(JSON.stringify(v), 200)}
+			</span>
+		{/if}
+	</span>
+{/snippet}
+
 <Portal name="object-viewer">
 	<S3FilePicker bind:this={s3FileViewer} readOnlyMode={true} />
 </Portal>
@@ -118,10 +175,11 @@
 		{#if searchOpen}
 			<div class="px-1 relative">
 				<input
-					on:keydown|stopPropagation={(event) => {
-						if (event?.key === 'Escape') {
+					onkeydown={(event) => {
+						if ((event as KeyboardEvent)?.key === 'Escape') {
 							clearSearch()
 						}
+						event.stopPropagation()
 					}}
 					type="text"
 					class="!h-6 !text-2xs mt-0.5"
@@ -130,7 +188,7 @@
 				/>
 				<button
 					class="absolute right-2 top-1 rounded-full hover:bg-surface-hover focus:bg-surface-hover text-secondary p-0.5"
-					on:click={() => {
+					onclick={() => {
 						clearSearch()
 					}}><X size={12} /></button
 				>
@@ -147,12 +205,13 @@
 		{/if}
 	</div>
 {/if}
+
 {#if keys.length > 0}
 	{#if !fullyCollapsed}
 		<span>
 			{#if level != 0 && keys.length > 1}
-				<!-- svelte-ignore a11y-click-events-have-key-events -->
-				<!-- svelte-ignore a11y-no-static-element-interactions -->
+				<!-- svelte-ignore a11y_click_events_have_key_events -->
+				<!-- svelte-ignore a11y_no_static_element_interactions -->
 
 				<Button
 					color="light"
@@ -198,7 +257,7 @@
 						<span class="text-2xs -ml-0.5 text-tertiary">:</span>
 
 						{#if getTypeAsString(jsonFiltered[key]) === 'object'}
-							<svelte:self
+							<ObjectViewer
 								{connecting}
 								json={jsonFiltered[key]}
 								level={level + 1}
@@ -212,50 +271,13 @@
 									: undefined}
 							/>
 						{:else}
-							<!-- svelte-ignore a11y-click-events-have-key-events -->
-							<!-- svelte-ignore a11y-no-static-element-interactions -->
-							<span
-								class="val inline text-left {pureViewer
-									? 'cursor-auto'
-									: ''} rounded pl-0.5 {getTypeAsString(jsonFiltered[key])}"
-								on:click={() => {
-									selectProp(key, jsonFiltered[key], true)
-								}}
-								title={JSON.stringify(jsonFiltered[key])}
-							>
-								{#if jsonFiltered[key] === NEVER_TESTED_THIS_FAR}
-									<span class="text-2xs text-tertiary font-normal">
-										Test the flow to see a value
-									</span>
-								{:else if jsonFiltered[key] == undefined}
-									<span class="text-2xs">undefined</span>
-								{:else if jsonFiltered[key] == null}
-									<span class="text-2xs">null</span>
-								{:else if typeof jsonFiltered[key] == 'string'}
-									<span class="text-2xs">"{truncate(jsonFiltered[key], 200)}"</span>
-								{:else if typeof jsonFiltered[key] == 'number' && Number.isInteger(jsonFiltered[key]) && !Number.isSafeInteger(jsonFiltered[key])}
-									<span class="inline-flex flex-row gap-1 items-center text-2xs">
-										{truncate(JSON.stringify(jsonFiltered[key]), 200)}
-										<Popover>
-											<TriangleAlertIcon size={14} class="text-yellow-500 mb-0.5" />
-											<svelte:fragment slot="text">
-												This number is too large for the frontend to handle correctly and may be
-												rounded.
-											</svelte:fragment>
-										</Popover>
-									</span>
-								{:else}
-									<span class="text-2xs">
-										{truncate(JSON.stringify(jsonFiltered[key]), 200)}
-									</span>
-								{/if}
-							</span>
+							{@render renderScalar(key, jsonFiltered[key])}
 						{/if}
 					</li>
 				{/each}
 				{#if keys.length > keyLimit}
 					{@const increment = Math.min(100, keys.length - keyLimit)}
-					<button on:click={() => (keyLimit += increment)} class="text-2xs px-2 text-secondary">
+					<button onclick={() => (keyLimit += increment)} class="text-2xs px-2 text-secondary">
 						{keyLimit}/{keys.length}: Load {increment} more...
 					</button>
 				{/if}
@@ -275,7 +297,7 @@
 						</a>
 						<button
 							class="text-secondary underline text-2xs whitespace-nowrap ml-1"
-							on:click={() => {
+							onclick={() => {
 								s3FileViewer?.open?.(jsonFiltered)
 							}}
 							><span class="flex items-center gap-1"><PanelRightOpen size={12} />open preview</span>
@@ -286,8 +308,8 @@
 		</span>
 	{/if}
 
-	<!-- svelte-ignore a11y-click-events-have-key-events -->
-	<!-- svelte-ignore a11y-no-static-element-interactions -->
+	<!-- svelte-ignore a11y_click_events_have_key_events -->
+	<!-- svelte-ignore a11y_no_static_element_interactions -->
 
 	{#if fullyCollapsed}
 		<span>
@@ -307,6 +329,8 @@
 	<span class="text-primary">{openBracket}{closeBracket}</span>
 {:else if jsonFiltered == undefined}
 	<span class="text-tertiary text-2xs ml-2">undefined</span>
+{:else if typeof jsonFiltered != 'object'}
+	{@render renderScalar('', jsonFiltered)}
 {:else}
 	<span class="text-tertiary text-2xs ml-2">No items</span>
 {/if}

frontend/src/lib/components/triggers/CaptureSection.svelte

@@ -307,7 +307,8 @@
 				<div class="mt-4 mb-2">
 					{#if displayAlert}
 						<Alert type="warning" title="Trigger deployed" size="xs" class="mb-4">
-							Capturing will suscribe to the trigger endpoint. Treat carefully.
+							Capturing on a deployed trigger can cause event loss on the deployed trigger. Treat
+							carefully.
 						</Alert>
 					{/if}
 					<Description>

frontend/src/lib/components/triggers/CaptureWrapper.svelte

@@ -6,7 +6,6 @@
 	import RouteCapture from './http/RouteCapture.svelte'
 	import type { ConnectionInfo } from '../common/alert/ConnectionIndicator.svelte'
 	import type { CaptureInfo } from './CaptureSection.svelte'
-	import { invalidRelations } from './postgres/utils'
 	import WebhooksCapture from './webhook/WebhooksCapture.svelte'
 	import EmailTriggerCaptures from '../details/EmailTriggerCaptures.svelte'
 	import WebsocketCapture from './websocket/WebsocketCapture.svelte'
@@ -53,21 +52,6 @@
 	const config: CaptureConfig | undefined = $derived(captureConfigs[captureType])
 
 	export async function setConfig(): Promise<boolean> {
-		if (captureType === 'postgres') {
-			if (!args?.publication?.table_to_track) {
-				sendUserToast('Table to track must be set', true)
-				return false
-			}
-
-			if (
-				invalidRelations(args.publication.table_to_track, {
-					showError: true,
-					trackSchemaTableError: true
-				}) !== ''
-			) {
-				return false
-			}
-		}
 		try {
 			await CaptureService.setCaptureConfig({
 				requestBody: {
@@ -224,7 +208,7 @@
 			<PostgresCapture
 				{captureInfo}
 				{captureLoading}
-				postgres_resource_path={args.postgres_resource_path}
+				{isValid}
 				{hasPreprocessor}
 				{isFlow}
 				{triggerDeployed}

frontend/src/lib/components/triggers/DeleteTriggerButton.svelte

@@ -0,0 +1,53 @@
+<script lang="ts">
+	import { Trash } from 'lucide-svelte'
+	import Button from '../common/button/Button.svelte'
+	import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte'
+	import { triggerIconMap, type Trigger } from './utils'
+	import TriggerLabel from './TriggerLabel.svelte'
+	import { twMerge } from 'tailwind-merge'
+
+	interface Props {
+		onDelete: (() => void) | undefined
+		trigger?: Trigger
+		small?: boolean
+	}
+
+	let { onDelete, trigger = undefined, small = false }: Props = $props()
+
+	let confirmationModalOpen = $state(false)
+</script>
+
+<ConfirmationModal
+	title="Are you sure you want to delete this draft trigger ?"
+	confirmationText="Delete"
+	open={confirmationModalOpen}
+	on:canceled={() => {
+		confirmationModalOpen = false
+	}}
+	on:confirmed={() => {
+		onDelete?.()
+		confirmationModalOpen = false
+	}}
+>
+	{#if trigger !== undefined}
+		{@const IconComponent = triggerIconMap[trigger.type]}
+		<div class="flex flex-row gap-2 items-center grow min-w-0 pr-2 rounded-md my-4">
+			<IconComponent
+				size={16}
+				class={twMerge(trigger.isDraft ? 'text-frost-400' : '', 'shrink-0')}
+			/>
+			<TriggerLabel {trigger} />
+		</div>
+	{/if}
+</ConfirmationModal>
+
+<Button
+	size="xs"
+	startIcon={{ icon: Trash }}
+	iconOnly
+	color={'light'}
+	on:click={() => {
+		confirmationModalOpen = true
+	}}
+	btnClasses={twMerge(small ? 'px-1 py-1' : '', 'bg-transparent hover:bg-red-500 hover:text-white')}
+/>

frontend/src/lib/components/triggers/TriggerEditorToolbar.svelte

@@ -1,13 +1,13 @@
 <script lang="ts">
 	import Button from '$lib/components/common/button/Button.svelte'
-	import { Trash, Save, RotateCcw } from 'lucide-svelte'
+	import { Save, RotateCcw } from 'lucide-svelte'
 	import { type Snippet } from 'svelte'
 	import Toggle from '$lib/components/Toggle.svelte'
 	import { Tooltip } from '../meltComponents'
+	import DeleteTriggerButton from './DeleteTriggerButton.svelte'
+	import type { Trigger } from './utils'
 
 	interface Props {
-		isDraftOnly: any
-		hasDraft: any
 		saveDisabled: any
 		enabled: boolean | undefined
 		allowDraft: any
@@ -21,12 +21,10 @@
 		onToggleEnabled?: (enabled: boolean) => void
 		onUpdate?: () => void
 		cloudDisabled?: boolean
-		triggerType?: string
+		trigger?: Trigger
 	}
 
 	let {
-		isDraftOnly,
-		hasDraft,
 		saveDisabled,
 		enabled,
 		allowDraft,
@@ -39,7 +37,8 @@
 		onReset,
 		onToggleEnabled,
 		onUpdate,
-		cloudDisabled = false
+		cloudDisabled = false,
+		trigger
 	}: Props = $props()
 
 	const canSave = $derived((permissions === 'write' && edit) || permissions === 'create')
@@ -73,7 +72,7 @@
 	{/if}
 {:else}
 	<div class="flex flex-row gap-2 items-center">
-		{#if !isDraftOnly && !hasDraft && enabled !== undefined}
+		{#if !trigger?.draftConfig && enabled !== undefined}
 			<div class="center-center">
 				<Toggle
 					size="2sm"
@@ -86,18 +85,9 @@
 				/>
 			</div>
 		{/if}
-		{#if isDraftOnly}
-			<Button
-				size="xs"
-				startIcon={{ icon: Trash }}
-				iconOnly
-				color={'light'}
-				on:click={() => {
-					onDelete?.()
-				}}
-				btnClasses="hover:bg-red-500 hover:text-white"
-			/>
-		{:else if hasDraft}
+		{#if trigger?.isDraft}
+			<DeleteTriggerButton {onDelete} {trigger} />
+		{:else if !trigger?.isDraft && trigger?.draftConfig}
 			<Button
 				size="xs"
 				startIcon={{ icon: RotateCcw }}
@@ -109,7 +99,7 @@
 				Reset changes
 			</Button>
 		{/if}
-		{#if canSave && (isDraftOnly || hasDraft)}
+		{#if canSave && trigger?.draftConfig}
 			<Tooltip placement="bottom-end" disablePopup={!saveDisabled && !cloudDisabled && isDeployed}>
 				<Button
 					size="xs"
@@ -120,7 +110,7 @@
 					}}
 					loading={isLoading}
 				>
-					{isDraftOnly ? 'Deploy' : 'Update'}
+					{trigger?.isDraft ? 'Deploy' : 'Update'}
 				</Button>
 				<span slot="text">
 					{#if !isDeployed}
@@ -128,7 +118,7 @@
 					{:else if cloudDisabled}
 						This trigger is disabled in the multi-tenant cloud
 					{:else}
-						Enter a valid config to {isDraftOnly ? 'deploy' : 'update'} the trigger
+						Enter a valid config to {trigger?.isDraft ? 'deploy' : 'update'} the trigger
 					{/if}
 				</span>
 			</Tooltip>

frontend/src/lib/components/triggers/TriggersTable.svelte

@@ -1,11 +1,12 @@
 <script lang="ts">
 	import DataTable from '$lib/components/table/DataTable.svelte'
 	import Button from '$lib/components/common/button/Button.svelte'
-	import { Plus, Star, Loader2, Trash, RotateCcw } from 'lucide-svelte'
+	import { Plus, Star, Loader2, RotateCcw } from 'lucide-svelte'
 	import { twMerge } from 'tailwind-merge'
 	import { triggerIconMap, type Trigger, type TriggerType } from './utils'
 	import AddTriggersButton from './AddTriggersButton.svelte'
 	import TriggerLabel from './TriggerLabel.svelte'
+	import DeleteTriggerButton from './DeleteTriggerButton.svelte'
 
 	interface Props {
 		// Props
@@ -94,14 +95,7 @@
 
 							{#if !['email', 'webhook', 'cli'].includes(trigger.type)}
 								{#if trigger.isDraft}
-									<Button
-										size="xs"
-										color="light"
-										btnClasses="transition-all duration-200 text-transparent hover:bg-surface group-hover:text-primary bg-transparent px-1 py-1"
-										startIcon={{ icon: Trash }}
-										iconOnly
-										on:click={() => onDeleteDraft?.(index)}
-									/>
+									<DeleteTriggerButton {trigger} onDelete={() => onDeleteDraft?.(index)} small />
 								{:else if !!trigger.draftConfig && !trigger.isDraft}
 									<Button
 										size="xs"

frontend/src/lib/components/triggers/TriggersWrapper.svelte

@@ -58,7 +58,7 @@
 		{isFlow}
 		path={initialPath || fakeInitialPath}
 		defaultValues={selectedTrigger.draftConfig ?? selectedTrigger.captureConfig ?? undefined}
-		customLabel={small ? customLabel : undefined}
+		{customLabel}
 		{...props}
 	/>
 {:else if selectedTrigger.type === 'webhook'}
@@ -86,7 +86,7 @@
 		{selectedTrigger}
 		defaultValues={selectedTrigger.draftConfig ?? selectedTrigger.captureConfig ?? undefined}
 		{schema}
-		customLabel={small ? customLabel : undefined}
+		{customLabel}
 		{...props}
 	/>
 {:else if selectedTrigger.type === 'websocket'}
@@ -95,7 +95,7 @@
 		path={initialPath || fakeInitialPath}
 		{selectedTrigger}
 		defaultValues={selectedTrigger.draftConfig ?? selectedTrigger.captureConfig ?? undefined}
-		customLabel={small ? customLabel : undefined}
+		{customLabel}
 		{...props}
 	/>
 {:else if selectedTrigger.type === 'kafka'}
@@ -104,7 +104,7 @@
 		path={initialPath || fakeInitialPath}
 		{selectedTrigger}
 		defaultValues={selectedTrigger.draftConfig ?? selectedTrigger.captureConfig ?? undefined}
-		customLabel={small ? customLabel : undefined}
+		{customLabel}
 		{...props}
 	/>
 {:else if selectedTrigger.type === 'postgres'}
@@ -113,7 +113,7 @@
 		path={initialPath || fakeInitialPath}
 		{selectedTrigger}
 		defaultValues={selectedTrigger.draftConfig ?? selectedTrigger.captureConfig ?? undefined}
-		customLabel={small ? customLabel : undefined}
+		{customLabel}
 		{...props}
 	/>
 {:else if selectedTrigger.type === 'nats'}
@@ -122,7 +122,7 @@
 		path={initialPath || fakeInitialPath}
 		{selectedTrigger}
 		defaultValues={selectedTrigger.draftConfig ?? selectedTrigger.captureConfig ?? undefined}
-		customLabel={small ? customLabel : undefined}
+		{customLabel}
 		{...props}
 	/>
 {:else if selectedTrigger.type === 'mqtt'}
@@ -131,7 +131,7 @@
 		path={initialPath || fakeInitialPath}
 		{selectedTrigger}
 		defaultValues={selectedTrigger.draftConfig ?? selectedTrigger.captureConfig ?? undefined}
-		customLabel={small ? customLabel : undefined}
+		{customLabel}
 		{...props}
 	/>
 {:else if selectedTrigger.type === 'sqs'}
@@ -140,7 +140,7 @@
 		path={initialPath || fakeInitialPath}
 		{selectedTrigger}
 		defaultValues={selectedTrigger.draftConfig ?? selectedTrigger.captureConfig ?? undefined}
-		customLabel={small ? customLabel : undefined}
+		{customLabel}
 		{...props}
 	/>
 {:else if selectedTrigger.type === 'gcp'}
@@ -149,7 +149,7 @@
 		path={initialPath || fakeInitialPath}
 		{selectedTrigger}
 		defaultValues={selectedTrigger.draftConfig ?? selectedTrigger.captureConfig ?? undefined}
-		customLabel={small ? customLabel : undefined}
+		{customLabel}
 		{...props}
 	/>
 {:else if selectedTrigger.type === 'poll'}

frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte

@@ -2,7 +2,7 @@
 	import Section from '$lib/components/Section.svelte'
 	import Required from '$lib/components/Required.svelte'
 	import ResourcePicker from '$lib/components/ResourcePicker.svelte'
-	import { emptyStringTrimmed } from '$lib/utils'
+	import { emptyStringTrimmed, sendUserToast } from '$lib/utils'
 	import TestTriggerConnection from '../TestTriggerConnection.svelte'
 	import Subsection from '$lib/components/Subsection.svelte'
 	import {
@@ -41,6 +41,8 @@
 	let topic_items: string[] = []
 	let subscription_items: string[] = []
 	let darkMode = false
+	let loadingTopic = false
+	let loadingSubscription = false
 
 	const DEFAULT_PUSH_CONFIG: PushConfig = {
 		audience: getBaseUrl(),
@@ -49,22 +51,34 @@
 
 	async function loadAllPubSubTopicsFromProject() {
 		if (!emptyStringTrimmed(gcp_resource_path)) {
-			topic_items = await GcpTriggerService.listGoogleTopics({
-				workspace: $workspaceStore!,
-				path: gcp_resource_path
-			})
+			try {
+				loadingTopic = true
+				topic_items = await GcpTriggerService.listGoogleTopics({
+					workspace: $workspaceStore!,
+					path: gcp_resource_path
+				})
+			} catch (error) {
+				sendUserToast(error.body, true)
+			}
+			loadingTopic = false
 		}
 	}
 
 	async function loadAllSubscriptionFromGooglePubSubTopic() {
 		if (!emptyStringTrimmed(gcp_resource_path) && !emptyStringTrimmed(topic_id)) {
-			subscription_items = await GcpTriggerService.listAllTgoogleTopicSubscriptions({
-				workspace: $workspaceStore!,
-				path: gcp_resource_path,
-				requestBody: {
-					topic_id
-				}
-			})
+			try {
+				loadingSubscription = true
+				subscription_items = await GcpTriggerService.listAllTgoogleTopicSubscriptions({
+					workspace: $workspaceStore!,
+					path: gcp_resource_path,
+					requestBody: {
+						topic_id
+					}
+				})
+			} catch (error) {
+				sendUserToast(error.body, true)
+			}
+			loadingSubscription = false
 		}
 	}
 
@@ -120,42 +134,45 @@
 				</div>
 			</Subsection>
 
-			<div class="flex flex-col gap-1">
-				<Subsection
-					label="Topic"
-					tooltip="Select the Pub/Sub topic that this subscription will be attached to. Messages published to this topic will be delivered to your subscription."
-				>
-					<div class="flex gap-1 mt-2">
-						<Select
-							class="grow shrink max-w-full"
-							on:change={(e) => {
-								topic_id = e.detail.value
-								loadAllSubscriptionFromGooglePubSubTopic()
-							}}
-							on:clear={() => {
-								topic_id = ''
-							}}
-							value={topic_id}
-							items={topic_items}
-							placeholder="Choose a topic"
-							inputStyles={SELECT_INPUT_DEFAULT_STYLE.inputStyles}
-							containerStyles={darkMode
-								? SELECT_INPUT_DEFAULT_STYLE.containerStylesDark
-								: SELECT_INPUT_DEFAULT_STYLE.containerStyles}
-							portal={false}
-						/>
-						<Button
-							disabled={!can_write}
-							variant="border"
-							color="light"
-							wrapperClasses="self-stretch"
-							on:click={loadAllPubSubTopicsFromProject}
-							startIcon={{ icon: RefreshCw }}
-							iconOnly
-						/>
-					</div>
-				</Subsection>
-			</div>
+			{#if gcp_resource_path}
+				<div class="flex flex-col gap-1">
+					<Subsection
+						label="Topic"
+						tooltip="Select the Pub/Sub topic that this subscription will be attached to. Messages published to this topic will be delivered to your subscription."
+					>
+						<div class="flex flex-row gap-1 mt-2">
+							<Select
+								loading={loadingTopic}
+								class="grow shrink max-w-full"
+								on:change={(e) => {
+									topic_id = e.detail.value
+									loadAllSubscriptionFromGooglePubSubTopic()
+								}}
+								on:clear={() => {
+									topic_id = ''
+								}}
+								value={topic_id}
+								items={topic_items}
+								placeholder="Choose a topic"
+								inputStyles={SELECT_INPUT_DEFAULT_STYLE.inputStyles}
+								containerStyles={darkMode
+									? SELECT_INPUT_DEFAULT_STYLE.containerStylesDark
+									: SELECT_INPUT_DEFAULT_STYLE.containerStyles}
+								portal={false}
+							/>
+							<Button
+								disabled={!can_write}
+								variant="border"
+								color="light"
+								wrapperClasses="self-stretch"
+								on:click={loadAllPubSubTopicsFromProject}
+								startIcon={{ icon: RefreshCw }}
+								iconOnly
+							/>
+						</div>
+					</Subsection>
+				</div>
+			{/if}
 			{#if !emptyStringTrimmed(gcp_resource_path) && !emptyStringTrimmed(topic_id)}
 				<Section
 					label="Subscription"
@@ -264,6 +281,7 @@
 							<div class="flex flex-col gap-3">
 								<div class="flex gap-1">
 									<Select
+										loading={loadingSubscription}
 										class="grow shrink max-w-full"
 										on:change={(e) => {
 											subscription_id = e.detail.value

frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte

@@ -21,7 +21,7 @@
 	import type { Snippet } from 'svelte'
 	import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte'
 	import { saveGcpTriggerFromCfg } from './utils'
-	import { handleConfigChange } from '../utils'
+	import { handleConfigChange, type Trigger } from '../utils'
 
 	let drawer: Drawer | undefined = $state(undefined)
 	let is_flow: boolean = $state(false)
@@ -55,8 +55,7 @@
 		hideTooltips = false,
 		isEditor = false,
 		allowDraft = false,
-		hasDraft = false,
-		isDraftOnly = false,
+		trigger = undefined,
 		isDeployed = false,
 		customLabel = undefined,
 		onConfigChange = undefined,
@@ -72,8 +71,7 @@
 		hideTooltips?: boolean
 		isEditor?: boolean
 		allowDraft?: boolean
-		hasDraft?: boolean
-		isDraftOnly?: boolean
+		trigger?: Trigger
 		isDeployed?: boolean
 		customLabel?: Snippet
 		onConfigChange?: (cfg: Record<string, any>, saveDisabled: boolean, updated: boolean) => void
@@ -225,7 +223,7 @@
 
 	async function handleToggleEnabled(toggleEnabled: boolean) {
 		enabled = toggleEnabled
-		if (!isDraftOnly && !hasDraft) {
+		if (!trigger?.draftConfig) {
 			await GcpTriggerService.setGcpTriggerEnabled({
 				path: initialPath,
 				workspace: $workspaceStore ?? '',
@@ -279,8 +277,6 @@
 {#snippet actionsButtons()}
 	{#if !drawerLoading && can_write}
 		<TriggerEditorToolbar
-			{isDraftOnly}
-			{hasDraft}
 			permissions={drawerLoading || !can_write ? 'none' : 'create'}
 			{saveDisabled}
 			{enabled}
@@ -293,6 +289,7 @@
 			{onDelete}
 			onToggleEnabled={handleToggleEnabled}
 			{cloudDisabled}
+			{trigger}
 		/>
 	{/if}
 {/snippet}