starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-25 08:00:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into rf/migrateFromDepot

Browse Source
This commit is contained in:
Ruben Fiszel
2025-03-11 09:57:42 +01:00
committed by GitHub
parent a0104262fa 11ed81a6c7
commit 89f0bee990
440 changed files with 17069 additions and 8154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/backend-check.yml
+4 -4
View File
@@ -16,7 +16,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.83.0
toolchain: 1.85.0
- uses: Swatinem/rust-cache@v2
with:
workspaces: backend
@@ -40,7 +40,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.83.0
toolchain: 1.85.0
- uses: Swatinem/rust-cache@v2
with:
workspaces: backend
@@ -77,7 +77,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.83.0
toolchain: 1.85.0
- uses: Swatinem/rust-cache@v2
with:
workspaces: backend
@@ -117,7 +117,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.83.0
toolchain: 1.85.0
- uses: Swatinem/rust-cache@v2
with:
workspaces: backend
.github/workflows/backend-test.yml
+1 -1
View File
@@ -51,7 +51,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.83.0
toolchain: 1.85.0
- uses: Swatinem/rust-cache@v2
with:
workspaces: backend
.github/workflows/build-publish-rh-image.yml
+2 -2
View File
@@ -64,7 +64,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
@@ -81,7 +81,7 @@ jobs:
platforms: linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
.github/workflows/build_windows_worker_.yml
+7 -1
View File
@@ -30,6 +30,12 @@ jobs:
token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
fetch-depth: 0
- name: Setup Rust
uses: actions-rs/toolchain@v1
with:
toolchain: 1.85.0
override: true
- name: Substitute EE code
shell: bash
run: |
@@ -45,7 +51,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
- name: Rename binary with corresponding architecture
run: |
.github/workflows/docker-image-rpi4.yml
+1 -1
View File
@@ -67,7 +67,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,deno_core,license,http_trigger,zip,oauth2,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
features=embedding,parquet,openidconnect,deno_core,license,http_trigger,zip,oauth2,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
${{ steps.meta-public.outputs.tags }}
.github/workflows/docker-image.yml
+1 -1
View File
@@ -200,7 +200,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
${{ steps.meta-ee-public.outputs.tags }}
.github/workflows/publish_windows_worker.yml
+7 -1
View File
@@ -32,6 +32,12 @@ jobs:
token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
fetch-depth: 0
- name: Setup Rust
uses: actions-rs/toolchain@v1
with:
toolchain: 1.85.0
override: true
- name: Substitute EE code
shell: bash
run: |
@@ -47,7 +53,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
- name: Rename binary with corresponding architecture
run: |
CHANGELOG.md
+152
View File
@@ -1,5 +1,157 @@
# Changelog
## [1.475.0](https://github.com/windmill-labs/windmill/compare/v1.474.0...v1.475.0) (2025-03-06)
### Features
* **backend:** option to invalidate all sessions on logout ([#5419](https://github.com/windmill-labs/windmill/issues/5419)) ([e9044f0](https://github.com/windmill-labs/windmill/commit/e9044f0b9b1647e0fc74e5e3cce39a4fc2718672))
* deploy triggers to prod/staging workspace ([#5429](https://github.com/windmill-labs/windmill/issues/5429)) ([b210ae3](https://github.com/windmill-labs/windmill/commit/b210ae36f7c1fd8860241aa5908cefc0ac956b7b))
* **frontend:** improve flow suspend status display ([#5425](https://github.com/windmill-labs/windmill/issues/5425)) ([a845733](https://github.com/windmill-labs/windmill/commit/a8457337cec870e9e3d6a053f4fbe89f58229401))
* **frontend:** pick image from workspace storage bucket ([#5382](https://github.com/windmill-labs/windmill/issues/5382)) ([8dbe0fa](https://github.com/windmill-labs/windmill/commit/8dbe0fa6446a34bd60484f1b5ac828ff9f892735))
* kafka mTLS ([#5449](https://github.com/windmill-labs/windmill/issues/5449)) ([371c892](https://github.com/windmill-labs/windmill/commit/371c892f9aa43fe2757b4d23ee076781b237f11f))
* MQTT triggers ([#5277](https://github.com/windmill-labs/windmill/issues/5277)) ([5c39037](https://github.com/windmill-labs/windmill/commit/5c39037aea35f9bdf780f1946abbb384533ee547))
### Bug Fixes
* **cli:** fix wmill user create-token with email and password ([a16cab0](https://github.com/windmill-labs/windmill/commit/a16cab0923f3be46f181dffc209a10c711873f86))
* **frontend:** fix many s3 file picker bugs ([#5428](https://github.com/windmill-labs/windmill/issues/5428)) ([4fabc2a](https://github.com/windmill-labs/windmill/commit/4fabc2a8256b5088b1af61baf81355cd556d23e2))
* **frontend:** improve capture payload preview ([#5417](https://github.com/windmill-labs/windmill/issues/5417)) ([fd56a63](https://github.com/windmill-labs/windmill/commit/fd56a639d21366bea5129409d805feefb22659c1))
* improve objectviewer performance ([2444f4f](https://github.com/windmill-labs/windmill/commit/2444f4f23e5212fa4d2ff8542d864118d4b4feb9))
* s3 file picker delete + better s3 path handling ([#5454](https://github.com/windmill-labs/windmill/issues/5454)) ([ae618c7](https://github.com/windmill-labs/windmill/commit/ae618c79dff08e8e9fe1f4ce69665ad7faeac006))
## [1.474.0](https://github.com/windmill-labs/windmill/compare/v1.473.1...v1.474.0) (2025-03-04)
### Features
* add template script for all triggers ([#5424](https://github.com/windmill-labs/windmill/issues/5424)) ([0a9d8c6](https://github.com/windmill-labs/windmill/commit/0a9d8c6b8b95c0e1093aef89b2a1956dbe59bbee))
* **frontend:** global recompute helper function ([#5408](https://github.com/windmill-labs/windmill/issues/5408)) ([b961efa](https://github.com/windmill-labs/windmill/commit/b961efa8691f806d437ba2cb303ad1a50fa618c4))
* more controls on setting token duration ([#5421](https://github.com/windmill-labs/windmill/issues/5421)) ([534a824](https://github.com/windmill-labs/windmill/commit/534a8249d60f7de6b542c3b5b5ee0b5eda360f22))
### Bug Fixes
* do not depend on public schema anymore ([90b00f5](https://github.com/windmill-labs/windmill/commit/90b00f55011288115e00b7a30b3e8e91bc0b7f4b))
* **python:** windows worker fails to install 3.10 ([#5409](https://github.com/windmill-labs/windmill/issues/5409)) ([ebb58e0](https://github.com/windmill-labs/windmill/commit/ebb58e0dc7cc104e0bbfd90cf7f37e40ffd0bbf5))
## [1.473.1](https://github.com/windmill-labs/windmill/compare/v1.473.0...v1.473.1) (2025-03-03)
### Bug Fixes
* **backend:** copilot info exists_ai_resource ([#5415](https://github.com/windmill-labs/windmill/issues/5415)) ([844edd1](https://github.com/windmill-labs/windmill/commit/844edd1117bc5fdcc108adf53aec3974a7c66384))
* improve cancel performance ([fba9e7e](https://github.com/windmill-labs/windmill/commit/fba9e7ef03b91d3e2c78ff833ddcc5207b7436d2))
## [1.473.0](https://github.com/windmill-labs/windmill/compare/v1.472.1...v1.473.0) (2025-03-03)
### Features
* app s3 input anonymous delete ([#5401](https://github.com/windmill-labs/windmill/issues/5401)) ([46c7845](https://github.com/windmill-labs/windmill/commit/46c784574add176cb75d3627c9d3f55b6fb945f8))
* track workspace runnables used in flows ([#5369](https://github.com/windmill-labs/windmill/issues/5369)) ([7bf9e25](https://github.com/windmill-labs/windmill/commit/7bf9e25ede82486115eae71865202f85aa931a8d))
### Bug Fixes
* improve db loads by adding index on audit ([e1ff001](https://github.com/windmill-labs/windmill/commit/e1ff00117ca5b66dd0b9365e63b9ff7dc277bc2c))
* migrations do not refer to public schema anymore ([#5400](https://github.com/windmill-labs/windmill/issues/5400)) ([3063001](https://github.com/windmill-labs/windmill/commit/3063001491b49a4b6d0cd5985818b32aa4d3f16f))
* remove typings_extensions from python sdk ([04ffbf8](https://github.com/windmill-labs/windmill/commit/04ffbf8c266a06c3efcebcbbaee767f0ab0771e2))
## [1.472.1](https://github.com/windmill-labs/windmill/compare/v1.472.0...v1.472.1) (2025-02-26)
### Bug Fixes
* disable bundling using env var ([#5396](https://github.com/windmill-labs/windmill/issues/5396)) ([cb559d6](https://github.com/windmill-labs/windmill/commit/cb559d6083553c400e18e6077002c4891289a8a2))
## [1.472.0](https://github.com/windmill-labs/windmill/compare/v1.471.1...v1.472.0) (2025-02-26)
### Bug Fixes
* downgrade v8 to fix some rare panics ([5569e4d](https://github.com/windmill-labs/windmill/commit/5569e4d4953a01f2ad03ea8b71e695e833964bea))
* **frontend:** markdown shows single backtick in single line code block ([#5391](https://github.com/windmill-labs/windmill/issues/5391)) ([7f290bb](https://github.com/windmill-labs/windmill/commit/7f290bbf6a33e2811dbe2bd8ee905c0fa8e8db3b))
* migrate toggle to melt (4/4) ([#5329](https://github.com/windmill-labs/windmill/issues/5329)) ([69fc8a9](https://github.com/windmill-labs/windmill/commit/69fc8a98ae78bc01dc3d97f9732ee28864b323dd))
## [1.471.1](https://github.com/windmill-labs/windmill/compare/v1.471.0...v1.471.1) (2025-02-26)
### Bug Fixes
* update to rust 1.86.0 ([3ada264](https://github.com/windmill-labs/windmill/commit/3ada264c4ad49f666c3a053eb48c7df294bf085b))
## [1.471.0](https://github.com/windmill-labs/windmill/compare/v1.470.1...v1.471.0) (2025-02-26)
### Features
* add support for claude sonnet 3.7 thinking ([#5387](https://github.com/windmill-labs/windmill/issues/5387)) ([487d84b](https://github.com/windmill-labs/windmill/commit/487d84bd7fdc39a2401df4108fca6183189cf38a))
### Bug Fixes
* **frontend:** improve pagination handling and filter refreshes ([#5378](https://github.com/windmill-labs/windmill/issues/5378)) ([a85ebfb](https://github.com/windmill-labs/windmill/commit/a85ebfbbf48590812c0931ad93179c322f819849))
## [1.470.1](https://github.com/windmill-labs/windmill/compare/v1.470.0...v1.470.1) (2025-02-26)
### Bug Fixes
* multiple app initializations fixes ([630e54f](https://github.com/windmill-labs/windmill/commit/630e54f65c950ec0073b3cdac9974cb666c1ab3f))
## [1.470.0](https://github.com/windmill-labs/windmill/compare/v1.469.0...v1.470.0) (2025-02-26)
### Features
* **frontend:** set default app refesh interval ([#5380](https://github.com/windmill-labs/windmill/issues/5380)) ([478d3fb](https://github.com/windmill-labs/windmill/commit/478d3fbf4a7e52d19fcb5cf8d601b2eeb3487716))
### Bug Fixes
* multiple app initializations fixes ([24b6003](https://github.com/windmill-labs/windmill/commit/24b600378025632aecb2ca898b63d6032e08eb2e))
## [1.469.0](https://github.com/windmill-labs/windmill/compare/v1.468.0...v1.469.0) (2025-02-25)
### Features
* limit the number of times a job can be restarted (3) after loss of pings ([c8a9596](https://github.com/windmill-labs/windmill/commit/c8a959691c37350def37fe3eb9f24c6f7789960d))
* python-client now support mocked api via `WM_MOCKED_API_FILE` env ([#5372](https://github.com/windmill-labs/windmill/issues/5372)) ([50607c7](https://github.com/windmill-labs/windmill/commit/50607c7625e4a48fb397cff167b41bb6602716c0))
### Bug Fixes
* improve flow editor for vscode extension ([44b26d2](https://github.com/windmill-labs/windmill/commit/44b26d2ccec0c9dd65d1f53b057d031f841d7dba))
* improve infinite grid behavior ([56d1da7](https://github.com/windmill-labs/windmill/commit/56d1da78fd3424ae5b4abbb009c7437ea98765ef))
## [1.468.0](https://github.com/windmill-labs/windmill/compare/v1.467.1...v1.468.0) (2025-02-24)
### Features
* add audit logs scope filter in admins workspace ([#5352](https://github.com/windmill-labs/windmill/issues/5352)) ([b3e00b7](https://github.com/windmill-labs/windmill/commit/b3e00b7fdc3ad4c689fc30216accbed05822794c))
* add support for | None and Optional in python ([#5361](https://github.com/windmill-labs/windmill/issues/5361)) ([9736355](https://github.com/windmill-labs/windmill/commit/9736355d5f82615100212698c5537997e5a0de39))
* make flow lock deployment error visible in UI ([b8e6d0d](https://github.com/windmill-labs/windmill/commit/b8e6d0da79ca57b115e7cb0ccff9f5623b23f3f3))
### Bug Fixes
* add LOCALAPPDATA env variable to python execution on windows ([8806870](https://github.com/windmill-labs/windmill/commit/8806870b1bf67c2f77beaf04d986cf172c7b4bf4))
* fix confirmation modal check on deploy ([3028325](https://github.com/windmill-labs/windmill/commit/3028325615e2f7e5ee3d1b6278580121880db14f))
* **frontend:** make html app component content selectable ([#5359](https://github.com/windmill-labs/windmill/issues/5359)) ([f1c5b77](https://github.com/windmill-labs/windmill/commit/f1c5b77d7af8433905937d274b97c2d5cd6c1316))
* handle better forced value propagation in apps ([7c842c8](https://github.com/windmill-labs/windmill/commit/7c842c88bf5225b6bc39857109b1b1ba5f99d708))
* handle better optional chaining operator ([d45c1f6](https://github.com/windmill-labs/windmill/commit/d45c1f69d48a5ad93f4399ce0150bbff6fd4fa6b))
* improve app markdown rendering ([96597d3](https://github.com/windmill-labs/windmill/commit/96597d3d6b3d31298e5582a55e11e1d48edbf175))
* improve cancel/back behavior on editors ([0565981](https://github.com/windmill-labs/windmill/commit/05659816e722effcba27e71f855e819c606f8756))
* improve custom component rendering ([4ee4ff7](https://github.com/windmill-labs/windmill/commit/4ee4ff78d389d61c63952c84dee967113c783c45))
* improve webhook settings cache invalidation ([0456272](https://github.com/windmill-labs/windmill/commit/0456272e3f36996c5f223fc332b150c7a64c2f05))
* update bun t.1.43->1.2.3 ([4e477d1](https://github.com/windmill-labs/windmill/commit/4e477d1f589343980d7bd2953909ff6a6be30739))
* update deno 2.1.2->2.2.1 ([b102ff4](https://github.com/windmill-labs/windmill/commit/b102ff4a4643e2f06d44d493f2f776b44ae721cc))
## [1.467.1](https://github.com/windmill-labs/windmill/compare/v1.467.0...v1.467.1) (2025-02-22)
Dockerfile
+1 -1
View File
@@ -1,5 +1,5 @@
ARG DEBIAN_IMAGE=debian:bookworm-slim
ARG RUST_IMAGE=rust:1.83-slim-bookworm
ARG RUST_IMAGE=rust:1.85-slim-bookworm
FROM ${RUST_IMAGE} AS rust_base
LICENSE
+16 -4
View File
@@ -1,6 +1,6 @@
Source code in this repository is variously licensed under the Apache License
Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file
./LICENSE-AGPL)
./LICENSE-AGPL) and a proprietary license for certain enterprise features.
Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise
specified. Every file is under License AGPL unless otherwise specified or
@@ -12,11 +12,23 @@ and commercial license. The files under frontend/ are AGPLv3 Licensed, except
any snippets of code that require a positive license check to be activated.
Those snippets and files are under a proprietary and commercial license. Private
and public forks MUST not include any of the above proprietary and commercial
code. Windmill Labs, Inc. provide tools to clean the codebase from those
snippets upon demand. The files under python-client/ deno-client/ go-client/ powershell-client/ are
code. The files under python-client/ deno-client/ go-client/ powershell-client/
are Apache 2.0 Licensed. The openapi files, including the OpenFlow spec is
Apache 2.0 Licensed.
The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
The binary compilable from source code in this repository without the
"enterprise" feature flag is open-source under the AGPLv3 License terms and
conditions.
The "Community Edition" of Windmill available in the docker images hosted under
ghcr.io/windmill-labs/windmill and the github binary releases contains the files
under the AGPLv3 and Apache 2 sources but also includes proprietary and
non-public code and features which are not open source and under the following
terms: Windmill Labs, Inc. grants a right to use all the features of the
"Community Edition" for free without restrictions other than the limits and
quotas set in the software and a right to distribute the community edition as is
but not to sell, resell, serve as a managed service, modify or wrap under any
form without an explicit agreement.
All third party components incorporated into the Windmill Software are licensed
under the original license provided by the owner of the applicable component.
README.md
+87 -63
View File
@@ -110,8 +110,8 @@ You can build your entire infra on top of Windmill!
```typescript
//import any dependency from npm
import * as wmill from "windmill-client"
import * as cowsay from 'cowsay@1.5.0';
import * as wmill from "windmill-client";
import * as cowsay from "cowsay@1.5.0";
// fill the type, or use the +Resource type to get a type-safe reference to a resource
type Postgresql = {
@@ -146,7 +146,9 @@ export async function main(
## CLI
We have a powerful CLI to interact with the windmill platform and sync your scripts from local files, GitHub repos and to run scripts and flows on the instance from local commands. See
We have a powerful CLI to interact with the windmill platform and sync your
scripts from local files, GitHub repos and to run scripts and flows on the
instance from local commands. See
[more details](https://www.windmill.dev/docs/advanced/cli).
![CLI Screencast](./cli/vhs/output/setup.gif)
@@ -168,7 +170,8 @@ Code extension: <https://www.windmill.dev/docs/cli_local_dev/vscode-extension>.
Architecture:
- Stateless API backend.
- Workers that pull jobs from a queue in Postgres (and later, Kafka or Redis.
Upvote [#173](#https://github.com/windmill-labs/windmill/issues/173) if interested).
Upvote [#173](#https://github.com/windmill-labs/windmill/issues/173) if
interested).
- Frontend in Svelte.
- Scripts executions are sandboxed using Google's
[nsjail](https://github.com/google/nsjail).
@@ -284,22 +287,37 @@ edition.
### Commercial license
To self-host Windmill, you must respect the terms of the
[AGPLv3 license](https://www.gnu.org/licenses/agpl-3.0.en.html) which you do not
need to worry about for personal uses. For business uses, you should be fine if
you do not re-expose Windmill in any way to your users and are comfortable with
AGPLv3.
See the [LICENSE](https://github.com/windmill-labs/windmill/blob/main/LICENSE)
file for the full license text.
The "Community Edition" of Windmill available in the docker images hosted under
ghcr.io/windmill-labs/windmill and the github binary releases contains the files
under the AGPLv3 and Apache 2 sources but also includes proprietary and
non-public code and features which are not open source and under the following
terms: Windmill Labs, Inc. grants a right to use all the features of the
"Community Edition" for free without restrictions other than the limits and
quotas set in the software and a right to distribute the community edition as is
but not to sell, resell, serve Windmill as a managed service, modify or wrap
under any form without an explicit agreement.
The binary compilable from source code in this repository without the
"enterprise" feature flag is open-source under the
[LICENSE-AGPLv3](https://github.com/windmill-labs/windmill/blob/main/LICENSE-AGPL)
License terms and conditions.
To
[re-expose any Windmill parts to your users](https://www.windmill.dev/docs/misc/white_labelling)
as a feature of your product, or to build a feature on top of Windmill, to
comply with AGPLv3 your product must be AGPLv3 or you must get a commercial
license. Contact us at <ruben@windmill.dev> if you have any doubts.
[re-expose directly any Windmill parts to your users](https://www.windmill.dev/docs/misc/white_labelling)
as a feature of your product, with the exception of iframed public Windmill
"apps", or to build a feature on top of "Windmill Community Edition" that you
sell commercially or embed in a distributable product or binary, you must get a
commercial license. Contact us at <sales@windmill.dev> if you have any
questions. To do the same from the binary compiled from the source code in this
repository without the "enterprise" feature flag, you must comply with the
AGPLv3 license terms and conditions or get a commercial license from Windmill
Labs, Inc.
In addition, a commercial license grants you a dedicated engineer to transition
your current infrastructure to Windmill, support with tight SLA, and our global
cache sync for high-performance/no dependency cache miss of cluster from 10+
nodes to 200+ nodes.
To use Windmill "Community Edition" as is internally in your organization, or to
use its APIs as is, you do NOT need a commercial license.
### Integrations
@@ -314,70 +332,76 @@ you to have it being synced automatically everyday.
## Environment Variables
| Environment Variable name | Default | Description | Api Server/Worker/All |
| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
| DATABASE_URL | | The Postgres database url. | All |
| WORKER_GROUP | default | The worker group the worker belongs to and get its configuration pulled from | Worker |
| MODE | standalone | The mode if the binary. Possible values: standalone, worker, server, agent | All |
| METRICS_ADDR | None | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All |
| JSON_FMT | false | Output the logs in json format instead of logfmt | All |
| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance. Is overriden by the instance settings if any. | Server |
| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server |
| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server |
| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker |
| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker |
| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker |
| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server |
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server |
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
| PYTHON_PATH | | The path to the python binary if wanting to not have it managed by uv. | Worker |
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker |
| GOPROXY | | The GOPROXY env variable to use | Worker |
| NETRC | | The netrc content to use a private go registry | Worker | | Worker |
| PY_CONCURRENT_DOWNLOADS | 20 | Sets the maximum number of in-flight concurrent python downloads that windmill will perform at any given time. | Worker |
| PATH | None | The path environment variable, usually inherited | Worker |
| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker |
| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All |
| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server |
| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker |
| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker |
| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker |
| DISABLE_RESPONSE_LOGS | false | Disable response logs | Server |
| CREATE_WORKSPACE_REQUIRE_SUPERADMIN | true | If true, only superadmins can create new workspaces | Server |
| Environment Variable name | Default | Description | Api Server/Worker/All |
| ----------------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | --- | ------ |
| DATABASE_URL | | The Postgres database url. | All |
| WORKER_GROUP | default | The worker group the worker belongs to and get its configuration pulled from | Worker |
| MODE | standalone | The mode if the binary. Possible values: standalone, worker, server, agent | All |
| METRICS_ADDR | None | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All |
| JSON_FMT | false | Output the logs in json format instead of logfmt | All |
| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance. Is overriden by the instance settings if any. | Server |
| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server |
| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server |
| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker |
| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker |
| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker |
| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server |
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server |
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
| PYTHON_PATH | | The path to the python binary if wanting to not have it managed by uv. | Worker |
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker |
| GOPROXY | | The GOPROXY env variable to use | Worker |
| NETRC | | The netrc content to use a private go registry | Worker | | Worker |
| PY_CONCURRENT_DOWNLOADS | 20 | Sets the maximum number of in-flight concurrent python downloads that windmill will perform at any given time. | Worker |
| PATH | None | The path environment variable, usually inherited | Worker |
| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker |
| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All |
| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server |
| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker |
| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker |
| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker |
| DISABLE_RESPONSE_LOGS | false | Disable response logs | Server |
| CREATE_WORKSPACE_REQUIRE_SUPERADMIN | true | If true, only superadmins can create new workspaces | Server |
## Run a local dev setup
See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all
running options.
Using [Nix](./frontend/README_DEV.md#nix).
### only Frontend
This will use the backend of <https://app.windmill.dev> but your own frontend
with hot-code reloading. Note that you will need to use a username / password login due to CSRF checks using a different auth provider.
with hot-code reloading. Note that you will need to use a username / password
login due to CSRF checks using a different auth provider.
In the `frontend/` directory:
1. install the dependencies with `npm install` (or `pnpm install` or `yarn`)
2. generate the windmill client:
```
npm run generate-backend-client
## on mac use
npm run generate-backend-client-mac
```
```
npm run generate-backend-client
## on mac use
npm run generate-backend-client-mac
```
3. Run your dev server with `npm run dev`
4. Et voilà, windmill should be available at `http://localhost/`
### Backend + Frontend
See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all
running options.
1. Create a Postgres Database for Windmill and create an admin role inside your
Postgres setup.
The easiest way to get a working db is to run
```
Postgres setup. The easiest way to get a working db is to run
```
cargo install sqlx-cli
env DATABASE_URL=<YOUR_DATABASE_URL> sqlx migrate run
```
```
This will also avoid compile time issue with sqlx's `query!` macro
2. Install [nsjail](https://github.com/google/nsjail) and have it accessible in
your PATH
@@ -387,15 +411,15 @@ running options.
5. Install the [lld linker](https://lld.llvm.org/)
6. Go to `frontend/`:
1. `npm install`, `npm run generate-backend-client` then `npm run dev`
2. You might need to set some extra heap space for the node runtime `export NODE_OPTIONS="--max-old-space-size=4096"`
3. In another shell `npm run build` otherwise the backend will not find the `frontend/build` folder and will not compile.
2. You might need to set some extra heap space for the node runtime
`export NODE_OPTIONS="--max-old-space-size=4096"`
3. In another shell `npm run build` otherwise the backend will not find the
`frontend/build` folder and will not compile.
4. In another shell `sudo caddy run --config Caddyfile`
7. Go to `backend/`:
`env DATABASE_URL=<DATABASE_URL_TO_YOUR_WINDMILL_DB> RUST_LOG=info cargo run`
8. Et voilà, windmill should be available at `http://localhost/`
## Contributors
<a href="https://github.com/windmill-labs/windmill/graphs/contributors">
@@ -404,4 +428,4 @@ running options.
## Copyright
Windmill Labs, Inc 2023
Windmill Labs, Inc 2023
backend/.sqlx/query-07da723ce5c9ee2d7c236e8eabe254c783fc34b617c8a9a95a0eb0cda535dab5.json
Generated
+2 -1
View File
@@ -20,7 +20,8 @@
"email",
"nats",
"postgres",
"sqs"
"sqs",
"mqtt"
]
}
}
backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json
Generated
+19 -19
View File
@@ -130,28 +130,28 @@
},
{
"ordinal": 25,
"name": "teams_command_script",
"type_info": "Text"
},
{
"ordinal": 26,
"name": "teams_team_id",
"type_info": "Text"
},
{
"ordinal": 27,
"name": "teams_team_name",
"type_info": "Text"
},
{
"ordinal": 28,
"name": "ai_models",
"type_info": "VarcharArray"
},
{
"ordinal": 29,
"ordinal": 26,
"name": "code_completion_model",
"type_info": "Varchar"
},
{
"ordinal": 27,
"name": "teams_command_script",
"type_info": "Text"
},
{
"ordinal": 28,
"name": "teams_team_id",
"type_info": "Text"
},
{
"ordinal": 29,
"name": "teams_team_name",
"type_info": "Text"
}
],
"parameters": {
@@ -185,10 +185,10 @@
true,
true,
true,
true,
true,
true,
false,
true,
true,
true,
true
]
},
backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a"
}
backend/.sqlx/query-186aef850c2eeb89c186ac6b2934dd3a703e2b9428096801e1d2d61fdbb99c9e.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "186aef850c2eeb89c186ac6b2934dd3a703e2b9428096801e1d2d61fdbb99c9e"
}
backend/.sqlx/query-1ab0d1ba1fbfad31ffb28a01a6c9640d0ac142aabee8d288a4f9c56ad9dbeac4.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO job_logs (job_id, logs)\n VALUES ($1, $2)\n ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\n' || EXCLUDED.logs\n WHERE job_logs.job_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": []
},
"hash": "1ab0d1ba1fbfad31ffb28a01a6c9640d0ac142aabee8d288a4f9c56ad9dbeac4"
}
backend/.sqlx/query-1bdf186d3b99bbd913cbf95150105470cd5f1d4ddbb147cb8ce46f9d1da5dfaf.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "WITH email_lookup AS (\n SELECT email FROM token WHERE token = $1\n )\n DELETE FROM token\n WHERE email = (SELECT email FROM email_lookup) AND label = 'session'\n RETURNING email",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
true
]
},
"hash": "1bdf186d3b99bbd913cbf95150105470cd5f1d4ddbb147cb8ce46f9d1da5dfaf"
}
backend/.sqlx/query-2bb2cf6accb18d3e37a63388cca52a6591e7593b1a7c3d7a6848587679a48187.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE flow_workspace_runnables SET workspace_id = $1 WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "2bb2cf6accb18d3e37a63388cca52a6591e7593b1a7c3d7a6848587679a48187"
}
backend/.sqlx/query-93d0ee34c7b7c56ab9cae28071f8e5fdaff0e601e2760cc5653d197c51b106bb.json → backend/.sqlx/query-31b6fccad46b22bcbba6bbce22209ccb1825116004ed72682854ce3352f454a5.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n \n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\", \n \n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\",\n EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS \"sqs_used!\"\n ",
"query": "\n SELECT \n \n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\", \n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\",\n EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS \"mqtt_used!\",\n EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS \"sqs_used!\"\n ",
"describe": {
"columns": [
{
@@ -30,6 +30,11 @@
},
{
"ordinal": 5,
"name": "mqtt_used!",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "sqs_used!",
"type_info": "Bool"
}
@@ -45,8 +50,9 @@
null,
null,
null,
null,
null
]
},
"hash": "93d0ee34c7b7c56ab9cae28071f8e5fdaff0e601e2760cc5653d197c51b106bb"
"hash": "31b6fccad46b22bcbba6bbce22209ccb1825116004ed72682854ce3352f454a5"
}
backend/.sqlx/query-35795d27c4ca69d2f145b4dba08a6ed16c25aea4584103c1b9a3651eb31bfe53.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, runnable_is_flow, workspace_id) VALUES ($1, $2, TRUE, $3) ON CONFLICT DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar"
]
},
"nullable": []
},
"hash": "35795d27c4ca69d2f145b4dba08a6ed16c25aea4584103c1b9a3651eb31bfe53"
}
backend/.sqlx/query-3bbde0fa35d935ec2dd8bd1fb14cfecf48305f5f4b644b3c35355074e1ccce28.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "CREATE INDEX CONCURRENTLY queue_sort_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) WHERE running = false",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "3bbde0fa35d935ec2dd8bd1fb14cfecf48305f5f4b644b3c35355074e1ccce28"
}
backend/.sqlx/query-3e33469f448fa86c7e0d0deb65ee2c13b5dcc4cbc4fcb0fe2bde1fdcb6d20e74.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'mqtt' AND \n server_id IS NULL\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "3e33469f448fa86c7e0d0deb65ee2c13b5dcc4cbc4fcb0fe2bde1fdcb6d20e74"
}
backend/.sqlx/query-458e053b93b6e8fc56420fb1ad25cd2910d301b44b6e9c9a861dee3de352f929.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS queue_sort_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) WHERE running = false",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "458e053b93b6e8fc56420fb1ad25cd2910d301b44b6e9c9a861dee3de352f929"
}
backend/.sqlx/query-4844a797ddec207f1d2cbf41836e65d6435840f6d0740f300d7c2cf88f8fe46a.json
Generated
+25
View File
@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "4844a797ddec207f1d2cbf41836e65d6435840f6d0740f300d7c2cf88f8fe46a"
}
backend/.sqlx/query-4ed74bbda2ad0ca5e4648787fe2d9c89e9b83571e30059861b6998935b35f78b.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT \n 1 \n FROM \n mqtt_trigger \n WHERE \n path = $1 AND \n workspace_id = $2\n )",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "4ed74bbda2ad0ca5e4648787fe2d9c89e9b83571e30059861b6998935b35f78b"
}
backend/.sqlx/query-548a1424a6b9ac9998b8fc5312bcff671ee7fe59154e9ef32800c117e64bce19.json
Generated
+50
View File
@@ -0,0 +1,50 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'mqtt' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "owner",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "trigger_config!: _",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
false,
false,
true
]
},
"hash": "548a1424a6b9ac9998b8fc5312bcff671ee7fe59154e9ef32800c117e64bce19"
}
backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json
Generated
+19 -19
View File
@@ -130,28 +130,28 @@
},
{
"ordinal": 25,
"name": "teams_command_script",
"type_info": "Text"
},
{
"ordinal": 26,
"name": "teams_team_id",
"type_info": "Text"
},
{
"ordinal": 27,
"name": "teams_team_name",
"type_info": "Text"
},
{
"ordinal": 28,
"name": "ai_models",
"type_info": "VarcharArray"
},
{
"ordinal": 29,
"ordinal": 26,
"name": "code_completion_model",
"type_info": "Varchar"
},
{
"ordinal": 27,
"name": "teams_command_script",
"type_info": "Text"
},
{
"ordinal": 28,
"name": "teams_team_id",
"type_info": "Text"
},
{
"ordinal": 29,
"name": "teams_team_name",
"type_info": "Text"
}
],
"parameters": {
@@ -185,10 +185,10 @@
true,
true,
true,
true,
true,
true,
false,
true,
true,
true,
true
]
},
backend/.sqlx/query-5c1de8473e0e96c1063a9a735a064c5a91e3ed8d9260c72b783fc12542b88fbd.json
Generated
+4 -2
View File
@@ -28,7 +28,8 @@
"email",
"nats",
"postgres",
"sqs"
"sqs",
"mqtt"
]
}
}
@@ -62,7 +63,8 @@
"email",
"nats",
"postgres",
"sqs"
"sqs",
"mqtt"
]
}
}
backend/.sqlx/query-5d773db0a69c73e86ccd5fd978932e7ac64eb41a56d8ca9ff0a89c594432b531.json
Generated
+36
View File
@@ -0,0 +1,36 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger \n SET\n mqtt_resource_path = $1,\n subscribe_topics = $2,\n client_version = $3,\n client_id = $4,\n v3_config = $5,\n v5_config = $6,\n is_flow = $7, \n edited_by = $8, \n email = $9,\n script_path = $10,\n path = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL\n WHERE \n workspace_id = $12 AND \n path = $13\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"JsonbArray",
{
"Custom": {
"name": "mqtt_client_version",
"kind": {
"Enum": [
"v3",
"v5"
]
}
}
},
"Varchar",
"Jsonb",
"Jsonb",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "5d773db0a69c73e86ccd5fd978932e7ac64eb41a56d8ca9ff0a89c594432b531"
}
backend/.sqlx/query-62475252dcf54f32433b97ae011daf5d4205d160d2aedf463c7dfe944e93257a.json
Generated
+2 -1
View File
@@ -20,7 +20,8 @@
"email",
"nats",
"postgres",
"sqs"
"sqs",
"mqtt"
]
}
}
backend/.sqlx/query-653574b381a31548d82c1f6f3f44ec826597c42310ab78f9aacd7d9448206c6a.json
Generated
-34
View File
@@ -1,34 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH zombie_jobs AS (\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM v2_job j, v2_job_runtime r\n WHERE j.id = q.id AND j.id = r.id\n AND ping < now() - ($1 || ' seconds')::interval\n AND running = true\n AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')\n AND same_worker = false\n RETURNING q.id, q.workspace_id, ping\n ),\n update_concurrency AS (\n UPDATE concurrency_counter cc\n SET job_uuids = job_uuids - zj.id::text\n FROM zombie_jobs zj\n INNER JOIN concurrency_key ck ON ck.job_id = zj.id\n WHERE cc.concurrency_id = ck.key\n )\n SELECT id, workspace_id, ping FROM zombie_jobs",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "ping",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
true
]
},
"hash": "653574b381a31548d82c1f6f3f44ec826597c42310ab78f9aacd7d9448206c6a"
}
backend/.sqlx/query-65cd5a5a12eb79a2f21ed898616c5cd5c5e671ce1ed3d89639cd306c76a1d9db.json
Generated
+135
View File
@@ -0,0 +1,135 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n mqtt_resource_path,\n subscribe_topics as \"subscribe_topics!: Vec<SqlxJson<SubscribeTopic>>\",\n v3_config as \"v3_config!: Option<SqlxJson<MqttV3Config>>\",\n v5_config as \"v5_config!: Option<SqlxJson<MqttV5Config>>\",\n client_version AS \"client_version: _\",\n client_id,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM \n mqtt_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "mqtt_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "subscribe_topics!: Vec<SqlxJson<SubscribeTopic>>",
"type_info": "JsonbArray"
},
{
"ordinal": 2,
"name": "v3_config!: Option<SqlxJson<MqttV3Config>>",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "v5_config!: Option<SqlxJson<MqttV5Config>>",
"type_info": "Jsonb"
},
{
"ordinal": 4,
"name": "client_version: _",
"type_info": {
"Custom": {
"name": "mqtt_client_version",
"kind": {
"Enum": [
"v3",
"v5"
]
}
}
}
},
{
"ordinal": 5,
"name": "client_id",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 12,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 13,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 15,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 16,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 17,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
true,
true,
false,
true,
false,
false,
false,
false,
false,
false,
false,
true,
true,
false,
true,
false
]
},
"hash": "65cd5a5a12eb79a2f21ed898616c5cd5c5e671ce1ed3d89639cd306c76a1d9db"
}
backend/.sqlx/query-67afe352fc26dda9107c90e50e954642d877178ce2c0e73b72c3824135ef86f4.json
Generated
-14
View File
@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO job_logs (job_id, logs)\n VALUES ($1, 'Restarted job after not receiving job''s ping for too long the ' || now() || '\n\n')\n ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\n' || EXCLUDED.logs\n WHERE job_logs.job_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "67afe352fc26dda9107c90e50e954642d877178ce2c0e73b72c3824135ef86f4"
}
backend/.sqlx/query-71d51bbc35da7b9930e3ea3a634451217ccb9f1bc35b1ad6e10d16bc19c41447.json
Generated
+2 -1
View File
@@ -31,7 +31,8 @@
"email",
"nats",
"postgres",
"sqs"
"sqs",
"mqtt"
]
}
}
backend/.sqlx/query-7b6e8dac5f83fcbae95056c4206bf4de1b12d6c8aab32f1f55d7f2c230c5c08e.json
Generated
+132
View File
@@ -0,0 +1,132 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n mqtt_resource_path,\n subscribe_topics as \"subscribe_topics!: Vec<SqlxJson<SubscribeTopic>>\",\n v3_config as \"v3_config!: Option<SqlxJson<MqttV3Config>>\",\n v5_config as \"v5_config!: Option<SqlxJson<MqttV5Config>>\",\n client_version as \"client_version: _\",\n client_id,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM\n mqtt_trigger\n WHERE\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "mqtt_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "subscribe_topics!: Vec<SqlxJson<SubscribeTopic>>",
"type_info": "JsonbArray"
},
{
"ordinal": 2,
"name": "v3_config!: Option<SqlxJson<MqttV3Config>>",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "v5_config!: Option<SqlxJson<MqttV5Config>>",
"type_info": "Jsonb"
},
{
"ordinal": 4,
"name": "client_version: _",
"type_info": {
"Custom": {
"name": "mqtt_client_version",
"kind": {
"Enum": [
"v3",
"v5"
]
}
}
}
},
{
"ordinal": 5,
"name": "client_id",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 12,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 13,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 15,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 16,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 17,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
true,
true,
false,
true,
false,
false,
false,
false,
false,
false,
false,
true,
true,
false,
true,
false
]
},
"hash": "7b6e8dac5f83fcbae95056c4206bf4de1b12d6c8aab32f1f55d7f2c230c5c08e"
}
backend/.sqlx/query-7db681e86f8332c636d0f29b389860292b704788827edfebc467a92486f9e14f.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM token WHERE email = $1 AND label = 'session'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "7db681e86f8332c636d0f29b389860292b704788827edfebc467a92486f9e14f"
}
backend/.sqlx/query-b05c5f62ef4aa21d33369130cced0e9d7d128727eb58a9be7ae69cbb16bcbb27.json → backend/.sqlx/query-8aebd7f7fd1374f1c3d5389e953ebf080df3f76ac3e6e6373a89c8d46388125d.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO token\n (token, email, label, expiration, super_admin)\n VALUES ($1, $2, $3, now() + ($4 || ' hours')::interval, $5)",
"query": "INSERT INTO token\n (token, email, label, expiration, super_admin)\n VALUES ($1, $2, $3, now() + ($4 || ' seconds')::interval, $5)",
"describe": {
"columns": [],
"parameters": {
@@ -14,5 +14,5 @@
},
"nullable": []
},
"hash": "b05c5f62ef4aa21d33369130cced0e9d7d128727eb58a9be7ae69cbb16bcbb27"
"hash": "8aebd7f7fd1374f1c3d5389e953ebf080df3f76ac3e6e6373a89c8d46388125d"
}
backend/.sqlx/query-4115a8e75ad5ed74f737f24aa64edc68e71b29ddc32a00a754678ea4f2c167e5.json → backend/.sqlx/query-8be2919c3511575c89b882b112b987fd5724c299cb285f819a2561260404e513.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at, scopes FROM token WHERE email = $1 AND label != 'ephemeral-script'\n ORDER BY created_at DESC LIMIT $2 OFFSET $3",
"query": "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at, scopes FROM token WHERE email = $1 AND (label != 'ephemeral-script' OR label IS NULL)\n ORDER BY created_at DESC LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
@@ -50,5 +50,5 @@
true
]
},
"hash": "4115a8e75ad5ed74f737f24aa64edc68e71b29ddc32a00a754678ea4f2c167e5"
"hash": "8be2919c3511575c89b882b112b987fd5724c299cb285f819a2561260404e513"
}
backend/.sqlx/query-8d31b4a531c59a2385210d1213c205100d6673a94e90000c8db4eb5809f17365.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger \n SET \n enabled = $1, \n email = $2, \n edited_by = $3, \n edited_at = now(), \n server_id = NULL, \n error = NULL\n WHERE \n path = $4 AND \n workspace_id = $5 \n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Bool",
"Varchar",
"Varchar",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "8d31b4a531c59a2385210d1213c205100d6673a94e90000c8db4eb5809f17365"
}
backend/.sqlx/query-9aeee333b1dbe58ba819ba3b2713242b54d77b46b5f785ae66b8104f89f43219.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT f.path\n FROM flow_workspace_runnables fwr \n JOIN flow f \n ON fwr.flow_path = f.path AND fwr.workspace_id = f.workspace_id\n WHERE fwr.runnable_path = $1 AND fwr.runnable_is_flow = $2 AND fwr.workspace_id = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Bool",
"Text"
]
},
"nullable": [
false
]
},
"hash": "9aeee333b1dbe58ba819ba3b2713242b54d77b46b5f785ae66b8104f89f43219"
}
backend/.sqlx/query-9e88a43de7315052668619002321aadbc27ff5bd0c554bf5339060cc35797d3f.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n DELETE \n FROM \n mqtt_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "9e88a43de7315052668619002321aadbc27ff5bd0c554bf5339060cc35797d3f"
}
backend/.sqlx/query-abfc6b530565f125bd0b8ac99cd67fd22f14f8fce89e59890ba21e87fe323af5.json
Generated
+41
View File
@@ -0,0 +1,41 @@
{
"db_name": "PostgreSQL",
"query": "WITH to_update AS (\n SELECT q.id, q.workspace_id, r.ping, COALESCE(zjc.counter, 0) as counter\n FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_runtime r ON r.id = j.id\n LEFT JOIN zombie_job_counter zjc ON zjc.job_id = q.id\n WHERE ping < now() - ($1 || ' seconds')::interval\n AND running = true\n AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')\n AND same_worker = false\n AND (zjc.counter IS NULL OR zjc.counter <= $2)\n FOR UPDATE of q SKIP LOCKED\n ),\n zombie_jobs AS (\n UPDATE v2_job_queue q\n SET running = false, started_at = null\n FROM to_update tu\n WHERE q.id = tu.id AND (tu.counter IS NULL OR tu.counter < $2)\n RETURNING q.id, q.workspace_id, ping, tu.counter\n ),\n update_ping AS (\n UPDATE v2_job_runtime r\n SET ping = null\n FROM zombie_jobs zj\n WHERE r.id = zj.id\n ),\n increment_counter AS (\n INSERT INTO zombie_job_counter (job_id, counter)\n SELECT id, 1 FROM to_update WHERE counter < $2\n ON CONFLICT (job_id) DO UPDATE \n SET counter = zombie_job_counter.counter + 1\n ),\n update_concurrency AS (\n UPDATE concurrency_counter cc\n SET job_uuids = job_uuids - zj.id::text\n FROM zombie_jobs zj\n INNER JOIN concurrency_key ck ON ck.job_id = zj.id\n WHERE cc.concurrency_id = ck.key\n )\n SELECT id AS \"id!\", workspace_id AS \"workspace_id!\", ping, counter + 1 AS counter FROM to_update",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id!",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "ping",
"type_info": "Timestamptz"
},
{
"ordinal": 3,
"name": "counter",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Int4"
]
},
"nullable": [
false,
false,
true,
null
]
},
"hash": "abfc6b530565f125bd0b8ac99cd67fd22f14f8fce89e59890ba21e87fe323af5"
}
backend/.sqlx/query-acbea8740b28c26942c50edcf5618cd141e68cf83a7dcae7d3c1b8a7ba94425b.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM flow_workspace_runnables WHERE flow_path = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "acbea8740b28c26942c50edcf5618cd141e68cf83a7dcae7d3c1b8a7ba94425b"
}
backend/.sqlx/query-b656927cd70b6667f3c72186ec04f0bf040da3af9e2eac3229264ec95b4755d8.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE flow_workspace_runnables SET flow_path = REGEXP_REPLACE(flow_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE flow_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "b656927cd70b6667f3c72186ec04f0bf040da3af9e2eac3229264ec95b4755d8"
}
backend/.sqlx/query-bb3b644d630dc2040fdbebfaf7649617c1e5a22fc418a2a6ee3276a09b9a8cf8.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_audit_timestamps ON audit (timestamp DESC)",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "bb3b644d630dc2040fdbebfaf7649617c1e5a22fc418a2a6ee3276a09b9a8cf8"
}
backend/.sqlx/query-c223f8b7fa4ef1aa06e1ba2a56d677774aa237508d5610714efd2e9b8b93c7b8.json
Generated
+2 -1
View File
@@ -23,7 +23,8 @@
"email",
"nats",
"postgres",
"sqs"
"sqs",
"mqtt"
]
}
}
backend/.sqlx/query-c35f44f91b08fa57e29a2b4a685706f62e700695810f23108e975dfcd1fee7a3.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, FALSE, $4) ON CONFLICT DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Int8",
"Varchar"
]
},
"nullable": []
},
"hash": "c35f44f91b08fa57e29a2b4a685706f62e700695810f23108e975dfcd1fee7a3"
}
backend/.sqlx/query-c5063e79aafa70b974276f5bea43ad71135d44b2e84c9efac43a6678f0cf9a18.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "c5063e79aafa70b974276f5bea43ad71135d44b2e84c9efac43a6678f0cf9a18"
}
backend/.sqlx/query-c5270ee815689e42b65df507b850da43239c9a5aaea41c9aed7ed33a6219a534.json
Generated
+2 -1
View File
@@ -20,7 +20,8 @@
"email",
"nats",
"postgres",
"sqs"
"sqs",
"mqtt"
]
}
}
backend/.sqlx/query-cff764601318bfecb8e0f15b64ff9f430d7b1efe8ba863cbcc4a49bab4951794.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'mqtt' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Bool",
"Text"
]
},
"nullable": [
null
]
},
"hash": "cff764601318bfecb8e0f15b64ff9f430d7b1efe8ba863cbcc4a49bab4951794"
}
backend/.sqlx/query-d8186f0cee285aa50db7626409aec7e0504b068ffa8bb185d9384ce1422fd3d1.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM audit WHERE timestamp <= now() - ($1::bigint::text || ' s')::interval",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "d8186f0cee285aa50db7626409aec7e0504b068ffa8bb185d9384ce1422fd3d1"
}
backend/.sqlx/query-dc58e5b4715601a93b3c01a2564a4420f232867a23cacb9a62b386f129a86a4b.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE flow_workspace_runnables SET runnable_path = REGEXP_REPLACE(runnable_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE runnable_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "dc58e5b4715601a93b3c01a2564a4420f232867a23cacb9a62b386f129a86a4b"
}
backend/.sqlx/query-e17ec84003e2ec414622d100f5dfdda86bee33f31835317df512a20c805b35d7.json
Generated
+2 -1
View File
@@ -28,7 +28,8 @@
"email",
"nats",
"postgres",
"sqs"
"sqs",
"mqtt"
]
}
}
backend/.sqlx/query-e23e110e1f0438d21534fc4323e0e7bc1f0dbeca2e4f44ced05bae0ca5ca1039.json
Generated
+2 -1
View File
@@ -36,7 +36,8 @@
"email",
"nats",
"postgres",
"sqs"
"sqs",
"mqtt"
]
}
}
backend/.sqlx/query-c5259e37703c3e48104438bad6e1f3615f4439c090a75e6fde03702a21589b25.json → backend/.sqlx/query-eb68469026be39048c5f42a80a2c538fbb54ad269ec81aea89e431a511245a1e.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id AS \"id!\" FROM v2_job WHERE parent_job = $1 AND workspace_id = $2",
"query": "SELECT id AS \"id!\" FROM v2_job_queue INNER JOIN v2_job USING (id) WHERE parent_job = $1 AND v2_job.workspace_id = $2",
"describe": {
"columns": [
{
@@ -19,5 +19,5 @@
false
]
},
"hash": "c5259e37703c3e48104438bad6e1f3615f4439c090a75e6fde03702a21589b25"
"hash": "eb68469026be39048c5f42a80a2c538fbb54ad269ec81aea89e431a511245a1e"
}
backend/.sqlx/query-f050be8da0d99aa1af64f5d32a8df01c5a59b036b2669878cc557537efb492c8.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'mqtt'\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "f050be8da0d99aa1af64f5d32a8df01c5a59b036b2669878cc557537efb492c8"
}
backend/.sqlx/query-f82974abc7da71fac397f754fe3b3c9c84b6b438b2d9f8cbba61192e999971f4.json
Generated
+36
View File
@@ -0,0 +1,36 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO mqtt_trigger (\n mqtt_resource_path,\n subscribe_topics,\n client_version,\n client_id,\n v3_config,\n v5_config,\n workspace_id,\n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4, \n $5, \n $6, \n $7,\n $8,\n $9,\n $10,\n $11,\n $12,\n $13\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"JsonbArray",
{
"Custom": {
"name": "mqtt_client_version",
"kind": {
"Enum": [
"v3",
"v5"
]
}
}
},
"Varchar",
"Jsonb",
"Jsonb",
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Bool",
"Varchar"
]
},
"nullable": []
},
"hash": "f82974abc7da71fac397f754fe3b3c9c84b6b438b2d9f8cbba61192e999971f4"
}
backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json
Generated
+25
View File
@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'mqtt' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text",
"Bool"
]
},
"nullable": [
null
]
},
"hash": "fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96"
}
backend/.vscode/settings.json
Vendored
+1 -1
View File
@@ -11,5 +11,5 @@
"remote.autoForwardPorts": true,
"conventionalCommits.scopes": [
"restructring triggers, decoding trigger message on work"
]
],
}
backend/Cargo.lock
Generated
+749 -873
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+19 -17
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.467.1"
version = "1.475.0"
authors.workspace = true
edition.workspace = true
@@ -30,7 +30,7 @@ members = [
]
[workspace.package]
version = "1.467.1"
version = "1.475.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -59,7 +59,7 @@ flow_testing = ["windmill-worker/flow_testing"]
openidconnect = ["windmill-api/openidconnect"]
cloud = ["windmill-queue/cloud", "windmill-worker/cloud"]
jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "enterprise", "parquet"]
tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"]
sqlx = ["windmill-worker/sqlx"]
deno_core = ["windmill-worker/deno_core", "dep:deno_core", "dep:v8"]
kafka = ["windmill-api/kafka"]
@@ -74,6 +74,7 @@ mssql = ["windmill-worker/mssql"]
bigquery = ["windmill-worker/bigquery"]
websocket = ["windmill-api/websocket"]
postgres_trigger = ["windmill-api/postgres_trigger"]
mqtt_trigger = ["windmill-api/mqtt_trigger"]
sqs_trigger = ["windmill-api/sqs_trigger"]
python = ["windmill-worker/python"]
smtp = ["windmill-api/smtp", "windmill-common/smtp"]
@@ -173,7 +174,7 @@ serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
uuid = { version = "^1", features = ["serde", "v4"] }
thiserror = "^2"
anyhow = "^1"
chrono = { version = "0.4.35", features = ["serde"] }
chrono = { version = "=0.4.39", features = ["serde"] }
chrono-tz = "^0.10.1"
tracing = "^0"
tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
@@ -211,20 +212,20 @@ itertools = "^0"
regex = "^1"
semver = "^1"
v8 = "=134.4.0" # Exact version
deno_fetch = "0.216.0"
deno_tls = "0.179.0"
deno_console = "0.192.0"
deno_url = "0.192.0"
deno_webidl = "0.192.0"
deno_web = "0.223.0"
deno_io = "0.102.0"
deno_net = "0.184.0"
deno_core = "0.338.0"
v8 = "=130.0.7" # Exact version
deno_fetch = "0.214.0"
deno_tls = "0.177.0"
deno_console = "0.190.0"
deno_url = "0.190.0"
deno_webidl = "0.190.0"
deno_web = "0.221.0"
deno_io = "0.100.0"
deno_net = "0.182.0"
deno_core = "0.336.0"
deno_ast = { version = "=0.44.0", features = ["transpiling"] }
deno_permissions = "0.51.0"
deno_runtime = { version = "0.200.0", features = ["transpile"] }
deno_telemetry = "0.14.0"
deno_permissions = "0.49.0"
deno_runtime = { version = "0.198.0", features = ["transpile"] }
deno_telemetry = "0.12.0"
deno_error = "=0.5.5"
swc_common = "=0.37.5"
@@ -355,3 +356,4 @@ tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] }
tree-sitter = {version = "0.23.0", features = []}
tree-sitter-c-sharp = "0.23.0"
oracle = { version = "0.6.3", features = ["chrono"] }
rumqttc = { version = "0.24.0", features = ["use-native-tls"]}
backend/custom_migrations/grant_all_current_schema.sql
+27
View File
@@ -0,0 +1,27 @@
DO
$$
DECLARE
tbl_name text;
policy_exists boolean;
current_sch text;
tbl_names text[] := ARRAY['account', 'app', 'audit', 'capture', 'completed_job', 'flow', 'folder', 'http_trigger', 'queue', 'raw_app', 'resource', 'schedule', 'script', 'usr_to_group', 'variable'];
BEGIN
-- Get the current schema
SELECT current_schema() INTO current_sch;
FOR tbl_name IN SELECT unnest(tbl_names)
LOOP
SELECT EXISTS (
SELECT 1
FROM pg_policies
WHERE schemaname = current_sch
AND tablename = tbl_name
AND policyname = 'admin_policy'
) INTO policy_exists;
IF NOT policy_exists THEN
EXECUTE format('CREATE POLICY admin_policy ON %I.%I TO windmill_admin USING (true);', current_sch, tbl_name);
END IF;
END LOOP;
END;
$$;
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
2dca712cde013e7dda96dc3162e47a148973a0dc
d3e20c73b06a7c6820868769a49b5fa621591653
backend/migrations/20250205131521_clear_cache_on_webhook_changes.down.sql
+4
View File
@@ -0,0 +1,4 @@
-- Add down migration script here
DROP TRIGGER webhook_change_trigger ON workspace_settings;
DROP FUNCTION notify_webhook_change();
backend/migrations/20250205131521_clear_cache_on_webhook_changes.up.sql
+15
View File
@@ -0,0 +1,15 @@
-- Add up migration script here
CREATE OR REPLACE FUNCTION notify_webhook_change()
RETURNS TRIGGER AS $$
BEGIN
PERFORM pg_notify('notify_webhook_change', NEW.workspace_id);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER webhook_change_trigger
AFTER UPDATE OF webhook ON workspace_settings
FOR EACH ROW
WHEN (OLD.webhook IS DISTINCT FROM NEW.webhook)
EXECUTE FUNCTION notify_webhook_change();
backend/migrations/20250205131522_add_zombie_job_counter.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add down migration script here
DROP TABLE zombie_job_counter;
backend/migrations/20250205131522_add_zombie_job_counter.up.sql
+7
View File
@@ -0,0 +1,7 @@
-- Add up migration script here
CREATE TABLE IF NOT EXISTS zombie_job_counter (
job_id UUID PRIMARY KEY REFERENCES v2_job (id) ON DELETE CASCADE,
counter INTEGER NOT NULL DEFAULT 0
);
backend/migrations/20250205131523_grant_all_in_current_schema.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20250205131523_grant_all_in_current_schema.up.sql
+37
View File
@@ -0,0 +1,37 @@
DO
$do$
DECLARE
current_schema_name TEXT;
BEGIN
-- Get the current schema for the session
SELECT current_schema() INTO current_schema_name;
-- Lock the roles table to prevent race conditions
LOCK TABLE pg_catalog.pg_roles;
EXECUTE format('GRANT USAGE ON SCHEMA %I TO windmill_user', current_schema_name);
EXECUTE format('GRANT USAGE ON SCHEMA %I TO windmill_admin', current_schema_name);
-- Grant privileges dynamically to the current schema
EXECUTE format('GRANT ALL ON ALL TABLES IN SCHEMA %I TO windmill_user', current_schema_name);
EXECUTE format('GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA %I TO windmill_user', current_schema_name);
-- Alter default privileges dynamically
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON TABLES TO windmill_user', current_schema_name);
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON SEQUENCES TO windmill_user', current_schema_name);
-- Grant privileges dynamically to the current schema
EXECUTE format('GRANT ALL ON ALL TABLES IN SCHEMA %I TO windmill_admin', current_schema_name);
EXECUTE format('GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA %I TO windmill_admin', current_schema_name);
-- Alter default privileges dynamically
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON TABLES TO windmill_admin', current_schema_name);
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON SEQUENCES TO windmill_admin', current_schema_name);
EXCEPTION WHEN OTHERS THEN
RAISE NOTICE 'Error granting proper permissions to windmill users: %', SQLERRM;
END
$do$;
backend/migrations/20250207172929_mqtt_trigger.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add down migration script here
DROP TABLE mqtt_trigger;
DROP TYPE MQTT_CLIENT_VERSION;
backend/migrations/20250207172929_mqtt_trigger.up.sql
+72
View File
@@ -0,0 +1,72 @@
-- Add up migration script here
CREATE TYPE MQTT_CLIENT_VERSION AS ENUM ('v3', 'v5');
CREATE TABLE mqtt_trigger (
mqtt_resource_path VARCHAR(255) NOT NULL,
subscribe_topics JSONB[] NOT NULL,
client_version MQTT_CLIENT_VERSION DEFAULT 'v5' NOT NULL,
v5_config JSONB NULL,
v3_config JSONB NULL,
client_id VARCHAR(65535) DEFAULT NULL,
path VARCHAR(255) NOT NULL,
script_path VARCHAR(255) NOT NULL,
is_flow BOOLEAN NOT NULL,
workspace_id VARCHAR(50) NOT NULL,
edited_by VARCHAR(50) NOT NULL,
email VARCHAR(255) NOT NULL,
edited_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
extra_perms JSONB NOT NULL DEFAULT '{}',
server_id VARCHAR(50) NULL,
last_server_ping TIMESTAMPTZ NULL,
error TEXT NULL,
enabled BOOLEAN NOT NULL,
PRIMARY KEY (path, workspace_id)
);
GRANT ALL ON mqtt_trigger TO windmill_user;
GRANT ALL ON mqtt_trigger TO windmill_admin;
ALTER TABLE mqtt_trigger ENABLE ROW LEVEL SECURITY;
CREATE POLICY admin_policy ON mqtt_trigger FOR ALL TO windmill_admin USING (true);
CREATE POLICY see_folder_extra_perms_user_select ON mqtt_trigger FOR SELECT TO windmill_user
USING (SPLIT_PART(mqtt_trigger.path, '/', 1) = 'f' AND SPLIT_PART(mqtt_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_insert ON mqtt_trigger FOR INSERT TO windmill_user
WITH CHECK (SPLIT_PART(mqtt_trigger.path, '/', 1) = 'f' AND SPLIT_PART(mqtt_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_update ON mqtt_trigger FOR UPDATE TO windmill_user
USING (SPLIT_PART(mqtt_trigger.path, '/', 1) = 'f' AND SPLIT_PART(mqtt_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_delete ON mqtt_trigger FOR DELETE TO windmill_user
USING (SPLIT_PART(mqtt_trigger.path, '/', 1) = 'f' AND SPLIT_PART(mqtt_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_own ON mqtt_trigger FOR ALL TO windmill_user
USING (SPLIT_PART(mqtt_trigger.path, '/', 1) = 'u' AND SPLIT_PART(mqtt_trigger.path, '/', 2) = current_setting('session.user'));
CREATE POLICY see_member ON mqtt_trigger FOR ALL TO windmill_user
USING (SPLIT_PART(mqtt_trigger.path, '/', 1) = 'g' AND SPLIT_PART(mqtt_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
CREATE POLICY see_extra_perms_user_select ON mqtt_trigger FOR SELECT TO windmill_user
USING (extra_perms ? CONCAT('u/', current_setting('session.user')));
CREATE POLICY see_extra_perms_user_insert ON mqtt_trigger FOR INSERT TO windmill_user
WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_user_update ON mqtt_trigger FOR UPDATE TO windmill_user
USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_user_delete ON mqtt_trigger FOR DELETE TO windmill_user
USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_groups_select ON mqtt_trigger FOR SELECT TO windmill_user
USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]);
CREATE POLICY see_extra_perms_groups_insert ON mqtt_trigger FOR INSERT TO windmill_user
WITH CHECK (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
CREATE POLICY see_extra_perms_groups_update ON mqtt_trigger FOR UPDATE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
CREATE POLICY see_extra_perms_groups_delete ON mqtt_trigger FOR DELETE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
backend/migrations/20250207173304_add_mqtt_type_value_to_trigger_kind_type.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20250207173304_add_mqtt_type_value_to_trigger_kind_type.up.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add up migration script here
ALTER TYPE TRIGGER_KIND ADD VALUE IF NOT EXISTS 'mqtt';
backend/migrations/20250224131521_flow_workspace_runnables.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE flow_workspace_runnables;
backend/migrations/20250224131521_flow_workspace_runnables.up.sql
+12
View File
@@ -0,0 +1,12 @@
CREATE TABLE flow_workspace_runnables (
flow_path VARCHAR(255) NOT NULL,
runnable_path VARCHAR(255) NOT NULL,
script_hash BIGINT NULL,
runnable_is_flow BOOLEAN NOT NULL,
workspace_id VARCHAR(50) NOT NULL,
FOREIGN KEY (workspace_id, flow_path) REFERENCES flow (workspace_id, path) ON DELETE CASCADE
);
CREATE UNIQUE INDEX flow_workspace_without_hash_unique_idx ON flow_workspace_runnables (flow_path, runnable_path, runnable_is_flow, workspace_id) WHERE script_hash IS NULL;
CREATE UNIQUE INDEX flow_workspace_with_hash_unique_idx ON flow_workspace_runnables (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) WHERE script_hash IS NOT NULL;
CREATE INDEX flow_workspace_runnable_path_is_flow_idx ON flow_workspace_runnables (runnable_path, runnable_is_flow, workspace_id);
backend/package-lock.json
Generated
-6
View File
@@ -1,6 +0,0 @@
{
"name": "backend",
"lockfileVersion": 3,
"requires": true,
"packages": {}
}
backend/parsers/windmill-parser-py/src/lib.rs
+72 -12
View File
@@ -96,11 +96,11 @@ pub fn parse_python_signature(
.iter()
.enumerate()
.map(|(i, x)| {
let mut typ = x
let (mut typ, has_default) = x
.as_arg()
.annotation
.as_ref()
.map_or(Typ::Unknown, |e| parse_expr(e));
.map_or((Typ::Unknown, false), |e| parse_expr(e));
let default = if i >= def_arg_start {
params
@@ -138,7 +138,7 @@ pub fn parse_python_signature(
otyp: None,
name: x.as_arg().arg.to_string(),
typ,
has_default: default.is_some(),
has_default: has_default || default.is_some(),
default,
oidx: None,
}
@@ -158,17 +158,27 @@ pub fn parse_python_signature(
}
}
fn parse_expr(e: &Box<Expr>) -> Typ {
fn parse_expr(e: &Box<Expr>) -> (Typ, bool) {
match e.as_ref() {
Expr::Name(ExprName { id, .. }) => parse_typ(id.as_ref()),
Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref()), false),
Expr::Attribute(x) => {
if x.value
.as_name_expr()
.is_some_and(|x| x.id.as_str() == "wmill")
{
parse_typ(x.attr.as_str())
(parse_typ(x.attr.as_str()), false)
} else {
Typ::Unknown
(Typ::Unknown, false)
}
}
Expr::BinOp(x) => {
if matches!(
x.right.as_ref(),
Expr::Constant(ExprConstant { value: Constant::None, .. })
) {
(parse_expr(&x.left).0, true)
} else {
(Typ::Unknown, false)
}
}
Expr::Subscript(x) => match x.value.as_ref() {
@@ -193,14 +203,15 @@ fn parse_expr(e: &Box<Expr>) -> Typ {
}
_ => None,
};
Typ::Str(values)
(Typ::Str(values), false)
}
"List" => Typ::List(Box::new(parse_expr(&x.slice))),
_ => Typ::Unknown,
"List" => (Typ::List(Box::new(parse_expr(&x.slice).0)), false),
"Optional" => (parse_expr(&x.slice).0, true),
_ => (Typ::Unknown, false),
},
_ => Typ::Unknown,
_ => (Typ::Unknown, false),
},
_ => Typ::Unknown,
_ => (Typ::Unknown, false),
}
}
@@ -676,4 +687,53 @@ def main(a: list, e: List[int], b: list = [1,2,3,4], c = [1,2,3,4], d = ["a", "b
Ok(())
}
#[test]
fn test_parse_python_sig_9() -> anyhow::Result<()> {
let code = r#"
from typing import Optional
def main(a: str, b: Optional[str], c: str | None): return
"#;
println!(
"{}",
serde_json::to_string(&parse_python_signature(code, None, false)?)?
);
assert_eq!(
parse_python_signature(code, None, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![
Arg {
otyp: None,
name: "a".to_string(),
typ: Typ::Str(None),
default: None,
has_default: false,
oidx: None
},
Arg {
otyp: None,
name: "b".to_string(),
typ: Typ::Str(None),
default: None,
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "c".to_string(),
typ: Typ::Str(None),
default: None,
has_default: true,
oidx: None
},
],
no_main_func: Some(false),
has_preprocessor: Some(false)
}
);
Ok(())
}
}
backend/parsers/windmill-parser-wasm/tests/wasm.rs
+9
View File
@@ -4,6 +4,7 @@ use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
use windmill_parser_bash::parse_powershell_sig;
use windmill_parser_ts::{parse_deno_signature, parse_expr_for_ids, parse_expr_for_imports};
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_deno_sig() -> anyhow::Result<()> {
let code = "
@@ -146,6 +147,8 @@ export function main(test1?: string, test2: string = \"burkina\",
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_deno_sig_implicit_types() -> anyhow::Result<()> {
let code = "
@@ -224,6 +227,7 @@ export function main(test2 = \"burkina\",
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_deno_types() -> anyhow::Result<()> {
let code = "
@@ -274,6 +278,7 @@ export function main(foo: FooBar, {a, b}: FooBar, {c, d}: FooBar = {a: \"foo\",
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_enum_list() -> anyhow::Result<()> {
let code = "
@@ -305,6 +310,7 @@ export function main(foo: (\"foo\" | \"bar\")[]) {
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_extract_ident() -> anyhow::Result<()> {
let code = "
@@ -324,6 +330,7 @@ fn test_parse_extract_ident() -> anyhow::Result<()> {
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_imports() -> anyhow::Result<()> {
let code = "
@@ -347,6 +354,7 @@ fn test_parse_imports() -> anyhow::Result<()> {
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_imports_dts() -> anyhow::Result<()> {
let code = "
@@ -359,6 +367,7 @@ export type foo = number
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_powershell_sig() -> anyhow::Result<()> {
let code = "
backend/src/main.rs
+40 -61
View File
@@ -44,7 +44,7 @@ use windmill_common::{
},
scripts::ScriptLang,
stats_ee::schedule_stats,
utils::{hostname, rd_string, Mode, GIT_VERSION},
utils::{hostname, rd_string, Mode, GIT_VERSION, MODE_AND_ADDONS},
worker::{reload_custom_tags_setting, HUB_CACHE_DIR, TMP_DIR, TMP_LOGS_DIR, WORKER_GROUP},
DB, METRICS_ENABLED,
};
@@ -115,6 +115,25 @@ where
}
pub fn main() -> anyhow::Result<()> {
// https://github.com/denoland/deno/blob/main/cli/main.rs#L477
#[cfg(feature = "deno_core")]
let unrecognized_v8_flags = deno_core::v8_set_flags(vec![
"--stack-size=1024".to_string(),
// TODO(bartlomieju): I think this can be removed as it's handled by `deno_core`
// and its settings.
// deno_ast removes TypeScript `assert` keywords, so this flag only affects JavaScript
// TODO(petamoriken): Need to check TypeScript `assert` keywords in deno_ast
"--no-harmony-import-assertions".to_string(),
])
.into_iter()
.skip(1)
.collect::<Vec<_>>();
#[cfg(feature = "deno_core")]
if !unrecognized_v8_flags.is_empty() {
println!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags);
}
#[cfg(feature = "deno_core")]
deno_core::JsRuntime::init_platform(None, false);
create_and_run_current_thread_inner(windmill_main())
@@ -219,63 +238,12 @@ async fn windmill_main() -> anyhow::Result<()> {
let hostname = hostname();
let mut enable_standalone_indexer: bool = false;
let mode_and_addons = MODE_AND_ADDONS.clone();
let mode = mode_and_addons.mode;
let mode = std::env::var("MODE")
.map(|x| x.to_lowercase())
.map(|x| {
if &x == "server" {
println!("server mode");
Mode::Server
} else if &x == "worker" {
tracing::info!("worker mode");
#[cfg(windows)]
{
println!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL");
}
Mode::Worker
} else if &x == "agent" {
println!("worker agent mode");
if std::env::var("BASE_INTERNAL_URL").is_err() {
panic!("BASE_INTERNAL_URL is required in agent mode")
}
if std::env::var("JOB_TOKEN").is_err() {
println!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table")
}
#[cfg(not(feature = "enterprise"))]
{
panic!("Agent mode is only available in the EE, ignoring...");
}
#[cfg(feature = "enterprise")]
Mode::Agent
} else if &x == "indexer" {
tracing::info!("indexer mode");
#[cfg(not(feature = "tantivy"))]
{
eprintln!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features.");
panic!("Indexer mode requires compiling with the tantivy feature flag.");
}
#[cfg(feature = "tantivy")]
Mode::Indexer
} else if &x == "standalone+search"{
enable_standalone_indexer = true;
println!("standalone mode with search enabled");
Mode::Standalone
}
else {
if &x != "standalone" {
eprintln!("mode not recognized, defaulting to standalone: {x}");
} else {
println!("standalone mode");
}
Mode::Standalone
}
})
.unwrap_or_else(|_| {
tracing::info!("Mode not specified, defaulting to standalone");
Mode::Standalone
});
if mode == Mode::Standalone {
println!("Running in standalone mode");
}
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
println!("jemalloc enabled");
@@ -518,8 +486,7 @@ Windmill Community Edition {GIT_VERSION}
.expect("could not create initial server dir");
#[cfg(feature = "tantivy")]
let should_index_jobs =
mode == Mode::Indexer || (enable_standalone_indexer && mode == Mode::Standalone);
let should_index_jobs = mode == Mode::Indexer || mode_and_addons.indexer;
reload_indexer_config(&db).await;
@@ -722,6 +689,11 @@ Windmill Community Edition {GIT_VERSION}
}
}
},
"notify_webhook_change" => {
let workspace_id = n.payload();
tracing::info!("Webhook change detected, invalidating webhook cache: {}", workspace_id);
windmill_api::webhook_util::WEBHOOK_CACHE.remove(workspace_id);
},
"notify_global_setting_change" => {
tracing::info!("Global setting change detected: {}", n.payload());
match n.payload() {
@@ -960,7 +932,11 @@ async fn listen_pg(db: &DB) -> Option<PgListener> {
};
if let Err(e) = listener
.listen_all(vec!["notify_config_change", "notify_global_setting_change"])
.listen_all(vec![
"notify_config_change",
"notify_global_setting_change",
"notify_webhook_change",
])
.await
{
tracing::error!(error = %e, "Could not listen to database");
@@ -1119,9 +1095,12 @@ pub async fn run_workers(
async fn send_delayed_killpill(
tx: &tokio::sync::broadcast::Sender<()>,
max_delay_secs: u64,
mut max_delay_secs: u64,
context: &str,
) {
if max_delay_secs == 0 {
max_delay_secs = 1;
}
// Random delay to avoid all servers/workers shutting down simultaneously
let rd_delay = rand::rng().random_range(0..max_delay_secs);
tracing::info!("Scheduling {context} shutdown in {rd_delay}s");
backend/src/monitor.rs
+189 -40
View File
@@ -34,7 +34,6 @@ use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts};
#[cfg(feature = "oauth2")]
use windmill_common::global_settings::OAUTH_SETTING;
use windmill_common::{
auth::JWT_SECRET,
ee::CriticalErrorChannel,
error,
flow_status::{FlowStatus, FlowStatusModule},
@@ -51,6 +50,7 @@ use windmill_common::{
},
indexer::load_indexer_config,
jobs::QueuedJob,
jwt::JWT_SECRET,
oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH,
server::load_smtp_config,
tracing_init::JSON_FMT,
@@ -242,19 +242,40 @@ pub async fn load_otel(db: &DB) {
if let Some(v) = v {
let deser = serde_json::from_value::<OtelSetting>(v);
if let Ok(o) = deser {
let metrics_enabled = o.metrics_enabled.unwrap_or(false);
let logs_enabled = o.logs_enabled.unwrap_or(false);
let tracing_enabled = o.tracing_enabled.unwrap_or(false);
let metrics_enabled = o.metrics_enabled.unwrap_or_else(|| {
std::env::var("OTEL_METRICS_ENABLED")
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false)
});
let logs_enabled = o.logs_enabled.unwrap_or_else(|| {
std::env::var("OTEL_LOGS_ENABLED")
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false)
});
let tracing_enabled = o.tracing_enabled.unwrap_or_else(|| {
std::env::var("OTEL_TRACING_ENABLED")
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false)
});
OTEL_METRICS_ENABLED.store(metrics_enabled, Ordering::Relaxed);
OTEL_LOGS_ENABLED.store(logs_enabled, Ordering::Relaxed);
OTEL_TRACING_ENABLED.store(tracing_enabled, Ordering::Relaxed);
if let Some(endpoint) = o.otel_exporter_otlp_endpoint.as_ref() {
std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
}
if let Some(headers) = o.otel_exporter_otlp_headers.as_ref() {
std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", headers);
}
let endpoint = if let Some(endpoint) = o.otel_exporter_otlp_endpoint {
std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint.clone());
Some(endpoint.clone())
} else {
std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok()
};
let headers = if let Some(headers) = o.otel_exporter_otlp_headers {
std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", headers.clone());
Some(headers.clone())
} else {
std::env::var("OTEL_EXPORTER_OTLP_HEADERS").ok()
};
if let Some(protocol) = o.otel_exporter_otlp_protocol {
std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", protocol);
}
@@ -262,7 +283,7 @@ pub async fn load_otel(db: &DB) {
std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", compression);
}
println!("OTEL settings loaded: tracing ({tracing_enabled}), logs ({logs_enabled}), metrics ({metrics_enabled}), endpoint ({:?}), headers defined: ({})",
o.otel_exporter_otlp_endpoint, o.otel_exporter_otlp_headers.is_some());
endpoint, headers.is_some());
} else {
tracing::error!("Error deserializing otel settings");
}
@@ -731,6 +752,22 @@ pub async fn delete_expired_items(db: &DB) -> () {
Err(e) => tracing::error!("Error deleting log file: {:?}", e),
}
#[cfg(not(feature = "enterprise"))]
let audit_retention_secs = 1 * 60 * 60 * 24 * 14;
#[cfg(feature = "enterprise")]
let audit_retention_secs = 1 * 60 * 60 * 24 * 365;
if let Err(e) = sqlx::query_scalar!(
"DELETE FROM audit WHERE timestamp <= now() - ($1::bigint::text || ' s')::interval",
audit_retention_secs,
)
.fetch_all(db)
.await
{
tracing::error!("Error deleting audit log on CE: {:?}", e);
}
let job_retention_secs = *JOB_RETENTION_SECS.read().await;
if job_retention_secs > 0 {
match db.begin().await {
@@ -1517,18 +1554,44 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
Ok(())
}
const RESTART_LIMIT: i32 = 3;
async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker_name: &str) {
let mut zombie_jobs_uuid_restart_limit_reached = vec![];
if *RESTART_ZOMBIE_JOBS {
let restarted = sqlx::query!(
"WITH zombie_jobs AS (
UPDATE v2_job_queue q SET running = false, started_at = null
FROM v2_job j, v2_job_runtime r
WHERE j.id = q.id AND j.id = r.id
AND ping < now() - ($1 || ' seconds')::interval
"WITH to_update AS (
SELECT q.id, q.workspace_id, r.ping, COALESCE(zjc.counter, 0) as counter
FROM v2_job_queue q
JOIN v2_job j ON j.id = q.id
JOIN v2_job_runtime r ON r.id = j.id
LEFT JOIN zombie_job_counter zjc ON zjc.job_id = q.id
WHERE ping < now() - ($1 || ' seconds')::interval
AND running = true
AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')
AND same_worker = false
RETURNING q.id, q.workspace_id, ping
AND (zjc.counter IS NULL OR zjc.counter <= $2)
FOR UPDATE of q SKIP LOCKED
),
zombie_jobs AS (
UPDATE v2_job_queue q
SET running = false, started_at = null
FROM to_update tu
WHERE q.id = tu.id AND (tu.counter IS NULL OR tu.counter < $2)
RETURNING q.id, q.workspace_id, ping, tu.counter
),
update_ping AS (
UPDATE v2_job_runtime r
SET ping = null
FROM zombie_jobs zj
WHERE r.id = zj.id
),
increment_counter AS (
INSERT INTO zombie_job_counter (job_id, counter)
SELECT id, 1 FROM to_update WHERE counter < $2
ON CONFLICT (job_id) DO UPDATE
SET counter = zombie_job_counter.counter + 1
),
update_concurrency AS (
UPDATE concurrency_counter cc
@@ -1537,8 +1600,9 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
INNER JOIN concurrency_key ck ON ck.job_id = zj.id
WHERE cc.concurrency_id = ck.key
)
SELECT id, workspace_id, ping FROM zombie_jobs",
SELECT id AS \"id!\", workspace_id AS \"workspace_id!\", ping, counter + 1 AS counter FROM to_update",
*ZOMBIE_JOB_TIMEOUT,
RESTART_LIMIT
)
.fetch_all(db)
.await
@@ -1558,22 +1622,61 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
"no last ping".to_string()
};
let url = format!("{}/run/{}?workspace={}", base_url, r.id, r.workspace_id,);
let error_message = format!(
"Zombie job {} on {} ({}) detected, restarting it, {}",
r.id, r.workspace_id, url, last_ping
);
let restart = r.counter.is_none_or(|x| x < RESTART_LIMIT);
let (critical_error_message, restart_message) = if restart {
(
format!(
"Zombie job {} on {} ({}) detected, restarting it ({}/{} attempts), last ping: {}",
r.id,
r.workspace_id,
url,
r.counter.unwrap_or(0) + 1,
RESTART_LIMIT,
last_ping
),
format!(
"Restarted job after not receiving job's ping for too long the {} ({}/{} attempts)\n\n",
last_ping,
r.counter.unwrap_or(0) + 1,
RESTART_LIMIT
)
)
} else {
(
format!(
"Zombie job {} on {} ({}) detected, but restart limit ({}) reached, job will be processed as an error, last ping: {}",
r.id, r.workspace_id, url, RESTART_LIMIT, last_ping
),
format!(
"job's ping was received last at {}, job will be processed as an error since all {} restart attempts failed",
last_ping, RESTART_LIMIT
)
)
};
let _ = sqlx::query!("
let _ = sqlx::query!(
"
INSERT INTO job_logs (job_id, logs)
VALUES ($1, 'Restarted job after not receiving job''s ping for too long the ' || now() || '\n\n')
VALUES ($1, $2)
ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\n' || EXCLUDED.logs
WHERE job_logs.job_id = $1",
r.id
r.id,
restart_message
)
.execute(db)
.await;
tracing::error!(error_message);
report_critical_error(error_message, db.clone(), Some(&r.workspace_id), None).await;
tracing::error!(critical_error_message);
report_critical_error(
critical_error_message,
db.clone(),
Some(&r.workspace_id),
None,
)
.await;
if !restart {
zombie_jobs_uuid_restart_limit_reached.push(r.id);
}
}
}
@@ -1648,9 +1751,43 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
.unwrap_or_else(|| vec![])
};
enum ErrorMessage {
RestartLimit,
SameWorker,
RestartDisabled,
}
impl ErrorMessage {
fn to_string(&self) -> String {
match self {
ErrorMessage::RestartLimit => format!("RestartLimit ({})", RESTART_LIMIT),
ErrorMessage::SameWorker => "SameWorker".to_string(),
ErrorMessage::RestartDisabled => "RestartDisabled".to_string(),
}
}
}
let zombie_jobs_restart_limit_reached =
sqlx::query_as::<_, QueuedJob>("SELECT * FROM v2_as_queue WHERE id = ANY($1)")
.bind(&zombie_jobs_uuid_restart_limit_reached[..])
.fetch_all(db)
.await
.ok()
.unwrap_or_else(|| vec![]);
let timeouts = non_restartable_jobs
.into_iter()
.chain(same_worker_timeout_jobs)
.map(|x| (x, ErrorMessage::RestartDisabled))
.chain(
same_worker_timeout_jobs
.into_iter()
.map(|x| (x, ErrorMessage::SameWorker)),
)
.chain(
zombie_jobs_restart_limit_reached
.into_iter()
.map(|x| (x, ErrorMessage::RestartLimit)),
)
.collect::<Vec<_>>();
#[cfg(feature = "prometheus")]
@@ -1658,7 +1795,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _);
}
for job in timeouts {
for (job, error_kind) in timeouts {
// since the job is unrecoverable, the same worker queue should never be sent anything
let (same_worker_tx_never_used, _same_worker_rx_never_used) =
mpsc::channel::<SameWorkerPayload>(1);
@@ -1693,20 +1830,21 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
};
let last_ping = job.last_ping.clone();
let error_message = format!(
"Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, reason: {:?})",
last_ping
.map(|x| x.to_string())
.unwrap_or_else(|| "no ping".to_string()),
*ZOMBIE_JOB_TIMEOUT,
error_kind.to_string()
);
let _ = handle_job_error(
db,
&client,
&job,
0,
None,
error::Error::ExecutionErr(format!(
"Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, same_worker: {})",
last_ping
.map(|x| x.to_string())
.unwrap_or_else(|| "no ping".to_string()),
*ZOMBIE_JOB_TIMEOUT,
job.same_worker
)),
error::Error::ExecutionErr(error_message),
true,
same_worker_tx_never_used,
"",
@@ -1785,16 +1923,27 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
let id = flow.id.clone();
let last_ping = flow.last_ping.clone();
let now = now_from_db(db).await?;
let base_url = BASE_URL.read().await;
let workspace_id = flow.workspace_id.clone();
let reason = format!(
"{} was hanging in between 2 steps. Last ping: {last_ping:?} (now: {now})",
if flow.is_flow_step.unwrap_or(false) && flow.parent_job.is_some() {
format!("Flow was cancelled because subflow {id}")
format!("Flow was cancelled because subflow {id} ({base_url}/run/{id}?workspace={workspace_id})")
} else {
format!("Flow {id} was cancelled because it")
format!("Flow {id} ({base_url}/run/{id}?workspace={workspace_id}) was cancelled because it")
}
);
report_critical_error(reason.clone(), db.clone(), Some(&flow.workspace_id), None).await;
cancel_zombie_flow_job(db, flow.id, &flow.workspace_id, reason).await?;
cancel_zombie_flow_job(db, flow.id, &flow.workspace_id,
format!(r#"{reason}
This would happen if a worker was interrupted, killed or crashed while doing a state transition at the end of a job which is always an unexpected behavior that should never happen.
Please check your worker logs for more details and feel free to report it to the Windmill team on our Discord or support@windmill.dev (response for non EE customers will be best effort) with as much context as possible, ideally:
- Windmill version
- Worker logs right after the job referenced has finished running
- Is the error consistent when running the same flow
- A minimal flow and its flow.yaml that reproduces the error and that is importable in a fresh workspace
- Your infra setup (helm, docker-compose, configuration of the workers and their number, memory of the database, etc.)
"#)).await?;
}
}
backend/tests/worker.rs
+1 -1
View File
@@ -22,12 +22,12 @@ use windmill_api_client::types::{CreateFlowBody, RawScript};
use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs};
use serde::Serialize;
use windmill_common::auth::JWT_SECRET;
use windmill_common::worker::WORKER_CONFIG;
use windmill_common::{
flow_status::{FlowStatus, FlowStatusModule, RestartedFrom},
flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
jobs::{JobKind, JobPayload, RawCode},
jwt::JWT_SECRET,
scripts::{ScriptHash, ScriptLang},
worker::{
MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440,
backend/update_sqlx.sh
+19
View File
@@ -1,3 +1,22 @@
./substitute_ee_code.sh --dir ../windmill-ee-private
# Check if running on macOS
if [[ "$(uname)" == "Darwin" ]]; then
echo "Running on macOS - substituting samael..."
# Comment out the version-based samael dependency
sed -i '' 's/^samael = { version="0.0.14", features = \["xmlsec"\] }/#samael = { version="0.0.14", features = ["xmlsec"] }/' Cargo.toml
# Uncomment the git-based samael dependency
sed -i '' 's/^# \(samael = { git="https:\/\/github.com\/njaremko\/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = \["xmlsec"\] }\)/\1/' Cargo.toml
fi
cargo sqlx prepare --workspace -- --all-targets --all-features
./substitute_ee_code.sh -r --dir ../windmill-ee-private
# Undo the samael changes on macOS
if [[ "$(uname)" == "Darwin" ]]; then
echo "Reverting samael changes..."
# Uncomment the version-based samael dependency
sed -i '' 's/^#samael = { version="0.0.14", features = \["xmlsec"\] }/samael = { version="0.0.14", features = ["xmlsec"] }/' Cargo.toml
# Comment out the git-based samael dependency
sed -i '' 's/^\(samael = { git="https:\/\/github.com\/njaremko\/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = \["xmlsec"\] }\)/# \1/' Cargo.toml
fi
backend/windmill-api/Cargo.toml
+3 -1
View File
@@ -29,6 +29,7 @@ oauth2 = ["dep:async-oauth2"]
http_trigger = ["dep:matchit"]
static_frontend = ["dep:rust-embed"]
postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:thiserror", "dep:rust_decimal", "dep:rust-postgres-native-tls"]
mqtt_trigger = ["dep:thiserror", "dep:rumqttc"]
sqs_trigger = ["dep:aws-sdk-sqs", "dep:thiserror", "dep:aws-config"]
[dependencies]
@@ -120,5 +121,6 @@ byteorder = { workspace = true, optional = true }
thiserror = { workspace = true, optional = true }
rust_decimal = { workspace = true, optional = true }
rust-postgres-native-tls = { workspace = true, optional = true}
rumqttc = { workspace = true, optional = true }
aws-sdk-sqs = { workspace = true, optional = true }
aws-config = { workspace = true, optional = true}
aws-config = { workspace = true, optional = true}
backend/windmill-api/openapi.yaml
+473 -4
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.467.1
version: 1.475.0
title: Windmill API
contact:
@@ -126,6 +126,11 @@ paths:
type: string
- $ref: "#/components/parameters/ResourceName"
- $ref: "#/components/parameters/ActionKind"
- name: all_workspaces
in: query
description: get audit logs for all workspaces
schema:
type: boolean
responses:
"200":
@@ -2611,6 +2616,8 @@ paths:
type: boolean
postgres_used:
type: boolean
mqtt_used:
type: boolean
sqs_used:
type: boolean
required:
@@ -2619,6 +2626,7 @@ paths:
- kafka_used
- nats_used
- postgres_used
- mqtt_used
- sqs_used
/w/{workspace}/users/list:
get:
@@ -5287,6 +5295,26 @@ paths:
schema:
$ref: "#/components/schemas/FlowVersion"
/w/{workspace}/flows/list_paths_from_workspace_runnable/{runnable_kind}/{path}:
get:
summary: list flow paths from workspace runnable
operationId: listFlowPathsFromWorkspaceRunnable
tags:
- flow
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/RunnableKind"
- $ref: "#/components/parameters/ScriptPath"
responses:
"200":
description: list of flow paths
content:
application/json:
schema:
type: array
items:
type: string
/w/{workspace}/flows/get/v/{version}/p/{path}:
get:
summary: get flow version
@@ -6207,6 +6235,92 @@ paths:
schema:
type: string
/w/{workspace}/apps_u/upload_s3_file/{path}:
post:
summary: upload s3 file from app
operationId: uploadS3FileFromApp
tags:
- app
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
- name: file_key
in: query
required: false
schema:
type: string
- name: file_extension
in: query
required: false
schema:
type: string
- name: s3_resource_path
in: query
required: false
schema:
type: string
- name: resource_type
in: query
required: false
schema:
type: string
- name: storage
in: query
schema:
type: string
- name: content_type
in: query
schema:
type: string
- name: content_disposition
in: query
schema:
type: string
requestBody:
description: File content
required: true
content:
application/octet-stream:
schema:
type: string
format: binary
responses:
"200":
description: file uploaded
content:
application/json:
schema:
type: object
properties:
file_key:
type: string
delete_token:
type: string
required:
- file_key
- delete_token
/w/{workspace}/apps_u/delete_s3_file:
delete:
summary: delete s3 file from app
operationId: deleteS3FileFromApp
tags:
- app
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: delete_token
in: query
required: true
schema:
type: string
responses:
"200":
description: file deleted
content:
text/plain:
schema:
type: string
/w/{workspace}/jobs/run/f/{path}:
post:
summary: run flow by path
@@ -8777,6 +8891,195 @@ paths:
schema:
type: string
/w/{workspace}/mqtt_triggers/create:
post:
summary: create mqtt trigger
operationId: createMqttTrigger
tags:
- mqtt_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: new mqtt trigger
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/NewMqttTrigger"
responses:
"201":
description: mqtt trigger created
content:
text/plain:
schema:
type: string
/w/{workspace}/mqtt_triggers/update/{path}:
post:
summary: update mqtt trigger
operationId: updateMqttTrigger
tags:
- mqtt_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
requestBody:
description: updated trigger
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/EditMqttTrigger"
responses:
"200":
description: mqtt trigger updated
content:
text/plain:
schema:
type: string
/w/{workspace}/mqtt_triggers/delete/{path}:
delete:
summary: delete mqtt trigger
operationId: deleteMqttTrigger
tags:
- mqtt_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: mqtt trigger deleted
content:
text/plain:
schema:
type: string
/w/{workspace}/mqtt_triggers/get/{path}:
get:
summary: get mqtt trigger
operationId: getMqttTrigger
tags:
- mqtt_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: mqtt trigger deleted
content:
application/json:
schema:
$ref: "#/components/schemas/MqttTrigger"
/w/{workspace}/mqtt_triggers/list:
get:
summary: list mqtt triggers
operationId: listMqttTriggers
tags:
- mqtt_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
required: true
- $ref: "#/components/parameters/Page"
- $ref: "#/components/parameters/PerPage"
- name: path
description: filter by path
in: query
schema:
type: string
- name: is_flow
in: query
schema:
type: boolean
- name: path_start
in: query
schema:
type: string
responses:
"200":
description: mqtt trigger list
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/MqttTrigger"
/w/{workspace}/mqtt_triggers/exists/{path}:
get:
summary: does mqtt trigger exists
operationId: existsMqttTrigger
tags:
- mqtt_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: mqtt trigger exists
content:
application/json:
schema:
type: boolean
/w/{workspace}/mqtt_triggers/setenabled/{path}:
post:
summary: set enabled mqtt trigger
operationId: setMqttTriggerEnabled
tags:
- mqtt_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
requestBody:
description: updated mqtt trigger enable
required: true
content:
application/json:
schema:
type: object
properties:
enabled:
type: boolean
required:
- enabled
responses:
"200":
description: mqtt trigger enabled set
content:
text/plain:
schema:
type: string
/w/{workspace}/mqtt_triggers/test:
post:
summary: test mqtt connection
operationId: testMqttConnection
tags:
- mqtt_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: test mqtt connection
required: true
content:
application/json:
schema:
type: object
properties:
connection:
type: object
required:
- connection
responses:
"200":
description: successfully connected to mqtt
content:
text/plain:
schema:
type: string
/w/{workspace}/postgres_triggers/is_valid_postgres_configuration/{path}:
get:
@@ -9730,6 +10033,23 @@ paths:
application/json:
schema:
$ref: "#/components/schemas/Folder"
/w/{workspace}/folders/exists/{name}:
get:
summary: exists folder
operationId: existsFolder
tags:
- folder
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Name"
responses:
"200":
description: folder exists
content:
application/json:
schema:
type: boolean
/w/{workspace}/folders/getusage/{name}:
get:
@@ -10065,6 +10385,7 @@ paths:
kafka_trigger,
nats_trigger,
postgres_trigger,
mqtt_trigger,
sqs_trigger
]
responses:
@@ -10107,6 +10428,7 @@ paths:
kafka_trigger,
nats_trigger,
postgres_trigger,
mqtt_trigger,
sqs_trigger
]
requestBody:
@@ -10160,6 +10482,7 @@ paths:
kafka_trigger,
nats_trigger,
postgres_trigger,
mqtt_trigger,
sqs_trigger
]
requestBody:
@@ -11092,7 +11415,7 @@ paths:
/w/{workspace}/job_helpers/download_s3_file:
get:
summary: Download file to S3 bucket
summary: Download file from S3 bucket
operationId: fileDownload
tags:
- helpers
@@ -13569,6 +13892,8 @@ components:
type: number
nats_count:
type: number
mqtt_count:
type: number
sqs_count:
type: number
@@ -13718,6 +14043,140 @@ components:
- is_flow
required:
- runnable_result
MqttQoS:
type: string
enum: ['qos0', 'qos1', 'qos2']
MqttV3Config:
type: object
properties:
clean_session:
type: boolean
MqttV5Config:
type: object
properties:
clean_start:
type: boolean
topic_alias:
type: number
session_expiry_interval:
type: number
MqttSubscribeTopic:
type: object
properties:
qos:
$ref: "#/components/schemas/MqttQoS"
topic:
type: string
required:
- qos
- topic
MqttClientVersion:
type: string
enum: ["v3", "v5"]
MqttTrigger:
allOf:
- $ref: "#/components/schemas/TriggerExtraProperty"
type: object
properties:
mqtt_resource_path:
type: string
subscribe_topics:
type: array
items:
$ref: "#/components/schemas/MqttSubscribeTopic"
v3_config:
$ref: "#/components/schemas/MqttV3Config"
v5_config:
$ref: "#/components/schemas/MqttV5Config"
client_id:
type: string
client_version:
$ref: "#/components/schemas/MqttClientVersion"
server_id:
type: string
last_server_ping:
type: string
format: date-time
error:
type: string
enabled:
type: boolean
required:
- enabled
- subscribe_topics
- mqtt_resource_path
NewMqttTrigger:
type: object
properties:
mqtt_resource_path:
type: string
subscribe_topics:
type: array
items:
$ref: "#/components/schemas/MqttSubscribeTopic"
client_id:
type: string
v3_config:
$ref: "#/components/schemas/MqttV3Config"
v5_config:
$ref: "#/components/schemas/MqttV5Config"
client_version:
$ref: "#/components/schemas/MqttClientVersion"
path:
type: string
script_path:
type: string
is_flow:
type: boolean
enabled:
type: boolean
required:
- path
- script_path
- is_flow
- subscribe_topics
- mqtt_resource_path
EditMqttTrigger:
type: object
properties:
mqtt_resource_path:
type: string
subscribe_topics:
type: array
items:
$ref: "#/components/schemas/MqttSubscribeTopic"
client_id:
type: string
v3_config:
$ref: "#/components/schemas/MqttV3Config"
v5_config:
$ref: "#/components/schemas/MqttV5Config"
client_version:
$ref: "#/components/schemas/MqttClientVersion"
path:
type: string
script_path:
type: string
is_flow:
type: boolean
enabled:
type: boolean
required:
- path
- script_path
- is_flow
- enabled
- subscribe_topics
- mqtt_resource_path
SqsTrigger:
allOf:
@@ -14477,6 +14936,15 @@ components:
type: array
items:
type: object
allowed_s3_keys:
type: array
items:
type: object
properties:
s3_path:
type: string
resource:
type: string
execution_mode:
type: string
enum: [viewer, publisher, anonymous]
@@ -14824,6 +15292,7 @@ components:
- resource
- variable
- secret
- trigger
WorkspaceDefaultScripts:
type: object
@@ -15094,7 +15563,7 @@ components:
CaptureTriggerKind:
type: string
enum: [webhook, http, websocket, kafka, email, nats, postgres, sqs]
enum: [webhook, http, websocket, kafka, email, nats, postgres, sqs, mqtt]
Capture:
type: object
@@ -15213,4 +15682,4 @@ components:
service_url:
type: string
description: The service URL for the channel
example: "https://smba.trafficmanager.net/amer/12345678-1234-1234-1234-123456789012/"
example: "https://smba.trafficmanager.net/amer/12345678-1234-1234-1234-123456789012/"
backend/windmill-api/src/apps.rs
+221 -36
View File
@@ -21,7 +21,7 @@ use crate::{
job_helpers_ee::{
download_s3_file_internal, get_random_file_name, get_s3_resource,
get_workspace_s3_resource, load_image_preview_internal, upload_file_from_req,
DownloadFileQuery, LoadImagePreviewQuery, UploadFileResponse,
DownloadFileQuery, LoadImagePreviewQuery,
},
users::fetch_api_authed_from_permissioned_as,
};
@@ -51,8 +51,6 @@ use sqlx::{types::Uuid, FromRow};
use std::str;
use windmill_audit::audit_ee::audit_log;
use windmill_audit::ActionKind;
#[cfg(feature = "parquet")]
use windmill_common::s3_helpers::build_object_store_client;
use windmill_common::variables::encrypt;
use windmill_common::{
apps::{AppScriptId, ListAppQuery},
@@ -69,6 +67,8 @@ use windmill_common::{
worker::{to_raw_value, CLOUD_HOSTED},
HUB_BASE_URL,
};
#[cfg(feature = "parquet")]
use windmill_common::{jwt, s3_helpers::build_object_store_client};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel};
@@ -96,6 +96,7 @@ pub fn unauthed_service() -> Router {
Router::new()
.route("/execute_component/*path", post(execute_component))
.route("/upload_s3_file/*path", post(upload_s3_file_from_app))
.route("/delete_s3_file", delete(delete_s3_file_from_app))
.route("/download_s3_file/*path", get(download_s3_file_from_app))
.route(
"/load_image_preview/*path",
@@ -224,6 +225,12 @@ pub struct S3Input {
file_key_regex: String,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct S3Key {
s3_path: String,
resource: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct Policy {
pub on_behalf_of: Option<String>,
@@ -238,6 +245,7 @@ pub struct Policy {
pub triggerables_v2: Option<HashMap<String, PolicyTriggerableInputs>>,
pub execution_mode: ExecutionMode,
pub s3_inputs: Option<Vec<S3Input>>,
pub allowed_s3_keys: Option<Vec<S3Key>>,
}
#[derive(Deserialize)]
@@ -1497,6 +1505,13 @@ async fn upload_s3_file_from_app() -> Result<()> {
));
}
#[cfg(not(feature = "parquet"))]
async fn delete_s3_file_from_app() -> Result<()> {
return Err(Error::BadRequest(
"This endpoint requires the parquet feature to be enabled".to_string(),
));
}
#[cfg(feature = "parquet")]
#[derive(Debug, Deserialize, Clone)]
struct UploadFileToS3Query {
@@ -1511,6 +1526,24 @@ struct UploadFileToS3Query {
force_viewer_allowed_resources: Option<String>,
}
#[cfg(feature = "parquet")]
#[derive(Serialize, Deserialize)]
struct DeleteTokenClaims {
file_key: String,
on_behalf_of_email: String,
permissioned_as: String,
username: String,
s3_resource_path: Option<String>,
pub exp: usize,
}
#[cfg(feature = "parquet")]
#[derive(Serialize)]
struct AppUploadFileResponse {
file_key: String,
delete_token: String,
}
#[cfg(feature = "parquet")]
async fn upload_s3_file_from_app(
OptAuthed(opt_authed): OptAuthed,
@@ -1518,7 +1551,7 @@ async fn upload_s3_file_from_app(
Path((w_id, path)): Path<(String, StripPath)>,
Query(query): Query<UploadFileToS3Query>,
request: axum::extract::Request,
) -> JsonResult<UploadFileResponse> {
) -> JsonResult<AppUploadFileResponse> {
let policy = if let Some(file_key_regex) = query.force_viewer_file_key_regex {
Some(Policy {
execution_mode: ExecutionMode::Viewer,
@@ -1537,6 +1570,7 @@ async fn upload_s3_file_from_app(
.map(|s| s.split(',').map(|s| s.to_string()).collect())
.unwrap_or_default(),
}]),
allowed_s3_keys: None,
})
} else {
let policy_o = sqlx::query_scalar!(
@@ -1554,7 +1588,10 @@ async fn upload_s3_file_from_app(
let user_db = UserDB::new(db.clone());
let (s3_resource_opt, file_key) = if policy.as_ref().is_some_and(|p| p.s3_inputs.is_some()) {
let (s3_resource_opt, file_key, on_behalf_of_email, permissioned_as, username) = if policy
.as_ref()
.is_some_and(|p| p.s3_inputs.is_some())
{
let policy = policy.unwrap();
let s3_inputs = policy.s3_inputs.as_ref().unwrap();
@@ -1562,11 +1599,11 @@ async fn upload_s3_file_from_app(
get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?;
let on_behalf_authed = fetch_api_authed_from_permissioned_as(
permissioned_as,
email,
permissioned_as.clone(),
email.clone(),
&w_id,
&db,
Some(username),
Some(username.clone()),
)
.await?;
@@ -1617,6 +1654,9 @@ async fn upload_s3_file_from_app(
.await?,
),
file_key,
email,
permissioned_as,
username,
)
} else {
return Err(Error::BadRequest(
@@ -1640,13 +1680,16 @@ async fn upload_s3_file_from_app(
.await?,
),
file_key,
email,
permissioned_as,
username,
)
}
} else {
let (_, s3_resource_opt) =
get_workspace_s3_resource(&on_behalf_authed, &db, None, "", &w_id, None)
.await?;
(s3_resource_opt, file_key)
(s3_resource_opt, file_key, email, permissioned_as, username)
}
} else {
return Err(Error::BadRequest(
@@ -1672,7 +1715,7 @@ async fn upload_s3_file_from_app(
let (_, s3_resource_opt) =
get_workspace_s3_resource(&on_behalf_authed, &db, None, "", &w_id, None).await?;
(s3_resource_opt, file_key)
(s3_resource_opt, file_key, email, permissioned_as, username)
}
} else {
// backward compatibility (no policy)
@@ -1682,6 +1725,12 @@ async fn upload_s3_file_from_app(
.file_key
.unwrap_or_else(|| get_random_file_name(query.file_extension));
let (on_behalf_of_email, permissioned_as, username) = (
authed.email.clone(),
username_to_permissioned_as(&authed.username),
authed.display_username().to_string(),
);
if let Some(ref s3_resource_path) = query.s3_resource_path {
(
Some(
@@ -1698,12 +1747,21 @@ async fn upload_s3_file_from_app(
.await?,
),
file_key,
on_behalf_of_email,
permissioned_as,
username,
)
} else {
let (_, s3_resource) =
get_workspace_s3_resource(&authed, &db, None, "", &w_id, None).await?;
(s3_resource, file_key)
(
s3_resource,
file_key,
on_behalf_of_email,
permissioned_as,
username,
)
}
} else {
return Err(Error::BadRequest("Missing s3 policy".to_string()));
@@ -1733,7 +1791,82 @@ async fn upload_s3_file_from_app(
upload_file_from_req(s3_client, &file_key, request, options).await?;
return Ok(Json(UploadFileResponse { file_key }));
let delete_token = jwt::encode_with_internal_secret(DeleteTokenClaims {
file_key: file_key.clone(),
on_behalf_of_email,
permissioned_as,
username,
s3_resource_path: query.s3_resource_path,
exp: (chrono::Utc::now() + chrono::Duration::seconds(3600 * 24)).timestamp() as usize,
})
.await?;
return Ok(Json(AppUploadFileResponse { file_key, delete_token }));
}
#[cfg(feature = "parquet")]
#[derive(Deserialize)]
struct DeleteS3FileQuery {
delete_token: String,
}
#[cfg(feature = "parquet")]
async fn delete_s3_file_from_app(
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(query): Query<DeleteS3FileQuery>,
) -> Result<()> {
let DeleteTokenClaims {
file_key,
on_behalf_of_email,
permissioned_as,
username,
s3_resource_path,
..
} = jwt::decode_with_internal_secret::<DeleteTokenClaims>(&query.delete_token).await?;
let on_behalf_authed = fetch_api_authed_from_permissioned_as(
permissioned_as,
on_behalf_of_email,
&w_id,
&db,
Some(username),
)
.await?;
let s3_resource = if let Some(s3_resource_path) = s3_resource_path {
get_s3_resource(
&on_behalf_authed,
&db,
Some(user_db),
"",
&w_id,
s3_resource_path.as_str(),
None,
None,
)
.await?
} else {
let (_, s3_resource) =
get_workspace_s3_resource(&on_behalf_authed, &db, None, "", &w_id, None).await?;
s3_resource.ok_or(Error::internal_err(
"No files storage resource defined at the workspace level".to_string(),
))?
};
let s3_client = build_object_store_client(&s3_resource).await?;
let path = object_store::path::Path::parse(file_key.as_str())
.map_err(|e| Error::internal_err(format!("Error parsing file key: {}", e)))?;
s3_client.delete(&path).await.map_err(|err| {
tracing::error!("Error deleting file: {:?}", err);
Error::internal_err(format!("Error deleting file: {}", err.to_string()))
})?;
Ok(())
}
#[cfg(not(feature = "parquet"))]
@@ -1749,26 +1882,41 @@ async fn get_on_behalf_authed_from_app(
path: &str,
w_id: &str,
opt_authed: &Option<ApiAuthed>,
) -> Result<ApiAuthed> {
let policy_o = sqlx::query_scalar!(
"SELECT policy from app WHERE path = $1 AND workspace_id = $2",
path,
w_id
)
.fetch_optional(db)
.await?;
let policy = policy_o
.map(|p| serde_json::from_value::<Policy>(p).map_err(to_anyhow))
.transpose()?
.unwrap_or_else(|| Policy {
force_allowed_s3_keys: Option<Vec<S3Key>>,
) -> Result<(ApiAuthed, Policy)> {
let policy = if let Some(force_allowed_s3_keys) = force_allowed_s3_keys {
Policy {
execution_mode: ExecutionMode::Viewer,
triggerables: None,
triggerables_v2: None,
on_behalf_of: None,
on_behalf_of_email: None,
s3_inputs: None,
});
allowed_s3_keys: Some(force_allowed_s3_keys),
}
} else {
// TODO: improve db query to not return uneeded fields
let policy_o = sqlx::query_scalar!(
"SELECT policy from app WHERE path = $1 AND workspace_id = $2",
path,
w_id
)
.fetch_optional(db)
.await?;
policy_o
.map(|p| serde_json::from_value::<Policy>(p).map_err(to_anyhow))
.transpose()?
.unwrap_or_else(|| Policy {
execution_mode: ExecutionMode::Viewer,
triggerables: None,
triggerables_v2: None,
on_behalf_of: None,
on_behalf_of_email: None,
s3_inputs: None,
allowed_s3_keys: None,
})
};
let (username, permissioned_as, email) =
get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?;
@@ -1777,7 +1925,7 @@ async fn get_on_behalf_authed_from_app(
fetch_api_authed_from_permissioned_as(permissioned_as, email, &w_id, &db, Some(username))
.await?;
Ok(on_behalf_authed)
Ok((on_behalf_authed, policy))
}
#[cfg(feature = "parquet")]
@@ -1787,6 +1935,7 @@ async fn check_if_allowed_to_access_s3_file_from_app(
file_key: &str,
w_id: &str,
path: &str,
policy: &Policy,
) -> Result<()> {
// if anonymous, check that the file was the result of an app script ran by an anonymous user in the last 3 hours
// otherwise, if logged in, allow any file (TODO: change that when we implement better s3 policy)
@@ -1808,7 +1957,10 @@ async fn check_if_allowed_to_access_s3_file_from_app(
)
.fetch_one(db)
.await?
.unwrap_or(false);
.unwrap_or(false)
// check if the file is allowed by the allowed_s3_keys policy
|| policy.allowed_s3_keys.as_ref().unwrap().iter().any(|key| key.s3_path == file_key);
if !allowed {
Err(Error::BadRequest("File restricted".to_string()))
@@ -1817,21 +1969,46 @@ async fn check_if_allowed_to_access_s3_file_from_app(
}
}
#[cfg(feature = "parquet")]
#[derive(Deserialize)]
pub struct DownloadFileQueryWithForceViewerAllowedS3Keys {
#[serde(flatten)]
pub file_query: DownloadFileQuery,
pub force_viewer_allowed_s3_keys: Option<String>,
}
#[cfg(feature = "parquet")]
async fn download_s3_file_from_app(
OptAuthed(opt_authed): OptAuthed,
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
Query(query): Query<DownloadFileQuery>,
Query(query): Query<DownloadFileQueryWithForceViewerAllowedS3Keys>,
) -> Result<Response> {
let path = path.to_path();
let on_behalf_authed = get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed).await?;
let force_viewer_allowed_s3_keys = if let Some(force_viewer_allowed_s3_keys) =
query.force_viewer_allowed_s3_keys.clone()
{
Some(serde_json::from_str::<Vec<S3Key>>(&force_viewer_allowed_s3_keys).unwrap_or_default())
} else {
None
};
check_if_allowed_to_access_s3_file_from_app(&db, &opt_authed, &query.file_key, &w_id, &path)
.await?;
let (on_behalf_authed, policy) =
get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed, force_viewer_allowed_s3_keys)
.await?;
download_s3_file_internal(on_behalf_authed, &db, None, "", &w_id, query).await
check_if_allowed_to_access_s3_file_from_app(
&db,
&opt_authed,
&query.file_query.file_key,
&w_id,
&path,
&policy,
)
.await?;
download_s3_file_internal(on_behalf_authed, &db, None, "", &w_id, query.file_query).await
}
#[cfg(not(feature = "parquet"))]
@@ -1850,10 +2027,18 @@ async fn load_s3_file_image_preview_from_app(
) -> Result<Response> {
let path = path.to_path();
let on_behalf_authed = get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed).await?;
let (on_behalf_authed, policy) =
get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed, None).await?;
check_if_allowed_to_access_s3_file_from_app(&db, &opt_authed, &query.file_key, &w_id, &path)
.await?;
check_if_allowed_to_access_s3_file_from_app(
&db,
&opt_authed,
&query.file_key,
&w_id,
&path,
&policy,
)
.await?;
load_image_preview_internal(on_behalf_authed, &db, "", &w_id, query).await
}
backend/windmill-api/src/auth.rs
+36 -46
View File
@@ -21,7 +21,8 @@ use std::sync::{
use tokio::sync::RwLock;
use windmill_common::{
auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, JWT_SECRET},
auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims},
jwt,
users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL},
};
@@ -99,55 +100,44 @@ impl AuthCache {
}
}
_ if token.starts_with("jwt_") => {
let jwt_secret = JWT_SECRET.read().await;
if !jwt_secret.is_empty() {
let jwt_token = token.trim_start_matches("jwt_");
let jwt_token = token.trim_start_matches("jwt_");
let jwt_result = jsonwebtoken::decode::<JWTAuthClaims>(
jwt_token,
&jsonwebtoken::DecodingKey::from_secret(jwt_secret.as_bytes()),
&jsonwebtoken::Validation::new(jsonwebtoken::Algorithm::HS256),
);
let jwt_result = jwt::decode_with_internal_secret::<JWTAuthClaims>(jwt_token).await;
match jwt_result {
Ok(payload) => {
if w_id.is_some_and(|w_id| w_id != payload.claims.workspace_id) {
tracing::error!("JWT auth error: workspace_id mismatch");
return None;
}
let username_override =
username_override_from_label(payload.claims.label);
let authed = crate::db::ApiAuthed {
email: payload.claims.email,
username: payload.claims.username,
is_admin: payload.claims.is_admin,
is_operator: payload.claims.is_operator,
groups: payload.claims.groups,
folders: payload.claims.folders,
scopes: None,
username_override,
};
self.cache.insert(
key,
ExpiringAuthCache {
authed: authed.clone(),
expiry: chrono::Utc
.timestamp_nanos(payload.claims.exp as i64 * 1_000_000_000),
},
);
Some(authed)
}
Err(err) => {
tracing::error!("JWT auth error: {:?}", err);
None
match jwt_result {
Ok(claims) => {
if w_id.is_some_and(|w_id| w_id != claims.workspace_id) {
tracing::error!("JWT auth error: workspace_id mismatch");
return None;
}
let username_override = username_override_from_label(claims.label);
let authed = crate::db::ApiAuthed {
email: claims.email,
username: claims.username,
is_admin: claims.is_admin,
is_operator: claims.is_operator,
groups: claims.groups,
folders: claims.folders,
scopes: None,
username_override,
};
self.cache.insert(
key,
ExpiringAuthCache {
authed: authed.clone(),
expiry: chrono::Utc
.timestamp_nanos(claims.exp as i64 * 1_000_000_000),
},
);
Some(authed)
}
Err(err) => {
tracing::error!("JWT auth error: {:?}", err);
None
}
} else {
tracing::error!("JWT auth error: no jwt secret set");
None
}
}
_ => {
backend/windmill-api/src/capture.rs
+44 -36
View File
@@ -6,36 +6,12 @@
* LICENSE-AGPL for a copy of the license.
*/
use axum::{
extract::{Extension, Path, Query},
routing::{delete, get, head, post},
Json, Router,
};
#[cfg(feature = "http_trigger")]
use http::HeaderMap;
use hyper::StatusCode;
#[cfg(feature = "http_trigger")]
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use sqlx::types::Json as SqlxJson;
#[cfg(feature = "http_trigger")]
use std::collections::HashMap;
use std::fmt;
#[cfg(feature = "http_trigger")]
use windmill_common::error::Error;
use windmill_common::{
db::UserDB,
error::{JsonResult, Result},
utils::{not_found_if_none, paginate, Pagination, StripPath},
worker::{to_raw_value, CLOUD_HOSTED},
};
use windmill_queue::{PushArgs, PushArgsOwned};
#[cfg(feature = "http_trigger")]
use crate::http_triggers::{build_http_trigger_extra, HttpMethod};
#[cfg(all(feature = "enterprise", feature = "kafka"))]
use crate::kafka_triggers_ee::KafkaTriggerConfigConnection;
#[cfg(feature = "mqtt_trigger")]
use crate::mqtt_triggers::{MqttClientVersion, MqttV3Config, MqttV5Config, SubscribeTopic};
#[cfg(all(feature = "enterprise", feature = "nats"))]
use crate::nats_triggers_ee::NatsTriggerConfigConnection;
#[cfg(feature = "postgres_trigger")]
@@ -43,16 +19,42 @@ use crate::postgres_triggers::{
create_logical_replication_slot_query, create_publication_query, drop_publication_query,
generate_random_string, get_database_connection, PublicationData,
};
#[cfg(feature = "http_trigger")]
use http::HeaderMap;
#[cfg(feature = "postgres_trigger")]
use itertools::Itertools;
#[cfg(feature = "postgres_trigger")]
use pg_escape::quote_literal;
#[cfg(feature = "http_trigger")]
use serde::de::DeserializeOwned;
#[cfg(feature = "http_trigger")]
use std::collections::HashMap;
#[cfg(feature = "http_trigger")]
use windmill_common::error::Error;
use crate::{
args::WebhookArgs,
db::{ApiAuthed, DB},
users::fetch_api_authed,
utils::RunnableKind,
};
use axum::{
extract::{Extension, Path, Query},
routing::{delete, get, head, post},
Json, Router,
};
use hyper::StatusCode;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use sqlx::types::Json as SqlxJson;
use std::fmt;
use windmill_common::{
db::UserDB,
error::{JsonResult, Result},
utils::{not_found_if_none, paginate, Pagination, StripPath},
worker::{to_raw_value, CLOUD_HOSTED},
};
use windmill_queue::{PushArgs, PushArgsOwned};
const KEEP_LAST: i64 = 20;
@@ -98,6 +100,7 @@ pub enum TriggerKind {
Kafka,
Email,
Nats,
Mqtt,
Sqs,
Postgres,
}
@@ -111,6 +114,7 @@ impl fmt::Display for TriggerKind {
TriggerKind::Kafka => "kafka",
TriggerKind::Email => "email",
TriggerKind::Nats => "nats",
TriggerKind::Mqtt => "mqtt",
TriggerKind::Sqs => "sqs",
TriggerKind::Postgres => "postgres",
};
@@ -139,7 +143,7 @@ pub struct KafkaTriggerConfig {
pub struct SqsTriggerConfig {
pub queue_url: String,
pub aws_resource_path: String,
pub message_attributes: Option<Vec<String>>
pub message_attributes: Option<Vec<String>>,
}
#[cfg(all(feature = "enterprise", feature = "nats"))]
@@ -155,6 +159,16 @@ pub struct NatsTriggerConfig {
pub use_jetstream: bool,
}
#[cfg(feature = "mqtt_trigger")]
#[derive(Debug, Serialize, Deserialize)]
pub struct MqttTriggerConfig {
pub mqtt_resource_path: String,
pub subscribe_topics: Vec<SubscribeTopic>,
pub v3_config: Option<MqttV3Config>,
pub v5_config: Option<MqttV5Config>,
pub client_version: Option<MqttClientVersion>,
pub client_id: Option<String>,
}
#[cfg(feature = "postgres_trigger")]
#[derive(Serialize, Deserialize, Debug)]
pub struct PostgresTriggerConfig {
@@ -187,6 +201,8 @@ enum TriggerConfig {
Kafka(KafkaTriggerConfig),
#[cfg(all(feature = "enterprise", feature = "nats"))]
Nats(NatsTriggerConfig),
#[cfg(feature = "mqtt_trigger")]
Mqtt(MqttTriggerConfig),
}
#[derive(Serialize, Deserialize)]
@@ -300,8 +316,7 @@ async fn set_config(
#[cfg(feature = "postgres_trigger")]
let nc = if let TriggerKind::Postgres = nc.trigger_kind {
set_postgres_trigger_config(&w_id, authed.clone(), &db, user_db.clone(), nc).await?
}
else {
} else {
nc
};
@@ -362,13 +377,6 @@ struct Capture {
trigger_extra: Option<SqlxJson<Box<serde_json::value::RawValue>>>,
}
#[derive(Deserialize)]
#[serde(rename_all = "lowercase")]
enum RunnableKind {
Script,
Flow,
}
#[derive(Deserialize)]
struct ListCapturesQuery {
trigger_kind: Option<TriggerKind>,
backend/windmill-api/src/db.rs
+40 -6
View File
@@ -32,6 +32,30 @@ async fn current_database(conn: &mut PgConnection) -> Result<String, MigrateErro
.await?)
}
lazy_static::lazy_static! {
pub static ref OVERRIDDEN_MIGRATIONS: std::collections::HashMap<i64, String> = vec![(20221207103910, include_str!(
"../../custom_migrations/create_workspace_without_md5.sql"
).to_string()),
(20240216100535, include_str!(
"../../migrations/20240216100535_improve_policies.up.sql"
).replace("public.", "")),
(20240403083110, include_str!(
"../../migrations/20240403083110_remove_team_id_constraint.up.sql"
).replace("public.", "")),
(20240613150524, include_str!(
"../../migrations/20240613150524_add_job_perms.up.sql"
).replace("public.", "")),
(20250102145420, include_str!(
"../../migrations/20250102145420_more_captures.up.sql"
).replace("public.", "")),
(20241006144414, include_str!(
"../../custom_migrations/grant_all_current_schema.sql"
).to_string()),
(20221105003256, "DELETE FROM workspace_invite WHERE workspace_id = 'demo' AND email = 'ruben@windmill.dev';".to_string()),
(20221123151919, "".to_string()),
].into_iter().collect();
}
struct CustomMigrator {
inner: PoolConnection<Postgres>,
}
@@ -132,12 +156,13 @@ impl Migrate for CustomMigrator {
migration.version,
migration.description
);
if migration.version == 20221207103910 {
tracing::info!("Skipping migration 20221207103910 to avoid using md5");
if let Some(migration_sql) = OVERRIDDEN_MIGRATIONS.get(&migration.version) {
tracing::info!("Using custom migration for version {}", migration.version);
self.inner
.execute(include_str!(
"../../custom_migrations/create_workspace_without_md5.sql"
))
.execute(&**migration_sql)
.await?;
let _ = sqlx::query(
r#"
@@ -528,6 +553,7 @@ async fn v2_finalize(db: &DB) -> Result<(), Error> {
)
.await?;
});
Ok(())
}
@@ -729,7 +755,7 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> {
});
run_windmill_migration!("v2_improve_v2_queued_jobs_indices", &db, |tx| {
sqlx::query!("CREATE INDEX CONCURRENTLY queue_sort_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) WHERE running = false")
sqlx::query!("CREATE INDEX CONCURRENTLY IF NOT EXISTS queue_sort_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) WHERE running = false")
.execute(db)
.await?;
@@ -745,6 +771,14 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> {
.execute(db)
.await?;
});
run_windmill_migration!("audit_timestamps", db, |tx| {
sqlx::query!(
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_audit_timestamps ON audit (timestamp DESC)"
)
.execute(db)
.await?;
});
Ok(())
}
backend/windmill-api/src/flows.rs
+27 -1
View File
@@ -12,7 +12,7 @@ use crate::db::ApiAuthed;
use crate::triggers::{
get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail,
};
use crate::utils::WithStarredInfoQuery;
use crate::utils::{RunnableKind, WithStarredInfoQuery};
use crate::{
db::DB,
schedule::clear_schedule,
@@ -65,6 +65,10 @@ pub fn workspaced_service() -> Router {
.route("/list_paths", get(list_paths))
.route("/history/p/*path", get(get_flow_history))
.route("/get_latest_version/*path", get(get_latest_version))
.route(
"/list_paths_from_workspace_runnable/:runnable_kind/*path",
get(list_paths_from_workspace_runnable),
)
.route(
"/history_update/v/:version/p/*path",
post(update_flow_history),
@@ -324,6 +328,28 @@ async fn check_path_conflict<'c>(
return Ok(());
}
async fn list_paths_from_workspace_runnable(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, StripPath)>,
) -> JsonResult<Vec<String>> {
let mut tx = user_db.begin(&authed).await?;
let runnables = sqlx::query_scalar!(
r#"SELECT f.path
FROM flow_workspace_runnables fwr
JOIN flow f
ON fwr.flow_path = f.path AND fwr.workspace_id = f.workspace_id
WHERE fwr.runnable_path = $1 AND fwr.runnable_is_flow = $2 AND fwr.workspace_id = $3"#,
path.to_path(),
matches!(runnable_kind, RunnableKind::Flow),
w_id
)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
Ok(Json(runnables))
}
async fn create_flow(
authed: ApiAuthed,
Extension(db): Extension<DB>,
backend/windmill-api/src/folders.rs
+17
View File
@@ -42,6 +42,7 @@ pub fn workspaced_service() -> Router {
.route("/listnames", get(list_foldernames))
.route("/create", post(create_folder))
.route("/get/:name", get(get_folder))
.route("/exists/:name", get(exists_folder))
.route("/update/:name", post(update_folder))
.route("/getusage/:name", get(get_folder_usage))
.route("/delete/:name", delete(delete_folder))
@@ -426,6 +427,22 @@ async fn get_folder(
Ok(Json(folder))
}
async fn exists_folder(
Extension(db): Extension<DB>,
Path((w_id, name)): Path<(String, String)>,
) -> JsonResult<bool> {
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2)",
name,
w_id
)
.fetch_one(&db)
.await?
.unwrap_or(false);
Ok(Json(exists))
}
#[derive(Serialize)]
struct FolderUsage {
pub scripts: i64,
backend/windmill-api/src/granular_acls.rs
+2 -1
View File
@@ -23,7 +23,7 @@ use windmill_common::{
utils::{not_found_if_none, StripPath},
};
const KINDS: [&str; 13] = [
const KINDS: [&str; 14] = [
"script",
"group_",
"resource",
@@ -37,6 +37,7 @@ const KINDS: [&str; 13] = [
"websocket_trigger",
"kafka_trigger",
"nats_trigger",
"mqtt_trigger"
];
pub fn workspaced_service() -> Router {
backend/windmill-api/src/jobs.rs
+40 -24
View File
@@ -28,7 +28,6 @@ use windmill_common::flow_status::{JobResult, RestartedFrom};
use windmill_common::jobs::{format_completed_job_result, format_result, ENTRYPOINT_OVERRIDE};
use windmill_common::worker::{CLOUD_HOSTED, TMP_DIR};
#[cfg(all(feature = "enterprise", feature = "parquet"))]
use windmill_common::scripts::PREVIEW_IS_CODEBASE_HASH;
use windmill_common::variables::get_workspace_key;
@@ -4397,7 +4396,6 @@ async fn run_preview_script(
Ok((StatusCode::CREATED, uuid.to_string()))
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
async fn run_bundle_preview_script(
authed: ApiAuthed,
Extension(db): Extension<DB>,
@@ -4408,8 +4406,6 @@ async fn run_bundle_preview_script(
) -> error::Result<(StatusCode, String)> {
use windmill_common::scripts::PREVIEW_IS_TAR_CODEBASE_HASH;
check_license_key_valid().await?;
check_scopes(&authed, || format!("jobs:runscript"))?;
if authed.is_operator {
return Err(error::Error::NotAuthorized(
@@ -4457,8 +4453,8 @@ async fn run_bundle_preview_script(
path: preview.path,
language: preview.language.unwrap_or(ScriptLang::Deno),
lock: preview.lock,
concurrent_limit: None, // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here
concurrency_time_window_s: None, // TODO(gbouv): same as above
concurrent_limit: None,
concurrency_time_window_s: None,
cache_ttl: None,
dedicated_worker: preview.dedicated_worker,
custom_concurrency_key: None,
@@ -4504,21 +4500,48 @@ async fn run_bundle_preview_script(
uploaded = true;
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
#[cfg(all(feature = "enterprise", feature = "parquet"))]
let object_store = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
.clone();
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
let object_store: Option<()> = None;
if &windmill_common::utils::MODE_AND_ADDONS.mode
== &windmill_common::utils::Mode::Standalone
&& object_store.is_none()
{
let path = windmill_common::s3_helpers::bundle(&w_id, &id);
if let Err(e) = os
.put(&object_store::path::Path::from(path.clone()), data.into())
.await
{
tracing::info!("Failed to put snapshot to s3 at {path}: {:?}", e);
return Err(Error::ExecutionErr(format!("Failed to put {path} to s3")));
}
std::fs::create_dir_all(
windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR.clone(),
)?;
windmill_common::worker::write_file_bytes(
&windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR,
&id,
&data,
)?;
} else {
return Err(Error::BadConfig("Object store is required for snapshot script and is not configured for servers".to_string()));
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
{
return Err(Error::ExecutionErr("codebase is an EE feature".to_string()));
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(os) = object_store {
check_license_key_valid().await?;
let path = windmill_common::s3_helpers::bundle(&w_id, &id);
if let Err(e) = os
.put(&object_store::path::Path::from(path.clone()), data.into())
.await
{
tracing::info!("Failed to put snapshot to s3 at {path}: {:?}", e);
return Err(Error::ExecutionErr(format!("Failed to put {path} to s3")));
}
} else {
return Err(Error::BadConfig("Object store is required for snapshot script and is not configured for servers".to_string()));
}
}
}
// println!("Length of `{}` is {} bytes", name, data.len());
@@ -4537,13 +4560,6 @@ async fn run_bundle_preview_script(
Ok((StatusCode::CREATED, job_id.unwrap().to_string()))
}
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
async fn run_bundle_preview_script() -> error::Result<(StatusCode, String)> {
return Err(Error::BadRequest(
"bundle preview is an ee feature".to_string(),
));
}
#[derive(Deserialize)]
pub struct RunDependenciesRequest {
pub raw_scripts: Vec<RawScriptForDependencies>,
backend/windmill-api/src/lib.rs
+22 -2
View File
@@ -89,6 +89,8 @@ pub mod job_metrics;
pub mod jobs;
#[cfg(all(feature = "enterprise", feature = "kafka"))]
mod kafka_triggers_ee;
#[cfg(feature = "mqtt_trigger")]
mod mqtt_triggers;
#[cfg(all(feature = "enterprise", feature = "nats"))]
mod nats_triggers_ee;
#[cfg(feature = "oauth2")]
@@ -116,7 +118,7 @@ mod users;
mod users_ee;
mod utils;
mod variables;
mod webhook_util;
pub mod webhook_util;
#[cfg(feature = "websocket")]
mod websocket_triggers;
mod workers;
@@ -321,6 +323,18 @@ pub async fn run_server(
}
};
let mqtt_triggers_service = {
#[cfg(all(feature = "mqtt_trigger"))]
{
mqtt_triggers::workspaced_service()
}
#[cfg(not(feature = "mqtt_trigger"))]
{
Router::new()
}
};
let sqs_triggers_service = {
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
{
@@ -387,6 +401,12 @@ pub async fn run_server(
let db_killpill_rx = rx.resubscribe();
postgres_triggers::start_database(db.clone(), db_killpill_rx);
}
#[cfg(feature = "mqtt_trigger")]
{
let mqtt_killpill_rx = rx.resubscribe();
mqtt_triggers::start_mqtt_consumer(db.clone(), mqtt_killpill_rx);
}
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
{
@@ -447,6 +467,7 @@ pub async fn run_server(
.nest("/websocket_triggers", websocket_triggers_service)
.nest("/kafka_triggers", kafka_triggers_service)
.nest("/nats_triggers", nats_triggers_service)
.nest("/mqtt_triggers", mqtt_triggers_service)
.nest("/sqs_triggers", sqs_triggers_service)
.nest("/postgres_triggers", postgres_triggers_service),
)
@@ -579,7 +600,6 @@ pub async fn run_server(
.on_failure(MyOnFailure {}),
)
};
let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
let port = listener.local_addr().map(|x| x.port()).unwrap_or(8000);
let ip = listener
backend/windmill-api/src/mqtt_triggers.rs
+1802
View File
File diff suppressed because it is too large Load Diff
backend/windmill-api/src/postgres_triggers/trigger.rs
+4 -3
View File
@@ -11,7 +11,8 @@ use crate::{
},
run_job,
},
users::fetch_api_authed, resources::try_get_resource_from_db_as,
resources::try_get_resource_from_db_as,
users::fetch_api_authed,
};
use bytes::{BufMut, Bytes, BytesMut};
use chrono::TimeZone;
@@ -878,8 +879,8 @@ impl CaptureConfigForPostgresTrigger {
args: Option<HashMap<String, Box<RawValue>>>,
extra: Option<HashMap<String, Box<RawValue>>>,
) -> () {
let args = PushArgsOwned { args: args.unwrap_or_default(), extra };
let extra = args.extra.as_ref().map(to_raw_value);
let args = PushArgsOwned { args: args.unwrap_or_default(), extra: None };
let extra = extra.as_ref().map(to_raw_value);
if let Err(err) = insert_capture_payload(
db,
&self.workspace_id,
backend/windmill-api/src/resources.rs
+1 -4
View File
@@ -1208,10 +1208,7 @@ async fn update_resource_type(
Ok(format!("resource_type {} updated", name))
}
#[cfg(any(
feature = "postgres_trigger",
all(feature = "sqs_trigger", feature = "enterprise")
))]
#[cfg(any(feature = "postgres_trigger", feature = "mqtt_trigger", all(feature = "sqs_trigger", feature = "enterprise")))]
pub async fn try_get_resource_from_db_as<T>(
authed: ApiAuthed,
user_db: Option<UserDB>,
backend/windmill-api/src/scripts.rs
+38 -19
View File
@@ -18,7 +18,6 @@ use crate::{
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
};
#[cfg(all(feature = "enterprise", feature = "parquet"))]
use axum::extract::Multipart;
use axum::{
@@ -42,7 +41,6 @@ use std::{
use windmill_audit::audit_ee::audit_log;
use windmill_audit::ActionKind;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
use windmill_common::error::to_anyhow;
use windmill_common::{
@@ -358,12 +356,6 @@ fn hash_script(ns: &NewScript) -> i64 {
dh.finish() as i64
}
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
async fn create_snapshot_script() -> Result<(StatusCode, String)> {
Err(Error::BadRequest("Upgrade to EE to use bundle".to_string()))
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
async fn create_snapshot_script(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -404,21 +396,48 @@ async fn create_snapshot_script(
})?;
uploaded = true;
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
#[cfg(all(feature = "enterprise", feature = "parquet"))]
let object_store = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
.clone();
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
let object_store: Option<()> = None;
if &windmill_common::utils::MODE_AND_ADDONS.mode
== &windmill_common::utils::Mode::Standalone
&& object_store.is_none()
{
let path = windmill_common::s3_helpers::bundle(&w_id, &hash);
if let Err(e) = os
.put(&object_store::path::Path::from(path.clone()), data.into())
.await
{
tracing::info!("Failed to put snapshot to s3 at {path}: {:?}", e);
return Err(Error::ExecutionErr(format!("Failed to put {path} to s3")));
}
std::fs::create_dir_all(
windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR.clone(),
)?;
windmill_common::worker::write_file(
&windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR,
&hash,
&String::from_utf8_lossy(&data),
)?;
} else {
return Err(Error::BadConfig("Object store is required for snapshot script and is not configured for servers".to_string()));
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
{
return Err(Error::ExecutionErr("codebase is an EE feature".to_string()));
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(os) = object_store {
let path = windmill_common::s3_helpers::bundle(&w_id, &hash);
if let Err(e) = os
.put(&object_store::path::Path::from(path.clone()), data.into())
.await
{
tracing::info!("Failed to put snapshot to s3 at {path}: {:?}", e);
return Err(Error::ExecutionErr(format!("Failed to put {path} to s3")));
}
} else {
return Err(Error::BadConfig("Object store is required for snapshot script and is not configured for servers".to_string()));
}
}
}
// println!("Length of `{}` is {} bytes", name, data.len());
backend/windmill-api/src/users.rs
+83 -8
View File
@@ -55,7 +55,6 @@ use windmill_common::{
utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath},
};
use windmill_git_sync::handle_deployment_metadata;
pub const TTL_TOKEN_DB_H: u32 = 72;
const COOKIE_PATH: &str = "/";
@@ -606,6 +605,13 @@ async fn list_invites(
Ok(Json(rows))
}
lazy_static::lazy_static! {
static ref INVALIDATE_ALL_SESSIONS_ON_LOGOUT: bool = std::env::var("INVALIDATE_ALL_SESSIONS_ON_LOGOUT")
.unwrap_or("false".to_string())
.parse::<bool>()
.unwrap_or(false);
}
#[derive(Deserialize)]
struct LogoutQuery {
rd: Option<String>,
@@ -623,15 +629,36 @@ async fn logout(
}
cookies.remove(cookie);
let mut tx = db.begin().await?;
let email = sqlx::query_scalar!("DELETE FROM token WHERE token = $1 RETURNING email", token)
let email = if *INVALIDATE_ALL_SESSIONS_ON_LOGOUT {
sqlx::query_scalar!(
"WITH email_lookup AS (
SELECT email FROM token WHERE token = $1
)
DELETE FROM token
WHERE email = (SELECT email FROM email_lookup) AND label = 'session'
RETURNING email",
token
)
.fetch_optional(&mut *tx)
.await?;
.await?
} else {
sqlx::query_scalar!("DELETE FROM token WHERE token = $1 RETURNING email", token)
.fetch_optional(&mut *tx)
.await?
};
if let Some(email) = email {
let email = email.unwrap_or("noemail".to_string());
let audit_message = if *INVALIDATE_ALL_SESSIONS_ON_LOGOUT {
"users.logout_all"
} else {
"users.logout"
};
audit_log(
&mut *tx,
&AuditAuthor { email: email.clone(), username: email, username_override: None },
"users.logout",
audit_message,
ActionKind::Delete,
"global",
Some(&truncate_token(&token)),
@@ -1696,6 +1723,11 @@ async fn refresh_token(
Ok("token refreshed".to_string())
}
lazy_static::lazy_static! {
static ref MAX_SESSION_VALIDITY_SECONDS: i64 = std::env::var("MAX_SESSION_VALIDITY_SECONDS").ok().unwrap_or_else(|| String::new()).parse::<i64>().unwrap_or(3 * 24 * 60 * 60);
static ref INVALIDATE_OLD_SESSIONS: bool = std::env::var("INVALIDATE_OLD_SESSIONS").ok().unwrap_or_else(|| String::new()).parse::<bool>().unwrap_or(false);
}
pub async fn create_session_token<'c>(
email: &str,
super_admin: bool,
@@ -1703,18 +1735,45 @@ pub async fn create_session_token<'c>(
cookies: Cookies,
) -> Result<String> {
let token = rd_string(32);
if *INVALIDATE_OLD_SESSIONS {
sqlx::query!(
"DELETE FROM token WHERE email = $1 AND label = 'session'",
email
)
.execute(&mut **tx)
.await?;
audit_log(
&mut **tx,
&AuditAuthor {
email: email.to_string(),
username: email.to_string(),
username_override: None,
},
"users.token.invalidate_old_sessions",
ActionKind::Delete,
&"global",
None,
None,
)
.instrument(tracing::info_span!("token", email))
.await?;
}
sqlx::query!(
"INSERT INTO token
(token, email, label, expiration, super_admin)
VALUES ($1, $2, $3, now() + ($4 || ' hours')::interval, $5)",
VALUES ($1, $2, $3, now() + ($4 || ' seconds')::interval, $5)",
token,
email,
"session",
TTL_TOKEN_DB_H.to_string(),
&MAX_SESSION_VALIDITY_SECONDS.to_string(),
super_admin
)
.execute(&mut **tx)
.await?;
let mut cookie = Cookie::new(COOKIE_NAME, token.clone());
cookie.set_secure(IS_SECURE.read().await.clone());
cookie.set_same_site(Some(tower_cookies::cookie::SameSite::Lax));
@@ -1725,7 +1784,7 @@ pub async fn create_session_token<'c>(
}
let mut expire: OffsetDateTime = time::OffsetDateTime::now_utc();
expire += time::Duration::days(3);
expire += time::Duration::seconds(*MAX_SESSION_VALIDITY_SECONDS);
cookie.set_expires(expire);
cookies.add(cookie);
Ok(token)
@@ -1845,7 +1904,7 @@ async fn list_tokens(
sqlx::query_as!(
TruncatedToken,
"SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, \
last_used_at, scopes FROM token WHERE email = $1 AND label != 'ephemeral-script'
last_used_at, scopes FROM token WHERE email = $1 AND (label != 'ephemeral-script' OR label IS NULL)
ORDER BY created_at DESC LIMIT $2 OFFSET $3",
email,
per_page as i64,
@@ -2478,6 +2537,22 @@ async fn update_username_in_workpsace<'c>(
.execute(&mut **tx)
.await?;
sqlx::query!(
r#"UPDATE flow_workspace_runnables SET flow_path = REGEXP_REPLACE(flow_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE flow_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#,
new_username,
old_username,
w_id
).execute(&mut **tx)
.await?;
sqlx::query!(
r#"UPDATE flow_workspace_runnables SET runnable_path = REGEXP_REPLACE(runnable_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE runnable_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#,
new_username,
old_username,
w_id
).execute(&mut **tx)
.await?;
sqlx::query!(
r#"UPDATE flow_node SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#,
new_username,
backend/windmill-api/src/utils.rs
+7
View File
@@ -29,6 +29,13 @@ pub struct WithStarredInfoQuery {
pub with_starred_info: Option<bool>,
}
#[derive(Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum RunnableKind {
Script,
Flow,
}
pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> {
let is_admin = is_super_admin_email(db, email).await?;
backend/windmill-api/src/variables.rs
+1
View File
@@ -19,6 +19,7 @@ use axum::{
};
use hyper::StatusCode;
use serde_json::Value;
use windmill_audit::audit_ee::{audit_log, AuditAuthorable};
use windmill_audit::ActionKind;
use windmill_common::{

Some files were not shown because too many files have changed in this diff Show More