starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-04 08:01:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into glm/improve-input-connection

Browse Source
This commit is contained in:
Ruben Fiszel
2024-11-02 19:47:36 +01:00
committed by GitHub
parent c48946b2b3 41acd56eb6
commit 8feb0bbc93
158 changed files with 6623 additions and 2130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/DockerfileBackendTests
+2 -2
View File
@@ -40,11 +40,11 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER
RUN /usr/local/bin/python3 -m pip install pip-tools
COPY --from=oven/bun:1.1.30 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.31 /usr/local/bin/bun /usr/bin/bun
ARG TARGETPLATFORM
RUN curl -Lsf https://github.com/denoland/deno/releases/download/v2.0.0/deno-x86_64-unknown-linux-gnu.zip -o deno.zip
RUN curl -Lsf https://github.com/denoland/deno/releases/download/v2.0.2/deno-x86_64-unknown-linux-gnu.zip -o deno.zip
# RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v2.0.0/deno-aarch64-unknown-linux-gnu.zip -o deno.zip || true
RUN unzip deno.zip && rm deno.zip && mv deno /usr/bin/deno
.github/workflows/docker-image.yml
+17 -14
View File
@@ -26,6 +26,7 @@ permissions: write-all
jobs:
build:
runs-on: ubicloud
if: (github.event_name != 'issue_comment') || (contains(github.event.comment.body, '/buildimage_all') || contains(github.event.comment.body, '/buildimage_base'))
steps:
- uses: actions/checkout@v4
with:
@@ -87,6 +88,7 @@ jobs:
build_ee:
runs-on: ubicloud
if: (github.event_name != 'issue_comment') || (contains(github.event.comment.body, '/buildimage_ee') || contains(github.event.comment.body, '/buildimage_nsjail')) || contains(github.event.comment.body, '/buildimage_all')
steps:
- uses: actions/checkout@v4
with:
@@ -148,7 +150,7 @@ jobs:
build_ee_312:
runs-on: ubicloud
if: ${{ startsWith(github.ref, 'refs/tags/') }}
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
steps:
- uses: actions/checkout@v4
with:
@@ -244,7 +246,7 @@ jobs:
attach_amd64_binary_to_release:
needs: [build, build_ee]
runs-on: ubicloud
if: ${{ startsWith(github.ref, 'refs/tags/') }}
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
env:
ARCH: amd64
steps:
@@ -337,10 +339,10 @@ jobs:
with:
fetch-depth: 0
- name: Prepare test run
if: ${{ ! startsWith(github.ref, 'refs/tags/') }}
if: ${{ ! startsWith(github.ref, 'refs/tags/v') }}
run: cd integration_tests && ./build.sh
- name: Test run
if: ${{ ! startsWith(github.ref, 'refs/tags/') }}
if: ${{ ! startsWith(github.ref, 'refs/tags/v') }}
timeout-minutes: 15
env:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
@@ -356,7 +358,7 @@ jobs:
tag_latest:
runs-on: ubicloud
needs: [run_integration_test, build]
if: github.event_name != 'pull_request'
if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
steps:
- uses: actions/checkout@v4
with:
@@ -375,7 +377,7 @@ jobs:
tag_latest_ee:
runs-on: ubicloud
needs: [run_integration_test, build_ee]
if: github.event_name != 'pull_request'
if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
steps:
- uses: actions/checkout@v4
with:
@@ -394,7 +396,7 @@ jobs:
verify_ee_image_vulnerabilities:
runs-on: ubicloud
needs: [tag_latest_ee]
if: ${{ startsWith(github.ref, 'refs/tags/') }}
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -436,7 +438,7 @@ jobs:
build_ee_nsjail:
needs: [build_ee]
runs-on: ubicloud
if: github.event_name != 'pull_request'
if: (github.event_name != 'issue_comment') || (github.event_name != 'pull_request') || (contains(github.event.comment.body, '/buildimage_nsjail') || contains(github.event.comment.body, '/buildimage_all'))
steps:
- uses: actions/checkout@v4
with:
@@ -459,6 +461,8 @@ jobs:
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha,enable=true,priority=100,prefix=,suffix=,format=short
type=ref,event=branch
type=ref,event=pr
- name: Login to registry
uses: docker/login-action@v3
@@ -476,7 +480,6 @@ jobs:
file: "./docker/DockerfileNsjail"
tags: |
${{ steps.meta-ee-public.outputs.tags }}
ghcr.io/windmill-labs/windmill-ee-nsjail:main
labels: |
${{ steps.meta-ee-public.outputs.labels }}
org.opencontainers.image.licenses=Windmill-Enterprise-License
@@ -570,7 +573,7 @@ jobs:
bucket-region: us-east-1
build_ee_cuda:
if: ${{ startsWith(github.ref, 'refs/tags/') }}
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
needs: [build_ee]
runs-on: ubicloud
steps:
@@ -614,7 +617,7 @@ jobs:
org.opencontainers.image.licenses=Windmill-Enterprise-License
build_slim:
if: ${{ startsWith(github.ref, 'refs/tags/') }}
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
needs: [build]
runs-on: ubicloud
steps:
@@ -657,7 +660,7 @@ jobs:
${{ steps.meta-ee-public.outputs.labels }}
build_ee_slim:
if: ${{ startsWith(github.ref, 'refs/tags/') }}
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
needs: [build_ee]
runs-on: ubicloud
steps:
@@ -701,7 +704,7 @@ jobs:
org.opencontainers.image.licenses=Windmill-Enterprise-License
build_full:
if: ${{ startsWith(github.ref, 'refs/tags/') }}
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
needs: [build]
runs-on: ubicloud
steps:
@@ -744,7 +747,7 @@ jobs:
${{ steps.meta-public.outputs.labels }}
build_ee_full:
if: ${{ startsWith(github.ref, 'refs/tags/') }}
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
needs: [build_ee]
runs-on: ubicloud
steps:
CHANGELOG.md
+139
View File
@@ -1,5 +1,144 @@
# Changelog
## [1.416.2](https://github.com/windmill-labs/windmill/compare/v1.416.1...v1.416.2) (2024-11-02)
### Bug Fixes
* apply NO_PROXY and HTTP_PROXY, HTTPS_PROXY more consistently ([567d621](https://github.com/windmill-labs/windmill/commit/567d6216d2631a90fbe59ec6142c38b3b352eea7))
## [1.416.1](https://github.com/windmill-labs/windmill/compare/v1.416.0...v1.416.1) (2024-11-01)
### Bug Fixes
* **prometheus:** fix incorrect worker_busy set to 1 ([53f9136](https://github.com/windmill-labs/windmill/commit/53f9136658b9fc1795793d82408c1f1f04adcf06))
## [1.416.0](https://github.com/windmill-labs/windmill/compare/v1.415.2...v1.416.0) (2024-11-01)
### Features
* private hub user accessible url setting ([#4617](https://github.com/windmill-labs/windmill/issues/4617)) ([79edf89](https://github.com/windmill-labs/windmill/commit/79edf89bd17827d5f1d946739385327b6c0520bf))
### Bug Fixes
* **frontend:** improve tag selector for workspace script drawer ([66f6985](https://github.com/windmill-labs/windmill/commit/66f69859ad2de51aef5a133df4ab4397d0f61ccf))
## [1.415.2](https://github.com/windmill-labs/windmill/compare/v1.415.1...v1.415.2) (2024-11-01)
### Bug Fixes
* **s3:** align s3 handler additional creds providers ([984c6dd](https://github.com/windmill-labs/windmill/commit/984c6dd10c63097eb195883c4d8a9681ab1b49e0))
## [1.415.1](https://github.com/windmill-labs/windmill/compare/v1.415.0...v1.415.1) (2024-10-31)
### Bug Fixes
* **cli:** improve --instance handling wmill instance push ([cb005a1](https://github.com/windmill-labs/windmill/commit/cb005a15baef4272bc58c7e80a43e44723556d31))
## [1.415.0](https://github.com/windmill-labs/windmill/compare/v1.414.2...v1.415.0) (2024-10-31)
### Features
* **cli:** opts.instance as instace name and prefix ([#4609](https://github.com/windmill-labs/windmill/issues/4609)) ([a07f57e](https://github.com/windmill-labs/windmill/commit/a07f57e698107056d045d8d5c2458e04c809fcc8))
### Bug Fixes
* improve express oauth setup ([ba4aed5](https://github.com/windmill-labs/windmill/commit/ba4aed5bf51c65204332cfc158d0ffd9c7095ec7))
* improve user resource input ([8c7f53b](https://github.com/windmill-labs/windmill/commit/8c7f53b2ebe0990cd93879258d004ac89dc8b24c))
## [1.414.2](https://github.com/windmill-labs/windmill/compare/v1.414.1...v1.414.2) (2024-10-29)
### Bug Fixes
* **cli:** improve instance sync for CI/CD + --folder-per-instance ([212579a](https://github.com/windmill-labs/windmill/commit/212579a514d070355fe0d9e0215593bacfa05e1f))
## [1.414.1](https://github.com/windmill-labs/windmill/compare/v1.414.0...v1.414.1) (2024-10-29)
### Bug Fixes
* **apps:** enable text selection on aggrid tables by default ([b0b9180](https://github.com/windmill-labs/windmill/commit/b0b9180fb907c92b95a48ff286eb1dae59bb4981))
* **apps:** public apps can take full height ([703db7d](https://github.com/windmill-labs/windmill/commit/703db7d4412795b4323a2eefcc39ae3cf43bc748))
* **bun:** handle bun lockfile created with windows ([#4602](https://github.com/windmill-labs/windmill/issues/4602)) ([dcf5e2f](https://github.com/windmill-labs/windmill/commit/dcf5e2f03f977e241f6785530dad61a70c5bdd79))
* **frontend:** make script and schema scrollable on script detail page ([6e222b3](https://github.com/windmill-labs/windmill/commit/6e222b3b1a419e5543fdf35420a7413699688b41))
* **frontend:** new approval steps default to timeout 1800 ([b86de62](https://github.com/windmill-labs/windmill/commit/b86de6280e03e014e8ddf85b2b5f8fd030d0467a))
## [1.414.0](https://github.com/windmill-labs/windmill/compare/v1.413.2...v1.414.0) (2024-10-29)
* Issue with previous release, re-releasing
## [1.413.2](https://github.com/windmill-labs/windmill/compare/v1.413.1...v1.413.2) (2024-10-29)
### Bug Fixes
* **backend:** in flows, workspace scripts should use their set tags instead of the default one ([5b7c6d7](https://github.com/windmill-labs/windmill/commit/5b7c6d7d62dcfd09fec374e781bdf5c5bafe4a9d))
* **cli:** fix wmill instance pull --instance ([3c62f5e](https://github.com/windmill-labs/windmill/commit/3c62f5ea83d1da8bd3705468969d56e2fe680751))
* **frontend:** fix script and flow renaming ([d743e00](https://github.com/windmill-labs/windmill/commit/d743e0056353a4fca445a7089e3afc1fd4e8c219))
## [1.413.1](https://github.com/windmill-labs/windmill/compare/v1.413.0...v1.413.1) (2024-10-28)
### Bug Fixes
* **cli:** fix wmill instance push --base-url and --instance ([8298710](https://github.com/windmill-labs/windmill/commit/82987105a6fd6ec272c170fb094453a0267143be))
## [1.413.0](https://github.com/windmill-labs/windmill/compare/v1.412.0...v1.413.0) (2024-10-28)
### Features
* autoscaling v0 ([#4593](https://github.com/windmill-labs/windmill/issues/4593)) ([fe7d044](https://github.com/windmill-labs/windmill/commit/fe7d044a66e8ec223a337cb704d3e58942dd1502))
### Bug Fixes
* add run immediately popover to run again ([e54d253](https://github.com/windmill-labs/windmill/commit/e54d25368541dc6109a0f99022a120d28455f9bd))
* **docs:** smtp setup documentation link ([#4590](https://github.com/windmill-labs/windmill/issues/4590)) ([bac3205](https://github.com/windmill-labs/windmill/commit/bac32057259d893140d649c9dfec2ca75e395ad4))
## [1.412.0](https://github.com/windmill-labs/windmill/compare/v1.411.1...v1.412.0) (2024-10-25)
### Features
* add Spotify oauth provider ([#4581](https://github.com/windmill-labs/windmill/issues/4581)) ([a46aa64](https://github.com/windmill-labs/windmill/commit/a46aa644b096e71e75b59507224ed92f7d2f99ba))
### Bug Fixes
* **app builder:** date input default value improvements ([9f43d5d](https://github.com/windmill-labs/windmill/commit/9f43d5dcd92ddcd0c0baeaa5271779520af652f4))
* **bash:** correctly propagate sigterm for cancelled bash scripts ([134cfdb](https://github.com/windmill-labs/windmill/commit/134cfdb30eb8d29c2ecb1b78b8fabae2b6e10700))
* do not update created_at of scripts on lockfile generation ([d1a28eb](https://github.com/windmill-labs/windmill/commit/d1a28eb7cac5f465e2b07f870a507b0cc5cc722a))
* initialize empty smtp settings correctly ([84e0524](https://github.com/windmill-labs/windmill/commit/84e05249505c3d2f7eb7a4917f6bca503df81017))
## [1.411.1](https://github.com/windmill-labs/windmill/compare/v1.411.0...v1.411.1) (2024-10-22)
### Bug Fixes
* update bun to 1.1.32 ([#4568](https://github.com/windmill-labs/windmill/issues/4568)) ([0586446](https://github.com/windmill-labs/windmill/commit/058644667129f0d79ec147aacdda449142ae0ab9))
## [1.411.0](https://github.com/windmill-labs/windmill/compare/v1.410.3...v1.411.0) (2024-10-21)
### Features
* **cli:** encrypt sensitive instance settings ([#4561](https://github.com/windmill-labs/windmill/issues/4561)) ([b8a6a11](https://github.com/windmill-labs/windmill/commit/b8a6a116354b10f5977e54edb365d6711e160538))
### Bug Fixes
* Do not ignore file resources with json file ext ([#4562](https://github.com/windmill-labs/windmill/issues/4562)) ([2079b2e](https://github.com/windmill-labs/windmill/commit/2079b2e7e19aa2fe327f2ae66d1b5eba988b9b0a))
* update bun to 1.1.31 and deno to 2.0.2 ([0d90396](https://github.com/windmill-labs/windmill/commit/0d9039641b3348e75599937188c35dd89a000584))
## [1.410.3](https://github.com/windmill-labs/windmill/compare/v1.410.2...v1.410.3) (2024-10-20)
Dockerfile
+3 -3
View File
@@ -162,7 +162,7 @@ ENV GO_PATH=/usr/local/go/bin/go
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv
RUN curl -sL https://deb.nodesource.com/setup_20.x | bash -
RUN apt-get -y update && apt-get install -y curl nodejs awscli && apt-get clean \
RUN apt-get -y update && apt-get install -y curl procps nodejs awscli && apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# go build is slower the first time it is ran, so we prewarm it in the build
@@ -175,9 +175,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
COPY --from=builder /frontend/build /static_frontend
COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=denoland/deno:2.0.0 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=denoland/deno:2.0.2 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.1.30 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.32 /usr/local/bin/bun /usr/bin/bun
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
backend/.sqlx/query-02b516dac764662194db1bc33e365c01f40bae70af3683f1f09748f6020f0d49.json
Generated
-26
View File
@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT tag, count(*) as count FROM queue WHERE\n scheduled_for <= now() - ('3 seconds')::interval AND running = false\n GROUP BY tag",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "tag",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
null
]
},
"hash": "02b516dac764662194db1bc33e365c01f40bae70af3683f1f09748f6020f0d49"
}
backend/.sqlx/query-19f0abd79372698f378cb6deea3ee6d098a2758d16ede000809bd9a09660b604.json
Generated
-20
View File
@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT created_at FROM metrics WHERE id = 'telemetry' ORDER BY created_at DESC LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "19f0abd79372698f378cb6deea3ee6d098a2758d16ede000809bd9a09660b604"
}
backend/.sqlx/query-c2060e8cacef6c3b5ce51ed203a2dbafc18d66f2924d1fe518c6728997647db2.json → backend/.sqlx/query-1a4d291c2f239f7b50c116594cebb031862e1a18ad9204e02a0194817db26d6a.json
Generated
+9 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT email, login_type::text, verified, super_admin, name, company, username from password ORDER BY super_admin DESC, email LIMIT $1 OFFSET $2",
"query": "SELECT email, login_type::text, verified, super_admin, name, company, username, NULL::bool as operator_only FROM password ORDER BY super_admin DESC, email LIMIT $1 OFFSET $2",
"describe": {
"columns": [
{
@@ -37,6 +37,11 @@
"ordinal": 6,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "operator_only",
"type_info": "Bool"
}
],
"parameters": {
@@ -52,8 +57,9 @@
false,
true,
true,
true
true,
null
]
},
"hash": "c2060e8cacef6c3b5ce51ed203a2dbafc18d66f2924d1fe518c6728997647db2"
"hash": "1a4d291c2f239f7b50c116594cebb031862e1a18ad9204e02a0194817db26d6a"
}
backend/.sqlx/query-1f5f0858909eb5bac63c4e3b1add95226bd94ca3facb92a620ffa59dacad6705.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO autoscaling_event (worker_group, event_type, desired_workers, reason) VALUES ($1, $2, $3, $4)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
{
"Custom": {
"name": "autoscaling_event_type",
"kind": {
"Enum": [
"full_scaleout",
"scalein",
"scaleout"
]
}
}
},
"Int4",
"Text"
]
},
"nullable": []
},
"hash": "1f5f0858909eb5bac63c4e3b1add95226bd94ca3facb92a620ffa59dacad6705"
}
backend/.sqlx/query-8813665f8adfcab0daefbac2cc6b50e427dfd1c12d12895451affd307dc59c37.json → backend/.sqlx/query-2c14d3a88193f16ad3b8cd590749cb5537995f2499f6cb8f0f316fb62902d542.json
Generated
+9 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT email, login_type::TEXT, super_admin, verified, name, company, username FROM password WHERE email = $1",
"query": "SELECT email, login_type::TEXT, super_admin, verified, name, company, username, NULL::bool as operator_only FROM password WHERE email = $1",
"describe": {
"columns": [
{
@@ -37,6 +37,11 @@
"ordinal": 6,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "operator_only",
"type_info": "Bool"
}
],
"parameters": {
@@ -51,8 +56,9 @@
false,
true,
true,
true
true,
null
]
},
"hash": "8813665f8adfcab0daefbac2cc6b50e427dfd1c12d12895451affd307dc59c37"
"hash": "2c14d3a88193f16ad3b8cd590749cb5537995f2499f6cb8f0f316fb62902d542"
}
backend/.sqlx/query-2eec077cc9e27d7ccd160cbaac118c321c422705f79e69550bb60f377083bcef.json
Generated
+65
View File
@@ -0,0 +1,65 @@
{
"db_name": "PostgreSQL",
"query": "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, name, company, username\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC\n LIMIT $1 OFFSET $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "operator_only",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "login_type",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "verified",
"type_info": "Bool"
},
{
"ordinal": 4,
"name": "super_admin",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "company",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "username",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8",
"Int8"
]
},
"nullable": [
false,
null,
null,
false,
false,
true,
true,
true
]
},
"hash": "2eec077cc9e27d7ccd160cbaac118c321c422705f79e69550bb60f377083bcef"
}
backend/.sqlx/query-4eca060026a0cb19c5794cd56ace89fc04765191f251945d14bbe78718714f6e.json
Generated
+39
View File
@@ -0,0 +1,39 @@
{
"db_name": "PostgreSQL",
"query": "SELECT event_type::AUTOSCALING_EVENT_TYPE AS \"event_type: _\", EXTRACT(EPOCH FROM (NOW() - applied_at))::int as seconds_ago FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "event_type: _",
"type_info": {
"Custom": {
"name": "autoscaling_event_type",
"kind": {
"Enum": [
"full_scaleout",
"scalein",
"scaleout"
]
}
}
}
},
{
"ordinal": 1,
"name": "seconds_ago",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "4eca060026a0cb19c5794cd56ace89fc04765191f251945d14bbe78718714f6e"
}
backend/.sqlx/query-6776dc50f184188756ad7fe263b0304333536768527525a43bdd45aedffa3c4f.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT pg_try_advisory_xact_lock($1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "pg_try_advisory_xact_lock",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
null
]
},
"hash": "6776dc50f184188756ad7fe263b0304333536768527525a43bdd45aedffa3c4f"
}
backend/.sqlx/query-6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6.json
Generated
+52
View File
@@ -0,0 +1,52 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, worker_group, event_type::text, desired_workers, reason, applied_at FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT 5",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "worker_group",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "event_type",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "desired_workers",
"type_info": "Int4"
},
{
"ordinal": 4,
"name": "reason",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "applied_at",
"type_info": "Timestamp"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
null,
false,
true,
false
]
},
"hash": "6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6"
}
backend/.sqlx/query-9ed77e78e6295c62745ac3ac3b7e5f544f654d7cddc13acb5d3f4fdc12a8875f.json
Generated
-20
View File
@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT created_at FROM metrics WHERE id = 'license_key_renewal' ORDER BY created_at DESC LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "9ed77e78e6295c62745ac3ac3b7e5f544f654d7cddc13acb5d3f4fdc12a8875f"
}
backend/.sqlx/query-ec47955683d811b12e82ff3e6aeafa0df46a320c436bad5eb1acc127df138a61.json → backend/.sqlx/query-d697b7311430e7bd5375ec5494179f4071e3bfe123d9600249cfa80c1103edd8.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE script SET lock = $1, created_at = now() WHERE hash = $2 AND workspace_id = $3",
"query": "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
@@ -12,5 +12,5 @@
},
"nullable": []
},
"hash": "ec47955683d811b12e82ff3e6aeafa0df46a320c436bad5eb1acc127df138a61"
"hash": "d697b7311430e7bd5375ec5494179f4071e3bfe123d9600249cfa80c1103edd8"
}
backend/Cargo.lock
Generated
+217 -200
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+10 -7
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.410.3"
version = "1.416.2"
authors.workspace = true
edition.workspace = true
@@ -13,6 +13,7 @@ members = [
"./windmill-common",
"./windmill-audit",
"./windmill-git-sync",
"./windmill-autoscaling",
"./windmill-indexer",
"./windmill-macros",
"./parsers/windmill-parser",
@@ -28,7 +29,7 @@ members = [
]
[workspace.package]
version = "1.410.3"
version = "1.416.2"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -45,7 +46,7 @@ lto = "thin"
[features]
default = []
enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"]
enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"]
enterprise_saml = ["windmill-api/enterprise_saml"]
stripe = ["windmill-api/stripe"]
benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"]
@@ -72,6 +73,7 @@ windmill-git-sync.workspace = true
windmill-api = { workspace = true, default-features = false }
windmill-worker.workspace = true
windmill-indexer = { workspace = true, optional = true }
windmill-autoscaling = { workspace = true, optional = true }
futures.workspace = true
tracing.workspace = true
sqlx.workspace = true
@@ -116,6 +118,7 @@ windmill-worker = { path = "./windmill-worker" }
windmill-common = { path = "./windmill-common", default-features = false }
windmill-audit = { path = "./windmill-audit" }
windmill-git-sync = { path = "./windmill-git-sync" }
windmill-autoscaling = { path = "./windmill-autoscaling" }
windmill-indexer = {path = "./windmill-indexer"}
windmill-macros = {path = "./windmill-macros"}
windmill-parser = { path = "./parsers/windmill-parser" }
@@ -157,7 +160,7 @@ hex = "^0"
sql-builder = "^3"
argon2 = "^0"
quick_cache = "^0"
rand = "0.8.5"
rand = "^0"
rand_core = { version = "^0", features = ["std"] }
magic-crypt = "^3"
git-version = "^0"
@@ -169,7 +172,7 @@ urlencoding = "^2"
url = "^2"
async-oauth2 = "^0"
reqwest = { version = "^0.12", features = ["json", "stream", "gzip"] }
time = "0.3.16"
time = "^0"
serde_urlencoded = "^0"
tokio-tar = "^0"
tempfile = "^3"
@@ -195,7 +198,7 @@ swc_ecma_visit = "=0.104.8"
async-recursion = "^1"
base64 = "0.21.0"
base64 = "^0"
base32 = "^0"
hmac = "0.12.1"
sha2 = "0.10.6"
@@ -219,7 +222,7 @@ serde_derive = "1.0.147"
const_format = { version = "0.2", features = ["rust_1_64", "rust_1_51"] }
dyn-iter = "0.2.0"
rsa = "0.7.2"
async-stripe = { version = "0.34.1", features = [
async-stripe = { version = "0.39.1", features = [
"runtime-tokio-hyper",
"checkout",
"billing",
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
ce38a7f85ca257c48471c46e3811a1281d9e2f27
f136a2f499e0fe7c10c54c79488851980d796eb2
backend/migrations/20241024125924_autoscaling.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add down migration script here
DROP TABLE autoscaling_event;
DROP TYPE autoscaling_event_type;
backend/migrations/20241024125924_autoscaling.up.sql
+13
View File
@@ -0,0 +1,13 @@
-- Add up migration script here
CREATE TYPE AUTOSCALING_EVENT_TYPE AS ENUM ('full_scaleout', 'scalein', 'scaleout');
CREATE TABLE autoscaling_event (
id SERIAL PRIMARY KEY,
worker_group TEXT NOT NULL,
event_type AUTOSCALING_EVENT_TYPE NOT NULL,
desired_workers INTEGER NOT NULL,
applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
reason TEXT
);
CREATE INDEX autoscaling_event_worker_group_idx ON autoscaling_event (worker_group, applied_at);
backend/oauth_connect.json
+19
View File
@@ -152,5 +152,24 @@
"vismanet_erp_interactive_api:read",
"vismanet_erp_interactive_api:update"
]
},
"spotify": {
"auth_url": "https://accounts.spotify.com/authorize",
"token_url": "https://accounts.spotify.com/api/token",
"scopes": [
"user-read-playback-state",
"user-modify-playback-state",
"user-read-currently-playing",
"playlist-read-private",
"playlist-read-collaborative",
"playlist-modify-private",
"playlist-modify-public",
"user-follow-read",
"user-read-playback-position",
"user-read-recently-played",
"user-top-read",
"user-library-modify",
"user-library-read"
]
}
}
backend/parsers/windmill-parser-py-imports/src/lib.rs
+1
View File
@@ -61,6 +61,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
"opensearchpy" => "opensearch-py",
"lokalise" => "python-lokalise-api",
"msgraph" => "msgraph-sdk",
"pythonjsonlogger" => "python-json-logger",
};
fn replace_import(x: String) -> String {
backend/parsers/windmill-parser-yaml/src/lib.rs
+26 -3
View File
@@ -188,9 +188,15 @@ pub struct AnsiblePlaybookOptions {
pub force_handlers: Option<()>,
}
#[derive(Debug, Clone)]
pub enum ResourceOrVariablePath {
Resource(String),
Variable(String),
}
#[derive(Debug, Clone)]
pub struct FileResource {
pub resource_path: String,
pub resource_path: ResourceOrVariablePath,
pub target_path: String,
}
@@ -309,7 +315,7 @@ pub fn parse_ansible_reqs(
}
}
}
Yaml::String(key) if key == "file_resources" => {
Yaml::String(key) if key == "files" || key == "file_resources" => {
if let Yaml::Array(file_resources) = value {
let resources: anyhow::Result<Vec<FileResource>> =
file_resources.iter().map(parse_file_resource).collect();
@@ -440,7 +446,24 @@ fn parse_file_resource(yaml: &Yaml) -> anyhow::Result<FileResource> {
"No `target` provided for file resource {}. Please input a target relative path for the ansible playbook to see this file.",
resource_path
))?;
return Ok(FileResource { resource_path: resource_path.clone(), target_path });
return Ok(FileResource {
resource_path: ResourceOrVariablePath::Resource(resource_path.clone()),
target_path,
});
}
if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("variable".to_string())) {
let target_path = f
.get(&Yaml::String("target".to_string()))
.and_then(|x| x.as_str())
.map(|x| x.to_string())
.ok_or(anyhow!(
"No `target` provided for file resource {}. Please input a target relative path for the ansible playbook to see this file.",
resource_path
))?;
return Ok(FileResource {
resource_path: ResourceOrVariablePath::Variable(resource_path.clone()),
target_path,
});
}
return Err(anyhow!(
"File resource should have a `resource` field, linking to a text file resource"
backend/src/ee.rs
+2 -10
View File
@@ -1,16 +1,8 @@
use anyhow::anyhow;
#[cfg(feature = "enterprise")]
use windmill_common::error::{Error, Result};
pub async fn set_license_key(_license_key: String) -> anyhow::Result<()> {
pub async fn set_license_key(_license_key: String) -> () {
// Implementation is not open source
Err(anyhow!("License cannot be set in Windmill CE"))
}
#[cfg(feature = "enterprise")]
pub async fn verify_license_key() -> Result<()> {
pub async fn verify_license_key() -> () {
// Implementation is not open source
Err(Error::InternalErr(
"License always invalid in Windmill CE".to_string(),
))
}
backend/src/main.rs
+77 -14
View File
@@ -27,7 +27,7 @@ use uuid::Uuid;
use windmill_api::HTTP_CLIENT;
#[cfg(feature = "enterprise")]
use windmill_common::ee::schedule_key_renewal;
use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICENSE_KEY_VALID};
use windmill_common::{
global_settings::{
@@ -284,7 +284,8 @@ async fn windmill_main() -> anyhow::Result<()> {
tracing::info!("Binary is in 'indexer' mode");
#[cfg(not(feature = "tantivy"))]
{
panic!("Indexer mode requires the tantivy feature flag");
tracing::error!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features.");
panic!("Indexer mode requires compiling with the tantivy feature flag.");
}
#[cfg(feature = "tantivy")]
Mode::Indexer
@@ -307,7 +308,8 @@ async fn windmill_main() -> anyhow::Result<()> {
Mode::Standalone
});
let num_workers = if mode == Mode::Server || mode == Mode::Indexer {
#[allow(unused_mut)]
let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer {
0
} else {
std::env::var("NUM_WORKERS")
@@ -422,6 +424,50 @@ Windmill Community Edition {GIT_VERSION}
display_config(&ENV_SETTINGS);
if let Err(e) = reload_base_url_setting(&db).await {
tracing::error!("Error loading base url: {:?}", e)
}
if let Err(e) = reload_critical_error_channels_setting(&db).await {
tracing::error!("Could loading critical error emails setting: {:?}", e);
}
#[cfg(feature = "enterprise")]
{
// load the license key and check if it's valid
// if not valid and not server mode just quit
// if not expired and server mode then force renewal
// if key still invalid and num_workers > 0, set to 0
if let Err(err) = reload_license_key(&db).await {
tracing::error!("Failed to reload license key: {err:#}");
}
let valid_key = *LICENSE_KEY_VALID.read().await;
if !valid_key && !server_mode {
panic!("Invalid license key, workers require a valid license key");
}
if server_mode {
// only force renewal if invalid but not empty (= expired)
let renewed_now = maybe_renew_license_key_on_start(
&HTTP_CLIENT,
&db,
!valid_key && !LICENSE_KEY_ID.read().await.is_empty(),
)
.await;
if renewed_now {
if let Err(err) = reload_license_key(&db).await {
tracing::error!("Failed to reload license key: {err:#}");
}
}
if num_workers > 0 {
let valid_key = *LICENSE_KEY_VALID.read().await;
if !valid_key {
tracing::warn!("License key invalid, setting num_workers to 0");
num_workers = 0;
}
}
}
}
let worker_mode = num_workers > 0;
if server_mode || worker_mode || indexer_mode {
@@ -448,7 +494,16 @@ Windmill Community Edition {GIT_VERSION}
initial_load(&db, killpill_tx.clone(), worker_mode, server_mode, is_agent).await;
monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode, true).await;
monitor_db(
&db,
&base_internal_url,
rsmq.clone(),
server_mode,
worker_mode,
true,
killpill_tx.clone(),
)
.await;
monitor_pool(&db).await;
@@ -548,8 +603,11 @@ Windmill Community Edition {GIT_VERSION}
rx.recv().await?;
}
}
tracing::info!("Starting phase 2 of shutdown");
killpill_phase2_tx.send(())?;
if killpill_phase2_tx.receiver_count() > 0 {
tracing::info!("Starting phase 2 of shutdown");
killpill_phase2_tx.send(())?;
tracing::info!("Phase 2 of shutdown completed");
}
Ok(()) as anyhow::Result<()>
};
@@ -575,7 +633,9 @@ Windmill Community Edition {GIT_VERSION}
&base_internal_url,
rsmq.clone(),
server_mode,
false
worker_mode,
false,
tx.clone(),
)
.await;
},
@@ -618,7 +678,15 @@ Windmill Community Edition {GIT_VERSION}
},
LICENSE_KEY_SETTING => {
if let Err(e) = reload_license_key(&db).await {
tracing::error!(error = %e, "Could not reload license key setting");
tracing::error!("Failed to reload license key: {e:#}");
}
#[cfg(feature = "enterprise")]
if worker_mode {
let valid_key = *LICENSE_KEY_VALID.read().await;
if !valid_key {
tracing::error!("Invalid license key, exiting...");
tx.send(()).expect("send");
}
}
},
DEFAULT_TAGS_PER_WORKSPACE_SETTING => {
@@ -764,15 +832,10 @@ Windmill Community Edition {GIT_VERSION}
Ok(()) as anyhow::Result<()>
};
if mode == Mode::Server || mode == Mode::Standalone {
if server_mode {
schedule_stats(&db, &HTTP_CLIENT).await;
}
#[cfg(feature = "enterprise")]
if mode == Mode::Server || mode == Mode::Standalone {
schedule_key_renewal(&HTTP_CLIENT, &db).await;
}
futures::try_join!(
shutdown_signal,
workers_f,
backend/src/monitor.rs
+29 -42
View File
@@ -27,7 +27,7 @@ use windmill_api::{
DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN,
};
#[cfg(feature = "enterprise")]
use windmill_common::ee::{worker_groups_alerts, jobs_waiting_alerts};
use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts, LICENSE_KEY_VALID};
use windmill_common::{
auth::JWT_SECRET,
ee::CriticalErrorChannel,
@@ -76,9 +76,6 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
#[cfg(feature = "enterprise")]
use crate::ee::verify_license_key;
#[cfg(feature = "enterprise")]
use windmill_common::ee::LICENSE_KEY_VALID;
use crate::ee::set_license_key;
lazy_static::lazy_static! {
@@ -152,18 +149,10 @@ pub async fn initial_load(
tracing::error!("Error reloading custom tags: {:?}", e)
}
if let Err(e) = reload_base_url_setting(db).await {
tracing::error!("Error reloading base url: {:?}", e)
}
if let Err(e) = reload_hub_base_url_setting(db, server_mode).await {
tracing::error!("Error reloading hub base url: {:?}", e)
}
if let Err(e) = reload_critical_error_channels_setting(&db).await {
tracing::error!("Could not reload critical error emails setting: {:?}", e);
}
if let Err(e) = reload_jwt_secret_setting(&db).await {
tracing::error!("Could not reload jwt secret setting: {:?}", e);
}
@@ -182,11 +171,6 @@ pub async fn initial_load(
reload_scim_token_setting(&db).await;
}
#[cfg(feature = "enterprise")]
if let Err(e) = reload_license_key(&db).await {
tracing::error!("Error reloading license key: {:?}", e)
}
if worker_mode {
reload_extra_pip_index_url_setting(&db).await;
reload_pip_index_url_setting(&db).await;
@@ -854,8 +838,10 @@ pub async fn reload_request_size(db: &DB) {
}
}
pub async fn reload_license_key(db: &DB) -> error::Result<()> {
let q = load_value_from_global_settings(db, LICENSE_KEY_SETTING).await?;
pub async fn reload_license_key(db: &DB) -> anyhow::Result<()> {
let q = load_value_from_global_settings(db, LICENSE_KEY_SETTING)
.await
.map_err(|err| anyhow::anyhow!("Error reloading license key: {}", err.to_string()))?;
let mut value = std::env::var("LICENSE_KEY")
.ok()
@@ -873,9 +859,7 @@ pub async fn reload_license_key(db: &DB) -> error::Result<()> {
tracing::error!("Could not parse LICENSE_KEY found: {:#?}", &q);
}
};
set_license_key(value).await?;
set_license_key(value).await;
Ok(())
}
@@ -1014,7 +998,9 @@ pub async fn monitor_db(
base_internal_url: &str,
rsmq: Option<MultiplexedRsmq>,
server_mode: bool,
_worker_mode: bool,
initial_load: bool,
_killpill_tx: tokio::sync::broadcast::Sender<()>,
) {
let zombie_jobs_f = async {
if server_mode && !initial_load {
@@ -1035,15 +1021,14 @@ pub async fn monitor_db(
let verify_license_key_f = async {
#[cfg(feature = "enterprise")]
if let Err(e) = verify_license_key().await {
tracing::error!("Error verifying license key: {:?}", e);
let mut l = LICENSE_KEY_VALID.write().await;
*l = false;
} else {
let is_valid = LICENSE_KEY_VALID.read().await.clone();
if !is_valid {
let mut l = LICENSE_KEY_VALID.write().await;
*l = true;
if !initial_load {
verify_license_key().await;
if _worker_mode {
let valid_key = *LICENSE_KEY_VALID.read().await;
if !valid_key {
tracing::error!("Invalid license key, exiting...");
_killpill_tx.send(()).expect("send");
}
}
}
};
@@ -1068,6 +1053,15 @@ pub async fn monitor_db(
}
};
let apply_autoscaling_f = async {
#[cfg(feature = "enterprise")]
if server_mode && !initial_load {
if let Err(e) = windmill_autoscaling::apply_all_autoscaling(db).await {
tracing::error!("Error applying autoscaling: {:?}", e);
}
}
};
join!(
expired_items_f,
zombie_jobs_f,
@@ -1075,6 +1069,7 @@ pub async fn monitor_db(
verify_license_key_f,
worker_groups_alerts_f,
jobs_waiting_alerts_f,
apply_autoscaling_f,
);
}
@@ -1092,19 +1087,11 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
.unwrap_or(true);
if metrics_enabled || save_metrics {
let queue_counts = sqlx::query!(
"SELECT tag, count(*) as count FROM queue WHERE
scheduled_for <= now() - ('3 seconds')::interval AND running = false
GROUP BY tag"
)
.fetch_all(db)
.await
.ok()
.unwrap_or_else(|| vec![]);
let queue_counts = windmill_common::queue::get_queue_counts(db).await;
for q in queue_counts {
let count = q.count.unwrap_or(0);
let tag = q.tag;
let count = q.1;
let tag = q.0;
if metrics_enabled {
let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
metric.set(count as i64);
backend/windmill-api/openapi.yaml
+51 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.410.3
version: 1.416.2
title: Windmill API
contact:
@@ -1467,6 +1467,11 @@ paths:
parameters:
- $ref: "#/components/parameters/Page"
- $ref: "#/components/parameters/PerPage"
- name: active_only
in: query
description: filter only active users
schema:
type: boolean
responses:
"200":
description: user
@@ -8198,6 +8203,29 @@ paths:
items:
$ref: "#/components/schemas/Config"
/configs/list_autoscaling_events/{worker_group}:
get:
summary: List autoscaling events
operationId: listAutoscalingEvents
tags:
- config
parameters:
- name: worker_group
in: path
required: true
schema:
type: string
responses:
"200":
description: List of autoscaling events
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/AutoscalingEvent"
/w/{workspace}/acls/get/{kind}/{path}:
get:
summary: get granular acls
@@ -10735,6 +10763,7 @@ components:
- "users.delete"
- "users.update"
- "users.login"
- "users.login_failure"
- "users.logout"
- "users.accept_invite"
- "users.decline_invite"
@@ -10746,6 +10775,7 @@ components:
- "users.impersonate"
- "users.leave_workspace"
- "oauth.login"
- "oauth.login_failure"
- "oauth.signup"
- "variables.create"
- "variables.delete"
@@ -11766,6 +11796,8 @@ components:
type: string
username:
type: string
operator_only:
type: boolean
required:
- email
@@ -12467,3 +12499,21 @@ components:
properties:
dancer:
type: string
AutoscalingEvent:
type: object
properties:
id:
type: integer
format: int64
worker_group:
type: string
event_type:
type: string
desired_workers:
type: integer
reason:
type: string
applied_at:
type: string
format: date-time
backend/windmill-api/src/configs.rs
+28
View File
@@ -29,6 +29,10 @@ pub fn global_service() -> Router {
.route("/update/:name", post(update_config).delete(delete_config))
.route("/get/:name", get(get_config))
.route("/list", get(list_configs))
.route(
"/list_autoscaling_events/:worker_group",
get(list_autoscaling_events),
)
}
#[derive(Serialize, Deserialize, FromRow)]
@@ -177,6 +181,30 @@ async fn delete_config(
Ok(format!("Deleted config {name}"))
}
#[derive(Serialize, Deserialize, FromRow)]
struct AutoscalingEvent {
id: i64,
worker_group: String,
event_type: Option<String>,
desired_workers: i32,
reason: Option<String>,
applied_at: chrono::NaiveDateTime,
}
async fn list_autoscaling_events(
Extension(db): Extension<DB>,
Path(worker_group): Path<String>,
) -> error::JsonResult<Vec<AutoscalingEvent>> {
let events = sqlx::query_as!(
AutoscalingEvent,
"SELECT id, worker_group, event_type::text, desired_workers, reason, applied_at FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT 5",
worker_group
)
.fetch_all(&db)
.await?;
Ok(Json(events))
}
#[cfg(feature = "enterprise")]
async fn list_configs(
authed: ApiAuthed,
backend/windmill-api/src/db.rs
+13 -13
View File
@@ -173,17 +173,6 @@ pub async fn migrate(db: &DB) -> Result<(), Error> {
let migrator = db.acquire().await?;
let mut custom_migrator = CustomMigrator { inner: migrator };
if let Err(err) = fix_flow_versioning_migration(&mut custom_migrator, db).await {
tracing::error!("Could not apply flow versioning fix migration: {err:#}");
}
let db2 = db.clone();
let _ = tokio::task::spawn(async move {
if let Err(err) = fix_job_completed_index(&db2).await {
tracing::error!("Could not apply job completed index fix migration: {err:#}");
}
});
match sqlx::migrate!("../migrations")
.run_direct(&mut custom_migrator)
.await
@@ -199,6 +188,17 @@ pub async fn migrate(db: &DB) -> Result<(), Error> {
Err(err) => Err(err),
}?;
if let Err(err) = fix_flow_versioning_migration(&mut custom_migrator, db).await {
tracing::error!("Could not apply flow versioning fix migration: {err:#}");
}
let db2 = db.clone();
let _ = tokio::task::spawn(async move {
if let Err(err) = fix_job_completed_index(&db2).await {
tracing::error!("Could not apply job completed index fix migration: {err:#}");
}
});
Ok(())
}
@@ -302,7 +302,7 @@ macro_rules! run_windmill_migration {
.await?;
tracing::info!("Finished applying {migration_job_name} migration");
} else {
tracing::info!("migration {migration_job_name} already done");
tracing::debug!("migration {migration_job_name} already done");
}
let _ = sqlx::query("SELECT pg_advisory_unlock(4242)")
@@ -311,7 +311,7 @@ macro_rules! run_windmill_migration {
tx.commit().await?;
tracing::info!("released lock for {migration_job_name}");
} else {
tracing::info!("migration {migration_job_name} already done");
tracing::debug!("migration {migration_job_name} already done");
}
}
backend/windmill-api/src/ee.rs
+1 -1
View File
@@ -4,7 +4,7 @@ use std::sync::Arc;
#[cfg(feature = "enterprise")]
use tokio::sync::RwLock;
pub async fn validate_license_key(_license_key: String) -> anyhow::Result<String> {
pub async fn validate_license_key(_license_key: String) -> anyhow::Result<(String, bool)> {
// Implementation is not open source
Err(anyhow!("License can't be validated in Windmill CE"))
}
backend/windmill-api/src/embeddings.rs
+5 -1
View File
@@ -53,6 +53,7 @@ use crate::{resources::ResourceType, HTTP_CLIENT};
lazy_static::lazy_static! {
pub static ref EMBEDDINGS_DB: Arc<RwLock<Option<EmbeddingsDb>>> = Arc::new(RwLock::new(None));
pub static ref MODEL_INSTANCE: Arc<RwLock<Option<Arc<ModelInstance>>>> = Arc::new(RwLock::new(None));
pub static ref HUB_EMBEDDINGS_PULLING_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_PULLING_INTERVAL_SECS").ok().map(|x| x.parse::<u64>().ok()).flatten().unwrap_or(3600 * 24);
}
#[cfg(feature = "embedding")]
@@ -607,7 +608,10 @@ pub fn load_embeddings_db(db: &Pool<Postgres>) -> () {
drop(model_instance_lock);
loop {
update_embeddings_db(&db_clone).await;
tokio::time::sleep(std::time::Duration::from_secs(3600 * 24)).await;
tokio::time::sleep(std::time::Duration::from_secs(
*HUB_EMBEDDINGS_PULLING_INTERVAL_SECS,
))
.await;
}
} else {
tracing::error!(
backend/windmill-api/src/flows.rs
-4
View File
@@ -544,7 +544,6 @@ async fn get_latest_version(
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Option<FlowVersion>> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
@@ -1333,7 +1332,6 @@ mod tests {
},
"type": "script",
"path": "test",
"tag_override": Option::<String>::None,
},
},
{
@@ -1377,14 +1375,12 @@ mod tests {
"input_transforms": {},
"type": "script",
"path": "test",
"tag_override": Option::<String>::None,
},
"stop_after_if": {
"expr": "previous.isEmpty()",
"skip_if_stopped": false
}
},
"preprocessor_module": Option::<String>::None
});
assert_eq!(dbg!(serde_json::json!(fv)), dbg!(expect));
}
backend/windmill-api/src/jobs.rs
+2 -29
View File
@@ -2815,7 +2815,6 @@ pub async fn run_flow_by_path_inner(
let flow_path = flow_path.to_path();
check_scopes(&authed, || format!("run:flow/{flow_path}"))?;
let (tag, dedicated_worker, has_preprocessor) = sqlx::query!(
"SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor
FROM flow
@@ -2910,7 +2909,6 @@ pub async fn restart_flow(
) -> error::Result<(StatusCode, String)> {
check_license_key_valid().await?;
let completed_job = sqlx::query_as::<_, CompletedJob>(
"SELECT *, result->'wm_labels' as labels from completed_job WHERE id = $1 and workspace_id = $2",
)
@@ -3010,7 +3008,6 @@ pub async fn run_script_by_path_inner(
check_scopes(&authed, || format!("run:script/{script_path}"))?;
let (job_payload, tag, _delete_after_use, timeout) =
script_path_to_payload(script_path, &db, &w_id, run_query.skip_preprocessor).await?;
let scheduled_for = run_query.get_scheduled_for(&db).await?;
@@ -3018,7 +3015,6 @@ pub async fn run_script_by_path_inner(
let tag = run_query.tag.clone().or(tag);
check_tag_available_for_workspace(&w_id, &tag).await?;
let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq);
let (uuid, tx) = push(
@@ -3067,7 +3063,6 @@ pub async fn run_workflow_as_code(
Query(wkflow_query): Query<WorkflowAsCodeQuery>,
Json(task): Json<WorkflowTask>,
) -> error::Result<(StatusCode, String)> {
let mut i = 1;
if *CLOUD_HOSTED {
@@ -3079,23 +3074,18 @@ pub async fn run_workflow_as_code(
check_license_key_valid().await?;
check_tag_available_for_workspace(&w_id, &run_query.tag).await?;
if *CLOUD_HOSTED {
tracing::info!("workflow_as_code_tracing id {i} ");
i += 1;
}
let job = get_queued_job(&job_id, &w_id, &db).await?;
if *CLOUD_HOSTED {
tracing::info!("workflow_as_code_tracing id {i} ");
i += 1;
}
let job = not_found_if_none(job, "Queued Job", &job_id.to_string())?;
let (job_payload, tag, _delete_after_use, timeout) = match job.job_kind {
JobKind::Preview => (
@@ -3118,18 +3108,12 @@ pub async fn run_workflow_as_code(
run_query.timeout,
),
JobKind::Script => {
script_path_to_payload(
job.script_path(),
&db,
&w_id,
run_query.skip_preprocessor,
)
.await?
script_path_to_payload(job.script_path(), &db, &w_id, run_query.skip_preprocessor)
.await?
}
_ => return Err(anyhow::anyhow!("Not supported").into()),
};
if *CLOUD_HOSTED {
tracing::info!("workflow_as_code_tracing id {i} ");
i += 1;
@@ -3143,22 +3127,18 @@ pub async fn run_workflow_as_code(
let tag = run_query.tag.clone().or(tag).or(Some(job.tag));
if *CLOUD_HOSTED {
tracing::info!("workflow_as_code_tracing id {i} ");
i += 1;
}
let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq);
if *CLOUD_HOSTED {
tracing::info!("workflow_as_code_tracing id {i} ");
i += 1;
}
let (uuid, mut tx) = push(
&db,
tx,
@@ -3185,7 +3165,6 @@ pub async fn run_workflow_as_code(
)
.await?;
if *CLOUD_HOSTED {
tracing::info!("workflow_as_code_tracing id {i} ");
i += 1;
@@ -3203,16 +3182,13 @@ pub async fn run_workflow_as_code(
tracing::info!("Skipping update of flow status for job {job_id} in workspace {w_id}");
}
if *CLOUD_HOSTED {
tracing::info!("workflow_as_code_tracing id {i} ");
i += 1;
}
tx.commit().await?;
if *CLOUD_HOSTED {
tracing::info!("workflow_as_code_tracing id {i} ");
}
@@ -3866,7 +3842,6 @@ pub async fn run_wait_result_flow_by_path_internal(
let flow_path = flow_path.to_path();
check_scopes(&authed, || format!("run:flow/{flow_path}"))?;
let scheduled_for = run_query.get_scheduled_for(&db).await?;
let (tag, dedicated_worker, early_return, has_preprocessor) = sqlx::query!(
@@ -4362,7 +4337,6 @@ async fn add_batch_jobs(
}
}
"flow" => {
let mut uuids: Vec<Uuid> = Vec::new();
let payload = if let Some(ref fv) = batch_info.flow_value {
JobPayload::RawFlow { value: fv.clone(), path: None, restarted_from: None }
@@ -4580,7 +4554,6 @@ pub async fn run_job_by_hash_inner(
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
let hash = script_hash.0;
let (
path,
backend/windmill-api/src/lib.rs
+8 -2
View File
@@ -84,12 +84,14 @@ mod stripe_ee;
mod tracing_init;
mod triggers;
mod users;
mod users_ee;
mod utils;
mod variables;
mod webhook_util;
mod websocket_triggers;
mod workers;
mod workspaces;
mod workspaces_ee;
pub const GIT_VERSION: &str =
git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
@@ -456,9 +458,13 @@ async fn ee_license() -> &'static str {
#[cfg(feature = "enterprise")]
async fn ee_license() -> String {
use windmill_common::ee::LICENSE_KEY_ID;
use windmill_common::ee::{LICENSE_KEY_ID, LICENSE_KEY_VALID};
LICENSE_KEY_ID.read().await.clone()
if *LICENSE_KEY_VALID.read().await {
LICENSE_KEY_ID.read().await.clone()
} else {
"".to_string()
}
}
async fn openapi() -> &'static str {
backend/windmill-api/src/settings.rs
+10 -5
View File
@@ -28,13 +28,13 @@ use serde::Deserialize;
#[cfg(feature = "enterprise")]
use windmill_common::ee::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel};
use windmill_common::{
email_ee::send_email,
error::{self, JsonResult, Result},
global_settings::{
AUTOMATE_USERNAME_CREATION_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
HUB_BASE_URL_SETTING,
HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING,
},
server::Smtp,
utils::send_email,
};
#[cfg(feature = "parquet")]
@@ -163,8 +163,13 @@ pub async fn test_license_key(
Json(TestKey { license_key }): Json<TestKey>,
) -> error::Result<String> {
require_super_admin(&db, &authed.email).await?;
validate_license_key(license_key).await?;
Ok("Sent test email".to_string())
let (_, expired) = validate_license_key(license_key).await?;
if expired {
Err(error::Error::BadRequest("Expired license key".to_string()))
} else {
Ok("Valid license key".to_string())
}
}
pub async fn get_local_settings(
@@ -257,6 +262,7 @@ pub async fn get_global_setting(
&& !key.starts_with("default_success_handler_")
&& key != AUTOMATE_USERNAME_CREATION_SETTING
&& key != HUB_BASE_URL_SETTING
&& key != HUB_ACCESSIBLE_URL_SETTING
&& key != EMAIL_DOMAIN_SETTING
{
require_super_admin(&db, &authed.email).await?;
@@ -301,7 +307,6 @@ pub async fn send_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Resu
windmill_common::stats_ee::send_stats(
&HTTP_CLIENT,
&db,
true,
windmill_common::stats_ee::SendStatsReason::Manual,
)
.await?;
backend/windmill-api/src/users.rs
+3169
View File
File diff suppressed because it is too large Load Diff
backend/windmill-api/src/users_ee.rs
+41
View File
@@ -0,0 +1,41 @@
use std::sync::Arc;
use crate::db::ApiAuthed;
use crate::users::{EditPassword, NewUser};
use crate::{db::DB, webhook_util::WebhookShared};
use argon2::Argon2;
use http::StatusCode;
use windmill_common::error::{Error, Result};
pub async fn create_user(
_authed: ApiAuthed,
_db: DB,
_webhook: WebhookShared,
_argon2: Arc<Argon2<'_>>,
_rsmq: Option<rsmq_async::MultiplexedRsmq>,
mut _nu: NewUser,
) -> Result<(StatusCode, String)> {
Err(Error::InternalErr(
"Not implemented in Windmill's Open Source repository".to_string(),
))
}
pub async fn set_password(
_db: DB,
_argon2: Arc<Argon2<'_>>,
_authed: ApiAuthed,
_ep: EditPassword,
) -> Result<String> {
Err(Error::InternalErr(
"Not implemented in Windmill's Open Source repository".to_string(),
))
}
pub fn send_email_if_possible(_subject: &str, _content: &str, _to: &str) {
tracing::warn!(
"send_email_if_possible is not implemented in Windmill's Open Source repository"
);
}
backend/windmill-api/src/utils.rs
-19
View File
@@ -155,25 +155,6 @@ pub async fn get_instance_username_or_create_pending<'c>(
}
}
pub async fn get_and_delete_pending_username_or_generate<'c>(
tx: &mut Transaction<'c, Postgres>,
email: &str,
) -> error::Result<String> {
let username = sqlx::query_scalar!("SELECT username FROM pending_user WHERE email = $1", email)
.fetch_optional(&mut **tx)
.await?;
if let Some(username) = username {
sqlx::query!("DELETE FROM pending_user WHERE email = $1", email)
.execute(&mut **tx)
.await?;
Ok(username)
} else {
let username = generate_instance_wide_unique_username(&mut *tx, email).await?;
Ok(username)
}
}
pub fn content_plain(body: Body) -> Response {
use axum::http::header;
Response::builder()
backend/windmill-api/src/workspaces.rs
+11 -293
View File
@@ -9,14 +9,15 @@
use std::collections::HashMap;
use crate::db::ApiAuthed;
use crate::utils::{get_instance_username_or_create_pending, INVALID_USERNAME_CHARS};
use crate::users_ee::send_email_if_possible;
use crate::utils::get_instance_username_or_create_pending;
use crate::BASE_URL;
use crate::{
apps::AppWithLastVersion,
db::DB,
folders::Folder,
resources::{Resource, ResourceType},
users::{send_email_if_possible, WorkspaceInvite, VALID_USERNAME},
users::{WorkspaceInvite, VALID_USERNAME},
utils::require_super_admin,
webhook_util::WebhookShared,
};
@@ -34,7 +35,7 @@ use itertools::Itertools;
use regex::Regex;
use uuid::Uuid;
use windmill_audit::audit_ee::{audit_log, AuditAuthor, AuditAuthorable};
use windmill_audit::audit_ee::audit_log;
use windmill_audit::ActionKind;
use windmill_common::db::UserDB;
use windmill_common::s3_helpers::LargeFileStorage;
@@ -218,11 +219,12 @@ struct EditDeployTo {
deploy_to: Option<String>,
}
#[allow(dead_code)]
#[derive(Deserialize)]
struct EditAutoInvite {
operator: Option<bool>,
invite_all: Option<bool>,
auto_add: Option<bool>,
pub struct EditAutoInvite {
pub operator: Option<bool>,
pub invite_all: Option<bool>,
pub auto_add: Option<bool>,
}
#[derive(Deserialize)]
@@ -573,250 +575,21 @@ async fn edit_deploy_to() -> Result<String> {
));
}
const BANNED_DOMAINS: &str = include_str!("../banned_domains.txt");
pub const BANNED_DOMAINS: &str = include_str!("../banned_domains.txt");
async fn is_allowed_auto_domain(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<bool> {
let domain = email.split('@').last().unwrap();
return Ok(Json(!BANNED_DOMAINS.contains(domain)));
}
async fn auto_add_user(
email: &str,
w_id: &str,
operator: &bool,
tx: &mut Transaction<'_, Postgres>,
authorable: &impl AuditAuthorable,
) -> Result<String> {
let automate_username_creation = sqlx::query_scalar!(
"SELECT value FROM global_settings WHERE name = $1",
AUTOMATE_USERNAME_CREATION_SETTING,
)
.fetch_optional(&mut **tx)
.await?
.map(|v| v.as_bool())
.flatten()
.unwrap_or(false);
let username = if automate_username_creation {
get_instance_username_or_create_pending(&mut *tx, &email).await?
} else {
let mut username = email
.split('@')
.next()
.unwrap()
.to_string()
.replace(".", "");
username = INVALID_USERNAME_CHARS
.replace_all(&mut username, "")
.to_string();
if username.is_empty() {
username = "user".to_string()
}
let base_username = username.clone();
let mut username_conflict = true;
let mut i = 1;
while username_conflict {
if i > 1000 {
return Err(Error::InternalErr(format!(
"too many username conflicts for {}",
email
)));
}
if i > 1 {
username = format!("{}{}", base_username, i)
}
username_conflict = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 AND workspace_id = $2)",
&username,
&w_id
)
.fetch_one(&mut **tx)
.await?
.unwrap_or(false);
i += 1;
}
username
};
sqlx::query!(
"INSERT INTO usr (workspace_id, username, email, is_admin, operator) VALUES ($1, $2, $3, false, $4) ON CONFLICT DO NOTHING",
&w_id,
&username,
&email,
&operator
)
.execute(&mut **tx)
.await?;
sqlx::query_as!(
Group,
"INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING",
&w_id,
username,
"all",
)
.execute(&mut **tx)
.await?;
let audit_author = if authorable.username() == authorable.email() && authorable.email() == email
{
// if the user is auto adding themselves (e.g. by joining the instance), we use their newly created workspace username for audit logs
AuditAuthor {
username: username.clone(),
email: email.to_string(),
username_override: None,
}
} else {
AuditAuthor {
username: authorable.username().to_string(),
email: authorable.email().to_string(),
username_override: authorable.username_override().map(|x| x.to_string()),
}
};
audit_log(
&mut **tx,
&audit_author,
"users.auto_invite_add",
ActionKind::Create,
&w_id,
Some(email),
None,
)
.await?;
Ok(username)
}
async fn edit_auto_invite(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path(w_id): Path<String>,
ApiAuthed { is_admin, email, username, .. }: ApiAuthed,
Json(ea): Json<EditAutoInvite>,
) -> Result<String> {
require_admin(is_admin, &username)?;
// #[cfg(not(feature = "enterprise"))]
// {
// return Err(Error::BadRequest(
// "Auto-invite is only available on enterprise".to_string(),
// ));
// }
let domain = if ea.invite_all.is_some_and(|x| x) {
if *CLOUD_HOSTED {
return Err(Error::BadRequest(
"invite_all is only available locally".to_string(),
));
} else {
"*"
}
} else {
email.split('@').last().unwrap()
};
let mut tx = db.begin().await?;
let mut users_to_auto_add = Option::None;
if let (Some(operator), Some(auto_add)) = (ea.operator, ea.auto_add) {
if BANNED_DOMAINS.contains(domain) {
return Err(Error::BadRequest(format!(
"Domain {} is not allowed",
domain
)));
}
sqlx::query!(
"UPDATE workspace_settings SET auto_invite_domain = $1, auto_invite_operator = $2, auto_add = $4 WHERE workspace_id = $3",
domain,
operator,
&w_id,
auto_add,
)
.execute(&mut *tx)
.await?;
if auto_add {
users_to_auto_add = Some(sqlx::query!(
"SELECT email FROM password WHERE ($2::text = '*' OR email LIKE CONCAT('%', $2::text)) AND NOT EXISTS (
SELECT 1 FROM usr WHERE workspace_id = $1::text AND email = password.email
)",
&w_id,
domain
)
.fetch_all(&mut *tx).await?);
for user in users_to_auto_add.as_ref().unwrap() {
auto_add_user(&user.email, &w_id, &operator, &mut tx, &authed).await?;
send_email_if_possible(
&format!("Added to Windmill's workspace: {w_id}"),
&format!(
"You have been granted access to Windmill's workspace {w_id} by {email}.
Access the workspace at {}/?workspace={w_id}",
BASE_URL.read().await.clone()
),
&user.email,
);
}
} else {
sqlx::query!(
"INSERT INTO workspace_invite
(workspace_id, email, is_admin, operator)
SELECT $1::text, email, false, $3 FROM password WHERE ($2::text = '*' OR email LIKE CONCAT('%', $2::text)) AND NOT EXISTS (
SELECT 1 FROM usr WHERE workspace_id = $1::text AND email = password.email
)
ON CONFLICT DO NOTHING",
&w_id,
domain,
operator
)
.execute(&mut *tx)
.await?;
}
} else {
sqlx::query!(
"UPDATE workspace_settings SET auto_invite_domain = NULL, auto_invite_operator = NULL, auto_add = NULL WHERE workspace_id = $1",
&w_id,
)
.execute(&mut *tx)
.await?;
}
audit_log(
&mut *tx,
&authed,
"workspaces.edit_auto_invite_domain",
ActionKind::Update,
&w_id,
Some(&authed.email),
Some([("operator", &format!("{:?}", ea.operator)[..])].into()),
)
.await?;
tx.commit().await?;
if let Some(users) = users_to_auto_add {
for user in users {
handle_deployment_metadata(
&email,
&username,
&db,
&w_id,
windmill_git_sync::DeployedObject::User { email: user.email.clone() },
Some(format!("Auto-added user '{}' to workspace", &user.email)),
rsmq.clone(),
true,
)
.await?;
}
}
Ok(format!(
"Edit auto-invite for workspace {} to {}",
&w_id, domain
))
crate::workspaces_ee::edit_auto_invite(authed, db, rsmq, w_id, ea).await
}
async fn edit_webhook(
@@ -2001,61 +1774,6 @@ async fn delete_workspace(
Ok(format!("Deleted workspace {}", &w_id))
}
pub async fn invite_user_to_all_auto_invite_worspaces(
db: &DB,
email: &str,
rsmq: Option<rsmq_async::MultiplexedRsmq>,
authorable: &impl AuditAuthorable,
) -> Result<()> {
let mut tx = db.begin().await?;
let domain = email.split('@').last().unwrap();
let workspaces = sqlx::query!(
"SELECT workspace_id, auto_invite_operator, auto_add FROM workspace_settings ws WHERE (auto_invite_domain = $1 OR auto_invite_domain = '*') AND NOT EXISTS (SELECT 1 FROM usr WHERE workspace_id = ws.workspace_id AND email = $2)",
domain,
email
)
.fetch_all(&mut *tx)
.await?;
let mut auto_added_workspace_usernames: Vec<(String, String)> = vec![];
for r in workspaces {
if r.auto_add.is_some() && r.auto_add.unwrap() {
let operator = r.auto_invite_operator.unwrap_or(false);
let username =
auto_add_user(email, &r.workspace_id, &operator, &mut tx, authorable).await?;
auto_added_workspace_usernames.push((r.workspace_id, username));
} else {
sqlx::query!(
"INSERT INTO workspace_invite
(workspace_id, email, is_admin, operator)
VALUES ($1, $2, false, $3)
ON CONFLICT DO NOTHING",
r.workspace_id,
email,
r.auto_invite_operator
)
.execute(&mut *tx)
.await?;
}
}
tx.commit().await?;
for workspace_username_tuple in auto_added_workspace_usernames {
let (w_id, username) = workspace_username_tuple;
handle_deployment_metadata(
&email,
&username,
db,
&w_id,
windmill_git_sync::DeployedObject::User { email: email.to_string() },
Some(format!("Auto-added user '{}' to workspace", email)),
rsmq.clone(),
true,
)
.await?;
}
Ok(())
}
async fn invite_user(
ApiAuthed { username, is_admin, .. }: ApiAuthed,
Extension(db): Extension<DB>,
backend/windmill-api/src/workspaces_ee.rs
+16
View File
@@ -0,0 +1,16 @@
use crate::{
db::{ApiAuthed, DB},
workspaces::EditAutoInvite,
};
pub async fn edit_auto_invite(
_authed: ApiAuthed,
_db: DB,
_rsmq: Option<rsmq_async::MultiplexedRsmq>,
_w_id: String,
_ea: EditAutoInvite,
) -> windmill_common::error::Result<String> {
Err(windmill_common::error::Error::InternalErr(
"Not implemented on OSS".to_string(),
))
}
backend/windmill-autoscaling/Cargo.toml
+24
View File
@@ -0,0 +1,24 @@
[package]
name = "windmill-autoscaling"
version.workspace = true
authors.workspace = true
edition.workspace = true
[lib]
name = "windmill_autoscaling"
path = "./src/lib.rs"
[features]
enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"]
default = []
[dependencies]
uuid.workspace = true
serde.workspace = true
sqlx.workspace = true
serde_json.workspace = true
tracing.workspace = true
windmill-common = { workspace = true, default-features = false }
windmill-queue.workspace = true
rsmq_async.workspace = true
anyhow.workspace = true
backend/windmill-autoscaling/src/autoscaling_ee.rs
+6
View File
@@ -0,0 +1,6 @@
use windmill_common::DB;
pub async fn apply_all_autoscaling(_db: &DB) -> anyhow::Result<()> {
// Autoscaling is an ee feature
Ok(())
}
backend/windmill-autoscaling/src/lib.rs
+2
View File
@@ -0,0 +1,2 @@
mod autoscaling_ee;
pub use autoscaling_ee::*;
backend/windmill-common/src/ee.rs
+6 -1
View File
@@ -48,8 +48,13 @@ pub async fn send_critical_alert(
}
#[cfg(feature = "enterprise")]
pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
pub async fn maybe_renew_license_key_on_start(
_http_client: &reqwest::Client,
_db: &crate::db::DB,
force_renew_now: bool,
) -> bool {
// Implementation is not open source
force_renew_now
}
#[cfg(feature = "enterprise")]
backend/windmill-common/src/ee.rs~main
-73
View File
@@ -1,73 +0,0 @@
#[cfg(feature = "enterprise")]
use crate::db::DB;
use crate::ee::LicensePlan::Community;
#[cfg(feature = "enterprise")]
use crate::error;
use serde::Deserialize;
use std::sync::Arc;
use tokio::sync::RwLock;
lazy_static::lazy_static! {
pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
}
pub enum LicensePlan {
Community,
Pro,
Enterprise,
}
pub async fn get_license_plan() -> LicensePlan {
// Implementation is not open source
return Community;
}
#[derive(Deserialize)]
#[serde(untagged)]
pub enum CriticalErrorChannel {}
pub enum CriticalAlertKind {
#[cfg(feature = "enterprise")]
CriticalError,
#[cfg(feature = "enterprise")]
RecoveredCriticalError,
}
#[cfg(feature = "enterprise")]
pub async fn send_critical_alert(
_error_message: String,
_db: &DB,
_kind: CriticalAlertKind,
_channels: Option<Vec<CriticalErrorChannel>>,
) {
}
#[cfg(feature = "enterprise")]
pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
// Implementation is not open source
}
#[cfg(feature = "enterprise")]
pub async fn renew_license_key(
_http_client: &reqwest::Client,
_db: &crate::db::DB,
_key: Option<String>,
_manual: bool,
) -> String {
// Implementation is not open source
"".to_string()
}
#[cfg(feature = "enterprise")]
pub async fn create_customer_portal_session(
_http_client: &reqwest::Client,
_key: Option<String>,
) -> error::Result<String> {
// Implementation is not open source
Ok("".to_string())
}
#[cfg(feature = "enterprise")]
pub async fn worker_groups_alerts(_db: &DB) {}
backend/windmill-common/src/ee.rs~main_0
-76
View File
@@ -1,76 +0,0 @@
#[cfg(feature = "enterprise")]
use crate::db::DB;
use crate::ee::LicensePlan::Community;
#[cfg(feature = "enterprise")]
use crate::error;
use serde::Deserialize;
use std::sync::Arc;
use tokio::sync::RwLock;
lazy_static::lazy_static! {
pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
}
pub enum LicensePlan {
Community,
Pro,
Enterprise,
}
pub async fn get_license_plan() -> LicensePlan {
// Implementation is not open source
return Community;
}
#[derive(Deserialize)]
#[serde(untagged)]
pub enum CriticalErrorChannel {
Email { email: String },
Slack { slack_channel: String },
}
pub enum CriticalAlertKind {
#[cfg(feature = "enterprise")]
CriticalError,
#[cfg(feature = "enterprise")]
RecoveredCriticalError,
}
#[cfg(feature = "enterprise")]
pub async fn send_critical_alert(
_error_message: String,
_db: &DB,
_kind: CriticalAlertKind,
_channels: Option<Vec<CriticalErrorChannel>>,
) {
}
#[cfg(feature = "enterprise")]
pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
// Implementation is not open source
}
#[cfg(feature = "enterprise")]
pub async fn renew_license_key(
_http_client: &reqwest::Client,
_db: &crate::db::DB,
_key: Option<String>,
_manual: bool,
) -> String {
// Implementation is not open source
"".to_string()
}
#[cfg(feature = "enterprise")]
pub async fn create_customer_portal_session(
_http_client: &reqwest::Client,
_key: Option<String>,
) -> error::Result<String> {
// Implementation is not open source
Ok("".to_string())
}
#[cfg(feature = "enterprise")]
pub async fn worker_groups_alerts(_db: &DB) {}
backend/windmill-common/src/email_ee.rs
+11
View File
@@ -0,0 +1,11 @@
use crate::server::Smtp;
pub async fn send_email(
_subject: &str,
_content: &str,
_to: Vec<String>,
_smtp: Smtp,
_client_timeout: Option<tokio::time::Duration>,
) -> crate::error::Result<()> {
Ok(())
}
backend/windmill-common/src/flows.rs
+2
View File
@@ -101,6 +101,7 @@ pub struct FlowValue {
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub failure_module: Option<Box<FlowModule>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub preprocessor_module: Option<Box<FlowModule>>,
#[serde(default)]
@@ -423,6 +424,7 @@ pub enum FlowModuleValue {
path: String,
#[serde(skip_serializing_if = "Option::is_none")]
hash: Option<ScriptHash>,
#[serde(skip_serializing_if = "Option::is_none")]
tag_override: Option<String>,
},
Flow {
backend/windmill-common/src/global_settings.rs
+1
View File
@@ -27,6 +27,7 @@ pub const OBJECT_STORE_CACHE_CONFIG_SETTING: &str = "object_store_cache_config";
pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation";
pub const HUB_BASE_URL_SETTING: &str = "hub_base_url";
pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url";
pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels";
pub const DEV_INSTANCE_SETTING: &str = "dev_instance";
pub const JWT_SECRET_SETTING: &str = "jwt_secret";
backend/windmill-common/src/lib.rs
+3 -2
View File
@@ -17,10 +17,12 @@ use scripts::ScriptLang;
use sqlx::{Pool, Postgres};
pub mod apps;
pub mod auth;
#[cfg(feature = "benchmark")]
pub mod bench;
pub mod db;
pub mod ee;
pub mod email_ee;
pub mod error;
pub mod external_ip;
pub mod flow_status;
@@ -32,9 +34,8 @@ pub mod job_s3_helpers_ee;
pub mod jobs;
pub mod more_serde;
pub mod oauth2;
pub mod queue;
pub mod s3_helpers;
pub mod auth;
pub mod schedule;
pub mod scripts;
pub mod server;
backend/windmill-common/src/queue.rs
+16
View File
@@ -0,0 +1,16 @@
use std::collections::HashMap;
use sqlx::{Pool, Postgres};
pub async fn get_queue_counts(db: &Pool<Postgres>) -> HashMap<String, u32> {
sqlx::query_as::<_, (String, i64)>(
"SELECT tag, count(*) as count FROM queue WHERE
scheduled_for <= now() - ('3 seconds')::interval AND running = false
GROUP BY tag",
)
.fetch_all(db)
.await
.ok()
.map(|v| v.into_iter().map(|(k, v)| (k, v as u32)).collect())
.unwrap_or_else(|| HashMap::new())
}
backend/windmill-common/src/s3_helpers.rs
+24 -24
View File
@@ -117,10 +117,10 @@ pub struct S3Object {
#[cfg(feature = "parquet")]
pub async fn get_etag_or_empty(
object_store_resource: &ObjectStoreResource,
object_store_resource: &mut ObjectStoreResource,
s3_object: S3Object,
) -> Option<String> {
let object_store_client = build_object_store_client(object_store_resource);
let object_store_client = build_object_store_client(object_store_resource).await;
if object_store_client.is_err() {
return None;
}
@@ -166,11 +166,11 @@ pub fn render_endpoint(
}
#[cfg(feature = "parquet")]
pub fn build_object_store_client(
pub async fn build_object_store_client(
resource_ref: &ObjectStoreResource,
) -> error::Result<Arc<dyn ObjectStore>> {
match resource_ref {
ObjectStoreResource::S3(s3_resource_ref) => build_s3_client(&s3_resource_ref, None),
ObjectStoreResource::S3(s3_resource_ref) => build_s3_client(&s3_resource_ref).await,
ObjectStoreResource::Azure(azure_blob_resource_ref) => {
build_azure_blob_client(&azure_blob_resource_ref)
}
@@ -225,10 +225,21 @@ use aws_config::{default_provider::credentials::DefaultCredentialsChain, Region}
use object_store::CredentialProvider;
#[cfg(feature = "parquet")]
pub fn build_s3_client(
s3_resource_ref: &S3Resource,
credential_providers: Option<DefaultCredentialsChain>,
) -> error::Result<Arc<dyn ObjectStore>> {
pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result<Arc<dyn ObjectStore>> {
let static_creds = s3_resource_ref.access_key.as_ref().is_some_and(|x| x != "")
|| s3_resource_ref.secret_key.as_ref().is_some_and(|x| x != "");
let credentials_provider = if !static_creds {
Some(
DefaultCredentialsChain::builder()
.region(Region::new(s3_resource_ref.region.clone()))
.build()
.await,
)
} else {
None
};
let s3_resource = s3_resource_ref.clone();
let endpoint = render_endpoint(
s3_resource.endpoint,
@@ -244,7 +255,7 @@ pub fn build_s3_client(
.with_bucket_name(s3_resource.bucket)
.with_endpoint(endpoint);
if let Some(credentials_provider) = credential_providers {
if let Some(credentials_provider) = credentials_provider {
store_builder = store_builder.with_credentials(Arc::new(AwsCredentialAdapter {
inner: credentials_provider,
}));
@@ -399,18 +410,7 @@ pub async fn build_s3_client_from_settings(
) -> error::Result<Arc<dyn ObjectStore>> {
let region = none_if_empty(settings.region)
.unwrap_or_else(|| std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string()));
let access_key = none_if_empty(settings.access_key);
let secret_key = none_if_empty(settings.secret_key);
let credentials_provider = if access_key.is_none() && secret_key.is_none() {
Some(
DefaultCredentialsChain::builder()
.region(Region::new(region.clone()))
.build()
.await,
)
} else {
None
};
let s3_resource = S3Resource {
endpoint: none_if_empty(settings.endpoint).unwrap_or_else(|| {
std::env::var("S3_ENDPOINT").unwrap_or_else(|_| format!("s3.{region}.amazonaws.com"))
@@ -419,15 +419,15 @@ pub async fn build_s3_client_from_settings(
std::env::var("S3_CACHE_BUCKET").unwrap_or_else(|_| "missingbucket".to_string())
}),
region,
access_key,
secret_key,
access_key: settings.access_key,
secret_key: settings.secret_key,
use_ssl: !settings.allow_http.unwrap_or(true),
path_style: settings.path_style,
port: settings.port,
token: None,
};
build_s3_client(&s3_resource, credentials_provider)
build_s3_client(&s3_resource).await
}
#[cfg(feature = "parquet")]
backend/windmill-common/src/stats_ee.rs
-1
View File
@@ -28,7 +28,6 @@ pub enum SendStatsReason {
pub async fn send_stats(
_http_client: &reqwest::Client,
_db: &DB,
_skip_job_usage: bool,
_reason: SendStatsReason,
) -> Result<()> {
// stats details are closed source
backend/windmill-common/src/utils.rs
+9 -55
View File
@@ -11,13 +11,11 @@ use crate::ee::LICENSE_KEY_ID;
use crate::ee::{send_critical_alert, CriticalAlertKind};
use crate::error::{to_anyhow, Error, Result};
use crate::global_settings::UNIQUE_ID_SETTING;
use crate::server::Smtp;
use crate::DB;
use anyhow::Context;
use gethostname::gethostname;
use git_version::git_version;
use mail_send::mail_builder::MessageBuilder;
use mail_send::SmtpClientBuilder;
use rand::{distributions::Alphanumeric, thread_rng, Rng};
use reqwest::Client;
use serde::{Deserialize, Serialize};
@@ -206,6 +204,14 @@ pub async fn get_uid<'c, E: sqlx::Executor<'c, Database = Postgres>>(db: E) -> R
Ok(uid)
}
pub fn map_string_to_number(s: &str, max_number: u64) -> u64 {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let mut hasher = DefaultHasher::new();
s.hash(&mut hasher);
hasher.finish() % (max_number + 1)
}
#[derive(Serialize, Deserialize, PartialEq, Clone)]
#[serde(rename_all = "lowercase")]
pub enum Mode {
@@ -235,58 +241,6 @@ pub fn generate_lock_id(database_name: &str) -> i64 {
0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64)
}
pub async fn send_email(
subject: &str,
content: &str,
to: Vec<String>,
smtp: Smtp,
client_timeout: Option<tokio::time::Duration>,
) -> Result<()> {
let mut client = SmtpClientBuilder::new(smtp.host, smtp.port)
.implicit_tls(smtp.tls_implicit.unwrap_or(false));
if std::env::var("ACCEPT_INVALID_CERTS").is_ok() {
client = client.allow_invalid_certs();
}
let client = if let (Some(username), Some(password)) = (smtp.username, smtp.password) {
if !username.is_empty() {
client.credentials((username, password))
} else {
client
}
} else {
client
};
let message = MessageBuilder::new()
.from(("Windmill", smtp.from.as_str()))
.to(to.clone())
.subject(subject)
.text_body(content);
match client_timeout {
Some(timeout) => {
tokio::time::timeout(timeout, client.connect())
.await
.map_err(to_anyhow)?
.map_err(to_anyhow)?
.send(message)
.await
.map_err(to_anyhow)?;
}
None => {
client
.connect()
.await
.map_err(to_anyhow)?
.send(message)
.await
.map_err(to_anyhow)?;
}
}
tracing::info!("Sent email to {:#?}: {subject}", to);
return Ok(());
}
pub async fn report_critical_error(error_message: String, _db: DB) -> () {
tracing::error!("CRITICAL ERROR: {error_message}");
backend/windmill-worker/src/ansible_executor.rs
+35 -20
View File
@@ -23,7 +23,7 @@ use windmill_common::{
jobs::QueuedJob,
worker::{to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG},
};
use windmill_parser_yaml::AnsibleRequirements;
use windmill_parser_yaml::{AnsibleRequirements, ResourceOrVariablePath};
use windmill_queue::{append_logs, CanceledBy};
use crate::{
@@ -35,7 +35,7 @@ use crate::{
handle_child::handle_child,
python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile},
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
TZ_ENV,
PROXY_ENVS, TZ_ENV,
};
lazy_static::lazy_static! {
@@ -147,6 +147,7 @@ async fn install_galaxy_collections(
galaxy_command
.current_dir(job_dir)
.env_clear()
.envs(PROXY_ENVS.clone())
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
// .env("BASE_INTERNAL_URL", base_internal_url)
@@ -185,7 +186,7 @@ async fn install_galaxy_collections(
#[cfg(not(feature = "enterprise"))]
fn check_ansible_exists() -> Result<(), error::Error> {
if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() {
let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str());
let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run Ansible jobs.", ANSIBLE_PLAYBOOK_PATH.as_str());
return Err(error::Error::NotFound(msg));
}
Ok(())
@@ -194,7 +195,7 @@ fn check_ansible_exists() -> Result<(), error::Error> {
#[cfg(feature = "enterprise")]
fn check_ansible_exists() -> Result<(), error::Error> {
if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() {
let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str());
let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-ee-full` for your instance in order to run Ansible jobs.", ANSIBLE_PLAYBOOK_PATH.as_str());
return Err(error::Error::NotFound(msg));
}
Ok(())
@@ -395,6 +396,7 @@ fi
nsjail_cmd
.current_dir(job_dir)
.env_clear()
.envs(PROXY_ENVS.clone())
// inject PYTHONPATH here - for some reason I had to do it in nsjail conf
.envs(reserved_variables)
.env("PATH", PATH_ENV.as_str())
@@ -556,22 +558,12 @@ async fn create_file_resources(
}
for file_res in &r.file_resources {
let r = client
.get_resource_value_interpolated::<serde_json::Value>(
&file_res.resource_path,
Some(job_id.to_string()),
)
.await?;
let r =
get_resource_or_variable_content(client, &file_res.resource_path, job_id.to_string())
.await?;
let path = file_res.target_path.clone();
let validated_path = write_file_at_user_defined_location(
job_dir,
path.as_str(),
r.get("content").and_then(|v| v.as_str()).ok_or(anyhow!(
"Invalid text file resource {}, `content` field absent or invalid",
&file_res.resource_path
))?,
)
.map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?;
let validated_path = write_file_at_user_defined_location(job_dir, path.as_str(), &r)
.map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?;
nsjail_mounts.push(
define_nsjail_mount(job_dir, &validated_path)
@@ -579,7 +571,7 @@ async fn create_file_resources(
);
logs.push_str(&format!(
"\nCreated {} from {}",
"\nCreated {} from {:?}",
file_res.target_path, file_res.resource_path
));
}
@@ -587,3 +579,26 @@ async fn create_file_resources(
Ok(nsjail_mounts)
}
async fn get_resource_or_variable_content(
client: &crate::AuthedClient,
path: &ResourceOrVariablePath,
job_id: String,
) -> anyhow::Result<String> {
Ok(match path {
ResourceOrVariablePath::Resource(p) => {
let r = client
.get_resource_value_interpolated::<serde_json::Value>(&p, Some(job_id))
.await?;
r.get("content")
.and_then(|v| v.as_str())
.ok_or(anyhow!(
"Invalid text file resource {}, `content` field absent or invalid",
p
))?
.to_string()
}
ResourceOrVariablePath::Variable(p) => client.get_variable_value(&p).await?,
})
}
backend/windmill-worker/src/bash_executor.rs
+39 -6
View File
@@ -29,7 +29,7 @@ use crate::{
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV,
POWERSHELL_CACHE_DIR, POWERSHELL_PATH, PROXY_ENVS, TZ_ENV,
};
#[cfg(windows)]
@@ -59,11 +59,42 @@ pub async fn handle_bash_job(
append_logs(&job.id, &job.workspace_id, logs1, db).await;
write_file(job_dir, "main.sh", &format!("set -e\n{content}"))?;
write_file(
job_dir,
"wrapper.sh",
&format!("set -o pipefail\nset -e\nmkfifo bp\ncat bp | tail -1 > ./result2.out &\n {bash} ./main.sh \"$@\" 2>&1 | tee bp\nwait $!", bash = BIN_BASH.as_str()),
)?;
let script = format!(
r#"
set -o pipefail
set -e
# Function to kill child processes
cleanup() {{
echo "Terminating child processes..."
# Ignore SIGTERM and SIGINT
trap '' SIGTERM SIGINT
# Kill the process group of the script (negative PID value)
pkill -P $$
exit
}}
# Trap SIGTERM (or other signals) and call cleanup function
trap cleanup SIGTERM SIGINT
# Create a named pipe
mkfifo bp
# Start background processes
cat bp | tail -1 >> ./result2.out &
# Run main.sh in the same process group
{bash} ./main.sh "$@" 2>&1 | tee bp &
# Wait for all background processes to finish
wait
"#,
bash = BIN_BASH.as_str(),
);
write_file(job_dir, "wrapper.sh", &script)?;
let token = client.get_token().await;
let mut reserved_variables = get_reserved_variables(job, &token, db).await?;
@@ -112,6 +143,7 @@ pub async fn handle_bash_job(
.current_dir(job_dir)
.env_clear()
.envs(reserved_variables)
.envs(PROXY_ENVS.clone())
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.args(cmd_args)
@@ -404,6 +436,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
Command::new(NSJAIL_PATH.as_str())
.current_dir(job_dir)
.env_clear()
.envs(PROXY_ENVS.clone())
.envs(reserved_variables)
.env("TZ", TZ_ENV.as_str())
.env("PATH", PATH_ENV.as_str())
backend/windmill-worker/src/bun_executor.rs
+45 -17
View File
@@ -26,7 +26,7 @@ use crate::{
handle_child::handle_child,
AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH,
NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV,
NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV,
};
#[cfg(windows)]
@@ -63,8 +63,26 @@ const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js");
const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.config.proto");
pub const BUN_LOCKB_SPLIT: &str = "\n//bun.lockb\n";
pub const BUN_LOCKB_SPLIT_WINDOWS: &str = "\r\n//bun.lockb\r\n";
pub const EMPTY_FILE: &str = "<empty>";
fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool) {
if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT) {
// Split using "\n//bun.lockb\n"
let (before, after_with_sep) = lockfile.split_at(index);
let after = &after_with_sep[BUN_LOCKB_SPLIT.len()..];
(before, Some(after), after == EMPTY_FILE)
} else if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT_WINDOWS) {
// Split using "\r\n//bun.lockb\r\n"
let (before, after_with_sep) = lockfile.split_at(index);
let after = &after_with_sep[BUN_LOCKB_SPLIT_WINDOWS.len()..];
(before, Some(after), after == EMPTY_FILE)
} else {
(lockfile, None, false)
}
}
pub async fn gen_bun_lockfile(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
@@ -180,7 +198,12 @@ pub async fn gen_bun_lockfile(
file.read_to_string(&mut content).await?;
}
if !npm_mode {
#[cfg(any(target_os = "linux", target_os = "macos"))]
content.push_str(BUN_LOCKB_SPLIT);
#[cfg(target_os = "windows")]
content.push_str(BUN_LOCKB_SPLIT_WINDOWS);
{
let file = format!("{job_dir}/bun.lockb");
if !empty_deps && tokio::fs::metadata(&file).await.is_ok() {
@@ -252,6 +275,7 @@ pub async fn install_bun_lockfile(
child_cmd
.current_dir(job_dir)
.env_clear()
.envs(PROXY_ENVS.clone())
.envs(common_bun_proc_envs)
.args(vec!["install"])
.stdout(Stdio::piped())
@@ -762,12 +786,15 @@ async fn compute_bundle_local_and_remote_path(
}
pub async fn prepare_job_dir(reqs: &str, job_dir: &str) -> Result<()> {
let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::<Vec<&str>>();
let _ = write_file(job_dir, "package.json", &splitted[0])?;
let (pkg, lock, empty) = split_lockfile(reqs);
let _ = write_file(job_dir, "package.json", pkg)?;
if splitted[1] != EMPTY_FILE {
let _ = write_lockb(splitted[1], job_dir).await?;
if !empty {
if let Some(lock) = lock {
let _ = write_lockb(lock, job_dir).await?;
}
}
Ok(())
}
async fn write_lockb(splitted_lockb_2: &str, job_dir: &str) -> Result<()> {
@@ -874,22 +901,23 @@ pub async fn handle_bun_job(
} else if let Some(codebase) = codebase.as_ref() {
pull_codebase(&job.workspace_id, codebase, job_dir).await?;
} else if let Some(reqs) = requirements_o.as_ref() {
let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::<Vec<&str>>();
if splitted.len() != 2 && !annotation.npm {
let (pkg, lock, empty) = split_lockfile(reqs);
if lock.is_none() && !annotation.npm {
return Err(error::Error::ExecutionErr(
format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|")
));
}
let _ = write_file(job_dir, "package.json", &splitted[0])?;
let lockb = if annotation.npm { "" } else { splitted[1] };
if lockb != EMPTY_FILE {
let _ = write_file(job_dir, "package.json", pkg)?;
let lockb = if annotation.npm { "" } else { lock.unwrap() };
if !empty {
let mut skip_install = false;
let mut create_buntar = false;
let mut buntar_path = "".to_string();
if !annotation.npm {
let _ = write_lockb(&splitted[1], job_dir).await?;
let _ = write_lockb(lockb, job_dir).await?;
let mut sha_path = sha2::Sha256::new();
sha_path.update(lockb.as_bytes());
@@ -1561,20 +1589,20 @@ pub async fn start_worker(
if let Some(codebase) = codebase.as_ref() {
pull_codebase(w_id, codebase, job_dir).await?;
} else if let Some(reqs) = requirements_o {
let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::<Vec<&str>>();
if splitted.len() != 2 {
let (pkg, lock, empty) = split_lockfile(&reqs);
if lock.is_none() {
return Err(error::Error::ExecutionErr(
format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|")
));
}
let _ = write_file(job_dir, "package.json", &splitted[0])?;
let lockb = splitted[1];
if lockb != EMPTY_FILE {
let _ = write_file(job_dir, "package.json", pkg)?;
let lockb = lock.unwrap();
if !empty {
let _ = write_file_binary(
job_dir,
"bun.lockb",
&base64::engine::general_purpose::STANDARD
.decode(&splitted[1])
.decode(lockb)
.map_err(|_| {
error::Error::InternalErr("Could not decode bun.lockb".to_string())
})?,
backend/windmill-worker/src/common.rs
+4 -4
View File
@@ -736,8 +736,8 @@ async fn arg_value_hash_additions(
.await;
storage = s3_object.storage.clone();
if let Some(s3_resource) = s3_resource_opt.ok().flatten() {
let etag = get_etag_or_empty(&s3_resource, s3_object.clone()).await;
if let Some(mut s3_resource) = s3_resource_opt.ok().flatten() {
let etag = get_etag_or_empty(&mut s3_resource, s3_object.clone()).await;
tracing::warn!("Enriching s3 arg value with etag: {:?}", etag);
result.insert(s3_object.s3.clone(), etag.unwrap_or_default()); // TODO: maybe inject a random value to invalidate the cache?
}
@@ -793,9 +793,9 @@ pub async fn get_cached_resource_value_if_valid(
return None;
}
for (s3_file_key, s3_file_etag) in s3_etags {
if let Some(object_store_resource) = object_store_resource_opt.clone() {
if let Some(mut object_store_resource) = object_store_resource_opt.clone() {
let etag = get_etag_or_empty(
&object_store_resource,
&mut object_store_resource,
S3Object {
s3: s3_file_key.clone(),
storage: cached_resource.storage.clone(),
backend/windmill-worker/src/dedicated_worker.rs
+2 -1
View File
@@ -77,7 +77,7 @@ pub async fn handle_dedicated_process(
) -> std::result::Result<(), error::Error> {
//do not cache local dependencies
use crate::handle_child::process_status;
use crate::{handle_child::process_status, PROXY_ENVS};
let mut child = {
let mut cmd = Command::new(command_path);
@@ -85,6 +85,7 @@ pub async fn handle_dedicated_process(
.env_clear()
.envs(context_envs)
.envs(envs)
.envs(PROXY_ENVS.clone())
.envs(
reserved_variables
.iter()
backend/windmill-worker/src/go_executor.rs
+2
View File
@@ -1,3 +1,4 @@
use crate::PROXY_ENVS;
use std::{collections::HashMap, fs::DirBuilder, process::Stdio};
use itertools::Itertools;
@@ -189,6 +190,7 @@ func Run(req Req) (interface{{}}, error){{
.env("BASE_INTERNAL_URL", base_internal_url)
.env("GOPATH", GO_CACHE_DIR)
.env("HOME", HOME_ENV.as_str())
.envs(PROXY_ENVS.clone())
.args(vec!["build", "main.go"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
backend/windmill-worker/src/pg_executor.rs
+158 -103
View File
@@ -18,12 +18,8 @@ use serde_json::value::RawValue;
use serde_json::Map;
use serde_json::Value;
use tokio::sync::Mutex;
use tokio_postgres::types::IsNull;
use tokio_postgres::Client;
use tokio_postgres::{
types::{to_sql_checked, ToSql},
NoTls, Row,
};
use tokio_postgres::{types::ToSql, NoTls, Row};
use tokio_postgres::{
types::{FromSql, Type},
Column,
@@ -41,7 +37,7 @@ use windmill_queue::CanceledBy;
use crate::common::{build_args_values, sizeof_val, OccupancyMetrics};
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::{AuthedClientBackgroundTask, MAX_RESULT_SIZE};
use bytes::{Buf, BytesMut};
use bytes::Buf;
use lazy_static::lazy_static;
use urlencoding::encode;
@@ -85,7 +81,7 @@ fn do_postgresql_inner<'a>(
let arg_t = arg
.otyp
.as_ref()
.ok_or_else(|| anyhow::anyhow!("Missing otyp for pg arg"))?;
.ok_or_else(|| anyhow::anyhow!("Missing otzyp for pg arg"))?;
let typ = &arg.typ;
let param = convert_val(value, arg_t, typ)?;
query_params.push(param);
@@ -98,6 +94,11 @@ fn do_postgresql_inner<'a>(
let mut res: Vec<serde_json::Value> = vec![];
let query_params = query_params
.iter()
.map(|p| &**p as &(dyn ToSql + Sync))
.collect_vec();
if skip_collect {
client
.execute_raw(&query, query_params)
@@ -414,133 +415,176 @@ pub async fn do_postgresql(
return Ok(raw_result);
}
#[derive(Debug)]
enum PgType {
String(String),
Bool(bool),
I8(i8),
I16(i16),
I32(i32),
I64(i64),
U32(u32),
F32(f32),
F64(f64),
Uuid(Uuid),
Decimal(Decimal),
Date(chrono::NaiveDate),
Time(chrono::NaiveTime),
Timestamp(chrono::NaiveDateTime),
None(Option<bool>),
Array(Vec<PgType>),
Json(serde_json::Value),
Bytea(Vec<u8>),
fn map_as_single_type<T>(
vec: &Vec<Value>,
f: impl Fn(&Value) -> Option<T>,
) -> anyhow::Result<Vec<Option<T>>> {
vec.into_iter()
.map(|v| {
// allow nulls in arrays
if matches!(v, Value::Null) {
Some(None)
} else {
f(v).map(Some)
}
})
.collect::<Option<Vec<Option<T>>>>()
.ok_or_else(|| anyhow::anyhow!("Mixed types in array"))
}
impl ToSql for PgType {
fn to_sql(
&self,
ty: &Type,
out: &mut BytesMut,
) -> Result<IsNull, Box<dyn std::error::Error + Sync + Send>> {
match *self {
PgType::String(ref val) => val.to_sql(ty, out),
PgType::Bool(ref val) => val.to_sql(ty, out),
PgType::I8(ref val) => val.to_sql(ty, out),
PgType::I16(ref val) => val.to_sql(ty, out),
PgType::I32(ref val) => val.to_sql(ty, out),
PgType::I64(ref val) => val.to_sql(ty, out),
PgType::U32(ref val) => val.to_sql(ty, out),
PgType::F32(ref val) => val.to_sql(ty, out),
PgType::F64(ref val) => val.to_sql(ty, out),
PgType::Uuid(ref val) => val.to_sql(ty, out),
PgType::Decimal(ref val) => val.to_sql(ty, out),
PgType::Date(ref val) => val.to_sql(ty, out),
PgType::Time(ref val) => val.to_sql(ty, out),
PgType::Timestamp(ref val) => val.to_sql(ty, out),
PgType::None(ref val) => val.to_sql(ty, out),
PgType::Array(ref val) => val.to_sql(ty, out),
PgType::Json(ref val) => val.to_sql(ty, out),
PgType::Bytea(ref val) => val.to_sql(ty, out),
fn convert_vec_val(
vec: &Vec<Value>,
arg_t: &String,
) -> windmill_common::error::Result<Box<dyn ToSql + Sync + Send>> {
match arg_t.as_str() {
"bool" | "boolean" => Ok(Box::new(map_as_single_type(vec, |v| v.as_bool())?)),
"char" | "character" => Ok(Box::new(map_as_single_type(vec, |v| {
v.as_i64().map(|x| x as i8)
})?)),
"smallint" | "smallserial" | "int2" | "serial2" => {
Ok(Box::new(map_as_single_type(vec, |v| {
v.as_i64().map(|x| x as i16)
})?))
}
"int" | "integer" | "int4" | "serial" => Ok(Box::new(map_as_single_type(vec, |v| {
v.as_i64().map(|x| x as i32)
})?)),
"numeric" | "decimal" => Ok(Box::new(map_as_single_type(vec, |v| {
if v.is_i64() {
Decimal::from_i64(v.as_i64().unwrap())
} else if v.is_f64() {
Decimal::from_f64(v.as_f64().unwrap())
} else {
None
}
})?)),
"oid" => Ok(Box::new(map_as_single_type(vec, |v| {
v.as_u64().map(|x| x as u32)
})?)),
"bigint" | "bigserial" | "int8" | "serial8" => {
Ok(Box::new(map_as_single_type(vec, |v| {
v.as_u64().map(|x| x as i64)
})?))
}
"real" | "float4" => Ok(Box::new(map_as_single_type(vec, |v| {
v.as_f64().map(|x| x as f32)
})?)),
"double" | "float8" => Ok(Box::new(map_as_single_type(vec, |v| v.as_f64())?)),
"uuid" => Ok(Box::new(map_as_single_type(vec, |v| {
v.as_str().map(|x| Uuid::parse_str(x).ok()).flatten()
})?)),
"date" => Ok(Box::new(map_as_single_type(vec, |v| {
v.as_str().map(|x| {
chrono::NaiveDate::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default()
})
})?)),
"time" | "timetz" => Ok(Box::new(map_as_single_type(vec, |v| {
v.as_str().map(|x| {
chrono::NaiveTime::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default()
})
})?)),
"timestamp" | "timestamptz" => Ok(Box::new(map_as_single_type(vec, |v| {
v.as_str().map(|x| {
chrono::NaiveDateTime::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ")
.unwrap_or_default()
})
})?)),
"jsonb" | "json" => Ok(Box::new(vec.clone().into_iter().map(Some).collect_vec())),
"bytea" => Ok(Box::new(map_as_single_type(vec, |v| {
v.as_str().map(|x| {
engine::general_purpose::STANDARD
.decode(x)
.unwrap_or(vec![])
})
})?)),
"text" | "varchar" => Ok(Box::new(map_as_single_type(vec, |v| {
v.as_str().map(|x| x.to_string())
})?)),
_ => Err(anyhow::anyhow!("Unsupported JSON array type"))?,
}
fn accepts(_: &Type) -> bool {
true
}
to_sql_checked!();
}
fn convert_val(value: &Value, arg_t: &String, typ: &Typ) -> windmill_common::error::Result<PgType> {
fn convert_val(
value: &Value,
arg_t: &String,
typ: &Typ,
) -> windmill_common::error::Result<Box<dyn ToSql + Sync + Send>> {
match value {
Value::Array(vec) if arg_t.ends_with("[]") => {
let arg_t = arg_t.trim_end_matches("[]").to_string();
let mut result = vec![];
for val in vec {
result.push(convert_val(val, &arg_t, typ)?);
}
Ok(PgType::Array(result))
convert_vec_val(vec, &arg_t)
}
Value::Null => Ok(PgType::None(None::<bool>)),
Value::Bool(b) => Ok(PgType::Bool(b.clone())),
Value::Number(n) if matches!(typ, Typ::Str(_)) => Ok(PgType::String(n.to_string())),
Value::Number(n) if n.is_i64() && arg_t == "char" => {
Ok(PgType::I8(n.as_i64().unwrap() as i8))
}
Value::Number(n) if n.is_i64() && (arg_t == "smallint" || arg_t == "smallserial") => {
Ok(PgType::I16(n.as_i64().unwrap() as i16))
Value::Null => Ok(Box::new(None::<bool>)),
Value::Bool(b) => Ok(Box::new(b.clone())),
Value::Number(n) if matches!(typ, Typ::Str(_)) => Ok(Box::new(n.to_string())),
Value::Number(n) if arg_t == "char" && n.is_i64() => {
Ok(Box::new(n.as_i64().unwrap() as i8))
}
Value::Number(n)
if n.is_i64()
&& (arg_t == "int"
|| arg_t == "integer"
|| arg_t == "int4"
|| arg_t == "serial") =>
if (arg_t == "smallint"
|| arg_t == "smallserial"
|| arg_t == "int2"
|| arg_t == "serial2")
&& n.is_i64() =>
{
Ok(PgType::I32(n.as_i64().unwrap() as i32))
Ok(Box::new(n.as_i64().unwrap() as i16))
}
Value::Number(n) if n.is_i64() && (arg_t == "numeric" || arg_t == "decimal") => Ok(
PgType::Decimal(Decimal::from_i64(n.as_i64().unwrap()).unwrap()),
Value::Number(n)
if (arg_t == "int" || arg_t == "integer" || arg_t == "int4" || arg_t == "serial")
&& n.is_i64() =>
{
Ok(Box::new(n.as_i64().unwrap() as i32))
}
Value::Number(n) if (arg_t == "real" || arg_t == "float4") && n.as_f64().is_some() => {
Ok(Box::new(n.as_f64().unwrap() as f32))
}
Value::Number(n) if (arg_t == "double" || arg_t == "float8") && n.as_f64().is_some() => {
Ok(Box::new(n.as_f64().unwrap()))
}
Value::Number(n) if (arg_t == "numeric" || arg_t == "decimal") && n.is_i64() => Ok(
Box::new(Decimal::from_i64(n.as_i64().unwrap()).unwrap_or_default()),
),
Value::Number(n) if n.is_i64() => Ok(PgType::I64(n.as_i64().unwrap())),
Value::Number(n) if n.is_u64() && arg_t == "oid" => {
Ok(PgType::U32(n.as_u64().unwrap() as u32))
}
Value::Number(n) if n.is_u64() && (arg_t == "bigint" || arg_t == "bigserial") => {
Ok(PgType::I64(n.as_u64().unwrap() as i64))
}
Value::Number(n) if n.is_f64() && arg_t == "real" => {
Ok(PgType::F32(n.as_f64().unwrap() as f32))
}
Value::Number(n) if n.is_f64() && arg_t == "double" => Ok(PgType::F64(n.as_f64().unwrap())),
Value::Number(n) if n.is_f64() && (arg_t == "numeric" || arg_t == "decimal") => Ok(
PgType::Decimal(Decimal::from_f64(n.as_f64().unwrap()).unwrap()),
Value::Number(n) if (arg_t == "numeric" || arg_t == "decimal") && n.is_f64() => Ok(
Box::new(Decimal::from_f64(n.as_f64().unwrap()).unwrap_or_default()),
),
Value::Number(n) => Ok(PgType::F64(n.as_f64().unwrap())),
Value::String(s) if arg_t == "uuid" => Ok(PgType::Uuid(Uuid::parse_str(s)?)),
Value::Number(n) if arg_t == "oid" && n.is_u64() => {
Ok(Box::new(n.as_u64().unwrap() as u32))
}
Value::Number(n)
if (arg_t == "bigint"
|| arg_t == "bigserial"
|| arg_t == "int8"
|| arg_t == "serial8")
&& n.is_u64() =>
{
Ok(Box::new(n.as_u64().unwrap() as i64))
}
Value::Number(n) if n.is_i64() => Ok(Box::new(n.as_i64().unwrap())),
Value::Number(n) => Ok(Box::new(n.as_f64().unwrap())),
Value::String(s) if arg_t == "uuid" => Ok(Box::new(Uuid::parse_str(s)?)),
Value::String(s) if arg_t == "date" => {
let date =
chrono::NaiveDate::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default();
Ok(PgType::Date(date))
Ok(Box::new(date))
}
Value::String(s) if arg_t == "time" || arg_t == "timetz" => {
let time =
chrono::NaiveTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default();
Ok(PgType::Time(time))
Ok(Box::new(time))
}
Value::String(s) if arg_t == "timestamp" || arg_t == "timestamptz" => {
let datetime = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ")
.unwrap_or_default();
Ok(PgType::Timestamp(datetime))
Ok(Box::new(datetime))
}
Value::String(s) if arg_t == "bytea" => {
let bytes = engine::general_purpose::STANDARD
.decode(s)
.unwrap_or(vec![]);
Ok(PgType::Bytea(bytes))
Ok(Box::new(bytes))
}
Value::Object(_) => Ok(PgType::Json(value.clone())),
Value::String(s) => Ok(PgType::String(s.clone())),
Value::Object(_) => Ok(Box::new(value.clone())),
Value::String(s) => Ok(Box::new(s.clone())),
_ => Err(Error::ExecutionErr(format!(
"Unsupported type in query: {:?} and signature {arg_t:?}",
value
@@ -624,6 +668,9 @@ pub fn pg_cell_to_json_value(
Type::TS_VECTOR => get_basic(row, column, column_i, |a: StringCollector| {
Ok(JSONValue::String(a.0))
})?,
Type::OID => get_basic(row, column, column_i, |a: u32| {
Ok(JSONValue::Number(serde_json::Number::from(a)))
})?,
// array types
Type::BOOL_ARRAY => get_array(row, column, column_i, |a: bool| Ok(JSONValue::Bool(a)))?,
Type::BIT_ARRAY => get_array(row, column, column_i, |a: bit_vec::BitVec| match a.len() {
@@ -655,6 +702,10 @@ pub fn pg_cell_to_json_value(
Type::FLOAT8_ARRAY => {
get_array(row, column, column_i, |a: f64| Ok(f64_to_json_number(a)?))?
}
Type::NUMERIC_ARRAY => get_array(row, column, column_i, |a: Decimal| {
Ok(serde_json::to_value(a)
.map_err(|_| anyhow::anyhow!("Cannot convert decimal to json"))?)
})?,
// these types require a custom StringCollector struct as an intermediary (see struct at bottom)
Type::TS_VECTOR_ARRAY => get_array(row, column, column_i, |a: StringCollector| {
Ok(JSONValue::String(a.0))
@@ -766,7 +817,7 @@ fn get_array<'a, T: FromSql<'a>>(
val_to_json_val: impl Fn(T) -> Result<JSONValue, Error>,
) -> Result<JSONValue, Error> {
let raw_val_array = row
.try_get::<_, Option<Vec<T>>>(column_i)
.try_get::<_, Option<Vec<Option<T>>>>(column_i)
.with_context(|| {
format!(
"conversion issue for array at column_name `{}`",
@@ -777,7 +828,11 @@ fn get_array<'a, T: FromSql<'a>>(
Some(val_array) => {
let mut result = vec![];
for val in val_array {
result.push(val_to_json_val(val)?);
result.push(
val.map(|v| val_to_json_val(v))
.transpose()?
.unwrap_or(Value::Null),
);
}
JSONValue::Array(result)
}
backend/windmill-worker/src/php_executor.rs
+1 -1
View File
@@ -143,7 +143,7 @@ fn check_php_exists() -> error::Result<()> {
#[cfg(feature = "enterprise")]
fn check_php_exists() -> error::Result<()> {
if !Path::new(PHP_PATH.as_str()).exists() {
let msg = format!("Couldn't find php at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run php jobs.", PHP_PATH.as_str());
let msg = format!("Couldn't find php at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-ee-full` for your instance in order to run php jobs.", PHP_PATH.as_str());
return Err(error::Error::NotFound(msg));
}
Ok(())
backend/windmill-worker/src/python_executor.rs
+10 -21
View File
@@ -65,9 +65,9 @@ use crate::{
read_result, start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY,
LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL,
PIP_INDEX_URL, TZ_ENV, UV_CACHE_DIR,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, LOCK_CACHE_DIR,
NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, TZ_ENV,
UV_CACHE_DIR,
};
#[cfg(windows)]
@@ -565,6 +565,7 @@ mount {{
.env_clear()
// inject PYTHONPATH here - for some reason I had to do it in nsjail conf
.envs(reserved_variables)
.envs(PROXY_ENVS.clone())
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
@@ -1003,15 +1004,6 @@ pub async fn handle_python_reqs(
if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
vars.push(("TRUSTED_HOST", host));
}
if let Some(http_proxy) = HTTP_PROXY.as_ref() {
vars.push(("HTTP_PROXY", http_proxy));
}
if let Some(https_proxy) = HTTPS_PROXY.as_ref() {
vars.push(("HTTPS_PROXY", https_proxy));
}
if let Some(no_proxy) = NO_PROXY.as_ref() {
vars.push(("NO_PROXY", no_proxy));
}
let _ = write_file(
job_dir,
@@ -1026,6 +1018,9 @@ pub async fn handle_python_reqs(
let mut req_with_penv: Vec<(String, String)> = vec![];
for req in requirements {
if req.starts_with('#') {
continue;
}
let venv_p = format!(
"{PIP_CACHE_DIR}/{}",
req.replace(' ', "").replace('/', "").replace(':', "")
@@ -1137,6 +1132,7 @@ pub async fn handle_python_reqs(
.current_dir(job_dir)
.env_clear()
.envs(vars)
.envs(PROXY_ENVS.clone())
.args(vec!["--config", "download.config.proto"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
@@ -1190,15 +1186,6 @@ pub async fn handle_python_reqs(
}
let mut envs = vec![("PATH", PATH_ENV.as_str())];
if let Some(http_proxy) = HTTP_PROXY.as_ref() {
envs.push(("HTTP_PROXY", http_proxy));
}
if let Some(https_proxy) = HTTPS_PROXY.as_ref() {
envs.push(("HTTPS_PROXY", https_proxy));
}
if let Some(no_proxy) = NO_PROXY.as_ref() {
envs.push(("NO_PROXY", no_proxy));
}
envs.push(("HOME", HOME_ENV.as_str()));
@@ -1209,6 +1196,7 @@ pub async fn handle_python_reqs(
let mut flock_cmd = Command::new(FLOCK_PATH.as_str());
flock_cmd
.env_clear()
.envs(PROXY_ENVS.clone())
.envs(envs)
.args([
"-x",
@@ -1227,6 +1215,7 @@ pub async fn handle_python_reqs(
pip_cmd
.env_clear()
.envs(envs)
.envs(PROXY_ENVS.clone())
.env("SystemRoot", SYSTEM_ROOT.as_str())
.args(&command_args[1..])
.stdout(Stdio::piped())
backend/windmill-worker/src/rust_executor.rs
+3 -2
View File
@@ -20,7 +20,7 @@ use crate::{
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
RUST_CACHE_DIR, TZ_ENV,
PROXY_ENVS, RUST_CACHE_DIR, TZ_ENV,
};
#[cfg(windows)]
@@ -192,6 +192,7 @@ pub async fn build_rust_crate(
build_rust_cmd
.current_dir(job_dir)
.env_clear()
.envs(PROXY_ENVS.clone())
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.env("HOME", HOME_ENV.as_str())
@@ -281,7 +282,7 @@ fn check_cargo_exists() -> Result<(), Error> {
#[cfg(feature = "enterprise")]
fn check_cargo_exists() -> Result<(), Error> {
if !Path::new(CARGO_PATH.as_str()).exists() {
let msg = format!("Couldn't find cargo at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run rust jobs.", CARGO_PATH.as_str());
let msg = format!("Couldn't find cargo at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-ee-full` for your instance in order to run rust jobs.", CARGO_PATH.as_str());
return Err(Error::NotFound(msg));
}
Ok(())
backend/windmill-worker/src/worker.rs
+25 -16
View File
@@ -324,6 +324,20 @@ lazy_static::lazy_static! {
pub static ref NO_PROXY: Option<String> = std::env::var("no_proxy").ok().or(std::env::var("NO_PROXY").ok());
pub static ref HTTP_PROXY: Option<String> = std::env::var("http_proxy").ok().or(std::env::var("HTTP_PROXY").ok());
pub static ref HTTPS_PROXY: Option<String> = std::env::var("https_proxy").ok().or(std::env::var("HTTPS_PROXY").ok());
pub static ref PROXY_ENVS: Vec<(&'static str, String)> = {
let mut proxy_env = Vec::new();
if let Some(no_proxy) = NO_PROXY.as_ref() {
proxy_env.push(("NO_PROXY", no_proxy.to_string()));
}
if let Some(http_proxy) = HTTP_PROXY.as_ref() {
proxy_env.push(("HTTP_PROXY", http_proxy.to_string()));
}
if let Some(https_proxy) = HTTPS_PROXY.as_ref() {
proxy_env.push(("HTTPS_PROXY", https_proxy.to_string()));
}
proxy_env
};
pub static ref DENO_PATH: String = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string());
pub static ref BUN_PATH: String = std::env::var("BUN_PATH").unwrap_or_else(|_| "/usr/bin/bun".to_string());
pub static ref NPM_PATH: String = std::env::var("NPM_PATH").unwrap_or_else(|_| "/usr/bin/npm".to_string());
@@ -1273,16 +1287,16 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
}
};
#[cfg(feature = "prometheus")]
if let Some(wb) = worker_busy.as_ref() {
wb.set(1);
tracing::debug!("set worker busy to 1");
}
occupancy_metrics.running_job_started_at = Some(Instant::now());
match next_job {
Ok(Some(job)) => {
#[cfg(feature = "prometheus")]
if let Some(wb) = worker_busy.as_ref() {
wb.set(1);
tracing::debug!("set worker busy to 1");
}
occupancy_metrics.running_job_started_at = Some(Instant::now());
last_executed_job = None;
jobs_executed += 1;
@@ -2067,15 +2081,10 @@ pub fn build_envs(
hm
};
if let Some(ref env) = *HTTPS_PROXY {
envs.insert("HTTPS_PROXY".to_string(), env.to_string());
}
if let Some(ref env) = *HTTP_PROXY {
envs.insert("HTTP_PROXY".to_string(), env.to_string());
}
if let Some(ref env) = *NO_PROXY {
envs.insert("NO_PROXY".to_string(), env.to_string());
for (k, v) in PROXY_ENVS.iter() {
envs.insert(k.to_string(), v.to_string());
}
Ok(envs)
}
backend/windmill-worker/src/worker_flow.rs
+15 -10
View File
@@ -2403,15 +2403,15 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
}
tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushing job {i} of {len}");
let payload_tag = match &job_payloads {
ContinuePayload::SingleJob(payload) => payload.clone(),
ContinuePayload::BranchAllJobs(payloads) => payloads[i].clone(),
ContinuePayload::SingleJob(payload) => payload,
ContinuePayload::BranchAllJobs(payloads) => &payloads[i],
ContinuePayload::ForloopJobs { flow_value, delete_after_use, .. } => {
let mut fv = flow_value.clone();
if let Some(failure_module) = fv.failure_module.as_mut() {
failure_module.id_append(&format!("{}-{i}", &status.step.to_string()));
}
JobPayloadWithTag {
&JobPayloadWithTag {
payload: JobPayload::RawFlow {
value: fv,
path: Some(format!("{}/forloop-{i}", flow_job.script_path())),
@@ -2590,13 +2590,17 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
};
tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed perms for job {i} of {len}");
let tag = if flow_job.tag == "flow" || flow_job.tag == format!("flow-{}", flow_job.workspace_id) {
payload_tag.tag.clone()
} else {
Some(flow_job.tag.clone())
};
let tx2 = PushIsolationLevel::Transaction(tx);
let (uuid, mut inner_tx) = push(
&db,
tx2,
&flow_job.workspace_id,
payload_tag.payload,
payload_tag.payload.clone(),
push_args,
&flow_job.created_by,
&flow_job.email,
@@ -2610,11 +2614,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
continue_on_same_worker,
err,
flow_job.visible_to_owner,
if flow_job.tag == "flow" || flow_job.tag == format!("flow-{}", flow_job.workspace_id) {
payload_tag.tag
} else {
Some(flow_job.tag.clone())
},
tag,
payload_tag.timeout,
Some(module.id.clone()),
new_job_priority_override,
@@ -3761,6 +3761,11 @@ async fn script_to_payload(
module: &FlowModule,
tag_override: &Option<String>,
) -> Result<JobPayloadWithTag, Error> {
let tag_override = if tag_override.as_ref().is_some_and(|x| x.trim().is_empty()) {
None
} else {
tag_override.clone()
};
let (payload, tag, delete_after_use, script_timeout) = if script_hash.is_none() {
let (jp, tag, delete_after_use, script_timeout) =
script_path_to_payload(script_path, db, &flow_job.workspace_id, Some(true)).await?;
backend/windmill-worker/src/worker_lockfiles.rs
+1 -1
View File
@@ -291,7 +291,7 @@ pub async fn handle_dependency_job<R: rsmq_async::RsmqConnection + Send + Sync +
let hash = job.script_hash.unwrap_or(ScriptHash(0));
let w_id = &job.workspace_id;
sqlx::query!(
"UPDATE script SET lock = $1, created_at = now() WHERE hash = $2 AND workspace_id = $3",
"UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3",
&content,
&hash.0,
w_id
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.410.3";
export const VERSION = "v1.416.2";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/instance.ts
+83 -23
View File
@@ -181,7 +181,9 @@ export type InstanceSyncOptions = {
instance?: string;
baseUrl?: string;
token?: string;
folderPerInstance?: boolean;
yes?: boolean;
prefix?: string;
};
export async function pickInstance(
@@ -189,6 +191,21 @@ export async function pickInstance(
allowNew: boolean
) {
const instances = await allInstances();
if (opts.baseUrl && opts.token && opts.instance) {
log.info("Using instance defined by --instance, --base-url and --token");
setClient(
opts.token,
opts.baseUrl.endsWith("/") ? opts.baseUrl.slice(0, -1) : opts.baseUrl
);
return {
name: opts.instance,
remote: opts.baseUrl,
token: opts.token,
prefix: opts.prefix ?? opts.instance,
};
}
if (opts.baseUrl && opts.token) {
log.info("Using instance fully defined by --base-url and --token");
@@ -201,7 +218,7 @@ export async function pickInstance(
name: "custom",
remote: opts.baseUrl,
token: opts.token,
prefix: "custom",
prefix: opts.prefix ?? "custom",
};
}
if (!allowNew && instances.length < 1) {
@@ -297,7 +314,11 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
if (opts.includeWorkspaces) {
log.info("\nPulling all workspaces");
const rootDir = Deno.cwd();
const localWorkspaces = await getLocalWorkspaces(rootDir, instance.prefix);
const localWorkspaces = await getLocalWorkspaces(
rootDir,
instance.prefix,
opts.folderPerInstance
);
const previousActiveWorkspace = await getActiveWorkspace(undefined);
const remoteWorkspaces = await wmill.listWorkspacesAsSuperAdmin({
@@ -306,7 +327,9 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
});
for (const remoteWorkspace of remoteWorkspaces) {
log.info("\nPulling workspace " + remoteWorkspace.id);
const workspaceName = instance.prefix + "_" + remoteWorkspace.id;
const workspaceName = opts?.folderPerInstance
? instance.prefix + "/" + remoteWorkspace.id
: instance.prefix + "_" + remoteWorkspace.id;
await Deno.mkdir(path.join(rootDir, workspaceName), {
recursive: true,
});
@@ -425,16 +448,21 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) {
instances = await allInstances();
const rootDir = Deno.cwd();
const localPrefix = (await Select.prompt({
message: "What is the prefix of the local workspaces you want to sync?",
options: [
...instances.map((i) => ({
name: `${i.prefix} (${i.name} - ${i.remote})`,
value: i.prefix,
})),
],
default: instance.prefix as unknown,
})) as unknown as string;
let localPrefix;
if (opts.prefix) {
localPrefix = opts.prefix;
} else {
localPrefix = (await Select.prompt({
message: "What is the prefix of the local workspaces you want to sync?",
options: [
...instances.map((i) => ({
name: `${i.prefix} (${i.name} - ${i.remote})`,
value: i.prefix,
})),
],
default: instance.prefix as unknown,
})) as unknown as string;
}
const remoteWorkspaces = await wmill.listWorkspacesAsSuperAdmin({
page: 1,
@@ -443,7 +471,11 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) {
const previousActiveWorkspace = await getActiveWorkspace(undefined);
const localWorkspaces = await getLocalWorkspaces(rootDir, localPrefix);
const localWorkspaces = await getLocalWorkspaces(
rootDir,
localPrefix,
opts.folderPerInstance
);
log.info(
`\nPushing all workspaces: ${localWorkspaces.map((x) => x.id).join(", ")}`
@@ -521,17 +553,35 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) {
}
}
async function getLocalWorkspaces(rootDir: string, localPrefix: string) {
async function getLocalWorkspaces(
rootDir: string,
localPrefix: string,
folderPerInstance?: boolean
) {
const localWorkspaces: { dir: string; id: string }[] = [];
for await (const dir of Deno.readDir(rootDir)) {
const dirName = dir.name;
if (dirName.startsWith(localPrefix + "_")) {
if (!(await Deno.stat(localPrefix).catch(() => null))) {
await Deno.mkdir(localPrefix);
}
if (folderPerInstance) {
for await (const dir of Deno.readDir(rootDir + "/" + localPrefix)) {
const dirName = dir.name;
localWorkspaces.push({
dir: dirName,
id: dirName.substring(localPrefix.length + 1),
dir: localPrefix + "/" + dirName,
id: dirName,
});
}
log.info(localWorkspaces);
} else {
for await (const dir of Deno.readDir(rootDir)) {
const dirName = dir.name;
if (dirName.startsWith(localPrefix + "_")) {
localWorkspaces.push({
dir: dirName,
id: dirName.substring(localPrefix.length + 1),
});
}
}
}
return localWorkspaces;
}
@@ -650,6 +700,15 @@ const command = new Command()
.option("--skip-configs", "Skip pulling configs (worker groups and SMTP)")
.option("--skip-groups", "Skip pulling instance groups")
.option("--include-workspaces", "Also pull workspaces")
.option("--folder-per-instance", "Create a folder per instance")
.option(
"--instance <instance:string>",
"Name of the instance to pull from, override the active instance"
)
.option(
"--prefix <prefix:string>",
"Prefix of the local workspaces to pull, used to create the folders when using --include-workspaces"
)
.action(instancePull as any)
.command("push")
@@ -662,13 +721,14 @@ const command = new Command()
.option("--skip-configs", "Skip pushing configs (worker groups and SMTP)")
.option("--skip-groups", "Skip pushing instance groups")
.option("--include-workspaces", "Also push workspaces")
.option("--folder-per-instance", "Create a folder per instance")
.option(
"--instance",
"--instance <instance:string>",
"Name of the instance to push to, override the active instance"
)
.option(
"--base-url",
"If used with --token, will be used as the base url for the instance"
"--prefix <prefix:string>",
"Prefix of the local workspaces folders to push"
)
.action(instancePush as any)
.command("whoami")
cli/local_encryption.ts
+101
View File
@@ -0,0 +1,101 @@
import crypto from "node:crypto";
// Helper function to convert strings to Uint8Array (binary)
function encode(input: string): Uint8Array {
return new TextEncoder().encode(input);
}
// Helper function to convert Uint8Array (binary) to base64
function toBase64(arr: Uint8Array): string {
return btoa(String.fromCharCode(...arr));
}
// Helper function to convert base64 to Uint8Array (binary)
function fromBase64(base64: string): Uint8Array {
return new Uint8Array(
atob(base64)
.split("")
.map((char) => char.charCodeAt(0))
);
}
// Function to derive a 256-bit key from any input string using SHA-256
async function deriveKey(
keyString: string
): Promise<crypto.webcrypto.CryptoKey> {
const keyMaterial = encode(keyString);
const keyHash = await crypto.subtle.digest("SHA-256", keyMaterial); // Generate SHA-256 hash
// Import the hash as a CryptoKey for AES-GCM
return crypto.subtle.importKey("raw", keyHash, { name: "AES-GCM" }, false, [
"encrypt",
"decrypt",
]);
}
// Encrypt function
export async function encrypt(
plaintext: string,
keyString: string
): Promise<string> {
const key = await deriveKey(keyString); // Derive a 256-bit AES key from any input string
const iv = crypto.getRandomValues(new Uint8Array(12)); // AES-GCM needs a 12-byte IV
const encrypted = await crypto.subtle.encrypt(
{
name: "AES-GCM",
iv,
tagLength: 128,
},
key,
encode(plaintext) // convert plaintext to binary
);
// Concatenate IV and encrypted data
const combined = new Uint8Array(iv.length + encrypted.byteLength);
combined.set(iv, 0); // first part is the IV
combined.set(new Uint8Array(encrypted), iv.length); // second part is the ciphertext
// Convert to base64 for storage/transmission
return toBase64(combined);
}
// Decrypt function
export async function decrypt(
combinedCiphertext: string,
keyString: string
): Promise<string> {
const key = await deriveKey(keyString); // Derive the same 256-bit AES key from the input string
const combined = fromBase64(combinedCiphertext); // decode base64 to binary
// Split the IV and the ciphertext
const iv = combined.slice(0, 12); // First 12 bytes are the IV
const ciphertext = combined.slice(12); // The rest is the encrypted data
console.log();
// log.info({keyString, key, ciphertext})
// Perform decryption
const decrypted = await crypto.subtle.decrypt(
{
name: "AES-GCM",
iv,
tagLength: 128,
},
key,
ciphertext
);
// Convert decrypted data from binary to string
return new TextDecoder().decode(decrypted);
}
// // Example usage:
// const key = "any-length-key-you-want"; // Now can be any length
// const message = "This is a secret message.";
// encrypt(message, key).then((combinedCiphertext) => {
// console.log("Encrypted message:", combinedCiphertext);
// // Now decrypt it
// decrypt(combinedCiphertext, key).then((decryptedMessage) => {
// console.log("Decrypted message:", decryptedMessage);
// });
// });
cli/main.ts
+1 -1
View File
@@ -60,7 +60,7 @@ export {
// }
// });
export const VERSION = "1.410.3";
export const VERSION = "1.416.2";
const command = new Command()
.name("wmill")
cli/settings.ts
+63 -18
View File
@@ -9,6 +9,7 @@ import { deepEqual } from "./utils.ts";
import * as wmill from "./gen/services.gen.ts";
import { Config, GlobalSetting } from "./gen/types.gen.ts";
import { removeWorkerPrefix } from "./worker_groups.ts";
import process from "node:process";
export interface SimplifiedSettings {
// slack_team_id?: string;
@@ -112,10 +113,10 @@ export async function pushWorkspaceSettings(
workspace,
requestBody: localSettings.auto_invite_enabled
? {
operator: localSettings.auto_invite_as === "operator",
invite_all: true,
auto_add: localSettings.auto_invite_mode === "add",
}
operator: localSettings.auto_invite_as === "operator",
invite_all: true,
auto_add: localSettings.auto_invite_mode === "add",
}
: {},
});
} catch (_) {
@@ -127,10 +128,10 @@ export async function pushWorkspaceSettings(
workspace,
requestBody: localSettings.auto_invite_enabled
? {
operator: localSettings.auto_invite_as === "operator",
invite_all: false,
auto_add: localSettings.auto_invite_mode === "add",
}
operator: localSettings.auto_invite_as === "operator",
invite_all: false,
auto_add: localSettings.auto_invite_mode === "add",
}
: {},
});
}
@@ -155,7 +156,7 @@ export async function pushWorkspaceSettings(
settings.error_handler_extra_args
) ||
localSettings.error_handler_muted_on_cancel !==
settings.error_handler_muted_on_cancel
settings.error_handler_muted_on_cancel
) {
log.debug(`Updating error handler...`);
await wmill.editErrorHandler({
@@ -259,27 +260,69 @@ export async function pushWorkspaceKey(
}
}
const INSTANCE_SETTINGS_PATH = "instance_settings.yaml";
export async function readInstanceSettings() {
let localSettings: GlobalSetting[] = [];
try {
localSettings = (await yamlParseFile(
"instance_settings.yaml"
)) as GlobalSetting[];
localSettings = (await yamlParseFile(INSTANCE_SETTINGS_PATH)) as GlobalSetting[];
} catch {
log.warn("No instance_settings.yaml found");
log.warn(`No ${INSTANCE_SETTINGS_PATH} found`);
}
return localSettings;
}
import { decrypt, encrypt } from "./local_encryption.ts";
const SENSITIVE_FIELD: string[] = ["license_key", "jwt_secret"]
async function processInstanceSettings(settings: GlobalSetting[], mode: "encode" | "decode"): Promise<GlobalSetting[]> {
const encKey = process.env.WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY;
if (encKey) {
const res: GlobalSetting[] = []
for (const s of settings) {
if (SENSITIVE_FIELD.includes(s.name) && typeof s.value === "string") {
res.push(await processField(s, "value", encKey, mode) as GlobalSetting);
} else if (s.name == "oauths") {
if (typeof s.value === "object") {
const oauths = s.value as { [key: string]: any };
for (const [k, v] of Object.entries(oauths)) {
oauths[k] = await processField(v, "secret", encKey, mode);
}
res.push(s);
} else {
log.warn(`Unexpected oauths value type: ${typeof s.value}`);
res.push(s);
}
} else {
res.push(s);
}
}
return res;
} else {
log.warn("No encryption key found, skipping encryption. Recommend setting WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY");
}
return settings;
}
async function processField(obj: { [key: string]: any }, field: string, encKey: string, mode: "encode" | "decode"): Promise<{ [key: string]: any }> {
return {
...obj,
[field]: mode === "encode" ? await encrypt(obj[field], encKey) : await decrypt(obj[field], encKey) as any,
}
}
export async function pullInstanceSettings(preview = false) {
const remoteSettings = await wmill.listGlobalSettings();
if (preview) {
const localSettings: GlobalSetting[] = await readInstanceSettings();
const processedSettings = await processInstanceSettings(remoteSettings, "encode");
return compareInstanceObjects(
remoteSettings,
processedSettings,
localSettings,
"name",
"setting"
@@ -287,12 +330,13 @@ export async function pullInstanceSettings(preview = false) {
} else {
log.info("Pulling settings from instance");
const processedSettings = await processInstanceSettings(remoteSettings, "encode");
await Deno.writeTextFile(
"instance_settings.yaml",
yamlStringify(remoteSettings as any)
INSTANCE_SETTINGS_PATH,
yamlStringify(processedSettings)
);
log.info(colors.green("Settings written to instance_settings.yaml"));
log.info(colors.green(`Settings written to ${INSTANCE_SETTINGS_PATH}`));
}
}
@@ -302,6 +346,7 @@ export async function pushInstanceSettings(
) {
const remoteSettings = await wmill.listGlobalSettings();
let localSettings: GlobalSetting[] = await readInstanceSettings();
localSettings = await processInstanceSettings(localSettings, "decode");
if (baseUrl) {
localSettings = localSettings.filter((s) => s.name !== "base_url");
cli/sync.ts
+2 -2
View File
@@ -618,8 +618,8 @@ export async function elementsToMap(
for await (const entry of readDirRecursiveWithIgnore(ignore, els)) {
if (entry.isDirectory || entry.ignored) continue;
const path = entry.path;
if (json && path.endsWith(".yaml")) continue;
if (!json && path.endsWith(".json")) continue;
if (json && path.endsWith(".yaml") && !isFileResource(path)) continue;
if (!json && path.endsWith(".json") && !isFileResource(path)) continue;
const ext = json ? ".json" : ".yaml";
if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue;
if (!skips.includeUsers && path.endsWith(".user" + ext)) continue;
docker/DockerfileSlim
+2 -2
View File
@@ -11,7 +11,7 @@ RUN apt-get update \
&& rm -rf /var/lib/apt/lists/*
RUN curl -sL https://deb.nodesource.com/setup_20.x | bash -
RUN apt-get -y update && apt-get install -y curl nodejs awscli
RUN apt-get -y update && apt-get install -y curl procps nodejs awscli
ENV TZ=Etc/UTC
@@ -20,7 +20,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
# Install UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv
COPY --from=oven/bun:1.1.27 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.32 /usr/local/bin/bun /usr/bin/bun
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
docker/DockerfileSlimEe
+2 -2
View File
@@ -11,7 +11,7 @@ RUN apt-get update \
&& rm -rf /var/lib/apt/lists/*
RUN curl -sL https://deb.nodesource.com/setup_20.x | bash -
RUN apt-get -y update && apt-get install -y curl nodejs awscli
RUN apt-get -y update && apt-get install -y curl procps nodejs awscli
ENV TZ=Etc/UTC
@@ -19,7 +19,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
# Install UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv
COPY --from=oven/bun:1.1.27 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.32 /usr/local/bin/bun /usr/bin/bun
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
frontend/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-components",
"version": "1.410.3",
"version": "1.416.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.410.3",
"version": "1.416.2",
"license": "AGPL-3.0",
"dependencies": {
"@aws-crypto/sha256-js": "^4.0.0",
frontend/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.410.3",
"version": "1.416.2",
"scripts": {
"dev": "vite dev",
"build": "vite build",
frontend/src/lib/components/AppConnectDrawer.svelte
+4 -4
View File
@@ -6,7 +6,7 @@
import AppConnectInner from './AppConnectInner.svelte'
import DarkModeObserver from './DarkModeObserver.svelte'
let expressOAuthSetup = false
export let expressOAuthSetup = false
let drawer: Drawer
let resourceType = ''
@@ -18,8 +18,7 @@
let appConnectInner: AppConnectInner | undefined = undefined
let rtToLoad: string | undefined = ''
export async function open(rt?: string, express?: boolean) {
expressOAuthSetup = express ?? false
export async function open(rt?: string) {
rtToLoad = rt
drawer.openDrawer?.()
}
@@ -27,7 +26,7 @@
$: appConnectInner && onRtToLoadChange(rtToLoad)
function onRtToLoadChange(rtToLoad: string | undefined) {
appConnectInner?.open(rtToLoad, expressOAuthSetup)
appConnectInner?.open(rtToLoad)
}
const dispatch = createEventDispatcher()
@@ -60,6 +59,7 @@
bind:manual
on:close={drawer?.closeDrawer}
on:refresh
express={expressOAuthSetup}
/>
<div slot="actions" class="flex gap-1">
{#if step > 1}
frontend/src/lib/components/AppConnectInner.svelte
+247 -248
View File
@@ -31,6 +31,7 @@
export let isGoogleSignin = false
export let disabled = false
export let manual = true
export let express = false
let isValid = true
@@ -81,10 +82,7 @@
let pathError = ''
let expressOAuthSetup = false
export async function open(rt?: string, express?: boolean) {
expressOAuthSetup = express ?? false
export async function open(rt?: string) {
if (!rt) {
loadResourceTypes()
}
@@ -95,12 +93,12 @@
valueToken = undefined
await loadConnects()
manual = !connects?.includes(resourceType)
if (manual && expressOAuthSetup) {
if (manual && express) {
dispatch('error', 'Express OAuth setup is not available for non OAuth resource types')
return
}
if (rt) {
if (!manual && expressOAuthSetup) {
if (!manual && express) {
await getScopesAndParams()
step = 2
}
@@ -171,7 +169,6 @@
}
function popupListener(event) {
console.log('popupListener', event.data, event.origin, window.location.origin)
let data = event.data
if (event.origin == null || event.origin !== window.location.origin) {
return
@@ -187,7 +184,7 @@
value = data.res.access_token!
valueToken = data.res
step = 4
if (expressOAuthSetup) {
if (express) {
path = `u/${$userStore?.username}/${resourceType}_${new Date().getTime()}`
next()
}
@@ -329,252 +326,254 @@
let editScopes = false
</script>
<SearchItems
{filter}
items={connects
? connects
.sort((a, b) => a.localeCompare(b))
.map((key) => ({
key
}))
: undefined}
bind:filteredItems={filteredConnects}
f={(x) => x.key}
/>
<SearchItems
{filter}
items={connectsManual?.sort((a, b) => a[0].localeCompare(b[0]))}
bind:filteredItems={filteredConnectsManual}
f={(x) => x[0]}
/>
{#if step == 1}
<div class="w-12/12 pb-2 flex flex-row my-1 gap-1">
<input
type="text"
placeholder="Search resource type"
bind:value={filter}
class="text-2xl grow"
/>
</div>
<h2 class="mb-4">OAuth APIs</h2>
<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
{#if filteredConnects}
{#each filteredConnects as { key }}
<Button
size="sm"
variant="border"
color={key === resourceType ? 'blue' : 'light'}
btnClasses={key === resourceType ? '!border-2' : 'm-[1px]'}
on:click={() => {
manual = false
resourceType = key
next()
}}
>
<IconedResourceType name={key} after={true} width="20px" height="20px" />
</Button>
{/each}
{:else}
{#each new Array(3) as _}
<Skeleton layout={[[2]]} />
{/each}
{/if}
</div>
{#if connects && connects.length == 0}
<div class="text-secondary text-sm w-full"
>No OAuth APIs has been setup on the instance. To add oauth APIs, first sync the resource
types with the hub, then add oauth configuration. See <a
href="https://www.windmill.dev/docs/misc/setup_oauth">documentation</a
>
</div>
{/if}
<h2 class="mt-8 mb-4">Others</h2>
{#if connectsManual && connectsManual?.length < 10}
<div class="text-secondary p-2">
Resource Types have not been synced with the hub. Go to the admins workspace to sync them (and
add a schedule to do daily):
<p class="mt-4"
>1. Go to the "admins" workspaces:
<img src="{base}/sync_resource_types.png" alt="sync resource types" class="mt-2" />
</p>
<p class="mt-4">
2: Run the synchronization script:
<img src="{base}/sync_resource_types2.png" alt="sync resource types" class="mt-2" />
</p>
</div>
{/if}
<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
{#if filteredConnectsManual}
{#each filteredConnectsManual as [key, _]}
{#if nativeLanguagesCategory.includes(key)}
<Button
size="sm"
variant="border"
color={key === resourceType ? 'blue' : 'light'}
btnClasses={key === resourceType ? '!border-2' : 'm-[1px]'}
on:click={() => {
manual = true
resourceType = key
next()
}}
>
<IconedResourceType name={key} after={true} width="20px" height="20px" />
</Button>
{/if}
{/each}
{/if}
</div>
<div class="mt-8 mb-4" />
<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
{#if filteredConnectsManual}
{#each filteredConnectsManual as [key, _]}
{#if !nativeLanguagesCategory.includes(key)}
<!-- Exclude specific items -->
<Button
size="sm"
variant="border"
color={key === resourceType ? 'blue' : 'light'}
btnClasses={key === resourceType ? '!border-2' : 'm-[1px]'}
on:click={() => {
manual = true
resourceType = key
next()
}}
>
<IconedResourceType name={key} after={true} width="20px" height="20px" />
</Button>
{/if}
{/each}
{:else}
{#each new Array(9) as _}
<Skeleton layout={[[2]]} />
{/each}
{/if}
</div>
{:else if step == 2 && manual}
<Path
bind:error={pathError}
bind:path
initialPath=""
namePlaceholder={resourceType}
kind="resource"
{#if !express}
<SearchItems
{filter}
items={connects
? connects
.sort((a, b) => a.localeCompare(b))
.map((key) => ({
key
}))
: undefined}
bind:filteredItems={filteredConnects}
f={(x) => x.key}
/>
{#if apiTokenApps[resourceType]}
<h2 class="mt-4 mb-2">Instructions</h2>
<div class="pl-10">
<ol class="list-decimal">
{#each apiTokenApps[resourceType].instructions as step}
<li>
{@html step}
</li>
{/each}
</ol>
</div>
{#if apiTokenApps[resourceType].img}
<div class="mt-4 w-full overflow-hidden">
<img class="m-auto max-h-60" alt="connect" src={base + apiTokenApps[resourceType].img} />
</div>
{/if}
{:else if !emptyString(resourceTypeInfo?.description)}
<h4 class="mt-8 mb-2">{resourceTypeInfo?.name} description</h4>
<div class="text-sm">
<Markdown md={urlize(resourceTypeInfo?.description ?? '', 'md')} />
</div>
{/if}
{#if resourceType == 'postgresql' || resourceType == 'mysql' || resourceType == 'mongodb'}
<WhitelistIp />
{/if}
<h4 class="mt-8 inline-flex items-center gap-4"
>Resource description <Required required={false} />
<div class="flex gap-1 items-center">
<Toggle size="xs" bind:checked={renderDescription} />
<Pen size={14} />
</div>
</h4>
{#if renderDescription}
<div>
<div class="flex flex-row-reverse text-2xs text-tertiary -mt-1">GH Markdown</div>
<textarea use:autosize bind:value={description} placeholder={'Resource description'} />
</div>
{:else if description == undefined || description == ''}
<div class="text-sm text-tertiary">No description provided</div>
{:else}
<div class="mt-2" />
<GfmMarkdown md={description} />
{/if}
<div class="mt-12">
{#key resourceTypeInfo}
<ApiConnectForm
bind:linkedSecret
{linkedSecretCandidates}
{resourceType}
{resourceTypeInfo}
bind:args
bind:isValid
<SearchItems
{filter}
items={connectsManual?.sort((a, b) => a[0].localeCompare(b[0]))}
bind:filteredItems={filteredConnectsManual}
f={(x) => x[0]}
/>
{#if step == 1}
<div class="w-12/12 pb-2 flex flex-row my-1 gap-1">
<input
type="text"
placeholder="Search resource type"
bind:value={filter}
class="text-2xl grow"
/>
{/key}
</div>
{:else if step == 2 && !manual}
{#if manual == false && resourceType != ''}
<h1 class="mb-4">{resourceType}</h1>
<div class="my-4 text-secondary"
>Click connect to create a resource backed by an oauth connection, whose token is fetched from
the external services and refreshed automatically if needed before expiration (using its
refresh token)</div
>
<h4 class="mb-2">Description</h4>
<div class="text-sm mb-8">
<Markdown md={urlize(resourceTypeInfo?.description ?? '', 'md')} />
</div>
<h3 class="mb-4 flex gap-4"
>Scopes <button
on:click={() => {
editScopes = !editScopes
}}><Pen size={14} /></button
></h3
>
{#if editScopes}
<OauthScopes bind:scopes />
{:else}
<div class="flex flex-col gap-1">
{#each scopes as scope}
<div class="py-0.5 pl-2 text-xs">- {scope}</div>
<h2 class="mb-4">OAuth APIs</h2>
<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
{#if filteredConnects}
{#each filteredConnects as { key }}
<Button
size="sm"
variant="border"
color={key === resourceType ? 'blue' : 'light'}
btnClasses={key === resourceType ? '!border-2' : 'm-[1px]'}
on:click={() => {
manual = false
resourceType = key
next()
}}
>
<IconedResourceType name={key} after={true} width="20px" height="20px" />
</Button>
{/each}
{:else}
{#each new Array(3) as _}
<Skeleton layout={[[2]]} />
{/each}
{/if}
</div>
{#if connects && connects.length == 0}
<div class="text-secondary text-sm w-full"
>No OAuth APIs has been setup on the instance. To add oauth APIs, first sync the resource
types with the hub, then add oauth configuration. See <a
href="https://www.windmill.dev/docs/misc/setup_oauth">documentation</a
>
</div>
{/if}
{/if}
{:else if step == 3 && !manual}
Finish connection in popup window
{:else}
<Path
initialPath=""
namePlaceholder={resourceType}
bind:error={pathError}
bind:path
kind="resource"
/>
{#if apiTokenApps[resourceType] || !manual}
<ul class="mt-10">
<li>
1. A secret variable containing the {apiTokenApps[resourceType]?.linkedSecret ?? 'token'}
<span class="font-bold">{truncateRev(value, 5, '*****')}</span>
will be stored a
<span class="font-mono whitespace-nowrap">{path}</span>.
</li>
<li class="mt-4">
2. The resource containing that token will be stored at the same path <span
class="font-mono whitespace-nowrap">{path}</span
>. The Variable and Resource will be "linked together", they will be deleted and renamed
together.
</li></ul
>
<h2 class="mt-8 mb-4">Others</h2>
{#if connectsManual && connectsManual?.length < 10}
<div class="text-secondary p-2">
Resource Types have not been synced with the hub. Go to the admins workspace to sync them
(and add a schedule to do daily):
<p class="mt-4"
>1. Go to the "admins" workspaces:
<img src="{base}/sync_resource_types.png" alt="sync resource types" class="mt-2" />
</p>
<p class="mt-4">
2: Run the synchronization script:
<img src="{base}/sync_resource_types2.png" alt="sync resource types" class="mt-2" />
</p>
</div>
{/if}
<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
{#if filteredConnectsManual}
{#each filteredConnectsManual as [key, _]}
{#if nativeLanguagesCategory.includes(key)}
<Button
size="sm"
variant="border"
color={key === resourceType ? 'blue' : 'light'}
btnClasses={key === resourceType ? '!border-2' : 'm-[1px]'}
on:click={() => {
manual = true
resourceType = key
next()
}}
>
<IconedResourceType name={key} after={true} width="20px" height="20px" />
</Button>
{/if}
{/each}
{/if}
</div>
<div class="mt-8 mb-4" />
<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
{#if filteredConnectsManual}
{#each filteredConnectsManual as [key, _]}
{#if !nativeLanguagesCategory.includes(key)}
<!-- Exclude specific items -->
<Button
size="sm"
variant="border"
color={key === resourceType ? 'blue' : 'light'}
btnClasses={key === resourceType ? '!border-2' : 'm-[1px]'}
on:click={() => {
manual = true
resourceType = key
next()
}}
>
<IconedResourceType name={key} after={true} width="20px" height="20px" />
</Button>
{/if}
{/each}
{:else}
{#each new Array(9) as _}
<Skeleton layout={[[2]]} />
{/each}
{/if}
</div>
{:else if step == 2 && manual}
<Path
bind:error={pathError}
bind:path
initialPath=""
namePlaceholder={resourceType}
kind="resource"
/>
{#if apiTokenApps[resourceType]}
<h2 class="mt-4 mb-2">Instructions</h2>
<div class="pl-10">
<ol class="list-decimal">
{#each apiTokenApps[resourceType].instructions as step}
<li>
{@html step}
</li>
{/each}
</ol>
</div>
{#if apiTokenApps[resourceType].img}
<div class="mt-4 w-full overflow-hidden">
<img class="m-auto max-h-60" alt="connect" src={base + apiTokenApps[resourceType].img} />
</div>
{/if}
{:else if !emptyString(resourceTypeInfo?.description)}
<h4 class="mt-8 mb-2">{resourceTypeInfo?.name} description</h4>
<div class="text-sm">
<Markdown md={urlize(resourceTypeInfo?.description ?? '', 'md')} />
</div>
{/if}
{#if resourceType == 'postgresql' || resourceType == 'mysql' || resourceType == 'mongodb'}
<WhitelistIp />
{/if}
<h4 class="mt-8 inline-flex items-center gap-4"
>Resource description <Required required={false} />
<div class="flex gap-1 items-center">
<Toggle size="xs" bind:checked={renderDescription} />
<Pen size={14} />
</div>
</h4>
{#if renderDescription}
<div>
<div class="flex flex-row-reverse text-2xs text-tertiary -mt-1">GH Markdown</div>
<textarea use:autosize bind:value={description} placeholder={'Resource description'} />
</div>
{:else if description == undefined || description == ''}
<div class="text-sm text-tertiary">No description provided</div>
{:else}
<div class="mt-2" />
<GfmMarkdown md={description} />
{/if}
<div class="mt-12">
{#key resourceTypeInfo}
<ApiConnectForm
bind:linkedSecret
{linkedSecretCandidates}
{resourceType}
{resourceTypeInfo}
bind:args
bind:isValid
/>
{/key}
</div>
{:else if step == 2 && !manual}
{#if manual == false && resourceType != ''}
<h1 class="mb-4">{resourceType}</h1>
<div class="my-4 text-secondary"
>Click connect to create a resource backed by an oauth connection, whose token is fetched
from the external services and refreshed automatically if needed before expiration (using
its refresh token)</div
>
<h4 class="mb-2">Description</h4>
<div class="text-sm mb-8">
<Markdown md={urlize(resourceTypeInfo?.description ?? '', 'md')} />
</div>
<h3 class="mb-4 flex gap-4"
>Scopes <button
on:click={() => {
editScopes = !editScopes
}}><Pen size={14} /></button
></h3
>
{#if editScopes}
<OauthScopes bind:scopes />
{:else}
<div class="flex flex-col gap-1">
{#each scopes as scope}
<div class="py-0.5 pl-2 text-xs">- {scope}</div>
{/each}
</div>
{/if}
{/if}
{:else if step == 3 && !manual && !express}
Finish connection in popup window
{:else}
<Path
initialPath=""
namePlaceholder={resourceType}
bind:error={pathError}
bind:path
kind="resource"
/>
{#if apiTokenApps[resourceType] || !manual}
<ul class="mt-10">
<li>
1. A secret variable containing the {apiTokenApps[resourceType]?.linkedSecret ?? 'token'}
<span class="font-bold">{truncateRev(value, 5, '*****')}</span>
will be stored a
<span class="font-mono whitespace-nowrap">{path}</span>.
</li>
<li class="mt-4">
2. The resource containing that token will be stored at the same path <span
class="font-mono whitespace-nowrap">{path}</span
>. The Variable and Resource will be "linked together", they will be deleted and renamed
together.
</li></ul
>
{/if}
{/if}
{/if}
frontend/src/lib/components/AppConnectLightweightResourcePicker.svelte
+19 -16
View File
@@ -23,7 +23,7 @@
onMount(async () => {
if (resourceType) {
appConnect?.open(resourceType, express)
appConnect?.open(resourceType)
}
})
</script>
@@ -31,24 +31,27 @@
<DarkModeObserver bind:darkMode />
<div>
<div class="flex flex-row-reverse w-full">
<div class="flex gap-2">
{#if step > 2}
<Button variant="border" on:click={appConnect?.back}>Back</Button>
{/if}
<Button {disabled} on:click={appConnect?.next}>
{#if step == 2 && !manual}
Connect
{:else if step == 1}
Next
{:else}
Save
{#if !express}
<div class="flex flex-row-reverse w-full pb-2">
<div class="flex gap-2">
{#if step > 2}
<Button variant="border" on:click={appConnect?.back}>Back</Button>
{/if}
</Button>
<Button {disabled} on:click={appConnect?.next}>
{#if step == 2 && !manual}
Connect
{:else if step == 1}
Next
{:else}
Save
{/if}
</Button>
</div>
</div>
</div>
{/if}
<AppConnectInner
{express}
bind:this={appConnect}
bind:step
bind:resourceType
frontend/src/lib/components/ArgInfo.svelte
+12 -10
View File
@@ -75,16 +75,18 @@
}}>{value}</button
>
{:else if typeof value !== 'object'}
{truncate(JSON.stringify(value), 100)}
{#if JSON.stringify(value).length > 100}
<button
class="text-xs text-blue-500"
on:click={() => {
jsonViewerContent = value
jsonViewer.toggleDrawer()
}}>See expanded</button
>
{/if}
<span>
{truncate(JSON.stringify(value), 80)}
{#if JSON.stringify(value).length > 80}
<button
class="text-xs text-blue-500"
on:click={() => {
jsonViewerContent = value
jsonViewer.toggleDrawer()
}}>See expanded</button
>
{/if}
</span>
{:else}
<div class="relative">
{#if JSON.stringify(value).length > 120}
frontend/src/lib/components/AutoscalingConfigEditor.svelte
+364
View File
@@ -0,0 +1,364 @@
<script lang="ts">
import type { AutoscalingConfig } from './worker_group'
import Toggle from './Toggle.svelte'
import Section from './Section.svelte'
import Tooltip from './Tooltip.svelte'
import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte'
import { Button } from './common'
import { ExternalLink } from 'lucide-svelte'
import { createEventDispatcher } from 'svelte'
import Label from './Label.svelte'
import MultiSelect from 'svelte-multiselect'
export let config: AutoscalingConfig | undefined
export let worker_tags: string[] | undefined
const dispatch = createEventDispatcher()
let test_input: number = 3
</script>
<div class="flex flex-row gap-16 pt-2">
<div class="space-y-4 flex flex-col gap-1 max-w-xs text-sm">
<h5>Rules</h5>
<Toggle
checked={config?.enabled ?? false}
options={{ right: 'Enabled' }}
on:change={(e) => {
dispatch('dirty')
if (e.detail) {
if (!config) {
config = {
enabled: true,
min_workers: 3,
max_workers: 10,
integration: { type: 'dryrun' }
}
} else {
config.enabled = true
}
} else {
config = {
...(config ?? {
min_workers: 3,
max_workers: 10,
integration: { type: 'dryrun' }
}),
enabled: false
}
}
}}
/>
<!-- svelte-ignore a11y-label-has-associated-control -->
<label>
Min # of Workers
{#if config !== undefined}
<input on:input={() => dispatch('dirty')} type="number" bind:value={config.min_workers} />
{#if config.min_workers !== undefined && config.min_workers != undefined && config.min_workers > config.max_workers}
<div class="text-red-600 text-xs whitespace-nowrap"
>Minimum cannot be {'>'} to Maximum</div
>
{/if}
{:else}
<input type="number" disabled />
{/if}
</label>
<!-- svelte-ignore a11y-label-has-associated-control -->
<label>
Max # of Workers
{#if config !== undefined}
<input on:input={() => dispatch('dirty')} type="number" bind:value={config.max_workers} />
{:else}
<input type="number" disabled />
{/if}
</label>
<div class="p-2">
<Section label="Advanced" small collapsable={true}>
<div class="flex flex-col gap-2 text-2xs">
<!-- svelte-ignore a11y-label-has-associated-control -->
<label>
Cooldown seconds after an incremental scale-in/out
{#if config !== undefined}
<input
on:input={() => dispatch('dirty')}
type="number"
step="1"
min="30"
placeholder="300"
bind:value={config.cooldown_seconds}
/>
{:else}
<input type="number" disabled />
{/if}
</label>
<!-- svelte-ignore a11y-label-has-associated-control -->
<label>
Cooldown seconds after a full scale out
{#if config !== undefined}
<input
on:input={() => dispatch('dirty')}
type="number"
step="1"
min="30"
placeholder="1500"
bind:value={config.full_scale_cooldown_seconds}
/>
{:else}
<input type="number" disabled />
{/if}
</label>
<!-- svelte-ignore a11y-label-has-associated-control -->
<label>
Num jobs waiting to trigger an incremental scale-out
{#if config !== undefined}
<input
on:input={() => dispatch('dirty')}
type="number"
bind:value={config.inc_scale_num_jobs_waiting}
placeholder="1"
/>
{:else}
<input type="number" disabled />
{/if}
</label>
<!-- svelte-ignore a11y-label-has-associated-control -->
<label>
Num jobs waiting to trigger a full scale out <Tooltip
>Default: max_workers, full scale out = scale out to max workers</Tooltip
>
{#if config !== undefined}
<input
on:input={() => dispatch('dirty')}
type="number"
placeholder="max workers"
bind:value={config.full_scale_jobs_waiting}
/>
{:else}
<input type="number" disabled />
{/if}
</label>
<!-- svelte-ignore a11y-label-has-associated-control -->
<label>
Occupancy rate % threhsold to go below to trigger a scale-in (decrease) <Tooltip
>Default: 25%, need to go below average of all of 15s, 5m and 30m occupancy rates</Tooltip
>
{#if config !== undefined}
<input
on:input={() => dispatch('dirty')}
type="number"
step="1"
min="0"
max="100"
placeholder="25"
bind:value={config.dec_scale_occupancy_rate}
/>
{:else}
<input type="number" step="0.01" disabled />
{/if}
</label>
<!-- svelte-ignore a11y-label-has-associated-control -->
<label>
Occupancy rate threshold to exceed to trigger an incremental scale-out (increase) <Tooltip
>Default: 75%, need to exceed average of all of 15s, 5m and 30m occupancy rates</Tooltip
>
{#if config !== undefined}
<input
on:input={() => dispatch('dirty')}
type="number"
step="1"
min="0"
max="100"
placeholder="75"
bind:value={config.inc_scale_occupancy_rate}
/>
{:else}
<input type="number" step="0.01" disabled />
{/if}
</label>
<!-- svelte-ignore a11y-label-has-associated-control -->
<label>
Num workers to scale-in/out by when incremental <Tooltip
>Default: (max_workers - min_workers) / 5</Tooltip
>
{#if config !== undefined}
<input
on:input={() => dispatch('dirty')}
type="number"
step="1"
min="1"
placeholder="(max_workers - min_workers) / 5"
bind:value={config.inc_num_workers}
/>
{:else}
<input type="number" disabled />
{/if}
</label>
<Label label="Custom tags to autoscale on">
<svelte:fragment slot="header">
<Tooltip>
By default, autoscaling will apply to the tags the worker group is assigned to but
you can override this here.
</Tooltip>
</svelte:fragment>
{#if config}
{#if config.custom_tags}
<MultiSelect
outerDivClass="text-secondary !bg-surface-disabled !border-0"
selected={config.custom_tags}
on:change={(e) => {
console.log(e.detail.type, config?.custom_tags)
if (e.detail && config?.custom_tags) {
if (e.detail.type === 'add') {
config.custom_tags = [...config.custom_tags, e.detail.option]
} else if (e.detail.type === 'remove') {
config.custom_tags = config.custom_tags.filter((t) => t !== e.detail.option)
if (config?.custom_tags && config.custom_tags.length == 0) {
config.custom_tags = undefined
}
} else if (e.detail.type === 'removeAll') {
config.custom_tags = undefined
} else {
console.error(
`Priority tags multiselect - unknown event type: '${e.detail.type}'`
)
}
dispatch('dirty')
}
}}
on:clear={() => {
if (config) {
config.custom_tags = undefined
}
dispatch('dirty')
}}
options={worker_tags ?? []}
selectedOptionsDraggable={false}
ulOptionsClass={'!bg-surface-secondary'}
placeholder="Tags"
/>
{:else}
<Button
color="light"
size="xs"
variant="contained"
on:click={() => {
if (config) {
config.custom_tags = []
dispatch('dirty')
}
}}>Add custom tags</Button
>
{/if}
{/if}
</Label>
</div>
</Section>
</div>
</div>
<div class="flex flex-col gap-1 max-w-xs text-sm">
<h5>Integration</h5>
{#if config?.integration}
<ToggleButtonGroup
on:selected={(e) => dispatch('dirty')}
bind:selected={config.integration.type}
class="mb-4 mt-2"
>
<ToggleButton
value="dryrun"
size="sm"
label="Dry run"
tooltip="See autoscaling events but not actual scaling actions will be performed"
/>
<ToggleButton
value="script"
size="sm"
label="Custom Script"
tooltip="Run a custom script to scale your worker group"
/>
<ToggleButton position="center" disabled value="ecs" size="sm" label="ECS (soon)" />
<ToggleButton position="right" disabled value="nomad" size="sm" label="Nomad (soon)" />
<ToggleButton
position="right"
disabled
value="kubernetes"
size="sm"
label="Kubernetes (soon)"
/>
</ToggleButtonGroup>
{#if config.integration.type === 'script'}
<label>
Script path on the 'admins' workspace
<input
on:input={() => dispatch('dirty')}
type="text"
bind:value={config.integration.path}
/>
</label>
<label>
Custom tag for executing script (optional)
{#if config.integration.tag}
<input
on:input={() => dispatch('dirty')}
type="text"
bind:value={config.integration.tag}
/>
{:else}
<Button
color="light"
size="xs"
variant="contained"
on:click={() => {
if (config?.integration?.type === 'script') {
config.integration.tag = 'bash'
dispatch('dirty')
}
}}>Set tag</Button
>
{/if}
</label>
<div class="flex mt-6 gap-2">
<Button
color="dark"
target="_blank"
endIcon={{ icon: ExternalLink }}
href="/scripts/add?hub=hub%2F9204%2Fhelper%2FScale%20a%20worker%20group%20deployed%20as%20a%20kubernetes%20service&workspace=admins"
>Create from template</Button
>
<Button
color="dark"
target="_blank"
href={`/runs/${config.integration.path}?workspace=admins`}
endIcon={{ icon: ExternalLink }}
>
See jobs
</Button>
</div>
<div class="flex flex-row gap-2 mt-4">
<Button color="light" size="xs" variant="contained">Test scaling</Button>
<div class="flex text-xs flex-row gap-2 items-center">
<input class="!w-16" type="number" bind:value={test_input} />
workers
</div>
</div>
{/if}
{:else}
<ToggleButtonGroup selected={'script'} disabled class="mb-4 mt-2">
<ToggleButton value="dryrun" size="sm" label="Dry run" />
<ToggleButton value="script" size="sm" label="Custom Script" />
<ToggleButton position="center" value="ecs" size="sm" label="ECS (soon)" />
<ToggleButton position="right" value="nomad" size="sm" label="Nomad (soon)" />
<ToggleButton position="right" value="kubernetes" size="sm" label="Kubernetes (soon)" />
</ToggleButtonGroup>
<label>
Script path on the 'admins' workspace
<input type="text" disabled />
</label>
{/if}
</div>
</div>
frontend/src/lib/components/AutoscalingEvents.svelte
+73
View File
@@ -0,0 +1,73 @@
<script lang="ts">
import { ConfigService, type AutoscalingEvent } from '$lib/gen'
import { LoaderIcon, RefreshCw } from 'lucide-svelte'
import { Button, Skeleton } from './common'
import { twMerge } from 'tailwind-merge'
import TimeAgo from './TimeAgo.svelte'
import { enterpriseLicense } from '$lib/stores'
export let worker_group: string
let loading = true
let events: AutoscalingEvent[] | undefined = undefined
$: worker_group && loadEvents()
async function loadEvents() {
loading = true
try {
events = await ConfigService.listAutoscalingEvents({ workerGroup: worker_group })
} catch (e) {
events = []
console.error(e)
} finally {
loading = false
}
}
</script>
<div>
<h6
class={!$enterpriseLicense || (events != undefined && events.length == 0)
? 'text-tertiary'
: ''}
>Autoscaling events {#if $enterpriseLicense}<span class="text-xs text-tertiary">(5 last)</span>
<span class="inline-flex ml-6">
<Button
startIcon={{
icon: loading ? LoaderIcon : RefreshCw,
classes: twMerge(
loading ? 'animate-spin text-blue-800' : '',
'transition-all text-gray-500 dark:text-white'
)
}}
color="light"
size="xs2"
btnClasses={twMerge(loading ? ' bg-blue-100 dark:bg-blue-400' : '', 'transition-all')}
on:click={() => loadEvents()}
iconOnly
/>
</span>{/if}
</h6>
{#if !$enterpriseLicense}
<div class="text-xs pt-2 text-tertiary">Autoscaling is an EE feature</div>
{:else if loading}
<Skeleton layout={[[12], 1]} />
{:else if events}
{#if events.length == 0}
<div class="text-xs pt-2 text-tertiary"
>No events, is autoscaling set in the worker group config?</div
>
{:else}
<div class="flex flex-col gap-2 text-xs text-tertiary pt-4">
{#each events as event}
<div class="flex flex-row gap-4">
<div class="text-primary">{event.event_type} to {event.desired_workers}</div>
<div class="text-secondary">{event.reason}</div>
<div class="text-tertiary"><TimeAgo date={event.applied_at ?? ''} /></div>
</div>
{/each}
</div>
{/if}
{/if}
</div>
frontend/src/lib/components/CronInput.svelte
+9 -9
View File
@@ -112,26 +112,26 @@
// If using the basic editor, set the cron string based on the selected options
if (executeEvery === 'second') {
if (seconds > 0) {
nschedule = `*/${seconds} * * * *`
nschedule = `*/${seconds} * * * * *`
} else {
nschedule = `* * * * *`
nschedule = `* * * * * *`
}
} else if (executeEvery === 'minute') {
if (minutes > 0) {
nschedule = `0 */${minutes} * * *`
nschedule = `0 */${minutes} * * * *`
} else {
nschedule = `* * * * *`
nschedule = `* * * * * *`
}
} else if (executeEvery === 'hour') {
if (hours > 0) {
nschedule = `0 0 */${hours} * *`
nschedule = `0 0 */${hours} * * *`
} else {
nschedule = `* * * * *`
nschedule = `* * * * * *`
}
} else if (executeEvery === 'day-month') {
nschedule = `0 ${s_AtUTCMinutes} ${s_AtUTCHours} ${s_daysOfMonth} *`
nschedule = `0 ${s_AtUTCMinutes} ${s_AtUTCHours} ${s_daysOfMonth} * *`
} else if (executeEvery === 'month') {
nschedule = `0 ${s_AtUTCMinutes} ${s_AtUTCHours} ${s_daysOfMonth} ${s_months}`
nschedule = `0 ${s_AtUTCMinutes} ${s_AtUTCHours} ${s_daysOfMonth} ${s_months} *`
} else if (executeEvery === 'day-week') {
nschedule = `0 ${s_AtUTCMinutes} ${s_AtUTCHours} * * ${s_daysOfWeek}`
}
@@ -214,7 +214,7 @@
type="text"
id="cron-schedule"
name="cron-schedule"
placeholder="*/30 * * * *"
placeholder="0 0 */1 * * *"
bind:value={schedule}
{disabled}
/>
frontend/src/lib/components/CustomPopover.svelte
+2 -1
View File
@@ -9,6 +9,7 @@
export let disappearTimeout = 100
export let appearTimeout = 300
export let style: string | undefined = undefined
export let noPadding = false
export let focusEl: HTMLElement | undefined = undefined
@@ -73,7 +74,7 @@
use:popperContent={popperOptions}
on:mouseenter={open}
on:mouseleave={close}
class="z-[5001] border rounded-lg shadow-lg p-4 bg-surface"
class="z-[5001] border rounded-lg shadow-lg {noPadding ? '' : 'p-4'} bg-surface"
>
<slot name="overlay" />
</div>
frontend/src/lib/components/EditorBar.svelte
+1
View File
@@ -666,6 +666,7 @@ $res = json_decode(curl_exec($ch));`)
</div>
<div class="flex flex-row items-center gap-2">
<slot name="right" />
{#if scriptPath && !noHistory}
<Button
btnClasses="!font-medium text-tertiary"
frontend/src/lib/components/FlowBuilder.svelte
+35 -29
View File
@@ -119,13 +119,21 @@
if (version === undefined) {
return
}
const flowVersion = await FlowService.getFlowLatestVersion({
workspace: $workspaceStore!,
path: $pathStore
})
onLatest = version === flowVersion?.id
try {
if (initialPath && initialPath != '') {
const flowVersion = await FlowService.getFlowLatestVersion({
workspace: $workspaceStore!,
path: initialPath
})
onLatest = version === flowVersion?.id
} else {
onLatest = true
}
} catch (err) {
console.error('Error comparing versions', err)
onLatest = true
}
}
const dispatch = createEventDispatcher()
@@ -277,9 +285,8 @@
}
async function handleSaveFlow(deploymentMsg?: string) {
await compareVersions();
if (onLatest) {
await compareVersions()
if (onLatest || initialPath == '' || savedFlow?.draft_only) {
// Handle directly
await saveFlow(deploymentMsg)
} else {
@@ -294,28 +301,27 @@
open = true
}
}
async function syncWithDeployed(){
const flow = await FlowService.getFlowByPath({
workspace: $workspaceStore!,
path: $pathStore,
withStarredInfo: true
})
deployedValue = {
...flow,
starred: undefined,
id: undefined,
edited_at: undefined,
edited_by: undefined,
workspace_id: undefined,
archived: undefined,
same_worker: undefined,
visible_to_runner_only: undefined,
ws_error_handler_muted: undefined,
}
deployedBy = flow.edited_by
async function syncWithDeployed() {
const flow = await FlowService.getFlowByPath({
workspace: $workspaceStore!,
path: initialPath,
withStarredInfo: true
})
deployedValue = {
...flow,
starred: undefined,
id: undefined,
edited_at: undefined,
edited_by: undefined,
workspace_id: undefined,
archived: undefined,
same_worker: undefined,
visible_to_runner_only: undefined,
ws_error_handler_muted: undefined
}
deployedBy = flow.edited_by
}
async function saveFlow(deploymentMsg?: string): Promise<void> {
loadingSave = true
try {
frontend/src/lib/components/FlowGraphViewerStep.svelte
+4 -2
View File
@@ -133,8 +133,10 @@
Error handler
{:else if stepDetail.id === 'preprocessor'}
Preprocessor
{:else}
Anonymous step
{:else if stepDetail.value.type == 'rawscript'}
Inline {stepDetail.value.language} script
{:else if stepDetail.value.type == 'script'}
Workspace script
{/if}
</span>
</div>
frontend/src/lib/components/FlowTutorials.svelte
+32 -17
View File
@@ -1,28 +1,28 @@
<script lang="ts">
import { skipAllTodos } from '$lib/tutorialUtils'
import FlowBranchAll from './tutorials/FlowBranchAll.svelte'
import FlowBranchOne from './tutorials/FlowBranchOne.svelte'
import FlowBuilderTutorialBranchAll from './tutorials/FlowBuilderTutorialBranchAll.svelte'
import FlowBuilderTutorialBranchOne from './tutorials/FlowBuilderTutorialBranchOne.svelte'
import FlowBuilderTutorialSimpleFlow from './tutorials/FlowBuilderTutorialSimpleFlow.svelte'
import FlowBuilderTutorialsForLoop from './tutorials/FlowBuilderTutorialsForLoop.svelte'
import FlowErrorHandler from './tutorials/FlowErrorHandler.svelte'
import FlowBuilderTutorialForLoop from './tutorials/FlowBuilderTutorialForLoop.svelte'
import FlowBuilderTutorialErrorHandler from './tutorials/FlowBuilderTutorialErrorHandler.svelte'
let flowBuilderTutorialSimpleFlow: FlowBuilderTutorialSimpleFlow | undefined = undefined
let flowBuilderTutorialsForLoop: FlowBuilderTutorialsForLoop | undefined = undefined
let flowBranchOne: FlowBranchOne | undefined = undefined
let flowBranchAll: FlowBranchAll | undefined = undefined
let flowErrorHandler: FlowErrorHandler | undefined = undefined
let flowBuilderTutorialForLoop: FlowBuilderTutorialForLoop | undefined = undefined
let flowBuilderTutorialBranchOne: FlowBuilderTutorialBranchOne | undefined = undefined
let flowBuilderTutorialBranchAll: FlowBuilderTutorialBranchAll | undefined = undefined
let flowBuilderTutorialErrorHandler: FlowBuilderTutorialErrorHandler | undefined = undefined
export function runTutorialById(id: string, indexToInsertAt?: number | undefined) {
if (id === 'forloop') {
flowBuilderTutorialsForLoop?.runTutorial(indexToInsertAt)
flowBuilderTutorialForLoop?.runTutorial(indexToInsertAt)
} else if (id === 'branchone') {
flowBranchOne?.runTutorial()
flowBuilderTutorialBranchOne?.runTutorial()
} else if (id === 'branchall') {
flowBranchAll?.runTutorial()
flowBuilderTutorialBranchAll?.runTutorial()
} else if (id === 'action') {
flowBuilderTutorialSimpleFlow?.runTutorial()
} else if (id === 'error-handler') {
flowErrorHandler?.runTutorial()
flowBuilderTutorialErrorHandler?.runTutorial()
}
}
@@ -37,12 +37,27 @@
on:skipAll={skipAll}
on:reload
/>
<FlowBuilderTutorialsForLoop
bind:this={flowBuilderTutorialsForLoop}
<FlowBuilderTutorialForLoop
bind:this={flowBuilderTutorialForLoop}
on:error
on:skipAll={skipAll}
on:reload
/>
<FlowBuilderTutorialBranchOne
bind:this={flowBuilderTutorialBranchOne}
on:error
on:skipAll={skipAll}
on:reload
/>
<FlowBuilderTutorialBranchAll
bind:this={flowBuilderTutorialBranchAll}
on:error
on:skipAll={skipAll}
on:reload
/>
<FlowBuilderTutorialErrorHandler
bind:this={flowBuilderTutorialErrorHandler}
on:error
on:skipAll={skipAll}
on:reload
/>
<FlowBranchOne bind:this={flowBranchOne} on:error on:skipAll={skipAll} on:reload />
<FlowBranchAll bind:this={flowBranchAll} on:error on:skipAll={skipAll} on:reload />
<FlowErrorHandler bind:this={flowErrorHandler} on:error on:skipAll={skipAll} on:reload />
frontend/src/lib/components/InstanceSettings.svelte
+95 -28
View File
@@ -13,7 +13,7 @@
import KeycloakSetting from './KeycloakSetting.svelte'
import Alert from './common/alert/Alert.svelte'
import { isCloudHosted } from '$lib/cloud'
import { capitalize, classNames } from '$lib/utils'
import { capitalize, classNames, sleep } from '$lib/utils'
import { enterpriseLicense } from '$lib/stores'
import CustomOauth from './CustomOauth.svelte'
import {
@@ -102,8 +102,8 @@
if (values['base_url'] == undefined) {
values['base_url'] = 'http://localhost'
}
if (values['smtp_connect'] == undefined) {
values['smtp_connect'] = {}
if (values['smtp_settings'] == undefined) {
values['smtp_settings'] = {}
}
loading = false
@@ -111,6 +111,7 @@
}
export async function saveSettings() {
let shouldReloadPage = false
if (values) {
const allSettings = Object.values(settings).flatMap((x) => Object.entries(x))
let licenseKeySet = false
@@ -129,6 +130,9 @@
if (x.key == 'license_key') {
licenseKeySet = true
}
if (x.requiresReloadOnChange) {
shouldReloadPage = true
}
return await SettingService.setGlobal({
key: x.key,
requestBody: { value: values?.[x.key] }
@@ -158,8 +162,14 @@
} else {
console.error('Values not loaded')
}
sendUserToast('Settings updated')
dispatch('saved')
if (shouldReloadPage) {
sendUserToast('Settings updated, reloading page...')
await sleep(1000)
window.location.reload()
} else {
sendUserToast('Settings updated')
dispatch('saved')
}
}
let oauths: Record<string, any> = {}
@@ -206,7 +216,8 @@
'basecamp',
'linkedin',
'quickbooks',
'visma'
'visma',
'spotify'
]
let oauth_name = undefined
@@ -274,22 +285,34 @@
<TabContent value={category}>
{#if category == 'SMTP'}
<div class="text-secondary pb-4 text-xs"
>Setting SMTP unlocks sending emails upon adding new users to the workspace or the instance or sending critical alerts.
<a target="_blank" href="https://www.windmill.dev/docs/misc/setup_smtp">Learn more</a></div
>Setting SMTP unlocks sending emails upon adding new users to the workspace or the
instance or sending critical alerts.
<a target="_blank" href="https://www.windmill.dev/docs/misc/setup_smtp">Learn more</a
></div
>
{:else if category == 'Registries'}
<div class="text-secondary pb-4 text-xs">
Add private registries for Pip, Bun and npm. <a target="_blank" href="https://www.windmill.dev/docs/advanced/imports">Learn more</a>
Add private registries for Pip, Bun and npm. <a
target="_blank"
href="https://www.windmill.dev/docs/advanced/imports">Learn more</a
>
</div>
{:else if category == 'Slack'}
<div class="text-secondary pb-4 text-xs">
Connecting your instance to a Slack workspace enables critical alerts to be sent to a Slack channel.
<a target="_blank" href="https://www.windmill.dev/docs/misc/saml_and_scim">Learn more</a>
Connecting your instance to a Slack workspace enables critical alerts to be sent to a
Slack channel.
<a target="_blank" href="https://www.windmill.dev/docs/misc/saml_and_scim"
>Learn more</a
>
</div>
{:else if category == 'SCIM/SAML'}
<div class="text-secondary pb-4 text-xs">
Setting up SAML and SCIM allows you to authenticate users using your identity provider.
<a target="_blank" href="https://www.windmill.dev/docs/advanced/instance_settings#slack">Learn more</a>
Setting up SAML and SCIM allows you to authenticate users using your identity
provider.
<a
target="_blank"
href="https://www.windmill.dev/docs/advanced/instance_settings#slack">Learn more</a
>
</div>
{:else if category == 'Debug'}
<div class="text-secondary pb-4 text-xs">
@@ -310,9 +333,11 @@
</ul>
</div>
{#if $enterpriseLicense}
<div class="text-secondary pb-4 text-xs">
On Enterprise Edition, you must send data to check that usage is in line with the terms of the subscription. You can either enable telemetry or regularly send usage data by clicking the button below.
</div>
<div class="text-secondary pb-4 text-xs">
On Enterprise Edition, you must send data to check that usage is in line with the
terms of the subscription. You can either enable telemetry or regularly send usage
data by clicking the button below.
</div>
<Button
on:click={sendStats}
variant="border"
@@ -340,12 +365,17 @@
{/if}
<div class="py-1" />
<div class="mb-2">
<span class="text-primary text-sm">When at least one of the below options is set, users will be able to login to
Windmill via their third-party account.
<br /> To test SSO, the recommended workflow is to to save the settings and try to
login in an incognito window. <a target="_blank" href="https://www.windmill.dev/docs/misc/setup_oauth#sso">Learn more</a></span>
</div>
<div class="mb-2">
<span class="text-primary text-sm"
>When at least one of the below options is set, users will be able to login to
Windmill via their third-party account.
<br /> To test SSO, the recommended workflow is to to save the settings and try
to login in an incognito window.
<a target="_blank" href="https://www.windmill.dev/docs/misc/setup_oauth#sso"
>Learn more</a
></span
>
</div>
<div class="flex flex-col gap-2 py-4">
<OAuthSetting name="google" bind:value={oauths['google']} />
<OAuthSetting name="microsoft" bind:value={oauths['microsoft']} />
@@ -374,7 +404,11 @@
<div class="p-2 border rounded">
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Custom Name</span>
<input type="text" placeholder="Custom Name" bind:value={oauths[k]['display_name']} />
<input
type="text"
placeholder="Custom Name"
bind:value={oauths[k]['display_name']}
/>
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Client Id</span>
@@ -429,12 +463,18 @@
</div>
{:else if ssoOrOauth === 'oauth'}
<div class="mb-2">
<span class="text-primary text-sm">When one of the below options is set, you will be able to create a specific
<span class="text-primary text-sm"
>When one of the below options is set, you will be able to create a specific
resource containing a token automatically generated by the third-party provider.
<br />
To test it after setting an oauth client, go to the Resources menu and create a new
one of the type of your oauth client (i.e. a 'github' resource if you set Github OAuth).
<br /><a target="_blank" href="https://www.windmill.dev/docs/misc/setup_oauth#oauth">Learn more</a></span>
one of the type of your oauth client (i.e. a 'github' resource if you set Github
OAuth).
<br /><a
target="_blank"
href="https://www.windmill.dev/docs/misc/setup_oauth#oauth">Learn more</a
></span
>
</div>
<div class="py-1" />
<OAuthSetting login={false} name="slack" bind:value={oauths['slack']} />
@@ -518,7 +558,7 @@
<div>
<div class="flex-col flex gap-2 pb-4">
{#each settings[category] as setting}
{#if (!setting.cloudonly || isCloudHosted()) && showSetting(setting.key, values)}
{#if (!setting.cloudonly || isCloudHosted()) && showSetting(setting.key, values) && !(setting.hiddenIfNull && values[setting.key] == null)}
{#if setting.ee_only != undefined && !$enterpriseLicense}
<div class="flex text-xs items-center gap-1 text-yellow-500 whitespace-nowrap">
<AlertTriangle size={16} />
@@ -549,6 +589,20 @@
: ''}
bind:value={values[setting.key]}
/>
{#if setting.advancedToggle}
<div class="mt-1">
<Toggle
size="xs"
options={{ right: setting.advancedToggle.label }}
checked={setting.advancedToggle.checked(values)}
on:change={() => {
if (setting.advancedToggle) {
values = setting.advancedToggle.onChange(values)
}
}}
/>
</div>
{/if}
{:else if setting.fieldType == 'textarea'}
<textarea
rows="2"
@@ -621,24 +675,33 @@
{@const attemptedAt = new Date(
latestKeyRenewalAttempt.attempted_at
).toLocaleString()}
{@const isTrial =
latestKeyRenewalAttempt.result.startsWith('error: trial:')}
<div class="relative">
<Popover notClickable>
<div class="flex flex-row items-center gap-1">
{#if latestKeyRenewalAttempt.result === 'success'}
<BadgeCheck class="text-green-600" size={12} />
{:else}
<BadgeX class="text-red-600" size={12} />
<BadgeX
class={isTrial ? 'text-yellow-600' : 'text-red-600'}
size={12}
/>
{/if}
<span
class={classNames(
'text-xs',
latestKeyRenewalAttempt.result === 'success'
? 'text-green-600'
: isTrial
? 'text-yellow-600'
: 'text-red-600'
)}
>
{latestKeyRenewalAttempt.result === 'success'
? 'Latest key renewal succeeded'
: isTrial
? 'Latest key renewal ignored because in trial'
: 'Latest key renewal failed'}
on {attemptedAt}
</span>
@@ -648,6 +711,10 @@
<span class="text-green-300">
Latest key renewal succeeded on {attemptedAt}
</span>
{:else if isTrial}
<span class="text-yellow-300">
License key cannot be renewed during trial ({attemptedAt})
</span>
{:else}
<span class="text-red-300">
Latest key renewal failed on {attemptedAt}: {latestKeyRenewalAttempt.result.replace(
frontend/src/lib/components/JobArgs.svelte
+4 -3
View File
@@ -12,6 +12,7 @@
import Row from './table/Row.svelte'
import HighlightTheme from './HighlightTheme.svelte'
import { deepEqual } from 'fast-equals'
import { isWindmillTooBigObject } from './job_args'
export let id: string | undefined = undefined
export let args: any
@@ -63,7 +64,7 @@ ${Object.entries(args)
<div class="relative">
<DataTable size="sm">
<Head>
<tr>
<tr class="w-full">
<Cell head first>{argLabel ?? 'Arg'}</Cell>
<Cell head last>Value</Cell>
</tr>
@@ -79,7 +80,7 @@ ${Object.entries(args)
</svelte:fragment>
</Head>
<tbody class="divide-y">
<tbody class="divide-y w-full">
{#if args && Object.keys(args).length > 0}
{#each Object.entries(args).sort((a, b) => a[0].localeCompare(b[0])) as [arg, value]}
<Row>
@@ -137,7 +138,7 @@ ${Object.entries(args)
Copy to clipboard
</Button>
</svelte:fragment>
{#if jsonStr.length > 100000 || (id && workspace && args && typeof args === 'object' && deepEqual( Object.keys(args), ['reason'] ) && args['reason'] == 'WINDMILL_TOO_BIG')}
{#if jsonStr.length > 100000 || (id && workspace && args && isWindmillTooBigObject(args))}
<div class="text-sm mb-2 text-tertiary">
<a
download="windmill-args.json"
frontend/src/lib/components/LightweightResourcePicker.svelte
+119 -52
View File
@@ -7,7 +7,7 @@
import DarkModeObserver from './DarkModeObserver.svelte'
import { Button, Drawer, DrawerContent } from './common'
import { Plus, Loader2 } from 'lucide-svelte'
import { Plus, Loader2, Link2Off } from 'lucide-svelte'
import type { AppViewerContext } from './apps/types'
import { sendUserToast } from '$lib/toast'
@@ -30,26 +30,37 @@
value: value ?? initialValue,
label: value ?? initialValue
}
: ''
: undefined
let collection = [valueSelect]
let loading = true
async function loadResources(resourceType: string | undefined) {
const nc = (
await ResourceService.listResource({
workspace: $workspaceStore!,
resourceType
})
).map((x) => ({
value: x.path,
label: x.path
}))
loading = true
try {
const nc = (
await ResourceService.listResource({
workspace: appViewerContext?.workspace ?? $workspaceStore,
resourceType
})
).map((x) => ({
value: x.path,
label: x.path
}))
// TODO check if this is needed
if (!nc.find((x) => x.value == value) && (initialValue || value)) {
nc.push({ value: value ?? initialValue!, label: value ?? initialValue! })
// TODO check if this is needed
if (!nc.find((x) => x.value == value) && (initialValue || value)) {
nc.push({ value: value ?? initialValue!, label: value ?? initialValue! })
}
collection = nc
console.log('collection', collection)
if (expressOAuthSetup && nc.length > 0) {
value = nc[0].value
valueSelect = nc[0]
}
} finally {
loading = false
}
collection = nc
}
$: $workspaceStore && loadResources(resourceType)
@@ -71,13 +82,21 @@
{#if expressOAuthSetup}
{#if open}
{#key refreshCount}
{#await import('./AppConnectLightweightResourcePicker.svelte')}
<Loader2 class="animate-spin" />
{:then Module}
{#await import('./AppConnectLightweightResourcePicker.svelte') then Module}
<Module.default
workspace={appViewerContext?.workspace ?? $workspaceStore}
{resourceType}
express={true}
on:error={(e) => {
sendUserToast(e.detail, true)
open = false
}}
on:refresh={async (e) => {
await loadResources(resourceType)
value = e.detail
valueSelect = { value, label: value }
open = false
}}
/>
{/await}
{/key}
@@ -100,7 +119,8 @@
on:error={(e) => {
sendUserToast(e.detail, true)
}}
on:refresh={(e) => {
on:refresh={async (e) => {
await loadResources(resourceType)
value = e.detail
valueSelect = { value, label: value }
drawer?.closeDrawer?.()
@@ -120,42 +140,89 @@
{/if}
<div class="flex flex-col w-full items-start">
<div class="flex flex-row gap-x-1 w-full">
<Select
{disabled}
portal={!disablePortal}
value={valueSelect}
on:change={(e) => {
value = e.detail.value
valueSelect = e.detail
}}
on:clear={() => {
value = undefined
valueSelect = ''
}}
items={collection}
class="text-clip grow min-w-0"
placeholder="{resourceType ?? 'any'} resource"
inputStyles={SELECT_INPUT_DEFAULT_STYLE.inputStyles}
containerStyles={darkMode
? SELECT_INPUT_DEFAULT_STYLE.containerStylesDark
: SELECT_INPUT_DEFAULT_STYLE.containerStyles}
/>
{#if resourceType}
<Button
<div class="flex flex-row gap-x-1 w-full justify-between items-center">
{#if loading}
<Loader2 size={12} class="animate-spin" />
{:else if expressOAuthSetup && collection.length < 2}
{#if collection.length == 0}
<div class="text-2xs text-primary pr-2">Connect {resourceType}</div>
{:else if collection.length == 1 && value == collection[0]?.value}
<div class="text-2xs text-primary pr-2 flex items-center gap-1">
{resourceType} <div class="rounded-full w-2 h-2 bg-green-600 animate-pulse" />
</div>
{:else}
<Button
color="light"
size="sm"
on:click={() => {
const item = collection[0]
if (item) {
value = item.value
valueSelect = item
}
}}>Use existing</Button
>
{/if}
{:else}
<Select
{disabled}
color="light"
variant="contained"
btnClasses="w-8 px-0.5 py-1.5"
size="sm"
on:click={() => {
open = true
drawer?.openDrawer?.()
portal={!disablePortal}
value={valueSelect}
on:change={(e) => {
value = e.detail.value
valueSelect = e.detail
}}
startIcon={{ icon: Plus }}
iconOnly
on:clear={() => {
value = undefined
valueSelect = undefined
}}
items={collection}
class="text-clip grow min-w-0"
placeholder="{resourceType ?? 'any'} resource"
inputStyles={SELECT_INPUT_DEFAULT_STYLE.inputStyles}
containerStyles={darkMode
? SELECT_INPUT_DEFAULT_STYLE.containerStylesDark
: SELECT_INPUT_DEFAULT_STYLE.containerStyles}
/>
{/if}
<div class="flex gap-1 items-center">
{#if valueSelect && expressOAuthSetup}
<Button
{disabled}
color="light"
variant="contained"
size="sm"
btnClasses="w-8 px-0.5 py-1.5"
iconOnly
on:click={async () => {
if (valueSelect && valueSelect.label) {
value = undefined
await ResourceService.deleteResource({
workspace: appViewerContext?.workspace ?? $workspaceStore,
path: valueSelect.label
})
await loadResources(resourceType)
valueSelect = undefined
}
}}
startIcon={{ icon: Link2Off }}
/>
{/if}
{#if resourceType}
<Button
{disabled}
color="light"
variant="contained"
btnClasses="w-8 px-0.5 py-1.5"
size="sm"
on:click={() => {
open = true
drawer?.openDrawer?.()
}}
startIcon={{ icon: Plus }}
iconOnly
/>
{/if}
</div>
</div>
</div>
frontend/src/lib/components/ModulePreview.svelte
+12 -9
View File
@@ -32,7 +32,7 @@
getContext<FlowEditorContext>('FlowEditorContext')
// Test
let scriptProgress = undefined;
let scriptProgress = undefined
let testJobLoader: TestJobLoader
let testIsLoading = false
let testJob: Job | undefined = undefined
@@ -54,7 +54,7 @@
export async function runTest(args: any) {
// Not defined if JobProgressBar not loaded
if (jobProgressReset) jobProgressReset();
if (jobProgressReset) jobProgressReset()
const val = mod.value
// let jobId: string | undefined = undefined
@@ -75,7 +75,7 @@
script.content,
script.language,
args,
$flowStore?.tag ?? script.tag
$flowStore?.tag ?? (val.tag_override ? val.tag_override : script.tag)
)
} else if (val.type == 'flow') {
await testJobLoader?.abstractRun(() =>
@@ -103,8 +103,6 @@
let forceJson = false
</script>
<TestJobLoader
toastError={noEditor}
on:done={() => jobDone()}
@@ -158,9 +156,14 @@
/>
</Pane>
<Pane size={50} minSize={10} class="text-sm text-tertiary">
{#if scriptProgress}
<JobProgressBar job={testJob} bind:scriptProgress bind:reset={jobProgressReset} compact={true} />
{/if}
{#if scriptProgress}
<JobProgressBar
job={testJob}
bind:scriptProgress
bind:reset={jobProgressReset}
compact={true}
/>
{/if}
{#if testJob != undefined && 'result' in testJob && testJob.result != undefined}
<div class="break-words relative h-full p-2">
<DisplayResult
@@ -186,7 +189,7 @@
<div class="p-2">
{#if testIsLoading}
{#if !scriptProgress}
<Loader2 class="animate-spin" />
<Loader2 class="animate-spin" />
{/if}
{:else}
Test to see the result here
frontend/src/lib/components/OAuthSetting.svelte
+22 -4
View File
@@ -3,6 +3,7 @@
import CollapseLink from './CollapseLink.svelte'
import IconedResourceType from './IconedResourceType.svelte'
import Toggle from './Toggle.svelte'
import { onMount } from 'svelte'
export let name: string
export let value: any
@@ -13,6 +14,21 @@
let tenant: string = ''
$: name == 'microsoft' && changeTenantId(tenant)
onMount(() => {
try {
if (
name == 'microsoft' &&
value &&
value['login_config'] &&
typeof value['login_config']['auth_url'] == 'string'
) {
tenant = value['login_config']['auth_url'].split('/')[3]
}
} catch (e) {
console.error('Could not set tenantId', e)
}
})
function changeTenantId(tenant: string) {
if (value && tenant) {
if (tenant != '') {
@@ -50,10 +66,12 @@
>
{#if enabled}
<div class="p-2 rounded border">
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Custom Name</span>
<input type="text" placeholder="Custom Name" bind:value={value['display_name']} />
</label>
{#if name != 'slack'}
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Custom Name</span>
<input type="text" placeholder="Custom Name" bind:value={value['display_name']} />
</label>
{/if}
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Client Id</span>
<input type="text" placeholder="Client Id" bind:value={value['id']} />
frontend/src/lib/components/QueueMetricsDrawer.svelte
+58 -37
View File
@@ -9,6 +9,18 @@
import Tooltip from './Tooltip.svelte'
import { enterpriseLicense } from '$lib/stores'
function updateChangesMade() {
changesMade = JSON.stringify(alerts) !== JSON.stringify(originalAlerts)
}
function handleInput(event) {
const target = event.target
console.log(target)
if (target.tagName.toLowerCase() === 'input') {
updateChangesMade()
}
}
type Alert = {
name: string
tags_to_monitor: string[]
@@ -70,16 +82,20 @@
}
function startEditing(index) {
if (editingIndex !== -1) {
const success = saveAlert(editingIndex)
if (!success) return
}
editingIndex = index
updateWorkerTags()
}
function saveAlert(index) {
function saveAlert(index): boolean {
const newAlert = alerts[index]
if (newAlert.tags_to_monitor.length === 0) {
sendUserToast('Please add at least one tag before saving.', true)
return
return false
}
if (
@@ -88,7 +104,7 @@
newAlert.alert_time_threshold_seconds <= 0
) {
sendUserToast('All numeric values must be strictly positive.', true)
return
return false
}
const alertExists = originalAlerts.some(
@@ -99,13 +115,15 @@
)
if (alertExists) {
sendUserToast('An identical alert already exists.', true)
return
sendUserToast('You can only define one alert per identical set of tags', true)
return false
}
editingIndex = -1
changesMade = JSON.stringify(alerts) !== JSON.stringify(originalAlerts)
updateChangesMade()
stagedNewAlert = false
return true
}
function stageDeleteAlert(index) {
@@ -126,16 +144,18 @@
}
newTag = ''
filteredTags = []
updateChangesMade()
}
function removeTag(alertIndex, tag) {
alerts[alertIndex].tags_to_monitor = alerts[alertIndex].tags_to_monitor.filter((t) => t !== tag)
updateChangesMade()
}
async function applyConfig() {
if (stagedNewAlert) {
sendUserToast('New alert staged. Please save or cancel the changes before applying.', true)
return
if (editingIndex !== -1) {
const success = saveAlert(editingIndex)
if (!success) return
}
try {
@@ -146,6 +166,7 @@
removedAlerts = []
changesMade = false
stagedNewAlert = false
editingIndex = -1
} catch (error) {
console.error('Failed to update config:', error)
}
@@ -167,7 +188,7 @@
}
const newAlert = {
name: 'New Alert',
name: 'Job Queue Alert',
tags_to_monitor: [],
jobs_num_threshold: 3,
alert_cooldown_seconds: 600,
@@ -177,6 +198,7 @@
alerts = [...alerts, newAlert]
editingIndex = alerts.length - 1
stagedNewAlert = true
updateChangesMade()
updateWorkerTags()
}
@@ -188,28 +210,35 @@
alerts[alertIndex].tags_to_monitor = [
...new Set([...alerts[alertIndex].tags_to_monitor, ...workerTags])
]
alerts = alerts // Trigger reactivity
alerts = [...alerts]
updateChangesMade()
}
</script>
<Drawer bind:this={drawer} size="800px">
<DrawerContent title="Queue Metrics" on:close={drawer.closeDrawer}>
<DrawerContent
title="Queues"
on:close={drawer.closeDrawer}
documentationLink="https://www.windmill.dev/docs/core_concepts/worker_groups#queue-metrics"
>
<Section
label="Queue Alert Settings"
collapsable={true}
tooltip="Manage queue alerts for monitoring queue metrics. These alerts will trigger a critical alert notification if the queue metrics are met."
tooltip="A critical alert is triggered when the number of jobs in the queue exceeds the set threshold and they have been waiting for at least the specified time. After an alert, no new alerts will be triggered during the cooldown period."
eeOnly={true}
>
{#if $enterpriseLicense}
{#if changesMade}
<div class="text-red-600 text-xs whitespace-nowrap pb-2">Non applied changes</div>
<div class="flex gap-2 pb-2">
<Button color="blue" size="xs" on:click={applyConfig}>
<SaveIcon size={16} /> Apply Config
</Button>
<Button color="light" size="xs" on:click={cancelChanges}>Cancel</Button>
</div>
{/if}
<div class="flex gap-2 pb-2">
<Button color="blue" size="xs" on:click={applyConfig} disabled={!changesMade}>
<SaveIcon size={16} /> Apply Config
</Button>
<Button color="light" size="xs" on:click={cancelChanges} disabled={!changesMade}>
Cancel
</Button>
</div>
{#if alerts.length > 0}
<div>
@@ -217,12 +246,6 @@
<table class="w-full border-collapse mb-2 text-xs table-auto">
<thead class="bg-gray-200 dark:bg-slate-600 text-left text-xs">
<tr>
<th class="p-2 min-w-[120px]">
Name
<Tooltip
markdownTooltip="The name of the alert. This will be used in the crticial alert notification and help identify the alert."
/>
</th>
<th class="p-2 w-full">
Queue Tags to Monitor
<Tooltip markdownTooltip="Queue tags to monitor for this alert." />
@@ -230,38 +253,31 @@
<th class="p-2 min-w-[65px]">
Jobs
<Tooltip
markdownTooltip="Number of jobs threshold. An alert will be triggerd if more than this number of jobs are in the queue for at least the time threshold."
markdownTooltip="Number of jobs threshold: An alert will be triggered if the number of jobs in the queue exceeds this threshold and they have been waiting for at least the specified time threshold."
/>
</th>
<th class="p-2 min-w-[115px]">
Cooldown (s)
<Tooltip
markdownTooltip="Cooldown period in seconds. The cooldown period is the time after an alert is triggered that no new alerts will be triggered."
markdownTooltip="Cooldown period in seconds: This defines the time interval after an alert is triggered during which no additional alerts will be sent."
/>
</th>
<th class="p-2 min-w-[105px]">
Time (s)
<Tooltip
markdownTooltip="Time threshold in seconds. An alert will be triggered if more than 'n jobs threshold' have been in the queue for at least this amount of time."
markdownTooltip="Time threshold in seconds: An alert will be triggered if the number of jobs in the queue exceeds the job threshold and they have remained in the queue for at least this duration."
/>
</th>
<th class="p-2 min-w-[100px]"> Actions </th>
</tr>
</thead>
<tbody>
<tbody on:input={handleInput}>
{#each alerts as alert, index}
<tr
class={removedAlerts.includes(alert)
? 'bg-red-100 dark:bg-red-900 pointer-events-none opacity-50'
: ''}
>
<td class="border p-2">
{#if editingIndex === index}
<input type="text" bind:value={alert.name} class="w-full p-1" />
{:else}
{alert.name}
{/if}
</td>
<td class="border p-2">
{#if editingIndex === index}
<div class="flex flex-wrap gap-1 mb-2">
@@ -282,8 +298,9 @@
<input
type="text"
bind:value={newTag}
placeholder="Add tag"
placeholder="{workerTags.length === alert.tags_to_monitor.length ? 'All tags already added' : 'Add tag from dropdown' }"
on:input={(e) => filterTags(e)}
disabled={workerTags.length === alert.tags_to_monitor.length}
class="p-1 flex-grow mr-1"
/>
<button on:click={() => addTag(index, newTag)} aria-label="Add Tag">
@@ -294,6 +311,7 @@
<button
on:click={() => addAllTags(index)}
class="text-xs hover:bg-gray-200 dark:hover:bg-gray-700 rounded px-2 py-1 mt-1"
disabled={workerTags.length === alert.tags_to_monitor.length}
>
Add All Tags
</button>
@@ -302,6 +320,7 @@
class="autocomplete-list border max-h-36 overflow-y-auto absolute z-50"
>
{#each filteredTags as tag}
{#if !alert.tags_to_monitor.includes(tag)}
<li>
<button
type="button"
@@ -311,6 +330,7 @@
{tag}
</button>
</li>
{/if}
{/each}
</ul>
{/if}
@@ -401,6 +421,7 @@
</p>
{/if}
</Section>
<h1 class="pt-4">Queue Metrics</h1>
<div class="p-8">
<QueueMetricsDrawerInner />
</div>
frontend/src/lib/components/ResourcePicker.svelte
+3 -2
View File
@@ -40,7 +40,7 @@
let collection = valueSelect ? [valueSelect] : []
export async function askNewResource() {
appConnect?.open?.(resourceType, expressOAuthSetup)
appConnect?.open?.(resourceType)
}
let loading = true
@@ -101,6 +101,7 @@
}
}}
bind:this={appConnect}
{expressOAuthSetup}
/>
<ResourceEditorDrawer
@@ -178,7 +179,7 @@
color="light"
variant="border"
size="sm"
on:click={() => appConnect?.open?.(resourceType, expressOAuthSetup)}
on:click={() => appConnect?.open?.(resourceType)}
startIcon={{ icon: Plus }}
iconOnly={collection?.length > 0}
>{#if collection?.length == 0}
frontend/src/lib/components/RunPageSchedules.svelte
+1 -1
View File
@@ -218,7 +218,7 @@
$primarySchedule = {
summary: '',
args: {},
cron: '0 0 0 /1 * * *',
cron: '0 0 */1 * * *',
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
enabled: true
}
frontend/src/lib/components/ScriptBuilder.svelte
+64 -52
View File
@@ -18,9 +18,7 @@
encodeState,
formatCron,
orderedJsonStringify,
type Value
} from '$lib/utils'
import Path from './Path.svelte'
import ScriptEditor from './ScriptEditor.svelte'
@@ -92,7 +90,6 @@
let confirmCallback: () => void = () => {} // What happens when user clicks `override` in warning
let open: boolean = false // Is confirmation modal open
let metadataOpen =
!neverShowMeta &&
(showMeta ||
@@ -251,62 +248,78 @@
}
}
async function handleEditScript(stay: boolean, deployMsg?: string): Promise<void>{
// Fetch latest version and fetch entire script after if needed
let actual_parent_hash = (await ScriptService.getScriptLatestVersion({
workspace: $workspaceStore!,
path: script.path,
}))?.script_hash;
async function handleEditScript(stay: boolean, deployMsg?: string): Promise<void> {
// Fetch latest version and fetch entire script after if needed
let actual_parent_hash: string | undefined = undefined
// Usually when we create new script, we put current hash as a parent_hash
// But if we specify parent_hash that is already used, than we get error
// In order to fix it we make sure that client's understanding of parent_hash
// is aligns with understanding of backend.
if (script.parent_hash == actual_parent_hash) {
// Handle directly
await editScript(stay, actual_parent_hash, deployMsg);
} else {
// Fetch entire script, since we need it to show Diff
await syncWithDeployed()
// Handle through confirmation modal
confirmCallback = async () => {
open = false
await editScript(stay, actual_parent_hash, deployMsg);
}
// Open confirmation modal
open = true
try {
if (initialPath && initialPath != '') {
actual_parent_hash = (
await ScriptService.getScriptLatestVersion({
workspace: $workspaceStore!,
path: initialPath
})
)?.script_hash
}
} catch (error) {
//
}
// Usually when we create new script, we put current hash as a parent_hash
// But if we specify parent_hash that is already used, than we get error
// In order to fix it we make sure that client's understanding of parent_hash
// is aligns with understanding of backend.
if (actual_parent_hash == undefined || script.parent_hash == actual_parent_hash) {
// Handle directly
await editScript(stay, script.parent_hash!, deployMsg)
} else {
// Fetch entire script, since we need it to show Diff
await syncWithDeployed()
// Handle through confirmation modal
confirmCallback = async () => {
open = false
if (actual_parent_hash) {
await editScript(stay, actual_parent_hash, deployMsg)
} else {
sendUserToast('Could not fetch latest version of the script', true)
}
}
// Open confirmation modal
open = true
}
}
async function syncWithDeployed(){
const latestScript = await ScriptService.getScriptByPath({
workspace: $workspaceStore!,
path: script.path,
withStarredInfo: true
});
async function syncWithDeployed() {
const latestScript = await ScriptService.getScriptByPath({
workspace: $workspaceStore!,
path: initialPath,
withStarredInfo: true
})
deployedValue = {
...latestScript,
starred: undefined,
workspace_id: undefined,
archived: undefined,
created_at: undefined,
created_by: undefined,
deleted: undefined,
extra_perms: undefined,
is_template: undefined,
lock: undefined,
lock_error_logs: undefined,
parent_hashes: undefined,
};
deployedValue = {
...latestScript,
starred: undefined,
workspace_id: undefined,
archived: undefined,
created_at: undefined,
created_by: undefined,
deleted: undefined,
extra_perms: undefined,
is_template: undefined,
lock: undefined,
lock_error_logs: undefined,
parent_hashes: undefined
}
deployedBy = latestScript.created_by;
deployedBy = latestScript.created_by
}
async function editScript(stay: boolean, parentHash: string, deploymentMsg?: string, ): Promise<void> {
async function editScript(
stay: boolean,
parentHash: string,
deploymentMsg?: string
): Promise<void> {
loadingSave = true
try {
try {
@@ -354,7 +367,6 @@
}
})
console.log('initialPath', initialPath)
const scheduleExists =
initialPath != '' &&
(await ScheduleService.existsSchedule({
frontend/src/lib/components/ScriptEditor.svelte
+3 -1
View File
@@ -293,7 +293,9 @@
{args}
{noHistory}
{saveToWorkspace}
/>
>
<slot name="editor-bar-right" slot="right" />
</EditorBar>
{#if !noSyncFromGithub && customUi?.editorBar?.useVsCode != false}
<div class="py-1">
<Button
frontend/src/lib/components/Scrollable.svelte
+2 -1
View File
@@ -5,6 +5,7 @@
let isAtBottom: boolean = false
let isScrollable = false
export let id: string | null | undefined = undefined
export let scrollableClass: string = ''
export let shiftedShadow: boolean = false
let mutationObserver: MutationObserver
@@ -43,7 +44,7 @@
})
</script>
<div class={twMerge('relative pb-1', scrollableClass)}>
<div {id} class={twMerge('relative pb-1', scrollableClass)}>
<div bind:this={el} on:scroll={handleScroll} class="w-full h-full overflow-y-auto">
<slot />
</div>

Some files were not shown because too many files have changed in this diff Show More