Compare commits

...
52 Commits
Author SHA1 Message Date
Ruben Fiszelandrubenfiszel 3f49d3e061 chore(main): release 1.335.0 (#3792)
* chore(main): release 1.335.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-05-22 23:20:31 +02:00
Ruben Fiszel ebe2212982 add concurrency_key to exported workspace scripts 2024-05-22 23:12:13 +02:00
Ruben Fiszel 4d28a38750 fix(cli): add concurrency key to script 2024-05-22 22:22:19 +02:00
wendrulandRuben Fiszel 59b0adea8a Add warning for omitted obscured jobs (#3790)
* Add warning for omitted obscured jobs

* Update RunsTable.svelte

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2024-05-22 21:14:52 +02:00
Faton Ramadani 85cd8b4c0e fix(frontend): Fix aggrid infinite refresh (#3789)
* fix(frontend): Fix AgGrid infinit refresh when args changed

* fix(frontend): Fix AgGrid infinit refresh when args changed

* fix(frontend): remove dead code
2024-05-22 16:57:45 +02:00
HugoCasa 2f370de628 feat: improve user audit logs and stats (#3786)
* fix: improve user audit logs and stats

* chore: sqlx

* fix: no features build

* fix: sqlx

* fix: no features build

* chore: update ee ref
2024-05-22 16:51:04 +02:00
Ruben Fiszel 2883fc9f8e make drop cache not an error 2024-05-22 13:16:53 +02:00
Ruben Fiszel 4d88dadb1f remove jemalloc specific image 2024-05-22 11:44:37 +02:00
Ruben Fiszelandrubenfiszel 686c188ad5 chore(main): release 1.334.0 (#3785)
* chore(main): release 1.334.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-05-22 00:44:47 +02:00
Ruben Fiszel b30d239a2b nit 2024-05-22 00:44:25 +02:00
Ruben Fiszel 4631c3fd08 fix: add allow http toggle to object store settings 2024-05-22 00:36:04 +02:00
Faton Ramadani 59c0abae3d feat(frontend): Added support for title and placeholder for ArgInputs (#3779)
* feat(frontend): Added support for customLabel and placeholder

* feat(frontend): correctly handle labels

* feat(frontend): rename customLabel into title

* feat(frontend): fix layout

* feat(frontend): properly implement placeholder

* feat(frontend): properly implement placeholder

* feat(frontend): add missing undefined check

* feat(frontend): remove code duplication
2024-05-21 21:58:48 +02:00
Faton Ramadani 36df8389cd feat(frontend): Improve multiselect perf (#3770)
* feat(frontend): wip

* feat(frontend): Fork multiselect

* feat(frontend): cleanup

* feat(frontend): fix design

* feat(frontend): trie search

* feat(frontend): simplify search

* feat(frontend): simplify search

* feat(frontend): fix open
2024-05-21 19:47:19 +02:00
Faton Ramadani 518d8b916b fix(frontend): Fix path overflow on the Runs page (#3781) 2024-05-21 19:46:53 +02:00
wendrul 2efe5f9f10 Win 130 concurrency observability fixes (#3782)
* Resize badge and add filter by key button

* Make status filters icononly with tooltip

* nit: replace current job None with empty

* tooltip should be on the same row with flex

* Expand selected on ToggleButtonMore

* Make a common definition of UnifiedJob fields
2024-05-21 19:28:05 +02:00
Ruben Fiszelandrubenfiszel 7a8a08f31e chore(main): release 1.333.5 (#3776)
* chore(main): release 1.333.5

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-05-21 12:26:03 +02:00
wendrulandRuben Fiszel 1001b1e664 Fix list jobs and intervals mismatch (#3767)
* fix: mismatch when filtering on concurrency key

Fix a mismatch between to endpoints when filtering by concurrency key

* Add ObscuredJob type on openapi spec

* Correct unit for graph

* Finish frontend changes

* Add field indicating if obscured jobs were omitted

* impl from for ListQueueQuery and factor logic

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2024-05-21 12:21:25 +02:00
Ruben Fiszel b56fbdf898 fix: improve concurrency key migration 2024-05-21 12:18:05 +02:00
Ruben Fiszel 0268dd1206 fix: drop the linux file cache regularly to avoid triggering OOM killer 2024-05-21 10:21:14 +02:00
Faton Ramadani edf3015cc4 fix(frontend): Fix full height component for public apps (#3777)
* fix(frontend): fix fullHeight component for public apps

* fix(frontend): fix fullHeight component for public apps

* fix(frontend): fix fullHeight component for public apps
2024-05-21 10:07:00 +02:00
Faton Ramadani 31c47774cd fix(frontend): Fix selected value when the default value of a select component is null (#3778) 2024-05-21 09:06:12 +02:00
Ruben Fiszel da3b0434e9 fix: lighten watermark on public apps for ee 2024-05-21 07:50:47 +02:00
Ruben Fiszel aa60952ac3 frontend nits 2024-05-21 07:28:31 +02:00
Ruben Fiszelandrubenfiszel 240a96d4f1 chore(main): release 1.333.4 (#3775)
* chore(main): release 1.333.4

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-05-20 23:30:48 +02:00
Ruben Fiszel b2958be7c1 fix(cli): make bun the default language in absence of wmill defaultTs 2 2024-05-20 23:27:08 +02:00
Ruben Fiszelandrubenfiszel 158be28a52 chore(main): release 1.333.3 (#3774)
* chore(main): release 1.333.3

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-05-20 23:16:05 +02:00
Ruben Fiszel fbfad5e654 update git sync hub script 2024-05-20 23:15:05 +02:00
Ruben Fiszel 83990869bc fix(cli): make bun the default language in absence of wmill defaultTs 2024-05-20 23:09:38 +02:00
Ruben Fiszel 8cee11e32c use jemalloc for normal ee 2024-05-20 20:03:01 +02:00
Ruben Fiszel 0d78a31415 nit compile 2024-05-20 19:45:00 +02:00
Ruben Fiszelandrubenfiszel e7e7360259 chore(main): release 1.333.2 (#3772)
* chore(main): release 1.333.2

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-05-20 19:19:42 +02:00
d10sfanandellipsis-dev[bot] a3d81dfdce AgGrid - Allow to de-select all rows with setSelectedIndex (#3760)
* AgGrid - Allow to de-select all rows with setSelectedIndex

* Update frontend/src/lib/components/apps/components/display/table/AppAggridTable.svelte

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

* Else & check for null instead of -1

---------

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>
2024-05-20 18:59:56 +02:00
Ruben Fiszel da091cbad7 only enable profiling if env variable is present 2024-05-20 18:57:26 +02:00
Ruben Fiszel f451748faa allow dynamic heap profiling using jemalloc live prof_active 2024-05-20 18:44:03 +02:00
Faton Ramadani ccc2699d4e fix(frontend): fix AgGrid pagination refresh (#3773)
* feat(frontend): fix AgGrid pagination refresh

* feat(frontend): fix AgGrid pagination refresh
2024-05-20 15:25:18 +02:00
Ruben Fiszelandrubenfiszel 73ddc46ce7 chore(main): release 1.333.1 (#3771)
* chore(main): release 1.333.1

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-05-20 12:35:28 +02:00
Ruben Fiszel d167053878 remove extra tracing 2024-05-20 12:34:15 +02:00
Ruben Fiszel cd8b14a2fa further memory reduction 2024-05-20 11:38:11 +02:00
Ruben Fiszel 46c440f41a use RawValue for Flow value and hub script schema 2024-05-20 10:45:52 +02:00
d10sfan 158a83b0a9 Add support for form schema property setting textarea rows value (#3759) 2024-05-20 10:24:46 +02:00
Ruben Fiszel f1bb7cfcbd fix(backend): improve memory usage by making schema RawValue instead of serde_json::Value 2024-05-20 10:11:47 +02:00
Ruben Fiszelandrubenfiszel 4a49dc6df4 chore(main): release 1.333.0 (#3769)
* chore(main): release 1.333.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-05-19 22:31:57 +02:00
Ruben Fiszel f63c8349f4 sleep handle f64 2024-05-19 22:21:18 +02:00
Ruben Fiszel 5eeadb9240 handle early stop for simple flows 2024-05-19 22:07:06 +02:00
Ruben Fiszel 7a1808e951 feat: reduce memory usage (#3768)
* all

* all

* optimize deserialize

* all

* all

* fix issue

* fix compile

* fix compile

* fix compile

* fix compile

* fix compile

* fix compile

* fix compile

* fix compile

* fix compile

* fix compile
2024-05-19 21:58:38 +02:00
Ruben Fiszelandrubenfiszel 03519f5cac chore(main): release 1.332.1 (#3766)
* chore(main): release 1.332.1

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-05-18 10:12:45 +02:00
Ruben Fiszel 32bf061627 fix(cli): improve handling of schema for script bundles 2024-05-18 10:00:19 +02:00
HugoCasaandellipsis-dev[bot] f0b3527e53 fix(frontend): improve json display for large table objects (#3765)
* fix(frontend): fix check for displaying table-col

* fix: improve display result json limit for tables

* fix: nit

* Update frontend/src/lib/components/DisplayResult.svelte

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

* fix: nit

---------

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>
2024-05-17 18:45:13 +02:00
Ruben Fiszel ed54974ebd chore: fix tag_latest to wait for build 2024-05-17 01:05:50 +02:00
Ruben Fiszelandrubenfiszel 583e132da9 chore(main): release 1.332.0 (#3755)
* chore(main): release 1.332.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-05-17 00:35:22 +02:00
Faton RamadaniandRuben Fiszel e11a3751c3 feat(frontend): Fix App Select component initial value (#3752)
* feat(frontend): Fix App Select component initial value

* Update AppSelect.svelte

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2024-05-17 00:31:32 +02:00
Ruben Fiszel 2b06d9ae79 fix: fix args interpolation for tag and concurrency key for non string values 2024-05-17 00:27:38 +02:00
136 changed files with 3856 additions and 3908 deletions
+3 -64
View File
@@ -136,7 +136,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
${{ steps.meta-ee-public.outputs.tags }}
@@ -207,67 +207,6 @@ jobs:
${{ steps.meta-ee-public-py312.outputs.labels }}
org.opencontainers.image.licenses=Windmill-Enterprise-License
build_ee_jemalloc:
runs-on: ubicloud
if: ${{ startsWith(github.ref, 'refs/tags/') }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Read EE repo commit hash
run: |
echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
- uses: actions/checkout@v3
with:
repository: windmill-labs/windmill-ee-private
path: ./windmill-ee-private
ref: ${{ env.ee_repo_ref }}
token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
- uses: depot/setup-action@v1
- name: Docker meta
id: meta-ee-public-jemalloc
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-jemalloc
flavor: |
latest=false
tags: |
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
- name: Login to registry
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Substitute EE code
run: |
./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
- name: Build and push publicly ee
uses: depot/build-push-action@v1
with:
context: .
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc
tags: |
${{ steps.meta-ee-public-jemalloc.outputs.tags }}
labels: |
${{ steps.meta-ee-public-jemalloc.outputs.labels }}
org.opencontainers.image.licenses=Windmill-Enterprise-License
# disabled until we make it 100% reliable and add more meaningful tests
# playwright:
# runs-on: [self-hosted, new]
@@ -415,7 +354,7 @@ jobs:
tag_latest:
runs-on: ubicloud
needs: [run_integration_test]
needs: [run_integration_test, build]
steps:
- uses: actions/checkout@v3
with:
@@ -433,7 +372,7 @@ jobs:
tag_latest_ee:
runs-on: ubicloud
needs: [run_integration_test]
needs: [run_integration_test, build_ee]
steps:
- uses: actions/checkout@v3
with:
+93
View File
@@ -1,5 +1,98 @@
# Changelog
## [1.335.0](https://github.com/windmill-labs/windmill/compare/v1.334.0...v1.335.0) (2024-05-22)
### Features
* improve user audit logs and stats ([#3786](https://github.com/windmill-labs/windmill/issues/3786)) ([2f370de](https://github.com/windmill-labs/windmill/commit/2f370de62805e1291901cf19368ae4270d70a916))
### Bug Fixes
* **cli:** add concurrency key to script ([4d28a38](https://github.com/windmill-labs/windmill/commit/4d28a38750c94dc4362778ace6a227472bd4cd9b))
* **frontend:** Fix aggrid infinite refresh ([#3789](https://github.com/windmill-labs/windmill/issues/3789)) ([85cd8b4](https://github.com/windmill-labs/windmill/commit/85cd8b4c0ef7aa0b8325db2f99b31ed31a051804))
## [1.334.0](https://github.com/windmill-labs/windmill/compare/v1.333.5...v1.334.0) (2024-05-21)
### Features
* **frontend:** Added support for title and placeholder for ArgInputs ([#3779](https://github.com/windmill-labs/windmill/issues/3779)) ([59c0aba](https://github.com/windmill-labs/windmill/commit/59c0abae3df7397f32b65360b6740ca71a91444b))
* **frontend:** Improve multiselect perf ([#3770](https://github.com/windmill-labs/windmill/issues/3770)) ([36df838](https://github.com/windmill-labs/windmill/commit/36df8389cd5d75c4332e0f0a1da61433dac88195))
### Bug Fixes
* add allow http toggle to object store settings ([4631c3f](https://github.com/windmill-labs/windmill/commit/4631c3fd08341858b97a1a9673664e7220bbef63))
* **frontend:** Fix path overflow on the Runs page ([#3781](https://github.com/windmill-labs/windmill/issues/3781)) ([518d8b9](https://github.com/windmill-labs/windmill/commit/518d8b916b41ba35d623deeb27c6f6f82e171060))
## [1.333.5](https://github.com/windmill-labs/windmill/compare/v1.333.4...v1.333.5) (2024-05-21)
### Bug Fixes
* drop the linux file cache regularly to avoid triggering OOM killer ([0268dd1](https://github.com/windmill-labs/windmill/commit/0268dd1206ac3e84eec571a6ff53f921f05ed6c4))
* **frontend:** Fix full height component for public apps ([#3777](https://github.com/windmill-labs/windmill/issues/3777)) ([edf3015](https://github.com/windmill-labs/windmill/commit/edf3015cc4944340e52b5d65a2c72f421b97b548))
* **frontend:** Fix selected value when the default value of a select component is null ([#3778](https://github.com/windmill-labs/windmill/issues/3778)) ([31c4777](https://github.com/windmill-labs/windmill/commit/31c47774cd64279100a5d23a076af0519cf6ee89))
* improve concurrency key migration ([b56fbdf](https://github.com/windmill-labs/windmill/commit/b56fbdf898792c68c535ca491b79c501993173a4))
* lighten watermark on public apps for ee ([da3b043](https://github.com/windmill-labs/windmill/commit/da3b0434e92f1ce95e3e7ede07d4bce0460de9af))
## [1.333.4](https://github.com/windmill-labs/windmill/compare/v1.333.3...v1.333.4) (2024-05-20)
### Bug Fixes
* **cli:** make bun the default language in absence of wmill defaultTs 2 ([b2958be](https://github.com/windmill-labs/windmill/commit/b2958be7c10d932ae4076e0cde49fff37d5e9f41))
## [1.333.3](https://github.com/windmill-labs/windmill/compare/v1.333.2...v1.333.3) (2024-05-20)
### Bug Fixes
* **cli:** make bun the default language in absence of wmill defaultTs ([8399086](https://github.com/windmill-labs/windmill/commit/83990869bc5edb98f633944410242de19dbe5d01))
## [1.333.2](https://github.com/windmill-labs/windmill/compare/v1.333.1...v1.333.2) (2024-05-20)
### Bug Fixes
* **frontend:** fix AgGrid pagination refresh ([#3773](https://github.com/windmill-labs/windmill/issues/3773)) ([ccc2699](https://github.com/windmill-labs/windmill/commit/ccc2699d4ef18a765c3f9e968b4cd5ea1f600fa1))
## [1.333.1](https://github.com/windmill-labs/windmill/compare/v1.333.0...v1.333.1) (2024-05-20)
### Bug Fixes
* **backend:** improve memory usage by making schema RawValue instead of serde_json::Value ([f1bb7cf](https://github.com/windmill-labs/windmill/commit/f1bb7cfcbdbfee7488f59256522be18f8d2eb0a3))
## [1.333.0](https://github.com/windmill-labs/windmill/compare/v1.332.1...v1.333.0) (2024-05-19)
### Features
* reduce memory usage ([#3768](https://github.com/windmill-labs/windmill/issues/3768)) ([7a1808e](https://github.com/windmill-labs/windmill/commit/7a1808e951a2f359902138d1bb964cf52986665d))
## [1.332.1](https://github.com/windmill-labs/windmill/compare/v1.332.0...v1.332.1) (2024-05-18)
### Bug Fixes
* **cli:** improve handling of schema for script bundles ([32bf061](https://github.com/windmill-labs/windmill/commit/32bf061627f087e1ec4e88a3e03be0ca1b7fa9f7))
* **frontend:** improve json display for large table objects ([#3765](https://github.com/windmill-labs/windmill/issues/3765)) ([f0b3527](https://github.com/windmill-labs/windmill/commit/f0b3527e53e077f9c9ab4c9c6373f35cee77e566))
## [1.332.0](https://github.com/windmill-labs/windmill/compare/v1.331.2...v1.332.0) (2024-05-16)
### Features
* **frontend:** Fix App Select component initial value ([#3752](https://github.com/windmill-labs/windmill/issues/3752)) ([e11a375](https://github.com/windmill-labs/windmill/commit/e11a3751c3d188522360010a03e1fbc92311f554))
### Bug Fixes
* fix args interpolation for tag and concurrency key for non string values ([2b06d9a](https://github.com/windmill-labs/windmill/commit/2b06d9ae793f9e5fddee4ca0889634429ff53416))
## [1.331.2](https://github.com/windmill-labs/windmill/compare/v1.331.1...v1.331.2) (2024-05-16)
+2 -1
View File
@@ -3,4 +3,5 @@ target/
oauth.json
oauth2.json
windmill-api/openapi-deref.yaml
tracing.folded
tracing.folded
heaptrack*
@@ -1,70 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n app.extra_perms, app_version.value, \n app_version.created_at, app_version.created_by from app, app_version \n WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "versions",
"type_info": "Int8Array"
},
{
"ordinal": 4,
"name": "policy",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"name": "value",
"type_info": "Json"
},
{
"ordinal": 7,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 8,
"name": "created_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false
]
},
"hash": "023fffd0a042a28b5be991169a506aff92f64f84e49b4c041cd369b045c31e73"
}
@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT owner FROM workspace WHERE owner NOT LIKE '%@windmill.dev'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "owner",
"type_info": "Varchar"
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "2fa6e9c2cfc4f4ddbaa91793686feb9773b36993dd417019db51fa59fd4952c3"
}
@@ -1,71 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n app.extra_perms, app_version.value, \n app_version.created_at, app_version.created_by from app, app_version \n WHERE app_version.id = $1 AND app.id = app_version.app_id AND app.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "versions",
"type_info": "Int8Array"
},
{
"ordinal": 4,
"name": "policy",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"name": "value",
"type_info": "Json"
},
{
"ordinal": 7,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 8,
"name": "created_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false
]
},
"hash": "438fb925ee90d5115bd3c3be8ae48b56ba86017af3ca519bd3a15829edaa7d1b"
}
@@ -1,38 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT success, result, started_at FROM completed_job WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4 ORDER BY created_at DESC LIMIT $5",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "success",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "result",
"type_info": "Jsonb"
},
{
"ordinal": 2,
"name": "started_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Uuid",
"Int8"
]
},
"nullable": [
false,
true,
false
]
},
"hash": "4a3917318d967d96c4b6e70786a034302f04da9d1e9eec59b1eb6052c2d947be"
}
@@ -1,187 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry, summary, no_flow_overlap, tag ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22 ) RETURNING *",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "schedule",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "args",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "timezone",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "on_failure",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "on_recovery",
"type_info": "Varchar"
},
{
"ordinal": 15,
"name": "on_failure_times",
"type_info": "Int4"
},
{
"ordinal": 16,
"name": "on_failure_exact",
"type_info": "Bool"
},
{
"ordinal": 17,
"name": "on_failure_extra_args",
"type_info": "Json"
},
{
"ordinal": 18,
"name": "on_recovery_times",
"type_info": "Int4"
},
{
"ordinal": 19,
"name": "on_recovery_extra_args",
"type_info": "Json"
},
{
"ordinal": 20,
"name": "ws_error_handler_muted",
"type_info": "Bool"
},
{
"ordinal": 21,
"name": "retry",
"type_info": "Jsonb"
},
{
"ordinal": 22,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 23,
"name": "no_flow_overlap",
"type_info": "Bool"
},
{
"ordinal": 24,
"name": "tag",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Jsonb",
"Bool",
"Varchar",
"Varchar",
"Int4",
"Bool",
"Json",
"Varchar",
"Int4",
"Json",
"Bool",
"Jsonb",
"Varchar",
"Bool",
"Varchar"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
false,
true,
false,
true,
true,
true,
true,
true,
true,
true,
false,
true,
true,
false,
true
]
},
"hash": "597335a4a1eda7799303e0d930b5468af2364f5d2075aab48787b93e5774336d"
}
@@ -1,29 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, app_version.value from app LEFT JOIN app_version ON app_version.id = versions[array_upper(versions, 1)] WHERE workspace_id = $1 LIMIT $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "value",
"type_info": "Json"
}
],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": [
false,
false
]
},
"hash": "644335d376b6554ab222d25dfa8722234661f2456b469f21b175e8607584614e"
}
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT email FROM password WHERE super_admin IS true AND email NOT LIKE '%@windmill.dev'",
"query": "SELECT email FROM password WHERE super_admin IS true AND email NOT LIKE '%@windmill.dev' LIMIT 1",
"describe": {
"columns": [
{
@@ -16,5 +16,5 @@
false
]
},
"hash": "3dd6ba886b214274d9453667e3732421251c3a0d84b0909206447a3927bad8b3"
"hash": "71ad5448cba8da6677edd02e090cc158e09af72dd31585ed2ddc1cc06cbfd189"
}
@@ -1,182 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, on_recovery_extra_args = $10, ws_error_handler_muted = $11, retry = $12, summary = $13, no_flow_overlap = $14, tag = $15\n WHERE path = $16 AND workspace_id = $17 RETURNING *",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "schedule",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "args",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "timezone",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "on_failure",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "on_recovery",
"type_info": "Varchar"
},
{
"ordinal": 15,
"name": "on_failure_times",
"type_info": "Int4"
},
{
"ordinal": 16,
"name": "on_failure_exact",
"type_info": "Bool"
},
{
"ordinal": 17,
"name": "on_failure_extra_args",
"type_info": "Json"
},
{
"ordinal": 18,
"name": "on_recovery_times",
"type_info": "Int4"
},
{
"ordinal": 19,
"name": "on_recovery_extra_args",
"type_info": "Json"
},
{
"ordinal": 20,
"name": "ws_error_handler_muted",
"type_info": "Bool"
},
{
"ordinal": 21,
"name": "retry",
"type_info": "Jsonb"
},
{
"ordinal": 22,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 23,
"name": "no_flow_overlap",
"type_info": "Bool"
},
{
"ordinal": 24,
"name": "tag",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Jsonb",
"Varchar",
"Int4",
"Bool",
"Json",
"Varchar",
"Int4",
"Json",
"Bool",
"Jsonb",
"Varchar",
"Bool",
"Varchar",
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
false,
true,
false,
true,
true,
true,
true,
true,
true,
true,
false,
true,
true,
false,
true
]
},
"hash": "90435687152ccd37db7e0a6a5baeb558db50f5f96b1e8eff0b10e4398566f325"
}
@@ -1,168 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "schedule",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "args",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "timezone",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "on_failure",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "on_recovery",
"type_info": "Varchar"
},
{
"ordinal": 15,
"name": "on_failure_times",
"type_info": "Int4"
},
{
"ordinal": 16,
"name": "on_failure_exact",
"type_info": "Bool"
},
{
"ordinal": 17,
"name": "on_failure_extra_args",
"type_info": "Json"
},
{
"ordinal": 18,
"name": "on_recovery_times",
"type_info": "Int4"
},
{
"ordinal": 19,
"name": "on_recovery_extra_args",
"type_info": "Json"
},
{
"ordinal": 20,
"name": "ws_error_handler_muted",
"type_info": "Bool"
},
{
"ordinal": 21,
"name": "retry",
"type_info": "Jsonb"
},
{
"ordinal": 22,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 23,
"name": "no_flow_overlap",
"type_info": "Bool"
},
{
"ordinal": 24,
"name": "tag",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
false,
true,
false,
true,
true,
true,
true,
true,
true,
true,
false,
true,
true,
false,
true
]
},
"hash": "911b1e1f2a5ba6d5159916e5598020e680c45043b0736ad0153ee261a151dd90"
}
@@ -1,169 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE schedule SET enabled = $1, email = $2 WHERE path = $3 AND workspace_id = $4 RETURNING *",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "schedule",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "args",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "timezone",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "on_failure",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "on_recovery",
"type_info": "Varchar"
},
{
"ordinal": 15,
"name": "on_failure_times",
"type_info": "Int4"
},
{
"ordinal": 16,
"name": "on_failure_exact",
"type_info": "Bool"
},
{
"ordinal": 17,
"name": "on_failure_extra_args",
"type_info": "Json"
},
{
"ordinal": 18,
"name": "on_recovery_times",
"type_info": "Int4"
},
{
"ordinal": 19,
"name": "on_recovery_extra_args",
"type_info": "Json"
},
{
"ordinal": 20,
"name": "ws_error_handler_muted",
"type_info": "Bool"
},
{
"ordinal": 21,
"name": "retry",
"type_info": "Jsonb"
},
{
"ordinal": 22,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 23,
"name": "no_flow_overlap",
"type_info": "Bool"
},
{
"ordinal": 24,
"name": "tag",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Bool",
"Varchar",
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
false,
true,
false,
true,
true,
true,
true,
true,
true,
true,
false,
true,
true,
false,
true
]
},
"hash": "96dc1cd308f3c21cfb50b88048054dc03f93e261d25969d66aa48e9d0502960f"
}
@@ -1,71 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n app.extra_perms, app_version.value, \n app_version.created_at, app_version.created_by from app, app_version \n WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "versions",
"type_info": "Int8Array"
},
{
"ordinal": 4,
"name": "policy",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"name": "value",
"type_info": "Json"
},
{
"ordinal": 7,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 8,
"name": "created_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false
]
},
"hash": "9ae98fbcea508dfc7113621c00b856f98f38dfb701a4660ea1a4058a6d7564f2"
}
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT concurrency_key FROM script WHERE hash = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "concurrency_key",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8",
"Text"
]
},
"nullable": [
true
]
},
"hash": "a3ccf362b4f6df400b3c7a084795dbf541eb14c5c374656ffb96da7283a2a6f1"
}
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value->>'concurrency_key' FROM flow WHERE path = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "a875cb56485b812e9d4739afd0915067f7e5abe0ca0adf264b792fccf21e005b"
}
@@ -1,167 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT * FROM schedule WHERE path = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "schedule",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "args",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "timezone",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "on_failure",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "on_recovery",
"type_info": "Varchar"
},
{
"ordinal": 15,
"name": "on_failure_times",
"type_info": "Int4"
},
{
"ordinal": 16,
"name": "on_failure_exact",
"type_info": "Bool"
},
{
"ordinal": 17,
"name": "on_failure_extra_args",
"type_info": "Json"
},
{
"ordinal": 18,
"name": "on_recovery_times",
"type_info": "Int4"
},
{
"ordinal": 19,
"name": "on_recovery_extra_args",
"type_info": "Json"
},
{
"ordinal": 20,
"name": "ws_error_handler_muted",
"type_info": "Bool"
},
{
"ordinal": 21,
"name": "retry",
"type_info": "Jsonb"
},
{
"ordinal": 22,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 23,
"name": "no_flow_overlap",
"type_info": "Bool"
},
{
"ordinal": 24,
"name": "tag",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
false,
true,
false,
true,
true,
true,
true,
true,
true,
true,
false,
true,
true,
false,
true
]
},
"hash": "aa2800113a8a8805f47cdc1dd0f29d94c546fe531e7edd3e91da4978af5442fb"
}
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "WITH active_users as (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login')),\n active_user_count as (SELECT count(email)::INT as count from active_users),\n active_author_count as (SELECT count(distinct email)::INT as count FROM usr WHERE usr.operator IS false AND email IN (SELECT email FROM active_users))\n SELECT active_author_count.count as author_count, active_user_count.count - active_author_count.count as operator_count FROM active_user_count, active_author_count",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "author_count",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "operator_count",
"type_info": "Int4"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "ad8d9c00bd2d3515ad4afa87aa24b243615484bbad5a793438e23908c3f20c2f"
}
@@ -1,83 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n app.extra_perms, app_version.value, \n app_version.created_at, app_version.created_by,\n app.draft_only, draft.value as \"draft?\"\n from app\n INNER JOIN app_version ON\n app_version.id = app.versions[array_upper(app.versions, 1)]\n LEFT JOIN draft ON \n app.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'app' \n WHERE app.path = $1 AND app.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "versions",
"type_info": "Int8Array"
},
{
"ordinal": 4,
"name": "policy",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"name": "value",
"type_info": "Json"
},
{
"ordinal": 7,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 8,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "draft_only",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "draft?",
"type_info": "Json"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false,
true,
false
]
},
"hash": "aedba9cff117a7069dfd56602110b7f66903d6afb4f87fbffe0175982ea8741f"
}
@@ -18,8 +18,8 @@
"Left": []
},
"nullable": [
false,
true
true,
false
]
},
"hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76"
@@ -1,168 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS false RETURNING *",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "schedule",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "args",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "timezone",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "on_failure",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "on_recovery",
"type_info": "Varchar"
},
{
"ordinal": 15,
"name": "on_failure_times",
"type_info": "Int4"
},
{
"ordinal": 16,
"name": "on_failure_exact",
"type_info": "Bool"
},
{
"ordinal": 17,
"name": "on_failure_extra_args",
"type_info": "Json"
},
{
"ordinal": 18,
"name": "on_recovery_times",
"type_info": "Int4"
},
{
"ordinal": 19,
"name": "on_recovery_extra_args",
"type_info": "Json"
},
{
"ordinal": 20,
"name": "ws_error_handler_muted",
"type_info": "Bool"
},
{
"ordinal": 21,
"name": "retry",
"type_info": "Jsonb"
},
{
"ordinal": 22,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 23,
"name": "no_flow_overlap",
"type_info": "Bool"
},
{
"ordinal": 24,
"name": "tag",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
false,
true,
false,
true,
true,
true,
true,
true,
true,
true,
false,
true,
true,
false,
true
]
},
"hash": "c10348d26e3e3dc9e345d9044db35db9906617eb9c98aaf58a55d9681ffb85e3"
}
@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH all_users as (SELECT count(*)::INT as count FROM usr WHERE disabled IS false),\n authors as (SELECT count(distinct email)::INT as count FROM usr WHERE usr.operator IS false AND disabled IS false)\n SELECT authors.count as author_count, all_users.count - authors.count as operator_count FROM all_users, authors",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "author_count",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "operator_count",
"type_info": "Int4"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "c584aeec21716f7405c355d5684ff3ac1be3f70849d67b8c2d4787e809b7ea2b"
}
@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) FROM worker_ping WHERE ping_at > NOW() - INTERVAL '2 minutes'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "cdfc8e9af4414b7ee7525bc74d9caad74b5479c64337c401a7a9550a3434cdfa"
}
@@ -1,71 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n app.extra_perms, app_version.value, \n app_version.created_at, app_version.created_by from app, app_version \n WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "versions",
"type_info": "Int8Array"
},
{
"ordinal": 4,
"name": "policy",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"name": "value",
"type_info": "Json"
},
{
"ordinal": 7,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 8,
"name": "created_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false
]
},
"hash": "d07faf1c9501d1bc5f6fd9588bb3da7b7f4897fcfd26dcb3b8ce406b1f653077"
}
@@ -1,168 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS false RETURNING *",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "schedule",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "args",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "timezone",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "on_failure",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "on_recovery",
"type_info": "Varchar"
},
{
"ordinal": 15,
"name": "on_failure_times",
"type_info": "Int4"
},
{
"ordinal": 16,
"name": "on_failure_exact",
"type_info": "Bool"
},
{
"ordinal": 17,
"name": "on_failure_extra_args",
"type_info": "Json"
},
{
"ordinal": 18,
"name": "on_recovery_times",
"type_info": "Int4"
},
{
"ordinal": 19,
"name": "on_recovery_extra_args",
"type_info": "Json"
},
{
"ordinal": 20,
"name": "ws_error_handler_muted",
"type_info": "Bool"
},
{
"ordinal": 21,
"name": "retry",
"type_info": "Jsonb"
},
{
"ordinal": 22,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 23,
"name": "no_flow_overlap",
"type_info": "Bool"
},
{
"ordinal": 24,
"name": "tag",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
false,
true,
false,
true,
true,
true,
true,
true,
true,
true,
false,
true,
true,
false,
true
]
},
"hash": "dd74fa9468b5fe8c8ad657ded06076c11a78d0206af2af20685c691d4d9520bb"
}
@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT author.count as author_count, operator.count as operator_count FROM (SELECT count(*)::INT FROM usr where usr.operator IS false) as author, (SELECT count(*)::INT FROM usr where usr.operator IS true) as operator",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "author_count",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "operator_count",
"type_info": "Int4"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "e70835de2b70a01a2e85f44f3cb80f41288d8e68b762de80ae724ed69ef1908c"
}
@@ -1,168 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "schedule",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "args",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "timezone",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "on_failure",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "on_recovery",
"type_info": "Varchar"
},
{
"ordinal": 15,
"name": "on_failure_times",
"type_info": "Int4"
},
{
"ordinal": 16,
"name": "on_failure_exact",
"type_info": "Bool"
},
{
"ordinal": 17,
"name": "on_failure_extra_args",
"type_info": "Json"
},
{
"ordinal": 18,
"name": "on_recovery_times",
"type_info": "Int4"
},
{
"ordinal": 19,
"name": "on_recovery_extra_args",
"type_info": "Json"
},
{
"ordinal": 20,
"name": "ws_error_handler_muted",
"type_info": "Bool"
},
{
"ordinal": 21,
"name": "retry",
"type_info": "Jsonb"
},
{
"ordinal": 22,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 23,
"name": "no_flow_overlap",
"type_info": "Bool"
},
{
"ordinal": 24,
"name": "tag",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
false,
true,
false,
true,
true,
true,
true,
true,
true,
true,
false,
true,
true,
false,
true
]
},
"hash": "eced0a09ba547ce1dccb54a5419b22373603c9d01f77047b3553bde125bf71e8"
}
@@ -1,29 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, value from flow WHERE workspace_id = $1 LIMIT $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "value",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": [
false,
false
]
},
"hash": "efbd9dc28ab5e53d070684b2e99d945872791171a1c8cef6088ea662f3b2cebb"
}
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT payload\n FROM capture\n WHERE workspace_id = $1\n AND path = $2\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "payload",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "f4960efbb4189b595541608c9ab5cbbd56eb42059e961b6338b685105626a5c7"
}
@@ -1,166 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT * FROM schedule\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "schedule",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "args",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "timezone",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "on_failure",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "on_recovery",
"type_info": "Varchar"
},
{
"ordinal": 15,
"name": "on_failure_times",
"type_info": "Int4"
},
{
"ordinal": 16,
"name": "on_failure_exact",
"type_info": "Bool"
},
{
"ordinal": 17,
"name": "on_failure_extra_args",
"type_info": "Json"
},
{
"ordinal": 18,
"name": "on_recovery_times",
"type_info": "Int4"
},
{
"ordinal": 19,
"name": "on_recovery_extra_args",
"type_info": "Json"
},
{
"ordinal": 20,
"name": "ws_error_handler_muted",
"type_info": "Bool"
},
{
"ordinal": 21,
"name": "retry",
"type_info": "Jsonb"
},
{
"ordinal": 22,
"name": "summary",
"type_info": "Varchar"
},
{
"ordinal": 23,
"name": "no_flow_overlap",
"type_info": "Bool"
},
{
"ordinal": 24,
"name": "tag",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
false,
true,
false,
true,
true,
true,
true,
true,
true,
true,
false,
true,
true,
false,
true
]
},
"hash": "fdd3710a381dac33ef5ee5ec5564a4874e6d7807d43fa6ea7b8408cad9e97480"
}
+165 -152
View File
File diff suppressed because it is too large Load Diff
+6 -4
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.331.2"
version = "1.335.0"
authors.workspace = true
edition.workspace = true
@@ -24,7 +24,7 @@ members = [
]
[workspace.package]
version = "1.331.2"
version = "1.335.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -51,7 +51,7 @@ prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill
flow_testing = ["windmill-worker/flow_testing"]
openidconnect = ["windmill-api/openidconnect"]
cloud = ["windmill-queue/cloud"]
jemalloc = ["dep:tikv-jemallocator"]
jemalloc = ["dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
[dependencies]
anyhow.workspace = true
@@ -84,7 +84,9 @@ deno_core.workspace = true
pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false, features = ['rt_tokio']}
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = { optional = true, version = "0.5" }
tikv-jemallocator = { optional = true, version = "0.5", features = ["profiling"] }
tikv-jemalloc-sys = { optional = true, version = "^0.5", features = ["profiling"] }
tikv-jemalloc-ctl = { optional = true, version = "^0.5" }
[dev-dependencies]
serde_json.workspace = true
+1 -1
View File
@@ -1 +1 @@
e0b0494c26b63efa4aa3f41a9fb42c7733dcca2e
229b7acc60691f7e6577c44f782676d6871defdd
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+22 -18
View File
@@ -32,6 +32,9 @@ use windmill_common::{
DB, METRICS_ENABLED,
};
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
use monitor::monitor_mem;
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
use tikv_jemallocator::Jemalloc;
@@ -52,13 +55,13 @@ use windmill_worker::{
};
use crate::monitor::{
initial_load, load_keep_job_dir, load_require_preexisting_user, load_tag_per_workspace_enabled,
monitor_db, monitor_pool, reload_base_url_setting, reload_bunfig_install_scopes_setting,
reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_license_key,
reload_npm_config_registry_setting, reload_pip_index_url_setting,
reload_retention_period_setting, reload_scim_token_setting, reload_server_config,
reload_worker_config,
initial_load, load_keep_job_dir, load_metrics_debug_enabled, load_require_preexisting_user,
load_tag_per_workspace_enabled, monitor_db, monitor_pool, reload_base_url_setting,
reload_bunfig_install_scopes_setting, reload_critical_error_channels_setting,
reload_extra_pip_index_url_setting, reload_hub_base_url_setting,
reload_job_default_timeout_setting, reload_license_key, reload_npm_config_registry_setting,
reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting,
reload_server_config, reload_worker_config,
};
#[cfg(feature = "parquet")]
@@ -321,6 +324,9 @@ Windmill Community Edition {GIT_VERSION}
monitor_pool(&db).await;
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
monitor_mem().await;
let addr = SocketAddr::from((server_bind_address, port));
let rsmq2 = rsmq.clone();
@@ -481,8 +487,8 @@ Windmill Community Edition {GIT_VERSION}
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => {
load_require_preexisting_user(&db).await;
},
EXPOSE_METRICS_SETTING | EXPOSE_DEBUG_METRICS_SETTING => {
if n.payload() != EXPOSE_DEBUG_METRICS_SETTING || worker_mode {
EXPOSE_METRICS_SETTING => {
if worker_mode {
tracing::info!("Metrics setting changed, restarting");
// we wait a bit randomly to avoid having all serverss and workers shutdown at same time
let rd_delay = rand::thread_rng().gen_range(0..4);
@@ -492,6 +498,11 @@ Windmill Community Edition {GIT_VERSION}
}
}
},
EXPOSE_DEBUG_METRICS_SETTING => {
if let Err(e) = load_metrics_debug_enabled(&db).await {
tracing::error!(error = %e, "Could not reload debug metrics setting");
}
},
REQUEST_SIZE_LIMIT_SETTING => {
if server_mode {
tracing::info!("Request limit size change detected, killing server expecting to be restarted");
@@ -564,15 +575,8 @@ Windmill Community Edition {GIT_VERSION}
};
let instance_name = rd_string(8);
if mode != Mode::Agent {
schedule_stats(
instance_name,
mode.clone(),
&db,
&HTTP_CLIENT,
cfg!(feature = "enterprise"),
)
.await;
if mode == Mode::Server || mode == Mode::Standalone {
schedule_stats(instance_name, &db, &HTTP_CLIENT).await;
}
futures::try_join!(shutdown_signal, workers_f, monitor_f, server_f, metrics_f)?;
+79 -25
View File
@@ -22,10 +22,7 @@ use windmill_api::{
DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN,
};
use windmill_common::{
ee::CriticalErrorChannel,
error,
flow_status::FlowStatusModule,
global_settings::{
ee::CriticalErrorChannel, error, flow_status::FlowStatusModule, global_settings::{
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
DEFAULT_TAGS_PER_WORKSPACE_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
@@ -33,17 +30,10 @@ use windmill_common::{
PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
},
jobs::QueuedJob,
oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH,
server::load_server_config,
users::truncate_token,
worker::{
}, jobs::QueuedJob, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_server_config, stats_ee::get_user_usage, users::truncate_token, worker::{
load_worker_config, reload_custom_tags_setting, DEFAULT_TAGS_PER_WORKSPACE, SERVER_CONFIG,
WORKER_CONFIG,
},
BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL,
METRICS_DEBUG_ENABLED, METRICS_ENABLED,
}, BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, METRICS_DEBUG_ENABLED, METRICS_ENABLED
};
use windmill_queue::cancel_job;
use windmill_worker::{
@@ -201,12 +191,83 @@ pub async fn load_tag_per_workspace_enabled(db: &DB) -> error::Result<()> {
pub async fn load_metrics_debug_enabled(db: &DB) -> error::Result<()> {
let metrics_enabled = load_value_from_global_settings(db, EXPOSE_DEBUG_METRICS_SETTING).await;
match metrics_enabled {
Ok(Some(serde_json::Value::Bool(t))) => METRICS_DEBUG_ENABLED.store(t, Ordering::Relaxed),
Ok(Some(serde_json::Value::Bool(t))) => {
METRICS_DEBUG_ENABLED.store(t, Ordering::Relaxed);
//_RJEM_MALLOC_CONF=prof:true,prof_active:false,lg_prof_interval:30,lg_prof_sample:21,prof_prefix:/tmp/jeprof
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
if std::env::var("_RJEM_MALLOC_CONF").is_ok() {
if let Err(e) = set_prof_active(t) {
tracing::error!("Error setting jemalloc prof_active: {e:?}");
}
}
},
_ => (),
};
Ok(())
}
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
#[derive(Debug, Clone)]
pub struct MallctlError { pub code: i32 }
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
fn set_prof_active(new_value: bool) -> Result<(), MallctlError> {
let option_name = std::ffi::CString::new("prof.active").unwrap();
tracing::info!("Setting jemalloc prof_active to {}", new_value);
let result = unsafe {
tikv_jemalloc_sys::mallctl(
option_name.as_ptr(), // const char *name
std::ptr::null_mut(), // void *oldp
std::ptr::null_mut(), // size_t *oldlenp
&new_value as *const _ as *mut _, // void *newp
std::mem::size_of_val(&new_value) // size_t newlen
)
};
if result != 0 {
return Err(MallctlError { code: result });
}
Ok(())
}
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
pub fn bytes_to_mb(bytes: u64) -> f64 {
const BYTES_PER_MB: f64 = 1_048_576.0;
bytes as f64 / BYTES_PER_MB
}
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
pub async fn monitor_mem() {
use std::time::Duration;
use tikv_jemalloc_ctl::{stats, epoch};
tokio::spawn(async move {
// Obtain a MIB for the `epoch`, `stats.allocated`, and
// `atats.resident` keys:
let e = epoch::mib().unwrap();
let allocated = stats::allocated::mib().unwrap();
let resident = stats::resident::mib().unwrap();
loop {
// Many statistics are cached and only updated
// when the epoch is advanced:
e.advance().unwrap();
// Read statistics using MIB key:
let allocated = allocated.read().unwrap();
let resident = resident.read().unwrap();
tracing::info!("{} mb allocated/{} mb resident", bytes_to_mb(allocated as u64), bytes_to_mb(resident as u64));
tokio::time::sleep(Duration::from_secs(10)).await;
}
});
}
pub async fn load_keep_job_dir(db: &DB) {
let value = load_value_from_global_settings(db, KEEP_JOB_DIR_SETTING).await;
match value {
@@ -794,20 +855,13 @@ pub async fn save_usage_metrics(db: &Pool<Postgres>) {
.map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::hours(24) - chrono::Duration::minutes(random_nb % 60))
.unwrap_or(true)
{
let counts = sqlx::query!(
"WITH all_users as (SELECT count(*)::INT as count FROM usr WHERE disabled IS false),
authors as (SELECT count(distinct email)::INT as count FROM usr WHERE usr.operator IS false AND disabled IS false)
SELECT authors.count as author_count, all_users.count - authors.count as operator_count FROM all_users, authors"
)
.fetch_one(&mut *tx)
.await
.ok();
let user_usage = get_user_usage(&mut *tx).await.ok();
if let Some(counts) = counts {
if let Some(user_usage) = user_usage {
sqlx::query!(
"INSERT INTO metrics (id, value) VALUES ('author_count', $1), ('operator_count', $2)",
serde_json::json!(counts.author_count),
serde_json::json!(counts.operator_count)
serde_json::json!(user_usage.author_count.unwrap_or(0)),
serde_json::json!(user_usage.operator_count.unwrap_or(0))
)
.execute(&mut *tx)
.await
+24 -21
View File
@@ -8,7 +8,6 @@ use futures::StreamExt;
use futures::{stream, Stream};
use serde::Deserialize;
use serde_json::json;
use sqlx::types::Json;
use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres};
#[cfg(feature = "enterprise")]
@@ -868,13 +867,17 @@ impl RunJob {
async fn push(self, db: &Pool<Postgres>) -> Uuid {
let RunJob { payload, args } = self;
let mut hm_args = std::collections::HashMap::new();
for (k, v) in args {
hm_args.insert(k, windmill_common::worker::to_raw_value(&v));
}
let tx = PushIsolationLevel::IsolatedRoot(db.clone(), None);
let (uuid, tx) = windmill_queue::push::<_, rsmq_async::MultiplexedRsmq>(
let (uuid, tx) = windmill_queue::push::<rsmq_async::MultiplexedRsmq>(
&db,
tx,
"test-workspace",
payload,
Json(args),
hm_args.into(),
/* user */ "test-user",
/* email */ "test@windmill.dev",
/* permissioned_as */ "u/test-user".to_string(),
@@ -1070,7 +1073,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
custom_concurrency_key: None,
concurrent_limit: None,
concurrency_time_window_s: None,
},
}.into(),
stop_after_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
@@ -1108,7 +1111,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
custom_concurrency_key: None,
concurrent_limit: None,
concurrency_time_window_s: None,
},
}.into(),
stop_after_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
@@ -1121,7 +1124,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
delete_after_use: None,
continue_on_error: None,
}],
},
}.into(),
stop_after_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
@@ -1207,12 +1210,12 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
input_transforms: [
(
"loop".to_string(),
InputTransform::Static { value: json!(false) },
InputTransform::Static { value: windmill_common::worker::to_raw_value(&false) },
),
("i".to_string(), InputTransform::Static { value: json!(1) }),
("i".to_string(), InputTransform::Static { value: windmill_common::worker::to_raw_value(&1) }),
(
"path".to_string(),
InputTransform::Static { value: json!("outer.txt") },
InputTransform::Static { value: windmill_common::worker::to_raw_value(&"outer.txt") },
),
]
.into(),
@@ -1224,7 +1227,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
custom_concurrency_key: None,
concurrent_limit: None,
concurrency_time_window_s: None,
},
}.into(),
stop_after_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
@@ -1240,7 +1243,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
FlowModule {
id: "b".to_string(),
value: FlowModuleValue::ForloopFlow {
iterator: InputTransform::Static { value: json!([1, 2, 3]) },
iterator: InputTransform::Static { value: windmill_common::worker::to_raw_value(&[1, 2, 3]) },
skip_failures: false,
parallel: false,
parallelism: None,
@@ -1257,11 +1260,11 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
),
(
"loop".to_string(),
InputTransform::Static { value: json!(true) },
InputTransform::Static { value: windmill_common::worker::to_raw_value(&true) },
),
(
"path".to_string(),
InputTransform::Static { value: json!("inner.txt") },
InputTransform::Static { value: windmill_common::worker::to_raw_value(&"inner.txt") },
),
]
.into(),
@@ -1273,7 +1276,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
custom_concurrency_key: None,
concurrent_limit: None,
concurrency_time_window_s: None,
},
}.into(),
stop_after_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
@@ -1291,10 +1294,10 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
value: FlowModuleValue::RawScript {
input_transforms: [(
"path".to_string(),
InputTransform::Static { value: json!("inner.txt") },
InputTransform::Static { value: windmill_common::worker::to_raw_value(&"inner.txt") },
), (
"path2".to_string(),
InputTransform::Static { value: json!("outer.txt") },
InputTransform::Static { value: windmill_common::worker::to_raw_value(&"outer.txt") },
)]
.into(),
language: ScriptLang::Deno,
@@ -1308,7 +1311,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
custom_concurrency_key: None,
concurrent_limit: None,
concurrency_time_window_s: None,
},
}.into(),
stop_after_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
@@ -1323,7 +1326,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
},
],
},
}.into(),
stop_after_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
@@ -1346,11 +1349,11 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
),
(
"path".to_string(),
InputTransform::Static { value: json!("outer.txt") },
InputTransform::Static { value: windmill_common::worker::to_raw_value(&"outer.txt") },
),
(
"path2".to_string(),
InputTransform::Static { value: json!("inner.txt") },
InputTransform::Static { value: windmill_common::worker::to_raw_value(&"inner.txt") },
),
]
.into(),
@@ -1365,7 +1368,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
custom_concurrency_key: None,
concurrent_limit: None,
concurrency_time_window_s: None,
},
}.into(),
stop_after_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
+27 -33
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.331.2
version: 1.335.0
title: Windmill API
contact:
@@ -8005,12 +8005,13 @@ paths:
application/json:
schema:
type: string
/w/{workspace}/concurrency_groups/intervals:
/w/{workspace}/concurrency_groups/list_jobs:
get:
summary: Get intervals of job runtime concurrency
operationId: getConcurrencyIntervals
operationId: listExtendedJobs
tags:
- concurrencyGroups
- job
parameters:
- name: concurrency_key
in: query
@@ -8078,7 +8079,7 @@ paths:
content:
application/json:
schema:
$ref: "#/components/schemas/ConcurrencyIntervals"
$ref: "#/components/schemas/ExtendedJobs"
components:
securitySchemes:
@@ -8516,6 +8517,8 @@ components:
type: integer
concurrency_time_window_s:
type: integer
concurrency_key:
type: string
cache_ttl:
type: number
dedicated_worker:
@@ -8955,6 +8958,16 @@ components:
- visible_to_owner
- tag
ObscuredJob:
type: object
properties:
typ:
type: string
started_at:
type: string
format: date-time
duration_ms:
type: number
Job:
oneOf:
- allOf:
@@ -10492,39 +10505,20 @@ components:
- concurrency_key
- total_running
ConcurrencyIntervals:
ExtendedJobs:
type: object
properties:
concurrency_key:
type: string
running_jobs:
jobs:
type: array
items:
type: object
properties:
job_id:
type: string
concurrency_key:
type: string
started_at:
type: string
format: date-time
completed_jobs:
$ref: "#/components/schemas/Job"
obscured_jobs:
type: array
items:
type: object
properties:
job_id:
type: string
concurrency_key:
type: string
started_at:
type: string
format: date-time
ended_at:
type: string
format: date-time
$ref : "#/components/schemas/ObscuredJob"
omitted_obscured_jobs:
description: "Obscured jobs omitted for security because of too specific filtering"
type: boolean
required:
- concurrency_key
- running_jobs
- completed_jobs
- jobs
- obscured_jobs
+42 -48
View File
@@ -39,6 +39,7 @@ use windmill_common::{
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath,
},
variables::build_crypt,
worker::to_raw_value,
HUB_BASE_URL,
};
@@ -94,37 +95,37 @@ pub struct ListableApp {
pub struct AppVersion {
pub id: i64,
pub app_id: Uuid,
pub value: serde_json::Value,
pub value: sqlx::types::Json<Box<RawValue>>,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
}
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, FromRow)]
pub struct AppWithLastVersion {
pub id: i64,
pub path: String,
pub summary: String,
pub policy: serde_json::Value,
pub policy: sqlx::types::Json<Box<RawValue>>,
pub versions: Vec<i64>,
pub value: serde_json::Value,
pub value: sqlx::types::Json<Box<RawValue>>,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub extra_perms: serde_json::Value,
pub extra_perms: Option<serde_json::Value>,
}
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, FromRow)]
pub struct AppWithLastVersionAndDraft {
pub id: i64,
pub path: String,
pub summary: String,
pub policy: serde_json::Value,
pub policy: sqlx::types::Json<Box<RawValue>>,
pub versions: Vec<i64>,
pub value: serde_json::Value,
pub value: sqlx::types::Json<Box<RawValue>>,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub extra_perms: serde_json::Value,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft: Option<serde_json::Value>,
pub draft: Option<sqlx::types::Json<Box<RawValue>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_only: Option<bool>,
}
@@ -178,7 +179,7 @@ pub struct Policy {
pub struct CreateApp {
pub path: String,
pub summary: String,
pub value: serde_json::Value,
pub value: sqlx::types::Json<Box<RawValue>>,
pub policy: Policy,
pub draft_only: Option<bool>,
pub deployment_message: Option<String>,
@@ -188,7 +189,7 @@ pub struct CreateApp {
pub struct EditApp {
pub path: Option<String>,
pub summary: Option<String>,
pub value: Option<serde_json::Value>,
pub value: Option<sqlx::types::Json<Box<RawValue>>>,
pub policy: Option<Policy>,
pub deployment_message: Option<String>,
}
@@ -196,7 +197,7 @@ pub struct EditApp {
#[derive(Serialize, FromRow)]
pub struct SearchApp {
path: String,
value: serde_json::Value,
value: sqlx::types::Json<Box<RawValue>>,
}
async fn list_search_apps(
authed: ApiAuthed,
@@ -210,12 +211,11 @@ async fn list_search_apps(
let n = 3;
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query_as!(
SearchApp,
let rows = sqlx::query_as::<_, SearchApp>(
"SELECT path, app_version.value from app LEFT JOIN app_version ON app_version.id = versions[array_upper(versions, 1)] WHERE workspace_id = $1 LIMIT $2",
&w_id,
n
)
.bind(&w_id)
.bind(n)
.fetch_all(&mut *tx)
.await?
.into_iter()
@@ -293,15 +293,14 @@ async fn get_app(
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let app_o = sqlx::query_as!(
AppWithLastVersion,
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by from app, app_version
WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
path.to_owned(),
&w_id
)
.bind(path.to_owned())
.bind(&w_id)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
@@ -318,21 +317,20 @@ async fn get_app_w_draft(
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let app_o = sqlx::query_as!(
AppWithLastVersionAndDraft,
let app_o = sqlx::query_as::<_, AppWithLastVersionAndDraft>(
r#"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by,
app.draft_only, draft.value as "draft?"
app.draft_only, draft.value as "draft"
from app
INNER JOIN app_version ON
app_version.id = app.versions[array_upper(app.versions, 1)]
LEFT JOIN draft ON
app.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'app'
WHERE app.path = $1 AND app.workspace_id = $2"#,
path.to_owned(),
&w_id
)
.bind(path.to_owned())
.bind(&w_id)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
@@ -406,15 +404,14 @@ async fn get_app_by_id(
) -> JsonResult<AppWithLastVersion> {
let mut tx = user_db.begin(&authed).await?;
let app_o = sqlx::query_as!(
AppWithLastVersion,
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by from app, app_version
WHERE app_version.id = $1 AND app.id = app_version.app_id AND app.workspace_id = $2",
id,
&w_id
)
.bind(&id)
.bind(&w_id)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
@@ -438,22 +435,20 @@ async fn get_public_app_by_secret(
let id: i64 = bytes.parse().map_err(to_anyhow)?;
let app_o = sqlx::query_as!(
AppWithLastVersion,
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, app_version.value,
null as extra_perms, app_version.value,
app_version.created_at, app_version.created_by from app, app_version
WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
id,
&w_id
)
WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]")
.bind(&id)
.bind(&w_id)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
let app = not_found_if_none(app_o, "App", id.to_string())?;
let policy = serde_json::from_value::<Policy>(app.policy.clone()).map_err(to_anyhow)?;
let policy = serde_json::from_str::<Policy>(app.policy.0.get()).map_err(to_anyhow)?;
if !matches!(policy.execution_mode, ExecutionMode::Anonymous) {
return Err(Error::NotAuthorized(
@@ -599,9 +594,9 @@ async fn create_app(
)
.await?;
let mut args: HashMap<String, serde_json::Value> = HashMap::new();
let mut args: HashMap<String, Box<serde_json::value::RawValue>> = HashMap::new();
if let Some(dm) = app.deployment_message {
args.insert("deployment_message".to_string(), json!(dm));
args.insert("deployment_message".to_string(), to_raw_value(&dm));
}
let tx = PushIsolationLevel::Transaction(tx);
@@ -610,7 +605,7 @@ async fn create_app(
tx,
&w_id,
JobPayload::AppDependencies { path: app.path.clone(), version: v_id },
args,
PushArgs { args, extra: HashMap::new() },
&authed.username,
&authed.email,
windmill_common::users::username_to_permissioned_as(&authed.username),
@@ -655,7 +650,7 @@ async fn list_hub_apps(Extension(db): Extension<DB>) -> impl IntoResponse {
pub async fn get_hub_app_by_id(
Path(id): Path<i32>,
Extension(db): Extension<DB>,
) -> JsonResult<serde_json::Value> {
) -> JsonResult<Box<serde_json::value::RawValue>> {
let value = http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/apps/{}/json", *HUB_BASE_URL.read().await, id),
@@ -889,18 +884,18 @@ async fn update_app(
let tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
PushIsolationLevel::Transaction(tx);
let mut args: HashMap<String, serde_json::Value> = HashMap::new();
let mut args: HashMap<String, Box<serde_json::value::RawValue>> = HashMap::new();
if let Some(dm) = ns.deployment_message {
args.insert("deployment_message".to_string(), json!(dm));
args.insert("deployment_message".to_string(), to_raw_value(&dm));
}
args.insert("parent_path".to_string(), json!(path));
args.insert("parent_path".to_string(), to_raw_value(&path));
let (dependency_job_uuid, new_tx) = push(
&db,
tx,
&w_id,
JobPayload::AppDependencies { path: npath.clone(), version: v_id },
args,
PushArgs { args, extra: HashMap::new() },
&authed.username,
&authed.email,
windmill_common::users::username_to_permissioned_as(&authed.username),
@@ -1160,8 +1155,7 @@ fn build_args(
component: &str,
path: String,
args: HashMap<String, Box<RawValue>>,
) -> Result<PushArgs<HashMap<String, Box<RawValue>>>> {
) -> Result<PushArgs> {
let key = format!("{}:{}", component, &path);
let (static_inputs, one_of_inputs) = match policy {
Policy { triggerables_v2: Some(t), .. } => {
@@ -1276,5 +1270,5 @@ fn build_args(
for (k, v) in static_inputs {
extra.insert(k.to_string(), v.to_owned());
}
Ok(PushArgs { extra, args: sqlx::types::Json(safe_args) })
Ok(PushArgs { extra, args: safe_args })
}
+11 -10
View File
@@ -6,15 +6,12 @@
* LICENSE-AGPL for a copy of the license.
*/
use std::collections::HashMap;
use axum::{
extract::{Extension, Path},
routing::{get, post, put},
Router,
};
use hyper::StatusCode;
use serde_json::value::RawValue;
use sqlx::types::Json;
use windmill_common::{
db::UserDB,
@@ -89,7 +86,7 @@ pub async fn new_payload(
pub async fn update_payload(
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
args: PushArgs<HashMap<String, Box<RawValue>>>,
args: PushArgs,
) -> Result<StatusCode> {
let mut tx = db.begin().await?;
@@ -102,7 +99,7 @@ pub async fn update_payload(
",
&w_id,
&path.to_path(),
Json(args) as Json<PushArgs<HashMap<String, Box<RawValue>>>>,
Json(args) as Json<PushArgs>,
)
.execute(&mut *tx)
.await?;
@@ -112,27 +109,31 @@ pub async fn update_payload(
Ok(StatusCode::NO_CONTENT)
}
#[derive(sqlx::FromRow)]
struct Payload {
payload: sqlx::types::Json<Box<serde_json::value::RawValue>>,
}
pub async fn get_payload(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<serde_json::Value> {
) -> JsonResult<Box<serde_json::value::RawValue>> {
let mut tx = user_db.begin(&authed).await?;
let payload = sqlx::query_scalar!(
let payload = sqlx::query_as::<_, Payload>(
"
SELECT payload
FROM capture
WHERE workspace_id = $1
AND path = $2
",
&w_id,
&path.to_path(),
)
.bind(&w_id)
.bind(&path.to_path())
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
not_found_if_none(payload, "capture", path.to_path()).map(axum::Json)
not_found_if_none(payload.map(|x| x.payload.0), "capture", path.to_path()).map(axum::Json)
}
+147 -162
View File
@@ -1,6 +1,7 @@
use crate::db::{ApiAuthed, DB};
use crate::jobs::{
filter_list_completed_query, filter_list_queue_query, ListCompletedQuery, ListQueueQuery,
filter_list_completed_query, filter_list_queue_query, Job, ListCompletedQuery, ListQueueQuery,
UnifiedJob,
};
use crate::users::check_scopes;
use axum::extract::Path;
@@ -13,12 +14,10 @@ use axum::Router;
use serde::Serialize;
use sql_builder::bind::Bind;
use sql_builder::SqlBuilder;
use sqlx::postgres::PgRow;
use sqlx::{FromRow, Row};
use uuid::Uuid;
use windmill_common::db::UserDB;
use windmill_common::error::Error::{InternalErr, PermissionDenied};
use windmill_common::error::JsonResult;
use windmill_common::error::{self, JsonResult};
use windmill_common::utils::require_admin;
pub fn global_service() -> Router {
@@ -29,7 +28,7 @@ pub fn global_service() -> Router {
}
pub fn workspaced_service() -> Router {
Router::new().route("/intervals", get(get_concurrent_intervals))
Router::new().route("/list_jobs", get(get_concurrent_intervals))
}
#[derive(Serialize)]
@@ -107,58 +106,30 @@ async fn prune_concurrency_group(
}
#[derive(Serialize)]
struct ConcurrencyIntervals {
concurrency_key: Option<String>,
running_jobs: Vec<RunningJobDuration>,
completed_jobs: Vec<CompletedJobDuration>,
struct ExtendedJobs {
jobs: Vec<Job>,
obscured_jobs: Vec<ObscuredJob>,
omitted_obscured_jobs: bool,
}
#[derive(Serialize)]
struct CompletedJobDuration {
job_id: Option<Uuid>,
concurrency_key: Option<String>,
started_at: chrono::DateTime<chrono::Utc>,
ended_at: chrono::DateTime<chrono::Utc>,
}
impl<'r> FromRow<'r, PgRow> for CompletedJobDuration {
fn from_row(row: &'r PgRow) -> Result<Self, sqlx::Error> {
let duration_ms: i64 = row.try_get("duration_ms")?;
let started_at = row.try_get("started_at")?;
let ended_at: chrono::DateTime<chrono::Utc> =
started_at + std::time::Duration::from_millis(duration_ms.try_into().unwrap());
Ok(Self {
job_id: row.try_get("id")?,
concurrency_key: row.try_get("key")?,
started_at,
ended_at,
})
}
}
#[derive(Serialize, FromRow)]
struct RunningJobDuration {
#[sqlx(rename = "id")]
job_id: Option<Uuid>,
#[sqlx(rename = "key")]
concurrency_key: Option<String>,
struct ObscuredJob {
typ: String,
started_at: Option<chrono::DateTime<chrono::Utc>>,
duration_ms: Option<i64>,
}
#[derive(Deserialize)]
struct ConcurrentIntervalsParams {
struct ExtendedJobsParams {
concurrency_key: Option<String>,
row_limit: Option<i64>,
}
fn join_concurrency_key<'c>(concurrency_key: Option<&String>, mut sqlb: SqlBuilder) -> SqlBuilder {
match concurrency_key {
Some(key) => sqlb
.join("concurrency_key")
if let Some(key) = concurrency_key {
sqlb.join("concurrency_key")
.on_eq("id", "job_id")
.and_where_eq("key", "?".bind(key)),
None => sqlb.left().join("concurrency_key").on_eq("id", "job_id"),
};
.and_where_eq("key", "?".bind(key));
}
sqlb
}
@@ -168,100 +139,49 @@ async fn get_concurrent_intervals(
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(iq): Query<ConcurrentIntervalsParams>,
Query(iq): Query<ExtendedJobsParams>,
Query(lq): Query<ListCompletedQuery>,
) -> JsonResult<ConcurrencyIntervals> {
) -> JsonResult<ExtendedJobs> {
check_scopes(&authed, || format!("listjobs"))?;
if lq.success.is_some() && lq.running.is_some_and(|x| x) {
return Err(error::Error::BadRequest(
"cannot specify both success and running".to_string(),
));
}
let row_limit = iq.row_limit.unwrap_or(1000);
let concurrency_key = iq.concurrency_key;
let lq_copy = lq.clone();
let lqq = ListQueueQuery {
script_path_start: lq_copy.script_path_start,
script_path_exact: lq_copy.script_path_exact,
script_hash: lq_copy.script_hash,
created_by: lq_copy.created_by,
started_before: lq_copy.started_before,
started_after: lq_copy.started_after,
created_before: lq_copy.created_before,
created_after: lq_copy.created_after,
created_or_started_before: lq_copy.created_or_started_before,
created_or_started_after: lq_copy.created_or_started_after,
running: lq_copy.running,
parent_job: lq_copy.parent_job,
order_desc: Some(true),
job_kinds: lq_copy.job_kinds,
suspended: lq_copy.suspended,
args: lq_copy.args,
tag: lq_copy.tag,
schedule_path: lq_copy.schedule_path,
scheduled_for_before_now: lq_copy.scheduled_for_before_now,
all_workspaces: lq_copy.all_workspaces,
is_flow_step: lq_copy.is_flow_step,
has_null_parent: lq_copy.has_null_parent,
is_not_schedule: lq_copy.is_not_schedule,
};
let lq = ListCompletedQuery { order_desc: Some(true), ..lq };
let lqc = lq.clone();
let lqq: ListQueueQuery = lqc.into();
let mut sqlb_q = SqlBuilder::select_from("queue")
.fields(&["id", "key", "started_at"])
.fields(UnifiedJob::queued_job_fields())
.order_by("created_at", lq.order_desc.unwrap_or(true))
.limit(row_limit)
.clone();
let mut sqlb_c = SqlBuilder::select_from("completed_job")
.fields(&["id", "key", "started_at", "duration_ms"])
.fields(UnifiedJob::completed_job_fields())
.order_by("started_at", lq.order_desc.unwrap_or(true))
.limit(row_limit)
.clone();
let mut sqlb_q_user = SqlBuilder::select_from("queue")
.fields(&["id"])
.order_by("created_at", lq.order_desc.unwrap_or(true))
.limit(row_limit)
.clone();
let mut sqlb_c_user = SqlBuilder::select_from("completed_job")
.fields(&["id"])
.order_by("started_at", lq.order_desc.unwrap_or(true))
.limit(row_limit)
.clone();
sqlb_q = join_concurrency_key(concurrency_key.as_ref(), sqlb_q);
sqlb_c = join_concurrency_key(concurrency_key.as_ref(), sqlb_c);
sqlb_q_user = join_concurrency_key(concurrency_key.as_ref(), sqlb_q_user);
sqlb_c_user = join_concurrency_key(concurrency_key.as_ref(), sqlb_c_user);
sqlb_q.and_where_is_not_null("started_at");
// When we have a concurrency key defined, fetch jobs from other workspaces
// as obscured unless we're in the admins workspace. This is to show the
// potential concurrency races without showing jobs that don't belong to
// the workspace.
let sqlb_all_workspaces: Option<(SqlBuilder, SqlBuilder)> =
if concurrency_key.is_some() && w_id != "admins" {
Some((
filter_list_queue_query(
sqlb_q.clone(),
&ListQueueQuery { all_workspaces: Some(true), ..lqq.clone() },
"admins",
),
filter_list_completed_query(
sqlb_c.clone(),
&ListCompletedQuery { all_workspaces: Some(true), ..lq.clone() },
"admins",
),
))
} else {
None
};
sqlb_q = filter_list_queue_query(sqlb_q, &lqq, w_id.as_str());
sqlb_c = filter_list_completed_query(sqlb_c, &lq, w_id.as_str());
let sql_q = sqlb_q.query()?;
let sql_c = sqlb_c.query()?;
let mut tx = user_db.begin(&authed).await?;
let running_jobs_user: Vec<RunningJobDuration> = if lq.success.is_none() {
sqlx::query_as(&sql_q).fetch_all(&mut *tx).await?
} else {
vec![]
};
let completed_jobs_user: Vec<CompletedJobDuration> = if lq.running.is_none() {
sqlx::query_as(&sql_c).fetch_all(&mut *tx).await?
} else {
vec![]
};
tx.commit().await?;
// To avoid infering information through filtering, don't return obscured
// jobs if the filters are too specific
let should_fetch_obscured_jobs = match lq {
ListCompletedQuery {
script_path_start: None,
@@ -291,69 +211,134 @@ async fn get_concurrent_intervals(
job_kinds: _,
is_flow_step: _,
all_workspaces: _,
} => true,
} => concurrency_key.is_some(),
_ => false,
};
// This second transaction using the raw db lets us get info for jobs that
// the user has no access to. Before returning that, we will hide the ids
if should_fetch_obscured_jobs && concurrency_key.is_some() {
let (sql_q, sql_c) = if let Some(sqlb) = sqlb_all_workspaces {
(sqlb.0.query()?, sqlb.1.query()?)
// When we have a concurrency key defined, fetch jobs from other workspaces
// as obscured unless we're in the admins workspace. This is to show the
// potential concurrency races without showing jobs that don't belong to
// the workspace.
// To avoid infering information through filtering, don't return obscured
// jobs if the filters are too specific
if should_fetch_obscured_jobs {
let (sqlb_q, sqlb_c) = if w_id != "admin" {
// By default get obscured jobs from all workspaces, unless in
// admin workspace where admins can select to get all or not
(
filter_list_queue_query(
sqlb_q,
&ListQueueQuery { all_workspaces: Some(true), ..lqq.clone() },
"admins",
),
filter_list_completed_query(
sqlb_c,
&ListCompletedQuery { all_workspaces: Some(true), ..lq.clone() },
"admins",
),
)
} else {
(sql_q, sql_c)
(
filter_list_queue_query(sqlb_q, &lqq, w_id.as_str()),
filter_list_completed_query(sqlb_c, &lq, w_id.as_str()),
)
};
let running_jobs_db: Vec<RunningJobDuration> = if lq.success.is_none() {
sqlb_q_user = filter_list_queue_query(sqlb_q_user, &lqq, w_id.as_str());
sqlb_c_user = filter_list_completed_query(sqlb_c_user, &lq, w_id.as_str());
let sql_q_user = sqlb_q_user.query()?;
let sql_c_user = sqlb_c_user.query()?;
let sql_q = sqlb_q.query()?;
let sql_c = sqlb_c.query()?;
// This first transaction uses the user_db to know which uuids are
// accessible to the user.
let mut tx = user_db.begin(&authed).await?;
let running_jobs_user: Vec<Uuid> = if lq.success.is_none() {
sqlx::query_scalar(&sql_q_user).fetch_all(&mut *tx).await?
} else {
vec![]
};
let completed_jobs_user: Vec<Uuid> = if lq.running.is_none() {
sqlx::query_scalar(&sql_c_user).fetch_all(&mut *tx).await?
} else {
vec![]
};
tx.commit().await?;
// This second transaction uses the db, so it will fetch information
// potentially forbidden to the user. It must be obscured before
// returning it
let running_jobs_db: Vec<UnifiedJob> = if lq.success.is_none() {
sqlx::query_as(&sql_q).fetch_all(&db).await?
} else {
vec![]
};
let completed_jobs_db: Vec<CompletedJobDuration> = if lq.running.is_none() {
let completed_jobs_db: Vec<UnifiedJob> = if lq.running.is_none() {
sqlx::query_as(&sql_c).fetch_all(&db).await?
} else {
vec![]
};
let running_jobs = running_jobs_db
.into_iter()
.map(|r| {
if running_jobs_user
let obscured_jobs = running_jobs_db
.iter()
.filter(|j| !running_jobs_user.iter().any(|id| j.id == *id))
.chain(
completed_jobs_db
.iter()
.any(|u| u.job_id.unwrap() == r.job_id.unwrap())
{
RunningJobDuration { ..r }
} else {
RunningJobDuration { job_id: None, ..r }
}
.filter(|j| !completed_jobs_user.iter().any(|id| j.id == *id)),
)
.map(|j| ObscuredJob {
typ: j.typ.clone(),
started_at: j.started_at,
duration_ms: j.duration_ms,
})
.collect();
let completed_jobs = completed_jobs_db
let jobs = running_jobs_db
.into_iter()
.map(|r| {
if completed_jobs_user
.iter()
.any(|u| u.job_id.unwrap() == r.job_id.unwrap())
{
CompletedJobDuration { ..r }
} else {
CompletedJobDuration { job_id: None, ..r }
}
})
.filter(|j| running_jobs_user.iter().any(|id| j.id == *id))
.chain(
completed_jobs_db
.into_iter()
.filter(|j| completed_jobs_user.iter().any(|id| j.id == *id)),
)
.map(From::from)
.collect();
Ok(Json(ExtendedJobs {
jobs,
obscured_jobs,
omitted_obscured_jobs: !should_fetch_obscured_jobs,
}))
} else {
let sql_q = sqlb_q.query()?;
let sql_c = sqlb_c.query()?;
let mut tx = user_db.begin(&authed).await?;
let running_jobs: Vec<UnifiedJob> = if lq.success.is_none() {
sqlx::query_as(&sql_q).fetch_all(&mut *tx).await?
} else {
vec![]
};
let completed_jobs: Vec<UnifiedJob> = if lq.running.is_none() {
sqlx::query_as(&sql_c).fetch_all(&mut *tx).await?
} else {
vec![]
};
tx.commit().await?;
let jobs = running_jobs
.into_iter()
.chain(completed_jobs.into_iter())
.map(From::from)
.collect();
return Ok(Json(ConcurrencyIntervals {
concurrency_key,
running_jobs,
completed_jobs,
}));
Ok(Json(ExtendedJobs {
jobs,
obscured_jobs: vec![],
omitted_obscured_jobs: !should_fetch_obscured_jobs,
}))
}
Ok(Json(ConcurrencyIntervals {
concurrency_key,
running_jobs: running_jobs_user,
completed_jobs: completed_jobs_user,
}))
}
async fn get_concurrency_key(
+1 -1
View File
@@ -38,7 +38,7 @@ pub enum DraftType {
#[derive(Deserialize, Serialize, Debug)]
pub struct Draft {
pub path: String,
pub value: serde_json::Value,
pub value: sqlx::types::Json<Box<serde_json::value::Value>>,
pub typ: DraftType,
}
+39 -38
View File
@@ -25,12 +25,12 @@ use axum::{
use hyper::StatusCode;
use serde::{Deserialize, Serialize};
use serde_json::json;
use sql_builder::prelude::*;
use sqlx::{FromRow, Postgres, Transaction};
use windmill_audit::audit_ee::audit_log;
use windmill_audit::ActionKind;
use windmill_common::utils::query_elems_from_hub;
use windmill_common::worker::to_raw_value;
use windmill_common::HUB_BASE_URL;
use windmill_common::{
db::UserDB,
@@ -71,7 +71,7 @@ pub fn global_service() -> Router {
#[derive(Serialize, FromRow)]
pub struct SearchFlow {
path: String,
value: serde_json::Value,
value: sqlx::types::Json<Box<serde_json::value::RawValue>>,
}
async fn list_search_flows(
authed: ApiAuthed,
@@ -85,12 +85,11 @@ async fn list_search_flows(
let n = 3;
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query_as!(
SearchFlow,
let rows = sqlx::query_as::<_, SearchFlow>(
"SELECT path, value from flow WHERE workspace_id = $1 LIMIT $2",
&w_id,
n
)
.bind(&w_id)
.bind(n)
.fetch_all(&mut *tx)
.await?
.into_iter()
@@ -200,7 +199,7 @@ async fn list_paths(
pub async fn get_hub_flow_by_id(
Path(id): Path<i32>,
Extension(db): Extension<DB>,
) -> JsonResult<serde_json::Value> {
) -> JsonResult<Box<serde_json::value::RawValue>> {
let value = http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/flows/{}/json", *HUB_BASE_URL.read().await, id),
@@ -363,9 +362,9 @@ async fn create_flow(
)
.await?;
let mut args: HashMap<String, serde_json::Value> = HashMap::new();
let mut args: HashMap<String, Box<serde_json::value::RawValue>> = HashMap::new();
if let Some(dm) = nf.deployment_message {
args.insert("deployment_message".to_string(), json!(dm));
args.insert("deployment_message".to_string(), to_raw_value(&dm));
}
let tx = PushIsolationLevel::Transaction(tx);
@@ -377,7 +376,7 @@ async fn create_flow(
path: nf.path.clone(),
dedicated_worker: nf.dedicated_worker,
},
args,
args.into(),
&authed.username,
&authed.email,
windmill_common::users::username_to_permissioned_as(&authed.username),
@@ -515,22 +514,19 @@ async fn update_flow(
}
}
let mut schedulables: Vec<Schedule> = sqlx::query_as!(
Schedule,
"UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
nf.path,
flow_path,
w_id,
)
let mut schedulables: Vec<Schedule> = sqlx::query_as::<_, Schedule>(
"UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *")
.bind(&nf.path)
.bind(&flow_path)
.bind(&w_id)
.fetch_all(&mut tx)
.await?;
let schedule = sqlx::query_as!(Schedule,
"UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
nf.path,
flow_path,
w_id,
)
let schedule = sqlx::query_as::<_, Schedule>(
"UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *")
.bind(&nf.path)
.bind(&flow_path)
.bind(&w_id)
.fetch_optional(&mut tx)
.await?;
@@ -582,11 +578,11 @@ async fn update_flow(
let tx = PushIsolationLevel::Transaction(tx);
let mut args: HashMap<String, serde_json::Value> = HashMap::new();
let mut args: HashMap<String, Box<serde_json::value::RawValue>> = HashMap::new();
if let Some(dm) = nf.deployment_message {
args.insert("deployment_message".to_string(), json!(dm));
args.insert("deployment_message".to_string(), to_raw_value(&dm));
}
args.insert("parent_path".to_string(), json!(flow_path));
args.insert("parent_path".to_string(), to_raw_value(&flow_path));
let (dependency_job_uuid, mut new_tx) = push(
&db,
@@ -596,7 +592,7 @@ async fn update_flow(
path: nf.path.clone(),
dedicated_worker: nf.dedicated_worker,
},
args,
windmill_queue::PushArgs { args, extra: HashMap::new() },
&authed.username,
&authed.email,
windmill_common::users::username_to_permissioned_as(&authed.username),
@@ -663,10 +659,10 @@ pub struct FlowWDraft {
pub summary: String,
pub description: String,
pub schema: Option<Schema>,
pub value: serde_json::Value,
pub value: sqlx::types::Json<Box<serde_json::value::RawValue>>,
pub extra_perms: serde_json::Value,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft: Option<serde_json::Value>,
pub draft: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_only: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -881,16 +877,18 @@ mod tests {
modules: vec![
FlowModule {
id: "a".to_string(),
value: FlowModuleValue::Script {
value: windmill_common::worker::to_raw_value(&FlowModuleValue::Script {
path: "test".to_string(),
input_transforms: [(
"test".to_string(),
InputTransform::Static { value: serde_json::json!("test2") },
InputTransform::Static {
value: windmill_common::worker::to_raw_value(&"test2".to_string()),
},
)]
.into(),
hash: None,
tag_override: None,
},
}),
stop_after_if: None,
summary: None,
suspend: Default::default(),
@@ -905,7 +903,7 @@ mod tests {
},
FlowModule {
id: "b".to_string(),
value: FlowModuleValue::RawScript {
value: windmill_common::worker::to_raw_value(&FlowModuleValue::RawScript {
input_transforms: HashMap::new(),
content: "test".to_string(),
language: scripts::ScriptLang::Deno,
@@ -915,7 +913,7 @@ mod tests {
custom_concurrency_key: None,
concurrent_limit: None,
concurrency_time_window_s: None,
},
}),
stop_after_if: Some(StopAfterIf {
expr: "foo = 'bar'".to_string(),
skip_if_stopped: false,
@@ -933,13 +931,15 @@ mod tests {
},
FlowModule {
id: "c".to_string(),
value: FlowModuleValue::ForloopFlow {
iterator: InputTransform::Static { value: serde_json::json!([1, 2, 3]) },
value: windmill_common::worker::to_raw_value(&FlowModuleValue::ForloopFlow {
iterator: InputTransform::Static {
value: windmill_common::worker::to_raw_value(&[1, 2, 3]),
},
modules: vec![],
skip_failures: true,
parallel: false,
parallelism: None,
},
}),
stop_after_if: Some(StopAfterIf {
expr: "previous.isEmpty()".to_string(),
skip_if_stopped: false,
@@ -963,7 +963,8 @@ mod tests {
input_transforms: HashMap::new(),
hash: None,
tag_override: None,
},
}
.into(),
stop_after_if: Some(StopAfterIf {
expr: "previous.isEmpty()".to_string(),
skip_if_stopped: false,
+9 -7
View File
@@ -47,7 +47,7 @@ pub struct InputRow {
pub runnable_id: String,
pub runnable_type: RunnableType,
pub name: String,
pub args: Value,
pub args: sqlx::types::Json<Box<serde_json::value::RawValue>>,
pub created_at: DateTime<Utc>,
pub created_by: String,
pub is_public: bool,
@@ -95,12 +95,12 @@ pub struct RunnableParams {
pub runnable_type: RunnableType,
}
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)]
pub struct Input {
id: Uuid,
name: String,
created_at: chrono::DateTime<chrono::Utc>,
args: serde_json::Value,
args: sqlx::types::Json<Box<serde_json::value::RawValue>>,
created_by: String,
is_public: bool,
success: bool,
@@ -110,7 +110,7 @@ pub struct Input {
pub struct CompletedJobMini {
id: Uuid,
created_at: chrono::DateTime<chrono::Utc>,
args: Option<serde_json::Value>,
args: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
created_by: String,
success: bool,
}
@@ -161,7 +161,9 @@ async fn get_input_history(
row.created_by
),
created_at: row.created_at,
args: row.args.unwrap_or(serde_json::json!({})),
args: row.args.unwrap_or(sqlx::types::Json(
serde_json::value::RawValue::from_string("null".to_string()).unwrap(),
)),
created_by: row.created_by,
is_public: true,
success: row.success,
@@ -240,7 +242,7 @@ async fn list_saved_inputs(
#[derive(Debug, Serialize, Deserialize)]
pub struct CreateInput {
name: String,
args: serde_json::Value,
args: Box<serde_json::value::RawValue>,
}
async fn create_input(
@@ -262,7 +264,7 @@ async fn create_input(
.bind(&r.runnable_id)
.bind(&r.runnable_type)
.bind(&input.name)
.bind(&input.args)
.bind(sqlx::types::Json(&input.args))
.bind(&authed.username)
.execute(&mut *tx)
.await?;
+202 -162
View File
@@ -903,7 +903,42 @@ pub struct ListQueueQuery {
pub is_not_schedule: Option<bool>,
}
pub fn filter_list_queue_query(mut sqlb: SqlBuilder, lq: &ListQueueQuery, w_id: &str) -> SqlBuilder {
impl From<ListCompletedQuery> for ListQueueQuery {
fn from(lcq: ListCompletedQuery) -> Self {
Self {
script_path_start: lcq.script_path_start,
script_path_exact: lcq.script_path_exact,
script_hash: lcq.script_hash,
created_by: lcq.created_by,
started_before: lcq.started_before,
started_after: lcq.started_after,
created_before: lcq.created_before,
created_after: lcq.created_after,
created_or_started_before: lcq.created_or_started_before,
created_or_started_after: lcq.created_or_started_after,
running: lcq.running,
parent_job: lcq.parent_job,
order_desc: lcq.order_desc,
job_kinds: lcq.job_kinds,
suspended: lcq.suspended,
args: lcq.args,
tag: lcq.tag,
schedule_path: lcq.schedule_path,
scheduled_for_before_now: lcq.scheduled_for_before_now,
all_workspaces: lcq.all_workspaces,
is_flow_step: lcq.is_flow_step,
has_null_parent: lcq.has_null_parent,
is_not_schedule: lcq.is_not_schedule,
}
}
}
pub fn filter_list_queue_query(
mut sqlb: SqlBuilder,
lq: &ListQueueQuery,
w_id: &str,
) -> SqlBuilder {
if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) {
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
}
@@ -1004,7 +1039,6 @@ pub fn filter_list_queue_query(mut sqlb: SqlBuilder, lq: &ListQueueQuery, w_id:
}
sqlb
}
pub fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> SqlBuilder {
@@ -1213,40 +1247,7 @@ async fn list_jobs(
per_page + offset,
0,
&ListCompletedQuery { order_desc: Some(true), ..lqc },
&[
"'CompletedJob' as typ",
"id",
"workspace_id",
"parent_job",
"created_by",
"created_at",
"started_at",
"null as scheduled_for",
"null as running",
"script_hash",
"script_path",
"null as args",
"duration_ms",
"success",
"deleted",
"canceled",
"canceled_by",
"job_kind",
"schedule_path",
"permissioned_as",
"is_flow_step",
"language",
"is_skipped",
"email",
"visible_to_owner",
"null as suspend",
"mem_peak",
"tag",
"null as concurrent_limit",
"null as concurrency_time_window_s",
"priority",
"result->'wm_labels' as labels",
],
UnifiedJob::completed_job_fields(),
))
} else {
None
@@ -1255,65 +1256,8 @@ async fn list_jobs(
let sql = if lq.success.is_none() && lq.label.is_none() {
let sqlq = list_queue_jobs_query(
&w_id,
&ListQueueQuery {
script_path_start: lq.script_path_start,
script_path_exact: lq.script_path_exact,
script_hash: lq.script_hash,
created_by: lq.created_by,
started_before: lq.started_before,
started_after: lq.started_after,
created_before: lq.created_before,
created_after: lq.created_after,
created_or_started_before: lq.created_or_started_before,
created_or_started_after: lq.created_or_started_after,
running: lq.running,
parent_job: lq.parent_job,
order_desc: Some(true),
job_kinds: lq.job_kinds,
suspended: lq.suspended,
args: lq.args,
tag: lq.tag,
schedule_path: lq.schedule_path,
scheduled_for_before_now: lq.scheduled_for_before_now,
all_workspaces: lq.all_workspaces,
is_flow_step: lq.is_flow_step,
has_null_parent: lq.has_null_parent,
is_not_schedule: lq.is_not_schedule,
},
&[
"'QueuedJob' as typ",
"id",
"workspace_id",
"parent_job",
"created_by",
"created_at",
"started_at",
"scheduled_for",
"running",
"script_hash",
"script_path",
"null as args",
"null as duration_ms",
"null as success",
"false as deleted",
"canceled",
"canceled_by",
"job_kind",
"schedule_path",
"permissioned_as",
"is_flow_step",
"language",
"false as is_skipped",
"email",
"visible_to_owner",
"suspend",
"mem_peak",
"tag",
"concurrent_limit",
"concurrency_time_window_s",
"priority",
"null as labels",
],
&ListQueueQuery { order_desc: Some(true), ..lq.into() },
UnifiedJob::queued_job_fields(),
);
if let Some(sqlc) = sqlc {
@@ -2054,7 +1998,10 @@ impl Job {
)
}
pub async fn concurrency_key(&self, db: &Pool<Postgres>) -> Result<Option<String>, sqlx::Error> {
pub async fn concurrency_key(
&self,
db: &Pool<Postgres>,
) -> Result<Option<String>, sqlx::Error> {
sqlx::query_scalar!(
"SELECT key FROM concurrency_key WHERE job_id = $1",
self.id()
@@ -2065,38 +2012,117 @@ impl Job {
}
#[derive(sqlx::FromRow)]
struct UnifiedJob {
workspace_id: String,
typ: String,
id: Uuid,
parent_job: Option<Uuid>,
created_by: String,
created_at: chrono::DateTime<chrono::Utc>,
started_at: Option<chrono::DateTime<chrono::Utc>>,
scheduled_for: Option<chrono::DateTime<chrono::Utc>>,
running: Option<bool>,
script_hash: Option<ScriptHash>,
script_path: Option<String>,
duration_ms: Option<i64>,
success: Option<bool>,
deleted: bool,
canceled: bool,
canceled_by: Option<String>,
job_kind: JobKind,
schedule_path: Option<String>,
permissioned_as: String,
is_flow_step: bool,
language: Option<ScriptLang>,
is_skipped: bool,
email: String,
visible_to_owner: bool,
suspend: Option<i32>,
mem_peak: Option<i32>,
tag: String,
concurrent_limit: Option<i32>,
concurrency_time_window_s: Option<i32>,
priority: Option<i16>,
labels: Option<serde_json::Value>,
pub struct UnifiedJob {
pub workspace_id: String,
pub typ: String,
pub id: Uuid,
pub parent_job: Option<Uuid>,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
pub scheduled_for: Option<chrono::DateTime<chrono::Utc>>,
pub running: Option<bool>,
pub script_hash: Option<ScriptHash>,
pub script_path: Option<String>,
pub duration_ms: Option<i64>,
pub success: Option<bool>,
pub deleted: bool,
pub canceled: bool,
pub canceled_by: Option<String>,
pub job_kind: JobKind,
pub schedule_path: Option<String>,
pub permissioned_as: String,
pub is_flow_step: bool,
pub language: Option<ScriptLang>,
pub is_skipped: bool,
pub email: String,
pub visible_to_owner: bool,
pub suspend: Option<i32>,
pub mem_peak: Option<i32>,
pub tag: String,
pub concurrent_limit: Option<i32>,
pub concurrency_time_window_s: Option<i32>,
pub priority: Option<i16>,
pub labels: Option<serde_json::Value>,
}
const CJ_FIELDS: &[&str] = &[
"'CompletedJob' as typ",
"id",
"workspace_id",
"parent_job",
"created_by",
"created_at",
"started_at",
"null as scheduled_for",
"null as running",
"script_hash",
"script_path",
"null as args",
"duration_ms",
"success",
"deleted",
"canceled",
"canceled_by",
"job_kind",
"schedule_path",
"permissioned_as",
"is_flow_step",
"language",
"is_skipped",
"email",
"visible_to_owner",
"null as suspend",
"mem_peak",
"tag",
"null as concurrent_limit",
"null as concurrency_time_window_s",
"priority",
"result->'wm_labels' as labels",
];
const QJ_FIELDS: &[&str] = &[
"'QueuedJob' as typ",
"id",
"workspace_id",
"parent_job",
"created_by",
"created_at",
"started_at",
"scheduled_for",
"running",
"script_hash",
"script_path",
"null as args",
"null as duration_ms",
"null as success",
"false as deleted",
"canceled",
"canceled_by",
"job_kind",
"schedule_path",
"permissioned_as",
"is_flow_step",
"language",
"false as is_skipped",
"email",
"visible_to_owner",
"suspend",
"mem_peak",
"tag",
"concurrent_limit",
"concurrency_time_window_s",
"priority",
"null as labels",
];
impl UnifiedJob {
pub fn completed_job_fields() -> &'static [&'static str] {
CJ_FIELDS
}
pub fn queued_job_fields() -> &'static [&'static str] {
QJ_FIELDS
}
}
impl<'a> From<UnifiedJob> for Job {
@@ -2202,7 +2228,7 @@ struct Preview {
content: Option<String>,
kind: Option<PreviewKind>,
path: Option<String>,
args: Option<Box<JsonRawValue>>,
args: Option<HashMap<String, Box<JsonRawValue>>>,
language: Option<ScriptLang>,
tag: Option<String>,
dedicated_worker: Option<bool>,
@@ -2218,7 +2244,7 @@ pub struct WorkflowTask {
struct PreviewFlow {
value: FlowValue,
path: Option<String>,
args: Option<Box<JsonRawValue>>,
args: Option<HashMap<String, Box<JsonRawValue>>>,
tag: Option<String>,
restarted_from: Option<RestartedFrom>,
}
@@ -2331,7 +2357,7 @@ pub async fn run_flow_by_path(
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, flow_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
args: PushArgs,
) -> error::Result<(StatusCode, String)> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
@@ -2432,7 +2458,8 @@ pub async fn restart_flow(
let push_args = completed_job
.args
.map(|json| PushArgs { args: json.clone(), extra: json.0 });
.map(|json| PushArgs { args: json.0, extra: HashMap::new() })
.unwrap_or_else(PushArgs::empty);
let scheduled_for = run_query.get_scheduled_for(&db).await?;
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
@@ -2476,7 +2503,7 @@ pub async fn run_script_by_path(
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, script_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
args: PushArgs,
) -> error::Result<(StatusCode, String)> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
@@ -2563,7 +2590,7 @@ pub async fn run_workflow_as_code(
let mut extra = HashMap::new();
extra.insert(ENTRYPOINT_OVERRIDE.to_string(), to_raw_value(&entrypoint));
let args = PushArgs { args: sqlx::types::Json(task.args), extra };
let args = PushArgs { args: task.args.unwrap_or_else(HashMap::new), extra };
let scheduled_for = run_query.get_scheduled_for(&db).await?;
let tag = run_query.tag.clone().or(tag).or(Some(job.tag));
@@ -2854,7 +2881,7 @@ pub async fn run_wait_result_job_by_path_get(
});
let inner_args: HashMap<String, Box<RawValue>> = HashMap::new();
let args = PushArgs { extra: payload_args, args: sqlx::types::Json(inner_args) };
let args = PushArgs { extra: payload_args, args: inner_args };
check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
let script_path = script_path.to_path();
@@ -2932,7 +2959,7 @@ pub async fn run_wait_result_flow_by_path_get(
payload_args.insert(k.to_string(), v.clone());
});
let args = PushArgs { extra: payload_args, args: sqlx::types::Json(HashMap::new()) };
let args = PushArgs { extra: payload_args, args: HashMap::new() };
run_wait_result_flow_by_path_internal(
db, run_query, flow_path, authed, rsmq, user_db, args, w_id,
@@ -2947,7 +2974,7 @@ pub async fn run_wait_result_script_by_path(
Extension(db): Extension<DB>,
Path((w_id, script_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
args: PushArgs,
) -> error::Result<Response> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
@@ -2973,7 +3000,7 @@ async fn run_wait_result_script_by_path_internal(
rsmq: Option<rsmq_async::MultiplexedRsmq>,
user_db: UserDB,
w_id: String,
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
args: PushArgs,
) -> error::Result<Response> {
check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
let script_path = script_path.to_path();
@@ -3026,7 +3053,7 @@ pub async fn run_wait_result_script_by_hash(
Extension(db): Extension<DB>,
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
Query(run_query): Query<RunJobQuery>,
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
args: PushArgs,
) -> error::Result<Response> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
@@ -3106,7 +3133,7 @@ pub async fn run_wait_result_flow_by_path(
Extension(db): Extension<DB>,
Path((w_id, flow_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
args: PushArgs,
) -> error::Result<Response> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
@@ -3124,7 +3151,7 @@ async fn run_wait_result_flow_by_path_internal(
authed: ApiAuthed,
rsmq: Option<rsmq_async::MultiplexedRsmq>,
user_db: UserDB,
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
args: PushArgs,
w_id: String,
) -> error::Result<Response> {
check_queue_too_long(&db, run_query.queue_limit).await?;
@@ -3220,7 +3247,7 @@ async fn run_preview_script(
dedicated_worker: preview.dedicated_worker,
}),
},
preview.args.unwrap_or_default(),
preview.args.unwrap_or_default().into(),
authed.display_username(),
&authed.email,
username_to_permissioned_as(&authed.username),
@@ -3301,7 +3328,7 @@ async fn run_bundle_preview_script(
custom_concurrency_key: None,
}),
},
args,
args.into(),
authed.display_username(),
&authed.email,
username_to_permissioned_as(&authed.username),
@@ -3421,10 +3448,7 @@ pub async fn run_dependencies_job(
"raw_deps".to_string(),
JsonRawValue::from_string("true".to_string()).unwrap(),
);
(
PushArgs { extra: hm, args: sqlx::types::Json(HashMap::new()) },
deps,
)
(PushArgs { extra: hm, args: HashMap::new() }, deps)
} else {
(
PushArgs::empty(),
@@ -3668,7 +3692,7 @@ async fn run_preview_flow_job(
path: raw_flow.path,
restarted_from: raw_flow.restarted_from,
},
raw_flow.args.unwrap_or_default(),
raw_flow.args.unwrap_or_default().into(),
authed.display_username(),
&authed.email,
username_to_permissioned_as(&authed.username),
@@ -3700,7 +3724,7 @@ pub async fn run_job_by_hash(
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
Query(run_query): Query<RunJobQuery>,
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
args: PushArgs,
) -> error::Result<(StatusCode, String)> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
@@ -3781,7 +3805,7 @@ pub struct JobUpdate {
pub new_logs: Option<String>,
pub log_offset: Option<i32>,
pub mem_peak: Option<i32>,
pub flow_status: Option<serde_json::Value>,
pub flow_status: Option<Box<serde_json::value::RawValue>>,
}
async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::Result<Response> {
@@ -3834,22 +3858,30 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R
)));
}
#[derive(Deserialize, sqlx::FromRow)]
pub struct JobUpdateRow {
pub running: bool,
pub logs: Option<String>,
pub mem_peak: Option<i32>,
pub flow_status: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
pub log_offset: Option<i32>,
}
async fn get_job_update(
Extension(db): Extension<DB>,
Path((w_id, job_id)): Path<(String, Uuid)>,
Query(JobUpdateQuery { running, log_offset }): Query<JobUpdateQuery>,
) -> error::JsonResult<JobUpdate> {
let record = sqlx::query!(
let record = sqlx::query_as::<_, JobUpdateRow>(
"SELECT running, substr(concat(coalesce(queue.logs, ''), job_logs.logs), greatest($1 - job_logs.log_offset, 0)) as logs, mem_peak,
CASE WHEN is_flow_step is true then NULL else flow_status END as flow_status,
job_logs.log_offset + char_length(job_logs.logs) + 1 as log_offset
FROM queue
LEFT JOIN job_logs ON job_logs.job_id = queue.id
WHERE queue.workspace_id = $2 AND queue.id = $3",
log_offset,
&w_id,
&job_id
)
.bind(log_offset)
.bind(&w_id)
.bind(&job_id)
.fetch_optional(&db)
.await?;
@@ -3864,20 +3896,22 @@ async fn get_job_update(
completed: None,
new_logs: record.logs,
mem_peak: record.mem_peak,
flow_status: record.flow_status,
flow_status: record
.flow_status
.map(|x: sqlx::types::Json<Box<RawValue>>| x.0),
}))
} else {
let record = sqlx::query!(
"SELECT substr(concat(coalesce(completed_job.logs, ''), job_logs.logs), greatest($1 - job_logs.log_offset, 0)) as logs, mem_peak,
let record = sqlx::query_as::<_, JobUpdateRow>(
"SELECT false as running, substr(concat(coalesce(completed_job.logs, ''), job_logs.logs), greatest($1 - job_logs.log_offset, 0)) as logs, mem_peak,
CASE WHEN is_flow_step is true then NULL else flow_status END as flow_status,
job_logs.log_offset + char_length(job_logs.logs) + 1 as log_offset
FROM completed_job
LEFT JOIN job_logs ON job_logs.job_id = completed_job.id
WHERE completed_job.workspace_id = $2 AND id = $3",
log_offset,
&w_id,
&job_id
)
.bind(log_offset)
.bind(&w_id)
.bind(&job_id)
.fetch_optional(&db)
.await?;
if let Some(record) = record {
@@ -3887,7 +3921,9 @@ async fn get_job_update(
log_offset: record.log_offset,
new_logs: record.logs,
mem_peak: record.mem_peak,
flow_status: record.flow_status,
flow_status: record
.flow_status
.map(|x: sqlx::types::Json<Box<RawValue>>| x.0),
}))
} else {
Err(error::Error::NotFound(format!("Job not found: {}", job_id)))
@@ -3895,7 +3931,11 @@ async fn get_job_update(
}
}
pub fn filter_list_completed_query(mut sqlb: SqlBuilder, lq: &ListCompletedQuery, w_id: &str) -> SqlBuilder {
pub fn filter_list_completed_query(
mut sqlb: SqlBuilder,
lq: &ListCompletedQuery,
w_id: &str,
) -> SqlBuilder {
if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) {
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
}
+49 -55
View File
@@ -153,38 +153,36 @@ async fn create_schedule(
)
.await?;
let schedule = sqlx::query_as!(
Schedule,
let schedule = sqlx::query_as::<_, Schedule>(
"INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, \
is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, \
on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, \
ws_error_handler_muted, retry, summary, no_flow_overlap, tag \
) VALUES ( \
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22 \
) RETURNING *",
w_id,
ns.path,
ns.schedule,
ns.timezone,
&authed.username,
ns.script_path,
ns.is_flow,
ns.args,
ns.enabled.unwrap_or(false),
&authed.email,
ns.on_failure,
ns.on_failure_times,
ns.on_failure_exact,
ns.on_failure_extra_args,
ns.on_recovery,
ns.on_recovery_times,
ns.on_recovery_extra_args,
ns.ws_error_handler_muted.unwrap_or(false),
ns.retry,
ns.summary,
ns.no_flow_overlap.unwrap_or(false),
ns.tag,
)
) RETURNING *")
.bind(&w_id)
.bind(&ns.path)
.bind(&ns.schedule)
.bind(&ns.timezone)
.bind(&authed.username)
.bind(&ns.script_path)
.bind(&ns.is_flow)
.bind(&ns.args)
.bind(&ns.enabled.unwrap_or(false))
.bind(&authed.email)
.bind(&ns.on_failure)
.bind(&ns.on_failure_times)
.bind(&ns.on_failure_exact)
.bind(&ns.on_failure_extra_args)
.bind(&ns.on_recovery)
.bind(&ns.on_recovery_times)
.bind(&ns.on_recovery_extra_args)
.bind(&ns.ws_error_handler_muted.unwrap_or(false))
.bind(&ns.retry)
.bind(&ns.summary)
.bind(&ns.no_flow_overlap.unwrap_or(false))
.bind(&ns.tag)
.fetch_one(&mut tx)
.await
.map_err(|e| Error::InternalErr(format!("inserting schedule in {w_id}: {e}")))?;
@@ -245,31 +243,29 @@ async fn edit_schedule(
cron::Schedule::from_str(&es.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
clear_schedule(tx.transaction_mut(), path, &w_id).await?;
let schedule = sqlx::query_as!(
Schedule,
let schedule = sqlx::query_as::<_, Schedule>(
"UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, \
on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, \
on_recovery_extra_args = $10, ws_error_handler_muted = $11, retry = $12, summary = $13, \
no_flow_overlap = $14, tag = $15
WHERE path = $16 AND workspace_id = $17 RETURNING *",
es.schedule,
es.timezone,
es.args,
es.on_failure,
es.on_failure_times,
es.on_failure_exact,
es.on_failure_extra_args,
es.on_recovery,
es.on_recovery_times,
es.on_recovery_extra_args,
es.ws_error_handler_muted.unwrap_or(false),
es.retry,
es.summary,
es.no_flow_overlap.unwrap_or(false),
es.tag,
path,
w_id,
)
WHERE path = $16 AND workspace_id = $17 RETURNING *")
.bind(&es.schedule)
.bind(&es.timezone)
.bind(&es.args)
.bind(&es.on_failure)
.bind(&es.on_failure_times)
.bind(&es.on_failure_exact)
.bind(&es.on_failure_extra_args)
.bind(&es.on_recovery)
.bind(&es.on_recovery_times)
.bind(&es.on_recovery_extra_args)
.bind(&es.ws_error_handler_muted.unwrap_or(false))
.bind(&es.retry)
.bind(&es.summary)
.bind(&es.no_flow_overlap.unwrap_or(false))
.bind(&es.tag)
.bind(&path)
.bind(&w_id)
.fetch_one(&mut tx)
.await
.map_err(|e| Error::InternalErr(format!("updating schedule in {w_id}: {e}")))?;
@@ -473,14 +469,12 @@ pub async fn set_enabled(
let mut tx: QueueTransaction<'_, rsmq_async::MultiplexedRsmq> =
(rsmq.clone(), user_db.begin(&authed).await?).into();
let path = path.to_path();
let schedule_o = sqlx::query_as!(
Schedule,
"UPDATE schedule SET enabled = $1, email = $2 WHERE path = $3 AND workspace_id = $4 RETURNING *",
&payload.enabled,
authed.email,
path,
w_id
)
let schedule_o = sqlx::query_as::<_, Schedule>(
"UPDATE schedule SET enabled = $1, email = $2 WHERE path = $3 AND workspace_id = $4 RETURNING *")
.bind(&payload.enabled)
.bind(&authed.email)
.bind(&path)
.bind(&w_id)
.fetch_optional(&mut tx)
.await?;
+17 -18
View File
@@ -25,6 +25,7 @@ use axum::{
use hyper::StatusCode;
use serde::{Deserialize, Serialize};
use serde_json::json;
use serde_json::value::RawValue;
use sql_builder::prelude::*;
use sqlx::{FromRow, Postgres, Transaction};
use std::{
@@ -51,6 +52,7 @@ use windmill_common::{
utils::{
not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
},
worker::to_raw_value,
HUB_BASE_URL,
};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
@@ -70,7 +72,7 @@ pub struct ScriptWDraft {
pub kind: ScriptKind,
pub tag: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft: Option<serde_json::Value>,
pub draft: Option<sqlx::types::Json<Box<RawValue>>>,
pub schema: Option<Schema>,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_only: Option<bool>,
@@ -622,22 +624,19 @@ async fn create_script_internal<'c>(
.execute(&mut tx)
.await?;
let mut schedulables = sqlx::query_as!(
Schedule,
"UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS false RETURNING *",
ns.path,
p_path,
w_id,
)
let mut schedulables = sqlx::query_as::<_, Schedule>(
"UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS false RETURNING *")
.bind(&ns.path)
.bind(&p_path)
.bind(&w_id)
.fetch_all(&mut tx)
.await?;
let schedule = sqlx::query_as!(Schedule,
"UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS false RETURNING *",
ns.path,
p_path,
w_id,
)
let schedule = sqlx::query_as::<_, Schedule>(
"UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS false RETURNING *")
.bind(&ns.path)
.bind(&p_path)
.bind(&w_id)
.fetch_optional(&mut tx)
.await?;
@@ -716,12 +715,12 @@ async fn create_script_internal<'c>(
ns.tag
};
let mut args: HashMap<String, serde_json::Value> = HashMap::new();
let mut args: HashMap<String, Box<serde_json::value::RawValue>> = HashMap::new();
if let Some(dm) = ns.deployment_message {
args.insert("deployment_message".to_string(), json!(dm));
args.insert("deployment_message".to_string(), to_raw_value(&dm));
}
if let Some(ref p_path) = p_path_opt {
args.insert("parent_path".to_string(), json!(p_path));
args.insert("parent_path".to_string(), to_raw_value(&p_path));
}
let tx = PushIsolationLevel::Transaction(tx);
@@ -735,7 +734,7 @@ async fn create_script_internal<'c>(
path: ns.path,
dedicated_worker: ns.dedicated_worker,
},
args,
args.into(),
&authed.username,
&authed.email,
permissioned_as,
+12 -9
View File
@@ -96,6 +96,7 @@ pub async fn test_s3_bucket(
Json(test_s3_bucket): Json<ObjectSettings>,
) -> error::Result<String> {
use bytes::Bytes;
use futures::StreamExt;
use windmill_common::ee::{get_license_plan, LicensePlan};
if matches!(get_license_plan().await, LicensePlan::Pro) {
@@ -107,6 +108,15 @@ pub async fn test_s3_bucket(
require_super_admin(&db, &authed.email).await?;
let client = build_object_store_from_settings(test_s3_bucket).await?;
let mut list = client.list(Some(&object_store::path::Path::from("".to_string())));
let first_file = list
.next()
.await
.ok_or_else(|| {
error::Error::InternalErr("Failed to list files in blob storage".to_string())
})?
.map_err(|e| anyhow::anyhow!("error listing bucket: {e}"))?;
tracing::info!("Listed files: {:?}", first_file);
let path = object_store::path::Path::from(format!(
"/test-s3-bucket-{uuid}",
uuid = uuid::Uuid::new_v4()
@@ -115,7 +125,7 @@ pub async fn test_s3_bucket(
client
.put(&path, Bytes::from_static(b"hello"))
.await
.map_err(to_anyhow)?;
.map_err(|e| anyhow::anyhow!("error writing file to {path}: {e}"))?;
let content = client
.get(&path)
.await
@@ -249,14 +259,7 @@ pub async fn get_global_setting(
pub async fn send_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Result<String> {
require_super_admin(&db, &authed.email).await?;
windmill_common::stats_ee::send_stats(
&"manual".to_string(),
&windmill_common::utils::Mode::Server,
&HTTP_CLIENT,
&db,
cfg!(feature = "enterprise"),
)
.await?;
windmill_common::stats_ee::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?;
Ok("Sent stats".to_string())
}
+11
View File
@@ -2139,6 +2139,17 @@ async fn login(
let token = create_session_token(&email, super_admin, &mut tx, cookies).await?;
audit_log(
&mut *tx,
&email,
"users.login",
ActionKind::Create,
"global",
Some(&truncate_token(&token)),
None,
)
.await?;
tx.commit().await?;
Ok(token)
}
+16 -15
View File
@@ -41,7 +41,7 @@ use windmill_common::s3_helpers::LargeFileStorage;
use windmill_common::schedule::Schedule;
use windmill_common::users::username_to_permissioned_as;
use windmill_common::variables::build_crypt;
use windmill_common::worker::CLOUD_HOSTED;
use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
use windmill_common::workspaces::WorkspaceGitSyncSettings;
use windmill_common::{
error::{to_anyhow, Error, JsonResult, Result},
@@ -58,7 +58,7 @@ use crate::oauth2_ee::InstanceEvent;
use crate::variables::{decrypt, encrypt};
use hyper::{header, StatusCode};
use serde::{Deserialize, Serialize};
use serde_json::{json, Map, Value};
use serde_json::Value;
use sqlx::{FromRow, Postgres, Transaction};
use tempfile::TempDir;
use tokio::fs::File;
@@ -476,15 +476,15 @@ async fn run_slack_message_test_job(
Path(w_id): Path<String>,
Json(req): Json<RunSlackMessageTestJobRequest>,
) -> JsonResult<RunSlackMessageTestJobResponse> {
let mut fake_result = Map::new();
fake_result.insert("error".to_string(), json!(req.test_msg));
fake_result.insert("success_result".to_string(), json!(req.test_msg));
let mut fake_result = HashMap::new();
fake_result.insert("error".to_string(), to_raw_value(&req.test_msg));
fake_result.insert("success_result".to_string(), to_raw_value(&req.test_msg));
let mut extra_args = Map::new();
extra_args.insert("channel".to_string(), json!(req.channel));
let mut extra_args = HashMap::new();
extra_args.insert("channel".to_string(), to_raw_value(&req.channel));
extra_args.insert(
"slack".to_string(),
json!(format!("$res:{WORKSPACE_SLACK_BOT_TOKEN_PATH}")),
to_raw_value(&format!("$res:{WORKSPACE_SLACK_BOT_TOKEN_PATH}")),
);
let uuid = windmill_queue::push_error_handler(
@@ -499,7 +499,7 @@ async fn run_slack_message_test_job(
sqlx::types::Json(&fake_result),
None,
Some(Utc::now()),
Some(json!(extra_args)),
Some(sqlx::types::Json(to_raw_value(&extra_args))),
authed.email.as_str(),
false,
false,
@@ -2220,6 +2220,8 @@ struct ScriptMetadata {
pub no_main_func: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub codebase: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub concurrency_key: Option<String>,
}
pub fn is_none_or_false(val: &Option<bool>) -> bool {
@@ -2499,6 +2501,7 @@ async fn tarball_workspace(
visible_to_runner_only: script.visible_to_runner_only,
no_main_func: script.no_main_func,
codebase: script.codebase,
concurrency_key: script.concurrency_key,
};
let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
archive
@@ -2588,14 +2591,13 @@ async fn tarball_workspace(
}
{
let apps = sqlx::query_as!(
AppWithLastVersion,
let apps = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by from app, app_version
WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
&w_id
)
.bind(&w_id)
.fetch_all(&mut *tx)
.await?;
@@ -2608,12 +2610,11 @@ async fn tarball_workspace(
}
if include_schedules.unwrap_or(false) {
let schedules = sqlx::query_as!(
Schedule,
let schedules = sqlx::query_as::<_, Schedule>(
"SELECT * FROM schedule
WHERE workspace_id = $1",
&w_id
)
.bind(&w_id)
.fetch_all(&mut *tx)
.await?;
+101 -5
View File
@@ -13,7 +13,6 @@ use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::flows::FlowValue;
use crate::more_serde::default_false;
const MINUTES: Duration = Duration::from_secs(60);
const HOURS: Duration = MINUTES.saturating_mul(60);
@@ -71,7 +70,7 @@ pub struct RestartedFrom {
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Iterator {
pub index: usize,
pub itered: Vec<serde_json::Value>,
pub itered: Vec<Box<serde_json::value::RawValue>>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
@@ -111,7 +110,23 @@ pub struct FlowCleanupModule {
pub flow_jobs_to_clean: Vec<Uuid>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Deserialize)]
struct UntaggedFlowStatusModule {
#[serde(rename = "type")]
type_: String,
id: Option<String>,
count: Option<u16>,
job: Option<Uuid>,
iterator: Option<Iterator>,
flow_jobs: Option<Vec<Uuid>>,
branch_chosen: Option<BranchChosen>,
branchall: Option<BranchAllStatus>,
parallel: Option<bool>,
while_loop: Option<bool>,
approvers: Option<Vec<Approval>>,
}
#[derive(Serialize, Debug, Clone)]
#[serde(tag = "type")]
pub enum FlowStatusModule {
WaitingForPriorSteps {
@@ -138,10 +153,8 @@ pub enum FlowStatusModule {
#[serde(skip_serializing_if = "Option::is_none")]
branchall: Option<BranchAllStatus>,
#[serde(skip_serializing_if = "std::ops::Not::not")]
#[serde(default = "default_false")]
parallel: bool,
#[serde(skip_serializing_if = "std::ops::Not::not")]
#[serde(default = "default_false")]
while_loop: bool,
},
Success {
@@ -165,6 +178,89 @@ pub enum FlowStatusModule {
},
}
impl<'de> Deserialize<'de> for FlowStatusModule {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let untagged: UntaggedFlowStatusModule =
UntaggedFlowStatusModule::deserialize(deserializer)?;
match untagged.type_.as_str() {
"WaitingForPriorSteps" => Ok(FlowStatusModule::WaitingForPriorSteps {
id: untagged
.id
.ok_or_else(|| serde::de::Error::missing_field("id"))?,
}),
"WaitingForEvents" => Ok(FlowStatusModule::WaitingForEvents {
id: untagged
.id
.ok_or_else(|| serde::de::Error::missing_field("id"))?,
count: untagged
.count
.ok_or_else(|| serde::de::Error::missing_field("count"))?,
job: untagged
.job
.ok_or_else(|| serde::de::Error::missing_field("job"))?,
}),
"WaitingForExecutor" => Ok(FlowStatusModule::WaitingForExecutor {
id: untagged
.id
.ok_or_else(|| serde::de::Error::missing_field("id"))?,
job: untagged
.job
.ok_or_else(|| serde::de::Error::missing_field("job"))?,
}),
"InProgress" => Ok(FlowStatusModule::InProgress {
id: untagged
.id
.ok_or_else(|| serde::de::Error::missing_field("id"))?,
job: untagged
.job
.ok_or_else(|| serde::de::Error::missing_field("job"))?,
iterator: untagged.iterator,
flow_jobs: untagged.flow_jobs,
branch_chosen: untagged.branch_chosen,
branchall: untagged.branchall,
parallel: untagged.parallel.unwrap_or(false),
while_loop: untagged.while_loop.unwrap_or(false),
}),
"Success" => Ok(FlowStatusModule::Success {
id: untagged
.id
.ok_or_else(|| serde::de::Error::missing_field("id"))?,
job: untagged
.job
.ok_or_else(|| serde::de::Error::missing_field("job"))?,
flow_jobs: untagged.flow_jobs,
branch_chosen: untagged.branch_chosen,
approvers: untagged.approvers.unwrap_or_default(),
}),
"Failure" => Ok(FlowStatusModule::Failure {
id: untagged
.id
.ok_or_else(|| serde::de::Error::missing_field("id"))?,
job: untagged
.job
.ok_or_else(|| serde::de::Error::missing_field("job"))?,
flow_jobs: untagged.flow_jobs,
branch_chosen: untagged.branch_chosen,
}),
other => Err(serde::de::Error::unknown_variant(
other,
&[
"WaitingForPriorSteps",
"WaitingForEvents",
"WaitingForExecutor",
"InProgress",
"Success",
"Failure",
],
)),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum JobResult {
SingleJob(Uuid),
+197 -20
View File
@@ -16,9 +16,7 @@ use rand::Rng;
use serde::{Deserialize, Serialize, Serializer};
use crate::{
more_serde::{
default_empty_string, default_false, default_id, default_null, default_true, is_default,
},
more_serde::{default_empty_string, default_id, default_null, default_true, is_default},
scripts::{Schema, ScriptHash, ScriptLang},
};
@@ -28,7 +26,7 @@ pub struct Flow {
pub path: String,
pub summary: String,
pub description: String,
pub value: serde_json::Value,
pub value: sqlx::types::Json<Box<serde_json::value::RawValue>>,
pub edited_by: String,
pub edited_at: chrono::DateTime<chrono::Utc>,
pub archived: bool,
@@ -220,7 +218,7 @@ pub struct Mock {
pub struct FlowModule {
#[serde(default = "default_id")]
pub id: String,
pub value: FlowModuleValue,
pub value: Box<serde_json::value::RawValue>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stop_after_if: Option<StopAfterIf>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -246,13 +244,82 @@ pub struct FlowModule {
pub continue_on_error: Option<bool>,
}
#[derive(Deserialize)]
pub struct FlowModuleValueType {
#[serde(rename = "type")]
pub type_: String,
}
#[derive(Deserialize)]
pub struct FlowModuleValueWithParallel {
#[serde(rename = "type")]
pub type_: String,
pub parallel: Option<bool>,
pub parallelism: Option<u16>,
}
impl FlowModule {
pub fn id_append(&mut self, s: &str) {
self.id = format!("{}-{}", self.id, s);
}
pub fn get_value(&self) -> anyhow::Result<FlowModuleValue> {
serde_json::from_str::<FlowModuleValue>(self.value.get()).map_err(crate::error::to_anyhow)
}
pub fn is_flow(&self) -> bool {
self.get_type().is_ok_and(|x| x == "flow")
}
pub fn get_value_with_parallel(&self) -> anyhow::Result<FlowModuleValueWithParallel> {
serde_json::from_str::<FlowModuleValueWithParallel>(self.value.get())
.map_err(crate::error::to_anyhow)
}
pub fn is_simple(&self) -> bool {
self.get_type()
.is_ok_and(|x| x == "flow" || x == "script" || x == "rawscript")
}
pub fn get_type(&self) -> anyhow::Result<String> {
serde_json::from_str::<FlowModuleValueType>(self.value.get())
.map_err(crate::error::to_anyhow)
.map(|x| x.type_)
}
}
#[derive(Deserialize, Serialize, Debug, Clone)]
#[derive(Deserialize)]
pub struct UntaggedInputTransform {
#[serde(rename = "type")]
pub type_: String,
pub value: Option<Box<serde_json::value::RawValue>>,
pub expr: Option<String>,
}
impl<'de> Deserialize<'de> for InputTransform {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let untagged: UntaggedInputTransform = UntaggedInputTransform::deserialize(deserializer)?;
match untagged.type_.as_str() {
"static" => {
let value = untagged.value.unwrap_or_else(default_null);
Ok(InputTransform::Static { value })
}
"javascript" => {
let expr = untagged.expr.unwrap_or_else(default_empty_string);
Ok(InputTransform::Javascript { expr })
}
other => Err(serde::de::Error::unknown_variant(
other,
&["static", "javascript"],
)),
}
}
}
#[derive(Serialize, Debug, Clone)]
#[serde(
tag = "type",
rename_all(serialize = "lowercase", deserialize = "lowercase")
@@ -260,7 +327,7 @@ impl FlowModule {
pub enum InputTransform {
Static {
#[serde(default = "default_null")]
value: serde_json::Value,
value: Box<serde_json::value::RawValue>,
},
Javascript {
#[serde(default = "default_empty_string")]
@@ -269,25 +336,19 @@ pub enum InputTransform {
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BranchOneModules {
pub struct Branch {
#[serde(skip_serializing_if = "Option::is_none")]
pub summary: Option<String>,
#[serde(default = "default_empty_string")]
pub expr: String,
pub modules: Vec<FlowModule>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BranchAllModules {
#[serde(skip_serializing_if = "Option::is_none")]
pub summary: Option<String>,
pub modules: Vec<FlowModule>,
#[serde(default = "default_true")]
pub skip_failure: bool,
#[serde(default = "default_true")]
pub parallel: bool,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Serialize, Debug, Clone)]
#[serde(
tag = "type",
rename_all(serialize = "lowercase", deserialize = "lowercase")
@@ -313,7 +374,6 @@ pub enum FlowModuleValue {
modules: Vec<FlowModule>,
#[serde(default = "default_true")]
skip_failures: bool,
#[serde(default = "default_false")]
parallel: bool,
#[serde(skip_serializing_if = "Option::is_none")]
parallelism: Option<u16>,
@@ -324,11 +384,11 @@ pub enum FlowModuleValue {
skip_failures: bool,
},
BranchOne {
branches: Vec<BranchOneModules>,
branches: Vec<Branch>,
default: Vec<FlowModule>,
},
BranchAll {
branches: Vec<BranchAllModules>,
branches: Vec<Branch>,
#[serde(default = "default_true")]
parallel: bool,
},
@@ -354,6 +414,123 @@ pub enum FlowModuleValue {
Identity,
}
#[derive(Deserialize)]
struct UntaggedFlowModuleValue {
#[serde(rename = "type")]
type_: String,
#[serde(alias = "input_transform")]
input_transforms: Option<HashMap<String, InputTransform>>,
path: Option<String>,
hash: Option<ScriptHash>,
tag_override: Option<String>,
iterator: Option<InputTransform>,
modules: Option<Vec<FlowModule>>,
skip_failures: Option<bool>,
parallel: Option<bool>,
parallelism: Option<u16>,
branches: Option<Vec<Branch>>,
default: Option<Vec<FlowModule>>,
content: Option<String>,
lock: Option<String>,
tag: Option<String>,
language: Option<ScriptLang>,
custom_concurrency_key: Option<String>,
concurrent_limit: Option<i32>,
concurrency_time_window_s: Option<i32>,
}
impl<'de> Deserialize<'de> for FlowModuleValue {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let untagged: UntaggedFlowModuleValue = UntaggedFlowModuleValue::deserialize(deserializer)?;
match untagged.type_.as_str() {
"script" => Ok(FlowModuleValue::Script {
input_transforms: untagged.input_transforms.unwrap_or_default(),
path: untagged
.path
.ok_or_else(|| serde::de::Error::missing_field("path"))?,
hash: untagged.hash,
tag_override: untagged.tag_override,
}),
"flow" => Ok(FlowModuleValue::Flow {
input_transforms: untagged.input_transforms.unwrap_or_default(),
path: untagged
.path
.ok_or_else(|| serde::de::Error::missing_field("path"))?,
}),
"forloopflow" => Ok(FlowModuleValue::ForloopFlow {
iterator: untagged
.iterator
.ok_or_else(|| serde::de::Error::missing_field("iterator"))?,
modules: untagged
.modules
.ok_or_else(|| serde::de::Error::missing_field("modules"))?,
skip_failures: untagged.skip_failures.unwrap_or(true),
parallel: untagged.parallel.unwrap_or(false),
parallelism: untagged.parallelism,
}),
"whileloopflow" => Ok(FlowModuleValue::WhileloopFlow {
modules: untagged
.modules
.ok_or_else(|| serde::de::Error::missing_field("modules"))?,
skip_failures: untagged.skip_failures.unwrap_or(false),
}),
"branchone" => Ok(FlowModuleValue::BranchOne {
branches: untagged
.branches
.ok_or_else(|| serde::de::Error::missing_field("branches"))?,
default: untagged
.default
.ok_or_else(|| serde::de::Error::missing_field("default"))?,
}),
"branchall" => Ok(FlowModuleValue::BranchAll {
branches: untagged
.branches
.ok_or_else(|| serde::de::Error::missing_field("branches"))?,
parallel: untagged.parallel.unwrap_or(true),
}),
"rawscript" => Ok(FlowModuleValue::RawScript {
input_transforms: untagged.input_transforms.unwrap_or_default(),
content: untagged
.content
.ok_or_else(|| serde::de::Error::missing_field("content"))?,
lock: untagged.lock,
path: untagged.path,
tag: untagged.tag,
language: untagged
.language
.ok_or_else(|| serde::de::Error::missing_field("language"))?,
custom_concurrency_key: untagged.custom_concurrency_key,
concurrent_limit: untagged.concurrent_limit,
concurrency_time_window_s: untagged.concurrency_time_window_s,
}),
"identity" => Ok(FlowModuleValue::Identity),
other => Err(serde::de::Error::unknown_variant(
other,
&[
"script",
"flow",
"forloopflow",
"whileloopflow",
"branchone",
"branchall",
"rawscript",
"identity",
],
)),
}
}
}
impl Into<Box<serde_json::value::RawValue>> for FlowModuleValue {
fn into(self) -> Box<serde_json::value::RawValue> {
crate::worker::to_raw_value(&self)
}
}
impl FlowModuleValue {
pub fn is_simple(&self) -> bool {
match self {
@@ -391,7 +568,7 @@ pub fn add_virtual_items_if_necessary(modules: &mut Vec<FlowModule>) {
{
modules.push(FlowModule {
id: format!("{}-v", modules[modules.len() - 1].id),
value: FlowModuleValue::Identity,
value: crate::worker::to_raw_value(&FlowModuleValue::Identity),
stop_after_if: None,
summary: Some("Virtual module needed for suspend/sleep when last module".to_string()),
mock: None,
+6 -4
View File
@@ -137,9 +137,11 @@ impl QueuedJob {
}
pub fn parse_raw_flow(&self) -> Option<FlowValue> {
self.raw_flow
.as_ref()
.and_then(|v| serde_json::from_str::<FlowValue>((**v).get()).ok())
self.raw_flow.as_ref().and_then(|v| {
let str = (**v).get();
// tracing::error!("raw_flow: {}", str);
return serde_json::from_str::<FlowValue>(str).ok();
})
}
pub fn parse_flow_status(&self) -> Option<FlowStatus> {
@@ -327,7 +329,7 @@ pub enum JobPayload {
SingleScriptFlow {
path: String,
hash: ScriptHash,
args: HashMap<String, serde_json::Value>,
args: HashMap<String, Box<serde_json::value::RawValue>>,
retry: Retry, // for now only used to retry the script, so retry is necessarily present
custom_concurrency_key: Option<String>,
concurrent_limit: Option<i32>,
+3 -2
View File
@@ -10,6 +10,7 @@
use crate::utils::rd_string;
use serde::{Deserialize, Deserializer};
use serde_json::value::RawValue;
use std::{fmt::Display, str::FromStr};
pub fn default_true() -> bool {
@@ -20,8 +21,8 @@ pub fn default_false() -> bool {
false
}
pub fn default_null() -> serde_json::Value {
serde_json::Value::Null
pub fn default_null() -> Box<RawValue> {
RawValue::from_string("null".to_string()).unwrap()
}
pub fn default_empty_string() -> String {
+3 -3
View File
@@ -23,7 +23,7 @@ pub struct Schedule {
pub enabled: bool,
pub script_path: String,
pub is_flow: bool,
pub args: Option<serde_json::Value>,
pub args: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
pub extra_perms: serde_json::Value,
pub email: String,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -35,13 +35,13 @@ pub struct Schedule {
#[serde(skip_serializing_if = "Option::is_none")]
pub on_failure_exact: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub on_failure_extra_args: Option<serde_json::Value>,
pub on_failure_extra_args: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub on_recovery: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub on_recovery_times: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub on_recovery_extra_args: Option<serde_json::Value>,
pub on_recovery_extra_args: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
pub ws_error_handler_muted: bool,
#[serde(skip_serializing_if = "Option::is_none")]
+3 -6
View File
@@ -18,7 +18,6 @@ use crate::{
};
use serde::de::Error as _;
use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize};
use serde_json::to_string_pretty;
use crate::utils::StripPath;
@@ -227,13 +226,11 @@ pub struct ScriptHistoryUpdate {
#[derive(Serialize, Deserialize, Debug, sqlx::Type)]
#[sqlx(transparent)]
#[serde(transparent)]
pub struct Schema(pub serde_json::Value);
pub struct Schema(pub sqlx::types::Json<Box<serde_json::value::RawValue>>);
impl Hash for Schema {
fn hash<H: Hasher>(&self, state: &mut H) {
if let Ok(s) = to_string_pretty(&self.0) {
s.hash(state);
}
self.0.get().hash(state);
}
}
@@ -413,6 +410,6 @@ pub struct HubScript {
pub content: String,
pub lockfile: Option<String>,
pub language: ScriptLang,
pub schema: serde_json::Value,
pub schema: Box<serde_json::value::RawValue>,
pub summary: Option<String>,
}
+15 -5
View File
@@ -1,4 +1,6 @@
use crate::{error::Result, scripts::ScriptLang, utils::Mode, DB};
use sqlx::Postgres;
use crate::{error::Result, scripts::ScriptLang, DB};
pub async fn get_disable_stats_setting(_db: &DB) -> bool {
// stats details are closed source
@@ -8,10 +10,8 @@ pub async fn get_disable_stats_setting(_db: &DB) -> bool {
pub async fn schedule_stats(
_instance_name: String,
_mode: Mode,
_db: &DB,
_http_client: &reqwest::Client,
_is_enterprise: bool,
) -> () {
// stats details are closed source
}
@@ -25,11 +25,21 @@ struct JobsUsage {
pub async fn send_stats(
_instance_name: &String,
_mode: &Mode,
_http_client: &reqwest::Client,
_db: &DB,
_is_enterprise: bool,
) -> Result<()> {
// stats details are closed source
Ok(())
}
pub struct ActiveUserUsage {
pub author_count: Option<i32>,
pub operator_count: Option<i32>,
}
pub async fn get_user_usage<'c, E: sqlx::Executor<'c, Database = Postgres>>(
_db: E,
) -> Result<ActiveUserUsage> {
let usage = ActiveUserUsage { author_count: None, operator_count: None };
Ok(usage)
}
+1 -1
View File
@@ -161,7 +161,7 @@ pub async fn get_reserved_variables(
)
} else if let Some(script_path) = path.clone() {
let script_path = if script_path.ends_with("/") {
"noname".to_string()
format!("{script_path}state")
} else {
script_path
};
+155 -114
View File
@@ -762,13 +762,18 @@ pub async fn add_completed_job<
}
}
if queued_job.concurrent_limit.is_some() {
let concurrency_key = concurrency_key(db, queued_job).await.unwrap_or_else(|e| {
tracing::error!(
"Could not get concurrency key for job {} defaulting to default key: {e:?}",
queued_job.id
);
return queued_job.full_path_with_workspace();
});
let concurrency_key = match concurrency_key(db, queued_job).await {
Ok(c) => c,
Err(e) => {
tracing::error!(
"Could not get concurrency key for job {} defaulting to default key: {e:?}",
queued_job.id
);
legacy_concurrency_key(db, queued_job)
.await
.unwrap_or_else(|| queued_job.full_path_with_workspace())
}
};
if let Err(e) = sqlx::query_scalar!(
"UPDATE concurrency_counter SET job_uuids = job_uuids - $2 WHERE concurrency_id = $1",
concurrency_key,
@@ -982,7 +987,11 @@ pub async fn add_completed_job<
.unwrap_or_else(|| ScriptLang::Deno),
priority: queued_job.priority,
},
queued_job.args.clone(),
queued_job
.args
.as_ref()
.map(|x| PushArgs { args: x.0.clone(), extra: HashMap::new() })
.unwrap_or_else(PushArgs::empty),
&queued_job.created_by,
&queued_job.email,
queued_job.permissioned_as.clone(),
@@ -1063,7 +1072,7 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel<
error_message: String,
) -> () {
let w_id = &queued_job.workspace_id;
let (error_handler, error_handler_extra_args) = sqlx::query_as::<_, (Option<String>, Option<serde_json::Value>)>(
let (error_handler, error_handler_extra_args) = sqlx::query_as::<_, (Option<String>, Option<Json<Box<RawValue>>>)>(
"SELECT error_handler, error_handler_extra_args FROM workspace_settings WHERE workspace_id = $1",
).bind(&w_id)
.fetch_optional(db)
@@ -1122,7 +1131,7 @@ pub async fn send_error_to_workspace_handler<
result: Json<&'a T>,
) -> Result<(), Error> {
let w_id = &queued_job.workspace_id;
let (error_handler, error_handler_extra_args, error_handler_muted_on_cancel) = sqlx::query_as::<_, (Option<String>, Option<serde_json::Value>, bool)>(
let (error_handler, error_handler_extra_args, error_handler_muted_on_cancel) = sqlx::query_as::<_, (Option<String>, Option<Json<Box<RawValue>>>, bool)>(
"SELECT error_handler, error_handler_extra_args, error_handler_muted_on_cancel FROM workspace_settings WHERE workspace_id = $1",
).bind(&w_id)
.fetch_optional(db)
@@ -1256,10 +1265,10 @@ pub async fn handle_maybe_scheduled_job<'c, R: rsmq_async::RsmqConnection + Clon
}
}
#[derive(Clone, Serialize)]
#[derive(Clone, Serialize, FromRow)]
struct CompletedJobSubset {
success: bool,
result: Option<serde_json::Value>,
result: Option<sqlx::types::Json<Box<RawValue>>>,
started_at: chrono::DateTime<chrono::Utc>,
}
async fn apply_schedule_handlers<
@@ -1285,15 +1294,15 @@ async fn apply_schedule_handlers<
let times = schedule.on_failure_times.unwrap_or(1).max(1);
let exact = schedule.on_failure_exact.unwrap_or(false);
if times > 1 || exact {
let past_jobs = sqlx::query_as!(
CompletedJobSubset,
let past_jobs = sqlx::query_as::<_, CompletedJobSubset>(
"SELECT success, result, started_at FROM completed_job WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4 ORDER BY created_at DESC LIMIT $5",
&schedule.workspace_id,
&schedule.path,
script_path,
job_id,
if exact { times } else { times - 1 } as i64,
).fetch_all(db).await?;
)
.bind(&schedule.workspace_id)
.bind(&schedule.path)
.bind(script_path)
.bind(job_id)
.bind(if exact { times } else { times - 1 } as i64,)
.fetch_all(db).await?;
let match_times = if exact {
past_jobs.len() == times as usize
@@ -1334,15 +1343,15 @@ async fn apply_schedule_handlers<
if let Some(on_recovery_path) = schedule.on_recovery.clone() {
let mut tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into();
let times = schedule.on_recovery_times.unwrap_or(1).max(1);
let past_jobs = sqlx::query_as!(
CompletedJobSubset,
let past_jobs = sqlx::query_as::<_, CompletedJobSubset>(
"SELECT success, result, started_at FROM completed_job WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4 ORDER BY created_at DESC LIMIT $5",
&schedule.workspace_id,
&schedule.path,
script_path,
job_id,
times as i64,
).fetch_all(db).await?;
)
.bind(&schedule.workspace_id)
.bind(&schedule.path)
.bind(script_path)
.bind(job_id)
.bind(times as i64)
.fetch_all(db).await?;
if past_jobs.len() < times as usize {
return Ok(());
@@ -1399,7 +1408,7 @@ pub async fn push_error_handler<
result: Json<&'a T>,
failed_times: Option<i32>,
started_at: Option<DateTime<Utc>>,
extra_args: Option<serde_json::Value>,
extra_args: Option<Json<Box<RawValue>>>,
email: &str,
is_schedule_error_handler: bool,
is_global_error_handler: bool,
@@ -1428,10 +1437,8 @@ pub async fn push_error_handler<
}
if let Some(args_v) = extra_args {
if let serde_json::Value::Object(args_m) = args_v {
for (k, v) in args_m {
extra.insert(k, to_raw_value(&v));
}
if let Ok(args_m) = serde_json::from_str::<HashMap<String, Box<RawValue>>>(args_v.get()) {
extra.extend(args_m);
} else {
return Err(error::Error::ExecutionErr(
"args of scripts needs to be dict".to_string(),
@@ -1457,7 +1464,7 @@ pub async fn push_error_handler<
tx,
handler_w_id,
payload,
PushArgs { extra, args: Json(&result) },
PushArgs { extra, args: result },
if is_global_error_handler {
"global"
} else if is_schedule_error_handler {
@@ -1496,16 +1503,10 @@ pub async fn push_error_handler<
return Ok(uuid);
}
fn sanitize_result<T: Serialize + Send + Sync>(result: Json<&T>) -> serde_json::Value {
let result =
serde_json::from_str(&serde_json::to_string(result.0).unwrap_or_else(|_| "{}".to_string()))
.unwrap_or_else(|_| json!({}));
let result = if result.is_object() || result.is_null() {
result
} else {
json!({ "error": result })
};
result
fn sanitize_result<T: Serialize + Send + Sync>(result: Json<&T>) -> HashMap<String, Box<RawValue>> {
let as_str = serde_json::to_string(result.0).unwrap_or_else(|_| "{}".to_string());
serde_json::from_str::<HashMap<String, Box<RawValue>>>(&as_str)
.unwrap_or_else(|_| [("error".to_string(), RawValue::from_string(as_str).unwrap())].into())
}
// #[derive(Serialize)]
@@ -1534,33 +1535,33 @@ async fn handle_recovered_schedule<
successful_job_result: Json<&'a T>,
successful_times: i32,
successful_job_started_at: DateTime<Utc>,
extra_args: Option<serde_json::Value>,
extra_args: Option<Json<Box<RawValue>>>,
) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
let (payload, tag) = get_payload_tag_from_prefixed_path(on_recovery_path, db, w_id).await?;
let mut args = error_job
.result
.unwrap_or(json!({}))
.as_object()
.unwrap()
.clone();
args.insert("error_started_at".to_string(), json!(error_job.started_at));
args.insert("schedule_path".to_string(), json!(schedule_path));
args.insert("path".to_string(), json!(script_path));
args.insert("is_flow".to_string(), json!(is_flow));
args.insert(
"success_result".to_string(),
serde_json::from_str(&serde_json::to_string(&successful_job_result).unwrap())
.unwrap_or_else(|_| json!("{}")),
let mut extra = HashMap::new();
extra.insert(
"error_started_at".to_string(),
to_raw_value(&error_job.started_at),
);
args.insert("success_times".to_string(), json!(successful_times));
args.insert(
extra.insert("schedule_path".to_string(), to_raw_value(&schedule_path));
extra.insert("path".to_string(), to_raw_value(&script_path));
extra.insert("is_flow".to_string(), to_raw_value(&is_flow));
extra.insert(
"success_result".to_string(),
serde_json::from_str::<Box<RawValue>>(
&serde_json::to_string(&successful_job_result).unwrap(),
)
.unwrap_or_else(|_| serde_json::value::RawValue::from_string("{}".to_string()).unwrap()),
);
extra.insert("success_times".to_string(), to_raw_value(&successful_times));
extra.insert(
"success_started_at".to_string(),
json!(successful_job_started_at),
to_raw_value(&successful_job_started_at),
);
if let Some(args_v) = extra_args {
if let serde_json::Value::Object(args_m) = args_v {
args.extend(args_m);
if let Ok(args_m) = serde_json::from_str::<HashMap<String, Box<RawValue>>>(args_v.get()) {
extra.extend(args_m);
} else {
return Err(error::Error::ExecutionErr(
"args of scripts needs to be dict".to_string(),
@@ -1574,15 +1575,21 @@ async fn handle_recovered_schedule<
{
// default slack error handler being used -> we need to inject the slack token
let slack_resource = format!("$res:{WORKSPACE_SLACK_BOT_TOKEN_PATH}");
args.insert("slack".to_string(), json!(slack_resource));
extra.insert("slack".to_string(), to_raw_value(&slack_resource));
}
let args = error_job
.result
.and_then(|x| serde_json::from_str::<HashMap<String, Box<RawValue>>>(x.0.get()).ok())
.unwrap_or_else(HashMap::new);
let tx = PushIsolationLevel::Transaction(tx);
let (uuid, tx) = push(
&db,
tx,
w_id,
payload,
args,
PushArgs { extra: extra, args: args },
SCHEDULE_RECOVERY_HANDLER_USERNAME,
SCHEDULE_RECOVERY_HANDLER_USER_EMAIL,
ERROR_HANDLER_USER_GROUP.to_string(),
@@ -1653,13 +1660,18 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
// Else the job is subject to concurrency limits
let job_script_path = pulled_job.script_path.clone().unwrap();
let job_concurrency_key = concurrency_key(db, &pulled_job).await.unwrap_or_else(|e| {
tracing::error!(
"Could not get concurrency key for job {} defaulting to default key: {e:?}",
pulled_job.id
);
return pulled_job.full_path_with_workspace();
});
let job_concurrency_key = match concurrency_key(db, &pulled_job).await {
Ok(key) => key,
Err(e) => {
tracing::error!(
"Could not get concurrency key for job {} defaulting to default key: {e:?}",
pulled_job.id
);
legacy_concurrency_key(db, &pulled_job)
.await
.unwrap_or_else(|| pulled_job.full_path_with_workspace())
}
};
tracing::debug!("Concurrency key is '{}'", job_concurrency_key);
let job_custom_concurrent_limit = pulled_job.concurrent_limit.unwrap();
// setting concurrency_time_window to 0 will count only the currently running jobs
@@ -2017,6 +2029,42 @@ pub async fn custom_concurrency_key(
.await
}
async fn legacy_concurrency_key(db: &Pool<Postgres>, queued_job: &QueuedJob) -> Option<String> {
let r = if queued_job.is_flow() {
sqlx::query_scalar!(
"SELECT value->>'concurrency_key' FROM flow WHERE path = $1 AND workspace_id = $2",
queued_job.script_path,
queued_job.workspace_id
)
.fetch_optional(db)
.await
} else {
sqlx::query_scalar!(
"SELECT concurrency_key FROM script WHERE hash = $1 AND workspace_id = $2",
queued_job.script_hash.unwrap_or(ScriptHash(0)).0,
queued_job.workspace_id
)
.fetch_optional(db)
.await
}
.ok()
.flatten()
.flatten();
r.map(|x| {
interpolate_args(
x,
&queued_job
.args
.clone()
.map(|x| x.0)
.unwrap_or_default()
.into(),
&queued_job.workspace_id,
)
})
}
async fn concurrency_key(
db: &Pool<Postgres>,
queued_job: &QueuedJob,
@@ -2028,27 +2076,21 @@ async fn concurrency_key(
)
}
fn interpolate_args<T: Serialize>(
x: String,
args: &T,
workspace_id: &str,
parsed_args: &mut Option<serde_json::Value>,
) -> String {
fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> String {
// Save this value to avoid parsing twice
if parsed_args.is_none() {
*parsed_args = Some(serde_json::to_value(args).unwrap_or_default());
}
let value = parsed_args.as_ref().unwrap();
let workspaced = x.as_str().replace("$workspace", workspace_id).to_string();
if RE_ARG_TAG.is_match(&workspaced) {
let mut interpolated = workspaced.clone();
for cap in RE_ARG_TAG.captures_iter(&workspaced) {
let arg_name = cap.get(1).unwrap().as_str();
let arg_value = value
let arg_value = args
.args
.get(arg_name)
.and_then(|x| x.as_str())
.or(args.extra.get(arg_name))
.map(|x| x.get())
.unwrap_or_default();
interpolated = interpolated.replace(format!("$args[{}]", arg_name).as_str(), arg_value);
interpolated =
interpolated.replace(format!("$args[{}]", arg_name).as_str(), &arg_value);
}
interpolated
} else {
@@ -2492,11 +2534,11 @@ macro_rules! fetch_scalar_isolated {
use sqlx::types::JsonRawValue;
#[derive(Serialize, Debug)]
pub struct PushArgs<T> {
pub struct PushArgs {
#[serde(flatten)]
pub extra: HashMap<String, Box<RawValue>>,
#[serde(flatten)]
pub args: Json<T>,
pub args: HashMap<String, Box<RawValue>>,
}
#[derive(Deserialize)]
@@ -2553,7 +2595,7 @@ impl DecodeQueries {
}
}
impl<T> PushArgs<T> {
impl PushArgs {
pub fn insert<K: Into<String>, V: Into<Box<RawValue>>>(&mut self, k: K, v: V) {
self.extra.insert(k.into(), v.into());
}
@@ -2566,7 +2608,7 @@ pub struct RequestQuery {
}
#[axum::async_trait]
impl<S> FromRequest<S, axum::body::Body> for PushArgs<HashMap<String, Box<RawValue>>>
impl<S> FromRequest<S, axum::body::Body> for PushArgs
where
S: Send + Sync,
{
@@ -2610,12 +2652,12 @@ where
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null));
let mut hm = HashMap::new();
hm.insert("body".to_string(), args);
Ok(PushArgs { extra, args: Json(hm) })
Ok(PushArgs { extra, args: hm })
} else {
let hm = serde_json::from_str::<Option<HashMap<String, Box<JsonRawValue>>>>(&str)
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())?
.unwrap_or_else(HashMap::new);
Ok(PushArgs { extra, args: Json(hm) })
Ok(PushArgs { extra, args: hm })
}
} else if content_type
.unwrap()
@@ -2641,7 +2683,7 @@ where
.map(|(k, v)| (k, to_raw_value(&v)))
.collect::<HashMap<_, _>>();
return Ok(PushArgs { extra, args: Json(payload) });
return Ok(PushArgs { extra, args: payload });
} else {
Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response())
}
@@ -2679,9 +2721,9 @@ pub fn build_extra(
args
}
impl PushArgs<HashMap<String, Box<RawValue>>> {
impl PushArgs {
pub fn empty() -> Self {
PushArgs { extra: HashMap::new(), args: Json(HashMap::new()) }
PushArgs { extra: HashMap::new(), args: HashMap::new() }
}
}
@@ -2689,9 +2731,9 @@ pub fn empty_result() -> Box<RawValue> {
return JsonRawValue::from_string("{}".to_string()).unwrap();
}
impl From<HashMap<String, Box<JsonRawValue>>> for PushArgs<HashMap<String, Box<JsonRawValue>>> {
impl From<HashMap<String, Box<JsonRawValue>>> for PushArgs {
fn from(value: HashMap<String, Box<JsonRawValue>>) -> Self {
PushArgs { extra: HashMap::new(), args: Json(value) }
PushArgs { extra: HashMap::new(), args: value }
}
}
@@ -2706,12 +2748,12 @@ lazy_static::lazy_static! {
}
// #[instrument(level = "trace", skip_all)]
pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection + Send + 'c>(
pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
_db: &Pool<Postgres>,
mut tx: PushIsolationLevel<'c, R>,
workspace_id: &str,
job_payload: JobPayload,
args: T,
args: PushArgs,
user: &str,
mut email: &str,
mut permissioned_as: String,
@@ -3143,12 +3185,14 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
let flow_value = FlowValue {
modules: vec![FlowModule {
id: "a".to_string(),
value: windmill_common::flows::FlowModuleValue::Script {
input_transforms: input_transforms,
path: path.clone(),
hash: Some(hash),
tag_override: tag_override,
},
value: windmill_common::worker::to_raw_value(
&windmill_common::flows::FlowModuleValue::Script {
input_transforms: input_transforms,
path: path.clone(),
hash: Some(hash),
tag_override: tag_override,
},
),
stop_after_if: None,
summary: None,
suspend: None,
@@ -3375,8 +3419,6 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
let per_workspace: bool = DEFAULT_TAGS_PER_WORKSPACE.load(std::sync::atomic::Ordering::Relaxed);
let mut parsed_args: Option<serde_json::Value> = None;
let tag = if dedicated_worker.is_some_and(|x| x) {
format!(
"{}:{}{}",
@@ -3399,8 +3441,7 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
tag = None;
}
let interpolated_tag =
tag.map(|x| interpolate_args(x, &args, workspace_id, &mut parsed_args));
let interpolated_tag = tag.map(|x| interpolate_args(x, &args, workspace_id));
let default = || {
let ntag = if job_kind == JobKind::Flow
@@ -3468,7 +3509,7 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
if concurrent_limit.is_some() {
let concurrency_key = custom_concurrency_key
.map(|x| interpolate_args(x, &args, workspace_id, &mut parsed_args))
.map(|x| interpolate_args(x, &args, workspace_id))
.unwrap_or(fullpath_with_workspace(
workspace_id,
script_path.as_ref(),
@@ -3504,7 +3545,7 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
script_path.clone(),
raw_code,
raw_lock,
Json(args) as Json<T>,
Json(args) as Json<PushArgs>,
job_kind.clone() as JobKind,
schedule_path,
raw_flow.map(|f| serde_json::json!(f)),
@@ -3668,8 +3709,8 @@ async fn restarted_flows_resolution(
module.id()
)))?;
match module_definition.value.clone() {
FlowModuleValue::BranchAll { branches, parallel, .. } => {
match module_definition.get_value() {
Ok(FlowModuleValue::BranchAll { branches, parallel, .. }) => {
if parallel {
return Err(Error::InternalErr(format!(
"Module {} is a parallel branchall. It can only be restarted at a given branch if it's sequential",
@@ -3701,7 +3742,7 @@ async fn restarted_flows_resolution(
while_loop: false,
});
}
FlowModuleValue::ForloopFlow { parallel, .. } => {
Ok(FlowModuleValue::ForloopFlow { parallel, .. }) => {
if parallel {
return Err(Error::InternalErr(format!(
"Module {} is not parallel loop. It can only be restarted at a given iteration if it's sequential",
+9 -8
View File
@@ -66,10 +66,12 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
return Ok(tx);
}
let mut args: serde_json::Map<String, serde_json::Value> = serde_json::Map::new();
let mut args: HashMap<String, Box<serde_json::value::RawValue>> = HashMap::new();
if let Some(args_v) = &schedule.args {
if let serde_json::Value::Object(args_m) = args_v {
if let Ok(args_m) =
serde_json::from_str::<HashMap<String, Box<serde_json::value::RawValue>>>(args_v.get())
{
args = args_m.clone()
} else {
return Err(error::Error::ExecutionErr(
@@ -121,7 +123,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
err.to_string(),
))
})?;
let mut static_args = HashMap::<String, serde_json::Value>::new();
let mut static_args = HashMap::<String, Box<serde_json::value::RawValue>>::new();
for (arg_name, arg_value) in args.clone() {
static_args.insert(arg_name, arg_value);
}
@@ -186,7 +188,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
tx,
&schedule.workspace_id,
payload,
args,
crate::PushArgs { args, extra: HashMap::new() },
&schedule_to_user(&schedule.path),
&schedule.email,
username_to_permissioned_as(&schedule.edited_by),
@@ -213,12 +215,11 @@ pub async fn get_schedule_opt<'c>(
w_id: &str,
path: &str,
) -> Result<Option<Schedule>> {
let schedule_opt = sqlx::query_as!(
Schedule,
let schedule_opt = sqlx::query_as::<_, Schedule>(
"SELECT * FROM schedule WHERE path = $1 AND workspace_id = $2",
path,
w_id
)
.bind(path)
.bind(w_id)
.fetch_optional(&mut **db)
.await?;
Ok(schedule_opt)
-6
View File
@@ -683,12 +683,6 @@ async fn compact_logs(
.unwrap_or(to_keep_in_db);
let stored_in_storage_to_newline = stored_in_storage_len + extra_to_newline;
tracing::error!(
"nlog_len: {nlogs_len}, modulo: {to_keep_in_db}, stored_in_storage_len: {stored_in_storage_len}, extra_to_newline: {extra_to_newline}, extra_split: {extra_split}, {}",
nlogs
.chars()
.skip(stored_in_storage_len).collect::<String>());
let (append_to_storage, stored_in_db) = if extra_split {
if stored_in_storage_to_newline == nlogs.len() {
(nlogs.as_ref(), "".to_string())
+7 -4
View File
@@ -399,7 +399,7 @@ async function variable(path) {{
return await Deno.core.ops.op_variable(path);
}}
async function resource(path) {{
return await Deno.core.ops.op_resource(path);
return JSON.parse(await Deno.core.ops.op_resource(path));
}}
"#,
);
@@ -518,14 +518,17 @@ async fn op_get_id(
}
#[op2(async)]
#[serde]
#[string]
async fn op_resource(
op_state: Rc<RefCell<OpState>>,
#[string] path: String,
) -> Result<serde_json::Value, anyhow::Error> {
) -> Result<Option<String>, anyhow::Error> {
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
if let Some(client) = client {
client.get_resource_value_interpolated(&path, None).await
client
.get_resource_value_interpolated::<Option<Box<RawValue>>>(&path, None)
.await
.map(|x| x.map(|x| x.get().to_string()))
} else {
anyhow::bail!("No client found in op state");
}
+145 -107
View File
@@ -219,6 +219,8 @@ pub const DEFAULT_NATIVE_JOBS: usize = 1;
const VACUUM_PERIOD: u32 = 50000;
const DROP_CACHE_PERIOD: u32 = 1000;
pub const MAX_BUFFERED_DEDICATED_JOBS: usize = 3;
#[cfg(feature = "prometheus")]
@@ -591,6 +593,30 @@ impl JobCompletedSender {
}
}
// on linux, we drop caches every DROP_CACHE_PERIOD to avoid OOM killer believing we are using too much memory just because we create lots of files when executing jobs
#[cfg(any(target_os = "linux"))]
pub async fn drop_cache() {
tracing::info!("Syncing and dropping linux file caches to reduce memory usage");
// Run the sync command
if let Err(e) = tokio::process::Command::new("sync").status().await {
tracing::error!("Failed to run sync command: {}", e);
return;
}
// Open /proc/sys/vm/drop_caches for writing asynchronously
match tokio::fs::File::create("/proc/sys/vm/drop_caches").await {
Ok(mut file) => {
// Write '3' to the file to drop caches
if let Err(e) = tokio::io::AsyncWriteExt::write_all(&mut file, b"3").await {
tracing::warn!("Failed to write to /proc/sys/vm/drop_caches (expected to not work in not in privileged mode, only required to forcefully drop the cache to avoid spurrious oom killer): {}", e);
}
}
Err(e) => {
tracing::warn!("Failed to open /proc/sys/vm/drop_caches (expected to not work in not in privileged mode, only required to forcefully drop the cache to avoid spurrious oom killer):: {}", e);
}
}
}
#[tracing::instrument(name = "worker", level = "info", skip_all, fields(worker = %worker_name))]
pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
db: &Pool<Postgres>,
@@ -1426,6 +1452,12 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
jobs_executed += 1;
}
#[cfg(any(target_os = "linux"))]
if (jobs_executed as u32 + 1) % DROP_CACHE_PERIOD == 0 {
drop_cache().await;
jobs_executed += 1;
}
let next_job = {
// println!("2: {:?}", instant.elapsed());
#[cfg(feature = "benchmark")]
@@ -1847,75 +1879,40 @@ async fn spawn_dedicated_workers_for_flow(
let mut workers = vec![];
let mut script_path_to_worker: HashMap<String, Sender<Arc<QueuedJob>>> = HashMap::new();
for module in modules.iter() {
match &module.value {
FlowModuleValue::Script { path, hash, .. } => {
let key = format!(
"{}:{}",
path,
hash.clone()
.map(|x| x.to_string())
.unwrap_or_else(|| "".to_string())
);
if let Some(sender) = script_path_to_worker.get(&key) {
workers.push((module.id.clone(), sender.clone(), None));
} else {
if let Some(dedi_w) = spawn_dedicated_worker(
SpawnWorker::Script { path: path.to_string(), hash: hash.clone() },
w_id,
killpill_tx.clone(),
killpill_rx,
db,
worker_dir,
base_internal_url,
worker_name,
job_completed_tx,
Some(module.id.clone()),
)
.await
{
script_path_to_worker.insert(key, dedi_w.1.clone());
workers.push(dedi_w);
let value = module.get_value();
if let Ok(value) = value {
match &value {
FlowModuleValue::Script { path, hash, .. } => {
let key = format!(
"{}:{}",
path,
hash.clone()
.map(|x| x.to_string())
.unwrap_or_else(|| "".to_string())
);
if let Some(sender) = script_path_to_worker.get(&key) {
workers.push((module.id.clone(), sender.clone(), None));
} else {
if let Some(dedi_w) = spawn_dedicated_worker(
SpawnWorker::Script { path: path.to_string(), hash: hash.clone() },
w_id,
killpill_tx.clone(),
killpill_rx,
db,
worker_dir,
base_internal_url,
worker_name,
job_completed_tx,
Some(module.id.clone()),
)
.await
{
script_path_to_worker.insert(key, dedi_w.1.clone());
workers.push(dedi_w);
}
}
}
}
FlowModuleValue::ForloopFlow { modules, .. } => {
let w = spawn_dedicated_workers_for_flow(
&modules,
path,
w_id,
killpill_tx.clone(),
killpill_rx,
db,
worker_dir,
base_internal_url,
worker_name,
job_completed_tx,
)
.await;
workers.extend(w);
}
FlowModuleValue::WhileloopFlow { modules, .. } => {
let w = spawn_dedicated_workers_for_flow(
&modules,
path,
w_id,
killpill_tx.clone(),
killpill_rx,
db,
worker_dir,
base_internal_url,
worker_name,
job_completed_tx,
)
.await;
workers.extend(w);
}
FlowModuleValue::BranchOne { branches, default } => {
for modules in branches
.iter()
.map(|x| &x.modules)
.chain(std::iter::once(default))
{
FlowModuleValue::ForloopFlow { modules, .. } => {
let w = spawn_dedicated_workers_for_flow(
&modules,
path,
@@ -1931,11 +1928,9 @@ async fn spawn_dedicated_workers_for_flow(
.await;
workers.extend(w);
}
}
FlowModuleValue::BranchAll { branches, .. } => {
for branch in branches {
FlowModuleValue::WhileloopFlow { modules, .. } => {
let w = spawn_dedicated_workers_for_flow(
&branch.modules,
&modules,
path,
w_id,
killpill_tx.clone(),
@@ -1949,32 +1944,74 @@ async fn spawn_dedicated_workers_for_flow(
.await;
workers.extend(w);
}
}
FlowModuleValue::RawScript { content, lock, path: spath, language, .. } => {
if let Some(dedi_w) = spawn_dedicated_worker(
SpawnWorker::RawScript {
path: spath.clone().unwrap_or(path.to_string()),
content: content.to_string(),
lock: lock.clone(),
lang: language.clone(),
},
w_id,
killpill_tx.clone(),
killpill_rx,
db,
worker_dir,
base_internal_url,
worker_name,
job_completed_tx,
Some(module.id.clone()),
)
.await
{
workers.push(dedi_w);
FlowModuleValue::BranchOne { branches, default } => {
for modules in branches
.iter()
.map(|x| &x.modules)
.chain(std::iter::once(default))
{
let w = spawn_dedicated_workers_for_flow(
&modules,
path,
w_id,
killpill_tx.clone(),
killpill_rx,
db,
worker_dir,
base_internal_url,
worker_name,
job_completed_tx,
)
.await;
workers.extend(w);
}
}
FlowModuleValue::BranchAll { branches, .. } => {
for branch in branches {
let w = spawn_dedicated_workers_for_flow(
&branch.modules,
path,
w_id,
killpill_tx.clone(),
killpill_rx,
db,
worker_dir,
base_internal_url,
worker_name,
job_completed_tx,
)
.await;
workers.extend(w);
}
}
FlowModuleValue::RawScript { content, lock, path: spath, language, .. } => {
if let Some(dedi_w) = spawn_dedicated_worker(
SpawnWorker::RawScript {
path: spath.clone().unwrap_or(path.to_string()),
content: content.to_string(),
lock: lock.clone(),
lang: language.clone(),
},
w_id,
killpill_tx.clone(),
killpill_rx,
db,
worker_dir,
base_internal_url,
worker_name,
job_completed_tx,
Some(module.id.clone()),
)
.await
{
workers.push(dedi_w);
}
}
FlowModuleValue::Flow { .. } => (),
FlowModuleValue::Identity => (),
}
FlowModuleValue::Flow { .. } => (),
FlowModuleValue::Identity => (),
} else {
tracing::error!("failed to get value for module: {:?}", module);
}
}
workers
@@ -3668,12 +3705,12 @@ async fn trigger_python_dependents_to_recompute_dependencies<
PushIsolationLevel::IsolatedRoot(db.clone(), rsmq.clone());
let r = get_latest_deployed_hash_for_path(db, w_id, s.as_str()).await;
if let Ok(r) = r {
let mut args: HashMap<String, serde_json::Value> = HashMap::new();
let mut args: HashMap<String, Box<RawValue>> = HashMap::new();
if let Some(ref dm) = deployment_message {
args.insert("deployment_message".to_string(), json!(dm));
args.insert("deployment_message".to_string(), to_raw_value(&dm));
}
if let Some(ref p_path) = parent_path {
args.insert("common_dependency_path".to_string(), json!(p_path));
args.insert("common_dependency_path".to_string(), to_raw_value(&p_path));
}
let (job_uuid, new_tx) = windmill_queue::push(
@@ -3686,7 +3723,7 @@ async fn trigger_python_dependents_to_recompute_dependencies<
language: r.6,
dedicated_worker: r.7,
},
args,
windmill_queue::PushArgs { args, extra: HashMap::new() },
&created_by,
email,
permissioned_as.to_string(),
@@ -3856,9 +3893,9 @@ async fn lock_modules(
custom_concurrency_key,
concurrent_limit,
concurrency_time_window_s,
} = e.value.clone()
} = e.get_value()?
else {
match e.value {
match e.get_value()? {
FlowModuleValue::ForloopFlow {
iterator,
modules,
@@ -3886,6 +3923,7 @@ async fn lock_modules(
parallel,
parallelism,
}
.into()
}
FlowModuleValue::BranchAll { branches, parallel } => {
let mut nbranches = vec![];
@@ -3906,7 +3944,7 @@ async fn lock_modules(
.await?;
nbranches.push(b)
}
e.value = FlowModuleValue::BranchAll { branches: nbranches, parallel }
e.value = FlowModuleValue::BranchAll { branches: nbranches, parallel }.into()
}
FlowModuleValue::BranchOne { branches, default } => {
let mut nbranches = vec![];
@@ -3941,7 +3979,7 @@ async fn lock_modules(
token,
)
.await?;
e.value = FlowModuleValue::BranchOne { branches: nbranches, default };
e.value = FlowModuleValue::BranchOne { branches: nbranches, default }.into();
}
_ => (),
};
@@ -3971,7 +4009,7 @@ async fn lock_modules(
.await;
match new_lock {
Ok(new_lock) => {
e.value = FlowModuleValue::RawScript {
e.value = windmill_common::worker::to_raw_value(&FlowModuleValue::RawScript {
lock: Some(new_lock),
path,
input_transforms,
@@ -3981,7 +4019,7 @@ async fn lock_modules(
custom_concurrency_key,
concurrent_limit,
concurrency_time_window_s,
};
});
new_flow_modules.push(e);
continue;
}
@@ -3993,7 +4031,7 @@ async fn lock_modules(
error = ?error,
"Failed to generate flow lock for raw script"
);
e.value = FlowModuleValue::RawScript {
e.value = windmill_common::worker::to_raw_value(&FlowModuleValue::RawScript {
lock: None,
path,
input_transforms,
@@ -4003,7 +4041,7 @@ async fn lock_modules(
custom_concurrency_key,
concurrent_limit,
concurrency_time_window_s,
};
});
new_flow_modules.push(e);
continue;
}
+124 -97
View File
@@ -243,7 +243,8 @@ pub async fn update_flow_status_after_job_completion_internal<
.await?
.ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
let module = get_module(&flow_job, module_index);
if module.is_some_and(|x| matches!(x.value, FlowModuleValue::Flow { .. })) {
if module.is_some_and(|x| x.is_flow()) {
(false, false, false)
} else {
(true, se, false)
@@ -1255,7 +1256,7 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
#[derive(Serialize, Debug)]
pub struct Iter {
index: i32,
value: serde_json::Value,
value: Box<RawValue>,
}
#[derive(Serialize)]
@@ -1518,7 +1519,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
if suspend.user_groups_required.is_some() {
match suspend.user_groups_required.unwrap() {
InputTransform::Static { value } => {
user_groups_required = serde_json::from_value::<Vec<String>>(value)
user_groups_required = serde_json::from_str::<Vec<String>>(value.get())
.expect("Unable to deserialize group names");
}
InputTransform::Javascript { expr } => {
@@ -1740,18 +1741,24 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
)
}
};
match json_value {
match json_value.and_then(|x| serde_json::from_str::<serde_json::Value>(x.get())) {
Ok(serde_json::Value::Number(n)) => {
if !n.is_u64() {
if n.is_f64() {
n.as_f64()
.map(|x: f64| from_now(Duration::from_millis((x * 1000.0) as u64)))
} else if n.is_u64() {
n.as_u64().map(|x: u64| from_now(Duration::from_secs(x)))
} else {
return Err(Error::ExecutionErr(format!(
"Expected an integer, found: {n}"
)));
}
n.as_u64().map(|x| from_now(Duration::from_secs(x)))
}
_ => Err(Error::ExecutionErr(format!(
"Expected a number value, found: {json_value:?}"
Ok(x @ _) => Err(Error::ExecutionErr(format!(
"Expected an integer, found: {x:?}"
)))?,
Err(e) => Err(Error::ExecutionErr(format!(
"Expected a number value, had error instead: {e:?}",
)))?,
}
} else {
@@ -1893,10 +1900,12 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
Ok(HashMap::new())
}
} else {
match &module.value {
FlowModuleValue::Script { input_transforms, .. }
| FlowModuleValue::RawScript { input_transforms, .. }
| FlowModuleValue::Flow { input_transforms, .. } => {
match &module.get_value() {
Ok(
FlowModuleValue::Script { input_transforms, .. }
| FlowModuleValue::RawScript { input_transforms, .. }
| FlowModuleValue::Flow { input_transforms, .. },
) => {
let ctx = get_transform_context(&flow_job, &previous_id, &status).await?;
transform_context = Some(ctx);
let by_id = transform_context.as_ref().unwrap();
@@ -1912,7 +1921,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
)
.await
}
FlowModuleValue::Identity => serde_json::from_str(
Ok(FlowModuleValue::Identity) => serde_json::from_str(
&serde_json::to_string(&PreviousResult {
previous_result: Some(&arc_last_job_result),
})
@@ -1920,7 +1929,12 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
)
.map_err(|e| error::Error::InternalErr(format!("identity: {e}"))),
_ => Ok(flow_job_args),
Ok(_) => Ok(flow_job_args),
Err(e) => {
return Err(error::Error::InternalErr(format!(
"module was not convertible to acceptable value {e:?}"
)))
}
}
};
tracing::debug!(id = %flow_job.id, root_id = %job_root, "flow job args computed");
@@ -2085,10 +2099,14 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
Ok(v) => (Some(v), None),
Err(e) => (None, Some(e)),
};
let root_job = if matches!(
module.value,
FlowModuleValue::Flow { .. } | FlowModuleValue::ForloopFlow { parallel: true, .. }
) {
let value_with_parallel = module.get_value_with_parallel()?;
let root_job = if {
value_with_parallel.type_ == "flow"
|| (value_with_parallel.type_ == "forloopflow"
&& value_with_parallel.parallel.is_some_and(|x| x))
} {
None
} else {
flow_job.root_job.or_else(|| Some(flow_job.id))
@@ -2100,7 +2118,10 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
tx2,
&flow_job.workspace_id,
payload_tag.payload,
ok.unwrap_or_else(|| serde_json::from_str("{}").unwrap()),
windmill_queue::PushArgs {
args: ok.unwrap_or_else(|| serde_json::from_str("{}").unwrap()),
extra: HashMap::new(),
},
&flow_job.created_by,
&flow_job.email,
flow_job.permissioned_as.to_owned(),
@@ -2126,17 +2147,19 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushed next flow job: {uuid}");
if let FlowModuleValue::ForloopFlow { parallelism: Some(p), .. } = &module.value {
if i as u16 >= *p {
sqlx::query!(
"UPDATE queue
if value_with_parallel.type_ == "forloopflow" {
if let Some(p) = value_with_parallel.parallelism {
if i as u16 >= p {
sqlx::query!(
"UPDATE queue
SET suspend = $1, suspend_until = now() + interval '14 day', running = true
WHERE id = $2",
(i as u16 - p + 1) as i32,
uuid,
)
.execute(&mut inner_tx)
.await?;
(i as u16 - p + 1) as i32,
uuid,
)
.execute(&mut inner_tx)
.await?;
}
}
}
@@ -2370,14 +2393,14 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
#[derive(Debug)]
struct ForloopNextIteration {
index: usize,
itered: Vec<serde_json::Value>,
itered: Vec<Box<RawValue>>,
flow_jobs: Vec<Uuid>,
new_args: Iter,
while_loop: bool,
}
enum ForLoopStatus {
ParallelIteration { itered: Vec<serde_json::Value> },
ParallelIteration { itered: Vec<Box<RawValue>> },
NextIteration(ForloopNextIteration),
EmptyIterator,
}
@@ -2482,7 +2505,7 @@ async fn compute_next_flow_transform(
let delete_after_use = module.delete_after_use.unwrap_or(false);
tracing::debug!(id = %flow_job.id, "computing next flow transform for {:?}", &module.value);
match &module.value {
match &module.get_value()? {
FlowModuleValue::Identity => trivial_next_job(JobPayload::Identity),
FlowModuleValue::Flow { path, .. } => {
let payload = flow_to_payload(path, &delete_after_use);
@@ -2548,7 +2571,10 @@ async fn compute_next_flow_transform(
index: next_loop_idx,
itered: vec![],
flow_jobs: flow_jobs.clone(),
new_args: Iter { index: next_loop_idx as i32, value: json!(next_loop_idx) },
new_args: Iter {
index: next_loop_idx as i32,
value: windmill_common::worker::to_raw_value(&next_loop_idx),
},
while_loop: true,
},
modules,
@@ -2599,65 +2625,66 @@ async fn compute_next_flow_transform(
.await
}
ForLoopStatus::ParallelIteration { itered, .. } => {
let inner_path = Some(format!("{}/loop-parrallel", flow_job.script_path(),));
let continue_payload = if is_simple {
let payload = payload_from_simple_module(
&modules[0].value,
db,
flow_job,
module,
inner_path,
)
.await?;
ContinuePayload::ForloopJobs { n: itered.len(), payload: payload }
if modules.is_empty() {
return Ok(NextFlowTransform::EmptyInnerFlows);
} else {
let payload = {
JobPayloadWithTag {
payload: JobPayload::RawFlow {
value: FlowValue {
modules: (*modules).clone(),
failure_module: flow.failure_module.clone(),
same_worker: flow.same_worker,
concurrent_limit: None,
concurrency_time_window_s: None,
skip_expr: None,
cache_ttl: None,
priority: None,
early_return: None,
concurrency_key: None,
let inner_path =
Some(format!("{}/loop-parrallel", flow_job.script_path(),));
let value = &modules[0].get_value()?;
let continue_payload = if is_simple {
let payload =
payload_from_simple_module(value, db, flow_job, module, inner_path)
.await?;
ContinuePayload::ForloopJobs { n: itered.len(), payload: payload }
} else {
let payload = {
JobPayloadWithTag {
payload: JobPayload::RawFlow {
value: FlowValue {
modules: (*modules).clone(),
failure_module: flow.failure_module.clone(),
same_worker: flow.same_worker,
concurrent_limit: None,
concurrency_time_window_s: None,
skip_expr: None,
cache_ttl: None,
priority: None,
early_return: None,
concurrency_key: None,
},
path: Some(format!("{}/forloop", flow_job.script_path())),
restarted_from: None,
},
path: Some(format!("{}/forloop", flow_job.script_path())),
restarted_from: None,
},
tag: None,
delete_after_use: delete_after_use,
timeout: None,
}
};
ContinuePayload::ForloopJobs { n: itered.len(), payload }
};
Ok(NextFlowTransform::Continue(
continue_payload,
NextStatus::AllFlowJobs {
branchall: None,
iterator: Some(windmill_common::flow_status::Iterator {
index: 0,
itered,
}),
simple_input_transforms: if is_simple {
match &modules[0].value {
FlowModuleValue::Script { input_transforms, .. }
| FlowModuleValue::RawScript { input_transforms, .. }
| FlowModuleValue::Flow { input_transforms, .. } => {
Some(input_transforms.clone())
}
_ => None,
tag: None,
delete_after_use: delete_after_use,
timeout: None,
}
} else {
None
};
ContinuePayload::ForloopJobs { n: itered.len(), payload }
};
Ok(NextFlowTransform::Continue(
continue_payload,
NextStatus::AllFlowJobs {
branchall: None,
iterator: Some(windmill_common::flow_status::Iterator {
index: 0,
itered,
}),
simple_input_transforms: if is_simple {
match value {
FlowModuleValue::Script { input_transforms, .. }
| FlowModuleValue::RawScript { input_transforms, .. }
| FlowModuleValue::Flow { input_transforms, .. } => {
Some(input_transforms.clone())
}
_ => None,
}
} else {
None
},
},
},
))
))
}
}
}
}
@@ -2882,14 +2909,14 @@ async fn next_loop_iteration(
add_virtual_items_if_necessary(&mut modules);
let inner_path = Some(format!("{}/loop-{}", flow_job.script_path(), ns.index));
if is_simple {
let payload =
payload_from_simple_module(&modules[0].value, db, flow_job, module, inner_path).await?;
let value = &modules[0].get_value()?;
let payload = payload_from_simple_module(value, db, flow_job, module, inner_path).await?;
Ok(NextFlowTransform::Continue(
ContinuePayload::SingleJob(payload),
NextStatus::NextLoopIteration {
next: ns,
simple_input_transforms: if is_simple {
match &modules[0].value {
match value {
FlowModuleValue::Script { input_transforms, .. }
| FlowModuleValue::RawScript { input_transforms, .. }
| FlowModuleValue::Flow { input_transforms, .. } => {
@@ -2932,13 +2959,13 @@ async fn next_loop_iteration(
fn is_simple_modules(modules: &Vec<FlowModule>, flow: &FlowValue) -> bool {
let is_simple = modules.len() == 1
&& modules[0].value.is_simple()
&& modules[0].is_simple()
&& modules[0].sleep.is_none()
&& modules[0].suspend.is_none()
&& modules[0].cache_ttl.is_none()
&& (modules[0].mock.is_none()
&& modules[0].mock.as_ref().is_some_and(|m| !m.enabled)
&& flow.failure_module.is_none());
&& modules[0].stop_after_if.is_none()
&& (modules[0].mock.is_none() || modules[0].mock.as_ref().is_some_and(|m| !m.enabled))
&& flow.failure_module.is_none();
is_simple
}
@@ -2987,7 +3014,7 @@ async fn next_forloop_status(
.await?
}
};
let itered = serde_json::from_str::<Vec<serde_json::Value>>(itered_raw.get()).map_err(
let itered = serde_json::from_str::<Vec<Box<RawValue>>>(itered_raw.get()).map_err(
|not_array| {
Error::ExecutionErr(format!(
"Expected an array value in the iterator expression, found: {not_array}"
@@ -3045,7 +3072,7 @@ async fn next_forloop_status(
.await?
}
};
serde_json::from_str::<Vec<serde_json::Value>>(itered_raw.get()).map_err(
serde_json::from_str::<Vec<Box<RawValue>>>(itered_raw.get()).map_err(
|not_array| {
Error::ExecutionErr(format!("Expected an array value, found: {not_array}"))
},
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.331.2";
export const VERSION = "v1.335.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
+2 -2
View File
@@ -36,13 +36,13 @@ export async function readConfigFile(): Promise<SyncOptions> {
) as SyncOptions;
if (conf?.defaultTs == undefined) {
log.warning(
"No defaultTs defined in your wmill.yaml, using deno as default typescript language. Use 'wmill init' to bootstrap it."
"No defaultTs defined in your wmill.yaml. Using 'bun' as default."
);
}
return typeof conf == "object" ? conf : ({} as SyncOptions);
} catch (e) {
log.warning(
"No wmill.yaml found, using deno as default typescript language. Use 'wmill init' to bootstrap it."
"No wmill.yaml found. Use 'wmill init' to bootstrap it. Using 'bun' as default typescript runtime."
);
return {};
}
+2 -2
View File
@@ -33,7 +33,7 @@ addEventListener("error", (event) => {
}
});
export const VERSION = "v1.331.2";
export const VERSION = "v1.335.0";
let command: any = new Command()
.name("wmill")
@@ -70,7 +70,7 @@ let command: any = new Command()
"wmill.yaml",
yamlStringify({
defaultTs: "bun",
includes: [],
includes: ["**"],
excludes: [],
codebases: [],
})
+53 -36
View File
@@ -24,8 +24,9 @@ import { SchemaProperty } from "./bootstrap/common.ts";
import { ScriptLanguage } from "./script_common.ts";
import { inferContentTypeFromFilePath } from "./script_common.ts";
import { GlobalDeps } from "./script.ts";
import { yamlOptions } from "./sync.ts";
import { findCodebase, yamlOptions } from "./sync.ts";
import { generateHash } from "./utils.ts";
import { SyncCodebase } from "./codebase.ts";
export async function generateAllMetadata() {}
@@ -77,7 +78,8 @@ export async function generateMetadataInternal(
},
dryRun: boolean,
noStaleMessage: boolean,
globalDeps: GlobalDeps
globalDeps: GlobalDeps,
codebases: SyncCodebase[]
): Promise<string | undefined> {
const remotePath = scriptPath
.substring(0, scriptPath.indexOf("."))
@@ -100,7 +102,8 @@ export async function generateMetadataInternal(
const metadataWithType = await parseMetadataFile(
remotePath,
undefined,
globalDeps
globalDeps,
codebases
);
// read script content
@@ -138,7 +141,9 @@ export async function generateMetadataInternal(
);
}
if (!opts.schemaOnly) {
const c = findCodebase(scriptPath, codebases);
if (!opts.schemaOnly && !c) {
await updateScriptLock(
workspace,
scriptContent,
@@ -163,7 +168,7 @@ export async function generateMetadataInternal(
return `${remotePath} (${language})`;
}
async function updateScriptSchema(
export async function updateScriptSchema(
scriptContent: string,
language: ScriptLanguage,
metadataContent: Record<string, any>,
@@ -355,8 +360,8 @@ function sortObject(obj: any): any {
);
}
function argSigToJsonSchemaType(
typ:
export function argSigToJsonSchemaType(
t:
| string
| { resource: string | null }
| {
@@ -371,55 +376,55 @@ function argSigToJsonSchemaType(
oldS: SchemaProperty
): void {
const newS: SchemaProperty = { type: "" };
if (typ === "int") {
if (t === "int") {
newS.type = "integer";
} else if (typ === "float") {
} else if (t === "float") {
newS.type = "number";
} else if (typ === "bool") {
} else if (t === "bool") {
newS.type = "boolean";
} else if (typ === "email") {
} else if (t === "email") {
newS.type = "string";
newS.format = "email";
} else if (typ === "sql") {
} else if (t === "sql") {
newS.type = "string";
newS.format = "sql";
} else if (typ === "yaml") {
} else if (t === "yaml") {
newS.type = "string";
newS.format = "yaml";
} else if (typ === "bytes") {
} else if (t === "bytes") {
newS.type = "string";
newS.contentEncoding = "base64";
} else if (typ === "datetime") {
} else if (t === "datetime") {
newS.type = "string";
newS.format = "date-time";
} else if (typeof typ !== "string" && `object` in typ) {
} else if (typeof t !== "string" && `object` in t) {
newS.type = "object";
if (typ.object) {
const properties: Record<string, SchemaProperty> = {};
for (const prop of typ.object) {
properties[prop.key] = { type: undefined };
if (t.object) {
const properties: Record<string, any> = {};
for (const prop of t.object) {
properties[prop.key] = {};
argSigToJsonSchemaType(prop.typ, properties[prop.key]);
}
newS.properties = properties;
}
} else if (typeof typ !== "string" && `str` in typ) {
} else if (typeof t !== "string" && `str` in t) {
newS.type = "string";
if (typ.str) {
newS.enum = typ.str;
if (t.str) {
newS.enum = t.str;
}
} else if (typeof typ !== "string" && `resource` in typ) {
} else if (typeof t !== "string" && `resource` in t) {
newS.type = "object";
newS.format = `resource-${typ.resource}`;
} else if (typeof typ !== "string" && `list` in typ) {
newS.format = `resource-${t.resource}`;
} else if (typeof t !== "string" && `list` in t) {
newS.type = "array";
if (typ.list === "int" || typ.list === "float") {
if (t.list === "int" || t.list === "float") {
newS.items = { type: "number" };
} else if (typ.list === "bytes") {
} else if (t.list === "bytes") {
newS.items = { type: "string", contentEncoding: "base64" };
} else if (typ.list == "string") {
} else if (t.list == "string") {
newS.items = { type: "string" };
} else if (typ.list && typeof typ.list == "object" && "str" in typ.list) {
newS.items = { type: "string", enum: typ.list.str };
} else if (t.list && typeof t.list == "object" && "str" in t.list) {
newS.items = { type: "string", enum: t.list.str };
} else {
newS.items = { type: "object" };
}
@@ -430,17 +435,27 @@ function argSigToJsonSchemaType(
if (oldS.type != newS.type) {
for (const prop of Object.getOwnPropertyNames(newS)) {
if (prop != "description") {
// @ts-ignore: fix
delete oldS[prop];
}
}
} else if (oldS.format == "date-time" && newS.format != "date-time") {
delete oldS.format;
} else if (
(oldS.format == "date" || oldS.format === "date-time") &&
newS.format == "string"
) {
newS.format = oldS.format;
} else if (newS.format == "date-time" && oldS.format == "date") {
newS.format = "date";
} else if (oldS.items?.type != newS.items?.type) {
delete oldS.items;
}
Object.assign(oldS, newS);
// if (sameItems && savedItems != undefined && savedItems.enum != undefined) {
// sendUserToast(JSON.stringify(savedItems))
// oldS.items = savedItems
// }
if (oldS.format?.startsWith("resource-") && newS.type != "object") {
oldS.format = undefined;
}
@@ -474,7 +489,8 @@ export async function parseMetadataFile(
schemaOnly?: boolean;
})
| undefined,
globalDeps: GlobalDeps
globalDeps: GlobalDeps,
codebases: SyncCodebase[]
): Promise<{ isJson: boolean; payload: any; path: string }> {
let metadataFilePath = scriptPath + ".script.json";
try {
@@ -522,7 +538,8 @@ export async function parseMetadataFile(
generateMetadataIfMissing,
false,
false,
globalDeps
globalDeps,
codebases
);
scriptInitialMetadata = yamlParse(
await Deno.readTextFile(metadataFilePath)
+1 -1
View File
@@ -41,7 +41,7 @@ export async function downloadZip(
}&include_users=${includeUsers ?? false}&include_groups=${
includeGroups ?? false
}&include_settings=${includeSettings ?? false}&default_ts=${
defaultTs ?? "deno"
defaultTs ?? "bun"
}`,
{
headers: requestHeaders,
+53 -43
View File
@@ -1,5 +1,5 @@
// deno-lint-ignore-file no-explicit-any
import { GlobalOptions } from "./types.ts";
import { GlobalOptions, showDiff } from "./types.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import {
colors,
@@ -7,6 +7,7 @@ import {
Confirm,
JobService,
log,
NewScript,
readAll,
Script,
ScriptService,
@@ -21,7 +22,11 @@ import {
} from "./bootstrap/script_bootstrap.ts";
import { Workspace } from "./workspace.ts";
import { generateMetadataInternal, parseMetadataFile } from "./metadata.ts";
import {
generateMetadataInternal,
parseMetadataFile,
updateScriptSchema,
} from "./metadata.ts";
import {
ScriptLanguage,
inferContentTypeFromFilePath,
@@ -74,7 +79,7 @@ async function push(opts: PushOptions, filePath: string) {
await requireLogin(opts);
const codebases = await listSyncCodebases(opts as SyncOptions);
const globalDeps = await findGlobalDeps(codebases);
const globalDeps = await findGlobalDeps();
await handleFile(
filePath,
workspace,
@@ -169,7 +174,8 @@ export async function handleFile(
schemaOnly: codebase ? true : undefined,
}
: undefined,
globalDeps
globalDeps,
codebases
)
)?.payload;
@@ -187,10 +193,26 @@ export async function handleFile(
}
const content = await Deno.readTextFile(path);
const requestBodyCommon = {
if (codebase) {
const typedBefore = JSON.parse(JSON.stringify(typed.schema));
await updateScriptSchema(content, language, typed, path);
if (typedBefore != typed.schema) {
log.info(`Updated metadata for bundle ${path}`);
showDiff(
yamlStringify(typedBefore, yamlOptions),
yamlStringify(typed.schema, yamlOptions)
);
await Deno.writeTextFile(
remotePath + ".script.yaml",
yamlStringify(typed as Record<string, any>, yamlOptions)
);
}
}
const requestBodyCommon: NewScript = {
content,
description: typed?.description ?? "",
language: language,
language: language as NewScript["language"],
path: remotePath.replaceAll("\\", "/"),
summary: typed?.summary ?? "",
kind: typed?.kind,
@@ -207,6 +229,8 @@ export async function handleFile(
visible_to_runner_only: typed?.visible_to_runner_only,
no_main_func: typed?.no_main_func,
priority: typed?.priority,
concurrency_key: typed?.concurrency_key,
//@ts-ignore
codebase: codebase?.digest,
timeout: typed?.timeout,
};
@@ -238,7 +262,9 @@ export async function handleFile(
Boolean(remote.visible_to_runner_only) &&
Boolean(typed.no_main_func) == Boolean(remote.no_main_func) &&
typed.priority == Boolean(remote.priority) &&
typed.timeout == remote.timeout)
typed.timeout == remote.timeout &&
//@ts-ignore
typed.concurrency_key == remote["concurrency_key"])
) {
log.info(colors.green(`Script ${remotePath} is up to date`));
return true;
@@ -272,28 +298,7 @@ export async function handleFile(
async function createScript(
bundleContent: string | undefined,
workspaceId: string,
body: {
parent_hash: string | undefined;
content: string;
description: any;
language: ScriptLanguage;
path: string;
summary: any;
kind: any;
lock: any;
schema: any;
tag: any;
ws_error_handler_muted: any;
dedicated_worker: any;
cache_ttl: any;
concurrency_time_window_s: any;
concurrent_limit: any;
deployment_message: string | undefined;
restart_unless_cancelled: any;
visible_to_runner_only: any;
no_main_func: any;
priority: any;
},
body: NewScript,
workspace: Workspace
) {
if (!bundleContent) {
@@ -371,13 +376,13 @@ export function filePathExtensionFromContentType(
} else if (language === "nativets") {
return ".fetch.ts";
} else if (language === "bun") {
if (defaultTs == undefined || defaultTs == "deno") {
if (defaultTs == "deno") {
return ".bun.ts";
} else {
return ".ts";
}
} else if (language === "deno") {
if (defaultTs == "bun") {
if (defaultTs == undefined || defaultTs == "bun") {
return ".deno.ts";
} else {
return ".ts";
@@ -662,12 +667,16 @@ async function bootstrap(
yamlOptions
);
Deno.writeTextFile(scriptCodeFileFullPath, scriptInitialCode, {
createNew: true,
});
Deno.writeTextFile(scriptMetadataFileFullPath, scriptInitialMetadataYaml, {
await Deno.writeTextFile(scriptCodeFileFullPath, scriptInitialCode, {
createNew: true,
});
await Deno.writeTextFile(
scriptMetadataFileFullPath,
scriptInitialMetadataYaml,
{
createNew: true,
}
);
}
export type GlobalDeps = {
@@ -675,13 +684,11 @@ export type GlobalDeps = {
reqs: Record<string, string>;
composers: Record<string, string>;
};
export async function findGlobalDeps(
codebases: SyncCodebase[]
): Promise<GlobalDeps> {
export async function findGlobalDeps(): Promise<GlobalDeps> {
const pkgs: { [key: string]: string } = {};
const reqs: { [key: string]: string } = {};
const composers: { [key: string]: string } = {};
const els = await FSFSElement(Deno.cwd(), codebases);
const els = await FSFSElement(Deno.cwd(), []);
for await (const entry of readDirRecursiveWithIgnore((p, isDir) => {
p = "/" + p;
return (
@@ -725,7 +732,7 @@ async function generateMetadata(
opts = await mergeConfigWithConfigFile(opts);
const codebases = await listSyncCodebases(opts);
const globalDeps = await findGlobalDeps(codebases);
const globalDeps = await findGlobalDeps();
if (scriptPath) {
// read script metadata file
await generateMetadataInternal(
@@ -734,7 +741,8 @@ async function generateMetadata(
opts,
false,
false,
globalDeps
globalDeps,
codebases
);
} else {
const ignore = await ignoreF(opts);
@@ -760,7 +768,8 @@ async function generateMetadata(
opts,
true,
true,
globalDeps
globalDeps,
codebases
);
if (candidate) {
hasAny = true;
@@ -788,7 +797,8 @@ async function generateMetadata(
opts,
false,
true,
globalDeps
globalDeps,
codebases
);
}
}
+1 -1
View File
@@ -27,7 +27,7 @@ export function inferContentTypeFromFilePath(
} else if (contentPath.endsWith("deno.ts")) {
return "deno";
} else if (contentPath.endsWith(".ts")) {
return defaultTs ?? "deno";
return defaultTs ?? "bun";
} else if (contentPath.endsWith(".go")) {
return "go";
} else if (contentPath.endsWith(".my.sql")) {
+62 -39
View File
@@ -190,7 +190,11 @@ export const yamlOptions = {
skipInvalid: true,
};
function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
function ZipFSElement(
zip: JSZip,
useYaml: boolean,
defaultTs: "bun" | "deno"
): DynFSElement {
async function _internal_file(
p: string,
f: JSZip.JSZipObject
@@ -214,7 +218,8 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
const seen_names = new Set<string>();
function assignPath(
summary: string | undefined,
language: RawScript["language"]
language: RawScript["language"],
defaultTs: "bun" | "deno"
): [string, string] {
let name;
@@ -236,7 +241,9 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
let ext;
if (language == "python3") ext = "py";
else if (language == "deno") ext = "ts";
else if (language == defaultTs) ext = "ts";
else if (language == "bun") ext = "bun.ts";
else if (language == "deno") ext = "deno.ts";
else if (language == "go") ext = "go";
else if (language == "bash") ext = "sh";
else if (language == "powershell") ext = "ps1";
@@ -246,7 +253,6 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
else if (language == "snowflake") ext = "sf.sql";
else if (language == "mssql") ext = "ms.sql";
else if (language == "graphql") ext = "gql";
else if (language == "bun") ext = "bun.ts";
else if (language == "nativets") ext = "native.ts";
else if (language == "frontend") ext = "frontend.js";
else if (language == "php") ext = "php";
@@ -260,13 +266,17 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
): InlineScript[] {
return modules.flatMap((m) => {
if (m.value.type == "rawscript") {
const [basePath, ext] = assignPath(m.summary, m.value.language);
const [basePath, ext] = assignPath(
m.summary,
m.value.language,
defaultTs
);
const path = basePath + ext;
const content = m.value.content;
const r = [{ path: path, content: content }];
m.value.content = "!inline " + path;
const lock = m.value.lock;
if (lock) {
if (lock && lock != "") {
const lockPath = basePath + "lock";
m.value.lock = "!inline " + lockPath;
r.push({ path: lockPath, content: lock });
@@ -302,7 +312,7 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
if (k == "inlineScript" && typeof v == "object") {
const o: Record<string, any> = v as any;
const name = rec["name"];
const [basePath, ext] = assignPath(name, o["language"]);
const [basePath, ext] = assignPath(name, o["language"], defaultTs);
const r = [];
if (o["content"]) {
const content = o["content"];
@@ -312,7 +322,7 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
content: content,
});
}
if (o["lock"]) {
if (o["lock"] && o["lock"] != "") {
const lock = o["lock"];
o["lock"] = "!inline " + basePath + "lock";
r.push({
@@ -403,8 +413,14 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
if (kind == "script") {
const parsed = JSON.parse(content);
if (parsed["lock"]) {
if (
parsed["lock"] &&
parsed["lock"] != "" &&
parsed["codebase"] == undefined
) {
parsed["lock"] = "!inline " + removeSuffix(p, ".json") + ".lock";
} else {
parsed["lock"] = undefined;
}
return useYaml
? yamlStringify(parsed, yamlOptions)
@@ -421,7 +437,7 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
const content = await f.async("text");
const parsed = JSON.parse(content);
const lock = parsed["lock"];
if (lock) {
if (lock && lock != "") {
r.push({
isDirectory: false,
path: removeSuffix(finalPath, ".json") + ".lock",
@@ -832,7 +848,8 @@ async function pull(opts: GlobalOptions & SyncOptions) {
opts.includeSettings,
opts.defaultTs
))!,
!opts.json
!opts.json,
opts.defaultTs ?? "bun"
);
const local = !opts.stateful
? await FSFSElement(Deno.cwd(), codebases)
@@ -867,40 +884,45 @@ async function pull(opts: GlobalOptions & SyncOptions) {
const target = path.join(Deno.cwd(), change.path);
const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
if (change.name === "edited") {
try {
const currentLocal = await Deno.readTextFile(target);
if (currentLocal !== change.before && currentLocal !== change.after) {
log.info(
colors.red(
`Conflict detected on ${change.path}\nBoth local and remote have been modified.`
)
);
if (opts.failConflicts) {
conflicts.push({
local: currentLocal,
change,
path: change.path,
});
continue;
} else if (opts.yes) {
if (opts.stateful) {
try {
const currentLocal = await Deno.readTextFile(target);
if (
currentLocal !== change.before &&
currentLocal !== change.after
) {
log.info(
colors.red(
`Override local version with remote since --yes was passed and no --fail-conflicts.`
`Conflict detected on ${change.path}\nBoth local and remote have been modified.`
)
);
} else {
showConflict(change.path, currentLocal, change.after);
if (
await Confirm.prompt(
"Preserve local (push to change remote and avoid seeing this again)?"
)
) {
if (opts.failConflicts) {
conflicts.push({
local: currentLocal,
change,
path: change.path,
});
continue;
} else if (opts.yes) {
log.info(
colors.red(
`Override local version with remote since --yes was passed and no --fail-conflicts.`
)
);
} else {
showConflict(change.path, currentLocal, change.after);
if (
await Confirm.prompt(
"Preserve local (push to change remote and avoid seeing this again)?"
)
) {
continue;
}
}
}
} catch {
// ignore
}
} catch {
// ignore
}
if (!change.path.endsWith(".json") && !change.path.endsWith(".yaml")) {
log.info(`Editing script content of ${change.path}`);
@@ -1047,7 +1069,8 @@ async function push(opts: GlobalOptions & SyncOptions) {
opts.includeSettings,
opts.defaultTs
))!,
!opts.json
!opts.json,
opts.defaultTs ?? "bun"
);
const local = await FSFSElement(path.join(Deno.cwd(), ""), codebases);
@@ -1083,7 +1106,7 @@ async function push(opts: GlobalOptions & SyncOptions) {
log.info(colors.gray(`Applying changes to files ...`));
const alreadySynced: string[] = [];
const globalDeps = await findGlobalDeps(codebases);
const globalDeps = await findGlobalDeps();
for await (const change of changes) {
const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-components",
"version": "1.331.2",
"version": "1.335.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.331.2",
"version": "1.335.0",
"license": "AGPL-3.0",
"dependencies": {
"@aws-crypto/sha256-js": "^4.0.0",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.331.2",
"version": "1.335.0",
"scripts": {
"dev": "vite dev",
"build": "vite build",
+7 -1
View File
@@ -41,6 +41,8 @@ export interface SchemaProperty {
order?: string[]
nullable?: boolean
dateFormat?: string
title?: string
placeholder?: string
}
export interface ModalSchemaProperty {
@@ -65,6 +67,8 @@ export interface ModalSchemaProperty {
password?: boolean
nullable?: boolean
dateFormat?: string
title?: string
placeholder?: string
}
export function modalToSchema(schema: ModalSchemaProperty): SchemaProperty {
@@ -88,7 +92,9 @@ export function modalToSchema(schema: ModalSchemaProperty): SchemaProperty {
showExpr: schema.showExpr,
password: schema.password,
nullable: schema.nullable,
dateFormat: schema.dateFormat
dateFormat: schema.dateFormat,
title: schema.title,
placeholder: schema.placeholder
}
}
export type Schema = {
+30 -8
View File
@@ -1,6 +1,6 @@
<script lang="ts">
import type { SchemaProperty } from '$lib/common'
import { setInputCat as computeInputCat, emptyString } from '$lib/utils'
import { setInputCat as computeInputCat, emptyString, shouldDisplayPlaceholder } from '$lib/utils'
import { ChevronDown, DollarSign, Pipette, Plus, X } from 'lucide-svelte'
import { createEventDispatcher } from 'svelte'
import Multiselect from 'svelte-multiselect'
@@ -30,6 +30,8 @@
import autosize from '$lib/autosize'
import PasswordArgInput from './PasswordArgInput.svelte'
import Password from './Password.svelte'
import Label from './Label.svelte'
import Tooltip from './Tooltip.svelte'
export let label: string = ''
export let value: any
@@ -74,6 +76,8 @@
export let customErrorMessage: string | undefined = undefined
export let onlyMaskPassword = false
export let nullable: boolean = false
export let title: string | undefined = undefined
export let placeholder: string | undefined = undefined
let seeEditable: boolean = enum_ != undefined || pattern != undefined
const dispatch = createEventDispatcher()
@@ -233,7 +237,7 @@
{#if displayHeader}
<FieldHeader
prettify={prettifyHeader}
{label}
label={title && !emptyString(title) ? title : label}
{disabled}
{required}
{type}
@@ -241,6 +245,7 @@
{format}
{simpleTooltip}
/>
{#if editableSchema}
<span class="mx-8" />
{#if editableSchema.i > 0}
@@ -260,7 +265,7 @@
{/if}
{/if}
{#if editableSchema}
<label class="text-secondary">
<Label label="Description">
<textarea
class="mb-1"
use:autosize
@@ -269,7 +274,15 @@
on:keydown={onKeyDown}
placeholder="Field description"
/>
</label>
</Label>
<div class="flex flex-row gap-2 w-full">
<Label label="Custom Title" class="w-full">
<svelte:fragment slot="header">
<Tooltip light>Will be displayed in the UI instead of the field name.</Tooltip>
</svelte:fragment>
<input class="mb-1" bind:value={title} on:keydown={onKeyDown} placeholder="Field title" />
</Label>
</div>
{#if type == 'array'}
<ArrayTypeNarrowing bind:itemsType />
@@ -317,6 +330,11 @@
<ObjectTypeNarrowing bind:format />
{/if}
</div>
{#if shouldDisplayPlaceholder(type, format, enum_, contentEncoding, pattern)}
<Label label="Placeholder" class="pt-2">
<textarea placeholder="Enter a placeholder" rows="1" bind:value={placeholder} />
</Label>
{/if}
{#if !required && type === 'string'}
<div class="mt-2 border-t pt-4">
<Toggle
@@ -381,7 +399,7 @@
class={valid
? ''
: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'}
placeholder={defaultValue ?? ''}
placeholder={placeholder ?? defaultValue ?? ''}
bind:value
min={extra['min']}
max={extra['max']}
@@ -680,7 +698,7 @@
class={valid
? ''
: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-3'}
placeholder={defaultValue ?? ''}
placeholder={placeholder ?? defaultValue ?? ''}
bind:value
/>
{:else if inputCat == 'string'}
@@ -688,7 +706,11 @@
<div class="flex flex-row w-full items-center justify-between relative">
{#if password || extra?.['password'] == true}
{#if onlyMaskPassword}
<Password {disabled} bind:password={value} placeholder={defaultValue ?? ''} />
<Password
{disabled}
bind:password={value}
placeholder={placeholder ?? defaultValue ?? ''}
/>
{:else}
<PasswordArgInput {disabled} bind:value />
{/if}
@@ -713,7 +735,7 @@
? ''
: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-3'
)}
placeholder={defaultValue ?? ''}
placeholder={placeholder ?? defaultValue ?? ''}
bind:value
/>
{/key}
@@ -13,47 +13,61 @@
Title,
Tooltip
} from 'chart.js'
import type { ConcurrencyIntervals } from '$lib/gen'
import type { CompletedJob, ExtendedJobs } from '$lib/gen'
import { createEventDispatcher } from 'svelte'
import { getDbClockNow } from '$lib/forLater'
export let concurrencyIntervals: ConcurrencyIntervals | undefined = undefined
export let extendedJobs: ExtendedJobs | undefined = undefined
export let maxIsNow: boolean = false
export let minTimeSet: string | undefined = undefined
export let maxTimeSet: string | undefined = undefined
const dispatch = createEventDispatcher()
function calculateTimeSeries(concurrencyIntervals: ConcurrencyIntervals): AggregatedInterval[] {
function calculateTimeSeries(extendedJobs: ExtendedJobs): AggregatedInterval[] {
const timeline = new Map<number, { count: number; id_started: string[]; id_ended: string[] }>()
concurrencyIntervals.completed_jobs?.forEach(({ job_id, started_at, ended_at }) => {
if (started_at != undefined && ended_at != undefined) {
const startTime = new Date(started_at).getTime()
const endTime = new Date(ended_at).getTime()
if (!timeline.has(startTime))
extendedJobs.jobs.forEach((j) => {
if (j.started_at != undefined) {
const startTime = new Date(j.started_at).getTime()
if (!timeline.has(startTime)) {
timeline.set(startTime, { count: 0, id_started: [], id_ended: [] })
if (!timeline.has(endTime))
timeline.set(endTime, { count: 0, id_started: [], id_ended: [] })
}
const s = timeline.get(startTime)!
const e = timeline.get(endTime)!
s.count += 1
s.id_started.push(job_id ?? 'unknown')
e.count -= 1
e.id_ended.push(job_id ?? 'unknown')
s.id_started.push(j.id)
if (j.type === 'CompletedJob') {
const jc = j as CompletedJob
const endTime = startTime + jc.duration_ms
if (!timeline.has(endTime)) {
timeline.set(endTime, { count: 0, id_started: [], id_ended: [] })
}
const e = timeline.get(endTime)!
e.count -= 1
e.id_ended.push(j.id)
}
}
})
concurrencyIntervals.running_jobs?.forEach(({ job_id, started_at }) => {
if (started_at != undefined) {
const startTime = new Date(started_at).getTime()
if (!timeline.has(startTime))
extendedJobs.obscured_jobs.forEach((j) => {
if (j.started_at != undefined) {
const startTime = new Date(j.started_at).getTime()
if (!timeline.has(startTime)) {
timeline.set(startTime, { count: 0, id_started: [], id_ended: [] })
}
const s = timeline.get(startTime)!
s.count += 1
s.id_started.push(job_id ?? 'unknown')
s.id_started.push('unknoww')
if (j.duration_ms != undefined) {
const jc = j as CompletedJob
const endTime = startTime + jc.duration_ms
if (!timeline.has(endTime)) {
timeline.set(endTime, { count: 0, id_started: [], id_ended: [] })
}
const e = timeline.get(endTime)!
e.count -= 1
e.id_ended.push('unknown')
}
}
})
@@ -90,7 +104,7 @@
type AggregatedInterval = { time: Date; count: number; msg?: string }
let intervals: AggregatedInterval[] | undefined = undefined
$: intervals = concurrencyIntervals ? calculateTimeSeries(concurrencyIntervals) : undefined
$: intervals = extendedJobs ? calculateTimeSeries(extendedJobs) : undefined
ChartJS.register(
Title,
@@ -38,6 +38,10 @@
export let noControls: boolean = false
export let drawerOpen = false
const IMG_MAX_SIZE = 10000000
const TABLE_MAX_SIZE = 5000000
const DISPLAY_MAX_SIZE = 100000
let resultKind:
| 'json'
| 'table-col'
@@ -65,18 +69,46 @@
let enableHtml = false
let s3FileDisplayRawMode = false
function isArrayOfArrays(result: any): boolean {
function isTableRow(result: any): boolean {
return Array.isArray(result) && result.every((x) => Array.isArray(x))
}
function isObjectOfArrays(result: any[], keys: string[]): boolean {
function isTableCol(result: any, keys: string[]): boolean {
return (
!Array.isArray(result) && keys.map((k) => Array.isArray(result[k])).reduce((a, b) => a && b)
!Array.isArray(result) &&
keys.map((k) => Array.isArray(result[k])).reduce((a, b) => a && b, true)
)
}
function isTableRowObject(json) {
// check array of objects (with possible a first row of headers)
return (
Array.isArray(json) &&
json.length > 0 &&
(json.every(
(item) =>
item && typeof item === 'object' && Object.keys(item).length > 0 && !Array.isArray(item)
) ||
(Array.isArray(json[0]) &&
json[0].every((item) => typeof item === 'string') &&
json
.slice(1)
.every(
(item) =>
item &&
typeof item === 'object' &&
Object.keys(item).length > 0 &&
!Array.isArray(item)
)))
)
}
let largeObject: boolean | undefined = undefined
function checkIfS3(result: any, keys: string[]) {
return keys.length === 1 && keys.includes('s3') && typeof result.s3 === 'string'
}
let is_render_all = false
function inferResultKind(result: any) {
if (result == 'WINDMILL_TOO_BIG') {
@@ -97,71 +129,52 @@
keys.length == 1 && keys.includes('render_all') && Array.isArray(result['render_all'])
// Check if the result is an image
if (
[
'png',
'svg',
'jpeg',
'html',
'gif',
'table-row',
'table-col',
'table-row-object'
].includes(keys[0]) &&
keys.length == 1
) {
if (['png', 'svg', 'jpeg', 'html', 'gif'].includes(keys[0]) && keys.length == 1) {
// Check if the image is too large (10mb)
largeObject = roughSizeOfObject(result) > 10000000
largeObject = roughSizeOfObject(result) > IMG_MAX_SIZE
return keys[0] as
| 'png'
| 'svg'
| 'jpeg'
| 'html'
| 'gif'
| 'table-row'
| 'table-col'
| 'table-row-object'
return keys[0] as 'png' | 'svg' | 'jpeg' | 'html' | 'gif'
}
const tableLargeObject = roughSizeOfObject(result) > TABLE_MAX_SIZE
// Otherwise, check if the result is too large (10kb) for json
largeObject = tableLargeObject || roughSizeOfObject(result) > DISPLAY_MAX_SIZE
if (tableLargeObject) {
return 'json'
}
if (Array.isArray(result)) {
largeObject = roughSizeOfObject(result) > 5000000
if (largeObject) {
if (result.length === 0) {
return 'json'
} else if (result.every((elt) => inferResultKind(elt) === 's3object')) {
} else if (
result.every((elt) => typeof elt === 'object' && checkIfS3(elt, Object.keys(elt)))
) {
largeObject = result.length > 100
if (largeObject) {
return 'json'
}
return 's3object-list'
} else if (isArrayOfArrays(result)) {
} else if (isTableRow(result)) {
return 'table-row'
} else if (isObjectOfArrays(result, keys)) {
return 'table-col'
} else if (isArrayOfObjects(result)) {
} else if (isTableRowObject(result)) {
return 'table-row-object'
} else {
return 'json'
}
} else if (
keys.length === 1 &&
['table-row', 'table-row-object', 'table-col'].includes(keys[0])
) {
return keys[0] as 'table-row' | 'table-row-object' | 'table-col'
}
let length = roughSizeOfObject(result)
// Otherwise, check if the result is too large (10kb) for json
largeObject = length > 100000
if (largeObject) {
return 'json'
}
if (keys.length != 0) {
if (keys.length == 1 && keys[0] == 'table-row') {
return 'table-row'
} else if (keys.length == 1 && keys[0] == 'table-col') {
return 'table-col'
} else if (keys.length == 1 && keys[0] == 'table-row-object') {
return 'table-row-object'
} else if (keys.length == 1 && keys[0] == 'html') {
if (keys.length == 1 && keys[0] == 'html') {
return 'html'
} else if (keys.length == 1 && keys[0] == 'map') {
return 'map'
@@ -196,10 +209,12 @@
keys.includes('approvalPage')
) {
return 'approval'
} else if (keys.length === 1 && keys.includes('s3') && typeof result.s3 === 'string') {
} else if (checkIfS3(result, keys)) {
return 's3object'
} else if (keys.length === 1 && (keys.includes('md') || keys.includes('markdown'))) {
return 'markdown'
} else if (isTableCol(result, keys)) {
return 'table-col'
}
}
} catch (err) {}
@@ -225,29 +240,6 @@
}
}
function isArrayOfObjects(json) {
// check array of objects (with possible a first row of headers)
return (
Array.isArray(json) &&
json.length > 0 &&
(json.every(
(item) =>
item && typeof item === 'object' && Object.keys(item).length > 0 && !Array.isArray(item)
) ||
(Array.isArray(json[0]) &&
json[0].every((item) => typeof item === 'string') &&
json
.slice(1)
.every(
(item) =>
item &&
typeof item === 'object' &&
Object.keys(item).length > 0 &&
!Array.isArray(item)
)))
)
}
function handleArrayOfObjectsHeaders(json: any) {
// handle possible a first row of headers
if (
@@ -152,6 +152,8 @@
bind:extra={schema.properties[argName]}
simpleTooltip={schemaFieldTooltip[argName]}
nullable={schema.properties[argName].nullable}
bind:title={schema.properties[argName].title}
bind:placeholder={schema.properties[argName].placeholder}
/>
{/if}
{/if}
+37 -31
View File
@@ -1,7 +1,7 @@
<script lang="ts">
import { type Job } from '$lib/gen'
import JobStatus from '$lib/components/JobStatus.svelte'
import { displayDate } from '$lib/utils'
import { displayDate, truncateRev } from '$lib/utils'
import ScheduleEditor from './ScheduleEditor.svelte'
import TimeAgo from './TimeAgo.svelte'
import { workspaceStore } from '$lib/stores'
@@ -15,42 +15,42 @@
</script>
<div
class="rounded-md p-3 bg-surface-secondary shadow-sm sm:text-sm md:text-base"
class="rounded-md p-3 bg-surface-secondary shadow-sm sm:text-sm md:text-base overflow-x-auto"
style="min-height: 150px;"
>
<JobStatus {job} />
<div class="flex flex-row gap-2 items-center">
<Clock size={SMALL_ICON_SIZE} class="text-secondary" />
<span class="text-2xs text-secondary">
<Clock size={SMALL_ICON_SIZE} class="text-secondary min-w-3.5" />
<span class="text-2xs text-secondary whitespace-nowrap">
{#if job['success'] != undefined}
Received job: {displayDate(job.created_at ?? '')}
{:else}
Received job <TimeAgo date={job.created_at ?? ''} />
{/if}
<Tooltip>{job?.created_at}</Tooltip>
<Tooltip small>{job?.created_at}</Tooltip>
</span>
</div>
{#if job && 'started_at' in job && job.started_at}
<div class="flex flex-row gap-2 items-center">
<Clock size={SMALL_ICON_SIZE} class="text-secondary" />
<span>
<div class="flex flex-row gap-2 items-center text-sm">
<Clock size={SMALL_ICON_SIZE} class="text-secondary min-w-3.5" />
<span class="whitespace-nowrap">
Started <TimeAgo withDate agoOnlyIfRecent date={job.started_at ?? ''} />
<Tooltip>{job?.started_at}</Tooltip>
<Tooltip small>{job?.started_at}</Tooltip>
</span>
</div>
{/if}
{#if job && job['mem_peak']}
<div class="flex flex-row gap-2 items-center">
<MemoryStick size={SMALL_ICON_SIZE} class="text-secondary" />
<div class="flex flex-row gap-2 items-center text-sm">
<MemoryStick size={SMALL_ICON_SIZE} class="text-secondary min-w-3.5" />
<span> Mem peak: {(job['mem_peak'] / 1024).toPrecision(5)}MB</span>
</div>
{/if}
<div>
{#if job && job.parent_job}
{#if job.is_flow_step}
<div class="flex flex-row gap-2 items-center">
<BarsStaggered size={SMALL_ICON_SIZE} class="text-secondary" />
<span>
<div class="flex flex-row gap-2 items-center text-sm">
<BarsStaggered size={SMALL_ICON_SIZE} class="text-secondary min-w-3.5" />
<span class="whitespace-nowrap">
Step of flow
<a href={`/run/${job.parent_job}?workspace=${$workspaceStore}`}>
{job.parent_job}
@@ -58,26 +58,27 @@
</span>
</div>
{:else}
<div class="flex flex-row gap-2 items-center">
<Bot size={SMALL_ICON_SIZE} class="text-secondary" />
<span>
<div class="flex flex-row gap-2 items-center text-sm">
<Bot size={SMALL_ICON_SIZE} class="text-secondary min-w-3.5" />
<span class="whitespace-nowrap">
Triggered by parent
<a href={`/run/${job.parent_job}?workspace=${$workspaceStore}`}> {job.parent_job}</a>
</span>
</div>
{/if}
{:else if job && job.schedule_path}
<div class="flex flex-row gap-2 items-center">
<Calendar size={SMALL_ICON_SIZE} class="text-secondary" />
<span>
Triggered by the schedule:
<button
class="break-words text-sm text-blue-600 font-normal"
<div class="flex flex-row gap-2 w-full items-center text-sm">
<Calendar size={SMALL_ICON_SIZE} class="text-secondary min-w-3.5" />
<span class="whitespace-nowrap">
Schedule:
<a
href="#"
class="break-words text-blue-600 font-normal"
on:click={() =>
scheduleEditor?.openEdit(job.schedule_path ?? '', job.job_kind == 'flow')}
>
{job.schedule_path}
</button>
{truncateRev(job.schedule_path, 40)}
</a>
</span>
</div>
{/if}
@@ -86,9 +87,9 @@
{@const stem = `/${job?.job_kind}s`}
{@const isScript = job?.job_kind === 'script'}
{@const viewHref = `${stem}/get/${isScript ? job?.script_hash : job?.script_path}`}
<div class="flex flex-row gap-2 items-center ml-0.5">
<div class="flex flex-row gap-2 items-center">
{#if isScript}
<Code2 size={SMALL_ICON_SIZE} class="text-secondary" />
<Code2 size={SMALL_ICON_SIZE} class="text-secondary min-w-3.5" />
{:else}
<BarsStaggered size={SMALL_ICON_SIZE} class="text-secondary" />
{/if}
@@ -98,18 +99,23 @@
</div>
{/if}
<div class="flex flex-row gap-2 items-center text-sm">
<User size={SMALL_ICON_SIZE} class="text-secondary" />
<div class="flex items-center gap-2 text-sm">
<User size={SMALL_ICON_SIZE} class="text-secondary min-w-3.5" />
<span>
By {job.created_by}
By {truncateRev(
job.created_by ?? 'unknown',
40
)}{#if (job?.created_by?.length ?? 0) > 40}<Tooltip small
><div class="break-all">{job.created_by}</div></Tooltip
>{/if}
{#if job.permissioned_as !== `u/${job.created_by}` && job.permissioned_as != job.created_by}
but permissioned as {job.permissioned_as}
{/if}
</span>
</div>
</div>
<div class="text-secondary text-2xs pt-2">
<div class="text-secondary text-2xs pt-2 whitespace-nowrap">
run id:
<a href={`/run/${job.id}?workspace=${job.workspace_id}`}> {job.id} </a>
</div>
@@ -204,11 +204,12 @@
console.error(e)
}
}
if (job?.type !== 'CompletedJob' && errorCount < 4) {
if (job?.type !== 'CompletedJob' && errorCount < 4 && !destroyed) {
timeout = setTimeout(() => loadJobInProgress(), reducedPolling ? 5000 : 1000)
}
}
let destroyed = false
async function updateJobId() {
if (jobId !== job?.id) {
$localModuleStates = {}
@@ -244,6 +245,7 @@
$: isListJob = flowJobIds != undefined && Array.isArray(flowJobIds?.flowJobs)
onDestroy(() => {
destroyed = true
timeout && clearTimeout(timeout)
})
@@ -407,6 +407,8 @@
bind:pickForField
showSchemaExplorer
nullable={schema.properties[argName].nullable}
bind:title={schema.properties[argName].title}
bind:placeholder={schema.properties[argName].placeholder}
/>
{:else if arg.expr != undefined}
<div class="border mt-2">
@@ -51,6 +51,8 @@
export let hideResourceInput: boolean = false
export let resourceInputUnsupported: boolean = false
export let render = true
export let title: string | undefined = undefined
export let placeholder: string | undefined = undefined
const dispatch = createEventDispatcher()
@@ -180,8 +182,8 @@
<div>
{#if displayHeader}
<FieldHeader
prettify
{label}
prettify={emptyString(title)}
label={title && !emptyString(title) ? title : label}
{required}
{type}
{contentEncoding}
@@ -231,7 +233,7 @@
? ''
: 'border !border-red-700 !border-opacity-70 focus:!border-red-700 focus:!border-opacity-30'
)}
placeholder={defaultValue ?? ''}
placeholder={placeholder ?? defaultValue ?? ''}
bind:value
min={extra['min']}
max={extra['max']}
@@ -430,7 +432,7 @@
class={valid
? ''
: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-3'}
placeholder={defaultValue ?? ''}
placeholder={placeholder ?? defaultValue ?? ''}
bind:value
/>
{:else if inputCat == 'currency'}
@@ -439,7 +441,7 @@
class={valid
? ''
: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-3'}
placeholder={defaultValue ?? ''}
placeholder={placeholder ?? defaultValue ?? ''}
bind:value
/>
{:else if inputCat == 'string'}
@@ -449,7 +451,7 @@
<Password bind:password={value} />
{:else}
<textarea
rows="1"
rows={extra?.['rows'] || 1}
bind:this={el}
on:focus={(e) => {
dispatch('focus')
@@ -458,7 +460,7 @@
class="col-span-10 {valid && error == ''
? ''
: 'border !border-red-700 !border-opacity-70 focus:!border-red-700 focus:!border-opacity-30'}"
placeholder={defaultValue ?? ''}
placeholder={placeholder ?? defaultValue ?? ''}
bind:value
on:pointerdown|stopPropagation={(e) => {
dispatch('inputClicked', e)
@@ -86,6 +86,8 @@
nestedRequired={schema.properties[argName].required}
itemsType={schema.properties[argName].items}
extra={schema.properties[argName]}
title={schema.properties[argName].title}
placeholder={schema.properties[argName].placeholder}
on:inputClicked
{displayType}
{css}
@@ -96,6 +96,8 @@
itemsType={schema.properties[argName].items}
extra={schema.properties[argName]}
nullable={schema.properties[argName].nullable}
title={schema.properties[argName].title}
placeholder={schema.properties[argName].placeholder}
/>
{/if}
<div class="pt-6 mt-0.5">
@@ -13,6 +13,7 @@
access_key: string
secret_key: string
endpoint: string
allow_http?: boolean
}
type AzureConfig = {
@@ -28,13 +29,18 @@
export let bucket_config: S3Config | AzureConfig | undefined = undefined
$: bucket_config?.type == 'S3' &&
bucket_config.allow_http == undefined &&
(bucket_config.allow_http = true)
let loading = false
async function testConnection() {
loading = true
try {
if (bucket_config) {
await SettingService.testObjectStorageConfig({ requestBody: bucket_config })
await SettingService.testObjectStorageConfig({
requestBody: bucket_config
})
sendUserToast('Connection successful', false)
}
} catch (e) {
@@ -151,6 +157,12 @@
>
<input type="text" bind:value={bucket_config.endpoint} />
</label>
<div class="block pb-2">
<span class="text-tertiary text-2xs">Disable if using https only policy</span>
<div>
<Toggle bind:checked={bucket_config.allow_http} options={{ right: 'allow http' }} />
</div>
</div>
{:else if bucket_config.type === 'Azure'}
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Account Name</span>
@@ -77,6 +77,7 @@
args = {}
runnable = undefined
is_flow = nis_flow
schedule = '0 0 12 * *'
let defaultErrorHandlerMaybe = undefined
let defaultRecoveryHandlerMaybe = undefined
if ($workspaceStore) {
@@ -489,7 +490,7 @@
readonly
value={path}
size={path?.length || 50}
class="font-mono !text-xs max-w-[calc(100%-70px)] !w-auto !h-[24px] !py-0 !border-l-0 !rounded-l-none"
class="font-mono !text-xs grow shrink overflow-x-auto !h-[24px] !py-0 !border-l-0 !rounded-l-none"
on:focus={({ currentTarget }) => {
currentTarget.select()
}}
@@ -159,6 +159,8 @@
simpleTooltip={schemaFieldTooltip[argName]}
{onlyMaskPassword}
nullable={schema.properties[argName].nullable}
title={schema.properties[argName].title}
placeholder={schema.properties[argName].placeholder}
>
<svelte:fragment slot="actions">
{#if linkedSecretCandidates?.includes(argName)}
+34 -1
View File
@@ -62,7 +62,9 @@
showExpr: schema.showExpr,
password: schema.password,
nullable: schema.nullable,
dateFormat: schema.format
dateFormat: schema.format,
title: schema.title,
placeholder: schema.placeholder
}
}
@@ -88,6 +90,8 @@
import LightweightSchemaForm from './LightweightSchemaForm.svelte'
import NumberTypeNarrowing from './NumberTypeNarrowing.svelte'
import SimpleEditor from './SimpleEditor.svelte'
import Label from './Label.svelte'
import { shouldDisplayPlaceholder } from '$lib/utils'
export let error = ''
export let editing = false
@@ -138,6 +142,8 @@
property.password = undefined
property.nullable = false
property.dateFormat = undefined
property.title = undefined
property.placeholder = undefined
drawer.closeDrawer()
}
@@ -157,6 +163,16 @@
return []
}
function shouldDisplayPlaceholderForProperty(property: ModalSchemaProperty): boolean {
return shouldDisplayPlaceholder(
property.selectedType,
property.format,
property.enum_,
property.contentEncoding,
property.pattern
)
}
</script>
<Drawer bind:this={drawer} placement="right">
@@ -192,6 +208,11 @@
</div>
<textarea placeholder="Enter a description" rows="3" bind:value={property.description} />
</label>
<label class="block">
<div class="mb-1 font-semibold text-secondary"> Custom Title </div>
<textarea placeholder="Enter a custom title" rows="1" bind:value={property.title} />
</label>
<div>
<div class="mb-1 font-semibold text-secondary">Type<Required required={true} /></div>
<div class="grid sm:grid-cols-3 md:grid-cols-4 gap-x-2 gap-y-1 items-center mb-2 w-full">
@@ -266,6 +287,8 @@
extra={property}
disabled={property.password}
nullable={property.nullable}
title={property.title}
placeholder={property.placeholder}
/>
<div>
<Toggle
@@ -341,6 +364,16 @@
</svelte:fragment>
</Tabs>
{/if}
{#if property && shouldDisplayPlaceholderForProperty(property)}
<Label label="Placeholder" class="pt-2">
<textarea
placeholder="Enter a placeholder"
rows="1"
bind:value={property.placeholder}
/>
</Label>
{/if}
<div class="pt-2">
<Toggle
options={{ right: 'Show this field only when conditions are met' }}
@@ -1,4 +1,5 @@
<script lang="ts">
import { computeKind } from '$lib/utils'
import Label from './Label.svelte'
import RadioButton from './RadioButton.svelte'
import ResourceTypePicker from './ResourceTypePicker.svelte'
@@ -21,7 +22,13 @@
export let noExtra = false
export let dateFormat: string | undefined
let kind: 'none' | 'pattern' | 'enum' | 'resource' | 'format' | 'base64' = computeKind()
let kind: 'none' | 'pattern' | 'enum' | 'resource' | 'format' | 'base64' = computeKind(
enum_,
contentEncoding,
pattern,
format
)
let patternStr: string = pattern ?? ''
let resource: string | undefined
@@ -64,24 +71,6 @@
}
}
function computeKind(): 'base64' | 'none' | 'pattern' | 'enum' | 'resource' | 'format' {
if (enum_ != undefined) {
return 'enum'
}
if (contentEncoding == 'base64') {
return 'base64'
}
if (pattern != undefined) {
return 'pattern'
}
if (format != undefined && format != '') {
if (format.startsWith('resource')) {
return 'resource'
}
return 'format'
}
return 'none'
}
const presetOptions = [
{ label: 'ISO Format', format: 'yyyy-MM-dd' },
{ label: 'US Format', format: 'MM/dd/yyyy' },
@@ -31,17 +31,11 @@
let runnableComponent: RunnableComponent | undefined = undefined
function clear() {
lastComponentInput = componentInput
setTimeout(() => {
aggrid?.clearRows()
}, 0)
}
let lastComponentInput = componentInput
$: JSON.stringify(lastComponentInput) !== JSON.stringify(componentInput) && clear()
const context = getContext<AppViewerContext>('AppViewerContext')
const { app, worldStore } = context
@@ -182,6 +176,9 @@
{render}
autoRefresh={true}
allowConcurentRequests
on:argsChanged={() => {
clear()
}}
>
<div class="flex flex-col h-full">
{#if resolvedConfig.searchEnabled}
@@ -284,6 +284,7 @@
: undefined,
onPaginationChanged: (event) => {
outputs?.page.set(event.api.paginationGetCurrentPage())
footerRenderCount++
},
initialState: state,
suppressRowDeselection: true,
@@ -303,7 +304,13 @@
$componentControl[id] = {
agGrid: { api: e.api, columnApi: e.columnApi },
setSelectedIndex: (index) => {
e.api.getRowNode(index.toString())?.setSelected(true)
if(index === null) {
e.api.deselectAll();
outputs?.selectedRow?.set({})
outputs?.selectedRowIndex.set(0)
} else {
e.api.getRowNode(index.toString())?.setSelected(true)
}
}
}
api = e.api
@@ -422,6 +429,7 @@
}
let loading = false
let refreshCount: number = 0
let footerRenderCount: number = 0
</script>
{#each Object.keys(components['aggridcomponent'].initialData.configuration) as key (key)}
@@ -488,88 +496,91 @@
{/key}
</div>
{#if resolvedConfig.footer}
<div class="flex gap-1 w-full justify-between items-center text-sm text-secondary/80 p-2">
<div>
<Popover>
<svelte:fragment slot="text">Download</svelte:fragment>
<Button
startIcon={{ icon: Download }}
color="light"
size="xs2"
on:click={() => {
api?.exportDataAsCsv()
}}
iconOnly
/>
</Popover>
</div>
<div class="flex flex-row gap-1 items-center">
{#if resolvedConfig?.pagination}
{#key refreshCount}
<div class="text-xs mx-2 text-primary">
{(api?.paginationGetPageSize() ?? 0) * (api?.paginationGetCurrentPage() ?? 0) + 1}
to {Math.min(
api?.paginationGetRowCount() ?? 0,
((api?.paginationGetCurrentPage() ?? 0) + 1) *
(api?.paginationGetPageSize() ?? 0)
)}
of {api?.paginationGetRowCount()}
</div>
{#key footerRenderCount}
<div class="flex gap-1 w-full justify-between items-center text-sm text-secondary/80 p-2">
<div>
<Popover>
<svelte:fragment slot="text">Download</svelte:fragment>
<Button
startIcon={{ icon: Download }}
color="light"
size="xs2"
on:click={() => {
api?.exportDataAsCsv()
}}
iconOnly
/>
</Popover>
</div>
<div class="flex flex-row gap-1 items-center">
{#if resolvedConfig?.pagination}
{#key refreshCount}
<div class="text-xs mx-2 text-primary">
{(api?.paginationGetPageSize() ?? 0) * (api?.paginationGetCurrentPage() ?? 0) +
1}
to {Math.min(
api?.paginationGetRowCount() ?? 0,
((api?.paginationGetCurrentPage() ?? 0) + 1) *
(api?.paginationGetPageSize() ?? 0)
)}
of {api?.paginationGetRowCount()}
</div>
<Button
iconOnly
startIcon={{ icon: SkipBack }}
color="light"
size="xs2"
disabled={api?.paginationGetCurrentPage() == 0}
on:click={() => {
api?.paginationGoToFirstPage()
refreshCount++
}}
/>
<Button
iconOnly
startIcon={{ icon: ChevronLeft }}
color="light"
size="xs2"
disabled={api?.paginationGetCurrentPage() == 0}
on:click={() => {
api?.paginationGoToPreviousPage()
refreshCount++
}}
/>
<div class="text-xs mx-2 text-primary">
Page {(api?.paginationGetCurrentPage() ?? 0) + 1} of {api?.paginationGetTotalPages() ??
0}
</div>
<Button
iconOnly
startIcon={{ icon: ChevronRight }}
color="light"
size="xs2"
disabled={(api?.paginationGetCurrentPage() ?? 0) + 1 ==
api?.paginationGetTotalPages()}
on:click={() => {
api?.paginationGoToNextPage()
refreshCount++
}}
/>
<Button
iconOnly
startIcon={{ icon: SkipForward }}
color="light"
size="xs2"
disabled={(api?.paginationGetCurrentPage() ?? 0) + 1 ==
api?.paginationGetTotalPages()}
on:click={() => {
api?.paginationGoToLastPage()
refreshCount++
}}
/>
{/key}
{/if}
<Button
iconOnly
startIcon={{ icon: SkipBack }}
color="light"
size="xs2"
disabled={api?.paginationGetCurrentPage() == 0}
on:click={() => {
api?.paginationGoToFirstPage()
refreshCount++
}}
/>
<Button
iconOnly
startIcon={{ icon: ChevronLeft }}
color="light"
size="xs2"
disabled={api?.paginationGetCurrentPage() == 0}
on:click={() => {
api?.paginationGoToPreviousPage()
refreshCount++
}}
/>
<div class="text-xs mx-2 text-primary">
Page {(api?.paginationGetCurrentPage() ?? 0) + 1} of {api?.paginationGetTotalPages() ??
0}
</div>
<Button
iconOnly
startIcon={{ icon: ChevronRight }}
color="light"
size="xs2"
disabled={(api?.paginationGetCurrentPage() ?? 0) + 1 ==
api?.paginationGetTotalPages()}
on:click={() => {
api?.paginationGoToNextPage()
refreshCount++
}}
/>
<Button
iconOnly
startIcon={{ icon: SkipForward }}
color="light"
size="xs2"
disabled={(api?.paginationGetCurrentPage() ?? 0) + 1 ==
api?.paginationGetTotalPages()}
on:click={() => {
api?.paginationGoToLastPage()
refreshCount++
}}
/>
{/key}
{/if}
</div>
</div>
</div>
{/key}
{/if}
</div>
</SyncColumnDefs>
@@ -137,6 +137,8 @@
resultJobLoader &&
refreshIfAutoRefresh('arg changed')
$: runnableInputValues && dispatch('argsChanged')
$: refreshOn =
runnable && runnable.type === 'runnableByName' ? runnable.inlineScript?.refreshOn ?? [] : []
@@ -275,6 +275,7 @@
on:done
on:doneError
on:cancel
on:argsChanged
on:resultSet={() => (initializing = false)}
on:success={(e) => {
onSuccess(e.detail)
@@ -14,7 +14,8 @@
export let customCss: ComponentCustomCSS<'fileinputcomponent'> | undefined = undefined
export let render: boolean
const { app, worldStore, componentControl } = getContext<AppViewerContext>('AppViewerContext')
const { app, worldStore, componentControl, mode } =
getContext<AppViewerContext>('AppViewerContext')
let acceptedFileTypes: string[] | undefined = undefined
let allowMultiple: boolean | undefined = undefined
@@ -48,6 +49,16 @@
}
let files: File[] | undefined = undefined
function preFillFiles() {
const data = outputs?.result?.peak()
if (data && Array.isArray(data) && data.length > 0) {
files = data.map((file: { name: any }) => new File([], file?.name))
}
}
$: outputs.result && files === undefined && $mode === 'dnd' && preFillFiles()
</script>
{#each Object.keys(css ?? {}) as key (key)}

Some files were not shown because too many files have changed in this diff Show More