starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-25 16:02:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into operator-menu

Browse Source
This commit is contained in:
Faton Ramadani
2024-01-08 11:35:59 +01:00
committed by GitHub
parent 37744ff583 d340fbc02f
commit 2cdda619de
132 changed files with 5921 additions and 1847 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/docker-image.yml
+18 -23
View File
@@ -200,25 +200,25 @@ jobs:
- run: |
# pulling docker image with desired arch so that actions-docker-extract doesn't do it
docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
- run: |
# Checks the image is in docker prior to running actions-docker-extract. It fails if not
# Also useful to visually check that the arch is the right opencontainers
docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
- uses: shrink/actions-docker-extract@v3
id: extract
with:
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
path: "/usr/src/app/windmill"
- uses: shrink/actions-docker-extract@v3
id: extract-ee
with:
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
path: "/usr/src/app/windmill"
- name: Rename binary with corresponding architecture
@@ -244,25 +244,25 @@ jobs:
- run: |
# pulling docker image with desired arch so that actions-docker-extract doesn't do it
docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
- run: |
# Checks the image is in docker prior to running actions-docker-extract. It fails if not
# Also useful to visually check that the arch is the right opencontainers
docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
- uses: shrink/actions-docker-extract@v3
id: extract
with:
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
path: "/usr/src/app/windmill"
- uses: shrink/actions-docker-extract@v3
id: extract-ee
with:
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
path: "/usr/src/app/windmill"
- name: Rename binary with corresponding architecture
@@ -307,7 +307,7 @@ jobs:
git_hash=$(git rev-parse --short "$GITHUB_SHA")
docker buildx imagetools create \
--tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${git_hash:0:7} \
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
echo "GIT_HASH=${git_hash:0:7}" >> "$GITHUB_OUTPUT"
- uses: shrink/actions-docker-extract@v3
@@ -349,7 +349,6 @@ jobs:
path: |
integration_tests/logs
tag_latest:
runs-on: ubicloud
needs: [run_integration_test]
@@ -365,10 +364,8 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}
- name: Tag main and latest
run: |
docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main
docker push -a ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main
tag_latest_ee:
runs-on: ubicloud
@@ -384,8 +381,6 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Tag main and latest for ee
run: |
docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main
docker push -a ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
run: |
docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main
CHANGELOG.md
+80
View File
@@ -1,6 +1,86 @@
# Changelog
## [1.236.2](https://github.com/windmill-labs/windmill/compare/v1.236.1...v1.236.2) (2024-01-08)
### Bug Fixes
* handle better non object result for app result loader ([269afe4](https://github.com/windmill-labs/windmill/commit/269afe41f7e02da8d3a364e649e5fef00847d24a))
## [1.236.1](https://github.com/windmill-labs/windmill/compare/v1.236.0...v1.236.1) (2024-01-08)
### Bug Fixes
* fix app initialization ([6b075e9](https://github.com/windmill-labs/windmill/commit/6b075e92ef4e312e6a4ce582dc7c4d88456b2f5d))
## [1.236.0](https://github.com/windmill-labs/windmill/compare/v1.235.1...v1.236.0) (2024-01-08)
### Features
* improve git sync and recompute dependents when common python script is redeployed ([#2967](https://github.com/windmill-labs/windmill/issues/2967)) ([9449381](https://github.com/windmill-labs/windmill/commit/94493812ac0030cc6bba468d8ee24a14661716a8))
### Bug Fixes
* reduce stack size needed for workflow transitions ([4384617](https://github.com/windmill-labs/windmill/commit/4384617f1ec4c27de88ef6c895d6ef060a79f6bd))
## [1.235.1](https://github.com/windmill-labs/windmill/compare/v1.235.0...v1.235.1) (2024-01-07)
### Bug Fixes
* fix resultjobloader ([2e02300](https://github.com/windmill-labs/windmill/commit/2e023003da8bea9efa86c0fbc52bf9723c73907d))
## [1.235.0](https://github.com/windmill-labs/windmill/compare/v1.234.1...v1.235.0) (2024-01-06)
### Features
* **frontend:** DB Explorer ([#2892](https://github.com/windmill-labs/windmill/issues/2892)) ([fffc533](https://github.com/windmill-labs/windmill/commit/fffc5338ce76fda9a68b19f03cefe199cc09a905))
## [1.234.1](https://github.com/windmill-labs/windmill/compare/v1.234.0...v1.234.1) (2024-01-05)
### Bug Fixes
* fix double view runs button ([9efcb53](https://github.com/windmill-labs/windmill/commit/9efcb53bcd903bf756c4beb92a5c7634e8e016ad))
* fix typo "Wourker" -> "Worker" ([4385edc](https://github.com/windmill-labs/windmill/commit/4385edc6ffdb1b2e4544347928b4cf08faf1eaa1))
## [1.234.0](https://github.com/windmill-labs/windmill/compare/v1.233.0...v1.234.0) (2024-01-05)
### Features
* Detailed job memory footprint on job run page ([#2956](https://github.com/windmill-labs/windmill/issues/2956)) ([0710ce6](https://github.com/windmill-labs/windmill/commit/0710ce6ef25d8f2344ac1350e96aa906f16d1be1))
* git sync can now push commits to individual branches ([#2959](https://github.com/windmill-labs/windmill/issues/2959)) ([fb5cd6a](https://github.com/windmill-labs/windmill/commit/fb5cd6a4298c88e26481b76258bf5b09a11cf4b0))
### Bug Fixes
* Deleting workspace delete all workspace related values in DB ([#2961](https://github.com/windmill-labs/windmill/issues/2961)) ([c05d119](https://github.com/windmill-labs/windmill/commit/c05d119ae5dfb6f1bae88fc9d53b48a707cd2f15))
## [1.233.0](https://github.com/windmill-labs/windmill/compare/v1.232.0...v1.233.0) (2024-01-03)
### Features
* **frontend:** add the view runs buttons for operators ([#2932](https://github.com/windmill-labs/windmill/issues/2932)) ([821d2d7](https://github.com/windmill-labs/windmill/commit/821d2d7ea15e828dbb3683c85025dbc09c7e8dc7))
* Job caching now works with S3 files ([#2934](https://github.com/windmill-labs/windmill/issues/2934)) ([b44618f](https://github.com/windmill-labs/windmill/commit/b44618f35b6fe4558a5cd9fddec1e82216a2cf5e))
* PIP_LOCAL_DEPENDENCIES now accepts regex and is configurable from the UI ([#2940](https://github.com/windmill-labs/windmill/issues/2940)) ([e0140f1](https://github.com/windmill-labs/windmill/commit/e0140f175d371e239f315613a537820b9a6ad25f))
### Bug Fixes
* allow extraConfig for agGrid to set onGridReady ([74fc2d3](https://github.com/windmill-labs/windmill/commit/74fc2d3f4397d976fec326893f8196008cb8b332))
* decision tree fix wording ([#2941](https://github.com/windmill-labs/windmill/issues/2941)) ([0299c65](https://github.com/windmill-labs/windmill/commit/0299c656c368876581dc228c76513f1171ffd682))
* decision tree nits ([#2936](https://github.com/windmill-labs/windmill/issues/2936)) ([787017f](https://github.com/windmill-labs/windmill/commit/787017f53c8227e1246b09e412e92316977937ae))
* **frontend:** handle case where large json has a file and a filename ([#2951](https://github.com/windmill-labs/windmill/issues/2951)) ([2494171](https://github.com/windmill-labs/windmill/commit/2494171b9cb3d1857235b6558d276451d5550560))
* persist state for agGrid ([c085f5f](https://github.com/windmill-labs/windmill/commit/c085f5fa7ec598d0abf6aad98de0e5c7b1c92985))
* Tag all docker images with latest and main tags ([#2953](https://github.com/windmill-labs/windmill/issues/2953)) ([9655718](https://github.com/windmill-labs/windmill/commit/965571853ca253b130bcc4473190b6c9c98f44e1))
## [1.232.0](https://github.com/windmill-labs/windmill/compare/v1.231.0...v1.232.0) (2023-12-27)
README.md
+1 -1
View File
@@ -2,7 +2,7 @@
<a href="https://www.windmill.dev/"><img src="./imgs/windmill-banner.png" alt="windmill.dev"></a>
</p>
<p align=center>
Open-source developer infrastructure for internal tools (APIs, background jobs, workflows and UIs). Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps.
Open-source developer infrastructure for internal tools (APIs, background jobs, workflows and UIs). Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs and custom UIs to trigger workflows and scripts as internal apps.
<p align=center>
Scripts are turned into sharable UIs automatically, and can be composed together into flows or used into richer apps built with low-code. Supported script languages supported are: Python, TypeScript, Go, Bash, SQL, and GraphQL.
backend/.sqlx/query-08c943c4b9b76da43b8b754e5c66ab3b6397e7edc82666662a8ea466496290fb.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM account WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "08c943c4b9b76da43b8b754e5c66ab3b6397e7edc82666662a8ea466496290fb"
}
backend/.sqlx/query-266741894b9a6ed650cf999b6e5b7c797424f8291517d3abf096fe45548a0273.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "git_sync",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
true
]
},
"hash": "266741894b9a6ed650cf999b6e5b7c797424f8291517d3abf096fe45548a0273"
}
backend/.sqlx/query-2ce7a562b1de63f767758642a4864ed982b74b53a143901480058f957b9641dc.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE job_stats SET timestamps = timestamps || $4, timeseries_float = timeseries_float || $5 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid",
"Text",
"TimestamptzArray",
"Float4Array"
]
},
"nullable": []
},
"hash": "2ce7a562b1de63f767758642a4864ed982b74b53a143901480058f957b9641dc"
}
backend/.sqlx/query-30e9250edd0c4c6025322f51b758174435d6df4f835cb0883ba40897cb99a410.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM dependency_map\n WHERE importer_path = $1 AND importer_kind = 'script'\n AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "30e9250edd0c4c6025322f51b758174435d6df4f835cb0883ba40897cb99a410"
}
backend/.sqlx/query-4c5276bbf6aaad004a58aaff6b2abd7e78903a0354dd041fcc0cc7cca2b60cc1.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM token WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "4c5276bbf6aaad004a58aaff6b2abd7e78903a0354dd041fcc0cc7cca2b60cc1"
}
backend/.sqlx/query-5fbd5dfb8df9aecce023c5d482ee58efcbc23c8682a3f0ec94977eb56023ed72.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path)\n VALUES ($1, $2, 'script', $3)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar"
]
},
"nullable": []
},
"hash": "5fbd5dfb8df9aecce023c5d482ee58efcbc23c8682a3f0ec94977eb56023ed72"
}
backend/.sqlx/query-60cd5ea09b2eba9df97ab011dd8575940a9dd5e244f3e877d3802fc88087234b.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM input WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "60cd5ea09b2eba9df97ab011dd8575940a9dd5e244f3e877d3802fc88087234b"
}
backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT true FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Uuid",
"Text"
]
},
"nullable": [
null
]
},
"hash": "661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a"
}
backend/.sqlx/query-699427e7e3e6cd3ba4f61f77e171c5c27234cfa30460696ac619760667d7553c.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE job_stats SET scalar_int = $4 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid",
"Text",
"Int4"
]
},
"nullable": []
},
"hash": "699427e7e3e6cd3ba4f61f77e171c5c27234cfa30460696ac619760667d7553c"
}
backend/.sqlx/query-73a3a742702c1c8237a3216523dffe487146b14512d4882179978efe280956aa.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE job_stats SET timestamps = timestamps || $4, timeseries_int = timeseries_int || $5 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid",
"Text",
"TimestamptzArray",
"Int4Array"
]
},
"nullable": []
},
"hash": "73a3a742702c1c8237a3216523dffe487146b14512d4882179978efe280956aa"
}
backend/.sqlx/query-74949628ea78a5fecfd415e6406636a9442edd5b7939f61894288103be0c03ef.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM job_stats WHERE job_id IN (SELECT id FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now())",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "74949628ea78a5fecfd415e6406636a9442edd5b7939f61894288103be0c03ef"
}
backend/.sqlx/query-76d73f2fe5ac75e5033fb93d60f2ef5ab2b0b73a80dcb6a250c17c3b2f15c6b4.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM dependency_map WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "76d73f2fe5ac75e5033fb93d60f2ef5ab2b0b73a80dcb6a250c17c3b2f15c6b4"
}
backend/.sqlx/query-813d3c65d4c4bc10f1ec16ed4ee0954f4b938f45751ac03fd45479e665db1d40.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM deployment_metadata WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "813d3c65d4c4bc10f1ec16ed4ee0954f4b938f45751ac03fd45479e665db1d40"
}
backend/.sqlx/query-96a77e839d6210629234879eeeaa23d9bddf5f751fe27d999659f6597f9759f1.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM raw_app WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "96a77e839d6210629234879eeeaa23d9bddf5f751fe27d999659f6597f9759f1"
}
backend/.sqlx/query-aa44311f9bae8253c6c483d122597ce7e470c6ba1905e7e877dca35d9d68968f.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT importer_path FROM dependency_map\n WHERE imported_path = $1 AND importer_kind = 'script'\n AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "importer_path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "aa44311f9bae8253c6c483d122597ce7e470c6ba1905e7e877dca35d9d68968f"
}
backend/.sqlx/query-b56111c5b81de2449a1b687656a4636457da851a4db5e916cf4f6847c48c6bd3.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE job_stats SET scalar_float = $4 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid",
"Text",
"Float4"
]
},
"nullable": []
},
"hash": "b56111c5b81de2449a1b687656a4636457da851a4db5e916cf4f6847c48c6bd3"
}
backend/.sqlx/query-d77223fc94257711512b6ff21ce109b97324f2917113f04492f53b20860e9588.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM job_stats WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "d77223fc94257711512b6ff21ce109b97324f2917113f04492f53b20860e9588"
}
backend/.sqlx/query-deb2e4c21d4f9d0b9439780ca6ebee13450609b578c469cb437c5420ab520258.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT content FROM script WHERE hash = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "content",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8",
"Text"
]
},
"nullable": [
false
]
},
"hash": "deb2e4c21d4f9d0b9439780ca6ebee13450609b578c469cb437c5420ab520258"
}
backend/Cargo.lock
Generated
+224 -378
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+8 -5
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.232.0"
version = "1.236.2"
authors.workspace = true
edition.workspace = true
@@ -11,6 +11,7 @@ members = [
"./windmill-worker",
"./windmill-common",
"./windmill-audit",
"./windmill-git-sync",
"./parsers/windmill-parser",
"./parsers/windmill-parser-ts",
"./parsers/windmill-parser-wasm",
@@ -21,7 +22,7 @@ members = [
]
[workspace.package]
version = "1.232.0"
version = "1.236.2"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -34,7 +35,7 @@ opt-level = 0
incremental = true
[features]
enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise"]
enterprise_saml = ["windmill-api/enterprise_saml"]
benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"]
flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
@@ -46,6 +47,7 @@ anyhow.workspace = true
tokio.workspace = true
dotenv.workspace = true
windmill-common = { workspace = true, features = ["tracing_init"] }
windmill-git-sync.workspace = true
windmill-api.workspace = true
windmill-worker.workspace = true
futures.workspace = true
@@ -66,7 +68,7 @@ uuid.workspace = true
gethostname.workspace = true
serde_json.workspace = true
serde.workspace = true
pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true}
pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false}
[dev-dependencies]
serde_json.workspace = true
@@ -83,6 +85,7 @@ windmill-queue = { path = "./windmill-queue" }
windmill-worker = { path = "./windmill-worker" }
windmill-common = { path = "./windmill-common" }
windmill-audit = { path = "./windmill-audit" }
windmill-git-sync = { path = "./windmill-git-sync" }
windmill-parser = { path = "./parsers/windmill-parser" }
windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
windmill-parser-py = { path = "./parsers/windmill-parser-py" }
@@ -153,7 +156,7 @@ swc_ecma_visit = "0.93.7"
base64 = "0.21.0"
hmac = "0.12.1"
sha2 = "0.10.6"
sqlx = { version = "^0", features = [
sqlx = { version = "0.7.3", features = [
"macros",
"migrate",
"uuid",
backend/migrations/20231227014107_add_dependency_map.down.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add down migration script here
DROP TABLE dependency_map;
backend/migrations/20231227014107_add_dependency_map.up.sql
+14
View File
@@ -0,0 +1,14 @@
-- Add up migration script here
CREATE TYPE IMPORTER_KIND AS ENUM ('script', 'flow', 'app');
CREATE TABLE IF NOT EXISTS dependency_map (
workspace_id VARCHAR(50) NOT NULL,
importer_path VARCHAR(510) NOT NULL,
importer_kind IMPORTER_KIND NOT NULL,
imported_path VARCHAR(510) NOT NULL,
PRIMARY KEY (workspace_id, importer_path, importer_kind, imported_path)
);
CREATE UNIQUE INDEX IF NOT EXISTS dependency_map_imported_path_idx ON dependency_map (workspace_id, imported_path);
backend/migrations/20240102133412_job_stats_table.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add down migration script here
DROP TABLE job_stats;
DROP TYPE METRIC_KIND;
backend/migrations/20240102133412_job_stats_table.up.sql
+16
View File
@@ -0,0 +1,16 @@
-- Add up migration script here
CREATE TYPE METRIC_KIND AS ENUM ('scalar_int', 'scalar_float', 'timeseries_int', 'timeseries_float');
CREATE TABLE IF NOT EXISTS job_stats (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
job_id UUID NOT NULL,
metric_id VARCHAR(50) NOT NULL,
metric_name VARCHAR(255),
metric_kind METRIC_KIND NOT NULL,
scalar_int INTEGER,
scalar_float REAL,
timestamps TIMESTAMP WITH TIME ZONE[],
timeseries_int INTEGER[],
timeseries_float REAL[],
PRIMARY KEY (workspace_id, job_id, metric_id)
);
backend/parsers/windmill-parser-py-imports/src/lib.rs
+54 -40
View File
@@ -63,7 +63,7 @@ lazy_static! {
fn process_import(module: Option<String>, path: &str, level: usize) -> Vec<String> {
if level > 0 {
let mut imports = vec!["requests".to_string()];
let mut imports = vec![];
let splitted_path = path.split("/");
let base = splitted_path
.clone()
@@ -78,10 +78,7 @@ fn process_import(module: Option<String>, path: &str, level: usize) -> Vec<Strin
} else if let Some(module) = module {
let imprt = module.split('.').next().unwrap_or("").replace("_", "-");
if imprt == "u" || imprt == "f" {
vec![
"requests".to_string(),
format!("relative:{}", module.replace(".", "/")),
]
vec![format!("relative:{}", module.replace(".", "/"))]
} else {
vec![imprt]
}
@@ -90,6 +87,57 @@ fn process_import(module: Option<String>, path: &str, level: usize) -> Vec<Strin
}
}
pub fn parse_relative_imports(code: &str, path: &str) -> error::Result<Vec<String>> {
let nimports = parse_code_for_imports(code, path)?;
return Ok(nimports
.into_iter()
.filter_map(|x| {
if x.starts_with("relative:") {
Some(x.replace("relative:", ""))
} else {
None
}
})
.collect());
}
fn parse_code_for_imports(code: &str, path: &str) -> error::Result<Vec<String>> {
let code = code.split(DEF_MAIN).next().unwrap_or("");
let ast = Suite::parse(code, "main.py").map_err(|e| {
error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
})?;
let nimports: Vec<String> = ast
.into_iter()
.filter_map(|x| match x {
Stmt::Import(StmtImport { names, .. }) => Some(
names
.into_iter()
.map(|x| {
let name = x.name.to_string();
process_import(Some(name), path, 0)
})
.flatten()
.collect::<Vec<String>>(),
),
Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => {
Some(process_import(
module.map(|x| x.to_string()),
path,
i.to_usize(),
))
}
Stmt::ImportFrom(StmtImportFrom { level: _, module, .. }) => {
Some(process_import(module.map(|x| x.to_string()), path, 0))
}
_ => None,
})
.flatten()
.filter(|x| !STDIMPORTS.contains(&x.as_str()))
.unique()
.collect();
return Ok(nimports);
}
#[async_recursion]
pub async fn parse_python_imports(
code: &str,
@@ -127,41 +175,7 @@ pub async fn parse_python_imports(
imports.extend(lines);
}
let code = code.split(DEF_MAIN).next().unwrap_or("");
let ast = Suite::parse(code, "main.py").map_err(|e| {
error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
})?;
let nimports: Vec<String> = ast
.into_iter()
.filter_map(|x| match x {
Stmt::Import(StmtImport { names, .. }) => Some(
names
.into_iter()
.map(|x| {
let name = x.name.to_string();
process_import(Some(name), path, 0)
})
.flatten()
.collect::<Vec<String>>(),
),
Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. })
if i.to_u32() > 0 =>
{
Some(process_import(
module.map(|x| x.to_string()),
path,
i.to_usize(),
))
}
Stmt::ImportFrom(StmtImportFrom { level: _, module, .. }) => {
Some(process_import(module.map(|x| x.to_string()), path, 0))
}
_ => None,
})
.flatten()
.filter(|x| !STDIMPORTS.contains(&x.as_str()))
.unique()
.collect();
let nimports = parse_code_for_imports(code, path)?;
for n in nimports.iter() {
let nested = if n.starts_with("relative:") {
let rpath = n.replace("relative:", "");
backend/parsers/windmill-parser-py-imports/tests/tests.rs
+1 -2
View File
@@ -20,7 +20,7 @@ def main():
";
let r = parse_python_imports(code, "test-workspace", "f/foo/bar", &db).await?;
// println!("{}", serde_json::to_string(&r)?);
assert_eq!(r, vec!["matplotlib", "requests", "wmill", "zanzibar"]);
assert_eq!(r, vec!["matplotlib", "wmill", "zanzibar"]);
Ok(())
}
@@ -73,7 +73,6 @@ def main():
"numpy",
"pandas",
"pandas2",
"requests"
]
);
backend/src/main.rs
+1 -1
View File
@@ -431,7 +431,7 @@ Windmill Community Edition {GIT_VERSION}
},
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => {
load_require_preexisting_user(&db).await;
}
},
EXPOSE_METRICS_SETTING | EXPOSE_DEBUG_METRICS_SETTING => {
if n.payload() != EXPOSE_DEBUG_METRICS_SETTING || worker_mode {
tracing::info!("Metrics setting changed, restarting");
backend/src/monitor.rs
+48 -18
View File
@@ -35,8 +35,8 @@ use windmill_common::{
BASE_URL, DB, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
};
use windmill_worker::{
create_token_for_owner, handle_job_error, AuthedClient, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR,
NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
create_token_for_owner, handle_job_error, AuthedClient, SendResult, JOB_DEFAULT_TIMEOUT,
KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
};
#[cfg(feature = "enterprise")]
@@ -167,6 +167,7 @@ pub async fn load_metrics_debug_enabled(db: &DB) -> error::Result<()> {
};
Ok(())
}
pub async fn load_keep_job_dir(db: &DB) {
let value = sqlx::query_scalar!(
"SELECT value FROM global_settings WHERE name = $1",
@@ -250,25 +251,52 @@ pub async fn delete_expired_items(db: &DB) -> () {
let job_retention_secs = *JOB_RETENTION_SECS.read().await;
if job_retention_secs > 0 {
let deleted_jobs = sqlx::query_scalar!(
"DELETE FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now() RETURNING id",
job_retention_secs
)
.fetch_all(db)
.await;
match db.begin().await {
Ok(mut tx) => {
let r = sqlx::query!(
"DELETE FROM job_stats WHERE job_id IN (SELECT id FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now())",
job_retention_secs
)
.fetch_all(&mut *tx)
.await;
match r {
Ok(_) => {
let deleted_jobs = sqlx::query_scalar!(
"DELETE FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now() RETURNING id",
job_retention_secs
)
.fetch_all(&mut *tx)
.await;
match deleted_jobs {
Ok(deleted_jobs) => {
if deleted_jobs.len() > 0 {
tracing::info!(
"deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
deleted_jobs.len(),
job_retention_secs,
deleted_jobs,
)
match deleted_jobs {
Ok(deleted_jobs) => {
if deleted_jobs.len() > 0 {
tracing::info!(
"deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
deleted_jobs.len(),
job_retention_secs,
deleted_jobs,
)
}
}
Err(e) => {
tracing::error!("Error deleting expired jobs: {:?}", e)
}
}
}
Err(err) => {
tracing::error!("Error deleting expired job stats: {:?}", err)
}
}
match tx.commit().await {
Ok(_) => (),
Err(err) => tracing::error!("Error deleting expired jobs: {:?}", err),
}
}
Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
Err(err) => {
tracing::error!("Error deleting expired jobs: {:?}", err)
}
}
}
}
@@ -729,6 +757,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
// since the job is unrecoverable, the same worker queue should never be sent anything
let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::<Uuid>(1);
let (send_result_never_used, _send_result_rx_never_used) = mpsc::channel::<SendResult>(1);
let token = create_token_for_owner(
&db,
@@ -767,6 +796,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
"",
rsmq.clone(),
worker_name,
send_result_never_used,
)
.await;
}
backend/tests/worker.rs
+1 -1
View File
@@ -1805,7 +1805,7 @@ def main():
}
#[sqlx::test(fixtures("base"))]
async fn test_empty_loop(db: Pool<Postgres>) {
async fn test_empty_loop_1(db: Pool<Postgres>) {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await;
let port = server.addr.port();
backend/windmill-api/Cargo.toml
+2 -1
View File
@@ -9,7 +9,7 @@ name = "windmill_api"
path = "src/lib.rs"
[features]
enterprise = ["windmill-queue/enterprise", "async-stripe", "windmill-audit/enterprise"]
enterprise = ["windmill-queue/enterprise", "async-stripe", "windmill-audit/enterprise", "windmill-git-sync/enterprise"]
enterprise_saml = ["samael"]
benchmark = []
@@ -27,6 +27,7 @@ windmill-common = { workspace = true, features = [
windmill-audit.workspace = true
windmill-parser.workspace = true
windmill-parser-py-imports.workspace = true
windmill-git-sync.workspace = true
tokio.workspace = true
anyhow.workspace = true
argon2.workspace = true
backend/windmill-api/openapi-deref.yaml
+164 -49
View File
@@ -1,6 +1,6 @@
openapi: 3.0.3
info:
version: 1.230.0
version: 1.233.0
title: Windmill API
contact:
name: Windmill Team
@@ -1531,6 +1531,8 @@ paths:
type: string
git_repo_resource_path:
type: string
use_individual_branch:
type: boolean
required: &ref_17
- script_path
- git_repo_resource_path
@@ -3352,32 +3354,32 @@ paths:
id:
type: string
value:
oneOf: &ref_196
oneOf: &ref_204
- type: object
properties: &ref_180
properties: &ref_188
input_transforms:
type: object
additionalProperties:
oneOf: &ref_26
- type: object
properties: &ref_176
properties: &ref_184
value: {}
type:
type: string
enum:
- javascript
required: &ref_177
required: &ref_185
- expr
- type
- type: object
properties: &ref_178
properties: &ref_186
expr:
type: string
type:
type: string
enum:
- javascript
required: &ref_179
required: &ref_187
- expr
- type
discriminator: &ref_27
@@ -3417,13 +3419,13 @@ paths:
type: number
concurrency_time_window_s:
type: number
required: &ref_181
required: &ref_189
- type
- content
- language
- input_transforms
- type: object
properties: &ref_182
properties: &ref_190
input_transforms:
type: object
additionalProperties:
@@ -3437,12 +3439,12 @@ paths:
type: string
enum:
- script
required: &ref_183
required: &ref_191
- type
- path
- input_transforms
- type: object
properties: &ref_184
properties: &ref_192
input_transforms:
type: object
additionalProperties:
@@ -3454,12 +3456,12 @@ paths:
type: string
enum:
- flow
required: &ref_185
required: &ref_193
- type
- path
- input_transforms
- type: object
properties: &ref_186
properties: &ref_194
modules:
type: array
items:
@@ -3481,13 +3483,13 @@ paths:
type: boolean
parallelism:
type: integer
required: &ref_187
required: &ref_195
- modules
- iterator
- skip_failures
- type
- type: object
properties: &ref_188
properties: &ref_196
branches:
type: array
items:
@@ -3518,12 +3520,12 @@ paths:
type: string
enum:
- branchone
required: &ref_189
required: &ref_197
- branches
- default
- type
- type: object
properties: &ref_190
properties: &ref_198
branches:
type: array
items:
@@ -3548,28 +3550,28 @@ paths:
- branchall
parallel:
type: boolean
required: &ref_191
required: &ref_199
- branches
- type
- type: object
properties: &ref_192
properties: &ref_200
type:
type: string
enum:
- identity
flow:
type: boolean
required: &ref_193
required: &ref_201
- type
- type: object
properties: &ref_194
properties: &ref_202
type:
type: string
enum:
- graphql
required: &ref_195
required: &ref_203
- type
discriminator: &ref_197
discriminator: &ref_205
propertyName: type
mapping:
rawscript: '#/components/schemas/RawScript'
@@ -8240,6 +8242,10 @@ paths:
retry:
type: object
properties: *ref_78
no_flow_overlap:
type: boolean
summary:
type: string
required: &ref_148
- path
- schedule
@@ -8305,6 +8311,10 @@ paths:
retry:
type: object
properties: *ref_78
no_flow_overlap:
type: boolean
summary:
type: string
required: &ref_150
- schedule
- timezone
@@ -8446,6 +8456,10 @@ paths:
retry:
type: object
properties: *ref_78
summary:
type: string
no_flow_overlap:
type: boolean
required: &ref_80
- path
- edited_by
@@ -10309,6 +10323,91 @@ paths:
- upload_id
- parts
- is_done
/w/{workspace}/job_metrics/get/{id}:
post:
summary: get job metrics
operationId: getJobMetrics
tags:
- metrics
parameters:
- name: workspace
in: path
required: true
schema: *ref_0
- name: id
in: path
required: true
schema: *ref_74
requestBody:
description: parameters for statistics retrieval
required: true
content:
application/json:
schema:
type: object
properties:
timeseries_max_datapoints:
type: integer
from_timestamp:
type: string
format: date-time
to_timestamp:
type: string
format: date-time
responses:
'200':
description: job details
content:
application/json:
schema:
type: object
properties:
metrics_metadata:
type: array
items:
type: object
properties: &ref_176
id:
type: string
name:
type: string
required: &ref_177
- id
scalar_metrics:
type: array
items:
type: object
properties: &ref_178
metric_id:
type: string
value:
type: number
required: &ref_179
- id
- value
timeseries_metrics:
type: array
items:
type: object
properties: &ref_180
metric_id:
type: string
values:
type: array
items:
type: object
properties: &ref_182
timestamp:
type: string
format: date-time
value:
type: number
required: &ref_183
- timestamp
- value
required: &ref_181
- id
- values
components:
securitySchemes:
bearerAuth:
@@ -10960,56 +11059,72 @@ components:
type: object
properties: *ref_96
required: *ref_97
StaticTransform:
MetricMetadata:
type: object
properties: *ref_176
required: *ref_177
JavascriptTransform:
ScalarMetric:
type: object
properties: *ref_178
required: *ref_179
TimeseriesMetric:
type: object
properties: *ref_180
required: *ref_181
MetricDataPoint:
type: object
properties: *ref_182
required: *ref_183
StaticTransform:
type: object
properties: *ref_184
required: *ref_185
JavascriptTransform:
type: object
properties: *ref_186
required: *ref_187
InputTransform:
oneOf: *ref_26
discriminator: *ref_27
RawScript:
type: object
properties: *ref_180
required: *ref_181
properties: *ref_188
required: *ref_189
PathScript:
type: object
properties: *ref_182
required: *ref_183
properties: *ref_190
required: *ref_191
PathFlow:
type: object
properties: *ref_184
required: *ref_185
properties: *ref_192
required: *ref_193
FlowModule:
type: object
properties: *ref_28
required: *ref_29
ForloopFlow:
type: object
properties: *ref_186
required: *ref_187
BranchOne:
type: object
properties: *ref_188
required: *ref_189
BranchAll:
type: object
properties: *ref_190
required: *ref_191
Identity:
type: object
properties: *ref_192
required: *ref_193
Graphql:
type: object
properties: *ref_194
required: *ref_195
BranchOne:
type: object
properties: *ref_196
required: *ref_197
BranchAll:
type: object
properties: *ref_198
required: *ref_199
Identity:
type: object
properties: *ref_200
required: *ref_201
Graphql:
type: object
properties: *ref_202
required: *ref_203
FlowModuleValue:
oneOf: *ref_196
discriminator: *ref_197
oneOf: *ref_204
discriminator: *ref_205
Retry:
type: object
properties: *ref_78
backend/windmill-api/openapi.yaml
+119 -47
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.232.0
version: 1.236.2
title: Windmill API
contact:
@@ -2231,8 +2231,7 @@ paths:
/w/{workspace}/resources/get_value_interpolated/{path}:
get:
summary:
get resource interpolated (variables and resources are fully unrolled)
summary: get resource interpolated (variables and resources are fully unrolled)
operationId: getResourceValueInterpolated
tags:
- resource
@@ -2920,8 +2919,7 @@ paths:
schema:
type: string
- name: first_parent_hash
description:
mask to filter scripts whom first direct parent has exact hash
description: mask to filter scripts whom first direct parent has exact hash
in: query
schema:
type: string
@@ -3116,8 +3114,7 @@ paths:
/workers/custom_tags:
get:
summary:
get all instance custom tags (tags are used to dispatch jobs to
summary: get all instance custom tags (tags are used to dispatch jobs to
different worker groups)
operationId: getCustomTags
tags:
@@ -3168,8 +3165,7 @@ paths:
/w/{workspace}/scripts/delete/h/{hash}:
post:
summary:
delete script by hash (erase content but keep hash, require admin)
summary: delete script by hash (erase content but keep hash, require admin)
operationId: deleteScriptByHash
tags:
- script
@@ -3408,8 +3404,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -3417,8 +3412,7 @@ paths:
- $ref: "#/components/parameters/WorkerTag"
- $ref: "#/components/parameters/NewJobId"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4451,8 +4445,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4461,8 +4454,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the flow owner (default false)
description: make the run invisible to the the flow owner (default false)
in: query
schema:
type: boolean
@@ -4514,8 +4506,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4524,8 +4515,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the flow owner (default false)
description: make the run invisible to the the flow owner (default false)
in: query
schema:
type: boolean
@@ -4563,8 +4553,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4573,8 +4562,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4605,8 +4593,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4639,8 +4626,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4857,8 +4843,7 @@ paths:
- job
responses:
"200":
description:
the timestamp of the db that can be used to compute the drift
description: the timestamp of the db that can be used to compute the drift
content:
application/json:
schema:
@@ -5122,8 +5107,7 @@ paths:
/w/{workspace}/jobs/resume_urls/{id}/{resume_id}:
get:
summary:
get resume urls given a job_id, resume_id and a nonce to resume a flow
summary: get resume urls given a job_id, resume_id and a nonce to resume a flow
operationId: getResumeUrls
tags:
- job
@@ -5674,8 +5658,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- name: only_member_of
in: query
description:
only list the groups the user is member of (default false)
description: only list the groups the user is member of (default false)
schema:
type: boolean
responses:
@@ -5863,8 +5846,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- name: only_member_of
in: query
description:
only list the folders the user is member of (default false)
description: only list the folders the user is member of (default false)
schema:
type: boolean
responses:
@@ -6692,8 +6674,7 @@ paths:
- storage_options
/w/{workspace}/job_helpers/v2/s3_resource_info:
post:
summary:
Returns the s3 resource associated to the provided path, or the
summary: Returns the s3 resource associated to the provided path, or the
workspace default S3 resource
operationId: s3ResourceInfo
tags:
@@ -6939,6 +6920,53 @@ paths:
- parts
- is_done
/w/{workspace}/job_metrics/get/{id}:
post:
summary: get job metrics
operationId: getJobMetrics
tags:
- metrics
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/JobId"
requestBody:
description: parameters for statistics retrieval
required: true
content:
application/json:
schema:
type: object
properties:
timeseries_max_datapoints:
type: integer
from_timestamp:
type: string
format: date-time
to_timestamp:
type: string
format: date-time
responses:
"200":
description: job details
content:
application/json:
schema:
type: object
properties:
metrics_metadata:
type: array
items:
$ref: "#/components/schemas/MetricMetadata"
scalar_metrics:
type: array
items:
$ref: "#/components/schemas/ScalarMetric"
timeseries_metrics:
type: array
items:
$ref: "#/components/schemas/TimeseriesMetric"
components:
securitySchemes:
bearerAuth:
@@ -7037,8 +7065,7 @@ components:
type: integer
PerPage:
name: per_page
description:
number of items to return for a given page (default 30, max 100)
description: number of items to return for a given page (default 30, max 100)
in: query
schema:
type: integer
@@ -7166,8 +7193,7 @@ components:
type: boolean
ScheduledForBeforeNow:
name: scheduled_for_before_now
description:
filter on jobs scheduled_for before now (hence waitinf for a worker)
description: filter on jobs scheduled_for before now (hence waitinf for a worker)
in: query
schema:
type: boolean
@@ -7185,8 +7211,7 @@ components:
type: boolean
ArgsFilter:
name: args
description:
filter on jobs containing those args as a json subset (@> in postgres)
description: filter on jobs containing those args as a json subset (@> in postgres)
in: query
schema:
type: string
@@ -7198,8 +7223,7 @@ components:
type: string
ResultFilter:
name: result
description:
filter on jobs containing those result as a json subset (@> in postgres)
description: filter on jobs containing those result as a json subset (@> in postgres)
in: query
schema:
type: string
@@ -9088,6 +9112,8 @@ components:
type: string
git_repo_resource_path:
type: string
use_individual_branch:
type: boolean
required:
- script_path
- git_repo_resource_path
@@ -9102,3 +9128,49 @@ components:
required:
- part_number
- tag
MetricMetadata:
type: object
properties:
id:
type: string
name:
type: string
required:
- id
ScalarMetric:
type: object
properties:
metric_id:
type: string
value:
type: number
required:
- id
- value
TimeseriesMetric:
type: object
properties:
metric_id:
type: string
values:
type: array
items:
$ref: "#/components/schemas/MetricDataPoint"
required:
- id
- values
MetricDataPoint:
type: object
properties:
timestamp:
type: string
format: date-time
value:
type: number
required:
- timestamp
- value
backend/windmill-api/src/apps.rs
+14 -44
View File
@@ -9,7 +9,6 @@ use std::collections::HashMap;
*/
use crate::{
db::{ApiAuthed, DB},
deployment_metadata_helpers,
users::{require_owner_of_path, OptAuthed},
variables::build_crypt,
webhook_util::{WebhookMessage, WebhookShared},
@@ -588,12 +587,16 @@ async fn create_app(
)
.await?;
let mut tx = PushIsolationLevel::Transaction(tx);
let tx = PushIsolationLevel::Transaction(tx);
let (dependency_job_uuid, new_tx) = push(
&db,
tx,
&w_id,
JobPayload::AppDependencies { path: app.path.clone(), version: v_id },
JobPayload::AppDependencies {
path: app.path.clone(),
version: v_id,
deployment_message: app.deployment_message,
},
PushArgs::empty(),
&authed.username,
&authed.email,
@@ -615,25 +618,7 @@ async fn create_app(
.await?;
tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
tx = PushIsolationLevel::Transaction(new_tx);
tx = deployment_metadata_helpers::handle_deployment_metadata(
tx,
&authed,
&db,
&w_id,
deployment_metadata_helpers::DeployedObject::App { path: app.path.clone(), version: v_id },
app.deployment_message,
)
.await?;
match tx {
PushIsolationLevel::Transaction(tx) => tx.commit().await?,
_ => {
return Err(Error::InternalErr(
"Expected a transaction here".to_string(),
));
}
}
new_tx.commit().await?;
webhook.send_message(
w_id.clone(),
@@ -849,14 +834,18 @@ async fn update_app(
)
.await?;
let mut tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
let tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
PushIsolationLevel::Transaction(tx);
if let Some(v_id) = v_id {
let (dependency_job_uuid, new_tx) = push(
&db,
tx,
&w_id,
JobPayload::AppDependencies { path: npath.clone(), version: v_id },
JobPayload::AppDependencies {
path: npath.clone(),
version: v_id,
deployment_message: ns.deployment_message,
},
PushArgs::empty(),
&authed.username,
&authed.email,
@@ -877,26 +866,7 @@ async fn update_app(
)
.await?;
tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
tx = PushIsolationLevel::Transaction(new_tx);
tx = deployment_metadata_helpers::handle_deployment_metadata(
tx,
&authed,
&db,
&w_id,
deployment_metadata_helpers::DeployedObject::App { path: npath.clone(), version: v_id },
ns.deployment_message,
)
.await?;
}
match tx {
PushIsolationLevel::Transaction(tx) => tx.commit().await?,
_ => {
return Err(Error::InternalErr(
"Expected a transaction here".to_string(),
));
}
new_tx.commit().await?;
}
webhook.send_message(
backend/windmill-api/src/flows.rs
+6 -42
View File
@@ -7,7 +7,6 @@
*/
use crate::db::ApiAuthed;
use crate::deployment_metadata_helpers;
use crate::{
db::DB,
schedule::clear_schedule,
@@ -355,7 +354,7 @@ async fn create_flow(
)
.await?;
let mut tx = PushIsolationLevel::Transaction(tx);
let tx = PushIsolationLevel::Transaction(tx);
let (dependency_job_uuid, mut new_tx) = push(
&db,
tx,
@@ -363,6 +362,7 @@ async fn create_flow(
JobPayload::FlowDependencies {
path: nf.path.clone(),
dedicated_worker: nf.dedicated_worker,
deployment_message: nf.deployment_message,
},
PushArgs::empty(),
&authed.username,
@@ -393,26 +393,7 @@ async fn create_flow(
.execute(&mut new_tx)
.await?;
tx = PushIsolationLevel::Transaction(new_tx);
tx = deployment_metadata_helpers::handle_deployment_metadata(
tx,
&authed,
&db,
&w_id,
deployment_metadata_helpers::DeployedObject::Flow { path: nf.path.clone() },
nf.deployment_message,
)
.await?;
match tx {
PushIsolationLevel::Transaction(tx) => tx.commit().await?,
_ => {
return Err(Error::InternalErr(
"Expected a transaction here".to_string(),
));
}
}
new_tx.commit().await?;
webhook.send_message(
w_id.clone(),
WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() },
@@ -585,7 +566,7 @@ async fn update_flow(
},
);
let mut tx = PushIsolationLevel::Transaction(tx);
let tx = PushIsolationLevel::Transaction(tx);
let (dependency_job_uuid, mut new_tx) = push(
&db,
@@ -594,6 +575,7 @@ async fn update_flow(
JobPayload::FlowDependencies {
path: nf.path.clone(),
dedicated_worker: nf.dedicated_worker,
deployment_message: nf.deployment_message,
},
PushArgs::empty(),
&authed.username,
@@ -631,25 +613,7 @@ async fn update_flow(
.await?;
}
tx = PushIsolationLevel::Transaction(new_tx);
tx = deployment_metadata_helpers::handle_deployment_metadata(
tx,
&authed,
&db,
&w_id,
deployment_metadata_helpers::DeployedObject::Flow { path: nf.path.clone() },
nf.deployment_message,
)
.await?;
match tx {
PushIsolationLevel::Transaction(tx) => tx.commit().await?,
_ => {
return Err(Error::InternalErr(
"Expected a transaction here".to_string(),
));
}
}
new_tx.commit().await?;
Ok(nf.path.to_string())
}
backend/windmill-api/src/job_metrics.rs
+165
View File
@@ -0,0 +1,165 @@
use crate::db::DB;
use axum::{extract::Path, routing::post, Extension, Json, Router};
use hyper::http;
use serde::{Deserialize, Serialize};
use tower_http::cors::{Any, CorsLayer};
use uuid::Uuid;
use windmill_common::{
error,
job_metrics::{JobStatsRecord, MetricKind},
};
pub fn workspaced_service() -> Router {
let cors = CorsLayer::new()
.allow_methods([http::Method::GET, http::Method::POST])
.allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION])
.allow_origin(Any);
Router::new().route("/get/:id", post(get_job_metrics).layer(cors.clone()))
}
#[derive(Deserialize)]
struct JobStatsRequest {
from_timestamp: Option<chrono::DateTime<chrono::Utc>>,
to_timestamp: Option<chrono::DateTime<chrono::Utc>>,
timeseries_max_datapoints: Option<u32>, // default to 100, any value lower than that will be ignored. Set to 0 to retrieve all
}
#[derive(Serialize)]
struct JobStatsResponse {
metrics_metadata: Vec<MetricsMetadata>,
scalar_metrics: Vec<ScalarMetric>,
timeseries_metrics: Vec<TimeseriesMetric>,
}
#[derive(Serialize)]
pub struct MetricsMetadata {
pub id: String,
pub name: Option<String>,
}
#[derive(Serialize)]
struct ScalarMetric {
metric_id: String,
value: f64,
}
#[derive(Serialize)]
struct TimeseriesMetric {
metric_id: String,
values: Vec<DataPoint>,
}
#[derive(Debug, Clone, Serialize)]
pub struct DataPoint {
pub timestamp: chrono::DateTime<chrono::Utc>,
pub value: f64,
}
async fn get_job_metrics(
Extension(db): Extension<DB>,
Path((w_id, job_id)): Path<(String, Uuid)>,
Json(JobStatsRequest { from_timestamp, to_timestamp, timeseries_max_datapoints }): Json<
JobStatsRequest,
>,
) -> error::JsonResult<JobStatsResponse> {
let records = sqlx::query_as::<_, JobStatsRecord>(
"SELECT * FROM job_stats where workspace_id = $1 and job_id = $2",
)
.bind(w_id)
.bind(job_id)
.fetch_all(&db)
.await?;
let mut metrics_metadata: Vec<MetricsMetadata> = vec![];
let mut scalar_metrics: Vec<ScalarMetric> = vec![];
let mut timeseries_metrics: Vec<TimeseriesMetric> = vec![];
for record in records {
let metric_id = record.metric_id;
match record.metric_kind {
MetricKind::ScalarInt => {
let value = record.scalar_int.unwrap_or_default() as f64;
scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value });
}
MetricKind::ScalarFloat => {
let value = record.scalar_float.unwrap_or_default() as f64;
scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value });
}
MetricKind::TimeseriesInt => {
if record.timestamps.clone().unwrap_or_default().len()
!= record.timeseries_int.clone().unwrap_or_default().len()
{
tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_int)
}
let (timestamps, timeseries_int) = timeseries_sample(
from_timestamp,
to_timestamp,
timeseries_max_datapoints,
record.timestamps.unwrap_or_default(),
record.timeseries_int.unwrap_or_default(),
);
let mut values: Vec<DataPoint> = vec![];
for (idx, value) in timeseries_int.iter().enumerate() {
values.push(DataPoint {
timestamp: timestamps[idx],
value: value.to_owned() as f64,
});
}
timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values });
}
MetricKind::TimeseriesFloat => {
if record.timestamps.clone().unwrap_or_default().len()
!= record.timeseries_int.clone().unwrap_or_default().len()
{
tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_float)
}
let (timestamps, timeseries_float) = timeseries_sample(
from_timestamp,
to_timestamp,
timeseries_max_datapoints,
record.timestamps.unwrap_or_default(),
record.timeseries_float.unwrap_or_default(),
);
let mut values: Vec<DataPoint> = vec![];
for (idx, value) in timeseries_float.iter().enumerate() {
values.push(DataPoint {
timestamp: timestamps[idx],
value: value.to_owned() as f64,
});
}
timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values });
}
};
metrics_metadata.push(MetricsMetadata { id: metric_id, name: record.metric_name });
}
let response = JobStatsResponse { metrics_metadata, scalar_metrics, timeseries_metrics };
Ok(Json(response))
}
fn timeseries_sample<T: Copy>(
from: Option<chrono::DateTime<chrono::Utc>>,
to: Option<chrono::DateTime<chrono::Utc>>,
_datapoints: Option<u32>,
timestamps: Vec<chrono::DateTime<chrono::Utc>>,
values: Vec<T>,
) -> (Vec<chrono::DateTime<chrono::Utc>>, Vec<T>) {
if timestamps.len() != values.len() {
tracing::warn!("Timeseries metric has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", timestamps.len(), values.len());
return (vec![], vec![]);
}
let mut filtered_timestamp: Vec<chrono::DateTime<chrono::Utc>> = vec![];
let mut filtered_values: Vec<T> = vec![];
for (idx, timestamp) in timestamps.iter().enumerate() {
if *timestamp > from.unwrap_or(chrono::DateTime::<chrono::Utc>::MIN_UTC)
&& *timestamp < to.unwrap_or(chrono::DateTime::<chrono::Utc>::MAX_UTC)
{
filtered_timestamp.push(timestamps[idx]);
filtered_values.push(values[idx]);
}
}
// TODO: implement sampling
return (filtered_timestamp, filtered_values);
}
backend/windmill-api/src/jobs.rs
+4 -4
View File
@@ -2826,14 +2826,14 @@ pub struct JobUpdate {
async fn get_job_update(
Extension(db): Extension<DB>,
Path((w_id, id)): Path<(String, Uuid)>,
Path((w_id, job_id)): Path<(String, Uuid)>,
Query(JobUpdateQuery { running, log_offset }): Query<JobUpdateQuery>,
) -> error::JsonResult<JobUpdate> {
let record = sqlx::query!(
"SELECT running, substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3",
log_offset,
&w_id,
&id
&job_id
)
.fetch_optional(&db)
.await?;
@@ -2855,11 +2855,11 @@ async fn get_job_update(
$3",
log_offset,
&w_id,
&id
&job_id
)
.fetch_optional(&db)
.await?;
let logs = not_found_if_none(logs, "Job Update", id.to_string())?;
let logs = not_found_if_none(logs, "Job Update", job_id.to_string())?;
Ok(Json(JobUpdate {
running: Some(false),
completed: Some(true),
backend/windmill-api/src/lib.rs
+2 -1
View File
@@ -49,7 +49,6 @@ mod audit;
mod capture;
mod configs;
mod db;
mod deployment_metadata_helpers;
mod drafts;
pub mod ee;
pub mod embeddings;
@@ -61,6 +60,7 @@ mod groups;
mod inputs;
mod integration;
pub mod job_helpers;
pub mod job_metrics;
pub mod jobs;
pub mod oauth2;
mod openai;
@@ -198,6 +198,7 @@ pub async fn run_server(
.nest("/folders", folders::workspaced_service())
.nest("/groups", groups::workspaced_service())
.nest("/inputs", inputs::workspaced_service())
.nest("/job_metrics", job_metrics::workspaced_service())
.nest("/job_helpers", job_helpers::workspaced_service())
.nest("/jobs", jobs::workspaced_service())
.nest("/oauth", oauth2::workspaced_service())
backend/windmill-api/src/scripts.rs
+19 -34
View File
@@ -8,7 +8,6 @@
use crate::{
db::{ApiAuthed, DB},
deployment_metadata_helpers,
schedule::clear_schedule,
users::{maybe_refresh_folders, require_owner_of_path, AuthCache},
webhook_util::{WebhookMessage, WebhookShared},
@@ -47,6 +46,7 @@ use windmill_common::{
not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
},
};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
use windmill_queue::{
self, schedule::push_scheduled_job, PushArgs, PushIsolationLevel, QueueTransaction,
};
@@ -325,7 +325,7 @@ async fn create_script(
let script_path = ns.path.clone();
let hash = ScriptHash(hash_script(&ns));
let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), user_db.begin(&authed).await?).into();
if sqlx::query_scalar!(
"SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
@@ -604,37 +604,29 @@ async fn create_script(
);
}
let mut tx = PushIsolationLevel::Transaction(tx);
let permissioned_as = username_to_permissioned_as(&authed.username);
if needs_lock_gen {
let dependencies = match ns.language {
ScriptLang::Python3 => {
windmill_parser_py_imports::parse_python_imports(&ns.content, &w_id, &ns.path, &db)
.await?
.join("\n")
}
_ => ns.content,
};
let tag = if ns.dedicated_worker.is_some_and(|x| x) {
Some(format!("{}:{}", &w_id, &ns.path,))
} else {
ns.tag
};
let tx = PushIsolationLevel::Transaction(tx);
let (_, new_tx) = windmill_queue::push(
&db,
tx,
&w_id,
JobPayload::Dependencies {
hash,
dependencies,
language: ns.language,
path: ns.path,
dedicated_worker: ns.dedicated_worker,
deployment_message: ns.deployment_message,
},
PushArgs::empty(),
&authed.username,
&authed.email,
username_to_permissioned_as(&authed.username),
permissioned_as,
None,
None,
None,
@@ -650,26 +642,19 @@ async fn create_script(
None,
)
.await?;
tx = PushIsolationLevel::Transaction(new_tx);
}
tx = deployment_metadata_helpers::handle_deployment_metadata(
tx,
&authed,
&db,
&w_id,
deployment_metadata_helpers::DeployedObject::Script { hash: hash, path: script_path },
ns.deployment_message,
)
.await?;
match tx {
PushIsolationLevel::Transaction(tx) => tx.commit().await?,
_ => {
return Err(Error::InternalErr(
"Expected a transaction here".to_string(),
));
}
new_tx.commit().await?;
} else {
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::Script { hash: hash, path: script_path },
ns.deployment_message,
rsmq,
)
.await?;
tx.commit().await?;
}
Ok((StatusCode::CREATED, format!("{}", hash)))
backend/windmill-api/src/workspaces.rs
+29 -6
View File
@@ -44,6 +44,7 @@ use windmill_common::s3_helpers::LargeFileStorage;
use windmill_common::schedule::Schedule;
use windmill_common::users::username_to_permissioned_as;
use windmill_common::worker::CLOUD_HOSTED;
use windmill_common::workspaces::WorkspaceGitRepo;
use windmill_common::{
error::{to_anyhow, Error, JsonResult, Result},
flows::Flow,
@@ -1045,12 +1046,6 @@ struct EditGitSyncConfig {
git_sync_settings: Option<WorkspaceGitRepo>,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct WorkspaceGitRepo {
pub script_path: String,
pub git_repo_resource_path: String,
}
async fn edit_git_sync_config(
authed: ApiAuthed,
Extension(db): Extension<DB>,
@@ -1504,6 +1499,9 @@ async fn delete_workspace(
let mut tx = db.begin().await?;
require_super_admin(&db, &email).await?;
sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM queue WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
@@ -1522,6 +1520,12 @@ async fn delete_workspace(
sqlx::query!("DELETE FROM app WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM raw_app WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM input WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM variable WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
@@ -1537,6 +1541,17 @@ async fn delete_workspace(
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM job_stats WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
sqlx::query!(
"DELETE FROM deployment_metadata WHERE workspace_id = $1",
&w_id
)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM usr WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
@@ -1564,6 +1579,10 @@ async fn delete_workspace(
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM account WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM workspace_key WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
@@ -1575,6 +1594,10 @@ async fn delete_workspace(
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM token WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM workspace WHERE id = $1", &w_id)
.execute(&mut *tx)
.await?;
backend/windmill-common/src/job_metrics.rs
+187
View File
@@ -0,0 +1,187 @@
use crate::{db::DB, error};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
#[derive(sqlx::FromRow)]
pub struct JobStatsRecord {
pub workspace_id: String,
pub job_id: Uuid,
pub metric_id: String,
pub metric_name: Option<String>,
pub metric_kind: MetricKind,
pub scalar_int: Option<i32>,
pub scalar_float: Option<f32>,
pub timestamps: Option<Vec<chrono::DateTime<chrono::Utc>>>,
pub timeseries_int: Option<Vec<i32>>,
pub timeseries_float: Option<Vec<f32>>,
}
#[derive(sqlx::Type, Debug, PartialEq, Deserialize, Serialize)]
#[sqlx(type_name = "METRIC_KIND", rename_all = "snake_case")]
pub enum MetricKind {
ScalarInt,
ScalarFloat,
TimeseriesInt,
TimeseriesFloat,
}
pub enum MetricNumericValue {
Integer(i32),
Float(f32),
}
pub async fn register_metric_for_job(
db: &DB,
workspace_id: String,
job_id: Uuid,
metric_id: String,
metric_kind: MetricKind,
metric_name: Option<String>,
) -> error::Result<String> {
let exists = sqlx::query_scalar!(
"SELECT true FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
workspace_id,
job_id,
metric_id
)
.fetch_optional(db)
.await?
.flatten();
if exists.unwrap_or(false) {
return Err(error::Error::BadRequest(format!(
"Metric {} is already registered for job {}",
metric_id, job_id
)));
}
let (scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) = match metric_kind
{
MetricKind::ScalarInt | MetricKind::ScalarFloat => {
(None as Option<i32>, None as Option<f32>, None, None, None)
}
MetricKind::TimeseriesInt => (
None,
None,
Some(&[] as &[chrono::DateTime<chrono::Utc>]),
Some(&[] as &[i32]),
None,
),
MetricKind::TimeseriesFloat => (
None,
None,
Some(&[] as &[chrono::DateTime<chrono::Utc>]),
None,
Some(&[] as &[f32]),
),
};
sqlx::query(
"INSERT INTO job_stats (workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
)
.bind(workspace_id)
.bind(job_id)
.bind(&metric_id)
.bind(metric_name)
.bind(metric_kind)
.bind(scalar_int)
.bind(scalar_float)
.bind(timestamps)
.bind(timeseries_int)
.bind(timeseries_float)
.execute(db)
.await?;
Ok(metric_id)
}
pub async fn record_metric(
db: &DB,
workspace_id: String,
job_id: Uuid,
metric_id: String,
value: MetricNumericValue,
) -> error::Result<()> {
let metric_kind_opt: Option<MetricKind> = sqlx::query_scalar(
"SELECT metric_kind FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
)
.bind(&workspace_id)
.bind(&job_id)
.bind(&metric_id)
.fetch_optional(db)
.await?;
if metric_kind_opt.is_none() {
return Err(error::Error::BadRequest(format!(
"Metric {} not yet registered for job {}.",
metric_id, job_id
)));
}
let metric_kind = metric_kind_opt.unwrap();
let (value_int, value_float) = match value {
MetricNumericValue::Integer(val) => {
if metric_kind != MetricKind::TimeseriesInt && metric_kind != MetricKind::ScalarInt {
return Err(error::Error::BadRequest(format!(
"Metric {} is not a timeseries int metric.",
metric_id
)));
}
(val, 0 as f32)
}
MetricNumericValue::Float(val) => {
if metric_kind != MetricKind::TimeseriesFloat && metric_kind != MetricKind::ScalarFloat
{
return Err(error::Error::BadRequest(format!(
"Metric {} is not a timeseries float metric.",
metric_id
)));
}
(0 as i32, val)
}
};
let timestamp = chrono::Utc::now();
match metric_kind {
MetricKind::ScalarInt => {
sqlx::query!(
"UPDATE job_stats SET scalar_int = $4 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
&workspace_id,
&job_id,
&metric_id,
value_int,
).execute(db).await?;
}
MetricKind::ScalarFloat => {
sqlx::query!(
"UPDATE job_stats SET scalar_float = $4 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
&workspace_id,
&job_id,
&metric_id,
value_float,
).execute(db).await?;
}
MetricKind::TimeseriesInt => {
sqlx::query!(
"UPDATE job_stats SET timestamps = timestamps || $4, timeseries_int = timeseries_int || $5 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
&workspace_id,
&job_id,
&metric_id,
&[timestamp],
&[value_int]
).execute(db).await?;
}
MetricKind::TimeseriesFloat => {
sqlx::query!(
"UPDATE job_stats SET timestamps = timestamps || $4, timeseries_float = timeseries_float || $5 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
&workspace_id,
&job_id,
&metric_id,
&[timestamp],
&[value_float]
).execute(db).await?;
}
}
Ok(())
}
backend/windmill-common/src/jobs.rs
+3 -1
View File
@@ -283,17 +283,19 @@ pub enum JobPayload {
Dependencies {
path: String,
hash: ScriptHash,
dependencies: String,
language: ScriptLang,
dedicated_worker: Option<bool>,
deployment_message: Option<String>,
},
FlowDependencies {
path: String,
dedicated_worker: Option<bool>,
deployment_message: Option<String>,
},
AppDependencies {
path: String,
version: i64,
deployment_message: Option<String>,
},
Flow {
path: String,
backend/windmill-common/src/lib.rs
+2
View File
@@ -23,6 +23,7 @@ pub mod external_ip;
pub mod flow_status;
pub mod flows;
pub mod global_settings;
pub mod job_metrics;
pub mod jobs;
pub mod more_serde;
pub mod oauth2;
@@ -35,6 +36,7 @@ pub mod users;
pub mod utils;
pub mod variables;
pub mod worker;
pub mod workspaces;
#[cfg(feature = "tracing_init")]
pub mod tracing_init;
backend/windmill-common/src/users.rs
+1
View File
@@ -8,6 +8,7 @@
pub const SUPERADMIN_SECRET_EMAIL: &str = "superadmin_secret@windmill.dev";
pub const SUPERADMIN_NOTIFICATION_EMAIL: &str = "superadmin_notification@windmill.dev";
pub const SUPERADMIN_SYNC_EMAIL: &str = "superadmin_sync@windmill.dev";
pub fn username_to_permissioned_as(user: &str) -> String {
if user.contains('@') {
backend/windmill-common/src/workspaces.rs
+8
View File
@@ -0,0 +1,8 @@
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Debug)]
pub struct WorkspaceGitRepo {
pub script_path: String,
pub git_repo_resource_path: String,
pub use_individual_branch: Option<bool>,
}
backend/windmill-git-sync/Cargo.toml
+22
View File
@@ -0,0 +1,22 @@
[package]
name = "windmill-git-sync"
version.workspace = true
authors.workspace = true
edition.workspace = true
[lib]
name = "windmill_git_sync"
path = "./src/lib.rs"
[features]
enterprise = ["windmill-queue/enterprise"]
default = []
[dependencies]
serde.workspace = true
sqlx.workspace = true
serde_json.workspace = true
tracing.workspace = true
windmill-common = { workspace = true, features = ["axum"] }
windmill-queue.workspace = true
rsmq_async.workspace = true
backend/windmill-api/src/deployment_metadata_helpers.rs → backend/windmill-git-sync/src/lib.rs
+70 -55
View File
@@ -8,17 +8,18 @@
use std::collections::HashMap;
use crate::db::ApiAuthed;
use crate::db::DB;
use crate::workspaces::{WorkspaceGitRepo, WorkspaceSettings};
use sqlx::{Pool, Postgres};
use windmill_common::users::SUPERADMIN_SYNC_EMAIL;
use windmill_common::workspaces::WorkspaceGitRepo;
use serde_json::json;
use windmill_common::error::{Error, Result};
use windmill_common::jobs::JobPayload;
use windmill_common::scripts::ScriptHash;
use windmill_common::users::username_to_permissioned_as;
use windmill_queue::PushIsolationLevel;
pub type DB = Pool<Postgres>;
#[derive(Clone)]
pub enum DeployedObject {
Script { hash: ScriptHash, path: String },
@@ -37,13 +38,14 @@ impl DeployedObject {
}
pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
mut tx: PushIsolationLevel<'c, R>,
authed: &ApiAuthed,
email: &str,
created_by: &str,
db: &DB,
w_id: &str,
obj: DeployedObject,
deployment_message: Option<String>,
) -> Result<PushIsolationLevel<'c, R>> {
rsmq: Option<R>,
) -> Result<()> {
let skip_git_sync = if obj.get_path().starts_with("u/") {
tracing::debug!(
"Ignoring {} from git sync as it's in a private user folder",
@@ -54,10 +56,10 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
false
};
let workspace_git_repo_setting = sqlx::query_as::<_, WorkspaceSettings>(
"SELECT * FROM workspace_settings WHERE workspace_id = $1",
let workspace_git_repo_setting = sqlx::query_scalar!(
"SELECT git_sync FROM workspace_settings WHERE workspace_id = $1",
w_id
)
.bind(&w_id)
.fetch_optional(db)
.await?;
@@ -69,11 +71,10 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
let workspace_git_repo = workspace_git_repo_setting
.unwrap()
.git_sync
.map(|conf| serde_json::from_value::<WorkspaceGitRepo>(conf).ok())
.flatten();
let (git_sync_job_uuid, mut new_tx) = if !skip_git_sync && workspace_git_repo.is_some() {
let git_sync_job_uuid = if !skip_git_sync && workspace_git_repo.is_some() {
let workspace_git_repo = workspace_git_repo.unwrap();
let mut args: HashMap<String, serde_json::Value> = HashMap::new();
@@ -84,35 +85,56 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
.strip_prefix("$res:")),
);
let default_commit_msg: String;
match obj.clone() {
let message = match obj.clone() {
DeployedObject::Script { path, .. } => {
args.insert("path".to_string(), json!(path.to_string()));
default_commit_msg = format!("Script '{}' deployed", path);
if deployment_message.as_ref().is_none()
|| deployment_message.as_ref().is_some_and(|x| x.is_empty())
{
format!("Script '{}' deployed", path)
} else {
deployment_message.clone().unwrap()
}
}
DeployedObject::Flow { path } => {
args.insert("path".to_string(), json!(path.to_string()));
default_commit_msg = format!("Flow '{}' deployed", path);
if deployment_message.as_ref().is_none()
|| deployment_message.as_ref().is_some_and(|x| x.is_empty())
{
format!("Flow '{}' deployed", path)
} else {
deployment_message.clone().unwrap()
}
}
DeployedObject::App { path, .. } => {
args.insert("path".to_string(), json!(path.to_string()));
default_commit_msg = format!("App '{}' deployed", path);
if deployment_message.as_ref().is_none()
|| deployment_message.as_ref().is_some_and(|x| x.is_empty())
{
format!("App '{}' deployed", path)
} else {
deployment_message.clone().unwrap()
}
}
}
};
args.insert("commit_msg".to_string(), json!(message));
args.insert(
"commit_msg".to_string(),
json!(deployment_message.clone().unwrap_or(default_commit_msg)),
"use_individual_branch".to_string(),
json!(workspace_git_repo.use_individual_branch.unwrap_or(false)),
);
let tx: PushIsolationLevel<'_, R> = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
let (job_uuid, new_tx) = windmill_queue::push(
&db,
db,
tx,
&w_id,
JobPayload::DeploymentCallback { path: workspace_git_repo.script_path.clone() },
args,
&authed.username,
&authed.email,
username_to_permissioned_as(&authed.username),
created_by,
&email,
SUPERADMIN_SYNC_EMAIL.to_string(),
None,
None,
None,
@@ -128,16 +150,10 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
None,
)
.await?;
(Some(job_uuid), new_tx)
new_tx.commit().await?;
Some(job_uuid)
} else {
let new_tx = match tx {
PushIsolationLevel::Isolated(user_db, authed, rsmq) => {
(rsmq, user_db.begin(&authed).await?).into()
}
PushIsolationLevel::IsolatedRoot(db, rsmq) => (rsmq, db.begin().await?).into(),
PushIsolationLevel::Transaction(tx) => tx,
};
(None, new_tx)
None
};
// We're not persisting the default commit msg as it's pretty useless. We will persist the ones manually set by users
@@ -149,28 +165,27 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
if deployment_message.is_some() || job_uuids.len() > 0 {
// if the git sync job hasn't been triggered, and there is not custom deployment message, there's not point adding an entry to the table
match obj.clone() {
DeployedObject::Script { path, hash, .. } => {
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
w_id, path, hash.0, &job_uuids, deployment_message,
)
},
DeployedObject::Flow { path } => {
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL DO UPDATE SET callback_job_ids = $3, deployment_msg = $4",
w_id, path, &job_uuids, deployment_message,
)
}
DeployedObject::App { path, version } => {
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
w_id, path, version, &job_uuids, deployment_message,
)
}
}.execute(&mut new_tx)
.await?;
DeployedObject::Script { path, hash, .. } => {
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
w_id, path, hash.0, &job_uuids, deployment_message,
)
},
DeployedObject::Flow { path } => {
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL DO UPDATE SET callback_job_ids = $3, deployment_msg = $4",
w_id, path, &job_uuids, deployment_message,
)
}
DeployedObject::App { path, version } => {
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
w_id, path, version, &job_uuids, deployment_message,
)
}
}.execute(db)
.await?;
}
tx = PushIsolationLevel::Transaction(new_tx);
return Ok(tx);
return Ok(());
}
backend/windmill-queue/src/jobs.rs
+20 -18
View File
@@ -2496,21 +2496,23 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
dedicated_worker,
None,
),
JobPayload::Dependencies { hash, dependencies, language, path, dedicated_worker } => (
Some(hash.0),
Some(path),
Some((dependencies, None)),
JobKind::Dependencies,
None,
None,
Some(language),
None,
None,
None,
dedicated_worker,
None,
),
JobPayload::FlowDependencies { path, dedicated_worker } => {
JobPayload::Dependencies { hash, language, path, dedicated_worker, deployment_message } => {
(
Some(hash.0),
Some(path),
Some((deployment_message.unwrap_or_else(String::new), None)),
JobKind::Dependencies,
None,
None,
Some(language),
None,
None,
None,
dedicated_worker,
None,
)
}
JobPayload::FlowDependencies { path, dedicated_worker, deployment_message } => {
let value_json = fetch_scalar_isolated!(
sqlx::query_scalar!(
"SELECT value FROM flow WHERE path = $1 AND workspace_id = $2",
@@ -2528,7 +2530,7 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
(
None,
Some(path),
None,
Some((deployment_message.unwrap_or_else(String::new), None)),
JobKind::FlowDependencies,
Some(value.clone()),
Some(FlowStatus::new(&value)), // this is a new flow being pushed, flow_status is set to flow_value
@@ -2540,10 +2542,10 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
None,
)
}
JobPayload::AppDependencies { path, version } => (
JobPayload::AppDependencies { path, version, deployment_message } => (
Some(version),
Some(path),
None,
Some((deployment_message.unwrap_or_else(String::new), None)),
JobKind::AppDependencies,
None,
None,
backend/windmill-worker/Cargo.toml
+2 -1
View File
@@ -10,7 +10,7 @@ path = "src/lib.rs"
[features]
default = []
enterprise = ["windmill-queue/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:pem", "dep:sha2", "dep:tiberius", "dep:tokio-util"]
enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:pem", "dep:sha2", "dep:tiberius", "dep:tokio-util"]
benchmark = ["windmill-queue/benchmark"]
flamegraph = []
@@ -31,6 +31,7 @@ windmill-parser-py-imports.workspace = true
windmill-parser-bash.workspace = true
windmill-parser-sql.workspace = true
windmill-parser-graphql.workspace = true
windmill-git-sync.workspace = true
sqlx.workspace = true
uuid.workspace = true
tracing.workspace = true
backend/windmill-worker/loader.py
+14 -13
View File
@@ -28,22 +28,23 @@ class WindmillFinder(MetaPathFinder):
return ModuleSpec(name, WindmillLoader(name))
elif l > 2:
script_path = "/".join(splitted)
import requests
import urllib.parse
import urllib.request
headers = {"Authorization": f"Bearer {os.environ.get('WM_TOKEN')}"}
url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py"
r = requests.get(
url, headers={"Authorization": f"Bearer {os.environ.get('WM_TOKEN')}"}
)
if r.status_code == 200:
folder = os.getcwd() + "/tmp/" + "/".join(splitted[:-1])
fullpath = folder + "/" + splitted[-1] + ".py"
os.makedirs(folder, exist_ok=True)
with open(fullpath, "w+") as f:
f.write(r.text)
return ModuleSpec(name, SourceFileLoader(name, fullpath))
else:
req = urllib.request.Request(url, None, headers)
try:
with urllib.request.urlopen(req) as response:
r = response.read().decode("utf-8")
folder = os.getcwd() + "/tmp/" + "/".join(splitted[:-1])
fullpath = folder + "/" + splitted[-1] + ".py"
os.makedirs(folder, exist_ok=True)
with open(fullpath, "w+") as f:
f.write(r)
return ModuleSpec(name, SourceFileLoader(name, fullpath))
except:
# raise ImportError(f"Script {script_path} not found")
return ModuleSpec(name, WindmillLoader(name))
backend/windmill-worker/src/common.rs
+29 -4
View File
@@ -32,7 +32,7 @@ use std::{
use tracing::{trace_span, Instrument};
use uuid::Uuid;
use windmill_common::{variables, DB};
use windmill_common::{job_metrics, variables, DB};
use tokio::{
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
@@ -408,6 +408,7 @@ async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
}
-2
} else {
// rand::random::<i32>() % 100 // to remove - used to fake memory data on MacOS
-3
}
}
@@ -427,7 +428,7 @@ pub async fn handle_child(
mut child: Child,
nsjail: bool,
worker_name: &str,
_w_id: &str,
w_id: &str,
child_name: &str,
custom_timeout: Option<i32>,
sigterm: bool,
@@ -465,6 +466,8 @@ pub async fn handle_child(
interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
let mut i = 0;
let mut memory_metric_id: Result<String, Error> =
Err(Error::NotFound("not yet initialized".to_string()));
loop {
tokio::select!(
@@ -485,7 +488,29 @@ pub async fn handle_child(
if current_mem > *mem_peak {
*mem_peak = current_mem
}
tracing::info!("{worker_name}/{job_id} in {_w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB");
tracing::info!("{worker_name}/{job_id} in {w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB");
#[cfg(feature = "enterprise")]
{
// tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
if i == 2 {
memory_metric_id = job_metrics::register_metric_for_job(
&db,
w_id.to_string(),
job_id,
"memory_kb".to_string(),
job_metrics::MetricKind::TimeseriesInt,
Some("Job Memory Footprint (kB)".to_string()),
)
.await;
}
if let Ok(ref metric_id) = memory_metric_id {
if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
}
}
}
let (canceled, canceled_by, canceled_reason) = sqlx::query_as::<_, (bool, Option<String>, Option<String>)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason")
.bind(*mem_peak)
.bind(job_id)
@@ -516,7 +541,7 @@ pub async fn handle_child(
}
let (timeout_duration, timeout_warn_msg) =
resolve_job_timeout(&db, _w_id, job_id, custom_timeout).await;
resolve_job_timeout(&db, w_id, job_id, custom_timeout).await;
if let Some(msg) = timeout_warn_msg {
logs.push_str(msg.as_str());
append_logs(job_id, msg.as_str(), db).await;
backend/windmill-worker/src/pg_executor.rs
+7 -2
View File
@@ -322,7 +322,13 @@ fn convert_val(value: &Value, arg_t: &String) -> windmill_common::error::Result<
Value::Number(n) if n.is_i64() && (arg_t == "smallint" || arg_t == "smallserial") => {
Ok(PgType::I16(n.as_i64().unwrap() as i16))
}
Value::Number(n) if n.is_i64() && (arg_t == "int" || arg_t == "serial") => {
Value::Number(n)
if n.is_i64()
&& (arg_t == "int"
|| arg_t == "integer"
|| arg_t == "int4"
|| arg_t == "serial") =>
{
Ok(PgType::I32(n.as_i64().unwrap() as i32))
}
Value::Number(n) if n.is_i64() && (arg_t == "numeric" || arg_t == "decimal") => Ok(
@@ -429,7 +435,6 @@ pub fn pg_cell_to_json_value(
Type::TS_VECTOR => get_basic(row, column, column_i, |a: StringCollector| {
Ok(JSONValue::String(a.0))
})?,
// array types
Type::BOOL_ARRAY => get_array(row, column, column_i, |a: bool| Ok(JSONValue::Bool(a)))?,
Type::INT2_ARRAY => get_array(row, column, column_i, |a: i16| {
backend/windmill-worker/src/worker.rs
+422 -150
View File
@@ -25,6 +25,8 @@ use std::{
},
time::Duration,
};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
use windmill_parser_py_imports::parse_relative_imports;
use uuid::Uuid;
#[cfg(feature = "enterprise")]
@@ -33,9 +35,9 @@ use windmill_common::{
error::{self, to_anyhow, Error},
flows::{FlowModule, FlowModuleValue, FlowValue},
get_latest_deployed_hash_for_path,
jobs::{JobKind, QueuedJob},
jobs::{JobKind, JobPayload, QueuedJob},
scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL},
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL},
utils::{rd_string, StripPath},
worker::{
to_raw_value, to_raw_value_owned, update_ping, CLOUD_HOSTED, WORKER_CONFIG, WORKER_GROUP,
@@ -149,7 +151,8 @@ pub async fn create_token_for_owner(
.await?
.unwrap_or(false)
|| email == SUPERADMIN_SECRET_EMAIL
|| email == SUPERADMIN_NOTIFICATION_EMAIL;
|| email == SUPERADMIN_NOTIFICATION_EMAIL
|| owner == SUPERADMIN_SYNC_EMAIL;
sqlx::query_scalar!(
"INSERT INTO token
@@ -472,6 +475,7 @@ async fn handle_receive_completed_job<
worker_name: &str,
worker_save_completed_job_duration: Option<Arc<prometheus::Histogram>>,
worker_flow_transition_duration: Option<Arc<prometheus::Histogram>>,
job_completed_tx: Sender<SendResult>,
) {
let token = jc.token.clone();
let workspace = jc.job.workspace_id.clone();
@@ -494,6 +498,7 @@ async fn handle_receive_completed_job<
worker_name,
worker_save_completed_job_duration,
worker_flow_transition_duration,
job_completed_tx.clone(),
)
.await
{
@@ -509,6 +514,7 @@ async fn handle_receive_completed_job<
&worker_dir,
rsmq.clone(),
worker_name,
job_completed_tx,
)
.await;
}
@@ -516,7 +522,7 @@ async fn handle_receive_completed_job<
#[derive(Clone)]
pub struct JobCompletedSender(
Sender<JobCompleted>,
Sender<SendResult>,
Option<Arc<GenericGauge<AtomicI64>>>,
Option<Arc<prometheus::Histogram>>,
);
@@ -525,12 +531,12 @@ impl JobCompletedSender {
pub async fn send(
&self,
jc: JobCompleted,
) -> Result<(), tokio::sync::mpsc::error::SendError<JobCompleted>> {
) -> Result<(), tokio::sync::mpsc::error::SendError<SendResult>> {
if let Some(wj) = self.1.as_ref() {
wj.inc()
}
let timer = self.2.as_ref().map(|x| x.start_timer());
let r = self.0.send(jc).await;
let r = self.0.send(SendResult::JobCompleted(jc)).await;
timer.map(|x| x.stop_and_record());
r
}
@@ -876,7 +882,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
let (same_worker_tx, mut same_worker_rx) = mpsc::channel::<Uuid>(5);
let (job_completed_tx, mut job_completed_rx) = mpsc::channel::<JobCompleted>(3);
let (job_completed_tx, mut job_completed_rx) = mpsc::channel::<SendResult>(3);
let job_completed_tx = JobCompletedSender(
job_completed_tx,
@@ -964,119 +970,167 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
let worker_name2 = worker_name.clone();
let killpill_tx2 = killpill_tx.clone();
let job_completed_sender = job_completed_tx.0.clone();
let send_result = tokio::spawn(async move {
while let Some(jc) = job_completed_rx.recv().await {
if let Some(wj) = worker_job_completed_channel_queue2.as_ref() {
wj.dec();
}
let base_internal_url2 = base_internal_url2.clone();
let worker_dir2 = worker_dir2.clone();
let db2 = db2.clone();
let same_worker_tx2 = same_worker_tx2.clone();
let rsmq2 = rsmq2.clone();
let worker_name = worker_name2.clone();
if matches!(jc.job.job_kind, JobKind::Noop) || is_dedicated_worker {
thread_count.fetch_add(1, Ordering::SeqCst);
let thread_count = thread_count.clone();
loop {
if thread_count.load(Ordering::Relaxed) < 4 {
break;
while let Some(sr) = job_completed_rx.recv().await {
match sr {
SendResult::JobCompleted(jc) => {
if let Some(wj) = worker_job_completed_channel_queue2.as_ref() {
wj.dec();
}
tokio::time::sleep(Duration::from_millis(3)).await;
}
let base_internal_url2 = base_internal_url2.clone();
let worker_dir2 = worker_dir2.clone();
let db2 = db2.clone();
let same_worker_tx2 = same_worker_tx2.clone();
let rsmq2 = rsmq2.clone();
let worker_name = worker_name2.clone();
if matches!(jc.job.job_kind, JobKind::Noop) || is_dedicated_worker {
thread_count.fetch_add(1, Ordering::SeqCst);
let thread_count = thread_count.clone();
#[cfg(feature = "benchmark")]
let send_duration = send_duration2.clone();
#[cfg(feature = "benchmark")]
let process_duration = process_duration.clone();
#[cfg(feature = "benchmark")]
let completed_jobs = completed_jobs.clone();
#[cfg(feature = "benchmark")]
let main_duration = main_duration2.clone();
let worker_save_completed_job_duration2 =
worker_save_completed_job_duration.clone();
let worker_flow_transition_duration2 = worker_flow_transition_duration.clone();
let killpill_tx = killpill_tx2.clone();
tokio::spawn(async move {
#[cfg(feature = "benchmark")]
let process_start = Instant::now();
let is_dependency_job = matches!(
jc.job.job_kind,
JobKind::Dependencies | JobKind::FlowDependencies
);
handle_receive_completed_job(
jc,
base_internal_url2,
db2.clone(),
worker_dir2,
same_worker_tx2,
rsmq2,
&worker_name,
worker_save_completed_job_duration2.clone(),
worker_flow_transition_duration2.clone(),
)
.await;
#[cfg(feature = "benchmark")]
{
let n = completed_jobs.fetch_add(1, Ordering::SeqCst);
if (n + 1) % 1000 == 0 || n == (jobs - 1) as usize {
let duration_s = start.elapsed().as_secs_f64();
let jobs_per_sec = n as f64 / duration_s;
tracing::info!(
"completed {} jobs in {}s, {} jobs/s",
n + 1,
duration_s,
jobs_per_sec
);
tracing::info!(
"main loop without send {}s",
main_duration.load(Ordering::SeqCst) as f64 / 1000.0
);
tracing::info!(
"send job completed / send dedicated job duration {}s",
send_duration.load(Ordering::SeqCst) as f64 / 1000.0
);
tracing::info!(
"job completed process duration {}s",
process_duration.load(Ordering::SeqCst) as f64 / 1000.0
);
loop {
if thread_count.load(Ordering::Relaxed) < 4 {
break;
}
tokio::time::sleep(Duration::from_millis(3)).await;
}
process_duration.fetch_add(
process_start.elapsed().as_millis() as usize,
Ordering::SeqCst,
);
}
#[cfg(feature = "benchmark")]
let send_duration = send_duration2.clone();
#[cfg(feature = "benchmark")]
let process_duration = process_duration.clone();
#[cfg(feature = "benchmark")]
let completed_jobs = completed_jobs.clone();
#[cfg(feature = "benchmark")]
let main_duration = main_duration2.clone();
thread_count.fetch_sub(1, Ordering::SeqCst);
if is_dependency_job && is_dedicated_worker {
tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted.");
sqlx::query!("UPDATE config SET config = config WHERE name = $1", format!("worker__{}", *WORKER_GROUP))
let worker_save_completed_job_duration2 =
worker_save_completed_job_duration.clone();
let worker_flow_transition_duration2 =
worker_flow_transition_duration.clone();
let killpill_tx = killpill_tx2.clone();
let job_completed_sender = job_completed_sender.clone();
tokio::spawn(async move {
#[cfg(feature = "benchmark")]
let process_start = Instant::now();
let is_dependency_job = matches!(
jc.job.job_kind,
JobKind::Dependencies | JobKind::FlowDependencies
);
handle_receive_completed_job(
jc,
base_internal_url2,
db2.clone(),
worker_dir2,
same_worker_tx2,
rsmq2,
&worker_name,
worker_save_completed_job_duration2.clone(),
worker_flow_transition_duration2.clone(),
job_completed_sender.clone(),
)
.await;
#[cfg(feature = "benchmark")]
{
let n = completed_jobs.fetch_add(1, Ordering::SeqCst);
if (n + 1) % 1000 == 0 || n == (jobs - 1) as usize {
let duration_s = start.elapsed().as_secs_f64();
let jobs_per_sec = n as f64 / duration_s;
tracing::info!(
"completed {} jobs in {}s, {} jobs/s",
n + 1,
duration_s,
jobs_per_sec
);
tracing::info!(
"main loop without send {}s",
main_duration.load(Ordering::SeqCst) as f64 / 1000.0
);
tracing::info!(
"send job completed / send dedicated job duration {}s",
send_duration.load(Ordering::SeqCst) as f64 / 1000.0
);
tracing::info!(
"job completed process duration {}s",
process_duration.load(Ordering::SeqCst) as f64 / 1000.0
);
}
process_duration.fetch_add(
process_start.elapsed().as_millis() as usize,
Ordering::SeqCst,
);
}
thread_count.fetch_sub(1, Ordering::SeqCst);
if is_dependency_job && is_dedicated_worker {
tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted.");
sqlx::query!("UPDATE config SET config = config WHERE name = $1", format!("worker__{}", *WORKER_GROUP))
.execute(&db2)
.await
.expect("update config to trigger restart of all dedicated workers at that config");
killpill_tx.send(()).unwrap_or_default();
killpill_tx.send(()).unwrap_or_default();
}
});
} else {
handle_receive_completed_job(
jc,
base_internal_url2,
db2,
worker_dir2,
same_worker_tx2,
rsmq2,
&worker_name,
worker_save_completed_job_duration2.clone(),
worker_flow_transition_duration2.clone(),
job_completed_sender.clone(),
)
.await;
}
});
} else {
handle_receive_completed_job(
jc,
base_internal_url2,
db2,
worker_dir2,
same_worker_tx2,
rsmq2,
&worker_name,
worker_save_completed_job_duration2.clone(),
worker_flow_transition_duration2.clone(),
)
.await;
}
SendResult::UpdateFlow {
flow,
w_id,
success,
result,
worker_dir,
stop_early_override,
token,
} => {
// let r;
if let Err(e) = update_flow_status_after_job_completion(
&db2,
&AuthedClient {
base_internal_url: base_internal_url2.to_string(),
workspace: w_id.clone(),
token: token.clone(),
force_client: None,
},
flow,
&Uuid::nil(),
&w_id,
success,
&result,
true,
same_worker_tx2.clone(),
&worker_dir,
stop_early_override,
rsmq2.clone(),
&worker_name2,
job_completed_sender.clone(),
)
.await
{
tracing::error!("Error updating flow status after job completion: {e}");
}
}
SendResult::Kill => {
break;
}
}
}
@@ -1333,6 +1387,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
}
}
println!("received killpill for worker {}", i_worker);
job_completed_tx.0.send(SendResult::Kill).await.unwrap();
break
},
_ = copy_to_bucket_rx.recv() => {
@@ -1600,6 +1655,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
&worker_dir,
rsmq.clone(),
&worker_name,
(&job_completed_tx.0).clone(),
)
.await;
};
@@ -2083,6 +2139,7 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
worker_name: &str,
worker_save_completed_job_duration: Option<Arc<prometheus::Histogram>>,
worker_flow_transition_duration: Option<Arc<prometheus::Histogram>>,
job_completed_tx: Sender<SendResult>,
) -> windmill_common::error::Result<()> {
if success {
// println!("bef completed job{:?}", SystemTime::now());
@@ -2126,6 +2183,7 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
None,
rsmq.clone(),
worker_name,
job_completed_tx,
)
.await?;
timer.map(|x| x.stop_and_record());
@@ -2161,6 +2219,7 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
None,
rsmq,
worker_name,
job_completed_tx,
)
.await?;
}
@@ -2216,6 +2275,7 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
worker_dir: &str,
rsmq: Option<R>,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
) {
let err = match err {
Error::JsonErr(err) => err,
@@ -2263,6 +2323,7 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
None,
rsmq.clone(),
worker_name,
job_completed_tx.clone(),
)
.await;
@@ -2305,6 +2366,34 @@ fn extract_error_value(log_lines: &str, i: i32) -> Box<RawValue> {
);
}
pub enum SendResult {
JobCompleted(JobCompleted),
UpdateFlow {
flow: Uuid,
w_id: String,
success: bool,
result: Box<RawValue>,
worker_dir: String,
stop_early_override: Option<bool>,
token: String,
},
Kill,
}
// db: &DB,
// client: &AuthedClient,
// flow: uuid::Uuid,
// job_id_for_status: &Uuid,
// w_id: &str,
// success: bool,
// result: &'a RawValue,
// unrecoverable: bool,
// same_worker_tx: Sender<Uuid>,
// worker_dir: &str,
// stop_early_override: Option<bool>,
// rsmq: Option<R>,
// worker_name: &str,
#[derive(Debug, Clone)]
pub struct JobCompleted {
pub job: Arc<QueuedJob>,
@@ -2467,7 +2556,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
same_worker_tx,
worker_dir,
rsmq,
worker_name,
job_completed_tx.0.clone(),
)
.await?;
timer.map(|x| x.stop_and_record());
@@ -2515,6 +2604,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
worker_dir,
base_internal_url,
&client.get_token().await,
rsmq.clone(),
)
.await
}
@@ -2529,6 +2619,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
worker_dir,
base_internal_url,
&client.get_token().await,
rsmq.clone(),
)
.await
.map(|()| serde_json::from_str("{}").unwrap()),
@@ -2543,6 +2634,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
worker_dir,
base_internal_url,
&client.get_token().await,
rsmq.clone(),
)
.await
.map(|()| serde_json::from_str("{}").unwrap()),
@@ -3044,7 +3136,7 @@ mount {{
}
#[tracing::instrument(level = "trace", skip_all)]
async fn handle_dependency_job(
async fn handle_dependency_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
job: &QueuedJob,
logs: &mut String,
mem_peak: &mut i32,
@@ -3055,7 +3147,17 @@ async fn handle_dependency_job(
worker_dir: &str,
base_internal_url: &str,
token: &str,
rsmq: Option<R>,
) -> error::Result<Box<RawValue>> {
let raw_code = sqlx::query_scalar!(
"SELECT content FROM script WHERE hash = $1 AND workspace_id = $2",
&job.script_hash.unwrap_or(ScriptHash(0)).0,
&job.workspace_id
)
.fetch_optional(db)
.await?
.unwrap_or_else(|| "No script found at this hash".to_string());
let script_path = job.script_path();
let content = capture_dependency_job(
&job.id,
job.language.as_ref().map(|v| Ok(v)).unwrap_or_else(|| {
@@ -3063,10 +3165,7 @@ async fn handle_dependency_job(
"Job Language required for dependency jobs".to_owned(),
))
})?,
job.raw_code
.as_ref()
.map(|a| a.as_str())
.unwrap_or_else(|| "no raw code"),
&raw_code,
logs,
mem_peak,
canceled_by,
@@ -3077,19 +3176,83 @@ async fn handle_dependency_job(
worker_dir,
base_internal_url,
token,
job.script_path(),
script_path,
)
.await;
match content {
Ok(content) => {
let hash = job.script_hash.unwrap_or(ScriptHash(0));
let w_id = &job.workspace_id;
sqlx::query!(
"UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3",
&content,
&job.script_hash.unwrap_or(ScriptHash(0)).0,
&job.workspace_id
&hash.0,
w_id
)
.execute(db)
.await?;
let deployment_message = job.raw_code.as_ref().map(|x| x.clone());
if let Err(e) = handle_deployment_metadata(
&job.email,
&job.created_by,
&db,
&w_id,
DeployedObject::Script { hash, path: script_path.to_string() },
deployment_message.clone(),
rsmq.clone(),
)
.await
{
tracing::error!(%e, "error handling deployment metadata");
}
if &job.language == &Some(ScriptLang::Python3) {
if let Ok(relative_imports) = parse_relative_imports(&raw_code, script_path) {
logs.push_str("\n--- RELATIVE IMPORTS ---\n\n");
logs.push_str(&relative_imports.join("\n"));
if !relative_imports.is_empty() {
let mut tx = db.begin().await?;
sqlx::query!(
"DELETE FROM dependency_map
WHERE importer_path = $1 AND importer_kind = 'script'
AND workspace_id = $2",
script_path,
w_id
)
.execute(&mut *tx)
.await?;
for import in relative_imports {
sqlx::query!(
"INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path)
VALUES ($1, $2, 'script', $3)",
w_id,
script_path,
import
)
.execute(&mut *tx)
.await?;
logs.push_str(&format!("{}\n", import));
}
tx.commit().await?;
}
if let Err(e) = trigger_python_dependents_to_recompute_dependencies(
w_id,
script_path,
deployment_message,
&job.email,
&job.created_by,
&job.permissioned_as,
db,
rsmq,
)
.await
{
tracing::error!(%e, "error triggering python dependents to recompute dependencies");
}
}
}
Ok(to_raw_value_owned(
json!({ "success": "Successful lock file generation", "lock": content }),
))
@@ -3108,7 +3271,80 @@ async fn handle_dependency_job(
}
}
async fn handle_flow_dependency_job(
async fn trigger_python_dependents_to_recompute_dependencies<
R: rsmq_async::RsmqConnection + Send + Sync + Clone,
>(
w_id: &str,
script_path: &str,
deployment_message: Option<String>,
email: &str,
created_by: &str,
permissioned_as: &str,
db: &sqlx::Pool<sqlx::Postgres>,
rsmq: Option<R>,
) -> error::Result<()> {
let script_importers = sqlx::query_scalar!(
"SELECT importer_path FROM dependency_map
WHERE imported_path = $1 AND importer_kind = 'script'
AND workspace_id = $2",
script_path,
w_id
)
.fetch_all(db)
.await?;
for s in script_importers.iter() {
let tx: PushIsolationLevel<'_, R> =
PushIsolationLevel::IsolatedRoot(db.clone(), rsmq.clone());
let r = get_latest_deployed_hash_for_path(db, w_id, s.as_str()).await;
if let Ok(r) = r {
let (job_uuid, new_tx) = windmill_queue::push(
db,
tx,
&w_id,
JobPayload::Dependencies {
path: s.clone(),
hash: r.0,
language: r.5,
dedicated_worker: r.6,
deployment_message: deployment_message.clone(),
},
PushArgs::empty(),
&created_by,
email,
permissioned_as.to_string(),
None,
None,
None,
None,
None,
false,
false,
None,
true,
None,
None,
None,
None,
)
.await?;
tracing::info!(
"pushed dependency job due to common python path: {job_uuid} for path {path} with hash {hash}",
path = s,
hash = r.0
);
new_tx.commit().await?;
} else {
tracing::error!(
"error getting latest deployed hash for path {path}: {err}",
path = s,
err = r.unwrap_err()
);
}
}
Ok(())
}
async fn handle_flow_dependency_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
job: &QueuedJob,
logs: &mut String,
mem_peak: &mut i32,
@@ -3119,6 +3355,7 @@ async fn handle_flow_dependency_job(
worker_dir: &str,
base_internal_url: &str,
token: &str,
rsmq: Option<R>,
) -> error::Result<()> {
let job_path = job.script_path.clone().ok_or_else(|| {
error::Error::InternalErr(
@@ -3170,6 +3407,21 @@ async fn handle_flow_dependency_job(
)
.execute(db)
.await?;
if let Err(e) = handle_deployment_metadata(
&job.email,
&job.created_by,
&db,
&job.workspace_id,
DeployedObject::Flow { path: job_path },
job.raw_code.as_ref().map(|x| x.clone()),
rsmq.clone(),
)
.await
{
tracing::error!(%e, "error handling deployment metadata");
}
Ok(())
}
@@ -3295,22 +3547,11 @@ async fn lock_modules(
new_flow_modules.push(e);
continue;
};
// sync with windmill-api/scripts
let dependencies = match language {
ScriptLang::Python3 => windmill_parser_py_imports::parse_python_imports(
&content,
&job.workspace_id,
&path.clone().unwrap_or_else(|| job_path.to_string()),
&db,
)
.await?
.join("\n"),
_ => content.clone(),
};
let new_lock = capture_dependency_job(
&job.id,
&language,
&dependencies,
&content,
logs,
mem_peak,
canceled_by,
@@ -3321,7 +3562,7 @@ async fn lock_modules(
worker_dir,
base_internal_url,
token,
job.script_path(),
&path.clone().unwrap_or_else(|| job_path.to_string()),
)
.await;
match new_lock {
@@ -3396,24 +3637,11 @@ async fn lock_modules_app(
.as_str()
.unwrap_or_default()
.to_string();
let dependencies = match language {
ScriptLang::Python3 => {
windmill_parser_py_imports::parse_python_imports(
&content,
&job.workspace_id,
job_path,
&db,
)
.await?
.join("\n")
}
_ => content.clone(),
};
logs.push_str("Found lockable inline script. Generating lock...\n");
let new_lock = capture_dependency_job(
&job.id,
&language,
&dependencies,
&content,
logs,
mem_peak,
canceled_by,
@@ -3498,7 +3726,7 @@ async fn lock_modules_app(
}
}
async fn handle_app_dependency_job(
async fn handle_app_dependency_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
job: &QueuedJob,
logs: &mut String,
mem_peak: &mut i32,
@@ -3509,6 +3737,7 @@ async fn handle_app_dependency_job(
worker_dir: &str,
base_internal_url: &str,
token: &str,
rsmq: Option<R>,
) -> error::Result<()> {
let job_path = job.script_path.clone().ok_or_else(|| {
error::Error::InternalErr(
@@ -3558,6 +3787,41 @@ async fn handle_app_dependency_job(
sqlx::query!("UPDATE app_version SET value = $1 WHERE id = $2", value, id,)
.execute(db)
.await?;
if let Err(e) = handle_deployment_metadata(
&job.email,
&job.created_by,
&db,
&job.workspace_id,
DeployedObject::App { path: job_path, version: id },
job.raw_code.as_ref().map(|x| x.clone()),
rsmq.clone(),
)
.await
{
tracing::error!(%e, "error handling deployment metadata");
}
// tx = PushIsolationLevel::Transaction(new_tx);
// tx = handle_deployment_metadata(
// tx,
// &authed,
// &db,
// &w_id,
// DeployedObject::App { path: app.path.clone(), version: v_id },
// app.deployment_message,
// )
// .await?;
// match tx {
// PushIsolationLevel::Transaction(tx) => tx.commit().await?,
// _ => {
// return Err(Error::InternalErr(
// "Expected a transaction here".to_string(),
// ));
// }
// }
Ok(())
} else {
Ok(())
@@ -3582,10 +3846,18 @@ async fn capture_dependency_job(
) -> error::Result<String> {
match job_language {
ScriptLang::Python3 => {
let reqs = windmill_parser_py_imports::parse_python_imports(
job_raw_code,
&w_id,
script_path,
&db,
)
.await?
.join("\n");
create_dependencies_dir(job_dir).await;
let req: std::result::Result<String, Error> = pip_compile(
job_id,
job_raw_code,
&reqs,
logs,
mem_peak,
canceled_by,
backend/windmill-worker/src/worker_flow.rs
+171 -81
View File
@@ -15,9 +15,8 @@ use std::time::Duration;
use crate::common::{hash_args, save_in_cache};
use crate::js_eval::{eval_timeout, IdContext};
use crate::{AuthedClient, PreviousResult, KEEP_JOB_DIR};
use crate::{AuthedClient, PreviousResult, SendResult, KEEP_JOB_DIR};
use anyhow::Context;
use async_recursion::async_recursion;
use serde::Serialize;
use serde_json::value::RawValue;
use serde_json::{json, Value};
@@ -70,8 +69,10 @@ pub async fn update_flow_status_after_job_completion<
stop_early_override: Option<bool>,
rsmq: Option<R>,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
) -> error::Result<()> {
// this is manual tailrecursion because async_recursion blows up the stack
// todo!();
let mut rec = update_flow_status_after_job_completion_internal(
db,
client,
@@ -87,6 +88,7 @@ pub async fn update_flow_status_after_job_completion<
false,
rsmq.clone(),
worker_name,
job_completed_tx.clone(),
)
.await?;
while let Some(nrec) = rec {
@@ -105,6 +107,7 @@ pub async fn update_flow_status_after_job_completion<
nrec.skip_error_handler,
rsmq.clone(),
worker_name,
job_completed_tx.clone(),
)
.await
{
@@ -125,6 +128,7 @@ pub async fn update_flow_status_after_job_completion<
nrec.skip_error_handler,
rsmq.clone(),
worker_name,
job_completed_tx.clone(),
)
.await?
}
@@ -167,6 +171,7 @@ pub async fn update_flow_status_after_job_completion_internal<
skip_error_handler: bool,
rsmq: Option<R>,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
let (
should_continue_flow,
@@ -575,6 +580,8 @@ pub async fn update_flow_status_after_job_completion_internal<
}
tx.commit().await?;
tracing::debug!(id = %flow_job.id, root_id = %job_root, "flow status updated");
(
should_continue_flow,
flow_job,
@@ -686,6 +693,7 @@ pub async fn update_flow_status_after_job_completion_internal<
}
true
} else {
tracing::debug!(id = %flow_job.id, "start handle flow");
match handle_flow(
&flow_job,
db,
@@ -694,7 +702,7 @@ pub async fn update_flow_status_after_job_completion_internal<
same_worker_tx.clone(),
worker_dir,
rsmq.clone(),
worker_name,
job_completed_tx,
)
.await
{
@@ -1037,7 +1045,7 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
same_worker_tx: Sender<Uuid>,
worker_dir: &str,
rsmq: Option<R>,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
) -> anyhow::Result<()> {
let flow = flow_job
.parse_raw_flow()
@@ -1056,7 +1064,7 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
same_worker_tx,
worker_dir,
rsmq,
worker_name,
job_completed_tx,
)
.await?;
Ok(())
@@ -1089,7 +1097,7 @@ pub struct RawArgs {
pub args: Option<Json<HashMap<String, Box<RawValue>>>>,
}
#[async_recursion]
// #[async_recursion]
// #[instrument(level = "trace", skip_all)]
async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
flow_job: &QueuedJob,
@@ -1101,7 +1109,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
same_worker_tx: Sender<Uuid>,
worker_dir: &str,
rsmq: Option<R>,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
) -> error::Result<()> {
let job_root = flow_job
.root_job
@@ -1122,29 +1130,51 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
// if this is an empty module of if the module has already been completed, successfully, update the parent flow
if flow.modules.is_empty() || matches!(status_module, FlowStatusModule::Success { .. }) {
let r;
return update_flow_status_after_job_completion(
db,
client,
flow_job.id,
&Uuid::nil(),
flow_job.workspace_id.as_str(),
true,
if flow.modules.is_empty() {
r = to_raw_value(&flow_job_args);
&r
} else {
// it has to be an empty for loop event
serde_json::from_str("[]").unwrap()
},
true,
same_worker_tx,
worker_dir,
None,
rsmq,
worker_name,
)
.await;
job_completed_tx
.send(SendResult::UpdateFlow {
flow: flow_job.id,
success: true,
result: if flow.modules.is_empty() {
to_raw_value(&flow_job_args)
} else {
// it has to be an empty for loop event
serde_json::from_str("[]").unwrap()
},
stop_early_override: None,
w_id: flow_job.workspace_id.clone(),
worker_dir: worker_dir.to_string(),
token: client.token.clone(),
})
.await
.map_err(|e| {
Error::InternalErr(format!(
"error sending update flow message to job completed channel: {e}"
))
})?;
// let r;
// return update_flow_status_after_job_completion(
// db,
// client,
// flow_job.id,
// &Uuid::nil(),
// flow_job.workspace_id.as_str(),
// true,
// if flow.modules.is_empty() {
// r = to_raw_value(&flow_job_args);
// &r
// } else {
// // it has to be an empty for loop event
// serde_json::from_str("[]").unwrap()
// },
// true,
// same_worker_tx,
// worker_dir,
// None,
// rsmq,
// worker_name,
// )
// .await;
return Ok(());
}
let arc_flow_job_args = Arc::new(flow_job_args.clone());
@@ -1166,25 +1196,45 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
flow_job.id
).fetch_one(db).await?.unwrap_or(0);
if count > 0 {
return update_flow_status_after_job_completion(
db,
client,
flow_job.id,
&Uuid::nil(),
flow_job.workspace_id.as_str(),
true,
serde_json::from_str(
"\"not allowed to overlap, scheduling next iteration\"",
)
.unwrap(),
true,
same_worker_tx,
worker_dir,
Some(true),
rsmq,
worker_name,
)
.await;
job_completed_tx
.send(SendResult::UpdateFlow {
flow: flow_job.id,
success: true,
result: serde_json::from_str(
"\"not allowed to overlap, scheduling next iteration\"",
)
.unwrap(),
stop_early_override: Some(true),
w_id: flow_job.workspace_id.clone(),
worker_dir: worker_dir.to_string(),
token: client.token.clone(),
})
.await
.map_err(|e| {
Error::InternalErr(format!(
"error sending update flow message to job completed channel: {e}"
))
})?;
// return update_flow_status_after_job_completion(
// db,
// client,
// flow_job.id,
// &Uuid::nil(),
// flow_job.workspace_id.as_str(),
// true,
// serde_json::from_str(
// "\"not allowed to overlap, scheduling next iteration\"",
// )
// .unwrap(),
// true,
// same_worker_tx,
// worker_dir,
// Some(true),
// rsmq,
// worker_name,
// )
// .await;
return Ok(());
}
}
}
@@ -1199,22 +1249,40 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
)
.await?;
if skip {
return update_flow_status_after_job_completion(
db,
client,
flow_job.id,
&Uuid::nil(),
flow_job.workspace_id.as_str(),
true,
serde_json::from_str("\"stopped early\"").unwrap(),
true,
same_worker_tx,
worker_dir,
Some(true),
rsmq,
worker_name,
)
.await;
job_completed_tx
.send(SendResult::UpdateFlow {
flow: flow_job.id,
success: true,
result: serde_json::from_str("\"stopped early\"").unwrap(),
stop_early_override: Some(true),
w_id: flow_job.workspace_id.clone(),
worker_dir: worker_dir.to_string(),
token: client.token.clone(),
})
.await
.map_err(|e| {
Error::InternalErr(format!(
"error sending update flow message to job completed channel: {e}"
))
})?;
// return update_flow_status_after_job_completion(
// db,
// client,
// flow_job.id,
// &Uuid::nil(),
// flow_job.workspace_id.as_str(),
// true,
// serde_json::from_str("\"stopped early\"").unwrap(),
// true,
// same_worker_tx,
// worker_dir,
// Some(true),
// rsmq,
// worker_name,
// )
// .await;
return Ok(());
}
}
}
@@ -1436,22 +1504,39 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
.await?;
if flow_job.is_flow_step {
if let Some(parent_job) = flow_job.parent_job {
update_flow_status_after_job_completion(
db,
client,
parent_job,
&flow_job.id,
&flow_job.workspace_id,
true,
&to_raw_value(&result),
false,
same_worker_tx.clone(),
&worker_dir,
None,
rsmq,
worker_name,
)
.await?;
job_completed_tx
.send(SendResult::UpdateFlow {
flow: parent_job,
success: true,
result: to_raw_value(&result),
stop_early_override: Some(true),
w_id: flow_job.workspace_id.clone(),
worker_dir: worker_dir.to_string(),
token: client.token.clone(),
})
.await
.map_err(|e| {
Error::InternalErr(format!(
"error sending update flow message to job completed channel: {e}"
))
})?;
// update_flow_status_after_job_completion(
// db,
// client,
// parent_job,
// &flow_job.id,
// &flow_job.workspace_id,
// true,
// &to_raw_value(&result),
// false,
// same_worker_tx.clone(),
// &worker_dir,
// None,
// rsmq,
// worker_name,
// )
// .await?;
return Ok(());
}
}
return Ok(());
@@ -1689,6 +1774,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
_ => Ok(flow_job_args),
}
};
tracing::debug!(id = %flow_job.id, root_id = %job_root, "flow job args computed");
let next_flow_transform = compute_next_flow_transform(
arc_flow_job_args.clone(),
@@ -1889,6 +1975,8 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
)
.await?;
tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushed next flow job: {uuid}");
if let FlowModuleValue::ForloopFlow { parallelism: Some(p), .. } = &module.value {
if i as u16 >= *p {
sqlx::query!(
@@ -2211,6 +2299,8 @@ async fn compute_next_flow_transform(
))
};
let delete_after_use = module.delete_after_use.unwrap_or(false);
tracing::debug!(id = %flow_job.id, "computing next flow transform for {:?}", &module.value);
match &module.value {
FlowModuleValue::Identity => trivial_next_job(JobPayload::Identity),
FlowModuleValue::Flow { path, .. } => {
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.232.0";
export const VERSION = "v1.236.2";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/main.ts
+1 -1
View File
@@ -31,7 +31,7 @@ addEventListener("error", (event) => {
}
});
export const VERSION = "v1.232.0";
export const VERSION = "v1.236.2";
let command: any = new Command()
.name("wmill")
cli/script.ts
+6 -3
View File
@@ -14,6 +14,7 @@ import {
writeAllSync,
yamlParse,
} from "./deps.ts";
import { deepEqual } from "./utils.ts";
export interface ScriptFile {
parent_hash?: string;
@@ -66,7 +67,7 @@ export async function handleFile(
path: string,
workspace: string,
alreadySynced: string[],
message?: string,
message?: string
): Promise<boolean> {
if (
!path.includes(".inline_script.") &&
@@ -125,8 +126,9 @@ export async function handleFile(
typed.is_template === remote.is_template &&
typed.kind == remote.kind &&
!remote.archived &&
(remote?.lock ?? "") == (typed.lock?.join("\n") ?? "") &&
JSON.stringify(typed.schema) == JSON.stringify(remote.schema) &&
(remote?.lock ?? "").trim() ==
(typed.lock?.join("\n") ?? "").trim() &&
deepEqual(typed.schema, remote.schema) &&
typed.tag == remote.tag &&
(typed.ws_error_handler_muted ?? false) ==
remote.ws_error_handler_muted &&
@@ -140,6 +142,7 @@ export async function handleFile(
return true;
}
}
log.info(
colors.yellow.bold(`Creating script with a parent ${remotePath}`)
);
cli/sync.ts
+14 -15
View File
@@ -478,7 +478,7 @@ async function pull(
!opts.json
);
const local = opts.raw
? undefined
? await FSFSElement(Deno.cwd())
: await FSFSElement(path.join(Deno.cwd(), ".wmill"));
const changes = await compareDynFSElement(
remote,
@@ -685,19 +685,18 @@ async function push(
"Computing the files to update on the remote to match local (taking .wmillignore into account)"
)
);
const remote = opts.raw
? undefined
: ZipFSElement(
(await downloadZip(
workspace,
opts.plainSecrets,
opts.skipVariables,
opts.skipResources,
opts.skipSecrets,
opts.includeSchedules
))!,
!opts.json
);
const remote = ZipFSElement(
(await downloadZip(
workspace,
opts.plainSecrets,
opts.skipVariables,
opts.skipResources,
opts.skipSecrets,
opts.includeSchedules
))!,
!opts.json
);
const local = await FSFSElement(path.join(Deno.cwd(), ""));
const changes = await compareDynFSElement(
local,
@@ -850,7 +849,7 @@ async function push(
case "flow":
await FlowService.deleteFlowByPath({
workspace: workspaceId,
path: removeSuffix(change.path, ".flow.json"),
path: removeSuffix(change.path, ".flow/flow.json"),
});
break;
case "app":
frontend/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-components",
"version": "1.232.0",
"version": "1.236.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.232.0",
"version": "1.236.2",
"license": "AGPL-3.0",
"dependencies": {
"@aws-crypto/sha256-js": "^4.0.0",
frontend/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.232.0",
"version": "1.236.2",
"scripts": {
"dev": "vite dev",
"build": "vite build",
frontend/src/lib/components/AddUser.svelte
+4 -4
View File
@@ -61,7 +61,7 @@
dispatch('new')
}
let selected: 'operator' | 'author' | 'admin' = 'author'
let selected: 'operator' | 'developer' | 'admin' = 'developer'
</script>
<Popup
@@ -92,10 +92,10 @@
/>
<ToggleButton
position="center"
value="author"
value="developer"
size="sm"
label="Author"
tooltip="An Author can execute and view scripts/flows/apps, but they can also create new ones."
label="Developer"
tooltip="A Developer can execute and view scripts/flows/apps, but they can also create new ones and edit those they are allowed to by their path (either u/ or Writer or Admin of their folder found at /f)."
/>
<ToggleButton
position="right"
frontend/src/lib/components/AppConnect.svelte
+1 -1
View File
@@ -460,7 +460,7 @@
{/if}
</div>
<h2 class="mt-8 mb-4" />
<div class="mt-8 mb-4" />
<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
{#if filteredConnectsManual}
{#each filteredConnectsManual as [key, _]}
frontend/src/lib/components/ChartHighlightTheme.svelte
+30 -57
View File
@@ -5,7 +5,6 @@
import nord from 'svelte-highlight/styles/nord'
import { each } from 'chart.js/helpers'
import { Chart } from 'chart.js'
import { deepMergeWithPriority } from '$lib/utils'
let darkMode: boolean = false
@@ -13,67 +12,41 @@
if (document.documentElement.classList.contains('dark')) {
darkMode = true
each(Chart.instances, (instance) => {
instance.options = deepMergeWithPriority(instance.options, {
scales: {
y: {
ticks: {
color: '#e0e7ed'
},
grid: {
color: '#4a5568'
}
},
x: {
ticks: {
color: '#e0e7ed'
},
grid: {
color: '#4a5568'
}
}
},
plugins: {
legend: {
labels: {
color: '#e0e7ed'
}
}
}
})
if (instance.options.scales?.y?.ticks?.color !== undefined) {
instance.options.scales.y.ticks.color = '#e0e7ed'
}
if (instance.options.scales?.y?.grid?.color !== undefined) {
instance.options.scales.y.grid.color = '#4a5568'
}
if (instance.options.scales?.x?.ticks?.color !== undefined) {
instance.options.scales.x.ticks.color = '#e0e7ed'
}
if (instance.options.scales?.x?.grid?.color !== undefined) {
instance.options.scales.x.grid.color = '#4a5568'
}
if (instance.options.plugins?.legend?.labels?.color !== undefined) {
instance.options.plugins.legend.labels.color = '#e0e7ed'
}
instance.update()
})
} else {
darkMode = false
each(Chart.instances, (instance) => {
instance.options = deepMergeWithPriority(instance.options, {
scales: {
y: {
ticks: {
color: '#4a5568'
},
grid: {
color: '#e0e7ed'
}
},
x: {
ticks: {
color: '#4a5568'
},
grid: {
color: '#e0e7ed'
}
}
},
plugins: {
legend: {
labels: {
color: '#4a5568'
}
}
}
})
if (instance.options.scales?.y?.ticks?.color !== undefined) {
instance.options.scales.y.ticks.color = '#4a5568'
}
if (instance.options.scales?.y?.grid?.color !== undefined) {
instance.options.scales.y.grid.color = '#e0e7ed'
}
if (instance.options.scales?.x?.ticks?.color !== undefined) {
instance.options.scales.x.ticks.color = '#4a5568'
}
if (instance.options.scales?.x?.grid?.color !== undefined) {
instance.options.scales.x.grid.color = '#e0e7ed'
}
if (instance.options.plugins?.legend?.labels?.color !== undefined) {
instance.options.plugins.legend.labels.color = '#4a5568'
}
instance.update()
})
}
frontend/src/lib/components/DBSchemaExplorer.svelte
+37 -284
View File
@@ -1,22 +1,21 @@
<script lang="ts">
import { JobService, Preview } from '$lib/gen'
import {
dbSchemas,
workspaceStore,
type DBSchema,
type GraphqlSchema,
type SQLSchema
} from '$lib/stores'
import { dbSchemas, workspaceStore, type DBSchema } from '$lib/stores'
import Button from './common/button/Button.svelte'
import Drawer from './common/drawer/Drawer.svelte'
import DrawerContent from './common/drawer/DrawerContent.svelte'
import ObjectViewer from './propertyPicker/ObjectViewer.svelte'
import { sendUserToast, tryEvery } from '$lib/utils'
import { sendUserToast } from '$lib/utils'
import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte'
import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte'
import { buildClientSchema, getIntrospectionQuery, printSchema } from 'graphql'
import GraphqlSchemaViewer from './GraphqlSchemaViewer.svelte'
import { RefreshCcw } from 'lucide-svelte'
import { Loader2, RefreshCcw } from 'lucide-svelte'
import {
formatGraphqlSchema,
formatSchema,
getDbSchemas,
scripts
} from './apps/components/display/dbtable/utils'
import Alert from './common/alert/Alert.svelte'
export let resourceType: string | undefined
export let resourcePath: string | undefined = undefined
@@ -25,292 +24,42 @@
let drawer: Drawer | undefined
const scripts: Record<
string,
{
code: string
lang: string
processingFn?: (any: any) => SQLSchema['schema']
argName: string
}
> = {
postgresql: {
code: `SELECT table_name, column_name, udt_name, column_default, is_nullable, table_schema FROM information_schema.columns WHERE table_schema != 'pg_catalog' AND table_schema != 'information_schema'`,
processingFn: (rows) => {
const schemas = rows.reduce((acc, a) => {
const table_schema = a.table_schema
delete a.table_schema
acc[table_schema] = acc[table_schema] || []
acc[table_schema].push(a)
return acc
}, {})
const data = {}
for (const key in schemas) {
data[key] = schemas[key].reduce((acc, a) => {
const table_name = a.table_name
delete a.table_name
acc[table_name] = acc[table_name] || {}
const p: {
type: string
required: boolean
default?: string
} = {
type: a.udt_name,
required: a.is_nullable === 'NO'
}
if (a.column_default) {
p.default = a.column_default
}
acc[table_name][a.column_name] = p
return acc
}, {})
}
return data
},
lang: 'postgresql',
argName: 'database'
},
mysql: {
code: "select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT from information_schema.columns where table_schema != 'information_schema'",
processingFn: (rows) => {
const schemas = rows.reduce((acc, a) => {
const table_schema = a.TABLE_SCHEMA
delete a.TABLE_SCHEMA
acc[table_schema] = acc[table_schema] || []
acc[table_schema].push(a)
return acc
}, {})
const data = {}
for (const key in schemas) {
data[key] = schemas[key].reduce((acc, a) => {
const table_name = a.TABLE_NAME
delete a.TABLE_NAME
acc[table_name] = acc[table_name] || {}
const p: {
type: string
required: boolean
default?: string
} = {
type: a.DATA_TYPE,
required: a.is_nullable === 'NO'
}
if (a.column_default) {
p.default = a.COLUMN_DEFAULT
}
acc[table_name][a.COLUMN_NAME] = p
return acc
}, {})
}
return data
},
lang: 'mysql',
argName: 'database'
},
graphql: {
code: getIntrospectionQuery(),
lang: 'graphql',
argName: 'api'
},
bigquery: {
code: `import { BigQuery } from 'npm:@google-cloud/bigquery@7.2.0';
export async function main(args: bigquery) {
const bq = new BigQuery({
credentials: args
})
const [datasets] = await bq.getDatasets();
const schema = {}
for (const dataset of datasets) {
schema[dataset.id] = {}
const query = "SELECT table_name, ARRAY_AGG(STRUCT(if(is_nullable = 'YES', true, false) AS required, column_name AS name, data_type AS type, if(column_default = 'NULL', null, column_default) AS \`default\`) ORDER BY ordinal_position) AS schema \
FROM \`{dataset.id}\`.INFORMATION_SCHEMA.COLUMNS \
GROUP BY table_name".replace('{dataset.id}', dataset.id)
const [rows] = await bq.query(query)
for (const row of rows) {
schema[dataset.id][row.table_name] = {}
for (const col of row.schema) {
const colName = col.name
delete col.name
if (col.default === null) {
delete col.default
}
schema[dataset.id][row.table_name][colName] = col
}
}
}
return schema
}`, // nested template literals
lang: 'deno',
argName: 'args'
},
snowflake: {
code: `select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT, IS_NULLABLE from information_schema.columns where table_schema != 'INFORMATION_SCHEMA'`,
lang: 'snowflake',
processingFn: (rows) => {
const schema = {}
for (const row of rows) {
if (!(row.TABLE_SCHEMA in schema)) {
schema[row.TABLE_SCHEMA] = {}
}
if (!(row.TABLE_NAME in schema[row.TABLE_SCHEMA])) {
schema[row.TABLE_SCHEMA][row.TABLE_NAME] = {}
}
schema[row.TABLE_SCHEMA][row.TABLE_NAME][row.COLUMN_NAME] = {
type: row.DATA_TYPE,
required: row.IS_NULLABLE === 'YES'
}
if (row.COLUMN_DEFAULT !== null) {
schema[row.TABLE_SCHEMA][row.TABLE_NAME][row.COLUMN_NAME]['default'] =
row.COLUMN_DEFAULT
}
}
return schema
},
argName: 'database'
},
ms_sql_server: {
argName: 'database',
code: `select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT from information_schema.columns where table_schema != 'sys'`,
lang: 'mssql',
processingFn: (rows) => {
const schemas = rows[0].reduce((acc, a) => {
const table_schema = a.TABLE_SCHEMA
delete a.TABLE_SCHEMA
acc[table_schema] = acc[table_schema] || []
acc[table_schema].push(a)
return acc
}, {})
const data = {}
for (const key in schemas) {
data[key] = schemas[key].reduce((acc, a) => {
const table_name = a.TABLE_NAME
delete a.TABLE_NAME
acc[table_name] = acc[table_name] || {}
const p: {
type: string
required: boolean
default?: string
} = {
type: a.DATA_TYPE,
required: a.is_nullable === 'NO'
}
if (a.column_default) {
p.default = a.COLUMN_DEFAULT
}
acc[table_name][a.COLUMN_NAME] = p
return acc
}, {})
}
return data
}
}
}
function resourceTypeToLang(rt: string) {
if (rt === 'ms_sql_server') {
return 'mssql'
} else {
return rt
}
}
async function getSchema() {
if (!resourceType || !resourcePath) return
if ($dbSchemas[resourcePath]) return
loading = true
const job = await JobService.runScriptPreview({
workspace: $workspaceStore!,
requestBody: {
language: scripts[resourceType].lang as Preview.language,
content: scripts[resourceType].code,
args: {
[scripts[resourceType].argName]: '$res:' + resourcePath
}
}
})
tryEvery({
tryCode: async () => {
if (resourcePath) {
const testResult = await JobService.getCompletedJob({
workspace: $workspaceStore!,
id: job
})
if (!testResult.success) {
console.error(testResult.result?.['error']?.['message'])
} else {
if (resourceType !== undefined) {
if (resourceType !== 'graphql') {
const { processingFn } = scripts[resourceType]
const schema =
processingFn !== undefined ? processingFn(testResult.result) : testResult.result
$dbSchemas[resourcePath] = {
lang: resourceTypeToLang(resourceType) as SQLSchema['lang'],
schema,
publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo
}
} else {
if (typeof testResult.result !== 'object' || !('__schema' in testResult.result)) {
console.error('Invalid GraphQL schema')
if (drawer?.isOpen()) {
sendUserToast('Invalid GraphQL schema', true)
}
} else {
$dbSchemas[resourcePath] = {
lang: 'graphql',
schema: testResult.result
}
}
}
}
try {
await getDbSchemas(
resourceType,
resourcePath,
$workspaceStore,
$dbSchemas,
(message: string) => {
if (drawer?.isOpen()) {
sendUserToast(message, true)
}
}
loading = false
},
timeoutCode: async () => {
loading = false
console.error('Could not query schema within 5s')
if (drawer?.isOpen()) {
sendUserToast('Could not query schema within 5s', true)
}
try {
await JobService.cancelQueuedJob({
workspace: $workspaceStore!,
id: job,
requestBody: {
reason: 'Could not query schema within 5s'
}
})
} catch (err) {
console.error(err)
}
},
interval: 500,
timeout: 5000
})
}
function formatSchema(dbSchema: DBSchema) {
if (dbSchema.lang !== 'graphql' && dbSchema.publicOnly) {
return dbSchema.schema.public || dbSchema.schema.PUBLIC || dbSchema.schema.dbo || dbSchema
} else if (dbSchema.lang === 'mysql' && Object.keys(dbSchema.schema).length === 1) {
return dbSchema.schema[Object.keys(dbSchema.schema)[0]]
} else {
return dbSchema.schema
)
$dbSchemas = $dbSchemas
} catch (e) {
console.error(e)
}
loading = false
}
function formatGraphqlSchema(dbSchema: GraphqlSchema) {
return printSchema(buildClientSchema(dbSchema.schema))
}
$: resourcePath &&
Object.keys(scripts).includes(resourceType || '') &&
!$dbSchemas[resourcePath] &&
getSchema()
$: resourcePath && Object.keys(scripts).includes(resourceType || '') && getSchema()
$: dbSchema = resourcePath && resourcePath in $dbSchemas ? $dbSchemas[resourcePath] : undefined
$: shouldDisplayError = resourcePath && resourcePath in $dbSchemas && !$dbSchemas[resourcePath]
</script>
{#if loading}
<Loader2 size={14} class="animate-spin " />
{/if}
{#if dbSchema}
<Button
size="xs"
@@ -350,4 +99,8 @@ GROUP BY table_name".replace('{dataset.id}', dataset.id)
{/if}
</DrawerContent>
</Drawer>
{:else if shouldDisplayError}
<Alert type="error" size="xs" title="Schema not available" class="mt-2">
Schema could not be loaded. Please check the permissions of the resource.
</Alert>
{/if}
frontend/src/lib/components/DisplayResult.svelte
+19 -8
View File
@@ -338,14 +338,25 @@
</button>
{/each}
</div>
{:else if largeObject}<div class="text-sm text-tertiary"
><a
download="{filename ?? 'result'}.json"
href={workspaceId && jobId
? `/api/w/${workspaceId}/jobs_u/completed/get_result/${jobId}`
: `data:text/json;charset=utf-8,${encodeURIComponent(toJsonStr(result))}`}>Download</a
>
</div>
{:else if largeObject}
{#if 'filename' in result && 'file' in result}
<div
><a
download={result.filename ?? result.file?.filename ?? 'windmill.file'}
href="data:application/octet-stream;base64,{contentOrRootString(result.file)}"
>Download</a
>
</div>
{:else}
<div class="text-sm text-tertiary"
><a
download="{filename ?? 'result'}.json"
href={workspaceId && jobId
? `/api/w/${workspaceId}/jobs_u/completed/get_result/${jobId}`
: `data:text/json;charset=utf-8,${encodeURIComponent(toJsonStr(result))}`}>Download</a
>
</div>
{/if}
<div class="mb-21">JSON is too large to be displayed in full</div>
{#if result && result != 'WINDMILL_TOO_BIG'}
<ObjectViewer json={result} />
frontend/src/lib/components/InviteUser.svelte
+4 -4
View File
@@ -61,7 +61,7 @@
email = ''
}
let selected: 'operator' | 'author' | 'admin' = 'author'
let selected: 'operator' | 'developer' | 'admin' = 'developer'
</script>
<Popup floatingConfig={{ strategy: 'absolute', placement: 'bottom-end' }}>
@@ -89,10 +89,10 @@
<ToggleButton
position="center"
value="author"
value="developer"
size="sm"
label="Author"
tooltip="An Author can execute and view scripts/flows/apps, but they can also create new ones."
label="Developer"
tooltip="A Developer can execute and view scripts/flows/apps, but they can also create new ones and edit those they are allowed to by their path (either u/ or Writer or Admin of their folder found at /f)."
/>
<ToggleButton
frontend/src/lib/components/MemoryFootprintViewer.svelte
+107
View File
@@ -0,0 +1,107 @@
<script lang="ts">
import { type MetricDataPoint, MetricsService } from '$lib/gen'
import { displayTime } from '$lib/utils'
import { enterpriseLicense, workspaceStore } from '$lib/stores'
import {
CategoryScale,
Chart as ChartJS,
Legend,
LinearScale,
LineElement,
PointElement,
Title,
Tooltip
} from 'chart.js'
import { Line } from 'svelte-chartjs'
import { Alert } from './common'
ChartJS.register(Title, Tooltip, Legend, LineElement, LinearScale, PointElement, CategoryScale)
export let jobId: string
export let jobUpdateLastFetch: Date | undefined
let jobMetricsLastFetch: Date | undefined = undefined
let jobMemoryStats: MetricDataPoint[] | undefined = undefined
let data: {
x: number
y: number
}[] = []
let labels: string[] = []
async function loadMetricsData() {
try {
let jobStatsPromise = MetricsService.getJobMetrics({
workspace: $workspaceStore!,
id: jobId,
requestBody: {
from_timestamp: jobMetricsLastFetch?.toISOString(),
timeseries_max_datapoints: 0
}
})
jobMetricsLastFetch = new Date()
let jobStats = await jobStatsPromise
let memoryTimeseries =
jobStats.timeseries_metrics?.filter((ts) => ts.metric_id === 'memory_kb') ?? []
if (memoryTimeseries.length > 0) {
jobMemoryStats = (jobMemoryStats ?? []).concat(memoryTimeseries[0].values)
}
} catch {
console.error('Unable to load metrics data for job', jobId)
return
}
for (let dp of jobMemoryStats ?? []) {
let ts = new Date(dp.timestamp).valueOf()
if (data.length === 0 || ts > data[data.length - 1].x) {
data.push({
x: ts,
y: dp.value
})
labels.push(displayTime(dp.timestamp))
}
}
data = [...data]
}
$: jobUpdateLastFetch && loadMetricsData()
</script>
<div class="relative max-h-100">
{#if !$enterpriseLicense}
<Alert type="error" title="Enterprise Edition only feature">
Job metrics are only available on Windmill Enterprise Edition.
</Alert>
{:else if (jobMemoryStats?.length ?? 0) === 0}
<Alert type="info" title="No metric available"
>No data points available for this job. Metrics are recorded only for jobs running for more
than 500ms.</Alert
>
{/if}
<Line
class="w-full max-h-80"
data={{
labels: labels,
datasets: [
{
label: 'Job memory footprint (kB)',
data: data,
fill: false,
borderColor: 'rgb(59, 130, 246, 0.8)',
backgroundColor: 'rgb(59, 130, 246, 0.8)',
tension: 0.1
}
]
}}
options={{
animation: {
duration: 0
},
maintainAspectRatio: false,
responsive: true
}}
/>
</div>
frontend/src/lib/components/ResultJobLoader.svelte
+45 -17
View File
@@ -11,6 +11,7 @@
export let workspaceOverride: string | undefined = undefined
export let notfound = false
export let isEditor = false
export let allowConcurentRequests = false
const dispatch = createEventDispatcher()
@@ -27,22 +28,28 @@
$: isLoading = currentId !== undefined
type Callbacks = { done: (x: any[]) => void; cancel: () => void; error: () => void }
let running = false
export async function abstractRun(fn: () => Promise<string>) {
let lastCallbacks: Callbacks | undefined = undefined
let finished: string[] = []
export async function abstractRun(fn: () => Promise<string>, callbacks?: Callbacks) {
try {
running = false
isLoading = true
clearCurrentJob()
const startedAt = Date.now()
const testId = await fn()
if (lastStartedAt < startedAt) {
lastCallbacks = callbacks
if (lastStartedAt < startedAt || allowConcurentRequests) {
lastStartedAt = startedAt
if (testId) {
dispatch('started', testId)
try {
await watchJob(testId)
await watchJob(testId, callbacks)
} catch {
callbacks?.cancel()
dispatch('cancel', testId)
if (currentId === testId) {
currentId = undefined
@@ -52,6 +59,7 @@
}
return testId
} catch (err) {
callbacks?.error()
// if error happens on submitting the job, reset UI state so the user can try again
isLoading = false
currentId = undefined
@@ -110,6 +118,9 @@
export async function cancelJob() {
const id = currentId
if (id) {
lastCallbacks?.cancel()
lastCallbacks = undefined
// console.log('cancel', 2)
dispatch('cancel', id)
currentId = undefined
try {
@@ -125,29 +136,32 @@
}
export async function clearCurrentJob() {
if (currentId) {
if (currentId && !allowConcurentRequests) {
lastCallbacks?.cancel()
// console.log('cancel', 3)
dispatch('cancel', currentId)
lastCallbacks = undefined
job = undefined
await cancelJob()
}
}
export async function watchJob(testId: string) {
export async function watchJob(testId: string, callbacks?: Callbacks) {
syncIteration = 0
errorIteration = 0
currentId = testId
job = undefined
const isCompleted = await loadTestJob(testId)
const isCompleted = await loadTestJob(testId, callbacks)
if (!isCompleted) {
setTimeout(() => {
syncer(testId)
syncer(testId, callbacks)
}, 50)
}
}
async function loadTestJob(id: string): Promise<boolean> {
async function loadTestJob(id: string, callbacks?: Callbacks): Promise<boolean> {
let isCompleted = false
if (currentId === id) {
if (currentId === id || allowConcurentRequests) {
try {
let maybe_job = await JobService.getCompletedJobResultMaybe({
workspace: workspace ?? '',
@@ -160,19 +174,26 @@
}
if (maybe_job.completed) {
isCompleted = true
if (currentId === id) {
if (currentId === id || allowConcurentRequests) {
job = { ...maybe_job, id }
await tick()
if ('error' in job ?? {}) {
if (typeof job?.result == 'object' && 'error' in (job?.result ?? {})) {
callbacks?.error()
dispatch('doneError', {
id,
error: job.result.error
})
} else {
callbacks?.done(job.result)
dispatch('done', job)
}
currentId = undefined
finished.push(id)
if (!allowConcurentRequests) {
currentId = undefined
}
} else {
// console.log('cancel', 3)
callbacks?.cancel()
dispatch('cancel', id)
}
}
@@ -188,25 +209,32 @@
}
return isCompleted
} else {
// console.log('cancel', 6)
callbacks?.cancel()
dispatch('cancel', id)
return true
}
}
async function syncer(id: string): Promise<void> {
if (currentId != id) {
async function syncer(id: string, callbacks?: Callbacks): Promise<void> {
if ((currentId != id && !allowConcurentRequests) || finished.includes(id)) {
callbacks?.cancel()
// console.log('cancel', 7)
dispatch('cancel', id)
return
}
syncIteration++
await loadTestJob(id)
let r = await loadTestJob(id, callbacks)
if (r) {
return
}
let nextIteration = 50
if (syncIteration > ITERATIONS_BEFORE_SLOW_REFRESH) {
nextIteration = 500
} else if (syncIteration > ITERATIONS_BEFORE_SUPER_SLOW_REFRESH) {
nextIteration = 2000
}
setTimeout(() => syncer(id), nextIteration)
setTimeout(() => syncer(id, callbacks), nextIteration)
}
onDestroy(async () => {
frontend/src/lib/components/RunFormAdvancedPopup.svelte
+1 -1
View File
@@ -81,7 +81,7 @@
{#if overrideTag}
<option value="">reset to default</option>
{:else}
<option value="" disabled selected>Override Wourker Group</option>
<option value="" disabled selected>Override Worker Group Tag</option>
{/if}
{#each $workerTags ?? [] as tag (tag)}
<option value={tag}>{tag}</option>
frontend/src/lib/components/SchemaEditor.svelte
+1 -1
View File
@@ -273,7 +273,7 @@
}}
id="flow-editor-add-property"
>
Add Property
Add Argument
</Button>
</div>
frontend/src/lib/components/TestJobLoader.svelte
+5 -2
View File
@@ -10,6 +10,7 @@
export let job: Job | undefined = undefined
export let workspaceOverride: string | undefined = undefined
export let notfound = false
export let jobUpdateLastFetch: Date | undefined = undefined
const dispatch = createEventDispatcher()
@@ -150,7 +151,6 @@
running: job.running,
logOffset: job.logs?.length ? job.logs?.length + 1 : 0
})
if (previewJobUpdates.new_logs) {
job.logs = (job?.logs ?? '').concat(previewJobUpdates.new_logs)
}
@@ -163,6 +163,7 @@
} else {
job = await JobService.getJob({ workspace: workspace!, id })
}
jobUpdateLastFetch = new Date()
if (job?.type === 'CompletedJob') {
//only CompletedJob has success property
@@ -194,7 +195,9 @@
return
}
syncIteration++
await loadTestJob(id)
if (await loadTestJob(id)) {
return
}
let nextIteration = 50
if (syncIteration > ITERATIONS_BEFORE_SLOW_REFRESH) {
nextIteration = 500
frontend/src/lib/components/apps/components/buttons/AppButton.svelte
+2
View File
@@ -34,6 +34,7 @@
export let errorHandledByComponent: boolean | undefined = false
export let extraKey: string | undefined = undefined
export let isMenuItem: boolean = false
export let noInitialize = false
export let controls: { left: () => boolean; right: () => boolean | string } | undefined =
undefined
@@ -156,6 +157,7 @@
<!-- gotoNewTab={resolvedConfig.onSuccess.selected == 'goto'} -->
<RunnableWrapper
{noInitialize}
bind:this={runnableWrapper}
{recomputeIds}
bind:runnableComponent
frontend/src/lib/components/apps/components/display/dbtable/AppDbExplorer.svelte
+460
View File
@@ -0,0 +1,460 @@
<script lang="ts">
import type {
AppEditorContext,
AppViewerContext,
ComponentCustomCSS,
RichConfigurations
} from '../../../types'
import { components } from '$lib/components/apps/editor/component'
import ResolveConfig from '../../helpers/ResolveConfig.svelte'
import { findGridItem, initConfig, initOutput } from '$lib/components/apps/editor/appUtils'
import {
createPostgresInput,
getDbSchemas,
loadTableMetaData,
type ColumnMetadata,
type TableMetadata,
getPrimaryKeys
} from './utils'
import { getContext, tick } from 'svelte'
import UpdateCell from './UpdateCell.svelte'
import { workspaceStore, type DBSchemas } from '$lib/stores'
import Button from '$lib/components/common/button/Button.svelte'
import { Plus } from 'lucide-svelte'
import { Drawer, DrawerContent } from '$lib/components/common'
import InsertRow from './InsertRow.svelte'
import Portal from 'svelte-portal'
import { sendUserToast } from '$lib/toast'
import type { AppInput, StaticInput } from '$lib/components/apps/inputType'
import DbExplorerCount from './DbExplorerCount.svelte'
import AppAggridExplorerTable from '../table/AppAggridExplorerTable.svelte'
import type { IDatasource } from 'ag-grid-community'
import { RunnableWrapper } from '../../helpers'
import type RunnableComponent from '../../helpers/RunnableComponent.svelte'
import InsertRowRunnable from './InsertRowRunnable.svelte'
import DeleteRow from './DeleteRow.svelte'
export let id: string
export let configuration: RichConfigurations
export let customCss: ComponentCustomCSS<'dbexplorercomponent'> | undefined = undefined
export let render: boolean
export let initializing: boolean = true
const resolvedConfig = initConfig(
components['dbexplorercomponent'].initialData.configuration,
configuration
)
const { app, worldStore, mode, selectedComponent } =
getContext<AppViewerContext>('AppViewerContext')
const editorContext = getContext<AppEditorContext>('AppEditorContext')
let input: AppInput | undefined = undefined
let quicksearch = ''
let aggrid: AppAggridExplorerTable
$: computeInput(
resolvedConfig.columnDefs,
resolvedConfig.whereClause,
resolvedConfig.type.configuration.postgresql.resource
)
function computeInput(columnDefs: any, whereClause: string | undefined, resource: any) {
aggrid?.clearRows()
input = createPostgresInput(
resource,
resolvedConfig.type.configuration.postgresql.table,
columnDefs,
whereClause
)
}
$: editorContext != undefined && $mode == 'dnd' && resolvedConfig.type && listTableIfAvailable()
$: editorContext != undefined &&
$mode == 'dnd' &&
resolvedConfig.type.configuration?.postgresql?.table &&
listColumnsIfAvailable()
$: if (quicksearch) {
aggrid?.clearRows()
}
initializing = false
let updateCell: UpdateCell
let renderCount = 0
let insertDrawer: Drawer | undefined = undefined
let componentContainerHeight: number | undefined = undefined
let buttonContainerHeight: number | undefined = undefined
function onUpdate(
e: CustomEvent<{
row: number
columnDef: ColumnMetadata
column: string
value: any
data: any
oldValue: string | undefined
}>
) {
const { columnDef, value, data, oldValue } = e.detail
updateCell?.triggerUpdate(
resolvedConfig.type.configuration.postgresql.resource,
resolvedConfig.type.configuration.postgresql.table ?? 'unknown',
columnDef,
resolvedConfig.columnDefs,
value,
data,
oldValue
)
}
let args: Record<string, any> = {}
let outputs = initOutput($worldStore, id, {
selectedRowIndex: 0,
selectedRow: {},
selectedRows: [] as any[],
result: [] as any[],
loading: false,
page: 0,
newChange: { row: 0, column: '', value: undefined },
ready: undefined as boolean | undefined
})
let lastResource: string | undefined = undefined
async function listTableIfAvailable() {
let resource = resolvedConfig.type.configuration?.postgresql?.resource
if (lastResource === resource) return
lastResource = resource
const gridItem = findGridItem($app, id)
if (!gridItem) {
return
}
if (
'configuration' in gridItem.data?.configuration?.type &&
'selectOptions' in gridItem.data?.configuration?.type?.configuration?.postgresql?.table
) {
gridItem.data.configuration.type.configuration.postgresql.table.selectOptions = []
}
if (!resolvedConfig.type?.configuration?.postgresql?.resource) {
$app = {
...$app
}
return
}
if (
'configuration' in gridItem.data?.configuration?.type &&
gridItem.data.configuration.type.configuration.postgresql.table
) {
gridItem.data.configuration.type.configuration.postgresql.table.loading = true
}
try {
const dbSchemas: DBSchemas = {}
await getDbSchemas(
'postgresql',
resolvedConfig.type.configuration.postgresql.resource.split(':')[1],
$workspaceStore,
dbSchemas,
(message: string) => {}
)
if ('configuration' in gridItem.data.configuration.type) {
gridItem.data.configuration.type.configuration.postgresql.table['selectOptions'] = dbSchemas
? // @ts-ignore
Object.keys(Object.values(dbSchemas)?.[0]?.schema?.public ?? {})
: []
}
$app = {
...$app
}
} catch (e) {}
if (
'configuration' in gridItem.data?.configuration?.type &&
gridItem.data.configuration.type.configuration.postgresql.table
) {
gridItem.data.configuration.type.configuration.postgresql.table.loading = false
}
}
let datasource: IDatasource = {
rowCount: 0,
getRows: async function (params) {
refreshCount++
let uuid = await runnableComponent?.runComponent(
undefined,
undefined,
undefined,
{
offset: params.startRow,
limit: params.endRow - params.startRow,
quicksearch,
orderBy: params.sortModel?.[0]?.colId ?? resolvedConfig.columnDefs?.[0]?.field,
is_desc: params.sortModel?.[0]?.sort === 'desc'
},
{
done: (x) => {
if (x && Array.isArray(x)) {
params.successCallback(
x.map((x) => {
let primaryKeys = getPrimaryKeys(resolvedConfig.columnDefs)
let o = {}
primaryKeys.forEach((pk) => {
o[pk] = x[pk]
})
x['__index'] = JSON.stringify(o)
return x
}),
datasource.rowCount
)
} else {
params.failCallback()
}
},
cancel: () => {
console.log('cancel datasource request')
params.failCallback()
},
error: () => {
console.log('error datasource request')
params.failCallback()
}
}
)
console.log('asking for ' + params.startRow + ' to ' + params.endRow, uuid)
}
}
let lastTable: string | undefined = undefined
async function listColumnsIfAvailable() {
let table = resolvedConfig.type.configuration?.postgresql?.table
if (lastTable === table) return
lastTable = table
let tableMetadata = await loadTableMetaData(
resolvedConfig.type.configuration.postgresql.resource,
$workspaceStore,
resolvedConfig.type.configuration.postgresql.table
)
if (!tableMetadata) return
const gridItem = findGridItem($app, id)
if (!gridItem) return
let columnDefs = gridItem.data.configuration.columnDefs as StaticInput<TableMetadata>
let old: TableMetadata = (columnDefs?.value as TableMetadata) ?? []
if (!Array.isArray(old)) {
console.log('old is not an array RESET')
old = []
}
// console.log('OLD', old)
// console.log(tableMetadata)
const oldMap = Object.fromEntries(old.filter((x) => x != undefined).map((x) => [x.field, x]))
const newMap = Object.fromEntries(tableMetadata?.map((x) => [x.field, x]) ?? [])
let ncols: any[] = []
Object.entries(oldMap).forEach(([key, value]) => {
if (newMap[key]) {
ncols.push({
...value,
...newMap[key]
})
}
})
Object.entries(newMap).forEach(([key, value]) => {
if (!oldMap[key]) {
ncols.push(value)
}
})
state = undefined
//@ts-ignore
gridItem.data.configuration.columnDefs = { value: ncols, type: 'static' }
gridItem.data = gridItem.data
$app = $app
let oldS = $selectedComponent
$selectedComponent = []
await tick()
$selectedComponent = oldS
//@ts-ignore
resolvedConfig.columnDefs = ncols
renderCount += 1
}
let isInsertable: boolean = false
$: $worldStore && connectToComponents()
function connectToComponents() {
if ($worldStore) {
const outputs = $worldStore.outputsById[`${id}_count`]
if (outputs) {
outputs.result.subscribe(
{
id: 'dbexplorer-count-' + id,
next: (value) => {
datasource.rowCount = value?.[0]?.count
}
},
datasource.rowCount
)
}
}
}
async function insert() {
try {
await insertRowRunnable?.insertRow(
resolvedConfig.type.configuration.postgresql.resource,
$workspaceStore,
resolvedConfig.type.configuration.postgresql.table,
resolvedConfig.columnDefs,
args
)
insertDrawer?.closeDrawer()
renderCount++
} catch (e) {
sendUserToast(e.message, true)
}
args = {}
}
let runnableComponent: RunnableComponent
let state: any = undefined
let insertRowRunnable: InsertRowRunnable
let deleteRow: DeleteRow
function onDelete(e) {
const data = { ...e.detail }
delete data['__index']
let primaryColumns = getPrimaryKeys(resolvedConfig.columnDefs)
let getPrimaryKeysresolvedConfig = resolvedConfig.columnDefs?.filter((x) =>
primaryColumns.includes(x.field)
)
deleteRow?.triggerDelete(
resolvedConfig.type.configuration.postgresql.resource,
resolvedConfig.type.configuration.postgresql.table ?? 'unknown',
getPrimaryKeysresolvedConfig,
data
)
}
let refreshCount = 0
</script>
{#each Object.keys(components['dbexplorercomponent'].initialData.configuration) as key (key)}
<ResolveConfig
{id}
extraKey="db_explorer"
{key}
bind:resolvedConfig={resolvedConfig[key]}
configuration={configuration[key]}
/>
{/each}
<InsertRowRunnable
on:insert={() => {
aggrid?.clearRows()
refreshCount++
}}
{id}
bind:this={insertRowRunnable}
/>
{#if resolvedConfig.allowDelete}
<DeleteRow
on:deleted={() => {
aggrid?.clearRows()
refreshCount++
}}
{id}
bind:this={deleteRow}
/>
{/if}
<UpdateCell {id} bind:this={updateCell} />
<DbExplorerCount
renderCount={refreshCount}
{id}
{quicksearch}
table={resolvedConfig?.type?.configuration?.postgresql?.table ?? ''}
resource={resolvedConfig?.type?.configuration?.postgresql?.resource ?? ''}
/>
<RunnableWrapper
allowConcurentRequests
noInitialize
bind:runnableComponent
componentInput={input}
autoRefresh={false}
{render}
{id}
{outputs}
>
<div class="h-full" bind:clientHeight={componentContainerHeight}>
<div class="flex p-2 justify-between gap-4" bind:clientHeight={buttonContainerHeight}>
<input
on:pointerdown|stopPropagation
on:keydown|stopPropagation
class="w-full max-w-[300px]"
type="text"
bind:value={quicksearch}
placeholder="Quicksearch"
/>
<Button
startIcon={{ icon: Plus }}
color="dark"
size="xs2"
on:click={() => {
args = {}
insertDrawer?.openDrawer()
}}
>
Insert
</Button>
</div>
{#if resolvedConfig.type.configuration?.postgresql?.resource && resolvedConfig.type.configuration?.postgresql?.table}
<!-- {JSON.stringify(lastInput)} -->
<!-- <span class="text-xs">{JSON.stringify(configuration.columnDefs)}</span> -->
{#key renderCount}
<!-- {JSON.stringify(resolvedConfig.columnDefs)} -->
<AppAggridExplorerTable
bind:this={aggrid}
bind:state
{id}
{datasource}
{resolvedConfig}
{customCss}
{outputs}
allowDelete={resolvedConfig.allowDelete ?? false}
containerHeight={componentContainerHeight - buttonContainerHeight}
on:update={onUpdate}
on:delete={onDelete}
/>
{/key}
{/if}
</div>
</RunnableWrapper>
<Portal>
<Drawer bind:this={insertDrawer} size="800px">
<DrawerContent title="Insert row" on:close={insertDrawer.closeDrawer}>
<svelte:fragment slot="actions">
<Button color="dark" size="xs" on:click={insert} disabled={!isInsertable}>Insert</Button>
</svelte:fragment>
<InsertRow bind:args bind:isInsertable columnDefs={resolvedConfig.columnDefs} />
</DrawerContent>
</Drawer>
</Portal>
frontend/src/lib/components/apps/components/display/dbtable/DbExplorerCount.svelte
+71
View File
@@ -0,0 +1,71 @@
<script lang="ts">
import { getContext, tick } from 'svelte'
import type { AppInput } from '../../../inputType'
import type { AppViewerContext } from '../../../types'
import type RunnableComponent from '../../helpers/RunnableComponent.svelte'
import RunnableWrapper from '../../helpers/RunnableWrapper.svelte'
import { initOutput } from '../../../editor/appUtils'
import { getCountPostgresql } from './utils'
export let id: string
export let table: string
export let resource: string
export let renderCount: number
export let quicksearch: string
const { worldStore } = getContext<AppViewerContext>('AppViewerContext')
let outputs = initOutput($worldStore, `${id}_count`, {
result: undefined,
loading: false,
jobId: undefined
})
let runnableComponent: RunnableComponent
let loading = false
let input: AppInput | undefined = undefined
$: table && renderCount != undefined && quicksearch != undefined && computeCount()
let lastTableCount = ''
let renderCountLast = -1
let quicksearchLast: string | undefined = undefined
async function computeCount() {
if (
lastTableCount === table &&
renderCount == renderCountLast &&
quicksearch == quicksearchLast
)
return
if (table != '' && resource != '') {
renderCountLast = renderCount
lastTableCount = table
quicksearchLast = quicksearch
await getCount(resource, table, quicksearch)
}
}
async function getCount(resource: string, table: string, quicksearch: string) {
input = getCountPostgresql(resource, table)
await tick()
if (runnableComponent) {
await runnableComponent?.runComponent(undefined, undefined, undefined, {
quicksearch
})
}
}
</script>
<RunnableWrapper
noInitialize
bind:runnableComponent
bind:loading
componentInput={input}
autoRefresh={false}
render={false}
id={`${id}_count`}
{outputs}
/>
frontend/src/lib/components/apps/components/display/dbtable/DeleteRow.svelte
+76
View File
@@ -0,0 +1,76 @@
<script lang="ts">
import { createEventDispatcher, getContext, tick } from 'svelte'
import type { AppInput } from '../../../inputType'
import type { AppViewerContext } from '../../../types'
import type RunnableComponent from '../../helpers/RunnableComponent.svelte'
import RunnableWrapper from '../../helpers/RunnableWrapper.svelte'
import { initOutput } from '../../../editor/appUtils'
import { type ColumnMetadata, createDeletePostgresInput } from './utils'
import { sendUserToast } from '$lib/toast'
export let id: string
const { worldStore } = getContext<AppViewerContext>('AppViewerContext')
let outputs = initOutput($worldStore, `${id}_delete`, {
result: undefined,
loading: false,
jobId: undefined
})
let runnableComponent: RunnableComponent
let loading = false
let input: AppInput | undefined = undefined
const dispatch = createEventDispatcher()
export async function triggerDelete(
resource: string,
table: string,
columns: ColumnMetadata[],
data: Record<string, any>
) {
// const datatype = tableMetaData?.find((column) => column.isprimarykey)?.datatype
input = createDeletePostgresInput(resource, table, columns)
await tick()
if (runnableComponent) {
let ndata = {}
columns.forEach((x) => {
ndata[x.field] = data[x.field]
})
await runnableComponent?.runComponent(
undefined,
undefined,
undefined,
{ ...ndata },
{
done: (x) => {
sendUserToast('Row deleted', false)
dispatch('deleted')
},
cancel: () => {
sendUserToast('Error deleting row', true)
},
error: () => {
sendUserToast('Error updating row', true)
}
}
)
}
}
</script>
<RunnableWrapper
noInitialize
bind:runnableComponent
bind:loading
componentInput={input}
autoRefresh={false}
render={false}
id={`${id}_delete`}
{outputs}
/>
frontend/src/lib/components/apps/components/display/dbtable/InsertRow.svelte
+130
View File
@@ -0,0 +1,130 @@
<script lang="ts">
import type { Schema, SchemaProperty } from '$lib/common'
import LightweightSchemaForm from '$lib/components/LightweightSchemaForm.svelte'
import { ColumnIdentity, type ColumnMetadata } from './utils'
export let args: Record<string, any> = {}
export let columnDefs: Array<
{
field: string
ignored: boolean
hideInsert: boolean
overrideDefaultValue: boolean
defaultUserValue: any
defaultValueNull: boolean
} & ColumnMetadata
> = []
type FieldMetadata = {
type: string
name: string
isPrimaryKey: boolean
defaultValue: string | undefined
fieldType: 'text' | 'number' | 'checkbox' | 'date'
identity: ColumnIdentity
nullable: 'YES' | 'NO'
}
$: fields = columnDefs
?.filter((t) => {
const shouldFilter = t.isidentity !== ColumnIdentity.Always && t?.hideInsert === true
return !shouldFilter
})
.map((column) => {
const type = column.datatype
const name = column.field
const isPrimaryKey = column.isprimarykey
const defaultValue = column.defaultValueNull ? null : column.defaultUserValue
const baseType = type.split('(')[0]
const validTextTypes = ['character varying', 'text']
const validNumberTypes = ['integer', 'bigint', 'numeric', 'double precision']
const validDateTypes = ['date', 'timestamp without time zone', 'timestamp with time zone']
const fieldType = validTextTypes.includes(baseType)
? 'text'
: validNumberTypes.includes(baseType)
? 'number'
: baseType === 'boolean'
? 'checkbox'
: validDateTypes.includes(baseType)
? 'date'
: 'text'
return {
type,
name,
isPrimaryKey,
defaultValue,
fieldType,
identity: column.isidentity,
nullable: column.isnullable
}
}) as FieldMetadata[] | undefined
function builtSchema(fields: FieldMetadata[]): Schema {
const properties: { [name: string]: SchemaProperty } = {}
const required: string[] = []
fields.forEach((field) => {
const schemaProperty: SchemaProperty = {
type: field.fieldType
}
switch (field.fieldType) {
case 'number':
schemaProperty.type = 'number'
const extractedDefaultValue = field.defaultValue
schemaProperty.default = extractedDefaultValue ? Number(extractedDefaultValue) : undefined
break
case 'checkbox':
schemaProperty.type = 'boolean'
schemaProperty.default = field.defaultValue?.toLocaleLowerCase() === 'true'
break
case 'date':
schemaProperty.type = 'string'
schemaProperty.format = 'date-time'
schemaProperty.default = field.defaultValue
break
case 'text':
default:
schemaProperty.type = 'string'
schemaProperty.default = field.defaultValue
break
}
properties[field.name] = schemaProperty
const isRequired =
(field.isPrimaryKey || field.defaultValue === undefined || field.defaultValue === null) &&
field.nullable !== 'YES' &&
![ColumnIdentity.Always, ColumnIdentity.ByDefault].includes(field.identity)
if (isRequired) {
required.push(field.name)
}
})
return {
$schema: 'http://json-schema.org/draft-07/schema#',
type: 'object',
properties,
required
}
}
$: schema = builtSchema(fields ?? []) as Schema
export let isInsertable: boolean = false
$: if (schema) {
const requiredFields = schema.required ?? []
const filledFields = Object.keys(args).filter(
(key) => args[key] !== undefined && args[key] !== null
)
isInsertable = requiredFields.every((field) => filledFields.includes(field))
}
</script>
<LightweightSchemaForm {schema} bind:args />
frontend/src/lib/components/apps/components/display/dbtable/InsertRowRunnable.svelte
+70
View File
@@ -0,0 +1,70 @@
<script lang="ts">
import { createEventDispatcher, getContext, tick } from 'svelte'
import type { AppInput } from '../../../inputType'
import type { AppViewerContext } from '../../../types'
import type RunnableComponent from '../../helpers/RunnableComponent.svelte'
import RunnableWrapper from '../../helpers/RunnableWrapper.svelte'
import { initOutput } from '../../../editor/appUtils'
import { type ColumnDef, createPostgresInsert } from './utils'
import { sendUserToast } from '$lib/toast'
export let id: string
const { worldStore } = getContext<AppViewerContext>('AppViewerContext')
let outputs = initOutput($worldStore, `${id}_insert`, {
result: undefined,
loading: false,
jobId: undefined
})
let runnableComponent: RunnableComponent
let loading = false
let input: AppInput | undefined = undefined
const dispatch = createEventDispatcher()
export async function insertRow(
resource: string,
workspace: string | undefined,
table: string | undefined,
columns: ColumnDef[],
values: Record<string, any>
): Promise<boolean> {
if (!resource || !table || !workspace) {
return false
}
input = createPostgresInsert(table, columns, resource)
await tick()
if (runnableComponent) {
await runnableComponent?.runComponent(undefined, undefined, undefined, values, {
done: (x) => {
dispatch('insert')
sendUserToast('Row inserted', false)
},
cancel: () => {
sendUserToast('Error inserting row', true)
},
error: () => {
sendUserToast('Error inserting row', true)
}
})
}
return false
}
</script>
<RunnableWrapper
noInitialize
bind:runnableComponent
bind:loading
componentInput={input}
autoRefresh={false}
render={false}
id={`${id}_insert`}
{outputs}
/>
frontend/src/lib/components/apps/components/display/dbtable/UpdateCell.svelte
+80
View File
@@ -0,0 +1,80 @@
<script lang="ts">
import { getContext, tick } from 'svelte'
import type { AppInput } from '../../../inputType'
import type { AppViewerContext } from '../../../types'
import type RunnableComponent from '../../helpers/RunnableComponent.svelte'
import RunnableWrapper from '../../helpers/RunnableWrapper.svelte'
import { initOutput } from '../../../editor/appUtils'
import { createUpdatePostgresInput, type ColumnMetadata, getPrimaryKeys } from './utils'
import { sendUserToast } from '$lib/toast'
export let id: string
const { worldStore } = getContext<AppViewerContext>('AppViewerContext')
let outputs = initOutput($worldStore, `${id}_update`, {
result: undefined,
loading: false,
jobId: undefined
})
let runnableComponent: RunnableComponent
let loading = false
let input: AppInput | undefined = undefined
export async function triggerUpdate(
resource: string,
table: string,
column: ColumnMetadata,
allColumns: ColumnMetadata[],
valueToUpdate: string,
data: Record<string, any>,
oldValue: string | undefined = undefined
) {
// const datatype = tableMetaData?.find((column) => column.isprimarykey)?.datatype
let primaryColumns = getPrimaryKeys(allColumns)
let columns = allColumns?.filter((x) => primaryColumns.includes(x.field))
input = createUpdatePostgresInput(resource, table, column, columns)
await tick()
if (runnableComponent) {
let ndata = {}
columns.forEach((x) => {
ndata[x.field] = data[x.field]
})
ndata[column.field] = oldValue
await runnableComponent?.runComponent(
undefined,
undefined,
undefined,
{ valueToUpdate, ...ndata },
{
done: (x) => {
sendUserToast('Value updated', false)
},
cancel: () => {
sendUserToast('Error updating value', true)
},
error: () => {
sendUserToast('Error updating value', true)
}
}
)
}
}
</script>
<RunnableWrapper
noInitialize
bind:runnableComponent
bind:loading
componentInput={input}
autoRefresh={false}
render={false}
id={`${id}_update`}
{outputs}
/>
frontend/src/lib/components/apps/components/display/dbtable/utils.ts
+783
View File
@@ -0,0 +1,783 @@
import type { AppInput, RunnableByName } from '../../../inputType'
import { JobService, Preview } from '$lib/gen'
import type { DBSchema, DBSchemas, GraphqlSchema, SQLSchema } from '$lib/stores'
import { buildClientSchema, getIntrospectionQuery, printSchema } from 'graphql'
import { tryEvery } from '$lib/utils'
export function makeQuery(
table: string,
tableMetadata: TableMetadata,
whereClause: string | undefined
) {
if (!table) throw new Error('Table name is required')
const filteredColumns = tableMetadata
.filter((x) => x != undefined)
.map((column) => `${column?.field}::text`)
let selectClause = filteredColumns.join(', ')
let orderBy = `
${tableMetadata
.map(
(column) =>
`
(CASE WHEN $4 = '${column.field}' AND $5 IS false THEN ${column.field}::text END),
(CASE WHEN $4 = '${column.field}' AND $5 IS true THEN ${column.field}::text END) DESC`
)
.join(',\n')}`
let query = `
-- $1 limit
-- $2 offset
-- $3 quicksearch
-- $4 orderBy
-- $5 is_desc
SELECT ${selectClause} FROM ${table} WHERE `
if (whereClause) {
query += ` ${whereClause} AND `
}
query += ` ($3 = '' OR ${table}::text ILIKE '%' || $3 || '%')`
query += ` ORDER BY ${orderBy}`
query += ` LIMIT $1::INT OFFSET $2::INT`
return query
}
export function createPostgresInsert(
table: string,
columns: ColumnDef[],
resource: string
): AppInput {
return {
runnable: {
name: 'AppDbExplorer',
type: 'runnableByName',
inlineScript: {
content: makeInsertQuery(table, columns),
language: Preview.language.POSTGRESQL,
schema: {
$schema: 'https://json-schema.org/draft/2020-12/schema',
properties: {},
required: ['database'],
type: 'object'
}
}
},
fields: {
database: {
type: 'static',
value: resource,
fieldType: 'object',
format: 'resource-postgresql'
}
},
type: 'runnable',
fieldType: 'object'
}
}
export function makeInsertQuery(table: string, columns: ColumnDef[]) {
if (!table) throw new Error('Table name is required')
const columnsInsert = columns.filter((x) => !x.hideInsert)
const columnsDefault = columns.filter(
(x) => x.hideInsert && (x.overrideDefaultValue || x.defaultvalue === null)
)
const allInsertColumns = columnsInsert.concat(columnsDefault)
// Constructing the query
const query = `
${columnsInsert.map((column, i) => `-- $${i + 1} ${column.field}`).join('\n')}
INSERT INTO ${table} (${allInsertColumns.map((c) => c.field).join(', ')})
VALUES (${columnsInsert.map((c, i) => `$${i + 1}::${c.datatype}`).join(', ')}${
columnsDefault.length > 0 ? ',' : ''
} ${columnsDefault
.map((c) => (c.defaultValueNull ? 'NULL' : `${c.defaultUserValue}::${c.datatype}`))
.join(', ')})`
return query
}
export function getPrimaryKeys(tableMetadata?: TableMetadata): string[] {
let r = tableMetadata?.filter((x) => x.isprimarykey)?.map((x) => x.field) ?? []
if (r?.length === 0) {
r = tableMetadata?.map((x) => x.field) ?? []
}
return r ?? []
}
export function createPostgresInput(
resource: string,
table: string | undefined,
columns: TableMetadata,
whereClause: string | undefined
): AppInput | undefined {
if (!resource || !table || !columns) {
// Return undefined if resource or table is not defined
return undefined
}
const getRunnable: RunnableByName = {
name: 'AppDbExplorer',
type: 'runnableByName',
inlineScript: {
content: makeQuery(table, columns, whereClause),
language: Preview.language.POSTGRESQL
}
}
const getQuery: AppInput = {
runnable: getRunnable,
fields: {
database: {
type: 'static',
value: resource,
fieldType: 'object',
format: 'resource-postgresql'
}
},
type: 'runnable',
fieldType: 'object'
}
return getQuery
}
export function createUpdatePostgresInput(
resource: string,
table: string,
column: ColumnMetadata,
columns: ColumnMetadata[]
): AppInput | undefined {
if (!resource || !table) {
return undefined
}
const query = updateWithAllValues()
function updateWithAllValues() {
let query = `
-- $1 valueToUpdate
${columns.map((c, i) => `-- $${i + 2} ${c.field}`).join('\n')}
UPDATE ${table} SET ${column.field} = $1::text::${column.datatype} WHERE
${columns.map((c, i) => `${c.field} = $${i + 2}::text::${c.datatype} `).join(' AND ')}
RETURNING 1`
return query
}
const updateRunnable: RunnableByName = {
name: 'AppDbExplorer',
type: 'runnableByName',
inlineScript: {
content: query,
language: Preview.language.POSTGRESQL,
schema: {
$schema: 'https://json-schema.org/draft/2020-12/schema',
properties: {},
required: ['database'],
type: 'object'
}
}
}
const updateQuery: AppInput = {
runnable: updateRunnable,
fields: {
database: {
type: 'static',
value: resource,
fieldType: 'object',
format: 'resource-postgresql'
}
},
type: 'runnable',
fieldType: 'object'
}
return updateQuery
}
export function createDeletePostgresInput(
resource: string,
table: string,
columns: ColumnMetadata[]
): AppInput | undefined {
if (!resource || !table) {
return undefined
}
const query = updateWithAllValues()
function updateWithAllValues() {
let query = `
${columns.map((c, i) => `-- $${i + 1} ${c.field}`).join('\n')}
DELETE FROM ${table} WHERE ${columns
.map((c, i) => `${c.field} = $${i + 1}::text::${c.datatype}`)
.join(' AND ')} RETURNING 1;`
return query
}
const updateRunnable: RunnableByName = {
name: 'AppDbExplorer',
type: 'runnableByName',
inlineScript: {
content: query,
language: Preview.language.POSTGRESQL,
schema: {
$schema: 'https://json-schema.org/draft/2020-12/schema',
properties: {},
required: ['database'],
type: 'object'
}
}
}
const updateQuery: AppInput = {
runnable: updateRunnable,
fields: {
database: {
type: 'static',
value: resource,
fieldType: 'object',
format: 'resource-postgresql'
}
},
type: 'runnable',
fieldType: 'object'
}
return updateQuery
}
export function getCountPostgresql(resource: string, table: string): AppInput | undefined {
if (!resource || !table) {
return undefined
}
const query = `
-- $1 quicksearch
SELECT COUNT(*) FROM ${table} WHERE ($1 = '' OR ${table}::text ILIKE '%' || $1 || '%')`
const updateRunnable: RunnableByName = {
name: 'AppDbExplorer',
type: 'runnableByName',
inlineScript: {
content: query,
language: Preview.language.POSTGRESQL,
schema: {
$schema: 'https://json-schema.org/draft/2020-12/schema',
properties: {
database: {
description: 'Database name',
type: 'object',
format: 'resource-postgresql'
}
},
required: ['database'],
type: 'object'
}
}
}
const updateQuery: AppInput = {
runnable: updateRunnable,
fields: {
database: {
type: 'static',
value: resource,
fieldType: 'object',
format: 'resource-postgresql'
}
},
type: 'runnable',
fieldType: 'object'
}
return updateQuery
}
export enum ColumnIdentity {
ByDefault = 'By Default',
Always = 'Always',
No = 'No'
}
export type ColumnMetadata = {
field: string
datatype: string
defaultvalue: string
isprimarykey: boolean
isidentity: ColumnIdentity
isnullable: 'YES' | 'NO'
isenum: boolean
}
export type TableMetadata = ColumnMetadata[]
export type ColumnDef = {
minWidth: number
hide: boolean
flex: number
sort: 'asc' | 'desc'
sortIndex: number
aggFunc: string
pivot: boolean
pivotIndex: number
pinned: 'left' | 'right' | boolean
rowGroup: boolean
rowGroupIndex: number
valueFormatter: string
valueParser: string
field: string
headerName: string
// DBExplorer
ignored: boolean
hideInsert: boolean
editable: boolean
overrideDefaultValue: boolean
defaultUserValue: any
defaultValueNull: boolean
} & ColumnMetadata
export async function loadTableMetaData(
resource: string,
workspace: string | undefined,
table: string | undefined
): Promise<TableMetadata | undefined> {
if (!resource || !table || !workspace) {
return undefined
}
const code = `
SELECT
a.attname as field,
pg_catalog.format_type(a.atttypid, a.atttypmod) as DataType,
(SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) for 128)
FROM pg_catalog.pg_attrdef d
WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) as DefaultValue,
(SELECT CASE WHEN i.indisprimary THEN true ELSE 'NO' END
FROM pg_catalog.pg_class tbl, pg_catalog.pg_class idx, pg_catalog.pg_index i, pg_catalog.pg_attribute att
WHERE tbl.oid = a.attrelid AND idx.oid = i.indexrelid AND att.attrelid = tbl.oid
AND i.indrelid = tbl.oid AND att.attnum = any(i.indkey) AND att.attname = a.attname LIMIT 1) as IsPrimaryKey,
CASE a.attidentity
WHEN 'd' THEN 'By Default'
WHEN 'a' THEN 'Always'
ELSE 'No'
END as IsIdentity,
CASE a.attnotnull
WHEN false THEN 'YES'
ELSE 'NO'
END as IsNullable,
(SELECT true
FROM pg_catalog.pg_enum e
WHERE e.enumtypid = a.atttypid FETCH FIRST ROW ONLY) as IsEnum
FROM pg_catalog.pg_attribute a
WHERE a.attrelid = (SELECT oid FROM pg_catalog.pg_class WHERE relname = '${table}')
AND a.attnum > 0 AND NOT a.attisdropped
ORDER BY a.attnum;
`
const maxRetries = 3
let attempts = 0
while (attempts < maxRetries) {
try {
const job = await JobService.runScriptPreview({
workspace: workspace,
requestBody: {
language: Preview.language.POSTGRESQL,
content: code,
args: {
database: resource
}
}
})
await new Promise((resolve) => setTimeout(resolve, 3000))
const testResult = await JobService.getCompletedJob({
workspace: workspace,
id: job
})
if (testResult.success) {
attempts = maxRetries
return testResult.result
} else {
attempts++
}
} catch (error) {
attempts++
}
// Exponential back-off
await new Promise((resolve) => setTimeout(resolve, 2000 * attempts))
}
console.error('Failed to load table metadata after maximum retries.')
return undefined
}
export function resourceTypeToLang(rt: string) {
if (rt === 'ms_sql_server') {
return 'mssql'
} else {
return rt
}
}
const scripts: Record<
string,
{
code: string
lang: string
processingFn?: (any: any) => SQLSchema['schema']
argName: string
}
> = {
postgresql: {
code: `SELECT table_name, column_name, udt_name, column_default, is_nullable, table_schema FROM information_schema.columns WHERE table_schema != 'pg_catalog' AND table_schema != 'information_schema'`,
processingFn: (rows) => {
const schemas = rows.reduce((acc, a) => {
const table_schema = a.table_schema
delete a.table_schema
acc[table_schema] = acc[table_schema] || []
acc[table_schema].push(a)
return acc
}, {})
const data = {}
for (const key in schemas) {
data[key] = schemas[key].reduce((acc, a) => {
const table_name = a.table_name
delete a.table_name
acc[table_name] = acc[table_name] || {}
const p: {
type: string
required: boolean
default?: string
} = {
type: a.udt_name,
required: a.is_nullable === 'NO'
}
if (a.column_default) {
p.default = a.column_default
}
acc[table_name][a.column_name] = p
return acc
}, {})
}
return data
},
lang: 'postgresql',
argName: 'database'
},
mysql: {
code: "select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT from information_schema.columns where table_schema != 'information_schema'",
processingFn: (rows) => {
const schemas = rows.reduce((acc, a) => {
const table_schema = a.TABLE_SCHEMA
delete a.TABLE_SCHEMA
acc[table_schema] = acc[table_schema] || []
acc[table_schema].push(a)
return acc
}, {})
const data = {}
for (const key in schemas) {
data[key] = schemas[key].reduce((acc, a) => {
const table_name = a.TABLE_NAME
delete a.TABLE_NAME
acc[table_name] = acc[table_name] || {}
const p: {
type: string
required: boolean
default?: string
} = {
type: a.DATA_TYPE,
required: a.is_nullable === 'NO'
}
if (a.column_default) {
p.default = a.COLUMN_DEFAULT
}
acc[table_name][a.COLUMN_NAME] = p
return acc
}, {})
}
return data
},
lang: 'mysql',
argName: 'database'
},
graphql: {
code: getIntrospectionQuery(),
lang: 'graphql',
argName: 'api'
},
bigquery: {
code: `import { BigQuery } from 'npm:@google-cloud/bigquery@7.2.0';
export async function main(args: bigquery) {
const bq = new BigQuery({
credentials: args
})
const [datasets] = await bq.getDatasets();
const schema = {}
for (const dataset of datasets) {
schema[dataset.id] = {}
const query = "SELECT table_name, ARRAY_AGG(STRUCT(if(is_nullable = 'YES', true, false) AS required, column_name AS name, data_type AS type, if(column_default = 'NULL', null, column_default) AS \`default\`) ORDER BY ordinal_position) AS schema \
FROM \`{dataset.id}\`.INFORMATION_SCHEMA.COLUMNS \
GROUP BY table_name".replace('{dataset.id}', dataset.id)
const [rows] = await bq.query(query)
for (const row of rows) {
schema[dataset.id][row.table_name] = {}
for (const col of row.schema) {
const colName = col.name
delete col.name
if (col.default === null) {
delete col.default
}
schema[dataset.id][row.table_name][colName] = col
}
}
}
return schema
}`, // nested template literals
lang: 'deno',
argName: 'args'
},
snowflake: {
code: `select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT, IS_NULLABLE from information_schema.columns where table_schema != 'INFORMATION_SCHEMA'`,
lang: 'snowflake',
processingFn: (rows) => {
const schema = {}
for (const row of rows) {
if (!(row.TABLE_SCHEMA in schema)) {
schema[row.TABLE_SCHEMA] = {}
}
if (!(row.TABLE_NAME in schema[row.TABLE_SCHEMA])) {
schema[row.TABLE_SCHEMA][row.TABLE_NAME] = {}
}
schema[row.TABLE_SCHEMA][row.TABLE_NAME][row.COLUMN_NAME] = {
type: row.DATA_TYPE,
required: row.IS_NULLABLE === 'YES'
}
if (row.COLUMN_DEFAULT !== null) {
schema[row.TABLE_SCHEMA][row.TABLE_NAME][row.COLUMN_NAME]['default'] = row.COLUMN_DEFAULT
}
}
return schema
},
argName: 'database'
},
ms_sql_server: {
argName: 'database',
code: `select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT from information_schema.columns where table_schema != 'sys'`,
lang: 'mssql',
processingFn: (rows) => {
const schemas = rows[0].reduce((acc, a) => {
const table_schema = a.TABLE_SCHEMA
delete a.TABLE_SCHEMA
acc[table_schema] = acc[table_schema] || []
acc[table_schema].push(a)
return acc
}, {})
const data = {}
for (const key in schemas) {
data[key] = schemas[key].reduce((acc, a) => {
const table_name = a.TABLE_NAME
delete a.TABLE_NAME
acc[table_name] = acc[table_name] || {}
const p: {
type: string
required: boolean
default?: string
} = {
type: a.DATA_TYPE,
required: a.is_nullable === 'NO'
}
if (a.column_default) {
p.default = a.COLUMN_DEFAULT
}
acc[table_name][a.COLUMN_NAME] = p
return acc
}, {})
}
return data
}
}
}
export { scripts }
export async function getDbSchemas(
resourceType: string,
resourcePath: string,
workspace: string | undefined,
dbSchemas: DBSchemas,
errorCallback: (message: string) => void
): Promise<void> {
return new Promise(async (resolve, reject) => {
if (!resourceType || !resourcePath || !workspace) {
resolve()
return
}
const job = await JobService.runScriptPreview({
workspace: workspace,
requestBody: {
language: scripts[resourceType].lang as Preview.language,
content: scripts[resourceType].code,
args: {
[scripts[resourceType].argName]: '$res:' + resourcePath
}
}
})
tryEvery({
tryCode: async () => {
if (resourcePath) {
const testResult = await JobService.getCompletedJob({
workspace,
id: job
})
if (!testResult.success) {
console.error(testResult.result?.['error']?.['message'])
} else {
if (resourceType !== undefined) {
if (resourceType !== 'graphql') {
const { processingFn } = scripts[resourceType]
const schema =
processingFn !== undefined ? processingFn(testResult.result) : testResult.result
dbSchemas[resourcePath] = {
lang: resourceTypeToLang(resourceType) as SQLSchema['lang'],
schema,
publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo
}
} else {
if (typeof testResult.result !== 'object' || !('__schema' in testResult.result)) {
console.error('Invalid GraphQL schema')
errorCallback('Invalid GraphQL schema')
} else {
dbSchemas[resourcePath] = {
lang: 'graphql',
schema: testResult.result
}
}
}
}
}
resolve()
}
},
timeoutCode: async () => {
console.error('Could not query schema within 5s')
errorCallback('Could not query schema within 5s')
try {
await JobService.cancelQueuedJob({
workspace,
id: job,
requestBody: {
reason: 'Could not query schema within 5s'
}
})
} catch (err) {
console.error(err)
}
reject()
},
interval: 500,
timeout: 5000
})
})
}
export function formatSchema(dbSchema: DBSchema) {
if (dbSchema.lang !== 'graphql' && dbSchema.publicOnly) {
return dbSchema.schema.public || dbSchema.schema.PUBLIC || dbSchema.schema.dbo || dbSchema
} else if (dbSchema.lang === 'mysql' && Object.keys(dbSchema.schema).length === 1) {
return dbSchema.schema[Object.keys(dbSchema.schema)[0]]
} else {
return dbSchema.schema
}
}
export function formatGraphqlSchema(dbSchema: GraphqlSchema): string {
return printSchema(buildClientSchema(dbSchema.schema))
}
/**
* Base class for embedding a svelte component within an AGGrid call.
* See: https://stackoverflow.com/a/72608215
*/
import type { ICellRendererComp, ICellRendererParams } from 'ag-grid-community'
/**
* Class for defining a cell renderer.
* If you don't need to define a separate class you could use cellRendererFactory
* to create a component with the column definitions.
*/
export abstract class AbstractCellRenderer implements ICellRendererComp {
eGui: any
protected value: any
protected params: any
constructor(parentElement = 'span') {
// create empty span (or other element) to place svelte component in
this.eGui = document.createElement(parentElement)
}
init(params: ICellRendererParams & { onClick?: (data: any) => void }) {
this.value = params.value
this.createComponent(params)
this.eGui.addEventListener('click', () => params.onClick?.(params.data))
this.params = params
}
getGui() {
return this.eGui
}
refresh(params: ICellRendererParams) {
this.value = params.value
this.eGui.innerHTML = ''
return true
}
/**
* Define and create the svelte component to use in the cell
* @example
* // This is all you need to do within this method: create the component with new, specify the target
* // is the class, and pass in props via the params.
* new CampusIcon({
* target: this.eGui,
* props: {
* color: params.data?.color,
* name: params.data?.name
* }
* @param params params for rendering the call, including the value for the cell
*/
abstract createComponent(params: ICellRendererParams): void
}
/**
* Creates a cell renderer using the given callback for how to initialise a svelte component.
* See AbstractCellRenderer.createComponent
* @param svelteComponent function for how to create the svelte component
* @returns
*/
export function cellRendererFactory(
svelteComponent: (cell: AbstractCellRenderer, params: ICellRendererParams) => void
) {
class Renderer extends AbstractCellRenderer {
createComponent(params: ICellRendererParams<any, any>): void {
svelteComponent(this, params)
}
}
return Renderer
}
frontend/src/lib/components/apps/components/display/table/AppAggridExplorerTable.svelte
+316
View File
@@ -0,0 +1,316 @@
<script lang="ts">
import { GridApi, createGrid, type IDatasource } from 'ag-grid-community'
import { isObject } from '$lib/utils'
import { createEventDispatcher, getContext } from 'svelte'
import type { AppViewerContext, ComponentCustomCSS } from '../../../types'
import type { components } from '$lib/components/apps/editor/component'
import Alert from '$lib/components/common/alert/Alert.svelte'
import { deepEqual } from 'fast-equals'
import 'ag-grid-community/styles/ag-grid.css'
import 'ag-grid-community/styles/ag-theme-alpine.css'
import { twMerge } from 'tailwind-merge'
import { initCss } from '$lib/components/apps/utils'
import ResolveStyle from '../../helpers/ResolveStyle.svelte'
import type { RunnableComponent } from '../..'
import type { Output } from '$lib/components/apps/rx'
import type { InitConfig } from '$lib/components/apps/editor/appUtils'
import { Button } from '$lib/components/common'
import { cellRendererFactory } from '../dbtable/utils'
import { Trash2 } from 'lucide-svelte'
// import 'ag-grid-community/dist/styles/ag-theme-alpine-dark.css'
export let id: string
export let customCss: ComponentCustomCSS<'aggridcomponent'> | undefined = undefined
export let containerHeight: number | undefined = undefined
export let resolvedConfig: InitConfig<
(typeof components)['dbexplorercomponent']['initialData']['configuration']
>
export let datasource: IDatasource
export let state: any = undefined
export let outputs: Record<string, Output<any>>
export let allowDelete: boolean
const { app, selectedComponent, componentControl, darkMode } =
getContext<AppViewerContext>('AppViewerContext')
let css = initCss($app.css?.aggridcomponent, customCss)
// let result: any[] | undefined = undefined
// $: result && setValues()
// let value: any[] = Array.isArray(result)
// ? (result as any[]).map((x, i) => ({ ...x, __index: i.toString() }))
// : [{ error: 'input was not an array' }]
// let loaded = false
// async function setValues() {
// value = Array.isArray(result)
// ? (result as any[]).map((x, i) => ({ ...x, __index: i.toString() }))
// : [{ error: 'input was not an array' }]
// if (api && loaded) {
// let selected = api.getSelectedNodes()
// if (selected && selected.length > 0 && resolvedConfig?.selectFirstRowByDefault != false) {
// let data = { ...selected[0].data }
// delete data['__index']
// outputs?.selectedRow?.set(data)
// }
// }
// if (!loaded) {
// loaded = true
// }
// }
let selectedRowIndex = -1
function toggleRow(row: any) {
if (row) {
let rowIndex = row.rowIndex
let data = { ...row.data }
delete data['__index']
if (selectedRowIndex !== rowIndex) {
selectedRowIndex = rowIndex
outputs?.selectedRowIndex.set(rowIndex)
}
if (!deepEqual(outputs?.selectedRow?.peak(), data)) {
outputs?.selectedRow.set(data)
}
}
}
function toggleRows(rows: any[]) {
if (rows.length === 0) {
outputs?.selectedRows.set([])
}
toggleRow(rows[0])
outputs?.selectedRows.set(
rows.map((x) => {
let data = { ...x.data }
delete data['__index']
return data
})
)
}
let clientHeight
let clientWidth
const dispatch = createEventDispatcher()
function onCellValueChanged(event) {
let dataCell = event.newValue
outputs?.newChange?.set({
row: event.node.rowIndex,
column: event.colDef.field,
value: dataCell
})
// result[event.node.rowIndex][event.colDef.field] = dataCell
// let data = { ...result[event.node.rowIndex] }
// outputs?.selectedRow?.set(data)
dispatch('update', {
row: event.node.rowIndex,
column: event.colDef.field,
value: dataCell,
data: event.node.data,
oldValue: event.oldValue,
columnDef: event.colDef
})
}
let api: GridApi<any> | undefined = undefined
let eGui: HTMLDivElement
$: eGui && mountGrid()
function transformColumnDefs(columnDefs: any[]) {
let r = columnDefs?.filter((x) => x && !x.ignored) ?? []
if (allowDelete) {
r.push({
field: 'delete',
headerName: 'Delete',
cellRenderer: cellRendererFactory((c, p) => {
new Button({
target: c.eGui,
props: {
btnClasses: 'mt-1',
color: 'red',
variant: 'border',
iconOnly: true,
endIcon: { icon: Trash2 },
nonCaptureEvent: true
}
})
}),
cellRendererParams: {
onClick: (e) => {
dispatch('delete', e)
}
},
lockPosition: 'right',
editable: false,
flex: 0,
width: 100
})
}
return r
}
let firstRow = 0
let lastRow = 0
function mountGrid() {
if (eGui) {
createGrid(
eGui,
{
rowModelType: 'infinite',
datasource,
columnDefs: transformColumnDefs(resolvedConfig?.columnDefs),
pagination: false,
defaultColDef: {
flex: resolvedConfig.flex ? 1 : 0,
editable: resolvedConfig?.allEditable,
onCellValueChanged
},
suppressColumnMoveAnimation: true,
rowSelection: resolvedConfig?.multipleSelectable ? 'multiple' : 'single',
rowMultiSelectWithClick: resolvedConfig?.multipleSelectable
? resolvedConfig.rowMultiselectWithClick
: false,
initialState: state,
suppressRowDeselection: true,
...(resolvedConfig?.extraConfig ?? {}),
onViewportChanged: (e) => {
firstRow = e.firstRow
lastRow = e.lastRow
},
onStateUpdated: (e) => {
state = e?.api?.getState()
resolvedConfig?.extraConfig?.['onStateUpdated']?.(e)
},
onGridReady: (e) => {
outputs?.ready.set(true)
$componentControl[id] = {
agGrid: { api: e.api, columnApi: e.columnApi },
setSelectedIndex: (index) => {
e.api.getRowNode(index.toString())?.setSelected(true)
}
}
api = e.api
resolvedConfig?.extraConfig?.['onGridReady']?.(e)
},
onSelectionChanged: (e) => {
onSelectionChanged(e.api)
resolvedConfig?.extraConfig?.['onSelectionChanged']?.(e)
},
getRowId: (data) => {
return (data as any).data['__index']
}
},
{}
)
}
}
$: resolvedConfig && updateOptions()
$: datasource && api?.updateGridOptions({ datasource })
let extraConfig = resolvedConfig.extraConfig
$: if (!deepEqual(extraConfig, resolvedConfig.extraConfig)) {
extraConfig = resolvedConfig.extraConfig
if (extraConfig) {
api?.updateGridOptions(extraConfig)
}
}
export function clearRows() {
api?.purgeInfiniteCache()
}
function onSelectionChanged(api: GridApi<any>) {
if (resolvedConfig?.multipleSelectable) {
const rows = api.getSelectedNodes()
if (rows != undefined) {
toggleRows(rows)
}
} else {
const row = api.getSelectedNodes()?.[0]
if (row != undefined) {
toggleRow(row)
}
}
}
function updateOptions() {
api?.updateGridOptions({
columnDefs: transformColumnDefs(resolvedConfig?.columnDefs),
defaultColDef: {
flex: resolvedConfig.flex ? 1 : 0,
editable: resolvedConfig?.allEditable,
onCellValueChanged
},
rowSelection: resolvedConfig?.multipleSelectable ? 'multiple' : 'single',
rowMultiSelectWithClick: resolvedConfig?.multipleSelectable
? resolvedConfig.rowMultiselectWithClick
: false,
...(resolvedConfig?.extraConfig ?? {})
})
}
let runnableComponent: RunnableComponent
export function recompute() {
runnableComponent?.runComponent()
}
</script>
{#each Object.keys(css ?? {}) as key (key)}
<ResolveStyle
{id}
{customCss}
{key}
bind:css={css[key]}
componentStyle={$app.css?.tablecomponent}
/>
{/each}
{#if Array.isArray(resolvedConfig.columnDefs) && resolvedConfig.columnDefs.every(isObject)}
<div
class={twMerge(
'border shadow-sm divide-y flex flex-col h-full',
css?.container?.class,
'wm-aggrid-container'
)}
style={containerHeight ? `height: ${containerHeight}px;` : css?.container?.style}
bind:clientHeight
bind:clientWidth
>
<div
on:pointerdown|stopPropagation={() => {
$selectedComponent = [id]
}}
style:height="{clientHeight}px"
style:width="{clientWidth}px"
class="ag-theme-alpine"
class:ag-theme-alpine-dark={$darkMode}
>
<div bind:this={eGui} style:height="100%" />
</div>
</div>
<div class="flex gap-1 absolute bottom-1 right-2 text-sm text-secondary"
>{firstRow}{'->'}{lastRow + 1} of {datasource?.rowCount} rows</div
>
{:else if resolvedConfig.columnDefs != undefined}
<Alert title="Parsing issues" type="error" size="xs">
The columnDefs should be an array of objects, received:
<pre class="overflow-auto">
{JSON.stringify(resolvedConfig.columnDefs)}
</pre>
</Alert>
{:else}
<Alert title="Parsing issues" type="error" size="xs">The columnDefs are undefined</Alert>
{/if}
frontend/src/lib/components/apps/components/display/table/AppCell.svelte
+79 -16
View File
@@ -1,31 +1,94 @@
<script lang="ts">
import Badge from '$lib/components/common/badge/Badge.svelte'
export let type: 'text' | 'badge' | 'link' = 'text'
export let value: any
import { createEventDispatcher, tick } from 'svelte'
import { writable } from 'svelte/store'
import { twMerge } from 'tailwind-merge'
type LinkObject = {
href: string
label: string
}
export let type: 'text' | 'badge' | 'link' = 'text'
export let value: any
export let width: number
let isEditable = writable(false)
let tempValue = value
const dispatch = createEventDispatcher()
function isLinkObject(value: any): value is LinkObject {
return value && typeof value === 'object' && 'href' in value && 'label' in value
}
async function toggleEdit() {
$isEditable = !$isEditable
if ($isEditable) {
await tick()
const input = document.getElementById('cell') as HTMLInputElement
input?.focus()
input?.setSelectionRange(0, 9999)
}
}
function handleInput(event: Event) {
tempValue = (event.target as HTMLInputElement).value
}
function saveEdit() {
value = tempValue
dispatch('update', {
value
})
toggleEdit()
}
</script>
{#if type === 'badge'}
<Badge>
{value}
</Badge>
{:else if type === 'link'}
{#if isLinkObject(value)}
<a href={value.href} class="underline" target="_blank">
{value.label}
</a>
<td
on:keydown
on:click
class={twMerge(
'p-4 whitespace-pre-wrap truncate text-xs text-primary',
$isEditable && 'bg-gray-100'
)}
style={'width: ' + width + 'px'}
>
{#if type === 'badge'}
<Badge>
{value}
</Badge>
{:else if type === 'link'}
{#if isLinkObject(value)}
<a href={value.href} class="underline" target="_blank">
{value.label}
</a>
{:else}
<a href={value} class="underline" target="_blank">{value}</a>
{/if}
{:else if $isEditable}
<input
type="text"
value={tempValue}
on:input={handleInput}
on:blur={saveEdit}
id="cell"
class="!appearance-none !bg-transparent !border-none !p-0 !m-0 leading-normal !text-xs"
style="outline: none; box-shadow: none; height: auto; resize: none;"
on:keypress={(e) => {
if (e.key === 'Enter') {
saveEdit()
}
}}
/>
{:else}
<a href={value} class="underline" target="_blank">{value}</a>
<!-- svelte-ignore a11y-no-static-element-interactions -->
<div on:dblclick={toggleEdit}>
{value}
</div>
{/if}
{:else}
{value}
{/if}
</td>
frontend/src/lib/components/apps/components/display/table/AppTable.svelte
+29 -14
View File
@@ -70,7 +70,7 @@
selectedRowIndex: 0,
selectedRow: undefined,
loading: false,
result: [],
result: [] as Record<string, any>[],
inputs: {},
search: '',
page: 1
@@ -289,6 +289,16 @@
}
$: $table && updateTable(resolvedConfig, searchValue)
function updateCellValue(rowIndex: number, columnIndex: number, newCellValue: string) {
if (result && rowIndex < result.length) {
const updatedRow = { ...result[rowIndex] }
const columnName = Object.keys(updatedRow)[columnIndex]
updatedRow[columnName] = newCellValue
result[rowIndex] = updatedRow
outputs?.result.set([result])
}
}
</script>
{#each Object.keys(components['tablecomponent'].initialData.configuration) as key (key)}
@@ -396,21 +406,20 @@
{#if cell?.column?.columnDef?.cell}
{@const context = cell?.getContext()}
{#if context}
<td
<AppCell
on:keydown={() => toggleRow(row)}
on:click={() => toggleRow(row)}
class="p-4 whitespace-pre-wrap truncate text-xs text-primary"
style={'width: ' + cell.column.getSize() + 'px'}
>
<AppCell
type={resolvedConfig.columnDefs?.find(
// TS types are wrong here
// @ts-ignore
(c) => c.field === cell.column.columnDef.accessorKey
)?.type ?? 'text'}
value={cell.getValue()}
/>
</td>
type={resolvedConfig.columnDefs?.find(
// TS types are wrong here
// @ts-ignore
(c) => c.field === cell.column.columnDef.accessorKey
)?.type ?? 'text'}
value={cell.getValue()}
width={cell.column.getSize()}
on:update={(event) => {
updateCellValue(rowIndex, index, event.detail.value)
}}
/>
{/if}
{/if}
{/each}
@@ -546,6 +555,7 @@
}}
{#if actionButton.type == 'buttoncomponent'}
<AppButton
noInitialize
extraKey={'idx' + rowIndex}
{render}
noWFull
@@ -562,6 +572,7 @@
/>
{:else if actionButton.type == 'checkboxcomponent'}
<AppCheckbox
noInitialize
extraKey={'idx' + rowIndex}
{render}
id={actionButton.id}
@@ -576,6 +587,7 @@
{:else if actionButton.type == 'selectcomponent'}
<div class="w-40">
<AppSelect
noInitialize
extraKey={'idx' + rowIndex}
{render}
id={actionButton.id}
@@ -591,6 +603,7 @@
{/if}
{:else if actionButton.type == 'buttoncomponent'}
<AppButton
noInitialize
extraKey={'idx' + rowIndex}
{render}
noWFull
@@ -606,6 +619,7 @@
/>
{:else if actionButton.type == 'checkboxcomponent'}
<AppCheckbox
noInitialize
extraKey={'idx' + rowIndex}
{render}
id={actionButton.id}
@@ -619,6 +633,7 @@
{:else if actionButton.type == 'selectcomponent'}
<div class="w-40">
<AppSelect
noInitialize
--font-size="10px"
extraKey={'idx' + rowIndex}
{render}
frontend/src/lib/components/apps/components/helpers/NonRunnableComponent.svelte
+4 -1
View File
@@ -11,6 +11,7 @@
export let result: any
export let render: boolean
export let hasChildrens: boolean
export let noInitialize
// Sync the result to the output
const { worldStore } = getContext<AppViewerContext>('AppViewerContext')
@@ -27,7 +28,9 @@
$: result != undefined && outputs && setOutput(result)
</script>
<InitializeComponent {id} />
{#if !noInitialize}
<InitializeComponent {id} />
{/if}
{#if componentInput.type !== 'runnable'}
<InputValue key="nonrunnable" {id} input={componentInput} bind:value={result} />
frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte
+54 -22
View File
@@ -51,6 +51,7 @@
export let errorHandledByComponent: boolean = false
export let hideRefreshButton: boolean = false
export let hasChildrens: boolean
export let allowConcurentRequests = false
const {
worldStore,
@@ -97,6 +98,7 @@
function setDebouncedExecute() {
executeTimeout && clearTimeout(executeTimeout)
executeTimeout = setTimeout(() => {
console.debug('debounce execute')
executeComponent(true)
}, 200)
}
@@ -132,7 +134,7 @@
const refreshEnabled =
autoRefresh && ((recomputeOnInputChanged ?? true) || refreshOn?.length > 0)
if (refreshEnabled && $initialized.initialized) {
// console.debug(`Refreshing ${id} because ${_src} (enabled)`)
console.debug(`Refreshing ${id} because ${_src} (enabled)`)
setDebouncedExecute()
}
}
@@ -216,11 +218,15 @@
return njobs
})
}
async function executeComponent(
noToast = false,
inlineScriptOverride?: InlineScript,
setRunnableJobEditorPanel?: boolean
) {
setRunnableJobEditorPanel?: boolean,
dynamicArgsOverride?: Record<string, any>,
callbacks?: Callbacks
): Promise<string | undefined> {
let jobId: string | undefined
console.debug(`Executing ${id}`)
if (iterContext && $iterContext.disabled) {
console.debug(`Skipping execution of ${id} because it is part of a disabled list`)
@@ -288,8 +294,8 @@
}
try {
const jobId = await resultJobLoader?.abstractRun(async () => {
const nonStaticRunnableInputs = {}
jobId = await resultJobLoader?.abstractRun(async () => {
const nonStaticRunnableInputs = dynamicArgsOverride ?? {}
const staticRunnableInputs = {}
for (const k of Object.keys(fields ?? {})) {
let field = fields[k]
@@ -338,31 +344,50 @@
addJob(uuid)
}
return uuid
})
}, callbacks)
if (setRunnableJobEditorPanel && editorContext) {
editorContext.runnableJobEditorPanel.update((p) => {
return {
...p,
jobs: { ...p.jobs, [id]: jobId }
jobs: { ...p.jobs, [id]: jobId as string }
}
})
}
return jobId
} catch (e) {
updateResult({ error: e.body ?? e.message })
let error = e.body ?? e.message
updateResult({ error })
$errorByComponent[id] = { error }
loading = false
}
}
type Callbacks = { done: (x: any[]) => void; cancel: () => void; error: () => void }
export async function runComponent() {
export async function runComponent(
noToast = false,
inlineScriptOverride?: InlineScript,
setRunnableJobEditorPanel?: boolean,
dynamicArgsOverride?: Record<string, any>,
callbacks?: Callbacks
): Promise<string | undefined> {
try {
if (cancellableRun) {
if (cancellableRun && !dynamicArgsOverride) {
await cancellableRun()
} else {
console.log('Run component')
executeComponent()
return await executeComponent(
noToast,
inlineScriptOverride,
setRunnableJobEditorPanel,
dynamicArgsOverride,
callbacks
)
}
} catch (e) {
updateResult({ error: e.body ?? e.message })
let error = e.body ?? e.message
updateResult({ error })
$errorByComponent[id] = { error }
}
}
@@ -453,7 +478,7 @@
jobId: string | undefined,
setRunnableJobEditorPanel?: boolean
) {
dispatch('done')
dispatch('resultSet')
const errors = getResultErrors(res)
if (errors) {
@@ -580,6 +605,7 @@
{/if}
<ResultJobLoader
{allowConcurentRequests}
{isEditor}
on:started={(e) => {
console.log('started', e.detail)
@@ -592,9 +618,11 @@
lastJobId = e.detail.id
setResult(e.detail.result, e.detail.id)
loading = false
dispatch('done', { id: e.detail.id, result: e.detail.result })
}}
on:cancel={(e) => {
let jobId = e.detail
console.debug('cancel', jobId)
let job = $jobsById[jobId]
if (job && job.created_at && !job.duration_ms) {
$jobsById[jobId] = {
@@ -603,6 +631,7 @@
duration_ms: Date.now() - (job.started_at ?? job.created_at)
}
}
dispatch('cancel', { id: e.detail })
}}
on:running={(e) => {
let jobId = e.detail
@@ -614,6 +643,7 @@
on:doneError={(e) => {
setResult({ error: e.detail.error }, e.detail.id)
loading = false
dispatch('doneError', { id: e.detail.id, result: e.detail.result })
}}
bind:this={resultJobLoader}
/>
@@ -652,19 +682,21 @@
<span slot="text">
<div class="bg-surface">
<Alert type="error" title="Error during execution">
<div class="flex flex-col gap-2">
<div class="flex flex-col gap-2 overflow-auto">
An error occured, please contact the app author.
{#if lastJobId && $errorByComponent[id].error}
{#if $errorByComponent?.[id]?.error}
<div class="font-bold">{$errorByComponent[id].error}</div>
{/if}
<a
href={`/run/${lastJobId}?workspace=${workspace}`}
class="font-semibold text-red-800 underline"
target="_blank"
>
Job id: {lastJobId}
</a>
{#if lastJobId}
<a
href={`/run/${lastJobId}?workspace=${workspace}`}
class="font-semibold text-red-800 underline"
target="_blank"
>
Job id: {lastJobId}
</a>
{/if}
</div>
</Alert>
</div>
frontend/src/lib/components/apps/components/helpers/RunnableWrapper.svelte
+11 -3
View File
@@ -10,6 +10,7 @@
import InitializeComponent from './InitializeComponent.svelte'
export let componentInput: AppInput | undefined
export let noInitialize = false
type SideEffectAction =
| {
@@ -76,6 +77,7 @@
export let refreshOnStart: boolean = false
export let errorHandledByComponent: boolean = false
export let hasChildrens: boolean = false
export let allowConcurentRequests = false
export function setArgs(value: any) {
runnableComponent?.setArgs(value)
@@ -208,10 +210,13 @@
</script>
{#if componentInput === undefined}
<InitializeComponent {id} />
{#if !noInitialize}
<InitializeComponent {id} />
{/if}
<slot />
{:else if componentInput.type === 'runnable' && isRunnableDefined(componentInput)}
<RunnableComponent
{allowConcurentRequests}
{refreshOnStart}
{extraKey}
{hasChildrens}
@@ -233,7 +238,10 @@
wrapperStyle={runnableStyle}
{render}
on:started
on:done={() => (initializing = false)}
on:done
on:doneError
on:cancel
on:resultSet={() => (initializing = false)}
on:success={() => handleSideEffect(true)}
on:handleError={(e) => handleSideEffect(false, e.detail)}
{outputs}
@@ -242,7 +250,7 @@
<slot />
</RunnableComponent>
{:else}
<NonRunnableComponent {hasChildrens} {render} bind:result {id} {componentInput}>
<NonRunnableComponent {noInitialize} {hasChildrens} {render} bind:result {id} {componentInput}>
<slot />
</NonRunnableComponent>
{/if}
frontend/src/lib/components/apps/components/inputs/AppCheckbox.svelte
+5 -1
View File
@@ -26,6 +26,7 @@
export let render: boolean
export let extraKey: string | undefined = undefined
export let preclickAction: (() => Promise<void>) | undefined = undefined
export let noInitialize = false
export let controls: { left: () => boolean; right: () => boolean | string } | undefined =
undefined
@@ -111,7 +112,10 @@
/>
{/each}
<InitializeComponent {id} />
{#if !noInitialize}
<InitializeComponent {id} />
{/if}
<AlignWrapper
{render}
{horizontalAlignment}
frontend/src/lib/components/apps/components/inputs/AppSelect.svelte
+4 -1
View File
@@ -30,6 +30,7 @@
export let extraKey: string | undefined = undefined
export let preclickAction: (() => Promise<void>) | undefined = undefined
export let recomputeIds: string[] | undefined = undefined
export let noInitialize = false
export let controls: { left: () => boolean; right: () => boolean | string } | undefined =
undefined
@@ -189,7 +190,9 @@
/>
{/each}
<InitializeComponent {id} />
{#if !noInitialize}
<InitializeComponent {id} />
{/if}
<AlignWrapper {render} {verticalAlignment}>
<div
frontend/src/lib/components/apps/editor/AppEditorHeader.svelte
+196 -134
View File
@@ -83,6 +83,15 @@
import type DiffDrawer from '$lib/components/DiffDrawer.svelte'
import { cloneDeep } from 'lodash'
import AppReportsDrawer from './AppReportsDrawer.svelte'
import HighlightCode from '$lib/components/HighlightCode.svelte'
import {
type ColumnDef,
getCountPostgresql,
createPostgresInput,
createPostgresInsert,
createUpdatePostgresInput,
getPrimaryKeys
} from '../components/display/dbtable/utils'
async function hash(message) {
try {
@@ -187,6 +196,47 @@
if (c.type === 'menucomponent') {
r.push(...c.menuItems.map((x) => ({ input: x.componentInput, id: x.id })))
}
if (c.type === 'dbexplorercomponent') {
let nr: { id: string; input: AppInput }[] = []
let config = c.configuration as any
let pg = config?.type?.configuration?.postgresql
if (pg) {
const { table, resource } = pg
const tableValue = table.value
const resourceValue = resource.value
const columnDefs = (c.configuration.columnDefs as any).value as ColumnDef[]
const whereClause = (c.configuration.whereClause as any).value as unknown as
| string
| undefined
console.log(columnDefs)
if (tableValue && resourceValue && columnDefs) {
r.push({
input: createPostgresInput(resourceValue, tableValue, columnDefs, whereClause),
id: x.id
})
r.push({
input: getCountPostgresql(resourceValue, tableValue),
id: x.id + '_count'
})
r.push({
input: createPostgresInsert(tableValue, columnDefs, resourceValue),
id: x.id + '_insert'
})
let primaryColumns = getPrimaryKeys(columnDefs)
let columns = columnDefs?.filter((x) => primaryColumns.includes(x.field))
columnDefs
.filter((col) => col.editable || config.allEditable.value)
.forEach((column) => {
r.push({
input: createUpdatePostgresInput(resourceValue, tableValue, column, columns),
id: x.id + '_update'
})
})
}
}
r.push(...nr)
}
return r
.filter((x) => x.input)
.map(async (o) => {
@@ -213,13 +263,16 @@
): Promise<[string, Record<string, any>] | undefined> {
const staticInputs = collectStaticFields(fields)
if (runnable?.type == 'runnableByName') {
console.log(runnable.inlineScript?.content)
let hex = await hash(runnable.inlineScript?.content)
console.log('hex', hex, id)
return [`${id}:rawscript/${hex}`, staticInputs]
} else if (runnable?.type == 'runnableByPath') {
let prefix = runnable.runType !== 'hubscript' ? runnable.runType : 'script'
return [`${id}:${prefix}/${runnable.path}`, staticInputs]
}
}
async function createApp(path: string) {
await computeTriggerables()
try {
@@ -657,6 +710,7 @@
type="text"
placeholder="App summary"
class="text-sm w-full font-semibold"
on:keydown|stopPropagation
bind:value={$summary}
on:keyup={() => {
if (appPath == '' && $summary?.length > 0 && !dirtyPath) {
@@ -713,6 +767,7 @@
placeholder="App summary"
class="text-sm w-full"
bind:value={$summary}
on:keydown|stopPropagation
on:keyup={() => {
if (appPath == '' && $summary?.length > 0 && !dirtyPath) {
path?.setName(
@@ -731,7 +786,6 @@
<div class="w-full pt-2">
<!-- svelte-ignore a11y-autofocus -->
<input
autofocus
type="text"
placeholder="Optional deployment message"
class="text-sm w-full"
@@ -928,123 +982,108 @@
</PanelSection>
</Pane>
<Pane size={75}>
<Tabs bind:selected={rightColumnSelect}>
<Tab value="timeline"><span class="font-semibold text-md">Timeline</span></Tab>
<Tab value="detail"><span class="font-semibold">Details</span></Tab>
</Tabs>
{#if rightColumnSelect == 'timeline'}
<div class="p-2">
<AppTimeline />
</div>
{:else if rightColumnSelect == 'detail'}
<div class="h-full flex flex-col w-full overflow-auto">
{#if selectedJobId}
{#if selectedJobId?.includes('Frontend')}
{@const jobResult = $jobsById[selectedJobId]}
{#if jobResult?.error !== undefined}
<Splitpanes horizontal class="grow border w-full">
<Pane size={10} minSize={10}>
<LogViewer
content={`Logs are avaiable in the browser console directly`}
isLoading={false}
tag={undefined}
/>
</Pane>
<Pane size={90} minSize={10} class="text-sm text-secondary">
<div class="relative h-full px-2">
<DisplayResult
result={{
error: { name: 'Frontend execution error', message: jobResult.error }
}}
/>
</div>
</Pane>
</Splitpanes>
{:else if jobResult !== undefined}
<Splitpanes horizontal class="grow border w-full">
<Pane size={10} minSize={10}>
<LogViewer
content={`Logs are avaiable in the browser console directly`}
isLoading={false}
tag={undefined}
/>
</Pane>
<Pane size={90} minSize={10} class="text-sm text-secondary">
<div class="relative h-full px-2">
<DisplayResult
workspaceId={$workspaceStore}
jobId={selectedJobId}
result={jobResult.result}
/>
</div>
</Pane>
</Splitpanes>
{:else}
<Loader2 class="animate-spin" />
{/if}
{:else}
<div class="flex flex-col h-full w-full gap-4 mb-4">
{#if job?.['running']}
<div class="flex flex-row-reverse w-full">
<Button
color="red"
variant="border"
on:click={() => testJobLoader?.cancelJob()}
>
<Loader2 size={14} class="animate-spin mr-2" />
Cancel
</Button>
</div>
{/if}
{#if job?.args}
<div class="p-2">
<JobArgs args={job?.args} />
</div>
{/if}
{#if job?.job_kind !== 'flow' && job?.job_kind !== 'flowpreview'}
{@const jobResult = $jobsById[selectedJobId]}
<div class="w-full h-full flex flex-col">
<Tabs bind:selected={rightColumnSelect}>
<Tab value="timeline"><span class="font-semibold text-md">Timeline</span></Tab>
<Tab value="detail"><span class="font-semibold">Details</span></Tab>
</Tabs>
{#if rightColumnSelect == 'timeline'}
<div class="p-2">
<AppTimeline />
</div>
{:else if rightColumnSelect == 'detail'}
<div class="grow flex flex-col w-full overflow-auto">
{#if selectedJobId}
{#if selectedJobId?.includes('Frontend')}
{@const jobResult = $jobsById[selectedJobId]}
{#if jobResult?.error !== undefined}
<Splitpanes horizontal class="grow border w-full">
<Pane size={50} minSize={10}>
<Pane size={10} minSize={10}>
<LogViewer
duration={job?.['duration_ms']}
jobId={job?.id}
content={job?.logs}
isLoading={testIsLoading}
tag={job?.tag}
content={`Logs are avaiable in the browser console directly`}
isLoading={false}
tag={undefined}
/>
</Pane>
<Pane size={50} minSize={10} class="text-sm text-secondary">
{#if job != undefined && 'result' in job && job.result != undefined}
<div class="relative h-full px-2">
<DisplayResult
workspaceId={$workspaceStore}
jobId={selectedJobId}
result={job.result}
/></div
>
{:else if testIsLoading}
<div class="p-2"><Loader2 class="animate-spin" /> </div>
{:else if job != undefined && 'result' in job && job?.['result'] == undefined}
<div class="p-2 text-tertiary">Result is undefined</div>
{:else}
<div class="p-2 text-tertiary">
<Loader2 size={14} class="animate-spin mr-2" />
</div>
{/if}
<Pane size={90} minSize={10} class="text-sm text-secondary">
<div class="relative h-full px-2">
<DisplayResult
result={{
error: { name: 'Frontend execution error', message: jobResult.error }
}}
/>
</div>
</Pane>
{#if jobResult?.transformer}
<Pane size={50} minSize={10} class="text-sm text-secondary p-2">
<div class="font-bold">Transformer results</div>
</Splitpanes>
{:else if jobResult !== undefined}
<Splitpanes horizontal class="grow border w-full">
<Pane size={10} minSize={10}>
<LogViewer
content={`Logs are avaiable in the browser console directly`}
isLoading={false}
tag={undefined}
/>
</Pane>
<Pane size={90} minSize={10} class="text-sm text-secondary">
<div class="relative h-full px-2">
<DisplayResult
workspaceId={$workspaceStore}
jobId={selectedJobId}
result={jobResult.result}
/>
</div>
</Pane>
</Splitpanes>
{:else}
<Loader2 class="animate-spin" />
{/if}
{:else}
<div class="flex flex-col h-full w-full mb-4">
{#if job?.['running']}
<div class="flex flex-row-reverse w-full">
<Button
color="red"
variant="border"
on:click={() => testJobLoader?.cancelJob()}
>
<Loader2 size={14} class="animate-spin mr-2" />
Cancel
</Button>
</div>
{/if}
{#if job?.args}
<div class="p-2">
<JobArgs args={job?.args} />
</div>
{/if}
{#if job?.raw_code}
<div class="pb-2 pl-2 pr-2 w-full overflow-auto h-full max-h-[80px]">
<HighlightCode language={job?.language} code={job?.raw_code} />
</div>
{/if}
{#if job?.job_kind !== 'flow' && job?.job_kind !== 'flowpreview'}
{@const jobResult = $jobsById[selectedJobId]}
<Splitpanes horizontal class="grow border w-full">
<Pane size={50} minSize={10}>
<LogViewer
duration={job?.['duration_ms']}
jobId={job?.id}
content={job?.logs}
isLoading={testIsLoading}
tag={job?.tag}
/>
</Pane>
<Pane size={50} minSize={10} class="text-sm text-secondary">
{#if job != undefined && 'result' in job && job.result != undefined}
<div class="relative h-full px-2">
<DisplayResult
workspaceId={$workspaceStore}
jobId={selectedJobId}
result={jobResult?.transformer}
/>
</div>
result={job.result}
/></div
>
{:else if testIsLoading}
<div class="p-2"><Loader2 class="animate-spin" /> </div>
{:else if job != undefined && 'result' in job && job?.['result'] == undefined}
@@ -1055,27 +1094,49 @@
</div>
{/if}
</Pane>
{/if}
</Splitpanes>
{:else}
<div class="mt-10" />
<FlowProgressBar {job} class="py-4" />
<div class="w-full mt-10 mb-20">
<FlowStatusViewer
jobId={job.id}
on:jobsLoaded={({ detail }) => {
job = detail
}}
/>
</div>
{/if}
</div>
{#if jobResult?.transformer}
<Pane size={50} minSize={10} class="text-sm text-secondary p-2">
<div class="font-bold">Transformer results</div>
{#if job != undefined && 'result' in job && job.result != undefined}
<div class="relative h-full px-2">
<DisplayResult
workspaceId={$workspaceStore}
jobId={selectedJobId}
result={jobResult?.transformer}
/>
</div>
{:else if testIsLoading}
<div class="p-2"><Loader2 class="animate-spin" /> </div>
{:else if job != undefined && 'result' in job && job?.['result'] == undefined}
<div class="p-2 text-tertiary">Result is undefined</div>
{:else}
<div class="p-2 text-tertiary">
<Loader2 size={14} class="animate-spin mr-2" />
</div>
{/if}
</Pane>
{/if}
</Splitpanes>
{:else}
<div class="mt-10" />
<FlowProgressBar {job} class="py-4" />
<div class="w-full mt-10 mb-20">
<FlowStatusViewer
jobId={job.id}
on:jobsLoaded={({ detail }) => {
job = detail
}}
/>
</div>
{/if}
</div>
{/if}
{:else}
<div class="text-sm p-2 text-tertiary">Select a job to see its details</div>
{/if}
{:else}
<div class="text-sm p-2 text-tertiary">Select a job to see its details</div>
{/if}
</div>
{/if}
</div>
{/if}
</div>
</Pane>
</Splitpanes>
<svelte:fragment slot="actions">
@@ -1121,6 +1182,7 @@
placeholder="App summary"
class="text-sm w-full font-semibold"
bind:value={$summary}
on:keydown|stopPropagation
/>
</div>
<div class="flex gap-4 items-center justify-center">
@@ -1247,13 +1309,13 @@
size="xs"
dropdownItems={appPath != ''
? () => [
{
label: 'Fork',
onClick: () => {
window.open(`/apps/add?template=${appPath}`)
{
label: 'Fork',
onClick: () => {
window.open(`/apps/add?template=${appPath}`)
}
}
}
]
]
: undefined}
>
Deploy
frontend/src/lib/components/apps/editor/RecomputeAllComponents.svelte
+4 -1
View File
@@ -63,6 +63,7 @@
}
loading = true
console.log('refresh all')
const promises = Object.keys($runnableComponents)
.flatMap((id) => {
if (
@@ -72,6 +73,7 @@
return
}
console.log('refresh start', id)
return $runnableComponents?.[id]?.cb?.map((f) =>
f().then(() => console.log('refreshed', id))
)
@@ -106,7 +108,8 @@
]
</script>
<!-- {$initialized.initializedComponents?.join(', ')} -->
<!-- {$initialized.initializedComponents?.join(', ')}
{allItems($app.grid, $app.subgrids).length + $app.hiddenInlineScripts.length} -->
<!-- {allItems($app.grid, $app.subgrids).length + $app.hiddenInlineScripts.length}
{$initialized.initializedComponents}
{allItems($app.grid, $app.subgrids)
frontend/src/lib/components/apps/editor/appUtils.ts
+30 -17
View File
@@ -597,6 +597,35 @@ export function initOutput<I extends Record<string, any>>(
) as Outputtable<I>
}
export type InitConfig<
T extends Record<
string,
| StaticAppInput
| EvalAppInput
| {
type: 'oneOf'
selected: string
configuration: Record<string, Record<string, StaticAppInput | EvalAppInput>>
}
>
> = {
[Property in keyof T]: T[Property] extends StaticAppInput
? T[Property]['value'] | undefined
: T[Property] extends { type: 'oneOf' }
? {
type: 'oneOf'
selected: keyof T[Property]['configuration']
configuration: {
[Choice in keyof T[Property]['configuration']]: {
[IT in keyof T[Property]['configuration'][Choice]]: T[Property]['configuration'][Choice][IT] extends StaticAppInput
? T[Property]['configuration'][Choice][IT]['value'] | undefined
: undefined
}
}
}
: undefined
}
export function initConfig<
T extends Record<
string,
@@ -620,23 +649,7 @@ export function initConfig<
}
| any
>
): {
[Property in keyof T]: T[Property] extends StaticAppInput
? T[Property]['value'] | undefined
: T[Property] extends { type: 'oneOf' }
? {
type: 'oneOf'
selected: keyof T[Property]['configuration']
configuration: {
[Choice in keyof T[Property]['configuration']]: {
[IT in keyof T[Property]['configuration'][Choice]]: T[Property]['configuration'][Choice][IT] extends StaticAppInput
? T[Property]['configuration'][Choice][IT]['value'] | undefined
: undefined
}
}
}
: undefined
} {
): InitConfig<T> {
return JSON.parse(
JSON.stringify(
Object.fromEntries(
frontend/src/lib/components/apps/editor/component/Component.svelte
+9
View File
@@ -66,6 +66,7 @@
import AppStatCard from '../../components/display/AppStatCard.svelte'
import AppMenu from '../../components/display/AppMenu.svelte'
import AppDecisionTree from '../../components/layout/AppDecisionTree.svelte'
import AppDbExplorer from '../../components/display/dbtable/AppDbExplorer.svelte'
export let component: AppComponent
export let selected: boolean
@@ -305,6 +306,14 @@
actionButtons={component.actionButtons}
{render}
/>
{:else if component.type === 'dbexplorercomponent'}
<AppDbExplorer
configuration={component.configuration}
id={component.id}
customCss={component.customCss}
bind:initializing
{render}
/>
{:else if component.type === 'aggridcomponent'}
<AppAggridTable
id={component.id}
frontend/src/lib/components/apps/editor/component/components.ts
+107 -4
View File
@@ -43,7 +43,8 @@ import {
Heading1,
FileBarChart,
Menu,
Network
Network,
Database
} from 'lucide-svelte'
import type {
Aligned,
@@ -170,6 +171,10 @@ export type MenuComponent = BaseComponent<'menucomponent'> & {
menuItems: (BaseAppComponent & ButtonComponent & GridItem)[]
}
export type DBExplorerComponent = BaseComponent<'dbexplorercomponent'> & {
columns: RichConfiguration
}
export type DecisionTreeNode = {
id: string
label: string
@@ -185,6 +190,7 @@ export type DecisionTreeComponent = BaseComponent<'decisiontreecomponent'> & {
}
export type TypedComponent =
| DBExplorerComponent
| DisplayComponent
| LogComponent
| JobIdLogComponent
@@ -569,7 +575,7 @@ const aggridcomponentconst = {
type: 'static',
fieldType: 'boolean',
value: false,
hide: true,
tooltip: 'Configure all columns as Editable by users'
},
multipleSelectable: {
@@ -589,7 +595,7 @@ const aggridcomponentconst = {
pagination: {
type: 'static',
fieldType: 'boolean',
value: false
value: false as boolean | undefined
},
selectFirstRowByDefault: {
type: 'static',
@@ -2205,7 +2211,8 @@ This is a paragraph.
selectOptions: selectOptions.animationTimingFunctionOptions,
value: 'linear',
tooltip: 'Sets how an animation progresses through the duration of each cycle, see https://developer.mozilla.org/en-US/docs/Web/CSS/animation-timing-function',
tooltip:
'Sets how an animation progresses through the duration of each cycle, see https://developer.mozilla.org/en-US/docs/Web/CSS/animation-timing-function'
}
},
componentInput: {
@@ -2947,6 +2954,102 @@ This is a paragraph.
}
] as DecisionTreeNode[]
}
},
dbexplorercomponent: {
name: 'Database Studio',
icon: Database,
documentationLink: `${documentationBaseUrl}/dbexplorer`,
dims: '2:8-6:8' as AppComponentDimensions,
customCss: {
container: { class: '', style: '' }
},
initialData: {
configuration: {
type: {
type: 'oneOf',
selected: 'postgresql',
labels: {
postgresql: 'PostgreSQL',
msql: 'MySQL'
},
configuration: {
postgresql: {
resource: {
type: 'static',
fieldType: 'resource',
value: ''
} as StaticAppInput,
table: {
fieldType: 'select',
subfieldType: 'db-table',
type: 'static',
selectOptions: [],
value: undefined
}
}
}
} as const,
columnDefs: {
type: 'static',
fieldType: 'array',
subFieldType: 'db-explorer',
value: []
} as StaticAppInput,
whereClause: {
type: 'static',
fieldType: 'text',
value: ''
},
flex: {
type: 'static',
fieldType: 'boolean',
value: true,
tooltip: 'default col flex is 1 (see ag-grid docs)'
},
allEditable: {
type: 'static',
fieldType: 'boolean',
value: false,
hide: true,
tooltip: 'Configure all columns as Editable by users'
},
allowDelete: {
type: 'static',
fieldType: 'boolean',
value: false,
hide: true,
tooltip: 'Allow deleting rows'
},
multipleSelectable: {
type: 'static',
fieldType: 'boolean',
value: false,
tooltip: 'Make multiple rows selectable at once'
},
rowMultiselectWithClick: {
type: 'static',
fieldType: 'boolean',
value: true,
tooltip: 'If multiple selectable, allow multiselect with click'
},
selectFirstRowByDefault: {
type: 'static',
fieldType: 'boolean',
value: true as boolean,
tooltip: 'Select the first row by default on start'
},
extraConfig: {
type: 'static',
fieldType: 'object',
value: {},
tooltip: 'any configuration that can be passed to ag-grid top level'
}
},
componentInput: undefined
}
}
} as const
frontend/src/lib/components/apps/editor/component/sets.ts
+1 -1
View File
@@ -72,7 +72,7 @@ const display: ComponentSet = {
const tables: ComponentSet = {
title: 'Tables',
components: ['tablecomponent', 'aggridcomponent', 'aggridcomponentee']
components: ['tablecomponent', 'aggridcomponent', 'aggridcomponentee', 'dbexplorercomponent']
} as const
const charts: ComponentSet = {
frontend/src/lib/components/apps/editor/componentsPanel/quickStyleProperties.ts
+2 -1
View File
@@ -751,5 +751,6 @@ export const quickStyleProperties: Record<
},
decisiontreecomponent: {
container: containerDefaultProps
}
},
dbexplorercomponent: {}
}
frontend/src/lib/components/apps/editor/settingsPanel/ArrayStaticInputEditor.svelte
+80 -35
View File
@@ -10,13 +10,12 @@
import Toggle from '$lib/components/Toggle.svelte'
import QuickAddColumn from './QuickAddColumn.svelte'
const flipDurationMs = 200
export let componentInput: StaticInput<any[]>
export let subFieldType: InputType | undefined = undefined
export let selectOptions: StaticOptions['selectOptions'] | undefined = undefined
const dispatch = createEventDispatcher()
const flipDurationMs = 200
function addElementByType() {
if (!Array.isArray(componentInput.value)) {
@@ -82,10 +81,13 @@
componentInput = componentInput
if (componentInput.value) {
items.push({
value: componentInput.value[componentInput.value.length - 1],
id: generateRandomString()
})
let value = componentInput.value[componentInput.value.length - 1]
if (value) {
items.push({
value,
id: generateRandomString()
})
}
}
}
@@ -140,16 +142,16 @@
if ((e.key === 'Enter' || e.key === ' ') && dragDisabled) dragDisabled = false
}
let items = (Array.isArray(componentInput.value) ? componentInput.value : []).map(
(item, index) => {
let items = (Array.isArray(componentInput.value) ? componentInput.value : [])
.filter((x) => x != undefined)
.map((item, index) => {
return { value: item, id: generateRandomString() }
}
)
})
$: items != undefined && handleItemsChange()
function handleItemsChange() {
componentInput.value = items.map((item) => item.value)
componentInput.value = items.map((item) => item.value).filter((item) => item != undefined)
}
let raw: boolean = false
@@ -205,41 +207,84 @@
>
<GripVertical size={16} />
</div>
<button
class="z-10 rounded-full p-1 duration-200 hover:bg-gray-200"
aria-label="Remove item"
on:click|preventDefault|stopPropagation={() => deleteElementByType(index)}
>
<X size={14} />
</button>
{#if subFieldType !== 'db-explorer'}
<button
class="z-10 rounded-full p-1 duration-200 hover:bg-gray-200"
aria-label="Remove item"
on:click|preventDefault|stopPropagation={() => deleteElementByType(index)}
>
<X size={14} />
</button>
{/if}
</div>
</div>
</div>
{/each}
</section>
{/if}
<Button size="xs" color="light" startIcon={{ icon: Plus }} on:click={() => addElementByType()}>
Add
</Button>
{#if subFieldType === 'table-column' || subFieldType == 'ag-grid'}
<QuickAddColumn
{#if subFieldType !== 'db-explorer'}
<Button size="xs" color="light" startIcon={{ icon: Plus }} on:click={() => addElementByType()}>
Add
</Button>
{#if subFieldType === 'table-column' || subFieldType == 'ag-grid'}
<QuickAddColumn
columns={componentInput.value?.map((item) => item.field)}
on:add={({ detail }) => {
if (!componentInput.value) componentInput.value = []
if (subFieldType === 'table-column') {
componentInput.value.push({ field: detail, headerName: detail, type: 'text' })
} else if (subFieldType === 'ag-grid') {
componentInput.value.push({ field: detail, headerName: detail, flex: 1 })
}
componentInput = componentInput
if (componentInput.value) {
let value = componentInput.value[componentInput.value.length - 1]
if (value) {
items.push({
value,
id: generateRandomString()
})
}
}
}}
/>
{/if}
{/if}
<!-- {#if subFieldType === 'db-explorer'}
<SynchronizeColumns
columns={componentInput.value?.map((item) => item.field)}
on:add={({ detail }) => {
if (!componentInput.value) componentInput.value = []
if (subFieldType === 'table-column') {
componentInput.value.push({ field: detail, headerName: detail, type: 'text' })
} else if (subFieldType === 'ag-grid') {
componentInput.value.push({ field: detail, headerName: detail, flex: 1 })
if (!Array.isArray(detail)) {
return
}
componentInput = componentInput
if (componentInput.value) {
items.push({
value: componentInput.value[componentInput.value.length - 1],
id: generateRandomString()
})
if (detail.length === 0) {
return
}
componentInput.value = []
items = []
detail.forEach((col) => {
if (!componentInput.value) componentInput.value = []
if (subFieldType === 'table-column') {
componentInput.value.push({ field: col, headerName: col, type: 'text' })
} else if (subFieldType === 'ag-grid') {
componentInput.value.push({ field: col, headerName: col, flex: 1 })
} else if (subFieldType === 'db-explorer') {
componentInput.value.push({ field: col, headerName: col, flex: 1 })
}
componentInput = componentInput
if (componentInput.value) {
items.push({
value: componentInput.value[componentInput.value.length - 1],
id: generateRandomString()
})
}
})
}}
/>
{/if}
{/if} -->
</div>
frontend/src/lib/components/apps/editor/settingsPanel/ComponentPanel.svelte
-1
View File
@@ -373,7 +373,6 @@
{ccomponents[component.type].name} has no configuration
</div>
{/if}
{#if (`recomputeIds` in componentSettings.item.data && Array.isArray(componentSettings.item.data.recomputeIds)) || componentSettings.item.data.type === 'buttoncomponent' || componentSettings.item.data.type === 'formcomponent' || componentSettings.item.data.type === 'formbuttoncomponent' || componentSettings.item.data.type === 'checkboxcomponent'}
<Recompute
bind:recomputeIds={componentSettings.item.data.recomputeIds}
frontend/src/lib/components/apps/editor/settingsPanel/DbExplorerHelpers.svelte
+13
View File
@@ -0,0 +1,13 @@
<script lang="ts">
import PanelSection from './common/PanelSection.svelte'
import { Button } from '$lib/components/common'
import { RefreshCcw } from 'lucide-svelte'
</script>
<PanelSection title={'Helpers'}>
<div>
<Button size="xs" startIcon={{ icon: RefreshCcw }} color="dark">
Sync columns definitions with table
</Button>
</div>
</PanelSection>
frontend/src/lib/components/apps/editor/settingsPanel/GridTab.svelte
+1
View File
@@ -182,6 +182,7 @@
<!-- svelte-ignore a11y-no-noninteractive-tabindex -->
<!-- svelte-ignore a11y-no-static-element-interactions -->
<div
tabindex={dragDisabled ? 0 : -1}
class="w-4 h-4"

Some files were not shown because too many files have changed in this diff Show More