diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
index 534b1c8bf3..c95af782e0 100644
--- a/.github/workflows/docker-image.yml
+++ b/.github/workflows/docker-image.yml
@@ -200,25 +200,25 @@ jobs:
- run: |
# pulling docker image with desired arch so that actions-docker-extract doesn't do it
- docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
- docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+ docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
+ docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
- run: |
# Checks the image is in docker prior to running actions-docker-extract. It fails if not
# Also useful to visually check that the arch is the right opencontainers
- docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
- docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+ docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
+ docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
- uses: shrink/actions-docker-extract@v3
id: extract
with:
- image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+ image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
path: "/usr/src/app/windmill"
- uses: shrink/actions-docker-extract@v3
id: extract-ee
with:
- image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+ image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
path: "/usr/src/app/windmill"
- name: Rename binary with corresponding architecture
@@ -244,25 +244,25 @@ jobs:
- run: |
# pulling docker image with desired arch so that actions-docker-extract doesn't do it
- docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
- docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+ docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
+ docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
- run: |
# Checks the image is in docker prior to running actions-docker-extract. It fails if not
# Also useful to visually check that the arch is the right opencontainers
- docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
- docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+ docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
+ docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
- uses: shrink/actions-docker-extract@v3
id: extract
with:
- image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+ image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
path: "/usr/src/app/windmill"
- uses: shrink/actions-docker-extract@v3
id: extract-ee
with:
- image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+ image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
path: "/usr/src/app/windmill"
- name: Rename binary with corresponding architecture
@@ -307,7 +307,7 @@ jobs:
git_hash=$(git rev-parse --short "$GITHUB_SHA")
docker buildx imagetools create \
--tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${git_hash:0:7} \
- ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+ ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
echo "GIT_HASH=${git_hash:0:7}" >> "$GITHUB_OUTPUT"
- uses: shrink/actions-docker-extract@v3
@@ -349,7 +349,6 @@ jobs:
path: |
integration_tests/logs
-
tag_latest:
runs-on: ubicloud
needs: [run_integration_test]
@@ -365,10 +364,8 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}
- name: Tag main and latest
run: |
- docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
- docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
- docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main
- docker push -a ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+ docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+ docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main
tag_latest_ee:
runs-on: ubicloud
@@ -384,8 +381,6 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Tag main and latest for ee
- run: |
- docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
- docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
- docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main
- docker push -a ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
+ run: |
+ docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+ docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e83a5694ef..3038b1cdae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,86 @@
# Changelog
+## [1.236.2](https://github.com/windmill-labs/windmill/compare/v1.236.1...v1.236.2) (2024-01-08)
+
+
+### Bug Fixes
+
+* handle better non object result for app result loader ([269afe4](https://github.com/windmill-labs/windmill/commit/269afe41f7e02da8d3a364e649e5fef00847d24a))
+
+## [1.236.1](https://github.com/windmill-labs/windmill/compare/v1.236.0...v1.236.1) (2024-01-08)
+
+
+### Bug Fixes
+
+* fix app initialization ([6b075e9](https://github.com/windmill-labs/windmill/commit/6b075e92ef4e312e6a4ce582dc7c4d88456b2f5d))
+
+## [1.236.0](https://github.com/windmill-labs/windmill/compare/v1.235.1...v1.236.0) (2024-01-08)
+
+
+### Features
+
+* improve git sync and recompute dependents when common python script is redeployed ([#2967](https://github.com/windmill-labs/windmill/issues/2967)) ([9449381](https://github.com/windmill-labs/windmill/commit/94493812ac0030cc6bba468d8ee24a14661716a8))
+
+
+### Bug Fixes
+
+* reduce stack size needed for workflow transitions ([4384617](https://github.com/windmill-labs/windmill/commit/4384617f1ec4c27de88ef6c895d6ef060a79f6bd))
+
+## [1.235.1](https://github.com/windmill-labs/windmill/compare/v1.235.0...v1.235.1) (2024-01-07)
+
+
+### Bug Fixes
+
+* fix resultjobloader ([2e02300](https://github.com/windmill-labs/windmill/commit/2e023003da8bea9efa86c0fbc52bf9723c73907d))
+
+## [1.235.0](https://github.com/windmill-labs/windmill/compare/v1.234.1...v1.235.0) (2024-01-06)
+
+
+### Features
+
+* **frontend:** DB Explorer ([#2892](https://github.com/windmill-labs/windmill/issues/2892)) ([fffc533](https://github.com/windmill-labs/windmill/commit/fffc5338ce76fda9a68b19f03cefe199cc09a905))
+
+## [1.234.1](https://github.com/windmill-labs/windmill/compare/v1.234.0...v1.234.1) (2024-01-05)
+
+
+### Bug Fixes
+
+* fix double view runs button ([9efcb53](https://github.com/windmill-labs/windmill/commit/9efcb53bcd903bf756c4beb92a5c7634e8e016ad))
+* fix typo "Wourker" -> "Worker" ([4385edc](https://github.com/windmill-labs/windmill/commit/4385edc6ffdb1b2e4544347928b4cf08faf1eaa1))
+
+## [1.234.0](https://github.com/windmill-labs/windmill/compare/v1.233.0...v1.234.0) (2024-01-05)
+
+
+### Features
+
+* Detailed job memory footprint on job run page ([#2956](https://github.com/windmill-labs/windmill/issues/2956)) ([0710ce6](https://github.com/windmill-labs/windmill/commit/0710ce6ef25d8f2344ac1350e96aa906f16d1be1))
+* git sync can now push commits to individual branches ([#2959](https://github.com/windmill-labs/windmill/issues/2959)) ([fb5cd6a](https://github.com/windmill-labs/windmill/commit/fb5cd6a4298c88e26481b76258bf5b09a11cf4b0))
+
+
+### Bug Fixes
+
+* Deleting workspace delete all workspace related values in DB ([#2961](https://github.com/windmill-labs/windmill/issues/2961)) ([c05d119](https://github.com/windmill-labs/windmill/commit/c05d119ae5dfb6f1bae88fc9d53b48a707cd2f15))
+
+## [1.233.0](https://github.com/windmill-labs/windmill/compare/v1.232.0...v1.233.0) (2024-01-03)
+
+
+### Features
+
+* **frontend:** add the view runs buttons for operators ([#2932](https://github.com/windmill-labs/windmill/issues/2932)) ([821d2d7](https://github.com/windmill-labs/windmill/commit/821d2d7ea15e828dbb3683c85025dbc09c7e8dc7))
+* Job caching now works with S3 files ([#2934](https://github.com/windmill-labs/windmill/issues/2934)) ([b44618f](https://github.com/windmill-labs/windmill/commit/b44618f35b6fe4558a5cd9fddec1e82216a2cf5e))
+* PIP_LOCAL_DEPENDENCIES now accepts regex and is configurable from the UI ([#2940](https://github.com/windmill-labs/windmill/issues/2940)) ([e0140f1](https://github.com/windmill-labs/windmill/commit/e0140f175d371e239f315613a537820b9a6ad25f))
+
+
+### Bug Fixes
+
+* allow extraConfig for agGrid to set onGridReady ([74fc2d3](https://github.com/windmill-labs/windmill/commit/74fc2d3f4397d976fec326893f8196008cb8b332))
+* decision tree fix wording ([#2941](https://github.com/windmill-labs/windmill/issues/2941)) ([0299c65](https://github.com/windmill-labs/windmill/commit/0299c656c368876581dc228c76513f1171ffd682))
+* decision tree nits ([#2936](https://github.com/windmill-labs/windmill/issues/2936)) ([787017f](https://github.com/windmill-labs/windmill/commit/787017f53c8227e1246b09e412e92316977937ae))
+* **frontend:** handle case where large json has a file and a filename ([#2951](https://github.com/windmill-labs/windmill/issues/2951)) ([2494171](https://github.com/windmill-labs/windmill/commit/2494171b9cb3d1857235b6558d276451d5550560))
+* persist state for agGrid ([c085f5f](https://github.com/windmill-labs/windmill/commit/c085f5fa7ec598d0abf6aad98de0e5c7b1c92985))
+* Tag all docker images with latest and main tags ([#2953](https://github.com/windmill-labs/windmill/issues/2953)) ([9655718](https://github.com/windmill-labs/windmill/commit/965571853ca253b130bcc4473190b6c9c98f44e1))
+
## [1.232.0](https://github.com/windmill-labs/windmill/compare/v1.231.0...v1.232.0) (2023-12-27)
diff --git a/README.md b/README.md
index 79767d2cb6..c2c4a65b53 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
-Open-source developer infrastructure for internal tools (APIs, background jobs, workflows and UIs). Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps.
+Open-source developer infrastructure for internal tools (APIs, background jobs, workflows and UIs). Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs and custom UIs to trigger workflows and scripts as internal apps.
Scripts are turned into sharable UIs automatically, and can be composed together into flows or used into richer apps built with low-code. Supported script languages supported are: Python, TypeScript, Go, Bash, SQL, and GraphQL.
diff --git a/backend/.sqlx/query-08c943c4b9b76da43b8b754e5c66ab3b6397e7edc82666662a8ea466496290fb.json b/backend/.sqlx/query-08c943c4b9b76da43b8b754e5c66ab3b6397e7edc82666662a8ea466496290fb.json
new file mode 100644
index 0000000000..19fcb65125
--- /dev/null
+++ b/backend/.sqlx/query-08c943c4b9b76da43b8b754e5c66ab3b6397e7edc82666662a8ea466496290fb.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM account WHERE workspace_id = $1",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "08c943c4b9b76da43b8b754e5c66ab3b6397e7edc82666662a8ea466496290fb"
+}
diff --git a/backend/.sqlx/query-266741894b9a6ed650cf999b6e5b7c797424f8291517d3abf096fe45548a0273.json b/backend/.sqlx/query-266741894b9a6ed650cf999b6e5b7c797424f8291517d3abf096fe45548a0273.json
new file mode 100644
index 0000000000..41879871e1
--- /dev/null
+++ b/backend/.sqlx/query-266741894b9a6ed650cf999b6e5b7c797424f8291517d3abf096fe45548a0273.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "git_sync",
+ "type_info": "Jsonb"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ true
+ ]
+ },
+ "hash": "266741894b9a6ed650cf999b6e5b7c797424f8291517d3abf096fe45548a0273"
+}
diff --git a/backend/.sqlx/query-2ce7a562b1de63f767758642a4864ed982b74b53a143901480058f957b9641dc.json b/backend/.sqlx/query-2ce7a562b1de63f767758642a4864ed982b74b53a143901480058f957b9641dc.json
new file mode 100644
index 0000000000..fb379a41f2
--- /dev/null
+++ b/backend/.sqlx/query-2ce7a562b1de63f767758642a4864ed982b74b53a143901480058f957b9641dc.json
@@ -0,0 +1,18 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE job_stats SET timestamps = timestamps || $4, timeseries_float = timeseries_float || $5 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Uuid",
+ "Text",
+ "TimestamptzArray",
+ "Float4Array"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "2ce7a562b1de63f767758642a4864ed982b74b53a143901480058f957b9641dc"
+}
diff --git a/backend/.sqlx/query-30e9250edd0c4c6025322f51b758174435d6df4f835cb0883ba40897cb99a410.json b/backend/.sqlx/query-30e9250edd0c4c6025322f51b758174435d6df4f835cb0883ba40897cb99a410.json
new file mode 100644
index 0000000000..0efe3f75d5
--- /dev/null
+++ b/backend/.sqlx/query-30e9250edd0c4c6025322f51b758174435d6df4f835cb0883ba40897cb99a410.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM dependency_map\n WHERE importer_path = $1 AND importer_kind = 'script'\n AND workspace_id = $2",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "30e9250edd0c4c6025322f51b758174435d6df4f835cb0883ba40897cb99a410"
+}
diff --git a/backend/.sqlx/query-4c5276bbf6aaad004a58aaff6b2abd7e78903a0354dd041fcc0cc7cca2b60cc1.json b/backend/.sqlx/query-4c5276bbf6aaad004a58aaff6b2abd7e78903a0354dd041fcc0cc7cca2b60cc1.json
new file mode 100644
index 0000000000..85cd242a5a
--- /dev/null
+++ b/backend/.sqlx/query-4c5276bbf6aaad004a58aaff6b2abd7e78903a0354dd041fcc0cc7cca2b60cc1.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM token WHERE workspace_id = $1",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "4c5276bbf6aaad004a58aaff6b2abd7e78903a0354dd041fcc0cc7cca2b60cc1"
+}
diff --git a/backend/.sqlx/query-5fbd5dfb8df9aecce023c5d482ee58efcbc23c8682a3f0ec94977eb56023ed72.json b/backend/.sqlx/query-5fbd5dfb8df9aecce023c5d482ee58efcbc23c8682a3f0ec94977eb56023ed72.json
new file mode 100644
index 0000000000..c58167c782
--- /dev/null
+++ b/backend/.sqlx/query-5fbd5dfb8df9aecce023c5d482ee58efcbc23c8682a3f0ec94977eb56023ed72.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path)\n VALUES ($1, $2, 'script', $3)",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Varchar",
+ "Varchar",
+ "Varchar"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "5fbd5dfb8df9aecce023c5d482ee58efcbc23c8682a3f0ec94977eb56023ed72"
+}
diff --git a/backend/.sqlx/query-60cd5ea09b2eba9df97ab011dd8575940a9dd5e244f3e877d3802fc88087234b.json b/backend/.sqlx/query-60cd5ea09b2eba9df97ab011dd8575940a9dd5e244f3e877d3802fc88087234b.json
new file mode 100644
index 0000000000..ec835ce212
--- /dev/null
+++ b/backend/.sqlx/query-60cd5ea09b2eba9df97ab011dd8575940a9dd5e244f3e877d3802fc88087234b.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM input WHERE workspace_id = $1",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "60cd5ea09b2eba9df97ab011dd8575940a9dd5e244f3e877d3802fc88087234b"
+}
diff --git a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json
new file mode 100644
index 0000000000..75b8108281
--- /dev/null
+++ b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json
@@ -0,0 +1,24 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT true FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "?column?",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Uuid",
+ "Text"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a"
+}
diff --git a/backend/.sqlx/query-699427e7e3e6cd3ba4f61f77e171c5c27234cfa30460696ac619760667d7553c.json b/backend/.sqlx/query-699427e7e3e6cd3ba4f61f77e171c5c27234cfa30460696ac619760667d7553c.json
new file mode 100644
index 0000000000..fea6b54002
--- /dev/null
+++ b/backend/.sqlx/query-699427e7e3e6cd3ba4f61f77e171c5c27234cfa30460696ac619760667d7553c.json
@@ -0,0 +1,17 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE job_stats SET scalar_int = $4 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Uuid",
+ "Text",
+ "Int4"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "699427e7e3e6cd3ba4f61f77e171c5c27234cfa30460696ac619760667d7553c"
+}
diff --git a/backend/.sqlx/query-73a3a742702c1c8237a3216523dffe487146b14512d4882179978efe280956aa.json b/backend/.sqlx/query-73a3a742702c1c8237a3216523dffe487146b14512d4882179978efe280956aa.json
new file mode 100644
index 0000000000..ca57d29258
--- /dev/null
+++ b/backend/.sqlx/query-73a3a742702c1c8237a3216523dffe487146b14512d4882179978efe280956aa.json
@@ -0,0 +1,18 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE job_stats SET timestamps = timestamps || $4, timeseries_int = timeseries_int || $5 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Uuid",
+ "Text",
+ "TimestamptzArray",
+ "Int4Array"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "73a3a742702c1c8237a3216523dffe487146b14512d4882179978efe280956aa"
+}
diff --git a/backend/.sqlx/query-74949628ea78a5fecfd415e6406636a9442edd5b7939f61894288103be0c03ef.json b/backend/.sqlx/query-74949628ea78a5fecfd415e6406636a9442edd5b7939f61894288103be0c03ef.json
new file mode 100644
index 0000000000..e9f542da19
--- /dev/null
+++ b/backend/.sqlx/query-74949628ea78a5fecfd415e6406636a9442edd5b7939f61894288103be0c03ef.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM job_stats WHERE job_id IN (SELECT id FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now())",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "74949628ea78a5fecfd415e6406636a9442edd5b7939f61894288103be0c03ef"
+}
diff --git a/backend/.sqlx/query-76d73f2fe5ac75e5033fb93d60f2ef5ab2b0b73a80dcb6a250c17c3b2f15c6b4.json b/backend/.sqlx/query-76d73f2fe5ac75e5033fb93d60f2ef5ab2b0b73a80dcb6a250c17c3b2f15c6b4.json
new file mode 100644
index 0000000000..97c52e993e
--- /dev/null
+++ b/backend/.sqlx/query-76d73f2fe5ac75e5033fb93d60f2ef5ab2b0b73a80dcb6a250c17c3b2f15c6b4.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM dependency_map WHERE workspace_id = $1",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "76d73f2fe5ac75e5033fb93d60f2ef5ab2b0b73a80dcb6a250c17c3b2f15c6b4"
+}
diff --git a/backend/.sqlx/query-813d3c65d4c4bc10f1ec16ed4ee0954f4b938f45751ac03fd45479e665db1d40.json b/backend/.sqlx/query-813d3c65d4c4bc10f1ec16ed4ee0954f4b938f45751ac03fd45479e665db1d40.json
new file mode 100644
index 0000000000..6bda4d5cb5
--- /dev/null
+++ b/backend/.sqlx/query-813d3c65d4c4bc10f1ec16ed4ee0954f4b938f45751ac03fd45479e665db1d40.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM deployment_metadata WHERE workspace_id = $1",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "813d3c65d4c4bc10f1ec16ed4ee0954f4b938f45751ac03fd45479e665db1d40"
+}
diff --git a/backend/.sqlx/query-96a77e839d6210629234879eeeaa23d9bddf5f751fe27d999659f6597f9759f1.json b/backend/.sqlx/query-96a77e839d6210629234879eeeaa23d9bddf5f751fe27d999659f6597f9759f1.json
new file mode 100644
index 0000000000..3a66267a5f
--- /dev/null
+++ b/backend/.sqlx/query-96a77e839d6210629234879eeeaa23d9bddf5f751fe27d999659f6597f9759f1.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM raw_app WHERE workspace_id = $1",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "96a77e839d6210629234879eeeaa23d9bddf5f751fe27d999659f6597f9759f1"
+}
diff --git a/backend/.sqlx/query-aa44311f9bae8253c6c483d122597ce7e470c6ba1905e7e877dca35d9d68968f.json b/backend/.sqlx/query-aa44311f9bae8253c6c483d122597ce7e470c6ba1905e7e877dca35d9d68968f.json
new file mode 100644
index 0000000000..808f531923
--- /dev/null
+++ b/backend/.sqlx/query-aa44311f9bae8253c6c483d122597ce7e470c6ba1905e7e877dca35d9d68968f.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT importer_path FROM dependency_map\n WHERE imported_path = $1 AND importer_kind = 'script'\n AND workspace_id = $2",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "importer_path",
+ "type_info": "Varchar"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "aa44311f9bae8253c6c483d122597ce7e470c6ba1905e7e877dca35d9d68968f"
+}
diff --git a/backend/.sqlx/query-b56111c5b81de2449a1b687656a4636457da851a4db5e916cf4f6847c48c6bd3.json b/backend/.sqlx/query-b56111c5b81de2449a1b687656a4636457da851a4db5e916cf4f6847c48c6bd3.json
new file mode 100644
index 0000000000..a3e2229cd3
--- /dev/null
+++ b/backend/.sqlx/query-b56111c5b81de2449a1b687656a4636457da851a4db5e916cf4f6847c48c6bd3.json
@@ -0,0 +1,17 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE job_stats SET scalar_float = $4 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Uuid",
+ "Text",
+ "Float4"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "b56111c5b81de2449a1b687656a4636457da851a4db5e916cf4f6847c48c6bd3"
+}
diff --git a/backend/.sqlx/query-d77223fc94257711512b6ff21ce109b97324f2917113f04492f53b20860e9588.json b/backend/.sqlx/query-d77223fc94257711512b6ff21ce109b97324f2917113f04492f53b20860e9588.json
new file mode 100644
index 0000000000..52800c5369
--- /dev/null
+++ b/backend/.sqlx/query-d77223fc94257711512b6ff21ce109b97324f2917113f04492f53b20860e9588.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM job_stats WHERE workspace_id = $1",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "d77223fc94257711512b6ff21ce109b97324f2917113f04492f53b20860e9588"
+}
diff --git a/backend/.sqlx/query-deb2e4c21d4f9d0b9439780ca6ebee13450609b578c469cb437c5420ab520258.json b/backend/.sqlx/query-deb2e4c21d4f9d0b9439780ca6ebee13450609b578c469cb437c5420ab520258.json
new file mode 100644
index 0000000000..54e871eb13
--- /dev/null
+++ b/backend/.sqlx/query-deb2e4c21d4f9d0b9439780ca6ebee13450609b578c469cb437c5420ab520258.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT content FROM script WHERE hash = $1 AND workspace_id = $2",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "content",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "deb2e4c21d4f9d0b9439780ca6ebee13450609b578c469cb437c5420ab520258"
+}
diff --git a/backend/Cargo.lock b/backend/Cargo.lock
index 012e435173..cabfa07be7 100644
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -63,9 +63,9 @@ dependencies = [
[[package]]
name = "ahash"
-version = "0.8.6"
+version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
+checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01"
dependencies = [
"cfg-if",
"getrandom 0.2.11",
@@ -169,9 +169,9 @@ dependencies = [
[[package]]
name = "anyhow"
-version = "1.0.77"
+version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9d19de80eff169429ac1e9f48fffb163916b448a44e8e046186232046d9e1f9"
+checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca"
[[package]]
name = "archiver-rs"
@@ -245,7 +245,7 @@ dependencies = [
"proc-macro2",
"quote",
"swc_macros_common",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -266,7 +266,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c"
dependencies = [
"concurrent-queue",
- "event-listener 4.0.1",
+ "event-listener 4.0.3",
"event-listener-strategy",
"futures-core",
"pin-project-lite",
@@ -313,7 +313,7 @@ dependencies = [
"async-task",
"concurrent-queue",
"fastrand 2.0.1",
- "futures-lite 2.1.0",
+ "futures-lite 2.2.0",
"slab",
]
@@ -328,7 +328,7 @@ dependencies = [
"async-io 2.2.2",
"async-lock 3.2.0",
"blocking",
- "futures-lite 2.1.0",
+ "futures-lite 2.2.0",
"once_cell",
]
@@ -362,7 +362,7 @@ dependencies = [
"cfg-if",
"concurrent-queue",
"futures-io",
- "futures-lite 2.1.0",
+ "futures-lite 2.2.0",
"parking",
"polling 3.3.1",
"rustix 0.38.28",
@@ -386,7 +386,7 @@ version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c"
dependencies = [
- "event-listener 4.0.1",
+ "event-listener 4.0.3",
"event-listener-strategy",
"pin-project-lite",
]
@@ -418,7 +418,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -466,7 +466,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -496,19 +496,19 @@ dependencies = [
[[package]]
name = "async-task"
-version = "4.6.0"
+version = "4.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1d90cd0b264dfdd8eb5bad0a2c217c1f88fa96a8573f40e7b12de23fb468f46"
+checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799"
[[package]]
name = "async-trait"
-version = "0.1.75"
+version = "0.1.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98"
+checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -539,15 +539,6 @@ dependencies = [
"pin-project-lite",
]
-[[package]]
-name = "atoi"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7c57d12312ff59c811c0643f4d80830505833c9ffaebd193d819392b265be8e"
-dependencies = [
- "num-traits",
-]
-
[[package]]
name = "atoi"
version = "2.0.0"
@@ -559,9 +550,9 @@ dependencies = [
[[package]]
name = "atoi_simd"
-version = "0.15.5"
+version = "0.15.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccfc14f5c3e34de57539a7ba9c18ecde3d9bbde48d232ea1da3e468adb307fd0"
+checksum = "9ae037714f313c1353189ead58ef9eec30a8e8dc101b2622d461418fd59e28a9"
[[package]]
name = "atomic-waker"
@@ -1073,7 +1064,7 @@ dependencies = [
"async-trait",
"futures-channel",
"futures-util",
- "parking_lot 0.12.1",
+ "parking_lot",
"tokio",
]
@@ -1132,13 +1123,13 @@ dependencies = [
"lazycell",
"log",
"peeking_take_while",
- "prettyplease 0.2.15",
+ "prettyplease 0.2.16",
"proc-macro2",
"quote",
"regex",
"rustc-hash",
"shlex",
- "syn 2.0.43",
+ "syn 2.0.48",
"which",
]
@@ -1224,7 +1215,7 @@ dependencies = [
"async-task",
"fastrand 2.0.1",
"futures-io",
- "futures-lite 2.1.0",
+ "futures-lite 2.2.0",
"piper",
"tracing",
]
@@ -1249,7 +1240,7 @@ dependencies = [
"proc-macro-crate 2.0.0",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
"syn_derive",
]
@@ -1338,7 +1329,7 @@ checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -1445,7 +1436,7 @@ version = "9.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e11c675378efb449ed3ce8de78d75d0d80542fc98487c26aba28eb3b82feac72"
dependencies = [
- "semver 1.0.20",
+ "semver 1.0.21",
"serde",
"toml 0.7.8",
"url",
@@ -1540,9 +1531,9 @@ dependencies = [
[[package]]
name = "clang-sys"
-version = "1.6.1"
+version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
+checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1"
dependencies = [
"glob",
"libc",
@@ -1551,9 +1542,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.4.12"
+version = "4.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d"
+checksum = "52bdc885e4cacc7f7c9eedc1ef6da641603180c783c41a15c264944deeaab642"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1580,7 +1571,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -1745,9 +1736,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
[[package]]
name = "cpufeatures"
-version = "0.2.11"
+version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0"
+checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
dependencies = [
"libc",
]
@@ -1879,7 +1870,7 @@ dependencies = [
"bitflags 2.4.1",
"crossterm_winapi",
"libc",
- "parking_lot 0.12.1",
+ "parking_lot",
"winapi",
]
@@ -1975,7 +1966,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -1997,7 +1988,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
dependencies = [
"darling_core 0.20.3",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -2010,7 +2001,7 @@ dependencies = [
"hashbrown 0.14.3",
"lock_api",
"once_cell",
- "parking_lot_core 0.9.9",
+ "parking_lot_core",
]
[[package]]
@@ -2055,7 +2046,7 @@ checksum = "3c65c2ffdafc1564565200967edc4851c7b55422d3913466688907efd05ea26f"
dependencies = [
"deno-proc-macro-rules-macros",
"proc-macro2",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -2067,7 +2058,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -2129,7 +2120,7 @@ dependencies = [
"libc",
"log",
"once_cell",
- "parking_lot 0.12.1",
+ "parking_lot",
"pin-project",
"serde",
"serde_json",
@@ -2187,7 +2178,7 @@ dependencies = [
"strum",
"strum_macros",
"syn 1.0.109",
- "syn 2.0.43",
+ "syn 2.0.48",
"thiserror",
]
@@ -2268,9 +2259,9 @@ dependencies = [
[[package]]
name = "deranged"
-version = "0.3.10"
+version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc"
+checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
dependencies = [
"powerfmt",
"serde",
@@ -2352,22 +2343,13 @@ dependencies = [
"subtle",
]
-[[package]]
-name = "dirs"
-version = "4.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
-dependencies = [
- "dirs-sys 0.3.7",
-]
-
[[package]]
name = "dirs"
version = "5.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
dependencies = [
- "dirs-sys 0.4.1",
+ "dirs-sys",
]
[[package]]
@@ -2380,17 +2362,6 @@ dependencies = [
"dirs-sys-next",
]
-[[package]]
-name = "dirs-sys"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
-dependencies = [
- "libc",
- "redox_users",
- "winapi",
-]
-
[[package]]
name = "dirs-sys"
version = "0.4.1"
@@ -2611,7 +2582,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -2631,7 +2602,7 @@ checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -2684,9 +2655,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
[[package]]
name = "event-listener"
-version = "4.0.1"
+version = "4.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84f2cdcf274580f2d63697192d744727b3198894b1bf02923643bf59e2c26712"
+checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e"
dependencies = [
"concurrent-queue",
"parking",
@@ -2699,7 +2670,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3"
dependencies = [
- "event-listener 4.0.1",
+ "event-listener 4.0.3",
"pin-project-lite",
]
@@ -2832,7 +2803,7 @@ dependencies = [
"pmutil 0.6.1",
"proc-macro2",
"swc_macros_common",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -2860,7 +2831,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e"
dependencies = [
"frunk_proc_macro_helpers",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -2872,7 +2843,7 @@ dependencies = [
"frunk_core",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -2884,7 +2855,7 @@ dependencies = [
"frunk_core",
"frunk_proc_macro_helpers",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -2945,17 +2916,6 @@ dependencies = [
"futures-util",
]
-[[package]]
-name = "futures-intrusive"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a604f7a68fbf8103337523b1fadc8ade7361ee3f112f7c680ad179651616aed5"
-dependencies = [
- "futures-core",
- "lock_api",
- "parking_lot 0.11.2",
-]
-
[[package]]
name = "futures-intrusive"
version = "0.5.0"
@@ -2964,7 +2924,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f"
dependencies = [
"futures-core",
"lock_api",
- "parking_lot 0.12.1",
+ "parking_lot",
]
[[package]]
@@ -2990,9 +2950,9 @@ dependencies = [
[[package]]
name = "futures-lite"
-version = "2.1.0"
+version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aeee267a1883f7ebef3700f262d2d54de95dfaf38189015a74fdc4e0c7ad8143"
+checksum = "445ba825b27408685aaecefd65178908c36c6e96aaf6d8599419d46e624192ba"
dependencies = [
"fastrand 2.0.1",
"futures-core",
@@ -3009,7 +2969,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -3260,7 +3220,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -3365,7 +3325,7 @@ version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.7",
]
[[package]]
@@ -3374,7 +3334,7 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.7",
"allocator-api2",
"rayon",
]
@@ -3439,7 +3399,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b780635574b3d92f036890d8373433d6f9fc7abb320ee42a5c25897fc8ed732"
dependencies = [
- "dirs 5.0.1",
+ "dirs",
"indicatif",
"log",
"native-tls",
@@ -3599,9 +3559,9 @@ dependencies = [
[[package]]
name = "iana-time-zone"
-version = "0.1.58"
+version = "0.1.59"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20"
+checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539"
dependencies = [
"android_system_properties",
"core-foundation-sys",
@@ -3732,14 +3692,14 @@ dependencies = [
[[package]]
name = "is-macro"
-version = "0.3.4"
+version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b75828adcb53122ef5ea649a39f50f82d94b754099bf6331b32e255e1891e8fb"
+checksum = "59a85abdc13717906baccb5a1e435556ce0df215f242892f721dff62bf25288f"
dependencies = [
"Inflector",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -3910,9 +3870,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "libc"
-version = "0.2.151"
+version = "0.2.152"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
+checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"
[[package]]
name = "libgit2-sys"
@@ -3928,12 +3888,12 @@ dependencies = [
[[package]]
name = "libloading"
-version = "0.7.4"
+version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f"
+checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161"
dependencies = [
"cfg-if",
- "winapi",
+ "windows-sys 0.48.0",
]
[[package]]
@@ -4122,9 +4082,9 @@ dependencies = [
[[package]]
name = "mail-send"
-version = "0.4.4"
+version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e38e7453c5cb25cbeaa251e480c74592ec121360a6bcb822928926bf69e3d1e"
+checksum = "5f5982137dccf24ca038daa022eb0ea97a31e9739477445a787a8ef77e015ebb"
dependencies = [
"base64 0.21.5",
"gethostname",
@@ -4299,9 +4259,9 @@ dependencies = [
[[package]]
name = "monostate"
-version = "0.1.10"
+version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e404e13820ea0df0eda93aa294e0c80de76a0daa6bec590d376fbec6d7810394"
+checksum = "878c2a1f1c70e5724fa28f101ca787b6a7e8ad5c5e4ae4ca3b0fa4a419fa9075"
dependencies = [
"monostate-impl",
"serde",
@@ -4309,13 +4269,13 @@ dependencies = [
[[package]]
name = "monostate-impl"
-version = "0.1.10"
+version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "531c82a934da419bed3da09bd87d6e98c72f8d4aa755427b3b009c2b8b8c433c"
+checksum = "f686d68a09079e63b1d2c64aa305095887ce50565f00a922ebfaeeee0d9ba6ce"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -4353,7 +4313,7 @@ dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
"termcolor",
"thiserror",
]
@@ -4614,7 +4574,7 @@ dependencies = [
"humantime",
"hyper",
"itertools 0.11.0",
- "parking_lot 0.12.1",
+ "parking_lot",
"percent-encoding",
"quick-xml 0.31.0",
"rand 0.8.5",
@@ -4697,7 +4657,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -4753,17 +4713,6 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
-[[package]]
-name = "parking_lot"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
-dependencies = [
- "instant",
- "lock_api",
- "parking_lot_core 0.8.6",
-]
-
[[package]]
name = "parking_lot"
version = "0.12.1"
@@ -4771,21 +4720,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api",
- "parking_lot_core 0.9.9",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.8.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
-dependencies = [
- "cfg-if",
- "instant",
- "libc",
- "redox_syscall 0.2.16",
- "smallvec",
- "winapi",
+ "parking_lot_core",
]
[[package]]
@@ -4923,14 +4858,11 @@ dependencies = [
"archiver-rs",
"async-trait",
"bytes",
- "dirs 5.0.1",
+ "dirs",
"futures",
"lazy_static",
"log",
- "reqwest",
- "sqlx 0.6.3",
"thiserror",
- "tokio",
"zip",
]
@@ -5009,7 +4941,7 @@ dependencies = [
"phf_shared 0.11.2",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -5047,7 +4979,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -5150,7 +5082,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -5175,9 +5107,9 @@ version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd503430a6d9779b07915d858865fe998317ef3cfef8973881f578ac5d4baae7"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.7",
"arrow-format",
- "atoi 2.0.0",
+ "atoi",
"atoi_simd",
"bytemuck",
"chrono",
@@ -5209,7 +5141,7 @@ version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae73d5b8e55decde670caba1cc82b61f14bfb9a72503198f0997d657a98dcfd6"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.7",
"bitflags 2.4.1",
"bytemuck",
"chrono",
@@ -5253,7 +5185,7 @@ version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96e10a0745acd6009db64bef0ceb9e23a70b1c27b26a0a6517c91f3e6363bc06"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.7",
"async-trait",
"atoi_simd",
"bytes",
@@ -5293,7 +5225,7 @@ version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3555f759705be6dd0d3762d16a0b8787b2dc4da73b57465f3b2bf1a070ba8f20"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.7",
"bitflags 2.4.1",
"futures",
"glob",
@@ -5318,7 +5250,7 @@ version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a7eb218296aaa7f79945f08288ca32ca3cf25fa505649eeee689ec21eebf636"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.7",
"argminmax",
"bytemuck",
"either",
@@ -5342,7 +5274,7 @@ version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "146010e4b7dd4d2d0e58ddc762f6361f77d7a0385c54471199370c17164f67dd"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.7",
"async-stream",
"base64 0.21.5",
"brotli",
@@ -5393,7 +5325,7 @@ version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10e32a0958ef854b132bad7f8369cb3237254635d5e864c99505bc0bc1035fbc"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.7",
"bytemuck",
"once_cell",
"percent-encoding",
@@ -5445,7 +5377,7 @@ version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae56f79e9cedd617773c1c8f5ca84a31a8b1d593714959d5f799e7bdd98fe51"
dependencies = [
- "atoi 2.0.0",
+ "atoi",
"chrono",
"now",
"once_cell",
@@ -5464,7 +5396,7 @@ version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da6ce68169fe61d46958c8eab7447360f30f2f23f6e24a0ce703a14b0a3cfbfc"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.7",
"bytemuck",
"hashbrown 0.14.3",
"indexmap 2.1.0",
@@ -5610,12 +5542,12 @@ dependencies = [
[[package]]
name = "prettyplease"
-version = "0.2.15"
+version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d"
+checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5"
dependencies = [
"proc-macro2",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -5669,9 +5601,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
[[package]]
name = "proc-macro2"
-version = "1.0.71"
+version = "1.0.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8"
+checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
dependencies = [
"unicode-ident",
]
@@ -5725,7 +5657,7 @@ dependencies = [
"schemars",
"serde",
"serde_json",
- "syn 2.0.43",
+ "syn 2.0.48",
"thiserror",
"typify",
"unicode-ident",
@@ -5745,7 +5677,7 @@ dependencies = [
"serde_json",
"serde_tokenstream",
"serde_yaml",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -5764,7 +5696,7 @@ dependencies = [
"fnv",
"lazy_static",
"memchr",
- "parking_lot 0.12.1",
+ "parking_lot",
"thiserror",
]
@@ -5867,17 +5799,17 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f69f8d22fa3f34f3083d9a4375c038732c7a7e964de1beb81c544da92dfc40b8"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.7",
"equivalent",
"hashbrown 0.14.3",
- "parking_lot 0.12.1",
+ "parking_lot",
]
[[package]]
name = "quote"
-version = "1.0.33"
+version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
dependencies = [
"proc-macro2",
]
@@ -6043,15 +5975,6 @@ dependencies = [
"url",
]
-[[package]]
-name = "redox_syscall"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
-dependencies = [
- "bitflags 1.3.2",
-]
-
[[package]]
name = "redox_syscall"
version = "0.3.5"
@@ -6354,7 +6277,7 @@ dependencies = [
"proc-macro2",
"quote",
"rust-embed-utils",
- "syn 2.0.43",
+ "syn 2.0.48",
"walkdir",
]
@@ -6412,7 +6335,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
- "semver 1.0.20",
+ "semver 1.0.21",
]
[[package]]
@@ -6488,7 +6411,7 @@ dependencies = [
"log",
"ring 0.17.7",
"rustls-pki-types",
- "rustls-webpki 0.102.0",
+ "rustls-webpki 0.102.1",
"subtle",
"zeroize",
]
@@ -6532,9 +6455,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
-version = "0.102.0"
+version = "0.102.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de2635c8bc2b88d367767c5de8ea1d8db9af3f6219eba28442242d9ab81d1b89"
+checksum = "ef4ca26037c909dedb327b48c3327d0ba91d3dd3c4e05dad328f210ffb68e95b"
dependencies = [
"ring 0.17.7",
"rustls-pki-types",
@@ -6773,9 +6696,9 @@ dependencies = [
[[package]]
name = "semver"
-version = "1.0.20"
+version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
+checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0"
dependencies = [
"serde",
]
@@ -6794,9 +6717,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
[[package]]
name = "serde"
-version = "1.0.193"
+version = "1.0.195"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89"
+checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02"
dependencies = [
"serde_derive",
]
@@ -6825,22 +6748,22 @@ dependencies = [
[[package]]
name = "serde_bytes"
-version = "0.11.13"
+version = "0.11.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bb1879ea93538b78549031e2d54da3e901fd7e75f2e4dc758d760937b123d10"
+checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734"
dependencies = [
"serde",
]
[[package]]
name = "serde_derive"
-version = "1.0.193"
+version = "1.0.195"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"
+checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -6856,9 +6779,9 @@ dependencies = [
[[package]]
name = "serde_json"
-version = "1.0.108"
+version = "1.0.111"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
+checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4"
dependencies = [
"indexmap 2.1.0",
"itoa",
@@ -6868,9 +6791,9 @@ dependencies = [
[[package]]
name = "serde_path_to_error"
-version = "0.1.14"
+version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335"
+checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c"
dependencies = [
"itoa",
"serde",
@@ -6925,7 +6848,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -6958,9 +6881,9 @@ dependencies = [
[[package]]
name = "serde_yaml"
-version = "0.9.29"
+version = "0.9.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a15e0ef66bf939a7c890a0bf6d5a733c70202225f9888a89ed5c62298b019129"
+checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38"
dependencies = [
"indexmap 2.1.0",
"itoa",
@@ -7149,9 +7072,9 @@ dependencies = [
[[package]]
name = "smtp-proto"
-version = "0.1.2"
+version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f7278d054ed6844babe3d760d03ccae24e26207c3c27b68d8385a03d3324619"
+checksum = "20b37ae016fedcac2174dd0e3029870154830a4d3fb10f533317f2604e72b343"
[[package]]
name = "snafu"
@@ -7294,90 +7217,27 @@ dependencies = [
"log",
]
-[[package]]
-name = "sqlx"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8de3b03a925878ed54a954f621e64bf55a3c1bd29652d0d1a17830405350188"
-dependencies = [
- "sqlx-core 0.6.3",
- "sqlx-macros 0.6.3",
-]
-
[[package]]
name = "sqlx"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dba03c279da73694ef99763320dea58b51095dfe87d001b1d4b5fe78ba8763cf"
dependencies = [
- "sqlx-core 0.7.3",
- "sqlx-macros 0.7.3",
+ "sqlx-core",
+ "sqlx-macros",
"sqlx-mysql",
"sqlx-postgres",
"sqlx-sqlite",
]
-[[package]]
-name = "sqlx-core"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029"
-dependencies = [
- "ahash 0.7.7",
- "atoi 1.0.0",
- "base64 0.13.1",
- "bitflags 1.3.2",
- "byteorder",
- "bytes",
- "crc",
- "crossbeam-queue",
- "dirs 4.0.0",
- "dotenvy",
- "either",
- "event-listener 2.5.3",
- "futures-channel",
- "futures-core",
- "futures-intrusive 0.4.2",
- "futures-util",
- "hashlink",
- "hex",
- "hkdf",
- "hmac",
- "indexmap 1.9.3",
- "itoa",
- "libc",
- "log",
- "md-5 0.10.6",
- "memchr",
- "once_cell",
- "paste",
- "percent-encoding",
- "rand 0.8.5",
- "rustls 0.20.9",
- "rustls-pemfile",
- "serde",
- "serde_json",
- "sha1",
- "sha2 0.10.8",
- "smallvec",
- "sqlformat",
- "sqlx-rt",
- "stringprep",
- "thiserror",
- "tokio-stream",
- "url",
- "webpki-roots 0.22.6",
- "whoami",
-]
-
[[package]]
name = "sqlx-core"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd"
dependencies = [
- "ahash 0.8.6",
- "atoi 2.0.0",
+ "ahash 0.8.7",
+ "atoi",
"bigdecimal 0.3.1",
"byteorder",
"bytes",
@@ -7389,7 +7249,7 @@ dependencies = [
"event-listener 2.5.3",
"futures-channel",
"futures-core",
- "futures-intrusive 0.5.0",
+ "futures-intrusive",
"futures-io",
"futures-util",
"hashlink",
@@ -7416,25 +7276,6 @@ dependencies = [
"webpki-roots 0.25.3",
]
-[[package]]
-name = "sqlx-macros"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9966e64ae989e7e575b19d7265cb79d7fc3cbbdf179835cb0d716f294c2049c9"
-dependencies = [
- "dotenvy",
- "either",
- "heck",
- "once_cell",
- "proc-macro2",
- "quote",
- "sha2 0.10.8",
- "sqlx-core 0.6.3",
- "sqlx-rt",
- "syn 1.0.109",
- "url",
-]
-
[[package]]
name = "sqlx-macros"
version = "0.7.3"
@@ -7443,7 +7284,7 @@ checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5"
dependencies = [
"proc-macro2",
"quote",
- "sqlx-core 0.7.3",
+ "sqlx-core",
"sqlx-macros-core",
"syn 1.0.109",
]
@@ -7465,7 +7306,7 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.8",
- "sqlx-core 0.7.3",
+ "sqlx-core",
"sqlx-mysql",
"sqlx-postgres",
"sqlx-sqlite",
@@ -7481,7 +7322,7 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4"
dependencies = [
- "atoi 2.0.0",
+ "atoi",
"base64 0.21.5",
"bigdecimal 0.3.1",
"bitflags 2.4.1",
@@ -7512,7 +7353,7 @@ dependencies = [
"sha1",
"sha2 0.10.8",
"smallvec",
- "sqlx-core 0.7.3",
+ "sqlx-core",
"stringprep",
"thiserror",
"tracing",
@@ -7526,7 +7367,7 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24"
dependencies = [
- "atoi 2.0.0",
+ "atoi",
"base64 0.21.5",
"bigdecimal 0.3.1",
"bitflags 2.4.1",
@@ -7555,7 +7396,7 @@ dependencies = [
"sha1",
"sha2 0.10.8",
"smallvec",
- "sqlx-core 0.7.3",
+ "sqlx-core",
"stringprep",
"thiserror",
"tracing",
@@ -7563,36 +7404,25 @@ dependencies = [
"whoami",
]
-[[package]]
-name = "sqlx-rt"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024"
-dependencies = [
- "once_cell",
- "tokio",
- "tokio-rustls 0.23.4",
-]
-
[[package]]
name = "sqlx-sqlite"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "210976b7d948c7ba9fced8ca835b11cbb2d677c59c79de41ac0d397e14547490"
dependencies = [
- "atoi 2.0.0",
+ "atoi",
"chrono",
"flume",
"futures-channel",
"futures-core",
"futures-executor",
- "futures-intrusive 0.5.0",
+ "futures-intrusive",
"futures-util",
"libsqlite3-sys",
"log",
"percent-encoding",
"serde",
- "sqlx-core 0.7.3",
+ "sqlx-core",
"tracing",
"url",
"urlencoding",
@@ -7653,7 +7483,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"
dependencies = [
"new_debug_unreachable",
"once_cell",
- "parking_lot 0.12.1",
+ "parking_lot",
"phf_shared 0.10.0",
"precomputed-hash",
"serde",
@@ -7681,7 +7511,7 @@ dependencies = [
"proc-macro2",
"quote",
"swc_macros_common",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -7720,7 +7550,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -7802,7 +7632,7 @@ dependencies = [
"proc-macro2",
"quote",
"swc_macros_common",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -7812,7 +7642,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7191c8c57af059b75a2aadc927a2608c3962d19e4d09ce8f9c3f03739ddf833"
dependencies = [
"bitflags 2.4.1",
- "is-macro 0.3.4",
+ "is-macro 0.3.5",
"num-bigint",
"scoped-tls",
"serde",
@@ -7851,7 +7681,7 @@ dependencies = [
"proc-macro2",
"quote",
"swc_macros_common",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -7934,7 +7764,7 @@ dependencies = [
"proc-macro2",
"quote",
"swc_macros_common",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -8038,7 +7868,7 @@ dependencies = [
"pmutil 0.6.1",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -8050,7 +7880,7 @@ dependencies = [
"pmutil 0.6.1",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -8074,7 +7904,7 @@ dependencies = [
"proc-macro2",
"quote",
"swc_macros_common",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -8090,9 +7920,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.43"
+version = "2.0.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53"
+checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
dependencies = [
"proc-macro2",
"quote",
@@ -8108,7 +7938,7 @@ dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -8125,7 +7955,7 @@ checksum = "285ba80e733fac80aa4270fbcdf83772a79b80aa35c97075320abfee4a915b06"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
"unicode-xid",
]
@@ -8220,22 +8050,22 @@ dependencies = [
[[package]]
name = "thiserror"
-version = "1.0.52"
+version = "1.0.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83a48fd946b02c0a526b2e9481c8e2a17755e47039164a86c4070446e3a4614d"
+checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
-version = "1.0.52"
+version = "1.0.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7fbe9b594d6568a6a1443250a7e67d80b74e1e96f6d1715e1e21cc1888291d3"
+checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -8399,7 +8229,7 @@ dependencies = [
"libc",
"mio",
"num_cpus",
- "parking_lot 0.12.1",
+ "parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2 0.5.5",
@@ -8416,7 +8246,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -8453,7 +8283,7 @@ dependencies = [
"futures-channel",
"futures-util",
"log",
- "parking_lot 0.12.1",
+ "parking_lot",
"percent-encoding",
"phf 0.11.2",
"pin-project-lite",
@@ -8630,7 +8460,7 @@ dependencies = [
"home",
"once_cell",
"regex",
- "semver 1.0.20",
+ "semver 1.0.21",
"walkdir",
]
@@ -8661,7 +8491,7 @@ dependencies = [
"cookie",
"futures-util",
"http",
- "parking_lot 0.12.1",
+ "parking_lot",
"pin-project-lite",
"tower-layer",
"tower-service",
@@ -8718,7 +8548,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -8888,7 +8718,7 @@ dependencies = [
"regress",
"schemars",
"serde_json",
- "syn 2.0.43",
+ "syn 2.0.48",
"thiserror",
"unicode-ident",
]
@@ -8905,7 +8735,7 @@ dependencies = [
"serde",
"serde_json",
"serde_tokenstream",
- "syn 2.0.43",
+ "syn 2.0.48",
"typify-impl",
]
@@ -9171,9 +9001,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "value-bag"
-version = "1.4.2"
+version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe"
+checksum = "dd65ff0de3304a013e6dfeb7324e1cd389a8a000e582e3c9c9fae82ced778e26"
[[package]]
name = "vcpkg"
@@ -9251,7 +9081,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
"wasm-bindgen-shared",
]
@@ -9285,7 +9115,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -9318,7 +9148,7 @@ checksum = "794645f5408c9a039fd09f4d113cdfb2e7eba5ff1956b07bcf701cf4b394fe89"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -9442,7 +9272,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"axum",
@@ -9462,7 +9292,7 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.8",
- "sqlx 0.7.3",
+ "sqlx",
"tokio",
"tokio-metrics",
"tracing",
@@ -9471,13 +9301,14 @@ dependencies = [
"windmill-api",
"windmill-api-client",
"windmill-common",
+ "windmill-git-sync",
"windmill-queue",
"windmill-worker",
]
[[package]]
name = "windmill-api"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"argon2",
@@ -9524,7 +9355,7 @@ dependencies = [
"serde_urlencoded",
"sha2 0.10.8",
"sql-builder",
- "sqlx 0.7.3",
+ "sqlx",
"tempfile",
"time",
"tinyvector",
@@ -9541,6 +9372,7 @@ dependencies = [
"uuid 1.6.1",
"windmill-audit",
"windmill-common",
+ "windmill-git-sync",
"windmill-parser",
"windmill-parser-py-imports",
"windmill-queue",
@@ -9548,7 +9380,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"base64 0.21.5",
"chrono",
@@ -9566,20 +9398,20 @@ dependencies = [
[[package]]
name = "windmill-audit"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"chrono",
"serde",
"serde_json",
"sql-builder",
- "sqlx 0.7.3",
+ "sqlx",
"tracing",
"windmill-common",
]
[[package]]
name = "windmill-common"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"aws-config",
@@ -9600,7 +9432,7 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.8",
- "sqlx 0.7.3",
+ "sqlx",
"thiserror",
"tokio",
"tracing",
@@ -9610,9 +9442,22 @@ dependencies = [
"uuid 1.6.1",
]
+[[package]]
+name = "windmill-git-sync"
+version = "1.236.2"
+dependencies = [
+ "rsmq_async",
+ "serde",
+ "serde_json",
+ "sqlx",
+ "tracing",
+ "windmill-common",
+ "windmill-queue",
+]
+
[[package]]
name = "windmill-parser"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"serde",
"serde_json",
@@ -9620,7 +9465,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"lazy_static",
@@ -9631,7 +9476,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"gosyn",
@@ -9643,7 +9488,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"lazy_static",
@@ -9654,7 +9499,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"itertools 0.12.0",
@@ -9665,7 +9510,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"async-recursion",
@@ -9675,14 +9520,14 @@ dependencies = [
"regex",
"rustpython-parser",
"serde_json",
- "sqlx 0.7.3",
+ "sqlx",
"windmill-common",
"windmill-parser",
]
[[package]]
name = "windmill-parser-sql"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"lazy_static",
@@ -9693,7 +9538,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -9710,7 +9555,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"getrandom 0.2.11",
@@ -9728,7 +9573,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"async-recursion",
@@ -9749,7 +9594,7 @@ dependencies = [
"serde_json",
"serde_urlencoded",
"sql-builder",
- "sqlx 0.7.3",
+ "sqlx",
"tokio",
"tracing",
"ulid",
@@ -9760,7 +9605,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
-version = "1.232.0"
+version = "1.236.2"
dependencies = [
"anyhow",
"async-recursion",
@@ -9800,7 +9645,7 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.8",
- "sqlx 0.7.3",
+ "sqlx",
"tiberius",
"tokio",
"tokio-postgres",
@@ -9810,6 +9655,7 @@ dependencies = [
"uuid 1.6.1",
"windmill-audit",
"windmill-common",
+ "windmill-git-sync",
"windmill-parser",
"windmill-parser-bash",
"windmill-parser-go",
@@ -9823,11 +9669,11 @@ dependencies = [
[[package]]
name = "windows-core"
-version = "0.51.1"
+version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64"
+checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
dependencies = [
- "windows-targets 0.48.5",
+ "windows-targets 0.52.0",
]
[[package]]
@@ -10030,9 +9876,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
[[package]]
name = "winnow"
-version = "0.5.31"
+version = "0.5.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97a4882e6b134d6c28953a387571f1acdd3496830d5e36c5e3a1075580ea641c"
+checksum = "b7520bbdec7211caa7c4e682eb1fbe07abe20cee6756b6e00f537c82c11816aa"
dependencies = [
"memchr",
]
@@ -10108,7 +9954,7 @@ checksum = "9e6936f0cce458098a201c245a11bef556c6a0181129c7034d10d76d1ec3a2b8"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
"synstructure",
]
@@ -10129,7 +9975,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -10149,7 +9995,7 @@ checksum = "e6a647510471d372f2e6c2e6b7219e44d8c574d24fdc11c610a61455782f18c3"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
"synstructure",
]
diff --git a/backend/Cargo.toml b/backend/Cargo.toml
index 4fea5d9c43..12f6db752f 100644
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "windmill"
-version = "1.232.0"
+version = "1.236.2"
authors.workspace = true
edition.workspace = true
@@ -11,6 +11,7 @@ members = [
"./windmill-worker",
"./windmill-common",
"./windmill-audit",
+ "./windmill-git-sync",
"./parsers/windmill-parser",
"./parsers/windmill-parser-ts",
"./parsers/windmill-parser-wasm",
@@ -21,7 +22,7 @@ members = [
]
[workspace.package]
-version = "1.232.0"
+version = "1.236.2"
authors = ["Ruben Fiszel "]
edition = "2021"
@@ -34,7 +35,7 @@ opt-level = 0
incremental = true
[features]
-enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
+enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise"]
enterprise_saml = ["windmill-api/enterprise_saml"]
benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"]
flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
@@ -46,6 +47,7 @@ anyhow.workspace = true
tokio.workspace = true
dotenv.workspace = true
windmill-common = { workspace = true, features = ["tracing_init"] }
+windmill-git-sync.workspace = true
windmill-api.workspace = true
windmill-worker.workspace = true
futures.workspace = true
@@ -66,7 +68,7 @@ uuid.workspace = true
gethostname.workspace = true
serde_json.workspace = true
serde.workspace = true
-pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true}
+pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false}
[dev-dependencies]
serde_json.workspace = true
@@ -83,6 +85,7 @@ windmill-queue = { path = "./windmill-queue" }
windmill-worker = { path = "./windmill-worker" }
windmill-common = { path = "./windmill-common" }
windmill-audit = { path = "./windmill-audit" }
+windmill-git-sync = { path = "./windmill-git-sync" }
windmill-parser = { path = "./parsers/windmill-parser" }
windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
windmill-parser-py = { path = "./parsers/windmill-parser-py" }
@@ -153,7 +156,7 @@ swc_ecma_visit = "0.93.7"
base64 = "0.21.0"
hmac = "0.12.1"
sha2 = "0.10.6"
-sqlx = { version = "^0", features = [
+sqlx = { version = "0.7.3", features = [
"macros",
"migrate",
"uuid",
diff --git a/backend/migrations/20231227014107_add_dependency_map.down.sql b/backend/migrations/20231227014107_add_dependency_map.down.sql
new file mode 100644
index 0000000000..ec6a4f8b45
--- /dev/null
+++ b/backend/migrations/20231227014107_add_dependency_map.down.sql
@@ -0,0 +1,2 @@
+-- Add down migration script here
+DROP TABLE dependency_map;
\ No newline at end of file
diff --git a/backend/migrations/20231227014107_add_dependency_map.up.sql b/backend/migrations/20231227014107_add_dependency_map.up.sql
new file mode 100644
index 0000000000..7014d0bd63
--- /dev/null
+++ b/backend/migrations/20231227014107_add_dependency_map.up.sql
@@ -0,0 +1,14 @@
+-- Add up migration script here
+
+CREATE TYPE IMPORTER_KIND AS ENUM ('script', 'flow', 'app');
+
+CREATE TABLE IF NOT EXISTS dependency_map (
+ workspace_id VARCHAR(50) NOT NULL,
+ importer_path VARCHAR(510) NOT NULL,
+ importer_kind IMPORTER_KIND NOT NULL,
+ imported_path VARCHAR(510) NOT NULL,
+ PRIMARY KEY (workspace_id, importer_path, importer_kind, imported_path)
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS dependency_map_imported_path_idx ON dependency_map (workspace_id, imported_path);
+
diff --git a/backend/migrations/20240102133412_job_stats_table.down.sql b/backend/migrations/20240102133412_job_stats_table.down.sql
new file mode 100644
index 0000000000..42bb1aaa11
--- /dev/null
+++ b/backend/migrations/20240102133412_job_stats_table.down.sql
@@ -0,0 +1,3 @@
+-- Add down migration script here
+DROP TABLE job_stats;
+DROP TYPE METRIC_KIND;
\ No newline at end of file
diff --git a/backend/migrations/20240102133412_job_stats_table.up.sql b/backend/migrations/20240102133412_job_stats_table.up.sql
new file mode 100644
index 0000000000..b1357594d8
--- /dev/null
+++ b/backend/migrations/20240102133412_job_stats_table.up.sql
@@ -0,0 +1,16 @@
+-- Add up migration script here
+CREATE TYPE METRIC_KIND AS ENUM ('scalar_int', 'scalar_float', 'timeseries_int', 'timeseries_float');
+
+CREATE TABLE IF NOT EXISTS job_stats (
+ workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+ job_id UUID NOT NULL,
+ metric_id VARCHAR(50) NOT NULL,
+ metric_name VARCHAR(255),
+ metric_kind METRIC_KIND NOT NULL,
+ scalar_int INTEGER,
+ scalar_float REAL,
+ timestamps TIMESTAMP WITH TIME ZONE[],
+ timeseries_int INTEGER[],
+ timeseries_float REAL[],
+ PRIMARY KEY (workspace_id, job_id, metric_id)
+);
diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs
index f221595efe..12ad11d534 100644
--- a/backend/parsers/windmill-parser-py-imports/src/lib.rs
+++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs
@@ -63,7 +63,7 @@ lazy_static! {
fn process_import(module: Option, path: &str, level: usize) -> Vec {
if level > 0 {
- let mut imports = vec!["requests".to_string()];
+ let mut imports = vec![];
let splitted_path = path.split("/");
let base = splitted_path
.clone()
@@ -78,10 +78,7 @@ fn process_import(module: Option, path: &str, level: usize) -> Vec, path: &str, level: usize) -> Vec error::Result> {
+ let nimports = parse_code_for_imports(code, path)?;
+ return Ok(nimports
+ .into_iter()
+ .filter_map(|x| {
+ if x.starts_with("relative:") {
+ Some(x.replace("relative:", ""))
+ } else {
+ None
+ }
+ })
+ .collect());
+}
+
+fn parse_code_for_imports(code: &str, path: &str) -> error::Result> {
+ let code = code.split(DEF_MAIN).next().unwrap_or("");
+ let ast = Suite::parse(code, "main.py").map_err(|e| {
+ error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
+ })?;
+ let nimports: Vec = ast
+ .into_iter()
+ .filter_map(|x| match x {
+ Stmt::Import(StmtImport { names, .. }) => Some(
+ names
+ .into_iter()
+ .map(|x| {
+ let name = x.name.to_string();
+ process_import(Some(name), path, 0)
+ })
+ .flatten()
+ .collect::>(),
+ ),
+ Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => {
+ Some(process_import(
+ module.map(|x| x.to_string()),
+ path,
+ i.to_usize(),
+ ))
+ }
+ Stmt::ImportFrom(StmtImportFrom { level: _, module, .. }) => {
+ Some(process_import(module.map(|x| x.to_string()), path, 0))
+ }
+ _ => None,
+ })
+ .flatten()
+ .filter(|x| !STDIMPORTS.contains(&x.as_str()))
+ .unique()
+ .collect();
+ return Ok(nimports);
+}
+
#[async_recursion]
pub async fn parse_python_imports(
code: &str,
@@ -127,41 +175,7 @@ pub async fn parse_python_imports(
imports.extend(lines);
}
- let code = code.split(DEF_MAIN).next().unwrap_or("");
- let ast = Suite::parse(code, "main.py").map_err(|e| {
- error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
- })?;
- let nimports: Vec = ast
- .into_iter()
- .filter_map(|x| match x {
- Stmt::Import(StmtImport { names, .. }) => Some(
- names
- .into_iter()
- .map(|x| {
- let name = x.name.to_string();
- process_import(Some(name), path, 0)
- })
- .flatten()
- .collect::>(),
- ),
- Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. })
- if i.to_u32() > 0 =>
- {
- Some(process_import(
- module.map(|x| x.to_string()),
- path,
- i.to_usize(),
- ))
- }
- Stmt::ImportFrom(StmtImportFrom { level: _, module, .. }) => {
- Some(process_import(module.map(|x| x.to_string()), path, 0))
- }
- _ => None,
- })
- .flatten()
- .filter(|x| !STDIMPORTS.contains(&x.as_str()))
- .unique()
- .collect();
+ let nimports = parse_code_for_imports(code, path)?;
for n in nimports.iter() {
let nested = if n.starts_with("relative:") {
let rpath = n.replace("relative:", "");
diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs
index 18d995ba41..3f2212d58f 100644
--- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs
+++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs
@@ -20,7 +20,7 @@ def main():
";
let r = parse_python_imports(code, "test-workspace", "f/foo/bar", &db).await?;
// println!("{}", serde_json::to_string(&r)?);
- assert_eq!(r, vec!["matplotlib", "requests", "wmill", "zanzibar"]);
+ assert_eq!(r, vec!["matplotlib", "wmill", "zanzibar"]);
Ok(())
}
@@ -73,7 +73,6 @@ def main():
"numpy",
"pandas",
"pandas2",
- "requests"
]
);
diff --git a/backend/src/main.rs b/backend/src/main.rs
index 5b77fb379c..2339b7c7c2 100644
--- a/backend/src/main.rs
+++ b/backend/src/main.rs
@@ -431,7 +431,7 @@ Windmill Community Edition {GIT_VERSION}
},
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => {
load_require_preexisting_user(&db).await;
- }
+ },
EXPOSE_METRICS_SETTING | EXPOSE_DEBUG_METRICS_SETTING => {
if n.payload() != EXPOSE_DEBUG_METRICS_SETTING || worker_mode {
tracing::info!("Metrics setting changed, restarting");
diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs
index 7879f8f056..d372f7c5cb 100644
--- a/backend/src/monitor.rs
+++ b/backend/src/monitor.rs
@@ -35,8 +35,8 @@ use windmill_common::{
BASE_URL, DB, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
};
use windmill_worker::{
- create_token_for_owner, handle_job_error, AuthedClient, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR,
- NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
+ create_token_for_owner, handle_job_error, AuthedClient, SendResult, JOB_DEFAULT_TIMEOUT,
+ KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
};
#[cfg(feature = "enterprise")]
@@ -167,6 +167,7 @@ pub async fn load_metrics_debug_enabled(db: &DB) -> error::Result<()> {
};
Ok(())
}
+
pub async fn load_keep_job_dir(db: &DB) {
let value = sqlx::query_scalar!(
"SELECT value FROM global_settings WHERE name = $1",
@@ -250,25 +251,52 @@ pub async fn delete_expired_items(db: &DB) -> () {
let job_retention_secs = *JOB_RETENTION_SECS.read().await;
if job_retention_secs > 0 {
- let deleted_jobs = sqlx::query_scalar!(
- "DELETE FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now() RETURNING id",
- job_retention_secs
- )
- .fetch_all(db)
- .await;
+ match db.begin().await {
+ Ok(mut tx) => {
+ let r = sqlx::query!(
+ "DELETE FROM job_stats WHERE job_id IN (SELECT id FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now())",
+ job_retention_secs
+ )
+ .fetch_all(&mut *tx)
+ .await;
+ match r {
+ Ok(_) => {
+ let deleted_jobs = sqlx::query_scalar!(
+ "DELETE FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now() RETURNING id",
+ job_retention_secs
+ )
+ .fetch_all(&mut *tx)
+ .await;
- match deleted_jobs {
- Ok(deleted_jobs) => {
- if deleted_jobs.len() > 0 {
- tracing::info!(
- "deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
- deleted_jobs.len(),
- job_retention_secs,
- deleted_jobs,
- )
+ match deleted_jobs {
+ Ok(deleted_jobs) => {
+ if deleted_jobs.len() > 0 {
+ tracing::info!(
+ "deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
+ deleted_jobs.len(),
+ job_retention_secs,
+ deleted_jobs,
+ )
+ }
+ }
+ Err(e) => {
+ tracing::error!("Error deleting expired jobs: {:?}", e)
+ }
+ }
+ }
+ Err(err) => {
+ tracing::error!("Error deleting expired job stats: {:?}", err)
+ }
+ }
+
+ match tx.commit().await {
+ Ok(_) => (),
+ Err(err) => tracing::error!("Error deleting expired jobs: {:?}", err),
}
}
- Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
+ Err(err) => {
+ tracing::error!("Error deleting expired jobs: {:?}", err)
+ }
}
}
}
@@ -729,6 +757,7 @@ async fn handle_zombie_jobs
// since the job is unrecoverable, the same worker queue should never be sent anything
let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::(1);
+ let (send_result_never_used, _send_result_rx_never_used) = mpsc::channel::(1);
let token = create_token_for_owner(
&db,
@@ -767,6 +796,7 @@ async fn handle_zombie_jobs
"",
rsmq.clone(),
worker_name,
+ send_result_never_used,
)
.await;
}
diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs
index 143a9ffe08..ee77635d9f 100644
--- a/backend/tests/worker.rs
+++ b/backend/tests/worker.rs
@@ -1805,7 +1805,7 @@ def main():
}
#[sqlx::test(fixtures("base"))]
-async fn test_empty_loop(db: Pool) {
+async fn test_empty_loop_1(db: Pool) {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await;
let port = server.addr.port();
diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml
index 37d205c337..a3a3033d95 100644
--- a/backend/windmill-api/Cargo.toml
+++ b/backend/windmill-api/Cargo.toml
@@ -9,7 +9,7 @@ name = "windmill_api"
path = "src/lib.rs"
[features]
-enterprise = ["windmill-queue/enterprise", "async-stripe", "windmill-audit/enterprise"]
+enterprise = ["windmill-queue/enterprise", "async-stripe", "windmill-audit/enterprise", "windmill-git-sync/enterprise"]
enterprise_saml = ["samael"]
benchmark = []
@@ -27,6 +27,7 @@ windmill-common = { workspace = true, features = [
windmill-audit.workspace = true
windmill-parser.workspace = true
windmill-parser-py-imports.workspace = true
+windmill-git-sync.workspace = true
tokio.workspace = true
anyhow.workspace = true
argon2.workspace = true
diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml
index 072e300ab3..043dc34e75 100644
--- a/backend/windmill-api/openapi-deref.yaml
+++ b/backend/windmill-api/openapi-deref.yaml
@@ -1,6 +1,6 @@
openapi: 3.0.3
info:
- version: 1.230.0
+ version: 1.233.0
title: Windmill API
contact:
name: Windmill Team
@@ -1531,6 +1531,8 @@ paths:
type: string
git_repo_resource_path:
type: string
+ use_individual_branch:
+ type: boolean
required: &ref_17
- script_path
- git_repo_resource_path
@@ -3352,32 +3354,32 @@ paths:
id:
type: string
value:
- oneOf: &ref_196
+ oneOf: &ref_204
- type: object
- properties: &ref_180
+ properties: &ref_188
input_transforms:
type: object
additionalProperties:
oneOf: &ref_26
- type: object
- properties: &ref_176
+ properties: &ref_184
value: {}
type:
type: string
enum:
- javascript
- required: &ref_177
+ required: &ref_185
- expr
- type
- type: object
- properties: &ref_178
+ properties: &ref_186
expr:
type: string
type:
type: string
enum:
- javascript
- required: &ref_179
+ required: &ref_187
- expr
- type
discriminator: &ref_27
@@ -3417,13 +3419,13 @@ paths:
type: number
concurrency_time_window_s:
type: number
- required: &ref_181
+ required: &ref_189
- type
- content
- language
- input_transforms
- type: object
- properties: &ref_182
+ properties: &ref_190
input_transforms:
type: object
additionalProperties:
@@ -3437,12 +3439,12 @@ paths:
type: string
enum:
- script
- required: &ref_183
+ required: &ref_191
- type
- path
- input_transforms
- type: object
- properties: &ref_184
+ properties: &ref_192
input_transforms:
type: object
additionalProperties:
@@ -3454,12 +3456,12 @@ paths:
type: string
enum:
- flow
- required: &ref_185
+ required: &ref_193
- type
- path
- input_transforms
- type: object
- properties: &ref_186
+ properties: &ref_194
modules:
type: array
items:
@@ -3481,13 +3483,13 @@ paths:
type: boolean
parallelism:
type: integer
- required: &ref_187
+ required: &ref_195
- modules
- iterator
- skip_failures
- type
- type: object
- properties: &ref_188
+ properties: &ref_196
branches:
type: array
items:
@@ -3518,12 +3520,12 @@ paths:
type: string
enum:
- branchone
- required: &ref_189
+ required: &ref_197
- branches
- default
- type
- type: object
- properties: &ref_190
+ properties: &ref_198
branches:
type: array
items:
@@ -3548,28 +3550,28 @@ paths:
- branchall
parallel:
type: boolean
- required: &ref_191
+ required: &ref_199
- branches
- type
- type: object
- properties: &ref_192
+ properties: &ref_200
type:
type: string
enum:
- identity
flow:
type: boolean
- required: &ref_193
+ required: &ref_201
- type
- type: object
- properties: &ref_194
+ properties: &ref_202
type:
type: string
enum:
- graphql
- required: &ref_195
+ required: &ref_203
- type
- discriminator: &ref_197
+ discriminator: &ref_205
propertyName: type
mapping:
rawscript: '#/components/schemas/RawScript'
@@ -8240,6 +8242,10 @@ paths:
retry:
type: object
properties: *ref_78
+ no_flow_overlap:
+ type: boolean
+ summary:
+ type: string
required: &ref_148
- path
- schedule
@@ -8305,6 +8311,10 @@ paths:
retry:
type: object
properties: *ref_78
+ no_flow_overlap:
+ type: boolean
+ summary:
+ type: string
required: &ref_150
- schedule
- timezone
@@ -8446,6 +8456,10 @@ paths:
retry:
type: object
properties: *ref_78
+ summary:
+ type: string
+ no_flow_overlap:
+ type: boolean
required: &ref_80
- path
- edited_by
@@ -10309,6 +10323,91 @@ paths:
- upload_id
- parts
- is_done
+ /w/{workspace}/job_metrics/get/{id}:
+ post:
+ summary: get job metrics
+ operationId: getJobMetrics
+ tags:
+ - metrics
+ parameters:
+ - name: workspace
+ in: path
+ required: true
+ schema: *ref_0
+ - name: id
+ in: path
+ required: true
+ schema: *ref_74
+ requestBody:
+ description: parameters for statistics retrieval
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ timeseries_max_datapoints:
+ type: integer
+ from_timestamp:
+ type: string
+ format: date-time
+ to_timestamp:
+ type: string
+ format: date-time
+ responses:
+ '200':
+ description: job details
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ metrics_metadata:
+ type: array
+ items:
+ type: object
+ properties: &ref_176
+ id:
+ type: string
+ name:
+ type: string
+ required: &ref_177
+ - id
+ scalar_metrics:
+ type: array
+ items:
+ type: object
+ properties: &ref_178
+ metric_id:
+ type: string
+ value:
+ type: number
+ required: &ref_179
+ - id
+ - value
+ timeseries_metrics:
+ type: array
+ items:
+ type: object
+ properties: &ref_180
+ metric_id:
+ type: string
+ values:
+ type: array
+ items:
+ type: object
+ properties: &ref_182
+ timestamp:
+ type: string
+ format: date-time
+ value:
+ type: number
+ required: &ref_183
+ - timestamp
+ - value
+ required: &ref_181
+ - id
+ - values
components:
securitySchemes:
bearerAuth:
@@ -10960,56 +11059,72 @@ components:
type: object
properties: *ref_96
required: *ref_97
- StaticTransform:
+ MetricMetadata:
type: object
properties: *ref_176
required: *ref_177
- JavascriptTransform:
+ ScalarMetric:
type: object
properties: *ref_178
required: *ref_179
+ TimeseriesMetric:
+ type: object
+ properties: *ref_180
+ required: *ref_181
+ MetricDataPoint:
+ type: object
+ properties: *ref_182
+ required: *ref_183
+ StaticTransform:
+ type: object
+ properties: *ref_184
+ required: *ref_185
+ JavascriptTransform:
+ type: object
+ properties: *ref_186
+ required: *ref_187
InputTransform:
oneOf: *ref_26
discriminator: *ref_27
RawScript:
type: object
- properties: *ref_180
- required: *ref_181
+ properties: *ref_188
+ required: *ref_189
PathScript:
type: object
- properties: *ref_182
- required: *ref_183
+ properties: *ref_190
+ required: *ref_191
PathFlow:
type: object
- properties: *ref_184
- required: *ref_185
+ properties: *ref_192
+ required: *ref_193
FlowModule:
type: object
properties: *ref_28
required: *ref_29
ForloopFlow:
- type: object
- properties: *ref_186
- required: *ref_187
- BranchOne:
- type: object
- properties: *ref_188
- required: *ref_189
- BranchAll:
- type: object
- properties: *ref_190
- required: *ref_191
- Identity:
- type: object
- properties: *ref_192
- required: *ref_193
- Graphql:
type: object
properties: *ref_194
required: *ref_195
+ BranchOne:
+ type: object
+ properties: *ref_196
+ required: *ref_197
+ BranchAll:
+ type: object
+ properties: *ref_198
+ required: *ref_199
+ Identity:
+ type: object
+ properties: *ref_200
+ required: *ref_201
+ Graphql:
+ type: object
+ properties: *ref_202
+ required: *ref_203
FlowModuleValue:
- oneOf: *ref_196
- discriminator: *ref_197
+ oneOf: *ref_204
+ discriminator: *ref_205
Retry:
type: object
properties: *ref_78
diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml
index 6445a6c76a..b4b40062a4 100644
--- a/backend/windmill-api/openapi.yaml
+++ b/backend/windmill-api/openapi.yaml
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
- version: 1.232.0
+ version: 1.236.2
title: Windmill API
contact:
@@ -2231,8 +2231,7 @@ paths:
/w/{workspace}/resources/get_value_interpolated/{path}:
get:
- summary:
- get resource interpolated (variables and resources are fully unrolled)
+ summary: get resource interpolated (variables and resources are fully unrolled)
operationId: getResourceValueInterpolated
tags:
- resource
@@ -2920,8 +2919,7 @@ paths:
schema:
type: string
- name: first_parent_hash
- description:
- mask to filter scripts whom first direct parent has exact hash
+ description: mask to filter scripts whom first direct parent has exact hash
in: query
schema:
type: string
@@ -3116,8 +3114,7 @@ paths:
/workers/custom_tags:
get:
- summary:
- get all instance custom tags (tags are used to dispatch jobs to
+ summary: get all instance custom tags (tags are used to dispatch jobs to
different worker groups)
operationId: getCustomTags
tags:
@@ -3168,8 +3165,7 @@ paths:
/w/{workspace}/scripts/delete/h/{hash}:
post:
- summary:
- delete script by hash (erase content but keep hash, require admin)
+ summary: delete script by hash (erase content but keep hash, require admin)
operationId: deleteScriptByHash
tags:
- script
@@ -3408,8 +3404,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
- description:
- schedule the script to execute in the number of seconds starting now
+ description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -3417,8 +3412,7 @@ paths:
- $ref: "#/components/parameters/WorkerTag"
- $ref: "#/components/parameters/NewJobId"
- name: invisible_to_owner
- description:
- make the run invisible to the the script owner (default false)
+ description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4451,8 +4445,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
- description:
- schedule the script to execute in the number of seconds starting now
+ description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4461,8 +4454,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
- description:
- make the run invisible to the the flow owner (default false)
+ description: make the run invisible to the the flow owner (default false)
in: query
schema:
type: boolean
@@ -4514,8 +4506,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
- description:
- schedule the script to execute in the number of seconds starting now
+ description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4524,8 +4515,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
- description:
- make the run invisible to the the flow owner (default false)
+ description: make the run invisible to the the flow owner (default false)
in: query
schema:
type: boolean
@@ -4563,8 +4553,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
- description:
- schedule the script to execute in the number of seconds starting now
+ description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4573,8 +4562,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
- description:
- make the run invisible to the the script owner (default false)
+ description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4605,8 +4593,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
- description:
- make the run invisible to the the script owner (default false)
+ description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4639,8 +4626,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
- description:
- make the run invisible to the the script owner (default false)
+ description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4857,8 +4843,7 @@ paths:
- job
responses:
"200":
- description:
- the timestamp of the db that can be used to compute the drift
+ description: the timestamp of the db that can be used to compute the drift
content:
application/json:
schema:
@@ -5122,8 +5107,7 @@ paths:
/w/{workspace}/jobs/resume_urls/{id}/{resume_id}:
get:
- summary:
- get resume urls given a job_id, resume_id and a nonce to resume a flow
+ summary: get resume urls given a job_id, resume_id and a nonce to resume a flow
operationId: getResumeUrls
tags:
- job
@@ -5674,8 +5658,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- name: only_member_of
in: query
- description:
- only list the groups the user is member of (default false)
+ description: only list the groups the user is member of (default false)
schema:
type: boolean
responses:
@@ -5863,8 +5846,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- name: only_member_of
in: query
- description:
- only list the folders the user is member of (default false)
+ description: only list the folders the user is member of (default false)
schema:
type: boolean
responses:
@@ -6692,8 +6674,7 @@ paths:
- storage_options
/w/{workspace}/job_helpers/v2/s3_resource_info:
post:
- summary:
- Returns the s3 resource associated to the provided path, or the
+ summary: Returns the s3 resource associated to the provided path, or the
workspace default S3 resource
operationId: s3ResourceInfo
tags:
@@ -6939,6 +6920,53 @@ paths:
- parts
- is_done
+ /w/{workspace}/job_metrics/get/{id}:
+ post:
+ summary: get job metrics
+ operationId: getJobMetrics
+ tags:
+ - metrics
+ parameters:
+ - $ref: "#/components/parameters/WorkspaceId"
+ - $ref: "#/components/parameters/JobId"
+ requestBody:
+ description: parameters for statistics retrieval
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ timeseries_max_datapoints:
+ type: integer
+ from_timestamp:
+ type: string
+ format: date-time
+ to_timestamp:
+ type: string
+ format: date-time
+
+ responses:
+ "200":
+ description: job details
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ metrics_metadata:
+ type: array
+ items:
+ $ref: "#/components/schemas/MetricMetadata"
+ scalar_metrics:
+ type: array
+ items:
+ $ref: "#/components/schemas/ScalarMetric"
+ timeseries_metrics:
+ type: array
+ items:
+ $ref: "#/components/schemas/TimeseriesMetric"
+
components:
securitySchemes:
bearerAuth:
@@ -7037,8 +7065,7 @@ components:
type: integer
PerPage:
name: per_page
- description:
- number of items to return for a given page (default 30, max 100)
+ description: number of items to return for a given page (default 30, max 100)
in: query
schema:
type: integer
@@ -7166,8 +7193,7 @@ components:
type: boolean
ScheduledForBeforeNow:
name: scheduled_for_before_now
- description:
- filter on jobs scheduled_for before now (hence waitinf for a worker)
+ description: filter on jobs scheduled_for before now (hence waitinf for a worker)
in: query
schema:
type: boolean
@@ -7185,8 +7211,7 @@ components:
type: boolean
ArgsFilter:
name: args
- description:
- filter on jobs containing those args as a json subset (@> in postgres)
+ description: filter on jobs containing those args as a json subset (@> in postgres)
in: query
schema:
type: string
@@ -7198,8 +7223,7 @@ components:
type: string
ResultFilter:
name: result
- description:
- filter on jobs containing those result as a json subset (@> in postgres)
+ description: filter on jobs containing those result as a json subset (@> in postgres)
in: query
schema:
type: string
@@ -9088,6 +9112,8 @@ components:
type: string
git_repo_resource_path:
type: string
+ use_individual_branch:
+ type: boolean
required:
- script_path
- git_repo_resource_path
@@ -9102,3 +9128,49 @@ components:
required:
- part_number
- tag
+
+ MetricMetadata:
+ type: object
+ properties:
+ id:
+ type: string
+ name:
+ type: string
+ required:
+ - id
+
+ ScalarMetric:
+ type: object
+ properties:
+ metric_id:
+ type: string
+ value:
+ type: number
+ required:
+ - id
+ - value
+
+ TimeseriesMetric:
+ type: object
+ properties:
+ metric_id:
+ type: string
+ values:
+ type: array
+ items:
+ $ref: "#/components/schemas/MetricDataPoint"
+ required:
+ - id
+ - values
+
+ MetricDataPoint:
+ type: object
+ properties:
+ timestamp:
+ type: string
+ format: date-time
+ value:
+ type: number
+ required:
+ - timestamp
+ - value
diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs
index 7bacf8a222..710747f841 100644
--- a/backend/windmill-api/src/apps.rs
+++ b/backend/windmill-api/src/apps.rs
@@ -9,7 +9,6 @@ use std::collections::HashMap;
*/
use crate::{
db::{ApiAuthed, DB},
- deployment_metadata_helpers,
users::{require_owner_of_path, OptAuthed},
variables::build_crypt,
webhook_util::{WebhookMessage, WebhookShared},
@@ -588,12 +587,16 @@ async fn create_app(
)
.await?;
- let mut tx = PushIsolationLevel::Transaction(tx);
+ let tx = PushIsolationLevel::Transaction(tx);
let (dependency_job_uuid, new_tx) = push(
&db,
tx,
&w_id,
- JobPayload::AppDependencies { path: app.path.clone(), version: v_id },
+ JobPayload::AppDependencies {
+ path: app.path.clone(),
+ version: v_id,
+ deployment_message: app.deployment_message,
+ },
PushArgs::empty(),
&authed.username,
&authed.email,
@@ -615,25 +618,7 @@ async fn create_app(
.await?;
tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
- tx = PushIsolationLevel::Transaction(new_tx);
- tx = deployment_metadata_helpers::handle_deployment_metadata(
- tx,
- &authed,
- &db,
- &w_id,
- deployment_metadata_helpers::DeployedObject::App { path: app.path.clone(), version: v_id },
- app.deployment_message,
- )
- .await?;
-
- match tx {
- PushIsolationLevel::Transaction(tx) => tx.commit().await?,
- _ => {
- return Err(Error::InternalErr(
- "Expected a transaction here".to_string(),
- ));
- }
- }
+ new_tx.commit().await?;
webhook.send_message(
w_id.clone(),
@@ -849,14 +834,18 @@ async fn update_app(
)
.await?;
- let mut tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
+ let tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
PushIsolationLevel::Transaction(tx);
if let Some(v_id) = v_id {
let (dependency_job_uuid, new_tx) = push(
&db,
tx,
&w_id,
- JobPayload::AppDependencies { path: npath.clone(), version: v_id },
+ JobPayload::AppDependencies {
+ path: npath.clone(),
+ version: v_id,
+ deployment_message: ns.deployment_message,
+ },
PushArgs::empty(),
&authed.username,
&authed.email,
@@ -877,26 +866,7 @@ async fn update_app(
)
.await?;
tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
-
- tx = PushIsolationLevel::Transaction(new_tx);
- tx = deployment_metadata_helpers::handle_deployment_metadata(
- tx,
- &authed,
- &db,
- &w_id,
- deployment_metadata_helpers::DeployedObject::App { path: npath.clone(), version: v_id },
- ns.deployment_message,
- )
- .await?;
- }
-
- match tx {
- PushIsolationLevel::Transaction(tx) => tx.commit().await?,
- _ => {
- return Err(Error::InternalErr(
- "Expected a transaction here".to_string(),
- ));
- }
+ new_tx.commit().await?;
}
webhook.send_message(
diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs
index 6ff5d8ed8b..b69493dc0e 100644
--- a/backend/windmill-api/src/flows.rs
+++ b/backend/windmill-api/src/flows.rs
@@ -7,7 +7,6 @@
*/
use crate::db::ApiAuthed;
-use crate::deployment_metadata_helpers;
use crate::{
db::DB,
schedule::clear_schedule,
@@ -355,7 +354,7 @@ async fn create_flow(
)
.await?;
- let mut tx = PushIsolationLevel::Transaction(tx);
+ let tx = PushIsolationLevel::Transaction(tx);
let (dependency_job_uuid, mut new_tx) = push(
&db,
tx,
@@ -363,6 +362,7 @@ async fn create_flow(
JobPayload::FlowDependencies {
path: nf.path.clone(),
dedicated_worker: nf.dedicated_worker,
+ deployment_message: nf.deployment_message,
},
PushArgs::empty(),
&authed.username,
@@ -393,26 +393,7 @@ async fn create_flow(
.execute(&mut new_tx)
.await?;
- tx = PushIsolationLevel::Transaction(new_tx);
- tx = deployment_metadata_helpers::handle_deployment_metadata(
- tx,
- &authed,
- &db,
- &w_id,
- deployment_metadata_helpers::DeployedObject::Flow { path: nf.path.clone() },
- nf.deployment_message,
- )
- .await?;
-
- match tx {
- PushIsolationLevel::Transaction(tx) => tx.commit().await?,
- _ => {
- return Err(Error::InternalErr(
- "Expected a transaction here".to_string(),
- ));
- }
- }
-
+ new_tx.commit().await?;
webhook.send_message(
w_id.clone(),
WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() },
@@ -585,7 +566,7 @@ async fn update_flow(
},
);
- let mut tx = PushIsolationLevel::Transaction(tx);
+ let tx = PushIsolationLevel::Transaction(tx);
let (dependency_job_uuid, mut new_tx) = push(
&db,
@@ -594,6 +575,7 @@ async fn update_flow(
JobPayload::FlowDependencies {
path: nf.path.clone(),
dedicated_worker: nf.dedicated_worker,
+ deployment_message: nf.deployment_message,
},
PushArgs::empty(),
&authed.username,
@@ -631,25 +613,7 @@ async fn update_flow(
.await?;
}
- tx = PushIsolationLevel::Transaction(new_tx);
- tx = deployment_metadata_helpers::handle_deployment_metadata(
- tx,
- &authed,
- &db,
- &w_id,
- deployment_metadata_helpers::DeployedObject::Flow { path: nf.path.clone() },
- nf.deployment_message,
- )
- .await?;
-
- match tx {
- PushIsolationLevel::Transaction(tx) => tx.commit().await?,
- _ => {
- return Err(Error::InternalErr(
- "Expected a transaction here".to_string(),
- ));
- }
- }
+ new_tx.commit().await?;
Ok(nf.path.to_string())
}
diff --git a/backend/windmill-api/src/job_metrics.rs b/backend/windmill-api/src/job_metrics.rs
new file mode 100644
index 0000000000..6f95f11fb8
--- /dev/null
+++ b/backend/windmill-api/src/job_metrics.rs
@@ -0,0 +1,165 @@
+use crate::db::DB;
+
+use axum::{extract::Path, routing::post, Extension, Json, Router};
+use hyper::http;
+use serde::{Deserialize, Serialize};
+use tower_http::cors::{Any, CorsLayer};
+use uuid::Uuid;
+use windmill_common::{
+ error,
+ job_metrics::{JobStatsRecord, MetricKind},
+};
+
+pub fn workspaced_service() -> Router {
+ let cors = CorsLayer::new()
+ .allow_methods([http::Method::GET, http::Method::POST])
+ .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION])
+ .allow_origin(Any);
+
+ Router::new().route("/get/:id", post(get_job_metrics).layer(cors.clone()))
+}
+
+#[derive(Deserialize)]
+struct JobStatsRequest {
+ from_timestamp: Option>,
+ to_timestamp: Option>,
+ timeseries_max_datapoints: Option, // default to 100, any value lower than that will be ignored. Set to 0 to retrieve all
+}
+
+#[derive(Serialize)]
+struct JobStatsResponse {
+ metrics_metadata: Vec,
+ scalar_metrics: Vec,
+ timeseries_metrics: Vec,
+}
+
+#[derive(Serialize)]
+pub struct MetricsMetadata {
+ pub id: String,
+ pub name: Option,
+}
+
+#[derive(Serialize)]
+struct ScalarMetric {
+ metric_id: String,
+ value: f64,
+}
+
+#[derive(Serialize)]
+struct TimeseriesMetric {
+ metric_id: String,
+ values: Vec,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct DataPoint {
+ pub timestamp: chrono::DateTime,
+ pub value: f64,
+}
+
+async fn get_job_metrics(
+ Extension(db): Extension,
+ Path((w_id, job_id)): Path<(String, Uuid)>,
+ Json(JobStatsRequest { from_timestamp, to_timestamp, timeseries_max_datapoints }): Json<
+ JobStatsRequest,
+ >,
+) -> error::JsonResult {
+ let records = sqlx::query_as::<_, JobStatsRecord>(
+ "SELECT * FROM job_stats where workspace_id = $1 and job_id = $2",
+ )
+ .bind(w_id)
+ .bind(job_id)
+ .fetch_all(&db)
+ .await?;
+
+ let mut metrics_metadata: Vec = vec![];
+ let mut scalar_metrics: Vec = vec![];
+ let mut timeseries_metrics: Vec = vec![];
+
+ for record in records {
+ let metric_id = record.metric_id;
+ match record.metric_kind {
+ MetricKind::ScalarInt => {
+ let value = record.scalar_int.unwrap_or_default() as f64;
+ scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value });
+ }
+ MetricKind::ScalarFloat => {
+ let value = record.scalar_float.unwrap_or_default() as f64;
+ scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value });
+ }
+ MetricKind::TimeseriesInt => {
+ if record.timestamps.clone().unwrap_or_default().len()
+ != record.timeseries_int.clone().unwrap_or_default().len()
+ {
+ tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_int)
+ }
+ let (timestamps, timeseries_int) = timeseries_sample(
+ from_timestamp,
+ to_timestamp,
+ timeseries_max_datapoints,
+ record.timestamps.unwrap_or_default(),
+ record.timeseries_int.unwrap_or_default(),
+ );
+ let mut values: Vec = vec![];
+ for (idx, value) in timeseries_int.iter().enumerate() {
+ values.push(DataPoint {
+ timestamp: timestamps[idx],
+ value: value.to_owned() as f64,
+ });
+ }
+ timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values });
+ }
+ MetricKind::TimeseriesFloat => {
+ if record.timestamps.clone().unwrap_or_default().len()
+ != record.timeseries_int.clone().unwrap_or_default().len()
+ {
+ tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_float)
+ }
+ let (timestamps, timeseries_float) = timeseries_sample(
+ from_timestamp,
+ to_timestamp,
+ timeseries_max_datapoints,
+ record.timestamps.unwrap_or_default(),
+ record.timeseries_float.unwrap_or_default(),
+ );
+ let mut values: Vec = vec![];
+ for (idx, value) in timeseries_float.iter().enumerate() {
+ values.push(DataPoint {
+ timestamp: timestamps[idx],
+ value: value.to_owned() as f64,
+ });
+ }
+ timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values });
+ }
+ };
+ metrics_metadata.push(MetricsMetadata { id: metric_id, name: record.metric_name });
+ }
+
+ let response = JobStatsResponse { metrics_metadata, scalar_metrics, timeseries_metrics };
+ Ok(Json(response))
+}
+
+fn timeseries_sample(
+ from: Option>,
+ to: Option>,
+ _datapoints: Option,
+ timestamps: Vec>,
+ values: Vec,
+) -> (Vec>, Vec) {
+ if timestamps.len() != values.len() {
+ tracing::warn!("Timeseries metric has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", timestamps.len(), values.len());
+ return (vec![], vec![]);
+ }
+ let mut filtered_timestamp: Vec> = vec![];
+ let mut filtered_values: Vec = vec![];
+ for (idx, timestamp) in timestamps.iter().enumerate() {
+ if *timestamp > from.unwrap_or(chrono::DateTime::::MIN_UTC)
+ && *timestamp < to.unwrap_or(chrono::DateTime::::MAX_UTC)
+ {
+ filtered_timestamp.push(timestamps[idx]);
+ filtered_values.push(values[idx]);
+ }
+ }
+ // TODO: implement sampling
+ return (filtered_timestamp, filtered_values);
+}
diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs
index 6a78e25ca1..c1e8f176fa 100644
--- a/backend/windmill-api/src/jobs.rs
+++ b/backend/windmill-api/src/jobs.rs
@@ -2826,14 +2826,14 @@ pub struct JobUpdate {
async fn get_job_update(
Extension(db): Extension,
- Path((w_id, id)): Path<(String, Uuid)>,
+ Path((w_id, job_id)): Path<(String, Uuid)>,
Query(JobUpdateQuery { running, log_offset }): Query,
) -> error::JsonResult {
let record = sqlx::query!(
"SELECT running, substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3",
log_offset,
&w_id,
- &id
+ &job_id
)
.fetch_optional(&db)
.await?;
@@ -2855,11 +2855,11 @@ async fn get_job_update(
$3",
log_offset,
&w_id,
- &id
+ &job_id
)
.fetch_optional(&db)
.await?;
- let logs = not_found_if_none(logs, "Job Update", id.to_string())?;
+ let logs = not_found_if_none(logs, "Job Update", job_id.to_string())?;
Ok(Json(JobUpdate {
running: Some(false),
completed: Some(true),
diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs
index b7e5e7e8f6..7d7f49cbcc 100644
--- a/backend/windmill-api/src/lib.rs
+++ b/backend/windmill-api/src/lib.rs
@@ -49,7 +49,6 @@ mod audit;
mod capture;
mod configs;
mod db;
-mod deployment_metadata_helpers;
mod drafts;
pub mod ee;
pub mod embeddings;
@@ -61,6 +60,7 @@ mod groups;
mod inputs;
mod integration;
pub mod job_helpers;
+pub mod job_metrics;
pub mod jobs;
pub mod oauth2;
mod openai;
@@ -198,6 +198,7 @@ pub async fn run_server(
.nest("/folders", folders::workspaced_service())
.nest("/groups", groups::workspaced_service())
.nest("/inputs", inputs::workspaced_service())
+ .nest("/job_metrics", job_metrics::workspaced_service())
.nest("/job_helpers", job_helpers::workspaced_service())
.nest("/jobs", jobs::workspaced_service())
.nest("/oauth", oauth2::workspaced_service())
diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs
index 7831aef244..778df43c0b 100644
--- a/backend/windmill-api/src/scripts.rs
+++ b/backend/windmill-api/src/scripts.rs
@@ -8,7 +8,6 @@
use crate::{
db::{ApiAuthed, DB},
- deployment_metadata_helpers,
schedule::clear_schedule,
users::{maybe_refresh_folders, require_owner_of_path, AuthCache},
webhook_util::{WebhookMessage, WebhookShared},
@@ -47,6 +46,7 @@ use windmill_common::{
not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
},
};
+use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
use windmill_queue::{
self, schedule::push_scheduled_job, PushArgs, PushIsolationLevel, QueueTransaction,
};
@@ -325,7 +325,7 @@ async fn create_script(
let script_path = ns.path.clone();
let hash = ScriptHash(hash_script(&ns));
let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
- let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
+ let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), user_db.begin(&authed).await?).into();
if sqlx::query_scalar!(
"SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
@@ -604,37 +604,29 @@ async fn create_script(
);
}
- let mut tx = PushIsolationLevel::Transaction(tx);
-
+ let permissioned_as = username_to_permissioned_as(&authed.username);
if needs_lock_gen {
- let dependencies = match ns.language {
- ScriptLang::Python3 => {
- windmill_parser_py_imports::parse_python_imports(&ns.content, &w_id, &ns.path, &db)
- .await?
- .join("\n")
- }
- _ => ns.content,
- };
let tag = if ns.dedicated_worker.is_some_and(|x| x) {
Some(format!("{}:{}", &w_id, &ns.path,))
} else {
ns.tag
};
+ let tx = PushIsolationLevel::Transaction(tx);
let (_, new_tx) = windmill_queue::push(
&db,
tx,
&w_id,
JobPayload::Dependencies {
hash,
- dependencies,
language: ns.language,
path: ns.path,
dedicated_worker: ns.dedicated_worker,
+ deployment_message: ns.deployment_message,
},
PushArgs::empty(),
&authed.username,
&authed.email,
- username_to_permissioned_as(&authed.username),
+ permissioned_as,
None,
None,
None,
@@ -650,26 +642,19 @@ async fn create_script(
None,
)
.await?;
- tx = PushIsolationLevel::Transaction(new_tx);
- }
-
- tx = deployment_metadata_helpers::handle_deployment_metadata(
- tx,
- &authed,
- &db,
- &w_id,
- deployment_metadata_helpers::DeployedObject::Script { hash: hash, path: script_path },
- ns.deployment_message,
- )
- .await?;
-
- match tx {
- PushIsolationLevel::Transaction(tx) => tx.commit().await?,
- _ => {
- return Err(Error::InternalErr(
- "Expected a transaction here".to_string(),
- ));
- }
+ new_tx.commit().await?;
+ } else {
+ handle_deployment_metadata(
+ &authed.email,
+ &authed.username,
+ &db,
+ &w_id,
+ DeployedObject::Script { hash: hash, path: script_path },
+ ns.deployment_message,
+ rsmq,
+ )
+ .await?;
+ tx.commit().await?;
}
Ok((StatusCode::CREATED, format!("{}", hash)))
diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs
index fe36a655be..b1e0ba4a88 100644
--- a/backend/windmill-api/src/workspaces.rs
+++ b/backend/windmill-api/src/workspaces.rs
@@ -44,6 +44,7 @@ use windmill_common::s3_helpers::LargeFileStorage;
use windmill_common::schedule::Schedule;
use windmill_common::users::username_to_permissioned_as;
use windmill_common::worker::CLOUD_HOSTED;
+use windmill_common::workspaces::WorkspaceGitRepo;
use windmill_common::{
error::{to_anyhow, Error, JsonResult, Result},
flows::Flow,
@@ -1045,12 +1046,6 @@ struct EditGitSyncConfig {
git_sync_settings: Option,
}
-#[derive(Serialize, Deserialize, Debug)]
-pub struct WorkspaceGitRepo {
- pub script_path: String,
- pub git_repo_resource_path: String,
-}
-
async fn edit_git_sync_config(
authed: ApiAuthed,
Extension(db): Extension,
@@ -1504,6 +1499,9 @@ async fn delete_workspace(
let mut tx = db.begin().await?;
require_super_admin(&db, &email).await?;
+ sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = $1", &w_id)
+ .execute(&mut *tx)
+ .await?;
sqlx::query!("DELETE FROM queue WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
@@ -1522,6 +1520,12 @@ async fn delete_workspace(
sqlx::query!("DELETE FROM app WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
+ sqlx::query!("DELETE FROM raw_app WHERE workspace_id = $1", &w_id)
+ .execute(&mut *tx)
+ .await?;
+ sqlx::query!("DELETE FROM input WHERE workspace_id = $1", &w_id)
+ .execute(&mut *tx)
+ .await?;
sqlx::query!("DELETE FROM variable WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
@@ -1537,6 +1541,17 @@ async fn delete_workspace(
.execute(&mut *tx)
.await?;
+ sqlx::query!("DELETE FROM job_stats WHERE workspace_id = $1", &w_id)
+ .execute(&mut *tx)
+ .await?;
+
+ sqlx::query!(
+ "DELETE FROM deployment_metadata WHERE workspace_id = $1",
+ &w_id
+ )
+ .execute(&mut *tx)
+ .await?;
+
sqlx::query!("DELETE FROM usr WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
@@ -1564,6 +1579,10 @@ async fn delete_workspace(
.execute(&mut *tx)
.await?;
+ sqlx::query!("DELETE FROM account WHERE workspace_id = $1", &w_id)
+ .execute(&mut *tx)
+ .await?;
+
sqlx::query!("DELETE FROM workspace_key WHERE workspace_id = $1", &w_id)
.execute(&mut *tx)
.await?;
@@ -1575,6 +1594,10 @@ async fn delete_workspace(
.execute(&mut *tx)
.await?;
+ sqlx::query!("DELETE FROM token WHERE workspace_id = $1", &w_id)
+ .execute(&mut *tx)
+ .await?;
+
sqlx::query!("DELETE FROM workspace WHERE id = $1", &w_id)
.execute(&mut *tx)
.await?;
diff --git a/backend/windmill-common/src/job_metrics.rs b/backend/windmill-common/src/job_metrics.rs
new file mode 100644
index 0000000000..65aacd5e35
--- /dev/null
+++ b/backend/windmill-common/src/job_metrics.rs
@@ -0,0 +1,187 @@
+use crate::{db::DB, error};
+
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+#[derive(sqlx::FromRow)]
+pub struct JobStatsRecord {
+ pub workspace_id: String,
+ pub job_id: Uuid,
+ pub metric_id: String,
+ pub metric_name: Option,
+ pub metric_kind: MetricKind,
+ pub scalar_int: Option,
+ pub scalar_float: Option,
+ pub timestamps: Option>>,
+ pub timeseries_int: Option>,
+ pub timeseries_float: Option>,
+}
+
+#[derive(sqlx::Type, Debug, PartialEq, Deserialize, Serialize)]
+#[sqlx(type_name = "METRIC_KIND", rename_all = "snake_case")]
+pub enum MetricKind {
+ ScalarInt,
+ ScalarFloat,
+ TimeseriesInt,
+ TimeseriesFloat,
+}
+
+pub enum MetricNumericValue {
+ Integer(i32),
+ Float(f32),
+}
+
+pub async fn register_metric_for_job(
+ db: &DB,
+ workspace_id: String,
+ job_id: Uuid,
+ metric_id: String,
+ metric_kind: MetricKind,
+ metric_name: Option,
+) -> error::Result {
+ let exists = sqlx::query_scalar!(
+ "SELECT true FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+ workspace_id,
+ job_id,
+ metric_id
+ )
+ .fetch_optional(db)
+ .await?
+ .flatten();
+ if exists.unwrap_or(false) {
+ return Err(error::Error::BadRequest(format!(
+ "Metric {} is already registered for job {}",
+ metric_id, job_id
+ )));
+ }
+
+ let (scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) = match metric_kind
+ {
+ MetricKind::ScalarInt | MetricKind::ScalarFloat => {
+ (None as Option, None as Option, None, None, None)
+ }
+ MetricKind::TimeseriesInt => (
+ None,
+ None,
+ Some(&[] as &[chrono::DateTime]),
+ Some(&[] as &[i32]),
+ None,
+ ),
+ MetricKind::TimeseriesFloat => (
+ None,
+ None,
+ Some(&[] as &[chrono::DateTime]),
+ None,
+ Some(&[] as &[f32]),
+ ),
+ };
+
+ sqlx::query(
+ "INSERT INTO job_stats (workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
+ )
+ .bind(workspace_id)
+ .bind(job_id)
+ .bind(&metric_id)
+ .bind(metric_name)
+ .bind(metric_kind)
+ .bind(scalar_int)
+ .bind(scalar_float)
+ .bind(timestamps)
+ .bind(timeseries_int)
+ .bind(timeseries_float)
+ .execute(db)
+ .await?;
+
+ Ok(metric_id)
+}
+
+pub async fn record_metric(
+ db: &DB,
+ workspace_id: String,
+ job_id: Uuid,
+ metric_id: String,
+ value: MetricNumericValue,
+) -> error::Result<()> {
+ let metric_kind_opt: Option = sqlx::query_scalar(
+ "SELECT metric_kind FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+ )
+ .bind(&workspace_id)
+ .bind(&job_id)
+ .bind(&metric_id)
+ .fetch_optional(db)
+ .await?;
+
+ if metric_kind_opt.is_none() {
+ return Err(error::Error::BadRequest(format!(
+ "Metric {} not yet registered for job {}.",
+ metric_id, job_id
+ )));
+ }
+ let metric_kind = metric_kind_opt.unwrap();
+
+ let (value_int, value_float) = match value {
+ MetricNumericValue::Integer(val) => {
+ if metric_kind != MetricKind::TimeseriesInt && metric_kind != MetricKind::ScalarInt {
+ return Err(error::Error::BadRequest(format!(
+ "Metric {} is not a timeseries int metric.",
+ metric_id
+ )));
+ }
+ (val, 0 as f32)
+ }
+ MetricNumericValue::Float(val) => {
+ if metric_kind != MetricKind::TimeseriesFloat && metric_kind != MetricKind::ScalarFloat
+ {
+ return Err(error::Error::BadRequest(format!(
+ "Metric {} is not a timeseries float metric.",
+ metric_id
+ )));
+ }
+ (0 as i32, val)
+ }
+ };
+
+ let timestamp = chrono::Utc::now();
+ match metric_kind {
+ MetricKind::ScalarInt => {
+ sqlx::query!(
+ "UPDATE job_stats SET scalar_int = $4 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+ &workspace_id,
+ &job_id,
+ &metric_id,
+ value_int,
+ ).execute(db).await?;
+ }
+ MetricKind::ScalarFloat => {
+ sqlx::query!(
+ "UPDATE job_stats SET scalar_float = $4 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+ &workspace_id,
+ &job_id,
+ &metric_id,
+ value_float,
+ ).execute(db).await?;
+ }
+ MetricKind::TimeseriesInt => {
+ sqlx::query!(
+ "UPDATE job_stats SET timestamps = timestamps || $4, timeseries_int = timeseries_int || $5 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+ &workspace_id,
+ &job_id,
+ &metric_id,
+ &[timestamp],
+ &[value_int]
+ ).execute(db).await?;
+ }
+ MetricKind::TimeseriesFloat => {
+ sqlx::query!(
+ "UPDATE job_stats SET timestamps = timestamps || $4, timeseries_float = timeseries_float || $5 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+ &workspace_id,
+ &job_id,
+ &metric_id,
+ &[timestamp],
+ &[value_float]
+ ).execute(db).await?;
+ }
+ }
+
+ Ok(())
+}
diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs
index 2595b7af75..6f2fc5abfd 100644
--- a/backend/windmill-common/src/jobs.rs
+++ b/backend/windmill-common/src/jobs.rs
@@ -283,17 +283,19 @@ pub enum JobPayload {
Dependencies {
path: String,
hash: ScriptHash,
- dependencies: String,
language: ScriptLang,
dedicated_worker: Option,
+ deployment_message: Option,
},
FlowDependencies {
path: String,
dedicated_worker: Option,
+ deployment_message: Option,
},
AppDependencies {
path: String,
version: i64,
+ deployment_message: Option,
},
Flow {
path: String,
diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs
index f103bf46f2..091cbc0254 100644
--- a/backend/windmill-common/src/lib.rs
+++ b/backend/windmill-common/src/lib.rs
@@ -23,6 +23,7 @@ pub mod external_ip;
pub mod flow_status;
pub mod flows;
pub mod global_settings;
+pub mod job_metrics;
pub mod jobs;
pub mod more_serde;
pub mod oauth2;
@@ -35,6 +36,7 @@ pub mod users;
pub mod utils;
pub mod variables;
pub mod worker;
+pub mod workspaces;
#[cfg(feature = "tracing_init")]
pub mod tracing_init;
diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs
index a349d928f6..7675c8799e 100644
--- a/backend/windmill-common/src/users.rs
+++ b/backend/windmill-common/src/users.rs
@@ -8,6 +8,7 @@
pub const SUPERADMIN_SECRET_EMAIL: &str = "superadmin_secret@windmill.dev";
pub const SUPERADMIN_NOTIFICATION_EMAIL: &str = "superadmin_notification@windmill.dev";
+pub const SUPERADMIN_SYNC_EMAIL: &str = "superadmin_sync@windmill.dev";
pub fn username_to_permissioned_as(user: &str) -> String {
if user.contains('@') {
diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs
new file mode 100644
index 0000000000..0a5a99bb03
--- /dev/null
+++ b/backend/windmill-common/src/workspaces.rs
@@ -0,0 +1,8 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct WorkspaceGitRepo {
+ pub script_path: String,
+ pub git_repo_resource_path: String,
+ pub use_individual_branch: Option,
+}
diff --git a/backend/windmill-git-sync/Cargo.toml b/backend/windmill-git-sync/Cargo.toml
new file mode 100644
index 0000000000..074d3fec56
--- /dev/null
+++ b/backend/windmill-git-sync/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "windmill-git-sync"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+
+[lib]
+name = "windmill_git_sync"
+path = "./src/lib.rs"
+
+[features]
+enterprise = ["windmill-queue/enterprise"]
+default = []
+
+[dependencies]
+serde.workspace = true
+sqlx.workspace = true
+serde_json.workspace = true
+tracing.workspace = true
+windmill-common = { workspace = true, features = ["axum"] }
+windmill-queue.workspace = true
+rsmq_async.workspace = true
\ No newline at end of file
diff --git a/backend/windmill-api/src/deployment_metadata_helpers.rs b/backend/windmill-git-sync/src/lib.rs
similarity index 52%
rename from backend/windmill-api/src/deployment_metadata_helpers.rs
rename to backend/windmill-git-sync/src/lib.rs
index bd2702329f..e246add07a 100644
--- a/backend/windmill-api/src/deployment_metadata_helpers.rs
+++ b/backend/windmill-git-sync/src/lib.rs
@@ -8,17 +8,18 @@
use std::collections::HashMap;
-use crate::db::ApiAuthed;
-use crate::db::DB;
-use crate::workspaces::{WorkspaceGitRepo, WorkspaceSettings};
+use sqlx::{Pool, Postgres};
+use windmill_common::users::SUPERADMIN_SYNC_EMAIL;
+use windmill_common::workspaces::WorkspaceGitRepo;
use serde_json::json;
use windmill_common::error::{Error, Result};
use windmill_common::jobs::JobPayload;
use windmill_common::scripts::ScriptHash;
-use windmill_common::users::username_to_permissioned_as;
use windmill_queue::PushIsolationLevel;
+pub type DB = Pool;
+
#[derive(Clone)]
pub enum DeployedObject {
Script { hash: ScriptHash, path: String },
@@ -37,13 +38,14 @@ impl DeployedObject {
}
pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
- mut tx: PushIsolationLevel<'c, R>,
- authed: &ApiAuthed,
+ email: &str,
+ created_by: &str,
db: &DB,
w_id: &str,
obj: DeployedObject,
deployment_message: Option,
-) -> Result> {
+ rsmq: Option,
+) -> Result<()> {
let skip_git_sync = if obj.get_path().starts_with("u/") {
tracing::debug!(
"Ignoring {} from git sync as it's in a private user folder",
@@ -54,10 +56,10 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
false
};
- let workspace_git_repo_setting = sqlx::query_as::<_, WorkspaceSettings>(
- "SELECT * FROM workspace_settings WHERE workspace_id = $1",
+ let workspace_git_repo_setting = sqlx::query_scalar!(
+ "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1",
+ w_id
)
- .bind(&w_id)
.fetch_optional(db)
.await?;
@@ -69,11 +71,10 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
let workspace_git_repo = workspace_git_repo_setting
.unwrap()
- .git_sync
.map(|conf| serde_json::from_value::(conf).ok())
.flatten();
- let (git_sync_job_uuid, mut new_tx) = if !skip_git_sync && workspace_git_repo.is_some() {
+ let git_sync_job_uuid = if !skip_git_sync && workspace_git_repo.is_some() {
let workspace_git_repo = workspace_git_repo.unwrap();
let mut args: HashMap = HashMap::new();
@@ -84,35 +85,56 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
.strip_prefix("$res:")),
);
- let default_commit_msg: String;
- match obj.clone() {
+ let message = match obj.clone() {
DeployedObject::Script { path, .. } => {
args.insert("path".to_string(), json!(path.to_string()));
- default_commit_msg = format!("Script '{}' deployed", path);
+ if deployment_message.as_ref().is_none()
+ || deployment_message.as_ref().is_some_and(|x| x.is_empty())
+ {
+ format!("Script '{}' deployed", path)
+ } else {
+ deployment_message.clone().unwrap()
+ }
}
DeployedObject::Flow { path } => {
args.insert("path".to_string(), json!(path.to_string()));
- default_commit_msg = format!("Flow '{}' deployed", path);
+ if deployment_message.as_ref().is_none()
+ || deployment_message.as_ref().is_some_and(|x| x.is_empty())
+ {
+ format!("Flow '{}' deployed", path)
+ } else {
+ deployment_message.clone().unwrap()
+ }
}
DeployedObject::App { path, .. } => {
args.insert("path".to_string(), json!(path.to_string()));
- default_commit_msg = format!("App '{}' deployed", path);
+ if deployment_message.as_ref().is_none()
+ || deployment_message.as_ref().is_some_and(|x| x.is_empty())
+ {
+ format!("App '{}' deployed", path)
+ } else {
+ deployment_message.clone().unwrap()
+ }
}
- }
+ };
+
+ args.insert("commit_msg".to_string(), json!(message));
args.insert(
- "commit_msg".to_string(),
- json!(deployment_message.clone().unwrap_or(default_commit_msg)),
+ "use_individual_branch".to_string(),
+ json!(workspace_git_repo.use_individual_branch.unwrap_or(false)),
);
+ let tx: PushIsolationLevel<'_, R> = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
+
let (job_uuid, new_tx) = windmill_queue::push(
- &db,
+ db,
tx,
&w_id,
JobPayload::DeploymentCallback { path: workspace_git_repo.script_path.clone() },
args,
- &authed.username,
- &authed.email,
- username_to_permissioned_as(&authed.username),
+ created_by,
+ &email,
+ SUPERADMIN_SYNC_EMAIL.to_string(),
None,
None,
None,
@@ -128,16 +150,10 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
None,
)
.await?;
- (Some(job_uuid), new_tx)
+ new_tx.commit().await?;
+ Some(job_uuid)
} else {
- let new_tx = match tx {
- PushIsolationLevel::Isolated(user_db, authed, rsmq) => {
- (rsmq, user_db.begin(&authed).await?).into()
- }
- PushIsolationLevel::IsolatedRoot(db, rsmq) => (rsmq, db.begin().await?).into(),
- PushIsolationLevel::Transaction(tx) => tx,
- };
- (None, new_tx)
+ None
};
// We're not persisting the default commit msg as it's pretty useless. We will persist the ones manually set by users
@@ -149,28 +165,27 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
if deployment_message.is_some() || job_uuids.len() > 0 {
// if the git sync job hasn't been triggered, and there is not custom deployment message, there's not point adding an entry to the table
match obj.clone() {
- DeployedObject::Script { path, hash, .. } => {
- sqlx::query!(
- "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
- w_id, path, hash.0, &job_uuids, deployment_message,
- )
- },
- DeployedObject::Flow { path } => {
- sqlx::query!(
- "INSERT INTO deployment_metadata (workspace_id, path, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL DO UPDATE SET callback_job_ids = $3, deployment_msg = $4",
- w_id, path, &job_uuids, deployment_message,
- )
- }
- DeployedObject::App { path, version } => {
- sqlx::query!(
- "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
- w_id, path, version, &job_uuids, deployment_message,
- )
- }
- }.execute(&mut new_tx)
- .await?;
+ DeployedObject::Script { path, hash, .. } => {
+ sqlx::query!(
+ "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
+ w_id, path, hash.0, &job_uuids, deployment_message,
+ )
+ },
+ DeployedObject::Flow { path } => {
+ sqlx::query!(
+ "INSERT INTO deployment_metadata (workspace_id, path, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL DO UPDATE SET callback_job_ids = $3, deployment_msg = $4",
+ w_id, path, &job_uuids, deployment_message,
+ )
+ }
+ DeployedObject::App { path, version } => {
+ sqlx::query!(
+ "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
+ w_id, path, version, &job_uuids, deployment_message,
+ )
+ }
+ }.execute(db)
+ .await?;
}
- tx = PushIsolationLevel::Transaction(new_tx);
- return Ok(tx);
+ return Ok(());
}
diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs
index 44e6f04996..b1d306f1b5 100644
--- a/backend/windmill-queue/src/jobs.rs
+++ b/backend/windmill-queue/src/jobs.rs
@@ -2496,21 +2496,23 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
dedicated_worker,
None,
),
- JobPayload::Dependencies { hash, dependencies, language, path, dedicated_worker } => (
- Some(hash.0),
- Some(path),
- Some((dependencies, None)),
- JobKind::Dependencies,
- None,
- None,
- Some(language),
- None,
- None,
- None,
- dedicated_worker,
- None,
- ),
- JobPayload::FlowDependencies { path, dedicated_worker } => {
+ JobPayload::Dependencies { hash, language, path, dedicated_worker, deployment_message } => {
+ (
+ Some(hash.0),
+ Some(path),
+ Some((deployment_message.unwrap_or_else(String::new), None)),
+ JobKind::Dependencies,
+ None,
+ None,
+ Some(language),
+ None,
+ None,
+ None,
+ dedicated_worker,
+ None,
+ )
+ }
+ JobPayload::FlowDependencies { path, dedicated_worker, deployment_message } => {
let value_json = fetch_scalar_isolated!(
sqlx::query_scalar!(
"SELECT value FROM flow WHERE path = $1 AND workspace_id = $2",
@@ -2528,7 +2530,7 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
(
None,
Some(path),
- None,
+ Some((deployment_message.unwrap_or_else(String::new), None)),
JobKind::FlowDependencies,
Some(value.clone()),
Some(FlowStatus::new(&value)), // this is a new flow being pushed, flow_status is set to flow_value
@@ -2540,10 +2542,10 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
None,
)
}
- JobPayload::AppDependencies { path, version } => (
+ JobPayload::AppDependencies { path, version, deployment_message } => (
Some(version),
Some(path),
- None,
+ Some((deployment_message.unwrap_or_else(String::new), None)),
JobKind::AppDependencies,
None,
None,
diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml
index 74966c0a7e..de8fd33767 100644
--- a/backend/windmill-worker/Cargo.toml
+++ b/backend/windmill-worker/Cargo.toml
@@ -10,7 +10,7 @@ path = "src/lib.rs"
[features]
default = []
-enterprise = ["windmill-queue/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:pem", "dep:sha2", "dep:tiberius", "dep:tokio-util"]
+enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:pem", "dep:sha2", "dep:tiberius", "dep:tokio-util"]
benchmark = ["windmill-queue/benchmark"]
flamegraph = []
@@ -31,6 +31,7 @@ windmill-parser-py-imports.workspace = true
windmill-parser-bash.workspace = true
windmill-parser-sql.workspace = true
windmill-parser-graphql.workspace = true
+windmill-git-sync.workspace = true
sqlx.workspace = true
uuid.workspace = true
tracing.workspace = true
diff --git a/backend/windmill-worker/loader.py b/backend/windmill-worker/loader.py
index b94cece909..b04ac3580a 100644
--- a/backend/windmill-worker/loader.py
+++ b/backend/windmill-worker/loader.py
@@ -28,22 +28,23 @@ class WindmillFinder(MetaPathFinder):
return ModuleSpec(name, WindmillLoader(name))
elif l > 2:
script_path = "/".join(splitted)
- import requests
+ import urllib.parse
+ import urllib.request
+ headers = {"Authorization": f"Bearer {os.environ.get('WM_TOKEN')}"}
url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py"
- r = requests.get(
- url, headers={"Authorization": f"Bearer {os.environ.get('WM_TOKEN')}"}
- )
-
- if r.status_code == 200:
- folder = os.getcwd() + "/tmp/" + "/".join(splitted[:-1])
- fullpath = folder + "/" + splitted[-1] + ".py"
- os.makedirs(folder, exist_ok=True)
- with open(fullpath, "w+") as f:
- f.write(r.text)
- return ModuleSpec(name, SourceFileLoader(name, fullpath))
- else:
+ req = urllib.request.Request(url, None, headers)
+ try:
+ with urllib.request.urlopen(req) as response:
+ r = response.read().decode("utf-8")
+ folder = os.getcwd() + "/tmp/" + "/".join(splitted[:-1])
+ fullpath = folder + "/" + splitted[-1] + ".py"
+ os.makedirs(folder, exist_ok=True)
+ with open(fullpath, "w+") as f:
+ f.write(r)
+ return ModuleSpec(name, SourceFileLoader(name, fullpath))
+ except:
# raise ImportError(f"Script {script_path} not found")
return ModuleSpec(name, WindmillLoader(name))
diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs
index 4d9da070d6..5d8047c737 100644
--- a/backend/windmill-worker/src/common.rs
+++ b/backend/windmill-worker/src/common.rs
@@ -32,7 +32,7 @@ use std::{
use tracing::{trace_span, Instrument};
use uuid::Uuid;
-use windmill_common::{variables, DB};
+use windmill_common::{job_metrics, variables, DB};
use tokio::{
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
@@ -408,6 +408,7 @@ async fn get_mem_peak(pid: Option, nsjail: bool) -> i32 {
}
-2
} else {
+ // rand::random::() % 100 // to remove - used to fake memory data on MacOS
-3
}
}
@@ -427,7 +428,7 @@ pub async fn handle_child(
mut child: Child,
nsjail: bool,
worker_name: &str,
- _w_id: &str,
+ w_id: &str,
child_name: &str,
custom_timeout: Option,
sigterm: bool,
@@ -465,6 +466,8 @@ pub async fn handle_child(
interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
let mut i = 0;
+ let mut memory_metric_id: Result =
+ Err(Error::NotFound("not yet initialized".to_string()));
loop {
tokio::select!(
@@ -485,7 +488,29 @@ pub async fn handle_child(
if current_mem > *mem_peak {
*mem_peak = current_mem
}
- tracing::info!("{worker_name}/{job_id} in {_w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB");
+ tracing::info!("{worker_name}/{job_id} in {w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB");
+
+ #[cfg(feature = "enterprise")]
+ {
+ // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
+ if i == 2 {
+ memory_metric_id = job_metrics::register_metric_for_job(
+ &db,
+ w_id.to_string(),
+ job_id,
+ "memory_kb".to_string(),
+ job_metrics::MetricKind::TimeseriesInt,
+ Some("Job Memory Footprint (kB)".to_string()),
+ )
+ .await;
+ }
+ if let Ok(ref metric_id) = memory_metric_id {
+ if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
+ tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
+ }
+ }
+ }
+
let (canceled, canceled_by, canceled_reason) = sqlx::query_as::<_, (bool, Option, Option)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason")
.bind(*mem_peak)
.bind(job_id)
@@ -516,7 +541,7 @@ pub async fn handle_child(
}
let (timeout_duration, timeout_warn_msg) =
- resolve_job_timeout(&db, _w_id, job_id, custom_timeout).await;
+ resolve_job_timeout(&db, w_id, job_id, custom_timeout).await;
if let Some(msg) = timeout_warn_msg {
logs.push_str(msg.as_str());
append_logs(job_id, msg.as_str(), db).await;
diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs
index ba71f5424c..bee6887515 100644
--- a/backend/windmill-worker/src/pg_executor.rs
+++ b/backend/windmill-worker/src/pg_executor.rs
@@ -322,7 +322,13 @@ fn convert_val(value: &Value, arg_t: &String) -> windmill_common::error::Result<
Value::Number(n) if n.is_i64() && (arg_t == "smallint" || arg_t == "smallserial") => {
Ok(PgType::I16(n.as_i64().unwrap() as i16))
}
- Value::Number(n) if n.is_i64() && (arg_t == "int" || arg_t == "serial") => {
+ Value::Number(n)
+ if n.is_i64()
+ && (arg_t == "int"
+ || arg_t == "integer"
+ || arg_t == "int4"
+ || arg_t == "serial") =>
+ {
Ok(PgType::I32(n.as_i64().unwrap() as i32))
}
Value::Number(n) if n.is_i64() && (arg_t == "numeric" || arg_t == "decimal") => Ok(
@@ -429,7 +435,6 @@ pub fn pg_cell_to_json_value(
Type::TS_VECTOR => get_basic(row, column, column_i, |a: StringCollector| {
Ok(JSONValue::String(a.0))
})?,
-
// array types
Type::BOOL_ARRAY => get_array(row, column, column_i, |a: bool| Ok(JSONValue::Bool(a)))?,
Type::INT2_ARRAY => get_array(row, column, column_i, |a: i16| {
diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs
index b421b2ae28..b0c9d31195 100644
--- a/backend/windmill-worker/src/worker.rs
+++ b/backend/windmill-worker/src/worker.rs
@@ -25,6 +25,8 @@ use std::{
},
time::Duration,
};
+use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
+use windmill_parser_py_imports::parse_relative_imports;
use uuid::Uuid;
#[cfg(feature = "enterprise")]
@@ -33,9 +35,9 @@ use windmill_common::{
error::{self, to_anyhow, Error},
flows::{FlowModule, FlowModuleValue, FlowValue},
get_latest_deployed_hash_for_path,
- jobs::{JobKind, QueuedJob},
+ jobs::{JobKind, JobPayload, QueuedJob},
scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
- users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL},
+ users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL},
utils::{rd_string, StripPath},
worker::{
to_raw_value, to_raw_value_owned, update_ping, CLOUD_HOSTED, WORKER_CONFIG, WORKER_GROUP,
@@ -149,7 +151,8 @@ pub async fn create_token_for_owner(
.await?
.unwrap_or(false)
|| email == SUPERADMIN_SECRET_EMAIL
- || email == SUPERADMIN_NOTIFICATION_EMAIL;
+ || email == SUPERADMIN_NOTIFICATION_EMAIL
+ || owner == SUPERADMIN_SYNC_EMAIL;
sqlx::query_scalar!(
"INSERT INTO token
@@ -472,6 +475,7 @@ async fn handle_receive_completed_job<
worker_name: &str,
worker_save_completed_job_duration: Option>,
worker_flow_transition_duration: Option>,
+ job_completed_tx: Sender,
) {
let token = jc.token.clone();
let workspace = jc.job.workspace_id.clone();
@@ -494,6 +498,7 @@ async fn handle_receive_completed_job<
worker_name,
worker_save_completed_job_duration,
worker_flow_transition_duration,
+ job_completed_tx.clone(),
)
.await
{
@@ -509,6 +514,7 @@ async fn handle_receive_completed_job<
&worker_dir,
rsmq.clone(),
worker_name,
+ job_completed_tx,
)
.await;
}
@@ -516,7 +522,7 @@ async fn handle_receive_completed_job<
#[derive(Clone)]
pub struct JobCompletedSender(
- Sender,
+ Sender,
Option>>,
Option>,
);
@@ -525,12 +531,12 @@ impl JobCompletedSender {
pub async fn send(
&self,
jc: JobCompleted,
- ) -> Result<(), tokio::sync::mpsc::error::SendError> {
+ ) -> Result<(), tokio::sync::mpsc::error::SendError> {
if let Some(wj) = self.1.as_ref() {
wj.inc()
}
let timer = self.2.as_ref().map(|x| x.start_timer());
- let r = self.0.send(jc).await;
+ let r = self.0.send(SendResult::JobCompleted(jc)).await;
timer.map(|x| x.stop_and_record());
r
}
@@ -876,7 +882,7 @@ pub async fn run_worker(5);
- let (job_completed_tx, mut job_completed_rx) = mpsc::channel::(3);
+ let (job_completed_tx, mut job_completed_rx) = mpsc::channel::(3);
let job_completed_tx = JobCompletedSender(
job_completed_tx,
@@ -964,119 +970,167 @@ pub async fn run_worker {
+ if let Some(wj) = worker_job_completed_channel_queue2.as_ref() {
+ wj.dec();
}
- tokio::time::sleep(Duration::from_millis(3)).await;
- }
+ let base_internal_url2 = base_internal_url2.clone();
+ let worker_dir2 = worker_dir2.clone();
+ let db2 = db2.clone();
+ let same_worker_tx2 = same_worker_tx2.clone();
+ let rsmq2 = rsmq2.clone();
+ let worker_name = worker_name2.clone();
+ if matches!(jc.job.job_kind, JobKind::Noop) || is_dedicated_worker {
+ thread_count.fetch_add(1, Ordering::SeqCst);
+ let thread_count = thread_count.clone();
- #[cfg(feature = "benchmark")]
- let send_duration = send_duration2.clone();
- #[cfg(feature = "benchmark")]
- let process_duration = process_duration.clone();
- #[cfg(feature = "benchmark")]
- let completed_jobs = completed_jobs.clone();
- #[cfg(feature = "benchmark")]
- let main_duration = main_duration2.clone();
-
- let worker_save_completed_job_duration2 =
- worker_save_completed_job_duration.clone();
- let worker_flow_transition_duration2 = worker_flow_transition_duration.clone();
- let killpill_tx = killpill_tx2.clone();
- tokio::spawn(async move {
- #[cfg(feature = "benchmark")]
- let process_start = Instant::now();
-
- let is_dependency_job = matches!(
- jc.job.job_kind,
- JobKind::Dependencies | JobKind::FlowDependencies
- );
- handle_receive_completed_job(
- jc,
- base_internal_url2,
- db2.clone(),
- worker_dir2,
- same_worker_tx2,
- rsmq2,
- &worker_name,
- worker_save_completed_job_duration2.clone(),
- worker_flow_transition_duration2.clone(),
- )
- .await;
- #[cfg(feature = "benchmark")]
- {
- let n = completed_jobs.fetch_add(1, Ordering::SeqCst);
- if (n + 1) % 1000 == 0 || n == (jobs - 1) as usize {
- let duration_s = start.elapsed().as_secs_f64();
- let jobs_per_sec = n as f64 / duration_s;
- tracing::info!(
- "completed {} jobs in {}s, {} jobs/s",
- n + 1,
- duration_s,
- jobs_per_sec
- );
-
- tracing::info!(
- "main loop without send {}s",
- main_duration.load(Ordering::SeqCst) as f64 / 1000.0
- );
-
- tracing::info!(
- "send job completed / send dedicated job duration {}s",
- send_duration.load(Ordering::SeqCst) as f64 / 1000.0
- );
-
- tracing::info!(
- "job completed process duration {}s",
- process_duration.load(Ordering::SeqCst) as f64 / 1000.0
- );
+ loop {
+ if thread_count.load(Ordering::Relaxed) < 4 {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(3)).await;
}
- process_duration.fetch_add(
- process_start.elapsed().as_millis() as usize,
- Ordering::SeqCst,
- );
- }
+ #[cfg(feature = "benchmark")]
+ let send_duration = send_duration2.clone();
+ #[cfg(feature = "benchmark")]
+ let process_duration = process_duration.clone();
+ #[cfg(feature = "benchmark")]
+ let completed_jobs = completed_jobs.clone();
+ #[cfg(feature = "benchmark")]
+ let main_duration = main_duration2.clone();
- thread_count.fetch_sub(1, Ordering::SeqCst);
- if is_dependency_job && is_dedicated_worker {
- tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted.");
- sqlx::query!("UPDATE config SET config = config WHERE name = $1", format!("worker__{}", *WORKER_GROUP))
+ let worker_save_completed_job_duration2 =
+ worker_save_completed_job_duration.clone();
+ let worker_flow_transition_duration2 =
+ worker_flow_transition_duration.clone();
+ let killpill_tx = killpill_tx2.clone();
+ let job_completed_sender = job_completed_sender.clone();
+ tokio::spawn(async move {
+ #[cfg(feature = "benchmark")]
+ let process_start = Instant::now();
+
+ let is_dependency_job = matches!(
+ jc.job.job_kind,
+ JobKind::Dependencies | JobKind::FlowDependencies
+ );
+ handle_receive_completed_job(
+ jc,
+ base_internal_url2,
+ db2.clone(),
+ worker_dir2,
+ same_worker_tx2,
+ rsmq2,
+ &worker_name,
+ worker_save_completed_job_duration2.clone(),
+ worker_flow_transition_duration2.clone(),
+ job_completed_sender.clone(),
+ )
+ .await;
+ #[cfg(feature = "benchmark")]
+ {
+ let n = completed_jobs.fetch_add(1, Ordering::SeqCst);
+ if (n + 1) % 1000 == 0 || n == (jobs - 1) as usize {
+ let duration_s = start.elapsed().as_secs_f64();
+ let jobs_per_sec = n as f64 / duration_s;
+ tracing::info!(
+ "completed {} jobs in {}s, {} jobs/s",
+ n + 1,
+ duration_s,
+ jobs_per_sec
+ );
+
+ tracing::info!(
+ "main loop without send {}s",
+ main_duration.load(Ordering::SeqCst) as f64 / 1000.0
+ );
+
+ tracing::info!(
+ "send job completed / send dedicated job duration {}s",
+ send_duration.load(Ordering::SeqCst) as f64 / 1000.0
+ );
+
+ tracing::info!(
+ "job completed process duration {}s",
+ process_duration.load(Ordering::SeqCst) as f64 / 1000.0
+ );
+ }
+
+ process_duration.fetch_add(
+ process_start.elapsed().as_millis() as usize,
+ Ordering::SeqCst,
+ );
+ }
+
+ thread_count.fetch_sub(1, Ordering::SeqCst);
+ if is_dependency_job && is_dedicated_worker {
+ tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted.");
+ sqlx::query!("UPDATE config SET config = config WHERE name = $1", format!("worker__{}", *WORKER_GROUP))
.execute(&db2)
.await
.expect("update config to trigger restart of all dedicated workers at that config");
- killpill_tx.send(()).unwrap_or_default();
+ killpill_tx.send(()).unwrap_or_default();
+ }
+ });
+ } else {
+ handle_receive_completed_job(
+ jc,
+ base_internal_url2,
+ db2,
+ worker_dir2,
+ same_worker_tx2,
+ rsmq2,
+ &worker_name,
+ worker_save_completed_job_duration2.clone(),
+ worker_flow_transition_duration2.clone(),
+ job_completed_sender.clone(),
+ )
+ .await;
}
- });
- } else {
- handle_receive_completed_job(
- jc,
- base_internal_url2,
- db2,
- worker_dir2,
- same_worker_tx2,
- rsmq2,
- &worker_name,
- worker_save_completed_job_duration2.clone(),
- worker_flow_transition_duration2.clone(),
- )
- .await;
+ }
+ SendResult::UpdateFlow {
+ flow,
+ w_id,
+ success,
+ result,
+ worker_dir,
+ stop_early_override,
+ token,
+ } => {
+ // let r;
+ if let Err(e) = update_flow_status_after_job_completion(
+ &db2,
+ &AuthedClient {
+ base_internal_url: base_internal_url2.to_string(),
+ workspace: w_id.clone(),
+ token: token.clone(),
+ force_client: None,
+ },
+ flow,
+ &Uuid::nil(),
+ &w_id,
+ success,
+ &result,
+ true,
+ same_worker_tx2.clone(),
+ &worker_dir,
+ stop_early_override,
+ rsmq2.clone(),
+ &worker_name2,
+ job_completed_sender.clone(),
+ )
+ .await
+ {
+ tracing::error!("Error updating flow status after job completion: {e}");
+ }
+ }
+ SendResult::Kill => {
+ break;
+ }
}
}
@@ -1333,6 +1387,7 @@ pub async fn run_worker {
@@ -1600,6 +1655,7 @@ pub async fn run_worker>,
worker_flow_transition_duration: Option>,
+ job_completed_tx: Sender,
) -> windmill_common::error::Result<()> {
if success {
// println!("bef completed job{:?}", SystemTime::now());
@@ -2126,6 +2183,7 @@ pub async fn process_completed_job,
worker_name: &str,
+ job_completed_tx: Sender,
) {
let err = match err {
Error::JsonErr(err) => err,
@@ -2263,6 +2323,7 @@ pub async fn handle_job_error Box {
);
}
+pub enum SendResult {
+ JobCompleted(JobCompleted),
+ UpdateFlow {
+ flow: Uuid,
+ w_id: String,
+ success: bool,
+ result: Box,
+ worker_dir: String,
+ stop_early_override: Option,
+ token: String,
+ },
+ Kill,
+}
+
+// db: &DB,
+// client: &AuthedClient,
+// flow: uuid::Uuid,
+// job_id_for_status: &Uuid,
+// w_id: &str,
+// success: bool,
+// result: &'a RawValue,
+// unrecoverable: bool,
+// same_worker_tx: Sender,
+// worker_dir: &str,
+// stop_early_override: Option,
+// rsmq: Option,
+// worker_name: &str,
+
#[derive(Debug, Clone)]
pub struct JobCompleted {
pub job: Arc,
@@ -2467,7 +2556,7 @@ async fn handle_queued_job(
same_worker_tx,
worker_dir,
rsmq,
- worker_name,
+ job_completed_tx.0.clone(),
)
.await?;
timer.map(|x| x.stop_and_record());
@@ -2515,6 +2604,7 @@ async fn handle_queued_job(
worker_dir,
base_internal_url,
&client.get_token().await,
+ rsmq.clone(),
)
.await
}
@@ -2529,6 +2619,7 @@ async fn handle_queued_job(
worker_dir,
base_internal_url,
&client.get_token().await,
+ rsmq.clone(),
)
.await
.map(|()| serde_json::from_str("{}").unwrap()),
@@ -2543,6 +2634,7 @@ async fn handle_queued_job(
worker_dir,
base_internal_url,
&client.get_token().await,
+ rsmq.clone(),
)
.await
.map(|()| serde_json::from_str("{}").unwrap()),
@@ -3044,7 +3136,7 @@ mount {{
}
#[tracing::instrument(level = "trace", skip_all)]
-async fn handle_dependency_job(
+async fn handle_dependency_job(
job: &QueuedJob,
logs: &mut String,
mem_peak: &mut i32,
@@ -3055,7 +3147,17 @@ async fn handle_dependency_job(
worker_dir: &str,
base_internal_url: &str,
token: &str,
+ rsmq: Option,
) -> error::Result> {
+ let raw_code = sqlx::query_scalar!(
+ "SELECT content FROM script WHERE hash = $1 AND workspace_id = $2",
+ &job.script_hash.unwrap_or(ScriptHash(0)).0,
+ &job.workspace_id
+ )
+ .fetch_optional(db)
+ .await?
+ .unwrap_or_else(|| "No script found at this hash".to_string());
+ let script_path = job.script_path();
let content = capture_dependency_job(
&job.id,
job.language.as_ref().map(|v| Ok(v)).unwrap_or_else(|| {
@@ -3063,10 +3165,7 @@ async fn handle_dependency_job(
"Job Language required for dependency jobs".to_owned(),
))
})?,
- job.raw_code
- .as_ref()
- .map(|a| a.as_str())
- .unwrap_or_else(|| "no raw code"),
+ &raw_code,
logs,
mem_peak,
canceled_by,
@@ -3077,19 +3176,83 @@ async fn handle_dependency_job(
worker_dir,
base_internal_url,
token,
- job.script_path(),
+ script_path,
)
.await;
match content {
Ok(content) => {
+ let hash = job.script_hash.unwrap_or(ScriptHash(0));
+ let w_id = &job.workspace_id;
sqlx::query!(
"UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3",
&content,
- &job.script_hash.unwrap_or(ScriptHash(0)).0,
- &job.workspace_id
+ &hash.0,
+ w_id
)
.execute(db)
.await?;
+
+ let deployment_message = job.raw_code.as_ref().map(|x| x.clone());
+
+ if let Err(e) = handle_deployment_metadata(
+ &job.email,
+ &job.created_by,
+ &db,
+ &w_id,
+ DeployedObject::Script { hash, path: script_path.to_string() },
+ deployment_message.clone(),
+ rsmq.clone(),
+ )
+ .await
+ {
+ tracing::error!(%e, "error handling deployment metadata");
+ }
+
+ if &job.language == &Some(ScriptLang::Python3) {
+ if let Ok(relative_imports) = parse_relative_imports(&raw_code, script_path) {
+ logs.push_str("\n--- RELATIVE IMPORTS ---\n\n");
+ logs.push_str(&relative_imports.join("\n"));
+ if !relative_imports.is_empty() {
+ let mut tx = db.begin().await?;
+ sqlx::query!(
+ "DELETE FROM dependency_map
+ WHERE importer_path = $1 AND importer_kind = 'script'
+ AND workspace_id = $2",
+ script_path,
+ w_id
+ )
+ .execute(&mut *tx)
+ .await?;
+ for import in relative_imports {
+ sqlx::query!(
+ "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path)
+ VALUES ($1, $2, 'script', $3)",
+ w_id,
+ script_path,
+ import
+ )
+ .execute(&mut *tx)
+ .await?;
+ logs.push_str(&format!("{}\n", import));
+ }
+ tx.commit().await?;
+ }
+ if let Err(e) = trigger_python_dependents_to_recompute_dependencies(
+ w_id,
+ script_path,
+ deployment_message,
+ &job.email,
+ &job.created_by,
+ &job.permissioned_as,
+ db,
+ rsmq,
+ )
+ .await
+ {
+ tracing::error!(%e, "error triggering python dependents to recompute dependencies");
+ }
+ }
+ }
Ok(to_raw_value_owned(
json!({ "success": "Successful lock file generation", "lock": content }),
))
@@ -3108,7 +3271,80 @@ async fn handle_dependency_job(
}
}
-async fn handle_flow_dependency_job(
+async fn trigger_python_dependents_to_recompute_dependencies<
+ R: rsmq_async::RsmqConnection + Send + Sync + Clone,
+>(
+ w_id: &str,
+ script_path: &str,
+ deployment_message: Option,
+ email: &str,
+ created_by: &str,
+ permissioned_as: &str,
+ db: &sqlx::Pool,
+ rsmq: Option,
+) -> error::Result<()> {
+ let script_importers = sqlx::query_scalar!(
+ "SELECT importer_path FROM dependency_map
+ WHERE imported_path = $1 AND importer_kind = 'script'
+ AND workspace_id = $2",
+ script_path,
+ w_id
+ )
+ .fetch_all(db)
+ .await?;
+ for s in script_importers.iter() {
+ let tx: PushIsolationLevel<'_, R> =
+ PushIsolationLevel::IsolatedRoot(db.clone(), rsmq.clone());
+ let r = get_latest_deployed_hash_for_path(db, w_id, s.as_str()).await;
+ if let Ok(r) = r {
+ let (job_uuid, new_tx) = windmill_queue::push(
+ db,
+ tx,
+ &w_id,
+ JobPayload::Dependencies {
+ path: s.clone(),
+ hash: r.0,
+ language: r.5,
+ dedicated_worker: r.6,
+ deployment_message: deployment_message.clone(),
+ },
+ PushArgs::empty(),
+ &created_by,
+ email,
+ permissioned_as.to_string(),
+ None,
+ None,
+ None,
+ None,
+ None,
+ false,
+ false,
+ None,
+ true,
+ None,
+ None,
+ None,
+ None,
+ )
+ .await?;
+ tracing::info!(
+ "pushed dependency job due to common python path: {job_uuid} for path {path} with hash {hash}",
+ path = s,
+ hash = r.0
+ );
+ new_tx.commit().await?;
+ } else {
+ tracing::error!(
+ "error getting latest deployed hash for path {path}: {err}",
+ path = s,
+ err = r.unwrap_err()
+ );
+ }
+ }
+ Ok(())
+}
+
+async fn handle_flow_dependency_job(
job: &QueuedJob,
logs: &mut String,
mem_peak: &mut i32,
@@ -3119,6 +3355,7 @@ async fn handle_flow_dependency_job(
worker_dir: &str,
base_internal_url: &str,
token: &str,
+ rsmq: Option,
) -> error::Result<()> {
let job_path = job.script_path.clone().ok_or_else(|| {
error::Error::InternalErr(
@@ -3170,6 +3407,21 @@ async fn handle_flow_dependency_job(
)
.execute(db)
.await?;
+
+ if let Err(e) = handle_deployment_metadata(
+ &job.email,
+ &job.created_by,
+ &db,
+ &job.workspace_id,
+ DeployedObject::Flow { path: job_path },
+ job.raw_code.as_ref().map(|x| x.clone()),
+ rsmq.clone(),
+ )
+ .await
+ {
+ tracing::error!(%e, "error handling deployment metadata");
+ }
+
Ok(())
}
@@ -3295,22 +3547,11 @@ async fn lock_modules(
new_flow_modules.push(e);
continue;
};
- // sync with windmill-api/scripts
- let dependencies = match language {
- ScriptLang::Python3 => windmill_parser_py_imports::parse_python_imports(
- &content,
- &job.workspace_id,
- &path.clone().unwrap_or_else(|| job_path.to_string()),
- &db,
- )
- .await?
- .join("\n"),
- _ => content.clone(),
- };
+
let new_lock = capture_dependency_job(
&job.id,
&language,
- &dependencies,
+ &content,
logs,
mem_peak,
canceled_by,
@@ -3321,7 +3562,7 @@ async fn lock_modules(
worker_dir,
base_internal_url,
token,
- job.script_path(),
+ &path.clone().unwrap_or_else(|| job_path.to_string()),
)
.await;
match new_lock {
@@ -3396,24 +3637,11 @@ async fn lock_modules_app(
.as_str()
.unwrap_or_default()
.to_string();
- let dependencies = match language {
- ScriptLang::Python3 => {
- windmill_parser_py_imports::parse_python_imports(
- &content,
- &job.workspace_id,
- job_path,
- &db,
- )
- .await?
- .join("\n")
- }
- _ => content.clone(),
- };
logs.push_str("Found lockable inline script. Generating lock...\n");
let new_lock = capture_dependency_job(
&job.id,
&language,
- &dependencies,
+ &content,
logs,
mem_peak,
canceled_by,
@@ -3498,7 +3726,7 @@ async fn lock_modules_app(
}
}
-async fn handle_app_dependency_job(
+async fn handle_app_dependency_job(
job: &QueuedJob,
logs: &mut String,
mem_peak: &mut i32,
@@ -3509,6 +3737,7 @@ async fn handle_app_dependency_job(
worker_dir: &str,
base_internal_url: &str,
token: &str,
+ rsmq: Option,
) -> error::Result<()> {
let job_path = job.script_path.clone().ok_or_else(|| {
error::Error::InternalErr(
@@ -3558,6 +3787,41 @@ async fn handle_app_dependency_job(
sqlx::query!("UPDATE app_version SET value = $1 WHERE id = $2", value, id,)
.execute(db)
.await?;
+
+ if let Err(e) = handle_deployment_metadata(
+ &job.email,
+ &job.created_by,
+ &db,
+ &job.workspace_id,
+ DeployedObject::App { path: job_path, version: id },
+ job.raw_code.as_ref().map(|x| x.clone()),
+ rsmq.clone(),
+ )
+ .await
+ {
+ tracing::error!(%e, "error handling deployment metadata");
+ }
+
+ // tx = PushIsolationLevel::Transaction(new_tx);
+ // tx = handle_deployment_metadata(
+ // tx,
+ // &authed,
+ // &db,
+ // &w_id,
+ // DeployedObject::App { path: app.path.clone(), version: v_id },
+ // app.deployment_message,
+ // )
+ // .await?;
+
+ // match tx {
+ // PushIsolationLevel::Transaction(tx) => tx.commit().await?,
+ // _ => {
+ // return Err(Error::InternalErr(
+ // "Expected a transaction here".to_string(),
+ // ));
+ // }
+ // }
+
Ok(())
} else {
Ok(())
@@ -3582,10 +3846,18 @@ async fn capture_dependency_job(
) -> error::Result {
match job_language {
ScriptLang::Python3 => {
+ let reqs = windmill_parser_py_imports::parse_python_imports(
+ job_raw_code,
+ &w_id,
+ script_path,
+ &db,
+ )
+ .await?
+ .join("\n");
create_dependencies_dir(job_dir).await;
let req: std::result::Result = pip_compile(
job_id,
- job_raw_code,
+ &reqs,
logs,
mem_peak,
canceled_by,
diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs
index aa71fa640b..5468407f69 100644
--- a/backend/windmill-worker/src/worker_flow.rs
+++ b/backend/windmill-worker/src/worker_flow.rs
@@ -15,9 +15,8 @@ use std::time::Duration;
use crate::common::{hash_args, save_in_cache};
use crate::js_eval::{eval_timeout, IdContext};
-use crate::{AuthedClient, PreviousResult, KEEP_JOB_DIR};
+use crate::{AuthedClient, PreviousResult, SendResult, KEEP_JOB_DIR};
use anyhow::Context;
-use async_recursion::async_recursion;
use serde::Serialize;
use serde_json::value::RawValue;
use serde_json::{json, Value};
@@ -70,8 +69,10 @@ pub async fn update_flow_status_after_job_completion<
stop_early_override: Option,
rsmq: Option,
worker_name: &str,
+ job_completed_tx: Sender,
) -> error::Result<()> {
// this is manual tailrecursion because async_recursion blows up the stack
+ // todo!();
let mut rec = update_flow_status_after_job_completion_internal(
db,
client,
@@ -87,6 +88,7 @@ pub async fn update_flow_status_after_job_completion<
false,
rsmq.clone(),
worker_name,
+ job_completed_tx.clone(),
)
.await?;
while let Some(nrec) = rec {
@@ -105,6 +107,7 @@ pub async fn update_flow_status_after_job_completion<
nrec.skip_error_handler,
rsmq.clone(),
worker_name,
+ job_completed_tx.clone(),
)
.await
{
@@ -125,6 +128,7 @@ pub async fn update_flow_status_after_job_completion<
nrec.skip_error_handler,
rsmq.clone(),
worker_name,
+ job_completed_tx.clone(),
)
.await?
}
@@ -167,6 +171,7 @@ pub async fn update_flow_status_after_job_completion_internal<
skip_error_handler: bool,
rsmq: Option,
worker_name: &str,
+ job_completed_tx: Sender,
) -> error::Result> {
let (
should_continue_flow,
@@ -575,6 +580,8 @@ pub async fn update_flow_status_after_job_completion_internal<
}
tx.commit().await?;
+ tracing::debug!(id = %flow_job.id, root_id = %job_root, "flow status updated");
+
(
should_continue_flow,
flow_job,
@@ -686,6 +693,7 @@ pub async fn update_flow_status_after_job_completion_internal<
}
true
} else {
+ tracing::debug!(id = %flow_job.id, "start handle flow");
match handle_flow(
&flow_job,
db,
@@ -694,7 +702,7 @@ pub async fn update_flow_status_after_job_completion_internal<
same_worker_tx.clone(),
worker_dir,
rsmq.clone(),
- worker_name,
+ job_completed_tx,
)
.await
{
@@ -1037,7 +1045,7 @@ pub async fn handle_flow(
same_worker_tx: Sender,
worker_dir: &str,
rsmq: Option,
- worker_name: &str,
+ job_completed_tx: Sender,
) -> anyhow::Result<()> {
let flow = flow_job
.parse_raw_flow()
@@ -1056,7 +1064,7 @@ pub async fn handle_flow(
same_worker_tx,
worker_dir,
rsmq,
- worker_name,
+ job_completed_tx,
)
.await?;
Ok(())
@@ -1089,7 +1097,7 @@ pub struct RawArgs {
pub args: Option>>>,
}
-#[async_recursion]
+// #[async_recursion]
// #[instrument(level = "trace", skip_all)]
async fn push_next_flow_job(
flow_job: &QueuedJob,
@@ -1101,7 +1109,7 @@ async fn push_next_flow_job
same_worker_tx: Sender,
worker_dir: &str,
rsmq: Option,
- worker_name: &str,
+ job_completed_tx: Sender,
) -> error::Result<()> {
let job_root = flow_job
.root_job
@@ -1122,29 +1130,51 @@ async fn push_next_flow_job
// if this is an empty module of if the module has already been completed, successfully, update the parent flow
if flow.modules.is_empty() || matches!(status_module, FlowStatusModule::Success { .. }) {
- let r;
- return update_flow_status_after_job_completion(
- db,
- client,
- flow_job.id,
- &Uuid::nil(),
- flow_job.workspace_id.as_str(),
- true,
- if flow.modules.is_empty() {
- r = to_raw_value(&flow_job_args);
- &r
- } else {
- // it has to be an empty for loop event
- serde_json::from_str("[]").unwrap()
- },
- true,
- same_worker_tx,
- worker_dir,
- None,
- rsmq,
- worker_name,
- )
- .await;
+ job_completed_tx
+ .send(SendResult::UpdateFlow {
+ flow: flow_job.id,
+ success: true,
+ result: if flow.modules.is_empty() {
+ to_raw_value(&flow_job_args)
+ } else {
+ // it has to be an empty for loop event
+ serde_json::from_str("[]").unwrap()
+ },
+ stop_early_override: None,
+ w_id: flow_job.workspace_id.clone(),
+ worker_dir: worker_dir.to_string(),
+ token: client.token.clone(),
+ })
+ .await
+ .map_err(|e| {
+ Error::InternalErr(format!(
+ "error sending update flow message to job completed channel: {e}"
+ ))
+ })?;
+ // let r;
+ // return update_flow_status_after_job_completion(
+ // db,
+ // client,
+ // flow_job.id,
+ // &Uuid::nil(),
+ // flow_job.workspace_id.as_str(),
+ // true,
+ // if flow.modules.is_empty() {
+ // r = to_raw_value(&flow_job_args);
+ // &r
+ // } else {
+ // // it has to be an empty for loop event
+ // serde_json::from_str("[]").unwrap()
+ // },
+ // true,
+ // same_worker_tx,
+ // worker_dir,
+ // None,
+ // rsmq,
+ // worker_name,
+ // )
+ // .await;
+ return Ok(());
}
let arc_flow_job_args = Arc::new(flow_job_args.clone());
@@ -1166,25 +1196,45 @@ async fn push_next_flow_job
flow_job.id
).fetch_one(db).await?.unwrap_or(0);
if count > 0 {
- return update_flow_status_after_job_completion(
- db,
- client,
- flow_job.id,
- &Uuid::nil(),
- flow_job.workspace_id.as_str(),
- true,
- serde_json::from_str(
- "\"not allowed to overlap, scheduling next iteration\"",
- )
- .unwrap(),
- true,
- same_worker_tx,
- worker_dir,
- Some(true),
- rsmq,
- worker_name,
- )
- .await;
+ job_completed_tx
+ .send(SendResult::UpdateFlow {
+ flow: flow_job.id,
+ success: true,
+ result: serde_json::from_str(
+ "\"not allowed to overlap, scheduling next iteration\"",
+ )
+ .unwrap(),
+ stop_early_override: Some(true),
+ w_id: flow_job.workspace_id.clone(),
+ worker_dir: worker_dir.to_string(),
+ token: client.token.clone(),
+ })
+ .await
+ .map_err(|e| {
+ Error::InternalErr(format!(
+ "error sending update flow message to job completed channel: {e}"
+ ))
+ })?;
+ // return update_flow_status_after_job_completion(
+ // db,
+ // client,
+ // flow_job.id,
+ // &Uuid::nil(),
+ // flow_job.workspace_id.as_str(),
+ // true,
+ // serde_json::from_str(
+ // "\"not allowed to overlap, scheduling next iteration\"",
+ // )
+ // .unwrap(),
+ // true,
+ // same_worker_tx,
+ // worker_dir,
+ // Some(true),
+ // rsmq,
+ // worker_name,
+ // )
+ // .await;
+ return Ok(());
}
}
}
@@ -1199,22 +1249,40 @@ async fn push_next_flow_job
)
.await?;
if skip {
- return update_flow_status_after_job_completion(
- db,
- client,
- flow_job.id,
- &Uuid::nil(),
- flow_job.workspace_id.as_str(),
- true,
- serde_json::from_str("\"stopped early\"").unwrap(),
- true,
- same_worker_tx,
- worker_dir,
- Some(true),
- rsmq,
- worker_name,
- )
- .await;
+ job_completed_tx
+ .send(SendResult::UpdateFlow {
+ flow: flow_job.id,
+ success: true,
+ result: serde_json::from_str("\"stopped early\"").unwrap(),
+ stop_early_override: Some(true),
+ w_id: flow_job.workspace_id.clone(),
+ worker_dir: worker_dir.to_string(),
+ token: client.token.clone(),
+ })
+ .await
+ .map_err(|e| {
+ Error::InternalErr(format!(
+ "error sending update flow message to job completed channel: {e}"
+ ))
+ })?;
+
+ // return update_flow_status_after_job_completion(
+ // db,
+ // client,
+ // flow_job.id,
+ // &Uuid::nil(),
+ // flow_job.workspace_id.as_str(),
+ // true,
+ // serde_json::from_str("\"stopped early\"").unwrap(),
+ // true,
+ // same_worker_tx,
+ // worker_dir,
+ // Some(true),
+ // rsmq,
+ // worker_name,
+ // )
+ // .await;
+ return Ok(());
}
}
}
@@ -1436,22 +1504,39 @@ async fn push_next_flow_job
.await?;
if flow_job.is_flow_step {
if let Some(parent_job) = flow_job.parent_job {
- update_flow_status_after_job_completion(
- db,
- client,
- parent_job,
- &flow_job.id,
- &flow_job.workspace_id,
- true,
- &to_raw_value(&result),
- false,
- same_worker_tx.clone(),
- &worker_dir,
- None,
- rsmq,
- worker_name,
- )
- .await?;
+ job_completed_tx
+ .send(SendResult::UpdateFlow {
+ flow: parent_job,
+ success: true,
+ result: to_raw_value(&result),
+ stop_early_override: Some(true),
+ w_id: flow_job.workspace_id.clone(),
+ worker_dir: worker_dir.to_string(),
+ token: client.token.clone(),
+ })
+ .await
+ .map_err(|e| {
+ Error::InternalErr(format!(
+ "error sending update flow message to job completed channel: {e}"
+ ))
+ })?;
+ // update_flow_status_after_job_completion(
+ // db,
+ // client,
+ // parent_job,
+ // &flow_job.id,
+ // &flow_job.workspace_id,
+ // true,
+ // &to_raw_value(&result),
+ // false,
+ // same_worker_tx.clone(),
+ // &worker_dir,
+ // None,
+ // rsmq,
+ // worker_name,
+ // )
+ // .await?;
+ return Ok(());
}
}
return Ok(());
@@ -1689,6 +1774,7 @@ async fn push_next_flow_job
_ => Ok(flow_job_args),
}
};
+ tracing::debug!(id = %flow_job.id, root_id = %job_root, "flow job args computed");
let next_flow_transform = compute_next_flow_transform(
arc_flow_job_args.clone(),
@@ -1889,6 +1975,8 @@ async fn push_next_flow_job
)
.await?;
+ tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushed next flow job: {uuid}");
+
if let FlowModuleValue::ForloopFlow { parallelism: Some(p), .. } = &module.value {
if i as u16 >= *p {
sqlx::query!(
@@ -2211,6 +2299,8 @@ async fn compute_next_flow_transform(
))
};
let delete_after_use = module.delete_after_use.unwrap_or(false);
+
+ tracing::debug!(id = %flow_job.id, "computing next flow transform for {:?}", &module.value);
match &module.value {
FlowModuleValue::Identity => trivial_next_job(JobPayload::Identity),
FlowModuleValue::Flow { path, .. } => {
diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts
index 2f8a7394f1..911e8942b3 100644
--- a/benchmarks/lib.ts
+++ b/benchmarks/lib.ts
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
-export const VERSION = "v1.232.0";
+export const VERSION = "v1.236.2";
export async function login(email: string, password: string): Promise {
return await windmill.UserService.login({
diff --git a/cli/main.ts b/cli/main.ts
index 305679f718..8f2bd4690d 100644
--- a/cli/main.ts
+++ b/cli/main.ts
@@ -31,7 +31,7 @@ addEventListener("error", (event) => {
}
});
-export const VERSION = "v1.232.0";
+export const VERSION = "v1.236.2";
let command: any = new Command()
.name("wmill")
diff --git a/cli/script.ts b/cli/script.ts
index d12623ec8a..bf1dcadbda 100644
--- a/cli/script.ts
+++ b/cli/script.ts
@@ -14,6 +14,7 @@ import {
writeAllSync,
yamlParse,
} from "./deps.ts";
+import { deepEqual } from "./utils.ts";
export interface ScriptFile {
parent_hash?: string;
@@ -66,7 +67,7 @@ export async function handleFile(
path: string,
workspace: string,
alreadySynced: string[],
- message?: string,
+ message?: string
): Promise {
if (
!path.includes(".inline_script.") &&
@@ -125,8 +126,9 @@ export async function handleFile(
typed.is_template === remote.is_template &&
typed.kind == remote.kind &&
!remote.archived &&
- (remote?.lock ?? "") == (typed.lock?.join("\n") ?? "") &&
- JSON.stringify(typed.schema) == JSON.stringify(remote.schema) &&
+ (remote?.lock ?? "").trim() ==
+ (typed.lock?.join("\n") ?? "").trim() &&
+ deepEqual(typed.schema, remote.schema) &&
typed.tag == remote.tag &&
(typed.ws_error_handler_muted ?? false) ==
remote.ws_error_handler_muted &&
@@ -140,6 +142,7 @@ export async function handleFile(
return true;
}
}
+
log.info(
colors.yellow.bold(`Creating script with a parent ${remotePath}`)
);
diff --git a/cli/sync.ts b/cli/sync.ts
index a1fa8a139b..6e69635221 100644
--- a/cli/sync.ts
+++ b/cli/sync.ts
@@ -478,7 +478,7 @@ async function pull(
!opts.json
);
const local = opts.raw
- ? undefined
+ ? await FSFSElement(Deno.cwd())
: await FSFSElement(path.join(Deno.cwd(), ".wmill"));
const changes = await compareDynFSElement(
remote,
@@ -685,19 +685,18 @@ async function push(
"Computing the files to update on the remote to match local (taking .wmillignore into account)"
)
);
- const remote = opts.raw
- ? undefined
- : ZipFSElement(
- (await downloadZip(
- workspace,
- opts.plainSecrets,
- opts.skipVariables,
- opts.skipResources,
- opts.skipSecrets,
- opts.includeSchedules
- ))!,
- !opts.json
- );
+ const remote = ZipFSElement(
+ (await downloadZip(
+ workspace,
+ opts.plainSecrets,
+ opts.skipVariables,
+ opts.skipResources,
+ opts.skipSecrets,
+ opts.includeSchedules
+ ))!,
+ !opts.json
+ );
+
const local = await FSFSElement(path.join(Deno.cwd(), ""));
const changes = await compareDynFSElement(
local,
@@ -850,7 +849,7 @@ async function push(
case "flow":
await FlowService.deleteFlowByPath({
workspace: workspaceId,
- path: removeSuffix(change.path, ".flow.json"),
+ path: removeSuffix(change.path, ".flow/flow.json"),
});
break;
case "app":
diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index e9de0883d7..cd8aca509d 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "windmill-components",
- "version": "1.232.0",
+ "version": "1.236.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
- "version": "1.232.0",
+ "version": "1.236.2",
"license": "AGPL-3.0",
"dependencies": {
"@aws-crypto/sha256-js": "^4.0.0",
diff --git a/frontend/package.json b/frontend/package.json
index 3cd839a561..19725bb27a 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
{
"name": "windmill-components",
- "version": "1.232.0",
+ "version": "1.236.2",
"scripts": {
"dev": "vite dev",
"build": "vite build",
diff --git a/frontend/src/lib/components/AddUser.svelte b/frontend/src/lib/components/AddUser.svelte
index 2856c06399..6d2b901ecc 100644
--- a/frontend/src/lib/components/AddUser.svelte
+++ b/frontend/src/lib/components/AddUser.svelte
@@ -61,7 +61,7 @@
dispatch('new')
}
- let selected: 'operator' | 'author' | 'admin' = 'author'
+ let selected: 'operator' | 'developer' | 'admin' = 'developer'
-
{#if filteredConnectsManual}
{#each filteredConnectsManual as [key, _]}
diff --git a/frontend/src/lib/components/ChartHighlightTheme.svelte b/frontend/src/lib/components/ChartHighlightTheme.svelte
index d3947ef127..ee842b0084 100644
--- a/frontend/src/lib/components/ChartHighlightTheme.svelte
+++ b/frontend/src/lib/components/ChartHighlightTheme.svelte
@@ -5,7 +5,6 @@
import nord from 'svelte-highlight/styles/nord'
import { each } from 'chart.js/helpers'
import { Chart } from 'chart.js'
- import { deepMergeWithPriority } from '$lib/utils'
let darkMode: boolean = false
@@ -13,67 +12,41 @@
if (document.documentElement.classList.contains('dark')) {
darkMode = true
each(Chart.instances, (instance) => {
- instance.options = deepMergeWithPriority(instance.options, {
- scales: {
- y: {
- ticks: {
- color: '#e0e7ed'
- },
- grid: {
- color: '#4a5568'
- }
- },
- x: {
- ticks: {
- color: '#e0e7ed'
- },
- grid: {
- color: '#4a5568'
- }
- }
- },
- plugins: {
- legend: {
- labels: {
- color: '#e0e7ed'
- }
- }
- }
- })
+ if (instance.options.scales?.y?.ticks?.color !== undefined) {
+ instance.options.scales.y.ticks.color = '#e0e7ed'
+ }
+ if (instance.options.scales?.y?.grid?.color !== undefined) {
+ instance.options.scales.y.grid.color = '#4a5568'
+ }
+ if (instance.options.scales?.x?.ticks?.color !== undefined) {
+ instance.options.scales.x.ticks.color = '#e0e7ed'
+ }
+ if (instance.options.scales?.x?.grid?.color !== undefined) {
+ instance.options.scales.x.grid.color = '#4a5568'
+ }
+ if (instance.options.plugins?.legend?.labels?.color !== undefined) {
+ instance.options.plugins.legend.labels.color = '#e0e7ed'
+ }
instance.update()
})
} else {
darkMode = false
-
each(Chart.instances, (instance) => {
- instance.options = deepMergeWithPriority(instance.options, {
- scales: {
- y: {
- ticks: {
- color: '#4a5568'
- },
- grid: {
- color: '#e0e7ed'
- }
- },
- x: {
- ticks: {
- color: '#4a5568'
- },
- grid: {
- color: '#e0e7ed'
- }
- }
- },
- plugins: {
- legend: {
- labels: {
- color: '#4a5568'
- }
- }
- }
- })
-
+ if (instance.options.scales?.y?.ticks?.color !== undefined) {
+ instance.options.scales.y.ticks.color = '#4a5568'
+ }
+ if (instance.options.scales?.y?.grid?.color !== undefined) {
+ instance.options.scales.y.grid.color = '#e0e7ed'
+ }
+ if (instance.options.scales?.x?.ticks?.color !== undefined) {
+ instance.options.scales.x.ticks.color = '#4a5568'
+ }
+ if (instance.options.scales?.x?.grid?.color !== undefined) {
+ instance.options.scales.x.grid.color = '#e0e7ed'
+ }
+ if (instance.options.plugins?.legend?.labels?.color !== undefined) {
+ instance.options.plugins.legend.labels.color = '#4a5568'
+ }
instance.update()
})
}
diff --git a/frontend/src/lib/components/DBSchemaExplorer.svelte b/frontend/src/lib/components/DBSchemaExplorer.svelte
index dc49ac8fdb..6c95786a5d 100644
--- a/frontend/src/lib/components/DBSchemaExplorer.svelte
+++ b/frontend/src/lib/components/DBSchemaExplorer.svelte
@@ -1,22 +1,21 @@
+{#if loading}
+
+{/if}
+
{#if dbSchema}
+{:else if shouldDisplayError}
+
+ Schema could not be loaded. Please check the permissions of the resource.
+
{/if}
diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte
index 0419bb26a3..bbd2238c2e 100644
--- a/frontend/src/lib/components/DisplayResult.svelte
+++ b/frontend/src/lib/components/DisplayResult.svelte
@@ -338,14 +338,25 @@
{/each}
JSON is too large to be displayed in full
{#if result && result != 'WINDMILL_TOO_BIG'}
@@ -89,10 +89,10 @@
+ import { type MetricDataPoint, MetricsService } from '$lib/gen'
+ import { displayTime } from '$lib/utils'
+ import { enterpriseLicense, workspaceStore } from '$lib/stores'
+ import {
+ CategoryScale,
+ Chart as ChartJS,
+ Legend,
+ LinearScale,
+ LineElement,
+ PointElement,
+ Title,
+ Tooltip
+ } from 'chart.js'
+ import { Line } from 'svelte-chartjs'
+ import { Alert } from './common'
+
+ ChartJS.register(Title, Tooltip, Legend, LineElement, LinearScale, PointElement, CategoryScale)
+
+ export let jobId: string
+ export let jobUpdateLastFetch: Date | undefined
+
+ let jobMetricsLastFetch: Date | undefined = undefined
+ let jobMemoryStats: MetricDataPoint[] | undefined = undefined
+
+ let data: {
+ x: number
+ y: number
+ }[] = []
+ let labels: string[] = []
+
+ async function loadMetricsData() {
+ try {
+ let jobStatsPromise = MetricsService.getJobMetrics({
+ workspace: $workspaceStore!,
+ id: jobId,
+ requestBody: {
+ from_timestamp: jobMetricsLastFetch?.toISOString(),
+ timeseries_max_datapoints: 0
+ }
+ })
+ jobMetricsLastFetch = new Date()
+ let jobStats = await jobStatsPromise
+
+ let memoryTimeseries =
+ jobStats.timeseries_metrics?.filter((ts) => ts.metric_id === 'memory_kb') ?? []
+
+ if (memoryTimeseries.length > 0) {
+ jobMemoryStats = (jobMemoryStats ?? []).concat(memoryTimeseries[0].values)
+ }
+ } catch {
+ console.error('Unable to load metrics data for job', jobId)
+ return
+ }
+
+ for (let dp of jobMemoryStats ?? []) {
+ let ts = new Date(dp.timestamp).valueOf()
+ if (data.length === 0 || ts > data[data.length - 1].x) {
+ data.push({
+ x: ts,
+ y: dp.value
+ })
+ labels.push(displayTime(dp.timestamp))
+ }
+ }
+ data = [...data]
+ }
+
+ $: jobUpdateLastFetch && loadMetricsData()
+
+
+
+ {#if !$enterpriseLicense}
+
+ Job metrics are only available on Windmill Enterprise Edition.
+
+ {:else if (jobMemoryStats?.length ?? 0) === 0}
+
No data points available for this job. Metrics are recorded only for jobs running for more
+ than 500ms.
+ {/if}
+
+
+
) {
+ let lastCallbacks: Callbacks | undefined = undefined
+
+ let finished: string[] = []
+ export async function abstractRun(fn: () => Promise, callbacks?: Callbacks) {
try {
running = false
isLoading = true
clearCurrentJob()
const startedAt = Date.now()
const testId = await fn()
-
- if (lastStartedAt < startedAt) {
+ lastCallbacks = callbacks
+ if (lastStartedAt < startedAt || allowConcurentRequests) {
lastStartedAt = startedAt
if (testId) {
dispatch('started', testId)
try {
- await watchJob(testId)
+ await watchJob(testId, callbacks)
} catch {
+ callbacks?.cancel()
dispatch('cancel', testId)
if (currentId === testId) {
currentId = undefined
@@ -52,6 +59,7 @@
}
return testId
} catch (err) {
+ callbacks?.error()
// if error happens on submitting the job, reset UI state so the user can try again
isLoading = false
currentId = undefined
@@ -110,6 +118,9 @@
export async function cancelJob() {
const id = currentId
if (id) {
+ lastCallbacks?.cancel()
+ lastCallbacks = undefined
+ // console.log('cancel', 2)
dispatch('cancel', id)
currentId = undefined
try {
@@ -125,29 +136,32 @@
}
export async function clearCurrentJob() {
- if (currentId) {
+ if (currentId && !allowConcurentRequests) {
+ lastCallbacks?.cancel()
+ // console.log('cancel', 3)
dispatch('cancel', currentId)
+ lastCallbacks = undefined
job = undefined
await cancelJob()
}
}
- export async function watchJob(testId: string) {
+ export async function watchJob(testId: string, callbacks?: Callbacks) {
syncIteration = 0
errorIteration = 0
currentId = testId
job = undefined
- const isCompleted = await loadTestJob(testId)
+ const isCompleted = await loadTestJob(testId, callbacks)
if (!isCompleted) {
setTimeout(() => {
- syncer(testId)
+ syncer(testId, callbacks)
}, 50)
}
}
- async function loadTestJob(id: string): Promise {
+ async function loadTestJob(id: string, callbacks?: Callbacks): Promise {
let isCompleted = false
- if (currentId === id) {
+ if (currentId === id || allowConcurentRequests) {
try {
let maybe_job = await JobService.getCompletedJobResultMaybe({
workspace: workspace ?? '',
@@ -160,19 +174,26 @@
}
if (maybe_job.completed) {
isCompleted = true
- if (currentId === id) {
+ if (currentId === id || allowConcurentRequests) {
job = { ...maybe_job, id }
await tick()
- if ('error' in job ?? {}) {
+ if (typeof job?.result == 'object' && 'error' in (job?.result ?? {})) {
+ callbacks?.error()
dispatch('doneError', {
id,
error: job.result.error
})
} else {
+ callbacks?.done(job.result)
dispatch('done', job)
}
- currentId = undefined
+ finished.push(id)
+ if (!allowConcurentRequests) {
+ currentId = undefined
+ }
} else {
+ // console.log('cancel', 3)
+ callbacks?.cancel()
dispatch('cancel', id)
}
}
@@ -188,25 +209,32 @@
}
return isCompleted
} else {
+ // console.log('cancel', 6)
+ callbacks?.cancel()
dispatch('cancel', id)
return true
}
}
- async function syncer(id: string): Promise {
- if (currentId != id) {
+ async function syncer(id: string, callbacks?: Callbacks): Promise {
+ if ((currentId != id && !allowConcurentRequests) || finished.includes(id)) {
+ callbacks?.cancel()
+ // console.log('cancel', 7)
dispatch('cancel', id)
return
}
syncIteration++
- await loadTestJob(id)
+ let r = await loadTestJob(id, callbacks)
+ if (r) {
+ return
+ }
let nextIteration = 50
if (syncIteration > ITERATIONS_BEFORE_SLOW_REFRESH) {
nextIteration = 500
} else if (syncIteration > ITERATIONS_BEFORE_SUPER_SLOW_REFRESH) {
nextIteration = 2000
}
- setTimeout(() => syncer(id), nextIteration)
+ setTimeout(() => syncer(id, callbacks), nextIteration)
}
onDestroy(async () => {
diff --git a/frontend/src/lib/components/RunFormAdvancedPopup.svelte b/frontend/src/lib/components/RunFormAdvancedPopup.svelte
index 6e355ad5ba..12838a1e5f 100644
--- a/frontend/src/lib/components/RunFormAdvancedPopup.svelte
+++ b/frontend/src/lib/components/RunFormAdvancedPopup.svelte
@@ -81,7 +81,7 @@
{#if overrideTag}
reset to default
{:else}
- Override Wourker Group
+ Override Worker Group Tag
{/if}
{#each $workerTags ?? [] as tag (tag)}
{tag}
diff --git a/frontend/src/lib/components/SchemaEditor.svelte b/frontend/src/lib/components/SchemaEditor.svelte
index ff616aa7b5..6b60310017 100644
--- a/frontend/src/lib/components/SchemaEditor.svelte
+++ b/frontend/src/lib/components/SchemaEditor.svelte
@@ -273,7 +273,7 @@
}}
id="flow-editor-add-property"
>
- Add Property
+ Add Argument
diff --git a/frontend/src/lib/components/TestJobLoader.svelte b/frontend/src/lib/components/TestJobLoader.svelte
index 8c1bef8ed4..b2caff4882 100644
--- a/frontend/src/lib/components/TestJobLoader.svelte
+++ b/frontend/src/lib/components/TestJobLoader.svelte
@@ -10,6 +10,7 @@
export let job: Job | undefined = undefined
export let workspaceOverride: string | undefined = undefined
export let notfound = false
+ export let jobUpdateLastFetch: Date | undefined = undefined
const dispatch = createEventDispatcher()
@@ -150,7 +151,6 @@
running: job.running,
logOffset: job.logs?.length ? job.logs?.length + 1 : 0
})
-
if (previewJobUpdates.new_logs) {
job.logs = (job?.logs ?? '').concat(previewJobUpdates.new_logs)
}
@@ -163,6 +163,7 @@
} else {
job = await JobService.getJob({ workspace: workspace!, id })
}
+ jobUpdateLastFetch = new Date()
if (job?.type === 'CompletedJob') {
//only CompletedJob has success property
@@ -194,7 +195,9 @@
return
}
syncIteration++
- await loadTestJob(id)
+ if (await loadTestJob(id)) {
+ return
+ }
let nextIteration = 50
if (syncIteration > ITERATIONS_BEFORE_SLOW_REFRESH) {
nextIteration = 500
diff --git a/frontend/src/lib/components/apps/components/buttons/AppButton.svelte b/frontend/src/lib/components/apps/components/buttons/AppButton.svelte
index d7255415a0..d2e1f783a0 100644
--- a/frontend/src/lib/components/apps/components/buttons/AppButton.svelte
+++ b/frontend/src/lib/components/apps/components/buttons/AppButton.svelte
@@ -34,6 +34,7 @@
export let errorHandledByComponent: boolean | undefined = false
export let extraKey: string | undefined = undefined
export let isMenuItem: boolean = false
+ export let noInitialize = false
export let controls: { left: () => boolean; right: () => boolean | string } | undefined =
undefined
@@ -156,6 +157,7 @@
+ import type {
+ AppEditorContext,
+ AppViewerContext,
+ ComponentCustomCSS,
+ RichConfigurations
+ } from '../../../types'
+ import { components } from '$lib/components/apps/editor/component'
+ import ResolveConfig from '../../helpers/ResolveConfig.svelte'
+ import { findGridItem, initConfig, initOutput } from '$lib/components/apps/editor/appUtils'
+ import {
+ createPostgresInput,
+ getDbSchemas,
+ loadTableMetaData,
+ type ColumnMetadata,
+ type TableMetadata,
+ getPrimaryKeys
+ } from './utils'
+ import { getContext, tick } from 'svelte'
+ import UpdateCell from './UpdateCell.svelte'
+ import { workspaceStore, type DBSchemas } from '$lib/stores'
+ import Button from '$lib/components/common/button/Button.svelte'
+ import { Plus } from 'lucide-svelte'
+ import { Drawer, DrawerContent } from '$lib/components/common'
+ import InsertRow from './InsertRow.svelte'
+ import Portal from 'svelte-portal'
+ import { sendUserToast } from '$lib/toast'
+ import type { AppInput, StaticInput } from '$lib/components/apps/inputType'
+ import DbExplorerCount from './DbExplorerCount.svelte'
+ import AppAggridExplorerTable from '../table/AppAggridExplorerTable.svelte'
+ import type { IDatasource } from 'ag-grid-community'
+ import { RunnableWrapper } from '../../helpers'
+ import type RunnableComponent from '../../helpers/RunnableComponent.svelte'
+ import InsertRowRunnable from './InsertRowRunnable.svelte'
+ import DeleteRow from './DeleteRow.svelte'
+
+ export let id: string
+ export let configuration: RichConfigurations
+ export let customCss: ComponentCustomCSS<'dbexplorercomponent'> | undefined = undefined
+ export let render: boolean
+ export let initializing: boolean = true
+
+ const resolvedConfig = initConfig(
+ components['dbexplorercomponent'].initialData.configuration,
+ configuration
+ )
+
+ const { app, worldStore, mode, selectedComponent } =
+ getContext('AppViewerContext')
+ const editorContext = getContext('AppEditorContext')
+
+ let input: AppInput | undefined = undefined
+ let quicksearch = ''
+ let aggrid: AppAggridExplorerTable
+
+ $: computeInput(
+ resolvedConfig.columnDefs,
+ resolvedConfig.whereClause,
+ resolvedConfig.type.configuration.postgresql.resource
+ )
+
+ function computeInput(columnDefs: any, whereClause: string | undefined, resource: any) {
+ aggrid?.clearRows()
+ input = createPostgresInput(
+ resource,
+ resolvedConfig.type.configuration.postgresql.table,
+ columnDefs,
+ whereClause
+ )
+ }
+
+ $: editorContext != undefined && $mode == 'dnd' && resolvedConfig.type && listTableIfAvailable()
+
+ $: editorContext != undefined &&
+ $mode == 'dnd' &&
+ resolvedConfig.type.configuration?.postgresql?.table &&
+ listColumnsIfAvailable()
+
+ $: if (quicksearch) {
+ aggrid?.clearRows()
+ }
+
+ initializing = false
+
+ let updateCell: UpdateCell
+
+ let renderCount = 0
+ let insertDrawer: Drawer | undefined = undefined
+ let componentContainerHeight: number | undefined = undefined
+ let buttonContainerHeight: number | undefined = undefined
+
+ function onUpdate(
+ e: CustomEvent<{
+ row: number
+ columnDef: ColumnMetadata
+ column: string
+ value: any
+ data: any
+ oldValue: string | undefined
+ }>
+ ) {
+ const { columnDef, value, data, oldValue } = e.detail
+
+ updateCell?.triggerUpdate(
+ resolvedConfig.type.configuration.postgresql.resource,
+ resolvedConfig.type.configuration.postgresql.table ?? 'unknown',
+ columnDef,
+ resolvedConfig.columnDefs,
+ value,
+ data,
+ oldValue
+ )
+ }
+
+ let args: Record = {}
+
+ let outputs = initOutput($worldStore, id, {
+ selectedRowIndex: 0,
+ selectedRow: {},
+ selectedRows: [] as any[],
+ result: [] as any[],
+ loading: false,
+ page: 0,
+ newChange: { row: 0, column: '', value: undefined },
+ ready: undefined as boolean | undefined
+ })
+
+ let lastResource: string | undefined = undefined
+ async function listTableIfAvailable() {
+ let resource = resolvedConfig.type.configuration?.postgresql?.resource
+ if (lastResource === resource) return
+ lastResource = resource
+ const gridItem = findGridItem($app, id)
+
+ if (!gridItem) {
+ return
+ }
+
+ if (
+ 'configuration' in gridItem.data?.configuration?.type &&
+ 'selectOptions' in gridItem.data?.configuration?.type?.configuration?.postgresql?.table
+ ) {
+ gridItem.data.configuration.type.configuration.postgresql.table.selectOptions = []
+ }
+
+ if (!resolvedConfig.type?.configuration?.postgresql?.resource) {
+ $app = {
+ ...$app
+ }
+ return
+ }
+
+ if (
+ 'configuration' in gridItem.data?.configuration?.type &&
+ gridItem.data.configuration.type.configuration.postgresql.table
+ ) {
+ gridItem.data.configuration.type.configuration.postgresql.table.loading = true
+ }
+
+ try {
+ const dbSchemas: DBSchemas = {}
+
+ await getDbSchemas(
+ 'postgresql',
+ resolvedConfig.type.configuration.postgresql.resource.split(':')[1],
+ $workspaceStore,
+ dbSchemas,
+ (message: string) => {}
+ )
+
+ if ('configuration' in gridItem.data.configuration.type) {
+ gridItem.data.configuration.type.configuration.postgresql.table['selectOptions'] = dbSchemas
+ ? // @ts-ignore
+ Object.keys(Object.values(dbSchemas)?.[0]?.schema?.public ?? {})
+ : []
+ }
+
+ $app = {
+ ...$app
+ }
+ } catch (e) {}
+ if (
+ 'configuration' in gridItem.data?.configuration?.type &&
+ gridItem.data.configuration.type.configuration.postgresql.table
+ ) {
+ gridItem.data.configuration.type.configuration.postgresql.table.loading = false
+ }
+ }
+
+ let datasource: IDatasource = {
+ rowCount: 0,
+ getRows: async function (params) {
+ refreshCount++
+ let uuid = await runnableComponent?.runComponent(
+ undefined,
+ undefined,
+ undefined,
+ {
+ offset: params.startRow,
+ limit: params.endRow - params.startRow,
+ quicksearch,
+ orderBy: params.sortModel?.[0]?.colId ?? resolvedConfig.columnDefs?.[0]?.field,
+ is_desc: params.sortModel?.[0]?.sort === 'desc'
+ },
+ {
+ done: (x) => {
+ if (x && Array.isArray(x)) {
+ params.successCallback(
+ x.map((x) => {
+ let primaryKeys = getPrimaryKeys(resolvedConfig.columnDefs)
+ let o = {}
+ primaryKeys.forEach((pk) => {
+ o[pk] = x[pk]
+ })
+ x['__index'] = JSON.stringify(o)
+ return x
+ }),
+ datasource.rowCount
+ )
+ } else {
+ params.failCallback()
+ }
+ },
+ cancel: () => {
+ console.log('cancel datasource request')
+ params.failCallback()
+ },
+ error: () => {
+ console.log('error datasource request')
+ params.failCallback()
+ }
+ }
+ )
+ console.log('asking for ' + params.startRow + ' to ' + params.endRow, uuid)
+ }
+ }
+
+ let lastTable: string | undefined = undefined
+ async function listColumnsIfAvailable() {
+ let table = resolvedConfig.type.configuration?.postgresql?.table
+ if (lastTable === table) return
+ lastTable = table
+
+ let tableMetadata = await loadTableMetaData(
+ resolvedConfig.type.configuration.postgresql.resource,
+ $workspaceStore,
+ resolvedConfig.type.configuration.postgresql.table
+ )
+ if (!tableMetadata) return
+
+ const gridItem = findGridItem($app, id)
+ if (!gridItem) return
+
+ let columnDefs = gridItem.data.configuration.columnDefs as StaticInput
+ let old: TableMetadata = (columnDefs?.value as TableMetadata) ?? []
+ if (!Array.isArray(old)) {
+ console.log('old is not an array RESET')
+ old = []
+ }
+ // console.log('OLD', old)
+ // console.log(tableMetadata)
+ const oldMap = Object.fromEntries(old.filter((x) => x != undefined).map((x) => [x.field, x]))
+ const newMap = Object.fromEntries(tableMetadata?.map((x) => [x.field, x]) ?? [])
+
+ let ncols: any[] = []
+ Object.entries(oldMap).forEach(([key, value]) => {
+ if (newMap[key]) {
+ ncols.push({
+ ...value,
+ ...newMap[key]
+ })
+ }
+ })
+ Object.entries(newMap).forEach(([key, value]) => {
+ if (!oldMap[key]) {
+ ncols.push(value)
+ }
+ })
+
+ state = undefined
+
+ //@ts-ignore
+ gridItem.data.configuration.columnDefs = { value: ncols, type: 'static' }
+ gridItem.data = gridItem.data
+ $app = $app
+ let oldS = $selectedComponent
+ $selectedComponent = []
+ await tick()
+ $selectedComponent = oldS
+
+ //@ts-ignore
+ resolvedConfig.columnDefs = ncols
+ renderCount += 1
+ }
+
+ let isInsertable: boolean = false
+
+ $: $worldStore && connectToComponents()
+
+ function connectToComponents() {
+ if ($worldStore) {
+ const outputs = $worldStore.outputsById[`${id}_count`]
+ if (outputs) {
+ outputs.result.subscribe(
+ {
+ id: 'dbexplorer-count-' + id,
+ next: (value) => {
+ datasource.rowCount = value?.[0]?.count
+ }
+ },
+ datasource.rowCount
+ )
+ }
+ }
+ }
+
+ async function insert() {
+ try {
+ await insertRowRunnable?.insertRow(
+ resolvedConfig.type.configuration.postgresql.resource,
+ $workspaceStore,
+ resolvedConfig.type.configuration.postgresql.table,
+ resolvedConfig.columnDefs,
+ args
+ )
+
+ insertDrawer?.closeDrawer()
+ renderCount++
+ } catch (e) {
+ sendUserToast(e.message, true)
+ }
+
+ args = {}
+ }
+
+ let runnableComponent: RunnableComponent
+ let state: any = undefined
+ let insertRowRunnable: InsertRowRunnable
+ let deleteRow: DeleteRow
+
+ function onDelete(e) {
+ const data = { ...e.detail }
+ delete data['__index']
+ let primaryColumns = getPrimaryKeys(resolvedConfig.columnDefs)
+ let getPrimaryKeysresolvedConfig = resolvedConfig.columnDefs?.filter((x) =>
+ primaryColumns.includes(x.field)
+ )
+ deleteRow?.triggerDelete(
+ resolvedConfig.type.configuration.postgresql.resource,
+ resolvedConfig.type.configuration.postgresql.table ?? 'unknown',
+ getPrimaryKeysresolvedConfig,
+ data
+ )
+ }
+
+ let refreshCount = 0
+
+
+{#each Object.keys(components['dbexplorercomponent'].initialData.configuration) as key (key)}
+ {
+ aggrid?.clearRows()
+ refreshCount++
+ }}
+ {id}
+ bind:this={insertRowRunnable}
+/>
+{#if resolvedConfig.allowDelete}
+ {
+ aggrid?.clearRows()
+ refreshCount++
+ }}
+ {id}
+ bind:this={deleteRow}
+ />
+{/if}
+
+
+
+ {
+ args = {}
+ insertDrawer?.openDrawer()
+ }}
+ >
+ Insert
+
+
+ {#if resolvedConfig.type.configuration?.postgresql?.resource && resolvedConfig.type.configuration?.postgresql?.table}
+
+
+ {#key renderCount}
+
+
+ {/key}
+ {/if}
+
+
+
+
+
+ Insert
+
+
+
+
+
diff --git a/frontend/src/lib/components/apps/components/display/dbtable/DbExplorerCount.svelte b/frontend/src/lib/components/apps/components/display/dbtable/DbExplorerCount.svelte
new file mode 100644
index 0000000000..81d4347db2
--- /dev/null
+++ b/frontend/src/lib/components/apps/components/display/dbtable/DbExplorerCount.svelte
@@ -0,0 +1,71 @@
+
+
+ {
+ if (!resource || !table || !workspace) {
+ return undefined
+ }
+
+ const code = `
+ SELECT
+ a.attname as field,
+ pg_catalog.format_type(a.atttypid, a.atttypmod) as DataType,
+ (SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) for 128)
+ FROM pg_catalog.pg_attrdef d
+ WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) as DefaultValue,
+ (SELECT CASE WHEN i.indisprimary THEN true ELSE 'NO' END
+ FROM pg_catalog.pg_class tbl, pg_catalog.pg_class idx, pg_catalog.pg_index i, pg_catalog.pg_attribute att
+ WHERE tbl.oid = a.attrelid AND idx.oid = i.indexrelid AND att.attrelid = tbl.oid
+ AND i.indrelid = tbl.oid AND att.attnum = any(i.indkey) AND att.attname = a.attname LIMIT 1) as IsPrimaryKey,
+ CASE a.attidentity
+ WHEN 'd' THEN 'By Default'
+ WHEN 'a' THEN 'Always'
+ ELSE 'No'
+ END as IsIdentity,
+ CASE a.attnotnull
+ WHEN false THEN 'YES'
+ ELSE 'NO'
+ END as IsNullable,
+ (SELECT true
+ FROM pg_catalog.pg_enum e
+ WHERE e.enumtypid = a.atttypid FETCH FIRST ROW ONLY) as IsEnum
+FROM pg_catalog.pg_attribute a
+WHERE a.attrelid = (SELECT oid FROM pg_catalog.pg_class WHERE relname = '${table}')
+ AND a.attnum > 0 AND NOT a.attisdropped
+ORDER BY a.attnum;
+
+`
+
+ const maxRetries = 3
+ let attempts = 0
+
+ while (attempts < maxRetries) {
+ try {
+ const job = await JobService.runScriptPreview({
+ workspace: workspace,
+ requestBody: {
+ language: Preview.language.POSTGRESQL,
+ content: code,
+ args: {
+ database: resource
+ }
+ }
+ })
+
+ await new Promise((resolve) => setTimeout(resolve, 3000))
+
+ const testResult = await JobService.getCompletedJob({
+ workspace: workspace,
+ id: job
+ })
+
+ if (testResult.success) {
+ attempts = maxRetries
+
+ return testResult.result
+ } else {
+ attempts++
+ }
+ } catch (error) {
+ attempts++
+ }
+ // Exponential back-off
+ await new Promise((resolve) => setTimeout(resolve, 2000 * attempts))
+ }
+
+ console.error('Failed to load table metadata after maximum retries.')
+ return undefined
+}
+
+export function resourceTypeToLang(rt: string) {
+ if (rt === 'ms_sql_server') {
+ return 'mssql'
+ } else {
+ return rt
+ }
+}
+
+const scripts: Record<
+ string,
+ {
+ code: string
+ lang: string
+ processingFn?: (any: any) => SQLSchema['schema']
+ argName: string
+ }
+> = {
+ postgresql: {
+ code: `SELECT table_name, column_name, udt_name, column_default, is_nullable, table_schema FROM information_schema.columns WHERE table_schema != 'pg_catalog' AND table_schema != 'information_schema'`,
+ processingFn: (rows) => {
+ const schemas = rows.reduce((acc, a) => {
+ const table_schema = a.table_schema
+ delete a.table_schema
+ acc[table_schema] = acc[table_schema] || []
+ acc[table_schema].push(a)
+ return acc
+ }, {})
+ const data = {}
+ for (const key in schemas) {
+ data[key] = schemas[key].reduce((acc, a) => {
+ const table_name = a.table_name
+ delete a.table_name
+ acc[table_name] = acc[table_name] || {}
+ const p: {
+ type: string
+ required: boolean
+ default?: string
+ } = {
+ type: a.udt_name,
+ required: a.is_nullable === 'NO'
+ }
+ if (a.column_default) {
+ p.default = a.column_default
+ }
+ acc[table_name][a.column_name] = p
+ return acc
+ }, {})
+ }
+ return data
+ },
+ lang: 'postgresql',
+ argName: 'database'
+ },
+ mysql: {
+ code: "select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT from information_schema.columns where table_schema != 'information_schema'",
+ processingFn: (rows) => {
+ const schemas = rows.reduce((acc, a) => {
+ const table_schema = a.TABLE_SCHEMA
+ delete a.TABLE_SCHEMA
+ acc[table_schema] = acc[table_schema] || []
+ acc[table_schema].push(a)
+ return acc
+ }, {})
+ const data = {}
+ for (const key in schemas) {
+ data[key] = schemas[key].reduce((acc, a) => {
+ const table_name = a.TABLE_NAME
+ delete a.TABLE_NAME
+ acc[table_name] = acc[table_name] || {}
+ const p: {
+ type: string
+ required: boolean
+ default?: string
+ } = {
+ type: a.DATA_TYPE,
+ required: a.is_nullable === 'NO'
+ }
+ if (a.column_default) {
+ p.default = a.COLUMN_DEFAULT
+ }
+ acc[table_name][a.COLUMN_NAME] = p
+ return acc
+ }, {})
+ }
+ return data
+ },
+ lang: 'mysql',
+ argName: 'database'
+ },
+ graphql: {
+ code: getIntrospectionQuery(),
+ lang: 'graphql',
+ argName: 'api'
+ },
+ bigquery: {
+ code: `import { BigQuery } from 'npm:@google-cloud/bigquery@7.2.0';
+export async function main(args: bigquery) {
+const bq = new BigQuery({
+ credentials: args
+})
+const [datasets] = await bq.getDatasets();
+const schema = {}
+for (const dataset of datasets) {
+ schema[dataset.id] = {}
+ const query = "SELECT table_name, ARRAY_AGG(STRUCT(if(is_nullable = 'YES', true, false) AS required, column_name AS name, data_type AS type, if(column_default = 'NULL', null, column_default) AS \`default\`) ORDER BY ordinal_position) AS schema \
+FROM \`{dataset.id}\`.INFORMATION_SCHEMA.COLUMNS \
+GROUP BY table_name".replace('{dataset.id}', dataset.id)
+ const [rows] = await bq.query(query)
+ for (const row of rows) {
+ schema[dataset.id][row.table_name] = {}
+ for (const col of row.schema) {
+ const colName = col.name
+ delete col.name
+ if (col.default === null) {
+ delete col.default
+ }
+ schema[dataset.id][row.table_name][colName] = col
+ }
+ }
+}
+return schema
+}`, // nested template literals
+ lang: 'deno',
+ argName: 'args'
+ },
+ snowflake: {
+ code: `select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT, IS_NULLABLE from information_schema.columns where table_schema != 'INFORMATION_SCHEMA'`,
+ lang: 'snowflake',
+ processingFn: (rows) => {
+ const schema = {}
+ for (const row of rows) {
+ if (!(row.TABLE_SCHEMA in schema)) {
+ schema[row.TABLE_SCHEMA] = {}
+ }
+ if (!(row.TABLE_NAME in schema[row.TABLE_SCHEMA])) {
+ schema[row.TABLE_SCHEMA][row.TABLE_NAME] = {}
+ }
+ schema[row.TABLE_SCHEMA][row.TABLE_NAME][row.COLUMN_NAME] = {
+ type: row.DATA_TYPE,
+ required: row.IS_NULLABLE === 'YES'
+ }
+ if (row.COLUMN_DEFAULT !== null) {
+ schema[row.TABLE_SCHEMA][row.TABLE_NAME][row.COLUMN_NAME]['default'] = row.COLUMN_DEFAULT
+ }
+ }
+ return schema
+ },
+ argName: 'database'
+ },
+ ms_sql_server: {
+ argName: 'database',
+ code: `select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT from information_schema.columns where table_schema != 'sys'`,
+ lang: 'mssql',
+ processingFn: (rows) => {
+ const schemas = rows[0].reduce((acc, a) => {
+ const table_schema = a.TABLE_SCHEMA
+ delete a.TABLE_SCHEMA
+ acc[table_schema] = acc[table_schema] || []
+ acc[table_schema].push(a)
+ return acc
+ }, {})
+ const data = {}
+ for (const key in schemas) {
+ data[key] = schemas[key].reduce((acc, a) => {
+ const table_name = a.TABLE_NAME
+ delete a.TABLE_NAME
+ acc[table_name] = acc[table_name] || {}
+ const p: {
+ type: string
+ required: boolean
+ default?: string
+ } = {
+ type: a.DATA_TYPE,
+ required: a.is_nullable === 'NO'
+ }
+ if (a.column_default) {
+ p.default = a.COLUMN_DEFAULT
+ }
+ acc[table_name][a.COLUMN_NAME] = p
+ return acc
+ }, {})
+ }
+ return data
+ }
+ }
+}
+
+export { scripts }
+export async function getDbSchemas(
+ resourceType: string,
+ resourcePath: string,
+ workspace: string | undefined,
+ dbSchemas: DBSchemas,
+ errorCallback: (message: string) => void
+): Promise {
+ return new Promise(async (resolve, reject) => {
+ if (!resourceType || !resourcePath || !workspace) {
+ resolve()
+ return
+ }
+
+ const job = await JobService.runScriptPreview({
+ workspace: workspace,
+ requestBody: {
+ language: scripts[resourceType].lang as Preview.language,
+ content: scripts[resourceType].code,
+ args: {
+ [scripts[resourceType].argName]: '$res:' + resourcePath
+ }
+ }
+ })
+
+ tryEvery({
+ tryCode: async () => {
+ if (resourcePath) {
+ const testResult = await JobService.getCompletedJob({
+ workspace,
+ id: job
+ })
+ if (!testResult.success) {
+ console.error(testResult.result?.['error']?.['message'])
+ } else {
+ if (resourceType !== undefined) {
+ if (resourceType !== 'graphql') {
+ const { processingFn } = scripts[resourceType]
+ const schema =
+ processingFn !== undefined ? processingFn(testResult.result) : testResult.result
+ dbSchemas[resourcePath] = {
+ lang: resourceTypeToLang(resourceType) as SQLSchema['lang'],
+ schema,
+ publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo
+ }
+ } else {
+ if (typeof testResult.result !== 'object' || !('__schema' in testResult.result)) {
+ console.error('Invalid GraphQL schema')
+
+ errorCallback('Invalid GraphQL schema')
+ } else {
+ dbSchemas[resourcePath] = {
+ lang: 'graphql',
+ schema: testResult.result
+ }
+ }
+ }
+ }
+ }
+ resolve()
+ }
+ },
+ timeoutCode: async () => {
+ console.error('Could not query schema within 5s')
+ errorCallback('Could not query schema within 5s')
+ try {
+ await JobService.cancelQueuedJob({
+ workspace,
+ id: job,
+ requestBody: {
+ reason: 'Could not query schema within 5s'
+ }
+ })
+ } catch (err) {
+ console.error(err)
+ }
+ reject()
+ },
+ interval: 500,
+ timeout: 5000
+ })
+ })
+}
+
+export function formatSchema(dbSchema: DBSchema) {
+ if (dbSchema.lang !== 'graphql' && dbSchema.publicOnly) {
+ return dbSchema.schema.public || dbSchema.schema.PUBLIC || dbSchema.schema.dbo || dbSchema
+ } else if (dbSchema.lang === 'mysql' && Object.keys(dbSchema.schema).length === 1) {
+ return dbSchema.schema[Object.keys(dbSchema.schema)[0]]
+ } else {
+ return dbSchema.schema
+ }
+}
+
+export function formatGraphqlSchema(dbSchema: GraphqlSchema): string {
+ return printSchema(buildClientSchema(dbSchema.schema))
+}
+
+/**
+ * Base class for embedding a svelte component within an AGGrid call.
+ * See: https://stackoverflow.com/a/72608215
+ */
+import type { ICellRendererComp, ICellRendererParams } from 'ag-grid-community'
+
+/**
+ * Class for defining a cell renderer.
+ * If you don't need to define a separate class you could use cellRendererFactory
+ * to create a component with the column definitions.
+ */
+export abstract class AbstractCellRenderer implements ICellRendererComp {
+ eGui: any
+ protected value: any
+ protected params: any
+ constructor(parentElement = 'span') {
+ // create empty span (or other element) to place svelte component in
+ this.eGui = document.createElement(parentElement)
+ }
+
+ init(params: ICellRendererParams & { onClick?: (data: any) => void }) {
+ this.value = params.value
+ this.createComponent(params)
+ this.eGui.addEventListener('click', () => params.onClick?.(params.data))
+ this.params = params
+ }
+
+ getGui() {
+ return this.eGui
+ }
+
+ refresh(params: ICellRendererParams) {
+ this.value = params.value
+ this.eGui.innerHTML = ''
+
+ return true
+ }
+
+ /**
+ * Define and create the svelte component to use in the cell
+ * @example
+ * // This is all you need to do within this method: create the component with new, specify the target
+ * // is the class, and pass in props via the params.
+ * new CampusIcon({
+ * target: this.eGui,
+ * props: {
+ * color: params.data?.color,
+ * name: params.data?.name
+ * }
+ * @param params params for rendering the call, including the value for the cell
+ */
+ abstract createComponent(params: ICellRendererParams): void
+}
+
+/**
+ * Creates a cell renderer using the given callback for how to initialise a svelte component.
+ * See AbstractCellRenderer.createComponent
+ * @param svelteComponent function for how to create the svelte component
+ * @returns
+ */
+export function cellRendererFactory(
+ svelteComponent: (cell: AbstractCellRenderer, params: ICellRendererParams) => void
+) {
+ class Renderer extends AbstractCellRenderer {
+ createComponent(params: ICellRendererParams): void {
+ svelteComponent(this, params)
+ }
+ }
+ return Renderer
+}
diff --git a/frontend/src/lib/components/apps/components/display/table/AppAggridExplorerTable.svelte b/frontend/src/lib/components/apps/components/display/table/AppAggridExplorerTable.svelte
new file mode 100644
index 0000000000..9c026b0bb6
--- /dev/null
+++ b/frontend/src/lib/components/apps/components/display/table/AppAggridExplorerTable.svelte
@@ -0,0 +1,316 @@
+
+
+{#each Object.keys(css ?? {}) as key (key)}
+
+
{
+ $selectedComponent = [id]
+ }}
+ style:height="{clientHeight}px"
+ style:width="{clientWidth}px"
+ class="ag-theme-alpine"
+ class:ag-theme-alpine-dark={$darkMode}
+ >
+
+
+
{firstRow}{'->'}{lastRow + 1} of {datasource?.rowCount} rows
+{:else if resolvedConfig.columnDefs != undefined}
+
+ The columnDefs should be an array of objects, received:
+
+ {JSON.stringify(resolvedConfig.columnDefs)}
+
+
+{:else}
+ The columnDefs are undefined
+{/if}
diff --git a/frontend/src/lib/components/apps/components/display/table/AppCell.svelte b/frontend/src/lib/components/apps/components/display/table/AppCell.svelte
index 85f83f2fb2..7ee8c24ad4 100644
--- a/frontend/src/lib/components/apps/components/display/table/AppCell.svelte
+++ b/frontend/src/lib/components/apps/components/display/table/AppCell.svelte
@@ -1,31 +1,94 @@
-{#if type === 'badge'}
-
- {value}
-
-{:else if type === 'link'}
- {#if isLinkObject(value)}
-
- {value.label}
-
+
+ {#if type === 'badge'}
+
+ {value}
+
+ {:else if type === 'link'}
+ {#if isLinkObject(value)}
+
+ {value.label}
+
+ {:else}
+ {value}
+ {/if}
+ {:else if $isEditable}
+ {value}
+
+
+ {value}
+
{/if}
-{:else}
- {value}
-{/if}
+
diff --git a/frontend/src/lib/components/apps/components/display/table/AppTable.svelte b/frontend/src/lib/components/apps/components/display/table/AppTable.svelte
index e9fd19c4be..bcd9c4c668 100644
--- a/frontend/src/lib/components/apps/components/display/table/AppTable.svelte
+++ b/frontend/src/lib/components/apps/components/display/table/AppTable.svelte
@@ -70,7 +70,7 @@
selectedRowIndex: 0,
selectedRow: undefined,
loading: false,
- result: [],
+ result: [] as Record[],
inputs: {},
search: '',
page: 1
@@ -289,6 +289,16 @@
}
$: $table && updateTable(resolvedConfig, searchValue)
+
+ function updateCellValue(rowIndex: number, columnIndex: number, newCellValue: string) {
+ if (result && rowIndex < result.length) {
+ const updatedRow = { ...result[rowIndex] }
+ const columnName = Object.keys(updatedRow)[columnIndex]
+ updatedRow[columnName] = newCellValue
+ result[rowIndex] = updatedRow
+ outputs?.result.set([result])
+ }
+ }
{#each Object.keys(components['tablecomponent'].initialData.configuration) as key (key)}
@@ -396,21 +406,20 @@
{#if cell?.column?.columnDef?.cell}
{@const context = cell?.getContext()}
{#if context}
- toggleRow(row)}
on:click={() => toggleRow(row)}
- class="p-4 whitespace-pre-wrap truncate text-xs text-primary"
- style={'width: ' + cell.column.getSize() + 'px'}
- >
- c.field === cell.column.columnDef.accessorKey
- )?.type ?? 'text'}
- value={cell.getValue()}
- />
-
+ type={resolvedConfig.columnDefs?.find(
+ // TS types are wrong here
+ // @ts-ignore
+ (c) => c.field === cell.column.columnDef.accessorKey
+ )?.type ?? 'text'}
+ value={cell.getValue()}
+ width={cell.column.getSize()}
+ on:update={(event) => {
+ updateCellValue(rowIndex, index, event.detail.value)
+ }}
+ />
{/if}
{/if}
{/each}
@@ -546,6 +555,7 @@
}}
{#if actionButton.type == 'buttoncomponent'}
('AppViewerContext')
@@ -27,7 +28,9 @@
$: result != undefined && outputs && setOutput(result)
-,
+ callbacks?: Callbacks
+ ): Promise {
+ let jobId: string | undefined
console.debug(`Executing ${id}`)
if (iterContext && $iterContext.disabled) {
console.debug(`Skipping execution of ${id} because it is part of a disabled list`)
@@ -288,8 +294,8 @@
}
try {
- const jobId = await resultJobLoader?.abstractRun(async () => {
- const nonStaticRunnableInputs = {}
+ jobId = await resultJobLoader?.abstractRun(async () => {
+ const nonStaticRunnableInputs = dynamicArgsOverride ?? {}
const staticRunnableInputs = {}
for (const k of Object.keys(fields ?? {})) {
let field = fields[k]
@@ -338,31 +344,50 @@
addJob(uuid)
}
return uuid
- })
+ }, callbacks)
if (setRunnableJobEditorPanel && editorContext) {
editorContext.runnableJobEditorPanel.update((p) => {
return {
...p,
- jobs: { ...p.jobs, [id]: jobId }
+ jobs: { ...p.jobs, [id]: jobId as string }
}
})
}
+ return jobId
} catch (e) {
- updateResult({ error: e.body ?? e.message })
+ let error = e.body ?? e.message
+ updateResult({ error })
+ $errorByComponent[id] = { error }
+
loading = false
}
}
+ type Callbacks = { done: (x: any[]) => void; cancel: () => void; error: () => void }
- export async function runComponent() {
+ export async function runComponent(
+ noToast = false,
+ inlineScriptOverride?: InlineScript,
+ setRunnableJobEditorPanel?: boolean,
+ dynamicArgsOverride?: Record,
+ callbacks?: Callbacks
+ ): Promise {
try {
- if (cancellableRun) {
+ if (cancellableRun && !dynamicArgsOverride) {
await cancellableRun()
} else {
console.log('Run component')
- executeComponent()
+ return await executeComponent(
+ noToast,
+ inlineScriptOverride,
+ setRunnableJobEditorPanel,
+ dynamicArgsOverride,
+ callbacks
+ )
}
} catch (e) {
- updateResult({ error: e.body ?? e.message })
+ let error = e.body ?? e.message
+ updateResult({ error })
+ $errorByComponent[id] = { error }
}
}
@@ -453,7 +478,7 @@
jobId: string | undefined,
setRunnableJobEditorPanel?: boolean
) {
- dispatch('done')
+ dispatch('resultSet')
const errors = getResultErrors(res)
if (errors) {
@@ -580,6 +605,7 @@
{/if}
{
console.log('started', e.detail)
@@ -592,9 +618,11 @@
lastJobId = e.detail.id
setResult(e.detail.result, e.detail.id)
loading = false
+ dispatch('done', { id: e.detail.id, result: e.detail.result })
}}
on:cancel={(e) => {
let jobId = e.detail
+ console.debug('cancel', jobId)
let job = $jobsById[jobId]
if (job && job.created_at && !job.duration_ms) {
$jobsById[jobId] = {
@@ -603,6 +631,7 @@
duration_ms: Date.now() - (job.started_at ?? job.created_at)
}
}
+ dispatch('cancel', { id: e.detail })
}}
on:running={(e) => {
let jobId = e.detail
@@ -614,6 +643,7 @@
on:doneError={(e) => {
setResult({ error: e.detail.error }, e.detail.id)
loading = false
+ dispatch('doneError', { id: e.detail.id, result: e.detail.result })
}}
bind:this={resultJobLoader}
/>
@@ -652,19 +682,21 @@
-
+
An error occured, please contact the app author.
- {#if lastJobId && $errorByComponent[id].error}
+ {#if $errorByComponent?.[id]?.error}
{$errorByComponent[id].error}
{/if}
-
- Job id: {lastJobId}
-
+ {#if lastJobId}
+
+ Job id: {lastJobId}
+
+ {/if}
(initializing = false)}
+ on:done
+ on:doneError
+ on:cancel
+ on:resultSet={() => (initializing = false)}
on:success={() => handleSideEffect(true)}
on:handleError={(e) => handleSideEffect(false, e.detail)}
{outputs}
@@ -242,7 +250,7 @@
{:else}
-
+
{/if}
diff --git a/frontend/src/lib/components/apps/components/inputs/AppCheckbox.svelte b/frontend/src/lib/components/apps/components/inputs/AppCheckbox.svelte
index 2124c92fe8..54c2ba97d8 100644
--- a/frontend/src/lib/components/apps/components/inputs/AppCheckbox.svelte
+++ b/frontend/src/lib/components/apps/components/inputs/AppCheckbox.svelte
@@ -26,6 +26,7 @@
export let render: boolean
export let extraKey: string | undefined = undefined
export let preclickAction: (() => Promise) | undefined = undefined
+ export let noInitialize = false
export let controls: { left: () => boolean; right: () => boolean | string } | undefined =
undefined
@@ -111,7 +112,10 @@
/>
{/each}
- Promise) | undefined = undefined
export let recomputeIds: string[] | undefined = undefined
+ export let noInitialize = false
export let controls: { left: () => boolean; right: () => boolean | string } | undefined =
undefined
@@ -189,7 +190,9 @@
/>
{/each}
-
({ input: x.componentInput, id: x.id })))
}
+ if (c.type === 'dbexplorercomponent') {
+ let nr: { id: string; input: AppInput }[] = []
+ let config = c.configuration as any
+ let pg = config?.type?.configuration?.postgresql
+ if (pg) {
+ const { table, resource } = pg
+ const tableValue = table.value
+ const resourceValue = resource.value
+ const columnDefs = (c.configuration.columnDefs as any).value as ColumnDef[]
+ const whereClause = (c.configuration.whereClause as any).value as unknown as
+ | string
+ | undefined
+ console.log(columnDefs)
+ if (tableValue && resourceValue && columnDefs) {
+ r.push({
+ input: createPostgresInput(resourceValue, tableValue, columnDefs, whereClause),
+ id: x.id
+ })
+ r.push({
+ input: getCountPostgresql(resourceValue, tableValue),
+ id: x.id + '_count'
+ })
+ r.push({
+ input: createPostgresInsert(tableValue, columnDefs, resourceValue),
+ id: x.id + '_insert'
+ })
+ let primaryColumns = getPrimaryKeys(columnDefs)
+ let columns = columnDefs?.filter((x) => primaryColumns.includes(x.field))
+
+ columnDefs
+ .filter((col) => col.editable || config.allEditable.value)
+ .forEach((column) => {
+ r.push({
+ input: createUpdatePostgresInput(resourceValue, tableValue, column, columns),
+ id: x.id + '_update'
+ })
+ })
+ }
+ }
+ r.push(...nr)
+ }
return r
.filter((x) => x.input)
.map(async (o) => {
@@ -213,13 +263,16 @@
): Promise<[string, Record
] | undefined> {
const staticInputs = collectStaticFields(fields)
if (runnable?.type == 'runnableByName') {
+ console.log(runnable.inlineScript?.content)
let hex = await hash(runnable.inlineScript?.content)
+ console.log('hex', hex, id)
return [`${id}:rawscript/${hex}`, staticInputs]
} else if (runnable?.type == 'runnableByPath') {
let prefix = runnable.runType !== 'hubscript' ? runnable.runType : 'script'
return [`${id}:${prefix}/${runnable.path}`, staticInputs]
}
}
+
async function createApp(path: string) {
await computeTriggerables()
try {
@@ -657,6 +710,7 @@
type="text"
placeholder="App summary"
class="text-sm w-full font-semibold"
+ on:keydown|stopPropagation
bind:value={$summary}
on:keyup={() => {
if (appPath == '' && $summary?.length > 0 && !dirtyPath) {
@@ -713,6 +767,7 @@
placeholder="App summary"
class="text-sm w-full"
bind:value={$summary}
+ on:keydown|stopPropagation
on:keyup={() => {
if (appPath == '' && $summary?.length > 0 && !dirtyPath) {
path?.setName(
@@ -731,7 +786,6 @@
-
- Timeline Details
- {#if rightColumnSelect == 'timeline'}
-
- {:else if rightColumnSelect == 'detail'}
-
- {#if selectedJobId}
- {#if selectedJobId?.includes('Frontend')}
- {@const jobResult = $jobsById[selectedJobId]}
- {#if jobResult?.error !== undefined}
-
-
-
-
-
-
-
-
- {:else if jobResult !== undefined}
-
-
-
-
-
-
-
-
- {:else}
-
- {/if}
- {:else}
-
- {#if job?.['running']}
-
- testJobLoader?.cancelJob()}
- >
-
-
- {/if}
- {#if job?.args}
-
-
- {/if}
-
- {#if job?.job_kind !== 'flow' && job?.job_kind !== 'flowpreview'}
- {@const jobResult = $jobsById[selectedJobId]}
+
+
+ Timeline Details
+ {#if rightColumnSelect == 'timeline'}
+
+ {:else if rightColumnSelect == 'detail'}
+
+ {#if selectedJobId}
+ {#if selectedJobId?.includes('Frontend')}
+ {@const jobResult = $jobsById[selectedJobId]}
+ {#if jobResult?.error !== undefined}
-
+
-
- {#if job != undefined && 'result' in job && job.result != undefined}
-
-
- {:else if testIsLoading}
-
- {:else if job != undefined && 'result' in job && job?.['result'] == undefined}
- Result is undefined
- {:else}
-
-
- {/if}
+
+
+
- {#if jobResult?.transformer}
-
- Transformer results
+
+ {:else if jobResult !== undefined}
+
+
+
+
+
+
+
+
+ {:else}
+
+ {/if}
+ {:else}
+
+ {#if job?.['running']}
+
+ testJobLoader?.cancelJob()}
+ >
+
+
+ {/if}
+ {#if job?.args}
+
+
+ {/if}
+ {#if job?.raw_code}
+
+
+ {/if}
+
+ {#if job?.job_kind !== 'flow' && job?.job_kind !== 'flowpreview'}
+ {@const jobResult = $jobsById[selectedJobId]}
+
+
+
+
{#if job != undefined && 'result' in job && job.result != undefined}
+ result={job.result}
+ />
{:else if testIsLoading}
{:else if job != undefined && 'result' in job && job?.['result'] == undefined}
@@ -1055,27 +1094,49 @@
{/if}
- {/if}
-
- {:else}
-
-
-
- {
- job = detail
- }}
- />
-
- {/if}
-
+ {#if jobResult?.transformer}
+
+ Transformer results
+ {#if job != undefined && 'result' in job && job.result != undefined}
+
+
+ {:else if testIsLoading}
+
+ {:else if job != undefined && 'result' in job && job?.['result'] == undefined}
+ Result is undefined
+ {:else}
+
+
+ {/if}
+
+ {/if}
+
+ {:else}
+
+
+
+ {
+ job = detail
+ }}
+ />
+
+ {/if}
+
+ {/if}
+ {:else}
+
Select a job to see its details
{/if}
- {:else}
-
Select a job to see its details
- {/if}
-
+ {/if}
+
@@ -1121,6 +1182,7 @@
placeholder="App summary"
class="text-sm w-full font-semibold"
bind:value={$summary}
+ on:keydown|stopPropagation
/>
@@ -1247,13 +1309,13 @@
size="xs"
dropdownItems={appPath != ''
? () => [
- {
- label: 'Fork',
- onClick: () => {
- window.open(`/apps/add?template=${appPath}`)
+ {
+ label: 'Fork',
+ onClick: () => {
+ window.open(`/apps/add?template=${appPath}`)
+ }
}
- }
- ]
+ ]
: undefined}
>
Deploy
diff --git a/frontend/src/lib/components/apps/editor/RecomputeAllComponents.svelte b/frontend/src/lib/components/apps/editor/RecomputeAllComponents.svelte
index 077f18135d..3b26396542 100644
--- a/frontend/src/lib/components/apps/editor/RecomputeAllComponents.svelte
+++ b/frontend/src/lib/components/apps/editor/RecomputeAllComponents.svelte
@@ -63,6 +63,7 @@
}
loading = true
+ console.log('refresh all')
const promises = Object.keys($runnableComponents)
.flatMap((id) => {
if (
@@ -72,6 +73,7 @@
return
}
+ console.log('refresh start', id)
return $runnableComponents?.[id]?.cb?.map((f) =>
f().then(() => console.log('refreshed', id))
)
@@ -106,7 +108,8 @@
]
-
+
diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/ComponentPanel.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/ComponentPanel.svelte
index 2ec4d4db47..e01804f15f 100644
--- a/frontend/src/lib/components/apps/editor/settingsPanel/ComponentPanel.svelte
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/ComponentPanel.svelte
@@ -373,7 +373,6 @@
{ccomponents[component.type].name} has no configuration
{/if}
-
{#if (`recomputeIds` in componentSettings.item.data && Array.isArray(componentSettings.item.data.recomputeIds)) || componentSettings.item.data.type === 'buttoncomponent' || componentSettings.item.data.type === 'formcomponent' || componentSettings.item.data.type === 'formbuttoncomponent' || componentSettings.item.data.type === 'checkboxcomponent'}
+ import PanelSection from './common/PanelSection.svelte'
+ import { Button } from '$lib/components/common'
+ import { RefreshCcw } from 'lucide-svelte'
+
+
+
+
+
+ Sync columns definitions with table
+
+
+
diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/GridTab.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/GridTab.svelte
index 9391736375..0571792c13 100644
--- a/frontend/src/lib/components/apps/editor/settingsPanel/GridTab.svelte
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/GridTab.svelte
@@ -182,6 +182,7 @@
+
('AppViewerContext')
@@ -73,6 +74,9 @@
? capitalize(addWhitespaceBeforeCapitals(key))
: key}
+ {#if loading}
+
+ {/if}
{#if tooltip}
{tooltip}
diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecsEditor.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecsEditor.svelte
index e0a7c10f6e..7154f8d338 100644
--- a/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecsEditor.svelte
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecsEditor.svelte
@@ -51,6 +51,7 @@
fileUpload={meta?.['fileUpload']}
placeholder={meta?.['placeholder']}
customTitle={meta?.['customTitle']}
+ loading={meta?.['loading']}
{displayType}
/>
{#if deletable}
diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/OneOfInputSpecsEditor.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/OneOfInputSpecsEditor.svelte
index 0fbdf8596d..10821eb904 100644
--- a/frontend/src/lib/components/apps/editor/settingsPanel/OneOfInputSpecsEditor.svelte
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/OneOfInputSpecsEditor.svelte
@@ -73,7 +73,11 @@
{/if}
{#each Object.keys(inputSpecsConfiguration?.[oneOf.selected] ?? {}) as nestedKey}
- {@const config = inputSpecsConfiguration?.[oneOf.selected]?.[nestedKey]}
+ {@const config = {
+ ...inputSpecsConfiguration?.[oneOf.selected]?.[nestedKey],
+ ...oneOf.configuration?.[oneOf.selected]?.[nestedKey]
+ }}
+
{#if config && oneOf.configuration[oneOf.selected]}
{/if}
{/each}
diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/SynchronizeColumns.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/SynchronizeColumns.svelte
new file mode 100644
index 0000000000..bf72b5e300
--- /dev/null
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/SynchronizeColumns.svelte
@@ -0,0 +1,86 @@
+
+
+{#if shouldDisplaySyncButton}
+
+ {
+ dispatch('add', remainingColumns)
+ }}
+ size="xs2"
+ color="dark"
+ startIcon={{ icon: Columns }}
+ >
+ Synchronize columns
+
+
+{/if}
diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/StaticInputEditor.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/StaticInputEditor.svelte
index 8b2cd719ad..8fd6f70d34 100644
--- a/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/StaticInputEditor.svelte
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/StaticInputEditor.svelte
@@ -17,13 +17,13 @@
import TableColumnWizard from '$lib/components/wizards/TableColumnWizard.svelte'
import PlotlyWizard from '$lib/components/wizards/PlotlyWizard.svelte'
import ChartJSWizard from '$lib/components/wizards/ChartJSWizard.svelte'
+ import DBExplorerWizard from '$lib/components/wizards/DBExplorerWizard.svelte'
export let componentInput: StaticInput
| undefined
export let fieldType: InputType | undefined = undefined
export let subFieldType: InputType | undefined = undefined
export let selectOptions: StaticOptions['selectOptions'] | undefined = undefined
export let placeholder: string | undefined = undefined
-
export let format: string | undefined = undefined
const { onchange } = getContext('AppViewerContext')
@@ -41,7 +41,7 @@
{:else if fieldType === 'boolean'}
+
{#each selectOptions ?? [] as option}
{#if typeof option == 'string'}
@@ -58,6 +58,22 @@
{
+ let path = e.detail
+ if (componentInput) {
+ if (path) {
+ componentInput.value = `$res:${path}`
+ } else {
+ componentInput.value = undefined
+ }
+ }
+ }}
+ showSchemaExplorer
+ resourceType="postgresql"
+ />
{:else if fieldType === 'labeledresource'}
{#if componentInput?.value && typeof componentInput?.value == 'object' && 'label' in componentInput?.value && (componentInput.value?.['value'] == undefined || typeof componentInput.value?.['value'] == 'string')}
@@ -142,6 +158,28 @@
diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceFlowList.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceFlowList.svelte
index ec0c12e60e..7193de4b07 100644
--- a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceFlowList.svelte
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceFlowList.svelte
@@ -40,7 +40,8 @@
diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceScriptList.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceScriptList.svelte
index e04a72619c..62b6806e37 100644
--- a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceScriptList.svelte
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceScriptList.svelte
@@ -40,6 +40,7 @@
= {
fieldType: T
subFieldType?: V
format?: string | undefined
+ loading?: boolean
fileUpload?: {
/** Use `*` to accept anything. */
accept: string
@@ -173,7 +177,7 @@ export type AppInput =
| AppInputSpec<'any', any>
| AppInputSpec<'object', Record
>
| AppInputSpec<'object', string>
- | (AppInputSpec<'select', string> & StaticOptions)
+ | (AppInputSpec<'select', string, 'db-table'> & StaticOptions)
| AppInputSpec<'icon-select', string>
| AppInputSpec<'color', string>
| AppInputSpec<'array', string[], 'text'>
@@ -192,10 +196,12 @@ export type AppInput =
| AppInputSpec<'array', object[], 'tab-select'>
| AppInputSpec<'schema', object>
| AppInputSpec<'array', object[], 'ag-grid'>
+ | AppInputSpec<'array', object[], 'db-explorer'>
| AppInputSpec<'array', object[], 'table-column'>
| AppInputSpec<'array', object[], 'plotly'>
| AppInputSpec<'array', object[], 'chartjs'>
| AppInputSpec<'array', DecisionTreeNode, 'DecisionTreeNode'>
+ | AppInputSpec<'resource', string>
export type RowAppInput = Extract
export type StaticAppInput = Extract
diff --git a/frontend/src/lib/components/common/button/Button.svelte b/frontend/src/lib/components/common/button/Button.svelte
index 65645f2002..236b9c64a2 100644
--- a/frontend/src/lib/components/common/button/Button.svelte
+++ b/frontend/src/lib/components/common/button/Button.svelte
@@ -9,7 +9,7 @@
export let size: ButtonType.Size = 'md'
export let spacingSize: ButtonType.Size = size
- export let color: ButtonType.Color = 'blue'
+ export let color: ButtonType.Color | string = 'blue';
export let variant: ButtonType.Variant = 'contained'
export let btnClasses: string = ''
export let wrapperClasses: string = ''
@@ -115,18 +115,26 @@
}
}
- $: buttonClass = twMerge(
- 'w-full',
- colorVariants?.[color]?.[variant],
- variant === 'border' ? 'border' : '',
- ButtonType.FontSizeClasses[size],
- ButtonType.SpacingClasses[spacingSize][variant],
- 'focus:ring-2 font-semibold',
- dropdownItems ? 'rounded-l-md h-full' : 'rounded-md',
- 'justify-center items-center text-center whitespace-nowrap inline-flex gap-2',
- btnClasses,
- 'transition-all'
- )
+ function getColorClass(color, variant) {
+ if (color in colorVariants) {
+ return colorVariants[color][variant];
+ } else {
+ return color;
+ }
+ }
+
+ $: buttonClass = twMerge(
+ 'w-full',
+ getColorClass(color, variant),
+ variant === 'border' ? 'border' : '',
+ ButtonType.FontSizeClasses[size],
+ ButtonType.SpacingClasses[spacingSize][variant],
+ 'focus:ring-2 font-semibold',
+ dropdownItems ? 'rounded-l-md h-full' : 'rounded-md',
+ 'justify-center items-center text-center whitespace-nowrap inline-flex gap-2',
+ btnClasses,
+ 'transition-all'
+ );
const iconMap = {
xs: 14,
diff --git a/frontend/src/lib/components/common/button/model.ts b/frontend/src/lib/components/common/button/model.ts
index b9bea721bb..d8b479cbeb 100644
--- a/frontend/src/lib/components/common/button/model.ts
+++ b/frontend/src/lib/components/common/button/model.ts
@@ -2,7 +2,7 @@ export const BUTTON_COLORS = ['blue', 'red', 'dark', 'light', 'green', 'gray', '
export namespace ButtonType {
export type Size = 'xs2' | 'xs' | 'sm' | 'md' | 'lg' | 'xl'
- export type Color = (typeof BUTTON_COLORS)[number]
+ export type Color = string;
export type Variant = 'contained' | 'border' | 'divider'
export type Target = '_self' | '_blank'
export type Element = HTMLButtonElement | HTMLAnchorElement
diff --git a/frontend/src/lib/components/flows/content/FlowInput.svelte b/frontend/src/lib/components/flows/content/FlowInput.svelte
index 1cf00bb5cf..6de0fa6dba 100644
--- a/frontend/src/lib/components/flows/content/FlowInput.svelte
+++ b/frontend/src/lib/components/flows/content/FlowInput.svelte
@@ -91,7 +91,7 @@
-
Customize Inputs
+ Customize Flow Inputs
diff --git a/frontend/src/lib/components/flows/pickers/WorkspaceScriptPicker.svelte b/frontend/src/lib/components/flows/pickers/WorkspaceScriptPicker.svelte
index 671d9c5a53..7f8455a052 100644
--- a/frontend/src/lib/components/flows/pickers/WorkspaceScriptPicker.svelte
+++ b/frontend/src/lib/components/flows/pickers/WorkspaceScriptPicker.svelte
@@ -29,7 +29,11 @@
$: $workspaceStore && kind && loadItems()
async function loadItems(): Promise {
- items = await ScriptService.listScripts({ workspace: $workspaceStore!, kinds: kind, isTemplate })
+ items = await ScriptService.listScripts({
+ workspace: $workspaceStore!,
+ kinds: kind,
+ isTemplate
+ })
}
let ownerFilter: string | undefined = undefined
@@ -58,6 +62,7 @@
Result
Logs
- {#if job?.job_kind == 'dependencies'}
- Code
- {:else if job?.job_kind == 'preview'}
+ {#if job?.job_kind == 'preview'}
Code
{/if}
diff --git a/frontend/src/lib/components/settings/PremiumInfo.svelte b/frontend/src/lib/components/settings/PremiumInfo.svelte
index af6f6995b1..191f4b5e59 100644
--- a/frontend/src/lib/components/settings/PremiumInfo.svelte
+++ b/frontend/src/lib/components/settings/PremiumInfo.svelte
@@ -6,7 +6,7 @@
import { workspaceStore } from '$lib/stores'
import { Button } from '../common'
import Tooltip from '../Tooltip.svelte'
- import { Badge, ExternalLink } from 'lucide-svelte'
+ import { ExternalLink } from 'lucide-svelte'
import { twMerge } from 'tailwind-merge'
export let plan: string | undefined
@@ -19,7 +19,7 @@
premium: boolean
usage: number
seats: number
- authorNb: number
+ developerNb: number
operatorNb: number
seatsFromUsers: number
seatsFromComps: number
@@ -34,15 +34,14 @@
Team: [
`$10/mo per seat`,
`Every seat includes 10 000 executions`,
- `Every seat includes either 1 user OR 2 operators`
+ `Every seat includes either 1 developer OR 2 operators`
],
Enterprise: [
- `Dedicated and isolated database and workers available (EU/US/Asia)`,
- `Dedicated entire cluster available for (EU/US/Asia)`,
- `SAML support with group syncing`,
- `SLA `,
- `Priority Support 24/7 with 3h response time and automation engineer assistance `,
- `Design partners for Roadmap `,
+ `Dedicated and isolated database and workers available (EU/US/Asia)`,
+ `Dedicated entire cluster available for (EU/US/Asia)`,
+ `SAML support with group syncing`,
+ `SLA & Priority Support 24/7 with 3h response time and automation engineer assistance`,
+ `Design partners for Roadmap`,
`Self-hosted licenses also available
`
]
}
@@ -60,16 +59,16 @@
async function loadPremiumInfo() {
const info = await WorkspaceService.getPremiumInfo({ workspace: $workspaceStore! })
- const authorNb = users?.filter((x) => !x.operator)?.length ?? 0
+ const developerNb = users?.filter((x) => !x.operator)?.length ?? 0
const operatorNb = users?.filter((x) => x.operator)?.length ?? 0
- const seatsFromUsers = Math.ceil(authorNb + operatorNb / 2)
+ const seatsFromUsers = Math.ceil(developerNb + operatorNb / 2)
const seatsFromComps = Math.ceil((info.usage ?? 0) / 10000)
const usedSeats = Math.max(seatsFromUsers, seatsFromComps)
premiumInfo = {
...info,
usage: info.usage ?? 0,
seats: info.seats ?? 1,
- authorNb,
+ developerNb,
operatorNb,
seatsFromUsers,
seatsFromComps,
@@ -113,7 +112,7 @@
- Authors
+ Developers
Actual pricing is calculated on the MAXIMUM number of users in a given billing
period, see the customer portal for more info.
@@ -121,7 +120,7 @@
|
- {premiumInfo.authorNb}
+ {premiumInfo.developerNb}
|
@@ -145,7 +144,7 @@
>
- u = ceil({premiumInfo.authorNb} + {premiumInfo.operatorNb}/2) = {premiumInfo.seatsFromUsers}
+ u = ceil({premiumInfo.developerNb} + {premiumInfo.operatorNb}/2) = {premiumInfo.seatsFromUsers}
|
@@ -178,7 +177,7 @@
: ''
)}
>Used seats Highest between seats from authors + operators and seats from computations
+ >Highest between seats from developers + operators and seats from computations
{plan === 'team' && premiumInfo.usedSeats > premiumInfo.seats
? ' > Paid seats'
: ''}
{:else}
- This workspace is NOT on a team plan. Users use their global free-tier quotas when doing executions
+ This workspace is not on a team plan. Users use their global free-tier quotas when doing executions
in this workspace. Upgrade to a Team or Enterprise plan to unlock unlimited executions in this workspace.
{/if}
@@ -221,14 +220,16 @@
All plans
- {#each Object.entries(plans) as [planTitle, planDesc]}
-
-
{planTitle}
-
- {#each planDesc as item}
- {@html item}
- {/each}
-
+ {#each Object.entries(plans) as [planTitle, planDesc]}
+
+
+ {planTitle}
+
+
+ {#each planDesc as item}
+ {@html item}
+ {/each}
+
{#if planTitle == 'Team'}
@@ -236,13 +237,13 @@
{#if plan != 'enterprise'}
- Upgrade to the Team plan
{:else}
@@ -256,7 +257,7 @@
{:else if planTitle == 'Enterprise'}
{#if plan != 'enterprise'}
-
+
See more
@@ -269,18 +270,19 @@
Cancel your plan in the customer portal to downgrade to the free plan
{:else}
-
Workspace is on the free plan
+
+ Workspace is on the free plan
+
{/if}
{/if}
{/each}
-
Frequently asked questions
-
+
Frequently asked questions
-
What is an execution?
+
What is an execution?
The single credit-unit is called an "execution". An execution corresponds to a single job
whose duration is less than 1s. For any additional seconds of computation, an additional
@@ -289,11 +291,11 @@
-
- What is the difference between an author and an operator?
+
+ What is the difference between a developer and an operator?
- An author can write scripts/flows/apps/variables/resources. An operator can only run/view
+ A developer can write scripts/flows/apps/variables/resources. An operator can only run/view
them.
diff --git a/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte b/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte
index 089fe282aa..e98ca04657 100644
--- a/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte
+++ b/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte
@@ -152,7 +152,7 @@
{/key}
|
diff --git a/frontend/src/lib/components/wizards/DBExplorerWizard.svelte b/frontend/src/lib/components/wizards/DBExplorerWizard.svelte
new file mode 100644
index 0000000000..3b4e962847
--- /dev/null
+++ b/frontend/src/lib/components/wizards/DBExplorerWizard.svelte
@@ -0,0 +1,381 @@
+
+
+
+
+
+ {#if value}
+
+
+
+
+ {
+ if (value?.ignored) {
+ e?.stopPropagation()
+ }
+ }}
+ >
+
+
+
+
+ {
+ e?.stopPropagation()
+ }}
+ options={{ right: 'Editable' }}
+ bind:checked={value.editable}
+ size="xs"
+ />
+
+
+
+
+
+
+
+
+ It's often required that one or more columns fill the entire available space in the
+ grid. For this scenario, it is possible to use the flex config. Some columns could
+ be set with a regular width config, while other columns would have a flex config.
+ Flex sizing works by dividing the remaining space in the grid among all flex columns
+ in proportion to their flex value. For example, suppose the grid has a total width
+ of 450px and it has three columns: the first with width: 150; the second with flex:
+ 1; and third with flex: 2. The first column will be 150px wide, leaving 300px
+ remaining. The column with flex: 2 has twice the size with flex: 1. So final sizes
+ will be: 150px, 100px, 200px.
+
+
+
+ {value.flex}
+
+
+
+ {
+ e?.stopPropagation()
+ }}
+ options={{ right: 'Hide' }}
+ bind:checked={value.hide}
+ size="xs"
+ />
+
+
+
+
+
+ Value formatters allow you to format values for display. This is useful when data is
+ one type (e.g. numeric) but needs to be converted for human reading (e.g. putting in
+ currency symbols and number formatting).
+
+
+
+ {
+ // @ts-ignore
+ value.valueFormatter = null
+ renderCount++
+ }}
+ >
+ Clear
+
+
+
+
+ {#key renderCount}
+
+
+ {#if !presets.find((preset) => preset.value === value?.valueFormatter)}
+
+ {/if}
+
Presets
+
{
+ renderCount++
+ }}
+ placeholder="Code"
+ >
+ {#each presets as preset}
+ {preset.label}
+ {/each}
+
+
+
+
+
Use `value` in the formatter
+
+ {/key}
+
+
+
+
+ None
+ Ascending
+ Descending
+
+
+
+
diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts
index f47f3ea5df..aefcbe1a9f 100644
--- a/frontend/src/lib/stores.ts
+++ b/frontend/src/lib/stores.ts
@@ -102,6 +102,6 @@ export interface GraphqlSchema {
export type DBSchema = SQLSchema | GraphqlSchema
-type DBSchemas = Partial
>
+export type DBSchemas = Partial>
export const dbSchemas = writable({})
diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts
index 2767653504..397d42b4ee 100644
--- a/frontend/src/lib/utils.ts
+++ b/frontend/src/lib/utils.ts
@@ -49,6 +49,19 @@ export function displayDate(dateString: string | Date | undefined, displaySecond
}
}
+export function displayTime(dateString: string | Date | undefined): string {
+ const date = new Date(dateString ?? '')
+ if (date.toString() === 'Invalid Date') {
+ return ''
+ } else {
+ return `${date.toLocaleTimeString([], {
+ hour: '2-digit',
+ minute: '2-digit',
+ second: '2-digit',
+ })}.${date.getMilliseconds()}`
+ }
+}
+
export function displaySize(sizeInBytes: number | undefined): string | undefined {
if (sizeInBytes === undefined) {
return undefined
@@ -514,6 +527,7 @@ export function deepMergeWithPriority(target: T, source: T): T {
for (const key in source) {
if (source.hasOwnProperty(key) && merged?.hasOwnProperty(key)) {
+ console.log(target)
if (target?.hasOwnProperty(key)) {
merged[key] = deepMergeWithPriority(target[key], source[key])
} else {
diff --git a/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte
index 8343bb472a..bd34eabfbc 100644
--- a/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte
+++ b/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte
@@ -59,8 +59,11 @@
/>
{#if can_write && !hideEditBtn}
- Edit Edit
{/if}
diff --git a/frontend/src/routes/(root)/(logged)/resources/+page.svelte b/frontend/src/routes/(root)/(logged)/resources/+page.svelte
index 0569a049fe..02b7a4fce3 100644
--- a/frontend/src/routes/(root)/(logged)/resources/+page.svelte
+++ b/frontend/src/routes/(root)/(logged)/resources/+page.svelte
@@ -130,7 +130,6 @@
: true
})
: preFilteredItemsOwners?.filter((x) => {
- console.log(x.resource_type)
return (
x.resource_type === typeFilter &&
(tab === 'workspace'
diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte
index f7b12eff58..bc19d5595c 100644
--- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte
+++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte
@@ -49,10 +49,12 @@
import ButtonDropdown from '$lib/components/common/button/ButtonDropdown.svelte'
import PersistentScriptDrawer from '$lib/components/PersistentScriptDrawer.svelte'
import Portal from 'svelte-portal'
+ import MemoryFootprintViewer from '$lib/components/MemoryFootprintViewer.svelte'
let job: Job | undefined
+ let jobUpdateLastFetch: Date | undefined
- let viewTab: 'result' | 'logs' | 'code' = 'result'
+ let viewTab: 'result' | 'logs' | 'code' | 'stats' = 'result'
let selectedJobStep: string | undefined = undefined
let branchOrIterationN: number = 0
@@ -171,6 +173,7 @@
bind:this={testJobLoader}
bind:isLoading={testIsLoading}
bind:job
+ bind:jobUpdateLastFetch
workspaceOverride={$workspaceStore}
bind:notfound
/>
@@ -404,16 +407,17 @@
>
{/if}
{/if}
-
+ >
View {job?.job_kind}
-
+
{/if}
@@ -507,9 +511,8 @@
Result
Logs
- {#if job?.job_kind == 'dependencies'}
- Code
- {:else if job?.job_kind == 'preview'}
+ Metrics
+ {#if job?.job_kind == 'preview'}
Code
{/if}
@@ -538,6 +541,10 @@
{:else}
+
{:else if job !== undefined && 'result' in job && job.result !== undefined}
{
+ workspaceStore.set(undefined)
workspaceStore.set(workspace.id)
loading = true
await goto(rd ?? '/')
diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
index 2e901d72e4..6c2c24319f 100644
--- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
+++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
@@ -56,7 +56,11 @@
let errorHandlerMutedOnCancel: boolean | undefined = undefined
let openaiResourceInitialPath: string | undefined = undefined
let s3ResourceInitialPath: string | undefined = undefined
- let gitSyncResourcePath: string | undefined = undefined
+ let gitSyncSettings: {
+ script_path: string
+ git_repo_resource_path: string
+ use_individual_branch: boolean
+ }
let gitSyncTestJob:
| {
jobId: string
@@ -190,28 +194,36 @@
}
}
- async function editWindmillGitSyncSettings(newGitRepoResourcePath: string): Promise {
- gitSyncResourcePath = newGitRepoResourcePath
- if (newGitRepoResourcePath) {
- let resourcePathWithPrefix = `$res:${newGitRepoResourcePath}`
+ async function editWindmillGitSyncSettings(
+ gitRepoResourcePath: string,
+ useIndividualBranch: boolean
+ ): Promise {
+ if (!emptyString(gitRepoResourcePath)) {
+ gitSyncSettings = {
+ script_path: 'hub/7924/sync-script-to-git-repo-windmill',
+ git_repo_resource_path: `$res:${gitRepoResourcePath.replace('$res:', '')}`,
+ use_individual_branch: useIndividualBranch
+ }
await WorkspaceService.editWorkspaceGitSyncConfig({
workspace: $workspaceStore!,
requestBody: {
- git_sync_settings: {
- script_path: 'hub/7848/sync-script-to-git-repo-windmill',
- git_repo_resource_path: resourcePathWithPrefix
- }
+ git_sync_settings: gitSyncSettings
}
})
- sendUserToast(`Workspace Git sync settings updated`)
+ sendUserToast('Workspace Git sync settings updated')
} else {
+ gitSyncSettings = {
+ script_path: '',
+ git_repo_resource_path: '',
+ use_individual_branch: false
+ }
await WorkspaceService.editWorkspaceGitSyncConfig({
workspace: $workspaceStore!,
requestBody: {
git_sync_settings: undefined
}
})
- sendUserToast(`Workspace Git sync settings reset`)
+ sendUserToast('Workspace Git sync settings reset')
}
}
@@ -249,7 +261,19 @@
settings.large_file_storage?.type === LargeFileStorage.type.S3STORAGE
? settings.large_file_storage?.s3_resource_path?.replace('$res:', '')
: undefined
- gitSyncResourcePath = settings.git_sync?.git_repo_resource_path?.replace('$res:', '')
+ if (settings.git_sync !== undefined && settings.git_sync !== null) {
+ gitSyncSettings = {
+ git_repo_resource_path: settings.git_sync.git_repo_resource_path.replace('$res:', ''),
+ script_path: settings.git_sync.script_path,
+ use_individual_branch: settings.git_sync.use_individual_branch ?? false
+ }
+ } else {
+ gitSyncSettings = {
+ git_repo_resource_path: '',
+ script_path: '',
+ use_individual_branch: false
+ }
+ }
// check openai_client_credentials_oauth
usingOpenaiClientCredentialsOauth = await ResourceService.existsResourceType({
@@ -298,15 +322,15 @@
)
}
- async function runGitSyncTestJob(gitRepoResourcePath: string | undefined) {
- if (gitRepoResourcePath === undefined) {
+ async function runGitSyncTestJob() {
+ if (emptyString(gitSyncSettings.script_path)) {
return
}
let jobId = await JobService.runScriptByPath({
workspace: $workspaceStore!,
- path: 'hub/7846/git-repo-test-read-write-windmill',
+ path: 'hub/7925/git-repo-test-read-write-windmill',
requestBody: {
- repo_url_resource_path: gitRepoResourcePath
+ repo_url_resource_path: gitSyncSettings.git_repo_resource_path.replace('$res:', '')
}
})
gitSyncTestJob = {
@@ -365,8 +389,11 @@
Users
+
+ Git Sync
+
- Dev/Staging/Prod
+ Deployment UI
{#if WORKSPACE_SHOW_SLACK_CMD}
@@ -392,9 +419,6 @@
S3 Storage
-
- Git sync
-
Delete Workspace
@@ -681,7 +705,7 @@
/>
{:else if tab == 'windmill_lfs'}
-
+
{#if !$enterpriseLicense}
Windmill S3 bucket browser will not work for buckets containing more than 20 files.
@@ -731,6 +755,7 @@
Automatically saving scripts to a Git repository on each deploy is a Windmill EE feature.
+
{/if}
- {#key s3ResourceInitialPath}
- {
- editWindmillGitSyncSettings(ev.detail)
- }}
- />
- runGitSyncTestJob(gitSyncResourcePath)}
- size="xs">Test connection
- {/key}
+ {#if gitSyncSettings}
+ {#key gitSyncSettings}
+ {
+ editWindmillGitSyncSettings(ev.detail, gitSyncSettings?.use_individual_branch)
+ }}
+ />
+ runGitSyncTestJob()}
+ size="xs">Test connection
+ {/key}
+ {/if}
{#if gitSyncTestJob !== undefined}
@@ -776,6 +803,23 @@
{/if}
+
+ {#if gitSyncSettings}
+ {
+ editWindmillGitSyncSettings(gitSyncSettings.git_repo_resource_path, ev.detail)
+ }}
+ options={{
+ right: 'Create one branch per deployed script/flow/app',
+ rightTooltip:
+ "If set, Windmill will create a unique branch per script/flow/app being pushed, prefixed with 'wm_deploy/'."
+ }}
+ />
+ {/if}
+
+
Git repository initial setup
@@ -802,12 +846,12 @@
> wmill workspace add WORKSPACE_NAME WORKSPACE_ID WINDMILL_URL
-> echo 'u/' > .wmillignore
-> wmill sync pull --raw --skip-variables --skip-secrets --skip-resources
-> git add -A
-> git commit -m 'Initial commit'
-> git pushwmill workspace add {$workspaceStore} {$workspaceStore} {`${$page.url.protocol}//${$page.url.hostname}/`}
+echo 'u/' > .wmillignore
+wmill sync pull --raw --skip-variables --skip-secrets --skip-resources
+git add -A
+git commit -m 'Initial commit'
+git push
diff --git a/lsp/Pipfile b/lsp/Pipfile
index 6b6ebba507..a598b92449 100644
--- a/lsp/Pipfile
+++ b/lsp/Pipfile
@@ -4,8 +4,8 @@ verify_ssl = true
name = "pypi"
[packages]
-wmill = ">=1.232.0"
-wmill_pg = ">=1.232.0"
+wmill = ">=1.236.2"
+wmill_pg = ">=1.236.2"
sendgrid = "*"
mysql-connector-python = "*"
pymongo = "*"
diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml
index 2a8326e3c3..d77e7d3135 100644
--- a/openflow.openapi.yaml
+++ b/openflow.openapi.yaml
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
- version: 1.232.0
+ version: 1.236.2
title: OpenFlow Spec
contact:
name: Ruben Fiszel
diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml
index d6a9904b92..f885cef21e 100644
--- a/python-client/wmill/pyproject.toml
+++ b/python-client/wmill/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "wmill"
-version = "1.232.0"
+version = "1.236.2"
description = "A client library for accessing Windmill server wrapping the Windmill client API"
license = "Apache-2.0"
homepage = "https://windmill.dev"
diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py
index 2b587e8985..5f0f7a3b6a 100644
--- a/python-client/wmill/wmill/client.py
+++ b/python-client/wmill/wmill/client.py
@@ -321,7 +321,6 @@ class Windmill:
def get_duckdb_connection_settings(
self,
s3_resource_path: str = "",
- none_if_undefined: bool = False,
) -> DuckDbConnectionSettings | None:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
@@ -334,15 +333,12 @@ class Windmill:
).json()
return DuckDbConnectionSettings(raw_obj)
except JSONDecodeError as e:
- if none_if_undefined:
- return None
raise Exception("Could not generate DuckDB S3 connection settings from the provided resource") from e
def get_polars_connection_settings(
self,
s3_resource_path: str = "",
- none_if_undefined: bool = False,
- ) -> PolarsConnectionSettings | None:
+ ) -> PolarsConnectionSettings:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection from Polars
@@ -354,15 +350,12 @@ class Windmill:
).json()
return PolarsConnectionSettings(raw_obj)
except JSONDecodeError as e:
- if none_if_undefined:
- return None
raise Exception("Could not generate Polars S3 connection settings from the provided resource") from e
def get_boto3_connection_settings(
self,
s3_resource_path: str = "",
- none_if_undefined: bool = False,
- ) -> Boto3ConnectionSettings | None:
+ ) -> Boto3ConnectionSettings:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection using boto3
@@ -385,8 +378,6 @@ class Windmill:
)
return boto3_settings
except JSONDecodeError as e:
- if none_if_undefined:
- return None
raise Exception("Could not generate Polars S3 connection settings from the provided resource") from e
def whoami(self) -> dict:
@@ -592,38 +583,30 @@ def get_result(job_id: str, assert_result_is_not_none=True) -> Dict[str, Any]:
@init_global_client
-def duckdb_connection_settings(
- s3_resource_path: str = "", none_if_undefined: bool = False
-) -> DuckDbConnectionSettings | None:
+def duckdb_connection_settings(s3_resource_path: str = "") -> DuckDbConnectionSettings:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection from DuckDB
"""
- return _client.get_duckdb_connection_settings(
- s3_resource_path=s3_resource_path, none_if_undefined=none_if_undefined
- )
+ return _client.get_duckdb_connection_settings(s3_resource_path)
@init_global_client
-def polars_connection_settings(
- s3_resource_path: str = "", none_if_undefined: bool = False
-) -> PolarsConnectionSettings | None:
+def polars_connection_settings(s3_resource_path: str = "") -> PolarsConnectionSettings:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection from Polars
"""
- return _client.get_polars_connection_settings(s3_resource_path, none_if_undefined)
+ return _client.get_polars_connection_settings(s3_resource_path)
@init_global_client
-def boto3_connection_settings(
- s3_resource_path: str = "", none_if_undefined: bool = False
-) -> Boto3ConnectionSettings | None:
+def boto3_connection_settings(s3_resource_path: str = "") -> Boto3ConnectionSettings:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection using boto3
"""
- return _client.get_boto3_connection_settings(s3_resource_path, none_if_undefined)
+ return _client.get_boto3_connection_settings(s3_resource_path)
@init_global_client
diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml
index 7eecc7cf97..ecc2b9cd08 100644
--- a/python-client/wmill_pg/pyproject.toml
+++ b/python-client/wmill_pg/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "wmill-pg"
-version = "1.232.0"
+version = "1.236.2"
description = "An extension client for the wmill client library focused on pg"
license = "Apache-2.0"
homepage = "https://windmill.dev"
diff --git a/typescript-client/package.json b/typescript-client/package.json
index 66f35cd2ef..ba0adf1f60 100644
--- a/typescript-client/package.json
+++ b/typescript-client/package.json
@@ -1,7 +1,7 @@
{
"name": "windmill-client",
"description": "Windmill SDK client for browsers and Node.js",
- "version": "1.232.0",
+ "version": "1.236.2",
"author": "Ruben Fiszel",
"license": "Apache 2.0",
"devDependencies": {
diff --git a/version.txt b/version.txt
index a50fe3a802..ff83e10213 100644
--- a/version.txt
+++ b/version.txt
@@ -1 +1 @@
-1.232.0
+1.236.2
