starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-09 00:04:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into rust-inc-comp

Browse Source
This commit is contained in:
pyranota
2025-04-11 15:12:50 +02:00
committed by GitHub
parent 99fcacb2e6 5e054d3385
commit 9221a63edb
190 changed files with 9362 additions and 3113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-publish-rh-image.yml
+2 -2
View File
@@ -64,7 +64,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
@@ -81,7 +81,7 @@ jobs:
platforms: linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
.github/workflows/build_windows_worker_.yml
+1 -1
View File
@@ -51,7 +51,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
- name: Rename binary with corresponding architecture
run: |
Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe"
.github/workflows/docker-image.yml
+2 -2
View File
@@ -95,7 +95,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,agent_worker_server,all_languages
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
${{ steps.meta-public.outputs.tags }}
@@ -157,7 +157,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,agent_worker_server,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
${{ steps.meta-ee-public.outputs.tags }}
.github/workflows/publish_windows_worker.yml
+1 -1
View File
@@ -53,7 +53,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages
- name: Rename binary with corresponding architecture
run: |
Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe"
backend/.sqlx/query-0102308ffa1c0dbfba54d29246535bb81146a4cfae0ec408435570e0813a3bef.json
Generated
+128
View File
@@ -0,0 +1,128 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM \n gcp_trigger\n WHERE \n workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "gcp_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "subscription_id",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "topic_id",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "delivery_type: _",
"type_info": {
"Custom": {
"name": "delivery_mode",
"kind": {
"Enum": [
"push",
"pull"
]
}
}
}
},
{
"ordinal": 5,
"name": "delivery_config: _",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 12,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 14,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 15,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 16,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
true,
false,
false,
false,
false,
false,
false,
true,
true,
false,
true,
false
]
},
"hash": "0102308ffa1c0dbfba54d29246535bb81146a4cfae0ec408435570e0813a3bef"
}
backend/.sqlx/query-070b8ad0b59f485fa5bf68082b060f5c3561c37e9c6f2834d234a862a475a6eb.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n gcp_trigger \n SET \n enabled = $1, \n email = $2, \n edited_by = $3, \n edited_at = now(), \n server_id = NULL, \n error = NULL\n WHERE \n path = $4 AND \n workspace_id = $5 \n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Bool",
"Varchar",
"Varchar",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "070b8ad0b59f485fa5bf68082b060f5c3561c37e9c6f2834d234a862a475a6eb"
}
backend/.sqlx/query-0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET \n ping_at = now(), \n jobs_executed = 1, \n current_job_id = $1, \n current_job_workspace_id = 'admins' \n WHERE worker = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": []
},
"hash": "0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182"
}
backend/.sqlx/query-15fbe481789a7817bf37415fb935f9ed537fd7d3b266d928af4d5d3dd8bb5c18.json
Generated
+126
View File
@@ -0,0 +1,126 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM\n gcp_trigger\n WHERE\n delivery_type != 'push'::DELIVERY_MODE AND\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "gcp_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "subscription_id",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "topic_id",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "delivery_type: _",
"type_info": {
"Custom": {
"name": "delivery_mode",
"kind": {
"Enum": [
"push",
"pull"
]
}
}
}
},
{
"ordinal": 5,
"name": "delivery_config: _",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 12,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 14,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 15,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 16,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
false,
false,
true,
false,
false,
false,
false,
false,
false,
true,
true,
false,
true,
false
]
},
"hash": "15fbe481789a7817bf37415fb935f9ed537fd7d3b266d928af4d5d3dd8bb5c18"
}
backend/.sqlx/query-e23e110e1f0438d21534fc4323e0e7bc1f0dbeca2e4f44ced05bae0ca5ca1039.json → backend/.sqlx/query-23419adcd74c326d716527293eff518b42f4cdb33e034441015494bd26c172d2.json
Generated
+6 -4
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT trigger_config as \"trigger_config: _\", owner, email\n FROM capture_config\n WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4 AND last_client_ping > NOW() - INTERVAL '10 seconds'",
"query": "\n SELECT \n trigger_config AS \"trigger_config: _\", \n owner, \n email\n FROM \n capture_config\n WHERE \n workspace_id = $1\n AND path = $2\n AND is_flow = $3\n AND trigger_kind = $4\n AND last_client_ping > NOW() - INTERVAL '10 seconds'\n AND (\n $5::bool IS FALSE\n OR (\n trigger_config IS NOT NULL\n AND trigger_config ->> 'delivery_type' = 'push'\n )\n )\n ",
"describe": {
"columns": [
{
@@ -37,11 +37,13 @@
"nats",
"postgres",
"sqs",
"mqtt"
"mqtt",
"gcp"
]
}
}
}
},
"Bool"
]
},
"nullable": [
@@ -50,5 +52,5 @@
false
]
},
"hash": "e23e110e1f0438d21534fc4323e0e7bc1f0dbeca2e4f44ced05bae0ca5ca1039"
"hash": "23419adcd74c326d716527293eff518b42f4cdb33e034441015494bd26c172d2"
}
backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json
Generated
+25
View File
@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'gcp' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text",
"Bool"
]
},
"nullable": [
null
]
},
"hash": "299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63"
}
backend/.sqlx/query-2ebb0463b790ddf7ba0ee22d8c9afc88eb57c4110a202775003fb48b2f4e317f.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'started_at'],\n to_jsonb(now()::text)\n )\n WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid"
]
},
"nullable": []
},
"hash": "2ebb0463b790ddf7ba0ee22d8c9afc88eb57c4110a202775003fb48b2f4e317f"
}
backend/.sqlx/query-6ff7a025f529c077c1b6c9632a367aa29e2f0fdac3f1984550484d5a06a6ea21.json → backend/.sqlx/query-2faa27519624249f16cf89814ab5efe8f8daf928c1194cecacfa8223165fb9f2.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_runtime r SET\n memory_peak = $1,\n ping = now()\n FROM v2_job_queue q\n WHERE r.id = $2 AND q.id = r.id\n RETURNING canceled_by, canceled_reason",
"query": "UPDATE v2_job_runtime r SET\n memory_peak = $1,\n ping = now()\n FROM v2_job_queue q\n WHERE r.id = $2 AND q.id = r.id\n RETURNING canceled_by, canceled_reason",
"describe": {
"columns": [
{
@@ -25,5 +25,5 @@
true
]
},
"hash": "6ff7a025f529c077c1b6c9632a367aa29e2f0fdac3f1984550484d5a06a6ea21"
"hash": "2faa27519624249f16cf89814ab5efe8f8daf928c1194cecacfa8223165fb9f2"
}
backend/.sqlx/query-33ee913ce263600a3f94f90e4a42cf0e4086030f3b7994e4892392765cbe1517.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT \n 1 \n FROM \n gcp_trigger \n WHERE \n path = $1 AND \n workspace_id = $2\n )",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "33ee913ce263600a3f94f90e4a42cf0e4086030f3b7994e4892392765cbe1517"
}
backend/.sqlx/query-37011e7f4cdfc87294e44252cca4f4683a12b82b972842f88f5c01111580224d.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture\n SET \n path = $1\n WHERE \n path = $2 \n AND workspace_id = $3 \n AND is_flow = $4\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "37011e7f4cdfc87294e44252cca4f4683a12b82b972842f88f5c01111580224d"
}
backend/.sqlx/query-3f5520e0ea00569bf169da9abde31617043fcc0bea3ef62c50fcd881f3d48605.json
Generated
+35
View File
@@ -0,0 +1,35 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4,\n $5,\n $6, \n $7, \n $8, \n $9,\n $10,\n $11,\n $12\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
{
"Custom": {
"name": "delivery_mode",
"kind": {
"Enum": [
"push",
"pull"
]
}
}
},
"Jsonb",
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Bool",
"Varchar"
]
},
"nullable": []
},
"hash": "3f5520e0ea00569bf169da9abde31617043fcc0bea3ef62c50fcd881f3d48605"
}
backend/.sqlx/query-41e557e1b63b13c9fcc195901c0bd0de7e03c539ee046955543d9693551246f7.json
Generated
-14
View File
@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM capture WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "41e557e1b63b13c9fcc195901c0bd0de7e03c539ee046955543d9693551246f7"
}
backend/.sqlx/query-c5270ee815689e42b65df507b850da43239c9a5aaea41c9aed7ed33a6219a534.json → backend/.sqlx/query-42b4b73e9d60348e2d90fcade9dcad6d8995242dc20a4e14c1a8fae4fc6a9fd2.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE capture_config SET last_client_ping = now() WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4",
"query": "\n UPDATE \n capture_config\n SET \n last_client_ping = NOW()\n WHERE \n workspace_id = $1 \n AND path = $2 \n AND is_flow = $3 \n AND trigger_kind = $4\n ",
"describe": {
"columns": [],
"parameters": {
@@ -21,7 +21,8 @@
"nats",
"postgres",
"sqs",
"mqtt"
"mqtt",
"gcp"
]
}
}
@@ -30,5 +31,5 @@
},
"nullable": []
},
"hash": "c5270ee815689e42b65df507b850da43239c9a5aaea41c9aed7ed33a6219a534"
"hash": "42b4b73e9d60348e2d90fcade9dcad6d8995242dc20a4e14c1a8fae4fc6a9fd2"
}
backend/.sqlx/query-43c8cd9f8560412bb06d9966ccfa2524943ba277c5f9a37c06bd5ee14fd46bda.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'gcp'\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "43c8cd9f8560412bb06d9966ccfa2524943ba277c5f9a37c06bd5ee14fd46bda"
}
backend/.sqlx/query-44b9bea3651edc8ee732def1241b3d956c004376102ccc1707fc016801599dbd.json
Generated
+36
View File
@@ -0,0 +1,36 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n gcp_trigger \n SET \n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6, \n edited_by = $7, \n email = $8,\n script_path = $9,\n path = $10,\n enabled = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL\n WHERE \n workspace_id = $12 AND \n path = $13\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
{
"Custom": {
"name": "delivery_mode",
"kind": {
"Enum": [
"push",
"pull"
]
}
}
},
"Jsonb",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "44b9bea3651edc8ee732def1241b3d956c004376102ccc1707fc016801599dbd"
}
backend/.sqlx/query-45d616c92ebcbe30a563e1fa7d2d0e53392e238144b039cfe042587d7fe1dea3.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'started_at'],\n to_jsonb(now()::text)\n )\n WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid"
]
},
"nullable": []
},
"hash": "45d616c92ebcbe30a563e1fa7d2d0e53392e238144b039cfe042587d7fe1dea3"
}
backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json
Generated
+3 -1
View File
@@ -220,7 +220,9 @@
"schedule",
"app",
"ui",
"postgres"
"postgres",
"sqs",
"gcp"
]
}
}
backend/.sqlx/query-505250098ab003ff0ca30046df283e54bf44be74305070f10a5720a04c4789f3.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue\n SET canceled_by = $1\n , canceled_reason = $2\nWHERE id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Uuid"
]
},
"nullable": []
},
"hash": "505250098ab003ff0ca30046df283e54bf44be74305070f10a5720a04c4789f3"
}
backend/.sqlx/query-e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c.json → backend/.sqlx/query-506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
"query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
"describe": {
"columns": [],
"parameters": {
@@ -18,5 +18,5 @@
},
"nullable": []
},
"hash": "e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c"
"hash": "506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773"
}
backend/.sqlx/query-5303206bbed76ee3ddc56d8057d8b82359c182ee2a5da6df35a4375d5d2d1ef7.json
Generated
+65
View File
@@ -0,0 +1,65 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n gcp_resource_path, \n script_path,\n is_flow, \n workspace_id,\n path,\n edited_by,\n email,\n delivery_config AS \"delivery_config: _\"\n FROM\n gcp_trigger\n WHERE\n workspace_id = $1 AND\n path = $2 AND\n delivery_type = 'push'::DELIVERY_MODE \n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "gcp_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "delivery_config: _",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true
]
},
"hash": "5303206bbed76ee3ddc56d8057d8b82359c182ee2a5da6df35a4375d5d2d1ef7"
}
backend/.sqlx/query-673564e6c4dcf30dae3d7a75c397998ea800860ede856e5fc6cd1f57d5408333.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'gcp' AND \n server_id IS NULL\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "673564e6c4dcf30dae3d7a75c397998ea800860ede856e5fc6cd1f57d5408333"
}
backend/.sqlx/query-e17ec84003e2ec414622d100f5dfdda86bee33f31835317df512a20c805b35d7.json → backend/.sqlx/query-6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, created_at, trigger_kind as \"trigger_kind: _\", payload as \"payload!: _\", trigger_extra as \"trigger_extra: _\" FROM capture WHERE id = $1 AND workspace_id = $2",
"query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\", \n payload AS \"payload!: _\", \n trigger_extra AS \"trigger_extra: _\"\n FROM \n capture\n WHERE \n id = $1 \n AND workspace_id = $2\n ",
"describe": {
"columns": [
{
@@ -29,7 +29,8 @@
"nats",
"postgres",
"sqs",
"mqtt"
"mqtt",
"gcp"
]
}
}
@@ -60,5 +61,5 @@
true
]
},
"hash": "e17ec84003e2ec414622d100f5dfdda86bee33f31835317df512a20c805b35d7"
"hash": "6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414"
}
backend/.sqlx/query-77701b16ee1f6dd827372835db59bbffc7254af47a8d48b7ba3cf969c2f8398c.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue SET tag = $1, running = false WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Uuid"
]
},
"nullable": []
},
"hash": "77701b16ee1f6dd827372835db59bbffc7254af47a8d48b7ba3cf969c2f8398c"
}
backend/.sqlx/query-5c1de8473e0e96c1063a9a735a064c5a91e3ed8d9260c72b783fc12542b88fbd.json → backend/.sqlx/query-7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3.json
Generated
+6 -4
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, created_at, trigger_kind as \"trigger_kind: _\", CASE WHEN pg_column_size(payload) < 40000 THEN payload ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as \"payload!: _\", trigger_extra as \"trigger_extra: _\"\n FROM capture\n WHERE workspace_id = $1\n AND path = $2 AND is_flow = $3\n AND ($4::trigger_kind IS NULL OR trigger_kind = $4)\n ORDER BY created_at DESC\n OFFSET $5\n LIMIT $6",
"query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\",\n CASE \n WHEN pg_column_size(payload) < 40000 THEN payload \n ELSE '\"WINDMILL_TOO_BIG\"'::jsonb \n END AS \"payload!: _\",\n trigger_extra AS \"trigger_extra: _\"\n FROM \n capture\n WHERE \n workspace_id = $1 \n AND path = $2 \n AND is_flow = $3 \n AND ($4::trigger_kind IS NULL OR trigger_kind = $4)\n ORDER BY \n created_at DESC\n OFFSET $5\n LIMIT $6\n ",
"describe": {
"columns": [
{
@@ -29,7 +29,8 @@
"nats",
"postgres",
"sqs",
"mqtt"
"mqtt",
"gcp"
]
}
}
@@ -64,7 +65,8 @@
"nats",
"postgres",
"sqs",
"mqtt"
"mqtt",
"gcp"
]
}
}
@@ -81,5 +83,5 @@
true
]
},
"hash": "5c1de8473e0e96c1063a9a735a064c5a91e3ed8d9260c72b783fc12542b88fbd"
"hash": "7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3"
}
backend/.sqlx/query-7d5a161d916cb8d1485f8d72e6b4044f505f49edcd2ebeee4a77eae737f48795.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n DELETE FROM \n capture\n WHERE \n id = $1\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "7d5a161d916cb8d1485f8d72e6b4044f505f49edcd2ebeee4a77eae737f48795"
}
backend/.sqlx/query-8efd06387ded837d7849adafe5bc93acb882ef90fc58b023650c875e0fd17047.json → backend/.sqlx/query-7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n values ($1, $2, $3, $4, $5, $6, $7, $8) \n ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8",
"query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n values ($1, $2, $3, $4, $5, $6, $7, $8) \n ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8",
"describe": {
"columns": [],
"parameters": {
@@ -17,5 +17,5 @@
},
"nullable": []
},
"hash": "8efd06387ded837d7849adafe5bc93acb882ef90fc58b023650c875e0fd17047"
"hash": "7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f"
}
backend/.sqlx/query-7e891e053b8545c800d629421f239319a854f761a300970e2fd909f8058ec566.json
Generated
-17
View File
@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE capture_config SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow = $4",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "7e891e053b8545c800d629421f239319a854f761a300970e2fd909f8058ec566"
}
backend/.sqlx/query-c223f8b7fa4ef1aa06e1ba2a56d677774aa237508d5610714efd2e9b8b93c7b8.json → backend/.sqlx/query-7fbf72d9059fcd77e4c1112fa4fa22e4276c1da653475628889ce17dc904fbaa.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT trigger_config as \"trigger_config: _\", trigger_kind as \"trigger_kind: _\", error, last_server_ping\n FROM capture_config\n WHERE workspace_id = $1 AND path = $2 AND is_flow = $3",
"query": "\n SELECT \n trigger_config AS \"trigger_config: _\", \n trigger_kind AS \"trigger_kind: _\", \n error, \n last_server_ping\n FROM \n capture_config\n WHERE \n workspace_id = $1 \n AND path = $2 \n AND is_flow = $3\n ",
"describe": {
"columns": [
{
@@ -24,7 +24,8 @@
"nats",
"postgres",
"sqs",
"mqtt"
"mqtt",
"gcp"
]
}
}
@@ -55,5 +56,5 @@
true
]
},
"hash": "c223f8b7fa4ef1aa06e1ba2a56d677774aa237508d5610714efd2e9b8b93c7b8"
"hash": "7fbf72d9059fcd77e4c1112fa4fa22e4276c1da653475628889ce17dc904fbaa"
}
backend/.sqlx/query-71d51bbc35da7b9930e3ea3a634451217ccb9f1bc35b1ad6e10d16bc19c41447.json → backend/.sqlx/query-87564a196a1662f524407d853db506bf08c28efe82b68b3d44bafbd3d0e91c29.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT owner, email\n FROM capture_config\n WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4 AND last_client_ping > NOW() - INTERVAL '10 seconds'",
"query": "\n SELECT \n owner, \n email\n FROM \n capture_config\n WHERE \n workspace_id = $1 \n AND path = $2 \n AND is_flow = $3 \n AND trigger_kind = $4 \n AND last_client_ping > NOW() - INTERVAL '10 seconds'\n ",
"describe": {
"columns": [
{
@@ -32,7 +32,8 @@
"nats",
"postgres",
"sqs",
"mqtt"
"mqtt",
"gcp"
]
}
}
@@ -44,5 +45,5 @@
false
]
},
"hash": "71d51bbc35da7b9930e3ea3a634451217ccb9f1bc35b1ad6e10d16bc19c41447"
"hash": "87564a196a1662f524407d853db506bf08c28efe82b68b3d44bafbd3d0e91c29"
}
backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n gcp_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633"
}
backend/.sqlx/query-97942578df746c8c8103b403cfc4e44ef5a0f082bdde854900064325adc4dd77.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM capture\n WHERE workspace_id = $1\n AND created_at <=\n (\n SELECT created_at\n FROM capture\n WHERE workspace_id = $1\n ORDER BY created_at DESC\n OFFSET $2\n LIMIT 1\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": []
},
"hash": "97942578df746c8c8103b403cfc4e44ef5a0f082bdde854900064325adc4dd77"
}
backend/.sqlx/query-999edc6f54a9efb6dc6237992dad59f418ed6fc2a98ddb9bbc33dce5f029d904.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n gcp_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "999edc6f54a9efb6dc6237992dad59f418ed6fc2a98ddb9bbc33dce5f029d904"
}
backend/.sqlx/query-a2e86f169ffbf8acee5f7c7b71db5859ac94ffbad267c9cd6c652e8ce8fc5d3c.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text) RETURNING length(logs)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "length",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Uuid",
"Varchar"
]
},
"nullable": [
null
]
},
"hash": "a2e86f169ffbf8acee5f7c7b71db5859ac94ffbad267c9cd6c652e8ce8fc5d3c"
}
backend/.sqlx/query-a7ffc5b983d365159ef379ec6a2ab0dc7217d1b3f86f362c213982984d2cf652.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'gcp' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Bool",
"Text"
]
},
"nullable": [
null
]
},
"hash": "a7ffc5b983d365159ef379ec6a2ab0dc7217d1b3f86f362c213982984d2cf652"
}
backend/.sqlx/query-03ae5b1c912b13a8a7aadf50cb4984a2ea952e782fd52eb3088454690bd13dd1.json → backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar FROM script WHERE hash = $1 LIMIT 1",
"query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar FROM script WHERE hash = $1 LIMIT 1",
"describe": {
"columns": [
{
@@ -83,5 +83,5 @@
null
]
},
"hash": "03ae5b1c912b13a8a7aadf50cb4984a2ea952e782fd52eb3088454690bd13dd1"
"hash": "a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a"
}
backend/.sqlx/query-9ae20f2f29406cfa5337736ad710c496f1757d950cdec63c93c77fe79d5212cc.json → backend/.sqlx/query-aa0215d4174c1aeda8631bcd582c895329d2daf722d360fbcbdef6f04bb1400f.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\",\n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\",\n EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS \"mqtt_used!\",\n EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS \"sqs_used!\"\n ",
"query": "\n SELECT\n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\",\n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\",\n EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS \"mqtt_used!\",\n EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS \"sqs_used!\",\n EXISTS(SELECT 1 FROM gcp_trigger WHERE workspace_id = $1) AS \"gcp_used!\"\n ",
"describe": {
"columns": [
{
@@ -37,6 +37,11 @@
"ordinal": 6,
"name": "sqs_used!",
"type_info": "Bool"
},
{
"ordinal": 7,
"name": "gcp_used!",
"type_info": "Bool"
}
],
"parameters": {
@@ -51,8 +56,9 @@
null,
null,
null,
null,
null
]
},
"hash": "9ae20f2f29406cfa5337736ad710c496f1757d950cdec63c93c77fe79d5212cc"
"hash": "aa0215d4174c1aeda8631bcd582c895329d2daf722d360fbcbdef6f04bb1400f"
}
backend/.sqlx/query-a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb.json → backend/.sqlx/query-aa523c363186575b4bd2537b8e2430e6938e7cc35f8c9e2d1c5459a85443cbdd.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6",
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6",
"describe": {
"columns": [],
"parameters": {
@@ -20,5 +20,5 @@
},
"nullable": []
},
"hash": "a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb"
"hash": "aa523c363186575b4bd2537b8e2430e6938e7cc35f8c9e2d1c5459a85443cbdd"
}
backend/.sqlx/query-62475252dcf54f32433b97ae011daf5d4205d160d2aedf463c7dfe944e93257a.json → backend/.sqlx/query-b3f0595cacba194e08b9a3e244d9e637e9e156cd85b69126c87dfff89a47711d.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO capture_config\n (workspace_id, path, is_flow, trigger_kind, trigger_config, owner, email)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ON CONFLICT (workspace_id, path, is_flow, trigger_kind)\n DO UPDATE SET trigger_config = $5, owner = $6, email = $7, server_id = NULL, error = NULL",
"query": "\n INSERT INTO capture_config (\n workspace_id, path, is_flow, trigger_kind, trigger_config, owner, email\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ON CONFLICT (workspace_id, path, is_flow, trigger_kind)\n DO UPDATE \n SET \n trigger_config = $5, \n owner = $6, \n email = $7, \n server_id = NULL, \n error = NULL\n ",
"describe": {
"columns": [],
"parameters": {
@@ -21,7 +21,8 @@
"nats",
"postgres",
"sqs",
"mqtt"
"mqtt",
"gcp"
]
}
}
@@ -33,5 +34,5 @@
},
"nullable": []
},
"hash": "62475252dcf54f32433b97ae011daf5d4205d160d2aedf463c7dfe944e93257a"
"hash": "b3f0595cacba194e08b9a3e244d9e637e9e156cd85b69126c87dfff89a47711d"
}
backend/.sqlx/query-c00bae0d8c9bee37cbad4de4cb02c80d00f52a3fc32bf32271ebc90f7837abda.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue\n SET canceled_by = 'timeout'\n , canceled_reason = $1\n WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid"
]
},
"nullable": []
},
"hash": "c00bae0d8c9bee37cbad4de4cb02c80d00f52a3fc32bf32271ebc90f7837abda"
}
backend/.sqlx/query-c0e6dbce7a401b06e1bf45155c3f81572818177a6024e743b12f558b46edf74c.json
Generated
+50
View File
@@ -0,0 +1,50 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'gcp' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n trigger_config->>'delivery_type' IS DISTINCT FROM 'push' AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "owner",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "trigger_config!: _",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
false,
false,
true
]
},
"hash": "c0e6dbce7a401b06e1bf45155c3f81572818177a6024e743b12f558b46edf74c"
}
backend/.sqlx/query-c19a60a9dc3f95af218baf40c62f14572ac204cfe377166aa1d91cf58f731f50.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n DELETE \n FROM \n gcp_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "c19a60a9dc3f95af218baf40c62f14572ac204cfe377166aa1d91cf58f731f50"
}
backend/.sqlx/query-c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET \nping_at = now(), \njobs_executed = 1, \ncurrent_job_id = $1, \ncurrent_job_workspace_id = 'admins' \nWHERE worker = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": []
},
"hash": "c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa"
}
backend/.sqlx/query-c353aa9abe673749f0836fd16894d91ba89aab2318d85621ad2868017adfb48a.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config\n SET \n path = $1\n WHERE \n path = $2 \n AND workspace_id = $3 \n AND is_flow = $4\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "c353aa9abe673749f0836fd16894d91ba89aab2318d85621ad2868017adfb48a"
}
backend/.sqlx/query-f367a1c8f80dd414dcbcd949374eeb5770796f00b5b3d547163bcfdaed65d8ae.json → backend/.sqlx/query-c53e1c7133c8ae187656eef5999509fae17fb0ba43e084327accbb5b24c3dfbd.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO outstanding_wait_time(job_id, self_wait_time_ms) VALUES ($1, $2)\n ON CONFLICT (job_id) DO UPDATE SET self_wait_time_ms = EXCLUDED.self_wait_time_ms",
"query": "INSERT INTO outstanding_wait_time(job_id, self_wait_time_ms) VALUES ($1, $2)\n ON CONFLICT (job_id) DO UPDATE SET self_wait_time_ms = EXCLUDED.self_wait_time_ms",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
"hash": "f367a1c8f80dd414dcbcd949374eeb5770796f00b5b3d547163bcfdaed65d8ae"
"hash": "c53e1c7133c8ae187656eef5999509fae17fb0ba43e084327accbb5b24c3dfbd"
}
backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json
Generated
+1 -1
View File
@@ -15,7 +15,7 @@
]
},
"nullable": [
null
true
]
},
"hash": "ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927"
backend/.sqlx/query-df2a426658f0a36683cc2163fe70912d0221d191b8ab091601cb53cdd932ad58.json
Generated
+25
View File
@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n gcp_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "df2a426658f0a36683cc2163fe70912d0221d191b8ab091601cb53cdd932ad58"
}
backend/.sqlx/query-e04a8a9f1e9cc3bb5c990194585e08f4c248e1af1c6580bdf8f2735ae1388981.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n DELETE FROM \n capture\n WHERE \n workspace_id = $1\n AND created_at <= (\n SELECT \n created_at\n FROM \n capture\n WHERE \n workspace_id = $1\n ORDER BY \n created_at DESC\n OFFSET $2\n LIMIT 1\n )\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": []
},
"hash": "e04a8a9f1e9cc3bb5c990194585e08f4c248e1af1c6580bdf8f2735ae1388981"
}
backend/.sqlx/query-e70c3da5864b7946735b7ad6c416874ee8b277a2914dff6f9c33a1f5a2351114.json
Generated
-17
View File
@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE capture SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow = $4",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "e70c3da5864b7946735b7ad6c416874ee8b277a2914dff6f9c33a1f5a2351114"
}
backend/.sqlx/query-f4f6336fc671b00bed7835124892f7a4d3bbe673f7c48153819dab385a5cb357.json
Generated
+129
View File
@@ -0,0 +1,129 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM \n gcp_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "gcp_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "subscription_id",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "topic_id",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "delivery_type: _",
"type_info": {
"Custom": {
"name": "delivery_mode",
"kind": {
"Enum": [
"push",
"pull"
]
}
}
}
},
{
"ordinal": 5,
"name": "delivery_config: _",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 12,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 14,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 15,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 16,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
true,
false,
false,
false,
false,
false,
false,
true,
true,
false,
true,
false
]
},
"hash": "f4f6336fc671b00bed7835124892f7a4d3bbe673f7c48153819dab385a5cb357"
}
backend/.sqlx/query-07da723ce5c9ee2d7c236e8eabe254c783fc34b617c8a9a95a0eb0cda535dab5.json → backend/.sqlx/query-f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, payload, trigger_extra, created_by)\n VALUES ($1, $2, $3, $4, $5, $6, $7)",
"query": "\n INSERT INTO \n capture (\n workspace_id, path, is_flow, trigger_kind, payload, trigger_extra, created_by\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ",
"describe": {
"columns": [],
"parameters": {
@@ -21,7 +21,8 @@
"nats",
"postgres",
"sqs",
"mqtt"
"mqtt",
"gcp"
]
}
}
@@ -33,5 +34,5 @@
},
"nullable": []
},
"hash": "07da723ce5c9ee2d7c236e8eabe254c783fc34b617c8a9a95a0eb0cda535dab5"
"hash": "f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946"
}
backend/.sqlx/query-faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT tag FROM v2_job WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "tag",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
false
]
},
"hash": "faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a"
}
backend/.sqlx/query-fd0a0ca4a107dc813240ad71f372ab7e0bf26431158fbfb5f0023ed847ca7dc5.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n gcp_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "fd0a0ca4a107dc813240ad71f372ab7e0bf26431158fbfb5f0023ed847ca7dc5"
}
backend/.sqlx/query-fec6d5674dc6b5a6a0ece419c40508835affcb7679a48f2a443777e829bd1e74.json
Generated
+5 -5
View File
@@ -41,11 +41,11 @@
]
},
"nullable": [
true,
true,
true,
true,
true,
false,
false,
false,
false,
false,
true
]
},
backend/.vscode/settings.json
Vendored
+1 -1
View File
@@ -11,5 +11,5 @@
"remote.autoForwardPorts": true,
"conventionalCommits.scopes": [
"restructring triggers, decoding trigger message on work"
]
],
}
backend/Cargo.lock
Generated
+261 -26
View File
@@ -554,6 +554,17 @@ dependencies = [
"syn 2.0.100",
]
[[package]]
name = "async-channel"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35"
dependencies = [
"concurrent-queue",
"event-listener 2.5.3",
"futures-core",
]
[[package]]
name = "async-compression"
version = "0.4.19"
@@ -2572,7 +2583,7 @@ dependencies = [
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core",
"parking_lot_core 0.9.10",
]
[[package]]
@@ -2626,7 +2637,7 @@ dependencies = [
"log",
"num_cpus",
"object_store",
"parking_lot",
"parking_lot 0.12.3",
"parquet",
"paste",
"pin-project-lite",
@@ -2687,7 +2698,7 @@ dependencies = [
"hashbrown 0.14.5",
"log",
"object_store",
"parking_lot",
"parking_lot 0.12.3",
"rand 0.8.5",
"tempfile",
"url",
@@ -2867,7 +2878,7 @@ dependencies = [
"itertools 0.12.1",
"log",
"once_cell",
"parking_lot",
"parking_lot 0.12.3",
"pin-project-lite",
"rand 0.8.5",
"tokio",
@@ -2998,7 +3009,7 @@ dependencies = [
"indexmap 2.8.0",
"log",
"once_cell",
"parking_lot",
"parking_lot 0.12.3",
"serde",
"serde_json",
"sha2 0.10.8",
@@ -3080,7 +3091,7 @@ dependencies = [
"indexmap 2.8.0",
"libc",
"memoffset",
"parking_lot",
"parking_lot 0.12.3",
"percent-encoding",
"pin-project",
"serde",
@@ -3325,7 +3336,7 @@ dependencies = [
"log",
"once_cell",
"os_pipe",
"parking_lot",
"parking_lot 0.12.3",
"pin-project",
"rand 0.8.5",
"tokio",
@@ -3705,7 +3716,7 @@ dependencies = [
"log",
"node_resolver",
"once_cell",
"parking_lot",
"parking_lot 0.12.3",
"sys_traits",
"thiserror 2.0.12",
"url",
@@ -3862,7 +3873,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d774fd83f26b24f0805a6ab8b26834a0d06ceac0db517b769b1e4633c96a2057"
dependencies = [
"futures",
"parking_lot",
"parking_lot 0.12.3",
"tokio",
]
@@ -4602,6 +4613,12 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "event-listener"
version = "2.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
[[package]]
name = "event-listener"
version = "5.4.0"
@@ -4972,7 +4989,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f"
dependencies = [
"futures-core",
"lock_api",
"parking_lot",
"parking_lot 0.12.3",
]
[[package]]
@@ -5347,6 +5364,94 @@ dependencies = [
"gl_generator",
]
[[package]]
name = "google-cloud-auth"
version = "0.17.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e57a13fbacc5e9c41ded3ad8d0373175a6b7a6ad430d99e89d314ac121b7ab06"
dependencies = [
"async-trait",
"base64 0.21.7",
"google-cloud-metadata",
"google-cloud-token",
"home",
"jsonwebtoken 9.3.1",
"reqwest 0.12.15",
"serde",
"serde_json",
"thiserror 1.0.69",
"time",
"tokio",
"tracing",
"urlencoding",
]
[[package]]
name = "google-cloud-gax"
version = "0.19.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de13e62d7e0ffc3eb40a0113ddf753cf6ec741be739164442b08893db4f9bfca"
dependencies = [
"google-cloud-token",
"http 1.3.1",
"thiserror 1.0.69",
"tokio",
"tokio-retry2",
"tonic",
"tower 0.4.13",
"tracing",
]
[[package]]
name = "google-cloud-googleapis"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "886aa8ec755382a1fdf4651f6e6ec01f2f3bf49f2cb0f068b9a74cafd574a715"
dependencies = [
"prost",
"prost-types",
"tonic",
]
[[package]]
name = "google-cloud-metadata"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d901aeb453fd80e51d64df4ee005014f6cf39f2d736dd64f7239c132d9d39a6a"
dependencies = [
"reqwest 0.12.15",
"thiserror 1.0.69",
"tokio",
]
[[package]]
name = "google-cloud-pubsub"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bebc6e7327e49a66ffb40508c673b7643191bd6b509530193bda97f09272cdcf"
dependencies = [
"async-channel",
"async-stream",
"google-cloud-auth",
"google-cloud-gax",
"google-cloud-googleapis",
"google-cloud-token",
"prost-types",
"thiserror 1.0.69",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "google-cloud-token"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f49c12ba8b21d128a2ce8585955246977fbce4415f680ebf9199b6f9d6d725f"
dependencies = [
"async-trait",
]
[[package]]
name = "gosyn"
version = "0.2.9"
@@ -5638,7 +5743,7 @@ dependencies = [
"ipconfig",
"moka",
"once_cell",
"parking_lot",
"parking_lot 0.12.3",
"rand 0.9.0",
"resolv-conf",
"serde",
@@ -6427,6 +6532,21 @@ dependencies = [
"simple_asn1",
]
[[package]]
name = "jsonwebtoken"
version = "9.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde"
dependencies = [
"base64 0.22.1",
"js-sys",
"pem 3.0.5",
"ring 0.17.14",
"serde",
"serde_json",
"simple_asn1",
]
[[package]]
name = "junction"
version = "1.2.0"
@@ -7230,7 +7350,7 @@ dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
"loom",
"parking_lot",
"parking_lot 0.12.3",
"portable-atomic",
"rustc_version 0.4.1",
"smallvec",
@@ -7933,7 +8053,7 @@ dependencies = [
"hyper 1.6.0",
"itertools 0.13.0",
"md-5 0.10.6",
"parking_lot",
"parking_lot 0.12.3",
"percent-encoding",
"quick-xml 0.36.2",
"rand 0.8.5",
@@ -8349,6 +8469,17 @@ version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.6",
]
[[package]]
name = "parking_lot"
version = "0.12.3"
@@ -8356,7 +8487,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
dependencies = [
"lock_api",
"parking_lot_core",
"parking_lot_core 0.9.10",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall 0.2.16",
"smallvec",
"winapi",
]
[[package]]
@@ -8957,7 +9102,7 @@ dependencies = [
"fnv",
"lazy_static",
"memchr",
"parking_lot",
"parking_lot 0.12.3",
"thiserror 2.0.12",
]
@@ -9111,7 +9256,7 @@ dependencies = [
"ahash 0.8.11",
"equivalent",
"hashbrown 0.15.2",
"parking_lot",
"parking_lot 0.12.3",
]
[[package]]
@@ -9359,6 +9504,15 @@ version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430"
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_syscall"
version = "0.3.5"
@@ -9568,6 +9722,43 @@ dependencies = [
"windows-registry",
]
[[package]]
name = "reqwest-middleware"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64e8975513bd9a7a43aad01030e79b3498e05db14e9d945df6483e8cf9b8c4c4"
dependencies = [
"anyhow",
"async-trait",
"http 1.3.1",
"reqwest 0.12.15",
"serde",
"thiserror 1.0.69",
"tower-service",
]
[[package]]
name = "reqwest-retry"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178"
dependencies = [
"anyhow",
"async-trait",
"futures",
"getrandom 0.2.15",
"http 1.3.1",
"hyper 1.6.0",
"parking_lot 0.11.2",
"reqwest 0.12.15",
"reqwest-middleware",
"retry-policies",
"thiserror 1.0.69",
"tokio",
"tracing",
"wasm-timer",
]
[[package]]
name = "resolv-conf"
version = "0.7.1"
@@ -9577,6 +9768,15 @@ dependencies = [
"hostname",
]
[[package]]
name = "retry-policies"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c"
dependencies = [
"rand 0.8.5",
]
[[package]]
name = "rfc6979"
version = "0.4.0"
@@ -10945,7 +11145,7 @@ dependencies = [
"crc",
"crossbeam-queue",
"either",
"event-listener",
"event-listener 5.4.0",
"futures-core",
"futures-intrusive",
"futures-io",
@@ -12264,7 +12464,7 @@ dependencies = [
"bytes",
"libc",
"mio 1.0.3",
"parking_lot",
"parking_lot 0.12.3",
"pin-project-lite",
"signal-hook-registry",
"socket2",
@@ -12328,7 +12528,7 @@ dependencies = [
"futures-channel",
"futures-util",
"log",
"parking_lot",
"parking_lot 0.12.3",
"percent-encoding",
"phf",
"pin-project-lite",
@@ -12354,7 +12554,7 @@ dependencies = [
"futures-channel",
"futures-util",
"log",
"parking_lot",
"parking_lot 0.12.3",
"percent-encoding",
"phf",
"pin-project-lite",
@@ -12367,6 +12567,16 @@ dependencies = [
"whoami",
]
[[package]]
name = "tokio-retry2"
version = "0.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1264d076dd34560544a2799e40e457bd07c43d30f4a845686b031bcd8455c84f"
dependencies = [
"pin-project",
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.24.1"
@@ -12544,6 +12754,7 @@ dependencies = [
"axum",
"base64 0.22.1",
"bytes",
"flate2",
"h2 0.4.8",
"http 1.3.1",
"http-body 1.0.1",
@@ -12564,6 +12775,7 @@ dependencies = [
"tower-layer",
"tower-service",
"tracing",
"webpki-roots",
]
[[package]]
@@ -12613,7 +12825,7 @@ dependencies = [
"cookie 0.18.1",
"futures-util",
"http 1.3.1",
"parking_lot",
"parking_lot 0.12.3",
"pin-project-lite",
"tower-layer",
"tower-service",
@@ -13460,6 +13672,21 @@ dependencies = [
"web-sys",
]
[[package]]
name = "wasm-timer"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f"
dependencies = [
"futures",
"js-sys",
"parking_lot 0.11.2",
"pin-utils",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "wasm_dep_analyzer"
version = "0.2.0"
@@ -13533,7 +13760,7 @@ dependencies = [
"log",
"naga",
"once_cell",
"parking_lot",
"parking_lot 0.12.3",
"profiling",
"raw-window-handle",
"ron",
@@ -13575,7 +13802,7 @@ dependencies = [
"ndk-sys",
"objc",
"once_cell",
"parking_lot",
"parking_lot 0.12.3",
"profiling",
"range-alloc",
"raw-window-handle",
@@ -13748,6 +13975,8 @@ dependencies = [
"datafusion",
"futures",
"git-version",
"google-cloud-googleapis",
"google-cloud-pubsub",
"half",
"hex",
"hf-hub",
@@ -13755,7 +13984,7 @@ dependencies = [
"http 1.3.1",
"hyper 1.6.0",
"itertools 0.14.0",
"jsonwebtoken",
"jsonwebtoken 8.3.0",
"lazy_static",
"magic-crypt",
"mail-parser",
@@ -13798,6 +14027,7 @@ dependencies = [
"tokio-tar",
"tokio-tungstenite",
"tokio-util",
"tonic",
"tower 0.5.2",
"tower-cookies",
"tower-http",
@@ -13813,8 +14043,10 @@ dependencies = [
"windmill-indexer",
"windmill-parser",
"windmill-parser-py",
"windmill-parser-py-imports",
"windmill-parser-ts",
"windmill-queue",
"windmill-worker",
]
[[package]]
@@ -13885,7 +14117,7 @@ dependencies = [
"hyper 1.6.0",
"indexmap 2.8.0",
"itertools 0.14.0",
"jsonwebtoken",
"jsonwebtoken 8.3.0",
"lazy_static",
"magic-crypt",
"mail-send",
@@ -13901,6 +14133,8 @@ dependencies = [
"rand 0.9.0",
"regex",
"reqwest 0.12.15",
"reqwest-middleware",
"reqwest-retry",
"semver 1.0.26",
"serde",
"serde_json",
@@ -14251,7 +14485,7 @@ dependencies = [
"git-version",
"hex",
"itertools 0.14.0",
"jsonwebtoken",
"jsonwebtoken 8.3.0",
"lazy_static",
"mappable-rc",
"mysql_async",
@@ -14267,6 +14501,7 @@ dependencies = [
"rand 0.9.0",
"regex",
"reqwest 0.12.15",
"reqwest-middleware",
"rust_decimal",
"serde",
"serde_json",
backend/Cargo.toml
+8 -1
View File
@@ -49,6 +49,7 @@ lto = "thin"
[features]
default = []
agent_worker_server = ["windmill-api/agent_worker_server"]
enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"]
enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"]
stripe = ["windmill-api/stripe"]
@@ -68,13 +69,14 @@ nats = ["windmill-api/nats"]
otel = ["windmill-common/otel", "windmill-worker/otel"]
dind = ["windmill-worker/dind"]
websocket = ["windmill-api/websocket"]
http_trigger = ["windmill-api/http_trigger"]
postgres_trigger = ["windmill-api/postgres_trigger"]
mqtt_trigger = ["windmill-api/mqtt_trigger"]
sqs_trigger = ["windmill-api/sqs_trigger"]
gcp_trigger = ["windmill-api/gcp_trigger"]
smtp = ["windmill-api/smtp", "windmill-common/smtp"]
license = ["windmill-api/license"]
oauth2 = ["windmill-api/oauth2"]
http_trigger = ["windmill-api/http_trigger"]
zip = ["windmill-api/zip"]
static_frontend = ["windmill-api/static_frontend"]
scoped_cache = ["windmill-common/scoped_cache"]
@@ -170,6 +172,9 @@ windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
windmill-parser-php = { path = "./parsers/windmill-parser-php" }
windmill-api-client = { path = "./windmill-api-client" }
reqwest-retry = "^0"
reqwest-middleware = { version = "^0", features = ["json"] }
rustls = "0.23.0"
memchr = "2.7.4"
axum = { version = "^0.7", features = ["multipart"] }
@@ -238,6 +243,8 @@ deno_runtime = { version = "0.198.0", features = ["transpile"] }
deno_telemetry = "0.12.0"
deno_error = "=0.5.5"
google-cloud-pubsub = "0.30.0"
google-cloud-googleapis = {version = "0.16.1", features = ["pubsub"]}
# TODO: remove once deno fixes the issue on their end
# https://github.com/denoland/deno/issues/28557
winapi = { version = "0.3.9", features = ["sysinfoapi"] }
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
619f7dcd1097bfcd2f8b85f2075782219ff76be1
85c37983ffb8f622458425c182613206625c6cee
backend/migrations/20250323135044_add-gcp-trigger-table.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add down migration script here
DROP TABLE gcp_trigger;
DROP TYPE DELIVERY_MODE;
backend/migrations/20250323135044_add-gcp-trigger-table.up.sql
+80
View File
@@ -0,0 +1,80 @@
-- Add up migration script here
ALTER TYPE TRIGGER_KIND ADD VALUE IF NOT EXISTS 'gcp';
CREATE TYPE DELIVERY_MODE AS ENUM ('push', 'pull');
CREATE TABLE gcp_trigger (
gcp_resource_path VARCHAR(255) NOT NULL,
topic_id VARCHAR(255) NOT NULL CHECK (
CHAR_LENGTH(topic_id) BETWEEN 3 AND 255
),
subscription_id VARCHAR(255) NOT NULL CHECK (
CHAR_LENGTH(subscription_id) BETWEEN 3 AND 255
),
delivery_type DELIVERY_MODE NOT NULL,
delivery_config JSONB NULL CHECK (delivery_type != 'push'::DELIVERY_MODE OR (delivery_config IS NOT NULL)),
path VARCHAR(255) NOT NULL,
script_path VARCHAR(255) NOT NULL,
is_flow BOOLEAN NOT NULL,
workspace_id VARCHAR(50) NOT NULL,
edited_by VARCHAR(50) NOT NULL,
email VARCHAR(255) NOT NULL,
edited_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
extra_perms JSONB NOT NULL DEFAULT '{}',
server_id VARCHAR(50),
last_server_ping TIMESTAMPTZ,
error TEXT,
enabled BOOLEAN NOT NULL,
PRIMARY KEY (path, workspace_id)
);
CREATE UNIQUE INDEX unique_subscription_per_gcp_resource
ON gcp_trigger (subscription_id, gcp_resource_path, workspace_id);
GRANT ALL ON gcp_trigger TO windmill_user;
GRANT ALL ON gcp_trigger TO windmill_admin;
ALTER TABLE gcp_trigger ENABLE ROW LEVEL SECURITY;
CREATE POLICY admin_policy ON gcp_trigger FOR ALL TO windmill_admin USING (true);
CREATE POLICY see_folder_extra_perms_user_select ON gcp_trigger FOR SELECT TO windmill_user
USING (SPLIT_PART(gcp_trigger.path, '/', 1) = 'f' AND SPLIT_PART(gcp_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_insert ON gcp_trigger FOR INSERT TO windmill_user
WITH CHECK (SPLIT_PART(gcp_trigger.path, '/', 1) = 'f' AND SPLIT_PART(gcp_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_update ON gcp_trigger FOR UPDATE TO windmill_user
USING (SPLIT_PART(gcp_trigger.path, '/', 1) = 'f' AND SPLIT_PART(gcp_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_delete ON gcp_trigger FOR DELETE TO windmill_user
USING (SPLIT_PART(gcp_trigger.path, '/', 1) = 'f' AND SPLIT_PART(gcp_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_own ON gcp_trigger FOR ALL TO windmill_user
USING (SPLIT_PART(gcp_trigger.path, '/', 1) = 'u' AND SPLIT_PART(gcp_trigger.path, '/', 2) = current_setting('session.user'));
CREATE POLICY see_member ON gcp_trigger FOR ALL TO windmill_user
USING (SPLIT_PART(gcp_trigger.path, '/', 1) = 'g' AND SPLIT_PART(gcp_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
CREATE POLICY see_extra_perms_user_select ON gcp_trigger FOR SELECT TO windmill_user
USING (extra_perms ? CONCAT('u/', current_setting('session.user')));
CREATE POLICY see_extra_perms_user_insert ON gcp_trigger FOR INSERT TO windmill_user
WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_user_update ON gcp_trigger FOR UPDATE TO windmill_user
USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_user_delete ON gcp_trigger FOR DELETE TO windmill_user
USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_groups_select ON gcp_trigger FOR SELECT TO windmill_user
USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]);
CREATE POLICY see_extra_perms_groups_insert ON gcp_trigger FOR INSERT TO windmill_user
WITH CHECK (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
CREATE POLICY see_extra_perms_groups_update ON gcp_trigger FOR UPDATE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
CREATE POLICY see_extra_perms_groups_delete ON gcp_trigger FOR DELETE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
backend/migrations/20250323162033_add-missing-trigger-kind-to-job-trigger-kind-type.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20250323162033_add-missing-trigger-kind-to-job-trigger-kind-type.up.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add up migration script here
ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'sqs';
ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'gcp';
backend/migrations/20250409093642_add_grant.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20250409093642_add_grant.up.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add up migration script here
GRANT ALL on workspace_runnable_dependencies TO windmill_user;
GRANT ALL on workspace_runnable_dependencies TO windmill_admin;
backend/parsers/windmill-parser-bash/src/lib.rs
+3 -2
View File
@@ -45,7 +45,7 @@ pub fn parse_powershell_sig(code: &str) -> anyhow::Result<MainArgSignature> {
}
lazy_static::lazy_static! {
static ref RE_BASH: Regex = Regex::new(r#"(?m)^(\w+)="\$(?:(\d+)|\{(\d+):-(.*)\})"(?:[\t ]*)?(?:#.*)?$"#).unwrap();
static ref RE_BASH: Regex = Regex::new(r#"(?m)^(\w+)="\$(?:(\d+)|\{(\d+)\}|\{(\d+):-(.*)\})"(?:[\t ]*)?(?:#.*)?$"#).unwrap();
pub static ref RE_POWERSHELL_PARAM: Regex = Regex::new(r#"(?m)param[\t ]*\(([^)]*)\)"#).unwrap();
static ref RE_POWERSHELL_ARGS: Regex = Regex::new(r#"(?:\[(\w+)\])?\$(\w+)[\t ]*(?:=[\t ]*(?:(?:(?:"|')([^"\n\r\$]*)(?:"|'))|([\d.]+)))?"#).unwrap();
@@ -57,11 +57,12 @@ fn parse_bash_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
hm.insert(
cap.get(2)
.or(cap.get(3))
.or(cap.get(4))
.and_then(|x| x.as_str().parse::<i32>().ok())
.ok_or_else(|| anyhow!("Impossible to parse arg digit"))?,
(
cap[1].to_string(),
cap.get(4).map(|x| x.as_str().to_string()),
cap.get(5).map(|x| x.as_str().to_string()),
),
);
}
backend/src/main.rs
+468 -393
View File
File diff suppressed because it is too large Load Diff
backend/src/monitor.rs
+226 -157
View File
@@ -13,7 +13,7 @@ use std::{
use chrono::{NaiveDateTime, Utc};
use futures::{stream::FuturesUnordered, StreamExt};
use serde::{de::DeserializeOwned, Deserializer};
use serde::de::DeserializeOwned;
use sqlx::{Pool, Postgres};
use tokio::{
join,
@@ -34,7 +34,8 @@ use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts};
#[cfg(feature = "oauth2")]
use windmill_common::global_settings::OAUTH_SETTING;
use windmill_common::{
ee::CriticalErrorChannel, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{
utils::empty_string_as_none,
agent_workers::DECODED_AGENT_TOKEN, auth::create_token_for_owner, ee::CriticalErrorChannel, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
@@ -45,14 +46,12 @@ use windmill_common::{
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
}, indexer::load_indexer_config, jobs::QueuedJob, jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, tracing_init::JSON_FMT, users::truncate_token, utils::{now_from_db, rd_string, report_critical_error, Mode}, worker::{
load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting,
update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG,
SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP,
}, KillpillSender, BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS
load_worker_config, reload_custom_tags_setting, store_pull_query, store_suspended_pull_query, update_min_version, Connection, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP
}, KillpillSender, BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS,
};
use windmill_queue::{cancel_job, MiniPulledJob};
use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload};
use windmill_worker::{
create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender, SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY
handle_job_error, AuthedClient, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL
};
#[cfg(feature = "parquet")]
@@ -113,88 +112,120 @@ lazy_static::lazy_static! {
.unwrap_or(false);
static ref QUEUE_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true");
}
pub async fn initial_load(
db: &Pool<Postgres>,
conn: &Connection,
tx: KillpillSender,
worker_mode: bool,
server_mode: bool,
#[cfg(feature = "parquet")] disable_s3_store: bool,
) {
if let Err(e) = load_metrics_enabled(db).await {
if let Err(e) = reload_base_url_setting(&conn).await {
tracing::error!("Error loading base url: {:?}", e)
}
if let Some(db) = conn.as_sql() {
if let Err(e) = reload_critical_error_channels_setting(&db).await {
tracing::error!("Could loading critical error emails setting: {:?}", e);
}
}
if let Err(e) = load_metrics_enabled(conn).await {
tracing::error!("Error loading expose metrics: {e:#}");
}
if let Err(e) = load_metrics_debug_enabled(db).await {
if let Err(e) = load_metrics_debug_enabled(conn).await {
tracing::error!("Error loading expose debug metrics: {e:#}");
}
if let Err(e) = reload_critical_alert_mute_ui_setting(db).await {
if let Err(e) = reload_critical_alert_mute_ui_setting(conn).await {
tracing::error!("Error loading critical alert mute ui setting: {e:#}");
}
if let Err(e) = load_tag_per_workspace_enabled(db).await {
tracing::error!("Error loading default tag per workpsace: {e:#}");
}
if let Some(db) = conn.as_sql() {
if let Err(e) = load_tag_per_workspace_enabled(db).await {
tracing::error!("Error loading default tag per workpsace: {e:#}");
}
if let Err(e) = load_tag_per_workspace_workspaces(db).await {
tracing::error!("Error loading default tag per workpsace workspaces: {e:#}");
if let Err(e) = load_tag_per_workspace_workspaces(db).await {
tracing::error!("Error loading default tag per workpsace workspaces: {e:#}");
}
}
if server_mode {
load_require_preexisting_user(db).await;
if let Some(db) = conn.as_sql() {
load_require_preexisting_user(db).await;
}
}
if worker_mode {
load_keep_job_dir(db).await;
reload_worker_config(&db, tx, false).await;
load_keep_job_dir(conn).await;
match conn {
Connection::Sql(db) => {
reload_worker_config(&db, tx, false).await;
}
Connection::Http(_) => {
// TODO: reload worker config from http
WORKER_CONFIG.write().await.worker_tags = DECODED_AGENT_TOKEN.as_ref().map(|x| x.tags.clone()).unwrap_or_default();
}
}
}
if let Err(e) = reload_custom_tags_setting(db).await {
tracing::error!("Error reloading custom tags: {:?}", e)
}
if let Err(e) = reload_hub_base_url_setting(db, server_mode).await {
if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await {
tracing::error!("Error reloading hub base url: {:?}", e)
}
if let Err(e) = reload_jwt_secret_setting(&db).await {
tracing::error!("Could not reload jwt secret setting: {:?}", e);
if let Some(db) = conn.as_sql() {
if let Err(e) = reload_jwt_secret_setting(db).await {
tracing::error!("Could not reload jwt secret setting: {:?}", e);
}
if let Err(e) = reload_custom_tags_setting(db).await {
tracing::error!("Error reloading custom tags: {:?}", e)
}
}
#[cfg(feature = "parquet")]
if !disable_s3_store {
reload_s3_cache_setting(&db).await;
if let Some(db) = conn.as_sql() {
reload_s3_cache_setting(db).await;
}
}
reload_smtp_config(&db).await;
if let Some(db) = conn.as_sql() {
reload_smtp_config(db).await;
}
if server_mode {
reload_retention_period_setting(&db).await;
reload_request_size(&db).await;
reload_saml_metadata_setting(&db).await;
reload_scim_token_setting(&db).await;
reload_retention_period_setting(&conn).await;
reload_request_size(&conn).await;
reload_saml_metadata_setting(&conn).await;
reload_scim_token_setting(&conn).await;
}
if worker_mode {
reload_job_default_timeout_setting(&db).await;
reload_extra_pip_index_url_setting(&db).await;
reload_pip_index_url_setting(&db).await;
reload_npm_config_registry_setting(&db).await;
reload_bunfig_install_scopes_setting(&db).await;
reload_instance_python_version_setting(&db).await;
reload_nuget_config_setting(&db).await;
reload_maven_repos_setting(&db).await;
reload_no_default_maven_setting(&db).await;
reload_job_default_timeout_setting(&conn).await;
reload_extra_pip_index_url_setting(&conn).await;
reload_pip_index_url_setting(&conn).await;
reload_npm_config_registry_setting(&conn).await;
reload_bunfig_install_scopes_setting(&conn).await;
reload_instance_python_version_setting(&conn).await;
reload_nuget_config_setting(&conn).await;
reload_maven_repos_setting(&conn).await;
reload_no_default_maven_setting(&conn).await;
}
}
pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> {
let metrics_enabled = load_value_from_global_settings(db, EXPOSE_METRICS_SETTING).await;
pub async fn load_metrics_enabled(conn: &Connection) -> error::Result<()> {
let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_METRICS_SETTING, true).await;
match metrics_enabled {
Ok(Some(serde_json::Value::Bool(t))) => METRICS_ENABLED.store(t, Ordering::Relaxed),
_ => (),
@@ -202,14 +233,6 @@ pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> {
Ok(())
}
fn empty_string_as_none<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
where
D: Deserializer<'de>,
{
let option = <Option<String> as serde::Deserialize>::deserialize(deserializer)?;
Ok(option.filter(|s| !s.is_empty()))
}
#[derive(serde::Deserialize)]
struct OtelSetting {
metrics_enabled: Option<bool>,
@@ -317,9 +340,9 @@ pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> {
Ok(())
}
pub async fn reload_critical_alert_mute_ui_setting(db: &DB) -> error::Result<()> {
pub async fn reload_critical_alert_mute_ui_setting(conn: &Connection) -> error::Result<()> {
if let Ok(Some(serde_json::Value::Bool(t))) =
load_value_from_global_settings(db, CRITICAL_ALERT_MUTE_UI_SETTING).await
load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await
{
CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed);
@@ -327,8 +350,8 @@ pub async fn reload_critical_alert_mute_ui_setting(db: &DB) -> error::Result<()>
Ok(())
}
pub async fn load_metrics_debug_enabled(db: &DB) -> error::Result<()> {
let metrics_enabled = load_value_from_global_settings(db, EXPOSE_DEBUG_METRICS_SETTING).await;
pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> {
let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await;
match metrics_enabled {
Ok(Some(serde_json::Value::Bool(t))) => {
METRICS_DEBUG_ENABLED.store(t, Ordering::Relaxed);
@@ -479,8 +502,8 @@ fn get_worker_group(mode: &Mode) -> Option<String> {
}
}
pub fn send_logs_to_object_store(db: &DB, hostname: &str, mode: &Mode) {
let db = db.clone();
pub fn send_logs_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) {
let conn = conn.clone();
let hostname = hostname.to_string();
let mode = mode.clone();
let worker_group = get_worker_group(&mode);
@@ -495,7 +518,7 @@ pub fn send_logs_to_object_store(db: &DB, hostname: &str, mode: &Mode) {
&hostname,
&mode,
&worker_group,
&db,
&conn,
snd_highest_file,
false,
)
@@ -504,11 +527,11 @@ pub fn send_logs_to_object_store(db: &DB, hostname: &str, mode: &Mode) {
});
}
pub async fn send_current_log_file_to_object_store(db: &DB, hostname: &str, mode: &Mode) {
pub async fn send_current_log_file_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) {
tracing::info!("Sending current log file to object store");
let (highest_file, _) = find_two_highest_files(hostname).await;
let worker_group = get_worker_group(&mode);
send_log_file_to_object_store(hostname, mode, &worker_group, db, highest_file, true).await;
send_log_file_to_object_store(hostname, mode, &worker_group, conn, highest_file, true).await;
}
fn get_now_and_str() -> (NaiveDateTime, String) {
@@ -528,7 +551,7 @@ async fn send_log_file_to_object_store(
hostname: &str,
mode: &Mode,
worker_group: &Option<String>,
db: &Pool<Postgres>,
conn: &Connection,
snd_highest_file: Option<String>,
use_now: bool,
) {
@@ -588,18 +611,23 @@ async fn send_log_file_to_object_store(
let (ok_lines, err_lines) = read_log_counters(ts_str);
if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)",
hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT)
.execute(db)
.await {
tracing::error!("Error inserting log file: {:?}", e);
} else {
if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| {
last_log_file_sent.replace(ts);
}) {
tracing::error!("Error updating last log file sent: {:?}", e);
if let Some(db) = conn.as_sql() {
if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)",
hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT)
.execute(db)
.await {
tracing::error!("Error inserting log file: {:?}", e);
} else {
if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| {
last_log_file_sent.replace(ts);
}) {
tracing::error!("Error updating last log file sent: {:?}", e);
}
tracing::info!("Log file sent: {}", highest_file);
}
tracing::info!("Log file sent: {}", highest_file);
} else {
// tracing::warn!("Not sending log file to object store in agent mode");
()
}
}
}
@@ -622,8 +650,8 @@ fn read_log_counters(ts_str: String) -> (usize, usize) {
(ok_lines, err_lines)
}
pub async fn load_keep_job_dir(db: &DB) {
let value = load_value_from_global_settings(db, KEEP_JOB_DIR_SETTING).await;
pub async fn load_keep_job_dir(conn: &Connection) {
let value = load_value_from_global_settings_with_conn(conn, KEEP_JOB_DIR_SETTING, true).await;
match value {
Ok(Some(serde_json::Value::Bool(t))) => KEEP_JOB_DIR.store(t, Ordering::Relaxed),
Err(e) => {
@@ -890,23 +918,23 @@ async fn delete_log_files_from_disk_and_store(
let _: Vec<_> = delete_futures.collect().await;
}
pub async fn reload_scim_token_setting(db: &DB) {
reload_option_setting_with_tracing(db, SCIM_TOKEN_SETTING, "SCIM_TOKEN", SCIM_TOKEN.clone())
pub async fn reload_scim_token_setting(conn: &Connection) {
reload_option_setting_with_tracing(conn, SCIM_TOKEN_SETTING, "SCIM_TOKEN", SCIM_TOKEN.clone())
.await;
}
pub async fn reload_timeout_wait_result_setting(db: &DB) {
pub async fn reload_timeout_wait_result_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
TIMEOUT_WAIT_RESULT_SETTING,
"TIMEOUT_WAIT_RESULT",
TIMEOUT_WAIT_RESULT.clone(),
)
.await;
}
pub async fn reload_saml_metadata_setting(db: &DB) {
pub async fn reload_saml_metadata_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
SAML_METADATA_SETTING,
"SAML_METADATA",
SAML_METADATA.clone(),
@@ -914,9 +942,9 @@ pub async fn reload_saml_metadata_setting(db: &DB) {
.await;
}
pub async fn reload_extra_pip_index_url_setting(db: &DB) {
pub async fn reload_extra_pip_index_url_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
EXTRA_PIP_INDEX_URL_SETTING,
"PIP_EXTRA_INDEX_URL",
PIP_EXTRA_INDEX_URL.clone(),
@@ -924,9 +952,9 @@ pub async fn reload_extra_pip_index_url_setting(db: &DB) {
.await;
}
pub async fn reload_pip_index_url_setting(db: &DB) {
pub async fn reload_pip_index_url_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
PIP_INDEX_URL_SETTING,
"PIP_INDEX_URL",
PIP_INDEX_URL.clone(),
@@ -934,9 +962,9 @@ pub async fn reload_pip_index_url_setting(db: &DB) {
.await;
}
pub async fn reload_instance_python_version_setting(db: &DB) {
pub async fn reload_instance_python_version_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
INSTANCE_PYTHON_VERSION_SETTING,
"INSTANCE_PYTHON_VERSION",
INSTANCE_PYTHON_VERSION.clone(),
@@ -944,9 +972,9 @@ pub async fn reload_instance_python_version_setting(db: &DB) {
.await;
}
pub async fn reload_npm_config_registry_setting(db: &DB) {
pub async fn reload_npm_config_registry_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
NPM_CONFIG_REGISTRY_SETTING,
"NPM_CONFIG_REGISTRY",
NPM_CONFIG_REGISTRY.clone(),
@@ -954,9 +982,9 @@ pub async fn reload_npm_config_registry_setting(db: &DB) {
.await;
}
pub async fn reload_bunfig_install_scopes_setting(db: &DB) {
pub async fn reload_bunfig_install_scopes_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
BUNFIG_INSTALL_SCOPES_SETTING,
"BUNFIG_INSTALL_SCOPES",
BUNFIG_INSTALL_SCOPES.clone(),
@@ -964,21 +992,21 @@ pub async fn reload_bunfig_install_scopes_setting(db: &DB) {
.await;
}
pub async fn reload_nuget_config_setting(db: &DB) {
pub async fn reload_nuget_config_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
NUGET_CONFIG_SETTING,
"NUGET_CONFIG",
NUGET_CONFIG.clone(),
)
.await;
}
pub async fn reload_maven_repos_setting(db: &DB) {
reload_option_setting_with_tracing(db, windmill_common::global_settings::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone())
pub async fn reload_maven_repos_setting(conn: &Connection) {
reload_option_setting_with_tracing(conn, windmill_common::global_settings::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone())
.await;
}
pub async fn reload_no_default_maven_setting(db: &DB) {
let value = load_value_from_global_settings(db, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING).await;
pub async fn reload_no_default_maven_setting(conn: &Connection) {
let value = load_value_from_global_settings_with_conn(conn, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING, true).await;
match value {
Ok(Some(serde_json::Value::Bool(t))) => NO_DEFAULT_MAVEN.store(t, Ordering::Relaxed),
Err(e) => {
@@ -988,9 +1016,9 @@ pub async fn reload_no_default_maven_setting(db: &DB) {
};
}
pub async fn reload_retention_period_setting(db: &DB) {
pub async fn reload_retention_period_setting(conn: &Connection) {
if let Err(e) = reload_setting(
db,
conn,
RETENTION_PERIOD_SECS_SETTING,
"JOB_RETENTION_SECS",
60 * 60 * 24 * 30,
@@ -1002,9 +1030,9 @@ pub async fn reload_retention_period_setting(db: &DB) {
tracing::error!("Error reloading retention period: {:?}", e)
}
}
pub async fn reload_delete_logs_periodically_setting(db: &DB) {
pub async fn reload_delete_logs_periodically_setting(conn: &Connection) {
if let Err(e) = reload_setting(
db,
conn,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING,
"MONITOR_LOGS_ON_OBJECT_STORE",
false,
@@ -1072,9 +1100,9 @@ pub async fn reload_s3_cache_setting(db: &DB) {
}
}
pub async fn reload_job_default_timeout_setting(db: &DB) {
pub async fn reload_job_default_timeout_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
JOB_DEFAULT_TIMEOUT_SECS_SETTING,
"JOB_DEFAULT_TIMEOUT_SECS",
JOB_DEFAULT_TIMEOUT.clone(),
@@ -1082,9 +1110,9 @@ pub async fn reload_job_default_timeout_setting(db: &DB) {
.await;
}
pub async fn reload_request_size(db: &DB) {
pub async fn reload_request_size(conn: &Connection) {
if let Err(e) = reload_setting(
db,
conn,
REQUEST_SIZE_LIMIT_SETTING,
"REQUEST_SIZE_LIMIT",
DEFAULT_BODY_LIMIT,
@@ -1097,8 +1125,8 @@ pub async fn reload_request_size(db: &DB) {
}
}
pub async fn reload_license_key(db: &DB) -> anyhow::Result<()> {
let q = load_value_from_global_settings(db, LICENSE_KEY_SETTING)
pub async fn reload_license_key(conn: &Connection) -> anyhow::Result<()> {
let q = load_value_from_global_settings_with_conn(conn, LICENSE_KEY_SETTING, true)
.await
.map_err(|err| anyhow::anyhow!("Error reloading license key: {}", err.to_string()))?;
@@ -1123,12 +1151,12 @@ pub async fn reload_license_key(db: &DB) -> anyhow::Result<()> {
}
pub async fn reload_option_setting_with_tracing<T: FromStr + DeserializeOwned>(
db: &DB,
conn: &Connection,
setting_name: &str,
std_env_var: &str,
lock: Arc<RwLock<Option<T>>>,
) {
if let Err(e) = reload_option_setting(db, setting_name, std_env_var, lock.clone()).await {
if let Err(e) = reload_option_setting(conn, setting_name, std_env_var, lock.clone()).await {
tracing::error!("Error reloading setting {}: {:?}", setting_name, e)
}
}
@@ -1147,8 +1175,28 @@ pub async fn load_value_from_global_settings(
Ok(r)
}
pub async fn load_value_from_global_settings_with_conn(
conn: &Connection,
setting_name: &str,
load_from_http: bool,
) -> anyhow::Result<Option<serde_json::Value>> {
match conn {
Connection::Sql(db) => Ok(load_value_from_global_settings(db, setting_name).await?),
Connection::Http(client) => {
if load_from_http {
client.get::<Option<serde_json::Value>>(&format!("/api/agent_workers/get_global_setting/{}", setting_name)).await
.map_err(|e| anyhow::anyhow!("Error loading setting {}: {}", setting_name, e))
} else {
Ok(None)
}
}
}
}
pub async fn reload_option_setting<T: FromStr + DeserializeOwned>(
db: &DB,
conn: &Connection,
setting_name: &str,
std_env_var: &str,
lock: Arc<RwLock<Option<T>>>,
@@ -1163,7 +1211,7 @@ pub async fn reload_option_setting<T: FromStr + DeserializeOwned>(
return Ok(());
}
let q = load_value_from_global_settings(db, setting_name).await?;
let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?;
let mut value = std::env::var(std_env_var)
.ok()
@@ -1190,14 +1238,14 @@ pub async fn reload_option_setting<T: FromStr + DeserializeOwned>(
}
pub async fn reload_setting<T: FromStr + DeserializeOwned + Display>(
db: &DB,
conn: &Connection,
setting_name: &str,
std_env_var: &str,
default: T,
lock: Arc<RwLock<T>>,
transformer: fn(T) -> T,
) -> error::Result<()> {
let q = load_value_from_global_settings(db, setting_name).await?;
let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?;
let mut value = std::env::var(std_env_var)
.ok()
@@ -1255,27 +1303,32 @@ pub async fn monitor_pool(db: &DB) {
}
pub async fn monitor_db(
db: &Pool<Postgres>,
conn: &Connection,
base_internal_url: &str,
server_mode: bool,
_worker_mode: bool,
initial_load: bool,
_killpill_tx: KillpillSender,
) {
tracing::info!("Starting periodic monitor task");
let zombie_jobs_f = async {
if server_mode && !initial_load && !*DISABLE_ZOMBIE_JOBS_MONITORING {
if let Some(db) = conn.as_sql() {
handle_zombie_jobs(db, base_internal_url, "server").await;
match handle_zombie_flows(db).await {
Err(err) => {
tracing::error!("Error handling zombie flows: {:?}", err);
}
},
_ => {}
}
}
}
};
let expired_items_f = async {
if server_mode && !initial_load {
if let Some(db) = conn.as_sql() {
delete_expired_items(&db).await;
}
}
};
@@ -1288,35 +1341,43 @@ pub async fn monitor_db(
let expose_queue_metrics_f = async {
if !initial_load && server_mode {
expose_queue_metrics(&db).await;
if let Some(db) = conn.as_sql() {
expose_queue_metrics(&db).await;
}
}
};
let worker_groups_alerts_f = async {
#[cfg(feature = "enterprise")]
if server_mode && !initial_load {
worker_groups_alerts(&db).await;
if let Some(db) = conn.as_sql() {
worker_groups_alerts(&db).await;
}
}
};
let jobs_waiting_alerts_f = async {
#[cfg(feature = "enterprise")]
if server_mode {
jobs_waiting_alerts(&db).await;
if let Some(db) = conn.as_sql() {
jobs_waiting_alerts(&db).await;
}
}
};
let apply_autoscaling_f = async {
#[cfg(feature = "enterprise")]
if server_mode && !initial_load {
if let Err(e) = windmill_autoscaling::apply_all_autoscaling(db).await {
tracing::error!("Error applying autoscaling: {:?}", e);
if let Some(db) = conn.as_sql() {
if let Err(e) = windmill_autoscaling::apply_all_autoscaling(db).await {
tracing::error!("Error applying autoscaling: {:?}", e);
}
}
}
};
let update_min_worker_version_f = async {
update_min_version(db).await;
update_min_version(conn).await;
};
join!(
@@ -1329,6 +1390,7 @@ pub async fn monitor_db(
apply_autoscaling_f,
update_min_worker_version_f,
);
tracing::info!("Periodic monitor task completed");
}
pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
@@ -1439,7 +1501,7 @@ pub async fn reload_worker_config(
tx: KillpillSender,
kill_if_change: bool,
) {
let config = load_worker_config(&db, tx.clone()).await;
let config = load_worker_config(db, tx.clone()).await;
if let Err(e) = config {
tracing::error!("Error reloading worker config: {:?}", e)
} else {
@@ -1473,15 +1535,15 @@ pub async fn reload_worker_config(
let mut wc = WORKER_CONFIG.write().await;
tracing::info!("Reloading worker config...");
make_suspended_pull_query(&config).await;
make_pull_query(&config).await;
store_suspended_pull_query(&config).await;
store_pull_query(&config).await;
*wc = config
}
}
}
pub async fn load_base_url(db: &DB) -> error::Result<String> {
let q_base_url = load_value_from_global_settings(db, BASE_URL_SETTING).await?;
pub async fn load_base_url(conn: &Connection) -> error::Result<String> {
let q_base_url = load_value_from_global_settings_with_conn(conn, BASE_URL_SETTING, false).await?;
let std_base_url = std::env::var("BASE_URL")
.ok()
@@ -1511,34 +1573,39 @@ pub async fn load_base_url(db: &DB) -> error::Result<String> {
Ok(base_url)
}
pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
#[cfg(feature = "oauth2")]
let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?;
pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> {
#[cfg(feature = "oauth2")]
let oauths = if let Some(q) = q_oauth {
if let Ok(v) = serde_json::from_value::<
Option<HashMap<String, windmill_api::oauth2_ee::OAuthClient>>,
>(q.clone())
{
v
let oauths = if let Some(db) = conn.as_sql() {
let q_oauth = load_value_from_global_settings (db, OAUTH_SETTING).await?;
if let Some(q) = q_oauth {
if let Ok(v) = serde_json::from_value::<
Option<HashMap<String, windmill_api::oauth2_ee::OAuthClient>>,
>(q.clone())
{
v
} else {
tracing::error!("Could not parse oauth setting as a json, found: {:#?}", &q);
None
}
} else {
tracing::error!("Could not parse oauth setting as a json, found: {:#?}", &q);
None
}
} else {
None
};
let base_url = load_base_url(db).await?;
let base_url = load_base_url(conn).await?;
let is_secure = base_url.starts_with("https://");
#[cfg(feature = "oauth2")]
{
let mut l = windmill_api::OAUTH_CLIENTS.write().await;
*l = windmill_api::oauth2_ee::build_oauth_clients(&base_url, oauths, db).await
.map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
.unwrap();
if let Some(db) = conn.as_sql() {
let mut l = windmill_api::OAUTH_CLIENTS.write().await;
*l = windmill_api::oauth2_ee::build_oauth_clients(&base_url, oauths, db).await
.map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
.unwrap();
}
}
{
@@ -1796,7 +1863,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
mpsc::channel::<SameWorkerPayload>(1);
let same_worker_tx_never_used =
SameWorkerSender(same_worker_tx_never_used, Arc::new(AtomicU16::new(0)));
let (send_result_never_used, _send_result_rx_never_used) = mpsc::channel::<SendResult>(1);
let (send_result_never_used, _send_result_rx_never_used) = JobCompletedSender::new_never_used();
let label = if job.permissioned_as != format!("u/{}", job.created_by)
&& job.permissioned_as != job.created_by
@@ -2006,8 +2073,8 @@ async fn cancel_zombie_flow_job(
Ok(())
}
pub async fn reload_hub_base_url_setting(db: &DB, server_mode: bool) -> error::Result<()> {
let hub_base_url = load_value_from_global_settings(db, HUB_BASE_URL_SETTING).await?;
pub async fn reload_hub_base_url_setting(conn: &Connection, server_mode: bool) -> error::Result<()> {
let hub_base_url = load_value_from_global_settings_with_conn(conn, HUB_BASE_URL_SETTING, true).await?;
let base_url = if let Some(q) = hub_base_url {
if let Ok(v) = serde_json::from_value::<String>(q.clone()) {
@@ -2030,16 +2097,18 @@ pub async fn reload_hub_base_url_setting(db: &DB, server_mode: bool) -> error::R
let mut l = HUB_BASE_URL.write().await;
if server_mode {
#[cfg(feature = "embedding")]
if *l != base_url {
let disable_embedding = std::env::var("DISABLE_EMBEDDING")
.ok()
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false);
if !disable_embedding {
let db_clone = db.clone();
tokio::spawn(async move {
update_embeddings_db(&db_clone).await;
});
if let Some(db) = conn.as_sql() {
if *l != base_url {
let disable_embedding = std::env::var("DISABLE_EMBEDDING")
.ok()
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false);
if !disable_embedding {
let db_clone = db.clone();
tokio::spawn(async move {
update_embeddings_db(&db_clone).await;
});
}
}
}
}
@@ -2048,9 +2117,9 @@ pub async fn reload_hub_base_url_setting(db: &DB, server_mode: bool) -> error::R
Ok(())
}
pub async fn reload_critical_error_channels_setting(db: &DB) -> error::Result<()> {
pub async fn reload_critical_error_channels_setting(conn: &DB) -> error::Result<()> {
let critical_error_channels =
load_value_from_global_settings(db, CRITICAL_ERROR_CHANNELS_SETTING).await?;
load_value_from_global_settings(conn, CRITICAL_ERROR_CHANNELS_SETTING).await?;
let critical_error_channels = if let Some(q) = critical_error_channels {
if let Ok(v) = serde_json::from_value::<Vec<CriticalErrorChannel>>(q.clone()) {
backend/tests/worker.rs
+7 -9
View File
@@ -137,7 +137,6 @@ impl ApiServer {
rx,
port_tx,
false,
#[cfg(feature = "smtp")]
format!("http://localhost:{}", addr.port()),
));
@@ -315,7 +314,7 @@ mod suspend_resume {
let second = completed.next().await.unwrap();
// print_job(second, &db).await;
let token = windmill_worker::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "", &Uuid::nil(), None).await.unwrap();
let token = windmill_common::auth::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "", &Uuid::nil(), None).await.unwrap();
let secret = reqwest::get(format!(
"http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}&approver=ruben"
))
@@ -418,7 +417,7 @@ mod suspend_resume {
/* ... and send a request resume it. */
let second = completed.next().await.unwrap();
let token = windmill_worker::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "", &Uuid::nil(), None).await.unwrap();
let token = windmill_common::auth::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "", &Uuid::nil(), None).await.unwrap();
let secret = reqwest::get(format!(
"http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}"
))
@@ -1033,11 +1032,11 @@ fn spawn_test_worker(
priority: 0,
tags: (*wc).worker_tags.clone(),
}];
windmill_common::worker::make_suspended_pull_query(&wc).await;
windmill_common::worker::make_pull_query(&wc).await;
windmill_common::worker::store_suspended_pull_query(&wc).await;
windmill_common::worker::store_pull_query(&wc).await;
}
windmill_worker::run_worker(
&db,
&db.into(),
worker_instance,
worker_name,
1,
@@ -1046,7 +1045,6 @@ fn spawn_test_worker(
rx,
tx2,
&base_internal_url,
false,
)
.await
};
@@ -3843,7 +3841,7 @@ async fn test_result_format(db: Pool<Postgres>) {
let port = server.addr.port();
let token = windmill_worker::create_token_for_owner(
let token = windmill_common::auth::create_token_for_owner(
&db,
"test-workspace",
"u/test-user",
@@ -3884,7 +3882,7 @@ async fn test_result_format(db: Pool<Postgres>) {
assert_eq!(job_result.get(), correct_result);
let response = windmill_api::jobs::run_wait_result(
&db,
&db.into(),
Uuid::parse_str(ordered_result_job_id).unwrap(),
"test-workspace".to_string(),
None,
backend/windmill-api/Cargo.toml
+11 -4
View File
@@ -10,13 +10,14 @@ path = "src/lib.rs"
[features]
default = []
enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise"]
enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"]
stripe = []
agent_worker_server = []
enterprise_saml = ["dep:samael"]
benchmark = []
embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn", "dep:half"]
parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet"]
prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus"]
parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet", "windmill-worker/parquet"]
prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"]
openidconnect = ["dep:openidconnect"]
tantivy = ["dep:windmill-indexer"]
kafka = ["dep:rdkafka"]
@@ -31,6 +32,7 @@ static_frontend = ["dep:rust-embed"]
postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:thiserror", "dep:rust_decimal", "dep:rust-postgres-native-tls"]
mqtt_trigger = ["dep:thiserror", "dep:rumqttc"]
sqs_trigger = ["dep:aws-sdk-sqs", "dep:thiserror", "dep:aws-config"]
gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"]
cloud = ["windmill-common/cloud"]
[dependencies]
@@ -40,8 +42,10 @@ windmill-audit.workspace = true
windmill-parser.workspace = true
windmill-parser-ts.workspace = true
windmill-parser-py.workspace = true
windmill-parser-py-imports.workspace = true
windmill-git-sync.workspace = true
windmill-indexer = { workspace = true, optional = true }
windmill-worker.workspace = true
tokio.workspace = true
anyhow.workspace = true
argon2.workspace = true
@@ -126,4 +130,7 @@ rust_decimal = { workspace = true, optional = true }
rust-postgres-native-tls = { workspace = true, optional = true}
rumqttc = { workspace = true, optional = true }
aws-sdk-sqs = { workspace = true, optional = true }
aws-config = { workspace = true, optional = true}
aws-config = { workspace = true, optional = true }
google-cloud-pubsub = { workspace = true, optional = true }
google-cloud-googleapis = { workspace = true , optional = true }
tonic = { workspace = true, optional = true }
backend/windmill-api/openapi.yaml
+473 -1
View File
@@ -2702,6 +2702,8 @@ paths:
type: boolean
mqtt_used:
type: boolean
gcp_used:
type: boolean
sqs_used:
type: boolean
required:
@@ -2711,6 +2713,7 @@ paths:
- nats_used
- postgres_used
- mqtt_used
- gcp_used
- sqs_used
/w/{workspace}/users/list:
get:
@@ -9222,6 +9225,264 @@ paths:
schema:
type: string
/w/{workspace}/gcp_triggers/create:
post:
summary: create gcp trigger
operationId: createGcpTrigger
tags:
- gcp_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: new gcp trigger
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/NewGcpTrigger"
responses:
"201":
description: gcp trigger created
content:
text/plain:
schema:
type: string
/w/{workspace}/gcp_triggers/update/{path}:
post:
summary: update gcp trigger
operationId: updateGcpTrigger
tags:
- gcp_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
requestBody:
description: updated trigger
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/EditGcpTrigger"
responses:
"200":
description: gcp trigger updated
content:
text/plain:
schema:
type: string
/w/{workspace}/gcp_triggers/delete/{path}:
delete:
summary: delete gcp trigger
operationId: deleteGcpTrigger
tags:
- gcp_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: gcp trigger deleted
content:
text/plain:
schema:
type: string
/w/{workspace}/gcp_triggers/get/{path}:
get:
summary: get gcp trigger
operationId: getGcpTrigger
tags:
- gcp_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: gcp trigger deleted
content:
application/json:
schema:
$ref: "#/components/schemas/GcpTrigger"
/w/{workspace}/gcp_triggers/list:
get:
summary: list gcp triggers
operationId: listGcpTriggers
tags:
- gcp_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
required: true
- $ref: "#/components/parameters/Page"
- $ref: "#/components/parameters/PerPage"
- name: path
description: filter by path
in: query
schema:
type: string
- name: is_flow
in: query
schema:
type: boolean
- name: path_start
in: query
schema:
type: string
responses:
"200":
description: gcp trigger list
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/GcpTrigger"
/w/{workspace}/gcp_triggers/exists/{path}:
get:
summary: does gcp trigger exists
operationId: existsGcpTrigger
tags:
- gcp_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: gcp trigger exists
content:
application/json:
schema:
type: boolean
/w/{workspace}/gcp_triggers/setenabled/{path}:
post:
summary: set enabled gcp trigger
operationId: setGcpTriggerEnabled
tags:
- gcp_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
requestBody:
description: updated gcp trigger enable
required: true
content:
application/json:
schema:
type: object
properties:
enabled:
type: boolean
required:
- enabled
responses:
"200":
description: gcp trigger enabled set
content:
text/plain:
schema:
type: string
/w/{workspace}/gcp_triggers/test:
post:
summary: test gcp connection
operationId: testGcpConnection
tags:
- gcp_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: test gcp connection
required: true
content:
application/json:
schema:
type: object
properties:
connection:
type: object
required:
- connection
responses:
"200":
description: try to connect to a gcp broker
content:
text/plain:
schema:
type: string
/w/{workspace}/gcp_triggers/subscriptions/delete/{path}:
delete:
summary: delete gcp trigger
operationId: deleteGcpSubscription
tags:
- gcp_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
requestBody:
description: args to delete subscription from google cloud
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/DeleteGcpSubscription"
responses:
"200":
description: gcp trigger deleted
content:
text/plain:
schema:
type: string
/w/{workspace}/gcp_triggers/topics/list/{path}:
get:
summary: list all topics of google cloud service
operationId: listGoogleTopics
tags:
- gcp_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: get all google topics
content:
application/json:
schema:
type: array
items:
type: string
/w/{workspace}/gcp_triggers/subscriptions/list/{path}:
post:
summary: list all subscription of a give topic from google cloud service
operationId: listAllTGoogleTopicSubscriptions
tags:
- gcp_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
requestBody:
description: args to get subscription's topic from google cloud
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/GetAllTopicSubscription"
responses:
"200":
description: get all google topic subscriptions name
content:
application/json:
schema:
type: array
items:
type: string
/w/{workspace}/postgres_triggers/is_valid_postgres_configuration/{path}:
get:
@@ -10498,6 +10759,41 @@ paths:
items:
$ref: "#/components/schemas/AutoscalingEvent"
/agent_workers/create_agent_token:
post:
summary: create agent token
operationId: createAgentToken
tags:
- agent_workers
requestBody:
description: agent token
required: true
content:
application/json:
schema:
type: object
properties:
worker_group:
type: string
tags:
type: array
items:
type: string
exp:
type: integer
required:
- worker_group
- tags
- exp
responses:
"200":
description: agent token created
content:
application/json:
schema:
type: string
/w/{workspace}/acls/get/{kind}/{path}:
get:
summary: get granular acls
@@ -10528,6 +10824,7 @@ paths:
nats_trigger,
postgres_trigger,
mqtt_trigger,
gcp_trigger,
sqs_trigger
]
responses:
@@ -10571,6 +10868,7 @@ paths:
nats_trigger,
postgres_trigger,
mqtt_trigger,
gcp_trigger,
sqs_trigger
]
requestBody:
@@ -10625,6 +10923,7 @@ paths:
nats_trigger,
postgres_trigger,
mqtt_trigger,
gcp_trigger,
sqs_trigger
]
requestBody:
@@ -10677,6 +10976,10 @@ paths:
responses:
"200":
description: capture config set
content:
application/json:
schema:
type: object
/w/{workspace}/capture/ping_config/{trigger_kind}/{runnable_kind}/{path}:
post:
@@ -14155,6 +14458,8 @@ components:
type: number
mqtt_count:
type: number
gcp_count:
type: number
sqs_count:
type: number
@@ -14439,6 +14744,173 @@ components:
- subscribe_topics
- mqtt_resource_path
DeliveryType:
type: string
enum:
- push
- pull
PushConfig:
type: object
properties:
audience:
type: string
authenticate:
type: boolean
base_endpoint:
type: string
required:
- authenticate
- base_endpoint
GcpTrigger:
allOf:
- $ref: "#/components/schemas/TriggerExtraProperty"
type: object
properties:
gcp_resource_path:
type: string
topic_id:
type: string
subscription_id:
type: string
server_id:
type: string
delivery_type:
$ref: "#/components/schemas/DeliveryType"
delivery_config:
$ref: "#/components/schemas/PushConfig"
last_server_ping:
type: string
format: date-time
error:
type: string
enabled:
type: boolean
required:
- gcp_resource_path
- topic_id
- subscription_id
- enabled
- delivery_type
SubscriptionMode:
type: string
enum:
- existing
- create_update
description: "The mode of subscription. 'existing' means using an existing GCP subscription, while 'create_update' involves creating or updating a new subscription."
GcpExistingSubscription:
type: object
properties:
subscription_id:
type: string
base_endpoint:
type: string
required:
- subscription_id
- base_endpoint
GcpCreateUpdateSubscription:
type: object
properties:
subscription_id:
type: string
delivery_type:
$ref: "#/components/schemas/DeliveryType"
delivery_config:
$ref: "#/components/schemas/PushConfig"
required:
- delivery_type
GcpSubscriptionModeConfig:
type: object
properties:
subscription_mode:
$ref: "#/components/schemas/SubscriptionMode"
required:
- subscription_mode
allOf:
- oneOf:
- $ref: "#/components/schemas/GcpExistingSubscription"
- $ref: "#/components/schemas/GcpCreateUpdateSubscription"
description: |
"This is a union type representing the subscription mode.
- 'existing': Represents an existing GCP subscription, and should be accompanied by an 'ExistingGcpSubscription' object.
- 'create_update': Represents a new or updated GCP subscription, and should be accompanied by a 'CreateUpdateConfig' object."
NewGcpTrigger:
type: object
properties:
gcp_resource_path:
type: string
topic_id:
type: string
subscription_mode:
$ref: "#/components/schemas/GcpSubscriptionModeConfig"
path:
type: string
script_path:
type: string
is_flow:
type: boolean
enabled:
type: boolean
required:
- path
- script_path
- is_flow
- gcp_resource_path
- topic_id
- subscription_mode
EditGcpTrigger:
type: object
properties:
gcp_resource_path:
type: string
topic_id:
type: string
subscription_mode:
$ref: "#/components/schemas/GcpSubscriptionModeConfig"
path:
type: string
script_path:
type: string
is_flow:
type: boolean
enabled:
type: boolean
required:
- path
- script_path
- is_flow
- enabled
- mqtt_resource_path
- subscription_id
- delivery_type
- topic_id
- subscription_mode
GetAllTopicSubscription:
type: object
properties:
topic_id:
type: string
required:
- topic_id
DeleteGcpSubscription:
type: object
properties:
subscription_id:
type: string
required:
- subscription_id
SqsTrigger:
allOf:
- $ref: "#/components/schemas/TriggerExtraProperty"
@@ -15824,7 +16296,7 @@ components:
CaptureTriggerKind:
type: string
enum: [webhook, http, websocket, kafka, email, nats, postgres, sqs, mqtt]
enum: [webhook, http, websocket, kafka, email, nats, postgres, sqs, mqtt, gcp]
Capture:
type: object
backend/windmill-api/src/agent_workers_ee.rs
+52
View File
@@ -0,0 +1,52 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2042
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::db::DB;
use axum::Router;
use serde::{Deserialize, Serialize};
pub fn global_service() -> Router {
Router::new()
}
pub fn workspaced_service(
db: DB,
_base_internal_url: String,
) -> (
Router,
Option<tokio::task::JoinHandle<()>>,
windmill_worker::JobCompletedSender,
) {
use windmill_common::worker::Connection;
use windmill_worker::JobCompletedSender;
let (job_completed_tx, _job_completed_rx) =
JobCompletedSender::new(&Connection::Sql(db.clone()), 100);
let router = Router::new();
(router, None, job_completed_tx)
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct AgentAuth {
pub worker_group: String,
pub suffix: Option<String>,
pub tags: Vec<String>,
pub exp: Option<usize>,
}
pub struct AgentCache {}
impl AgentCache {
pub fn new() -> Self {
AgentCache {}
}
}
backend/windmill-api/src/auth.rs
-1
View File
@@ -509,7 +509,6 @@ where
.map(|x| x.0)
.unwrap_or_default();
let path_vec: Vec<&str> = original_uri.path().split("/").collect();
let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" {
Some(path_vec[3].to_owned())
} else {
backend/windmill-api/src/capture.rs
+366 -97
View File
@@ -7,34 +7,47 @@
*/
#[cfg(feature = "http_trigger")]
use crate::http_triggers::{build_http_trigger_extra, HttpMethod};
use {
crate::{
args::try_from_request_body,
http_triggers::{build_http_trigger_extra, HttpMethod},
},
axum::response::{IntoResponse, Response},
std::collections::HashMap,
};
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
use crate::gcp_triggers_ee::{
manage_google_subscription, process_google_push_request, validate_jwt_token, SubscriptionMode,
};
#[cfg(any(
feature = "http_trigger",
all(feature = "enterprise", feature = "gcp_trigger")
))]
use {
axum::extract::Request, http::HeaderMap, serde::de::DeserializeOwned,
windmill_common::error::Error,
};
#[cfg(all(feature = "enterprise", feature = "kafka"))]
use crate::kafka_triggers_ee::KafkaTriggerConfigConnection;
#[cfg(feature = "mqtt_trigger")]
use crate::mqtt_triggers::{MqttClientVersion, MqttV3Config, MqttV5Config, SubscribeTopic};
#[cfg(all(feature = "enterprise", feature = "nats"))]
use crate::nats_triggers_ee::NatsTriggerConfigConnection;
#[cfg(feature = "postgres_trigger")]
use crate::postgres_triggers::{
create_logical_replication_slot_query, create_publication_query, drop_publication_query,
generate_random_string, get_database_connection, PublicationData,
use {
crate::postgres_triggers::{
create_logical_replication_slot_query, create_publication_query, drop_publication_query,
generate_random_string, get_database_connection, PublicationData,
},
itertools::Itertools,
pg_escape::quote_literal,
};
#[cfg(feature = "http_trigger")]
use axum::extract::Request;
#[cfg(feature = "http_trigger")]
use axum::response::IntoResponse;
#[cfg(feature = "http_trigger")]
use http::HeaderMap;
#[cfg(feature = "postgres_trigger")]
use itertools::Itertools;
#[cfg(feature = "postgres_trigger")]
use pg_escape::quote_literal;
#[cfg(feature = "http_trigger")]
use serde::de::DeserializeOwned;
#[cfg(feature = "http_trigger")]
use std::collections::HashMap;
#[cfg(feature = "http_trigger")]
use windmill_common::error::Error;
use crate::{
args::WebhookArgs,
@@ -42,23 +55,26 @@ use crate::{
users::fetch_api_authed,
utils::RunnableKind,
};
use axum::{
extract::{Extension, Path, Query},
routing::{delete, get, head, post},
Json, Router,
};
use hyper::StatusCode;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use sqlx::types::Json as SqlxJson;
use windmill_common::{
db::UserDB,
error::{JsonResult, Result},
utils::{not_found_if_none, paginate, Pagination, StripPath},
worker::{to_raw_value, CLOUD_HOSTED},
};
use windmill_queue::TriggerKind;
use windmill_queue::{PushArgs, PushArgsOwned};
use windmill_queue::{PushArgs, PushArgsOwned, TriggerKind};
const KEEP_LAST: i64 = 20;
@@ -85,14 +101,26 @@ pub fn workspaced_unauthed_service() -> Router {
head(|| async {}).post(webhook_payload),
);
#[cfg(feature = "http_trigger")]
#[cfg(any(
feature = "http_trigger",
all(feature = "enterprise", feature = "gcp_trigger")
))]
{
router.route("/http/:runnable_kind/:path/*route_path", {
#[cfg(feature = "http_trigger")]
let router = router.route("/http/:runnable_kind/:path/*route_path", {
head(|| async {}).fallback(http_payload)
})
});
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
let router = router.route("/gcp/:runnable_kind/*path", post(gcp_payload));
router
}
#[cfg(not(feature = "http_trigger"))]
#[cfg(not(any(
feature = "http_trigger",
all(feature = "enterprise", feature = "gcp_trigger")
)))]
{
router
}
@@ -124,6 +152,15 @@ pub struct SqsTriggerConfig {
pub message_attributes: Option<Vec<String>>,
}
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
#[derive(Debug, Serialize, Deserialize)]
pub struct GcpTriggerConfig {
pub gcp_resource_path: String,
#[serde(flatten)]
pub subscription_mode: SubscriptionMode,
pub topic_id: String,
}
#[cfg(all(feature = "enterprise", feature = "nats"))]
#[derive(Serialize, Deserialize)]
pub struct NatsTriggerConfig {
@@ -181,6 +218,8 @@ enum TriggerConfig {
Nats(NatsTriggerConfig),
#[cfg(feature = "mqtt_trigger")]
Mqtt(MqttTriggerConfig),
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
Gcp(GcpTriggerConfig),
}
#[derive(Serialize, Deserialize)]
@@ -208,15 +247,26 @@ async fn get_configs(
let configs = sqlx::query_as!(
CaptureConfig,
r#"SELECT trigger_config as "trigger_config: _", trigger_kind as "trigger_kind: _", error, last_server_ping
FROM capture_config
WHERE workspace_id = $1 AND path = $2 AND is_flow = $3"#,
r#"
SELECT
trigger_config AS "trigger_config: _",
trigger_kind AS "trigger_kind: _",
error,
last_server_ping
FROM
capture_config
WHERE
workspace_id = $1
AND path = $2
AND is_flow = $3
"#,
&w_id,
&path.to_path(),
matches!(runnable_kind, RunnableKind::Flow),
)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
Ok(Json(configs))
@@ -284,33 +334,100 @@ async fn set_postgres_trigger_config(
Ok(capture_config)
}
#[inline]
#[cfg(not(feature = "postgres_trigger"))]
async fn set_postgres_trigger_config(
_w_id: &str,
_authed: ApiAuthed,
_db: &DB,
_user_db: UserDB,
capture_config: NewCaptureConfig,
) -> Result<NewCaptureConfig> {
Ok(capture_config)
}
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
async fn set_gcp_trigger_config(
w_id: &str,
authed: ApiAuthed,
db: &DB,
mut capture_config: NewCaptureConfig,
) -> Result<NewCaptureConfig> {
let Some(TriggerConfig::Gcp(mut gcp_config)) = capture_config.trigger_config else {
return Err(windmill_common::error::Error::BadRequest(
"Invalid GCP Pub/Sub config".to_string(),
));
};
let config = manage_google_subscription(
authed,
db,
w_id,
&gcp_config.gcp_resource_path,
&capture_config.path,
&gcp_config.topic_id,
gcp_config.subscription_mode,
)
.await?;
gcp_config.subscription_mode = SubscriptionMode::CreateUpdate(config);
capture_config.trigger_config = Some(TriggerConfig::Gcp(gcp_config));
Ok(capture_config)
}
#[inline]
#[cfg(not(all(feature = "enterprise", feature = "gcp_trigger")))]
async fn set_gcp_trigger_config(
_w_id: &str,
_authed: ApiAuthed,
_db: &DB,
capture_config: NewCaptureConfig,
) -> Result<NewCaptureConfig> {
Ok(capture_config)
}
async fn set_config(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
#[cfg(feature = "postgres_trigger")] Extension(db): Extension<DB>,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(nc): Json<NewCaptureConfig>,
) -> Result<()> {
#[cfg(feature = "postgres_trigger")]
let nc = if let TriggerKind::Postgres = nc.trigger_kind {
set_postgres_trigger_config(&w_id, authed.clone(), &db, user_db.clone(), nc).await?
} else {
nc
) -> JsonResult<Option<TriggerConfig>> {
let nc = match nc.trigger_kind {
TriggerKind::Postgres => {
set_postgres_trigger_config(&w_id, authed.clone(), &db, user_db.clone(), nc).await?
}
TriggerKind::Gcp => set_gcp_trigger_config(&w_id, authed.clone(), &db, nc).await?,
_ => nc,
};
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"INSERT INTO capture_config
(workspace_id, path, is_flow, trigger_kind, trigger_config, owner, email)
VALUES ($1, $2, $3, $4, $5, $6, $7)
r#"
INSERT INTO capture_config (
workspace_id, path, is_flow, trigger_kind, trigger_config, owner, email
)
VALUES (
$1, $2, $3, $4, $5, $6, $7
)
ON CONFLICT (workspace_id, path, is_flow, trigger_kind)
DO UPDATE SET trigger_config = $5, owner = $6, email = $7, server_id = NULL, error = NULL",
DO UPDATE
SET
trigger_config = $5,
owner = $6,
email = $7,
server_id = NULL,
error = NULL
"#,
&w_id,
&nc.path,
nc.is_flow,
nc.trigger_kind as TriggerKind,
nc.trigger_config.map(|x| SqlxJson(to_raw_value(&x))) as Option<SqlxJson<Box<RawValue>>>,
nc.trigger_config
.as_ref()
.map(|x| SqlxJson(to_raw_value(&x))) as Option<SqlxJson<Box<RawValue>>>,
&authed.username,
&authed.email,
)
@@ -319,7 +436,7 @@ async fn set_config(
tx.commit().await?;
Ok(())
Ok(Json(nc.trigger_config))
}
async fn ping_config(
@@ -333,8 +450,19 @@ async fn ping_config(
)>,
) -> Result<()> {
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"UPDATE capture_config SET last_client_ping = now() WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4",
r#"
UPDATE
capture_config
SET
last_client_ping = NOW()
WHERE
workspace_id = $1
AND path = $2
AND is_flow = $3
AND trigger_kind = $4
"#,
&w_id,
&path.to_path(),
matches!(runnable_kind, RunnableKind::Flow),
@@ -342,6 +470,7 @@ async fn ping_config(
)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(())
}
@@ -374,14 +503,28 @@ async fn list_captures(
let captures = sqlx::query_as!(
Capture,
r#"SELECT id, created_at, trigger_kind as "trigger_kind: _", CASE WHEN pg_column_size(payload) < 40000 THEN payload ELSE '"WINDMILL_TOO_BIG"'::jsonb END as "payload!: _", trigger_extra as "trigger_extra: _"
FROM capture
WHERE workspace_id = $1
AND path = $2 AND is_flow = $3
r#"
SELECT
id,
created_at,
trigger_kind AS "trigger_kind: _",
CASE
WHEN pg_column_size(payload) < 40000 THEN payload
ELSE '"WINDMILL_TOO_BIG"'::jsonb
END AS "payload!: _",
trigger_extra AS "trigger_extra: _"
FROM
capture
WHERE
workspace_id = $1
AND path = $2
AND is_flow = $3
AND ($4::trigger_kind IS NULL OR trigger_kind = $4)
ORDER BY created_at DESC
ORDER BY
created_at DESC
OFFSET $5
LIMIT $6"#,
LIMIT $6
"#,
&w_id,
&path.to_path(),
matches!(runnable_kind, RunnableKind::Flow),
@@ -403,14 +546,28 @@ async fn get_capture(
Path((w_id, id)): Path<(String, i64)>,
) -> JsonResult<Capture> {
let mut tx = user_db.begin(&authed).await?;
let capture = sqlx::query_as!(
Capture,
r#"SELECT id, created_at, trigger_kind as "trigger_kind: _", payload as "payload!: _", trigger_extra as "trigger_extra: _" FROM capture WHERE id = $1 AND workspace_id = $2"#,
r#"
SELECT
id,
created_at,
trigger_kind AS "trigger_kind: _",
payload AS "payload!: _",
trigger_extra AS "trigger_extra: _"
FROM
capture
WHERE
id = $1
AND workspace_id = $2
"#,
id,
&w_id,
)
.fetch_one(&mut *tx)
.await?;
.await?;
tx.commit().await?;
Ok(Json(capture))
}
@@ -421,9 +578,17 @@ async fn delete_capture(
Path((_, id)): Path<(String, i64)>,
) -> Result<()> {
let mut tx = user_db.begin(&authed).await?;
sqlx::query!("DELETE FROM capture WHERE id = $1", id)
.execute(&mut *tx)
.await?;
sqlx::query!(
r#"
DELETE FROM
capture
WHERE
id = $1
"#,
id
)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(())
}
@@ -441,8 +606,18 @@ async fn move_captures_and_configs(
) -> Result<()> {
let mut tx = user_db.begin(&authed).await?;
let old_path = old_path.to_path();
sqlx::query!(
"UPDATE capture_config SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow = $4",
r#"
UPDATE
capture_config
SET
path = $1
WHERE
path = $2
AND workspace_id = $3
AND is_flow = $4
"#,
body.new_path,
old_path,
&w_id,
@@ -450,8 +625,18 @@ async fn move_captures_and_configs(
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"UPDATE capture SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow = $4",
r#"
UPDATE
capture
SET
path = $1
WHERE
path = $2
AND workspace_id = $3
AND is_flow = $4
"#,
body.new_path,
old_path,
&w_id,
@@ -459,6 +644,7 @@ async fn move_captures_and_configs(
)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(())
}
@@ -478,9 +664,19 @@ pub async fn get_active_capture_owner_and_email(
) -> Result<(String, String)> {
let capture_config = sqlx::query_as!(
ActiveCaptureOwner,
"SELECT owner, email
FROM capture_config
WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4 AND last_client_ping > NOW() - INTERVAL '10 seconds'",
r#"
SELECT
owner,
email
FROM
capture_config
WHERE
workspace_id = $1
AND path = $2
AND is_flow = $3
AND trigger_kind = $4
AND last_client_ping > NOW() - INTERVAL '10 seconds'
"#,
&w_id,
&path,
is_flow,
@@ -498,7 +694,10 @@ pub async fn get_active_capture_owner_and_email(
Ok((capture_config.owner, capture_config.email))
}
#[cfg(feature = "http_trigger")]
#[cfg(any(
feature = "http_trigger",
all(feature = "enterprise", feature = "gcp_trigger")
))]
async fn get_capture_trigger_config_and_owner<T: DeserializeOwned>(
db: &DB,
w_id: &str,
@@ -512,16 +711,34 @@ async fn get_capture_trigger_config_and_owner<T: DeserializeOwned>(
owner: String,
email: String,
}
let capture_config = sqlx::query_as!(
CaptureTriggerConfigAndOwner,
r#"SELECT trigger_config as "trigger_config: _", owner, email
FROM capture_config
WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4 AND last_client_ping > NOW() - INTERVAL '10 seconds'"#,
r#"
SELECT
trigger_config AS "trigger_config: _",
owner,
email
FROM
capture_config
WHERE
workspace_id = $1
AND path = $2
AND is_flow = $3
AND trigger_kind = $4
AND last_client_ping > NOW() - INTERVAL '10 seconds'
AND (
$5::bool IS FALSE
OR (
trigger_config IS NOT NULL
AND trigger_config ->> 'delivery_type' = 'push'
)
)
"#,
&w_id,
&path,
is_flow,
kind as &TriggerKind,
matches!(kind, TriggerKind::Gcp)
)
.fetch_optional(db)
.await?;
@@ -554,17 +771,24 @@ async fn clear_captures_history(db: &DB, w_id: &str) -> Result<()> {
if *CLOUD_HOSTED {
/* Retain only KEEP_LAST most recent captures in this workspace. */
sqlx::query!(
"DELETE FROM capture
WHERE workspace_id = $1
AND created_at <=
(
SELECT created_at
FROM capture
WHERE workspace_id = $1
ORDER BY created_at DESC
OFFSET $2
LIMIT 1
)",
r#"
DELETE FROM
capture
WHERE
workspace_id = $1
AND created_at <= (
SELECT
created_at
FROM
capture
WHERE
workspace_id = $1
ORDER BY
created_at DESC
OFFSET $2
LIMIT 1
)
"#,
&w_id,
KEEP_LAST,
)
@@ -585,8 +809,15 @@ pub async fn insert_capture_payload(
owner: &str,
) -> Result<()> {
sqlx::query!(
"INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, payload, trigger_extra, created_by)
VALUES ($1, $2, $3, $4, $5, $6, $7)",
r#"
INSERT INTO
capture (
workspace_id, path, is_flow, trigger_kind, payload, trigger_extra, created_by
)
VALUES (
$1, $2, $3, $4, $5, $6, $7
)
"#,
&w_id,
path,
is_flow,
@@ -642,36 +873,74 @@ async fn webhook_payload(
Ok(StatusCode::NO_CONTENT)
}
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
async fn gcp_payload(
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, String)>,
headers: HeaderMap,
request: Request,
) -> Result<StatusCode> {
let is_flow = matches!(runnable_kind, RunnableKind::Flow);
let (gcp_trigger_config, owner, email): (GcpTriggerConfig, _, _) =
get_capture_trigger_config_and_owner(&db, &w_id, &path, is_flow, &TriggerKind::Gcp).await?;
let authed = fetch_api_authed(owner.clone(), email, &w_id, &db, None).await?;
let SubscriptionMode::CreateUpdate(config) = &gcp_trigger_config.subscription_mode else {
return Err(Error::BadConfig("Bad config".to_string()));
};
validate_jwt_token(
&db,
user_db.clone(),
authed.clone(),
&headers,
&gcp_trigger_config.gcp_resource_path,
&w_id,
config.delivery_config.as_ref().unwrap(),
)
.await?;
let (args, extra) = process_google_push_request(headers, request).await?;
let payload = PushArgsOwned { args, extra: None };
let _ = insert_capture_payload(
&db,
&w_id,
&path,
is_flow,
&TriggerKind::Gcp,
payload,
Some(to_raw_value(&extra)),
&owner,
)
.await?;
Ok(StatusCode::NO_CONTENT)
}
#[cfg(feature = "http_trigger")]
async fn http_payload(
Extension(db): Extension<DB>,
Path((w_id, kind, path, route_path)): Path<(String, RunnableKind, String, StripPath)>,
Path((w_id, runnable_kind, path, route_path)): Path<(String, RunnableKind, String, StripPath)>,
Query(query): Query<HashMap<String, String>>,
method: http::Method,
headers: HeaderMap,
request: Request,
) -> std::result::Result<StatusCode, impl IntoResponse> {
use axum::response::Response;
use crate::args::try_from_request_body;
let route_path = route_path.to_path();
) -> std::result::Result<StatusCode, Response> {
let path = path.replace(".", "/");
let is_flow = matches!(runnable_kind, RunnableKind::Flow);
let route_path = route_path.to_path();
let (http_trigger_config, owner, email): (HttpTriggerConfig, _, _) =
get_capture_trigger_config_and_owner(
&db,
&w_id,
&path,
matches!(kind, RunnableKind::Flow),
&TriggerKind::Http,
)
.await
.map_err(|e| e.into_response())?;
get_capture_trigger_config_and_owner(&db, &w_id, &path, is_flow, &TriggerKind::Http)
.await
.map_err(|e| e.into_response())?;
let args = try_from_request_body(
request,
&db,
&(),
http_trigger_config.raw_string,
http_trigger_config.wrap_body,
)
@@ -721,7 +990,7 @@ async fn http_payload(
&db,
&w_id,
&path,
matches!(kind, RunnableKind::Flow),
is_flow,
&TriggerKind::Http,
args,
extra,
@@ -730,5 +999,5 @@ async fn http_payload(
.await
.map_err(|e| e.into_response())?;
Ok::<_, Response>(StatusCode::NO_CONTENT)
Ok(StatusCode::NO_CONTENT)
}
backend/windmill-api/src/drafts.rs
+1 -1
View File
@@ -1,6 +1,6 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2042
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
backend/windmill-api/src/gcp_triggers_ee.rs
+133
View File
@@ -0,0 +1,133 @@
use crate::db::{ApiAuthed, DB};
use axum::{extract::Request, Router};
use http::HeaderMap;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use sqlx::prelude::FromRow;
use sqlx::types::Json as SqlxJson;
use std::collections::HashMap;
use windmill_common::db::UserDB;
use windmill_common::{
error::{Error as WindmillError, Result as WindmillResult},
utils::empty_string_as_none,
};
#[derive(sqlx::Type, Debug, Deserialize, Serialize)]
#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
#[sqlx(type_name = "DELIVERY_MODE", rename_all = "lowercase")]
#[allow(unused)]
pub enum DeliveryType {
Pull,
Push,
}
impl Default for DeliveryType {
fn default() -> Self {
Self::Pull
}
}
#[derive(FromRow, Deserialize, Serialize, Debug)]
#[allow(unused)]
pub struct PushConfig {
#[serde(deserialize_with = "empty_string_as_none")]
route_path: Option<String>,
#[serde(deserialize_with = "empty_string_as_none")]
audience: Option<String>,
authenticate: bool,
base_endpoint: String,
}
#[derive(Default, Debug, Serialize, Deserialize)]
#[allow(unused)]
pub struct CreateUpdateConfig {
pub delivery_type: DeliveryType,
#[serde(default, deserialize_with = "empty_string_as_none")]
pub subscription_id: Option<String>,
pub delivery_config: Option<SqlxJson<PushConfig>>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct ExistingGcpSubscription {
pub subscription_id: String,
pub base_endpoint: String,
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(tag = "subscription_mode", rename_all = "snake_case")]
pub enum SubscriptionMode {
Existing(ExistingGcpSubscription),
CreateUpdate(CreateUpdateConfig),
}
pub fn workspaced_service() -> Router {
Router::new()
}
pub fn start_consuming_gcp_pubsub_event(
_db: DB,
mut _killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> () {
// implementation is not open source
}
pub async fn manage_google_subscription(
_authed: ApiAuthed,
_db: &DB,
_workspace_id: &str,
_gcp_resource_path: &str,
_path: &str,
_topic_id: &str,
_subscription_mode: SubscriptionMode,
) -> WindmillResult<CreateUpdateConfig> {
Ok(CreateUpdateConfig::default())
}
pub async fn process_google_push_request(
_headers: HeaderMap,
_request: Request,
) -> Result<
(
HashMap<String, Box<RawValue>>,
Option<HashMap<String, Box<RawValue>>>,
),
WindmillError,
> {
Ok((HashMap::new(), None))
}
pub async fn validate_jwt_token(
_db: &DB,
_user_db: UserDB,
_authed: ApiAuthed,
_headers: &HeaderMap,
_gcp_resource_path: &str,
_workspace_id: &str,
_delivery_config: &PushConfig,
) -> Result<(), windmill_common::error::Error> {
Ok(())
}
pub fn gcp_push_route_handler() -> Router {
Router::new()
}
#[derive(FromRow, Deserialize, Serialize, Debug)]
pub struct GcpTrigger {
pub gcp_resource_path: String,
pub subscription_id: String,
pub delivery_type: DeliveryType,
pub delivery_config: Option<SqlxJson<PushConfig>>,
pub topic_id: String,
pub path: String,
pub script_path: String,
pub is_flow: bool,
pub workspace_id: String,
pub edited_by: String,
pub email: String,
pub edited_at: chrono::DateTime<chrono::Utc>,
pub extra_perms: Option<serde_json::Value>,
pub error: Option<String>,
pub server_id: Option<String>,
pub last_server_ping: Option<chrono::DateTime<chrono::Utc>>,
pub enabled: bool,
}
backend/windmill-api/src/http_trigger_auth.rs
+6 -2
View File
@@ -618,7 +618,9 @@ impl AuthenticationMethod {
api_key_header,
api_key_secret,
}) => {
let api_key_to_cmp = headers.try_get_webhook_header(&api_key_header)?;
let api_key_to_cmp = headers
.try_get_webhook_header(&api_key_header)
.map_err(|_| AuthenticationError::InvalidApiKey)?;
if api_key_to_cmp != api_key_secret {
return Err(AuthenticationError::InvalidApiKey);
}
@@ -734,7 +736,9 @@ impl IntoResponse for AuthenticationError {
)
.into_response()
}
AuthenticationError::InvalidApiKey => (StatusCode::FORBIDDEN, self.to_string()),
AuthenticationError::InvalidApiKey => {
return (StatusCode::UNAUTHORIZED, "Unauthorized").into_response()
}
};
let body = json!({ "error": error_message });
backend/windmill-api/src/http_triggers.rs
+1 -1
View File
@@ -953,7 +953,7 @@ async fn route_job(
let args = try_from_request_body(
request,
&db,
&(),
Some(match trigger.authentication_method {
AuthenticationMethod::CustomScript | AuthenticationMethod::Signature => true,
_ => trigger.raw_string,
backend/windmill-api/src/jobs.rs
+13 -9
View File
@@ -26,7 +26,7 @@ use tower::ServiceBuilder;
use windmill_common::error::JsonResult;
use windmill_common::flow_status::{JobResult, RestartedFrom};
use windmill_common::jobs::{format_completed_job_result, format_result, ENTRYPOINT_OVERRIDE};
use windmill_common::worker::{CLOUD_HOSTED, TMP_DIR};
use windmill_common::worker::{Connection, CLOUD_HOSTED, TMP_DIR};
use windmill_common::scripts::PREVIEW_IS_CODEBASE_HASH;
use windmill_common::variables::get_workspace_key;
@@ -372,7 +372,6 @@ async fn cancel_job_api(
email: "anonymous".to_string(),
},
};
let (mut tx, job_option) = tokio::time::timeout(
std::time::Duration::from_secs(120),
windmill_queue::cancel_job(
@@ -587,7 +586,9 @@ async fn get_flow_job_debug_info(
Extension(db): Extension<DB>,
Path((w_id, id)): Path<(String, Uuid)>,
) -> error::Result<Response> {
let job = GetQuery::new().fetch_queued(&db, id, &w_id).await?;
let job = GetQuery::new()
.fetch_queued((&db).into(), id, &w_id)
.await?;
if let Some(job) = job {
let is_flow = job.is_flow();
if job.is_flow_step || !is_flow {
@@ -926,8 +927,9 @@ impl<'a> GetQuery<'a> {
// Try to fetch the code from the cache, fallback to the preview code.
// NOTE: This could check for the job kinds instead of the `or_else` but it's not
// necessary as `fetch_script` return early if the job kind is not a preview one.
cache::job::fetch_script(db, kind, hash)
.or_else(|_| cache::job::fetch_preview_script(db, &id, raw_lock, raw_code))
let conn = Connection::from(db.clone());
cache::job::fetch_script(db.clone(), kind, hash)
.or_else(|_| cache::job::fetch_preview_script(&conn, &id, raw_lock, raw_code))
.await
.ok()
.inspect(|data| {
@@ -956,7 +958,7 @@ impl<'a> GetQuery<'a> {
self.check_auth(job.as_ref().map(|job| job.created_by.as_str()))?;
if let Some(job) = job.as_mut() {
self.resolve_raw_values(db, job.id, job.job_kind, job.script_hash, job)
self.resolve_raw_values(&db, job.id, job.job_kind, job.script_hash, job)
.await;
}
if self.with_flow {
@@ -993,12 +995,14 @@ impl<'a> GetQuery<'a> {
self.resolve_raw_values(db, job.id, job.job_kind, job.script_hash, job)
.await;
}
if self.with_flow {
cjob = resolve_maybe_value(db, workspace_id, self.with_code, cjob, |job| {
job.raw_flow.as_mut()
})
.await?;
}
if let Some(mut cjob) = cjob {
cjob.inner = format_completed_job_result(cjob.inner);
return Ok(Some(cjob));
@@ -1008,7 +1012,7 @@ impl<'a> GetQuery<'a> {
async fn fetch(self, db: &DB, job_id: Uuid, workspace_id: &str) -> error::Result<Job> {
let cjob = self
.fetch_completed(db, job_id, workspace_id)
.fetch_completed(db.into(), job_id, workspace_id)
.await?
.map(Job::CompletedJob);
@@ -1016,7 +1020,7 @@ impl<'a> GetQuery<'a> {
Some(cjob) => Ok(cjob),
None => {
let job_maybe = self
.fetch_queued(db, job_id, workspace_id)
.fetch_queued(db.into(), job_id, workspace_id)
.await?
.map(Job::QueuedJob);
// potential race condition here, if the job was in queue and completed right after the fetch completed, so we need to check one last time
@@ -1024,7 +1028,7 @@ impl<'a> GetQuery<'a> {
return Ok(job);
} else {
let cjob2 = self
.fetch_completed(db, job_id, workspace_id)
.fetch_completed(db.into(), job_id, workspace_id)
.await?
.map(Job::CompletedJob);
not_found_if_none(cjob2, "Job", job_id.to_string())
backend/windmill-api/src/lib.rs
+94 -13
View File
@@ -25,6 +25,8 @@ use crate::{
webhook_util::WebhookShared,
};
#[cfg(feature = "agent_worker_server")]
use agent_workers_ee::AgentCache;
use anyhow::Context;
use argon2::Argon2;
use axum::extract::DefaultBodyLimit;
@@ -56,6 +58,8 @@ use windmill_common::{utils::GIT_VERSION, BASE_URL, INSTANCE_NAME};
use crate::scim_ee::has_scim_token;
use windmill_common::error::AppError;
#[cfg(feature = "agent_worker_server")]
mod agent_workers_ee;
mod ai;
mod apps;
pub mod args;
@@ -74,9 +78,9 @@ mod folders;
mod granular_acls;
mod groups;
#[cfg(feature = "http_trigger")]
mod http_triggers;
#[cfg(feature = "http_trigger")]
mod http_trigger_auth;
#[cfg(feature = "http_trigger")]
mod http_triggers;
mod indexer_ee;
mod inputs;
mod integration;
@@ -85,6 +89,8 @@ mod postgres_triggers;
#[cfg(feature = "enterprise")]
mod apps_ee;
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
mod gcp_triggers_ee;
#[cfg(feature = "enterprise")]
mod git_sync_ee;
#[cfg(feature = "parquet")]
@@ -113,6 +119,7 @@ mod slack_approvals;
mod smtp_server_ee;
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
mod sqs_triggers_ee;
mod static_assets;
#[cfg(all(feature = "stripe", feature = "enterprise"))]
mod stripe_ee;
@@ -209,10 +216,10 @@ pub async fn run_server(
job_index_reader: Option<IndexReader>,
log_index_reader: Option<ServiceLogIndexReader>,
addr: SocketAddr,
mut rx: tokio::sync::broadcast::Receiver<()>,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
port_tx: tokio::sync::oneshot::Sender<String>,
server_mode: bool,
#[cfg(feature = "smtp")] base_internal_url: String,
_base_internal_url: String,
) -> anyhow::Result<()> {
let user_db = UserDB::new(db.clone());
@@ -246,7 +253,10 @@ pub async fn run_server(
.layer(Extension(log_index_reader))
// .layer(Extension(index_writer))
.layer(CookieManagerLayer::new())
.layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())))
.layer(Extension(WebhookShared::new(
killpill_rx.resubscribe(),
db.clone(),
)))
.layer(DefaultBodyLimit::max(
REQUEST_SIZE_LIMIT.read().await.clone(),
));
@@ -279,7 +289,7 @@ pub async fn run_server(
db: db.clone(),
user_db: user_db,
auth_cache: auth_cache.clone(),
base_internal_url: base_internal_url.clone(),
base_internal_url: _base_internal_url.clone(),
});
if let Err(err) = smtp_server.start_listener_thread(addr).await {
tracing::error!("Error starting SMTP server: {err:#}");
@@ -340,6 +350,18 @@ pub async fn run_server(
}
};
let gcp_triggers_service = {
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
{
gcp_triggers_ee::workspaced_service()
}
#[cfg(not(all(feature = "enterprise", feature = "gcp_trigger")))]
{
Router::new()
}
};
let sqs_triggers_service = {
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
{
@@ -385,41 +407,54 @@ pub async fn run_server(
if !*CLOUD_HOSTED && server_mode {
#[cfg(feature = "websocket")]
{
let ws_killpill_rx = rx.resubscribe();
let ws_killpill_rx = killpill_rx.resubscribe();
websocket_triggers::start_websockets(db.clone(), ws_killpill_rx);
}
#[cfg(all(feature = "enterprise", feature = "kafka"))]
{
let kafka_killpill_rx = rx.resubscribe();
let kafka_killpill_rx = killpill_rx.resubscribe();
kafka_triggers_ee::start_kafka_consumers(db.clone(), kafka_killpill_rx);
}
#[cfg(all(feature = "enterprise", feature = "nats"))]
{
let nats_killpill_rx = rx.resubscribe();
let nats_killpill_rx = killpill_rx.resubscribe();
nats_triggers_ee::start_nats_consumers(db.clone(), nats_killpill_rx);
}
#[cfg(feature = "postgres_trigger")]
{
let db_killpill_rx = rx.resubscribe();
let db_killpill_rx = killpill_rx.resubscribe();
postgres_triggers::start_database(db.clone(), db_killpill_rx);
}
#[cfg(feature = "mqtt_trigger")]
{
let mqtt_killpill_rx = rx.resubscribe();
let mqtt_killpill_rx = killpill_rx.resubscribe();
mqtt_triggers::start_mqtt_consumer(db.clone(), mqtt_killpill_rx);
}
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
{
let sqs_killpill_rx = rx.resubscribe();
let sqs_killpill_rx = killpill_rx.resubscribe();
sqs_triggers_ee::start_sqs(db.clone(), sqs_killpill_rx);
}
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
{
let gcp_killpill_rx = killpill_rx.resubscribe();
gcp_triggers_ee::start_consuming_gcp_pubsub_event(db.clone(), gcp_killpill_rx);
}
}
#[cfg(feature = "agent_worker_server")]
let (agent_workers_router, agent_workers_bg_processor, agent_workers_killpill_tx) =
agent_workers_ee::workspaced_service(db.clone(), _base_internal_url.clone());
#[cfg(feature = "agent_worker_server")]
let agent_cache = Arc::new(AgentCache::new());
// build our application with a route
let app = Router::new()
.nest(
@@ -474,6 +509,7 @@ pub async fn run_server(
.nest("/nats_triggers", nats_triggers_service)
.nest("/mqtt_triggers", mqtt_triggers_service)
.nest("/sqs_triggers", sqs_triggers_service)
.nest("/gcp_triggers", gcp_triggers_service)
.nest("/postgres_triggers", postgres_triggers_service),
)
.nest("/workspaces", workspaces::global_service())
@@ -495,6 +531,26 @@ pub async fn run_server(
.nest("/ai", ai::global_service())
.route_layer(from_extractor::<ApiAuthed>())
.route_layer(from_extractor::<users::Tokened>())
.nest("/agent_workers", {
#[cfg(feature = "agent_worker_server")]
{
agent_workers_ee::global_service().layer(Extension(agent_cache.clone()))
}
#[cfg(not(feature = "agent_worker_server"))]
{
Router::new()
}
})
.nest("/w/:workspace_id/agent_workers", {
#[cfg(feature = "agent_worker_server")]
{
agent_workers_router.layer(Extension(agent_cache.clone()))
}
#[cfg(not(feature = "agent_worker_server"))]
{
Router::new()
}
})
.nest("/jobs", jobs::global_root_service())
.nest(
"/srch/w/:workspace_id/index",
@@ -604,6 +660,20 @@ pub async fn run_server(
}
.layer(from_extractor::<OptAuthed>()),
)
.nest(
"/gcp/w/:workspace_id",
{
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
{
gcp_triggers_ee::gcp_push_route_handler()
}
#[cfg(not(all(feature = "enterprise", feature = "gcp_trigger")))]
{
Router::new()
}
}
.layer(from_extractor::<OptAuthed>()),
)
.route("/version", get(git_v))
.route("/uptodate", get(is_up_to_date))
.route("/ee_license", get(ee_license))
@@ -647,11 +717,22 @@ pub async fn run_server(
.expect("Failed to send port");
let server = server.with_graceful_shutdown(async move {
rx.recv().await.ok();
killpill_rx.recv().await.ok();
#[cfg(feature = "agent_worker_server")]
if let Err(e) = agent_workers_killpill_tx.kill().await {
tracing::error!("Error killing agent workers: {e:#}");
}
tracing::info!("Graceful shutdown of server");
});
server.await?;
#[cfg(feature = "agent_worker_server")]
if let Some(bg_processor) = agent_workers_bg_processor {
tracing::info!("server off. shutting down agent workers bg processor");
bg_processor.await?;
tracing::info!("agent workers bg processor shut down");
}
Ok(())
}
backend/windmill-api/src/resources.rs
+8 -6
View File
@@ -570,7 +570,7 @@ pub async fn transform_json_value<'c>(
};
let variables = variables::get_reserved_variables(
db,
&db.into(),
workspace,
token,
&job.email,
@@ -599,11 +599,10 @@ pub async fn transform_json_value<'c>(
}
Value::Object(mut m) => {
for (a, b) in m.clone().into_iter() {
m.insert(
a.clone(),
let v =
transform_json_value(authed, user_db.clone(), db, workspace, b, job_id, token)
.await?,
);
.await?;
m.insert(a.clone(), v);
}
Ok(Value::Object(m))
}
@@ -1212,7 +1211,10 @@ async fn update_resource_type(
feature = "http_trigger",
feature = "postgres_trigger",
feature = "mqtt_trigger",
all(feature = "sqs_trigger", feature = "enterprise")
all(
feature = "enterprise",
any(feature = "sqs_trigger", feature = "gcp_trigger")
)
))]
pub async fn try_get_resource_from_db_as<T>(
authed: ApiAuthed,
backend/windmill-api/src/scripts.rs
+14 -5
View File
@@ -44,12 +44,21 @@ use windmill_audit::ActionKind;
use windmill_common::error::to_anyhow;
use windmill_common::{
db::UserDB, error::{Error, JsonResult, Result}, jobs::JobPayload, schedule::Schedule, schema::should_validate_schema, scripts::{
db::UserDB,
error::{Error, JsonResult, Result},
jobs::JobPayload,
schedule::Schedule,
schema::should_validate_schema,
scripts::{
to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Schema, Script, ScriptHash,
ScriptHistory, ScriptHistoryUpdate, ScriptKind, ScriptLang, ScriptWithStarred,
}, users::username_to_permissioned_as, utils::{
},
users::username_to_permissioned_as,
utils::{
not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
}, worker::to_raw_value, HUB_BASE_URL
},
worker::to_raw_value,
HUB_BASE_URL,
};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
use windmill_parser_ts::remove_pinned_imports;
@@ -409,10 +418,10 @@ async fn create_snapshot_script(
std::fs::create_dir_all(
windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR.clone(),
)?;
windmill_common::worker::write_file(
windmill_common::worker::write_file_bytes(
&windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR,
&hash,
&String::from_utf8_lossy(&data),
&data,
)?;
} else {
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
backend/windmill-api/src/variables.rs
+1 -1
View File
@@ -61,7 +61,7 @@ async fn list_contextual_variables(
) -> JsonResult<Vec<ContextualVariable>> {
Ok(Json(
get_reserved_variables(
&db,
&db.into(),
&w_id,
"q1A0qcPuO00yxioll7iph76N9CJDqn",
&email,
backend/windmill-api/src/workspaces.rs
+3 -1
View File
@@ -1307,6 +1307,7 @@ struct UsedTriggers {
pub postgres_used: bool,
pub mqtt_used: bool,
pub sqs_used: bool,
pub gcp_used: bool,
}
async fn get_used_triggers(
@@ -1325,7 +1326,8 @@ async fn get_used_triggers(
EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as "nats_used!",
EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS "postgres_used!",
EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS "mqtt_used!",
EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS "sqs_used!"
EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS "sqs_used!",
EXISTS(SELECT 1 FROM gcp_trigger WHERE workspace_id = $1) AS "gcp_used!"
"#,
w_id
)
backend/windmill-api/src/workspaces_export.rs
+41
View File
@@ -637,6 +637,47 @@ pub(crate) async fn tarball_workspace(
}
}
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
{
let gcp_triggers = sqlx::query_as!(
crate::gcp_triggers_ee::GcpTrigger,
r#"
SELECT
gcp_resource_path,
subscription_id,
topic_id,
workspace_id,
delivery_type AS "delivery_type: _",
delivery_config AS "delivery_config: _",
path,
script_path,
is_flow,
edited_by,
email,
edited_at,
server_id,
last_server_ping,
extra_perms,
error,
enabled
FROM
gcp_trigger
WHERE
workspace_id = $1
"#,
&w_id
)
.fetch_all(&mut *tx)
.await?;
for trigger in gcp_triggers {
let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap();
archive
.write_to_archive(&trigger_str, &format!("{}.gcp_trigger.json", trigger.path))
.await?;
}
}
#[cfg(all(feature = "enterprise", feature = "nats"))]
{
let nats_triggers = sqlx::query_as!(
backend/windmill-common/Cargo.toml
+2
View File
@@ -34,6 +34,8 @@ serde_json.workspace = true
chrono.workspace = true
chrono-tz.workspace = true
hex.workspace = true
reqwest-middleware = { workspace = true }
reqwest-retry = { workspace = true }
rand.workspace = true
sqlx = { workspace = true, features = ["postgres"] }
uuid.workspace = true
backend/windmill-common/src/agent_workers.rs
+90
View File
@@ -0,0 +1,90 @@
use serde::{Deserialize, Serialize};
#[derive(Deserialize, Serialize)]
pub struct QueueInitJob {
pub content: String,
}
use lazy_static::lazy_static;
use std::time::Duration;
use reqwest_middleware::ClientBuilder;
use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware};
use crate::{jwt::decode_without_verify, worker::HttpClient};
lazy_static! {
pub static ref BASE_INTERNAL_URL: String =
std::env::var("BASE_INTERNAL_URL").unwrap_or("http://localhost:8080".to_string());
pub static ref AGENT_TOKEN: String = std::env::var("AGENT_TOKEN").unwrap_or_default();
pub static ref DECODED_AGENT_TOKEN: Option<AgentAuth> = {
if AGENT_TOKEN.is_empty() {
None
} else {
decode_without_verify::<AgentAuth>(AGENT_TOKEN.trim_start_matches(AGENT_JWT_PREFIX))
.ok()
}
};
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct AgentAuth {
pub worker_group: String,
pub suffix: Option<String>,
pub tags: Vec<String>,
pub exp: Option<usize>,
}
pub const AGENT_JWT_PREFIX: &str = "jwt_agent_";
pub fn build_agent_http_client(worker_suffix: &str) -> HttpClient {
let client = ClientBuilder::new(
reqwest::Client::builder()
.pool_max_idle_per_host(10)
.pool_idle_timeout(Duration::from_secs(60))
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(30))
.default_headers({
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
"User-Agent", // Replace with your desired header name
"Windmill-Agent/1.0".parse().unwrap(), // Replace with your desired header value
);
let token = format!(
"{}{}_{}",
AGENT_JWT_PREFIX,
worker_suffix,
AGENT_TOKEN.trim_start_matches(AGENT_JWT_PREFIX),
);
headers.insert(
"Authorization",
format!("Bearer {}", token).parse().unwrap(),
);
headers
})
.build()
.expect("Failed to create HTTP client"),
)
.with(RetryTransientMiddleware::new_with_policy(
ExponentialBackoff::builder().build_with_max_retries(5),
))
.build();
HttpClient(client)
}
#[derive(Deserialize, Serialize)]
pub struct PingJobStatus {
pub mem_peak: Option<i32>,
pub current_mem: Option<i32>,
}
#[derive(Deserialize, Serialize, Debug)]
pub struct PingJobStatusResponse {
pub canceled_by: Option<String>,
pub canceled_reason: Option<String>,
pub already_completed: bool,
}
// #[derive(Serialize, Deserialize)]
// pub struct PullJobRequest {
// pub worker_name: String,
// }
backend/windmill-common/src/auth.rs
+63 -1
View File
@@ -1,8 +1,11 @@
use anyhow::Context;
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::{
db::Authed,
error::{Error, Result},
jwt,
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL},
DB,
};
@@ -22,7 +25,7 @@ pub struct JWTAuthClaims {
pub scopes: Option<Vec<String>>,
}
#[derive(Deserialize)]
#[derive(Deserialize, Debug)]
pub struct JobPerms {
pub email: String,
pub username: String,
@@ -205,3 +208,62 @@ pub async fn get_groups_for_user(
.collect();
Ok(groups)
}
#[tracing::instrument(level = "trace", skip_all)]
pub async fn create_token_for_owner(
db: &DB,
w_id: &str,
owner: &str,
label: &str,
expires_in: u64,
email: &str,
job_id: &Uuid,
perms: Option<JobPerms>,
) -> crate::error::Result<String> {
let job_perms = if perms.is_some() {
Ok(perms)
} else {
sqlx::query_as!(
JobPerms,
"SELECT email, username, is_admin, is_operator, groups, folders FROM job_perms WHERE job_id = $1 AND workspace_id = $2",
job_id,
w_id
)
.fetch_optional(db)
.await
};
let job_authed = match job_perms {
Ok(Some(jp)) => jp.into(),
_ => {
tracing::warn!("Could not get permissions for job {job_id} from job_perms table, getting permissions directly...");
fetch_authed_from_permissioned_as(owner.to_string(), email.to_string(), w_id, db)
.await
.map_err(|e| {
Error::internal_err(format!(
"Could not get permissions directly for job {job_id}: {e:#}"
))
})?
}
};
let payload = JWTAuthClaims {
email: job_authed.email,
username: job_authed.username,
is_admin: job_authed.is_admin,
is_operator: job_authed.is_operator,
groups: job_authed.groups,
folders: job_authed.folders,
label: Some(label.to_string()),
workspace_id: w_id.to_string(),
exp: (chrono::Utc::now() + chrono::Duration::seconds(expires_in as i64)).timestamp()
as usize,
job_id: Some(job_id.to_string()),
scopes: None,
};
let token = jwt::encode_with_internal_secret(&payload)
.await
.with_context(|| format!("Could not encode JWT token for job {job_id}"))?;
Ok(format!("jwt_{}", token))
}
backend/windmill-common/src/cache.rs
+182 -116
View File
@@ -7,7 +7,6 @@
//! This shall only be used for testing, e.g. [`sqlx::test`] spawn a database per test,
//! and there is only one test per thread, so using thread-local cache avoid unexpected results.
use anyhow::anyhow;
use crate::{
apps::AppScriptId,
error,
@@ -15,6 +14,7 @@ use crate::{
schema::SchemaValidator,
scripts::{ScriptHash, ScriptLang},
};
use anyhow::anyhow;
#[cfg(feature = "scoped_cache")]
use std::thread::ThreadId;
@@ -322,6 +322,25 @@ pub struct RawScript {
pub meta: Option<ScriptMetadata>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct RawScriptApi {
pub content: String,
pub lock: Option<String>,
pub meta: Option<ScriptMetadata>,
}
impl From<RawScript> for RawScriptApi {
fn from(value: RawScript) -> Self {
RawScriptApi { content: value.content, lock: value.lock, meta: value.meta }
}
}
impl From<RawScriptApi> for RawScript {
fn from(value: RawScriptApi) -> Self {
RawScript { content: value.content, lock: value.lock, meta: value.meta }
}
}
#[derive(Debug)]
pub struct RawFlow {
pub raw_flow: Box<RawValue>,
@@ -334,6 +353,25 @@ pub struct RawNode {
pub raw_flow: Option<Box<RawValue>>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct RawNodeApi {
pub raw_code: Option<String>,
pub raw_lock: Option<String>,
pub raw_flow: Option<Box<RawValue>>,
}
impl From<RawNode> for RawNodeApi {
fn from(value: RawNode) -> Self {
RawNodeApi { raw_code: value.raw_code, raw_lock: value.raw_lock, raw_flow: value.raw_flow }
}
}
impl From<RawNodeApi> for RawNode {
fn from(value: RawNodeApi) -> Self {
RawNode { raw_code: value.raw_code, raw_lock: value.raw_lock, raw_flow: value.raw_flow }
}
}
#[derive(Debug, Clone)]
struct Entry<T>(Arc<T>);
@@ -343,7 +381,7 @@ struct ScriptFull {
pub meta: Arc<ScriptMetadata>,
}
fn unwrap_or_error<Key: std::fmt::Debug, Val>(
pub fn unwrap_or_error<Key: std::fmt::Debug, Val>(
at: &'static Location,
entity: &'static str,
key: Key,
@@ -363,6 +401,11 @@ pub fn clear() {
}
pub mod flow {
use crate::{
worker::{fetch_flow_node_query, Connection},
DB,
};
use super::*;
make_static! {
@@ -388,10 +431,10 @@ pub mod flow {
/// This should be preferred over fetching the database directly.
#[track_caller]
pub fn fetch_script<'c>(
e: impl PgExecutor<'c>,
conn: &'c Connection,
node: FlowNodeId,
) -> impl Future<Output = error::Result<Arc<ScriptData>>> {
let fetch_node = fetch_node(e, node);
) -> impl Future<Output = error::Result<Arc<ScriptData>>> + 'c {
let fetch_node = fetch_node(conn, node);
async move {
fetch_node.await.and_then(|data| match data {
RawData::Script(data) => Ok(data),
@@ -409,11 +452,12 @@ pub mod flow {
/// This should be preferred over fetching the database directly.
#[track_caller]
pub fn fetch_flow<'c>(
e: impl PgExecutor<'c>,
db: &'c DB,
node: FlowNodeId,
) -> impl Future<Output = error::Result<Arc<FlowData>>> {
let fetch_node = fetch_node(e, node);
) -> impl Future<Output = error::Result<Arc<FlowData>>> + 'c {
async move {
let conn = Connection::Sql(db.clone());
let fetch_node = fetch_node(&conn, node);
fetch_node.await.and_then(|data| match data {
RawData::Flow(data) => Ok(data),
RawData::Script(_) => Err(error::Error::internal_err(format!(
@@ -430,31 +474,23 @@ pub mod flow {
/// This should be preferred over fetching the database directly.
#[track_caller]
pub(super) fn fetch_node<'c>(
e: impl PgExecutor<'c>,
conn: &'c Connection,
node: FlowNodeId,
) -> impl Future<Output = error::Result<RawData>> {
) -> impl Future<Output = error::Result<RawData>> + 'c {
let loc = Location::caller();
// If not present, `get_or_insert_async` will lock the key until the future completes,
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
NODES.get_or_insert_async(node, async move {
sqlx::query!(
"SELECT \
code AS \"raw_code: String\", \
lock AS \"raw_lock: String\", \
flow AS \"raw_flow: Json<Box<RawValue>>\" \
FROM flow_node WHERE id = $1 LIMIT 1",
node.0,
)
.fetch_optional(e)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Flow node", node))
.map(|r| RawNode {
raw_code: r.raw_code,
raw_lock: r.raw_lock,
raw_flow: r.raw_flow.map(|Json(raw_flow)| raw_flow),
})
match conn {
Connection::Sql(db) => fetch_flow_node_query(db, node.0, loc).await,
Connection::Http(client) => {
let r = client
.get::<RawNodeApi>(&format!("/api/agent_workers/flow_script/{}", node.0))
.await?;
Ok(r.into())
}
}
})
}
@@ -502,6 +538,8 @@ pub mod flow {
}
pub mod script {
use crate::{worker::Connection, DB};
use super::*;
make_static! {
@@ -520,66 +558,80 @@ pub mod script {
/// it to the file system and cache.
/// This should be preferred over fetching the database directly.
#[track_caller]
pub fn fetch<'c>(
e: impl PgExecutor<'c>,
pub fn fetch(
conn: &Connection,
hash: ScriptHash,
) -> impl Future<Output = error::Result<(Arc<ScriptData>, Arc<ScriptMetadata>)>> {
// If not present, `get_or_insert_async` will lock the key until the future completes,
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
let loc = Location::caller();
let conn = conn.clone();
let fut = CACHE.get_or_insert_async(hash, async move {
sqlx::query!(
"SELECT \
content AS \"content!: String\",
lock AS \"lock: String\", \
language AS \"language: Option<ScriptLang>\", \
envs AS \"envs: Vec<String>\", \
schema AS \"schema: String\", \
schema_validation AS \"schema_validation: bool\", \
codebase LIKE '%.tar' as use_tar \
FROM script WHERE hash = $1 LIMIT 1",
hash.0
)
.fetch_optional(e)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Script", hash))
.and_then(|r| {
Ok(RawScript {
content: r.content,
lock: r.lock,
meta: Some(ScriptMetadata {
language: r.language,
envs: r.envs,
codebase: if let Some(use_tar) = r.use_tar {
let sh = hash.to_string();
if use_tar {
Some(format!("{sh}.tar"))
} else {
Some(sh)
}
} else {
None
},
schema_validator: if r.schema_validation {
r.schema
.as_ref()
.map(|schema_str| {
SchemaValidator::from_schema(schema_str).map_err(|e| anyhow!("Couldn't create schema validator for script requiring schema validation: {e}"))
})
.transpose()?
} else {
None
},
schema: r.schema,
}),
})
})
match conn {
Connection::Sql(db) => fetch_script_from_db(&db, hash, loc).await,
Connection::Http(_) => Err(error::Error::InternalErr(format!(
"Cannot fetch script in HTTP mode"
))),
}
});
fut.map_ok(|ScriptFull { data, meta }| (data, meta))
}
async fn fetch_script_from_db(
db: &DB,
hash: ScriptHash,
loc: &'static Location<'_>,
) -> error::Result<RawScript> {
sqlx::query!(
"SELECT \
content AS \"content!: String\",
lock AS \"lock: String\", \
language AS \"language: Option<ScriptLang>\", \
envs AS \"envs: Vec<String>\", \
schema AS \"schema: String\", \
schema_validation AS \"schema_validation: bool\", \
codebase LIKE '%.tar' as use_tar \
FROM script WHERE hash = $1 LIMIT 1",
hash.0
)
.fetch_optional(db)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Script", hash))
.and_then(|r| {
Ok(RawScript {
content: r.content,
lock: r.lock,
meta: Some(ScriptMetadata {
language: r.language,
envs: r.envs,
codebase: if let Some(use_tar) = r.use_tar {
let sh = hash.to_string();
if use_tar {
Some(format!("{sh}.tar"))
} else {
Some(sh)
}
} else {
None
},
schema_validator: if r.schema_validation {
r.schema
.as_ref()
.map(|schema_str| {
SchemaValidator::from_schema(schema_str).map_err(|e| anyhow!("Couldn't create schema validator for script requiring schema validation: {e}"))
})
.transpose()?
} else {
None
},
schema: r.schema,
}),
})
})
}
/// Invalidate the script cache for the given `hash`.
pub fn invalidate(hash: ScriptHash) {
let _ = CACHE.remove(&hash);
@@ -587,6 +639,8 @@ pub mod script {
}
pub mod app {
use crate::worker::{fetch_raw_script_from_app_query, Connection};
use super::*;
make_static! {
@@ -605,23 +659,23 @@ pub mod app {
/// This should be preferred over fetching the database directly.
#[track_caller]
pub fn fetch_script<'c>(
e: impl PgExecutor<'c>,
conn: &'c Connection,
id: AppScriptId,
) -> impl Future<Output = error::Result<Arc<ScriptData>>> {
) -> impl Future<Output = error::Result<Arc<ScriptData>>> + 'c {
// If not present, `get_or_insert_async` will lock the key until the future completes,
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
let loc = Location::caller();
let fut = CACHE.get_or_insert_async(id, async move {
sqlx::query!(
"SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1",
id.0,
)
.fetch_optional(e)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Application script", id))
.map(|r| RawScript { content: r.code, lock: r.lock, meta: None })
match conn {
Connection::Sql(db) => fetch_raw_script_from_app_query(db, id.0, loc).await,
Connection::Http(client) => {
let r = client
.get::<RawScriptApi>(&format!("/api/agent_workers/app_script/{}", id.0))
.await?;
Ok(r.into())
}
}
});
fut.map_ok(|Entry(data)| data)
}
@@ -629,7 +683,7 @@ pub mod app {
pub mod job {
use super::*;
use crate::jobs::JobKind;
use crate::{jobs::JobKind, worker::Connection, DB};
#[cfg(not(feature = "scoped_cache"))]
lazy_static! {
@@ -649,15 +703,18 @@ pub mod job {
}
#[track_caller]
pub fn fetch_preview_flow<'a, 'c>(
e: impl PgExecutor<'c> + 'a,
pub fn fetch_preview_flow<'a>(
db: &'a DB,
job: &'a Uuid,
// original raw values from `queue` or `completed_job` tables:
// kept for backward compatibility.
raw_flow: Option<Json<Box<RawValue>>>,
) -> impl Future<Output = error::Result<Arc<FlowData>>> + 'a {
let fetch_preview = fetch_preview(e, job, None, None, raw_flow);
// Create the Connection first so it lives for the entire scope
async move {
let conn = Connection::from(db);
let fetch_preview = fetch_preview(&conn, job, None, None, raw_flow);
fetch_preview.await.and_then(|data| match data {
RawData::Flow(data) => Ok(data),
RawData::Script(_) => Err(error::Error::internal_err(format!(
@@ -669,7 +726,7 @@ pub mod job {
#[track_caller]
pub fn fetch_preview_script<'a, 'c>(
e: impl PgExecutor<'c> + 'a,
e: &'a Connection,
job: &'a Uuid,
// original raw values from `queue` or `completed_job` tables:
// kept for backward compatibility.
@@ -689,7 +746,7 @@ pub mod job {
#[track_caller]
pub fn fetch_preview<'a, 'c>(
e: impl PgExecutor<'c> + 'a,
e: &'a Connection,
job: &'a Uuid,
// original raw values from `queue` or `completed_job` tables:
// kept for backward compatibility.
@@ -700,16 +757,21 @@ pub mod job {
let loc = Location::caller();
let fetch = async move {
match (raw_lock, raw_code, raw_flow) {
(None, None, None) => sqlx::query!(
"SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json<Box<RawValue>>\" \
FROM v2_job WHERE id = $1 LIMIT 1",
job
)
.fetch_optional(e)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Preview", job))
.map(|r| (r.raw_lock, r.raw_code, r.raw_flow)),
(None, None, None) => match e {
Connection::Sql(pool) => sqlx::query!(
"SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json<Box<RawValue>>\" \
FROM v2_job WHERE id = $1 LIMIT 1",
job
)
.fetch_optional(pool)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Preview", job))
.map(|r| (r.raw_lock, r.raw_code, r.raw_flow)),
Connection::Http(_) => Err(error::Error::InternalErr(format!(
"Cannot fetch preview in HTTP mode"
))),
},
(lock, code, flow) => Ok((lock, code, flow)),
}
.and_then(|(lock, code, flow)| match flow {
@@ -730,8 +792,8 @@ pub mod job {
}
#[track_caller]
pub fn fetch_script<'c>(
e: impl PgExecutor<'c>,
pub fn fetch_script(
db: DB,
kind: JobKind,
hash: Option<ScriptHash>,
) -> impl Future<Output = error::Result<Arc<ScriptData>>> {
@@ -739,11 +801,15 @@ pub mod job {
let loc = Location::caller();
async move {
match (kind, hash.map(|ScriptHash(id)| id)) {
(FlowScript, Some(id)) => flow::fetch_script(e, FlowNodeId(id)).await,
(Script | Dependencies, Some(hash)) => script::fetch(e, ScriptHash(hash))
(FlowScript, Some(id)) => {
flow::fetch_script(&Connection::Sql(db.clone()), FlowNodeId(id)).await
}
(Script | Dependencies, Some(hash)) => script::fetch(&db.into(), ScriptHash(hash))
.await
.map(|(data, _meta)| data),
(AppScript, Some(id)) => app::fetch_script(e, AppScriptId(id)).await,
(AppScript, Some(id)) => {
app::fetch_script(&Connection::Sql(db.clone()), AppScriptId(id)).await
}
_ => Err(error::Error::internal_err(format!(
"Isn't a script job: {:?}",
kind
@@ -755,19 +821,19 @@ pub mod job {
#[track_caller]
pub fn fetch_flow<'c>(
e: impl PgExecutor<'c> + Copy,
db: &'c DB,
kind: JobKind,
hash: Option<ScriptHash>,
) -> impl Future<Output = error::Result<Arc<FlowData>>> {
) -> impl Future<Output = error::Result<Arc<FlowData>>> + 'c {
use JobKind::*;
let loc = Location::caller();
async move {
match (kind, hash.map(|ScriptHash(id)| id)) {
(FlowDependencies, Some(id)) => flow::fetch_version(e, id).await,
(FlowNode, Some(id)) => flow::fetch_flow(e, FlowNodeId(id)).await,
(Flow, Some(id)) => match flow::fetch_version_lite(e, id).await {
(FlowDependencies, Some(id)) => flow::fetch_version(db, id).await,
(FlowNode, Some(id)) => flow::fetch_flow(db, FlowNodeId(id)).await,
(Flow, Some(id)) => match flow::fetch_version_lite(db, id).await {
Ok(raw_flow) => Ok(raw_flow),
Err(_) => flow::fetch_version(e, id).await,
Err(_) => flow::fetch_version(db, id).await,
},
_ => Err(error::Error::internal_err(format!(
"Isn't a flow job {:?}",
backend/windmill-common/src/error.rs
+6
View File
@@ -22,6 +22,8 @@ pub type JsonResult<T> = std::result::Result<Json<T>, Error>;
#[derive(Debug, Error)]
pub enum Error {
#[error("Bad gateway: {0}")]
BadGateway(String),
#[error("Bad config: {0}")]
BadConfig(String),
#[error("Connecting to database: {0}")]
@@ -76,6 +78,8 @@ pub enum Error {
FindPythonError(String),
#[error("Problem with arguments: {0}")]
ArgumentErr(String),
#[error("{1}")]
Generic(StatusCode, String),
}
fn prettify_location(location: &'static Location<'static>) -> String {
@@ -181,6 +185,8 @@ impl IntoResponse for Error {
| Self::BadRequest(_)
| Self::AIError(_)
| Self::QuotaExceeded(_) => axum::http::StatusCode::BAD_REQUEST,
Self::BadGateway(_) => axum::http::StatusCode::BAD_GATEWAY,
Self::Generic(status_code, _) => status_code,
_ => axum::http::StatusCode::INTERNAL_SERVER_ERROR,
};
backend/windmill-common/src/flows.rs
+8 -8
View File
@@ -22,7 +22,7 @@ use crate::{
error::Error,
more_serde::{default_empty_string, default_id, default_null, default_true, is_default},
scripts::{Schema, ScriptHash, ScriptLang},
worker::to_raw_value,
worker::{to_raw_value, Connection}, DB,
};
#[derive(Serialize, Deserialize, sqlx::FromRow)]
@@ -731,7 +731,7 @@ pub async fn resolve_maybe_value<T>(
}
/// Resolve modules recursively.
pub async fn resolve_value(
async fn resolve_value(
e: &sqlx::PgPool,
workspace_id: &str,
value: &mut Box<JsonRawValue>,
@@ -749,7 +749,7 @@ pub async fn resolve_value(
/// Resolve module value recursively.
pub async fn resolve_module(
e: &sqlx::PgPool,
db: &DB,
workspace_id: &str,
value: &mut Box<JsonRawValue>,
with_code: bool,
@@ -783,7 +783,7 @@ pub async fn resolve_module(
let (lock, content) = if !with_code {
(Some("...".to_string()), "...".to_string())
} else {
cache::flow::fetch_script(e, id)
cache::flow::fetch_script(&Connection::Sql(db.clone()), id)
.await
.map(|data| (data.lock.clone(), data.code.clone()))?
};
@@ -801,13 +801,13 @@ pub async fn resolve_module(
};
}
ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => {
resolve_modules(e, workspace_id, modules, modules_node.take(), with_code).await?;
resolve_modules(db, workspace_id, modules, modules_node.take(), with_code).await?;
}
BranchOne { branches, default, default_node } => {
resolve_modules(e, workspace_id, default, default_node.take(), with_code).await?;
resolve_modules(db, workspace_id, default, default_node.take(), with_code).await?;
for branch in branches {
resolve_modules(
e,
db,
workspace_id,
&mut branch.modules,
branch.modules_node.take(),
@@ -819,7 +819,7 @@ pub async fn resolve_module(
BranchAll { branches, .. } => {
for branch in branches {
resolve_modules(
e,
db,
workspace_id,
&mut branch.modules,
branch.modules_node.take(),
backend/windmill-common/src/jobs.rs
+9 -1
View File
@@ -10,6 +10,7 @@ use tokio::io::AsyncReadExt;
use uuid::Uuid;
pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE";
pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000;
use crate::{
apps::AppScriptId,
@@ -24,7 +25,7 @@ use crate::{
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)]
#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
#[serde(rename_all(serialize = "lowercase"))]
#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
pub enum JobKind {
Script,
#[allow(non_camel_case_types)]
@@ -51,6 +52,13 @@ impl JobKind {
JobKind::Flow | JobKind::FlowPreview | JobKind::SingleScriptFlow | JobKind::FlowNode
)
}
pub fn is_dependency(&self) -> bool {
matches!(
self,
JobKind::FlowDependencies | JobKind::AppDependencies | JobKind::Dependencies
)
}
}
#[derive(sqlx::FromRow, Debug, Serialize, Clone)]
backend/windmill-common/src/jwt.rs
+18 -1
View File
@@ -1,6 +1,6 @@
use crate::error::{self, to_anyhow, Error};
use serde::{de::DeserializeOwned, Serialize};
use std::sync::Arc;
use std::{collections::HashSet, sync::Arc};
use tokio::sync::RwLock;
lazy_static::lazy_static! {
@@ -40,3 +40,20 @@ pub async fn decode_with_internal_secret<T: DeserializeOwned>(token: &str) -> er
Ok(result.claims)
}
pub fn decode_without_verify<T: DeserializeOwned>(token: &str) -> anyhow::Result<T> {
// Create a validation that skips all checks
let mut validation = jsonwebtoken::Validation::default();
validation.insecure_disable_signature_validation();
validation.validate_exp = false;
validation.validate_nbf = false;
validation.required_spec_claims = HashSet::new();
// Use an empty key since we're not verifying
let key = jsonwebtoken::DecodingKey::from_secret(&[]);
// Decode the token
let token_data = jsonwebtoken::decode::<T>(token, &key, &validation)?;
Ok(token_data.claims)
}
backend/windmill-common/src/lib.rs
+2 -1
View File
@@ -22,6 +22,7 @@ use error::Error;
use scripts::ScriptLang;
use sqlx::{Pool, Postgres};
pub mod agent_workers;
pub mod apps;
pub mod auth;
pub mod cache;
@@ -45,6 +46,7 @@ pub mod otel_ee;
pub mod queue;
pub mod s3_helpers;
pub mod schedule;
pub mod schema;
pub mod scripts;
pub mod server;
pub mod stats_ee;
@@ -55,7 +57,6 @@ pub mod utils;
pub mod variables;
pub mod worker;
pub mod workspaces;
pub mod schema;
pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5;
backend/windmill-common/src/utils.rs
+35 -3
View File
@@ -23,7 +23,7 @@ use rand::distr::Alphanumeric;
use rand::{rng, Rng};
use reqwest::Client;
use semver::Version;
use serde::{Deserialize, Serialize};
use serde::{Deserialize, Deserializer, Serialize};
use sha2::{Digest, Sha256};
use sqlx::{Pool, Postgres};
use std::str::FromStr;
@@ -75,8 +75,8 @@ lazy_static::lazy_static! {
if std::env::var("BASE_INTERNAL_URL").is_err() {
panic!("BASE_INTERNAL_URL is required in agent mode")
}
if std::env::var("JOB_TOKEN").is_err() {
println!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table")
if std::env::var("AGENT_TOKEN").is_err() {
println!("AGENT_TOKEN is not passed. This is required for the agent to work and contains the JWT to authenticate with the server.")
}
#[cfg(not(feature = "enterprise"))]
@@ -175,6 +175,28 @@ pub fn hostname() -> String {
})
}
fn instance_name(hostname: &str) -> String {
hostname
.replace(" ", "")
.split("-")
.last()
.unwrap()
.to_ascii_lowercase()
.to_string()
}
pub fn worker_suffix(hostname: &str, rd_string: &str) -> String {
format!("{}-{}", instance_name(hostname), rd_string)
}
pub fn worker_name_with_suffix(is_agent: bool, worker_group: &str, suffix: &str) -> String {
if is_agent {
format!("ag-{}-{}", worker_group, suffix)
} else {
format!("wk-{}-{}", worker_group, suffix)
}
}
pub fn paginate(pagination: Pagination) -> (usize, usize) {
let per_page = pagination
.per_page
@@ -446,6 +468,16 @@ pub async fn report_recovered_critical_error(
}
}
pub fn empty_string_as_none<'de, D>(
deserializer: D,
) -> std::result::Result<Option<String>, D::Error>
where
D: Deserializer<'de>,
{
let option = <Option<String> as serde::Deserialize>::deserialize(deserializer)?;
Ok(option.filter(|s| !s.is_empty()))
}
pub async fn fetch_mute_workspace(_db: &DB, workspace_id: &str) -> Result<bool> {
match sqlx::query!(
"SELECT mute_critical_alerts FROM workspace_settings WHERE workspace_id = $1",
backend/windmill-common/src/variables.rs
+17 -13
View File
@@ -7,6 +7,7 @@
*/
use crate::error;
use crate::worker::Connection;
use crate::{worker::WORKER_GROUP, BASE_URL, DB};
use chrono::{SecondsFormat, Utc};
use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait};
@@ -166,7 +167,7 @@ lazy_static::lazy_static! {
}
pub async fn get_reserved_variables(
db: &DB,
conn: &Connection,
w_id: &str,
token: &str,
email: &str,
@@ -206,7 +207,7 @@ pub async fn get_reserved_variables(
}
};
let custom_envs = get_cached_workspace_envs(db, w_id).await;
let custom_envs = get_cached_workspace_envs(conn, w_id).await;
let joined_schedule_path = schedule_path
.clone()
@@ -354,10 +355,7 @@ pub async fn get_reserved_variables(
).collect()
}
async fn get_cached_workspace_envs(
db: &sqlx::Pool<sqlx::Postgres>,
w_id: &str,
) -> Vec<(String, String)> {
async fn get_cached_workspace_envs(conn: &Connection, w_id: &str) -> Vec<(String, String)> {
let cached_envs_o = CUSTOM_ENVS_CACHE.get(w_id).and_then(|(ts, envs)| {
if ts > chrono::Utc::now().timestamp() - (60 * 15) {
Some(envs)
@@ -369,13 +367,19 @@ async fn get_cached_workspace_envs(
let custom_envs = if let Some(cached_envs) = cached_envs_o {
cached_envs
} else {
let custom_envs = sqlx::query_as::<_, (String, String)>(
"SELECT name, value FROM workspace_env WHERE workspace_id = $1",
)
.bind(w_id)
.fetch_all(db)
.await
.unwrap_or_default();
let custom_envs = match conn {
Connection::Sql(db) => sqlx::query_as::<_, (String, String)>(
"SELECT name, value FROM workspace_env WHERE workspace_id = $1",
)
.bind(w_id)
.fetch_all(db)
.await
.unwrap_or_default(),
Connection::Http(client) => client
.get(&format!("/api/w/{w_id}/agent_workers/custom_envs"))
.await
.unwrap_or_default(),
};
CUSTOM_ENVS_CACHE.insert(
w_id.to_string(),
(chrono::Utc::now().timestamp(), custom_envs.clone()),
backend/windmill-common/src/worker.rs
+484 -54
View File
@@ -3,28 +3,55 @@ use bytes::Bytes;
use const_format::concatcp;
use itertools::Itertools;
use regex::Regex;
use reqwest_middleware::ClientWithMiddleware;
use semver::Version;
use serde::{Deserialize, Serialize};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde_json::value::RawValue;
use sqlx::{types::Json, Pool, Postgres};
use std::{
cmp::Reverse,
collections::{HashMap, HashSet},
fs::{self, File},
io::Write,
panic::Location,
path::{Component, Path, PathBuf},
str::FromStr,
sync::{atomic::AtomicBool, Arc},
};
use tokio::sync::RwLock;
use uuid::Uuid;
use windmill_macros::annotations;
use crate::{
error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp,
agent_workers::{PingJobStatusResponse, BASE_INTERNAL_URL},
cache::{unwrap_or_error, RawNode, RawScript},
error::{self, to_anyhow},
global_settings::CUSTOM_TAGS_SETTING,
indexer::TantivyIndexerSettings,
server::Smtp,
KillpillSender, DB,
};
pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900;
pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days
lazy_static::lazy_static! {
pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string());
pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| {
#[cfg(not(feature = "enterprise"))]
{
"default".to_string()
}
#[cfg(feature = "enterprise")]
{
if let Some(token) = crate::agent_workers::DECODED_AGENT_TOKEN.as_ref() {
token.worker_group.clone()
} else {
"default".to_string()
}
}
});
pub static ref NO_LOGS: bool = std::env::var("NO_LOGS").ok().is_some_and(|x| x == "1" || x == "true");
pub static ref CGROUP_V2_PATH_RE: Regex = Regex::new(r#"(?m)^0::(/.*)$"#).unwrap();
@@ -61,6 +88,15 @@ lazy_static::lazy_static! {
pub static ref DEFAULT_TAGS_PER_WORKSPACE: AtomicBool = AtomicBool::new(false);
pub static ref DEFAULT_TAGS_WORKSPACES: Arc<RwLock<Option<Vec<String>>>> = Arc::new(RwLock::new(None));
pub static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT")
.ok()
.and_then(|x| x.parse::<u64>().ok())
.unwrap_or_else(|| if *CLOUD_HOSTED { DEFAULT_CLOUD_TIMEOUT } else { DEFAULT_SELFHOSTED_TIMEOUT });
pub static ref SCRIPT_TOKEN_EXPIRY: u64 = std::env::var("SCRIPT_TOKEN_EXPIRY")
.ok()
.and_then(|x| x.parse::<u64>().ok())
.unwrap_or(*MAX_TIMEOUT);
pub static ref WORKER_CONFIG: Arc<RwLock<WorkerConfig>> = Arc::new(RwLock::new(WorkerConfig {
worker_tags: Default::default(),
@@ -115,6 +151,90 @@ pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/");
pub static MIN_VERSION_IS_LATEST: AtomicBool = AtomicBool::new(false);
#[derive(Clone)]
pub struct HttpClient(pub ClientWithMiddleware);
impl HttpClient {
pub async fn post<T: Serialize, R: DeserializeOwned>(
&self,
url: &str,
body: &T,
) -> anyhow::Result<R> {
let response = self
.0
.post(format!("{}{}", *BASE_INTERNAL_URL, url))
.json(body)
.send()
.await
.map_err(|e| anyhow::anyhow!(e))?;
let status = response.status();
if status.is_success() {
Ok(response.json().await?)
} else {
Err(anyhow::anyhow!(format!(
"HTTP agent request POST {} failed {}",
url,
response.status()
)))
}
}
pub async fn get<R: DeserializeOwned>(&self, url: &str) -> anyhow::Result<R> {
let response = self
.0
.get(format!("{}{}", *BASE_INTERNAL_URL, url))
.send()
.await
.map_err(|e| anyhow::anyhow!(e))?;
let status = response.status();
if status.is_success() {
Ok(response.json().await?)
} else {
Err(anyhow::anyhow!(format!(
"HTTP agent request GET {} failed {}",
url,
response.status()
)))
}
}
}
#[derive(Clone)]
pub enum Connection {
Sql(Pool<Postgres>),
Http(HttpClient),
}
impl std::fmt::Debug for Connection {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Connection::Sql(_) => write!(f, "Sql"),
Connection::Http(_) => write!(f, "Http"),
}
}
}
impl Connection {
pub fn as_sql(&self) -> Option<&Pool<Postgres>> {
match self {
Connection::Sql(db) => Some(db),
Connection::Http(_) => None,
}
}
}
impl From<Pool<Postgres>> for Connection {
fn from(value: Pool<Postgres>) -> Self {
Connection::Sql(value)
}
}
impl From<&Pool<Postgres>> for Connection {
fn from(value: &Pool<Postgres>) -> Self {
Connection::Sql(value.clone())
}
}
fn format_pull_query(peek: String) -> String {
let r = format!(
"WITH peek AS (
@@ -164,40 +284,48 @@ fn format_pull_query(peek: String) -> String {
r
}
pub async fn make_suspended_pull_query(wc: &WorkerConfig) {
if wc.worker_tags.len() == 0 {
tracing::error!("Empty tags in worker tags, skipping");
return;
}
let query = format_pull_query(format!(
pub fn make_suspended_pull_query(tags: &[String]) -> String {
format_pull_query(format!(
"SELECT id
FROM v2_job_queue
WHERE suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()) AND tag IN ({})
ORDER BY priority DESC NULLS LAST, created_at
FOR UPDATE SKIP LOCKED
LIMIT 1",
wc.worker_tags.iter().map(|x| format!("'{x}'")).join(", ")
));
tags.iter().map(|x| format!("'{x}'")).join(", ")
))
}
// pub async fn make_suspended
pub async fn store_suspended_pull_query(wc: &WorkerConfig) {
if wc.worker_tags.len() == 0 {
tracing::error!("Empty tags in worker tags, skipping");
return;
}
let query = make_suspended_pull_query(&wc.worker_tags);
let mut l = WORKER_SUSPENDED_PULL_QUERY.write().await;
*l = query;
}
pub async fn make_pull_query(wc: &WorkerConfig) {
pub fn make_pull_query(tags: &[String]) -> String {
format_pull_query(format!(
"SELECT id
FROM v2_job_queue
WHERE running = false AND tag IN ({}) AND scheduled_for <= now()
ORDER BY priority DESC NULLS LAST, scheduled_for
FOR UPDATE SKIP LOCKED
LIMIT 1",
tags.iter().map(|x| format!("'{x}'")).join(", ")
))
}
pub async fn store_pull_query(wc: &WorkerConfig) {
let mut queries = vec![];
for tags in wc.priority_tags_sorted.iter() {
if tags.tags.len() == 0 {
tracing::error!("Empty tags in priority tags, skipping");
continue;
}
let query = format_pull_query(format!(
"SELECT id
FROM v2_job_queue
WHERE running = false AND tag IN ({}) AND scheduled_for <= now()
ORDER BY priority DESC NULLS LAST, scheduled_for
FOR UPDATE SKIP LOCKED
LIMIT 1",
tags.tags.iter().map(|x| format!("'{x}'")).join(", ")
));
let query = make_pull_query(&tags.tags);
queries.push(query);
}
let mut l = WORKER_PULL_QUERIES.write().await;
@@ -758,24 +886,33 @@ pub fn get_windmill_memory_usage() -> Option<i64> {
}
}
pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>(
executor: E,
) -> bool {
pub async fn update_min_version(conn: &Connection) -> bool {
use crate::utils::{GIT_SEM_VERSION, GIT_VERSION};
// fetch all pings with a different version than self from the last 5 minutes.
let pings = sqlx::query_scalar!(
"SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'",
GIT_VERSION
).fetch_all(executor).await.unwrap_or_default();
let cur_version = GIT_SEM_VERSION.clone();
let min_version = pings
.iter()
.filter(|x| !x.is_empty())
.filter_map(|x| semver::Version::parse(if x.starts_with('v') { &x[1..] } else { x }).ok())
.min()
.unwrap_or_else(|| cur_version.clone());
let min_version = match conn {
Connection::Sql(pool) => {
// fetch all pings with a different version than self from the last 5 minutes.
let pings = sqlx::query_scalar!(
"SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'",
GIT_VERSION
).fetch_all(pool).await.unwrap_or_default();
pings
.iter()
.filter(|x| !x.is_empty())
.filter_map(|x| {
semver::Version::parse(if x.starts_with('v') { &x[1..] } else { x }).ok()
})
.min()
.unwrap_or_else(|| cur_version.clone())
}
Connection::Http(_) => {
// TODO: get min version from server, for now we use the current version. Min version should be of no interest for http mode workers
cur_version.clone()
}
};
if min_version != cur_version {
tracing::info!("Minimal worker version: {min_version}");
@@ -790,37 +927,330 @@ pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postg
min_version >= cur_version
}
pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: &DB) {
let (tags, dw) = {
let wc = WORKER_CONFIG.read().await.clone();
(
wc.worker_tags,
wc.dedicated_worker
.as_ref()
.map(|x| format!("{}:{}", x.workspace_id, x.path)),
)
};
#[derive(Serialize, Deserialize)]
pub enum PingType {
Initial,
MainLoop,
Job,
InitScript,
}
#[derive(Serialize, Deserialize)]
pub struct Ping {
pub last_job_executed: Option<Uuid>,
pub last_job_workspace_id: Option<String>,
pub worker_instance: Option<String>,
pub ip: Option<String>,
pub tags: Option<Vec<String>>,
pub dw: Option<String>,
pub version: Option<String>,
pub vcpus: Option<i64>,
pub memory: Option<i64>,
pub memory_usage: Option<i64>,
pub wm_memory_usage: Option<i64>,
pub jobs_executed: Option<i32>,
pub occupancy_rate: Option<f32>,
pub occupancy_rate_15s: Option<f32>,
pub occupancy_rate_5m: Option<f32>,
pub occupancy_rate_30m: Option<f32>,
pub ping_type: PingType,
}
pub async fn update_ping_http(
insert_ping: Ping,
worker_name: &str,
worker_group: &str,
db: &DB,
) -> anyhow::Result<()> {
// tracing::info!("update ping: {}", insert_ping.tags.join(","));
match insert_ping.ping_type {
PingType::MainLoop => {
update_worker_ping_main_loop_query(
worker_name,
insert_ping.tags.unwrap_or_default().as_slice(),
insert_ping.vcpus,
insert_ping.memory,
insert_ping.jobs_executed,
insert_ping.occupancy_rate,
insert_ping.memory_usage,
insert_ping.wm_memory_usage,
insert_ping.occupancy_rate_15s,
insert_ping.occupancy_rate_5m,
insert_ping.occupancy_rate_30m,
db,
)
.await?
}
PingType::Initial => {
if insert_ping.worker_instance.is_none()
|| insert_ping.version.is_none()
|| insert_ping.ip.is_none()
{
return Err(anyhow::anyhow!(
"Worker instance, version and ip are required"
));
}
let vcpus = get_vcpus();
let memory = get_memory();
insert_ping_query(
&insert_ping.worker_instance.unwrap(),
&worker_name,
worker_group,
&insert_ping.ip.unwrap(),
insert_ping.tags.unwrap_or_default().as_slice(),
insert_ping.dw,
&insert_ping.version.unwrap(),
insert_ping.vcpus,
insert_ping.memory,
db,
)
.await?;
}
PingType::Job => {
update_worker_ping_from_job_query(
&insert_ping.last_job_executed.unwrap_or_default(),
&insert_ping.last_job_workspace_id.unwrap_or_default(),
worker_name,
insert_ping.memory_usage,
insert_ping.wm_memory_usage,
insert_ping.occupancy_rate,
insert_ping.occupancy_rate_15s,
insert_ping.occupancy_rate_5m,
insert_ping.occupancy_rate_30m,
db,
)
.await?;
}
PingType::InitScript => {
update_ping_for_failed_init_script_query(
worker_name,
insert_ping.last_job_executed.unwrap_or_default(),
db,
)
.await?
}
}
Ok(())
}
#[derive(Serialize, Deserialize)]
pub struct JobCancelled {
pub canceled_by: String,
pub reason: String,
}
pub async fn set_job_cancelled_query(
job_id: Uuid,
db: &DB,
canceled_by: &str,
reason: &str,
) -> anyhow::Result<()> {
sqlx::query!(
"UPDATE v2_job_queue
SET canceled_by = $1
, canceled_reason = $2
WHERE id = $3",
canceled_by,
reason,
job_id
)
.execute(db)
.await?;
Ok(())
}
pub async fn update_ping_for_failed_init_script_query(
worker_name: &str,
last_job_id: Uuid,
db: &DB,
) -> anyhow::Result<()> {
sqlx::query!(
"UPDATE worker_ping SET
ping_at = now(),
jobs_executed = 1,
current_job_id = $1,
current_job_workspace_id = 'admins'
WHERE worker = $2",
last_job_id,
worker_name
)
.execute(db)
.await?;
Ok(())
}
pub async fn fetch_flow_node_query(
db: &DB,
id: i64,
loc: &'static Location<'_>,
) -> error::Result<RawNode> {
let r = sqlx::query!(
"SELECT \
code AS \"raw_code: String\", \
lock AS \"raw_lock: String\", \
flow AS \"raw_flow: Json<Box<RawValue>>\" \
FROM flow_node WHERE id = $1 LIMIT 1",
id,
)
.fetch_optional(db)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(loc, "Flow node", id))
.map(|r| RawNode {
raw_code: r.raw_code,
raw_lock: r.raw_lock,
raw_flow: r.raw_flow.map(|Json(raw_flow)| raw_flow),
})?;
Ok(r)
}
pub async fn fetch_raw_script_from_app_query(
db: &DB,
id: i64,
loc: &'static Location<'_>,
) -> error::Result<RawScript> {
sqlx::query!(
"SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1",
id,
)
.fetch_optional(db)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Application script", id))
.map(|r| RawScript { content: r.code, lock: r.lock, meta: None })
}
pub async fn insert_ping_query(
worker_instance: &str,
worker_name: &str,
worker_group: &str,
ip: &str,
tags: &[String],
dw: Option<String>,
version: &str,
vcpus: Option<i64>,
memory: Option<i64>,
db: &DB,
) -> anyhow::Result<()> {
sqlx::query!(
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
worker_instance,
worker_name,
ip,
tags.as_slice(),
*WORKER_GROUP,
tags,
worker_group,
dw,
crate::utils::GIT_VERSION,
version,
vcpus,
memory
)
.execute(db)
.await?;
Ok(())
}
pub async fn update_worker_ping_from_job_query(
job_id: &Uuid,
w_id: &str,
worker_name: &str,
memory_usage: Option<i64>,
wm_memory_usage: Option<i64>,
occupancy_rate: Option<f32>,
occupancy_rate_15s: Option<f32>,
occupancy_rate_5m: Option<f32>,
occupancy_rate_30m: Option<f32>,
db: &DB,
) -> anyhow::Result<()> {
sqlx::query!(
"UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,
occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
job_id,
w_id,
memory_usage,
wm_memory_usage,
worker_name,
occupancy_rate,
occupancy_rate_15s,
occupancy_rate_5m,
occupancy_rate_30m,
)
.execute(db)
.await
.expect("insert worker_ping initial value");
.await?;
Ok(())
}
pub async fn update_job_ping_query(
job_id: &Uuid,
db: &DB,
mem_peak: Option<i32>,
) -> anyhow::Result<PingJobStatusResponse> {
let ro = sqlx::query!(
"UPDATE v2_job_runtime r SET
memory_peak = $1,
ping = now()
FROM v2_job_queue q
WHERE r.id = $2 AND q.id = r.id
RETURNING canceled_by, canceled_reason",
mem_peak,
job_id
)
.map(|x| PingJobStatusResponse {
canceled_by: x.canceled_by,
canceled_reason: x.canceled_reason,
already_completed: false,
})
.fetch_optional(db)
.await;
// TODO: add memory metrics to memory time series
if let Ok(r) = ro {
if let Some(i) = r {
Ok(i)
} else {
Err(anyhow::anyhow!("Job not found"))
}
} else {
Err(to_anyhow(ro.unwrap_err()))
}
}
pub async fn update_worker_ping_main_loop_query(
worker_name: &str,
tags: &[String],
vcpus: Option<i64>,
memory: Option<i64>,
jobs_executed: Option<i32>,
occupancy_rate: Option<f32>,
memory_usage: Option<i64>,
wm_memory_usage: Option<i64>,
occupancy_rate_15s: Option<f32>,
occupancy_rate_5m: Option<f32>,
occupancy_rate_30m: Option<f32>,
db: &DB,
) -> anyhow::Result<()> {
sqlx::query!(
"UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,
occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),
memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6",
jobs_executed,
tags,
occupancy_rate,
memory_usage,
wm_memory_usage,
worker_name,
vcpus,
memory,
occupancy_rate_15s,
occupancy_rate_5m,
occupancy_rate_30m,
)
.execute(db)
.await?;
Ok(())
}
// "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,
// occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),
// memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6",
pub async fn load_worker_config(
db: &DB,
killpill_tx: KillpillSender,
@@ -1018,7 +1448,7 @@ pub struct WorkspacedPath {
pub path: String,
}
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, Debug)]
pub struct WorkerConfigOpt {
pub worker_tags: Option<Vec<String>>,
pub priority_tags: Option<HashMap<String, u8>>,
backend/windmill-queue/src/flow_status.rs
+135
View File
@@ -0,0 +1,135 @@
use uuid::Uuid;
use windmill_common::{
error::{self, Error},
utils::WarnAfterExt,
DB,
};
#[derive(Debug, Copy, Clone)]
pub enum Step {
Step(usize),
PreprocessorStep,
FailureStep,
}
impl Step {
pub fn from_i32_and_len(step: i32, len: usize) -> Self {
if step < 0 {
Step::PreprocessorStep
} else if (step as usize) < len {
Step::Step(step as usize)
} else {
Step::FailureStep
}
}
}
pub async fn update_flow_status_in_progress(
db: &DB,
_w_id: &str,
flow: Uuid,
job_in_progress: Uuid,
) -> error::Result<Step> {
let step = get_step_of_flow_status(db, flow).await?;
match step {
Step::Step(step) => {
sqlx::query!(
"UPDATE v2_job_status SET
flow_status = jsonb_set(
jsonb_set(flow_status, ARRAY['modules', $3::INTEGER::TEXT, 'job'], to_jsonb($1::UUID::TEXT)),
ARRAY['modules', $3::INTEGER::TEXT, 'type'],
to_jsonb('InProgress'::text)
)
WHERE id = $2",
job_in_progress,
flow,
step as i32
)
.execute(db)
.await?;
}
Step::PreprocessorStep => {
sqlx::query!(
"UPDATE v2_job_status SET
flow_status = jsonb_set(
jsonb_set(flow_status, ARRAY['preprocessor_module', 'job'], to_jsonb($1::UUID::TEXT)),
ARRAY['preprocessor_module', 'type'],
to_jsonb('InProgress'::text)
)
WHERE id = $2",
job_in_progress,
flow
)
.execute(db)
.await?;
}
Step::FailureStep => {
sqlx::query!(
"UPDATE v2_job_status SET
flow_status = jsonb_set(
jsonb_set(flow_status, ARRAY['failure_module', 'job'], to_jsonb($1::UUID::TEXT)),
ARRAY['failure_module', 'type'],
to_jsonb('InProgress'::text)
)
WHERE id = $2",
job_in_progress,
flow
)
.execute(db)
.await?;
}
}
Ok(step)
}
pub async fn update_workflow_as_code_status(
db: &DB,
id: &Uuid,
parent_job: &Uuid,
) -> error::Result<()> {
let _ = sqlx::query_scalar!(
"UPDATE v2_job_status SET
workflow_as_code_status = jsonb_set(
jsonb_set(
COALESCE(workflow_as_code_status, '{}'::jsonb),
array[$1],
COALESCE(workflow_as_code_status->$1, '{}'::jsonb)
),
array[$1, 'started_at'],
to_jsonb(now()::text)
)
WHERE id = $2",
id.to_string(),
parent_job
)
.execute(db)
.warn_after_seconds(5)
.await
.inspect_err(|e| {
tracing::error!(
"Could not update parent job `started_at` in workflow as code status: {}",
e
)
});
Ok(())
}
// TODO: merge as a CTE
#[tracing::instrument(level = "trace", skip_all)]
async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result<Step> {
let r = sqlx::query!(
"SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len
FROM v2_job_status WHERE id = $1",
id
)
.fetch_one(db)
.await
.map_err(|e| Error::internal_err(format!("fetching step flow status: {e:#}")))?;
if let Some(step) = r.step {
Ok(Step::from_i32_and_len(step, r.len.unwrap_or(0) as usize))
} else {
Err(Error::internal_err("step is null".to_string()))
}
}

Some files were not shown because too many files have changed in this diff Show More