starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-09 00:04:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: extract windmill-api into subcrates for parallel compilation (#7845)

Browse Source 
* refactor: extract windmill-api into 4 subcrates (api-auth, store, api-sse, api-jobs)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* refactor: eliminate refresh_token OnceLock bridge in windmill-store

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* refactor: eliminate FromRequestParts OnceLock bridge in windmill-api-auth

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* refactor: wire subcrates into workspace and clean up unused re-exports

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: resolve cargo check --all-features errors in subcrate wiring

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* sqlx

* all

* chore: update ee-repo-ref for warning fixes

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* refactor: extract windmill-trigger crate and expand windmill-api-jobs

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* refactor: extract windmill-trigger-kafka crate from windmill-api

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* refactor: extract windmill-trigger-postgres crate from windmill-api

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* refactor: extract windmill-trigger-websocket and windmill-trigger-mqtt crates

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* refactor: extract windmill-trigger-nats, sqs, gcp, and email crates

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* refactor: extract windmill-trigger-http crate from windmill-api

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* refactor: move token creation and permission helpers to windmill-api-auth

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* refactor: extract windmill-native-triggers crate from windmill-api

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* sqlx

* all

* refactor: extract windmill-api-embeddings crate and fix CI warnings

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: resolve type mismatch in oauth2_oss and remaining warnings

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: use correct HTTP_CLIENT config in embeddings crate (30s timeout, cert override)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* all

* fix: gate oauth_refresh_ee on oauth2 feature to fix warnings

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* all

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-02-07 22:12:55 +00:00
committed by GitHub
co-authored by Claude Opus 4.6
parent bbb397b6ad
commit 9ff8a85af6
217 changed files with 16761 additions and 14660 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.sqlx/query-023cdbc77ea9e2c17a1aa92a5b9001f29e58e81b3f782887db6e0a627dd8ad75.json
Generated
+2 -1
View File
@@ -12,7 +12,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-0574df3e18f626dd8b3f83fbff8b0ee99cf8483a8fe66fa9311cb96e3f5a0ee2.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n DELETE FROM\n capture\n WHERE\n workspace_id = $1\n AND created_at <= (\n SELECT\n created_at\n FROM\n capture\n WHERE\n workspace_id = $1\n ORDER BY\n created_at DESC\n OFFSET $2\n LIMIT 1\n )\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": []
},
"hash": "0574df3e18f626dd8b3f83fbff8b0ee99cf8483a8fe66fa9311cb96e3f5a0ee2"
}
backend/.sqlx/query-06072cfe26abe58629623a8b38382b33947c5a5c702ce586e6e6ea51430380bf.json
Generated
+2 -1
View File
@@ -17,7 +17,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json
Generated
+5 -5
View File
@@ -46,11 +46,11 @@
]
},
"nullable": [
true,
true,
true,
true,
true,
false,
false,
false,
false,
false,
true,
true
]
backend/.sqlx/query-0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd.json
Generated
-28
View File
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE nats_trigger \n SET \n nats_resource_path = $1,\n subjects = $2,\n stream_name = $3,\n consumer_name = $4,\n use_jetstream = $5,\n script_path = $6,\n path = $7,\n is_flow = $8,\n edited_by = $9,\n email = $10,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $13,\n error_handler_args = $14,\n retry = $15\n WHERE \n workspace_id = $11 AND path = $12\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"VarcharArray",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd"
}
backend/.sqlx/query-0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592.json
Generated
-39
View File
@@ -1,39 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger \n SET\n mqtt_resource_path = $1,\n subscribe_topics = $2,\n client_version = $3,\n client_id = $4,\n v3_config = $5,\n v5_config = $6,\n is_flow = $7, \n edited_by = $8, \n email = $9,\n script_path = $10,\n path = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE \n workspace_id = $12 AND \n path = $13\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"JsonbArray",
{
"Custom": {
"name": "mqtt_client_version",
"kind": {
"Enum": [
"v3",
"v5"
]
}
}
},
"Varchar",
"Jsonb",
"Jsonb",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592"
}
backend/.sqlx/query-73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca.json → backend/.sqlx/query-141a428bf866f2e3d4c6eff756a5fb8c6476d6b9b00b9a336b4616d3b69a491e.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO v2_job_completed (\n id,\n workspace_id,\n started_at,\n completed_at,\n duration_ms,\n result,\n deleted,\n canceled_by,\n canceled_reason,\n flow_status,\n memory_peak,\n status,\n worker,\n workflow_as_code_status,\n result_columns,\n retries,\n extras\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17\n )\n ON CONFLICT (id) DO NOTHING\n ",
"query": "\n INSERT INTO v2_job_completed (\n id, workspace_id, started_at, completed_at, duration_ms, result, deleted,\n canceled_by, canceled_reason, flow_status, memory_peak, status, worker,\n workflow_as_code_status, result_columns, retries, extras\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17\n )\n ON CONFLICT (id) DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
@@ -38,5 +38,5 @@
},
"nullable": []
},
"hash": "73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca"
"hash": "141a428bf866f2e3d4c6eff756a5fb8c6476d6b9b00b9a336b4616d3b69a491e"
}
backend/.sqlx/query-1a85e45df7fec414e3e167bad3472f455571e0dd3006fb717b36dcab36689cca.json
Generated
-35
View File
@@ -1,35 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE email_trigger \n SET \n script_path = $1,\n path = $2,\n is_flow = $3,\n edited_by = $4,\n email = $5,\n edited_at = now(),\n error_handler_path = $6,\n error_handler_args = $7,\n retry = $8,\n mode = $9\n WHERE \n workspace_id = $10 AND path = $11\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Jsonb",
"Jsonb",
{
"Custom": {
"name": "trigger_mode",
"kind": {
"Enum": [
"enabled",
"disabled",
"suspended"
]
}
}
},
"Text",
"Text"
]
},
"nullable": []
},
"hash": "1a85e45df7fec414e3e167bad3472f455571e0dd3006fb717b36dcab36689cca"
}
backend/.sqlx/query-1a996a7b9cd38a3b0574048a70ed75281dada61fd1c6da1fa52624ebe0b1c962.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO v2_job_status (id, flow_status, flow_leaf_jobs, workflow_as_code_status)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (id) DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Jsonb",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "1a996a7b9cd38a3b0574048a70ed75281dada61fd1c6da1fa52624ebe0b1c962"
}
backend/.sqlx/query-f2eebf58001aa792c5fbe9f24ac8290cc15d2865b6774f07a52dd9db7a7209c7.json → backend/.sqlx/query-1cf2eb1426e8be89c3649272103bcd029e99b029b7f1b71eda4411d1e24e790d.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n gcp_resource_path, \n script_path,\n is_flow,\n mode as \"mode: _\",\n workspace_id,\n path,\n edited_by,\n email,\n delivery_config AS \"delivery_config: _\",\n retry as \"retry: _\",\n error_handler_path,\n error_handler_args as \"error_handler_args: _\"\n FROM\n gcp_trigger\n WHERE\n workspace_id = $1 AND\n path = $2 AND\n delivery_type = 'push'::DELIVERY_MODE \n ",
"query": "\n SELECT\n gcp_resource_path,\n script_path,\n is_flow,\n mode as \"mode: _\",\n workspace_id,\n path,\n edited_by,\n email,\n delivery_config AS \"delivery_config: _\",\n retry as \"retry: _\",\n error_handler_path,\n error_handler_args as \"error_handler_args: _\"\n FROM\n gcp_trigger\n WHERE\n workspace_id = $1 AND\n path = $2 AND\n delivery_type = 'push'::DELIVERY_MODE\n ",
"describe": {
"columns": [
{
@@ -96,5 +96,5 @@
true
]
},
"hash": "f2eebf58001aa792c5fbe9f24ac8290cc15d2865b6774f07a52dd9db7a7209c7"
"hash": "1cf2eb1426e8be89c3649272103bcd029e99b029b7f1b71eda4411d1e24e790d"
}
backend/.sqlx/query-204998a6136091abe6bb8a503b25dc0d34fc38ee129d736b28219b84e6474229.json
Generated
-52
View File
@@ -1,52 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n gcp_trigger \n SET \n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6, \n edited_by = $7, \n email = $8,\n script_path = $9,\n path = $10,\n mode = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17,\n ack_deadline = $18\n WHERE \n workspace_id = $12 AND \n path = $13\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
{
"Custom": {
"name": "delivery_mode",
"kind": {
"Enum": [
"push",
"pull"
]
}
}
},
"Jsonb",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
{
"Custom": {
"name": "trigger_mode",
"kind": {
"Enum": [
"enabled",
"disabled",
"suspended"
]
}
}
},
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb",
"Bool",
"Int4"
]
},
"nullable": []
},
"hash": "204998a6136091abe6bb8a503b25dc0d34fc38ee129d736b28219b84e6474229"
}
backend/.sqlx/query-234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17.json
Generated
-23
View File
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1\n FROM \n http_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n )\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17"
}
backend/.sqlx/query-256118bb9c87675d6a8db2d13c4f446d88c7bc671d5e90307febfd60bb235031.json
Generated
+2 -1
View File
@@ -16,7 +16,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-2602402bedcdbc45cdc0d64a76ce075c6ae51404037b0a7d4d33faf6a7d6a6d8.json
Generated
+2 -1
View File
@@ -11,7 +11,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-26141a1c3c48184285d94a2e2aca94ff708ce8e0b0b4fb59e5de7c9698c75c25.json
Generated
+2 -1
View File
@@ -11,7 +11,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-27ada97cb533c8595f1d73987c7823d8e54c96889e06895c57cafae9ca27bf8b.json
Generated
+2 -1
View File
@@ -15,7 +15,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-28a5caa6e8a8a95d228908cc83e9109986a0803d50164a259aabcdee81014d07.json
Generated
-23
View File
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1 \n FROM email_trigger \n WHERE \n ((workspaced_local_part IS TRUE AND workspace_id || '-' || local_part = $1) \n OR (workspaced_local_part IS FALSE AND local_part = $1))\n AND ($2::TEXT IS NULL OR path != $2)\n )\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "28a5caa6e8a8a95d228908cc83e9109986a0803d50164a259aabcdee81014d07"
}
backend/.sqlx/query-2fd22c4ffa2d222bb116260994a748e0639c2f73cbc1d8be66420c70b14c96e1.json
Generated
+2 -1
View File
@@ -12,7 +12,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-388ff2abd495cf71e87cf0c4ddc73b6c84867fb966df91b320c54acdd5e61315.json
Generated
+37
View File
@@ -0,0 +1,37 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE email_trigger\n SET\n script_path = $1,\n path = $2,\n is_flow = $3,\n local_part = $4,\n workspaced_local_part = $5,\n edited_by = $6,\n email = $7,\n edited_at = now(),\n error_handler_path = $8,\n error_handler_args = $9,\n retry = $10,\n mode = $11\n WHERE\n workspace_id = $12 AND path = $13\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Jsonb",
"Jsonb",
{
"Custom": {
"name": "trigger_mode",
"kind": {
"Enum": [
"enabled",
"disabled",
"suspended"
]
}
}
},
"Text",
"Text"
]
},
"nullable": []
},
"hash": "388ff2abd495cf71e87cf0c4ddc73b6c84867fb966df91b320c54acdd5e61315"
}
backend/.sqlx/query-3ebf94f926fff2df4268dd76773eb1996e5f0bb679dca175737ace462432721f.json → backend/.sqlx/query-3b3f60623126626b52ca0a4a188655ddf728cd3f21ee308db7393694ccc5c7b3.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_integrations \n SET oauth_data = $1, updated_at = now()\n WHERE workspace_id = $2 AND service_name = $3\n ",
"query": "\n UPDATE workspace_integrations\n SET oauth_data = $1, updated_at = now()\n WHERE workspace_id = $2 AND service_name = $3\n ",
"describe": {
"columns": [],
"parameters": {
@@ -12,7 +12,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
@@ -21,5 +22,5 @@
},
"nullable": []
},
"hash": "3ebf94f926fff2df4268dd76773eb1996e5f0bb679dca175737ace462432721f"
"hash": "3b3f60623126626b52ca0a4a188655ddf728cd3f21ee308db7393694ccc5c7b3"
}
backend/.sqlx/query-3d8788d42d45cf27082f031a2ce29b496f1f2caae145c47db296b6ea0089f86c.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO v2_job_queue (\n id, workspace_id, started_at, scheduled_for, running, canceled_by,\n canceled_reason, suspend, suspend_until, worker, extras, tag, priority\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13\n )\n ON CONFLICT (id) DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Varchar",
"Timestamptz",
"Timestamptz",
"Bool",
"Varchar",
"Text",
"Int4",
"Timestamptz",
"Varchar",
"Jsonb",
"Varchar",
"Int2"
]
},
"nullable": []
},
"hash": "3d8788d42d45cf27082f031a2ce29b496f1f2caae145c47db296b6ea0089f86c"
}
backend/.sqlx/query-3e5bdc2e071fc2f1e3c7971736272f20bb5a0aa921a614bd02898d3f162660c2.json
Generated
+4 -2
View File
@@ -16,7 +16,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
@@ -51,7 +52,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947.json
Generated
-26
View File
@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO v2_job_queue (\n id,\n workspace_id,\n started_at,\n scheduled_for,\n running,\n canceled_by,\n canceled_reason,\n suspend,\n suspend_until,\n worker,\n extras,\n tag,\n priority\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13\n )\n ON CONFLICT (id) DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Varchar",
"Timestamptz",
"Timestamptz",
"Bool",
"Varchar",
"Text",
"Int4",
"Timestamptz",
"Varchar",
"Jsonb",
"Varchar",
"Int2"
]
},
"nullable": []
},
"hash": "3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947"
}
backend/.sqlx/query-013603454ee8335dc841a3637803532cf556bca5ab5a46c78fe2334c361a140e.json → backend/.sqlx/query-5368683c19f8d6744d5dbc53e5b2ab0f2348646d79f5306c6868e2c3a8f389ee.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n oauth_data as \"oauth_data!: sqlx::types::Json<WorkspaceOAuthConfig>\",\n service_name as \"service_name!: ServiceName\"\n FROM \n workspace_integrations \n WHERE \n workspace_id = $1\n ",
"query": "\n SELECT\n oauth_data as \"oauth_data!: sqlx::types::Json<WorkspaceOAuthConfig>\",\n service_name as \"service_name!: ServiceName\"\n FROM\n workspace_integrations\n WHERE\n workspace_id = $1\n ",
"describe": {
"columns": [
{
@@ -16,7 +16,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
@@ -33,5 +34,5 @@
false
]
},
"hash": "013603454ee8335dc841a3637803532cf556bca5ab5a46c78fe2334c361a140e"
"hash": "5368683c19f8d6744d5dbc53e5b2ab0f2348646d79f5306c6868e2c3a8f389ee"
}
backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json
Generated
+1 -1
View File
@@ -15,7 +15,7 @@
]
},
"nullable": [
null
true
]
},
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"
backend/.sqlx/query-66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147.json
Generated
+51
View File
@@ -0,0 +1,51 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n owner,\n email\n FROM\n capture_config\n WHERE\n workspace_id = $1\n AND path = $2\n AND is_flow = $3\n AND trigger_kind = $4\n AND last_client_ping > NOW() - INTERVAL '10 seconds'\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "owner",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "email",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Bool",
{
"Custom": {
"name": "trigger_kind",
"kind": {
"Enum": [
"webhook",
"http",
"websocket",
"kafka",
"email",
"nats",
"postgres",
"sqs",
"mqtt",
"gcp",
"default_email",
"nextcloud"
]
}
}
}
]
},
"nullable": [
false,
false
]
},
"hash": "66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147"
}
backend/.sqlx/query-6c3c38ed5a0e6de0c97954fe4581daa788a7551b4a00dd33f78d36e246b98dd7.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE\n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n initial_messages = $6,\n url_runnable_args = $7,\n edited_by = $8,\n email = $9,\n can_return_message = $10,\n can_return_error_result = $11,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND path = $13\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Bool",
"JsonbArray",
"JsonbArray",
"Jsonb",
"Varchar",
"Varchar",
"Bool",
"Bool",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "6c3c38ed5a0e6de0c97954fe4581daa788a7551b4a00dd33f78d36e246b98dd7"
}
backend/.sqlx/query-8642e4fc3efb011a8b7f98d2080a165ec4046c3a7172b5635b818c0fa133411f.json
Generated
+52
View File
@@ -0,0 +1,52 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE\n gcp_trigger\n SET\n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n script_path = $9,\n path = $10,\n mode = $11,\n edited_at = now(),\n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17,\n ack_deadline = $18\n WHERE\n workspace_id = $12 AND\n path = $13\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
{
"Custom": {
"name": "delivery_mode",
"kind": {
"Enum": [
"push",
"pull"
]
}
}
},
"Jsonb",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
{
"Custom": {
"name": "trigger_mode",
"kind": {
"Enum": [
"enabled",
"disabled",
"suspended"
]
}
}
},
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb",
"Bool",
"Int4"
]
},
"nullable": []
},
"hash": "8642e4fc3efb011a8b7f98d2080a165ec4046c3a7172b5635b818c0fa133411f"
}
backend/.sqlx/query-8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c.json
Generated
-18
View File
@@ -1,18 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO job_logs (\n job_id,\n workspace_id,\n logs,\n log_offset,\n log_file_index\n ) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (job_id) DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Varchar",
"Text",
"Int4",
"TextArray"
]
},
"nullable": []
},
"hash": "8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c"
}
backend/.sqlx/query-1ecb8a116bbfbcae8b9c502d58d98c838bf4de8d1048889b658d7e969d70a50f.json → backend/.sqlx/query-9052b7cd438ff029a37bd489190d98d365acec09f2f102b7de71dcc9d356900e.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n oauth_data \n FROM \n workspace_integrations \n WHERE \n workspace_id = $1 AND\n service_name = $2\n ",
"query": "\n SELECT\n oauth_data\n FROM\n workspace_integrations\n WHERE\n workspace_id = $1 AND\n service_name = $2\n ",
"describe": {
"columns": [
{
@@ -17,7 +17,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
@@ -28,5 +29,5 @@
false
]
},
"hash": "1ecb8a116bbfbcae8b9c502d58d98c838bf4de8d1048889b658d7e969d70a50f"
"hash": "9052b7cd438ff029a37bd489190d98d365acec09f2f102b7de71dcc9d356900e"
}
backend/.sqlx/query-94bb99ee062eb72850721f121a626dd1665b3434f8624710ea5931958f459707.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1\n FROM\n http_trigger\n WHERE\n workspace_id = $1 AND\n path = $2\n )\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "94bb99ee062eb72850721f121a626dd1665b3434f8624710ea5931958f459707"
}
backend/.sqlx/query-95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739.json
Generated
-16
View File
@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO v2_job_runtime (\n id, ping, memory_peak\n ) VALUES (\n $1, $2, $3\n )\n ON CONFLICT (id) DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Timestamptz",
"Int4"
]
},
"nullable": []
},
"hash": "95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739"
}
backend/.sqlx/query-92745b0ee2b42cc6f8ba58091b3845784ed840b2f5196b40661a453c1f2a4db5.json → backend/.sqlx/query-95c57fb921a2e3725b92cbafac6e3dc360b88429f03dd1e2b1b55cfabe208cb7.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS (\n SELECT 1\n FROM workspace_integrations\n WHERE workspace_id = $1\n AND service_name = $2 \n AND oauth_data IS NOT NULL\n )\n ",
"query": "\n SELECT EXISTS (\n SELECT 1\n FROM workspace_integrations\n WHERE workspace_id = $1\n AND service_name = $2\n AND oauth_data IS NOT NULL\n )\n ",
"describe": {
"columns": [
{
@@ -17,7 +17,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
@@ -28,5 +29,5 @@
null
]
},
"hash": "92745b0ee2b42cc6f8ba58091b3845784ed840b2f5196b40661a453c1f2a4db5"
"hash": "95c57fb921a2e3725b92cbafac6e3dc360b88429f03dd1e2b1b55cfabe208cb7"
}
backend/.sqlx/query-9f41ea5cbe4cffa74e4a283fe8f023c813e349956487f7b6599da452c068e9b9.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE nats_trigger\n SET\n nats_resource_path = $1,\n subjects = $2,\n stream_name = $3,\n consumer_name = $4,\n use_jetstream = $5,\n script_path = $6,\n path = $7,\n is_flow = $8,\n edited_by = $9,\n email = $10,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $13,\n error_handler_args = $14,\n retry = $15\n WHERE\n workspace_id = $11 AND path = $12\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"VarcharArray",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "9f41ea5cbe4cffa74e4a283fe8f023c813e349956487f7b6599da452c068e9b9"
}
backend/.sqlx/query-447457d93fa956d2f7364ff15038cc590473da374b5120921086a1ee557ac9ba.json → backend/.sqlx/query-aa59a96bf2d7edfa7c550e66c4d52ddc8e84eacc633e361e49a5219d0bec94b9.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n mode, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg,\n ack_deadline\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4,\n $5,\n $6, \n $7, \n $8, \n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16,\n $17\n )",
"query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id,\n path,\n script_path,\n is_flow,\n email,\n mode,\n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg,\n ack_deadline\n )\n VALUES (\n $1,\n $2,\n $3,\n $4,\n $5,\n $6,\n $7,\n $8,\n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16,\n $17\n )",
"describe": {
"columns": [],
"parameters": {
@@ -47,5 +47,5 @@
},
"nullable": []
},
"hash": "447457d93fa956d2f7364ff15038cc590473da374b5120921086a1ee557ac9ba"
"hash": "aa59a96bf2d7edfa7c550e66c4d52ddc8e84eacc633e361e49a5219d0bec94b9"
}
backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json
Generated
-46
View File
@@ -1,46 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5 AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text",
"Bool",
{
"Custom": {
"name": "trigger_kind",
"kind": {
"Enum": [
"webhook",
"http",
"websocket",
"kafka",
"email",
"nats",
"postgres",
"sqs",
"mqtt",
"gcp",
"default_email",
"nextcloud"
]
}
}
}
]
},
"nullable": [
null
]
},
"hash": "ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30"
}
backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json
Generated
+2 -2
View File
@@ -18,8 +18,8 @@
"Left": []
},
"nullable": [
true,
false
false,
true
]
},
"hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76"
backend/.sqlx/query-b4eb72b0274cbdce7490f63c36d0d16ee847294fadc138593a1baa417cbb3652.json
Generated
+1 -3
View File
@@ -59,9 +59,7 @@
"failure",
"command",
"approval",
"preprocessor",
"schedule_handler_old",
"dynamic_skip"
"preprocessor"
]
}
}
backend/.sqlx/query-b4fc94adfe55bb87d2c4b6b45ed2eb5ac25e84f3619a0b403a2d619a0eb51432.json
Generated
-29
View File
@@ -1,29 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n variable.account as account_id,\n (now() > account.expires_at) as \"is_expired: bool\"\n FROM variable\n LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2\n WHERE variable.path = $1 AND variable.workspace_id = $2\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "account_id",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "is_expired: bool",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true,
null
]
},
"hash": "b4fc94adfe55bb87d2c4b6b45ed2eb5ac25e84f3619a0b403a2d619a0eb51432"
}
backend/.sqlx/query-b5cda8eb32384f315689001f45676d4bf44cb4397dd0a722e7c9d035b58a09c1.json
Generated
+35
View File
@@ -0,0 +1,35 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE email_trigger\n SET\n script_path = $1,\n path = $2,\n is_flow = $3,\n edited_by = $4,\n email = $5,\n edited_at = now(),\n error_handler_path = $6,\n error_handler_args = $7,\n retry = $8,\n mode = $9\n WHERE\n workspace_id = $10 AND path = $11\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Jsonb",
"Jsonb",
{
"Custom": {
"name": "trigger_mode",
"kind": {
"Enum": [
"enabled",
"disabled",
"suspended"
]
}
}
},
"Text",
"Text"
]
},
"nullable": []
},
"hash": "b5cda8eb32384f315689001f45676d4bf44cb4397dd0a722e7c9d035b58a09c1"
}
backend/.sqlx/query-b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa.json
Generated
-29
View File
@@ -1,29 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n initial_messages = $6,\n url_runnable_args = $7,\n edited_by = $8,\n email = $9,\n can_return_message = $10,\n can_return_error_result = $11,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND path = $13\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Bool",
"JsonbArray",
"JsonbArray",
"Jsonb",
"Varchar",
"Varchar",
"Bool",
"Bool",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa"
}
backend/.sqlx/query-b615d73ddb43e9d655b86a0cf98f892bf40e629ee11ee4845199481755f2789d.json
Generated
+4 -2
View File
@@ -21,7 +21,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
@@ -71,7 +72,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-bb94e3105cb1bc5d90af2bc914b579a6e821c432eac38ad877d9ff362d8ab916.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1\n FROM email_trigger\n WHERE\n ((workspaced_local_part IS TRUE AND workspace_id || '-' || local_part = $1)\n OR (workspaced_local_part IS FALSE AND local_part = $1))\n AND ($2::TEXT IS NULL OR path != $2)\n )\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "bb94e3105cb1bc5d90af2bc914b579a6e821c432eac38ad877d9ff362d8ab916"
}
backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json → backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO v2_job (\n id,\n raw_code,\n raw_lock,\n raw_flow,\n tag,\n workspace_id,\n created_at,\n created_by,\n permissioned_as,\n permissioned_as_email,\n kind,\n runnable_id,\n runnable_path,\n parent_job,\n root_job,\n script_lang,\n script_entrypoint_override,\n flow_step,\n flow_step_id,\n flow_innermost_root_job,\n trigger,\n trigger_kind,\n same_worker,\n visible_to_owner,\n concurrent_limit,\n concurrency_time_window_s,\n cache_ttl,\n timeout,\n priority,\n preprocessed,\n args,\n labels,\n pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ",
"query": "\n INSERT INTO v2_job (\n id, raw_code, raw_lock, raw_flow, tag, workspace_id, created_at, created_by,\n permissioned_as, permissioned_as_email, kind, runnable_id, runnable_path,\n parent_job, root_job, script_lang, script_entrypoint_override, flow_step,\n flow_step_id, flow_innermost_root_job, trigger, trigger_kind, same_worker,\n visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl,\n timeout, priority, preprocessed, args, labels, pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
@@ -125,5 +125,5 @@
},
"nullable": []
},
"hash": "fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f"
"hash": "be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66"
}
backend/.sqlx/query-c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d.json
Generated
-17
View File
@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO v2_job_status (\n id, flow_status, flow_leaf_jobs, workflow_as_code_status\n ) VALUES (\n $1, $2, $3, $4\n )\n ON CONFLICT (id) DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Jsonb",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d"
}
backend/.sqlx/query-c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e.json
Generated
+46
View File
@@ -0,0 +1,46 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE\n capture_config\n SET\n server_id = $1,\n last_server_ping = now(),\n error = 'Connecting...'\n WHERE\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n workspace_id = $2 AND\n path = $3 AND\n is_flow = $4 AND\n trigger_kind = $5 AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n RETURNING true\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text",
"Bool",
{
"Custom": {
"name": "trigger_kind",
"kind": {
"Enum": [
"webhook",
"http",
"websocket",
"kafka",
"email",
"nats",
"postgres",
"sqs",
"mqtt",
"gcp",
"default_email",
"nextcloud"
]
}
}
}
]
},
"nullable": [
null
]
},
"hash": "c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e"
}
backend/.sqlx/query-c723c3a5066a487b93e2642993f3bf624a1f50d06c7de75157420d97cf144763.json
Generated
+37
View File
@@ -0,0 +1,37 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE sqs_trigger\n SET\n queue_url = $1,\n aws_resource_path = $2,\n message_attributes = $3,\n aws_auth_resource_type = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE\n workspace_id = $10 AND path = $11\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"TextArray",
{
"Custom": {
"name": "aws_auth_resource_type",
"kind": {
"Enum": [
"oidc",
"credentials"
]
}
}
},
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "c723c3a5066a487b93e2642993f3bf624a1f50d06c7de75157420d97cf144763"
}
backend/.sqlx/query-cc5919b087e3319045fc2242f22384514987b6a4eb807ee375fb8b5cd4e92307.json
Generated
+2 -1
View File
@@ -11,7 +11,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json → backend/.sqlx/query-ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config\n SET \n last_server_ping = NULL\n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = $4 AND \n server_id IS NULL\n ",
"query": "\n UPDATE\n capture_config\n SET\n last_server_ping = NULL\n WHERE\n workspace_id = $1 AND\n path = $2 AND\n is_flow = $3 AND\n trigger_kind = $4 AND\n server_id IS NULL\n ",
"describe": {
"columns": [],
"parameters": {
@@ -33,5 +33,5 @@
},
"nullable": []
},
"hash": "dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e"
"hash": "ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1"
}
backend/.sqlx/query-d0b493da8ff5b3b78a9a0c6972cd2ca5449dad0fe361bee1a9834989bfeabbae.json
Generated
+2 -1
View File
@@ -12,7 +12,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-d33810c502838ad7aeb1f45691a456ec563d346d99256ded52756cdb4a0d7ffd.json
Generated
-37
View File
@@ -1,37 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE email_trigger \n SET \n script_path = $1,\n path = $2,\n is_flow = $3,\n local_part = $4,\n workspaced_local_part = $5,\n edited_by = $6,\n email = $7,\n edited_at = now(),\n error_handler_path = $8,\n error_handler_args = $9,\n retry = $10,\n mode = $11\n WHERE \n workspace_id = $12 AND path = $13\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Jsonb",
"Jsonb",
{
"Custom": {
"name": "trigger_mode",
"kind": {
"Enum": [
"enabled",
"disabled",
"suspended"
]
}
}
},
"Text",
"Text"
]
},
"nullable": []
},
"hash": "d33810c502838ad7aeb1f45691a456ec563d346d99256ded52756cdb4a0d7ffd"
}
backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json → backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO v2_job (\n id,\n raw_code,\n raw_lock,\n raw_flow,\n tag,\n workspace_id,\n created_at,\n created_by,\n permissioned_as,\n permissioned_as_email,\n kind,\n runnable_id,\n runnable_path,\n parent_job,\n root_job,\n script_lang,\n script_entrypoint_override,\n flow_step,\n flow_step_id,\n flow_innermost_root_job,\n trigger,\n trigger_kind,\n same_worker,\n visible_to_owner,\n concurrent_limit,\n concurrency_time_window_s,\n cache_ttl,\n timeout,\n priority,\n preprocessed,\n args,\n labels,\n pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10,\n $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30,\n $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ",
"query": "\n INSERT INTO v2_job (\n id, raw_code, raw_lock, raw_flow, tag, workspace_id, created_at, created_by,\n permissioned_as, permissioned_as_email, kind, runnable_id, runnable_path,\n parent_job, root_job, script_lang, script_entrypoint_override, flow_step,\n flow_step_id, flow_innermost_root_job, trigger, trigger_kind, same_worker,\n visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl,\n timeout, priority, preprocessed, args, labels, pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10,\n $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30,\n $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
@@ -125,5 +125,5 @@
},
"nullable": []
},
"hash": "b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b"
"hash": "d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016"
}
backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json → backend/.sqlx/query-d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5\n ",
"query": "\n UPDATE\n capture_config\n SET\n error = $1,\n server_id = NULL,\n last_server_ping = NULL\n WHERE\n workspace_id = $2 AND\n path = $3 AND\n is_flow = $4 AND\n trigger_kind = $5\n ",
"describe": {
"columns": [],
"parameters": {
@@ -34,5 +34,5 @@
},
"nullable": []
},
"hash": "db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899"
"hash": "d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106"
}
backend/.sqlx/query-679a9159a5fca976a3de99fe26806faded2cc63e8f16c201e99ab1725dcff294.json → backend/.sqlx/query-d5d96f93769d2f51e340422f78bd9e78ca5db2c6677109b04dc477faf48de033.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1 \n FROM http_trigger \n WHERE \n route_path_key = $1\n AND workspace_id = $2 \n AND http_method = $3 \n AND ($4::TEXT IS NULL OR path != $4)\n )\n ",
"query": "\n SELECT EXISTS(\n SELECT 1\n FROM http_trigger\n WHERE\n route_path_key = $1\n AND workspace_id = $2\n AND http_method = $3\n AND ($4::TEXT IS NULL OR path != $4)\n )\n ",
"describe": {
"columns": [
{
@@ -34,5 +34,5 @@
null
]
},
"hash": "679a9159a5fca976a3de99fe26806faded2cc63e8f16c201e99ab1725dcff294"
"hash": "d5d96f93769d2f51e340422f78bd9e78ca5db2c6677109b04dc477faf48de033"
}
backend/.sqlx/query-80c8da5b571f8da7cde6da8557898e25fdfdf19ea7a85b5e55922883764df15f.json → backend/.sqlx/query-e25aa749cfc9f0bb1649d162e36f2c0ce2187e47d745aa5ba96f9453c722750c.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO mqtt_trigger (\n mqtt_resource_path,\n subscribe_topics,\n client_version,\n client_id,\n v3_config,\n v5_config,\n workspace_id,\n path, \n script_path, \n is_flow, \n email, \n mode, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry\n ) \n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16\n )",
"query": "\n INSERT INTO mqtt_trigger (\n mqtt_resource_path,\n subscribe_topics,\n client_version,\n client_id,\n v3_config,\n v5_config,\n workspace_id,\n path,\n script_path,\n is_flow,\n email,\n mode,\n edited_by,\n error_handler_path,\n error_handler_args,\n retry\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16\n )",
"describe": {
"columns": [],
"parameters": {
@@ -46,5 +46,5 @@
},
"nullable": []
},
"hash": "80c8da5b571f8da7cde6da8557898e25fdfdf19ea7a85b5e55922883764df15f"
"hash": "e25aa749cfc9f0bb1649d162e36f2c0ce2187e47d745aa5ba96f9453c722750c"
}
backend/.sqlx/query-54ee00c98dc50ca3da0e2195a78e6df77fd1e8c969667b267aed3347335c91ab.json → backend/.sqlx/query-e2b362d9e36be80ee92cf9897a86b61e8e4e92b80b336d1a861e68f9ef782f8a.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1 \n FROM email_trigger \n WHERE \n workspace_id = $1\n AND local_part = $2 \n AND ($3::TEXT IS NULL OR path != $3)\n )\n ",
"query": "\n SELECT EXISTS(\n SELECT 1\n FROM email_trigger\n WHERE\n workspace_id = $1\n AND local_part = $2\n AND ($3::TEXT IS NULL OR path != $3)\n )\n ",
"describe": {
"columns": [
{
@@ -20,5 +20,5 @@
null
]
},
"hash": "54ee00c98dc50ca3da0e2195a78e6df77fd1e8c969667b267aed3347335c91ab"
"hash": "e2b362d9e36be80ee92cf9897a86b61e8e4e92b80b336d1a861e68f9ef782f8a"
}
backend/.sqlx/query-e486a64b76da5de97e404c81dd6e29d333ada2dcfbbddb028f37794b85778ca8.json
Generated
+39
View File
@@ -0,0 +1,39 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE\n mqtt_trigger\n SET\n mqtt_resource_path = $1,\n subscribe_topics = $2,\n client_version = $3,\n client_id = $4,\n v3_config = $5,\n v5_config = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n script_path = $10,\n path = $11,\n edited_at = now(),\n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND\n path = $13\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"JsonbArray",
{
"Custom": {
"name": "mqtt_client_version",
"kind": {
"Enum": [
"v3",
"v5"
]
}
}
},
"Varchar",
"Jsonb",
"Jsonb",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "e486a64b76da5de97e404c81dd6e29d333ada2dcfbbddb028f37794b85778ca8"
}
backend/.sqlx/query-e4ffb20330f584f64d96423c0ecbfaf7f8740a02d1d0cc55aaa7d2a45acfb881.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO job_logs (\n job_id, workspace_id, logs, log_offset, log_file_index\n ) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (job_id) DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Varchar",
"Text",
"Int4",
"TextArray"
]
},
"nullable": []
},
"hash": "e4ffb20330f584f64d96423c0ecbfaf7f8740a02d1d0cc55aaa7d2a45acfb881"
}
backend/.sqlx/query-e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b.json
Generated
-26
View File
@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE postgres_trigger \n SET \n postgres_resource_path = $1,\n replication_slot_name = $2,\n publication_name = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $11,\n error_handler_args = $12,\n retry = $13\n WHERE \n workspace_id = $9 AND path = $10\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b"
}
backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json → backend/.sqlx/query-e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config\n SET \n last_server_ping = now(), error = $1\n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5 AND \n server_id = $6 AND \n last_client_ping > NOW() - INTERVAL '10 seconds'\n RETURNING 1\n ",
"query": "\n UPDATE\n capture_config\n SET\n last_server_ping = now(), error = $1\n WHERE\n workspace_id = $2 AND\n path = $3 AND\n is_flow = $4 AND\n trigger_kind = $5 AND\n server_id = $6 AND\n last_client_ping > NOW() - INTERVAL '10 seconds'\n RETURNING 1\n ",
"describe": {
"columns": [
{
@@ -43,5 +43,5 @@
null
]
},
"hash": "615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae"
"hash": "e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022"
}
backend/.sqlx/query-e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e.json
Generated
-37
View File
@@ -1,37 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE sqs_trigger \n SET \n queue_url = $1,\n aws_resource_path = $2,\n message_attributes = $3,\n aws_auth_resource_type = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE \n workspace_id = $10 AND path = $11\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"TextArray",
{
"Custom": {
"name": "aws_auth_resource_type",
"kind": {
"Enum": [
"oidc",
"credentials"
]
}
}
},
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e"
}
backend/.sqlx/query-ecab1af12a7afa685c056b9d0e526275203fc8ecddf83ca6d05c9fb77e46e7ee.json
Generated
+4 -2
View File
@@ -21,7 +21,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
@@ -71,7 +72,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd.json
Generated
+40
View File
@@ -0,0 +1,40 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO\n capture (\n workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bool",
{
"Custom": {
"name": "trigger_kind",
"kind": {
"Enum": [
"webhook",
"http",
"websocket",
"kafka",
"email",
"nats",
"postgres",
"sqs",
"mqtt",
"gcp",
"default_email",
"nextcloud"
]
}
}
},
"Jsonb",
"Jsonb",
"Varchar"
]
},
"nullable": []
},
"hash": "ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd"
}
backend/.sqlx/query-ee537def1ead8bee48bb9f5c1f57d42e7add6011c34d91761ba23e2c74c4032c.json
Generated
+4 -2
View File
@@ -21,7 +21,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
@@ -71,7 +72,8 @@
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud"
"nextcloud",
"google"
]
}
}
backend/.sqlx/query-f4340e31adddc9a37bab64957b2b1b9b29f978c73f003dfbf81d5c487f666358.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE postgres_trigger\n SET\n postgres_resource_path = $1,\n replication_slot_name = $2,\n publication_name = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $11,\n error_handler_args = $12,\n retry = $13\n WHERE\n workspace_id = $9 AND path = $10\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "f4340e31adddc9a37bab64957b2b1b9b29f978c73f003dfbf81d5c487f666358"
}
backend/.sqlx/query-0d8153986cea6166820f601f80d8e67156408b08360d628300b28221ea995a58.json → backend/.sqlx/query-fe464b8b3ade86743d82c5e3fb14f457e07f07e44c7b693d5d755899d4210dee.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1 \n FROM http_trigger \n WHERE \n ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1) \n OR (workspaced_route IS FALSE AND route_path_key = $1))\n AND http_method = $2 \n AND ($3::TEXT IS NULL OR path != $3)\n )\n ",
"query": "\n SELECT EXISTS(\n SELECT 1\n FROM http_trigger\n WHERE\n ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1)\n OR (workspaced_route IS FALSE AND route_path_key = $1))\n AND http_method = $2\n AND ($3::TEXT IS NULL OR path != $3)\n )\n ",
"describe": {
"columns": [
{
@@ -33,5 +33,5 @@
null
]
},
"hash": "0d8153986cea6166820f601f80d8e67156408b08360d628300b28221ea995a58"
"hash": "fe464b8b3ade86743d82c5e3fb14f457e07f07e44c7b693d5d755899d4210dee"
}
backend/.sqlx/query-fe71d74e996511ab7331c52704f2f03f5972394fd927b3da216de66dab9e3e5a.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO v2_job_runtime (id, ping, memory_peak)\n VALUES ($1, $2, $3)\n ON CONFLICT (id) DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Timestamptz",
"Int4"
]
},
"nullable": []
},
"hash": "fe71d74e996511ab7331c52704f2f03f5972394fd927b3da216de66dab9e3e5a"
}
backend/Cargo.lock
Generated
+440 -28
View File
@@ -15722,7 +15722,6 @@ dependencies = [
"anyhow",
"argon2",
"astral-tokio-tar",
"async-nats",
"async-recursion",
"async-stream",
"async-trait",
@@ -15732,25 +15731,15 @@ dependencies = [
"aws-sdk-bedrock",
"aws-sdk-bedrockruntime",
"aws-sdk-config",
"aws-sdk-sqs",
"aws-sdk-sso",
"aws-sdk-ssooidc",
"aws-sdk-sts",
"aws-sigv4",
"aws-smithy-types",
"axum 0.7.9",
"backon",
"base32",
"base64 0.22.1",
"byteorder",
"bytes",
"candle-core",
"candle-nn",
"candle-transformers",
"chrono",
"chrono-tz",
"const_format",
"constant_time_eq 0.3.1",
"cookie 0.17.0",
"cron",
"dashmap 6.1.0",
@@ -15761,10 +15750,7 @@ dependencies = [
"flate2",
"futures",
"git-version",
"google-cloud-googleapis",
"google-cloud-pubsub",
"hex",
"hf-hub",
"hmac",
"http 1.4.0",
"hyper 1.8.1",
@@ -15778,50 +15764,36 @@ dependencies = [
"matchit 0.7.3",
"mime_guess",
"native-tls",
"nkeys",
"object_store",
"openidconnect",
"openssl",
"pg_escape",
"pin-project",
"postgres-native-tls 0.5.0",
"postgres-native-tls 0.5.1",
"prometheus",
"quick_cache",
"rand 0.9.0",
"rdkafka",
"rdkafka-sys",
"regex",
"reqwest 0.13.1",
"rsa",
"rumqttc",
"rust-embed",
"rust_decimal",
"rustls 0.23.35",
"samael",
"serde",
"serde_json",
"serde_urlencoded",
"serde_yml",
"sha1",
"sha2 0.10.9",
"sql-builder",
"sqlx",
"strum 0.27.2",
"tar",
"tempfile",
"thiserror 2.0.18",
"time",
"tinyvector",
"tokenizers",
"tokio",
"tokio-native-tls",
"tokio-postgres 0.7.11",
"tokio-postgres 0.7.13",
"tokio-stream",
"tokio-tungstenite 0.24.0",
"tokio-util",
"tonic",
"tower 0.5.3",
"tower-cookies",
"tower-http",
@@ -15831,12 +15803,17 @@ dependencies = [
"url",
"urlencoding",
"uuid",
"windmill-api-auth",
"windmill-api-embeddings",
"windmill-api-jobs",
"windmill-api-sse",
"windmill-audit",
"windmill-autoscaling",
"windmill-common",
"windmill-git-sync",
"windmill-indexer",
"windmill-mcp",
"windmill-native-triggers",
"windmill-oauth",
"windmill-parser",
"windmill-parser-py",
@@ -15844,9 +15821,46 @@ dependencies = [
"windmill-parser-sql",
"windmill-parser-ts",
"windmill-queue",
"windmill-store",
"windmill-trigger",
"windmill-trigger-email",
"windmill-trigger-gcp",
"windmill-trigger-http",
"windmill-trigger-kafka",
"windmill-trigger-mqtt",
"windmill-trigger-nats",
"windmill-trigger-postgres",
"windmill-trigger-sqs",
"windmill-trigger-websocket",
"windmill-worker",
]
[[package]]
name = "windmill-api-auth"
version = "1.628.3"
dependencies = [
"anyhow",
"axum 0.7.9",
"chrono",
"http 1.4.0",
"itertools 0.14.0",
"jsonwebtoken 8.3.0",
"lazy_static",
"quick_cache",
"reqwest 0.13.1",
"serde",
"serde_json",
"sqlx",
"time",
"tokio",
"tower-cookies",
"tracing",
"uuid",
"windmill-audit",
"windmill-common",
"windmill-mcp",
]
[[package]]
name = "windmill-api-client"
version = "1.628.3"
@@ -15857,6 +15871,66 @@ dependencies = [
"urlencoding",
]
[[package]]
name = "windmill-api-embeddings"
version = "1.628.3"
dependencies = [
"anyhow",
"axum 0.7.9",
"candle-core",
"candle-nn",
"candle-transformers",
"hf-hub",
"lazy_static",
"reqwest 0.13.1",
"serde",
"serde_json",
"sqlx",
"tinyvector",
"tokenizers",
"tokio",
"tracing",
"windmill-common",
"windmill-store",
]
[[package]]
name = "windmill-api-jobs"
version = "1.628.3"
dependencies = [
"anyhow",
"axum 0.7.9",
"base64 0.22.1",
"chrono",
"http 1.4.0",
"hyper 1.8.1",
"lazy_static",
"serde",
"serde_json",
"sql-builder",
"sqlx",
"tokio",
"tower-http",
"tracing",
"uuid",
"windmill-api-auth",
"windmill-api-sse",
"windmill-common",
"windmill-queue",
]
[[package]]
name = "windmill-api-sse"
version = "1.628.3"
dependencies = [
"lazy_static",
"serde",
"serde_json",
"sqlx",
"uuid",
"windmill-common",
]
[[package]]
name = "windmill-audit"
version = "1.628.3"
@@ -16049,6 +16123,7 @@ version = "1.628.3"
dependencies = [
"anyhow",
"async-trait",
"chrono",
"futures",
"http 1.4.0",
"oauth2",
@@ -16063,6 +16138,37 @@ dependencies = [
"windmill-common",
]
[[package]]
name = "windmill-native-triggers"
version = "1.628.3"
dependencies = [
"anyhow",
"async-trait",
"axum 0.7.9",
"backon",
"base64 0.22.1",
"chrono",
"hmac",
"http 1.4.0",
"itertools 0.14.0",
"lazy_static",
"reqwest 0.13.1",
"serde",
"serde_json",
"sha2 0.10.9",
"sqlx",
"strum 0.27.2",
"tokio",
"tracing",
"urlencoding",
"uuid",
"windmill-api-auth",
"windmill-audit",
"windmill-common",
"windmill-oauth",
"windmill-queue",
]
[[package]]
name = "windmill-oauth"
version = "1.628.3"
@@ -16337,6 +16443,312 @@ dependencies = [
"windmill-parser-sql",
]
[[package]]
name = "windmill-store"
version = "1.628.3"
dependencies = [
"anyhow",
"async-recursion",
"axum 0.7.9",
"chrono",
"futures",
"http 1.4.0",
"hyper 1.8.1",
"lazy_static",
"quick_cache",
"reqwest 0.13.1",
"serde",
"serde_json",
"sql-builder",
"sqlx",
"tokio",
"tracing",
"uuid",
"windmill-api-auth",
"windmill-audit",
"windmill-common",
"windmill-git-sync",
"windmill-mcp",
"windmill-oauth",
]
[[package]]
name = "windmill-trigger"
version = "1.628.3"
dependencies = [
"anyhow",
"async-trait",
"axum 0.7.9",
"chrono",
"http 1.4.0",
"hyper 1.8.1",
"itertools 0.14.0",
"lazy_static",
"rand 0.9.0",
"reqwest 0.13.1",
"serde",
"serde_json",
"sql-builder",
"sqlx",
"tokio",
"tracing",
"uuid",
"windmill-api-auth",
"windmill-api-jobs",
"windmill-api-sse",
"windmill-audit",
"windmill-common",
"windmill-git-sync",
"windmill-parser",
"windmill-parser-py",
"windmill-parser-ts",
"windmill-queue",
]
[[package]]
name = "windmill-trigger-email"
version = "1.628.3"
dependencies = [
"anyhow",
"async-trait",
"axum 0.7.9",
"base64 0.22.1",
"lazy_static",
"regex",
"serde",
"serde_json",
"sqlx",
"tracing",
"windmill-api-auth",
"windmill-common",
"windmill-git-sync",
"windmill-trigger",
]
[[package]]
name = "windmill-trigger-gcp"
version = "1.628.3"
dependencies = [
"anyhow",
"async-trait",
"axum 0.7.9",
"base64 0.22.1",
"bytes",
"chrono",
"google-cloud-googleapis",
"google-cloud-pubsub",
"http 1.4.0",
"itertools 0.14.0",
"jsonwebtoken 8.3.0",
"lazy_static",
"quick_cache",
"reqwest 0.13.1",
"serde",
"serde_json",
"sqlx",
"thiserror 2.0.18",
"tokio",
"tokio-util",
"tonic",
"tower-http",
"tracing",
"windmill-api-auth",
"windmill-common",
"windmill-git-sync",
"windmill-store",
"windmill-trigger",
]
[[package]]
name = "windmill-trigger-http"
version = "1.628.3"
dependencies = [
"anyhow",
"async-trait",
"axum 0.7.9",
"base64 0.22.1",
"constant_time_eq 0.3.1",
"futures",
"hex",
"hmac",
"http 1.4.0",
"hyper 1.8.1",
"itertools 0.14.0",
"lazy_static",
"matchit 0.7.3",
"quick_cache",
"regex",
"serde",
"serde_json",
"sha1",
"sha2 0.10.9",
"sqlx",
"thiserror 2.0.18",
"tokio",
"tracing",
"windmill-api-auth",
"windmill-audit",
"windmill-common",
"windmill-git-sync",
"windmill-trigger",
]
[[package]]
name = "windmill-trigger-kafka"
version = "1.628.3"
dependencies = [
"anyhow",
"async-trait",
"axum 0.7.9",
"base64 0.22.1",
"itertools 0.14.0",
"rdkafka",
"serde",
"serde_json",
"sha2 0.10.9",
"sqlx",
"tokio",
"tracing",
"windmill-api-auth",
"windmill-common",
"windmill-git-sync",
"windmill-store",
"windmill-trigger",
]
[[package]]
name = "windmill-trigger-mqtt"
version = "1.628.3"
dependencies = [
"anyhow",
"async-trait",
"axum 0.7.9",
"base64 0.22.1",
"bytes",
"itertools 0.14.0",
"rumqttc",
"serde",
"serde_json",
"sqlx",
"thiserror 2.0.18",
"tokio",
"tracing",
"windmill-api-auth",
"windmill-common",
"windmill-git-sync",
"windmill-store",
"windmill-trigger",
]
[[package]]
name = "windmill-trigger-nats"
version = "1.628.3"
dependencies = [
"anyhow",
"async-nats",
"async-trait",
"axum 0.7.9",
"base64 0.22.1",
"itertools 0.14.0",
"nkeys",
"serde",
"serde_json",
"sqlx",
"tokio",
"tokio-stream",
"tracing",
"windmill-api-auth",
"windmill-common",
"windmill-git-sync",
"windmill-store",
"windmill-trigger",
]
[[package]]
name = "windmill-trigger-postgres"
version = "1.628.3"
dependencies = [
"anyhow",
"async-trait",
"axum 0.7.9",
"byteorder",
"bytes",
"chrono",
"futures",
"itertools 0.14.0",
"lazy_static",
"native-tls",
"pg_escape",
"postgres-native-tls 0.5.0",
"quick_cache",
"rand 0.9.0",
"rust_decimal",
"serde",
"serde_json",
"sqlx",
"thiserror 2.0.18",
"tokio",
"tokio-postgres 0.7.11",
"tokio-stream",
"tracing",
"uuid",
"windmill-api-auth",
"windmill-common",
"windmill-git-sync",
"windmill-store",
"windmill-trigger",
]
[[package]]
name = "windmill-trigger-sqs"
version = "1.628.3"
dependencies = [
"anyhow",
"async-trait",
"aws-config",
"aws-credential-types",
"aws-sdk-sqs",
"aws-sdk-sts",
"axum 0.7.9",
"backon",
"chrono",
"itertools 0.14.0",
"serde",
"serde_json",
"sqlx",
"thiserror 2.0.18",
"tokio",
"tracing",
"windmill-api-auth",
"windmill-common",
"windmill-git-sync",
"windmill-store",
"windmill-trigger",
]
[[package]]
name = "windmill-trigger-websocket"
version = "1.628.3"
dependencies = [
"anyhow",
"async-trait",
"axum 0.7.9",
"futures",
"http 1.4.0",
"itertools 0.14.0",
"serde",
"serde_json",
"sqlx",
"tokio",
"tokio-tungstenite 0.24.0",
"tracing",
"windmill-api-auth",
"windmill-common",
"windmill-git-sync",
"windmill-queue",
"windmill-trigger",
]
[[package]]
name = "windmill-worker"
version = "1.628.3"
backend/Cargo.toml
+32
View File
@@ -8,6 +8,22 @@ edition.workspace = true
resolver = "2"
members = [
"./windmill-api",
"./windmill-api-auth",
"./windmill-api-sse",
"./windmill-api-jobs",
"./windmill-trigger",
"./windmill-trigger-kafka",
"./windmill-trigger-postgres",
"./windmill-trigger-mqtt",
"./windmill-trigger-websocket",
"./windmill-trigger-email",
"./windmill-trigger-nats",
"./windmill-trigger-sqs",
"./windmill-trigger-gcp",
"./windmill-trigger-http",
"./windmill-native-triggers",
"./windmill-api-embeddings",
"./windmill-store",
"./windmill-queue",
"./windmill-worker",
"./windmill-common",
@@ -227,6 +243,22 @@ windmill-indexer = {path = "./windmill-indexer"}
windmill-mcp = {path = "./windmill-mcp"}
windmill-oauth = {path = "./windmill-oauth"}
windmill-macros = {path = "./windmill-macros"}
windmill-api-auth = { path = "./windmill-api-auth" }
windmill-api-sse = { path = "./windmill-api-sse" }
windmill-api-jobs = { path = "./windmill-api-jobs" }
windmill-trigger = { path = "./windmill-trigger" }
windmill-trigger-kafka = { path = "./windmill-trigger-kafka" }
windmill-trigger-postgres = { path = "./windmill-trigger-postgres" }
windmill-trigger-mqtt = { path = "./windmill-trigger-mqtt" }
windmill-trigger-websocket = { path = "./windmill-trigger-websocket" }
windmill-trigger-email = { path = "./windmill-trigger-email" }
windmill-trigger-nats = { path = "./windmill-trigger-nats" }
windmill-trigger-sqs = { path = "./windmill-trigger-sqs" }
windmill-trigger-gcp = { path = "./windmill-trigger-gcp" }
windmill-trigger-http = { path = "./windmill-trigger-http" }
windmill-native-triggers = { path = "./windmill-native-triggers" }
windmill-api-embeddings = { path = "./windmill-api-embeddings" }
windmill-store = { path = "./windmill-store" }
windmill-parser = { path = "./parsers/windmill-parser" }
windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
windmill-parser-py = { path = "./parsers/windmill-parser-py" }
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
7f93a13e96c77292ed4b1e63bc1e8ff1e341d283
ca79631be6c311c5300863634c539593f805e51e
backend/windmill-api-auth/Cargo.toml
+39
View File
@@ -0,0 +1,39 @@
[package]
name = "windmill-api-auth"
version.workspace = true
authors.workspace = true
edition.workspace = true
[lib]
name = "windmill_api_auth"
path = "src/lib.rs"
[features]
default = []
enterprise = ["windmill-common/enterprise", "dep:jsonwebtoken", "dep:reqwest"]
cloud = ["windmill-common/cloud"]
private = ["windmill-common/private"]
no_auth = []
mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth"]
[dependencies]
windmill-common = { workspace = true, default-features = false }
windmill-audit.workspace = true
windmill-mcp = { workspace = true, optional = true }
axum.workspace = true
http.workspace = true
serde.workspace = true
serde_json.workspace = true
uuid.workspace = true
itertools.workspace = true
tower-cookies.workspace = true
tracing.workspace = true
chrono.workspace = true
quick_cache.workspace = true
sqlx.workspace = true
lazy_static.workspace = true
tokio.workspace = true
time.workspace = true
anyhow.workspace = true
jsonwebtoken = { workspace = true, optional = true }
reqwest = { workspace = true, optional = true }
backend/windmill-api-auth/src/auth.rs
+745
View File
@@ -0,0 +1,745 @@
#[cfg(feature = "enterprise")]
use crate::ee_oss::ExternalJwks;
use axum::{
async_trait,
extract::{FromRequestParts, OriginalUri, Query},
Extension, Json,
};
use chrono::TimeZone;
use http::{request::Parts, StatusCode};
use quick_cache::sync::Cache;
use serde::{Deserialize, Serialize};
use sqlx::FromRow;
use tower_cookies::Cookies;
use tracing::Span;
use crate::{ApiAuthed, OptJobAuthed};
use std::{
str::FromStr,
sync::{
atomic::{AtomicI64, AtomicU64, Ordering},
Arc,
},
};
#[cfg(feature = "enterprise")]
use tokio::sync::RwLock;
use windmill_common::DB;
use windmill_common::{
auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, TOKEN_PREFIX_LEN},
error::{Error, JsonResult},
jwt,
users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL},
};
lazy_static::lazy_static! {
// Global auth cache accessible from main.rs for direct invalidation
pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300);
}
// Global function to invalidate a specific token from cache
pub fn invalidate_token_from_cache(token: &str) {
// Remove all cache entries for this token (across all workspaces)
AUTH_CACHE.retain(|(_workspace_id, cached_token), _cached_value| cached_token != token);
tracing::info!(
"Invalidated token from auth cache: {}...",
&token[..token.len().min(8)]
);
}
#[derive(Clone)]
pub struct ExpiringAuthCache {
pub authed: ApiAuthed,
pub expiry: chrono::DateTime<chrono::Utc>,
pub job_id: Option<uuid::Uuid>,
}
pub struct AuthCache {
db: DB,
superadmin_secret: Option<String>,
#[cfg(feature = "enterprise")]
ext_jwks: Option<Arc<RwLock<ExternalJwks>>>,
}
impl AuthCache {
pub fn new(
db: DB,
superadmin_secret: Option<String>,
#[cfg(feature = "enterprise")] ext_jwks: Option<Arc<RwLock<ExternalJwks>>>,
) -> Self {
AuthCache {
db,
superadmin_secret,
#[cfg(feature = "enterprise")]
ext_jwks,
}
}
pub async fn invalidate(&self, w_id: &str, token: String) {
AUTH_CACHE.remove(&(w_id.to_string(), token));
}
pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<ApiAuthed> {
Some(self.get_opt_job_authed(w_id, token).await?.authed)
}
pub async fn get_opt_job_authed(
&self,
w_id: Option<String>,
token: &str,
) -> Option<OptJobAuthed> {
let key = (
w_id.as_ref().unwrap_or(&"".to_string()).to_string(),
token.to_string(),
);
let s = AUTH_CACHE.get(&key).map(|c| c.to_owned());
match s {
Some(ExpiringAuthCache { authed, expiry, job_id }) if expiry > chrono::Utc::now() => {
Some(OptJobAuthed { authed, job_id })
}
#[cfg(feature = "enterprise")]
_ if token.starts_with("jwt_ext_") => {
let authed_and_exp = match crate::ee_oss::jwt_ext_auth(
w_id.as_ref(),
token.trim_start_matches("jwt_ext_"),
self.ext_jwks.clone(),
&self.db,
)
.await
{
Ok(r) => Some(r),
Err(e) => {
tracing::error!("JWT_EXT auth error: {:?}", e);
None
}
};
if let Some((authed, exp, job_id)) = authed_and_exp.clone() {
AUTH_CACHE.insert(
key,
ExpiringAuthCache {
authed: authed.clone(),
expiry: chrono::Utc.timestamp_nanos(exp as i64 * 1_000_000_000),
job_id,
},
);
Some(OptJobAuthed { authed, job_id })
} else {
None
}
}
_ if token.starts_with("jwt_") => {
let jwt_token = token.trim_start_matches("jwt_");
let jwt_result = jwt::decode_with_internal_secret::<JWTAuthClaims>(jwt_token).await;
match jwt_result {
Ok(claims) => {
if w_id.is_some_and(|w_id| !claims.allowed_in_workspace(&w_id)) {
tracing::error!("JWT auth error: workspace_id mismatch");
return None;
}
let username_override = username_override_from_label(claims.label);
let authed = ApiAuthed {
email: claims.email,
username: claims.username,
is_admin: claims.is_admin,
is_operator: claims.is_operator,
groups: claims.groups,
folders: claims.folders,
scopes: None,
username_override,
token_prefix: claims.audit_span,
};
let job_id = claims.job_id.and_then(|j| uuid::Uuid::from_str(&j).ok());
AUTH_CACHE.insert(
key,
ExpiringAuthCache {
authed: authed.clone(),
expiry: chrono::Utc
.timestamp_nanos(claims.exp as i64 * 1_000_000_000),
job_id,
},
);
Some(OptJobAuthed { authed, job_id })
}
Err(err) => {
tracing::error!("JWT auth error: {:?}", err);
None
}
}
}
_ => {
let user_o = sqlx::query!(
"UPDATE token SET last_used_at = now() WHERE
token = $1
AND (expiration > NOW() OR expiration IS NULL)
AND (workspace_id IS NULL OR workspace_id = $2)
RETURNING owner, email, super_admin, scopes, label",
token,
w_id.as_ref(),
)
.map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label))
.fetch_optional(&self.db)
.await
.ok()
.flatten();
if let Some(user) = user_o {
let authed_o = {
match user {
(Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => {
let username_override = username_override_from_label(label);
if let Some((prefix, name)) = owner.split_once('/') {
if prefix == "u" {
let (is_admin, is_operator) = if super_admin {
(true, false)
} else {
let r = sqlx::query!(
"SELECT is_admin, operator FROM usr where username = $1 AND \
workspace_id = $2 AND disabled = false",
name,
&w_id.as_ref().unwrap()
)
.fetch_one(&self.db)
.await
.ok();
if let Some(r) = r {
(r.is_admin, r.operator)
} else {
(false, true)
}
};
let w_id = &w_id.unwrap();
let groups =
get_groups_for_user(w_id, &name, &email, &self.db)
.await
.ok()
.unwrap_or_default();
let folders =
get_folders_for_user(w_id, &name, &groups, &self.db)
.await
.ok()
.unwrap_or_default();
Some(ApiAuthed {
email: email,
username: name.to_string(),
is_admin,
is_operator,
groups,
folders,
scopes: None,
username_override,
token_prefix: Some(
token[0..TOKEN_PREFIX_LEN].to_string(),
),
})
} else {
let groups = vec![name.to_string()];
let folders = get_folders_for_user(
&w_id.unwrap(),
"",
&groups,
&self.db,
)
.await
.ok()
.unwrap_or_default();
Some(ApiAuthed {
email: email,
username: format!("group-{name}"),
is_admin: false,
groups,
is_operator: false,
folders,
scopes: None,
username_override,
token_prefix: Some(
token[0..TOKEN_PREFIX_LEN].to_string(),
),
})
}
} else {
let groups = vec![];
let folders = vec![];
Some(ApiAuthed {
email: email,
username: owner,
is_admin: super_admin,
is_operator: true,
groups,
folders,
scopes: None,
username_override,
token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()),
})
}
}
(_, Some(email), super_admin, scopes, label) => {
let username_override = username_override_from_label(label);
if w_id.is_some() {
let row_o = sqlx::query!(
"SELECT username, is_admin, operator FROM usr WHERE
email = $1 AND workspace_id = $2 AND disabled = false",
&email,
w_id.as_ref().unwrap()
)
.map(|x| (x.username, x.is_admin, x.operator))
.fetch_optional(&self.db)
.await
.unwrap_or(Some(("error".to_string(), false, false)));
match row_o {
Some((username, is_admin, is_operator)) => {
let groups = get_groups_for_user(
&w_id.as_ref().unwrap(),
&username,
&email,
&self.db,
)
.await
.ok()
.unwrap_or_default();
let folders = get_folders_for_user(
&w_id.unwrap(),
&username,
&groups,
&self.db,
)
.await
.ok()
.unwrap_or_default();
Some(ApiAuthed {
email,
username,
is_admin: is_admin || super_admin,
is_operator,
groups,
folders,
scopes,
username_override,
token_prefix: Some(
token[0..TOKEN_PREFIX_LEN].to_string(),
),
})
}
None if super_admin => Some(ApiAuthed {
email: email.clone(),
username: email,
is_admin: super_admin,
is_operator: false,
groups: vec![],
folders: vec![],
scopes,
username_override,
token_prefix: Some(
token[0..TOKEN_PREFIX_LEN].to_string(),
),
}),
None => None,
}
} else {
Some(ApiAuthed {
email: email.to_string(),
username: email,
is_admin: super_admin,
is_operator: true,
groups: Vec::new(),
folders: Vec::new(),
scopes,
username_override,
token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()),
})
}
}
_ => None,
}
};
if let Some(authed) = authed_o.as_ref() {
AUTH_CACHE.insert(
key,
ExpiringAuthCache {
authed: authed.clone(),
expiry: chrono::Utc::now()
+ chrono::Duration::try_seconds(120).unwrap(),
job_id: None,
},
);
}
authed_o.map(|authed| OptJobAuthed { authed, job_id: None })
} else if self
.superadmin_secret
.as_ref()
.map(|x| x == token)
.unwrap_or(false)
{
let authed = ApiAuthed {
email: SUPERADMIN_SECRET_EMAIL.to_string(),
username: "superadmin_secret".to_string(),
is_admin: true,
is_operator: false,
groups: Vec::new(),
folders: Vec::new(),
scopes: None,
username_override: None,
token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()),
};
Some(OptJobAuthed { authed, job_id: None })
} else {
None
}
}
}
}
}
pub(crate) async fn extract_token<S: Send + Sync>(parts: &mut Parts, state: &S) -> Option<String> {
let auth_header = parts
.headers
.get(http::header::AUTHORIZATION)
.and_then(|value| value.to_str().ok())
.and_then(|s| s.strip_prefix("Bearer "));
let from_cookie = match auth_header {
Some(x) => Some(x.to_owned()),
None => Extension::<Cookies>::from_request_parts(parts, state)
.await
.ok()
.and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())),
};
#[derive(Deserialize)]
struct Token {
token: Option<String>,
}
match from_cookie {
Some(token) => Some(token),
None => Query::<Token>::from_request_parts(parts, state)
.await
.ok()
.and_then(|token| token.token.clone()),
}
}
#[derive(Clone, Debug)]
pub struct Tokened {
pub token: String,
}
#[derive(Clone, Debug)]
pub struct OptTokened {
#[allow(dead_code)]
pub token: Option<String>,
}
struct BruteForceCounter {
counter: AtomicU64,
last_reset: AtomicI64,
}
lazy_static::lazy_static! {
static ref BRUTE_FORCE_COUNTER: BruteForceCounter =
BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) };
}
impl BruteForceCounter {
async fn increment(&self) {
let now = time::OffsetDateTime::now_utc().unix_timestamp();
if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 {
tracing::error!(
"Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds"
);
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
if now - self.last_reset.load(Ordering::Relaxed) > 60 {
self.counter.store(0, Ordering::Relaxed);
self.last_reset.store(now, Ordering::Relaxed);
}
}
}
#[async_trait]
impl<S> FromRequestParts<S> for Tokened
where
S: Send + Sync,
{
type Rejection = (StatusCode, String);
async fn from_request_parts(
parts: &mut Parts,
state: &S,
) -> std::result::Result<Self, Self::Rejection> {
if parts.method == http::Method::OPTIONS {
return Ok(Tokened { token: "".to_string() });
};
let already_tokened = parts.extensions.get::<Tokened>();
if let Some(tokened) = already_tokened {
Ok(tokened.clone())
} else {
let token_o = extract_token(parts, state).await;
if let Some(token) = token_o {
let tokened = Self { token };
parts.extensions.insert(tokened.clone());
Ok(tokened)
} else {
BRUTE_FORCE_COUNTER.increment().await;
Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
}
}
}
}
#[async_trait]
impl<S> FromRequestParts<S> for OptTokened
where
S: Send + Sync,
{
type Rejection = (StatusCode, String);
async fn from_request_parts(
parts: &mut Parts,
state: &S,
) -> std::result::Result<Self, Self::Rejection> {
if parts.method == http::Method::OPTIONS {
return Ok(OptTokened { token: None });
};
let already_tokened = parts.extensions.get::<Tokened>();
if let Some(tokened) = already_tokened {
Ok(OptTokened { token: Some(tokened.token.clone()) })
} else {
let token_o = extract_token(parts, state).await;
Ok(OptTokened { token: token_o })
}
}
}
pub fn transform_old_scope_to_new_scope(scopes: Option<&mut Vec<String>>) {
if let Some(scopes) = scopes {
for scope in scopes.iter_mut() {
if scope.starts_with("run:") {
let (_, part_scope) = scope.split_once(":").unwrap();
if let Some((kind, path)) = part_scope.split_once("/") {
//appending a 's' as runnable kind is singular while new scope format expect it to be plural
*scope = format!("jobs:run:{}s:{}", kind, path);
}
} else if scope.starts_with("jobs:") {
// Map old jobs scopes to new format
let new_scope = match scope.as_str() {
"jobs:listjobs" => "jobs:read",
"jobs:runscript" => "jobs:run:scripts",
"jobs:runflow" => "jobs:run:flows",
"jobs:resumeflow" => "jobs:run:flows",
"jobs:deletejob" => "jobs:write",
_ => continue,
};
*scope = new_scope.to_string();
}
}
}
}
fn maybe_get_workspace_id_from_path(path_vec: &[&str]) -> Option<String> {
let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" {
Some(path_vec[3].to_owned())
} else if path_vec.len() >= 5
&& path_vec[0] == ""
&& path_vec[1] == "api"
&& path_vec[2] == "mcp"
&& path_vec[3] == "w"
{
Some(path_vec[4].to_owned())
} else {
if path_vec.len() >= 5 && path_vec[0] == "" && path_vec[2] == "srch" && path_vec[3] == "w" {
Some(path_vec[4].to_owned())
} else {
None
}
};
workspace_id
}
/// Resolves OptJobAuthed from request parts.
/// Takes ownership of Parts and returns them back.
#[allow(unreachable_code, unused_mut)]
pub async fn resolve_opt_job_authed(
mut parts: Parts,
) -> std::result::Result<(OptJobAuthed, Parts), (Error, Parts)> {
if parts.method == http::Method::OPTIONS {
return Ok((OptJobAuthed::default(), parts));
};
#[cfg(feature = "no_auth")]
{
let authed = ApiAuthed {
email: "admin@windmill.dev".to_string(),
username: "admin".to_string(),
is_admin: true,
is_operator: false,
groups: Vec::new(),
folders: Vec::new(),
scopes: None,
username_override: None,
token_prefix: None,
};
return Ok((OptJobAuthed { authed, job_id: None }, parts));
}
let already_authed = parts.extensions.get::<OptJobAuthed>().cloned();
if let Some(authed) = already_authed {
return Ok((authed, parts));
}
let already_tokened = parts.extensions.get::<Tokened>().cloned();
let token_o = if let Some(token) = already_tokened {
Some(token.token.clone())
} else {
extract_token(&mut parts, &()).await
};
if let Some(token) = token_o {
if let Ok(Extension(cache)) =
Extension::<Arc<AuthCache>>::from_request_parts(&mut parts, &()).await
{
let original_uri = OriginalUri::from_request_parts(&mut parts, &())
.await
.ok()
.map(|x| x.0)
.unwrap_or_default();
let path_vec: Vec<&str> = original_uri.path().split("/").collect();
let workspace_id = maybe_get_workspace_id_from_path(&path_vec);
if let Some(mut opt_job_authed) =
cache.get_opt_job_authed(workspace_id.clone(), &token).await
{
let authed = &mut opt_job_authed.authed;
if authed.scopes.is_some() {
transform_old_scope_to_new_scope(authed.scopes.as_mut());
let path = original_uri.path();
let method = parts.method.as_str();
if let Err(err) = crate::scopes::check_scopes_for_route(
authed.scopes.as_deref(),
path,
method,
) {
BRUTE_FORCE_COUNTER.increment().await;
return Err((err, parts));
}
}
parts.extensions.insert(authed.clone());
Span::current().record("username", &authed.username.as_str());
Span::current().record("email", &authed.email);
if let Some(workspace_id) = workspace_id {
Span::current().record("workspace_id", &workspace_id);
}
return Ok((opt_job_authed, parts));
}
}
}
BRUTE_FORCE_COUNTER.increment().await;
Err((Error::NotAuthorized("Unauthorized".to_string()), parts))
}
fn username_override_from_label(label: Option<String>) -> Option<String> {
match label {
Some(label)
if label.starts_with("webhook-")
|| label.starts_with("http-")
|| label.starts_with("email-")
|| label.starts_with("ws-") =>
{
Some(label)
}
Some(label) if label.starts_with("ephemeral-script-end-user-") => Some(
label
.trim_start_matches("ephemeral-script-end-user-")
.to_string(),
),
Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()),
Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => {
Some(format!("label-{label}"))
}
_ => None,
}
}
#[derive(FromRow, Serialize)]
pub struct TruncatedTokenWithEmail {
pub label: Option<String>,
pub token_prefix: Option<String>,
pub expiration: Option<chrono::DateTime<chrono::Utc>>,
pub created_at: chrono::DateTime<chrono::Utc>,
pub last_used_at: chrono::DateTime<chrono::Utc>,
pub scopes: Option<Vec<String>>,
pub email: Option<String>,
}
pub async fn list_tokens_internal(
db: &DB,
w_id: &str,
path: &str,
is_flow: bool,
) -> JsonResult<Vec<TruncatedTokenWithEmail>> {
let tokens = if is_flow {
sqlx::query_as!(
TruncatedTokenWithEmail,
r#"
SELECT label,
concat(substring(token for 10)) AS token_prefix,
expiration,
created_at,
last_used_at,
scopes,
email
FROM token
WHERE workspace_id = $1
AND (
scopes @> ARRAY['jobs:run:flows:' || $2]::text[]
OR scopes @> ARRAY['run:flow/' || $2]::text[]
)
"#,
w_id,
path
)
.fetch_all(db)
.await?
} else {
sqlx::query_as!(
TruncatedTokenWithEmail,
r#"
SELECT label,
concat(substring(token for 10)) AS token_prefix,
expiration,
created_at,
last_used_at,
scopes,
email
FROM token
WHERE workspace_id = $1
AND (
scopes @> ARRAY['jobs:run:scripts:' || $2]::text[]
OR scopes @> ARRAY['run:script/' || $2]::text[]
)
"#,
w_id,
path
)
.fetch_all(db)
.await?
};
Ok(Json(tokens))
}
backend/windmill-api-auth/src/ee_oss.rs
+36
View File
@@ -0,0 +1,36 @@
/*
* Author: Windmill Labs, Inc
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
#[cfg(feature = "private")]
#[allow(unused)]
pub use crate::ee::*;
#[cfg(all(feature = "enterprise", not(feature = "private")))]
use {std::sync::Arc, tokio::sync::RwLock};
#[cfg(all(feature = "enterprise", not(feature = "private")))]
pub async fn jwt_ext_auth(
_w_id: Option<&String>,
_token: &str,
_external_jwks: Option<Arc<RwLock<ExternalJwks>>>,
_db: &windmill_common::DB,
) -> anyhow::Result<(crate::ApiAuthed, usize, Option<uuid::Uuid>)> {
// Implementation is not open source
Err(anyhow::anyhow!("External JWT auth is not open source"))
}
#[cfg(all(feature = "enterprise", not(feature = "private")))]
pub struct ExternalJwks;
#[cfg(all(feature = "enterprise", not(feature = "private")))]
impl ExternalJwks {
pub async fn load() -> Option<Arc<RwLock<Self>>> {
// Implementation is not open source
None
}
}
backend/windmill-api-auth/src/lib.rs
+712
View File
@@ -0,0 +1,712 @@
/*
* Author: Windmill Labs, Inc
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
pub mod auth;
#[cfg(feature = "private")]
pub mod ee;
pub mod ee_oss;
pub mod scopes;
use axum::async_trait;
use axum::extract::FromRequestParts;
use http::request::Parts;
use windmill_audit::audit_oss::AuditAuthorable;
use windmill_common::{
auth::{fetch_authed_from_permissioned_as, is_devops_email, is_super_admin_email},
db::{Authable, Authed, AuthedRef},
error::{self, Error, Result},
users::username_to_permissioned_as,
DB,
};
use scopes::ScopeDefinition;
// Re-export key auth types and functions
pub use auth::{
invalidate_token_from_cache, AuthCache, ExpiringAuthCache, OptTokened, Tokened,
TruncatedTokenWithEmail, AUTH_CACHE,
};
// ------------ ApiAuthed & OptJobAuthed types ------------
#[derive(Default, Clone, Debug)]
pub struct OptJobAuthed {
pub job_id: Option<uuid::Uuid>,
pub authed: ApiAuthed,
}
#[derive(Clone, Debug, Default, Hash, Eq, PartialEq)]
pub struct ApiAuthed {
pub email: String,
pub username: String,
pub is_admin: bool,
pub is_operator: bool,
pub groups: Vec<String>,
// (folder name, can write, is owner)
pub folders: Vec<(String, bool, bool)>,
pub scopes: Option<Vec<String>>,
pub username_override: Option<String>,
pub token_prefix: Option<String>,
}
impl ApiAuthed {
pub fn to_authed_ref<'e>(&'e self) -> AuthedRef<'e> {
AuthedRef {
email: &self.email,
username: &self.username,
is_admin: &self.is_admin,
is_operator: &self.is_operator,
groups: &self.groups,
folders: &self.folders,
scopes: &self.scopes,
token_prefix: &self.token_prefix,
}
}
pub fn display_username(&self) -> &str {
self.username_override.as_ref().unwrap_or(&self.username)
}
}
impl From<ApiAuthed> for Authed {
fn from(value: ApiAuthed) -> Self {
Self {
email: value.email,
username: value.username,
is_admin: value.is_admin,
is_operator: value.is_operator,
groups: value.groups,
folders: value.folders,
scopes: value.scopes,
token_prefix: value.token_prefix,
}
}
}
impl From<Authed> for ApiAuthed {
fn from(value: Authed) -> Self {
Self {
email: value.email,
username: value.username,
is_admin: value.is_admin,
is_operator: value.is_operator,
groups: value.groups,
folders: value.folders,
scopes: value.scopes,
username_override: None,
token_prefix: value.token_prefix,
}
}
}
impl AuditAuthorable for ApiAuthed {
fn username(&self) -> &str {
self.username.as_str()
}
fn email(&self) -> &str {
self.email.as_str()
}
fn username_override(&self) -> Option<&str> {
self.username_override.as_deref()
}
fn token_prefix(&self) -> Option<&str> {
self.token_prefix.as_deref()
}
}
impl Authable for ApiAuthed {
fn is_admin(&self) -> bool {
self.is_admin
}
fn is_operator(&self) -> bool {
self.is_operator
}
fn groups(&self) -> &[String] {
&self.groups
}
fn folders(&self) -> &[(String, bool, bool)] {
&self.folders
}
fn scopes(&self) -> Option<&[std::string::String]> {
self.scopes.as_ref().map(|x| x.as_slice())
}
fn email(&self) -> &str {
&self.email
}
fn username(&self) -> &str {
&self.username
}
}
// ------------ McpAuth impl (feature-gated) ------------
#[cfg(feature = "mcp")]
impl windmill_mcp::server::McpAuth for ApiAuthed {
fn username(&self) -> &str {
&self.username
}
fn email(&self) -> &str {
&self.email
}
fn is_admin(&self) -> bool {
self.is_admin
}
fn is_operator(&self) -> bool {
self.is_operator
}
fn groups(&self) -> &[String] {
&self.groups
}
fn folders(&self) -> &[(String, bool, bool)] {
&self.folders
}
fn scopes(&self) -> Option<&[String]> {
self.scopes.as_deref()
}
}
// ------------ Utility functions ------------
pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> {
let is_admin = is_super_admin_email(db, email).await?;
if !is_admin {
Err(Error::NotAuthorized(
"This endpoint requires the caller to be a super admin".to_owned(),
))
} else {
Ok(())
}
}
pub fn check_scopes<F>(authed: &ApiAuthed, required: F) -> error::Result<()>
where
F: FnOnce() -> String,
{
if let Some(scopes) = authed.scopes.as_ref() {
let mut is_scoped_token = false;
let required_scope = ScopeDefinition::from_scope_string(&required())?;
for scope in scopes {
if !scope.starts_with("if_jobs:filter_tags:") {
if !is_scoped_token {
is_scoped_token = true;
}
match ScopeDefinition::from_scope_string(scope) {
Ok(scope) if scope.includes(&required_scope) => return Ok(()),
_ => {}
}
}
}
if is_scoped_token {
return Err(Error::NotAuthorized(format!(
"Required scope: {}",
required_scope.as_string()
)));
}
}
Ok(())
}
pub async fn require_devops_role(db: &DB, email: &str) -> error::Result<()> {
let is_devops = is_devops_email(db, email).await?;
if is_devops {
Ok(())
} else {
Err(Error::NotAuthorized(
"This endpoint requires the caller to have the `devops` role".to_string(),
))
}
}
// ------------ Folder ownership checks ------------
pub fn is_owner(ApiAuthed { is_admin, folders, .. }: &ApiAuthed, name: &str) -> bool {
if *is_admin {
true
} else {
folders.into_iter().any(|x| x.0 == name && x.2)
}
}
pub fn require_is_owner(authed: &ApiAuthed, name: &str) -> Result<()> {
if is_owner(authed, name) {
Ok(())
} else {
Err(Error::NotAuthorized(format!(
"You are not owner of the folder {}",
name
)))
}
}
pub fn require_owner_of_path(authed: &ApiAuthed, path: &str) -> Result<()> {
if authed.is_admin {
return Ok(());
}
if !path.is_empty() {
let splitted = path.split("/").collect::<Vec<&str>>();
if splitted[0] == "u" {
if splitted[1] == authed.username {
Ok(())
} else {
Err(Error::BadRequest(format!(
"only the owner {} is authorized to perform this operation",
splitted[1]
)))
}
} else if splitted[0] == "f" {
require_is_owner(authed, splitted[1])
} else {
Err(Error::BadRequest(format!(
"Not recognized path kind: {}",
path
)))
}
} else {
Err(Error::BadRequest(
"Cannot be owner of an empty path".to_string(),
))
}
}
// ------------ Scope tag helpers ------------
pub fn get_scope_tags(authed: &ApiAuthed) -> Option<Vec<&str>> {
authed.scopes.as_ref()?.iter().find_map(|s| {
if s.starts_with("if_jobs:filter_tags:") {
Some(
s.trim_start_matches("if_jobs:filter_tags:")
.split(",")
.collect::<Vec<_>>(),
)
} else {
None
}
})
}
// ------------ Maybe refresh folders ------------
pub async fn maybe_refresh_folders(
path: &str,
w_id: &str,
authed: ApiAuthed,
db: &DB,
) -> ApiAuthed {
use windmill_common::auth::{get_folders_for_user, get_groups_for_user};
if authed.is_admin {
return authed;
}
let splitted = path.split('/').collect::<Vec<_>>();
if splitted.len() >= 2
&& splitted[0] == "f"
&& !authed.folders.iter().any(|(f, _, _)| f == splitted[1])
{
let name = &authed.username;
let groups = get_groups_for_user(w_id, name, &authed.email, db)
.await
.ok()
.unwrap_or_default();
let folders = get_folders_for_user(w_id, name, &groups, db)
.await
.ok()
.unwrap_or_default();
ApiAuthed { folders, ..authed }
} else {
authed
}
}
// ------------ FromRequestParts impls (direct call to auth module) ------------
#[async_trait]
impl<S> FromRequestParts<S> for ApiAuthed
where
S: Send + Sync,
{
type Rejection = Error;
async fn from_request_parts(
parts: &mut Parts,
_state: &S,
) -> std::result::Result<Self, Self::Rejection> {
let opt_job_authed = OptJobAuthed::from_request_parts(parts, _state).await?;
Ok(opt_job_authed.authed)
}
}
#[async_trait]
impl<S> FromRequestParts<S> for OptJobAuthed
where
S: Send + Sync,
{
type Rejection = Error;
async fn from_request_parts(
parts: &mut Parts,
_state: &S,
) -> std::result::Result<Self, Self::Rejection> {
// Swap out parts so we can pass ownership to resolve_opt_job_authed
let owned_parts = std::mem::replace(parts, empty_parts());
match auth::resolve_opt_job_authed(owned_parts).await {
Ok((result, returned_parts)) => {
*parts = returned_parts;
Ok(result)
}
Err((err, returned_parts)) => {
*parts = returned_parts;
Err(err)
}
}
}
}
fn empty_parts() -> Parts {
let (parts, _body) = http::Request::new(()).into_parts();
parts
}
// ------------ OptAuthed (optional auth extractor) ------------
#[derive(Clone, Debug)]
pub struct OptAuthed(pub Option<ApiAuthed>);
#[async_trait]
impl<S> FromRequestParts<S> for OptAuthed
where
S: Send + Sync,
{
type Rejection = (http::StatusCode, String);
async fn from_request_parts(
parts: &mut Parts,
state: &S,
) -> std::result::Result<Self, Self::Rejection> {
ApiAuthed::from_request_parts(parts, state)
.await
.map(|authed| Self(Some(authed)))
.or_else(|_| Ok(Self(None)))
}
}
// ------------ fetch_api_authed helpers ------------
lazy_static::lazy_static! {
static ref API_AUTHED_CACHE: quick_cache::sync::Cache<(String,String,String), ExpiringAuthCache> = quick_cache::sync::Cache::new(300);
}
#[allow(unused)]
pub async fn fetch_api_authed(
username: String,
email: String,
w_id: &str,
db: &DB,
username_override: Option<String>,
) -> error::Result<ApiAuthed> {
let permissioned_as = username_to_permissioned_as(username.as_str());
fetch_api_authed_from_permissioned_as(permissioned_as, email, w_id, db, username_override).await
}
#[allow(unused)]
pub async fn fetch_api_authed_from_permissioned_as(
permissioned_as: String,
email: String,
w_id: &str,
db: &DB,
username_override: Option<String>,
) -> error::Result<ApiAuthed> {
let key = (w_id.to_string(), permissioned_as.clone(), email.clone());
let mut api_authed = match API_AUTHED_CACHE.get(&key) {
Some(expiring_authed) if expiring_authed.expiry > chrono::Utc::now() => {
tracing::debug!("API authed cache hit for user {}", email);
expiring_authed.authed
}
_ => {
tracing::debug!("API authed cache miss for user {}", email);
let authed =
fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?;
let api_authed = ApiAuthed {
username: authed.username,
email,
is_admin: authed.is_admin,
is_operator: authed.is_operator,
groups: authed.groups,
folders: authed.folders,
scopes: authed.scopes,
username_override: None,
token_prefix: authed.token_prefix,
};
API_AUTHED_CACHE.insert(
key,
ExpiringAuthCache {
authed: api_authed.clone(),
expiry: chrono::Utc::now() + chrono::Duration::try_seconds(120).unwrap(),
job_id: None,
},
);
api_authed
}
};
api_authed.username_override = username_override;
Ok(api_authed)
}
// ------------ Token creation ------------
#[derive(serde::Deserialize)]
pub struct NewToken {
pub label: Option<String>,
pub expiration: Option<chrono::DateTime<chrono::Utc>>,
pub impersonate_email: Option<String>,
pub scopes: Option<Vec<String>>,
pub workspace_id: Option<String>,
}
impl NewToken {
pub fn new(
label: Option<String>,
expiration: Option<chrono::DateTime<chrono::Utc>>,
impersonate_email: Option<String>,
scopes: Option<Vec<String>>,
workspace_id: Option<String>,
) -> Self {
Self { label, expiration, impersonate_email, scopes, workspace_id }
}
}
pub async fn create_token_internal(
tx: &mut sqlx::PgConnection,
db: &DB,
authed: &ApiAuthed,
token_config: NewToken,
) -> Result<String> {
use tracing::Instrument;
use windmill_audit::{audit_oss::audit_log, ActionKind};
use windmill_common::{utils::rd_string, worker::CLOUD_HOSTED};
let token = rd_string(32);
let is_super_admin = sqlx::query_scalar!(
"SELECT super_admin FROM password WHERE email = $1",
authed.email
)
.fetch_optional(&mut *tx)
.await?
.unwrap_or(false);
if *CLOUD_HOSTED {
let nb_tokens =
sqlx::query_scalar!("SELECT COUNT(*) FROM token WHERE email = $1", &authed.email)
.fetch_one(db)
.await?;
if nb_tokens.unwrap_or(0) >= 10000 {
return Err(Error::BadRequest(
"You have reached the maximum number of tokens (10000) on cloud. Contact support@windmill.dev to increase the limit"
.to_string(),
));
}
}
sqlx::query!(
"INSERT INTO token
(token, email, label, expiration, super_admin, scopes, workspace_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)",
token,
authed.email,
token_config.label,
token_config.expiration,
is_super_admin,
token_config.scopes.as_ref().map(|x| x.as_slice()),
token_config.workspace_id,
)
.execute(&mut *tx)
.await?;
audit_log(
&mut *tx,
authed,
"users.token.create",
ActionKind::Create,
&"global",
Some(&token[0..10]),
None,
)
.instrument(tracing::info_span!("token", email = &authed.email))
.await?;
Ok(token)
}
// ------------ Permission helpers ------------
pub fn get_perm_in_extra_perms_for_authed(
v: serde_json::Value,
authed: &ApiAuthed,
) -> Option<bool> {
match v {
serde_json::Value::Object(obj) => {
let mut keys = vec![format!("u/{}", authed.username)];
for g in authed.groups.iter() {
keys.push(format!("g/{}", g));
}
let mut res = None;
for k in keys {
if let Some(v) = obj.get(&k) {
if let Some(v) = v.as_bool() {
if v {
return Some(true);
}
res = Some(v);
}
}
}
res
}
_ => None,
}
}
pub async fn require_is_writer(
authed: &ApiAuthed,
path: &str,
w_id: &str,
db: DB,
query: &str,
kind: &str,
) -> Result<()> {
if authed.is_admin {
return Ok(());
}
if !path.is_empty() {
if require_owner_of_path(authed, path).is_ok() {
return Ok(());
}
if path.starts_with("f/") && path.split('/').count() >= 2 {
let folder = path.split('/').nth(1).unwrap();
let extra_perms = sqlx::query_scalar!(
"SELECT extra_perms FROM folder WHERE name = $1 AND workspace_id = $2",
folder,
w_id
)
.fetch_optional(&db)
.await?;
if let Some(perms) = extra_perms {
let is_folder_writer =
get_perm_in_extra_perms_for_authed(perms, authed).unwrap_or(false);
if is_folder_writer {
return Ok(());
}
}
}
let extra_perms = sqlx::query_scalar(query)
.bind(path)
.bind(w_id)
.fetch_optional(&db)
.await?;
if let Some(perms) = extra_perms {
let perm = get_perm_in_extra_perms_for_authed(perms, authed);
match perm {
Some(true) => Ok(()),
Some(false) => Err(Error::BadRequest(format!(
"User {} is not a writer of {kind} path {path}",
authed.username
))),
None => Err(Error::BadRequest(format!(
"User {} has neither read or write permission on {kind} {path}",
authed.username
))),
}
} else {
Err(Error::BadRequest(format!(
"{path} does not exist yet and user {} is not an owner of the parent folder",
authed.username
)))
}
} else {
Err(Error::BadRequest(format!(
"Cannot be writer of an empty path"
)))
}
}
// ------------ Preview access check ------------
pub fn require_path_read_access_for_preview(
authed: &ApiAuthed,
path: &Option<String>,
) -> Result<()> {
let Some(path) = path else {
return Ok(());
};
if authed.is_admin {
return Ok(());
}
if path.is_empty() {
return Ok(());
}
let splitted: Vec<&str> = path.split('/').collect();
if splitted.len() < 2 {
return Err(Error::BadRequest(format!(
"Invalid path format for preview job: {}",
path
)));
}
match splitted[0] {
"u" => {
if splitted[1] == authed.username {
Ok(())
} else {
Err(Error::BadRequest(format!(
"You can only run preview jobs in your own namespace (u/{}) or in folders you have read access to",
authed.username
)))
}
}
"f" => {
let folder = splitted[1];
if authed.folders.iter().any(|(f, _, _)| f == folder) {
Ok(())
} else {
Err(Error::BadRequest(format!(
"You do not have read access to folder '{}'. Preview jobs require at least read access to the target folder.",
folder
)))
}
}
"hub" => Ok(()),
_ => Err(Error::BadRequest(format!(
"Invalid path format for preview job: {}. Path must start with 'u/' or 'f/'",
path
))),
}
}
backend/windmill-api/src/scopes.rs → backend/windmill-api-auth/src/scopes.rs
+8 -8
View File
@@ -230,6 +230,10 @@ fn resource_matches_pattern(scope_resource: &str, accepted_resource: &str) -> bo
|| matches_wildcard(accepted_resource, scope_resource)
}
// ─────────────────────────────────────────────────────────────────
// Route-level scope checking
// ─────────────────────────────────────────────────────────────────
/// Available scope domains (top-level API categories)
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ScopeDomain {
@@ -883,8 +887,6 @@ mod tests {
#[test]
fn test_user_example_case() {
// User's example: scope has "u/dieri/*", required has ["u/dadad/wqdq", "u/*"]
// Should grant access because scope "u/dieri/*" falls under required pattern "u/*"
let user_scope =
ScopeDefinition::new("scripts", "read", None, Some(vec!["u/dieri/*".to_string()]));
let required_mixed = ScopeDefinition::new(
@@ -893,9 +895,8 @@ mod tests {
None,
Some(vec!["u/dadad/wqdq".to_string(), "u/*".to_string()]),
);
assert!(user_scope.includes(&required_mixed)); // Should match because u/dieri/* falls under u/*
assert!(user_scope.includes(&required_mixed));
// Another example: scope covers one but not both paths
let scope_specific = ScopeDefinition::new(
"scripts",
"read",
@@ -908,9 +909,8 @@ mod tests {
None,
Some(vec!["folder/file1".to_string(), "other/file2".to_string()]),
);
assert!(scope_specific.includes(&required_multi)); // Should match because folder/file1 matches exactly
assert!(scope_specific.includes(&required_multi));
// Test bidirectional pattern matching more explicitly
let scope_broad =
ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()]));
let required_specific = ScopeDefinition::new(
@@ -919,7 +919,7 @@ mod tests {
None,
Some(vec!["u/dieri/script.py".to_string()]),
);
assert!(scope_broad.includes(&required_specific)); // u/* covers u/dieri/script.py
assert!(scope_broad.includes(&required_specific));
let scope_specific_path = ScopeDefinition::new(
"scripts",
@@ -929,6 +929,6 @@ mod tests {
);
let required_broad =
ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()]));
assert!(scope_specific_path.includes(&required_broad)); // u/dieri/script.py satisfies u/*
assert!(scope_specific_path.includes(&required_broad));
}
}
backend/windmill-api-embeddings/Cargo.toml
+32
View File
@@ -0,0 +1,32 @@
[package]
name = "windmill-api-embeddings"
version.workspace = true
authors.workspace = true
edition.workspace = true
[lib]
name = "windmill_api_embeddings"
path = "src/lib.rs"
[features]
default = []
embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"]
[dependencies]
windmill-common = { workspace = true, default-features = false }
windmill-store.workspace = true
axum.workspace = true
serde.workspace = true
serde_json.workspace = true
sqlx.workspace = true
reqwest.workspace = true
tokio.workspace = true
tracing.workspace = true
lazy_static.workspace = true
anyhow.workspace = true
tinyvector = { workspace = true, optional = true }
hf-hub = { workspace = true, optional = true }
tokenizers = { workspace = true, optional = true }
candle-core = { workspace = true, optional = true }
candle-transformers = { workspace = true, optional = true }
candle-nn = { workspace = true, optional = true }
backend/windmill-api-embeddings/src/lib.rs
+637
View File
@@ -0,0 +1,637 @@
#[cfg(feature = "embedding")]
use anyhow::{anyhow, Error, Result};
#[cfg(feature = "embedding")]
use std::{collections::HashMap, path::PathBuf, sync::Arc};
#[cfg(feature = "embedding")]
use windmill_common::DEFAULT_HUB_BASE_URL;
#[cfg(feature = "embedding")]
use windmill_common::HUB_BASE_URL;
use axum::Router;
#[cfg(feature = "embedding")]
use axum::{
extract::{Path, Query},
Json,
};
#[cfg(feature = "embedding")]
use axum::routing::get;
#[cfg(feature = "embedding")]
use candle_core::{Device, Tensor};
#[cfg(feature = "embedding")]
use candle_nn::VarBuilder;
#[cfg(feature = "embedding")]
use candle_transformers::models::bert::{BertModel, Config, DTYPE};
#[cfg(feature = "embedding")]
use hf_hub::api::tokio::Api;
#[cfg(feature = "embedding")]
use serde::Deserialize;
#[cfg(feature = "embedding")]
use serde::Serialize;
#[cfg(feature = "embedding")]
use sqlx::{Pool, Postgres};
#[cfg(feature = "embedding")]
use tinyvector::{
db::{Db, Embedding},
similarity::Distance,
};
#[cfg(feature = "embedding")]
use tokenizers::Tokenizer;
#[cfg(feature = "embedding")]
use tokio::sync::RwLock;
#[cfg(feature = "embedding")]
use windmill_common::utils::http_get_from_hub;
#[cfg(feature = "embedding")]
use windmill_common::error::JsonResult;
#[cfg(feature = "embedding")]
use windmill_store::resources::ResourceType;
#[cfg(feature = "embedding")]
lazy_static::lazy_static! {
pub static ref EMBEDDINGS_DB: Arc<RwLock<Option<EmbeddingsDb>>> = Arc::new(RwLock::new(None));
pub static ref MODEL_INSTANCE: Arc<RwLock<Option<Arc<ModelInstance>>>> = Arc::new(RwLock::new(None));
pub static ref HUB_EMBEDDINGS_PULLING_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_PULLING_INTERVAL_SECS").ok().map(|x| x.parse::<u64>().ok()).flatten().unwrap_or(3600 * 24);
// Match windmill-api's HTTP_CLIENT config: 30s timeout, ACCEPT_INVALID_CERTS support
static ref HTTP_CLIENT: reqwest::Client = windmill_common::utils::configure_client(reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.connect_timeout(std::time::Duration::from_secs(10))
.timeout(std::time::Duration::from_secs(30))
.danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok()))
.build().unwrap();
}
#[cfg(feature = "embedding")]
#[derive(Deserialize)]
struct HubScriptsQuery {
text: String,
limit: Option<i64>,
kind: Option<String>,
app: Option<String>,
}
#[cfg(feature = "embedding")]
#[derive(Serialize)]
pub struct HubScriptResult {
ask_id: i64,
id: i64,
version_id: i64,
summary: String,
app: String,
kind: String,
score: f32,
}
#[cfg(feature = "embedding")]
async fn query_hub_scripts(
Query(query): Query<HubScriptsQuery>,
) -> JsonResult<Vec<HubScriptResult>> {
let embeddings_db = EMBEDDINGS_DB.read().await;
if let Some(embeddings_db) = embeddings_db.as_ref() {
let results = embeddings_db
.query_hub_scripts(&query.text, query.limit, query.kind, query.app)
.await?;
Ok(Json(results))
} else {
Err(windmill_common::error::Error::internal_err(
"Embeddings db not initialized".to_string(),
))
}
}
#[cfg(feature = "embedding")]
#[derive(Deserialize)]
struct ResourceTypesQuery {
text: String,
limit: Option<i64>,
}
#[cfg(feature = "embedding")]
#[derive(Serialize)]
pub struct ResourceTypeResult {
name: String,
score: f32,
schema: Option<serde_json::Value>,
}
#[cfg(feature = "embedding")]
async fn query_resource_types(
Query(query): Query<ResourceTypesQuery>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<ResourceTypeResult>> {
let embeddings_db = EMBEDDINGS_DB.read().await;
if let Some(embeddings_db) = embeddings_db.as_ref() {
let results = embeddings_db
.query_resource_types(w_id, &query.text, query.limit)
.await?;
Ok(Json(results))
} else {
Err(windmill_common::error::Error::internal_err(
"Embeddings db not initialized".to_string(),
))
}
}
#[cfg(feature = "embedding")]
#[derive(Deserialize, Debug, Clone)]
struct HubScript {
ask_id: i64,
id: i64,
version_id: i64,
summary: String,
app: String,
kind: String,
embedding: Vec<f32>,
}
#[cfg(feature = "embedding")]
#[derive(Deserialize, Debug)]
struct HubResourceType {
name: String,
embedding: Vec<f32>,
}
#[cfg(feature = "embedding")]
pub struct ModelInstance {
model: BertModel,
tokenizer: Tokenizer,
}
#[cfg(feature = "embedding")]
impl ModelInstance {
pub async fn load_model_files() -> Result<(PathBuf, PathBuf, PathBuf)> {
let api = Api::new()?;
let repo_api = api.model("thenlper/gte-small".to_string());
let (config_filename, tokenizer_filename, weights_filename) =
(
repo_api
.get("config.json")
.await
.map_err(|e| anyhow!("Failed to get config.json from hugging face: {}", e))?,
repo_api.get("tokenizer.json").await.map_err(|e| {
anyhow!("Failed to get tokenizer.json from hugging face: {}", e)
})?,
repo_api.get("model.safetensors").await.map_err(|e| {
anyhow!("Failed to get model.safetensors from hugging face: {}", e)
})?,
);
Ok((config_filename, tokenizer_filename, weights_filename))
}
pub async fn new() -> Result<Self> {
tracing::info!("Loading embedding model...");
let device = Device::Cpu;
let (config_filename, tokenizer_filename, weights_filename) =
Self::load_model_files().await?;
let config = std::fs::read_to_string(config_filename)?;
let config: Config = serde_json::from_str(&config)?;
let tokenizer = Tokenizer::from(
Tokenizer::from_file(tokenizer_filename)
.map_err(Error::msg)?
.with_padding(None)
.to_owned(),
);
let vb =
unsafe { VarBuilder::from_mmaped_safetensors(&[weights_filename], DTYPE, &device)? };
let model = BertModel::load(vb, &config)?;
tracing::info!("Loaded embedding model");
Ok(Self { model, tokenizer })
}
pub async fn create_embedding(self: Arc<Self>, sentence: &str) -> Result<Vec<f32>> {
let sentence = sentence.to_owned();
tokio::task::spawn_blocking(move || {
let tokens = self
.tokenizer
.encode(sentence, true)
.map_err(Error::msg)?
.get_ids()
.to_vec();
let token_ids = Tensor::new(&tokens[..], &Device::Cpu)?.unsqueeze(0)?;
let token_type_ids = token_ids.zeros_like()?;
let embedding = self.model.forward(&token_ids, &token_type_ids, None)?;
let embedding = (embedding.sum(1)? / embedding.dim(1)? as f64)?;
let embedding = normalize_l2(&embedding)?;
let embedding = embedding.get(0)?.to_vec1()?;
Ok(embedding)
})
.await?
}
}
#[cfg(feature = "embedding")]
pub struct EmbeddingsDb {
db: Db,
model_instance: Arc<ModelInstance>,
}
#[cfg(feature = "embedding")]
impl EmbeddingsDb {
pub async fn new(pg_db: &Pool<Postgres>, model_instance: Arc<ModelInstance>) -> Result<Self> {
let db = Db::new();
let mut embeddings_db = Self { db, model_instance: model_instance.clone() };
embeddings_db.fill_db(pg_db).await?;
Ok(embeddings_db)
}
async fn fill_db(&mut self, pg_db: &Pool<Postgres>) -> Result<()> {
if self.db.get_collection("scripts").is_some() {
self.db.delete_collection("scripts")?;
}
self.db
.create_collection("scripts".to_string(), 384, Distance::Cosine)?;
if self.db.get_collection("resource_types").is_some() {
self.db.delete_collection("resource_types")?;
}
self.db
.create_collection("resource_types".to_string(), 384, Distance::Cosine)?;
let hub_base_url = HUB_BASE_URL.read().await.clone();
let response = match hub_base_url.as_str() {
DEFAULT_HUB_BASE_URL => {
let response = HTTP_CLIENT
.get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json")
.send()
.await;
if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() {
tracing::warn!("Failed to get scripts embeddings from bucket, trying hub...");
http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/scripts/embeddings", hub_base_url),
false,
None,
Some(pg_db),
)
.await?
} else {
response.unwrap()
}
}
_ => {
http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/scripts/embeddings", hub_base_url),
false,
None,
Some(pg_db),
)
.await?
}
};
if response.error_for_status_ref().is_err() {
return Err(anyhow!(
"Failed to get scripts embeddings from hub with error code: {}",
response.status()
));
}
let hub_scripts = response.json::<Vec<HubScript>>().await?;
for script in &hub_scripts {
let mut hm = HashMap::new();
hm.insert("ask_id".to_string(), script.ask_id.clone().to_string());
hm.insert("summary".to_string(), script.summary.clone());
hm.insert("app".to_string(), script.app.clone());
hm.insert("kind".to_string(), script.kind.clone());
hm.insert("id".to_string(), script.id.clone().to_string());
hm.insert(
"version_id".to_string(),
script.version_id.clone().to_string(),
);
let embedding = Embedding {
id: script.ask_id.clone().to_string(),
vector: script.embedding.clone(),
metadata: Some(hm),
};
self.db.insert_into_collection("scripts", embedding)?;
}
let response = match hub_base_url.as_str() {
DEFAULT_HUB_BASE_URL => {
let response = HTTP_CLIENT
.get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json")
.send()
.await;
if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() {
tracing::warn!(
"Failed to get resource types embeddings from bucket, trying hub..."
);
http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/resource_types/embeddings", hub_base_url),
false,
None,
Some(pg_db),
)
.await?
} else {
response.unwrap()
}
}
_ => {
http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/resource_types/embeddings", hub_base_url),
false,
None,
Some(pg_db),
)
.await?
}
};
if response.error_for_status_ref().is_err() {
return Err(anyhow!(
"Failed to get resource types embeddings from hub with error code: {}",
response.status()
));
}
let hub_resource_types = response.json::<Vec<HubResourceType>>().await?;
let resource_types: Vec<ResourceType> =
sqlx::query_as!(ResourceType, "SELECT * from resource_type ORDER BY name",)
.fetch_all(pg_db)
.await?;
for rt in resource_types {
let mut hm = HashMap::new();
hm.insert("name".to_string(), rt.name.clone());
if let Some(schema) = rt.schema.clone() {
hm.insert("schema".to_string(), serde_json::to_string(&schema)?);
}
hm.insert("workspace".to_string(), rt.workspace_id.clone());
let hub_rt = hub_resource_types.iter().find(|hrt| hrt.name == rt.name);
let vector = if let Some(hub_rt) = hub_rt {
hub_rt.embedding.clone()
} else {
self.model_instance
.clone()
.create_embedding(&format!(
"{};{}",
rt.name,
rt.description.unwrap_or_default()
))
.await?
};
let embedding = Embedding {
id: format!("{}_{}", rt.workspace_id, rt.name),
vector,
metadata: Some(hm),
};
self.db
.insert_into_collection("resource_types", embedding)?;
}
Ok(())
}
pub async fn query_hub_scripts(
&self,
query: &str,
limit: Option<i64>,
kind: Option<String>,
app: Option<String>,
) -> Result<Vec<HubScriptResult>> {
let model_instance = self.model_instance.clone();
let query_embedding = model_instance.create_embedding(query).await?;
let collection = self.db.get_collection("scripts");
let collection = collection.ok_or(Error::msg("no collection found"))?;
let filter = |embedding: &Embedding| {
if let Some(metadata) = embedding.metadata.as_ref() {
match (
metadata.get("kind"),
kind.clone(),
metadata.get("app"),
app.clone(),
) {
(Some(script_kind), Some(kind), Some(script_app), Some(app)) => {
&kind == script_kind && &app == script_app
}
(Some(script_kind), Some(kind), _, _) => &kind == script_kind,
(_, _, Some(script_app), Some(app)) => &app == script_app,
(_, None, _, None) => true,
_ => false,
}
} else {
false
}
};
let results = collection.get_similarity(
&query_embedding,
limit.unwrap_or(10) as usize,
Some(&filter),
Some(0.8),
);
let results: Result<Vec<_>> = results
.iter()
.map(|r| {
let metadata = r
.embedding
.metadata
.as_ref()
.ok_or(Error::msg("no metadata"))?;
Ok(HubScriptResult {
ask_id: metadata
.get("ask_id")
.ok_or(Error::msg("no ask_id"))?
.parse::<i64>()?,
summary: metadata
.get("summary")
.ok_or(Error::msg("no summary"))?
.to_owned(),
app: metadata.get("app").ok_or(Error::msg("no app"))?.to_owned(),
kind: metadata
.get("kind")
.ok_or(Error::msg("no kind"))?
.to_owned(),
id: metadata
.get("id")
.ok_or(Error::msg("no id"))?
.parse::<i64>()?,
version_id: metadata
.get("version_id")
.ok_or(Error::msg("no version_id"))?
.parse::<i64>()?,
score: r.score,
})
})
.collect();
let mut results = results?;
if results.len() > 1 {
let top_score = results[0].score;
results = results
.into_iter()
.take_while(|r| (top_score - r.score) / top_score <= 0.05)
.collect();
}
Ok(results)
}
pub async fn query_resource_types(
&self,
workspace: String,
query: &str,
limit: Option<i64>,
) -> Result<Vec<ResourceTypeResult>> {
let model_instance = self.model_instance.clone();
let query_embedding = model_instance.create_embedding(query).await?;
let collection = self.db.get_collection("resource_types");
if collection.is_none() {
return Ok(vec![]);
}
let collection = collection.ok_or(Error::msg("no collection found"))?;
let filter = |embedding: &Embedding| {
if let Some(metadata) = embedding.metadata.as_ref() {
match metadata.get("workspace").map(|x| x.as_str()) {
Some("admins") => true,
Some(rt_workspace) => &workspace == rt_workspace,
_ => false,
}
} else {
false
}
};
let results = collection.get_similarity(
&query_embedding,
limit.unwrap_or(10) as usize,
Some(&filter),
Some(0.75),
);
let results: Result<_> = results
.iter()
.map(|r| {
let metadata = r
.embedding
.metadata
.as_ref()
.ok_or(Error::msg("no metadata"))?;
Ok(ResourceTypeResult {
name: metadata
.get("name")
.ok_or(Error::msg("no name"))?
.to_owned(),
schema: match metadata.get("schema") {
Some(schema) => serde_json::from_str(schema)?,
None => None,
},
score: r.score,
})
})
.collect();
results
}
}
#[cfg(feature = "embedding")]
fn normalize_l2(v: &Tensor) -> Result<Tensor> {
Ok(v.broadcast_div(&v.sqr()?.sum_keepdim(1)?.sqrt()?)?)
}
#[cfg(feature = "embedding")]
pub fn load_embeddings_db(db: &Pool<Postgres>) -> () {
let disable_embedding = std::env::var("DISABLE_EMBEDDING")
.ok()
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false);
if !disable_embedding {
let db_clone = db.clone();
tokio::spawn(async move {
let model_instance = ModelInstance::new().await;
if let Ok(model_instance) = model_instance {
let mut model_instance_lock = MODEL_INSTANCE.write().await;
*model_instance_lock = Some(Arc::new(model_instance));
drop(model_instance_lock);
loop {
update_embeddings_db(&db_clone).await;
tokio::time::sleep(std::time::Duration::from_secs(
*HUB_EMBEDDINGS_PULLING_INTERVAL_SECS,
))
.await;
}
} else {
tracing::error!(
"Failed to initialize model instance: {}",
model_instance.err().unwrap()
);
}
});
}
}
#[cfg(feature = "embedding")]
pub async fn update_embeddings_db(db: &Pool<Postgres>) -> () {
if let Some(model_instance) = MODEL_INSTANCE.read().await.as_ref() {
tracing::info!("Creating embeddings DB...");
let new_embeddings_db = EmbeddingsDb::new(&db, model_instance.clone()).await;
if let Err(e) = new_embeddings_db.as_ref() {
tracing::error!("Failed to create embeddings db: {}", e);
} else {
let mut embeddings_db = EMBEDDINGS_DB.write().await;
*embeddings_db = new_embeddings_db.ok();
tracing::info!("Created embeddings DB");
}
} else {
tracing::error!("Could not update embeddings DB, model instance not initialized");
}
}
#[cfg(feature = "embedding")]
pub fn workspaced_service() -> Router {
Router::new().route("/query_resource_types", get(query_resource_types))
}
#[cfg(feature = "embedding")]
pub fn global_service() -> Router {
Router::new().route("/query_hub_scripts", get(query_hub_scripts))
}
#[cfg(not(feature = "embedding"))]
pub fn workspaced_service() -> Router {
Router::new()
}
#[cfg(not(feature = "embedding"))]
pub fn global_service() -> Router {
Router::new()
}
backend/windmill-api-jobs/Cargo.toml
+34
View File
@@ -0,0 +1,34 @@
[package]
name = "windmill-api-jobs"
version.workspace = true
authors.workspace = true
edition.workspace = true
[lib]
name = "windmill_api_jobs"
path = "src/lib.rs"
[features]
default = []
enterprise = ["windmill-common/enterprise"]
[dependencies]
windmill-api-auth.workspace = true
windmill-api-sse.workspace = true
windmill-common = { workspace = true, default-features = false }
windmill-queue.workspace = true
axum.workspace = true
serde.workspace = true
serde_json.workspace = true
sqlx.workspace = true
uuid.workspace = true
chrono.workspace = true
http.workspace = true
tower-http.workspace = true
base64.workspace = true
anyhow.workspace = true
tracing.workspace = true
sql-builder.workspace = true
tokio.workspace = true
hyper.workspace = true
lazy_static.workspace = true
backend/windmill-api-jobs/src/concurrency_groups.rs
+364
View File
@@ -0,0 +1,364 @@
use windmill_api_auth::{check_scopes, ApiAuthed};
use windmill_common::{
db::{UserDB, DB},
error::Error::PermissionDenied,
error::{self, JsonResult},
utils::require_admin,
};
use crate::query::{filter_list_completed_query, filter_list_queue_query};
use crate::types::{ListCompletedQuery, ListQueueQuery, UnifiedJob};
use windmill_api_sse::Job;
use axum::extract::Path;
use axum::routing::{delete, get};
use axum::{extract::Query, Extension, Json};
use serde::Deserialize;
use axum::Router;
use serde::Serialize;
use sql_builder::bind::Bind;
use sql_builder::SqlBuilder;
use uuid::Uuid;
pub fn global_service() -> Router {
Router::new()
.route("/list", get(list_concurrency_groups))
.route("/prune/*concurrency_key", delete(prune_concurrency_group))
.route("/:job_id/key", get(get_concurrency_key))
}
pub fn workspaced_service() -> Router {
Router::new().route("/list_jobs", get(get_concurrent_intervals))
}
#[derive(Serialize)]
pub struct ConcurrencyGroups {
concurrency_key: String,
total_running: i64,
}
async fn list_concurrency_groups(
authed: ApiAuthed,
Extension(db): Extension<DB>,
) -> JsonResult<Vec<ConcurrencyGroups>> {
require_admin(authed.is_admin, &authed.username)?;
let concurrency_counts = sqlx::query_as::<_, (String, i64)>(
"SELECT concurrency_id, (select COUNT(*) from jsonb_object_keys(job_uuids)) as n_job_uuids FROM concurrency_counter",
).fetch_all(&db)
.await?;
let mut concurrency_groups: Vec<ConcurrencyGroups> = vec![];
for (concurrency_key, count) in concurrency_counts {
concurrency_groups.push(ConcurrencyGroups {
concurrency_key: concurrency_key.clone(),
total_running: count,
})
}
return Ok(Json(concurrency_groups));
}
async fn prune_concurrency_group(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(concurrency_key): Path<String>,
) -> JsonResult<()> {
if !authed.is_admin {
return Err(PermissionDenied(
"Only administrators can delete concurrency groups".to_string(),
));
}
let mut tx = db.begin().await?;
let concurrency_group = sqlx::query_as::<_, (String, i64)>(
"SELECT concurrency_id, (select COUNT(*) from jsonb_object_keys(job_uuids)) as n_job_uuids FROM concurrency_counter WHERE concurrency_id = $1 FOR UPDATE",
)
.bind(concurrency_key.clone())
.fetch_optional(&mut *tx)
.await?;
let n_job_uuids = concurrency_group.map(|cg| cg.1).unwrap_or_default();
if n_job_uuids > 0 {
tx.commit().await?;
return Err(error::Error::internal_err(
"Concurrency group is currently in use, unable to remove it. Retry later.".to_string(),
));
}
sqlx::query!(
"DELETE FROM concurrency_counter WHERE concurrency_id = $1",
concurrency_key.clone(),
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"DELETE FROM concurrency_key WHERE key = $1",
concurrency_key.clone(),
)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(Json(()))
}
#[derive(Serialize)]
struct ExtendedJobs {
jobs: Vec<Job>,
obscured_jobs: Vec<ObscuredJob>,
omitted_obscured_jobs: bool,
}
#[derive(Serialize)]
struct ObscuredJob {
typ: String,
started_at: Option<chrono::DateTime<chrono::Utc>>,
duration_ms: Option<i64>,
}
#[derive(Deserialize)]
struct ExtendedJobsParams {
row_limit: Option<i64>,
}
pub fn join_concurrency_key<'c>(
concurrency_key: Option<&String>,
mut sqlb: SqlBuilder,
) -> SqlBuilder {
if let Some(key) = concurrency_key {
sqlb.join("concurrency_key")
.on_eq("id", "concurrency_key.job_id")
.and_where_eq("key", "?".bind(key));
}
sqlb
}
async fn get_concurrent_intervals(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(iq): Query<ExtendedJobsParams>,
Query(lq): Query<ListCompletedQuery>,
) -> JsonResult<ExtendedJobs> {
check_scopes(&authed, || format!("jobs:read"))?;
if lq.success.is_some() && lq.running.is_some_and(|x| x) {
return Err(error::Error::BadRequest(
"cannot specify both success and running".to_string(),
));
}
let row_limit = iq.row_limit.unwrap_or(1000);
let lq = ListCompletedQuery { order_desc: Some(true), ..lq };
let lqc = lq.clone();
let lqq: ListQueueQuery = lqc.into();
let mut sqlb_q = SqlBuilder::select_from("v2_job_queue")
.fields(UnifiedJob::queued_job_fields())
.order_by("created_at", lq.order_desc.unwrap_or(true))
.limit(row_limit)
.clone();
let mut sqlb_c = SqlBuilder::select_from("v2_job_completed")
.fields(UnifiedJob::completed_job_fields())
.order_by("completed_at", lq.order_desc.unwrap_or(true))
.limit(row_limit)
.clone();
let mut sqlb_q_user = SqlBuilder::select_from("v2_job_queue")
.fields(&["id"])
.order_by("created_at", lq.order_desc.unwrap_or(true))
.limit(row_limit)
.clone();
let mut sqlb_c_user = SqlBuilder::select_from("v2_job_completed")
.fields(&["id"])
.order_by("completed_at", lq.order_desc.unwrap_or(true))
.limit(row_limit)
.clone();
sqlb_q = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_q);
sqlb_c = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_c);
sqlb_q_user = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_q_user);
sqlb_c_user = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_c_user);
let should_fetch_obscured_jobs = match lq {
ListCompletedQuery {
script_path_start: None,
script_path_exact: None,
script_hash: None,
created_by: None,
success: None,
running: None,
parent_job: None,
is_skipped: None | Some(false),
suspended: None,
schedule_path: None,
args: None,
result: None,
tag: None,
has_null_parent: None,
worker: None,
label: None,
trigger_path: None,
scheduled_for_before_now: _,
is_not_schedule: _,
started_before: _,
started_after: _,
created_before: _,
created_after: _,
created_before_queue: _,
created_after_queue: _,
completed_after: _,
completed_before: _,
created_or_started_before: _,
created_or_started_after: _,
created_or_started_after_completed_jobs: _,
order_desc: _,
job_kinds: _,
is_flow_step: _,
all_workspaces: _,
concurrency_key: Some(_),
allow_wildcards: None,
trigger_kind: _,
include_args: _,
} => true,
_ => false,
};
// When we have a concurrency key defined, fetch jobs from other workspaces
// as obscured unless we're in the admins workspace. This is to show the
// potential concurrency races without showing jobs that don't belong to
// the workspace.
// To avoid infering information through filtering, don't return obscured
// jobs if the filters are too specific
if should_fetch_obscured_jobs && w_id != "admins" {
// Get the obscured jobs from all workspaces (concurrency key could be global)
let (sqlb_q, sqlb_c) = (
filter_list_queue_query(
sqlb_q,
&ListQueueQuery { all_workspaces: Some(true), ..lqq.clone() },
"admins",
true,
),
filter_list_completed_query(
sqlb_c,
&ListCompletedQuery { all_workspaces: Some(true), ..lq.clone() },
"admins",
true,
),
);
sqlb_q_user = filter_list_queue_query(sqlb_q_user, &lqq, w_id.as_str(), true);
sqlb_c_user = filter_list_completed_query(sqlb_c_user, &lq, w_id.as_str(), true);
let sql_q_user = sqlb_q_user.query()?;
let sql_c_user = sqlb_c_user.query()?;
let sql_q = sqlb_q.query()?;
let sql_c = sqlb_c.query()?;
// This first transaction uses the user_db to know which uuids are
// accessible to the user.
let mut tx = user_db.begin(&authed).await?;
let running_jobs_user: Vec<Uuid> = if lq.success.is_none() {
sqlx::query_scalar(&sql_q_user).fetch_all(&mut *tx).await?
} else {
vec![]
};
let completed_jobs_user: Vec<Uuid> = if lq.running.is_none() {
sqlx::query_scalar(&sql_c_user).fetch_all(&mut *tx).await?
} else {
vec![]
};
tx.commit().await?;
// This second transaction uses the db, so it will fetch information
// potentially forbidden to the user. It must be obscured before
// returning it
let running_jobs_db: Vec<UnifiedJob> = if lq.success.is_none() {
sqlx::query_as(&sql_q).fetch_all(&db).await?
} else {
vec![]
};
let completed_jobs_db: Vec<UnifiedJob> = if lq.running.is_none() {
sqlx::query_as(&sql_c).fetch_all(&db).await?
} else {
vec![]
};
let obscured_jobs = running_jobs_db
.iter()
.filter(|j| !running_jobs_user.iter().any(|id| j.id == *id))
.chain(
completed_jobs_db
.iter()
.filter(|j| !completed_jobs_user.iter().any(|id| j.id == *id)),
)
.map(|j| ObscuredJob {
typ: j.typ.clone(),
started_at: j.started_at,
duration_ms: j.duration_ms,
})
.collect();
let jobs = running_jobs_db
.into_iter()
.filter(|j| running_jobs_user.iter().any(|id| j.id == *id))
.chain(
completed_jobs_db
.into_iter()
.filter(|j| completed_jobs_user.iter().any(|id| j.id == *id)),
)
.map(From::from)
.collect();
Ok(Json(ExtendedJobs {
jobs,
obscured_jobs,
omitted_obscured_jobs: !should_fetch_obscured_jobs,
}))
} else {
sqlb_q = filter_list_queue_query(sqlb_q, &lqq, w_id.as_str(), true);
sqlb_c = filter_list_completed_query(sqlb_c, &lq, w_id.as_str(), true);
let sql_q = sqlb_q.query()?;
let sql_c = sqlb_c.query()?;
let mut tx = user_db.begin(&authed).await?;
let running_jobs: Vec<UnifiedJob> = if lq.success.is_none() {
sqlx::query_as(&sql_q).fetch_all(&mut *tx).await?
} else {
vec![]
};
let completed_jobs: Vec<UnifiedJob> = if lq.running.is_none() {
sqlx::query_as(&sql_c).fetch_all(&mut *tx).await?
} else {
vec![]
};
tx.commit().await?;
let jobs = running_jobs
.into_iter()
.chain(completed_jobs.into_iter())
.map(From::from)
.collect();
Ok(Json(ExtendedJobs {
jobs,
obscured_jobs: vec![],
omitted_obscured_jobs: !should_fetch_obscured_jobs,
}))
}
}
async fn get_concurrency_key(
Extension(db): Extension<DB>,
Path(job_id): Path<Uuid>,
) -> JsonResult<Option<String>> {
let key = sqlx::query_scalar!("SELECT key FROM concurrency_key WHERE job_id = $1", job_id)
.fetch_optional(&db)
.await?;
Ok(Json(key))
}
backend/windmill-api-jobs/src/execution.rs
+892
View File
@@ -0,0 +1,892 @@
/*
* Author: Windmill Labs, Inc
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Job execution primitives: push, cancel, tag/license checks, wait-for-result.
use axum::{
response::{IntoResponse, Response},
Json,
};
use http::{HeaderMap, HeaderName, HeaderValue};
use hyper::StatusCode;
use serde::Deserialize;
use serde_json::value::RawValue;
use std::collections::HashMap;
use std::str::FromStr;
use std::sync::Arc;
use tokio::sync::RwLock;
use uuid::Uuid;
use windmill_api_auth::{check_scopes, get_scope_tags, ApiAuthed};
use windmill_common::{
db::{UserDB, UserDbWithAuthed},
error::{self, Error},
flow_conversations::{add_message_to_conversation_tx, MessageType},
get_latest_flow_version_info_for_path,
jobs::{
check_tag_available_for_workspace_internal, format_result, script_path_to_payload,
JobPayload,
},
triggers::TriggerMetadata,
users::username_to_permissioned_as,
utils::StripPath,
FlowVersionInfo, DB,
};
use windmill_queue::{
cancel_job, get_result_and_success_by_id_from_flow, push, PushArgs, PushArgsOwned,
PushIsolationLevel,
};
use crate::types::RunJobQuery;
// ---------------------------------------------------------------------------
// Tag / license checks
// ---------------------------------------------------------------------------
pub async fn check_tag_available_for_workspace(
db: &DB,
w_id: &str,
tag: &Option<String>,
authed: &ApiAuthed,
) -> error::Result<()> {
if let Some(tag) = tag.as_deref().filter(|t| !t.is_empty()) {
let tags = get_scope_tags(authed);
check_tag_available_for_workspace_internal(&db, w_id, tag, &authed.email, tags).await
} else {
Ok(())
}
}
#[cfg(feature = "enterprise")]
pub async fn check_license_key_valid() -> error::Result<()> {
use windmill_common::ee_oss::LICENSE_KEY_VALID;
let valid = *LICENSE_KEY_VALID.read().await;
if !valid {
return Err(error::Error::BadRequest(
"License key is not valid. Go to your superadmin settings to update your license key."
.to_string(),
));
}
Ok(())
}
// ---------------------------------------------------------------------------
// Cancel jobs
// ---------------------------------------------------------------------------
pub async fn cancel_jobs(
jobs: Vec<Uuid>,
db: &DB,
username: &str,
w_id: &str,
force_cancel: bool,
) -> error::JsonResult<Vec<Uuid>> {
let mut uuids = vec![];
tracing::info!("Cancelling jobs: {:?}", jobs);
let mut tx = db.begin().await?;
let trivial_jobs = sqlx::query!("INSERT INTO v2_job_completed AS cj
( workspace_id
, id
, duration_ms
, result
, canceled_by
, canceled_reason
, flow_status
, status
, worker
)
SELECT q.workspace_id
, q.id
, 0
, $4
, $1
, 'cancel all'
, (SELECT flow_status FROM v2_job_status WHERE id = q.id)
, 'canceled'::job_status
, worker
FROM v2_job_queue q
JOIN v2_job USING (id)
WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger_kind IS DISTINCT FROM 'schedule'
FOR UPDATE SKIP LOCKED
ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"", username, &jobs, w_id, serde_json::json!({"error": { "message": format!("Job canceled: cancel all by {username}"), "name": "Canceled", "reason": "cancel all", "canceler": username}}))
.fetch_all(&mut *tx)
.await?.into_iter().map(|x| x.id).collect::<Vec<Uuid>>();
sqlx::query!(
"DELETE FROM v2_job_queue WHERE id = any($1) AND workspace_id = $2",
&trivial_jobs,
w_id
)
.execute(&mut *tx)
.await?;
tx.commit().await?;
for job_id in jobs.into_iter() {
if trivial_jobs.contains(&job_id) {
continue;
}
match tokio::time::timeout(tokio::time::Duration::from_secs(5), async move {
let tx = db.begin().await?;
let (tx, _) = cancel_job(
username,
None,
job_id.clone(),
w_id,
tx,
db,
force_cancel,
false,
)
.await?;
tx.commit().await?;
Ok::<_, anyhow::Error>(())
})
.await
{
Ok(result) => match result {
Ok(_) => {
uuids.push(job_id);
}
Err(e) => {
tracing::error!("Failed to cancel job {:?}: {:?}", job_id, e);
}
},
Err(_) => {
tracing::error!(
"Timeout while trying to cancel job {:?} after 5 seconds",
job_id
);
}
}
}
uuids.extend(trivial_jobs);
Ok(Json(uuids))
}
// ---------------------------------------------------------------------------
// Wait-for-result infrastructure
// ---------------------------------------------------------------------------
struct Guard {
done: bool,
id: Uuid,
w_id: String,
db: DB,
username: String,
}
impl Drop for Guard {
fn drop(&mut self) {
if !&self.done {
let id = self.id;
let w_id = self.w_id.clone();
let db = self.db.clone();
let username = self.username.clone();
tracing::info!("http connection broke, marking job {id} as canceled");
tokio::spawn(async move {
let cancel_f = async {
let tx = db.begin().await?;
let (tx, _) = cancel_job(
&username,
Some("http connection broke".to_string()),
id,
&w_id,
tx,
&db,
false,
false,
)
.await?;
tx.commit().await?;
Ok::<_, anyhow::Error>(())
};
if let Err(e) = cancel_f.await {
tracing::error!(
"Error marking job as canceled after http connection broke: {e}"
);
}
});
}
}
}
lazy_static::lazy_static! {
pub static ref TIMEOUT_WAIT_RESULT: Arc<RwLock<Option<u64>>> = Arc::new(RwLock::new(
std::env::var("TIMEOUT_WAIT_RESULT")
.ok()
.and_then(|x| x.parse::<u64>().ok())
));
pub static ref QUEUE_LIMIT_WAIT_RESULT: Option<i64> = std::env::var("QUEUE_LIMIT_WAIT_RESULT")
.ok()
.and_then(|x| x.parse().ok());
pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS")
.ok()
.and_then(|x| x.parse().ok())
.unwrap_or(50);
pub static ref WAIT_RESULT_FAST_POLL_DURATION_SECS: u16 = std::env::var("WAIT_RESULT_FAST_POLL_DURATION_SECS")
.ok()
.and_then(|x| x.parse().ok())
.unwrap_or(2);
pub static ref WAIT_RESULT_SLOW_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_SLOW_POLL_INTERVAL_MS")
.ok()
.and_then(|x| x.parse().ok())
.unwrap_or(200);
}
#[derive(Deserialize)]
pub struct WindmillCompositeResult {
windmill_status_code: Option<u16>,
windmill_content_type: Option<String>,
windmill_headers: Option<HashMap<String, String>>,
result: Option<Box<RawValue>>,
}
pub async fn run_wait_result_internal(
db: &DB,
uuid: Uuid,
w_id: &str,
node_id_for_empty_return: Option<String>,
username: &str,
) -> error::Result<(Box<RawValue>, bool)> {
let mut result = None;
let mut success = false;
let timeout = TIMEOUT_WAIT_RESULT.read().await.clone().unwrap_or(600);
let timeout_ms = if timeout <= 0 {
2000
} else {
(timeout * 1000) as u64
};
let mut g = Guard {
done: false,
id: uuid,
w_id: w_id.to_string(),
db: db.clone(),
username: username.to_string(),
};
let fast_poll_duration = *WAIT_RESULT_FAST_POLL_DURATION_SECS as u64 * 1000;
let mut accumulated_delay = 0 as u64;
loop {
if let Some(node_id_for_empty_return) = node_id_for_empty_return.as_ref() {
let result_and_success = get_result_and_success_by_id_from_flow(
&db,
w_id,
&uuid,
node_id_for_empty_return,
None,
)
.await
.ok();
if let Some((r, s)) = result_and_success {
result = Some(r);
success = s;
}
}
if result.is_none() {
let row = sqlx::query!(
"
SELECT
result AS \"result: sqlx::types::Json<Box<RawValue>>\",
result_columns,
status = 'success' AS \"success!\"
FROM
v2_job_completed
WHERE
id = $1 AND
workspace_id = $2
",
uuid,
&w_id
)
.fetch_optional(db)
.await?;
if let Some(mut raw_result) = row {
format_result(
raw_result.result_columns.as_ref(),
raw_result.result.as_mut(),
);
result = raw_result.result.map(|x| x.0);
success = raw_result.success;
}
}
if result.is_some() {
break;
}
let delay = if accumulated_delay <= fast_poll_duration {
*WAIT_RESULT_FAST_POLL_INTERVAL_MS
} else {
*WAIT_RESULT_SLOW_POLL_INTERVAL_MS
};
accumulated_delay += delay;
if accumulated_delay > timeout_ms {
break;
};
tokio::time::sleep(core::time::Duration::from_millis(delay)).await;
}
if let Some(result) = result {
g.done = true;
Ok((result, success))
} else {
Err(Error::ExecutionErr(format!("timeout after {}s", timeout)))
}
}
pub fn result_to_response(result: Box<RawValue>, success: bool) -> error::Result<Response> {
let composite_result = serde_json::from_str::<WindmillCompositeResult>(result.get());
match composite_result {
Ok(WindmillCompositeResult {
windmill_status_code,
windmill_content_type,
windmill_headers,
result: result_value,
}) => {
if windmill_content_type.is_none()
&& windmill_status_code.is_none()
&& windmill_headers.is_none()
{
return Ok((
if success {
StatusCode::OK
} else {
StatusCode::UNPROCESSABLE_ENTITY
},
Json(result),
)
.into_response());
}
let status_code_or_default = windmill_status_code
.map(|val| match StatusCode::from_u16(val) {
Ok(sc) => Ok(sc),
Err(_) => Err(Error::ExecutionErr("Invalid status code".to_string())),
})
.unwrap_or_else(|| {
if !success {
Ok(StatusCode::UNPROCESSABLE_ENTITY)
} else if result_value.is_some() {
Ok(StatusCode::OK)
} else {
Ok(StatusCode::NO_CONTENT)
}
})?;
let mut headers = HeaderMap::new();
if let Some(windmill_headers) = windmill_headers {
for (k, v) in windmill_headers {
let k = HeaderName::from_str(k.as_str()).map_err(|err| {
Error::internal_err(format!("Invalid header name {k}: {err}"))
})?;
let v = HeaderValue::from_str(v.as_str()).map_err(|err| {
Error::internal_err(format!("Invalid header value {v}: {err}"))
})?;
headers.insert(k, v);
}
}
if let Some(content_type) = windmill_content_type {
let serialized_json_result = result_value
.map(|val| val.get().to_owned())
.unwrap_or_else(String::new);
let serialized_result =
serde_json::from_str::<String>(serialized_json_result.as_str())
.ok()
.unwrap_or(serialized_json_result);
headers.insert(
http::header::CONTENT_TYPE,
HeaderValue::from_str(content_type.as_str()).map_err(|err| {
Error::internal_err(format!("Invalid content type {content_type}: {err}"))
})?,
);
return Ok((status_code_or_default, headers, serialized_result).into_response());
}
if let Some(result_value) = result_value {
return Ok((status_code_or_default, headers, Json(result_value)).into_response());
} else {
Ok((status_code_or_default, headers).into_response())
}
}
_ => Ok((
if success {
StatusCode::OK
} else {
StatusCode::UNPROCESSABLE_ENTITY
},
Json(result),
)
.into_response()),
}
}
pub async fn run_wait_result(
db: &DB,
uuid: Uuid,
w_id: &str,
node_id_for_empty_return: Option<String>,
username: &str,
) -> error::Result<Response> {
let (result, success) =
run_wait_result_internal(db, uuid, w_id, node_id_for_empty_return, username).await?;
result_to_response(result, success)
}
// ---------------------------------------------------------------------------
// Delete job metadata
// ---------------------------------------------------------------------------
pub async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<(), Error> {
sqlx::query!(
"UPDATE v2_job SET args = '{}'::jsonb WHERE id = $1",
job_uuid,
)
.execute(db)
.await?;
sqlx::query!(
"UPDATE v2_job_completed SET result = '{}'::jsonb WHERE id = $1",
job_uuid,
)
.execute(db)
.await?;
sqlx::query!(
"UPDATE job_logs SET logs = '##DELETED##' WHERE job_id = $1",
job_uuid,
)
.execute(db)
.await?;
Ok(())
}
// ---------------------------------------------------------------------------
// Queue limit check
// ---------------------------------------------------------------------------
pub async fn check_queue_too_long(db: &DB, queue_limit: Option<i64>) -> error::Result<()> {
if let Some(limit) = queue_limit {
let count = sqlx::query_scalar!(
"SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now()
OR (q.suspend_until IS NOT NULL
AND (q.suspend <= 0 OR q.suspend_until <= now())))",
)
.fetch_one(db)
.await?
.unwrap_or(0);
if count > queue_limit.unwrap() {
return Err(Error::internal_err(format!(
"Number of queued job is too high: {count} > {limit}"
)));
}
}
Ok(())
}
// ---------------------------------------------------------------------------
// Flow run helpers
// ---------------------------------------------------------------------------
pub async fn set_flow_memory_id(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
job_id: Uuid,
memory_id: Uuid,
) -> error::Result<()> {
sqlx::query!(
"UPDATE v2_job_status
SET flow_status = jsonb_set(
flow_status,
'{memory_id}',
to_jsonb($2::uuid)
)
WHERE id = $1",
job_id,
memory_id
)
.execute(&mut **tx)
.await?;
Ok(())
}
/// Apply flow-specific query parameters after job creation
pub async fn process_flow_run_query_params(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
job_id: Uuid,
run_query: &RunJobQuery,
) -> error::Result<()> {
if let Some(memory_id) = run_query.memory_id {
set_flow_memory_id(tx, job_id, memory_id).await?;
}
Ok(())
}
pub async fn handle_chat_conversation_messages(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
authed: &ApiAuthed,
w_id: &str,
flow_path: &str,
run_query: &RunJobQuery,
user_message_raw: Option<&Box<serde_json::value::RawValue>>,
) -> error::Result<()> {
let memory_id = run_query.memory_id.ok_or_else(|| {
windmill_common::error::Error::BadRequest(
"memory_id is required for chat-enabled flows".to_string(),
)
})?;
let user_message_raw = user_message_raw.ok_or_else(|| {
windmill_common::error::Error::BadRequest(
"user_message argument is required for chat-enabled flows".to_string(),
)
})?;
let user_message: String = serde_json::from_str(user_message_raw.get()).map_err(|e| {
windmill_common::error::Error::BadRequest(format!(
"Failed to deserialize user_message: {}",
e
))
})?;
windmill_common::flow_conversations::get_or_create_conversation_with_id(
tx,
w_id,
flow_path,
&authed.username,
&user_message,
memory_id,
)
.await?;
add_message_to_conversation_tx(
tx,
memory_id,
None,
&user_message,
MessageType::User,
None,
true,
)
.await?;
Ok(())
}
pub async fn run_flow<'c>(
authed: &ApiAuthed,
db: &DB,
tx_o: Option<sqlx::Transaction<'c, sqlx::Postgres>>,
user_db: UserDB,
w_id: &str,
flow_path: &str,
flow_version_info: FlowVersionInfo,
run_query: RunJobQuery,
args: PushArgsOwned,
trigger: Option<TriggerMetadata>,
) -> error::Result<(
Uuid,
Option<String>,
Option<sqlx::Transaction<'c, sqlx::Postgres>>,
)> {
let FlowVersionInfo {
version,
tag,
dedicated_worker,
has_preprocessor,
chat_input_enabled,
on_behalf_of_email,
edited_by,
early_return,
..
} = flow_version_info;
let tag = run_query.tag.clone().or(tag);
check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?;
let scheduled_for = run_query.get_scheduled_for(&db).await?;
let return_tx = tx_o.is_some();
let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o {
(
&authed.email,
username_to_permissioned_as(&authed.username),
Some(authed.clone().into()),
PushIsolationLevel::Transaction(tx),
)
} else if let Some(on_behalf_of_email) = on_behalf_of_email.as_ref() {
(
on_behalf_of_email,
username_to_permissioned_as(&edited_by),
None,
PushIsolationLevel::IsolatedRoot(db.clone()),
)
} else {
(
&authed.email,
username_to_permissioned_as(&authed.username),
Some(authed.clone().into()),
PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()),
)
};
let (uuid, mut tx) = push(
&db,
tx,
&w_id,
JobPayload::Flow {
path: flow_path.to_string(),
dedicated_worker,
version,
apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
&& has_preprocessor.unwrap_or(false),
},
PushArgs { args: &args.args, extra: args.extra },
authed.display_username(),
email,
permissioned_as,
authed.token_prefix.as_deref(),
scheduled_for,
None,
run_query.parent_job,
None,
run_query.root_job,
run_query.job_id,
false,
false,
None,
!run_query.invisible_to_owner.unwrap_or(false),
tag,
None,
None,
None,
push_authed.as_ref(),
false,
None,
trigger,
run_query.suspended_mode,
)
.await?;
// Set memory_id if provided (for agent memory)
if let Some(memory_id) = run_query.memory_id {
set_flow_memory_id(&mut tx, uuid, memory_id).await?;
}
// Handle conversation messages for chat-enabled flows
if chat_input_enabled.unwrap_or(false) {
handle_chat_conversation_messages(
&mut tx,
&authed,
&w_id,
&flow_path.to_string(),
&run_query,
args.args.get("user_message"),
)
.await?;
}
// If we were given a transaction, return it; otherwise commit it
if return_tx {
Ok((uuid, early_return, Some(tx)))
} else {
tx.commit().await?;
Ok((uuid, early_return, None))
}
}
pub async fn run_flow_and_wait_result(
authed: &ApiAuthed,
db: &DB,
user_db: UserDB,
w_id: &str,
flow_path: &str,
flow_version_info: FlowVersionInfo,
run_query: RunJobQuery,
args: PushArgsOwned,
trigger: Option<TriggerMetadata>,
) -> error::Result<Response> {
let (uuid, early_return, _) = run_flow(
authed,
db,
None,
user_db,
w_id,
flow_path,
flow_version_info,
run_query,
args,
trigger,
)
.await?;
run_wait_result(&db, uuid, w_id, early_return, &authed.username).await
}
// ---------------------------------------------------------------------------
// Push job by path
// ---------------------------------------------------------------------------
pub async fn push_flow_job_by_path_into_queue<'c>(
authed: ApiAuthed,
db: DB,
tx_o: Option<sqlx::Transaction<'c, sqlx::Postgres>>,
user_db: UserDB,
w_id: String,
flow_path: StripPath,
run_query: RunJobQuery,
args: PushArgsOwned,
trigger: Option<TriggerMetadata>,
) -> error::Result<(
Uuid,
Option<String>,
Option<sqlx::Transaction<'c, sqlx::Postgres>>,
)> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
let flow_path = flow_path.to_path();
check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?;
let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() };
let flow_version_info =
get_latest_flow_version_info_for_path(Some(userdb_authed), &db, &w_id, &flow_path, true)
.await?;
run_flow(
&authed,
&db,
tx_o,
user_db,
&w_id,
flow_path,
flow_version_info,
run_query,
args,
trigger,
)
.await
}
pub async fn push_script_job_by_path_into_queue<'c>(
authed: ApiAuthed,
db: DB,
tx_o: Option<sqlx::Transaction<'c, sqlx::Postgres>>,
user_db: UserDB,
w_id: String,
script_path: StripPath,
run_query: RunJobQuery,
args: PushArgsOwned,
trigger: Option<TriggerMetadata>,
) -> error::Result<(
Uuid,
Option<bool>,
Option<sqlx::Transaction<'c, sqlx::Postgres>>,
)> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
let script_path = script_path.to_path();
check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?;
let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() };
let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = script_path_to_payload(
script_path,
Some(userdb_authed),
db.clone(),
&w_id,
run_query.skip_preprocessor,
)
.await?;
let scheduled_for = run_query.get_scheduled_for(&db).await?;
let tag = run_query.tag.clone().or(tag);
check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?;
let return_tx = tx_o.is_some();
let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o {
(
authed.email.as_str(),
username_to_permissioned_as(&authed.username),
Some(authed.clone().into()),
PushIsolationLevel::Transaction(tx),
)
} else if let Some(on_behalf_of) = on_behalf_of.as_ref() {
(
on_behalf_of.email.as_str(),
on_behalf_of.permissioned_as.clone(),
None,
PushIsolationLevel::IsolatedRoot(db.clone()),
)
} else {
(
authed.email.as_str(),
username_to_permissioned_as(&authed.username),
Some(authed.clone().into()),
PushIsolationLevel::Isolated(user_db, authed.clone().into()),
)
};
let (uuid, tx) = push(
&db,
tx,
&w_id,
job_payload,
PushArgs { args: &args.args, extra: args.extra },
authed.display_username(),
email,
permissioned_as,
authed.token_prefix.as_deref(),
scheduled_for,
None,
run_query.parent_job,
None,
run_query.root_job,
run_query.job_id,
false,
false,
None,
!run_query.invisible_to_owner.unwrap_or(false),
tag,
timeout,
None,
// If the job has a parent job, set priority to 2 as it may be ran synchronously and block a current worker until being executed. Flow steps have a priority of 1 so this is higher.
if run_query.parent_job.is_some() || run_query.root_job.is_some() {
Some(2)
} else {
None
},
push_authed.as_ref(),
false,
None,
trigger,
run_query.suspended_mode,
)
.await?;
// If we were given a transaction, return it; otherwise commit it
if return_tx {
Ok((uuid, delete_after_use, Some(tx)))
} else {
tx.commit().await?;
Ok((uuid, delete_after_use, None))
}
}
backend/windmill-api-jobs/src/job_metrics.rs
+266
View File
@@ -0,0 +1,266 @@
use windmill_common::DB;
use axum::{
extract::Path,
routing::{get, post},
Extension, Json, Router,
};
use serde::{Deserialize, Serialize};
use tower_http::cors::{Any, CorsLayer};
use uuid::Uuid;
use windmill_common::{
error::{self, Error},
job_metrics::{
record_metric, register_metric_for_job, JobStatsRecord, MetricKind, MetricNumericValue,
},
};
pub fn workspaced_service() -> Router {
let cors = CorsLayer::new()
.allow_methods([http::Method::GET, http::Method::POST])
.allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION])
.allow_origin(Any);
Router::new()
.route("/get/:id", post(get_job_metrics).layer(cors.clone()))
.route(
"/set_progress/:id",
post(set_job_progress).layer(cors.clone()),
)
.route(
"/get_progress/:id",
get(get_job_progress).layer(cors.clone()),
)
}
#[derive(Deserialize)]
struct JobStatsRequest {
from_timestamp: Option<chrono::DateTime<chrono::Utc>>,
to_timestamp: Option<chrono::DateTime<chrono::Utc>>,
timeseries_max_datapoints: Option<u32>, // default to 100, any value lower than that will be ignored. Set to 0 to retrieve all
}
#[derive(Serialize)]
struct JobStatsResponse {
metrics_metadata: Vec<MetricsMetadata>,
scalar_metrics: Vec<ScalarMetric>,
timeseries_metrics: Vec<TimeseriesMetric>,
}
#[derive(Serialize)]
pub struct MetricsMetadata {
pub id: String,
pub name: Option<String>,
}
#[derive(Serialize)]
struct ScalarMetric {
metric_id: String,
value: f64,
}
#[derive(Serialize)]
struct TimeseriesMetric {
metric_id: String,
values: Vec<DataPoint>,
}
#[derive(Debug, Clone, Serialize)]
pub struct DataPoint {
pub timestamp: chrono::DateTime<chrono::Utc>,
pub value: f64,
}
async fn get_job_metrics(
Extension(db): Extension<DB>,
Path((w_id, job_id)): Path<(String, Uuid)>,
Json(JobStatsRequest { from_timestamp, to_timestamp, timeseries_max_datapoints }): Json<
JobStatsRequest,
>,
) -> error::JsonResult<JobStatsResponse> {
let records = sqlx::query_as::<_, JobStatsRecord>(
"SELECT * FROM job_stats where workspace_id = $1 and job_id = $2",
)
.bind(w_id)
.bind(job_id)
.fetch_all(&db)
.await?;
let mut metrics_metadata: Vec<MetricsMetadata> = vec![];
let mut scalar_metrics: Vec<ScalarMetric> = vec![];
let mut timeseries_metrics: Vec<TimeseriesMetric> = vec![];
for record in records {
let metric_id = record.metric_id;
match record.metric_kind {
MetricKind::ScalarInt => {
let value = record.scalar_int.unwrap_or_default() as f64;
scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value });
}
MetricKind::ScalarFloat => {
let value = record.scalar_float.unwrap_or_default() as f64;
scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value });
}
MetricKind::TimeseriesInt => {
if record.timestamps.clone().unwrap_or_default().len()
!= record.timeseries_int.clone().unwrap_or_default().len()
{
tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_int)
}
let (timestamps, timeseries_int) = timeseries_sample(
from_timestamp,
to_timestamp,
timeseries_max_datapoints,
record.timestamps.unwrap_or_default(),
record.timeseries_int.unwrap_or_default(),
);
let mut values: Vec<DataPoint> = vec![];
for (idx, value) in timeseries_int.iter().enumerate() {
values.push(DataPoint {
timestamp: timestamps[idx],
value: value.to_owned() as f64,
});
}
timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values });
}
MetricKind::TimeseriesFloat => {
if record.timestamps.clone().unwrap_or_default().len()
!= record.timeseries_int.clone().unwrap_or_default().len()
{
tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_float)
}
let (timestamps, timeseries_float) = timeseries_sample(
from_timestamp,
to_timestamp,
timeseries_max_datapoints,
record.timestamps.unwrap_or_default(),
record.timeseries_float.unwrap_or_default(),
);
let mut values: Vec<DataPoint> = vec![];
for (idx, value) in timeseries_float.iter().enumerate() {
values.push(DataPoint {
timestamp: timestamps[idx],
value: value.to_owned() as f64,
});
}
timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values });
}
};
metrics_metadata.push(MetricsMetadata { id: metric_id, name: record.metric_name });
}
let response = JobStatsResponse { metrics_metadata, scalar_metrics, timeseries_metrics };
Ok(Json(response))
}
#[derive(Deserialize)]
struct JobProgressSetRequest {
percent: i32,
/// Optional parent flow id
/// Used to modify flow status
/// Specifically `progress` field in corresponding FlowStatusModule in `InProgress` state
flow_job_id: Option<Uuid>,
}
async fn set_job_progress(
Extension(db): Extension<DB>,
Path((w_id, job_id)): Path<(String, Uuid)>,
Json(JobProgressSetRequest { percent, flow_job_id }): Json<JobProgressSetRequest>,
) -> error::JsonResult<()> {
// If flow_job_id exists, than we should modify flow_status of corresponding module
// Individual jobs and flows are handled differently
if let Some(flow_job_id) = flow_job_id {
// TODO: Return error if trying to set completed job?
sqlx::query!(
"UPDATE v2_job_status
SET flow_status = JSONB_SET(flow_status, ARRAY['modules', flow_status->>'step', 'progress'], $1)
WHERE id = $2",
serde_json::json!(percent.clamp(0, 99)),
flow_job_id
)
.execute(&db)
.await?;
}
let record_progress = || {
record_metric(
&db,
w_id.clone(),
job_id,
"progress_perc".to_owned(),
MetricNumericValue::Integer(percent),
)
};
// Try to record
if let Err(err) = record_progress().await {
if matches!(err, Error::MetricNotFound(..)) {
// Register
// TODO: Reset progress after job is finished (in case it reruns same job)?
_ = register_metric_for_job(
&db,
w_id.clone(),
job_id,
"progress_perc".to_string(),
MetricKind::ScalarInt,
Some("Job Execution Progress (%)".to_owned()),
)
.await?;
// Retry recording progress
record_progress().await.map_err(|err| {
// If for some reason it still returns same error, this error will be converted to BadRequest and returned
if let Error::MetricNotFound(body) = err {
Error::BadRequest(body)
} else {
err
}
})?;
} else {
return Err(err);
}
};
return Ok(Json(()));
}
async fn get_job_progress(
Extension(db): Extension<DB>,
Path((w_id, job_id)): Path<(String, Uuid)>,
) -> error::JsonResult<Option<i32>> {
let progress: Option<Option<i32>> = sqlx::query_scalar!(
"SELECT (scalar_int)::int FROM job_stats WHERE job_id = $1 AND workspace_id = $2 AND metric_id = 'progress_perc'",
job_id, w_id)
.fetch_optional(&db)
.await?;
let respond_value = if let Some(Some(progress)) = progress {
Some(progress.clamp(0, 99))
} else {
None
};
Ok(Json(respond_value))
}
fn timeseries_sample<T: Copy>(
from: Option<chrono::DateTime<chrono::Utc>>,
to: Option<chrono::DateTime<chrono::Utc>>,
_datapoints: Option<u32>,
timestamps: Vec<chrono::DateTime<chrono::Utc>>,
values: Vec<T>,
) -> (Vec<chrono::DateTime<chrono::Utc>>, Vec<T>) {
if timestamps.len() != values.len() {
tracing::warn!("Timeseries metric has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", timestamps.len(), values.len());
return (vec![], vec![]);
}
let mut filtered_timestamp: Vec<chrono::DateTime<chrono::Utc>> = vec![];
let mut filtered_values: Vec<T> = vec![];
for (idx, timestamp) in timestamps.iter().enumerate() {
if *timestamp > from.unwrap_or(chrono::DateTime::<chrono::Utc>::MIN_UTC)
&& *timestamp < to.unwrap_or(chrono::DateTime::<chrono::Utc>::MAX_UTC)
{
filtered_timestamp.push(timestamps[idx]);
filtered_values.push(values[idx]);
}
}
// TODO: implement sampling
return (filtered_timestamp, filtered_values);
}
backend/windmill-api-jobs/src/jobs.rs
+4
View File
@@ -0,0 +1,4 @@
// benchmark_modification
// benchmark
pub fn _benchmark_test_fn() -> bool { true }
backend/windmill-api-jobs/src/jobs_export.rs
+690
View File
@@ -0,0 +1,690 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use axum::{
extract::{Path, Query},
Extension, Json,
};
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use uuid::Uuid;
use windmill_common::{
db::UserDB,
error,
jobs::{JobKind, JobStatus, JobTriggerKind},
scripts::ScriptLang,
utils::{paginate, paginate_without_limits, require_admin, Pagination},
};
use windmill_api_auth::ApiAuthed;
#[derive(Serialize, Deserialize)]
pub struct ExportableCompletedJob {
pub id: Uuid,
pub raw_code: Option<String>,
pub raw_lock: Option<String>,
pub raw_flow: Option<sqlx::types::Json<Box<RawValue>>>,
pub tag: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub created_by: String,
pub permissioned_as: String,
pub permissioned_as_email: String,
pub kind: JobKind,
pub runnable_id: Option<i64>,
pub runnable_path: Option<String>,
pub parent_job: Option<Uuid>,
pub root_job: Option<Uuid>,
pub script_lang: Option<ScriptLang>,
pub script_entrypoint_override: Option<String>,
pub flow_step: Option<i32>,
pub flow_step_id: Option<String>,
pub flow_innermost_root_job: Option<Uuid>,
pub trigger: Option<String>,
pub trigger_kind: Option<JobTriggerKind>,
pub same_worker: bool,
pub visible_to_owner: bool,
pub concurrent_limit: Option<i32>,
pub concurrency_time_window_s: Option<i32>,
pub cache_ttl: Option<i32>,
pub timeout: Option<i32>,
pub priority: Option<i16>,
pub preprocessed: Option<bool>,
pub args: Option<sqlx::types::Json<Box<RawValue>>>,
pub labels: Option<Vec<String>>,
pub pre_run_error: Option<String>,
pub duration_ms: i64,
pub result: Option<sqlx::types::Json<Box<RawValue>>>,
pub deleted: bool,
pub canceled_by: Option<String>,
pub canceled_reason: Option<String>,
pub flow_status: Option<sqlx::types::Json<Box<RawValue>>>,
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
pub memory_peak: Option<i32>,
pub status: JobStatus,
pub completed_at: chrono::DateTime<chrono::Utc>,
pub worker: Option<String>,
pub workflow_as_code_status: Option<sqlx::types::Json<Box<RawValue>>>,
pub result_columns: Option<Vec<String>>,
pub retries: Option<Vec<Uuid>>,
pub extras: Option<sqlx::types::Json<Box<RawValue>>>,
pub logs: Option<String>,
pub log_offset: Option<i32>,
pub log_file_index: Option<Vec<String>>,
}
#[derive(Serialize, Deserialize)]
pub struct ExportableQueuedJob {
// v2_job columns (excluding workspace_id)
pub id: Uuid,
pub raw_code: Option<String>,
pub raw_lock: Option<String>,
pub raw_flow: Option<sqlx::types::Json<Box<RawValue>>>,
pub tag: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub created_by: String,
pub permissioned_as: String,
pub permissioned_as_email: String,
pub kind: JobKind,
pub runnable_id: Option<i64>,
pub runnable_path: Option<String>,
pub parent_job: Option<Uuid>,
pub root_job: Option<Uuid>,
pub script_lang: Option<ScriptLang>,
pub script_entrypoint_override: Option<String>,
pub flow_step: Option<i32>,
pub flow_step_id: Option<String>,
pub flow_innermost_root_job: Option<Uuid>,
pub trigger: Option<String>,
pub trigger_kind: Option<JobTriggerKind>,
pub same_worker: bool,
pub visible_to_owner: bool,
pub concurrent_limit: Option<i32>,
pub concurrency_time_window_s: Option<i32>,
pub cache_ttl: Option<i32>,
pub timeout: Option<i32>,
pub priority: Option<i16>,
pub preprocessed: Option<bool>,
pub args: Option<sqlx::types::Json<Box<RawValue>>>,
pub labels: Option<Vec<String>>,
pub pre_run_error: Option<String>,
// v2_job_queue columns (excluding workspace_id and id/created_at/tag/priority)
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
pub scheduled_for: chrono::DateTime<chrono::Utc>,
pub running: bool,
pub canceled_by: Option<String>,
pub canceled_reason: Option<String>,
pub suspend: Option<i32>,
pub suspend_until: Option<chrono::DateTime<chrono::Utc>>,
pub worker: Option<String>,
pub extras: Option<sqlx::types::Json<Box<RawValue>>>,
// v2_job_runtime columns (excluding id)
pub ping: Option<chrono::DateTime<chrono::Utc>>,
pub memory_peak: Option<i32>,
// v2_job_status columns (excluding id)
pub flow_status: Option<sqlx::types::Json<Box<RawValue>>>,
pub flow_leaf_jobs: Option<sqlx::types::Json<Box<RawValue>>>,
pub workflow_as_code_status: Option<sqlx::types::Json<Box<RawValue>>>,
// concurrency_key table
pub concurrency_key: Option<String>,
}
pub async fn export_completed_jobs(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
) -> error::JsonResult<Vec<ExportableCompletedJob>> {
require_admin(authed.is_admin, &authed.username)?;
let (per_page, offset) = paginate(pagination);
let per_page = per_page as i64;
let offset = offset as i64;
let mut tx = user_db.begin(&authed).await?;
let jobs = sqlx::query_as!(
ExportableCompletedJob,
r#"
SELECT
v2_job.id,
v2_job.raw_code,
v2_job.raw_lock,
v2_job.raw_flow as "raw_flow: _",
v2_job.tag,
v2_job.created_at,
v2_job.created_by,
v2_job.permissioned_as,
v2_job.permissioned_as_email,
v2_job.kind as "kind: _",
v2_job.runnable_id,
v2_job.runnable_path,
v2_job.parent_job,
v2_job.root_job,
v2_job.script_lang as "script_lang: _",
v2_job.script_entrypoint_override,
v2_job.flow_step,
v2_job.flow_step_id,
v2_job.flow_innermost_root_job,
v2_job.trigger,
v2_job.trigger_kind as "trigger_kind: _",
v2_job.same_worker,
v2_job.visible_to_owner,
v2_job.concurrent_limit,
v2_job.concurrency_time_window_s,
v2_job.cache_ttl,
v2_job.timeout,
v2_job.priority,
v2_job.preprocessed,
v2_job.args as "args: _",
v2_job.labels,
v2_job.pre_run_error,
v2_job_completed.duration_ms,
v2_job_completed.result as "result: _",
v2_job_completed.deleted,
v2_job_completed.canceled_by,
v2_job_completed.canceled_reason,
v2_job_completed.flow_status as "flow_status: _",
v2_job_completed.started_at,
v2_job_completed.memory_peak,
v2_job_completed.status as "status: _",
v2_job_completed.completed_at,
v2_job_completed.worker,
v2_job_completed.workflow_as_code_status as "workflow_as_code_status: _",
v2_job_completed.result_columns,
v2_job_completed.retries,
v2_job_completed.extras as "extras: _",
job_logs.logs AS logs,
job_logs.log_offset,
job_logs.log_file_index
FROM v2_job_completed
INNER JOIN v2_job ON v2_job.id = v2_job_completed.id
LEFT JOIN v2_job_status ON v2_job_completed.id = v2_job_status.id
LEFT JOIN job_logs ON job_logs.job_id = v2_job.id
WHERE v2_job_completed.workspace_id = $1
ORDER BY v2_job.created_at DESC
LIMIT $2
OFFSET $3
"#,
w_id,
per_page,
offset
)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
Ok(Json(jobs))
}
pub async fn export_queued_jobs(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
) -> error::JsonResult<Vec<ExportableQueuedJob>> {
require_admin(authed.is_admin, &authed.username)?;
let (per_page, offset) = paginate_without_limits(pagination);
let per_page = per_page as i64;
let offset = offset as i64;
let mut tx = user_db.begin(&authed).await?;
let jobs = sqlx::query_as!(
ExportableQueuedJob,
r#"
SELECT
v2_job.id,
v2_job.raw_code,
v2_job.raw_lock,
v2_job.raw_flow as "raw_flow: _",
v2_job.tag,
v2_job.created_at,
v2_job.created_by,
v2_job.permissioned_as,
v2_job.permissioned_as_email,
v2_job.kind as "kind: _",
v2_job.runnable_id,
v2_job.runnable_path,
v2_job.parent_job,
v2_job.root_job,
v2_job.script_lang as "script_lang: _",
v2_job.script_entrypoint_override,
v2_job.flow_step,
v2_job.flow_step_id,
v2_job.flow_innermost_root_job,
v2_job."trigger",
v2_job.trigger_kind as "trigger_kind: _",
v2_job.same_worker,
v2_job.visible_to_owner,
v2_job.concurrent_limit,
v2_job.concurrency_time_window_s,
v2_job.cache_ttl,
v2_job.timeout,
v2_job.priority,
v2_job.preprocessed,
v2_job.args as "args: _",
v2_job.labels,
v2_job.pre_run_error,
v2_job_queue.started_at,
v2_job_queue.scheduled_for,
v2_job_queue.running,
v2_job_queue.canceled_by,
v2_job_queue.canceled_reason,
v2_job_queue.suspend,
v2_job_queue.suspend_until,
v2_job_queue.worker,
v2_job_queue.extras as "extras: _",
v2_job_runtime.ping,
v2_job_runtime.memory_peak,
v2_job_status.flow_status as "flow_status: _",
v2_job_status.flow_leaf_jobs as "flow_leaf_jobs: _",
v2_job_status.workflow_as_code_status as "workflow_as_code_status: _",
concurrency_key.key as "concurrency_key?"
FROM v2_job_queue
INNER JOIN v2_job ON v2_job.id = v2_job_queue.id
LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id
LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id
LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_job_queue.id
WHERE v2_job_queue.workspace_id = $1
AND v2_job_queue.running = false
AND v2_job.parent_job IS NULL
AND v2_job.trigger_kind IS DISTINCT FROM 'schedule'
ORDER BY v2_job.created_at DESC
LIMIT $2
OFFSET $3
"#,
w_id,
per_page,
offset
)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
Ok(Json(jobs))
}
pub async fn import_completed_jobs(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(jobs): Json<Vec<ExportableCompletedJob>>,
) -> error::Result<String> {
require_admin(authed.is_admin, &authed.username)?;
let mut tx = user_db.begin(&authed).await?;
for job in jobs {
sqlx::query!(
r#"
INSERT INTO v2_job (
id, raw_code, raw_lock, raw_flow, tag, workspace_id, created_at, created_by,
permissioned_as, permissioned_as_email, kind, runnable_id, runnable_path,
parent_job, root_job, script_lang, script_entrypoint_override, flow_step,
flow_step_id, flow_innermost_root_job, trigger, trigger_kind, same_worker,
visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl,
timeout, priority, preprocessed, args, labels, pre_run_error
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,
$21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33
)
ON CONFLICT (id) DO NOTHING
"#,
job.id, job.raw_code, job.raw_lock, job.raw_flow as _, &job.tag, &w_id,
job.created_at, &job.created_by, &job.permissioned_as, job.permissioned_as_email,
job.kind as _, job.runnable_id, job.runnable_path, job.parent_job, job.root_job,
job.script_lang as _, job.script_entrypoint_override, job.flow_step,
job.flow_step_id, job.flow_innermost_root_job, job.trigger, job.trigger_kind as _,
job.same_worker, job.visible_to_owner, job.concurrent_limit,
job.concurrency_time_window_s, job.cache_ttl, job.timeout, job.priority,
job.preprocessed, job.args as _, job.labels as _, job.pre_run_error
)
.execute(&mut *tx)
.await?;
sqlx::query!(
r#"
INSERT INTO v2_job_completed (
id, workspace_id, started_at, completed_at, duration_ms, result, deleted,
canceled_by, canceled_reason, flow_status, memory_peak, status, worker,
workflow_as_code_status, result_columns, retries, extras
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17
)
ON CONFLICT (id) DO NOTHING
"#,
job.id,
&w_id,
job.started_at,
job.completed_at,
job.duration_ms,
job.result as _,
job.deleted,
job.canceled_by,
job.canceled_reason,
job.flow_status as _,
job.memory_peak,
job.status as _,
job.worker,
job.workflow_as_code_status as _,
job.result_columns as _,
job.retries as _,
job.extras as _
)
.execute(&mut *tx)
.await?;
if let Some(logs) = &job.logs {
sqlx::query!(
r#"
INSERT INTO job_logs (
job_id, workspace_id, logs, log_offset, log_file_index
) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (job_id) DO NOTHING
"#,
job.id,
&w_id,
logs,
job.log_offset,
job.log_file_index as _
)
.execute(&mut *tx)
.await?;
}
}
tx.commit().await?;
Ok(format!("Successfully imported jobs"))
}
pub async fn import_queued_jobs(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(jobs): Json<Vec<ExportableQueuedJob>>,
) -> error::Result<String> {
require_admin(authed.is_admin, &authed.username)?;
let mut tx = user_db.begin(&authed).await?;
for job in jobs {
sqlx::query!(
r#"
INSERT INTO v2_job (
id, raw_code, raw_lock, raw_flow, tag, workspace_id, created_at, created_by,
permissioned_as, permissioned_as_email, kind, runnable_id, runnable_path,
parent_job, root_job, script_lang, script_entrypoint_override, flow_step,
flow_step_id, flow_innermost_root_job, trigger, trigger_kind, same_worker,
visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl,
timeout, priority, preprocessed, args, labels, pre_run_error
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
$11, $12, $13, $14, $15, $16, $17, $18, $19, $20,
$21, $22, $23, $24, $25, $26, $27, $28, $29, $30,
$31, $32, $33
)
ON CONFLICT (id) DO NOTHING
"#,
job.id,
job.raw_code,
job.raw_lock,
job.raw_flow as _,
&job.tag,
&w_id,
job.created_at,
&job.created_by,
&job.permissioned_as,
&job.permissioned_as_email,
job.kind as _,
job.runnable_id,
job.runnable_path,
job.parent_job,
job.root_job,
job.script_lang as _,
job.script_entrypoint_override,
job.flow_step,
job.flow_step_id,
job.flow_innermost_root_job,
job.trigger,
job.trigger_kind as _,
job.same_worker,
job.visible_to_owner,
job.concurrent_limit,
job.concurrency_time_window_s,
job.cache_ttl,
job.timeout,
job.priority,
job.preprocessed,
job.args as _,
job.labels as _,
job.pre_run_error,
)
.execute(&mut *tx)
.await?;
sqlx::query!(
r#"
INSERT INTO v2_job_queue (
id, workspace_id, started_at, scheduled_for, running, canceled_by,
canceled_reason, suspend, suspend_until, worker, extras, tag, priority
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13
)
ON CONFLICT (id) DO NOTHING
"#,
job.id,
&w_id,
job.started_at,
job.scheduled_for,
job.running,
job.canceled_by,
job.canceled_reason,
job.suspend,
job.suspend_until,
job.worker,
job.extras as _,
&job.tag,
job.priority
)
.execute(&mut *tx)
.await?;
if job.ping.is_some() || job.memory_peak.is_some() {
sqlx::query!(
r#"
INSERT INTO v2_job_runtime (id, ping, memory_peak)
VALUES ($1, $2, $3)
ON CONFLICT (id) DO NOTHING
"#,
job.id,
job.ping,
job.memory_peak
)
.execute(&mut *tx)
.await?;
}
if job.flow_status.is_some()
|| job.flow_leaf_jobs.is_some()
|| job.workflow_as_code_status.is_some()
{
sqlx::query!(
r#"
INSERT INTO v2_job_status (id, flow_status, flow_leaf_jobs, workflow_as_code_status)
VALUES ($1, $2, $3, $4)
ON CONFLICT (id) DO NOTHING
"#,
job.id,
job.flow_status as _,
job.flow_leaf_jobs as _,
job.workflow_as_code_status as _
)
.execute(&mut *tx)
.await?;
}
if let Some(ref concurrency_key) = job.concurrency_key {
sqlx::query!(
r#"
WITH inserted_concurrency_counter AS (
INSERT INTO concurrency_counter (concurrency_id, job_uuids)
VALUES ($1, '{}'::jsonb)
ON CONFLICT DO NOTHING
)
INSERT INTO concurrency_key(key, job_id)
VALUES ($1, $2)
ON CONFLICT (job_id) DO NOTHING
"#,
concurrency_key,
job.id
)
.execute(&mut *tx)
.await?;
}
}
tx.commit().await?;
Ok(format!("Successfully imported jobs"))
}
pub async fn delete_jobs(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(job_ids): Json<Vec<Uuid>>,
) -> error::Result<String> {
require_admin(authed.is_admin, &authed.username)?;
if job_ids.is_empty() {
return Ok("No jobs to delete".to_string());
}
let mut tx = user_db.begin(&authed).await?;
let logs_deleted = sqlx::query!(
"DELETE FROM job_logs WHERE workspace_id = $1 AND job_id = ANY($2)",
&w_id,
&job_ids
)
.execute(&mut *tx)
.await?
.rows_affected();
let perms_deleted = sqlx::query!(
"DELETE FROM job_perms WHERE workspace_id = $1 AND job_id = ANY($2)",
&w_id,
&job_ids
)
.execute(&mut *tx)
.await?
.rows_affected();
let stats_deleted = sqlx::query!(
"DELETE FROM job_stats WHERE workspace_id = $1 AND job_id = ANY($2)",
&w_id,
&job_ids
)
.execute(&mut *tx)
.await?
.rows_affected();
let resume_deleted = sqlx::query!("DELETE FROM resume_job WHERE job = ANY($1)", &job_ids)
.execute(&mut *tx)
.await?
.rows_affected();
let runtime_deleted = sqlx::query!("DELETE FROM v2_job_runtime WHERE id = ANY($1)", &job_ids)
.execute(&mut *tx)
.await?
.rows_affected();
let status_deleted = sqlx::query!("DELETE FROM v2_job_status WHERE id = ANY($1)", &job_ids)
.execute(&mut *tx)
.await?
.rows_affected();
let concurrency_key_deleted = sqlx::query!(
"DELETE FROM concurrency_key WHERE job_id = ANY($1)",
&job_ids
)
.execute(&mut *tx)
.await?
.rows_affected();
let queue_deleted = sqlx::query!(
"DELETE FROM v2_job_queue WHERE workspace_id = $1 AND id = ANY($2)",
&w_id,
&job_ids
)
.execute(&mut *tx)
.await?
.rows_affected();
let completed_deleted = sqlx::query!(
"DELETE FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)",
&w_id,
&job_ids
)
.execute(&mut *tx)
.await?
.rows_affected();
let zombie_deleted = sqlx::query!(
"DELETE FROM zombie_job_counter WHERE job_id = ANY($1)",
&job_ids
)
.execute(&mut *tx)
.await?
.rows_affected();
let jobs_deleted = sqlx::query!(
"DELETE FROM v2_job WHERE workspace_id = $1 AND id = ANY($2)",
&w_id,
&job_ids
)
.execute(&mut *tx)
.await?
.rows_affected();
tx.commit().await?;
let total_rows_deleted = logs_deleted
+ perms_deleted
+ stats_deleted
+ resume_deleted
+ runtime_deleted
+ status_deleted
+ concurrency_key_deleted
+ queue_deleted
+ completed_deleted
+ zombie_deleted
+ jobs_deleted;
tracing::info!(
"Successfully deleted {} jobs ({} total rows across all tables) from workspace {}",
job_ids.len(),
total_rows_deleted,
w_id
);
Ok(format!(
"Successfully deleted {} jobs ({} total rows across all tables)",
jobs_deleted, total_rows_deleted
))
}
backend/windmill-api-jobs/src/lib.rs
+18
View File
@@ -0,0 +1,18 @@
/*
* Author: Windmill Labs, Inc
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
pub mod concurrency_groups;
pub mod execution;
pub mod job_metrics;
pub mod jobs_export;
pub mod query;
pub mod types;
pub use execution::*;
pub use query::*;
pub use types::*;
backend/windmill-api-jobs/src/query.rs
+373
View File
@@ -0,0 +1,373 @@
/*
* Author: Windmill Labs, Inc
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Query builders for filtering job lists (queue and completed).
use sql_builder::prelude::*;
use sql_builder::SqlBuilder;
use windmill_common::utils::{paginate_without_limits, Pagination};
use crate::types::{ListCompletedQuery, ListQueueQuery};
pub fn filter_list_queue_query(
mut sqlb: SqlBuilder,
lq: &ListQueueQuery,
w_id: &str,
join_outstanding_wait_times: bool,
) -> SqlBuilder {
sqlb.join("v2_job").on_eq("v2_job_queue.id", "v2_job.id");
if join_outstanding_wait_times {
sqlb.left()
.join("outstanding_wait_time")
.on_eq("v2_job_queue.id", "outstanding_wait_time.job_id");
}
if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) {
sqlb.and_where_eq("v2_job_queue.workspace_id", "?".bind(&w_id));
}
if let Some(w) = &lq.worker {
if lq.allow_wildcards.unwrap_or(false) {
sqlb.and_where_like_left("v2_job_queue.worker", w.replace("*", "%"));
} else {
sqlb.and_where_eq("v2_job_queue.worker", "?".bind(w));
}
}
if let Some(ps) = &lq.script_path_start {
sqlb.and_where_like_left("runnable_path", ps);
}
if let Some(p) = &lq.script_path_exact {
sqlb.and_where_eq("runnable_path", "?".bind(p));
}
if let Some(p) = &lq.schedule_path {
sqlb.and_where_eq("trigger", "?".bind(p));
sqlb.and_where_eq("trigger_kind", "'schedule'");
}
if let Some(h) = &lq.script_hash {
sqlb.and_where_eq("runnable_id", "?".bind(h));
}
if let Some(cb) = &lq.created_by {
sqlb.and_where_eq("created_by", "?".bind(cb));
}
if let Some(t) = &lq.tag {
if lq.allow_wildcards.unwrap_or(false) {
sqlb.and_where_like_left("v2_job.tag", t.replace("*", "%"));
} else {
sqlb.and_where_eq("v2_job.tag", "?".bind(t));
}
}
if let Some(r) = &lq.running {
sqlb.and_where_eq("running", &r);
}
if let Some(pj) = &lq.parent_job {
sqlb.and_where_eq("parent_job", "?".bind(pj));
}
if let Some(dt) = &lq.started_before {
sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339()));
}
if let Some(dt) = &lq.started_after {
sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339()));
}
if let Some(fs) = &lq.is_flow_step {
if *fs {
sqlb.and_where_is_not_null("flow_step_id");
} else {
sqlb.and_where_is_null("flow_step_id");
}
}
if let Some(fs) = &lq.has_null_parent {
if *fs {
sqlb.and_where_is_null("parent_job");
}
}
if let Some(dt) = &lq.created_before {
sqlb.and_where_le("v2_job.created_at", "?".bind(&dt.to_rfc3339()));
}
if let Some(dt) = &lq.created_after {
sqlb.and_where_ge("v2_job.created_at", "?".bind(&dt.to_rfc3339()));
}
if let Some(dt) = &lq.created_or_started_after {
let ts = dt.timestamp_millis();
sqlb.and_where(format!("(started_at IS NOT NULL AND started_at >= to_timestamp({} / 1000.0)) OR (started_at IS NULL AND v2_job.created_at >= to_timestamp({} / 1000.0))", ts, ts));
}
if let Some(dt) = &lq.created_or_started_before {
let ts = dt.timestamp_millis();
sqlb.and_where(format!("(started_at IS NOT NULL AND started_at < to_timestamp({} / 1000.0)) OR (started_at IS NULL AND v2_job.created_at < to_timestamp({} / 1000.0))", ts, ts));
}
if let Some(s) = &lq.suspended {
if *s {
sqlb.and_where_gt("suspend", 0);
} else {
sqlb.and_where_is_null("suspend_until");
}
}
if let Some(jk) = &lq.job_kinds {
sqlb.and_where_in(
"kind",
&jk.split(',').into_iter().map(quote).collect::<Vec<_>>(),
);
}
if let Some(args) = &lq.args {
sqlb.and_where("args @> ?".bind(&args.replace("'", "''")));
}
if lq.scheduled_for_before_now.is_some_and(|x| x) {
sqlb.and_where_le("scheduled_for", "now()");
}
if lq.is_not_schedule.unwrap_or(false) {
sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'");
}
if let Some(tk) = &lq.trigger_kind {
sqlb.and_where_eq("trigger_kind", "?".bind(&format!("{}", tk)));
}
if let Some(tp) = &lq.trigger_path {
sqlb.and_where_eq("trigger", "?".bind(tp));
}
sqlb
}
pub fn list_queue_jobs_query(
w_id: &str,
lq: &ListQueueQuery,
fields: &[&str],
pagination: Pagination,
join_outstanding_wait_times: bool,
tags: Option<Vec<&str>>,
) -> SqlBuilder {
let (limit, offset) = paginate_without_limits(pagination);
let mut sqlb = SqlBuilder::select_from("v2_job_queue")
.fields(fields)
.order_by("v2_job.created_at", lq.order_desc.unwrap_or(true))
.limit(limit)
.offset(offset)
.clone();
if let Some(tags) = tags {
sqlb.and_where_in(
"v2_job.tag",
&tags.iter().map(|x| quote(x)).collect::<Vec<_>>(),
);
}
filter_list_queue_query(sqlb, lq, w_id, join_outstanding_wait_times)
}
pub fn filter_list_completed_query(
mut sqlb: SqlBuilder,
lq: &ListCompletedQuery,
w_id: &str,
join_outstanding_wait_times: bool,
) -> SqlBuilder {
sqlb.join("v2_job")
.on_eq("v2_job_completed.id", "v2_job.id");
if join_outstanding_wait_times {
sqlb.left()
.join("outstanding_wait_time")
.on_eq("v2_job_completed.id", "outstanding_wait_time.job_id");
}
if let Some(label) = &lq.label {
if lq.allow_wildcards.unwrap_or(false) {
let wh = format!(
"EXISTS (SELECT 1 FROM jsonb_array_elements_text(result->'wm_labels') label WHERE jsonb_typeof(result->'wm_labels') = 'array' AND label LIKE '{}')",
&label.replace("*", "%").replace("'", "''")
);
sqlb.and_where("result ? 'wm_labels'");
sqlb.and_where(&wh);
} else {
let mut wh = format!("result->'wm_labels' ? ");
wh.push_str(&format!("'{}'", &label.replace("'", "''")));
sqlb.and_where("result ? 'wm_labels'");
sqlb.and_where(&wh);
}
}
if let Some(worker) = &lq.worker {
if lq.allow_wildcards.unwrap_or(false) {
sqlb.and_where_like_left("v2_job_completed.worker", worker.replace("*", "%"));
} else {
sqlb.and_where_eq("v2_job_completed.worker", "?".bind(worker));
}
}
if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) {
sqlb.and_where_eq("v2_job_completed.workspace_id", "?".bind(&w_id))
.and_where_eq("v2_job.workspace_id", "?".bind(&w_id));
}
if let Some(p) = &lq.schedule_path {
sqlb.and_where_eq("trigger", "?".bind(p));
sqlb.and_where_eq("trigger_kind", "'schedule'");
}
if let Some(ps) = &lq.script_path_start {
sqlb.and_where_like_left("runnable_path", ps);
}
if let Some(p) = &lq.script_path_exact {
sqlb.and_where_eq("runnable_path", "?".bind(p));
}
if let Some(h) = &lq.script_hash {
sqlb.and_where_eq("runnable_id", "?".bind(h));
}
if let Some(t) = &lq.tag {
if lq.allow_wildcards.unwrap_or(false) {
sqlb.and_where_like_left("v2_job.tag", t.replace("*", "%"));
} else {
sqlb.and_where_eq("v2_job.tag", "?".bind(t));
}
}
if let Some(cb) = &lq.created_by {
sqlb.and_where_eq("created_by", "?".bind(cb));
}
if let Some(r) = &lq.success {
if *r {
sqlb.and_where_eq("status", "'success'")
.or_where_eq("status", "'skipped'");
} else {
sqlb.and_where_eq("status", "'failure'")
.or_where_eq("status", "'canceled'");
}
}
if let Some(pj) = &lq.parent_job {
sqlb.and_where_eq("parent_job", "?".bind(pj));
}
if let Some(dt) = &lq.started_before {
sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339()));
}
if let Some(dt) = &lq.started_after {
sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339()));
}
if let Some(dt) = &lq.created_or_started_before {
sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339()));
}
if let Some(dt) = &lq.created_or_started_after {
let ts = dt.to_rfc3339();
sqlb.and_where(format!(
"(created_at >= '{}' OR started_at >= '{}')",
ts.replace("'", "''"),
ts.replace("'", "''")
));
}
if let Some(dt) = &lq.created_before {
sqlb.and_where_le("created_at", "?".bind(&dt.to_rfc3339()));
}
if let Some(dt) = &lq.created_after {
sqlb.and_where_ge("created_at", "?".bind(&dt.to_rfc3339()));
}
if let Some(dt) = &lq.created_or_started_after_completed_jobs {
sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339()));
}
if let Some(dt) = &lq.completed_after {
sqlb.and_where_ge("completed_at", "?".bind(&dt.to_rfc3339()));
}
if let Some(dt) = &lq.completed_before {
sqlb.and_where_le("completed_at", "?".bind(&dt.to_rfc3339()));
}
if let Some(sk) = &lq.is_skipped {
if *sk {
sqlb.and_where_eq("status", "'skipped'");
} else {
sqlb.and_where_ne("status", "'skipped'");
}
}
if let Some(fs) = &lq.is_flow_step {
if *fs {
sqlb.and_where_is_not_null("flow_step_id");
} else {
sqlb.and_where_is_null("flow_step_id");
}
}
if let Some(fs) = &lq.has_null_parent {
if *fs {
sqlb.and_where_is_null("parent_job");
}
}
if let Some(jk) = &lq.job_kinds {
sqlb.and_where_in(
"kind",
&jk.split(',').into_iter().map(quote).collect::<Vec<_>>(),
);
}
if let Some(args) = &lq.args {
sqlb.and_where("args @> ?".bind(&args.replace("'", "''")));
}
if let Some(result) = &lq.result {
sqlb.and_where("result @> ?".bind(&result.replace("'", "''")));
}
if lq.is_not_schedule.unwrap_or(false) {
sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'");
}
if let Some(tk) = &lq.trigger_kind {
sqlb.and_where_eq("trigger_kind", "?".bind(&format!("{}", tk)));
}
if let Some(tp) = &lq.trigger_path {
sqlb.and_where_eq("trigger", "?".bind(tp));
}
sqlb
}
pub fn list_completed_jobs_query(
w_id: &str,
per_page: Option<usize>,
offset: usize,
lq: &ListCompletedQuery,
fields: &[&str],
join_outstanding_wait_times: bool,
tags: Option<Vec<&str>>,
) -> SqlBuilder {
let mut sqlb = SqlBuilder::select_from("v2_job_completed")
.fields(fields)
.order_by(
if lq.completed_before.is_some() || lq.completed_after.is_some() {
"v2_job_completed.completed_at"
} else {
"v2_job.created_at"
},
lq.order_desc.unwrap_or(true),
)
.offset(offset)
.clone();
if let Some(per_page) = per_page {
sqlb.limit(per_page);
}
if let Some(tags) = tags {
sqlb.and_where_in(
"v2_job.tag",
&tags.iter().map(|x| quote(x)).collect::<Vec<_>>(),
);
}
filter_list_completed_query(sqlb, lq, w_id, join_outstanding_wait_times)
}
backend/windmill-api-jobs/src/types.rs
+570
View File
@@ -0,0 +1,570 @@
/*
* Author: Windmill Labs, Inc
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Shared types for the jobs subsystem.
use axum::{
extract::{FromRequest, Json, Query, Request},
response::{IntoResponse, Response},
};
use http::StatusCode;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde_json::value::RawValue;
use std::collections::HashMap;
use uuid::Uuid;
use windmill_common::{
error,
jobs::{CompletedJob, JobKind, JobTriggerKind, QueuedJob},
scripts::{ScriptHash, ScriptLang},
utils::now_from_db,
DB,
};
use windmill_api_sse::{Job, JobExtended};
// ------------ RunJobQuery ------------
#[derive(Debug, Deserialize, Clone, Default)]
pub struct RunJobQuery {
pub scheduled_for: Option<chrono::DateTime<chrono::Utc>>,
pub scheduled_in_secs: Option<i64>,
pub parent_job: Option<Uuid>,
pub root_job: Option<Uuid>,
pub invisible_to_owner: Option<bool>,
pub queue_limit: Option<i64>,
pub payload: Option<String>,
pub job_id: Option<Uuid>,
pub tag: Option<String>,
pub timeout: Option<i32>,
pub cache_ttl: Option<i32>,
pub cache_ignore_s3_path: Option<bool>,
pub skip_preprocessor: Option<bool>,
pub poll_delay_ms: Option<u64>,
pub memory_id: Option<Uuid>,
pub trigger_external_id: Option<String>,
pub service_name: Option<String>,
pub suspended_mode: Option<bool>,
}
impl RunJobQuery {
pub async fn get_scheduled_for(
&self,
db: &DB,
) -> error::Result<Option<chrono::DateTime<chrono::Utc>>> {
if let Some(scheduled_for) = self.scheduled_for {
Ok(Some(scheduled_for))
} else if let Some(scheduled_in_secs) = self.scheduled_in_secs {
let now = now_from_db(db).await?;
Ok(Some(
now + chrono::Duration::try_seconds(scheduled_in_secs).unwrap_or_default(),
))
} else {
Ok(None)
}
}
pub fn payload_as_args(&self) -> error::Result<HashMap<String, Box<RawValue>>> {
let payload_r = self.payload.clone().map(decode_payload).map(|x| {
x.map_err(|e| {
error::Error::internal_err(format!("Impossible to decode query payload: {e:#?}"))
})
});
let payload_as_args = if let Some(payload) = payload_r {
payload?
} else {
HashMap::new()
};
Ok(payload_as_args)
}
}
// ------------ List query types ------------
#[derive(Deserialize, Clone)]
pub struct ListQueueQuery {
pub script_path_start: Option<String>,
pub script_path_exact: Option<String>,
pub script_hash: Option<String>,
pub created_by: Option<String>,
pub started_before: Option<chrono::DateTime<chrono::Utc>>,
pub started_after: Option<chrono::DateTime<chrono::Utc>>,
pub created_before: Option<chrono::DateTime<chrono::Utc>>,
pub created_after: Option<chrono::DateTime<chrono::Utc>>,
pub created_or_started_before: Option<chrono::DateTime<chrono::Utc>>,
pub created_or_started_after: Option<chrono::DateTime<chrono::Utc>>,
pub running: Option<bool>,
pub schedule_path: Option<String>,
pub parent_job: Option<String>,
pub order_desc: Option<bool>,
pub job_kinds: Option<String>,
pub suspended: Option<bool>,
pub worker: Option<String>,
// filter by matching a subset of the args using base64 encoded json subset
pub args: Option<String>,
pub tag: Option<String>,
pub scheduled_for_before_now: Option<bool>,
pub all_workspaces: Option<bool>,
pub is_flow_step: Option<bool>,
pub has_null_parent: Option<bool>,
pub is_not_schedule: Option<bool>,
pub concurrency_key: Option<String>,
pub allow_wildcards: Option<bool>,
pub trigger_kind: Option<JobTriggerKind>,
pub trigger_path: Option<String>,
pub include_args: Option<bool>,
}
#[derive(Deserialize, Clone)]
pub struct ListCompletedQuery {
pub script_path_start: Option<String>,
pub script_path_exact: Option<String>,
pub script_hash: Option<String>,
pub created_by: Option<String>,
pub started_before: Option<chrono::DateTime<chrono::Utc>>,
pub started_after: Option<chrono::DateTime<chrono::Utc>>,
pub created_before: Option<chrono::DateTime<chrono::Utc>>,
pub created_after: Option<chrono::DateTime<chrono::Utc>>,
pub created_or_started_before: Option<chrono::DateTime<chrono::Utc>>,
pub created_or_started_after: Option<chrono::DateTime<chrono::Utc>>,
pub created_or_started_after_completed_jobs: Option<chrono::DateTime<chrono::Utc>>,
pub created_before_queue: Option<chrono::DateTime<chrono::Utc>>,
pub created_after_queue: Option<chrono::DateTime<chrono::Utc>>,
pub completed_after: Option<chrono::DateTime<chrono::Utc>>,
pub completed_before: Option<chrono::DateTime<chrono::Utc>>,
pub success: Option<bool>,
pub running: Option<bool>,
pub parent_job: Option<String>,
pub order_desc: Option<bool>,
pub job_kinds: Option<String>,
pub is_skipped: Option<bool>,
pub is_flow_step: Option<bool>,
pub suspended: Option<bool>,
pub schedule_path: Option<String>,
// filter by matching a subset of the args using base64 encoded json subset
pub args: Option<String>,
// filter by matching a subset of the result using base64 encoded json subset
pub result: Option<String>,
pub tag: Option<String>,
pub scheduled_for_before_now: Option<bool>,
pub all_workspaces: Option<bool>,
pub has_null_parent: Option<bool>,
pub label: Option<String>,
pub is_not_schedule: Option<bool>,
pub concurrency_key: Option<String>,
pub worker: Option<String>,
pub allow_wildcards: Option<bool>,
pub trigger_kind: Option<JobTriggerKind>,
pub trigger_path: Option<String>,
pub include_args: Option<bool>,
}
impl From<ListCompletedQuery> for ListQueueQuery {
fn from(lcq: ListCompletedQuery) -> Self {
Self {
script_path_start: lcq.script_path_start,
script_path_exact: lcq.script_path_exact,
script_hash: lcq.script_hash,
created_by: lcq.created_by,
started_before: lcq.started_before,
started_after: lcq.started_after,
created_before: lcq.created_before_queue.or(lcq.created_before),
created_after: lcq.created_after_queue.or(lcq.created_after),
created_or_started_before: lcq.created_or_started_before,
created_or_started_after: lcq.created_or_started_after,
worker: lcq.worker,
running: lcq.running,
parent_job: lcq.parent_job,
order_desc: lcq.order_desc,
job_kinds: lcq.job_kinds,
suspended: lcq.suspended,
args: lcq.args,
tag: lcq.tag,
schedule_path: lcq.schedule_path,
scheduled_for_before_now: lcq.scheduled_for_before_now,
all_workspaces: lcq.all_workspaces,
is_flow_step: lcq.is_flow_step,
has_null_parent: lcq.has_null_parent,
is_not_schedule: lcq.is_not_schedule,
concurrency_key: lcq.concurrency_key,
allow_wildcards: lcq.allow_wildcards,
trigger_kind: lcq.trigger_kind,
trigger_path: lcq.trigger_path,
include_args: lcq.include_args,
}
}
}
// ------------ ListableCompletedJob ------------
#[derive(Debug, sqlx::FromRow, Serialize)]
pub struct ListableCompletedJob {
pub r#type: String,
pub workspace_id: String,
pub id: Uuid,
#[serde(skip_serializing_if = "Option::is_none")]
pub parent_job: Option<Uuid>,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
pub duration_ms: i64,
pub success: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub script_hash: Option<ScriptHash>,
#[serde(skip_serializing_if = "Option::is_none")]
pub script_path: Option<String>,
pub deleted: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub raw_code: Option<String>,
pub canceled: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub canceled_by: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub canceled_reason: Option<String>,
pub job_kind: JobKind,
#[serde(skip_serializing_if = "Option::is_none")]
pub schedule_path: Option<String>,
pub permissioned_as: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub flow_status: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub raw_flow: Option<serde_json::Value>,
pub is_flow_step: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub language: Option<ScriptLang>,
pub is_skipped: bool,
pub email: String,
pub visible_to_owner: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub mem_peak: Option<i32>,
pub tag: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub priority: Option<i16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub labels: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub args: Option<serde_json::Value>,
}
// ------------ UnifiedJob ------------
#[derive(sqlx::FromRow)]
pub struct UnifiedJob {
pub workspace_id: String,
pub typ: String,
pub id: Uuid,
pub parent_job: Option<Uuid>,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
pub completed_at: Option<chrono::DateTime<chrono::Utc>>,
pub scheduled_for: Option<chrono::DateTime<chrono::Utc>>,
pub running: Option<bool>,
pub script_hash: Option<ScriptHash>,
pub script_path: Option<String>,
pub args: Option<serde_json::Value>,
pub duration_ms: Option<i64>,
pub success: Option<bool>,
pub deleted: bool,
pub canceled: bool,
pub canceled_by: Option<String>,
pub job_kind: JobKind,
pub schedule_path: Option<String>,
pub permissioned_as: String,
pub is_flow_step: bool,
pub language: Option<ScriptLang>,
pub is_skipped: bool,
pub email: String,
pub visible_to_owner: bool,
pub suspend: Option<i32>,
pub mem_peak: Option<i32>,
pub tag: String,
pub concurrent_limit: Option<i32>,
pub concurrency_time_window_s: Option<i32>,
pub priority: Option<i16>,
pub labels: Option<serde_json::Value>,
pub self_wait_time_ms: Option<i64>,
pub aggregate_wait_time_ms: Option<i64>,
pub preprocessed: Option<bool>,
pub worker: Option<String>,
pub runnable_settings_handle: Option<i64>,
}
const CJ_FIELDS: &[&str] = &[
"'CompletedJob' as typ",
"v2_job_completed.id",
"v2_job_completed.workspace_id",
"v2_job.parent_job",
"v2_job.created_by",
"v2_job.created_at",
"v2_job_completed.started_at",
"null as scheduled_for",
"v2_job_completed.completed_at",
"null as running",
"v2_job.runnable_id as script_hash",
"v2_job.runnable_path as script_path",
"null as args",
"v2_job_completed.duration_ms",
"v2_job_completed.status = 'success' OR v2_job_completed.status = 'skipped' as success",
"false as deleted",
"v2_job_completed.status = 'canceled' as canceled",
"v2_job_completed.canceled_by",
"v2_job.kind as job_kind",
"CASE WHEN v2_job.trigger_kind = 'schedule' THEN v2_job.trigger END as schedule_path",
"v2_job.permissioned_as",
"v2_job.flow_step_id IS NOT NULL as is_flow_step",
"v2_job.script_lang as language",
"v2_job_completed.status = 'skipped' as is_skipped",
"v2_job.permissioned_as_email as email",
"v2_job.visible_to_owner",
"null as suspend",
"v2_job_completed.memory_peak as mem_peak",
"v2_job.tag",
"null as concurrent_limit",
"null as concurrency_time_window_s",
"v2_job.priority",
"v2_job_completed.result->'wm_labels' as labels",
"self_wait_time_ms",
"aggregate_wait_time_ms",
"v2_job.preprocessed",
"v2_job_completed.worker",
"null as runnable_settings_handle",
];
const QJ_FIELDS: &[&str] = &[
"'QueuedJob' as typ",
"v2_job_queue.id",
"v2_job_queue.workspace_id",
"v2_job.parent_job",
"v2_job.created_by",
"v2_job_queue.created_at",
"v2_job_queue.started_at",
"v2_job_queue.scheduled_for",
"null as completed_at",
"v2_job_queue.running",
"v2_job.runnable_id as script_hash",
"v2_job.runnable_path as script_path",
"null as args",
"null as duration_ms",
"null as success",
"false as deleted",
"v2_job_queue.canceled_by IS NOT NULL as canceled",
"v2_job_queue.canceled_by",
"v2_job.kind as job_kind",
"CASE WHEN v2_job.trigger_kind = 'schedule' THEN v2_job.trigger END as schedule_path",
"v2_job.permissioned_as",
"v2_job.flow_step_id IS NOT NULL as is_flow_step",
"v2_job.script_lang as language",
"false as is_skipped",
"v2_job.permissioned_as_email as email",
"v2_job.visible_to_owner",
"v2_job_queue.suspend",
"null as mem_peak",
"v2_job.tag",
"v2_job.concurrent_limit",
"v2_job.concurrency_time_window_s",
"v2_job.priority",
"null as labels",
"self_wait_time_ms",
"aggregate_wait_time_ms",
"v2_job.preprocessed",
"v2_job_queue.worker",
"v2_job_queue.runnable_settings_handle",
];
impl UnifiedJob {
pub fn completed_job_fields() -> &'static [&'static str] {
CJ_FIELDS
}
pub fn queued_job_fields() -> &'static [&'static str] {
QJ_FIELDS
}
}
impl From<UnifiedJob> for Job {
fn from(uj: UnifiedJob) -> Self {
let args = uj.args.and_then(|v| serde_json::from_value(v).ok());
match uj.typ.as_ref() {
"CompletedJob" => Job::CompletedJob(JobExtended::new(
uj.self_wait_time_ms,
uj.aggregate_wait_time_ms,
CompletedJob {
workspace_id: uj.workspace_id,
id: uj.id,
parent_job: uj.parent_job,
created_by: uj.created_by,
created_at: uj.created_at,
started_at: uj.started_at,
completed_at: uj.completed_at,
duration_ms: uj.duration_ms.unwrap(),
success: uj.success.unwrap(),
script_hash: uj.script_hash,
script_path: uj.script_path,
args: args.clone(),
result: None,
result_columns: None,
logs: None,
flow_status: None,
workflow_as_code_status: None,
deleted: uj.deleted,
canceled: uj.canceled,
canceled_by: uj.canceled_by,
canceled_reason: None,
job_kind: uj.job_kind,
schedule_path: uj.schedule_path,
permissioned_as: uj.permissioned_as,
is_flow_step: uj.is_flow_step,
language: uj.language,
is_skipped: uj.is_skipped,
email: uj.email,
visible_to_owner: uj.visible_to_owner,
mem_peak: uj.mem_peak,
tag: uj.tag,
priority: uj.priority,
labels: uj.labels,
preprocessed: uj.preprocessed,
},
)),
"QueuedJob" => Job::QueuedJob(JobExtended::new(
uj.self_wait_time_ms,
uj.aggregate_wait_time_ms,
QueuedJob {
workspace_id: uj.workspace_id,
id: uj.id,
parent_job: uj.parent_job,
created_by: uj.created_by,
created_at: uj.created_at,
started_at: uj.started_at,
scheduled_for: uj.scheduled_for.unwrap(),
running: uj.running.unwrap(),
script_hash: uj.script_hash,
script_path: uj.script_path,
script_entrypoint_override: None,
args,
logs: None,
canceled: uj.canceled,
canceled_by: uj.canceled_by,
canceled_reason: None,
last_ping: None,
job_kind: uj.job_kind,
schedule_path: uj.schedule_path,
permissioned_as: uj.permissioned_as,
flow_status: None,
workflow_as_code_status: None,
is_flow_step: uj.is_flow_step,
language: uj.language,
same_worker: false,
pre_run_error: None,
email: uj.email,
visible_to_owner: uj.visible_to_owner,
suspend: uj.suspend,
mem_peak: uj.mem_peak,
root_job: None,
leaf_jobs: None,
tag: uj.tag,
concurrent_limit: uj.concurrent_limit,
concurrency_time_window_s: uj.concurrency_time_window_s,
timeout: None,
flow_step_id: None,
cache_ttl: None,
cache_ignore_s3_path: None,
priority: uj.priority,
preprocessed: uj.preprocessed,
runnable_settings_handle: uj.runnable_settings_handle,
},
)),
t => panic!("job type {} not valid", t),
}
}
}
// ------------ Approval types ------------
#[derive(Deserialize, Debug)]
pub struct QueryApprover {
pub approver: Option<String>,
/// If true, generate/verify resume URLs for the parent flow instead of the specific step.
/// This allows pre-approvals that can be consumed by any later suspend step in the same flow.
pub flow_level: Option<bool>,
}
#[allow(non_snake_case)]
#[derive(Serialize, Debug)]
pub struct ResumeUrls {
pub approvalPage: String,
pub cancel: String,
pub resume: String,
}
// ------------ QueryOrBody extractor ------------
pub struct QueryOrBody<D>(pub Option<D>);
#[axum::async_trait]
impl<S, D> FromRequest<S, axum::body::Body> for QueryOrBody<D>
where
D: DeserializeOwned,
S: Send + Sync,
{
type Rejection = Response;
async fn from_request(
req: Request<axum::body::Body>,
state: &S,
) -> std::result::Result<Self, Self::Rejection> {
return if req.method() == axum::http::Method::GET {
let Query(InPayload { payload }) = Query::from_request(req, state)
.await
.map_err(IntoResponse::into_response)?;
payload
.map(|p| {
decode_payload(p)
.map(QueryOrBody)
.map_err(|err| (StatusCode::BAD_REQUEST, format!("{err:#?}")))
.map_err(IntoResponse::into_response)
})
.unwrap_or(Ok(QueryOrBody(None)))
} else {
Json::from_request(req, state)
.await
.map(|Json(v)| QueryOrBody(Some(v)))
.map_err(IntoResponse::into_response)
};
#[derive(Deserialize)]
struct InPayload {
payload: Option<String>,
}
}
}
// ------------ Utility functions ------------
pub fn decode_payload<D: DeserializeOwned>(t: String) -> anyhow::Result<D> {
let vec = base64::engine::general_purpose::STANDARD
.decode(t)
.context("invalid base64")?;
serde_json::from_slice(vec.as_slice()).context("invalid json")
}
pub fn add_raw_string(
raw_string: Option<String>,
mut args: serde_json::Map<String, serde_json::Value>,
) -> serde_json::Map<String, serde_json::Value> {
if let Some(raw_string) = raw_string {
args.insert(
"raw_string".to_string(),
serde_json::Value::String(raw_string),
);
}
return args;
}
use anyhow::Context;
use base64::Engine;
backend/windmill-api-sse/Cargo.toml
+21
View File
@@ -0,0 +1,21 @@
[package]
name = "windmill-api-sse"
version.workspace = true
authors.workspace = true
edition.workspace = true
[lib]
name = "windmill_api_sse"
path = "src/lib.rs"
[features]
default = []
enterprise = []
[dependencies]
windmill-common = { workspace = true, default-features = false }
serde.workspace = true
serde_json.workspace = true
sqlx.workspace = true
uuid.workspace = true
lazy_static.workspace = true
backend/windmill-api-sse/src/lib.rs
+341
View File
@@ -0,0 +1,341 @@
/*
* Author: Windmill Labs, Inc
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Shared SSE types for job update streaming.
//!
//! This crate contains the types used by the SSE job update system,
//! separated out so they can be depended on without pulling in the
//! full windmill-api-jobs crate.
use serde::Serialize;
use serde_json::value::RawValue;
use std::collections::HashMap;
use std::hash::{DefaultHasher, Hash, Hasher};
use std::ops::{Deref, DerefMut};
use uuid::Uuid;
use windmill_common::flow_status::FlowStatus;
use windmill_common::jobs::{CompletedJob, JobKind, QueuedJob};
// ------------ Job types (from jobs.rs) ------------
#[derive(sqlx::FromRow, Debug, Serialize)]
pub struct JobExtended<T: JobCommon> {
#[sqlx(flatten)]
#[serde(flatten)]
pub inner: T,
#[serde(skip_serializing_if = "Option::is_none")]
pub raw_code: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub raw_lock: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub raw_flow: Option<sqlx::types::Json<Box<RawValue>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub worker: Option<String>,
#[sqlx(skip)]
#[serde(skip_serializing_if = "Option::is_none")]
pub self_wait_time_ms: Option<i64>,
#[sqlx(skip)]
#[serde(skip_serializing_if = "Option::is_none")]
pub aggregate_wait_time_ms: Option<i64>,
}
pub trait JobCommon {
fn job_kind(&self) -> &JobKind;
}
impl JobCommon for QueuedJob {
fn job_kind(&self) -> &JobKind {
&self.job_kind
}
}
impl JobCommon for CompletedJob {
fn job_kind(&self) -> &JobKind {
&self.job_kind
}
}
impl<T: JobCommon> JobExtended<T> {
pub fn new(
self_wait_time_ms: Option<i64>,
aggregate_wait_time_ms: Option<i64>,
inner: T,
) -> Self {
Self {
inner,
raw_code: None,
raw_lock: None,
raw_flow: None,
worker: None,
self_wait_time_ms,
aggregate_wait_time_ms,
}
}
}
impl<T: JobCommon> Deref for JobExtended<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl<T: JobCommon> DerefMut for JobExtended<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner
}
}
#[derive(Serialize, Debug)]
#[serde(tag = "type")]
pub enum Job {
QueuedJob(JobExtended<QueuedJob>),
CompletedJob(JobExtended<CompletedJob>),
}
impl Job {
pub fn created_by(&self) -> &str {
match self {
Job::QueuedJob(job) => &job.created_by,
Job::CompletedJob(job) => &job.created_by,
}
}
pub fn append_to_logs(&mut self, logs: &str) {
match self {
Job::QueuedJob(job) => {
if let Some(ref mut l) = job.logs {
l.push_str(logs);
} else {
job.logs = Some(logs.to_string());
}
}
Job::CompletedJob(job) => {
if let Some(ref mut l) = job.logs {
l.push_str(logs);
} else {
job.logs = Some(logs.to_string());
}
}
}
}
pub fn log_len(&self) -> Option<usize> {
match self {
Job::QueuedJob(job) => job.logs.as_ref().map(|l| l.len()),
Job::CompletedJob(job) => job.logs.as_ref().map(|l| l.len()),
}
}
pub fn logs(&self) -> Option<String> {
match self {
Job::QueuedJob(job) => job.logs.clone(),
Job::CompletedJob(job) => job.logs.clone(),
}
}
pub fn flow_status(&self) -> Option<FlowStatus> {
match self {
Job::QueuedJob(job) => job
.flow_status
.as_ref()
.and_then(|rf| serde_json::from_str(rf.0.get()).ok()),
Job::CompletedJob(job) => job
.flow_status
.as_ref()
.and_then(|rf| serde_json::from_str(rf.0.get()).ok()),
}
}
pub fn is_flow_step(&self) -> bool {
match self {
Job::QueuedJob(job) => job.is_flow_step,
Job::CompletedJob(job) => job.is_flow_step,
}
}
pub fn is_flow(&self) -> bool {
self.job_kind().is_flow()
}
pub fn job_kind(&self) -> &JobKind {
match self {
Job::QueuedJob(job) => &job.job_kind,
Job::CompletedJob(job) => &job.job_kind,
}
}
pub fn id(&self) -> Uuid {
match self {
Job::QueuedJob(job) => job.id,
Job::CompletedJob(job) => job.id,
}
}
pub fn workspace_id(&self) -> &String {
match self {
Job::QueuedJob(job) => &job.workspace_id,
Job::CompletedJob(job) => &job.workspace_id,
}
}
pub fn script_path(&self) -> &str {
match self {
Job::QueuedJob(job) => job.script_path.as_ref(),
Job::CompletedJob(job) => job.script_path.as_ref(),
}
.map(String::as_str)
.unwrap_or("tmp/main")
}
pub fn args(&self) -> Option<&sqlx::types::Json<HashMap<String, Box<RawValue>>>> {
match self {
Job::QueuedJob(job) => job.args.as_ref(),
Job::CompletedJob(job) => job.args.as_ref(),
}
}
pub fn full_path_with_workspace(&self) -> String {
format!(
"{}/{}/{}",
self.workspace_id(),
if self.is_flow() { "flow" } else { "script" },
self.script_path(),
)
}
pub async fn concurrency_key(
&self,
db: &sqlx::Pool<sqlx::Postgres>,
) -> Result<Option<String>, sqlx::Error> {
sqlx::query_scalar!(
"SELECT key FROM concurrency_key WHERE job_id = $1",
self.id()
)
.fetch_optional(db)
.await
}
pub async fn fetch_outstanding_wait_time(
&mut self,
db: &sqlx::Pool<sqlx::Postgres>,
) -> Result<(), sqlx::Error> {
let r = sqlx::query!(
"SELECT self_wait_time_ms, aggregate_wait_time_ms FROM outstanding_wait_time WHERE job_id = $1",
self.id()
)
.fetch_optional(db)
.await?;
let (self_wait_time, aggregate_wait_time) = r
.map(|x| (x.self_wait_time_ms, x.aggregate_wait_time_ms))
.unwrap_or((None, None));
match self {
Job::QueuedJob(job) => {
job.self_wait_time_ms = self_wait_time;
job.aggregate_wait_time_ms = aggregate_wait_time;
}
Job::CompletedJob(job) => {
job.self_wait_time_ms = self_wait_time;
job.aggregate_wait_time_ms = aggregate_wait_time;
}
}
Ok(())
}
}
// ------------ SSE types ------------
#[derive(Serialize, Debug)]
pub struct JobUpdate {
#[serde(skip_serializing_if = "Option::is_none")]
pub running: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub completed: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub new_logs: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub new_result_stream: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub log_offset: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stream_offset: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mem_peak: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub progress: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub flow_status: Option<Box<RawValue>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub workflow_as_code_status: Option<Box<RawValue>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub job: Option<Job>,
#[serde(skip_serializing_if = "Option::is_none")]
pub only_result: Option<Box<RawValue>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub flow_stream_job_id: Option<Uuid>,
}
impl JobUpdate {
pub fn hash_str(&self) -> String {
let mut hasher = DefaultHasher::new();
self.hash(&mut hasher);
format!("{:x}", hasher.finish())
}
}
impl Hash for JobUpdate {
fn hash<H: Hasher>(&self, state: &mut H) {
self.running.hash(state);
self.completed.hash(state);
self.log_offset.hash(state);
self.mem_peak.hash(state);
self.progress.hash(state);
self.stream_offset.hash(state);
self.flow_stream_job_id.hash(state);
if !self.completed.unwrap_or(false) {
self.flow_status.as_ref().map(|x| x.get().hash(state));
self.workflow_as_code_status
.as_ref()
.map(|x| x.get().hash(state));
}
}
}
#[derive(serde::Deserialize)]
pub struct JobUpdateQuery {
pub running: Option<bool>,
pub log_offset: Option<i32>,
pub stream_offset: Option<i32>,
pub get_progress: Option<bool>,
pub no_logs: Option<bool>,
pub only_result: Option<bool>,
pub fast: Option<bool>,
pub is_flow: Option<bool>,
pub poll_delay_ms: Option<u64>,
}
#[derive(Serialize)]
#[serde(tag = "type", rename_all = "lowercase")]
pub enum JobUpdateSSEStream {
Update(JobUpdate),
Error { error: String },
NotFound,
Timeout,
Ping,
}
lazy_static::lazy_static! {
pub static ref TIMEOUT_SSE_STREAM: u64 =
std::env::var("TIMEOUT_SSE_STREAM").unwrap_or("60".to_string()).parse::<u64>().unwrap_or(60);
}
backend/windmill-api/Cargo.toml
+35 -48
View File
@@ -10,41 +10,46 @@ path = "src/lib.rs"
[features]
default = []
private = ["windmill-audit/private", "windmill-common/private"]
enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"]
private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private"]
enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise"]
stripe = []
agent_worker_server = []
enterprise_saml = ["dep:samael", "dep:libxml"]
benchmark = []
embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"]
embedding = ["windmill-api-embeddings/embedding"]
parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-worker/parquet"]
prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"]
openidconnect = ["dep:openidconnect", "windmill-common/openidconnect"]
openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"]
tantivy = ["dep:windmill-indexer"]
kafka = ["dep:rdkafka", "dep:rdkafka-sys"]
kafka-gssapi = ["kafka", "rdkafka/gssapi"]
nats = ["dep:async-nats", "dep:nkeys"]
websocket = ["dep:tokio-tungstenite"]
smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp"]
kafka = ["dep:windmill-trigger-kafka", "windmill-trigger-kafka/enterprise", "windmill-store/kafka"]
kafka-gssapi = ["kafka", "windmill-trigger-kafka/kafka-gssapi"]
nats = ["dep:windmill-trigger-nats", "windmill-trigger-nats/enterprise", "windmill-store/nats"]
websocket = ["dep:windmill-trigger-websocket", "windmill-trigger-websocket/enterprise"]
smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp", "dep:windmill-trigger-email", "windmill-trigger-email/enterprise"]
license = ["dep:rsa"]
zip = ["dep:async_zip"]
oauth2 = ["dep:windmill-oauth"]
http_trigger = ["dep:matchit", "dep:thiserror", "dep:sha1", "dep:constant_time_eq"]
oauth2 = ["dep:windmill-oauth", "windmill-store/oauth2"]
http_trigger = ["dep:matchit", "dep:windmill-trigger-http", "windmill-trigger-http/enterprise", "windmill-store/http_trigger"]
static_frontend = ["dep:rust-embed"]
postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:thiserror", "dep:rust_decimal", "dep:rust-postgres-native-tls"]
mqtt_trigger = ["dep:thiserror", "dep:rumqttc"]
native_trigger = ["dep:strum", "dep:backon", "oauth2"]
sqs_trigger = ["dep:aws-sdk-sqs", "dep:aws-sdk-sts", "dep:aws-sdk-sso", "dep:aws-sdk-ssooidc", "dep:thiserror", "dep:backon", "dep:aws-config"]
postgres_trigger = ["dep:windmill-trigger-postgres", "windmill-trigger-postgres/enterprise", "windmill-store/postgres_trigger"]
mqtt_trigger = ["dep:windmill-trigger-mqtt", "windmill-trigger-mqtt/enterprise", "windmill-store/mqtt_trigger"]
native_trigger = ["dep:windmill-native-triggers", "windmill-native-triggers/native_trigger", "windmill-native-triggers/enterprise", "dep:strum", "oauth2"]
sqs_trigger = ["dep:windmill-trigger-sqs", "windmill-trigger-sqs/enterprise", "windmill-store/sqs_trigger"]
deno_core = ["dep:deno_core", "dep:deno_error"]
gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"]
cloud = ["windmill-common/cloud"]
mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth"]
gcp_trigger = ["dep:windmill-trigger-gcp", "windmill-trigger-gcp/enterprise", "windmill-store/gcp_trigger"]
cloud = ["windmill-common/cloud", "windmill-api-auth/cloud", "windmill-store/cloud"]
mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth", "windmill-api-auth/mcp", "windmill-store/mcp"]
bedrock = ["dep:aws-sdk-bedrock", "dep:aws-sdk-bedrockruntime", "windmill-common/bedrock", "dep:aws-config"]
python = []
no_auth = []
no_auth = ["windmill-api-auth/no_auth", "windmill-store/no_auth"]
[dependencies]
windmill-mcp = { workspace = true, optional = true }
windmill-api-auth.workspace = true
windmill-api-sse.workspace = true
windmill-api-jobs.workspace = true
windmill-trigger.workspace = true
windmill-store.workspace = true
windmill-queue.workspace = true
windmill-common = { workspace = true, default-features = false }
windmill-audit.workspace = true
@@ -104,8 +109,6 @@ postgres-native-tls.workspace = true
hmac.workspace = true
cookie.workspace = true
sha2.workspace = true
sha1 = { workspace = true, optional = true }
constant_time_eq = { workspace = true, optional = true }
urlencoding.workspace = true
lazy_static.workspace = true
prometheus = { workspace = true, optional = true }
@@ -118,40 +121,28 @@ libxml = { workspace = true, optional = true }
async-recursion.workspace = true
rsa = { workspace = true, optional = true}
uuid.workspace = true
tinyvector = { workspace = true, optional = true}
hf-hub = { workspace = true, optional = true}
tokenizers = { workspace = true, optional = true}
candle-core = { workspace = true, optional = true}
candle-transformers = { workspace = true, optional = true}
candle-nn = { workspace = true, optional = true}
datafusion = { workspace = true, optional = true}
object_store = { workspace = true, optional = true}
openidconnect = { workspace = true, optional = true}
jsonwebtoken = { workspace = true }
matchit = { workspace = true, optional = true }
tokio-tungstenite = { workspace = true, optional = true}
rdkafka = { workspace = true, optional = true }
rdkafka-sys = { workspace = true, optional = true }
async-nats = { workspace = true, optional = true }
nkeys = { workspace = true, optional = true }
windmill-trigger-kafka = { workspace = true, optional = true }
windmill-trigger-postgres = { workspace = true, optional = true }
windmill-trigger-mqtt = { workspace = true, optional = true }
windmill-trigger-websocket = { workspace = true, optional = true }
windmill-trigger-email = { workspace = true, optional = true }
windmill-trigger-nats = { workspace = true, optional = true }
windmill-trigger-sqs = { workspace = true, optional = true }
windmill-trigger-gcp = { workspace = true, optional = true }
windmill-trigger-http = { workspace = true, optional = true }
windmill-native-triggers = { workspace = true, optional = true }
windmill-api-embeddings.workspace = true
const_format.workspace = true
pin-project.workspace = true
http.workspace = true
indexmap.workspace = true
async-stream.workspace = true
ulid.workspace = true
rust-postgres = { workspace = true, optional = true }
pg_escape = { workspace = true, optional = true }
byteorder = { workspace = true, optional = true }
thiserror = { workspace = true, optional = true }
rust_decimal = { workspace = true, optional = true }
rust-postgres-native-tls = { workspace = true, optional = true}
rumqttc = { workspace = true, optional = true }
aws-sdk-sqs = { workspace = true, optional = true }
aws-sdk-sso = { workspace = true, optional = true }
aws-sdk-ssooidc = { workspace = true, optional = true }
aws-sdk-sts = { workspace = true, optional = true }
rustls = { workspace = true }
aws-sigv4.workspace = true
@@ -162,14 +153,10 @@ aws-sdk-bedrock = { workspace = true, optional = true }
aws-sdk-bedrockruntime = { workspace = true, optional = true }
aws-smithy-types.workspace = true
async-trait.workspace = true
google-cloud-pubsub = { workspace = true, optional = true }
google-cloud-googleapis = { workspace = true , optional = true }
tonic = { workspace = true, optional = true }
deno_error = { workspace = true, optional = true }
deno_core = { workspace = true, optional = true }
tar.workspace = true
flate2.workspace = true
backon = {workspace = true, optional = true}
strum = { workspace = true, optional = true }
dashmap.workspace = true
backend/windmill-api/src/apps.rs
+1 -1
View File
@@ -1411,7 +1411,7 @@ async fn delete_app(
deployed_object,
Some(format!("App '{}' deleted", path)),
true,
None,
None,
)
.await?;
backend/windmill-api/src/auth.rs
+3 -765
View File
@@ -1,766 +1,4 @@
#[cfg(feature = "enterprise")]
use crate::ee_oss::ExternalJwks;
use axum::{
async_trait,
extract::{FromRequestParts, OriginalUri, Query},
Extension, Json,
pub use windmill_api_auth::auth::{
invalidate_token_from_cache, list_tokens_internal, transform_old_scope_to_new_scope, AuthCache,
ExpiringAuthCache, OptTokened, Tokened, TruncatedTokenWithEmail,
};
use chrono::TimeZone;
use http::{request::Parts, StatusCode};
use quick_cache::sync::Cache;
use serde::{Deserialize, Serialize};
use sqlx::FromRow;
use tower_cookies::Cookies;
use tracing::Span;
use crate::db::{ApiAuthed, OptJobAuthed, DB};
use std::{
str::FromStr,
sync::{
atomic::{AtomicI64, AtomicU64, Ordering},
Arc,
},
};
#[cfg(feature = "enterprise")]
use tokio::sync::RwLock;
use windmill_common::{
auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, TOKEN_PREFIX_LEN},
error::{Error, JsonResult},
jwt,
users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL},
};
lazy_static::lazy_static! {
// Global auth cache accessible from main.rs for direct invalidation
pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300);
}
// Global function to invalidate a specific token from cache
pub fn invalidate_token_from_cache(token: &str) {
// Remove all cache entries for this token (across all workspaces)
AUTH_CACHE.retain(|(_workspace_id, cached_token), _cached_value| cached_token != token);
tracing::info!(
"Invalidated token from auth cache: {}...",
&token[..token.len().min(8)]
);
}
#[derive(Clone)]
pub struct ExpiringAuthCache {
pub authed: ApiAuthed,
pub expiry: chrono::DateTime<chrono::Utc>,
pub job_id: Option<uuid::Uuid>,
}
pub struct AuthCache {
db: DB,
superadmin_secret: Option<String>,
#[cfg(feature = "enterprise")]
ext_jwks: Option<Arc<RwLock<ExternalJwks>>>,
}
impl AuthCache {
pub fn new(
db: DB,
superadmin_secret: Option<String>,
#[cfg(feature = "enterprise")] ext_jwks: Option<Arc<RwLock<ExternalJwks>>>,
) -> Self {
AuthCache {
db,
superadmin_secret,
#[cfg(feature = "enterprise")]
ext_jwks,
}
}
pub async fn invalidate(&self, w_id: &str, token: String) {
AUTH_CACHE.remove(&(w_id.to_string(), token));
}
pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<ApiAuthed> {
Some(self.get_opt_job_authed(w_id, token).await?.authed)
}
pub async fn get_opt_job_authed(
&self,
w_id: Option<String>,
token: &str,
) -> Option<OptJobAuthed> {
let key = (
w_id.as_ref().unwrap_or(&"".to_string()).to_string(),
token.to_string(),
);
let s = AUTH_CACHE.get(&key).map(|c| c.to_owned());
match s {
Some(ExpiringAuthCache { authed, expiry, job_id }) if expiry > chrono::Utc::now() => {
Some(OptJobAuthed { authed, job_id })
}
#[cfg(feature = "enterprise")]
_ if token.starts_with("jwt_ext_") => {
let authed_and_exp = match crate::ee_oss::jwt_ext_auth(
w_id.as_ref(),
token.trim_start_matches("jwt_ext_"),
self.ext_jwks.clone(),
&self.db,
)
.await
{
Ok(r) => Some(r),
Err(e) => {
tracing::error!("JWT_EXT auth error: {:?}", e);
None
}
};
if let Some((authed, exp, job_id)) = authed_and_exp.clone() {
AUTH_CACHE.insert(
key,
ExpiringAuthCache {
authed: authed.clone(),
expiry: chrono::Utc.timestamp_nanos(exp as i64 * 1_000_000_000),
job_id,
},
);
Some(OptJobAuthed { authed, job_id })
} else {
None
}
}
_ if token.starts_with("jwt_") => {
let jwt_token = token.trim_start_matches("jwt_");
let jwt_result = jwt::decode_with_internal_secret::<JWTAuthClaims>(jwt_token).await;
match jwt_result {
Ok(claims) => {
if w_id.is_some_and(|w_id| !claims.allowed_in_workspace(&w_id)) {
tracing::error!("JWT auth error: workspace_id mismatch");
return None;
}
let username_override = username_override_from_label(claims.label);
let authed = crate::db::ApiAuthed {
email: claims.email,
username: claims.username,
is_admin: claims.is_admin,
is_operator: claims.is_operator,
groups: claims.groups,
folders: claims.folders,
scopes: None,
username_override,
token_prefix: claims.audit_span,
};
let job_id = claims.job_id.and_then(|j| uuid::Uuid::from_str(&j).ok());
AUTH_CACHE.insert(
key,
ExpiringAuthCache {
authed: authed.clone(),
expiry: chrono::Utc
.timestamp_nanos(claims.exp as i64 * 1_000_000_000),
job_id,
},
);
Some(OptJobAuthed { authed, job_id })
}
Err(err) => {
tracing::error!("JWT auth error: {:?}", err);
None
}
}
}
_ => {
let user_o = sqlx::query!(
"UPDATE token SET last_used_at = now() WHERE
token = $1
AND (expiration > NOW() OR expiration IS NULL)
AND (workspace_id IS NULL OR workspace_id = $2)
RETURNING owner, email, super_admin, scopes, label",
token,
w_id.as_ref(),
)
.map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label))
.fetch_optional(&self.db)
.await
.ok()
.flatten();
if let Some(user) = user_o {
let authed_o = {
match user {
(Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => {
let username_override = username_override_from_label(label);
if let Some((prefix, name)) = owner.split_once('/') {
if prefix == "u" {
let (is_admin, is_operator) = if super_admin {
(true, false)
} else {
let r = sqlx::query!(
"SELECT is_admin, operator FROM usr where username = $1 AND \
workspace_id = $2 AND disabled = false",
name,
&w_id.as_ref().unwrap()
)
.fetch_one(&self.db)
.await
.ok();
if let Some(r) = r {
(r.is_admin, r.operator)
} else {
(false, true)
}
};
let w_id = &w_id.unwrap();
let groups =
get_groups_for_user(w_id, &name, &email, &self.db)
.await
.ok()
.unwrap_or_default();
let folders =
get_folders_for_user(w_id, &name, &groups, &self.db)
.await
.ok()
.unwrap_or_default();
Some(ApiAuthed {
email: email,
username: name.to_string(),
is_admin,
is_operator,
groups,
folders,
scopes: None,
username_override,
token_prefix: Some(
token[0..TOKEN_PREFIX_LEN].to_string(),
),
})
} else {
let groups = vec![name.to_string()];
let folders = get_folders_for_user(
&w_id.unwrap(),
"",
&groups,
&self.db,
)
.await
.ok()
.unwrap_or_default();
Some(ApiAuthed {
email: email,
username: format!("group-{name}"),
is_admin: false,
groups,
is_operator: false,
folders,
scopes: None,
username_override,
token_prefix: Some(
token[0..TOKEN_PREFIX_LEN].to_string(),
),
})
}
} else {
let groups = vec![];
let folders = vec![];
Some(ApiAuthed {
email: email,
username: owner,
is_admin: super_admin,
is_operator: true,
groups,
folders,
scopes: None,
username_override,
token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()),
})
}
}
(_, Some(email), super_admin, scopes, label) => {
let username_override = username_override_from_label(label);
if w_id.is_some() {
let row_o = sqlx::query!(
"SELECT username, is_admin, operator FROM usr WHERE
email = $1 AND workspace_id = $2 AND disabled = false",
&email,
w_id.as_ref().unwrap()
)
.map(|x| (x.username, x.is_admin, x.operator))
.fetch_optional(&self.db)
.await
.unwrap_or(Some(("error".to_string(), false, false)));
match row_o {
Some((username, is_admin, is_operator)) => {
let groups = get_groups_for_user(
&w_id.as_ref().unwrap(),
&username,
&email,
&self.db,
)
.await
.ok()
.unwrap_or_default();
let folders = get_folders_for_user(
&w_id.unwrap(),
&username,
&groups,
&self.db,
)
.await
.ok()
.unwrap_or_default();
Some(ApiAuthed {
email,
username,
is_admin: is_admin || super_admin,
is_operator,
groups,
folders,
scopes,
username_override,
token_prefix: Some(
token[0..TOKEN_PREFIX_LEN].to_string(),
),
})
}
None if super_admin => Some(ApiAuthed {
email: email.clone(),
username: email,
is_admin: super_admin,
is_operator: false,
groups: vec![],
folders: vec![],
scopes,
username_override,
token_prefix: Some(
token[0..TOKEN_PREFIX_LEN].to_string(),
),
}),
None => None,
}
} else {
Some(ApiAuthed {
email: email.to_string(),
username: email,
is_admin: super_admin,
is_operator: true,
groups: Vec::new(),
folders: Vec::new(),
scopes,
username_override,
token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()),
})
}
}
_ => None,
}
};
if let Some(authed) = authed_o.as_ref() {
AUTH_CACHE.insert(
key,
ExpiringAuthCache {
authed: authed.clone(),
expiry: chrono::Utc::now()
+ chrono::Duration::try_seconds(120).unwrap(),
job_id: None,
},
);
}
authed_o.map(|authed| OptJobAuthed { authed, job_id: None })
} else if self
.superadmin_secret
.as_ref()
.map(|x| x == token)
.unwrap_or(false)
{
let authed = ApiAuthed {
email: SUPERADMIN_SECRET_EMAIL.to_string(),
username: "superadmin_secret".to_string(),
is_admin: true,
is_operator: false,
groups: Vec::new(),
folders: Vec::new(),
scopes: None,
username_override: None,
token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()),
};
Some(OptJobAuthed { authed, job_id: None })
} else {
None
}
}
}
}
}
async fn extract_token<S: Send + Sync>(parts: &mut Parts, state: &S) -> Option<String> {
let auth_header = parts
.headers
.get(http::header::AUTHORIZATION)
.and_then(|value| value.to_str().ok())
.and_then(|s| s.strip_prefix("Bearer "));
let from_cookie = match auth_header {
Some(x) => Some(x.to_owned()),
None => Extension::<Cookies>::from_request_parts(parts, state)
.await
.ok()
.and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())),
};
#[derive(Deserialize)]
struct Token {
token: Option<String>,
}
match from_cookie {
Some(token) => Some(token),
None => Query::<Token>::from_request_parts(parts, state)
.await
.ok()
.and_then(|token| token.token.clone()),
}
}
#[derive(Clone, Debug)]
pub struct Tokened {
pub token: String,
}
#[derive(Clone, Debug)]
pub struct OptTokened {
#[allow(dead_code)]
pub token: Option<String>,
}
struct BruteForceCounter {
counter: AtomicU64,
last_reset: AtomicI64,
}
lazy_static::lazy_static! {
static ref BRUTE_FORCE_COUNTER: BruteForceCounter =
BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) };
}
impl BruteForceCounter {
async fn increment(&self) {
let now = time::OffsetDateTime::now_utc().unix_timestamp();
if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 {
tracing::error!(
"Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds"
);
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
if now - self.last_reset.load(Ordering::Relaxed) > 60 {
self.counter.store(0, Ordering::Relaxed);
self.last_reset.store(now, Ordering::Relaxed);
}
}
}
#[async_trait]
impl<S> FromRequestParts<S> for Tokened
where
S: Send + Sync,
{
type Rejection = (StatusCode, String);
async fn from_request_parts(
parts: &mut Parts,
state: &S,
) -> std::result::Result<Self, Self::Rejection> {
if parts.method == http::Method::OPTIONS {
return Ok(Tokened { token: "".to_string() });
};
let already_tokened = parts.extensions.get::<Tokened>();
if let Some(tokened) = already_tokened {
Ok(tokened.clone())
} else {
let token_o = extract_token(parts, state).await;
if let Some(token) = token_o {
let tokened = Self { token };
parts.extensions.insert(tokened.clone());
Ok(tokened)
} else {
BRUTE_FORCE_COUNTER.increment().await;
Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
}
}
}
}
#[async_trait]
impl<S> FromRequestParts<S> for OptTokened
where
S: Send + Sync,
{
type Rejection = (StatusCode, String);
async fn from_request_parts(
parts: &mut Parts,
state: &S,
) -> std::result::Result<Self, Self::Rejection> {
if parts.method == http::Method::OPTIONS {
return Ok(OptTokened { token: None });
};
let already_tokened = parts.extensions.get::<Tokened>();
if let Some(tokened) = already_tokened {
Ok(OptTokened { token: Some(tokened.token.clone()) })
} else {
let token_o = extract_token(parts, state).await;
Ok(OptTokened { token: token_o })
}
}
}
pub fn transform_old_scope_to_new_scope(scopes: Option<&mut Vec<String>>) {
if let Some(scopes) = scopes {
for scope in scopes.iter_mut() {
if scope.starts_with("run:") {
let (_, part_scope) = scope.split_once(":").unwrap();
if let Some((kind, path)) = part_scope.split_once("/") {
//appending a 's' as runnable kind is singular while new scope format expect it to be plural
*scope = format!("jobs:run:{}s:{}", kind, path);
}
} else if scope.starts_with("jobs:") {
// Map old jobs scopes to new format
let new_scope = match scope.as_str() {
"jobs:listjobs" => "jobs:read",
"jobs:runscript" => "jobs:run:scripts",
"jobs:runflow" => "jobs:run:flows",
"jobs:resumeflow" => "jobs:run:flows",
"jobs:deletejob" => "jobs:write",
_ => continue,
};
*scope = new_scope.to_string();
}
}
}
}
fn maybe_get_workspace_id_from_path(path_vec: &[&str]) -> Option<String> {
let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" {
Some(path_vec[3].to_owned())
} else if path_vec.len() >= 5
&& path_vec[0] == ""
&& path_vec[1] == "api"
&& path_vec[2] == "mcp"
&& path_vec[3] == "w"
{
Some(path_vec[4].to_owned())
} else {
if path_vec.len() >= 5 && path_vec[0] == "" && path_vec[2] == "srch" && path_vec[3] == "w" {
Some(path_vec[4].to_owned())
} else {
None
}
};
workspace_id
}
#[async_trait]
impl<S> FromRequestParts<S> for ApiAuthed
where
S: Send + Sync,
{
type Rejection = Error;
async fn from_request_parts(
parts: &mut Parts,
state: &S,
) -> std::result::Result<Self, Self::Rejection> {
let opt_job_authed = OptJobAuthed::from_request_parts(parts, state).await?;
Ok(opt_job_authed.authed)
}
}
#[async_trait]
impl<S> FromRequestParts<S> for OptJobAuthed
where
S: Send + Sync,
{
type Rejection = Error;
async fn from_request_parts(
parts: &mut Parts,
state: &S,
) -> std::result::Result<Self, Self::Rejection> {
if parts.method == http::Method::OPTIONS {
return Ok(OptJobAuthed::default());
};
#[cfg(feature = "no_auth")]
{
let authed = ApiAuthed {
email: "admin@windmill.dev".to_string(),
username: "admin".to_string(),
is_admin: true,
is_operator: false,
groups: Vec::new(),
folders: Vec::new(),
scopes: None,
username_override: None,
token_prefix: None,
};
return Ok(OptJobAuthed { authed, job_id: None });
}
let already_authed = parts.extensions.get::<OptJobAuthed>();
if let Some(authed) = already_authed {
return Ok(authed.clone());
}
let already_tokened = parts.extensions.get::<Tokened>();
let token_o = if let Some(token) = already_tokened {
Some(token.token.clone())
} else {
extract_token(parts, state).await
};
if let Some(token) = token_o {
if let Ok(Extension(cache)) =
Extension::<Arc<AuthCache>>::from_request_parts(parts, state).await
{
let original_uri = OriginalUri::from_request_parts(parts, state)
.await
.ok()
.map(|x| x.0)
.unwrap_or_default();
let path_vec: Vec<&str> = original_uri.path().split("/").collect();
let workspace_id = maybe_get_workspace_id_from_path(&path_vec);
if let Some(mut opt_job_authed) =
cache.get_opt_job_authed(workspace_id.clone(), &token).await
{
let authed = &mut opt_job_authed.authed;
if authed.scopes.is_some() {
transform_old_scope_to_new_scope(authed.scopes.as_mut());
let path = original_uri.path();
let method = parts.method.as_str();
if let Err(err) = crate::scopes::check_scopes_for_route(
authed.scopes.as_deref(),
path,
method,
) {
BRUTE_FORCE_COUNTER.increment().await;
return Err(err);
}
}
parts.extensions.insert(authed.clone());
Span::current().record("username", &authed.username.as_str());
Span::current().record("email", &authed.email);
if let Some(workspace_id) = workspace_id {
Span::current().record("workspace_id", &workspace_id);
}
return Ok(opt_job_authed);
}
}
}
BRUTE_FORCE_COUNTER.increment().await;
Err(Error::NotAuthorized("Unauthorized".to_string()))
}
}
fn username_override_from_label(label: Option<String>) -> Option<String> {
match label {
Some(label)
if label.starts_with("webhook-")
|| label.starts_with("http-")
|| label.starts_with("email-")
|| label.starts_with("ws-") =>
{
Some(label)
}
Some(label) if label.starts_with("ephemeral-script-end-user-") => Some(
label
.trim_start_matches("ephemeral-script-end-user-")
.to_string(),
),
Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()),
Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => {
Some(format!("label-{label}"))
}
_ => None,
}
}
#[derive(FromRow, Serialize)]
pub struct TruncatedTokenWithEmail {
pub label: Option<String>,
pub token_prefix: Option<String>,
pub expiration: Option<chrono::DateTime<chrono::Utc>>,
pub created_at: chrono::DateTime<chrono::Utc>,
pub last_used_at: chrono::DateTime<chrono::Utc>,
pub scopes: Option<Vec<String>>,
pub email: Option<String>,
}
pub async fn list_tokens_internal(
db: &DB,
w_id: &str,
path: &str,
is_flow: bool,
) -> JsonResult<Vec<TruncatedTokenWithEmail>> {
let tokens = if is_flow {
sqlx::query_as!(
TruncatedTokenWithEmail,
r#"
SELECT label,
concat(substring(token for 10)) AS token_prefix,
expiration,
created_at,
last_used_at,
scopes,
email
FROM token
WHERE workspace_id = $1
AND (
scopes @> ARRAY['jobs:run:flows:' || $2]::text[]
OR scopes @> ARRAY['run:flow/' || $2]::text[]
)
"#,
w_id,
path
)
.fetch_all(db)
.await?
} else {
sqlx::query_as!(
TruncatedTokenWithEmail,
r#"
SELECT label,
concat(substring(token for 10)) AS token_prefix,
expiration,
created_at,
last_used_at,
scopes,
email
FROM token
WHERE workspace_id = $1
AND (
scopes @> ARRAY['jobs:run:scripts:' || $2]::text[]
OR scopes @> ARRAY['run:script/' || $2]::text[]
)
"#,
w_id,
path
)
.fetch_all(db)
.await?
};
Ok(Json(tokens))
}
backend/windmill-api/src/bedrock.rs
+9 -18
View File
@@ -18,14 +18,16 @@
use axum::body::Bytes;
use serde::Deserialize;
use windmill_common::ai_bedrock::build_tool_config;
use windmill_common::ai_bedrock::{
bedrock_stream_event_is_block_stop, bedrock_stream_event_to_text,
bedrock_stream_event_to_tool_delta, bedrock_stream_event_to_tool_start, format_bedrock_error,
BedrockClient
BedrockClient,
};
use windmill_common::ai_types::{
OpenAIFunction, OpenAIMessage, OpenAIToolCall, ToolDef, ToolDefFunction,
};
use windmill_common::error::{Error, Result};
use windmill_common::ai_types::{OpenAIFunction, OpenAIMessage, OpenAIToolCall, ToolDef, ToolDefFunction};
use windmill_common::ai_bedrock::build_tool_config;
// ============================================================================
// Shared Request Types for SDK-Based Handlers
@@ -69,10 +71,7 @@ struct OpenAIToolFunction {
/// Authentication configuration for Bedrock clients
enum BedrockAuthConfig {
BearerToken(String),
IamCredentials {
access_key_id: String,
secret_access_key: String,
},
IamCredentials { access_key_id: String, secret_access_key: String },
Environment,
}
@@ -105,13 +104,8 @@ async fn create_bedrock_client(
region: &str,
) -> Result<BedrockClient> {
match determine_auth_config(api_key, aws_access_key_id, aws_secret_access_key) {
BedrockAuthConfig::BearerToken(key) => {
BedrockClient::from_bearer_token(key, region).await
}
BedrockAuthConfig::IamCredentials {
access_key_id,
secret_access_key,
} => {
BedrockAuthConfig::BearerToken(key) => BedrockClient::from_bearer_token(key, region).await,
BedrockAuthConfig::IamCredentials { access_key_id, secret_access_key } => {
BedrockClient::from_credentials(access_key_id, secret_access_key, None, region).await
}
BedrockAuthConfig::Environment => BedrockClient::from_env(region).await,
@@ -185,10 +179,7 @@ async fn create_bedrock_control_client(
.build();
Ok(aws_sdk_bedrock::Client::from_conf(config))
}
BedrockAuthConfig::IamCredentials {
access_key_id,
secret_access_key,
} => {
BedrockAuthConfig::IamCredentials { access_key_id, secret_access_key } => {
let credentials = aws_credential_types::Credentials::new(
access_key_id,
secret_access_key,
backend/windmill-api/src/concurrency_groups.rs
+1 -364
View File
@@ -1,364 +1 @@
use crate::{
db::{ApiAuthed, DB},
jobs::{
filter_list_completed_query, filter_list_queue_query, Job, ListCompletedQuery,
ListQueueQuery, UnifiedJob,
},
utils::check_scopes,
};
use axum::extract::Path;
use axum::routing::{delete, get};
use axum::{extract::Query, Extension, Json};
use serde::Deserialize;
use axum::Router;
use serde::Serialize;
use sql_builder::bind::Bind;
use sql_builder::SqlBuilder;
use uuid::Uuid;
use windmill_common::db::UserDB;
use windmill_common::error::Error::PermissionDenied;
use windmill_common::error::{self, JsonResult};
use windmill_common::utils::require_admin;
pub fn global_service() -> Router {
Router::new()
.route("/list", get(list_concurrency_groups))
.route("/prune/*concurrency_key", delete(prune_concurrency_group))
.route("/:job_id/key", get(get_concurrency_key))
}
pub fn workspaced_service() -> Router {
Router::new().route("/list_jobs", get(get_concurrent_intervals))
}
#[derive(Serialize)]
pub struct ConcurrencyGroups {
concurrency_key: String,
total_running: i64,
}
async fn list_concurrency_groups(
authed: ApiAuthed,
Extension(db): Extension<DB>,
) -> JsonResult<Vec<ConcurrencyGroups>> {
require_admin(authed.is_admin, &authed.username)?;
let concurrency_counts = sqlx::query_as::<_, (String, i64)>(
"SELECT concurrency_id, (select COUNT(*) from jsonb_object_keys(job_uuids)) as n_job_uuids FROM concurrency_counter",
).fetch_all(&db)
.await?;
let mut concurrency_groups: Vec<ConcurrencyGroups> = vec![];
for (concurrency_key, count) in concurrency_counts {
concurrency_groups.push(ConcurrencyGroups {
concurrency_key: concurrency_key.clone(),
total_running: count,
})
}
return Ok(Json(concurrency_groups));
}
async fn prune_concurrency_group(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(concurrency_key): Path<String>,
) -> JsonResult<()> {
if !authed.is_admin {
return Err(PermissionDenied(
"Only administrators can delete concurrency groups".to_string(),
));
}
let mut tx = db.begin().await?;
let concurrency_group = sqlx::query_as::<_, (String, i64)>(
"SELECT concurrency_id, (select COUNT(*) from jsonb_object_keys(job_uuids)) as n_job_uuids FROM concurrency_counter WHERE concurrency_id = $1 FOR UPDATE",
)
.bind(concurrency_key.clone())
.fetch_optional(&mut *tx)
.await?;
let n_job_uuids = concurrency_group.map(|cg| cg.1).unwrap_or_default();
if n_job_uuids > 0 {
tx.commit().await?;
return Err(error::Error::internal_err(
"Concurrency group is currently in use, unable to remove it. Retry later.".to_string(),
));
}
sqlx::query!(
"DELETE FROM concurrency_counter WHERE concurrency_id = $1",
concurrency_key.clone(),
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"DELETE FROM concurrency_key WHERE key = $1",
concurrency_key.clone(),
)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(Json(()))
}
#[derive(Serialize)]
struct ExtendedJobs {
jobs: Vec<Job>,
obscured_jobs: Vec<ObscuredJob>,
omitted_obscured_jobs: bool,
}
#[derive(Serialize)]
struct ObscuredJob {
typ: String,
started_at: Option<chrono::DateTime<chrono::Utc>>,
duration_ms: Option<i64>,
}
#[derive(Deserialize)]
struct ExtendedJobsParams {
row_limit: Option<i64>,
}
pub fn join_concurrency_key<'c>(
concurrency_key: Option<&String>,
mut sqlb: SqlBuilder,
) -> SqlBuilder {
if let Some(key) = concurrency_key {
sqlb.join("concurrency_key")
.on_eq("id", "concurrency_key.job_id")
.and_where_eq("key", "?".bind(key));
}
sqlb
}
async fn get_concurrent_intervals(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(iq): Query<ExtendedJobsParams>,
Query(lq): Query<ListCompletedQuery>,
) -> JsonResult<ExtendedJobs> {
check_scopes(&authed, || format!("jobs:read"))?;
if lq.success.is_some() && lq.running.is_some_and(|x| x) {
return Err(error::Error::BadRequest(
"cannot specify both success and running".to_string(),
));
}
let row_limit = iq.row_limit.unwrap_or(1000);
let lq = ListCompletedQuery { order_desc: Some(true), ..lq };
let lqc = lq.clone();
let lqq: ListQueueQuery = lqc.into();
let mut sqlb_q = SqlBuilder::select_from("v2_job_queue")
.fields(UnifiedJob::queued_job_fields())
.order_by("created_at", lq.order_desc.unwrap_or(true))
.limit(row_limit)
.clone();
let mut sqlb_c = SqlBuilder::select_from("v2_job_completed")
.fields(UnifiedJob::completed_job_fields())
.order_by("completed_at", lq.order_desc.unwrap_or(true))
.limit(row_limit)
.clone();
let mut sqlb_q_user = SqlBuilder::select_from("v2_job_queue")
.fields(&["id"])
.order_by("created_at", lq.order_desc.unwrap_or(true))
.limit(row_limit)
.clone();
let mut sqlb_c_user = SqlBuilder::select_from("v2_job_completed")
.fields(&["id"])
.order_by("completed_at", lq.order_desc.unwrap_or(true))
.limit(row_limit)
.clone();
sqlb_q = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_q);
sqlb_c = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_c);
sqlb_q_user = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_q_user);
sqlb_c_user = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_c_user);
let should_fetch_obscured_jobs = match lq {
ListCompletedQuery {
script_path_start: None,
script_path_exact: None,
script_hash: None,
created_by: None,
success: None,
running: None,
parent_job: None,
is_skipped: None | Some(false),
suspended: None,
schedule_path: None,
args: None,
result: None,
tag: None,
has_null_parent: None,
worker: None,
label: None,
trigger_path: None,
scheduled_for_before_now: _,
is_not_schedule: _,
started_before: _,
started_after: _,
created_before: _,
created_after: _,
created_before_queue: _,
created_after_queue: _,
completed_after: _,
completed_before: _,
created_or_started_before: _,
created_or_started_after: _,
created_or_started_after_completed_jobs: _,
order_desc: _,
job_kinds: _,
is_flow_step: _,
all_workspaces: _,
concurrency_key: Some(_),
allow_wildcards: None,
trigger_kind: _,
include_args: _,
} => true,
_ => false,
};
// When we have a concurrency key defined, fetch jobs from other workspaces
// as obscured unless we're in the admins workspace. This is to show the
// potential concurrency races without showing jobs that don't belong to
// the workspace.
// To avoid infering information through filtering, don't return obscured
// jobs if the filters are too specific
if should_fetch_obscured_jobs && w_id != "admins" {
// Get the obscured jobs from all workspaces (concurrency key could be global)
let (sqlb_q, sqlb_c) = (
filter_list_queue_query(
sqlb_q,
&ListQueueQuery { all_workspaces: Some(true), ..lqq.clone() },
"admins",
true,
),
filter_list_completed_query(
sqlb_c,
&ListCompletedQuery { all_workspaces: Some(true), ..lq.clone() },
"admins",
true,
),
);
sqlb_q_user = filter_list_queue_query(sqlb_q_user, &lqq, w_id.as_str(), true);
sqlb_c_user = filter_list_completed_query(sqlb_c_user, &lq, w_id.as_str(), true);
let sql_q_user = sqlb_q_user.query()?;
let sql_c_user = sqlb_c_user.query()?;
let sql_q = sqlb_q.query()?;
let sql_c = sqlb_c.query()?;
// This first transaction uses the user_db to know which uuids are
// accessible to the user.
let mut tx = user_db.begin(&authed).await?;
let running_jobs_user: Vec<Uuid> = if lq.success.is_none() {
sqlx::query_scalar(&sql_q_user).fetch_all(&mut *tx).await?
} else {
vec![]
};
let completed_jobs_user: Vec<Uuid> = if lq.running.is_none() {
sqlx::query_scalar(&sql_c_user).fetch_all(&mut *tx).await?
} else {
vec![]
};
tx.commit().await?;
// This second transaction uses the db, so it will fetch information
// potentially forbidden to the user. It must be obscured before
// returning it
let running_jobs_db: Vec<UnifiedJob> = if lq.success.is_none() {
sqlx::query_as(&sql_q).fetch_all(&db).await?
} else {
vec![]
};
let completed_jobs_db: Vec<UnifiedJob> = if lq.running.is_none() {
sqlx::query_as(&sql_c).fetch_all(&db).await?
} else {
vec![]
};
let obscured_jobs = running_jobs_db
.iter()
.filter(|j| !running_jobs_user.iter().any(|id| j.id == *id))
.chain(
completed_jobs_db
.iter()
.filter(|j| !completed_jobs_user.iter().any(|id| j.id == *id)),
)
.map(|j| ObscuredJob {
typ: j.typ.clone(),
started_at: j.started_at,
duration_ms: j.duration_ms,
})
.collect();
let jobs = running_jobs_db
.into_iter()
.filter(|j| running_jobs_user.iter().any(|id| j.id == *id))
.chain(
completed_jobs_db
.into_iter()
.filter(|j| completed_jobs_user.iter().any(|id| j.id == *id)),
)
.map(From::from)
.collect();
Ok(Json(ExtendedJobs {
jobs,
obscured_jobs,
omitted_obscured_jobs: !should_fetch_obscured_jobs,
}))
} else {
sqlb_q = filter_list_queue_query(sqlb_q, &lqq, w_id.as_str(), true);
sqlb_c = filter_list_completed_query(sqlb_c, &lq, w_id.as_str(), true);
let sql_q = sqlb_q.query()?;
let sql_c = sqlb_c.query()?;
let mut tx = user_db.begin(&authed).await?;
let running_jobs: Vec<UnifiedJob> = if lq.success.is_none() {
sqlx::query_as(&sql_q).fetch_all(&mut *tx).await?
} else {
vec![]
};
let completed_jobs: Vec<UnifiedJob> = if lq.running.is_none() {
sqlx::query_as(&sql_c).fetch_all(&mut *tx).await?
} else {
vec![]
};
tx.commit().await?;
let jobs = running_jobs
.into_iter()
.chain(completed_jobs.into_iter())
.map(From::from)
.collect();
Ok(Json(ExtendedJobs {
jobs,
obscured_jobs: vec![],
omitted_obscured_jobs: !should_fetch_obscured_jobs,
}))
}
}
async fn get_concurrency_key(
Extension(db): Extension<DB>,
Path(job_id): Path<Uuid>,
) -> JsonResult<Option<String>> {
let key = sqlx::query_scalar!("SELECT key FROM concurrency_key WHERE job_id = $1", job_id)
.fetch_optional(&db)
.await?;
Ok(Json(key))
}
pub use windmill_api_jobs::concurrency_groups::*;
backend/windmill-api/src/db.rs
+3 -123
View File
@@ -14,13 +14,10 @@ use sqlx::{
};
use tokio::task::JoinHandle;
use windmill_audit::audit_oss::AuditAuthorable;
pub use windmill_common::db::DB;
use windmill_common::{
db::{Authable, Authed, AuthedRef},
error::Error,
utils::generate_lock_id,
};
use windmill_common::{error::Error, utils::generate_lock_id};
pub use windmill_api_auth::{ApiAuthed, OptJobAuthed};
async fn current_database(conn: &mut PgConnection) -> Result<String, MigrateError> {
// language=SQL
@@ -291,120 +288,3 @@ pub async fn migrate(
crate::live_migrations::custom_migrations(&mut custom_migrator, db).await?;
Ok(None)
}
#[derive(Default, Clone, Debug)]
pub struct OptJobAuthed {
pub job_id: Option<uuid::Uuid>,
pub authed: ApiAuthed,
}
#[derive(Clone, Debug, Default, Hash, Eq, PartialEq)]
pub struct ApiAuthed {
pub email: String,
pub username: String,
pub is_admin: bool,
pub is_operator: bool,
pub groups: Vec<String>,
// (folder name, can write, is owner)
pub folders: Vec<(String, bool, bool)>,
pub scopes: Option<Vec<String>>,
pub username_override: Option<String>,
pub token_prefix: Option<String>,
}
impl ApiAuthed {
pub fn to_authed_ref<'e>(&'e self) -> AuthedRef<'e> {
AuthedRef {
email: &self.email,
username: &self.username,
is_admin: &self.is_admin,
is_operator: &self.is_operator,
groups: &self.groups,
folders: &self.folders,
scopes: &self.scopes,
token_prefix: &self.token_prefix,
}
}
}
impl From<ApiAuthed> for Authed {
fn from(value: ApiAuthed) -> Self {
Self {
email: value.email,
username: value.username,
is_admin: value.is_admin,
is_operator: value.is_operator,
groups: value.groups,
folders: value.folders,
scopes: value.scopes,
token_prefix: value.token_prefix,
}
}
}
impl From<Authed> for ApiAuthed {
fn from(value: Authed) -> Self {
Self {
email: value.email,
username: value.username,
is_admin: value.is_admin,
is_operator: value.is_operator,
groups: value.groups,
folders: value.folders,
scopes: value.scopes,
username_override: None, // Authed doesn't have this field, so default to None
token_prefix: value.token_prefix,
}
}
}
impl ApiAuthed {
pub fn display_username(&self) -> &str {
self.username_override.as_ref().unwrap_or(&self.username)
}
}
impl AuditAuthorable for ApiAuthed {
fn username(&self) -> &str {
self.username.as_str()
}
fn email(&self) -> &str {
self.email.as_str()
}
fn username_override(&self) -> Option<&str> {
self.username_override.as_deref()
}
fn token_prefix(&self) -> Option<&str> {
self.token_prefix.as_deref()
}
}
impl Authable for ApiAuthed {
fn is_admin(&self) -> bool {
self.is_admin
}
fn is_operator(&self) -> bool {
self.is_operator
}
fn groups(&self) -> &[String] {
&self.groups
}
fn folders(&self) -> &[(String, bool, bool)] {
&self.folders
}
fn scopes(&self) -> Option<&[std::string::String]> {
self.scopes.as_ref().map(|x| x.as_slice())
}
fn email(&self) -> &str {
&self.email
}
fn username(&self) -> &str {
&self.username
}
}
backend/windmill-api/src/debug.rs
+17 -12
View File
@@ -21,22 +21,23 @@
//! - A completed job entry in v2_job_completed
//! - An audit log entry identical to script preview runs
use axum::{extract::Path, routing::{get, post}, Extension, Json, Router};
use axum::{
extract::Path,
routing::{get, post},
Extension, Json, Router,
};
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
use chrono::Utc;
use ed25519_dalek::{SigningKey, Signer};
use ed25519_dalek::{Signer, SigningKey};
use serde::{Deserialize, Serialize};
use sha2::{Sha256, Digest};
use sha2::{Digest, Sha256};
use sqlx::types::Json as SqlxJson;
use std::sync::Arc;
use tokio::sync::RwLock;
use uuid::Uuid;
use windmill_audit::{audit_oss::audit_log, ActionKind};
use windmill_common::{
db::UserDB,
error::JsonResult,
jobs::JobKind,
scripts::ScriptLang,
db::UserDB, error::JsonResult, jobs::JobKind, scripts::ScriptLang,
users::username_to_permissioned_as,
};
@@ -279,7 +280,8 @@ async fn sign_debug_request(
job_id,
w_id,
now,
SqlxJson(serde_json::json!({"debug_session": true, "language": request.language})) as SqlxJson<serde_json::Value>,
SqlxJson(serde_json::json!({"debug_session": true, "language": request.language}))
as SqlxJson<serde_json::Value>,
)
.execute(&mut *tx)
.await?;
@@ -400,10 +402,13 @@ async fn sign_expression(
ActionKind::Execute,
&w_id,
Some(&resource),
Some([
("job_id", request.job_id.as_str()),
("expression", request.expression.as_str()),
].into()),
Some(
[
("job_id", request.job_id.as_str()),
("expression", request.expression.as_str()),
]
.into(),
),
)
.await?;
backend/windmill-api/src/ee_oss.rs
+5 -44
View File
@@ -1,17 +1,11 @@
#[cfg(feature = "private")]
#[allow(unused)]
pub use crate::ee::*;
#[cfg(all(
feature = "enterprise",
any(feature = "nats", feature = "kafka", feature = "sqs_trigger"),
not(feature = "private")
))]
use {crate::db::ApiAuthed, windmill_common::DB};
#[cfg(all(feature = "enterprise", not(feature = "private")))]
pub use windmill_api_auth::ee_oss::ExternalJwks;
#[cfg(not(feature = "private"))]
use anyhow::anyhow;
#[cfg(all(feature = "enterprise", not(feature = "private")))]
use {std::sync::Arc, tokio::sync::RwLock};
#[cfg(not(feature = "private"))]
pub async fn validate_license_key(
_license_key: String,
@@ -21,42 +15,9 @@ pub async fn validate_license_key(
Err(anyhow!("License can't be validated in Windmill CE"))
}
#[cfg(all(feature = "enterprise", not(feature = "private")))]
pub async fn jwt_ext_auth(
_w_id: Option<&String>,
_token: &str,
_external_jwks: Option<Arc<RwLock<ExternalJwks>>>,
_db: &crate::db::DB,
) -> anyhow::Result<(crate::db::ApiAuthed, usize, Option<uuid::Uuid>)> {
// Implementation is not open source
Err(anyhow!("External JWT auth is not open source"))
}
#[cfg(all(feature = "enterprise", not(feature = "private")))]
pub struct ExternalJwks;
#[cfg(all(feature = "enterprise", not(feature = "private")))]
impl ExternalJwks {
pub async fn load() -> Option<Arc<RwLock<Self>>> {
// Implementation is not open source
None
}
}
// interpolate moved to windmill-store/src/resources.rs
#[cfg(all(
feature = "enterprise",
any(feature = "nats", feature = "kafka", feature = "sqs_trigger"),
not(feature = "private")
any(feature = "nats", feature = "kafka", feature = "sqs_trigger")
))]
pub async fn interpolate(
_authed: &ApiAuthed,
_db: &DB,
_w_id: &str,
_s: String,
) -> Result<String, anyhow::Error> {
// Implementation is not open source
Err(anyhow!(
"Interpolation is not available in open source version"
))
}
pub use windmill_store::resources::interpolate;
backend/windmill-api/src/embeddings.rs
+1 -630
View File
@@ -1,630 +1 @@
#[cfg(feature = "embedding")]
use anyhow::{anyhow, Error, Result};
#[cfg(feature = "embedding")]
use std::{collections::HashMap, path::PathBuf, sync::Arc};
#[cfg(feature = "embedding")]
use windmill_common::DEFAULT_HUB_BASE_URL;
#[cfg(feature = "embedding")]
use windmill_common::HUB_BASE_URL;
use axum::Router;
#[cfg(feature = "embedding")]
use axum::{
extract::{Path, Query},
Json,
};
#[cfg(feature = "embedding")]
use axum::routing::get;
#[cfg(feature = "embedding")]
use candle_core::{Device, Tensor};
#[cfg(feature = "embedding")]
use candle_nn::VarBuilder;
#[cfg(feature = "embedding")]
use candle_transformers::models::bert::{BertModel, Config, DTYPE};
#[cfg(feature = "embedding")]
use hf_hub::api::tokio::Api;
#[cfg(feature = "embedding")]
use serde::Deserialize;
#[cfg(feature = "embedding")]
use serde::Serialize;
#[cfg(feature = "embedding")]
use sqlx::{Pool, Postgres};
#[cfg(feature = "embedding")]
use tinyvector::{
db::{Db, Embedding},
similarity::Distance,
};
#[cfg(feature = "embedding")]
use tokenizers::Tokenizer;
#[cfg(feature = "embedding")]
use tokio::sync::RwLock;
#[cfg(feature = "embedding")]
use windmill_common::utils::http_get_from_hub;
#[cfg(feature = "embedding")]
use windmill_common::error::JsonResult;
#[cfg(feature = "embedding")]
use crate::{resources::ResourceType, HTTP_CLIENT};
#[cfg(feature = "embedding")]
lazy_static::lazy_static! {
pub static ref EMBEDDINGS_DB: Arc<RwLock<Option<EmbeddingsDb>>> = Arc::new(RwLock::new(None));
pub static ref MODEL_INSTANCE: Arc<RwLock<Option<Arc<ModelInstance>>>> = Arc::new(RwLock::new(None));
pub static ref HUB_EMBEDDINGS_PULLING_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_PULLING_INTERVAL_SECS").ok().map(|x| x.parse::<u64>().ok()).flatten().unwrap_or(3600 * 24);
}
#[cfg(feature = "embedding")]
#[derive(Deserialize)]
struct HubScriptsQuery {
text: String,
limit: Option<i64>,
kind: Option<String>,
app: Option<String>,
}
#[cfg(feature = "embedding")]
#[derive(Serialize)]
pub struct HubScriptResult {
ask_id: i64,
id: i64,
version_id: i64,
summary: String,
app: String,
kind: String,
score: f32,
}
#[cfg(feature = "embedding")]
async fn query_hub_scripts(
Query(query): Query<HubScriptsQuery>,
) -> JsonResult<Vec<HubScriptResult>> {
let embeddings_db = EMBEDDINGS_DB.read().await;
if let Some(embeddings_db) = embeddings_db.as_ref() {
let results = embeddings_db
.query_hub_scripts(&query.text, query.limit, query.kind, query.app)
.await?;
Ok(Json(results))
} else {
Err(windmill_common::error::Error::internal_err(
"Embeddings db not initialized".to_string(),
))
}
}
#[cfg(feature = "embedding")]
#[derive(Deserialize)]
struct ResourceTypesQuery {
text: String,
limit: Option<i64>,
}
#[cfg(feature = "embedding")]
#[derive(Serialize)]
pub struct ResourceTypeResult {
name: String,
score: f32,
schema: Option<serde_json::Value>,
}
#[cfg(feature = "embedding")]
async fn query_resource_types(
Query(query): Query<ResourceTypesQuery>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<ResourceTypeResult>> {
let embeddings_db = EMBEDDINGS_DB.read().await;
if let Some(embeddings_db) = embeddings_db.as_ref() {
let results = embeddings_db
.query_resource_types(w_id, &query.text, query.limit)
.await?;
Ok(Json(results))
} else {
Err(windmill_common::error::Error::internal_err(
"Embeddings db not initialized".to_string(),
))
}
}
#[cfg(feature = "embedding")]
#[derive(Deserialize, Debug, Clone)]
struct HubScript {
ask_id: i64,
id: i64,
version_id: i64,
summary: String,
app: String,
kind: String,
embedding: Vec<f32>,
}
#[cfg(feature = "embedding")]
#[derive(Deserialize, Debug)]
struct HubResourceType {
name: String,
embedding: Vec<f32>,
}
#[cfg(feature = "embedding")]
pub struct ModelInstance {
model: BertModel,
tokenizer: Tokenizer,
}
#[cfg(feature = "embedding")]
impl ModelInstance {
pub async fn load_model_files() -> Result<(PathBuf, PathBuf, PathBuf)> {
let api = Api::new()?;
let repo_api = api.model("thenlper/gte-small".to_string());
let (config_filename, tokenizer_filename, weights_filename) =
(
repo_api
.get("config.json")
.await
.map_err(|e| anyhow!("Failed to get config.json from hugging face: {}", e))?,
repo_api.get("tokenizer.json").await.map_err(|e| {
anyhow!("Failed to get tokenizer.json from hugging face: {}", e)
})?,
repo_api.get("model.safetensors").await.map_err(|e| {
anyhow!("Failed to get model.safetensors from hugging face: {}", e)
})?,
);
Ok((config_filename, tokenizer_filename, weights_filename))
}
pub async fn new() -> Result<Self> {
tracing::info!("Loading embedding model...");
let device = Device::Cpu;
let (config_filename, tokenizer_filename, weights_filename) =
Self::load_model_files().await?;
let config = std::fs::read_to_string(config_filename)?;
let config: Config = serde_json::from_str(&config)?;
let tokenizer = Tokenizer::from(
Tokenizer::from_file(tokenizer_filename)
.map_err(Error::msg)?
.with_padding(None)
.to_owned(),
);
let vb =
unsafe { VarBuilder::from_mmaped_safetensors(&[weights_filename], DTYPE, &device)? };
let model = BertModel::load(vb, &config)?;
tracing::info!("Loaded embedding model");
Ok(Self { model, tokenizer })
}
pub async fn create_embedding(self: Arc<Self>, sentence: &str) -> Result<Vec<f32>> {
let sentence = sentence.to_owned();
tokio::task::spawn_blocking(move || {
let tokens = self
.tokenizer
.encode(sentence, true)
.map_err(Error::msg)?
.get_ids()
.to_vec();
let token_ids = Tensor::new(&tokens[..], &Device::Cpu)?.unsqueeze(0)?;
let token_type_ids = token_ids.zeros_like()?;
let embedding = self.model.forward(&token_ids, &token_type_ids, None)?;
let embedding = (embedding.sum(1)? / embedding.dim(1)? as f64)?;
let embedding = normalize_l2(&embedding)?;
let embedding = embedding.get(0)?.to_vec1()?;
Ok(embedding)
})
.await?
}
}
#[cfg(feature = "embedding")]
pub struct EmbeddingsDb {
db: Db,
model_instance: Arc<ModelInstance>,
}
#[cfg(feature = "embedding")]
impl EmbeddingsDb {
pub async fn new(pg_db: &Pool<Postgres>, model_instance: Arc<ModelInstance>) -> Result<Self> {
let db = Db::new();
let mut embeddings_db = Self { db, model_instance: model_instance.clone() };
embeddings_db.fill_db(pg_db).await?;
Ok(embeddings_db)
}
async fn fill_db(&mut self, pg_db: &Pool<Postgres>) -> Result<()> {
if self.db.get_collection("scripts").is_some() {
self.db.delete_collection("scripts")?;
}
self.db
.create_collection("scripts".to_string(), 384, Distance::Cosine)?;
if self.db.get_collection("resource_types").is_some() {
self.db.delete_collection("resource_types")?;
}
self.db
.create_collection("resource_types".to_string(), 384, Distance::Cosine)?;
let hub_base_url = HUB_BASE_URL.read().await.clone();
let response = match hub_base_url.as_str() {
DEFAULT_HUB_BASE_URL => {
let response = HTTP_CLIENT
.get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json")
.send()
.await;
if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() {
tracing::warn!("Failed to get scripts embeddings from bucket, trying hub...");
http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/scripts/embeddings", hub_base_url),
false,
None,
Some(pg_db),
)
.await?
} else {
response.unwrap()
}
}
_ => {
http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/scripts/embeddings", hub_base_url),
false,
None,
Some(pg_db),
)
.await?
}
};
if response.error_for_status_ref().is_err() {
return Err(anyhow!(
"Failed to get scripts embeddings from hub with error code: {}",
response.status()
));
}
let hub_scripts = response.json::<Vec<HubScript>>().await?;
for script in &hub_scripts {
let mut hm = HashMap::new();
hm.insert("ask_id".to_string(), script.ask_id.clone().to_string());
hm.insert("summary".to_string(), script.summary.clone());
hm.insert("app".to_string(), script.app.clone());
hm.insert("kind".to_string(), script.kind.clone());
hm.insert("id".to_string(), script.id.clone().to_string());
hm.insert(
"version_id".to_string(),
script.version_id.clone().to_string(),
);
let embedding = Embedding {
id: script.ask_id.clone().to_string(),
vector: script.embedding.clone(),
metadata: Some(hm),
};
self.db.insert_into_collection("scripts", embedding)?;
}
let response = match hub_base_url.as_str() {
DEFAULT_HUB_BASE_URL => {
let response = HTTP_CLIENT
.get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json")
.send()
.await;
if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() {
tracing::warn!(
"Failed to get resource types embeddings from bucket, trying hub..."
);
http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/resource_types/embeddings", hub_base_url),
false,
None,
Some(pg_db),
)
.await?
} else {
response.unwrap()
}
}
_ => {
http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/resource_types/embeddings", hub_base_url),
false,
None,
Some(pg_db),
)
.await?
}
};
if response.error_for_status_ref().is_err() {
return Err(anyhow!(
"Failed to get resource types embeddings from hub with error code: {}",
response.status()
));
}
let hub_resource_types = response.json::<Vec<HubResourceType>>().await?;
let resource_types: Vec<ResourceType> =
sqlx::query_as!(ResourceType, "SELECT * from resource_type ORDER BY name",)
.fetch_all(pg_db)
.await?;
for rt in resource_types {
let mut hm = HashMap::new();
hm.insert("name".to_string(), rt.name.clone());
if let Some(schema) = rt.schema.clone() {
hm.insert("schema".to_string(), serde_json::to_string(&schema)?);
}
hm.insert("workspace".to_string(), rt.workspace_id.clone());
let hub_rt = hub_resource_types.iter().find(|hrt| hrt.name == rt.name);
let vector = if let Some(hub_rt) = hub_rt {
hub_rt.embedding.clone()
} else {
self.model_instance
.clone()
.create_embedding(&format!(
"{};{}",
rt.name,
rt.description.unwrap_or_default()
))
.await?
};
let embedding = Embedding {
id: format!("{}_{}", rt.workspace_id, rt.name),
vector,
metadata: Some(hm),
};
self.db
.insert_into_collection("resource_types", embedding)?;
}
Ok(())
}
pub async fn query_hub_scripts(
&self,
query: &str,
limit: Option<i64>,
kind: Option<String>,
app: Option<String>,
) -> Result<Vec<HubScriptResult>> {
let model_instance = self.model_instance.clone();
let query_embedding = model_instance.create_embedding(query).await?;
let collection = self.db.get_collection("scripts");
let collection = collection.ok_or(Error::msg("no collection found"))?;
let filter = |embedding: &Embedding| {
if let Some(metadata) = embedding.metadata.as_ref() {
match (
metadata.get("kind"),
kind.clone(),
metadata.get("app"),
app.clone(),
) {
(Some(script_kind), Some(kind), Some(script_app), Some(app)) => {
&kind == script_kind && &app == script_app
}
(Some(script_kind), Some(kind), _, _) => &kind == script_kind,
(_, _, Some(script_app), Some(app)) => &app == script_app,
(_, None, _, None) => true,
_ => false,
}
} else {
false
}
};
let results = collection.get_similarity(
&query_embedding,
limit.unwrap_or(10) as usize,
Some(&filter),
Some(0.8),
);
let results: Result<Vec<_>> = results
.iter()
.map(|r| {
let metadata = r
.embedding
.metadata
.as_ref()
.ok_or(Error::msg("no metadata"))?;
Ok(HubScriptResult {
ask_id: metadata
.get("ask_id")
.ok_or(Error::msg("no ask_id"))?
.parse::<i64>()?,
summary: metadata
.get("summary")
.ok_or(Error::msg("no summary"))?
.to_owned(),
app: metadata.get("app").ok_or(Error::msg("no app"))?.to_owned(),
kind: metadata
.get("kind")
.ok_or(Error::msg("no kind"))?
.to_owned(),
id: metadata
.get("id")
.ok_or(Error::msg("no id"))?
.parse::<i64>()?,
version_id: metadata
.get("version_id")
.ok_or(Error::msg("no version_id"))?
.parse::<i64>()?,
score: r.score,
})
})
.collect();
let mut results = results?;
if results.len() > 1 {
let top_score = results[0].score;
results = results
.into_iter()
.take_while(|r| (top_score - r.score) / top_score <= 0.05)
.collect();
}
Ok(results)
}
pub async fn query_resource_types(
&self,
workspace: String,
query: &str,
limit: Option<i64>,
) -> Result<Vec<ResourceTypeResult>> {
let model_instance = self.model_instance.clone();
let query_embedding = model_instance.create_embedding(query).await?;
let collection = self.db.get_collection("resource_types");
if collection.is_none() {
return Ok(vec![]);
}
let collection = collection.ok_or(Error::msg("no collection found"))?;
let filter = |embedding: &Embedding| {
if let Some(metadata) = embedding.metadata.as_ref() {
match metadata.get("workspace").map(|x| x.as_str()) {
Some("admins") => true,
Some(rt_workspace) => &workspace == rt_workspace,
_ => false,
}
} else {
false
}
};
let results = collection.get_similarity(
&query_embedding,
limit.unwrap_or(10) as usize,
Some(&filter),
Some(0.75),
);
let results: Result<_> = results
.iter()
.map(|r| {
let metadata = r
.embedding
.metadata
.as_ref()
.ok_or(Error::msg("no metadata"))?;
Ok(ResourceTypeResult {
name: metadata
.get("name")
.ok_or(Error::msg("no name"))?
.to_owned(),
schema: match metadata.get("schema") {
Some(schema) => serde_json::from_str(schema)?,
None => None,
},
score: r.score,
})
})
.collect();
results
}
}
#[cfg(feature = "embedding")]
fn normalize_l2(v: &Tensor) -> Result<Tensor> {
Ok(v.broadcast_div(&v.sqr()?.sum_keepdim(1)?.sqrt()?)?)
}
#[cfg(feature = "embedding")]
pub fn load_embeddings_db(db: &Pool<Postgres>) -> () {
let disable_embedding = std::env::var("DISABLE_EMBEDDING")
.ok()
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false);
if !disable_embedding {
let db_clone = db.clone();
tokio::spawn(async move {
let model_instance = ModelInstance::new().await;
if let Ok(model_instance) = model_instance {
let mut model_instance_lock = MODEL_INSTANCE.write().await;
*model_instance_lock = Some(Arc::new(model_instance));
drop(model_instance_lock);
loop {
update_embeddings_db(&db_clone).await;
tokio::time::sleep(std::time::Duration::from_secs(
*HUB_EMBEDDINGS_PULLING_INTERVAL_SECS,
))
.await;
}
} else {
tracing::error!(
"Failed to initialize model instance: {}",
model_instance.err().unwrap()
);
}
});
}
}
#[cfg(feature = "embedding")]
pub async fn update_embeddings_db(db: &Pool<Postgres>) -> () {
if let Some(model_instance) = MODEL_INSTANCE.read().await.as_ref() {
tracing::info!("Creating embeddings DB...");
let new_embeddings_db = EmbeddingsDb::new(&db, model_instance.clone()).await;
if let Err(e) = new_embeddings_db.as_ref() {
tracing::error!("Failed to create embeddings db: {}", e);
} else {
let mut embeddings_db = EMBEDDINGS_DB.write().await;
*embeddings_db = new_embeddings_db.ok();
tracing::info!("Created embeddings DB");
}
} else {
tracing::error!("Could not update embeddings DB, model instance not initialized");
}
}
#[cfg(feature = "embedding")]
pub fn workspaced_service() -> Router {
Router::new().route("/query_resource_types", get(query_resource_types))
}
#[cfg(feature = "embedding")]
pub fn global_service() -> Router {
Router::new().route("/query_hub_scripts", get(query_hub_scripts))
}
#[cfg(not(feature = "embedding"))]
pub fn workspaced_service() -> Router {
Router::new()
}
#[cfg(not(feature = "embedding"))]
pub fn global_service() -> Router {
Router::new()
}
pub use windmill_api_embeddings::*;
backend/windmill-api/src/flow_conversations.rs
+1 -60
View File
@@ -10,6 +10,7 @@ use sqlx::{FromRow, Postgres};
use uuid::Uuid;
use crate::db::ApiAuthed;
pub use windmill_common::flow_conversations::FlowConversation;
use windmill_common::{
db::{UserDB, DB},
error::{JsonResult, Result},
@@ -24,17 +25,6 @@ pub fn workspaced_service() -> Router {
.route("/:conversation_id/messages", get(list_messages))
}
#[derive(Serialize, FromRow, Debug)]
pub struct FlowConversation {
pub id: Uuid,
pub workspace_id: String,
pub flow_path: String,
pub title: Option<String>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub created_by: String,
}
#[derive(Serialize, FromRow, Debug)]
pub struct FlowConversationMessage {
pub id: Uuid,
@@ -104,55 +94,6 @@ async fn list_conversations(
Ok(Json(conversations))
}
pub async fn get_or_create_conversation_with_id(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
w_id: &str,
flow_path: &str,
username: &str,
title: &str,
conversation_id: Uuid,
) -> Result<FlowConversation> {
// Check if conversation already exists
let existing_conversation = sqlx::query_as!(
FlowConversation,
"SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by
FROM flow_conversation
WHERE id = $1 AND workspace_id = $2",
conversation_id,
w_id
)
.fetch_optional(&mut **tx)
.await?;
if let Some(existing) = existing_conversation {
return Ok(existing);
}
// Truncate title to 25 char characters max
let title = if title.len() > 25 {
format!("{}...", &title[..25])
} else {
title.to_string()
};
// Create new conversation with provided ID
let conversation = sqlx::query_as!(
FlowConversation,
"INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title)
VALUES ($1, $2, $3, $4, $5)
RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by",
conversation_id,
w_id,
flow_path,
username,
title
)
.fetch_one(&mut **tx)
.await?;
Ok(conversation)
}
async fn delete_conversation(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
backend/windmill-api/src/folders.rs
+2 -18
View File
@@ -306,24 +306,8 @@ pub async fn is_owner_api(
Ok(Json(is_owner(&authed, &name)))
}
pub fn is_owner(ApiAuthed { is_admin, folders, .. }: &ApiAuthed, name: &str) -> bool {
if *is_admin {
true
} else {
folders.into_iter().any(|x| x.0 == name && x.2)
}
}
pub fn require_is_owner(authed: &ApiAuthed, name: &str) -> Result<()> {
if is_owner(authed, name) {
Ok(())
} else {
Err(windmill_common::error::Error::NotAuthorized(format!(
"You are not owner of the folder {}",
name
)))
}
}
use windmill_api_auth::is_owner;
pub use windmill_api_auth::require_is_owner;
async fn update_folder(
authed: ApiAuthed,
backend/windmill-api/src/groups.rs
+47 -19
View File
@@ -271,15 +271,8 @@ async fn create_group(
)
.await?;
log_group_permission_change(
&mut *tx,
&w_id,
&ng.name,
&authed.username,
"create",
None,
)
.await?;
log_group_permission_change(&mut *tx, &w_id, &ng.name, &authed.username, "create", None)
.await?;
tx.commit().await?;
@@ -606,7 +599,10 @@ async fn add_user(
.await?;
if result.rows_affected() == 0 {
return Ok(format!("{} is already a member of group {}", user_username, name));
return Ok(format!(
"{} is already a member of group {}",
user_username, name
));
}
audit_log(
@@ -693,12 +689,37 @@ async fn add_user_igroup(
WHERE auto_invite->'instance_groups' ? $1
"#,
&name
).fetch_all(&mut *tx).await?;
)
.fetch_all(&mut *tx)
.await?;
for ws in workspaces {
let role = ws.instance_groups_roles.and_then(|r| r.get(&name).and_then(|v| v.as_str().map(String::from))).unwrap_or_else(|| "developer".to_string());
let (is_admin, is_operator) = match role.as_str() { "admin" => (true, false), "operator" => (false, true), _ => (false, false) };
auto_add_user(&email, &ws.workspace_id, &is_operator, &mut tx, &authed, Some(serde_json::json!({"source": "instance_group", "group": &name}))).await?;
if is_admin { sqlx::query!("UPDATE usr SET is_admin = true WHERE workspace_id = $1 AND email = $2", &ws.workspace_id, &email).execute(&mut *tx).await?; }
let role = ws
.instance_groups_roles
.and_then(|r| r.get(&name).and_then(|v| v.as_str().map(String::from)))
.unwrap_or_else(|| "developer".to_string());
let (is_admin, is_operator) = match role.as_str() {
"admin" => (true, false),
"operator" => (false, true),
_ => (false, false),
};
auto_add_user(
&email,
&ws.workspace_id,
&is_operator,
&mut tx,
&authed,
Some(serde_json::json!({"source": "instance_group", "group": &name})),
)
.await?;
if is_admin {
sqlx::query!(
"UPDATE usr SET is_admin = true WHERE workspace_id = $1 AND email = $2",
&ws.workspace_id,
&email
)
.execute(&mut *tx)
.await?;
}
}
}
@@ -741,7 +762,9 @@ async fn list_igroups(Extension(db): Extension<DB>) -> JsonResult<Vec<IGroup>> {
return Ok(Json(groups));
}
async fn list_igroups_with_workspaces(Extension(db): Extension<DB>) -> JsonResult<Vec<IGroupWithWorkspaces>> {
async fn list_igroups_with_workspaces(
Extension(db): Extension<DB>,
) -> JsonResult<Vec<IGroupWithWorkspaces>> {
let mut tx: Transaction<'_, Postgres> = db.begin().await?;
// Get all instance groups with their emails first
@@ -771,9 +794,11 @@ async fn list_igroups_with_workspaces(Extension(db): Extension<DB>) -> JsonResul
.await?;
// Create a map of group_name -> Vec<WorkspaceInfo>
let mut workspaces_by_group: std::collections::HashMap<String, Vec<WorkspaceInfo>> = std::collections::HashMap::new();
let mut workspaces_by_group: std::collections::HashMap<String, Vec<WorkspaceInfo>> =
std::collections::HashMap::new();
for mapping in workspace_mappings {
let role = mapping.role
let role = mapping
.role
.and_then(|r| r.as_str().map(|s| s.to_string()))
.unwrap_or_else(|| "developer".to_string());
@@ -791,7 +816,10 @@ async fn list_igroups_with_workspaces(Extension(db): Extension<DB>) -> JsonResul
let mut result = Vec::new();
for group in groups {
let workspaces = workspaces_by_group.get(&group.name).cloned().unwrap_or_default();
let workspaces = workspaces_by_group
.get(&group.name)
.cloned()
.unwrap_or_default();
result.push(IGroupWithWorkspaces {
name: group.name,

Some files were not shown because too many files have changed in this diff Show More