From 86eb9074cc94f309f17ea72e9cecd0d502ffd2be Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 20 Jun 2025 12:07:33 +0200 Subject: [PATCH] feat: blacklist remote agent worker token (#5985) --- ...1e1f501a821d6143c5d8aece20814237a6b86.json | 38 ++ ...a01ebe289183526afab19c1390bbf5082f019.json | 23 ++ ...079ac1721fe7bd1844f339236379211defc78.json | 14 + ...7a9ee4b80cf3ade038a032c57ecff8fa3c6cf.json | 20 + ...141bee63339d4e3d5e2d68aabb12009f691c6.json | 16 + ...b83b28fa8cbb528a2cfc90684cb10a69e4375.json | 40 ++ backend/ee-repo-ref.txt | 2 +- ...8164000_add_agent_token_blacklist.down.sql | 2 + ...618164000_add_agent_token_blacklist.up.sql | 14 + backend/src/main.rs | 3 + backend/src/monitor.rs | 18 + backend/windmill-api/openapi.yaml | 92 +++++ backend/windmill-api/src/agent_workers_oss.rs | 12 - backend/windmill-worker/src/worker.rs | 4 +- .../components/HttpAgentWorkerDrawer.svelte | 362 +++++++++++++----- 15 files changed, 554 insertions(+), 106 deletions(-) create mode 100644 backend/.sqlx/query-1c5d3556fc8436ddd294f39c5431e1f501a821d6143c5d8aece20814237a6b86.json create mode 100644 backend/.sqlx/query-2bf99d540365c228e1776ee5d2ba01ebe289183526afab19c1390bbf5082f019.json create mode 100644 backend/.sqlx/query-54fee31b61d62598c89cf7d0729079ac1721fe7bd1844f339236379211defc78.json create mode 100644 backend/.sqlx/query-995b194da28092d5aa053df936e7a9ee4b80cf3ade038a032c57ecff8fa3c6cf.json create mode 100644 backend/.sqlx/query-c9c040ec228a8fe4fda08439420141bee63339d4e3d5e2d68aabb12009f691c6.json create mode 100644 backend/.sqlx/query-d56722c25877222af9affd5da5bb83b28fa8cbb528a2cfc90684cb10a69e4375.json create mode 100644 backend/migrations/20250618164000_add_agent_token_blacklist.down.sql create mode 100644 backend/migrations/20250618164000_add_agent_token_blacklist.up.sql diff --git a/backend/.sqlx/query-1c5d3556fc8436ddd294f39c5431e1f501a821d6143c5d8aece20814237a6b86.json b/backend/.sqlx/query-1c5d3556fc8436ddd294f39c5431e1f501a821d6143c5d8aece20814237a6b86.json new file mode 100644 index 0000000000..adf00884b8 --- /dev/null +++ b/backend/.sqlx/query-1c5d3556fc8436ddd294f39c5431e1f501a821d6143c5d8aece20814237a6b86.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT token, expires_at, blacklisted_at, blacklisted_by \n FROM agent_token_blacklist \n ORDER BY blacklisted_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "expires_at", + "type_info": "Timestamp" + }, + { + "ordinal": 2, + "name": "blacklisted_at", + "type_info": "Timestamp" + }, + { + "ordinal": 3, + "name": "blacklisted_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "1c5d3556fc8436ddd294f39c5431e1f501a821d6143c5d8aece20814237a6b86" +} diff --git a/backend/.sqlx/query-2bf99d540365c228e1776ee5d2ba01ebe289183526afab19c1390bbf5082f019.json b/backend/.sqlx/query-2bf99d540365c228e1776ee5d2ba01ebe289183526afab19c1390bbf5082f019.json new file mode 100644 index 0000000000..04acf05335 --- /dev/null +++ b/backend/.sqlx/query-2bf99d540365c228e1776ee5d2ba01ebe289183526afab19c1390bbf5082f019.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM agent_token_blacklist WHERE token = $1 AND expires_at > $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Timestamp" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2bf99d540365c228e1776ee5d2ba01ebe289183526afab19c1390bbf5082f019" +} diff --git a/backend/.sqlx/query-54fee31b61d62598c89cf7d0729079ac1721fe7bd1844f339236379211defc78.json b/backend/.sqlx/query-54fee31b61d62598c89cf7d0729079ac1721fe7bd1844f339236379211defc78.json new file mode 100644 index 0000000000..b41cb7bf8c --- /dev/null +++ b/backend/.sqlx/query-54fee31b61d62598c89cf7d0729079ac1721fe7bd1844f339236379211defc78.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM agent_token_blacklist WHERE token = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "54fee31b61d62598c89cf7d0729079ac1721fe7bd1844f339236379211defc78" +} diff --git a/backend/.sqlx/query-995b194da28092d5aa053df936e7a9ee4b80cf3ade038a032c57ecff8fa3c6cf.json b/backend/.sqlx/query-995b194da28092d5aa053df936e7a9ee4b80cf3ade038a032c57ecff8fa3c6cf.json new file mode 100644 index 0000000000..d5946b5114 --- /dev/null +++ b/backend/.sqlx/query-995b194da28092d5aa053df936e7a9ee4b80cf3ade038a032c57ecff8fa3c6cf.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "995b194da28092d5aa053df936e7a9ee4b80cf3ade038a032c57ecff8fa3c6cf" +} diff --git a/backend/.sqlx/query-c9c040ec228a8fe4fda08439420141bee63339d4e3d5e2d68aabb12009f691c6.json b/backend/.sqlx/query-c9c040ec228a8fe4fda08439420141bee63339d4e3d5e2d68aabb12009f691c6.json new file mode 100644 index 0000000000..c03a00eec3 --- /dev/null +++ b/backend/.sqlx/query-c9c040ec228a8fe4fda08439420141bee63339d4e3d5e2d68aabb12009f691c6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO agent_token_blacklist (token, expires_at, blacklisted_by) \n VALUES ($1, $2, $3) \n ON CONFLICT (token) DO UPDATE SET \n expires_at = EXCLUDED.expires_at,\n blacklisted_at = NOW(),\n blacklisted_by = EXCLUDED.blacklisted_by", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Timestamp", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "c9c040ec228a8fe4fda08439420141bee63339d4e3d5e2d68aabb12009f691c6" +} diff --git a/backend/.sqlx/query-d56722c25877222af9affd5da5bb83b28fa8cbb528a2cfc90684cb10a69e4375.json b/backend/.sqlx/query-d56722c25877222af9affd5da5bb83b28fa8cbb528a2cfc90684cb10a69e4375.json new file mode 100644 index 0000000000..a8256a8060 --- /dev/null +++ b/backend/.sqlx/query-d56722c25877222af9affd5da5bb83b28fa8cbb528a2cfc90684cb10a69e4375.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT token, expires_at, blacklisted_at, blacklisted_by \n FROM agent_token_blacklist \n WHERE expires_at > $1 \n ORDER BY blacklisted_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "expires_at", + "type_info": "Timestamp" + }, + { + "ordinal": 2, + "name": "blacklisted_at", + "type_info": "Timestamp" + }, + { + "ordinal": 3, + "name": "blacklisted_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Timestamp" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "d56722c25877222af9affd5da5bb83b28fa8cbb528a2cfc90684cb10a69e4375" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c41787fda1..425a17dc1a 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -67e727c618cf673850a0887931c803241abfcfe8 \ No newline at end of file +835a91c7c31ea749759cd8af0922ad837049ea2a \ No newline at end of file diff --git a/backend/migrations/20250618164000_add_agent_token_blacklist.down.sql b/backend/migrations/20250618164000_add_agent_token_blacklist.down.sql new file mode 100644 index 0000000000..aad208817c --- /dev/null +++ b/backend/migrations/20250618164000_add_agent_token_blacklist.down.sql @@ -0,0 +1,2 @@ +-- Remove agent token blacklist table +DROP TABLE IF EXISTS agent_token_blacklist; \ No newline at end of file diff --git a/backend/migrations/20250618164000_add_agent_token_blacklist.up.sql b/backend/migrations/20250618164000_add_agent_token_blacklist.up.sql new file mode 100644 index 0000000000..aefff3913d --- /dev/null +++ b/backend/migrations/20250618164000_add_agent_token_blacklist.up.sql @@ -0,0 +1,14 @@ +-- Add agent token blacklist table +CREATE TABLE agent_token_blacklist ( + token VARCHAR PRIMARY KEY, + expires_at TIMESTAMP NOT NULL, + blacklisted_at TIMESTAMP NOT NULL DEFAULT NOW(), + blacklisted_by VARCHAR NOT NULL +); + +-- Add index for efficient expiry cleanup +CREATE INDEX idx_agent_token_blacklist_expires_at ON agent_token_blacklist(expires_at); + +-- Grant permissions to windmill users +GRANT ALL ON agent_token_blacklist TO windmill_user; +GRANT ALL ON agent_token_blacklist TO windmill_admin; \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index be9a7578a3..aa3eee4a18 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1102,6 +1102,9 @@ Windmill Community Edition {GIT_VERSION} _ = tokio::time::sleep(Duration::from_secs(12 * 60 * 60)) => { tracing::info!("Reloading config after 12 hours"); initial_load(&conn, tx.clone(), worker_mode, server_mode, #[cfg(feature = "parquet")] disable_s3_store).await; + if let Err(e) = reload_license_key(&conn).await { + tracing::error!("Failed to reload license key on agent: {e:#}"); + } #[cfg(feature = "enterprise")] ee_oss::verify_license_key().await; } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 5c4677018c..4b4946e3e4 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -840,6 +840,24 @@ pub async fn delete_expired_items(db: &DB) -> () { tracing::error!("Error deleting audit log on CE: {:?}", e); } + match sqlx::query_scalar!( + "DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token", + ) + .fetch_all(db) + .await + { + Ok(deleted_tokens) => { + if deleted_tokens.len() > 0 { + tracing::info!( + "deleted {} expired blacklisted agent tokens: {:?}", + deleted_tokens.len(), + deleted_tokens + ); + } + } + Err(e) => tracing::error!("Error deleting expired blacklisted agent tokens: {:?}", e), + } + let job_retention_secs = *JOB_RETENTION_SECS.read().await; if job_retention_secs > 0 { match db.begin().await { diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 6922c98379..7f1fc67f5b 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -11240,6 +11240,98 @@ paths: schema: type: string + /agent_workers/blacklist_token: + post: + summary: blacklist agent token (requires super admin) + operationId: blacklistAgentToken + tags: + - agent_workers + requestBody: + description: token to blacklist + required: true + content: + application/json: + schema: + type: object + properties: + token: + type: string + description: The agent token to blacklist + expires_at: + type: string + format: date-time + description: Optional expiration date for the blacklist entry + required: + - token + responses: + "200": + description: token blacklisted successfully + + /agent_workers/remove_blacklist_token: + post: + summary: remove agent token from blacklist (requires super admin) + operationId: removeBlacklistAgentToken + tags: + - agent_workers + requestBody: + description: token to remove from blacklist + required: true + content: + application/json: + schema: + type: object + properties: + token: + type: string + description: The agent token to remove from blacklist + required: + - token + responses: + "200": + description: token removed from blacklist successfully + + /agent_workers/list_blacklisted_tokens: + get: + summary: list blacklisted agent tokens (requires super admin) + operationId: listBlacklistedAgentTokens + tags: + - agent_workers + parameters: + - name: include_expired + in: query + description: Whether to include expired blacklisted tokens + schema: + type: boolean + default: false + responses: + "200": + description: list of blacklisted tokens + content: + application/json: + schema: + type: array + items: + type: object + properties: + token: + type: string + description: The blacklisted token (without prefix) + expires_at: + type: string + format: date-time + description: When the blacklist entry expires + blacklisted_at: + type: string + format: date-time + description: When the token was blacklisted + blacklisted_by: + type: string + description: Email of the user who blacklisted the token + required: + - token + - expires_at + - blacklisted_at + - blacklisted_by /w/{workspace}/acls/get/{kind}/{path}: get: diff --git a/backend/windmill-api/src/agent_workers_oss.rs b/backend/windmill-api/src/agent_workers_oss.rs index 655058c493..dabb2cb079 100644 --- a/backend/windmill-api/src/agent_workers_oss.rs +++ b/backend/windmill-api/src/agent_workers_oss.rs @@ -16,9 +16,6 @@ use crate::db::DB; #[cfg(not(feature = "private"))] use axum::Router; -#[cfg(not(feature = "private"))] -use serde::{Deserialize, Serialize}; - #[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new() @@ -44,15 +41,6 @@ pub fn workspaced_service( (router, vec![], Some(job_completed_tx)) } -#[derive(Clone, Debug, Deserialize, Serialize)] -#[cfg(not(feature = "private"))] -pub struct AgentAuth { - pub worker_group: String, - pub suffix: Option, - pub tags: Vec, - pub exp: Option, -} - #[cfg(not(feature = "private"))] pub struct AgentCache {} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 56ac7c00e0..058ad19598 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -858,13 +858,14 @@ pub fn start_interactive_worker_shell( .await; } _ => { - tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE)).await; + tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE * 10)).await; } } } Err(err) => { tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err); + tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE * 20)).await; } }; } @@ -1861,6 +1862,7 @@ pub async fn run_worker( } Err(err) => { tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err); + tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE * 5)).await; } }; } diff --git a/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte b/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte index 9a505bad55..d9e43fe2a4 100644 --- a/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte +++ b/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte @@ -1,8 +1,8 @@ -
- Use HTTP agent workers only when the workers need to be deployed remotely OR with only HTTP - connectivity OR in untrusted environments. HTTP agent workers have more latency and less - capabilities than normal workers. -
- -
-
- {#if !$enterpriseLicense} -
- Agent workers are only available in the enterprise edition. For evaluation purposes, you can - only use the tag `agent_test` tag and it is limited to 100 jobs. -
- {/if} - -
+ + Create + Blacklist + {#snippet content()} +
+ {#if selectedTab === 'create'} + Use HTTP agent workers only when the workers need to be deployed remotely OR with only + HTTP connectivity OR in untrusted environments. HTTP agent workers have more latency and + less capabilities than normal workers. +
+
+ +
+
+ {#if !$enterpriseLicense} +
+ Agent workers are only available in the enterprise edition. For evaluation purposes, + you can only use the tag `agent_test` tag and it is limited to 100 jobs. +
+ {/if} + +
-
- {#if !$enterpriseLicense} -
- Agent workers are only available in the enterprise edition. For evaluation purposes, you can - only use the tag `agent_test` tag and it is limited to 100 jobs. -
- {/if} -
- - { - if (token) { - navigator.clipboard.writeText(token) - sendUserToast('Copied to clipboard') - } - }} - placeholder="Select tags to generate a JWT token" - type="text" - disabled - value={token} - class="w-full pr-10 pl-3 py-2 text-sm text-gray-600 bg-gray-50 border border-gray-300 rounded-lg cursor-pointer hover:bg-gray-100 transition truncate" - /> +
+ {#if !$enterpriseLicense} +
+ Agent workers are only available in the enterprise edition. For evaluation purposes, + you can only use the tag `agent_test` tag and it is limited to 100 jobs. +
+ {/if} +
+ { + e.preventDefault() + e.stopPropagation() + if (token) { + navigator.clipboard.writeText(token) + sendUserToast('Copied to clipboard') + } + }} + placeholder="Select tags to generate a JWT token" + type="text" + disabled + value={token} + class="w-full pr-10 pl-3 py-2 text-sm text-gray-600 bg-gray-50 border border-gray-300 rounded-lg cursor-pointer hover:bg-gray-100 transition truncatere" + /> - - -
+ +
-
- Set the following environment variables: -
    -
  • MODE=agent
    • -
  • AGENT_TOKEN=<token>
    • -
  • BASE_INTERNAL_URL=<base url>
    • -
-

- to a worker to have it act as an HTTP agent worker. - INIT_SCRIPT, if needed, must be passed as an env variable. -

- - Ensure at least one normal worker is running and listening to the tags - flow and dependency - (or flow-<workspace> and dependency-<workspace> if - using workspace-specific default tags), because agent workers - cannot run dependency jobs - nor execute the - flow state machine. They can, however, run subjobs within flows. - - -
- Use the following API endpoint with a superadmin bearer token: - POST /api/agent_workers/create_agent_token - 
+						

+							Set the following environment variables:
+							
    
+								
  • MODE=agent
    • 
+								
  • AGENT_TOKEN=<token>
    • 
+								
  • BASE_INTERNAL_URL=<base url>
    • 
+							

+							

+								to a worker to have it act as an HTTP agent worker.
+								INIT_SCRIPT, if needed, must be passed as an env variable.
+							

+							
+								Ensure at least one normal worker is running and listening to the tags
+								flow and dependency
+								(or flow-<workspace> and
+								dependency-<workspace>
+								if using workspace-specific default tags), because agent workers
+								cannot run dependency jobs
+								nor execute the
+								flow state machine. They can, however, run subjobs within flows.
+							
+							
+								

+									Use the following API endpoint with a superadmin bearer token:
+									POST /api/agent_workers/create_agent_token
+									
 	{`
+											>{`
 	  "worker_group": "agent",
 	  "tags": ["tag1", "tag2"],
 	  "exp": 1717334400
 	`}
-					

-					The JSON response will contain the generated JWT token.
+										>
+
+ The JSON response will contain the generated JWT token. +
+ +
+
- + {:else if selectedTab === 'blacklist'} +
+
+ {#if !$enterpriseLicense} +
+ Token blacklist management is only available in the enterprise edition. +
+ {:else} +
+ Add tokens to the blacklist to prevent them from being used by agent workers. + Blacklisted tokens may take up to 5 minutes to be effective because of caching. +
+ +
+
+ + +
+
+ +
+ + {#if !$superadmin} +
+ Only superadmins can manage the token blacklist. +
+ {/if} +
+ + +
+
+

Blacklisted Tokens

+ +
+ + {#if isLoadingBlacklist} +
Loading blacklisted tokens...
+ {:else if blacklistedTokens?.length === 0} +
+ No tokens are currently blacklisted. +
+ {:else} +
+ {#each blacklistedTokens ?? [] as blacklistedToken} +
+
+
+ {blacklistedToken.token} +
+ {#if blacklistedToken.expires_at} +
+ Expires: {new Date(blacklistedToken.expires_at).toLocaleString()} +
+ {/if} +
+ {#if $superadmin} + + {/if} +
+ {/each} +
+ {/if} +
+ {/if} +
+
+ {/if}
- -
+ {/snippet} +