feat: blacklist remote agent worker token (#5985)

This commit is contained in:
Ruben Fiszel
2025-06-20 12:07:33 +02:00
committed by GitHub
parent 06e61ee958
commit 86eb9074cc
15 changed files with 554 additions and 106 deletions
@@ -0,0 +1,38 @@
{
"db_name": "PostgreSQL",
"query": "SELECT token, expires_at, blacklisted_at, blacklisted_by \n FROM agent_token_blacklist \n ORDER BY blacklisted_at DESC",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "token",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "expires_at",
"type_info": "Timestamp"
},
{
"ordinal": 2,
"name": "blacklisted_at",
"type_info": "Timestamp"
},
{
"ordinal": 3,
"name": "blacklisted_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
false
]
},
"hash": "1c5d3556fc8436ddd294f39c5431e1f501a821d6143c5d8aece20814237a6b86"
}
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM agent_token_blacklist WHERE token = $1 AND expires_at > $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Timestamp"
]
},
"nullable": [
null
]
},
"hash": "2bf99d540365c228e1776ee5d2ba01ebe289183526afab19c1390bbf5082f019"
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM agent_token_blacklist WHERE token = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "54fee31b61d62598c89cf7d0729079ac1721fe7bd1844f339236379211defc78"
}
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "token",
"type_info": "Varchar"
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "995b194da28092d5aa053df936e7a9ee4b80cf3ade038a032c57ecff8fa3c6cf"
}
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO agent_token_blacklist (token, expires_at, blacklisted_by) \n VALUES ($1, $2, $3) \n ON CONFLICT (token) DO UPDATE SET \n expires_at = EXCLUDED.expires_at,\n blacklisted_at = NOW(),\n blacklisted_by = EXCLUDED.blacklisted_by",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Timestamp",
"Varchar"
]
},
"nullable": []
},
"hash": "c9c040ec228a8fe4fda08439420141bee63339d4e3d5e2d68aabb12009f691c6"
}
@@ -0,0 +1,40 @@
{
"db_name": "PostgreSQL",
"query": "SELECT token, expires_at, blacklisted_at, blacklisted_by \n FROM agent_token_blacklist \n WHERE expires_at > $1 \n ORDER BY blacklisted_at DESC",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "token",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "expires_at",
"type_info": "Timestamp"
},
{
"ordinal": 2,
"name": "blacklisted_at",
"type_info": "Timestamp"
},
{
"ordinal": 3,
"name": "blacklisted_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Timestamp"
]
},
"nullable": [
false,
false,
false,
false
]
},
"hash": "d56722c25877222af9affd5da5bb83b28fa8cbb528a2cfc90684cb10a69e4375"
}
+1 -1
View File
@@ -1 +1 @@
67e727c618cf673850a0887931c803241abfcfe8
835a91c7c31ea749759cd8af0922ad837049ea2a
@@ -0,0 +1,2 @@
-- Remove agent token blacklist table
DROP TABLE IF EXISTS agent_token_blacklist;
@@ -0,0 +1,14 @@
-- Add agent token blacklist table
CREATE TABLE agent_token_blacklist (
token VARCHAR PRIMARY KEY,
expires_at TIMESTAMP NOT NULL,
blacklisted_at TIMESTAMP NOT NULL DEFAULT NOW(),
blacklisted_by VARCHAR NOT NULL
);
-- Add index for efficient expiry cleanup
CREATE INDEX idx_agent_token_blacklist_expires_at ON agent_token_blacklist(expires_at);
-- Grant permissions to windmill users
GRANT ALL ON agent_token_blacklist TO windmill_user;
GRANT ALL ON agent_token_blacklist TO windmill_admin;
+3
View File
@@ -1102,6 +1102,9 @@ Windmill Community Edition {GIT_VERSION}
_ = tokio::time::sleep(Duration::from_secs(12 * 60 * 60)) => {
tracing::info!("Reloading config after 12 hours");
initial_load(&conn, tx.clone(), worker_mode, server_mode, #[cfg(feature = "parquet")] disable_s3_store).await;
if let Err(e) = reload_license_key(&conn).await {
tracing::error!("Failed to reload license key on agent: {e:#}");
}
#[cfg(feature = "enterprise")]
ee_oss::verify_license_key().await;
}
+18
View File
@@ -840,6 +840,24 @@ pub async fn delete_expired_items(db: &DB) -> () {
tracing::error!("Error deleting audit log on CE: {:?}", e);
}
match sqlx::query_scalar!(
"DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token",
)
.fetch_all(db)
.await
{
Ok(deleted_tokens) => {
if deleted_tokens.len() > 0 {
tracing::info!(
"deleted {} expired blacklisted agent tokens: {:?}",
deleted_tokens.len(),
deleted_tokens
);
}
}
Err(e) => tracing::error!("Error deleting expired blacklisted agent tokens: {:?}", e),
}
let job_retention_secs = *JOB_RETENTION_SECS.read().await;
if job_retention_secs > 0 {
match db.begin().await {
+92
View File
@@ -11240,6 +11240,98 @@ paths:
schema:
type: string
/agent_workers/blacklist_token:
post:
summary: blacklist agent token (requires super admin)
operationId: blacklistAgentToken
tags:
- agent_workers
requestBody:
description: token to blacklist
required: true
content:
application/json:
schema:
type: object
properties:
token:
type: string
description: The agent token to blacklist
expires_at:
type: string
format: date-time
description: Optional expiration date for the blacklist entry
required:
- token
responses:
"200":
description: token blacklisted successfully
/agent_workers/remove_blacklist_token:
post:
summary: remove agent token from blacklist (requires super admin)
operationId: removeBlacklistAgentToken
tags:
- agent_workers
requestBody:
description: token to remove from blacklist
required: true
content:
application/json:
schema:
type: object
properties:
token:
type: string
description: The agent token to remove from blacklist
required:
- token
responses:
"200":
description: token removed from blacklist successfully
/agent_workers/list_blacklisted_tokens:
get:
summary: list blacklisted agent tokens (requires super admin)
operationId: listBlacklistedAgentTokens
tags:
- agent_workers
parameters:
- name: include_expired
in: query
description: Whether to include expired blacklisted tokens
schema:
type: boolean
default: false
responses:
"200":
description: list of blacklisted tokens
content:
application/json:
schema:
type: array
items:
type: object
properties:
token:
type: string
description: The blacklisted token (without prefix)
expires_at:
type: string
format: date-time
description: When the blacklist entry expires
blacklisted_at:
type: string
format: date-time
description: When the token was blacklisted
blacklisted_by:
type: string
description: Email of the user who blacklisted the token
required:
- token
- expires_at
- blacklisted_at
- blacklisted_by
/w/{workspace}/acls/get/{kind}/{path}:
get:
@@ -16,9 +16,6 @@ use crate::db::DB;
#[cfg(not(feature = "private"))]
use axum::Router;
#[cfg(not(feature = "private"))]
use serde::{Deserialize, Serialize};
#[cfg(not(feature = "private"))]
pub fn global_service() -> Router {
Router::new()
@@ -44,15 +41,6 @@ pub fn workspaced_service(
(router, vec![], Some(job_completed_tx))
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[cfg(not(feature = "private"))]
pub struct AgentAuth {
pub worker_group: String,
pub suffix: Option<String>,
pub tags: Vec<String>,
pub exp: Option<usize>,
}
#[cfg(not(feature = "private"))]
pub struct AgentCache {}
+3 -1
View File
@@ -858,13 +858,14 @@ pub fn start_interactive_worker_shell(
.await;
}
_ => {
tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE)).await;
tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE * 10)).await;
}
}
}
Err(err) => {
tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err);
tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE * 20)).await;
}
};
}
@@ -1861,6 +1862,7 @@ pub async fn run_worker(
}
Err(err) => {
tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err);
tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE * 5)).await;
}
};
}
@@ -1,8 +1,8 @@
<script lang="ts">
import { AgentWorkersService } from '$lib/gen'
import { AgentWorkersService, type ListBlacklistedAgentTokensResponse } from '$lib/gen'
import { sendUserToast } from '$lib/toast'
import { Copy } from 'lucide-svelte'
import { Alert } from './common'
import { Copy, Trash2, RefreshCw } from 'lucide-svelte'
import { Alert, Button, Tab, Tabs } from './common'
import Section from './Section.svelte'
import TagsToListenTo from './TagsToListenTo.svelte'
import { enterpriseLicense, superadmin } from '$lib/stores'
@@ -15,6 +15,10 @@
let selectedTags: string[] = $state(!$enterpriseLicense ? ['agent_test'] : [])
let workerGroup: string = $state('agent')
let token: string = $state('')
let blacklistToken: string = $state('')
let selectedTab: 'create' | 'blacklist' = $state('create')
let blacklistedTokens: ListBlacklistedAgentTokensResponse | undefined = $state(undefined)
let isLoadingBlacklist: boolean = $state(false)
async function refreshToken(workerGroup: string, selectedTags: string[]) {
try {
@@ -32,109 +36,283 @@
}
}
async function loadBlacklistedTokens() {
isLoadingBlacklist = true
try {
blacklistedTokens = await AgentWorkersService.listBlacklistedAgentTokens({
includeExpired: true
})
} catch (error) {
sendUserToast('Error loading blacklisted tokens: ' + error.toString(), true)
blacklistedTokens = []
} finally {
isLoadingBlacklist = false
}
}
async function addToBlacklist() {
if (!blacklistToken.trim()) {
sendUserToast('Please enter a token to blacklist', true)
return
}
try {
await AgentWorkersService.blacklistAgentToken({
requestBody: {
token: blacklistToken
}
})
sendUserToast('Token successfully added to blacklist')
blacklistToken = ''
// Refresh the blacklist after adding a new token
await loadBlacklistedTokens()
} catch (error) {
sendUserToast('Error blacklisting token: ' + error.body, true)
}
}
async function removeFromBlacklist(tokenToRemove: string) {
try {
const response = await fetch('/api/agent_workers/remove_blacklist_token', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ token: tokenToRemove })
})
if (!response.ok) {
const errorText = await response.text()
throw new Error(errorText || 'Failed to remove token from blacklist')
}
sendUserToast('Token successfully removed from blacklist')
// Refresh the blacklist after removing a token
await loadBlacklistedTokens()
} catch (error) {
sendUserToast('Error removing token from blacklist: ' + error.toString(), true)
}
}
$effect(() => {
if (selectedTags.length > 0 && $superadmin) {
refreshToken(workerGroup, selectedTags)
}
})
$effect(() => {
if (selectedTab === 'blacklist' && $enterpriseLicense && $superadmin) {
loadBlacklistedTokens()
}
})
</script>
<div class="flex flex-col gap-y-4">
<Alert type="info" title="HTTP agent workers "
>Use HTTP agent workers only when the workers need to be deployed remotely OR with only HTTP
connectivity OR in untrusted environments. HTTP agent workers have more latency and less
capabilities than normal workers.</Alert
>
<Section
label="Worker group"
tooltip="This is only used to give a name prefix to the agent worker and to group workers in the workers page, no worker group config is passed to an agent worker."
>
<input class="max-w-md" type="text" bind:value={workerGroup} />
</Section>
<Section label="Tags to listen to" eeOnly>
{#if !$enterpriseLicense}
<div class="text-sm text-secondary mb-2 max-w-md">
Agent workers are only available in the enterprise edition. For evaluation purposes, you can
only use the tag `agent_test` tag and it is limited to 100 jobs.
</div>
{/if}
<TagsToListenTo disabled={!$enterpriseLicense} bind:worker_tags={selectedTags} {customTags} />
</Section>
<Tabs bind:selected={selectedTab}>
<Tab value="create">Create</Tab>
<Tab value="blacklist">Blacklist</Tab>
{#snippet content()}
<div class="flex flex-col gap-y-4 pt-2">
{#if selectedTab === 'create'}
<Alert type="info" title="HTTP agent workers "
>Use HTTP agent workers only when the workers need to be deployed remotely OR with only
HTTP connectivity OR in untrusted environments. HTTP agent workers have more latency and
less capabilities than normal workers.</Alert
>
<div class="flex flex-col gap-y-4 mt-4">
<Section
label="Worker group"
tooltip="This is only used to give a name prefix to the agent worker and to group workers in the workers page, no worker group config is passed to an agent worker."
>
<input class="max-w-md" type="text" bind:value={workerGroup} />
</Section>
<Section label="Tags to listen to" eeOnly>
{#if !$enterpriseLicense}
<div class="text-sm text-secondary mb-2 max-w-md">
Agent workers are only available in the enterprise edition. For evaluation purposes,
you can only use the tag `agent_test` tag and it is limited to 100 jobs.
</div>
{/if}
<TagsToListenTo
disabled={!$enterpriseLicense}
bind:worker_tags={selectedTags}
{customTags}
/>
</Section>
<Section label="Generated JWT token" primary>
{#if !$enterpriseLicense}
<div class="text-sm text-secondary mb-2 max-w-md">
Agent workers are only available in the enterprise edition. For evaluation purposes, you can
only use the tag `agent_test` tag and it is limited to 100 jobs.
</div>
{/if}
<div class="relative max-w-md group">
<!-- svelte-ignore event_directive_deprecated -->
<input
on:click|preventDefault|stopPropagation={() => {
if (token) {
navigator.clipboard.writeText(token)
sendUserToast('Copied to clipboard')
}
}}
placeholder="Select tags to generate a JWT token"
type="text"
disabled
value={token}
class="w-full pr-10 pl-3 py-2 text-sm text-gray-600 bg-gray-50 border border-gray-300 rounded-lg cursor-pointer hover:bg-gray-100 transition truncate"
/>
<Section label="Generated JWT token" primary>
{#if !$enterpriseLicense}
<div class="text-sm text-secondary mb-2 max-w-md">
Agent workers are only available in the enterprise edition. For evaluation purposes,
you can only use the tag `agent_test` tag and it is limited to 100 jobs.
</div>
{/if}
<div class="relative max-w-md group">
<input
onclick={(e) => {
e.preventDefault()
e.stopPropagation()
if (token) {
navigator.clipboard.writeText(token)
sendUserToast('Copied to clipboard')
}
}}
placeholder="Select tags to generate a JWT token"
type="text"
disabled
value={token}
class="w-full pr-10 pl-3 py-2 text-sm text-gray-600 bg-gray-50 border border-gray-300 rounded-lg cursor-pointer hover:bg-gray-100 transition truncatere"
/>
<!-- svelte-ignore event_directive_deprecated -->
<button
class="absolute right-2 top-1/2 -translate-y-1/2 text-gray-500 group-hover:text-blue-600 hover:scale-105 transition"
aria-label="Copy token to clipboard"
on:click|preventDefault|stopPropagation={() => {
if (token) {
navigator.clipboard.writeText(token)
sendUserToast('Copied to clipboard')
}
}}
>
<Copy size={18} />
</button>
</div>
<button
class="absolute right-2 top-1/2 -translate-y-1/2 text-gray-500 group-hover:text-blue-600 hover:scale-105 transition"
aria-label="Copy token to clipboard"
onclick={(e) => {
e.preventDefault()
e.stopPropagation()
if (token) {
navigator.clipboard.writeText(token)
sendUserToast('Copied to clipboard')
}
}}
>
<Copy size={18} />
</button>
</div>
<div class="flex flex-col gap-2 text-sm mt-3 leading-relaxed">
Set the following environment variables:
<ul class="list-disc list-inside mt-1">
<li><code>MODE=agent</code></li>
<li><code>AGENT_TOKEN=&lt;token&gt;</code></li>
<li><code>BASE_INTERNAL_URL=&lt;base url&gt;</code></li>
</ul>
<p class="text-sm leading-relaxed">
to a worker to have it act as an HTTP agent worker.
<code>INIT_SCRIPT</code>, if needed, must be passed as an env variable.
</p>
<Alert type="warning" size="sm" title="Agent Worker Limitations">
Ensure at least one normal worker is running and listening to the tags
<code>flow</code> and <code>dependency</code>
(or <code>flow-&lt;workspace&gt;</code> and <code>dependency-&lt;workspace&gt;</code> if
using workspace-specific default tags), because agent workers
<strong>cannot run dependency jobs</strong>
nor execute the
<strong>flow state machine</strong>. They can, however, run subjobs within flows.
</Alert>
<CollapseLink text="Automate JWT token generation" small>
<div class="text-xs mt-2">
Use the following API endpoint with a superadmin bearer token:
<code class="block mt-1 mb-2">POST /api/agent_workers/create_agent_token</code>
<pre class=" p-2 rounded-lg text-xs overflow-auto">
<div class="flex flex-col gap-2 text-sm mt-3 leading-relaxed">
Set the following environment variables:
<ul class="list-disc list-inside mt-1">
<li><code>MODE=agent</code></li>
<li><code>AGENT_TOKEN=&lt;token&gt;</code></li>
<li><code>BASE_INTERNAL_URL=&lt;base url&gt;</code></li>
</ul>
<p class="text-sm leading-relaxed">
to a worker to have it act as an HTTP agent worker.
<code>INIT_SCRIPT</code>, if needed, must be passed as an env variable.
</p>
<Alert type="warning" size="sm" title="Agent Worker Limitations">
Ensure at least one normal worker is running and listening to the tags
<code>flow</code> and <code>dependency</code>
(or <code>flow-&lt;workspace&gt;</code> and
<code>dependency-&lt;workspace&gt;</code>
if using workspace-specific default tags), because agent workers
<strong>cannot run dependency jobs</strong>
nor execute the
<strong>flow state machine</strong>. They can, however, run subjobs within flows.
</Alert>
<CollapseLink text="Automate JWT token generation" small>
<div class="text-xs mt-2">
Use the following API endpoint with a superadmin bearer token:
<code class="block mt-1 mb-2">POST /api/agent_workers/create_agent_token</code>
<pre class=" p-2 rounded-lg text-xs overflow-auto">
<code
>{`
>{`
"worker_group": "agent",
"tags": ["tag1", "tag2"],
"exp": 1717334400
`}</code
>
</pre>
The JSON response will contain the generated JWT token.
>
</pre>
The JSON response will contain the generated JWT token.
</div>
</CollapseLink>
</div>
</Section>
</div>
</CollapseLink>
{:else if selectedTab === 'blacklist'}
<div class="flex flex-col gap-y-4 mt-4">
<Section label="Agent Token Blacklist" eeOnly>
{#if !$enterpriseLicense}
<div class="text-sm text-secondary mb-2 max-w-md">
Token blacklist management is only available in the enterprise edition.
</div>
{:else}
<div class="text-sm text-secondary mb-4 max-w-md">
Add tokens to the blacklist to prevent them from being used by agent workers.
Blacklisted tokens may take up to 5 minutes to be effective because of caching.
</div>
<div class="flex flex-col gap-3 w-full mb-6">
<div>
<label class="block text-sm font-medium mb-1" for="blacklistTokenInput"
>Token</label
>
<input
id="blacklistTokenInput"
class="w-full"
type="text"
bind:value={blacklistToken}
placeholder="jwt_agent_eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ3b3JrZXJfZ3JvdXAiOiJhZ2VudCIsInN1ZmZpeCI6bnVsbCwidGFncyI6WyJiYXNoIl0sImV4cCI6MTg0NDk1NDYxMX0.JQWb-_ERGaomukbl_cEPPmmCAEepTR79d9oIrKREscE"
/>
</div>
<div class="flex">
<Button color="red" on:click={addToBlacklist} disabled={!$superadmin}
>Blacklist</Button
>
</div>
{#if !$superadmin}
<div class="text-xs text-amber-600">
Only superadmins can manage the token blacklist.
</div>
{/if}
</div>
<!-- Blacklisted Tokens List -->
<div class="border-t pt-6">
<div class="flex items-center justify-between mb-4">
<h3 class="text-lg font-medium">Blacklisted Tokens</h3>
<button
class="p-2 text-gray-500 hover:text-blue-600 hover:bg-gray-100 rounded-lg transition"
onclick={loadBlacklistedTokens}
disabled={isLoadingBlacklist}
title="Refresh blacklist"
>
<RefreshCw size={16} class={isLoadingBlacklist ? 'animate-spin' : ''} />
</button>
</div>
{#if isLoadingBlacklist}
<div class="text-center py-4 text-gray-500"> Loading blacklisted tokens... </div>
{:else if blacklistedTokens?.length === 0}
<div class="text-center py-4 text-gray-500">
No tokens are currently blacklisted.
</div>
{:else}
<div class="space-y-2">
{#each blacklistedTokens ?? [] as blacklistedToken}
<div
class="flex items-center justify-between p-3 bg-gray-50 rounded-lg border"
>
<div class="flex-1 min-w-0">
<div class="font-mono text-xs text-gray-700 pr-4 break-all">
{blacklistedToken.token}
</div>
{#if blacklistedToken.expires_at}
<div class="text-xs text-gray-500 mt-1">
Expires: {new Date(blacklistedToken.expires_at).toLocaleString()}
</div>
{/if}
</div>
{#if $superadmin}
<button
class="ml-3 p-2 text-red-600 hover:text-red-800 hover:bg-red-50 rounded-lg transition"
onclick={() => removeFromBlacklist(blacklistedToken.token)}
title="Remove from blacklist"
>
<Trash2 size={16} />
</button>
{/if}
</div>
{/each}
</div>
{/if}
</div>
{/if}
</Section>
</div>
{/if}
</div>
</Section>
</div>
{/snippet}
</Tabs>