Files
codex-proxy-rs/backend/crates/gateway-host/tests/workers.rs
T
ef11b72d8a fix(workers): recover daemon health after successful work (#499)
Report per-run daemon progress to reset health failures and restart backoff
after successful work, with stale-run isolation and recovery regression tests.

Co-authored-by: Zyy <zyycn@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 11:08:15 +08:00

1210 lines
40 KiB
Rust

//! 验证 Worker 注册、任务组合、禁用原因与监督生命周期
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::future::pending;
use std::num::NonZeroU64;
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use futures::future::BoxFuture;
use gateway_core::health::{WorkerHealthKey, WorkerHealthSnapshot, WorkerRuntimeState};
use gateway_core::lifecycle::CancellationToken;
use gateway_core::task::{
DaemonRestartPolicy, DaemonTask, ScheduledTask, WorkerContribution, WorkerCycleContext,
WorkerDaemonContext, WorkerDisabledReason, WorkerFencingToken, WorkerId, WorkerKind,
WorkerLeaderLeaseGuard, WorkerLeaderLeasePort, WorkerLeaseAcquisition, WorkerLeaseError,
WorkerLeaseRequest, WorkerRegistration, WorkerRunnable, WorkerSchedule, WorkerTaskError,
};
use gateway_host::workers::{WorkerStartError, WorkerSupervisor};
use tokio::sync::Notify;
const ACTIVE_KINDS: [WorkerKind; 11] = [
WorkerKind::OAuthRefresh,
WorkerKind::QuotaCatalogHealth,
WorkerKind::RuntimeSnapshotReconciliation,
WorkerKind::RuntimeChangeSubscription,
WorkerKind::StaleModelRequestRecovery,
WorkerKind::RequestAnalytics,
WorkerKind::Retention,
WorkerKind::Backup,
WorkerKind::AccountImport,
WorkerKind::AccountFreezeRecovery,
WorkerKind::ResetCreditAutomation,
];
#[derive(Clone)]
struct CountingTask {
runs: Arc<AtomicUsize>,
fail_until: usize,
}
impl CountingTask {
fn new(fail_until: usize) -> Self {
Self {
runs: Arc::new(AtomicUsize::new(0)),
fail_until,
}
}
}
impl ScheduledTask for CountingTask {
fn run_cycle(
&self,
_context: WorkerCycleContext,
) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
Box::pin(async move {
let run = self.runs.fetch_add(1, Ordering::SeqCst) + 1;
if run <= self.fail_until {
Err(WorkerTaskError::safe("expected test failure")
.with_source(std::io::Error::other("PRIVATE_WORKER_NATIVE_CAUSE")))
} else {
Ok(())
}
})
}
}
#[derive(Clone)]
struct PanicOnceTask {
runs: Arc<AtomicUsize>,
}
impl ScheduledTask for PanicOnceTask {
fn run_cycle(
&self,
_context: WorkerCycleContext,
) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
Box::pin(async move {
if self.runs.fetch_add(1, Ordering::SeqCst) == 0 {
panic!("expected isolated worker panic");
}
Ok(())
})
}
}
#[derive(Clone)]
struct CancellationTask {
entered: Arc<AtomicUsize>,
observed: Arc<AtomicUsize>,
}
impl ScheduledTask for CancellationTask {
fn run_cycle(&self, context: WorkerCycleContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
Box::pin(async move {
self.entered.fetch_add(1, Ordering::SeqCst);
context.cancellation().cancelled().await;
self.observed.fetch_add(1, Ordering::SeqCst);
Ok(())
})
}
}
#[derive(Clone)]
struct HungTask {
entered: Arc<AtomicBool>,
notification: Arc<Notify>,
}
impl ScheduledTask for HungTask {
fn run_cycle(
&self,
_context: WorkerCycleContext,
) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
Box::pin(async move {
self.entered.store(true, Ordering::SeqCst);
self.notification.notify_one();
pending::<()>().await;
Ok(())
})
}
}
#[derive(Clone)]
struct SucceedsThenHangsTask {
runs: Arc<AtomicUsize>,
second_cycle: Arc<Notify>,
}
impl ScheduledTask for SucceedsThenHangsTask {
fn run_cycle(
&self,
_context: WorkerCycleContext,
) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
Box::pin(async move {
if self.runs.fetch_add(1, Ordering::SeqCst) == 0 {
return Ok(());
}
self.second_cycle.notify_one();
pending::<()>().await;
Ok(())
})
}
}
#[derive(Clone)]
struct LeaseAwareTask {
entered: Arc<Notify>,
release: Arc<Notify>,
completed: Arc<AtomicBool>,
saw_live_guard: Arc<AtomicBool>,
expected_worker: WorkerId,
active_guards: Arc<Mutex<BTreeSet<WorkerId>>>,
fence: Arc<AtomicU64>,
}
impl ScheduledTask for LeaseAwareTask {
fn run_cycle(&self, context: WorkerCycleContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
Box::pin(async move {
self.saw_live_guard.store(
lock_unpoisoned(&self.active_guards).contains(context.worker()),
Ordering::SeqCst,
);
if context.worker() != &self.expected_worker {
return Err(WorkerTaskError::safe("unexpected worker identity"));
}
self.fence.store(
context
.fencing_token()
.expect("leased task fencing token")
.get(),
Ordering::SeqCst,
);
self.entered.notify_one();
self.release.notified().await;
self.completed.store(true, Ordering::SeqCst);
Ok(())
})
}
}
#[derive(Clone)]
struct ConcurrentCycleTask {
starts: Arc<AtomicUsize>,
active: Arc<AtomicUsize>,
maximum_active: Arc<AtomicUsize>,
}
impl ScheduledTask for ConcurrentCycleTask {
fn run_cycle(&self, context: WorkerCycleContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
Box::pin(async move {
self.starts.fetch_add(1, Ordering::SeqCst);
let active = self.active.fetch_add(1, Ordering::SeqCst) + 1;
self.maximum_active.fetch_max(active, Ordering::SeqCst);
let _active = ActiveCycle(Arc::clone(&self.active));
context.cancellation().cancelled().await;
Ok(())
})
}
}
struct ActiveCycle(Arc<AtomicUsize>);
impl Drop for ActiveCycle {
fn drop(&mut self) {
self.0.fetch_sub(1, Ordering::SeqCst);
}
}
#[derive(Debug, Clone)]
enum LeaseStep {
Acquired,
Busy(Option<Duration>),
Error,
Panic,
}
#[derive(Debug, Clone, Copy)]
enum RenewStep {
Success,
Error,
}
#[derive(Default)]
struct FakeLeasePort {
scripts: Mutex<BTreeMap<WorkerId, VecDeque<LeaseStep>>>,
renew_scripts: Arc<Mutex<BTreeMap<WorkerId, VecDeque<RenewStep>>>>,
requests: Mutex<Vec<WorkerLeaseRequest>>,
renewals: Arc<Mutex<Vec<WorkerId>>>,
active_guards: Arc<Mutex<BTreeSet<WorkerId>>>,
next_fence: AtomicU64,
}
impl FakeLeasePort {
fn script(&self, worker: WorkerId, steps: impl IntoIterator<Item = LeaseStep>) {
lock_unpoisoned(&self.scripts).insert(worker, steps.into_iter().collect());
}
fn renew_script(&self, worker: WorkerId, steps: impl IntoIterator<Item = RenewStep>) {
lock_unpoisoned(&self.renew_scripts).insert(worker, steps.into_iter().collect());
}
fn requests(&self) -> Vec<WorkerLeaseRequest> {
lock_unpoisoned(&self.requests).clone()
}
fn renewal_count(&self, worker: &WorkerId) -> usize {
lock_unpoisoned(&self.renewals)
.iter()
.filter(|renewed| *renewed == worker)
.count()
}
}
struct FakeLeaseGuard {
worker: WorkerId,
fencing_token: WorkerFencingToken,
active_guards: Arc<Mutex<BTreeSet<WorkerId>>>,
renew_scripts: Arc<Mutex<BTreeMap<WorkerId, VecDeque<RenewStep>>>>,
renewals: Arc<Mutex<Vec<WorkerId>>>,
}
impl WorkerLeaderLeaseGuard for FakeLeaseGuard {
fn fencing_token(&self) -> WorkerFencingToken {
self.fencing_token
}
fn renew(&mut self) -> BoxFuture<'_, Result<(), WorkerLeaseError>> {
Box::pin(async move {
lock_unpoisoned(&self.renewals).push(self.worker.clone());
match lock_unpoisoned(&self.renew_scripts)
.get_mut(&self.worker)
.and_then(VecDeque::pop_front)
.unwrap_or(RenewStep::Success)
{
RenewStep::Success => Ok(()),
RenewStep::Error => Err(WorkerLeaseError::safe("expected renewal failure")),
}
})
}
fn release(self: Box<Self>) -> BoxFuture<'static, Result<(), WorkerLeaseError>> {
Box::pin(async move {
drop(self);
Ok(())
})
}
}
impl Drop for FakeLeaseGuard {
fn drop(&mut self) {
lock_unpoisoned(&self.active_guards).remove(&self.worker);
}
}
impl WorkerLeaderLeasePort for FakeLeasePort {
fn try_acquire(
&self,
request: WorkerLeaseRequest,
) -> BoxFuture<'_, Result<WorkerLeaseAcquisition, WorkerLeaseError>> {
Box::pin(async move {
lock_unpoisoned(&self.requests).push(request.clone());
let step = lock_unpoisoned(&self.scripts)
.get_mut(request.worker())
.and_then(VecDeque::pop_front)
.unwrap_or(LeaseStep::Acquired);
match step {
LeaseStep::Acquired => {
let fence = self.next_fence.fetch_add(1, Ordering::SeqCst) + 1;
lock_unpoisoned(&self.active_guards).insert(request.worker().clone());
Ok(WorkerLeaseAcquisition::Acquired(Box::new(FakeLeaseGuard {
worker: request.worker().clone(),
fencing_token: WorkerFencingToken::new(
NonZeroU64::new(fence).expect("positive fence"),
),
active_guards: Arc::clone(&self.active_guards),
renew_scripts: Arc::clone(&self.renew_scripts),
renewals: Arc::clone(&self.renewals),
})))
}
LeaseStep::Busy(retry_after) => Ok(WorkerLeaseAcquisition::Busy { retry_after }),
LeaseStep::Error => Err(WorkerLeaseError::safe("expected lease failure")),
LeaseStep::Panic => panic!("expected isolated lease panic"),
}
})
}
}
#[tokio::test]
async fn registry_composes_multiple_owners_but_rejects_duplicate_identity() {
let duplicate = registration(
WorkerKind::OAuthRefresh,
"openai",
CountingTask::new(0),
test_schedule(),
true,
);
let mut plan = complete_plan(vec![registration(
WorkerKind::OAuthRefresh,
"openai",
CountingTask::new(0),
test_schedule(),
true,
)]);
plan.push(duplicate);
let supervisor = supervisor();
assert!(matches!(
supervisor.start(plan, Arc::new(FakeLeasePort::default()),
Arc::new(RecordingDiagnostics::default()),
),
Err(WorkerStartError::Duplicate(id)) if id.owner() == "openai"
));
}
#[tokio::test]
async fn only_database_fact_reasons_can_disable_nonexistent_workers() {
let mut plan = complete_plan(Vec::new());
plan.push(WorkerContribution::Registration(registration_value(
WorkerKind::OpsFlush,
"store",
CountingTask::new(0),
test_schedule(),
true,
)));
assert!(matches!(
supervisor().start(
plan,
Arc::new(FakeLeasePort::default()),
Arc::new(RecordingDiagnostics::default()),
),
Err(WorkerStartError::KindDisabled(WorkerKind::OpsFlush))
));
}
#[tokio::test]
async fn registry_refuses_start_when_any_real_owner_kind_is_missing() {
let plan = complete_plan(Vec::new())
.into_iter()
.filter(|item| item.kind() != WorkerKind::Retention)
.collect();
assert!(matches!(
supervisor().start(plan, Arc::new(FakeLeasePort::default()),
Arc::new(RecordingDiagnostics::default()),
),
Err(WorkerStartError::Missing(missing)) if missing == vec![WorkerKind::Retention]
));
}
#[tokio::test]
async fn disabled_database_facts_never_execute_or_acquire_a_lease() {
let leases = Arc::new(FakeLeasePort::default());
let supervisor = supervisor();
supervisor
.start(
complete_plan(Vec::new()),
leases.clone(),
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
wait_until(|| leases.requests().len() >= ACTIVE_KINDS.len()).await;
let snapshot = supervisor.health_source().snapshot();
assert!(snapshot.iter().any(|state| {
state.key == WorkerHealthKey::Disabled(WorkerKind::OpsFlush)
&& state.state == WorkerRuntimeState::Disabled
}));
assert!(
leases
.requests()
.iter()
.all(|request| { request.worker().kind() != WorkerKind::OpsFlush })
);
supervisor.shutdown(Duration::from_secs(1)).await;
}
#[tokio::test]
async fn one_kind_runs_all_registered_owner_tasks() {
let second = CountingTask::new(0);
let plan = complete_plan(vec![
registration(
WorkerKind::OAuthRefresh,
"openai",
CountingTask::new(0),
test_schedule(),
true,
),
registration(
WorkerKind::OAuthRefresh,
"xai",
second.clone(),
test_schedule(),
true,
),
]);
let supervisor = supervisor();
supervisor
.start(
plan,
Arc::new(FakeLeasePort::default()),
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
wait_until(|| second.runs.load(Ordering::SeqCst) >= 1).await;
assert_eq!(
supervisor
.health_source()
.snapshot()
.iter()
.filter(|state| matches!(&state.key, WorkerHealthKey::Task(id) if id.kind() == WorkerKind::OAuthRefresh))
.count(),
2
);
supervisor.shutdown(Duration::from_secs(1)).await;
}
#[tokio::test]
async fn leader_guard_lives_through_cycle_and_fence_reaches_owner() {
let worker = target_worker();
let leases = Arc::new(FakeLeasePort::default());
let task = LeaseAwareTask {
entered: Arc::new(Notify::new()),
release: Arc::new(Notify::new()),
completed: Arc::new(AtomicBool::new(false)),
saw_live_guard: Arc::new(AtomicBool::new(false)),
expected_worker: worker.clone(),
active_guards: Arc::clone(&leases.active_guards),
fence: Arc::new(AtomicU64::new(0)),
};
let supervisor = supervisor();
supervisor
.start(
target_plan(task.clone(), long_interval_schedule()),
leases.clone(),
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
task.entered.notified().await;
assert!(task.saw_live_guard.load(Ordering::SeqCst));
assert!(lock_unpoisoned(&leases.active_guards).contains(&worker));
task.release.notify_one();
wait_until(|| task.completed.load(Ordering::SeqCst)).await;
wait_until(|| !lock_unpoisoned(&leases.active_guards).contains(&worker)).await;
assert!(task.fence.load(Ordering::SeqCst) > 0);
supervisor.shutdown(Duration::from_secs(1)).await;
}
#[tokio::test(start_paused = true)]
async fn long_cycle_renews_leader_lease_before_ttl() {
let leases = Arc::new(FakeLeasePort::default());
let task = lease_aware_task(&leases);
let supervisor = supervisor();
supervisor
.start(
target_plan(task.clone(), renewal_schedule()),
leases.clone(),
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
task.entered.notified().await;
tokio::time::advance(Duration::from_millis(10)).await;
yield_until(|| leases.renewal_count(&target_worker()) >= 1).await;
assert!(lock_unpoisoned(&leases.active_guards).contains(&target_worker()));
task.release.notify_one();
supervisor.shutdown(Duration::from_secs(1)).await;
}
#[tokio::test(start_paused = true)]
async fn renewal_failure_cancels_old_cycle_before_another_leader_cycle_starts() {
let task = ConcurrentCycleTask {
starts: Arc::new(AtomicUsize::new(0)),
active: Arc::new(AtomicUsize::new(0)),
maximum_active: Arc::new(AtomicUsize::new(0)),
};
let leases = Arc::new(FakeLeasePort::default());
leases.renew_script(target_worker(), [RenewStep::Error]);
let supervisor = supervisor();
supervisor
.start(
target_plan(task.clone(), renewal_schedule()),
leases,
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
yield_until(|| task.starts.load(Ordering::SeqCst) == 1).await;
tokio::time::advance(Duration::from_millis(10)).await;
yield_until(|| task.active.load(Ordering::SeqCst) == 0).await;
tokio::time::advance(Duration::from_millis(5)).await;
yield_until(|| task.starts.load(Ordering::SeqCst) == 2).await;
assert_eq!(task.maximum_active.load(Ordering::SeqCst), 1);
supervisor.shutdown(Duration::from_secs(1)).await;
}
#[tokio::test(start_paused = true)]
async fn supervisor_uses_exponential_backoff_and_recovers_health() {
let task = CountingTask::new(2);
let supervisor = supervisor();
supervisor
.start(
target_plan(task.clone(), backoff_schedule()),
Arc::new(FakeLeasePort::default()),
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
yield_until(|| task.runs.load(Ordering::SeqCst) == 1).await;
tokio::time::advance(Duration::from_millis(10)).await;
yield_until(|| task.runs.load(Ordering::SeqCst) == 2).await;
tokio::time::advance(Duration::from_millis(20)).await;
yield_until(|| task.runs.load(Ordering::SeqCst) == 3).await;
let health = task_health(&supervisor, &target_worker());
assert_eq!(
(health.consecutive_failures, health.completed_cycles),
(0, 1)
);
supervisor.shutdown(Duration::from_secs(1)).await;
}
#[tokio::test(start_paused = true)]
async fn task_and_lease_port_panics_are_isolated_and_retried() {
let task = PanicOnceTask {
runs: Arc::new(AtomicUsize::new(0)),
};
let leases = Arc::new(FakeLeasePort::default());
leases.script(
target_worker(),
[LeaseStep::Error, LeaseStep::Panic, LeaseStep::Acquired],
);
let supervisor = supervisor();
supervisor
.start(
target_plan(task.clone(), backoff_schedule()),
leases,
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_millis(10)).await;
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_millis(20)).await;
yield_until(|| task.runs.load(Ordering::SeqCst) == 1).await;
tokio::time::advance(Duration::from_millis(20)).await;
yield_until(|| task.runs.load(Ordering::SeqCst) >= 2).await;
assert_eq!(
task_health(&supervisor, &target_worker()).consecutive_failures,
0
);
supervisor.shutdown(Duration::from_secs(1)).await;
}
#[tokio::test(start_paused = true)]
async fn lease_busy_is_healthy_standby_and_retries_without_running_owner() {
let task = CountingTask::new(0);
let leases = Arc::new(FakeLeasePort::default());
leases.script(
target_worker(),
[
LeaseStep::Busy(Some(Duration::from_millis(7))),
LeaseStep::Acquired,
],
);
let supervisor = supervisor();
supervisor
.start(
target_plan(task.clone(), backoff_schedule()),
leases,
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
yield_until(|| task_health(&supervisor, &target_worker()).state == WorkerRuntimeState::Standby)
.await;
assert_eq!(task.runs.load(Ordering::SeqCst), 0);
tokio::time::advance(Duration::from_millis(7)).await;
yield_until(|| task.runs.load(Ordering::SeqCst) == 1).await;
supervisor.shutdown(Duration::from_secs(1)).await;
}
#[tokio::test]
async fn cooperative_shutdown_propagates_cancel_before_dropping_all_guards() {
let task = CancellationTask {
entered: Arc::new(AtomicUsize::new(0)),
observed: Arc::new(AtomicUsize::new(0)),
};
let leases = Arc::new(FakeLeasePort::default());
let supervisor = supervisor();
supervisor
.start(
all_active_plan(task.clone()),
leases.clone(),
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
wait_until(|| task.entered.load(Ordering::SeqCst) == ACTIVE_KINDS.len()).await;
supervisor.shutdown(Duration::from_secs(1)).await;
assert_eq!(task.observed.load(Ordering::SeqCst), ACTIVE_KINDS.len());
assert!(lock_unpoisoned(&leases.active_guards).is_empty());
}
#[tokio::test]
async fn shutdown_timeout_aborts_uncooperative_task_and_drops_guard() {
let task = HungTask {
entered: Arc::new(AtomicBool::new(false)),
notification: Arc::new(Notify::new()),
};
let leases = Arc::new(FakeLeasePort::default());
let supervisor = supervisor();
supervisor
.start(
target_plan(task.clone(), long_interval_schedule()),
leases.clone(),
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
task.notification.notified().await;
supervisor.shutdown(Duration::from_millis(10)).await;
assert!(lock_unpoisoned(&leases.active_guards).is_empty());
assert_eq!(
task_health(&supervisor, &target_worker()).state,
WorkerRuntimeState::Stopped
);
}
#[tokio::test]
async fn dropping_shutdown_aborts_current_and_unjoined_workers() {
let task = HungTask {
entered: Arc::new(AtomicBool::new(false)),
notification: Arc::new(Notify::new()),
};
let leases = Arc::new(FakeLeasePort::default());
let supervisor = supervisor();
supervisor
.start(
all_active_plan(task),
leases.clone(),
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
wait_until(|| lock_unpoisoned(&leases.active_guards).len() == ACTIVE_KINDS.len()).await;
let mut shutdown = Box::pin(supervisor.shutdown(Duration::from_secs(60)));
assert!(futures::poll!(&mut shutdown).is_pending());
drop(shutdown);
// Supervisor 仍存活,shutdown 自己必须回收当前等待和尚未轮到的所有任务
wait_until(|| lock_unpoisoned(&leases.active_guards).is_empty()).await;
assert!(
supervisor
.health_source()
.snapshot()
.into_iter()
.all(|entry| matches!(
entry.state,
WorkerRuntimeState::Stopped | WorkerRuntimeState::Disabled
))
);
}
#[tokio::test]
async fn shutdown_timeout_must_not_rejoin_workers_that_already_exited() {
// 挂死任务放在 join 顺序最末的 kind(Retention),保证超时触发时
// 排在它前面的合作任务句柄都已被 join 过一次
let hung = HungTask {
entered: Arc::new(AtomicBool::new(false)),
notification: Arc::new(Notify::new()),
};
let cooperative = CancellationTask {
entered: Arc::new(AtomicUsize::new(0)),
observed: Arc::new(AtomicUsize::new(0)),
};
let mut plan = ACTIVE_KINDS
.into_iter()
.map(|kind| {
if kind == WorkerKind::Retention {
registration(kind, kind.as_str(), hung.clone(), test_schedule(), true)
} else {
registration(
kind,
kind.as_str(),
cooperative.clone(),
test_schedule(),
true,
)
}
})
.collect::<Vec<_>>();
plan.push(WorkerContribution::Disabled {
kind: WorkerKind::OpsFlush,
reason: WorkerDisabledReason::NoBufferedOpsEvents,
});
let leases = Arc::new(FakeLeasePort::default());
let supervisor = supervisor();
supervisor
.start(
plan,
leases.clone(),
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
hung.notification.notified().await;
wait_until(|| cooperative.entered.load(Ordering::SeqCst) == ACTIVE_KINDS.len() - 1).await;
supervisor.shutdown(Duration::from_millis(100)).await;
assert_eq!(
cooperative.observed.load(Ordering::SeqCst),
ACTIVE_KINDS.len() - 1
);
assert!(lock_unpoisoned(&leases.active_guards).is_empty());
}
#[tokio::test]
async fn running_worker_becomes_unhealthy_when_last_success_is_stale() {
let task = SucceedsThenHangsTask {
runs: Arc::new(AtomicUsize::new(0)),
second_cycle: Arc::new(Notify::new()),
};
let supervisor = supervisor();
supervisor
.start(
target_plan(task.clone(), freshness_schedule()),
Arc::new(FakeLeasePort::default()),
Arc::new(RecordingDiagnostics::default()),
)
.expect("complete plan");
task.second_cycle.notified().await;
tokio::time::sleep(Duration::from_millis(45)).await;
assert_eq!(
task_health(&supervisor, &target_worker()).state,
WorkerRuntimeState::BackingOff
);
supervisor.shutdown(Duration::from_millis(10)).await;
}
fn supervisor() -> WorkerSupervisor {
WorkerSupervisor::new(CancellationToken::new())
}
fn registration(
kind: WorkerKind,
owner: &str,
task: impl ScheduledTask + 'static,
schedule: WorkerSchedule,
leased: bool,
) -> WorkerContribution {
WorkerContribution::Registration(registration_value(kind, owner, task, schedule, leased))
}
fn registration_value(
kind: WorkerKind,
owner: &str,
task: impl ScheduledTask + 'static,
schedule: WorkerSchedule,
leased: bool,
) -> WorkerRegistration {
let id = WorkerId::try_new(kind, owner).expect("valid worker id");
let lease = leased
.then(|| WorkerLeaseRequest::try_new(id.clone(), schedule.leader_lease_ttl()))
.transpose()
.expect("valid lease");
WorkerRegistration::try_new(
id,
WorkerRunnable::Scheduled {
schedule,
lease,
task: Box::new(task),
},
)
.expect("valid registration")
}
fn complete_plan(mut custom: Vec<WorkerContribution>) -> Vec<WorkerContribution> {
let present = custom
.iter()
.map(WorkerContribution::kind)
.collect::<BTreeSet<_>>();
for kind in ACTIVE_KINDS {
if !present.contains(&kind) {
custom.push(registration(
kind,
kind.as_str(),
CountingTask::new(0),
long_interval_schedule(),
true,
));
}
}
custom.push(WorkerContribution::Disabled {
kind: WorkerKind::OpsFlush,
reason: WorkerDisabledReason::NoBufferedOpsEvents,
});
custom
}
fn target_plan(
task: impl ScheduledTask + 'static,
schedule: WorkerSchedule,
) -> Vec<WorkerContribution> {
complete_plan(vec![registration(
WorkerKind::OAuthRefresh,
"openai",
task,
schedule,
true,
)])
}
fn all_active_plan(task: impl ScheduledTask + Clone + 'static) -> Vec<WorkerContribution> {
let mut plan = ACTIVE_KINDS
.into_iter()
.map(|kind| registration(kind, kind.as_str(), task.clone(), test_schedule(), true))
.collect::<Vec<_>>();
plan.push(WorkerContribution::Disabled {
kind: WorkerKind::OpsFlush,
reason: WorkerDisabledReason::NoBufferedOpsEvents,
});
plan
}
fn lease_aware_task(leases: &Arc<FakeLeasePort>) -> LeaseAwareTask {
LeaseAwareTask {
entered: Arc::new(Notify::new()),
release: Arc::new(Notify::new()),
completed: Arc::new(AtomicBool::new(false)),
saw_live_guard: Arc::new(AtomicBool::new(false)),
expected_worker: target_worker(),
active_guards: Arc::clone(&leases.active_guards),
fence: Arc::new(AtomicU64::new(0)),
}
}
fn target_worker() -> WorkerId {
WorkerId::try_new(WorkerKind::OAuthRefresh, "openai").expect("valid target worker")
}
fn task_health(supervisor: &WorkerSupervisor, id: &WorkerId) -> WorkerHealthSnapshot {
supervisor
.health_source()
.snapshot()
.into_iter()
.find(|state| state.key == WorkerHealthKey::Task(id.clone()))
.expect("worker health")
}
fn schedule(
interval: Duration,
initial: Duration,
maximum: Duration,
ttl: Duration,
renewal: Duration,
) -> WorkerSchedule {
WorkerSchedule::try_new(interval, initial, maximum, ttl, renewal).expect("valid schedule")
}
fn test_schedule() -> WorkerSchedule {
schedule(
Duration::from_millis(20),
Duration::from_millis(5),
Duration::from_millis(20),
Duration::from_secs(30),
Duration::from_secs(10),
)
}
fn long_interval_schedule() -> WorkerSchedule {
schedule(
Duration::from_secs(60),
Duration::from_millis(5),
Duration::from_millis(20),
Duration::from_secs(30),
Duration::from_secs(10),
)
}
fn backoff_schedule() -> WorkerSchedule {
schedule(
Duration::from_secs(60),
Duration::from_millis(10),
Duration::from_millis(20),
Duration::from_secs(30),
Duration::from_secs(10),
)
}
fn renewal_schedule() -> WorkerSchedule {
schedule(
Duration::from_secs(60),
Duration::from_millis(5),
Duration::from_millis(20),
Duration::from_millis(30),
Duration::from_millis(10),
)
}
fn freshness_schedule() -> WorkerSchedule {
schedule(
Duration::from_millis(10),
Duration::from_millis(5),
Duration::from_millis(10),
Duration::from_millis(30),
Duration::from_millis(10),
)
}
async fn wait_until(mut condition: impl FnMut() -> bool) {
tokio::time::timeout(Duration::from_secs(1), async {
while !condition() {
tokio::task::yield_now().await;
}
})
.await
.expect("worker test timeout");
}
async fn yield_until(mut condition: impl FnMut() -> bool) {
for _ in 0..100 {
if condition() {
return;
}
tokio::task::yield_now().await;
}
assert!(condition(), "condition did not become true after yielding");
}
fn lock_unpoisoned<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
mutex
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
#[derive(Default)]
struct RecordingDiagnostics(std::sync::Mutex<Vec<gateway_core::diagnostics::OperationalFailure>>);
#[async_trait::async_trait]
impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics {
async fn record_failure(
&self,
failure: gateway_core::diagnostics::OperationalFailure,
) -> Result<(), gateway_core::error::StoreError> {
self.0.lock().unwrap().push(failure);
Ok(())
}
}
#[tokio::test]
async fn worker_boundary_records_one_native_failure_and_recovers_health() {
let supervisor = supervisor();
let task = CountingTask::new(1);
let diagnostics = Arc::new(RecordingDiagnostics::default());
supervisor
.start(
target_plan(task.clone(), test_schedule()),
Arc::new(FakeLeasePort::default()),
diagnostics.clone(),
)
.unwrap();
wait_until(|| task.runs.load(Ordering::SeqCst) >= 2).await;
supervisor.shutdown(Duration::from_secs(1)).await;
let failures = diagnostics.0.lock().unwrap();
assert_eq!(failures.len(), 1);
assert_eq!(
failures[0].correlation_id.as_deref(),
Some(target_worker().to_string().as_str())
);
assert!(
failures[0]
.details
.as_ref()
.unwrap()
.as_str()
.contains("PRIVATE_WORKER_NATIVE_CAUSE")
);
assert!(
!format!("{:?}", supervisor.health_source().snapshot())
.contains("PRIVATE_WORKER_NATIVE_CAUSE")
);
}
// 每次运行保存自己的上下文,测试可模拟已退出任务的迟到通知
struct ControlledDaemon {
contexts: Arc<Mutex<Vec<WorkerDaemonContext>>>,
commands: tokio::sync::Mutex<tokio::sync::mpsc::UnboundedReceiver<DaemonCommand>>,
}
enum DaemonCommand {
Succeed,
Fail,
SucceedThenFail,
Panic,
Exit,
}
impl DaemonTask for ControlledDaemon {
fn run(&self, context: WorkerDaemonContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
Box::pin(async move {
lock_unpoisoned(&self.contexts).push(context.clone());
let mut commands = self.commands.lock().await;
loop {
let command = tokio::select! {
() = context.cancellation().cancelled() => return Ok(()),
command = commands.recv() => command.expect("daemon command"),
};
match command {
DaemonCommand::Succeed => context.succeeded(),
DaemonCommand::SucceedThenFail => {
context.succeeded();
return Err(WorkerTaskError::safe("test daemon failure"));
}
DaemonCommand::Fail => {
return Err(WorkerTaskError::safe("test daemon failure"));
}
DaemonCommand::Panic => panic!("test daemon panic"),
DaemonCommand::Exit => return Ok(()),
}
}
})
}
}
fn daemon_plan(task: impl DaemonTask + 'static) -> Vec<WorkerContribution> {
complete_plan(vec![WorkerContribution::Registration(
WorkerRegistration::try_new(
WorkerId::try_new(WorkerKind::Backup, "backup").unwrap(),
WorkerRunnable::Daemon {
restart: DaemonRestartPolicy::try_new(
Duration::from_millis(10),
Duration::from_millis(20),
)
.unwrap(),
task: Box::new(task),
},
)
.unwrap(),
)])
}
#[tokio::test(start_paused = true)]
async fn daemon_recovers_only_after_current_run_succeeds_and_resets_backoff() {
let (commands, receiver) = tokio::sync::mpsc::unbounded_channel();
let contexts = Arc::new(Mutex::new(Vec::new()));
let supervisor = supervisor();
let id = WorkerId::try_new(WorkerKind::Backup, "backup").unwrap();
supervisor
.start(
daemon_plan(ControlledDaemon {
contexts: contexts.clone(),
commands: tokio::sync::Mutex::new(receiver),
}),
Arc::new(FakeLeasePort::default()),
Arc::new(RecordingDiagnostics::default()),
)
.unwrap();
yield_until(|| lock_unpoisoned(&contexts).len() == 1).await;
// 连续失败逐步退避;仅重新进入 run 不能恢复健康
for (attempt, delay) in [(1, 10), (2, 20)] {
commands.send(DaemonCommand::Fail).unwrap();
yield_until(|| task_health(&supervisor, &id).state == WorkerRuntimeState::BackingOff).await;
assert_eq!(task_health(&supervisor, &id).consecutive_failures, attempt);
tokio::time::advance(Duration::from_millis(delay - 1)).await;
assert_eq!(lock_unpoisoned(&contexts).len(), attempt as usize);
tokio::time::advance(Duration::from_millis(1)).await;
yield_until(|| lock_unpoisoned(&contexts).len() == attempt as usize + 1).await;
assert_eq!(task_health(&supervisor, &id).consecutive_failures, attempt);
}
lock_unpoisoned(&contexts)[0].succeeded();
tokio::task::yield_now().await;
assert_eq!(task_health(&supervisor, &id).consecutive_failures, 2);
commands.send(DaemonCommand::Succeed).unwrap();
yield_until(|| task_health(&supervisor, &id).consecutive_failures == 0).await;
let recovered = task_health(&supervisor, &id);
assert_eq!(recovered.state, WorkerRuntimeState::Running);
assert_eq!(recovered.completed_cycles, 1);
assert!(recovered.last_error.is_none());
assert!(recovered.last_success_at.is_some());
assert!(recovered.last_failure_at.is_some());
commands.send(DaemonCommand::Fail).unwrap();
yield_until(|| task_health(&supervisor, &id).state == WorkerRuntimeState::BackingOff).await;
assert_eq!(task_health(&supervisor, &id).consecutive_failures, 1);
tokio::time::advance(Duration::from_millis(10)).await;
yield_until(|| lock_unpoisoned(&contexts).len() == 4).await;
commands.send(DaemonCommand::Fail).unwrap();
yield_until(|| task_health(&supervisor, &id).state == WorkerRuntimeState::BackingOff).await;
tokio::time::advance(Duration::from_millis(20)).await;
yield_until(|| lock_unpoisoned(&contexts).len() == 5).await;
// 同一 poll 中先成功再退出,必须保留最新故障并重新从初始退避开始
commands.send(DaemonCommand::SucceedThenFail).unwrap();
yield_until(|| task_health(&supervisor, &id).state == WorkerRuntimeState::BackingOff).await;
assert_eq!(task_health(&supervisor, &id).consecutive_failures, 1);
assert_eq!(task_health(&supervisor, &id).completed_cycles, 2);
tokio::time::advance(Duration::from_millis(10)).await;
yield_until(|| lock_unpoisoned(&contexts).len() == 6).await;
supervisor.shutdown(Duration::from_secs(1)).await;
lock_unpoisoned(&contexts)[5].succeeded();
assert_eq!(
task_health(&supervisor, &id).state,
WorkerRuntimeState::Stopped
);
}
#[tokio::test]
async fn daemon_success_remains_healthy_while_idle_beyond_restart_backoff() {
let (commands, receiver) = tokio::sync::mpsc::unbounded_channel();
let supervisor = supervisor();
let id = WorkerId::try_new(WorkerKind::Backup, "backup").unwrap();
supervisor
.start(
daemon_plan(ControlledDaemon {
contexts: Arc::default(),
commands: tokio::sync::Mutex::new(receiver),
}),
Arc::new(FakeLeasePort::default()),
Arc::new(RecordingDiagnostics::default()),
)
.unwrap();
commands.send(DaemonCommand::Succeed).unwrap();
wait_until(|| task_health(&supervisor, &id).completed_cycles == 1).await;
tokio::time::sleep(Duration::from_millis(50)).await;
let health = task_health(&supervisor, &id);
assert_eq!(health.state, WorkerRuntimeState::Running);
assert_eq!(health.consecutive_failures, 0);
supervisor.shutdown(Duration::from_secs(1)).await;
}
#[tokio::test(start_paused = true)]
async fn daemon_panic_and_unexpected_exit_remain_unhealthy_until_success() {
for (command, error) in [
(DaemonCommand::Panic, "daemon panicked"),
(DaemonCommand::Exit, "daemon exited unexpectedly"),
] {
let (commands, receiver) = tokio::sync::mpsc::unbounded_channel();
let contexts = Arc::new(Mutex::new(Vec::new()));
let supervisor = supervisor();
let id = WorkerId::try_new(WorkerKind::Backup, "backup").unwrap();
supervisor
.start(
daemon_plan(ControlledDaemon {
contexts: contexts.clone(),
commands: tokio::sync::Mutex::new(receiver),
}),
Arc::new(FakeLeasePort::default()),
Arc::new(RecordingDiagnostics::default()),
)
.unwrap();
commands.send(command).unwrap();
yield_until(|| task_health(&supervisor, &id).state == WorkerRuntimeState::BackingOff).await;
assert_eq!(
task_health(&supervisor, &id).last_error.as_deref(),
Some(error)
);
tokio::time::advance(Duration::from_millis(10)).await;
yield_until(|| lock_unpoisoned(&contexts).len() == 2).await;
assert_eq!(task_health(&supervisor, &id).consecutive_failures, 1);
commands.send(DaemonCommand::Succeed).unwrap();
yield_until(|| task_health(&supervisor, &id).consecutive_failures == 0).await;
supervisor.shutdown(Duration::from_secs(1)).await;
}
}