mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-20 08:01:46 +00:00
Report per-run daemon progress to reset health failures and restart backoff after successful work, with stale-run isolation and recovery regression tests. Co-authored-by: Zyy <zyycn@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
1210 lines
40 KiB
Rust
1210 lines
40 KiB
Rust
//! 验证 Worker 注册、任务组合、禁用原因与监督生命周期
|
|
|
|
use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
|
use std::future::pending;
|
|
use std::num::NonZeroU64;
|
|
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::Duration;
|
|
|
|
use futures::future::BoxFuture;
|
|
use gateway_core::health::{WorkerHealthKey, WorkerHealthSnapshot, WorkerRuntimeState};
|
|
use gateway_core::lifecycle::CancellationToken;
|
|
use gateway_core::task::{
|
|
DaemonRestartPolicy, DaemonTask, ScheduledTask, WorkerContribution, WorkerCycleContext,
|
|
WorkerDaemonContext, WorkerDisabledReason, WorkerFencingToken, WorkerId, WorkerKind,
|
|
WorkerLeaderLeaseGuard, WorkerLeaderLeasePort, WorkerLeaseAcquisition, WorkerLeaseError,
|
|
WorkerLeaseRequest, WorkerRegistration, WorkerRunnable, WorkerSchedule, WorkerTaskError,
|
|
};
|
|
use gateway_host::workers::{WorkerStartError, WorkerSupervisor};
|
|
use tokio::sync::Notify;
|
|
|
|
const ACTIVE_KINDS: [WorkerKind; 11] = [
|
|
WorkerKind::OAuthRefresh,
|
|
WorkerKind::QuotaCatalogHealth,
|
|
WorkerKind::RuntimeSnapshotReconciliation,
|
|
WorkerKind::RuntimeChangeSubscription,
|
|
WorkerKind::StaleModelRequestRecovery,
|
|
WorkerKind::RequestAnalytics,
|
|
WorkerKind::Retention,
|
|
WorkerKind::Backup,
|
|
WorkerKind::AccountImport,
|
|
WorkerKind::AccountFreezeRecovery,
|
|
WorkerKind::ResetCreditAutomation,
|
|
];
|
|
|
|
#[derive(Clone)]
|
|
struct CountingTask {
|
|
runs: Arc<AtomicUsize>,
|
|
fail_until: usize,
|
|
}
|
|
|
|
impl CountingTask {
|
|
fn new(fail_until: usize) -> Self {
|
|
Self {
|
|
runs: Arc::new(AtomicUsize::new(0)),
|
|
fail_until,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl ScheduledTask for CountingTask {
|
|
fn run_cycle(
|
|
&self,
|
|
_context: WorkerCycleContext,
|
|
) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
|
Box::pin(async move {
|
|
let run = self.runs.fetch_add(1, Ordering::SeqCst) + 1;
|
|
if run <= self.fail_until {
|
|
Err(WorkerTaskError::safe("expected test failure")
|
|
.with_source(std::io::Error::other("PRIVATE_WORKER_NATIVE_CAUSE")))
|
|
} else {
|
|
Ok(())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
struct PanicOnceTask {
|
|
runs: Arc<AtomicUsize>,
|
|
}
|
|
|
|
impl ScheduledTask for PanicOnceTask {
|
|
fn run_cycle(
|
|
&self,
|
|
_context: WorkerCycleContext,
|
|
) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
|
Box::pin(async move {
|
|
if self.runs.fetch_add(1, Ordering::SeqCst) == 0 {
|
|
panic!("expected isolated worker panic");
|
|
}
|
|
Ok(())
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
struct CancellationTask {
|
|
entered: Arc<AtomicUsize>,
|
|
observed: Arc<AtomicUsize>,
|
|
}
|
|
|
|
impl ScheduledTask for CancellationTask {
|
|
fn run_cycle(&self, context: WorkerCycleContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
|
Box::pin(async move {
|
|
self.entered.fetch_add(1, Ordering::SeqCst);
|
|
context.cancellation().cancelled().await;
|
|
self.observed.fetch_add(1, Ordering::SeqCst);
|
|
Ok(())
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
struct HungTask {
|
|
entered: Arc<AtomicBool>,
|
|
notification: Arc<Notify>,
|
|
}
|
|
|
|
impl ScheduledTask for HungTask {
|
|
fn run_cycle(
|
|
&self,
|
|
_context: WorkerCycleContext,
|
|
) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
|
Box::pin(async move {
|
|
self.entered.store(true, Ordering::SeqCst);
|
|
self.notification.notify_one();
|
|
pending::<()>().await;
|
|
Ok(())
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
struct SucceedsThenHangsTask {
|
|
runs: Arc<AtomicUsize>,
|
|
second_cycle: Arc<Notify>,
|
|
}
|
|
|
|
impl ScheduledTask for SucceedsThenHangsTask {
|
|
fn run_cycle(
|
|
&self,
|
|
_context: WorkerCycleContext,
|
|
) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
|
Box::pin(async move {
|
|
if self.runs.fetch_add(1, Ordering::SeqCst) == 0 {
|
|
return Ok(());
|
|
}
|
|
self.second_cycle.notify_one();
|
|
pending::<()>().await;
|
|
Ok(())
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
struct LeaseAwareTask {
|
|
entered: Arc<Notify>,
|
|
release: Arc<Notify>,
|
|
completed: Arc<AtomicBool>,
|
|
saw_live_guard: Arc<AtomicBool>,
|
|
expected_worker: WorkerId,
|
|
active_guards: Arc<Mutex<BTreeSet<WorkerId>>>,
|
|
fence: Arc<AtomicU64>,
|
|
}
|
|
|
|
impl ScheduledTask for LeaseAwareTask {
|
|
fn run_cycle(&self, context: WorkerCycleContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
|
Box::pin(async move {
|
|
self.saw_live_guard.store(
|
|
lock_unpoisoned(&self.active_guards).contains(context.worker()),
|
|
Ordering::SeqCst,
|
|
);
|
|
if context.worker() != &self.expected_worker {
|
|
return Err(WorkerTaskError::safe("unexpected worker identity"));
|
|
}
|
|
self.fence.store(
|
|
context
|
|
.fencing_token()
|
|
.expect("leased task fencing token")
|
|
.get(),
|
|
Ordering::SeqCst,
|
|
);
|
|
self.entered.notify_one();
|
|
self.release.notified().await;
|
|
self.completed.store(true, Ordering::SeqCst);
|
|
Ok(())
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
struct ConcurrentCycleTask {
|
|
starts: Arc<AtomicUsize>,
|
|
active: Arc<AtomicUsize>,
|
|
maximum_active: Arc<AtomicUsize>,
|
|
}
|
|
|
|
impl ScheduledTask for ConcurrentCycleTask {
|
|
fn run_cycle(&self, context: WorkerCycleContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
|
Box::pin(async move {
|
|
self.starts.fetch_add(1, Ordering::SeqCst);
|
|
let active = self.active.fetch_add(1, Ordering::SeqCst) + 1;
|
|
self.maximum_active.fetch_max(active, Ordering::SeqCst);
|
|
let _active = ActiveCycle(Arc::clone(&self.active));
|
|
context.cancellation().cancelled().await;
|
|
Ok(())
|
|
})
|
|
}
|
|
}
|
|
|
|
struct ActiveCycle(Arc<AtomicUsize>);
|
|
|
|
impl Drop for ActiveCycle {
|
|
fn drop(&mut self) {
|
|
self.0.fetch_sub(1, Ordering::SeqCst);
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
enum LeaseStep {
|
|
Acquired,
|
|
Busy(Option<Duration>),
|
|
Error,
|
|
Panic,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy)]
|
|
enum RenewStep {
|
|
Success,
|
|
Error,
|
|
}
|
|
|
|
#[derive(Default)]
|
|
struct FakeLeasePort {
|
|
scripts: Mutex<BTreeMap<WorkerId, VecDeque<LeaseStep>>>,
|
|
renew_scripts: Arc<Mutex<BTreeMap<WorkerId, VecDeque<RenewStep>>>>,
|
|
requests: Mutex<Vec<WorkerLeaseRequest>>,
|
|
renewals: Arc<Mutex<Vec<WorkerId>>>,
|
|
active_guards: Arc<Mutex<BTreeSet<WorkerId>>>,
|
|
next_fence: AtomicU64,
|
|
}
|
|
|
|
impl FakeLeasePort {
|
|
fn script(&self, worker: WorkerId, steps: impl IntoIterator<Item = LeaseStep>) {
|
|
lock_unpoisoned(&self.scripts).insert(worker, steps.into_iter().collect());
|
|
}
|
|
|
|
fn renew_script(&self, worker: WorkerId, steps: impl IntoIterator<Item = RenewStep>) {
|
|
lock_unpoisoned(&self.renew_scripts).insert(worker, steps.into_iter().collect());
|
|
}
|
|
|
|
fn requests(&self) -> Vec<WorkerLeaseRequest> {
|
|
lock_unpoisoned(&self.requests).clone()
|
|
}
|
|
|
|
fn renewal_count(&self, worker: &WorkerId) -> usize {
|
|
lock_unpoisoned(&self.renewals)
|
|
.iter()
|
|
.filter(|renewed| *renewed == worker)
|
|
.count()
|
|
}
|
|
}
|
|
|
|
struct FakeLeaseGuard {
|
|
worker: WorkerId,
|
|
fencing_token: WorkerFencingToken,
|
|
active_guards: Arc<Mutex<BTreeSet<WorkerId>>>,
|
|
renew_scripts: Arc<Mutex<BTreeMap<WorkerId, VecDeque<RenewStep>>>>,
|
|
renewals: Arc<Mutex<Vec<WorkerId>>>,
|
|
}
|
|
|
|
impl WorkerLeaderLeaseGuard for FakeLeaseGuard {
|
|
fn fencing_token(&self) -> WorkerFencingToken {
|
|
self.fencing_token
|
|
}
|
|
|
|
fn renew(&mut self) -> BoxFuture<'_, Result<(), WorkerLeaseError>> {
|
|
Box::pin(async move {
|
|
lock_unpoisoned(&self.renewals).push(self.worker.clone());
|
|
match lock_unpoisoned(&self.renew_scripts)
|
|
.get_mut(&self.worker)
|
|
.and_then(VecDeque::pop_front)
|
|
.unwrap_or(RenewStep::Success)
|
|
{
|
|
RenewStep::Success => Ok(()),
|
|
RenewStep::Error => Err(WorkerLeaseError::safe("expected renewal failure")),
|
|
}
|
|
})
|
|
}
|
|
|
|
fn release(self: Box<Self>) -> BoxFuture<'static, Result<(), WorkerLeaseError>> {
|
|
Box::pin(async move {
|
|
drop(self);
|
|
Ok(())
|
|
})
|
|
}
|
|
}
|
|
|
|
impl Drop for FakeLeaseGuard {
|
|
fn drop(&mut self) {
|
|
lock_unpoisoned(&self.active_guards).remove(&self.worker);
|
|
}
|
|
}
|
|
|
|
impl WorkerLeaderLeasePort for FakeLeasePort {
|
|
fn try_acquire(
|
|
&self,
|
|
request: WorkerLeaseRequest,
|
|
) -> BoxFuture<'_, Result<WorkerLeaseAcquisition, WorkerLeaseError>> {
|
|
Box::pin(async move {
|
|
lock_unpoisoned(&self.requests).push(request.clone());
|
|
let step = lock_unpoisoned(&self.scripts)
|
|
.get_mut(request.worker())
|
|
.and_then(VecDeque::pop_front)
|
|
.unwrap_or(LeaseStep::Acquired);
|
|
match step {
|
|
LeaseStep::Acquired => {
|
|
let fence = self.next_fence.fetch_add(1, Ordering::SeqCst) + 1;
|
|
lock_unpoisoned(&self.active_guards).insert(request.worker().clone());
|
|
Ok(WorkerLeaseAcquisition::Acquired(Box::new(FakeLeaseGuard {
|
|
worker: request.worker().clone(),
|
|
fencing_token: WorkerFencingToken::new(
|
|
NonZeroU64::new(fence).expect("positive fence"),
|
|
),
|
|
active_guards: Arc::clone(&self.active_guards),
|
|
renew_scripts: Arc::clone(&self.renew_scripts),
|
|
renewals: Arc::clone(&self.renewals),
|
|
})))
|
|
}
|
|
LeaseStep::Busy(retry_after) => Ok(WorkerLeaseAcquisition::Busy { retry_after }),
|
|
LeaseStep::Error => Err(WorkerLeaseError::safe("expected lease failure")),
|
|
LeaseStep::Panic => panic!("expected isolated lease panic"),
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn registry_composes_multiple_owners_but_rejects_duplicate_identity() {
|
|
let duplicate = registration(
|
|
WorkerKind::OAuthRefresh,
|
|
"openai",
|
|
CountingTask::new(0),
|
|
test_schedule(),
|
|
true,
|
|
);
|
|
let mut plan = complete_plan(vec![registration(
|
|
WorkerKind::OAuthRefresh,
|
|
"openai",
|
|
CountingTask::new(0),
|
|
test_schedule(),
|
|
true,
|
|
)]);
|
|
plan.push(duplicate);
|
|
let supervisor = supervisor();
|
|
|
|
assert!(matches!(
|
|
supervisor.start(plan, Arc::new(FakeLeasePort::default()),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
),
|
|
Err(WorkerStartError::Duplicate(id)) if id.owner() == "openai"
|
|
));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn only_database_fact_reasons_can_disable_nonexistent_workers() {
|
|
let mut plan = complete_plan(Vec::new());
|
|
plan.push(WorkerContribution::Registration(registration_value(
|
|
WorkerKind::OpsFlush,
|
|
"store",
|
|
CountingTask::new(0),
|
|
test_schedule(),
|
|
true,
|
|
)));
|
|
|
|
assert!(matches!(
|
|
supervisor().start(
|
|
plan,
|
|
Arc::new(FakeLeasePort::default()),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
),
|
|
Err(WorkerStartError::KindDisabled(WorkerKind::OpsFlush))
|
|
));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn registry_refuses_start_when_any_real_owner_kind_is_missing() {
|
|
let plan = complete_plan(Vec::new())
|
|
.into_iter()
|
|
.filter(|item| item.kind() != WorkerKind::Retention)
|
|
.collect();
|
|
|
|
assert!(matches!(
|
|
supervisor().start(plan, Arc::new(FakeLeasePort::default()),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
),
|
|
Err(WorkerStartError::Missing(missing)) if missing == vec![WorkerKind::Retention]
|
|
));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn disabled_database_facts_never_execute_or_acquire_a_lease() {
|
|
let leases = Arc::new(FakeLeasePort::default());
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
complete_plan(Vec::new()),
|
|
leases.clone(),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
wait_until(|| leases.requests().len() >= ACTIVE_KINDS.len()).await;
|
|
let snapshot = supervisor.health_source().snapshot();
|
|
|
|
assert!(snapshot.iter().any(|state| {
|
|
state.key == WorkerHealthKey::Disabled(WorkerKind::OpsFlush)
|
|
&& state.state == WorkerRuntimeState::Disabled
|
|
}));
|
|
assert!(
|
|
leases
|
|
.requests()
|
|
.iter()
|
|
.all(|request| { request.worker().kind() != WorkerKind::OpsFlush })
|
|
);
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn one_kind_runs_all_registered_owner_tasks() {
|
|
let second = CountingTask::new(0);
|
|
let plan = complete_plan(vec![
|
|
registration(
|
|
WorkerKind::OAuthRefresh,
|
|
"openai",
|
|
CountingTask::new(0),
|
|
test_schedule(),
|
|
true,
|
|
),
|
|
registration(
|
|
WorkerKind::OAuthRefresh,
|
|
"xai",
|
|
second.clone(),
|
|
test_schedule(),
|
|
true,
|
|
),
|
|
]);
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
plan,
|
|
Arc::new(FakeLeasePort::default()),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
wait_until(|| second.runs.load(Ordering::SeqCst) >= 1).await;
|
|
|
|
assert_eq!(
|
|
supervisor
|
|
.health_source()
|
|
.snapshot()
|
|
.iter()
|
|
.filter(|state| matches!(&state.key, WorkerHealthKey::Task(id) if id.kind() == WorkerKind::OAuthRefresh))
|
|
.count(),
|
|
2
|
|
);
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn leader_guard_lives_through_cycle_and_fence_reaches_owner() {
|
|
let worker = target_worker();
|
|
let leases = Arc::new(FakeLeasePort::default());
|
|
let task = LeaseAwareTask {
|
|
entered: Arc::new(Notify::new()),
|
|
release: Arc::new(Notify::new()),
|
|
completed: Arc::new(AtomicBool::new(false)),
|
|
saw_live_guard: Arc::new(AtomicBool::new(false)),
|
|
expected_worker: worker.clone(),
|
|
active_guards: Arc::clone(&leases.active_guards),
|
|
fence: Arc::new(AtomicU64::new(0)),
|
|
};
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
target_plan(task.clone(), long_interval_schedule()),
|
|
leases.clone(),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
task.entered.notified().await;
|
|
|
|
assert!(task.saw_live_guard.load(Ordering::SeqCst));
|
|
assert!(lock_unpoisoned(&leases.active_guards).contains(&worker));
|
|
task.release.notify_one();
|
|
wait_until(|| task.completed.load(Ordering::SeqCst)).await;
|
|
wait_until(|| !lock_unpoisoned(&leases.active_guards).contains(&worker)).await;
|
|
assert!(task.fence.load(Ordering::SeqCst) > 0);
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn long_cycle_renews_leader_lease_before_ttl() {
|
|
let leases = Arc::new(FakeLeasePort::default());
|
|
let task = lease_aware_task(&leases);
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
target_plan(task.clone(), renewal_schedule()),
|
|
leases.clone(),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
task.entered.notified().await;
|
|
tokio::time::advance(Duration::from_millis(10)).await;
|
|
yield_until(|| leases.renewal_count(&target_worker()) >= 1).await;
|
|
|
|
assert!(lock_unpoisoned(&leases.active_guards).contains(&target_worker()));
|
|
task.release.notify_one();
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn renewal_failure_cancels_old_cycle_before_another_leader_cycle_starts() {
|
|
let task = ConcurrentCycleTask {
|
|
starts: Arc::new(AtomicUsize::new(0)),
|
|
active: Arc::new(AtomicUsize::new(0)),
|
|
maximum_active: Arc::new(AtomicUsize::new(0)),
|
|
};
|
|
let leases = Arc::new(FakeLeasePort::default());
|
|
leases.renew_script(target_worker(), [RenewStep::Error]);
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
target_plan(task.clone(), renewal_schedule()),
|
|
leases,
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
yield_until(|| task.starts.load(Ordering::SeqCst) == 1).await;
|
|
tokio::time::advance(Duration::from_millis(10)).await;
|
|
yield_until(|| task.active.load(Ordering::SeqCst) == 0).await;
|
|
tokio::time::advance(Duration::from_millis(5)).await;
|
|
yield_until(|| task.starts.load(Ordering::SeqCst) == 2).await;
|
|
|
|
assert_eq!(task.maximum_active.load(Ordering::SeqCst), 1);
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn supervisor_uses_exponential_backoff_and_recovers_health() {
|
|
let task = CountingTask::new(2);
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
target_plan(task.clone(), backoff_schedule()),
|
|
Arc::new(FakeLeasePort::default()),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
yield_until(|| task.runs.load(Ordering::SeqCst) == 1).await;
|
|
tokio::time::advance(Duration::from_millis(10)).await;
|
|
yield_until(|| task.runs.load(Ordering::SeqCst) == 2).await;
|
|
tokio::time::advance(Duration::from_millis(20)).await;
|
|
yield_until(|| task.runs.load(Ordering::SeqCst) == 3).await;
|
|
let health = task_health(&supervisor, &target_worker());
|
|
|
|
assert_eq!(
|
|
(health.consecutive_failures, health.completed_cycles),
|
|
(0, 1)
|
|
);
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn task_and_lease_port_panics_are_isolated_and_retried() {
|
|
let task = PanicOnceTask {
|
|
runs: Arc::new(AtomicUsize::new(0)),
|
|
};
|
|
let leases = Arc::new(FakeLeasePort::default());
|
|
leases.script(
|
|
target_worker(),
|
|
[LeaseStep::Error, LeaseStep::Panic, LeaseStep::Acquired],
|
|
);
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
target_plan(task.clone(), backoff_schedule()),
|
|
leases,
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
tokio::task::yield_now().await;
|
|
tokio::time::advance(Duration::from_millis(10)).await;
|
|
tokio::task::yield_now().await;
|
|
tokio::time::advance(Duration::from_millis(20)).await;
|
|
yield_until(|| task.runs.load(Ordering::SeqCst) == 1).await;
|
|
tokio::time::advance(Duration::from_millis(20)).await;
|
|
yield_until(|| task.runs.load(Ordering::SeqCst) >= 2).await;
|
|
|
|
assert_eq!(
|
|
task_health(&supervisor, &target_worker()).consecutive_failures,
|
|
0
|
|
);
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn lease_busy_is_healthy_standby_and_retries_without_running_owner() {
|
|
let task = CountingTask::new(0);
|
|
let leases = Arc::new(FakeLeasePort::default());
|
|
leases.script(
|
|
target_worker(),
|
|
[
|
|
LeaseStep::Busy(Some(Duration::from_millis(7))),
|
|
LeaseStep::Acquired,
|
|
],
|
|
);
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
target_plan(task.clone(), backoff_schedule()),
|
|
leases,
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
yield_until(|| task_health(&supervisor, &target_worker()).state == WorkerRuntimeState::Standby)
|
|
.await;
|
|
|
|
assert_eq!(task.runs.load(Ordering::SeqCst), 0);
|
|
tokio::time::advance(Duration::from_millis(7)).await;
|
|
yield_until(|| task.runs.load(Ordering::SeqCst) == 1).await;
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn cooperative_shutdown_propagates_cancel_before_dropping_all_guards() {
|
|
let task = CancellationTask {
|
|
entered: Arc::new(AtomicUsize::new(0)),
|
|
observed: Arc::new(AtomicUsize::new(0)),
|
|
};
|
|
let leases = Arc::new(FakeLeasePort::default());
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
all_active_plan(task.clone()),
|
|
leases.clone(),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
wait_until(|| task.entered.load(Ordering::SeqCst) == ACTIVE_KINDS.len()).await;
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
|
|
assert_eq!(task.observed.load(Ordering::SeqCst), ACTIVE_KINDS.len());
|
|
assert!(lock_unpoisoned(&leases.active_guards).is_empty());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn shutdown_timeout_aborts_uncooperative_task_and_drops_guard() {
|
|
let task = HungTask {
|
|
entered: Arc::new(AtomicBool::new(false)),
|
|
notification: Arc::new(Notify::new()),
|
|
};
|
|
let leases = Arc::new(FakeLeasePort::default());
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
target_plan(task.clone(), long_interval_schedule()),
|
|
leases.clone(),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
task.notification.notified().await;
|
|
supervisor.shutdown(Duration::from_millis(10)).await;
|
|
|
|
assert!(lock_unpoisoned(&leases.active_guards).is_empty());
|
|
assert_eq!(
|
|
task_health(&supervisor, &target_worker()).state,
|
|
WorkerRuntimeState::Stopped
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dropping_shutdown_aborts_current_and_unjoined_workers() {
|
|
let task = HungTask {
|
|
entered: Arc::new(AtomicBool::new(false)),
|
|
notification: Arc::new(Notify::new()),
|
|
};
|
|
let leases = Arc::new(FakeLeasePort::default());
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
all_active_plan(task),
|
|
leases.clone(),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
wait_until(|| lock_unpoisoned(&leases.active_guards).len() == ACTIVE_KINDS.len()).await;
|
|
|
|
let mut shutdown = Box::pin(supervisor.shutdown(Duration::from_secs(60)));
|
|
assert!(futures::poll!(&mut shutdown).is_pending());
|
|
drop(shutdown);
|
|
// Supervisor 仍存活,shutdown 自己必须回收当前等待和尚未轮到的所有任务
|
|
wait_until(|| lock_unpoisoned(&leases.active_guards).is_empty()).await;
|
|
assert!(
|
|
supervisor
|
|
.health_source()
|
|
.snapshot()
|
|
.into_iter()
|
|
.all(|entry| matches!(
|
|
entry.state,
|
|
WorkerRuntimeState::Stopped | WorkerRuntimeState::Disabled
|
|
))
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn shutdown_timeout_must_not_rejoin_workers_that_already_exited() {
|
|
// 挂死任务放在 join 顺序最末的 kind(Retention),保证超时触发时
|
|
// 排在它前面的合作任务句柄都已被 join 过一次
|
|
let hung = HungTask {
|
|
entered: Arc::new(AtomicBool::new(false)),
|
|
notification: Arc::new(Notify::new()),
|
|
};
|
|
let cooperative = CancellationTask {
|
|
entered: Arc::new(AtomicUsize::new(0)),
|
|
observed: Arc::new(AtomicUsize::new(0)),
|
|
};
|
|
let mut plan = ACTIVE_KINDS
|
|
.into_iter()
|
|
.map(|kind| {
|
|
if kind == WorkerKind::Retention {
|
|
registration(kind, kind.as_str(), hung.clone(), test_schedule(), true)
|
|
} else {
|
|
registration(
|
|
kind,
|
|
kind.as_str(),
|
|
cooperative.clone(),
|
|
test_schedule(),
|
|
true,
|
|
)
|
|
}
|
|
})
|
|
.collect::<Vec<_>>();
|
|
plan.push(WorkerContribution::Disabled {
|
|
kind: WorkerKind::OpsFlush,
|
|
reason: WorkerDisabledReason::NoBufferedOpsEvents,
|
|
});
|
|
let leases = Arc::new(FakeLeasePort::default());
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
plan,
|
|
leases.clone(),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
hung.notification.notified().await;
|
|
wait_until(|| cooperative.entered.load(Ordering::SeqCst) == ACTIVE_KINDS.len() - 1).await;
|
|
|
|
supervisor.shutdown(Duration::from_millis(100)).await;
|
|
|
|
assert_eq!(
|
|
cooperative.observed.load(Ordering::SeqCst),
|
|
ACTIVE_KINDS.len() - 1
|
|
);
|
|
assert!(lock_unpoisoned(&leases.active_guards).is_empty());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn running_worker_becomes_unhealthy_when_last_success_is_stale() {
|
|
let task = SucceedsThenHangsTask {
|
|
runs: Arc::new(AtomicUsize::new(0)),
|
|
second_cycle: Arc::new(Notify::new()),
|
|
};
|
|
let supervisor = supervisor();
|
|
supervisor
|
|
.start(
|
|
target_plan(task.clone(), freshness_schedule()),
|
|
Arc::new(FakeLeasePort::default()),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.expect("complete plan");
|
|
task.second_cycle.notified().await;
|
|
tokio::time::sleep(Duration::from_millis(45)).await;
|
|
|
|
assert_eq!(
|
|
task_health(&supervisor, &target_worker()).state,
|
|
WorkerRuntimeState::BackingOff
|
|
);
|
|
supervisor.shutdown(Duration::from_millis(10)).await;
|
|
}
|
|
|
|
fn supervisor() -> WorkerSupervisor {
|
|
WorkerSupervisor::new(CancellationToken::new())
|
|
}
|
|
|
|
fn registration(
|
|
kind: WorkerKind,
|
|
owner: &str,
|
|
task: impl ScheduledTask + 'static,
|
|
schedule: WorkerSchedule,
|
|
leased: bool,
|
|
) -> WorkerContribution {
|
|
WorkerContribution::Registration(registration_value(kind, owner, task, schedule, leased))
|
|
}
|
|
|
|
fn registration_value(
|
|
kind: WorkerKind,
|
|
owner: &str,
|
|
task: impl ScheduledTask + 'static,
|
|
schedule: WorkerSchedule,
|
|
leased: bool,
|
|
) -> WorkerRegistration {
|
|
let id = WorkerId::try_new(kind, owner).expect("valid worker id");
|
|
let lease = leased
|
|
.then(|| WorkerLeaseRequest::try_new(id.clone(), schedule.leader_lease_ttl()))
|
|
.transpose()
|
|
.expect("valid lease");
|
|
WorkerRegistration::try_new(
|
|
id,
|
|
WorkerRunnable::Scheduled {
|
|
schedule,
|
|
lease,
|
|
task: Box::new(task),
|
|
},
|
|
)
|
|
.expect("valid registration")
|
|
}
|
|
|
|
fn complete_plan(mut custom: Vec<WorkerContribution>) -> Vec<WorkerContribution> {
|
|
let present = custom
|
|
.iter()
|
|
.map(WorkerContribution::kind)
|
|
.collect::<BTreeSet<_>>();
|
|
for kind in ACTIVE_KINDS {
|
|
if !present.contains(&kind) {
|
|
custom.push(registration(
|
|
kind,
|
|
kind.as_str(),
|
|
CountingTask::new(0),
|
|
long_interval_schedule(),
|
|
true,
|
|
));
|
|
}
|
|
}
|
|
custom.push(WorkerContribution::Disabled {
|
|
kind: WorkerKind::OpsFlush,
|
|
reason: WorkerDisabledReason::NoBufferedOpsEvents,
|
|
});
|
|
custom
|
|
}
|
|
|
|
fn target_plan(
|
|
task: impl ScheduledTask + 'static,
|
|
schedule: WorkerSchedule,
|
|
) -> Vec<WorkerContribution> {
|
|
complete_plan(vec![registration(
|
|
WorkerKind::OAuthRefresh,
|
|
"openai",
|
|
task,
|
|
schedule,
|
|
true,
|
|
)])
|
|
}
|
|
|
|
fn all_active_plan(task: impl ScheduledTask + Clone + 'static) -> Vec<WorkerContribution> {
|
|
let mut plan = ACTIVE_KINDS
|
|
.into_iter()
|
|
.map(|kind| registration(kind, kind.as_str(), task.clone(), test_schedule(), true))
|
|
.collect::<Vec<_>>();
|
|
plan.push(WorkerContribution::Disabled {
|
|
kind: WorkerKind::OpsFlush,
|
|
reason: WorkerDisabledReason::NoBufferedOpsEvents,
|
|
});
|
|
plan
|
|
}
|
|
|
|
fn lease_aware_task(leases: &Arc<FakeLeasePort>) -> LeaseAwareTask {
|
|
LeaseAwareTask {
|
|
entered: Arc::new(Notify::new()),
|
|
release: Arc::new(Notify::new()),
|
|
completed: Arc::new(AtomicBool::new(false)),
|
|
saw_live_guard: Arc::new(AtomicBool::new(false)),
|
|
expected_worker: target_worker(),
|
|
active_guards: Arc::clone(&leases.active_guards),
|
|
fence: Arc::new(AtomicU64::new(0)),
|
|
}
|
|
}
|
|
|
|
fn target_worker() -> WorkerId {
|
|
WorkerId::try_new(WorkerKind::OAuthRefresh, "openai").expect("valid target worker")
|
|
}
|
|
|
|
fn task_health(supervisor: &WorkerSupervisor, id: &WorkerId) -> WorkerHealthSnapshot {
|
|
supervisor
|
|
.health_source()
|
|
.snapshot()
|
|
.into_iter()
|
|
.find(|state| state.key == WorkerHealthKey::Task(id.clone()))
|
|
.expect("worker health")
|
|
}
|
|
|
|
fn schedule(
|
|
interval: Duration,
|
|
initial: Duration,
|
|
maximum: Duration,
|
|
ttl: Duration,
|
|
renewal: Duration,
|
|
) -> WorkerSchedule {
|
|
WorkerSchedule::try_new(interval, initial, maximum, ttl, renewal).expect("valid schedule")
|
|
}
|
|
|
|
fn test_schedule() -> WorkerSchedule {
|
|
schedule(
|
|
Duration::from_millis(20),
|
|
Duration::from_millis(5),
|
|
Duration::from_millis(20),
|
|
Duration::from_secs(30),
|
|
Duration::from_secs(10),
|
|
)
|
|
}
|
|
|
|
fn long_interval_schedule() -> WorkerSchedule {
|
|
schedule(
|
|
Duration::from_secs(60),
|
|
Duration::from_millis(5),
|
|
Duration::from_millis(20),
|
|
Duration::from_secs(30),
|
|
Duration::from_secs(10),
|
|
)
|
|
}
|
|
|
|
fn backoff_schedule() -> WorkerSchedule {
|
|
schedule(
|
|
Duration::from_secs(60),
|
|
Duration::from_millis(10),
|
|
Duration::from_millis(20),
|
|
Duration::from_secs(30),
|
|
Duration::from_secs(10),
|
|
)
|
|
}
|
|
|
|
fn renewal_schedule() -> WorkerSchedule {
|
|
schedule(
|
|
Duration::from_secs(60),
|
|
Duration::from_millis(5),
|
|
Duration::from_millis(20),
|
|
Duration::from_millis(30),
|
|
Duration::from_millis(10),
|
|
)
|
|
}
|
|
|
|
fn freshness_schedule() -> WorkerSchedule {
|
|
schedule(
|
|
Duration::from_millis(10),
|
|
Duration::from_millis(5),
|
|
Duration::from_millis(10),
|
|
Duration::from_millis(30),
|
|
Duration::from_millis(10),
|
|
)
|
|
}
|
|
|
|
async fn wait_until(mut condition: impl FnMut() -> bool) {
|
|
tokio::time::timeout(Duration::from_secs(1), async {
|
|
while !condition() {
|
|
tokio::task::yield_now().await;
|
|
}
|
|
})
|
|
.await
|
|
.expect("worker test timeout");
|
|
}
|
|
|
|
async fn yield_until(mut condition: impl FnMut() -> bool) {
|
|
for _ in 0..100 {
|
|
if condition() {
|
|
return;
|
|
}
|
|
tokio::task::yield_now().await;
|
|
}
|
|
assert!(condition(), "condition did not become true after yielding");
|
|
}
|
|
|
|
fn lock_unpoisoned<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
|
|
mutex
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
}
|
|
|
|
#[derive(Default)]
|
|
struct RecordingDiagnostics(std::sync::Mutex<Vec<gateway_core::diagnostics::OperationalFailure>>);
|
|
#[async_trait::async_trait]
|
|
impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics {
|
|
async fn record_failure(
|
|
&self,
|
|
failure: gateway_core::diagnostics::OperationalFailure,
|
|
) -> Result<(), gateway_core::error::StoreError> {
|
|
self.0.lock().unwrap().push(failure);
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn worker_boundary_records_one_native_failure_and_recovers_health() {
|
|
let supervisor = supervisor();
|
|
let task = CountingTask::new(1);
|
|
let diagnostics = Arc::new(RecordingDiagnostics::default());
|
|
supervisor
|
|
.start(
|
|
target_plan(task.clone(), test_schedule()),
|
|
Arc::new(FakeLeasePort::default()),
|
|
diagnostics.clone(),
|
|
)
|
|
.unwrap();
|
|
wait_until(|| task.runs.load(Ordering::SeqCst) >= 2).await;
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
let failures = diagnostics.0.lock().unwrap();
|
|
assert_eq!(failures.len(), 1);
|
|
assert_eq!(
|
|
failures[0].correlation_id.as_deref(),
|
|
Some(target_worker().to_string().as_str())
|
|
);
|
|
assert!(
|
|
failures[0]
|
|
.details
|
|
.as_ref()
|
|
.unwrap()
|
|
.as_str()
|
|
.contains("PRIVATE_WORKER_NATIVE_CAUSE")
|
|
);
|
|
assert!(
|
|
!format!("{:?}", supervisor.health_source().snapshot())
|
|
.contains("PRIVATE_WORKER_NATIVE_CAUSE")
|
|
);
|
|
}
|
|
|
|
// 每次运行保存自己的上下文,测试可模拟已退出任务的迟到通知
|
|
struct ControlledDaemon {
|
|
contexts: Arc<Mutex<Vec<WorkerDaemonContext>>>,
|
|
commands: tokio::sync::Mutex<tokio::sync::mpsc::UnboundedReceiver<DaemonCommand>>,
|
|
}
|
|
|
|
enum DaemonCommand {
|
|
Succeed,
|
|
Fail,
|
|
SucceedThenFail,
|
|
Panic,
|
|
Exit,
|
|
}
|
|
|
|
impl DaemonTask for ControlledDaemon {
|
|
fn run(&self, context: WorkerDaemonContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
|
Box::pin(async move {
|
|
lock_unpoisoned(&self.contexts).push(context.clone());
|
|
let mut commands = self.commands.lock().await;
|
|
loop {
|
|
let command = tokio::select! {
|
|
() = context.cancellation().cancelled() => return Ok(()),
|
|
command = commands.recv() => command.expect("daemon command"),
|
|
};
|
|
match command {
|
|
DaemonCommand::Succeed => context.succeeded(),
|
|
DaemonCommand::SucceedThenFail => {
|
|
context.succeeded();
|
|
return Err(WorkerTaskError::safe("test daemon failure"));
|
|
}
|
|
DaemonCommand::Fail => {
|
|
return Err(WorkerTaskError::safe("test daemon failure"));
|
|
}
|
|
DaemonCommand::Panic => panic!("test daemon panic"),
|
|
DaemonCommand::Exit => return Ok(()),
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
fn daemon_plan(task: impl DaemonTask + 'static) -> Vec<WorkerContribution> {
|
|
complete_plan(vec![WorkerContribution::Registration(
|
|
WorkerRegistration::try_new(
|
|
WorkerId::try_new(WorkerKind::Backup, "backup").unwrap(),
|
|
WorkerRunnable::Daemon {
|
|
restart: DaemonRestartPolicy::try_new(
|
|
Duration::from_millis(10),
|
|
Duration::from_millis(20),
|
|
)
|
|
.unwrap(),
|
|
task: Box::new(task),
|
|
},
|
|
)
|
|
.unwrap(),
|
|
)])
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn daemon_recovers_only_after_current_run_succeeds_and_resets_backoff() {
|
|
let (commands, receiver) = tokio::sync::mpsc::unbounded_channel();
|
|
let contexts = Arc::new(Mutex::new(Vec::new()));
|
|
let supervisor = supervisor();
|
|
let id = WorkerId::try_new(WorkerKind::Backup, "backup").unwrap();
|
|
supervisor
|
|
.start(
|
|
daemon_plan(ControlledDaemon {
|
|
contexts: contexts.clone(),
|
|
commands: tokio::sync::Mutex::new(receiver),
|
|
}),
|
|
Arc::new(FakeLeasePort::default()),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.unwrap();
|
|
yield_until(|| lock_unpoisoned(&contexts).len() == 1).await;
|
|
|
|
// 连续失败逐步退避;仅重新进入 run 不能恢复健康
|
|
for (attempt, delay) in [(1, 10), (2, 20)] {
|
|
commands.send(DaemonCommand::Fail).unwrap();
|
|
yield_until(|| task_health(&supervisor, &id).state == WorkerRuntimeState::BackingOff).await;
|
|
assert_eq!(task_health(&supervisor, &id).consecutive_failures, attempt);
|
|
tokio::time::advance(Duration::from_millis(delay - 1)).await;
|
|
assert_eq!(lock_unpoisoned(&contexts).len(), attempt as usize);
|
|
tokio::time::advance(Duration::from_millis(1)).await;
|
|
yield_until(|| lock_unpoisoned(&contexts).len() == attempt as usize + 1).await;
|
|
assert_eq!(task_health(&supervisor, &id).consecutive_failures, attempt);
|
|
}
|
|
lock_unpoisoned(&contexts)[0].succeeded();
|
|
tokio::task::yield_now().await;
|
|
assert_eq!(task_health(&supervisor, &id).consecutive_failures, 2);
|
|
|
|
commands.send(DaemonCommand::Succeed).unwrap();
|
|
yield_until(|| task_health(&supervisor, &id).consecutive_failures == 0).await;
|
|
let recovered = task_health(&supervisor, &id);
|
|
assert_eq!(recovered.state, WorkerRuntimeState::Running);
|
|
assert_eq!(recovered.completed_cycles, 1);
|
|
assert!(recovered.last_error.is_none());
|
|
assert!(recovered.last_success_at.is_some());
|
|
assert!(recovered.last_failure_at.is_some());
|
|
|
|
commands.send(DaemonCommand::Fail).unwrap();
|
|
yield_until(|| task_health(&supervisor, &id).state == WorkerRuntimeState::BackingOff).await;
|
|
assert_eq!(task_health(&supervisor, &id).consecutive_failures, 1);
|
|
tokio::time::advance(Duration::from_millis(10)).await;
|
|
yield_until(|| lock_unpoisoned(&contexts).len() == 4).await;
|
|
commands.send(DaemonCommand::Fail).unwrap();
|
|
yield_until(|| task_health(&supervisor, &id).state == WorkerRuntimeState::BackingOff).await;
|
|
tokio::time::advance(Duration::from_millis(20)).await;
|
|
yield_until(|| lock_unpoisoned(&contexts).len() == 5).await;
|
|
|
|
// 同一 poll 中先成功再退出,必须保留最新故障并重新从初始退避开始
|
|
commands.send(DaemonCommand::SucceedThenFail).unwrap();
|
|
yield_until(|| task_health(&supervisor, &id).state == WorkerRuntimeState::BackingOff).await;
|
|
assert_eq!(task_health(&supervisor, &id).consecutive_failures, 1);
|
|
assert_eq!(task_health(&supervisor, &id).completed_cycles, 2);
|
|
tokio::time::advance(Duration::from_millis(10)).await;
|
|
yield_until(|| lock_unpoisoned(&contexts).len() == 6).await;
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
lock_unpoisoned(&contexts)[5].succeeded();
|
|
assert_eq!(
|
|
task_health(&supervisor, &id).state,
|
|
WorkerRuntimeState::Stopped
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn daemon_success_remains_healthy_while_idle_beyond_restart_backoff() {
|
|
let (commands, receiver) = tokio::sync::mpsc::unbounded_channel();
|
|
let supervisor = supervisor();
|
|
let id = WorkerId::try_new(WorkerKind::Backup, "backup").unwrap();
|
|
supervisor
|
|
.start(
|
|
daemon_plan(ControlledDaemon {
|
|
contexts: Arc::default(),
|
|
commands: tokio::sync::Mutex::new(receiver),
|
|
}),
|
|
Arc::new(FakeLeasePort::default()),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.unwrap();
|
|
commands.send(DaemonCommand::Succeed).unwrap();
|
|
wait_until(|| task_health(&supervisor, &id).completed_cycles == 1).await;
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
let health = task_health(&supervisor, &id);
|
|
assert_eq!(health.state, WorkerRuntimeState::Running);
|
|
assert_eq!(health.consecutive_failures, 0);
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn daemon_panic_and_unexpected_exit_remain_unhealthy_until_success() {
|
|
for (command, error) in [
|
|
(DaemonCommand::Panic, "daemon panicked"),
|
|
(DaemonCommand::Exit, "daemon exited unexpectedly"),
|
|
] {
|
|
let (commands, receiver) = tokio::sync::mpsc::unbounded_channel();
|
|
let contexts = Arc::new(Mutex::new(Vec::new()));
|
|
let supervisor = supervisor();
|
|
let id = WorkerId::try_new(WorkerKind::Backup, "backup").unwrap();
|
|
supervisor
|
|
.start(
|
|
daemon_plan(ControlledDaemon {
|
|
contexts: contexts.clone(),
|
|
commands: tokio::sync::Mutex::new(receiver),
|
|
}),
|
|
Arc::new(FakeLeasePort::default()),
|
|
Arc::new(RecordingDiagnostics::default()),
|
|
)
|
|
.unwrap();
|
|
commands.send(command).unwrap();
|
|
yield_until(|| task_health(&supervisor, &id).state == WorkerRuntimeState::BackingOff).await;
|
|
assert_eq!(
|
|
task_health(&supervisor, &id).last_error.as_deref(),
|
|
Some(error)
|
|
);
|
|
tokio::time::advance(Duration::from_millis(10)).await;
|
|
yield_until(|| lock_unpoisoned(&contexts).len() == 2).await;
|
|
assert_eq!(task_health(&supervisor, &id).consecutive_failures, 1);
|
|
commands.send(DaemonCommand::Succeed).unwrap();
|
|
yield_until(|| task_health(&supervisor, &id).consecutive_failures == 0).await;
|
|
supervisor.shutdown(Duration::from_secs(1)).await;
|
|
}
|
|
}
|