diff --git a/backend/src/main.rs b/backend/src/main.rs index 74f9df37db..70e05d2b16 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -543,7 +543,7 @@ pub async fn run_workers = if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some( @@ -1287,24 +1317,39 @@ pub async fn run_worker 3 { last_checked_suspended = Instant::now(); true } else { false }; - pull(&db, rsmq.clone(), suspend_first).map(|x| (x, timer)) + pull(&db, rsmq.clone(), suspend_first).map(|x| (x, pull_time)).await } => { add_time!(timing, loop_start, "post pull"); - - timer.map(|timer| { - let duration_pull_s = timer.stop_and_record(); - if let Some(wp) = worker_pull_duration_counter.as_ref() { - wp.inc_by(duration_pull_s); + let duration_pull_s = timer.elapsed().as_secs_f64(); + if duration_pull_s > 0.5 { + tracing::error!("pull took more than 0.5s, this is a sign that the database is VERY undersized for this load") + } else if duration_pull_s > 0.1 { + tracing::error!("pull took more than 0.5s, this is a sign that the database is undersized for this load") + } + if let Ok(j) = job.as_ref() { + if j.is_some() { + if let Some(wp) = worker_pull_duration_counter.as_ref() { + wp.inc_by(duration_pull_s); + } + if let Some(wp) = worker_pull_duration.as_ref() { + wp.observe(duration_pull_s); + } + } else { + if let Some(wp) = worker_pull_duration_counter_empty.as_ref() { + wp.inc_by(duration_pull_s); + } + if let Some(wp) = worker_pull_duration_empty.as_ref() { + wp.observe(duration_pull_s); + } } - }); + } job - }, } }; diff --git a/benchmarks/pulumi/index.ts b/benchmarks/pulumi/index.ts index a4183ddad7..ab7cfab190 100644 --- a/benchmarks/pulumi/index.ts +++ b/benchmarks/pulumi/index.ts @@ -242,7 +242,7 @@ const lb = new awsx.lb.ApplicationLoadBalancer("lb2", { // desiredCount: 2, // taskDefinitionArgs: { // container: { -// image: "nginx:latest", +// image: "nginx:main", // cpu: 512, // memory: 128, // essential: true, @@ -261,7 +261,7 @@ db.address.apply((address) => { containerDefinitions: JSON.stringify([ { name: "windmill-worker", - image: "ghcr.io/windmill-labs/windmill-ee:latest", + image: "ghcr.io/windmill-labs/windmill-ee:main", cpu: 1024, memory: 1800, essential: true, @@ -319,7 +319,7 @@ db.address.apply((address) => { containerDefinitions: JSON.stringify([ { name: "windmill-worker", - image: "ghcr.io/windmill-labs/windmill-ee:latest", + image: "ghcr.io/windmill-labs/windmill-ee:main", cpu: 1024, memory: 1800, essential: true, @@ -378,7 +378,7 @@ db.address.apply((address) => { containerDefinitions: JSON.stringify([ { name: "windmill-server", - image: "ghcr.io/windmill-labs/windmill-ee:latest", + image: "ghcr.io/windmill-labs/windmill-ee:main", cpu: 1024, memory: 1024, essential: true, @@ -447,11 +447,6 @@ db.address.apply((address) => { field: "cpu", }, ], - serviceRegistries: { - registryArn: "aws.servicediscovery.Service", - containerName: "windmill-worker", - containerPort: 8001 - } }); const service_worker2 = new aws.ecs.Service("service-worker-2", {