From 0e3456351fca1184bb4fd45c8cecdc3245ae77a0 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 6 Jul 2022 18:18:04 +0300 Subject: [PATCH] Shrink thread pools used for WAL receivers and background tasks. I noticed that the pageserver has a very large virtual memory size, several GB, even though it doesn't actually use that much memory. That's not much of a problem normally, but I hit it because I wanted to run tests with a limited virtual memory size, by calling setrlimit(RLIMIT_AS), but the highest limit you can set is 2 GB. I was not able to start pageserver with a limit of 2 GB. On Linux, each thread allocates 32 MB of virtual memory. I read this on some random forum on the Internet, but unfortunately could not find the source again now. Empirically, reducing the number of threads clearly helps to bring down the virtual memory size. Aside from the virtual memory usage, it seems excessive to launch 40 threads in both of those thread pools. The tokio default is to have as many worker threads as there are CPU cores in the system. That seems like a fine heuristic for us, too, so remove the explicit setting of the pool size and rely on the default. Note that the GC and compaction tasks are actually run with tokio spawn_blocking, so the threads that are actually doing the work, and possibly waiting on I/O, are not consuming threads from the thread pool. The WAL receiver work is done in the tokio worker threads, but the WAL receivers are more CPU bound so that seems OK. Also remove the explicit maxinum on blocking tasks. I'm not sure what the right value for that would be, or whether the value we set (100) would be better than the tokio default (512). Since the value was arbitrary, let's just rely on the tokio default for that, too. --- pageserver/src/tenant_tasks.rs | 2 -- pageserver/src/walreceiver.rs | 1 - 2 files changed, 3 deletions(-) diff --git a/pageserver/src/tenant_tasks.rs b/pageserver/src/tenant_tasks.rs index 6871ac3001..b0bb4953ca 100644 --- a/pageserver/src/tenant_tasks.rs +++ b/pageserver/src/tenant_tasks.rs @@ -119,8 +119,6 @@ pub fn start_compaction_loop(tenantid: ZTenantId) -> anyhow::Result<()> { pub fn init_tenant_task_pool() -> anyhow::Result<()> { let runtime = tokio::runtime::Builder::new_multi_thread() .thread_name("tenant-task-worker") - .worker_threads(40) // Way more than necessary - .max_blocking_threads(100) // Way more than necessary .enable_all() .build()?; diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index 2b5a3123c1..9f0f911e0c 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -91,7 +91,6 @@ pub fn init_wal_receiver_main_thread( let runtime = tokio::runtime::Builder::new_multi_thread() .thread_name("wal-receiver-runtime-thread") - .worker_threads(40) .enable_all() .on_thread_start(|| IS_WAL_RECEIVER.with(|c| c.set(true))) .build()