mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-06 13:02:55 +00:00
pageserver: reduce import memory utilization (#12086)
## Problem Imports can end up allocating too much. ## Summary of Changes Nerf them a bunch and add some logs.
This commit is contained in:
@@ -106,6 +106,8 @@ pub async fn doit(
|
||||
);
|
||||
}
|
||||
|
||||
tracing::info!("Import plan executed. Flushing remote changes and notifying storcon");
|
||||
|
||||
timeline
|
||||
.remote_client
|
||||
.schedule_index_upload_for_file_changes()?;
|
||||
|
||||
@@ -130,7 +130,15 @@ async fn run_v1(
|
||||
|
||||
pausable_failpoint!("import-timeline-pre-execute-pausable");
|
||||
|
||||
let jobs_count = import_progress.as_ref().map(|p| p.jobs);
|
||||
let start_from_job_idx = import_progress.map(|progress| progress.completed);
|
||||
|
||||
tracing::info!(
|
||||
start_from_job_idx=?start_from_job_idx,
|
||||
jobs=?jobs_count,
|
||||
"Executing import plan"
|
||||
);
|
||||
|
||||
plan.execute(timeline, start_from_job_idx, plan_hash, &import_config, ctx)
|
||||
.await
|
||||
}
|
||||
@@ -484,6 +492,8 @@ impl Plan {
|
||||
anyhow::anyhow!("Shut down while putting timeline import status")
|
||||
})?;
|
||||
}
|
||||
|
||||
tracing::info!(last_completed_job_idx, jobs=%jobs_in_plan, "Checkpointing import status");
|
||||
},
|
||||
Some(Err(_)) => {
|
||||
anyhow::bail!(
|
||||
@@ -760,7 +770,7 @@ impl ImportTask for ImportRelBlocksTask {
|
||||
layer_writer: &mut ImageLayerWriter,
|
||||
ctx: &RequestContext,
|
||||
) -> anyhow::Result<usize> {
|
||||
const MAX_BYTE_RANGE_SIZE: usize = 128 * 1024 * 1024;
|
||||
const MAX_BYTE_RANGE_SIZE: usize = 4 * 1024 * 1024;
|
||||
|
||||
debug!("Importing relation file");
|
||||
|
||||
|
||||
Reference in New Issue
Block a user