rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-10-28 22:46:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add test for GC of stairs layers

Browse Source
This commit is contained in:
Konstantin Knizhnik
2023-05-16 21:18:20 +03:00
parent f9999c84d9
commit a1c8e74fb9
2 changed files with 116 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pageserver/src/tenant/timeline.rs
+47 -29
View File
@@ -2913,7 +2913,7 @@ impl Timeline {
let layers = self.layers.read().unwrap();
let mut max_deltas = 0;
let wanted_image_layers = self.wanted_image_layers.lock().unwrap().take();
let wanted_image_layers = self.wanted_image_layers.lock().unwrap();
for part_range in &partition.ranges {
let image_coverage = layers.image_coverage(part_range, lsn)?;
@@ -2947,12 +2947,25 @@ impl Timeline {
);
return Ok(true);
}
if let Some(wanted) = &wanted_image_layers {
if let Some((_start, end)) = wanted.range(..img_range.end).next_back() {
if *end > img_range.start {
return Ok(true);
if num_deltas > 0 {
if let Some(wanted) = &*wanted_image_layers {
if let Some((_start, end)) = wanted.range(..img_range.end).next_back() {
if *end > img_range.start {
info!(
"Force generation of layer {}-{} wanted by GC ({} deltas)",
img_range.start, img_range.end, num_deltas
);
return Ok(true);
} else {
info!(
"Image {}-{} is not wanted",
img_range.start, img_range.end
);
}
}
}
} else {
info!("No deltas for range {}-{}", img_range.start, img_range.end);
}
}
}
@@ -3826,34 +3839,39 @@ impl Timeline {
let start = l.get_key_range().start;
let mut end = l.get_key_range().end;
// check if this range overlaps with exited ranges
let mut iter = wanted_image_layers.range_mut(..end);
while let Some((prev_start, prev_end)) = iter.next_back() {
if *prev_end >= start {
// two ranges overlap
if *prev_start <= start {
// combine with prev range
insert_new_range = false;
if *prev_end < end {
// extend prev range
*prev_end = end;
if start != Key::MIN || end != Key::MAX {
// ignore L0 layers
// check if this range overlaps with exited ranges
let mut iter = wanted_image_layers.range_mut(..end);
while let Some((prev_start, prev_end)) = iter.next_back() {
if *prev_end >= start {
// two ranges overlap
if *prev_start <= start {
// combine with prev range
insert_new_range = false;
if *prev_end < end {
// extend prev range
*prev_end = end;
info!("Extend wanted image {}..{}", *prev_start, end);
}
break;
} else {
to_remove.push(*prev_start);
if *prev_end > end {
end = *prev_end;
}
}
break;
} else {
to_remove.push(*prev_start);
if *prev_end > end {
end = *prev_end;
}
break;
}
} else {
break;
}
}
for key in to_remove {
wanted_image_layers.remove(&key);
}
if insert_new_range {
wanted_image_layers.insert(start, end);
for key in to_remove {
wanted_image_layers.remove(&key);
}
if insert_new_range {
info!("Wanted image {}..{}", start, end);
wanted_image_layers.insert(start, end);
}
}
result.layers_not_updated += 1;
continue 'outer;
test_runner/regress/test_gc_old_layers.py
+69
View File
@@ -0,0 +1,69 @@
import time
import pytest
from fixtures.log_helper import log
from fixtures.neon_fixtures import NeonEnvBuilder
# Test gc_old_layers
#
# This test sets fail point at the end of GC, and checks that pageserver
# normally restarts after it. Also, there should be GC ERRORs in the log,
# but the fixture checks the log for any unexpected ERRORs after every
# test anyway, so it doesn't need any special attention here.
@pytest.mark.timeout(10000)
def test_gc_old_layers(neon_env_builder: NeonEnvBuilder):
env = neon_env_builder.init_start()
client = env.pageserver.http_client()
# Use aggressive GC and checkpoint settings, so that we also exercise GC during the test
tenant_id, _ = env.neon_cli.create_tenant(
conf={
# disable default GC and compaction
"gc_period": "1000 m",
"compaction_period": "0 s",
"gc_horizon": f"{1024 ** 2}",
"checkpoint_distance": f"{1024 ** 2}",
"compaction_target_size": f"{1024 ** 2}",
# set PITR interval to be small, so we can do GC
"pitr_interval": "1 s",
# "compaction_threshold": "3",
# "image_creation_threshold": "2",
}
)
pg = env.postgres.create_start("main", tenant_id=tenant_id)
timeline_id = pg.safe_psql("show neon.timeline_id")[0][0]
n_steps = 10
n_update_iters = 100
step_size = 10000
with pg.cursor() as cur:
cur.execute("SET statement_timeout='1000s'")
cur.execute(
"CREATE TABLE t(pk bigint primary key, count bigint default 0, payload text default repeat(' ', 100)) with (fillfactor=50)"
)
for step in range(n_steps):
cur.execute(
f"INSERT INTO t (pk) values (generate_series({step*step_size+1},{(step+1)*step_size}))"
)
for i in range(n_update_iters):
cur.execute(
f"UPDATE t set count=count+1 where pk BETWEEN {(step-1)*step_size+1+i*step_size//n_update_iters} AND {step*step_size+i*step_size//n_update_iters}"
)
cur.execute("vacuum t")
# cur.execute("select pg_table_size('t')")
# logical_size = cur.fetchone()[0]
logical_size = client.timeline_detail(tenant_id, timeline_id)["current_logical_size"]
log.info(f"Logical storage size {logical_size}")
client.timeline_checkpoint(tenant_id, timeline_id)
# Do compaction and GC
client.timeline_gc(tenant_id, timeline_id, 0)
client.timeline_compact(tenant_id, timeline_id)
physical_size = client.timeline_detail(tenant_id, timeline_id)["current_physical_size"]
log.info(f"Physical storage size {physical_size}")
log.info("Done")
time.sleep(10000)