rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-08 05:52:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

bottom-most-compaction: use in test_gc_feedback + fix bugs (#8103)

Browse Source 
Adds manual compaction trigger; add gc compaction to test_gc_feedback

Part of https://github.com/neondatabase/neon/issues/8002

```
test_gc_feedback[debug-pg15].logical_size: 50 Mb
test_gc_feedback[debug-pg15].physical_size: 2269 Mb
test_gc_feedback[debug-pg15].physical/logical ratio: 44.5302 
test_gc_feedback[debug-pg15].max_total_num_of_deltas: 7 
test_gc_feedback[debug-pg15].max_num_of_deltas_above_image: 2 
test_gc_feedback[debug-pg15].logical_size_after_bottom_most_compaction: 50 Mb
test_gc_feedback[debug-pg15].physical_size_after_bottom_most_compaction: 287 Mb
test_gc_feedback[debug-pg15].physical/logical ratio after bottom_most_compaction: 5.6312 
test_gc_feedback[debug-pg15].max_total_num_of_deltas_after_bottom_most_compaction: 4 
test_gc_feedback[debug-pg15].max_num_of_deltas_above_image_after_bottom_most_compaction: 1
```

## Summary of changes

* Add the manual compaction trigger
* Use in test_gc_feedback
* Add a guard to avoid running it with retain_lsns
* Fix: Do `schedule_compaction_update` after compaction
* Fix: Supply deltas in the correct order to reconstruct value

---------

Signed-off-by: Alex Chi Z <chi@neon.tech>
This commit is contained in:
Alex Chi Z
2024-06-25 19:00:14 -04:00
committed by GitHub
parent 76864e6a2a
commit 9b98823d61
9 changed files with 92 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
test_runner/fixtures/pageserver/http.py
View File

@@ -573,6 +573,7 @@ class PageserverHttpClient(requests.Session, MetricsGetter):
force_repartition=False,
force_image_layer_creation=False,
wait_until_uploaded=False,
enhanced_gc_bottom_most_compaction=False,
):
self.is_testing_enabled_or_skip()
query = {}
@@ -582,6 +583,8 @@ class PageserverHttpClient(requests.Session, MetricsGetter):
query["force_image_layer_creation"] = "true"
if wait_until_uploaded:
query["wait_until_uploaded"] = "true"
if enhanced_gc_bottom_most_compaction:
query["enhanced_gc_bottom_most_compaction"] = "true"
log.info(f"Requesting compact: tenant {tenant_id}, timeline {timeline_id}")
res = self.put(

48
test_runner/performance/test_gc_feedback.py
View File

@@ -33,7 +33,7 @@ def test_gc_feedback(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenchma
"checkpoint_distance": f"{1024 ** 2}",
"compaction_target_size": f"{1024 ** 2}",
# set PITR interval to be small, so we can do GC
"pitr_interval": "10 s",
"pitr_interval": "60 s",
# "compaction_threshold": "3",
# "image_creation_threshold": "2",
}
@@ -99,6 +99,52 @@ def test_gc_feedback(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenchma
MetricReport.LOWER_IS_BETTER,
)
client.timeline_compact(tenant_id, timeline_id, enhanced_gc_bottom_most_compaction=True)
tline_detail = client.timeline_detail(tenant_id, timeline_id)
logical_size = tline_detail["current_logical_size"]
physical_size = tline_detail["current_physical_size"]
max_num_of_deltas_above_image = 0
max_total_num_of_deltas = 0
for key_range in client.perf_info(tenant_id, timeline_id):
max_total_num_of_deltas = max(max_total_num_of_deltas, key_range["total_num_of_deltas"])
max_num_of_deltas_above_image = max(
max_num_of_deltas_above_image, key_range["num_of_deltas_above_image"]
)
zenbenchmark.record(
"logical_size_after_bottom_most_compaction",
logical_size // MB,
"Mb",
MetricReport.LOWER_IS_BETTER,
)
zenbenchmark.record(
"physical_size_after_bottom_most_compaction",
physical_size // MB,
"Mb",
MetricReport.LOWER_IS_BETTER,
)
zenbenchmark.record(
"physical/logical ratio after bottom_most_compaction",
physical_size / logical_size,
"",
MetricReport.LOWER_IS_BETTER,
)
zenbenchmark.record(
"max_total_num_of_deltas_after_bottom_most_compaction",
max_total_num_of_deltas,
"",
MetricReport.LOWER_IS_BETTER,
)
zenbenchmark.record(
"max_num_of_deltas_above_image_after_bottom_most_compaction",
max_num_of_deltas_above_image,
"",
MetricReport.LOWER_IS_BETTER,
)
with endpoint.cursor() as cur:
cur.execute("SELECT * FROM t") # ensure data is not corrupted
layer_map_path = env.repo_dir / "layer-map.json"
log.info(f"Writing layer map to {layer_map_path}")
with layer_map_path.open("w") as f: