From afda4420bd660eec1d53d4d9f6e1f1ecba86bfa9 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 21 Feb 2024 17:03:55 +0000 Subject: [PATCH] test_sharding_ingress: bigger data, skip in debug mode (#6859) ## Problem Accidentally merged #6852 without this test stability change. The test as-written could sometimes fail on debug-pg14. ## Summary of changes - Write more data so that the test can more reliably assert on the ratio of total layers to small layers - Skip the test in debug mode, since writing any more than a tiny bit of data tends to result in a flaky test in the much slower debug environment. --- test_runner/regress/test_sharding.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/test_runner/regress/test_sharding.py b/test_runner/regress/test_sharding.py index 9e491d450c..5413b178a5 100644 --- a/test_runner/regress/test_sharding.py +++ b/test_runner/regress/test_sharding.py @@ -1,3 +1,6 @@ +import os + +import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, @@ -286,6 +289,12 @@ def test_sharding_split_smoke( env.attachment_service.consistency_check() +@pytest.mark.skipif( + # The quantity of data isn't huge, but debug can be _very_ slow, and the things we're + # validating in this test don't benefit much from debug assertions. + os.getenv("BUILD_TYPE") == "debug", + reason="Avoid running bulkier ingest tests in debug mode", +) def test_sharding_ingest( neon_env_builder: NeonEnvBuilder, ): @@ -319,10 +328,10 @@ def test_sharding_ingest( workload = Workload(env, tenant_id, timeline_id) workload.init() - workload.write_rows(512, upload=False) - workload.write_rows(512, upload=False) - workload.write_rows(512, upload=False) - workload.write_rows(512, upload=False) + workload.write_rows(4096, upload=False) + workload.write_rows(4096, upload=False) + workload.write_rows(4096, upload=False) + workload.write_rows(4096, upload=False) workload.validate() small_layer_count = 0 @@ -361,7 +370,12 @@ def test_sharding_ingest( # - Because we roll layers on checkpoint_distance * shard_count, we expect to obey the target # layer size on average, but it is still possible to write some tiny layers. log.info(f"Totals: {small_layer_count} small layers, {ok_layer_count} ok layers") - assert float(small_layer_count) / float(ok_layer_count) < 0.25 + if small_layer_count <= shard_count: + # If each shard has <= 1 small layer + pass + else: + # General case: + assert float(small_layer_count) / float(ok_layer_count) < 0.25 # Each shard may emit up to one huge layer, because initdb ingest doesn't respect checkpoint_distance. assert huge_layer_count <= shard_count