diff --git a/test_runner/performance/test_bulk_tenant_create.py b/test_runner/performance/test_bulk_tenant_create.py index 9b05903cfa..e8daa96be6 100644 --- a/test_runner/performance/test_bulk_tenant_create.py +++ b/test_runner/performance/test_bulk_tenant_create.py @@ -1,7 +1,10 @@ +import threading import timeit +from threading import BoundedSemaphore import pytest from fixtures.benchmark_fixture import MetricReport +from fixtures.compare_fixtures import NeonCompare from fixtures.neon_fixtures import NeonEnvBuilder # Run bulk tenant creation test. @@ -50,3 +53,57 @@ def test_bulk_tenant_create( "s", report=MetricReport.LOWER_IS_BETTER, ) + + +@pytest.mark.parametrize("tenants_count", [50]) +def test_parallel_tenant_create( + neon_compare: NeonCompare, + tenants_count: int, +): + """Create lots of tenants in parallel + + One important thing that this measures is the amount of prometheus + metrics per tenant. The pageserver exposes a lot of metrics for + each timeline, and this test gives some visibility to how much + exactly. (We've had to raise the prometheus scraper's limit on + the max metrics size several times, because we expose so many.) + """ + env = neon_compare.env + zenbenchmark = neon_compare.zenbenchmark + + max_concurrent = 5 + pool_sema = BoundedSemaphore(value=max_concurrent) + + def worker(i: int): + with pool_sema: + tenant, timeline_id = env.neon_cli.create_tenant() + + endpoint_tenant = env.endpoints.create_start("main", tenant_id=tenant) + + with endpoint_tenant.cursor() as cur: + cur.execute("select count(*) from pg_class") + endpoint_tenant.stop() + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(tenants_count)] + start = timeit.default_timer() + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + end = timeit.default_timer() + + zenbenchmark.record( + "tenant_creation_time", + end - start, + "s", + report=MetricReport.LOWER_IS_BETTER, + ) + + metrics = env.pageserver.http_client().get_metrics_str() + zenbenchmark.record( + "prometheus_metrics_size", + len(metrics) / tenants_count, + "bytes", + report=MetricReport.LOWER_IS_BETTER, + ) diff --git a/test_runner/regress/test_metric_collection.py b/test_runner/regress/test_metric_collection.py index ecbce1f8f7..2f6f8ffe04 100644 --- a/test_runner/regress/test_metric_collection.py +++ b/test_runner/regress/test_metric_collection.py @@ -51,7 +51,7 @@ metric_kinds_checked = set([]) # -# verify that metrics look minilally sane +# verify that metrics look minimally sane # def metrics_handler(request: Request) -> Response: if request.json is None: