mirror of
https://github.com/neondatabase/neon.git
synced 2025-12-22 21:59:59 +00:00
Move logic for ingest benchmark from GitHub workflow into python testcase (#9762)
## Problem The first version of the ingest benchmark had some parsing and reporting logic in shell script inside GitHub workflow. it is better to move that logic into a python testcase so that we can also run it locally. ## Summary of changes - Create new python testcase - invoke pgcopydb inside python test case - move the following logic into python testcase - determine backpressure - invoke pgcopydb and report its progress - parse pgcopydb log and extract metrics - insert metrics into perf test database - add additional column to perf test database that can receive endpoint ID used for pgcopydb run to have it available in grafana dashboard when retrieving other metrics for an endpoint ## Example run https://github.com/neondatabase/neon/actions/runs/11860622170/job/33056264386
This commit is contained in:
@@ -25,7 +25,8 @@ CREATE TABLE IF NOT EXISTS perf_test_results (
|
||||
metric_value NUMERIC,
|
||||
metric_unit VARCHAR(10),
|
||||
metric_report_type TEXT,
|
||||
recorded_at_timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
recorded_at_timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
labels JSONB with default '{}'
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -91,6 +92,7 @@ def ingest_perf_test_result(cursor, data_file: Path, recorded_at_timestamp: int)
|
||||
"metric_unit": metric["unit"],
|
||||
"metric_report_type": metric["report"],
|
||||
"recorded_at_timestamp": datetime.utcfromtimestamp(recorded_at_timestamp),
|
||||
"labels": json.dumps(metric.get("labels")),
|
||||
}
|
||||
args_list.append(values)
|
||||
|
||||
@@ -105,7 +107,8 @@ def ingest_perf_test_result(cursor, data_file: Path, recorded_at_timestamp: int)
|
||||
metric_value,
|
||||
metric_unit,
|
||||
metric_report_type,
|
||||
recorded_at_timestamp
|
||||
recorded_at_timestamp,
|
||||
labels
|
||||
) VALUES %s
|
||||
""",
|
||||
args_list,
|
||||
@@ -117,7 +120,8 @@ def ingest_perf_test_result(cursor, data_file: Path, recorded_at_timestamp: int)
|
||||
%(metric_value)s,
|
||||
%(metric_unit)s,
|
||||
%(metric_report_type)s,
|
||||
%(recorded_at_timestamp)s
|
||||
%(recorded_at_timestamp)s,
|
||||
%(labels)s
|
||||
)""",
|
||||
)
|
||||
return len(args_list)
|
||||
|
||||
Reference in New Issue
Block a user