feat(perf): add mixed read/write saturated case for scheduler validation

Add optional [scenario.write_measure.mix] to write_throughput: runs the
remote-write ingestion in a background thread while query-loop threads
(query_parallelism x query_interval_ms) hammer the same frontend, so
query and write tasks contend on the datanode runtime under a dual
backlog. Adds query gates (failure rate, p99 regression) and a
best-effort scheduler poll-share diagnostic scraped from the datanode
/metrics. Includes write_read_mixed_scheduler built-in case.

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
discord9
2026-08-06 14:19:52 +08:00
parent 19ac3bffdd
commit efc23d79bd
5 changed files with 929 additions and 4 deletions
+260
View File
@@ -153,6 +153,85 @@ pub(super) struct WriteMeasureConfig {
#[serde(default)]
pub(super) target_rps: f64,
pub(super) thresholds: WriteThroughputThresholds,
/// Optional concurrent read+write ("mix") measurement. When present the
/// runner runs the remote-write ingestion in a background thread while a
/// query loop hammers the same frontend for `duration_seconds`, so query
/// and write tasks genuinely contend on the datanode runtime under a dual
/// backlog.
#[serde(default)]
pub(super) mix: Option<MixMeasureConfig>,
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub(super) struct MixMeasureConfig {
/// SQL for the concurrent query loop. Defaults to a `count(*)` over the
/// remote-write physical table (`greptime_physical_table`), which scans
/// every ingested row and is therefore a real datanode-runtime contender.
#[serde(default)]
pub(super) query_sql: Option<String>,
/// Wall-clock interval between queries per loop thread, in milliseconds.
#[serde(default = "default_mix_query_interval_ms")]
pub(super) query_interval_ms: NonZeroU64,
/// Number of concurrent query-loop threads.
#[serde(default = "default_mix_query_parallelism")]
pub(super) query_parallelism: NonZeroU64,
pub(super) thresholds: MixMeasureThresholds,
}
pub(super) fn default_mix_query_interval_ms() -> NonZeroU64 {
NonZeroU64::new(100).expect("100 is non-zero")
}
pub(super) fn default_mix_query_parallelism() -> NonZeroU64 {
NonZeroU64::new(1).expect("1 is non-zero")
}
impl MixMeasureConfig {
pub(super) fn validate(&self) -> Result<(), String> {
if let Some(sql) = &self.query_sql
&& sql.trim().is_empty()
{
return Err(
"scenario.write_measure.mix.query_sql must be a non-empty SQL string".to_string(),
);
}
self.thresholds.validate()
}
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub(super) struct MixMeasureThresholds {
/// Max fraction of failed query attempts per target, in [0, 1] (0.05 = 5%).
pub(super) max_query_failure_rate: f64,
/// Max candidate-vs-base query p99 latency regression percent:
/// `(candidate_p99_ms - base_p99_ms) / base_p99_ms * 100`.
pub(super) max_query_p99_regression_pct: f64,
}
impl MixMeasureThresholds {
fn validate(&self) -> Result<(), String> {
for (name, value) in [
("max_query_failure_rate", self.max_query_failure_rate),
(
"max_query_p99_regression_pct",
self.max_query_p99_regression_pct,
),
] {
if !value.is_finite() || value < 0.0 {
return Err(format!(
"scenario.write_measure.mix.thresholds.{name} must be a finite non-negative number"
));
}
}
if self.max_query_failure_rate > 1.0 {
return Err(
"scenario.write_measure.mix.thresholds.max_query_failure_rate must be <= 1.0 (a rate in [0, 1])"
.to_string(),
);
}
Ok(())
}
}
#[derive(Debug, Deserialize, Serialize)]
@@ -185,6 +264,9 @@ impl WriteMeasureConfig {
.to_string(),
);
}
if let Some(mix) = &self.mix {
mix.validate()?;
}
self.thresholds.validate()
}
}
@@ -670,6 +752,10 @@ mod tests {
repo_root().join("tests/perf/query_cases/write_throughput_scheduler/case.toml")
}
fn builtin_write_read_mixed_case() -> PathBuf {
repo_root().join("tests/perf/query_cases/write_read_mixed_scheduler/case.toml")
}
fn parse_case(text: &str) -> Result<CaseFile, toml::de::Error> {
toml::from_str(text)
}
@@ -719,6 +805,31 @@ max_p99_latency_regression_pct = 10.0
max_concurrent_polls = 16
query_weight = 2
write_weight = 8
"#;
const MIXED_CASE: &str = r#"
[scenario]
kind = "write_throughput"
[scenario.remote_write]
metric = "write_read_mixed_test"
[scenario.write_measure]
duration_seconds = 60
window_seconds = 5
[scenario.write_measure.thresholds]
max_failure_rate = 0.05
max_mean_rps_regression_pct = 10.0
max_p99_latency_regression_pct = 10.0
[scenario.write_measure.mix]
query_interval_ms = 100
query_parallelism = 2
[scenario.write_measure.mix.thresholds]
max_query_failure_rate = 0.05
max_query_p99_regression_pct = 10.0
"#;
#[test]
@@ -884,4 +995,153 @@ write_weight = 8
.expect_err("max_failure_rate above 1.0 must be rejected");
assert!(err.contains("max_failure_rate"), "{err}");
}
#[test]
fn parses_builtin_write_read_mixed_case() {
let text = std::fs::read_to_string(builtin_write_read_mixed_case())
.expect("built-in write_read_mixed case.toml must exist");
let case = parse_case(&text).expect("built-in write_read_mixed case must parse");
assert_eq!(case.scenario.kind(), "write_throughput");
let scenario = write_throughput_scenario(&text);
assert_eq!(scenario.remote_write.series_count, 2048);
assert_eq!(scenario.remote_write.samples_per_series, 3600);
assert_eq!(scenario.write_measure.duration_seconds.get(), 60);
assert_eq!(scenario.write_measure.window_seconds.get(), 5);
let mix = scenario
.write_measure
.mix
.as_ref()
.expect("built-in mixed case must declare a mix section");
assert_eq!(mix.query_interval_ms.get(), 100);
assert_eq!(mix.query_parallelism.get(), 2);
assert_eq!(mix.thresholds.max_query_failure_rate, 0.05);
assert_eq!(mix.thresholds.max_query_p99_regression_pct, 10.0);
let scheduler = scenario
.scheduler
.as_ref()
.expect("built-in mixed case must declare a scheduler section");
assert_eq!(scheduler.query_weight, 2);
assert_eq!(scheduler.write_weight, 8);
scenario
.write_measure
.validate()
.expect("built-in mixed case must validate");
scheduler
.validate()
.expect("built-in scheduler must validate");
}
#[test]
fn write_throughput_mix_uses_defaults_and_validates() {
// Defaults apply only for keys the case does not set explicitly; drop
// the explicit query_parallelism to exercise the default.
let defaults_case = MIXED_CASE.replace("query_parallelism = 2\n", "");
let scenario = write_throughput_scenario(&defaults_case);
let mix = scenario
.write_measure
.mix
.as_ref()
.expect("mix section must parse into Some");
assert!(mix.query_sql.is_none());
assert_eq!(mix.query_interval_ms.get(), 100);
assert_eq!(mix.query_parallelism.get(), 1);
mix.validate().expect("mix config must validate");
scenario
.write_measure
.validate()
.expect("write_measure with mix must validate");
}
#[test]
fn write_throughput_mix_parses_explicit_values() {
let explicit = MIXED_CASE.replace(
"query_interval_ms = 100\nquery_parallelism = 2",
"query_sql = \"SELECT count(*) FROM greptime_physical_table\"\nquery_interval_ms = 250\nquery_parallelism = 4",
);
let scenario = write_throughput_scenario(&explicit);
let mix = scenario
.write_measure
.mix
.expect("mix section must parse into Some");
assert_eq!(
mix.query_sql.as_deref(),
Some("SELECT count(*) FROM greptime_physical_table")
);
assert_eq!(mix.query_interval_ms.get(), 250);
assert_eq!(mix.query_parallelism.get(), 4);
mix.validate().expect("explicit mix config must validate");
}
#[test]
fn write_throughput_mix_rejects_zero_interval_or_parallelism() {
let zero_interval = MIXED_CASE.replace("query_interval_ms = 100", "query_interval_ms = 0");
assert!(
parse_case(&zero_interval).is_err(),
"zero query_interval_ms must be rejected"
);
let zero_parallelism = MIXED_CASE.replace("query_parallelism = 2", "query_parallelism = 0");
assert!(
parse_case(&zero_parallelism).is_err(),
"zero query_parallelism must be rejected"
);
}
#[test]
fn write_throughput_mix_rejects_unknown_fields() {
let unknown = MIXED_CASE.replace(
"query_interval_ms = 100",
"query_interval_ms = 100\nbogus_mix_field = 1",
);
let err = parse_case(&unknown).expect_err("unknown mix field must be rejected");
assert!(err.to_string().contains("bogus_mix_field"), "{err}");
}
#[test]
fn write_throughput_mix_rejects_empty_query_sql() {
let empty_sql = MIXED_CASE.replace(
"query_interval_ms = 100",
"query_sql = \" \"\nquery_interval_ms = 100",
);
let scenario = write_throughput_scenario(&empty_sql);
let mix = scenario
.write_measure
.mix
.expect("mix section must parse into Some");
let err = mix
.validate()
.expect_err("blank query_sql must be rejected");
assert!(err.contains("query_sql"), "{err}");
}
#[test]
fn write_throughput_mix_rejects_invalid_thresholds() {
let negative_limit = MIXED_CASE.replace(
"max_query_p99_regression_pct = 10.0",
"max_query_p99_regression_pct = -1.0",
);
let scenario = write_throughput_scenario(&negative_limit);
let mix = scenario
.write_measure
.mix
.expect("mix section must parse into Some");
let err = mix
.validate()
.expect_err("negative query p99 regression limit must be rejected");
assert!(err.contains("max_query_p99_regression_pct"), "{err}");
let oversized_rate = MIXED_CASE.replace(
"max_query_failure_rate = 0.05",
"max_query_failure_rate = 1.5",
);
let scenario = write_throughput_scenario(&oversized_rate);
let mix = scenario
.write_measure
.mix
.expect("mix section must parse into Some");
let err = mix
.validate()
.expect_err("query failure rate above 1.0 must be rejected");
assert!(err.contains("max_query_failure_rate"), "{err}");
}
}
+60
View File
@@ -386,6 +386,66 @@ for a max-throughput run. For local results, run the case at least three times
on an otherwise idle machine and compare median regressions rather than relying
on one run.
### Mixed read/write (saturated) measurement
Adding `[scenario.write_measure.mix]` turns the pure-write scenario into a
concurrent read+write ("mix") measurement: the runner spawns the chunked
remote-write ingestion in a background thread (each chunk is still its own
`query_perf_fixture prom-remote-write` subprocess) and, while it runs, drives
`query_parallelism` query-loop threads against the same frontend HTTP
`/v1/sql` endpoint for `duration_seconds`, then joins the ingestion. Both
streams hit the same frontend/datanode, so query tasks and write tasks
genuinely contend on the datanode runtime under a dual backlog — the
workload-scheduler 2:8 allocation can be observed end to end.
```toml
[scenario.write_measure.mix]
# Default query is `SELECT count(*) FROM greptime_physical_table` (the
# physical table where every remote-write row lands); override with query_sql.
# query_interval_ms is the per-thread wall-clock interval between queries.
query_interval_ms = 100
query_parallelism = 2
[scenario.write_measure.mix.thresholds]
max_query_failure_rate = 0.05 # per-target failed query attempt fraction
max_query_p99_regression_pct = 10.0 # (candidate_p99 - base_p99) / base_p99 * 100
```
Rust owns the mix schema: `query_interval_ms` (default 100) and
`query_parallelism` (default 1) are positive integers (zero is rejected at
parse time), `query_sql` when present must be non-empty, and the mix
thresholds follow the same finite/non-negative rules as the write thresholds
with `max_query_failure_rate` in `[0, 1]`. Unknown mix fields are rejected by
`query_perf_fixture plan`.
For each target the mixed run records the existing `write_measurement` plus:
- `query_measurement`: `{samples, failures, failure_rate, p50_ms, p99_ms,
mean_ms, latency_samples}`. Latency percentiles/mean cover every attempt
that recorded a latency, including failed ones, so timeouts show up in p99.
- `scheduler_poll_deltas`: best-effort deltas of the datanode cumulative
`greptime_workload_scheduler_polls{workload="query"|"write"}` counters over
the window, scraped from `http://127.0.0.1:<datanode_http_port>/metrics`.
This is a diagnostic only, never a gate: the base target runs with the
scheduler disabled and typically exposes no such samples, and a workload
whose counter is missing or reset (decreased) is reported as `null`.
Threshold enforcement combines the existing write gates with the query gates:
per-target `max_query_failure_rate` and base-vs-candidate
`max_query_p99_regression_pct` (positive `actual_pct` is a candidate
regression, negative is an improvement). The case gates that neither side
collapses — it does not gate on the 80% write poll share, which is the
scheduler micro-benchmark's job.
`tests/perf/query_cases/write_read_mixed_scheduler/case.toml` is the built-in
mixed case: the same 2048 series × 3600 samples remote-write sizing as
`write_throughput_scheduler`, a 60s measurement window, mix query
`SELECT count(*) FROM greptime_physical_table` at 100ms interval with
parallelism 2, and gates of ≤ 5% write/query failure, ≤ 10% write RPS
regression, ≤ 10% write and query p99 regression, and a 15k rows/s absolute
floor. It is opt-in (not part of the default CI case set); run it the same way
as the pure write-throughput case, substituting the case path.
## Generator contract
The direct-SST generator should accept a case definition with:
@@ -0,0 +1,76 @@
# Mixed read/write (saturated) regression case for the workload scheduler
# (PR #8736).
#
# Runs remote-write ingestion in the background while a query loop hammers the
# same frontend concurrently for the whole measurement window, so query tasks
# and write tasks genuinely contend on the datanode runtime under a dual
# backlog. With the candidate scheduler's 2:8 query:write weights, admitted
# polls should skew ~80% write while neither side collapses; this case does
# NOT gate on the poll-share number (that is the scheduler micro-benchmark's
# job) but gates that neither side regresses: the existing write gates
# (failure, mean-RPS regression, p99 regression, absolute RPS floor) plus the
# query gates (failure rate, p99 regression).
#
# Sized for a local run: 2048 series x 3600 samples = 7,372,800 rows per
# target, same as write_throughput_scheduler. The ingestion is split into 20
# sample chunks of 180 samples per series; the measurement spans the first 60s
# (12 windows of 5s), during which the query loop (parallelism 2, one query
# every 100ms per thread) runs concurrently against the same frontend.
[case]
name = "write_read_mixed_scheduler"
description = "Concurrent saturated read+write throughput/latency for validating workload-scheduler 2:8 allocation under dual backlog"
issue = "https://github.com/GreptimeTeam/greptimedb/pull/8736"
[scenario]
kind = "write_throughput"
[scenario.scheduler]
# enable is not set here: the runner derives it per target (base = disabled,
# candidate = enabled). max_concurrent_polls 0 = 4 * global_rt_size.
max_concurrent_polls = 16
query_weight = 2
write_weight = 8
[scenario.remote_write]
database = "public"
metric = "write_read_mixed_scheduler"
physical_table = "greptime_physical_table"
series_count = 2048
samples_per_series = 3600
sample_chunk_size = 180
flush_every_sample_chunks = 1
start_unix_millis = 1_704_067_200_000 # 2024-01-01T00:00:00Z
step_millis = 1000
chunk_series_count = 256
timeout_seconds = 120
[scenario.remote_write.prom_store]
pending_rows_flush_interval = "1s"
max_batch_rows = 1_000_000
max_concurrent_flushes = 256
[scenario.write_measure]
duration_seconds = 60
window_seconds = 5
target_rps = 0 # 0 = max throughput; the achieved rate is measured
# Concurrent read+write measurement: the runner spawns the remote-write
# ingestion in a background thread and runs `query_parallelism` query-loop
# threads (one query every `query_interval_ms` each) against the same frontend
# for `duration_seconds`. The default query is
# `SELECT count(*) FROM greptime_physical_table` (the physical table where all
# remote-write rows land); override with `query_sql`.
[scenario.write_measure.mix]
query_interval_ms = 100
query_parallelism = 2
[scenario.write_measure.mix.thresholds]
max_query_failure_rate = 0.05 # per-target failed query attempt fraction
max_query_p99_regression_pct = 10.0 # (candidate_p99 - base_p99) / base_p99 * 100
[scenario.write_measure.thresholds]
max_failure_rate = 0.05
max_mean_rps_regression_pct = 10
max_p99_latency_regression_pct = 10
min_rps_absolute = 15000
+285 -4
View File
@@ -29,6 +29,7 @@ import socket
import statistics
import subprocess
import sys
import threading
import time
import tomllib
import urllib.error
@@ -47,6 +48,9 @@ OTLP_TRACE_METRICS = {
"greptime_servers_http_otlp_traces_elapsed_count",
}
PROMETHEUS_SAMPLE_RE = re.compile(r"^([A-Za-z_:][A-Za-z0-9_:]*)(?:\{.*\})?\s+([^\s]+)(?:\s+.*)?$")
# Workload-scheduler cumulative poll counters on the datanode /metrics
# endpoint, e.g. greptime_workload_scheduler_polls{workload="write"} 1234.
SCHEDULER_POLL_RE = re.compile(r'^greptime_workload_scheduler_polls\{[^}]*workload="(query|write)"[^}]*\}\s+(\d+)(?:\s+.*)?$')
@dataclass(frozen=True)
@@ -1125,6 +1129,55 @@ def metric_delta(after: dict[str, Any], before: dict[str, Any], name: str) -> fl
return delta
def parse_scheduler_poll_metrics(text: str) -> dict[str, int]:
"""Parse `greptime_workload_scheduler_polls{workload="..."}` samples.
Returns {workload: cumulative polls} for the workloads the datanode
scheduler tracks ("query", "write"); workloads absent from the scrape are
omitted (the base target runs with the scheduler disabled and typically
exposes no such samples at all).
"""
values: dict[str, int] = {}
for line in text.splitlines():
match = SCHEDULER_POLL_RE.match(line.strip())
if match:
values[match.group(1)] = int(match.group(2))
return values
def scrape_scheduler_polls(target: RunTarget, http_timeout: float) -> dict[str, Any]:
"""Best-effort datanode scheduler-poll snapshot (never a gate).
Returns {"captured_monotonic_seconds", "values": {workload: polls}} on
success or {"error", "values": {}} when the datanode /metrics endpoint is
unreachable or the scheduler metric is absent.
"""
try:
with urllib.request.urlopen(f"http://127.0.0.1:{target.datanode_http_port}/metrics", timeout=http_timeout) as response:
text = response.read().decode()
return {"captured_monotonic_seconds": time.monotonic(), "values": parse_scheduler_poll_metrics(text)}
except Exception as e: # noqa: BLE001 - best-effort diagnostic
return {"error": repr(e), "values": {}}
def scheduler_poll_deltas(after: dict[str, Any], before: dict[str, Any]) -> dict[str, Any]:
"""Delta of cumulative scheduler polls between two snapshots, per workload.
A workload whose counter is missing in either snapshot, or that decreased
(counter reset/restart), is reported as None because its delta is unknown.
"""
result: dict[str, Any] = {}
for workload in ("query", "write"):
before_value = before.get("values", {}).get(workload)
after_value = after.get("values", {}).get(workload)
if before_value is None or after_value is None:
result[workload] = None
else:
delta = after_value - before_value
result[workload] = delta if delta >= 0 else None
return result
def otelgen_command(otelgen_bin: Path, target: RunTarget, load: dict[str, Any]) -> list[str]:
return [
str(otelgen_bin),
@@ -1667,6 +1720,202 @@ def enforce_write_throughput_thresholds(write_measure: dict[str, Any], base: dic
return results
def mix_query_sql(mix: dict[str, Any], remote: dict[str, Any]) -> str:
"""Resolve the mixed-scenario query SQL.
Defaults to a count(*) over the remote-write physical table, which scans
every ingested row and therefore contends with the write path on the
datanode runtime.
"""
sql = mix.get("query_sql")
if sql:
return str(sql)
return f"SELECT count(*) FROM {sql_ident(remote['physical_table'])}"
def expected_mix_query_attempts(duration_seconds: int, interval_ms: int, parallelism: int) -> int:
"""Per-thread attempt count for a query loop: queries at t=0, interval,
2*interval, ... while t < duration. Wall-clock jitter may drop at most one
trailing query per thread, so the observed count is
[expected - parallelism, expected]."""
per_thread = math.ceil(max(0.0, float(duration_seconds)) * 1000.0 / max(1, int(interval_ms)))
return per_thread * max(1, int(parallelism))
def run_mix_query_loop(target: RunTarget, mix: dict[str, Any], db: str, duration_seconds: int, http_timeout: float, remote: dict[str, Any]) -> list[dict[str, Any]]:
"""Run `query_parallelism` query-loop threads for `duration_seconds`.
Each thread issues the mix query via HTTP SQL every `query_interval_ms`
(per-thread wall clock; if a query itself takes longer than the interval
the thread does not sleep). Returns the collected attempt results in
completion order; every attempt carries `ok`, `status`, `latency_ms`, and
`sql` from `http_post_sql`.
"""
query_sql = mix_query_sql(mix, remote)
interval_ms = int(mix.get("query_interval_ms", 100))
parallelism = max(1, int(mix.get("query_parallelism", 1)))
attempts: list[dict[str, Any]] = []
lock = threading.Lock()
stop = threading.Event()
deadline = time.monotonic() + max(0.0, float(duration_seconds))
threads = []
for _ in range(parallelism):
thread = threading.Thread(
target=_mix_query_worker,
args=(target, query_sql, db, interval_ms, deadline, http_timeout, attempts, lock, stop),
daemon=True,
)
threads.append(thread)
thread.start()
for thread in threads:
# Bounded join: workers may be stuck in a slow HTTP call beyond the
# deadline; their daemon status keeps process exit unblocked.
thread.join(timeout=max(10.0, float(interval_ms) / 1000.0 + 2.0))
return attempts
def _mix_query_worker(
target: RunTarget,
query_sql: str,
db: str,
interval_ms: int,
deadline: float,
http_timeout: float,
attempts: list[dict[str, Any]],
lock: threading.Lock,
stop: threading.Event,
) -> None:
interval_seconds = float(interval_ms) / 1000.0
while not stop.is_set():
started = time.monotonic()
if started >= deadline:
return
result = http_post_sql(target.http_port, query_sql, db, http_timeout)
with lock:
attempts.append(result)
remaining = interval_seconds - (time.monotonic() - started)
if remaining > 0 and stop.wait(remaining):
return
def mix_query_measurement(attempts: list[dict[str, Any]]) -> dict[str, Any]:
"""Aggregate query-loop attempts into {samples, failures, failure_rate,
p50_ms, p99_ms, mean_ms, latency_samples}.
Latency percentiles/mean cover every attempt that recorded a latency
(including failed ones, so timeouts show up in p99). `latency_samples`
counts those attempts. The input is snapshotted so a still-running daemon
query worker cannot mutate it mid-aggregation.
"""
attempts = list(attempts)
total = len(attempts)
failures = sum(1 for a in attempts if not a.get("ok"))
latencies = [float(a["latency_ms"]) for a in attempts if a.get("latency_ms") is not None]
return {
"samples": total,
"failures": failures,
"failure_rate": failures / total if total else None,
"p50_ms": percentile(latencies, 50) if latencies else None,
"p99_ms": percentile(latencies, 99) if latencies else None,
"mean_ms": statistics.mean(latencies) if latencies else None,
"latency_samples": len(latencies),
}
def planned_mix_query_measurement(mix: dict[str, Any], write_measure: dict[str, Any]) -> dict[str, Any]:
"""Dry-run query measurement: no query loop ran, so report planned counts."""
duration_seconds = int(write_measure["duration_seconds"])
interval_ms = int(mix.get("query_interval_ms", 100))
parallelism = int(mix.get("query_parallelism", 1))
return {
"status": "planned",
"planned_attempts": expected_mix_query_attempts(duration_seconds, interval_ms, parallelism),
"query_interval_ms": interval_ms,
"query_parallelism": parallelism,
"samples": 0,
"failures": 0,
"failure_rate": None,
"p50_ms": None,
"p99_ms": None,
"mean_ms": None,
"latency_samples": 0,
}
def run_mixed_ingestion_and_queries(generator: Path | None, target: RunTarget, remote: dict[str, Any], args: argparse.Namespace, mix: dict[str, Any], write_measure: dict[str, Any], *, dry_run: bool) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]:
"""Run remote-write ingestion in a background thread while a query loop
hammers the same frontend concurrently for `duration_seconds`, then join.
The ingestion is the same chunked `run_write_throughput_ingestion`
machinery, just executed from a thread; each chunk is its own
`query_perf_fixture prom-remote-write` subprocess (generate+send in one
shot), and the query loop issues HTTP /v1/sql requests from the main
process, so both hit the same frontend/datanode and genuinely contend on
the datanode runtime. Returns (rw, flushes, query_attempts); in dry-run
mode no thread and no query loop is started and attempts is empty.
"""
if dry_run:
rw, flushes = run_write_throughput_ingestion(generator, target, remote, args, dry_run=True)
return rw, flushes, []
results: dict[str, Any] = {}
errors: list[BaseException] = []
def ingest() -> None:
try:
results["rw"], results["flushes"] = run_write_throughput_ingestion(generator, target, remote, args, dry_run=False)
except BaseException as e: # noqa: BLE001 - re-raised in the caller
errors.append(e)
thread = threading.Thread(target=ingest, name=f"{target.name}-ingest", daemon=True)
thread.start()
try:
attempts = run_mix_query_loop(target, mix, remote["database"], int(write_measure["duration_seconds"]), args.http_timeout, remote)
finally:
thread.join()
if thread.is_alive():
# The datanode may be wedged; the ingestion thread is daemon so it
# cannot block process exit, but surface the lack of results.
raise RuntimeError(f"write ingestion thread did not finish for {target.name}")
if errors:
raise errors[0]
return results["rw"], results["flushes"], attempts
def planned_mix_query_thresholds(mix: dict[str, Any]) -> list[dict[str, Any]]:
thresholds = mix["thresholds"]
return [
{"threshold": "max_query_failure_rate", "status": "planned", "limit": thresholds["max_query_failure_rate"]},
{"threshold": "max_query_p99_regression_pct", "status": "planned", "limit_pct": thresholds["max_query_p99_regression_pct"]},
]
def enforce_mix_query_thresholds(mix: dict[str, Any], base: dict[str, Any], candidate: dict[str, Any]) -> list[dict[str, Any]]:
"""Enforce the query-side gates of the mixed read/write scenario.
Per-target: `max_query_failure_rate` (failed query attempts / total).
Base-vs-candidate: `max_query_p99_regression_pct`
`(candidate_p99_ms - base_p99_ms) / base_p99_ms * 100`; positive actual
means a candidate regression, negative is an improvement.
"""
thresholds = mix["thresholds"]
results: list[dict[str, Any]] = []
failure_limit = float(thresholds["max_query_failure_rate"])
for target_name, measurement in (("base", base), ("candidate", candidate)):
failure_rate = measurement.get("failure_rate")
results.append({"target": target_name, "threshold": "max_query_failure_rate", "status": "passed" if failure_rate is not None and failure_rate <= failure_limit else "failed", "actual": failure_rate, "limit": failure_limit})
base_p99 = base.get("p99_ms")
candidate_p99 = candidate.get("p99_ms")
p99_limit = float(thresholds["max_query_p99_regression_pct"])
if base_p99 in (None, 0) or candidate_p99 is None:
results.append({"threshold": "max_query_p99_regression_pct", "status": "failed", "reason": "missing or zero query p99 latency", "base": base_p99, "candidate": candidate_p99})
else:
actual = (candidate_p99 - base_p99) / base_p99 * 100.0
results.append({"threshold": "max_query_p99_regression_pct", "status": "passed" if actual <= p99_limit else "failed", "actual_pct": actual, "limit_pct": p99_limit, "base": base_p99, "candidate": candidate_p99})
return results
def extract_count_value(result: dict[str, Any]) -> int | None:
body = result.get("response")
if not isinstance(body, dict):
@@ -1790,6 +2039,7 @@ def run_write_throughput_scenario(args: argparse.Namespace, case: dict[str, Any]
clusters: list[DistributedCluster] = []
measurements: list[dict[str, Any]] = []
query_measurements: list[dict[str, Any]] = []
try:
for target in targets:
target.work_dir.mkdir(parents=True, exist_ok=True)
@@ -1802,7 +2052,19 @@ def run_write_throughput_scenario(args: argparse.Namespace, case: dict[str, Any]
create_database: dict[str, Any] = {"status": "dry-run", "database": db}
if not args.dry_run:
create_database = http_post_sql(target.http_port, f"CREATE DATABASE IF NOT EXISTS {sql_ident(db)}", "public", args.http_timeout)
rw, flushes = run_write_throughput_ingestion(helper, target, remote, args, dry_run=args.dry_run)
mix = write_measure.get("mix")
scheduler_polls_before: dict[str, Any] | None = None
scheduler_polls_after: dict[str, Any] | None = None
if mix is not None and not args.dry_run:
scheduler_polls_before = scrape_scheduler_polls(target, args.http_timeout)
if mix is not None:
rw, flushes, query_attempts = run_mixed_ingestion_and_queries(helper, target, remote, args, mix, write_measure, dry_run=args.dry_run)
query_measurement = planned_mix_query_measurement(mix, write_measure) if args.dry_run else mix_query_measurement(query_attempts)
if not args.dry_run:
scheduler_polls_after = scrape_scheduler_polls(target, args.http_timeout)
else:
rw, flushes = run_write_throughput_ingestion(helper, target, remote, args, dry_run=args.dry_run)
query_measurement = None
measurement = planned_write_throughput_measurement(remote, write_measure) if args.dry_run else write_throughput_measurement(rw, write_measure)
tr = {
"name": target.name,
@@ -1817,17 +2079,36 @@ def run_write_throughput_scenario(args: argparse.Namespace, case: dict[str, Any]
"scheduler": scheduler_report_entry(target.name, scenario_config.get("scheduler")),
"write_measurement": measurement,
}
if query_measurement is not None:
tr["query_measurement"] = query_measurement
if mix is not None:
tr["mix"] = mix
tr["scheduler_poll_deltas"] = scheduler_poll_deltas(scheduler_polls_after, scheduler_polls_before) if scheduler_polls_after is not None else {"status": "planned"}
flushes_ok = all(flush.get("ok") for flush in flushes)
measurement_ok = measurement.get("mean_rps") not in (None, 0)
checks_ok = args.dry_run or (create_database.get("ok") and flushes_ok and measurement_ok)
query_ok = query_measurement is None or query_measurement.get("samples", 0) > 0
checks_ok = args.dry_run or (create_database.get("ok") and flushes_ok and measurement_ok and query_ok)
if not checks_ok and not args.dry_run:
tr.setdefault("validation_errors", []).append({"phase": "write_throughput", "create_database_ok": create_database.get("ok"), "flushes_ok": flushes_ok, "measurement_ok": measurement_ok, "mean_rps": measurement.get("mean_rps"), "failure_rate": measurement.get("failure_rate")})
validation = {"phase": "write_throughput", "create_database_ok": create_database.get("ok"), "flushes_ok": flushes_ok, "measurement_ok": measurement_ok, "mean_rps": measurement.get("mean_rps"), "failure_rate": measurement.get("failure_rate")}
if query_measurement is not None:
validation["query_ok"] = query_ok
validation["query_samples"] = query_measurement.get("samples")
tr.setdefault("validation_errors", []).append(validation)
tr["status"] = "planned" if args.dry_run else ("measured" if checks_ok else "failed")
write_json(target.report_path, tr)
report["targets"].append(tr)
measurements.append(measurement)
if query_measurement is not None:
query_measurements.append(query_measurement)
cluster.stop_all()
report["thresholds"] = planned_write_throughput_thresholds(write_measure) if args.dry_run else enforce_write_throughput_thresholds(write_measure, measurements[0], measurements[1])
if args.dry_run:
report["thresholds"] = planned_write_throughput_thresholds(write_measure)
if write_measure.get("mix") is not None:
report["thresholds"] += planned_mix_query_thresholds(write_measure["mix"])
else:
report["thresholds"] = enforce_write_throughput_thresholds(write_measure, measurements[0], measurements[1])
if write_measure.get("mix") is not None:
report["thresholds"] += enforce_mix_query_thresholds(write_measure["mix"], query_measurements[0], query_measurements[1])
report["status"] = "planned" if args.dry_run else ("failed" if any(t["status"] == "failed" for t in report["thresholds"]) or any(t.get("status") == "failed" for t in report["targets"]) else "ok")
finally:
for cluster in reversed(clusters):
+248
View File
@@ -23,6 +23,7 @@ dry-run path. No live database, cluster, or generator subprocess is started.
import importlib.util
import sys
import tempfile
import time
import unittest
from pathlib import Path
from unittest.mock import patch
@@ -79,6 +80,19 @@ def make_write_measure(**overrides):
return write_measure
def make_mix(**overrides):
mix = {
"query_interval_ms": 100,
"query_parallelism": 2,
"thresholds": {
"max_query_failure_rate": 0.05,
"max_query_p99_regression_pct": 10.0,
},
}
mix.update(overrides)
return mix
class WriteThroughputDispatchTest(unittest.TestCase):
def test_scenario_accepts_write_throughput_kind(self) -> None:
case = {"scenario": {"kind": "write_throughput", "remote_write": {}, "write_measure": {}}}
@@ -406,5 +420,239 @@ class WriteThroughputScenarioDryRunTest(unittest.TestCase):
self.assertTrue(targets[0].report_path.exists())
class WriteThroughputMixQueryTest(unittest.TestCase):
"""Pure-function coverage for the mixed read/write query loop."""
def test_expected_attempts_math(self) -> None:
self.assertEqual(runner.expected_mix_query_attempts(60, 100, 2), 1200)
self.assertEqual(runner.expected_mix_query_attempts(10, 3000, 1), 4)
self.assertEqual(runner.expected_mix_query_attempts(60, 1000, 4), 240)
# Degenerate parallelism is clamped to 1 (interval is already
# guaranteed > 0 by the Rust schema; the clamp is defensive).
self.assertEqual(runner.expected_mix_query_attempts(60, 100, 0), 600)
def test_mix_query_sql_default_and_override(self) -> None:
remote = make_remote(physical_table="greptime_physical_table")
mix = make_mix()
self.assertEqual(runner.mix_query_sql(mix, remote), 'SELECT count(*) FROM "greptime_physical_table"')
mix["query_sql"] = "SELECT max(greptime_value) FROM greptime_physical_table"
self.assertEqual(runner.mix_query_sql(mix, remote), "SELECT max(greptime_value) FROM greptime_physical_table")
def test_query_loop_runs_and_collects_attempts(self) -> None:
target = runner.RunTarget("base", Path("/bin/true"), Path("/tmp/wt"), Path("/tmp/wt/data"), Path("/tmp/wt/fixture"), Path("/tmp/wt/report.json"), 4000, 4001, 4002, 4003, 4004, 4005, 4006, 4007, Path("/tmp/wt/datanode/data"))
mix = make_mix(query_interval_ms=100, query_parallelism=2)
def fake_http(port, sql, db, timeout):
return {"ok": True, "status": 200, "latency_ms": 5.0, "response": {"data": []}, "sql": sql}
with patch.object(runner, "http_post_sql", side_effect=fake_http):
attempts = runner.run_mix_query_loop(target, mix, "public", duration_seconds=1, http_timeout=5.0, remote=make_remote())
# 1s at 100ms per thread = 10 attempts/thread; a delayed thread may
# drop its final query, but both threads must have run repeatedly.
self.assertGreaterEqual(len(attempts), 12)
self.assertLessEqual(len(attempts), 20)
self.assertTrue(all(a["ok"] for a in attempts))
self.assertTrue(all(a["sql"] == 'SELECT count(*) FROM "greptime_physical_table"' for a in attempts))
def test_mixed_ingestion_and_queries_join_thread(self) -> None:
target = runner.RunTarget("base", Path("/bin/true"), Path("/tmp/wt"), Path("/tmp/wt/data"), Path("/tmp/wt/fixture"), Path("/tmp/wt/report.json"), 4000, 4001, 4002, 4003, 4004, 4005, 4006, 4007, Path("/tmp/wt/datanode/data"))
args = runner.argparse.Namespace(http_timeout=5.0)
remote = make_remote()
def fake_ingest(generator, target, remote, args, *, dry_run):
time.sleep(0.05)
return {"status": "ok", "chunks": []}, [{"ok": True}]
with (
patch.object(runner, "run_write_throughput_ingestion", side_effect=fake_ingest),
patch.object(runner, "run_mix_query_loop", return_value=[{"ok": True, "latency_ms": 1.0}]),
):
rw, flushes, attempts = runner.run_mixed_ingestion_and_queries(None, target, remote, args, make_mix(), make_write_measure(), dry_run=False)
self.assertEqual(rw["status"], "ok")
self.assertEqual(flushes, [{"ok": True}])
self.assertEqual(attempts, [{"ok": True, "latency_ms": 1.0}])
def test_mixed_dry_run_skips_threads(self) -> None:
target = runner.RunTarget("base", Path("/bin/true"), Path("/tmp/wt"), Path("/tmp/wt/data"), Path("/tmp/wt/fixture"), Path("/tmp/wt/report.json"), 4000, 4001, 4002, 4003, 4004, 4005, 4006, 4007, Path("/tmp/wt/datanode/data"))
args = runner.argparse.Namespace(http_timeout=5.0)
with (
patch.object(runner, "run_write_throughput_ingestion", return_value=({"status": "ok"}, [{"ok": True}])) as ingest,
patch.object(runner.threading, "Thread") as thread_cls,
):
rw, flushes, attempts = runner.run_mixed_ingestion_and_queries(None, target, make_remote(), args, make_mix(), make_write_measure(), dry_run=True)
thread_cls.assert_not_called()
ingest.assert_called_once()
self.assertEqual(attempts, [])
class WriteThroughputMixMeasurementTest(unittest.TestCase):
def test_measurement_aggregates_latency_and_failures(self) -> None:
attempts = [
{"ok": True, "latency_ms": 10.0},
{"ok": True, "latency_ms": 20.0},
{"ok": True, "latency_ms": 30.0},
{"ok": False, "latency_ms": 40.0},
{"ok": False, "latency_ms": 50.0},
]
measurement = runner.mix_query_measurement(attempts)
self.assertEqual(measurement["samples"], 5)
self.assertEqual(measurement["failures"], 2)
self.assertEqual(measurement["failure_rate"], 0.4)
self.assertEqual(measurement["latency_samples"], 5)
self.assertEqual(measurement["p50_ms"], 30.0)
self.assertEqual(measurement["p99_ms"], 50.0)
self.assertAlmostEqual(measurement["mean_ms"], 30.0)
def test_measurement_empty_attempts(self) -> None:
measurement = runner.mix_query_measurement([])
self.assertEqual(measurement["samples"], 0)
self.assertEqual(measurement["failures"], 0)
self.assertIsNone(measurement["failure_rate"])
self.assertIsNone(measurement["p50_ms"])
self.assertIsNone(measurement["p99_ms"])
self.assertIsNone(measurement["mean_ms"])
def test_planned_measurement_reports_expected_attempts(self) -> None:
measurement = runner.planned_mix_query_measurement(make_mix(), make_write_measure())
self.assertEqual(measurement["status"], "planned")
self.assertEqual(measurement["planned_attempts"], 1200)
self.assertEqual(measurement["query_interval_ms"], 100)
self.assertEqual(measurement["query_parallelism"], 2)
self.assertIsNone(measurement["p99_ms"])
class WriteThroughputMixThresholdTest(unittest.TestCase):
def test_combined_write_and_query_gates(self) -> None:
write_base = {"mean_rps": 100_000, "p99_latency_ms": 50.0, "failure_rate": 0.0}
write_candidate = {"mean_rps": 90_000, "p99_latency_ms": 55.0, "failure_rate": 0.02}
query_base = {"p99_ms": 200.0, "failure_rate": 0.0}
query_candidate = {"p99_ms": 220.0, "failure_rate": 0.01}
write_results = runner.enforce_write_throughput_thresholds(make_write_measure(), write_base, write_candidate)
query_results = runner.enforce_mix_query_thresholds(make_mix(), query_base, query_candidate)
combined = write_results + query_results
by_key = {(r.get("target"), r["threshold"]): r for r in combined}
self.assertEqual(len(combined), 9)
# Write gates unchanged.
self.assertEqual(by_key[(None, "max_mean_rps_regression_pct")]["status"], "passed")
self.assertEqual(by_key[(None, "max_p99_latency_regression_pct")]["status"], "passed")
# Query gates pass within limits.
self.assertEqual(by_key[("base", "max_query_failure_rate")]["status"], "passed")
self.assertEqual(by_key[("candidate", "max_query_failure_rate")]["status"], "passed")
q99 = by_key[(None, "max_query_p99_regression_pct")]
self.assertEqual(q99["status"], "passed")
self.assertAlmostEqual(q99["actual_pct"], 10.0)
self.assertAlmostEqual(q99["limit_pct"], 10.0)
def test_query_gates_fail_on_regression_and_failures(self) -> None:
query_base = {"p99_ms": 200.0, "failure_rate": 0.0}
query_candidate = {"p99_ms": 260.0, "failure_rate": 0.10}
results = runner.enforce_mix_query_thresholds(make_mix(), query_base, query_candidate)
by_key = {(r.get("target"), r["threshold"]): r for r in results}
self.assertEqual(by_key[("candidate", "max_query_failure_rate")]["status"], "failed")
self.assertEqual(by_key[(None, "max_query_p99_regression_pct")]["status"], "failed")
self.assertAlmostEqual(by_key[(None, "max_query_p99_regression_pct")]["actual_pct"], 30.0)
def test_query_gates_fail_on_missing_base(self) -> None:
query_base = {"p99_ms": None, "failure_rate": None}
query_candidate = {"p99_ms": 260.0, "failure_rate": 0.0}
results = runner.enforce_mix_query_thresholds(make_mix(), query_base, query_candidate)
by_key = {(r.get("target"), r["threshold"]): r for r in results}
self.assertEqual(by_key[("base", "max_query_failure_rate")]["status"], "failed")
self.assertEqual(by_key[(None, "max_query_p99_regression_pct")]["status"], "failed")
self.assertEqual(by_key[(None, "max_query_p99_regression_pct")]["reason"], "missing or zero query p99 latency")
def test_planned_mix_query_thresholds(self) -> None:
planned = runner.planned_mix_query_thresholds(make_mix())
self.assertEqual([p["threshold"] for p in planned], ["max_query_failure_rate", "max_query_p99_regression_pct"])
self.assertTrue(all(p["status"] == "planned" for p in planned))
class SchedulerPollMetricsTest(unittest.TestCase):
def test_parse_scheduler_poll_metrics(self) -> None:
text = (
"# HELP greptime_workload_scheduler_polls Cumulative task polls admitted by the workload scheduler\n"
"# TYPE greptime_workload_scheduler_polls gauge\n"
"greptime_workload_scheduler_polls{workload=\"query\"} 1234\n"
"greptime_workload_scheduler_polls{workload=\"write\"} 5678\n"
"greptime_workload_scheduler_queued_tasks{workload=\"query\"} 3\n"
"greptime_runtime_threads_alive{thread_name=\"global\"} 8\n"
)
self.assertEqual(runner.parse_scheduler_poll_metrics(text), {"query": 1234, "write": 5678})
# Scheduler disabled: no samples at all.
self.assertEqual(runner.parse_scheduler_poll_metrics("greptime_runtime_threads_alive 8\n"), {})
def test_scheduler_poll_deltas(self) -> None:
before = {"values": {"query": 100, "write": 900}}
after = {"values": {"query": 180, "write": 1780}}
self.assertEqual(runner.scheduler_poll_deltas(after, before), {"query": 80, "write": 880})
def test_scheduler_poll_deltas_unknown_on_missing_or_reset(self) -> None:
before = {"values": {"query": 100}}
after = {"values": {"query": 180, "write": 20}}
# write absent before -> unknown; write decreased -> unknown (reset).
self.assertEqual(runner.scheduler_poll_deltas(after, before), {"query": 80, "write": None})
before = {"values": {"query": 100, "write": 900}}
after = {"values": {}}
self.assertEqual(runner.scheduler_poll_deltas(after, before), {"query": None, "write": None})
class WriteThroughputMixScenarioDryRunTest(unittest.TestCase):
def test_dry_run_with_mix_plans_write_and_query_gates(self) -> None:
events = []
class FakeCluster:
def __init__(self, target):
self.target = target
self.stopped = False
events.append(f"create:{self.target.name}")
def component_report(self):
return {}
def stop_all(self):
if not self.stopped:
self.stopped = True
events.append(f"stop:{self.target.name}")
args = runner.argparse.Namespace(
fixture_only=False,
fixture_generator=Path("query_perf_fixture"),
remote_write_generator=None,
dry_run=True,
http_timeout=1.0,
)
write_measure = make_write_measure()
write_measure["mix"] = make_mix()
case = {"scenario": {"kind": "write_throughput", "remote_write": make_remote(), "write_measure": write_measure}}
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
targets = [
runner.make_target("base", Path("/bin/true"), root, list(range(10_000, 10_008))),
runner.make_target("candidate", Path("/bin/true"), root, list(range(10_008, 10_016))),
]
report = {"targets": []}
with (
patch.object(runner, "DistributedCluster", FakeCluster),
patch.object(runner, "run_command") as run_cmd,
):
runner.run_write_throughput_scenario(args, case, Path("case.toml"), targets, report)
run_cmd.assert_not_called()
self.assertEqual(report["status"], "planned")
self.assertEqual(len(report["thresholds"]), 6)
self.assertTrue(all(t["status"] == "planned" for t in report["thresholds"]))
self.assertEqual(
[t["threshold"] for t in report["thresholds"]],
["max_failure_rate", "max_mean_rps_regression_pct", "max_p99_latency_regression_pct", "min_rps_absolute", "max_query_failure_rate", "max_query_p99_regression_pct"],
)
for tr in report["targets"]:
self.assertEqual(tr["write_measurement"]["status"], "planned")
self.assertEqual(tr["query_measurement"]["status"], "planned")
self.assertEqual(tr["query_measurement"]["planned_attempts"], 1200)
self.assertEqual(tr["scheduler_poll_deltas"], {"status": "planned"})
self.assertEqual(tr["mix"]["query_parallelism"], 2)
if __name__ == "__main__":
unittest.main()