mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-14 14:56:54 +00:00
chore: merge narthex into feat/narthex-range-modifiers
This commit is contained in:
Dennis Zhuang
@@ -57,6 +57,7 @@ runs:
|
||||
# Without GT_S3_*, every remote object-store case is skipped and the
|
||||
# release only ever exercises the local file backend.
|
||||
env:
|
||||
RUST_MIN_STACK: 8388608 # 8 MiB, aligned with Rust CI tests.
|
||||
GT_S3_BUCKET: ${{ inputs.aws-ci-test-bucket }}
|
||||
GT_S3_REGION: ${{ inputs.aws-region }}
|
||||
GT_S3_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
|
||||
|
||||
@@ -64,6 +64,7 @@ runs:
|
||||
shell: bash
|
||||
env:
|
||||
CARGO_BUILD_RUSTFLAGS: "-Clink-arg=-Wl,-ld_classic"
|
||||
RUST_MIN_STACK: 8388608 # 8 MiB, aligned with Rust CI and release tests.
|
||||
SQLNESS_OPTS: "--preserve-state"
|
||||
run: |
|
||||
make test sqlness-test
|
||||
|
||||
@@ -22,7 +22,9 @@ inputs:
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
tool: protoc@3.23.4
|
||||
|
||||
- name: Install rust toolchain
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
@@ -48,6 +50,7 @@ runs:
|
||||
run: make test sqlness-test
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
RUST_MIN_STACK: 8388608 # 8 MiB, aligned with Rust CI tests.
|
||||
SQLNESS_OPTS: "--preserve-state"
|
||||
|
||||
- name: Upload sqlness logs
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
datanode:
|
||||
storage:
|
||||
storageClassName: standard
|
||||
|
||||
tracing:
|
||||
enabled: true
|
||||
endpoint: "http://my-greptimedb-monitor-standalone.my-greptimedb:4000/v1/otlp/v1/traces"
|
||||
@@ -15,9 +19,9 @@ monitoring:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 256Mi
|
||||
datanodeStorage:
|
||||
fs:
|
||||
storageSize: 5Gi
|
||||
datanodeStorage:
|
||||
fs:
|
||||
storageSize: 5Gi
|
||||
vector:
|
||||
resources:
|
||||
requests:
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Deterministic wide, heterogeneous Metric tables for snapshot benchmarks."""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
|
||||
GROUPS = 10
|
||||
TAG_POOL = 298
|
||||
ROWS = 32
|
||||
DATABASE = "metrics"
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, addr):
|
||||
self.addr = addr
|
||||
self.connections = threading.local()
|
||||
self.headers = {"Content-Type": "application/x-www-form-urlencoded"}
|
||||
if auth := os.environ.get("GREPTIME_BENCH_AUTH"):
|
||||
self.headers["Authorization"] = "Basic " + base64.b64encode(
|
||||
auth.encode()
|
||||
).decode()
|
||||
|
||||
def sql(self, query, database=DATABASE, batch=False):
|
||||
connection = getattr(self.connections, "connection", None)
|
||||
if connection is None:
|
||||
connection = http.client.HTTPConnection(self.addr, timeout=600)
|
||||
self.connections.connection = connection
|
||||
body = urllib.parse.urlencode({"db": database, "sql": query})
|
||||
try:
|
||||
connection.request(
|
||||
"POST", "/v1/ddl/logical-tables" if batch else "/v1/sql",
|
||||
body, self.headers,
|
||||
)
|
||||
response = connection.getresponse()
|
||||
raw = response.read()
|
||||
except (OSError, http.client.HTTPException):
|
||||
connection.close()
|
||||
self.connections.connection = None
|
||||
raise
|
||||
data = json.loads(raw)
|
||||
if response.status != 200 or "error" in data:
|
||||
raise RuntimeError(f"SQL failed (HTTP {response.status}): {data}")
|
||||
return data["output"]
|
||||
|
||||
def values(self, query, database=DATABASE):
|
||||
return self.sql(query, database)[0]["records"]["rows"]
|
||||
|
||||
|
||||
def tags(index, per_group):
|
||||
return sorted((index % per_group * 7 + offset) % TAG_POOL for offset in range(10))
|
||||
|
||||
|
||||
def column_names(tag_ids):
|
||||
return ["ts", *[f"tag_{tag:03d}" for tag in tag_ids], "val"]
|
||||
|
||||
|
||||
def columns(tag_ids):
|
||||
names = [f"tag_{tag:03d}" for tag in tag_ids]
|
||||
return (
|
||||
"ts TIMESTAMP TIME INDEX, "
|
||||
+ ", ".join(name + " STRING" for name in names)
|
||||
+ ", val DOUBLE, PRIMARY KEY(" + ",".join(names) + ")"
|
||||
)
|
||||
|
||||
|
||||
def table_ddl(index, per_group):
|
||||
return (
|
||||
f"CREATE TABLE m{index:06d} ({columns(tags(index, per_group))}) "
|
||||
f"ENGINE=metric WITH(on_physical_table='phy_{index // per_group:02d}')"
|
||||
)
|
||||
|
||||
|
||||
def row(index, row_index, per_group):
|
||||
values = []
|
||||
for tag in tags(index, per_group):
|
||||
if (row_index + tag) % 29 == 0:
|
||||
value = None
|
||||
elif (row_index + tag) % 17 == 0:
|
||||
value = ""
|
||||
elif tag % 10 == 0:
|
||||
value = f"instance-{index:06d}-{row_index % 4}"
|
||||
elif tag % 10 == 1:
|
||||
value = "区域-" + str(row_index % 3)
|
||||
elif tag % 10 == 2:
|
||||
value = "label-" + "x" * 48 + "-" + str(row_index % 2)
|
||||
else:
|
||||
value = f"tag{tag:03d}-v{row_index % (2 + tag % 7)}"
|
||||
values.append(value)
|
||||
return [row_index, *values, None if row_index % 7 == 0 else index % 97 + row_index / 4]
|
||||
|
||||
|
||||
def sql_value(value):
|
||||
if value is None:
|
||||
return "NULL"
|
||||
if isinstance(value, str):
|
||||
return "'" + value.replace("'", "''") + "'"
|
||||
return str(value)
|
||||
|
||||
|
||||
def canonical(value):
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def update_digest(digest, value):
|
||||
digest.update((canonical(value) + "\n").encode())
|
||||
|
||||
|
||||
def progress(phase, completed, total):
|
||||
if completed == total or completed % 4096 == 0:
|
||||
print(f"{phase}: {completed}/{total}", flush=True)
|
||||
|
||||
|
||||
def prepare(client, per_group):
|
||||
stages = {}
|
||||
started = time.monotonic()
|
||||
client.sql(f"CREATE DATABASE {DATABASE}", "public")
|
||||
for group in range(GROUPS):
|
||||
client.sql(
|
||||
f"CREATE TABLE phy_{group:02d} ({columns(range(TAG_POOL))}) "
|
||||
"ENGINE=metric WITH(physical_metric_table='')"
|
||||
)
|
||||
total = GROUPS * per_group
|
||||
for group in range(GROUPS):
|
||||
for base in range(group * per_group, (group + 1) * per_group, 128):
|
||||
end = min((group + 1) * per_group, base + 128)
|
||||
client.sql(";".join(table_ddl(i, per_group) for i in range(base, end)), batch=True)
|
||||
progress("create", end, total)
|
||||
stages["create_seconds"] = time.monotonic() - started
|
||||
|
||||
def insert(index):
|
||||
rows = ",".join(
|
||||
"(" + ",".join(sql_value(v) for v in row(index, j, per_group)) + ")"
|
||||
for j in range(ROWS)
|
||||
)
|
||||
names = ",".join(column_names(tags(index, per_group)))
|
||||
client.sql(f"INSERT INTO m{index:06d} ({names}) VALUES {rows}")
|
||||
|
||||
started = time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
||||
for base in range(0, total, 128):
|
||||
end = min(total, base + 128)
|
||||
list(pool.map(insert, range(base, end)))
|
||||
progress("insert", end, total)
|
||||
stages["insert_seconds"] = time.monotonic() - started
|
||||
started = time.monotonic()
|
||||
for group in range(GROUPS):
|
||||
client.sql(f"ADMIN FLUSH_TABLE('{DATABASE}.phy_{group:02d}')")
|
||||
stages["flush_seconds"] = time.monotonic() - started
|
||||
return stages
|
||||
|
||||
|
||||
def check_records(index, per_group, records):
|
||||
names = column_names(tags(index, per_group))
|
||||
types = ["TimestampMillisecond", *["String"] * 10, "Float64"]
|
||||
expected_columns = [dict(name=name, data_type=kind) for name, kind in zip(names, types)]
|
||||
actual_columns = records["schema"]["column_schemas"]
|
||||
if sorted(actual_columns, key=lambda c: c["name"]) != sorted(
|
||||
expected_columns, key=lambda c: c["name"]
|
||||
):
|
||||
raise RuntimeError(f"logical schema mismatch: m{index:06d}")
|
||||
# Metric tables canonicalize column order; SELECT * still checks every column.
|
||||
positions = [names.index(column["name"]) for column in actual_columns]
|
||||
expected_rows = [
|
||||
[values[position] for position in positions]
|
||||
for values in (row(index, j, per_group) for j in range(ROWS))
|
||||
]
|
||||
# Canonical JSON also distinguishes integers, floats, booleans, and NULL.
|
||||
if canonical(records["rows"]) != canonical(expected_rows):
|
||||
raise RuntimeError(f"typed values/NULL mismatch: m{index:06d}")
|
||||
|
||||
|
||||
def validate(client, per_group):
|
||||
total = GROUPS * per_group
|
||||
expected_names = {f"m{i:06d}" for i in range(total)} | {
|
||||
f"phy_{group:02d}" for group in range(GROUPS)
|
||||
}
|
||||
table_rows = client.values(
|
||||
"SELECT table_name, table_id FROM information_schema.tables "
|
||||
f"WHERE table_schema='{DATABASE}' ORDER BY table_name"
|
||||
)
|
||||
table_ids = dict(table_rows)
|
||||
if len(table_rows) != len(expected_names) or set(table_ids) != expected_names:
|
||||
raise RuntimeError("table membership mismatch")
|
||||
if len(set(table_ids.values())) != len(table_ids):
|
||||
raise RuntimeError("duplicate table IDs")
|
||||
digests = {name: hashlib.sha256() for name in ("schema", "data", "ddl")}
|
||||
|
||||
def check_ddl(names):
|
||||
outputs = client.sql(";".join(f"SHOW CREATE TABLE {name}" for name in names))
|
||||
if len(outputs) != len(names):
|
||||
raise RuntimeError("incomplete SHOW CREATE TABLE response")
|
||||
for name, output in zip(names, outputs):
|
||||
records = output["records"]["rows"]
|
||||
if len(records) != 1 or records[0][0] != name or not records[0][1]:
|
||||
raise RuntimeError(f"invalid SHOW CREATE TABLE response: {name}")
|
||||
update_digest(digests["ddl"], records)
|
||||
|
||||
for base in range(0, total, 64):
|
||||
end = min(total, base + 64)
|
||||
names = [f"m{i:06d}" for i in range(base, end)]
|
||||
outputs = client.sql(";".join(f"SELECT * FROM {name} ORDER BY ts" for name in names))
|
||||
if len(outputs) != len(names):
|
||||
raise RuntimeError("incomplete SELECT response")
|
||||
for index, output in enumerate(outputs, base):
|
||||
records = output["records"]
|
||||
check_records(index, per_group, records)
|
||||
update_digest(digests["schema"], [f"m{index:06d}", records["schema"]])
|
||||
update_digest(digests["data"], [f"m{index:06d}", records["rows"]])
|
||||
check_ddl(names)
|
||||
progress("validate", end, total)
|
||||
check_ddl([f"phy_{group:02d}" for group in range(GROUPS)])
|
||||
return {**{name + "_sha256": digest.hexdigest() for name, digest in digests.items()},
|
||||
"table_ids": table_ids}
|
||||
|
||||
|
||||
def compare_source(result, source):
|
||||
for field in ("fixture", "schema_sha256", "data_sha256", "ddl_sha256"):
|
||||
if result[field] != source[field]:
|
||||
raise RuntimeError(f"source/target mismatch: {field}")
|
||||
source_ids, target_ids = source["table_ids"], result["table_ids"]
|
||||
if source_ids.keys() != target_ids.keys():
|
||||
raise RuntimeError("source/target table membership mismatch")
|
||||
changed = sum(source_ids[name] != target_ids[name] for name in source_ids)
|
||||
if changed != len(source_ids):
|
||||
raise RuntimeError(f"table IDs not all remapped: {changed}/{len(source_ids)}")
|
||||
result["remapped_table_ids"] = changed
|
||||
|
||||
|
||||
def placeholders(client):
|
||||
database = "metric_bench_guard"
|
||||
client.sql(f"CREATE DATABASE {database}", "public")
|
||||
for index in range(17):
|
||||
client.sql(f"CREATE TABLE placeholder_{index} (ts TIMESTAMP TIME INDEX)", database)
|
||||
return {"placeholder_tables": 17}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("action", choices=("prepare", "validate", "placeholders"))
|
||||
parser.add_argument("--addr", default="127.0.0.1:4000")
|
||||
parser.add_argument("--tables-per-group", type=int, default=10000)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--source", type=Path, help="Source validation JSON, for target comparison")
|
||||
args = parser.parse_args()
|
||||
if not 50 <= args.tables_per_group <= 10000:
|
||||
parser.error("--tables-per-group must be between 50 and 10000")
|
||||
if args.source is not None and args.action != "validate":
|
||||
parser.error("--source only applies to validate")
|
||||
coverage = {tag for i in range(args.tables_per_group) for tag in tags(i, args.tables_per_group)}
|
||||
if coverage != set(range(TAG_POOL)):
|
||||
parser.error("fixture does not cover all 298 tags")
|
||||
client = Client(args.addr)
|
||||
started = time.monotonic()
|
||||
result = {"fixture": {"groups": GROUPS, "tables_per_group": args.tables_per_group,
|
||||
"rows_per_table": ROWS, "tag_pool": TAG_POOL}}
|
||||
if args.action == "placeholders":
|
||||
result.update(placeholders(client))
|
||||
elif args.action == "prepare":
|
||||
result.update(prepare(client, args.tables_per_group))
|
||||
else:
|
||||
result.update(validate(client, args.tables_per_group))
|
||||
result["wall_seconds"] = time.monotonic() - started
|
||||
if args.source is not None:
|
||||
compare_source(result, json.loads(args.source.read_text()))
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps({key: value for key, value in result.items() if key != "table_ids"}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,392 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Run one packed Metric snapshot smoke, then one full S3 round trip."""
|
||||
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def command(args, **kwargs):
|
||||
result = subprocess.run(args, text=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, **kwargs)
|
||||
if result.returncode:
|
||||
# CLI argv and error output can contain storage credentials.
|
||||
raise RuntimeError(f"{args[0]} exited with status {result.returncode}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def write_json(path, value):
|
||||
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def metric_values(text):
|
||||
values = {}
|
||||
for line in text.splitlines():
|
||||
if line and not line.startswith("#"):
|
||||
key, value = line.rsplit(" ", 1)
|
||||
values[key] = float(value)
|
||||
return values
|
||||
|
||||
|
||||
def docker_timestamp(value):
|
||||
# Docker emits nanoseconds; Python 3.9 accepts at most microseconds.
|
||||
return datetime.fromisoformat(re.sub(r"(\.\d{6})\d+", r"\1", value).replace("Z", "+00:00"))
|
||||
|
||||
|
||||
def host_memory():
|
||||
if not Path("/proc/vmstat").exists():
|
||||
return {"available": False}
|
||||
vm = dict(line.split() for line in Path("/proc/vmstat").read_text().splitlines())
|
||||
mem = dict(line.split(":", 1) for line in Path("/proc/meminfo").read_text().splitlines())
|
||||
return {"available": True, "pswpin_pages": int(vm["pswpin"]),
|
||||
"pswpout_pages": int(vm["pswpout"]), "oom_kill": int(vm.get("oom_kill", 0)),
|
||||
**{key + "_kib": int(mem[key].split()[0])
|
||||
for key in ("MemTotal", "MemAvailable", "SwapTotal", "SwapFree")}}
|
||||
|
||||
|
||||
def process_usage(pid):
|
||||
try:
|
||||
stat = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()
|
||||
return {"cpu_seconds": (int(stat[11]) + int(stat[12])) / os.sysconf("SC_CLK_TCK"),
|
||||
"rss_bytes": int(stat[21]) * os.sysconf("SC_PAGE_SIZE")}
|
||||
except (OSError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
class Benchmark:
|
||||
def __init__(self, local_smoke):
|
||||
self.local_smoke = local_smoke
|
||||
self.image = os.environ["GREPTIMEDB_IMAGE"]
|
||||
self.commit = os.environ["EXPECTED_COMMIT"]
|
||||
if not re.fullmatch(r"greptime/greptimedb(?:-nightly)?@sha256:[0-9a-f]{64}", self.image):
|
||||
raise ValueError("GREPTIMEDB_IMAGE must be an immutable official image digest")
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", self.commit):
|
||||
raise ValueError("EXPECTED_COMMIT must be a full commit SHA")
|
||||
self.per_group = int(os.environ.get("LOGICAL_TABLES_PER_GROUP", "10000"))
|
||||
if not 50 <= self.per_group <= 10000:
|
||||
raise ValueError("LOGICAL_TABLES_PER_GROUP must be in 50..10000")
|
||||
self.prefix = os.environ["RUN_PREFIX"]
|
||||
if not re.fullmatch(r"metric-snapshot-bench/[0-9]+-[0-9]+/", self.prefix):
|
||||
raise ValueError("RUN_PREFIX must be metric-snapshot-bench/<run>-<attempt>/")
|
||||
self.results = Path(os.environ["RESULTS"]).resolve()
|
||||
self.data = Path(os.environ["DATA_ROOT"]).resolve()
|
||||
self.results.mkdir(parents=True, exist_ok=False)
|
||||
self.data.mkdir(parents=True, exist_ok=False)
|
||||
self.network = "metric-snapshot-" + self.prefix.split("/")[1]
|
||||
self.server = self.network + "-db"
|
||||
self.cli_name = self.network + "-cli"
|
||||
self.port = int(os.environ.get("BENCH_HTTP_PORT", "19410"))
|
||||
self.base_url = f"http://127.0.0.1:{self.port}"
|
||||
self.endpoint = os.environ.get("S3_ENDPOINT", "")
|
||||
if self.endpoint and not local_smoke:
|
||||
raise ValueError("Custom S3 endpoints are restricted to local smoke")
|
||||
if not local_smoke and os.environ["S3_REGION"] != "ap-southeast-1":
|
||||
raise ValueError("Cloud benchmark requires ap-southeast-1")
|
||||
self.containers = set()
|
||||
self.network_created = False
|
||||
self.phases = []
|
||||
self.server_pid = 0
|
||||
self.round_dir = None
|
||||
|
||||
def docker(self, *args):
|
||||
return command(["docker", *args])
|
||||
|
||||
def metrics(self):
|
||||
try:
|
||||
with urllib.request.urlopen(self.base_url + "/metrics", timeout=5) as response:
|
||||
return response.read().decode()
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
def phase(self, name, action):
|
||||
print(f"Starting {self.round_dir.name}/{name}", flush=True)
|
||||
before = host_memory()
|
||||
metrics_before = self.metrics()
|
||||
samples = []
|
||||
done = threading.Event()
|
||||
|
||||
def sample():
|
||||
while not done.is_set():
|
||||
samples.append({"time": time.time(), "server": process_usage(self.server_pid),
|
||||
"cli": process_usage(getattr(self, "cli_pid", 0)),
|
||||
"query_memory": {k: v for k, v in metric_values(self.metrics()).items()
|
||||
if "query_memory" in k} if self.server_pid else {}})
|
||||
done.wait(0.5)
|
||||
|
||||
monitor = threading.Thread(target=sample, daemon=True)
|
||||
monitor.start()
|
||||
start = time.monotonic()
|
||||
outcome = "failed"
|
||||
result = None
|
||||
try:
|
||||
result = action()
|
||||
outcome = "passed"
|
||||
return result
|
||||
finally:
|
||||
elapsed = time.monotonic() - start
|
||||
done.set()
|
||||
monitor.join()
|
||||
after = host_memory()
|
||||
interfered = before["available"] and after["available"] and (
|
||||
after["pswpout_pages"] > before["pswpout_pages"] or after["oom_kill"] > before["oom_kill"])
|
||||
if interfered:
|
||||
outcome = "resource-invalid"
|
||||
record = {"round": self.round_dir.name, "phase": name, "outcome": outcome,
|
||||
"wall_seconds": elapsed, "host_before": before, "host_after": after,
|
||||
"process_samples": samples}
|
||||
if isinstance(result, dict):
|
||||
record.update(result)
|
||||
if before["available"] and after["available"]:
|
||||
record["new_swapout_pages"] = after["pswpout_pages"] - before["pswpout_pages"]
|
||||
record["new_oom_kills"] = after["oom_kill"] - before["oom_kill"]
|
||||
metrics_after = self.metrics()
|
||||
pre, post = metric_values(metrics_before), metric_values(metrics_after)
|
||||
record["server_counter_deltas"] = {
|
||||
key: value - pre.get(key, 0) for key, value in post.items()
|
||||
if ("opendal" in key and ("_count{" in key or "_sum{" in key))
|
||||
or key.startswith("greptime_servers_http_requests_total{")}
|
||||
write_json(self.round_dir / f"{name}-resources.json", record)
|
||||
(self.round_dir / f"{name}-before.prom").write_text(metrics_before)
|
||||
(self.round_dir / f"{name}-after.prom").write_text(metrics_after)
|
||||
self.phases.append({k: v for k, v in record.items() if k != "process_samples"})
|
||||
self.summary()
|
||||
print(f"Finished {name}: {outcome}, {elapsed:.3f}s", flush=True)
|
||||
if interfered:
|
||||
raise RuntimeError(f"Resource interference during {name}; evidence retained")
|
||||
|
||||
def fixture(self, operation, per_group, output, source=None):
|
||||
args = [sys.executable, str(HERE / "metric-snapshot-fixture.py"), operation,
|
||||
"--addr", f"127.0.0.1:{self.port}", "--tables-per-group", str(per_group),
|
||||
"--output", str(output)]
|
||||
if source:
|
||||
args += ["--source", str(source)]
|
||||
with (self.round_dir / f"{operation}-{output.stem}.log").open("w") as log:
|
||||
status = subprocess.run(args, stdout=log, stderr=subprocess.STDOUT).returncode
|
||||
if status:
|
||||
raise RuntimeError(f"Fixture {operation} failed; see its artifact log")
|
||||
|
||||
def start_server(self, role):
|
||||
root = self.data / self.round_dir.name / role
|
||||
root.mkdir(parents=True)
|
||||
evidence = self.round_dir / role
|
||||
evidence.mkdir()
|
||||
config = root / "config.toml"
|
||||
config.write_text('''enable_telemetry = false
|
||||
[http]
|
||||
addr = "0.0.0.0:4000"
|
||||
[grpc]
|
||||
bind_addr = "0.0.0.0:4001"
|
||||
[mysql]
|
||||
enable = false
|
||||
[postgres]
|
||||
enable = false
|
||||
[storage]
|
||||
type = "File"
|
||||
data_home = "/data/db"
|
||||
[query]
|
||||
parallelism = 4
|
||||
memory_pool_size = "2GiB"
|
||||
[logging]
|
||||
dir = "/evidence/logs"
|
||||
level = "info"
|
||||
''')
|
||||
self.containers.add(self.server)
|
||||
self.docker("run", "-d", "--name", self.server, "--network", self.network,
|
||||
"--network-alias", "metric-db", "--user", f"{os.getuid()}:{os.getgid()}",
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-p", f"127.0.0.1:{self.port}:4000",
|
||||
"-v", f"{root}:/data", "-v", f"{evidence}:/evidence",
|
||||
self.image, "standalone", "start", "-c", "/data/config.toml")
|
||||
self.server_pid = int(self.docker("inspect", "-f", "{{.State.Pid}}", self.server))
|
||||
deadline = time.monotonic() + 120
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(self.base_url + "/v1/capabilities", timeout=2) as response:
|
||||
caps = json.load(response)
|
||||
if any(caps.get(key) != 1 for key in
|
||||
("metric_packed_export", "metric_packed_import", "metric_batch_ddl")):
|
||||
raise RuntimeError("Required packed snapshot capabilities are absent")
|
||||
write_json(evidence / "capabilities.json", caps)
|
||||
return
|
||||
except OSError:
|
||||
if self.docker("inspect", "-f", "{{.State.Running}}", self.server).strip() != "true":
|
||||
break
|
||||
time.sleep(1)
|
||||
raise RuntimeError("Server did not become ready")
|
||||
|
||||
def stop_server(self, role):
|
||||
self.docker("stop", "-t", "60", self.server)
|
||||
state = json.loads(self.docker("inspect", "-f", "{{json .State}}", self.server))
|
||||
write_json(self.round_dir / role / "exit.json", state)
|
||||
(self.round_dir / role / "stdout.log").write_text(self.docker("logs", self.server))
|
||||
self.docker("rm", self.server)
|
||||
self.containers.remove(self.server)
|
||||
self.server_pid = 0
|
||||
if state["OOMKilled"] or state["ExitCode"] != 0:
|
||||
raise RuntimeError(f"{role} server did not shut down cleanly")
|
||||
|
||||
def cli(self, args, name):
|
||||
# Expand credentials only inside the container. Never persist argv or env.
|
||||
script = 'exec greptime "$@" --s3 --s3-region "$S3_REGION" --s3-access-key-id "$AWS_ACCESS_KEY_ID" --s3-secret-access-key "$AWS_SECRET_ACCESS_KEY"'
|
||||
if self.endpoint:
|
||||
args += ["--s3-endpoint", self.endpoint]
|
||||
self.containers.add(self.cli_name)
|
||||
self.docker("create", "--name", self.cli_name, "--network", self.network,
|
||||
"--user", f"{os.getuid()}:{os.getgid()}",
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-e", "AWS_ACCESS_KEY_ID", "-e", "AWS_SECRET_ACCESS_KEY", "-e", "S3_REGION",
|
||||
"-v", f"{self.round_dir}:/evidence", "--entrypoint", "/bin/sh", self.image,
|
||||
"-c", script, "sh", "--log-dir", f"/evidence/{name}-logs", "cli", "data", *args)
|
||||
|
||||
def execute():
|
||||
self.docker("start", self.cli_name)
|
||||
self.cli_pid = int(self.docker("inspect", "-f", "{{.State.Pid}}", self.cli_name))
|
||||
status = int(self.docker("wait", self.cli_name))
|
||||
state = json.loads(self.docker("inspect", "-f", "{{json .State}}", self.cli_name))
|
||||
(self.round_dir / f"{name}.log").write_text(self.docker("logs", self.cli_name))
|
||||
write_json(self.round_dir / f"{name}-exit.json", state)
|
||||
if status or state["ExitCode"] or state["OOMKilled"]:
|
||||
raise RuntimeError(f"{name} failed; see artifact log")
|
||||
return {"cli_wall_seconds": (docker_timestamp(state["FinishedAt"]) -
|
||||
docker_timestamp(state["StartedAt"])).total_seconds()}
|
||||
|
||||
try:
|
||||
self.phase(name, execute)
|
||||
finally:
|
||||
self.cli_pid = 0
|
||||
if not (self.round_dir / f"{name}-exit.json").exists():
|
||||
self.save_container(self.cli_name, name + "-failure")
|
||||
self.docker("rm", "-f", self.cli_name)
|
||||
self.containers.remove(self.cli_name)
|
||||
|
||||
def round(self, label, per_group):
|
||||
self.round_dir = self.results / label
|
||||
self.round_dir.mkdir()
|
||||
self.start_server("source")
|
||||
source = self.round_dir / "source-validation.json"
|
||||
self.phase("prepare", lambda: self.fixture("prepare", per_group, self.round_dir / "prepare.json"))
|
||||
self.phase("source-validation", lambda: self.fixture("validate", per_group, source))
|
||||
location = f"s3://{os.environ['S3_BUCKET']}/{self.prefix}{label}"
|
||||
self.cli(["export-v2", "create", "--addr", "metric-db:4000", "--to", location,
|
||||
"--schemas", "metrics", "--pack-metric-data", "--parallelism", "4",
|
||||
"--chunk-parallelism", "1", "--start-time", "1970-01-01T00:00:00Z",
|
||||
"--end-time", "1970-01-01T00:00:01Z", "--timeout", "1h",
|
||||
"--no-proxy", "--progress", "always"], "export")
|
||||
if not self.local_smoke:
|
||||
command(["uv", "run", str(HERE / "metric-snapshot-s3-ops.py"), "inventory", "--round", label,
|
||||
"--output", str(self.round_dir / "s3-inventory.json")])
|
||||
self.stop_server("source")
|
||||
self.cli(["export-v2", "verify", "--snapshot", location], "verify")
|
||||
self.start_server("target")
|
||||
self.fixture("placeholders", per_group, self.round_dir / "placeholders.json")
|
||||
self.cli(["import-v2", "--addr", "metric-db:4000", "--from", location,
|
||||
"--task-parallelism", "1", "--state-path", "/evidence/import-state.json",
|
||||
"--timeout", "1h", "--no-proxy", "--progress", "always"], "import")
|
||||
imported = json.loads((self.round_dir / "import-resources.json").read_text())
|
||||
batch_requests = sum(value for key, value in imported["server_counter_deltas"].items()
|
||||
if '/v1/ddl/logical-tables' in key and 'code="200"' in key)
|
||||
if batch_requests <= 0:
|
||||
raise RuntimeError("Import did not complete any successful batch DDL HTTP requests")
|
||||
write_json(self.round_dir / "import-completion.json", {"batch_ddl_requests": batch_requests,
|
||||
"resume_state_removed": not (self.round_dir / "import-state.json").exists()})
|
||||
if (self.round_dir / "import-state.json").exists():
|
||||
raise RuntimeError("Successful import left its resume state file")
|
||||
self.phase("target-validation", lambda: self.fixture(
|
||||
"validate", per_group, self.round_dir / "target-validation.json", source))
|
||||
self.stop_server("target")
|
||||
|
||||
def summary(self):
|
||||
lines = ["# Metric packed snapshot benchmark", "", f"Image: `{self.image}`",
|
||||
f"Source commit: `{self.commit}`", "",
|
||||
"Local smoke only; timings are not AWS results." if self.local_smoke else
|
||||
"AWS S3 in ap-southeast-1; source and target use local File storage.", "",
|
||||
"| Round | Phase | Seconds | Result |", "| --- | --- | ---: | --- |"]
|
||||
for phase in self.phases:
|
||||
lines.append(f"| {phase['round']} | {phase['phase']} | {phase['wall_seconds']:.3f} | {phase['outcome']} |")
|
||||
lines += ["", "Each logical table has 32 rows, 10 overlapping string tags, a timestamp and a DOUBLE.",
|
||||
"Phase durations include Docker start, wait and log collection. cli_wall_seconds records Docker StartedAt to FinishedAt separately; setup, loading and full validation are separate.",
|
||||
"Process CPU/RSS are sampled every 0.5 s on Linux, not exact process peaks. Linux host swap/OOM counters are checked per phase; configured swap is not RAM.",
|
||||
"Raw server Prometheus snapshots and CLI progress/logs are retained. OpenDAL operation counts are not AWS HTTP request counts: multipart requests, retries and CLI operations are not fully covered. Exact AWS request totals are unavailable."]
|
||||
(self.results / "summary.md").write_text("\n".join(lines) + "\n")
|
||||
|
||||
def run(self):
|
||||
self.docker("pull", self.image)
|
||||
version = self.docker("run", "--rm", self.image, "--version")
|
||||
(self.results / "version.txt").write_text(version)
|
||||
if self.commit not in version or "clean: true" not in version:
|
||||
raise RuntimeError("Image version does not match the expected clean source commit")
|
||||
for args, required in ((["export-v2", "create"], "--pack-metric-data"),
|
||||
(["import-v2"], "--task-parallelism")):
|
||||
help_text = self.docker("run", "--rm", self.image, "cli", "data", *args, "--help")
|
||||
(self.results / (args[0] + "-help.txt")).write_text(help_text)
|
||||
if required not in help_text:
|
||||
raise RuntimeError(f"Image CLI lacks {required}")
|
||||
write_json(self.results / "host.json", {"memory": host_memory(), "local_smoke": self.local_smoke,
|
||||
"platform": self.docker("image", "inspect", "-f", "{{.Os}}/{{.Architecture}}", self.image).strip()})
|
||||
self.docker("network", "create", self.network)
|
||||
self.network_created = True
|
||||
self.round("smoke", 50)
|
||||
if not self.local_smoke and self.per_group != 50:
|
||||
self.round("full", self.per_group)
|
||||
|
||||
def save_container(self, name, label):
|
||||
state = subprocess.run(["docker", "inspect", "-f", "{{json .State}}", name],
|
||||
capture_output=True, text=True)
|
||||
(self.round_dir / (label + "-state.json")).write_text(state.stdout)
|
||||
result = subprocess.run(["docker", "logs", name], capture_output=True, text=True)
|
||||
(self.round_dir / (label + ".log")).write_text(result.stdout + result.stderr)
|
||||
|
||||
def cleanup(self):
|
||||
for name in self.containers:
|
||||
if self.round_dir:
|
||||
self.save_container(name, name + "-failure")
|
||||
subprocess.run(["docker", "rm", "-f", name], stdout=subprocess.DEVNULL, check=False)
|
||||
if self.network_created:
|
||||
subprocess.run(["docker", "network", "rm", self.network], stdout=subprocess.DEVNULL, check=False)
|
||||
self.summary()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("command", choices=["run"])
|
||||
parser.add_argument("--local-smoke", action="store_true")
|
||||
args = parser.parse_args()
|
||||
bench = Benchmark(args.local_smoke)
|
||||
|
||||
def cancelled(signum, frame):
|
||||
raise RuntimeError(f"Received signal {signum}")
|
||||
|
||||
signal.signal(signal.SIGTERM, cancelled)
|
||||
try:
|
||||
bench.run()
|
||||
finally:
|
||||
bench.cleanup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = ["boto3>=1.35,<2"]
|
||||
# ///
|
||||
|
||||
"""Budget preflight and strictly scoped S3 operations for the metric snapshot benchmark."""
|
||||
|
||||
import argparse
|
||||
from decimal import Decimal
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
REGION = "ap-southeast-1"
|
||||
HOURLY_USD = Decimal("0.9408")
|
||||
OTHER_RESERVE_USD = Decimal("1.50")
|
||||
|
||||
|
||||
def owned_prefix(prefix):
|
||||
if not re.fullmatch(r"metric-snapshot-bench/[0-9]+-[0-9]+/", prefix):
|
||||
raise ValueError("prefix must be exactly metric-snapshot-bench/<run_id>-<attempt>/")
|
||||
return prefix
|
||||
|
||||
|
||||
def plan(env):
|
||||
image = env["GREPTIMEDB_IMAGE"]
|
||||
if not re.fullmatch(r"greptime/greptimedb(?:-nightly)?@sha256:[0-9a-f]{64}", image):
|
||||
raise ValueError("image must be an immutable greptime/greptimedb sha256 reference")
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", env["EXPECTED_COMMIT"]):
|
||||
raise ValueError("expected commit must be a full lowercase source SHA")
|
||||
tables, minutes = int(env["LOGICAL_TABLES_PER_GROUP"]), int(env["BENCHMARK_TIMEOUT_MINUTES"])
|
||||
if not 50 <= tables <= 10_000 or not 30 <= minutes <= 240:
|
||||
raise ValueError("logical tables per group must be 50..10000; timeout must be 30..240 minutes")
|
||||
if env["S3_REGION"] != REGION or env["RUNNER_REGION"] != REGION:
|
||||
raise ValueError("bucket and runner must both use ap-southeast-1")
|
||||
if not env["S3_BUCKET"]:
|
||||
raise ValueError("S3 bucket is required")
|
||||
prior = Decimal(env["PRIOR_SPEND_USD"])
|
||||
# Reserve 45m provision, 10m queue allowance and 20m cleanup beyond the job timeout.
|
||||
reservation = HOURLY_USD * Decimal(minutes + 75) / 60 + OTHER_RESERVE_USD
|
||||
if not prior.is_finite() or prior < 0 or prior >= 15 or prior + reservation > 20:
|
||||
raise ValueError("campaign budget exhausted: prior spend must be below $15 and leave the full attempt reserve within $20")
|
||||
return {"image": image, "expected_commit": env["EXPECTED_COMMIT"],
|
||||
"logical_tables_per_group": tables, "smoke_logical_tables": 500,
|
||||
"timeout_minutes": minutes, "prefix": owned_prefix(env["RUN_PREFIX"]),
|
||||
"instance_type": "c6id.4xlarge", "region": REGION,
|
||||
"prior_estimated_usd": str(prior), "attempt_reserve_usd": str(reservation),
|
||||
"ec2_hourly_usd": str(HOURLY_USD), "other_reserve_usd": str(OTHER_RESERVE_USD),
|
||||
"cost_note": "Estimate, not an AWS billing total or a hard spending cap."}
|
||||
|
||||
|
||||
def client(region):
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
if region != REGION:
|
||||
raise ValueError("S3 region must be ap-southeast-1")
|
||||
return boto3.client("s3", region_name=region, config=Config(
|
||||
connect_timeout=10, read_timeout=30, retries={"mode": "standard", "max_attempts": 3}))
|
||||
|
||||
|
||||
def entries(s3, operation, bucket, prefix, fields, scope=""):
|
||||
listed = owned_prefix(prefix) + scope
|
||||
for page in s3.get_paginator(operation).paginate(Bucket=bucket, Prefix=listed):
|
||||
for field in fields:
|
||||
for item in page.get(field, []):
|
||||
if not item["Key"].startswith(listed):
|
||||
raise ValueError("S3 returned an object outside the owned prefix")
|
||||
yield item
|
||||
|
||||
|
||||
def inventory(s3, bucket, prefix, round=None):
|
||||
# A round's snapshot lives under <run prefix><round>/; earlier rounds stay until cleanup.
|
||||
if round not in (None, "smoke", "full"):
|
||||
raise ValueError("round must be smoke or full")
|
||||
scope = round + "/" if round else ""
|
||||
objects = list(entries(s3, "list_objects_v2", bucket, prefix, ("Contents",), scope))
|
||||
return {"prefix": prefix + scope, "objects": len(objects), "bytes": sum(item["Size"] for item in objects),
|
||||
"measurement": "S3 listing inventory; not request counts or transferred bytes."}
|
||||
|
||||
|
||||
def probe(s3, bucket, prefix):
|
||||
if inventory(s3, bucket, prefix)["objects"]:
|
||||
raise ValueError("run prefix is already occupied; use a fresh workflow attempt")
|
||||
key = prefix + "_probe"
|
||||
s3.put_object(Bucket=bucket, Key=key, Body=b"metric snapshot benchmark")
|
||||
try:
|
||||
body = s3.get_object(Bucket=bucket, Key=key)["Body"]
|
||||
try:
|
||||
if body.read() != b"metric snapshot benchmark":
|
||||
raise ValueError("S3 probe returned unexpected content")
|
||||
finally:
|
||||
body.close()
|
||||
finally:
|
||||
s3.delete_object(Bucket=bucket, Key=key)
|
||||
return inventory(s3, bucket, prefix)
|
||||
|
||||
|
||||
def cleanup(s3, bucket, prefix):
|
||||
owned_prefix(prefix)
|
||||
for item in entries(s3, "list_objects_v2", bucket, prefix, ("Contents",)):
|
||||
s3.delete_object(Bucket=bucket, Key=item["Key"])
|
||||
after = inventory(s3, bucket, prefix)
|
||||
if after["objects"]:
|
||||
raise RuntimeError("owned S3 prefix is not empty after cleanup")
|
||||
return after
|
||||
|
||||
|
||||
def scrub(root, env):
|
||||
secrets = [env[name].encode() for name in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN")
|
||||
if env.get(name)]
|
||||
if not secrets:
|
||||
raise ValueError("credential scan requires the CI credential environment")
|
||||
overlap = max(map(len, secrets)) - 1
|
||||
for result in Path(root).rglob("*"):
|
||||
if result.is_file():
|
||||
with result.open("rb") as stream:
|
||||
previous = b""
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
data = previous + chunk
|
||||
if any(secret in data for secret in secrets):
|
||||
raise ValueError("credential detected in results; artifact upload is blocked")
|
||||
previous = data[-overlap:] if overlap else b""
|
||||
|
||||
|
||||
def wait_for_benchmark(env, started_at):
|
||||
"""Bound runner allocation even when the benchmark job never leaves its queue."""
|
||||
run_id, attempt = owned_prefix(env["RUN_PREFIX"]).split("/")[1].split("-")
|
||||
repo = env["GITHUB_REPOSITORY"]
|
||||
if run_id != env["GITHUB_RUN_ID"] or not re.fullmatch(r"[\w.-]+/[\w.-]+", repo):
|
||||
raise ValueError("workflow identity does not match its owned resource prefix")
|
||||
minutes = int(env["BENCHMARK_TIMEOUT_MINUTES"])
|
||||
if not 30 <= minutes <= 240 or started_at <= 0:
|
||||
raise ValueError("invalid benchmark allocation deadline")
|
||||
# Use the original allocation timestamp; a delayed or retried watcher cannot extend it.
|
||||
deadline = started_at + (45 + minutes + 10) * 60
|
||||
request = urllib.request.Request(
|
||||
f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100",
|
||||
headers={"Authorization": f"Bearer {env['GH_TOKEN']}", "Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28"})
|
||||
while True:
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
raise TimeoutError("benchmark allocation deadline reached; proceeding to resource cleanup")
|
||||
# API errors deliberately fail this step: subsequent always() steps clean up immediately.
|
||||
with urllib.request.urlopen(request, timeout=min(15, remaining)) as response:
|
||||
jobs = json.load(response)["jobs"]
|
||||
benchmark = next((job for job in jobs if job["name"] == "benchmark"), None)
|
||||
if benchmark and benchmark["status"] == "completed":
|
||||
return {"benchmark_conclusion": benchmark["conclusion"], "owned_attempt": attempt}
|
||||
remaining = deadline - time.time()
|
||||
if remaining > 0:
|
||||
time.sleep(min(30, remaining))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("command", choices=("plan", "probe", "inventory", "cleanup", "scrub", "cost", "wait-for-benchmark"))
|
||||
parser.add_argument("--output")
|
||||
parser.add_argument("--started-at", type=int)
|
||||
parser.add_argument("--round")
|
||||
args = parser.parse_args()
|
||||
if args.round is not None and args.command != "inventory":
|
||||
parser.error("--round applies only to inventory")
|
||||
if args.command == "plan":
|
||||
result = plan(os.environ)
|
||||
elif args.command == "scrub":
|
||||
scrub(os.environ["RESULTS"], os.environ)
|
||||
return
|
||||
elif args.command == "wait-for-benchmark":
|
||||
if args.started_at is None:
|
||||
parser.error("wait-for-benchmark requires --started-at")
|
||||
result = wait_for_benchmark(os.environ, args.started_at)
|
||||
elif args.command == "cost":
|
||||
if args.started_at is None:
|
||||
parser.error("cost requires --started-at")
|
||||
seconds = max(0, int(time.time()) - args.started_at)
|
||||
estimate = HOURLY_USD * Decimal(seconds) / 3600
|
||||
result = {"allocation_window_seconds": seconds, "ec2_estimate_usd": str(round(estimate, 4)),
|
||||
"campaign_estimate_with_other_reserve_usd": str(round(
|
||||
Decimal(os.environ["PRIOR_SPEND_USD"]) + estimate + OTHER_RESERVE_USD, 4)),
|
||||
"note": "Allocation wall-time estimate plus $1.50 reserve for S3, disk and IPv4; not an AWS bill."}
|
||||
else:
|
||||
prefix = owned_prefix(os.environ["RUN_PREFIX"])
|
||||
s3, bucket = client(os.environ["S3_REGION"]), os.environ["S3_BUCKET"]
|
||||
if args.command == "inventory":
|
||||
result = inventory(s3, bucket, prefix, args.round)
|
||||
else:
|
||||
result = globals()[args.command](s3, bucket, prefix)
|
||||
rendered = json.dumps(result, indent=2) + "\n"
|
||||
if args.output:
|
||||
destination = Path(args.output)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(rendered)
|
||||
print(rendered, end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Writer and oracle of the benchmark's crash cells.
|
||||
|
||||
Every writer owns the stripe `first + index, first + index + writers, ...` of
|
||||
batch sequence numbers, inserts `rows` rows per batch and stops at the first
|
||||
statement that is not acknowledged, so a stripe has at most one attempted but
|
||||
unacknowledged batch: the one in flight at the kill. Every request is logged
|
||||
with its status and the time it ended, on the same clock as the kill stamps.
|
||||
With `--seconds` the writers also stop once that long has passed, and the
|
||||
command fails unless every writer wrote until then without a failure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def insert(url: str, table: str, seq: int, rows: int) -> str:
|
||||
values = ", ".join(f"({seq}, {seq * rows + i})" for i in range(rows))
|
||||
body = urllib.parse.urlencode({"sql": f"INSERT INTO {table} (seq, ts) VALUES {values}"}).encode()
|
||||
try:
|
||||
with urllib.request.urlopen(urllib.request.Request(url, data=body), timeout=60) as response:
|
||||
return str(response.status)
|
||||
except urllib.error.HTTPError as error:
|
||||
return str(error.code)
|
||||
except (OSError, urllib.error.URLError):
|
||||
# The connection dropped: the request may or may not have been applied.
|
||||
return "000"
|
||||
|
||||
|
||||
def writer(args, index: int, deadline: float, failures: list) -> None:
|
||||
directory, seq = Path(args.dir), args.first_seq + index
|
||||
with open(directory / f"attempted-{index}.log", "a") as attempted, \
|
||||
open(directory / f"requests-{index}.log", "a") as requests, \
|
||||
open(directory / f"acked-{index}.log", "a") as acked:
|
||||
while time.monotonic() < deadline:
|
||||
attempted.write(f"{seq}\n")
|
||||
attempted.flush()
|
||||
code = insert(args.url, args.table, seq, args.rows)
|
||||
requests.write(f"{seq} {code} {time.time_ns()}\n")
|
||||
requests.flush()
|
||||
if code != "200":
|
||||
failures.append(f"writer {index}: batch {seq} ended with {code}")
|
||||
break
|
||||
acked.write(f"{seq}\n")
|
||||
acked.flush()
|
||||
seq += args.writers
|
||||
(directory / f"exit-{index}.log").write_text(f"{time.time_ns()}\n")
|
||||
|
||||
|
||||
def write(args) -> int:
|
||||
os.makedirs(args.dir, exist_ok=True)
|
||||
deadline = time.monotonic() + args.seconds if args.seconds else float("inf")
|
||||
failures = []
|
||||
|
||||
def run(index):
|
||||
try:
|
||||
writer(args, index, deadline, failures)
|
||||
except Exception as error:
|
||||
failures.append(f"writer {index}: {error!r}")
|
||||
raise
|
||||
|
||||
threads = [threading.Thread(target=run, args=(i,)) for i in range(args.writers)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
# Crash cells end their writers with a kill; timed writers must have run to the deadline.
|
||||
if args.seconds and (failures or not any(lines(Path(args.dir) / f"acked-{i}.log") for i in range(args.writers))):
|
||||
print("; ".join(failures) or "no batch was acknowledged", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def lines(path: Path) -> list:
|
||||
return [line.split() for line in path.read_text().splitlines() if line.strip()] if path.exists() else []
|
||||
|
||||
|
||||
def found_batches(path: Path) -> dict:
|
||||
"""GreptimeDB's JSON answer to SELECT seq, count(*) ... GROUP BY seq, as batch -> row count."""
|
||||
return {int(seq): int(count) for seq, count in json.loads(path.read_text())["output"][0]["records"]["rows"]}
|
||||
|
||||
|
||||
def check_kill(directory: Path, writers: int, before_ns: int, after_ns: int, window_max_ms: int) -> dict:
|
||||
"""A dropped connection is in flight when it ended at or after the stamp taken before the kill."""
|
||||
errors, in_flight, running = [], 0, 0
|
||||
if after_ns - before_ns > window_max_ms * 1_000_000:
|
||||
errors.append(f"kill window {after_ns - before_ns}ns exceeds {window_max_ms}ms")
|
||||
for index in range(writers):
|
||||
exit_path = directory / f"exit-{index}.log"
|
||||
if not exit_path.exists():
|
||||
errors.append(f"writer {index} left no exit time")
|
||||
continue
|
||||
running += int(exit_path.read_text()) >= before_ns
|
||||
for seq, code, end_ns in lines(directory / f"requests-{index}.log"):
|
||||
if code == "200":
|
||||
continue
|
||||
if code != "000":
|
||||
errors.append(f"writer {index}: batch {seq} rejected with HTTP {code}")
|
||||
elif int(end_ns) < before_ns:
|
||||
errors.append(f"writer {index}: batch {seq} lost its connection before the kill")
|
||||
else:
|
||||
in_flight += 1
|
||||
if running == 0:
|
||||
errors.append("no writer was running at the kill")
|
||||
return {"kill_window_ns": after_ns - before_ns, "in_flight": in_flight, "writers_at_kill": running,
|
||||
"kill_errors": errors}
|
||||
|
||||
|
||||
def verify(run_dir: Path, found: dict, rows: int, enqueued: bool, cycle: int) -> dict:
|
||||
"""Checks the table, read as batch -> row count, against every cycle's writers so far and
|
||||
against what every earlier restart recovered, and counts what the given cycle's writers
|
||||
acknowledged, left and lost on their own."""
|
||||
acked, stripes, cycle_attempted, cycle_acked = [], collections.OrderedDict(), set(), set()
|
||||
recovered_before = set()
|
||||
for cycle_dir in sorted(run_dir.glob("cycle-*"), key=lambda p: int(p.name.split("-")[1])):
|
||||
current = cycle_dir.name == f"cycle-{cycle}"
|
||||
if int(cycle_dir.name.split("-")[1]) < cycle and (cycle_dir / "found.json").exists():
|
||||
recovered_before.update(found_batches(cycle_dir / "found.json"))
|
||||
for path in sorted(cycle_dir.glob("attempted-*.log")):
|
||||
stripes[(cycle_dir.name, path.name)] = [int(seq) for (seq,) in lines(path)]
|
||||
if current:
|
||||
cycle_attempted.update(stripes[(cycle_dir.name, path.name)])
|
||||
for path in sorted(cycle_dir.glob("acked-*.log")):
|
||||
acked += [int(seq) for (seq,) in lines(path)]
|
||||
if current:
|
||||
cycle_acked.update(int(seq) for (seq,) in lines(path))
|
||||
acked_set = set(acked)
|
||||
attempted = {seq for stripe in stripes.values() for seq in stripe}
|
||||
errors = []
|
||||
duplicated = sorted(seq for seq, count in found.items() if count > rows)
|
||||
partial = sorted(seq for seq, count in found.items() if count < rows)
|
||||
if duplicated:
|
||||
errors.append(f"batches present more than once: {duplicated[:20]}")
|
||||
if partial:
|
||||
errors.append(f"batches present in part: {partial[:20]}")
|
||||
missing = sorted(acked_set - set(found))
|
||||
if missing and not enqueued:
|
||||
errors.append(f"acknowledged but absent: {missing[:20]}")
|
||||
# A batch an earlier restart recovered was durable then, so in either mode only the
|
||||
# current cycle's backlog that was not durable at its kill may be missing.
|
||||
vanished = sorted(recovered_before - set(found))
|
||||
if vanished:
|
||||
errors.append(f"recovered by an earlier restart but absent: {vanished[:20]}")
|
||||
unattempted = sorted(set(found) - attempted)
|
||||
if unattempted:
|
||||
errors.append(f"present but never attempted: {unattempted[:20]}")
|
||||
# The unacknowledged tail of each stripe was in flight at a kill.
|
||||
in_flight = set()
|
||||
for key, stripe in stripes.items():
|
||||
prefix = 0
|
||||
while prefix < len(stripe) and stripe[prefix] in acked_set:
|
||||
prefix += 1
|
||||
if any(seq in acked_set for seq in stripe[prefix:]):
|
||||
errors.append(f"{key}: acknowledged after a failure")
|
||||
in_flight.update(stripe[prefix:])
|
||||
extra = sorted(set(found) - acked_set)
|
||||
stray = [seq for seq in extra if seq not in in_flight]
|
||||
if stray and not enqueued:
|
||||
errors.append(f"present, unacknowledged and not in flight at a kill: {stray[:20]}")
|
||||
if len(acked) != len(acked_set):
|
||||
errors.append("a batch was acknowledged twice")
|
||||
present = set(found) & cycle_attempted
|
||||
return {"cycle_acked_batches": len(cycle_acked), "cycle_present_batches": len(present),
|
||||
"cycle_lost_batches": len(cycle_acked - present), "cycle_lost_rows": len(cycle_acked - present) * rows,
|
||||
"cycle_extra_batches": len(present - cycle_acked),
|
||||
"total_found_batches": len(found), "total_acked_batches": len(acked_set),
|
||||
"total_lost_batches": len(missing), "total_extra_batches": len(extra), "verify_errors": errors}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
w = commands.add_parser("write")
|
||||
w.add_argument("--dir", required=True)
|
||||
w.add_argument("--url", required=True)
|
||||
w.add_argument("--table", default="crash")
|
||||
for name in ("writers", "first-seq", "rows"):
|
||||
w.add_argument(f"--{name}", type=int, required=True)
|
||||
w.add_argument("--seconds", type=float, default=0, help="stop after this long; 0 writes until a failure")
|
||||
c = commands.add_parser("check")
|
||||
c.add_argument("--run-dir", type=Path, required=True)
|
||||
c.add_argument("--cycle", type=int, required=True)
|
||||
c.add_argument("--found", type=Path, required=True, help="GreptimeDB JSON output of SELECT seq, count(*)")
|
||||
c.add_argument("--mode", choices=("durable", "enqueued"), required=True)
|
||||
for name in ("writers", "rows", "before-ns", "after-ns", "window-max-ms"):
|
||||
c.add_argument(f"--{name}", type=int, required=True)
|
||||
c.add_argument("--pin-rows", type=int, help="rows of the pin table, whose one row was acknowledged first")
|
||||
args = parser.parse_args()
|
||||
if args.command == "write":
|
||||
return write(args)
|
||||
result = {"cycle": args.cycle}
|
||||
result |= check_kill(args.run_dir / f"cycle-{args.cycle}", args.writers, args.before_ns, args.after_ns,
|
||||
args.window_max_ms)
|
||||
result |= verify(args.run_dir, found_batches(args.found), args.rows,
|
||||
args.mode == "enqueued", args.cycle)
|
||||
if args.pin_rows is not None and (args.pin_rows > 1 or args.mode == "durable" and args.pin_rows != 1):
|
||||
result["verify_errors"].append(f"the pin table holds {args.pin_rows} rows, not 1")
|
||||
result["passed"] = not result["kill_errors"] and not result["verify_errors"]
|
||||
print(json.dumps(result))
|
||||
return 0 if result["passed"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs inside the o11ybench runtime image: the load of scripts/agent-observability/greptimedb.sh
|
||||
# with the same arguments plus the loader's --skip-create-table, for a table the cell created.
|
||||
set -euo pipefail
|
||||
|
||||
mkdir "/results/$RESULT_NAME"
|
||||
mkdir -p /results/corpus
|
||||
cp /work/corpus/summary.json /work/corpus/evidence.json /results/corpus/
|
||||
w="$PWD/workloads/logbench-agent-observability"
|
||||
r="/results/$RESULT_NAME"
|
||||
prepare_workers=${PREPARE_WORKERS:-$(python3 -c 'import sys; sys.path.insert(0, "scripts"); from benchmark_workers import effective_cpus; print(effective_cpus())')}
|
||||
o11ybench-greptimedb-loader --workload logbench-agent-observability --prepare-workers "$prepare_workers" \
|
||||
--skip-create-table \
|
||||
--grpc-endpoint "$GRPC_ENDPOINT" --sql-endpoint "$ENDPOINT" \
|
||||
--database public --table agent_observations --batch-size "${GREPTIMEDB_BATCH_SIZE:-10000}" --write-mode stream --max-in-flight "${MAX_IN_FLIGHT:-1}" \
|
||||
--flush-table --request-timeout-seconds 900 \
|
||||
--schema "$w/schema.json" --physical-index-profiles "$w/physical-index-profiles.json" \
|
||||
--data /work/corpus/agent_observations.jsonl \
|
||||
--profile-summary /work/corpus/summary.json --evidence /work/corpus/evidence.json \
|
||||
--transcript-output "$r/load-transcript.jsonl" \
|
||||
--target-metadata-output "$r/target-metadata.json" \
|
||||
--greptimedb-image "$TARGET_IMAGE" --greptimedb-version "$TARGET_VERSION" \
|
||||
--greptimedb-digest "$TARGET_DIGEST" \
|
||||
--startup-input=standalone --startup-input=start \
|
||||
--startup-input=--http-addr=0.0.0.0:4000 \
|
||||
--startup-input=--grpc-bind-addr=0.0.0.0:4001 \
|
||||
> "$r/load-result.json"
|
||||
@@ -10,24 +10,64 @@ runtime() {
|
||||
--security-opt no-new-privileges --cap-drop ALL "$@"
|
||||
}
|
||||
|
||||
# Pin the runtime image to a digest without pulling it: ECR Public's anonymous data limit can refuse a
|
||||
# hosted runner even a manifest read. A reference that names its digest is pinned as given; only a tag
|
||||
# is resolved from the registry's manifest. The benchmark host pulls the image and checks the contents.
|
||||
check_runtime() {
|
||||
local image=$1 resolved
|
||||
local image=$1 repository digest attempt component='[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*'
|
||||
[[ "$image" == public.ecr.aws/* ]] || { echo 'The runtime image must come from AWS ECR Public' >&2; return 1; }
|
||||
bash "$here/pull-benchmark-image.sh" "$image"
|
||||
resolved=$(docker image inspect "$image" | jq -er '.[0].RepoDigests[0]')
|
||||
docker run --rm --entrypoint sh "$resolved" -c \
|
||||
'test -f scripts/agent-observability/greptimedb.sh && test -f scripts/generate_agent_observations.py' \
|
||||
|| { echo 'The runtime image lacks the agent observability loader or generator' >&2; return 1; }
|
||||
printf 'runtime_image=%s\n' "$resolved" >> "${GITHUB_OUTPUT:-/dev/stdout}"
|
||||
# Docker's reference grammar, checked here because a named digest never reaches the registry.
|
||||
local grammar="^public\.ecr\.aws(/$component)+(:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?(@sha256:[0-9a-f]{64})?\$"
|
||||
[[ "$image" =~ $grammar ]] || { echo "$image is not a full ECR Public image reference" >&2; return 1; }
|
||||
if [[ "$image" == *@* ]]; then
|
||||
digest=${image##*@}
|
||||
else
|
||||
for attempt in 1 2 3; do
|
||||
digest=$(docker buildx imagetools inspect --format '{{json .Manifest}}' "$image" | jq -er .digest) && break
|
||||
((attempt < 3)) || { echo "Cannot read the manifest of $image" >&2; return 1; }
|
||||
sleep $((attempt * 10))
|
||||
done
|
||||
fi
|
||||
[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "$image has no sha256 digest" >&2; return 1; }
|
||||
repository=${image%@*}
|
||||
[[ "${repository##*/}" != *:* ]] || repository=${repository%:*}
|
||||
printf 'runtime_image=%s@%s\n' "$repository" "$digest" >> "${GITHUB_OUTPUT:-/dev/stdout}"
|
||||
}
|
||||
|
||||
# Pull every GreptimeDB image of the run, pin it to its digest and record the version and
|
||||
# commit it reports, before any instance is allocated.
|
||||
check_images() {
|
||||
local entry image resolved info version commit checked=()
|
||||
while IFS= read -r entry; do
|
||||
image=$(jq -r .image <<< "$entry")
|
||||
bash "$here/pull-benchmark-image.sh" "$image"
|
||||
resolved=$(docker image inspect "$image" | jq -er '[.[0].RepoDigests[]? | select(test("@sha256:[0-9a-f]{64}$"))][0]') \
|
||||
|| { echo "$image has no sha256 repository digest" >&2; return 1; }
|
||||
info=$(docker run --rm --entrypoint greptime "$resolved" --version)
|
||||
version=$(sed -n 's/^version: //p' <<< "$info") commit=$(sed -n 's/^commit: //p' <<< "$info")
|
||||
[[ -n "$version" && -n "$commit" ]] || { echo "$image reports no version or commit" >&2; return 1; }
|
||||
echo "${image}: $resolved, version $version, commit $commit"
|
||||
checked+=("$(jq -c --arg image "$resolved" --arg version "$version" --arg commit "$commit" \
|
||||
'{label, image: $image, requested: .image, version: $version, commit: $commit}' <<< "$entry")")
|
||||
done < <(jq -c '.[]' <<< "$1")
|
||||
((${#checked[@]})) || { echo 'No GreptimeDB image to check' >&2; return 1; }
|
||||
printf 'greptimedb_images=%s\n' "$(jq -sc . <<< "${checked[*]}")" >> "${GITHUB_OUTPUT:-/dev/stdout}"
|
||||
}
|
||||
|
||||
pull_images() {
|
||||
: "${RESULTS:?}" "${RUNTIME_IMAGE:?}" "${GREPTIMEDB_TAG:?}"
|
||||
: "${RESULTS:?}" "${RUNTIME_IMAGE:?}" "${GREPTIMEDB_IMAGES:?}"
|
||||
local image images=()
|
||||
mkdir -p "$RESULTS/images"
|
||||
bash "$here/pull-benchmark-image.sh" "$RUNTIME_IMAGE"
|
||||
bash "$here/pull-benchmark-image.sh" "greptime/greptimedb:$GREPTIMEDB_TAG"
|
||||
docker run --rm --entrypoint sh "$RUNTIME_IMAGE" -c \
|
||||
'test -f scripts/agent-observability/greptimedb.sh && test -f scripts/generate_agent_observations.py' \
|
||||
|| { echo 'The runtime image lacks the agent observability loader or generator' >&2; return 1; }
|
||||
while IFS= read -r image; do
|
||||
bash "$here/pull-benchmark-image.sh" "$image"
|
||||
images+=("$image")
|
||||
done < <(jq -r '.[].image' <<< "$GREPTIMEDB_IMAGES")
|
||||
docker image inspect "$RUNTIME_IMAGE" > "$RESULTS/images/runtime.json"
|
||||
docker image inspect "greptime/greptimedb:$GREPTIMEDB_TAG" > "$RESULTS/images/greptimedb.json"
|
||||
docker image inspect "${images[@]}" > "$RESULTS/images/greptimedb.json"
|
||||
}
|
||||
|
||||
wal_disk() {
|
||||
@@ -46,25 +86,242 @@ wal_disk() {
|
||||
findmnt -no SOURCE,FSTYPE -T "$root" | tee "$RESULTS/wal-disk.txt"
|
||||
}
|
||||
|
||||
# One corpus per row count the load cells use; crash cells write their own rows.
|
||||
generate() {
|
||||
: "${PLAN:?}" "${RESULTS:?}" "${CORPUS_DIR:?}" "${OWNER:?}" "${RUNTIME_IMAGE:?}"
|
||||
local args=() arg
|
||||
while IFS= read -r arg; do args+=("$arg"); done < <(jq -r '.generator_args[]' "$PLAN")
|
||||
mkdir -p "$CORPUS_DIR"
|
||||
runtime --name "$OWNER-generator" --mount "type=bind,src=$CORPUS_DIR,dst=/work/corpus" \
|
||||
"$RUNTIME_IMAGE" scripts/generate_agent_observations.py "${args[@]}" \
|
||||
--output /work/corpus/agent_observations.jsonl \
|
||||
--summary-output /work/corpus/summary.json --evidence-output /work/corpus/evidence.json \
|
||||
> "$RESULTS/generator.log" 2>&1
|
||||
jq -e --argjson rows "$(jq .rows "$PLAN")" '.line_count == $rows' "$CORPUS_DIR/summary.json" >/dev/null
|
||||
local corpus args arg dir rows
|
||||
while IFS= read -r corpus; do
|
||||
args=() dir="$CORPUS_DIR/$(jq -r .dir <<< "$corpus")" rows=$(jq .rows <<< "$corpus")
|
||||
while IFS= read -r arg; do args+=("$arg"); done < <(jq -r '.generator_args[]' <<< "$corpus")
|
||||
mkdir -p "$dir"
|
||||
runtime --name "$OWNER-generator-$rows" --mount "type=bind,src=$dir,dst=/work/corpus" \
|
||||
"$RUNTIME_IMAGE" scripts/generate_agent_observations.py "${args[@]}" \
|
||||
--output /work/corpus/agent_observations.jsonl \
|
||||
--summary-output /work/corpus/summary.json --evidence-output /work/corpus/evidence.json \
|
||||
>> "$RESULTS/generator.log" 2>&1
|
||||
jq -e --argjson rows "$rows" '.line_count == $rows' "$dir/summary.json" >/dev/null
|
||||
done < <(jq -c '.corpora[]' "$PLAN")
|
||||
}
|
||||
|
||||
# Scrape once, after the load succeeded or failed, while the server still runs.
|
||||
# Scrape once per server, after the load succeeded or failed, while it still runs:
|
||||
# the loaded server into metrics.prom, a restarted one into metrics-restart.prom.
|
||||
scrape_metrics() {
|
||||
curl --noproxy '*' -fsS --max-time 30 http://127.0.0.1:4000/metrics > "$dir/metrics.all" || { rm -f "$dir/metrics.all"; return 1; }
|
||||
grep -E '^(# (HELP|TYPE) )?greptime_logstore_object_store_wal_' "$dir/metrics.all" > "$dir/metrics.prom" || true
|
||||
grep -E '^(# (HELP|TYPE) )?(greptime_logstore_object_store_wal_|opendal_operation_(duration_seconds_count|errors_total))' \
|
||||
"$dir/metrics.all" > "$dir/metrics$phase.prom" || true
|
||||
rm -f "$dir/metrics.all"
|
||||
metrics_scraped=true
|
||||
case "$phase" in "") metrics_scraped=true ;; -restart) restart_scraped=true ;; esac
|
||||
}
|
||||
|
||||
# A crash cycle cannot repeat a scrape once its server is killed, so it retries.
|
||||
# The named WAL sample must be in the scrape, or it does not count.
|
||||
scrape_retry() {
|
||||
local required=$1 attempt
|
||||
for attempt in 1 2 3; do
|
||||
scrape_metrics && grep -q "^greptime_logstore_object_store_wal_$required " "$dir/metrics$phase.prom" && return 0
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
sql() { curl --noproxy '*' -fsS --max-time 30 --data-urlencode "sql=$1" 'http://127.0.0.1:4000/v1/sql?db=public'; }
|
||||
now_ns() { python3 -c 'import time; print(time.time_ns())'; }
|
||||
|
||||
now() { date +%s.%N; }
|
||||
elapsed() { awk -v a="$1" -v b="$(now)" 'BEGIN {printf "%.3f", b - a}'; }
|
||||
# CPU seconds the server's cgroup used, and busy and total jiffies of the host.
|
||||
server_cpu() { docker exec "$cid" cat /sys/fs/cgroup/cpu.stat 2>/dev/null | awk '/^usage_usec / {printf "%.3f", $2 / 1e6}' || true; }
|
||||
host_cpu() { awk '/^cpu / {print $2 + $3 + $4 + $7 + $8 + $9, $2 + $3 + $4 + $5 + $6 + $7 + $8 + $9}' /proc/stat 2>/dev/null || true; }
|
||||
# The host's 1, 5 and 15 minute load averages as a JSON list.
|
||||
host_load() { awk '{printf "[%s, %s, %s]", $1, $2, $3}' "${OSWAL_LOADAVG:-/proc/loadavg}" 2>/dev/null || true; }
|
||||
|
||||
# A server's log is named after the phase it started in, whatever phase stops it.
|
||||
start_server() {
|
||||
server_phase=$phase
|
||||
cid=$(docker run -d --init --name "$name$phase" --label "o11ybench.owner=$OWNER" \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
-p 127.0.0.1:4000:4000 -p 127.0.0.1:4001:4001 "${docker_env[@]}" ${mounts[@]+"${mounts[@]}"} \
|
||||
"$GREPTIMEDB_IMAGE" standalone start --http-addr=0.0.0.0:4000 --grpc-bind-addr=0.0.0.0:4001)
|
||||
local deadline=$((SECONDS + 300))
|
||||
until curl --noproxy '*' -fsS --max-time 5 http://127.0.0.1:4000/health >/dev/null 2>&1; do
|
||||
[[ $(docker inspect --format '{{.State.Running}}' "$cid") == true ]] || { echo "$id: server$phase exited" >&2; exit 1; }
|
||||
((SECONDS < deadline)) || { echo "$id: server$phase not healthy in 300s" >&2; exit 1; }
|
||||
sleep 0.2
|
||||
done
|
||||
}
|
||||
|
||||
# kill sends SIGKILL, killed collects a server the caller already killed, and
|
||||
# no argument stops it gracefully.
|
||||
stop_server() {
|
||||
case "${1:-}" in
|
||||
kill) docker kill --signal KILL "$cid" >/dev/null ;;
|
||||
killed) docker wait "$cid" >/dev/null ;;
|
||||
*) docker stop -t 120 "$cid" >/dev/null ;;
|
||||
esac
|
||||
docker logs "$cid" > "$dir/server$server_phase.log" 2>&1
|
||||
docker rm -fv "$cid" >/dev/null
|
||||
cid=
|
||||
}
|
||||
|
||||
# Kill the loaded server, start the same configuration on the same prefix and
|
||||
# data home, and time it until /health and until the loaded table answers.
|
||||
restart() {
|
||||
local table count started deadline
|
||||
table=$(jq -er .table "$dir/greptimedb-1/load-result.json")
|
||||
scrape_metrics || { echo "$id: metrics scrape failed" >&2; exit 4; }
|
||||
# The objects recovery has to read are those still indexed at the kill.
|
||||
indexed_before_kill=$(awk '/^greptime_logstore_object_store_wal_indexed_objects / {print $2}' "$dir/metrics.prom")
|
||||
stop_server kill
|
||||
phase=-restart started=$(now)
|
||||
start_server
|
||||
restart_seconds=$(elapsed "$started")
|
||||
deadline=$((SECONDS + 300))
|
||||
# Regions may still be opening after /health.
|
||||
until count=$(curl --noproxy '*' -fsS --max-time 30 --data-urlencode "sql=SELECT count(*) FROM $table" \
|
||||
'http://127.0.0.1:4000/v1/sql?db=public' | jq -er '.output[0].records.rows[0][0]'); do
|
||||
((SECONDS < deadline)) || { echo "$id: $table not readable in 300s after restart" >&2; exit 5; }
|
||||
sleep 0.2
|
||||
done
|
||||
restart_query_seconds=$(elapsed "$started") restart_count=$count
|
||||
[[ "$count" == "$(jq .rows <<< "$spec")" ]] || { echo "$id: $count rows after restart" >&2; exit 5; }
|
||||
# The restart was measured, but on fewer objects than the cell asked for.
|
||||
awk -v n="${indexed_before_kill:-0}" -v min="$(jq -r '.min_objects // 0' <<< "$spec")" 'BEGIN {exit !(n >= min)}' \
|
||||
|| { echo "$id: ${indexed_before_kill:-no} objects indexed at the kill, fewer than min_objects" >&2; exit 7; }
|
||||
}
|
||||
|
||||
# Pins the WAL with one row in a second table before the writers start, if asked,
|
||||
# and samples the object counts every 30 s while they run.
|
||||
sample_objects() {
|
||||
set +e
|
||||
local body
|
||||
while sleep "${OSWAL_SAMPLE_SECONDS:-30}"; do
|
||||
# A failed scrape is a missing sample, never zero objects.
|
||||
if ! body=$(curl --noproxy '*' -fsS --max-time 10 http://127.0.0.1:4000/metrics 2>/dev/null) \
|
||||
|| ! grep -q '^greptime_logstore_object_store_wal_indexed_objects ' <<< "$body"; then
|
||||
echo "$(date +%s) missing" >> "$dir/objects.tsv"
|
||||
continue
|
||||
fi
|
||||
awk -v t="$(date +%s)" '
|
||||
/^greptime_logstore_object_store_wal_indexed_objects / {i = $2}
|
||||
/^greptime_logstore_object_store_wal_deleted_objects_total / {d = $2}
|
||||
/^greptime_logstore_object_store_wal_created_objects_total / {c = $2}
|
||||
END {print t, i + 0, d + 0, c + 0}' <<< "$body" >> "$dir/objects.tsv"
|
||||
done
|
||||
}
|
||||
|
||||
# Writers insert striped batches until a SIGKILL at a random point stops the
|
||||
# server; it restarts on the same prefix and data home and the table is checked
|
||||
# against what was acknowledged. Nothing is cleaned between cycles.
|
||||
crash_cycles() {
|
||||
local writers rows cycles kill_min kill_max cycle cdir wpid spid= delay before after started restart_s query_s next=0 pin_count pin_args
|
||||
local load_start complete pin_left pin_base deleted sql_url=${OSWAL_SQL_URL:-http://127.0.0.1:4000/v1/sql?db=public}
|
||||
writers=$(jq -r .concurrency <<< "$spec") rows=$(jq -r .batch_rows "$PLAN") cycles=$(jq -r .cycles <<< "$spec")
|
||||
kill_min=$(jq -r '.kill_after[0]' <<< "$spec") kill_max=$(jq -r '.kill_after[1]' <<< "$spec")
|
||||
if jq -e .pin <<< "$spec" >/dev/null; then
|
||||
# Only the objects indexed before the pin row lie below it, such as the empty start object that
|
||||
# the row's write lets the store collect at any later point; the servers may collect no more.
|
||||
phase=-pin; scrape_retry indexed_objects || { echo "$id: metrics scrape failed" >&2; exit 4; }
|
||||
read -r pin_left pin_base < <(awk '/^greptime_logstore_object_store_wal_indexed_objects / {i = $2}
|
||||
/^greptime_logstore_object_store_wal_deleted_objects_total / {d = $2} END {printf "%d %d\n", i, d}' "$dir/metrics-pin.prom")
|
||||
# The pin table's own auto flush outlasts every cycle a crash group allows, so its row holds the WAL.
|
||||
sql "CREATE TABLE pin (v BIGINT, ts TIMESTAMP TIME INDEX) WITH ('auto_flush_interval' = '1d')" >/dev/null
|
||||
sql "INSERT INTO pin VALUES (1, 0)" >/dev/null
|
||||
fi
|
||||
sql "CREATE TABLE crash (seq BIGINT, ts TIMESTAMP TIME INDEX) WITH (append_mode = 'true')" >/dev/null
|
||||
for ((cycle = 1; cycle <= cycles; cycle++)); do
|
||||
cdir="$dir/cycle-$cycle" complete=true; mkdir -p "$cdir"
|
||||
# The counters of this server process before its writers start are the cycle's baseline.
|
||||
phase=-start-$cycle; scrape_retry indexed_objects || complete=false
|
||||
load_start=$(now_ns)
|
||||
python3 "$here/oswal-s3-bench-crash.py" write --dir "$cdir" --url "$sql_url" \
|
||||
--writers "$writers" --first-seq "$next" --rows "$rows" &
|
||||
wpid=$!
|
||||
if jq -e .pin <<< "$spec" >/dev/null; then sample_objects & spid=$!; fi
|
||||
delay=$(awk -v a="$kill_min" -v b="$kill_max" 'BEGIN {srand(); printf "%.3f", a + rand() * (b - a)}')
|
||||
sleep "$delay"
|
||||
phase=-cycle-$cycle; scrape_retry indexed_objects || complete=false
|
||||
before=$(now_ns); docker kill --signal KILL "$cid" >/dev/null; after=$(now_ns)
|
||||
wait "$wpid"
|
||||
if [[ -n "$spid" ]]; then kill "$spid" 2>/dev/null || true; wait "$spid" 2>/dev/null || true; spid=; fi
|
||||
stop_server killed
|
||||
phase=-restart-$cycle started=$(now)
|
||||
start_server
|
||||
restart_s=$(elapsed "$started")
|
||||
until sql "SELECT seq, count(*) FROM crash GROUP BY seq" > "$cdir/found.json" 2>/dev/null; do
|
||||
(($(elapsed "$started" | cut -d. -f1) < 300)) || { echo "$id: crash not readable after restart $cycle" >&2; exit 5; }
|
||||
sleep 0.2
|
||||
done
|
||||
query_s=$(elapsed "$started")
|
||||
scrape_retry recovery_seconds_count || complete=false
|
||||
pin_count=null pin_args=()
|
||||
if jq -e .pin <<< "$spec" >/dev/null; then
|
||||
pin_count=$(sql "SELECT count(*) FROM pin" | jq -er '.output[0].records.rows[0][0]') pin_args=(--pin-rows "$pin_count")
|
||||
fi
|
||||
local verdict=0
|
||||
python3 "$here/oswal-s3-bench-crash.py" check --run-dir "$dir" --cycle "$cycle" --found "$cdir/found.json" \
|
||||
--mode "$(jq -r .wal.ACK_MODE <<< "$spec")" --writers "$writers" --rows "$rows" \
|
||||
--before-ns "$before" --after-ns "$after" --window-max-ms 2000 ${pin_args[@]+"${pin_args[@]}"} \
|
||||
> "$cdir/check.json" || verdict=$?
|
||||
jq -c --arg delay "$delay" --arg restart "$restart_s" --arg query "$query_s" --argjson pin "$pin_count" \
|
||||
--argjson load "$(awk -v a="$load_start" -v b="$before" 'BEGIN {printf "%.3f", (b - a) / 1e9}')" \
|
||||
--argjson complete "$complete" \
|
||||
'. + {kill_after_seconds: ($delay | tonumber), load_seconds: $load, restart_seconds: ($restart | tonumber),
|
||||
restart_query_seconds: ($query | tonumber), pin_rows: $pin, metrics_complete: $complete}' \
|
||||
"$cdir/check.json" >> "$dir/cycles.jsonl"
|
||||
((verdict == 0)) || { echo "$id: cycle $cycle failed its check" >&2; exit 6; }
|
||||
# A cycle without its scrapes is not a measurement, whatever the table held.
|
||||
[[ "$complete" == true ]] || { echo "$id: cycle $cycle lost a metrics scrape" >&2; exit 4; }
|
||||
# Each server counts its collections from its own start: the first one from its pin scrape,
|
||||
# every restarted one from zero, and an absent counter has not moved.
|
||||
if jq -e .pin <<< "$spec" >/dev/null; then
|
||||
deleted=$(awk '/^greptime_logstore_object_store_wal_deleted_objects_total / {d = $2} END {printf "%d", d}' \
|
||||
"$dir/metrics-cycle-$cycle.prom")
|
||||
pin_left=$((pin_left - (deleted - pin_base))) pin_base=0
|
||||
((pin_left >= 0)) || { echo "$id: cycle $cycle collected WAL objects the pin held" >&2; exit 11; }
|
||||
fi
|
||||
next=$(cat "$cdir"/attempted-*.log | sort -n | tail -1); next=$((next + 1))
|
||||
done
|
||||
}
|
||||
|
||||
# Writes the table `dropped` for write_seconds, drops it and kills the server right
|
||||
# away, while most of its objects are still indexed; restarts on the same prefix and
|
||||
# data home, then writes and flushes the table `kept` and samples the indexed objects
|
||||
# until the dropped table's objects are collected.
|
||||
drop_restart() {
|
||||
local writers rows seconds started deadline indexed recovered sql_url=${OSWAL_SQL_URL:-http://127.0.0.1:4000/v1/sql?db=public}
|
||||
writers=$(jq -r .concurrency <<< "$spec") rows=$(jq -r .batch_rows "$PLAN") seconds=$(jq -r .write_seconds <<< "$spec")
|
||||
sql "CREATE TABLE dropped (seq BIGINT, ts TIMESTAMP TIME INDEX) WITH (append_mode = 'true')" >/dev/null
|
||||
sql "CREATE TABLE kept (seq BIGINT, ts TIMESTAMP TIME INDEX) WITH (append_mode = 'true')" >/dev/null
|
||||
python3 "$here/oswal-s3-bench-crash.py" write --dir "$dir/dropped" --url "$sql_url" --table dropped \
|
||||
--writers "$writers" --first-seq 0 --rows "$rows" --seconds "$seconds" \
|
||||
|| { echo "$id: the writers of dropped stopped before $seconds s" >&2; exit 10; }
|
||||
phase=-written; scrape_retry indexed_objects || { echo "$id: metrics scrape failed" >&2; exit 4; }
|
||||
sql "DROP TABLE dropped" >/dev/null
|
||||
stop_server kill
|
||||
phase=-restart started=$(now)
|
||||
start_server
|
||||
restart_seconds=$(elapsed "$started")
|
||||
scrape_retry recovery_seconds_count || { echo "$id: metrics scrape failed" >&2; exit 4; }
|
||||
recovered=$(awk '/^greptime_logstore_object_store_wal_recovered_objects_total / {print $2}' "$dir/metrics-restart.prom")
|
||||
python3 "$here/oswal-s3-bench-crash.py" write --dir "$dir/kept" --url "$sql_url" --table kept \
|
||||
--writers 1 --first-seq 0 --rows "$rows" --seconds 2 \
|
||||
|| { echo "$id: the writer of kept stopped before 2 s" >&2; exit 10; }
|
||||
sql "ADMIN FLUSH_TABLE('kept')" >/dev/null
|
||||
# The flushed watermark becomes durable with the next object, so every sample writes one row.
|
||||
deadline=$((SECONDS + ${OSWAL_COLLECT_SECONDS:-120}))
|
||||
while :; do
|
||||
sql "INSERT INTO kept VALUES (-1, $SECONDS)" >/dev/null
|
||||
# The samples have their own file; the final scrape keeps the restart's name.
|
||||
phase=-collected
|
||||
scrape_retry indexed_objects || { phase=-restart; echo "$id: metrics scrape failed" >&2; exit 4; }
|
||||
phase=-restart
|
||||
indexed=$(awk '/^greptime_logstore_object_store_wal_indexed_objects / {print $2}' "$dir/metrics-collected.prom")
|
||||
printf '%s %s\n' "$(date +%s)" "$indexed" >> "$dir/collected.tsv"
|
||||
# A start object, the newest object and kept's own may stay; the dropped table's go.
|
||||
awk -v i="$indexed" -v r="${recovered:-0}" 'BEGIN {exit !(i <= 3 || 2 * i < r)}' && break
|
||||
((SECONDS < deadline)) || { echo "$id: $indexed of ${recovered:-?} recovered objects still indexed" >&2; exit 9; }
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
finish() {
|
||||
@@ -74,39 +331,62 @@ finish() {
|
||||
if [[ -n "$cid" ]]; then
|
||||
# A load that succeeded without its metrics is not a measurement.
|
||||
scrape_metrics || { echo "$id: metrics scrape failed" >&2; [[ $rc != 0 ]] || rc=4; }
|
||||
docker stop -t 120 "$cid" >/dev/null
|
||||
docker logs "$cid" > "$dir/server.log" 2>&1
|
||||
docker rm -fv "$cid" >/dev/null
|
||||
stop_server
|
||||
fi
|
||||
[[ -z "$wal_dir" ]] || sudo -n rm -rf "$wal_dir"
|
||||
[[ -z "$home_dir" ]] || sudo -n rm -rf "$home_dir"
|
||||
jq -n --argjson spec "$spec" --arg device "$wal_device" --arg version "$version" \
|
||||
--arg host_load_start "$host_load_start" --arg host_load_end "$(host_load)" \
|
||||
--arg load_seconds "$load_seconds" --argjson exit_code "$rc" --argjson scraped "$metrics_scraped" \
|
||||
--argjson restart_scraped "$restart_scraped" --arg restart_seconds "$restart_seconds" \
|
||||
--arg restart_query_seconds "$restart_query_seconds" --arg restart_count "$restart_count" \
|
||||
--arg indexed_before_kill "$indexed_before_kill" \
|
||||
--arg server_cpu_seconds "$server_cpu_seconds" --arg host_cpu_busy "$host_cpu_busy" --arg host_cpus "$(nproc 2>/dev/null)" \
|
||||
'$spec + {wal_device: $device, version: $version, exit_code: $exit_code, metrics_scraped: $scraped,
|
||||
load_wall_seconds: ($load_seconds | tonumber? // null)}' > "$dir/cell.json"
|
||||
load_wall_seconds: ($load_seconds | tonumber? // null),
|
||||
server_cpu_seconds: ($server_cpu_seconds | tonumber? // null),
|
||||
host_cpu_busy: ($host_cpu_busy | tonumber? // null), host_cpus: ($host_cpus | tonumber? // null),
|
||||
restart_scraped: $restart_scraped, restart_seconds: ($restart_seconds | tonumber? // null),
|
||||
restart_query_seconds: ($restart_query_seconds | tonumber? // null),
|
||||
restart_count: ($restart_count | tonumber? // null),
|
||||
indexed_before_kill: ($indexed_before_kill | tonumber? // null),
|
||||
host_load_start: ($host_load_start | fromjson? // null), host_load_end: ($host_load_end | fromjson? // null)}' \
|
||||
> "$dir/cell.json"
|
||||
exit "$rc"
|
||||
}
|
||||
|
||||
# One cell in its own process, so errexit holds inside it. The cell's state is
|
||||
# global because the EXIT trap runs after this function returns.
|
||||
cell() {
|
||||
: "${PLAN:?}" "${RESULTS:?}" "${CORPUS_DIR:?}" "${OWNER:?}" "${RUNTIME_IMAGE:?}" "${GREPTIMEDB_IMAGE:?}"
|
||||
: "${PLAN:?}" "${RESULTS:?}" "${CORPUS_DIR:?}" "${OWNER:?}" "${RUNTIME_IMAGE:?}" "${GREPTIMEDB_IMAGES:?}"
|
||||
: "${RUN_PREFIX:?}" "${WAL_ROOT:?}" "${S3_BUCKET:?}" "${S3_REGION:?}" "${AWS_ACCESS_KEY_ID:?}" "${AWS_SECRET_ACCESS_KEY:?}"
|
||||
spec=$1 cid= wal_dir= version= load_seconds= metrics_scraped=false id=
|
||||
local concurrency name setting started ended
|
||||
id=$(jq -r .id <<< "$spec"); concurrency=$(jq -r .concurrency <<< "$spec")
|
||||
dir="$RESULTS/cells/$id"; name="$OWNER-$id"
|
||||
spec=$1 cid= wal_dir= home_dir= version= load_seconds= metrics_scraped=false id= phase= server_phase=
|
||||
restart_scraped=false restart_seconds= restart_query_seconds= restart_count= server_cpu_seconds= host_cpu_busy=
|
||||
indexed_before_kill= host_load_start=$(host_load)
|
||||
local concurrency setting started cpu_before host_before host_after corpus partitions load script label
|
||||
id=$(jq -r .id <<< "$spec"); concurrency=$(jq -r .concurrency <<< "$spec") label=$(jq -r .image <<< "$spec")
|
||||
dir="$RESULTS/cells/$id" name="$OWNER-$id"
|
||||
# The digest the image of the cell's label was pinned to before the run.
|
||||
GREPTIMEDB_IMAGE=$(jq -er --arg label "$label" '.[] | select(.label == $label) | .image' <<< "$GREPTIMEDB_IMAGES") \
|
||||
|| { echo "$id: no image labelled '$label'" >&2; exit 1; }
|
||||
mkdir -p "$dir"
|
||||
# The root holds data/ and the WAL's default prefix wal/ side by side.
|
||||
local settings=(STORAGE__TYPE=S3 "STORAGE__BUCKET=$S3_BUCKET" "STORAGE__REGION=$S3_REGION"
|
||||
"STORAGE__ROOT=$RUN_PREFIX/$id")
|
||||
[[ -z "${S3_CONTAINER_ENDPOINT:-}" ]] || settings+=("STORAGE__ENDPOINT=$S3_CONTAINER_ENDPOINT")
|
||||
# A restarted server needs the metadata the loaded one kept in its data home.
|
||||
jq -e '.restart or .mode != "load"' <<< "$spec" >/dev/null && settings+=(STORAGE__DATA_HOME=/oswal_home)
|
||||
while IFS= read -r setting; do settings+=("$setting"); done \
|
||||
< <(jq -r '.wal | to_entries[] | "WAL__\(.key)=\(.value)"' <<< "$spec")
|
||||
local docker_env=() mounts=()
|
||||
docker_env=() mounts=()
|
||||
for setting in "${settings[@]}"; do
|
||||
docker_env+=(-e "GREPTIMEDB_STANDALONE__$setting")
|
||||
printf 'GREPTIMEDB_STANDALONE__%s\n' "$setting" >> "$dir/config.env"
|
||||
done
|
||||
while IFS= read -r setting; do
|
||||
docker_env+=(-e "$setting")
|
||||
printf '%s\n' "$setting" >> "$dir/config.env"
|
||||
done < <(jq -r '.env // {} | to_entries[] | "\(.key)=\(.value)"' <<< "$spec")
|
||||
# Pass the two keys by name: their values never reach argv, a file or the artifact.
|
||||
export GREPTIMEDB_STANDALONE__STORAGE__ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
|
||||
export GREPTIMEDB_STANDALONE__STORAGE__SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
|
||||
@@ -119,33 +399,60 @@ cell() {
|
||||
else
|
||||
wal_device="s3://$S3_BUCKET/$RUN_PREFIX/$id/wal"
|
||||
fi
|
||||
if jq -e '.restart or .mode != "load"' <<< "$spec" >/dev/null; then
|
||||
home_dir="$WAL_ROOT/$id-home"; mkdir -p "$home_dir"
|
||||
mounts+=(--mount "type=bind,src=$home_dir,dst=/oswal_home")
|
||||
fi
|
||||
trap finish EXIT
|
||||
cid=$(docker run -d --init --name "$name" --label "o11ybench.owner=$OWNER" \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
-p 127.0.0.1:4000:4000 -p 127.0.0.1:4001:4001 "${docker_env[@]}" ${mounts[@]+"${mounts[@]}"} \
|
||||
"$GREPTIMEDB_IMAGE" standalone start --http-addr=0.0.0.0:4000 --grpc-bind-addr=0.0.0.0:4001)
|
||||
local deadline=$((SECONDS + 300))
|
||||
until curl --noproxy '*' -fsS --max-time 5 http://127.0.0.1:4000/health >/dev/null 2>&1; do
|
||||
[[ $(docker inspect --format '{{.State.Running}}' "$cid") == true ]] || { echo "$id: server exited" >&2; exit 1; }
|
||||
((SECONDS < deadline)) || { echo "$id: server not healthy in 300s" >&2; exit 1; }
|
||||
sleep 2
|
||||
done
|
||||
start_server
|
||||
version=$(docker exec "$cid" greptime --version | sed -n 's/^version: //p')
|
||||
[[ -n "$version" ]]
|
||||
started=$(date +%s.%N)
|
||||
# Only the load phase, run exactly as run-agent-observability.sh runs it.
|
||||
if jq -e '.mode == "crash"' <<< "$spec" >/dev/null; then crash_cycles; return; fi
|
||||
if jq -e '.mode == "drop"' <<< "$spec" >/dev/null; then drop_restart; return; fi
|
||||
corpus="$CORPUS_DIR/corpus-$(jq -r .rows <<< "$spec")" partitions=$(jq -r '.partitions // 1' <<< "$spec")
|
||||
# Only the load phase, run exactly as run-agent-observability.sh runs it, or with the
|
||||
# loader writing into a table split into regions that the cell created first.
|
||||
load=(scripts/agent-observability/greptimedb.sh load) script=()
|
||||
if ((partitions > 1)); then
|
||||
sql "$(python3 "$here/oswal-s3-bench.py" table --summary "$corpus/summary.json" --partitions "$partitions")" >/dev/null
|
||||
load=(/work/oswal-s3-bench-load.sh)
|
||||
script=(--mount "type=bind,src=$here/oswal-s3-bench-load.sh,dst=/work/oswal-s3-bench-load.sh,readonly")
|
||||
fi
|
||||
cpu_before=$(server_cpu) host_before=$(host_cpu)
|
||||
started=$(now)
|
||||
runtime --name "$name-load" --network host \
|
||||
--mount "type=bind,src=$CORPUS_DIR,dst=/work/corpus,readonly" --mount "type=bind,src=$dir,dst=/results" \
|
||||
-e GRPC_ENDPOINT=127.0.0.1:4001 -e "DATA_PROFILE=$(jq -r .data_profile "$PLAN")" -e RESULT_NAME=greptimedb-1 \
|
||||
--mount "type=bind,src=$corpus,dst=/work/corpus,readonly" ${script[@]+"${script[@]}"} \
|
||||
--mount "type=bind,src=$dir,dst=/results" \
|
||||
-e GRPC_ENDPOINT=127.0.0.1:4001 -e "DATA_PROFILE=$(jq -r .data_profile <<< "$spec")" -e RESULT_NAME=greptimedb-1 \
|
||||
-e "ENDPOINT=http://127.0.0.1:4000/v1/sql?db=public&format=greptimedb_v1" \
|
||||
-e "TARGET_IMAGE=$GREPTIMEDB_IMAGE" -e "TARGET_VERSION=$version" -e "TARGET_DIGEST=${GREPTIMEDB_IMAGE##*@}" \
|
||||
-e "TARGET_ID=$name" -e TARGET_HOST=127.0.0.1 -e PREPARE_WORKERS= -e "MAX_IN_FLIGHT=$concurrency" \
|
||||
-e "GREPTIMEDB_BATCH_SIZE=$(jq -r .batch_rows "$PLAN")" --entrypoint /bin/bash \
|
||||
"$RUNTIME_IMAGE" scripts/agent-observability/greptimedb.sh load > "$dir/load.log" 2>&1
|
||||
ended=$(date +%s.%N)
|
||||
load_seconds=$(awk -v a="$started" -v b="$ended" 'BEGIN {printf "%.3f", b - a}')
|
||||
jq -e --argjson rows "$(jq .rows "$PLAN")" '.count_before == 0 and .count_after == $rows' \
|
||||
"$RUNTIME_IMAGE" "${load[@]}" > "$dir/load.log" 2>&1
|
||||
load_seconds=$(elapsed "$started")
|
||||
host_after=$(host_cpu)
|
||||
server_cpu_seconds=$(awk -v a="$cpu_before" -v b="$(server_cpu)" 'BEGIN {if (a != "" && b != "") printf "%.3f", b - a}')
|
||||
# Busy share of all host CPUs during the load, the loader included.
|
||||
host_cpu_busy=$(awk -v a="$host_before" -v b="$host_after" \
|
||||
'BEGIN {split(a, x, " "); split(b, y, " "); if (y[2] > x[2]) printf "%.4f", (y[1] - x[1]) / (y[2] - x[2])}')
|
||||
jq -e --argjson rows "$(jq .rows <<< "$spec")" '.count_before == 0 and .count_after == $rows' \
|
||||
"$dir/greptimedb-1/load-result.json" >/dev/null
|
||||
if ((partitions > 1)); then
|
||||
# The server's own account of the rows each region took.
|
||||
sql "SELECT p.partition_name, s.region_id, s.region_rows, s.written_bytes_since_open
|
||||
FROM information_schema.partitions p JOIN information_schema.region_statistics s
|
||||
ON p.greptime_partition_id = s.region_id
|
||||
WHERE p.table_schema = 'public' AND p.table_name = 'agent_observations'
|
||||
ORDER BY s.region_id" > "$dir/regions.json"
|
||||
jq -e --argjson n "$partitions" '[.output[0].records.rows[] | select(.[2] > 0)] | length == $n' \
|
||||
"$dir/regions.json" >/dev/null || { echo "$id: fewer than $partitions regions took rows" >&2; exit 8; }
|
||||
fi
|
||||
# An idle window: what the server sends with no writes, between two scrapes.
|
||||
if jq -e '.idle_seconds > 0' <<< "$spec" >/dev/null; then
|
||||
phase=-idle; scrape_metrics || { echo "$id: metrics scrape failed" >&2; exit 4; }; phase=
|
||||
sleep "$(jq -r .idle_seconds <<< "$spec")"
|
||||
fi
|
||||
if jq -e '.restart' <<< "$spec" >/dev/null; then restart; fi
|
||||
}
|
||||
|
||||
scrub() {
|
||||
@@ -160,6 +467,7 @@ scrub() {
|
||||
|
||||
case "${1:-}" in
|
||||
check-runtime) check_runtime "${2:?}" ;;
|
||||
check-images) check_images "${2:?}" ;;
|
||||
pull) pull_images ;;
|
||||
wal-disk) wal_disk "${2:?}" ;;
|
||||
generate) generate ;;
|
||||
@@ -175,5 +483,5 @@ case "${1:-}" in
|
||||
done < <(jq -c '.cells[]' "$PLAN")
|
||||
exit "$failed" ;;
|
||||
scrub) scrub ;;
|
||||
*) echo "usage: $0 check-runtime IMAGE | pull | wal-disk DIR | generate | cells | scrub" >&2; exit 2 ;;
|
||||
*) echo "usage: $0 check-runtime IMAGE | check-images JSON | pull | wal-disk DIR | generate | cells | scrub" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
@@ -26,14 +26,30 @@ from pathlib import Path
|
||||
PREFIX = "greptime_logstore_object_store_wal_"
|
||||
COUNTERS = ("created_objects_total", "stalled_appends_total", "poisoned_total",
|
||||
"create_failures_total", "create_conflicts_total")
|
||||
# The Raft Engine controls every object store WAL cell is compared with: fsync per write and the default.
|
||||
BASELINES = ("R", "R0")
|
||||
SAMPLE = re.compile(r'([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)')
|
||||
# Mito's line for every region it opened, with the WAL range it replayed.
|
||||
REPLAY = re.compile(r"Replay WAL for region: (\d+)\(.*?provider: (\w+).*?rows recovered: (\d+), "
|
||||
r"replay from entry id: (\d+), last entry id: (\d+),.*?elapsed: ([0-9.]+)(ns|µs|us|ms|s)\b")
|
||||
UNITS = {"ns": 1e-9, "µs": 1e-6, "us": 1e-6, "ms": 1e-3, "s": 1.0}
|
||||
# An object store WAL entry id is the object sequence above a 20-bit position.
|
||||
POSITION_BITS = 20
|
||||
|
||||
|
||||
def parse_metrics(text: str) -> dict:
|
||||
"""Map each unlabeled sample to its value and each histogram to its buckets."""
|
||||
samples, buckets = {}, {}
|
||||
"""Map each unlabeled sample to its value, each histogram to its buckets, and the S3
|
||||
operations (data files and WAL together, as OpenDAL counts its calls: a paginated list or a
|
||||
multipart write is one operation over several HTTP requests) to their count per kind; the local cache
|
||||
stores report through the same layer under another scheme and are left out."""
|
||||
samples, buckets, operations = {}, {}, {}
|
||||
for line in text.splitlines():
|
||||
match = SAMPLE.fullmatch(line.strip())
|
||||
if match and match.group(1) == "opendal_operation_duration_seconds_count":
|
||||
operation = re.search(r'operation="([^"]+)"', match.group(2) or "")
|
||||
if operation and re.search(r'scheme="s3"', match.group(2) or ""):
|
||||
operations[operation.group(1)] = operations.get(operation.group(1), 0) + float(match.group(3))
|
||||
continue
|
||||
if not match or not match.group(1).startswith(PREFIX):
|
||||
continue
|
||||
name, labels, value = match.group(1)[len(PREFIX):], match.group(2) or "", float(match.group(3))
|
||||
@@ -42,7 +58,8 @@ def parse_metrics(text: str) -> dict:
|
||||
buckets.setdefault(name[:-len("_bucket")], []).append((float(le.group(1)), value))
|
||||
else:
|
||||
samples[name] = value
|
||||
return {"samples": samples, "buckets": {name: sorted(items) for name, items in buckets.items()}}
|
||||
return {"samples": samples, "buckets": {name: sorted(items) for name, items in buckets.items()},
|
||||
"operations": operations}
|
||||
|
||||
|
||||
def quantile(q: float, buckets: list) -> float | None:
|
||||
@@ -64,25 +81,151 @@ def geomean(values: list) -> float | None:
|
||||
return math.exp(sum(map(math.log, values)) / len(values)) if values and all(v > 0 for v in values) else None
|
||||
|
||||
|
||||
def read_metrics(path: Path) -> dict:
|
||||
return parse_metrics(path.read_text() if path.exists() else "")
|
||||
|
||||
|
||||
def ratio(a, b):
|
||||
return a / b if a is not None and b else None
|
||||
|
||||
|
||||
def replay(log: Path, metrics: dict) -> dict | None:
|
||||
"""What a restarted server replayed: every region it opened, the span of object sequences
|
||||
its WAL range covers, and the log store's time serving region reads (`read_seconds`). The span
|
||||
bounds the objects a region read from above: objects without its entries are skipped."""
|
||||
if not log.exists():
|
||||
return None
|
||||
regions = []
|
||||
for match in REPLAY.finditer(log.read_text(errors="replace")):
|
||||
region, provider, rows, first, last, elapsed, unit = match.groups()
|
||||
# The range runs from the first entry to replay to the last one; it is empty when first > last.
|
||||
span = None if provider != "ObjectStore" else 0 if int(first) > int(last) else \
|
||||
(int(last) >> POSITION_BITS) - (int(first) >> POSITION_BITS) + 1
|
||||
regions.append({"region": int(region), "rows": int(rows), "first_entry_id": int(first),
|
||||
"last_entry_id": int(last), "seconds": float(elapsed) * UNITS[unit], "object_span": span})
|
||||
samples = metrics["samples"]
|
||||
return {"regions": regions, "rows": sum(r["rows"] for r in regions),
|
||||
"seconds": max((r["seconds"] for r in regions), default=None),
|
||||
"object_span": sum(r["object_span"] for r in regions)
|
||||
if regions and regions[0]["object_span"] is not None else None,
|
||||
"read_seconds": samples.get("read_seconds_sum"), "reads": samples.get("read_seconds_count")}
|
||||
|
||||
|
||||
def replayed(regions: list) -> str:
|
||||
"""Each region that replayed rows: its rows, the object sequences its range spans and its time."""
|
||||
return "; ".join(f"{r['region']}: {r['rows']:,} rows" + (
|
||||
f", object sequences {r['first_entry_id'] >> POSITION_BITS:,}–{r['last_entry_id'] >> POSITION_BITS:,}"
|
||||
if r["object_span"] else "") + f", {r['seconds']:.2f} s" for r in regions if r["rows"]) or "—"
|
||||
|
||||
|
||||
def client_latency(path: Path) -> tuple:
|
||||
"""The 50th and 99th percentile of the loader's own time per write request."""
|
||||
durations = []
|
||||
if path.exists():
|
||||
for line in path.read_text().splitlines():
|
||||
record = json.loads(line) if line.strip() else {}
|
||||
if record.get("step") == "grpc_write_batch" and record.get("status") == "success":
|
||||
durations.append(record["duration_seconds"])
|
||||
durations.sort()
|
||||
return tuple(durations[max(0, math.ceil(q * len(durations)) - 1)] if durations else None for q in (0.5, 0.99))
|
||||
|
||||
|
||||
def region_rows(path: Path) -> list | None:
|
||||
"""Rows per region of a partitioned load cell, as the server reported them after the load."""
|
||||
if not path.exists():
|
||||
return None
|
||||
return [int(row[2] or 0) for row in json.loads(path.read_text())["output"][0]["records"]["rows"]]
|
||||
|
||||
|
||||
def interval_seconds(wal: dict) -> float:
|
||||
value = wal.get("FLUSH_INTERVAL", "100ms")
|
||||
return float(value[:-2]) / 1000 if value.endswith("ms") else float(value.rstrip("s"))
|
||||
|
||||
|
||||
def traffic(end: dict, start: dict, seconds, wal: dict) -> dict:
|
||||
"""WAL traffic between two scrapes of one process over `seconds`; more objects than seal
|
||||
ticks, or objects near the 8 MiB threshold, mean batches sealed on size rather than on the timer."""
|
||||
def delta(name):
|
||||
value = end["samples"].get(name)
|
||||
return None if value is None else value - start["samples"].get(name, 0)
|
||||
created, written, objects = delta("created_objects_total"), delta("object_bytes_sum"), delta("object_bytes_count")
|
||||
per_second = created / seconds if created and seconds else None
|
||||
mean = written / objects if written and objects else None
|
||||
return {"wal_mb_per_second": written / seconds / 1e6 if written and seconds else None,
|
||||
"objects_per_second": per_second, "mean_object_bytes": mean,
|
||||
"sealed_on_size": None if per_second is None else
|
||||
per_second > 1.05 / interval_seconds(wal) or (mean or 0) >= 0.9 * 8 * 2**20,
|
||||
"operations": {op: n - start["operations"].get(op, 0) for op, n in end["operations"].items()
|
||||
if n != start["operations"].get(op, 0)}}
|
||||
|
||||
|
||||
def crash_cycles(path: Path, wal: dict) -> list:
|
||||
"""Each cycle's check plus the loaded server's metrics before the kill and the restarted one's after."""
|
||||
cycles = [json.loads(line) for line in (path / "cycles.jsonl").read_text().splitlines()
|
||||
if line.strip()] if (path / "cycles.jsonl").exists() else []
|
||||
for cycle in cycles:
|
||||
start = read_metrics(path / f"metrics-start-{cycle['cycle']}.prom")
|
||||
killed = read_metrics(path / f"metrics-cycle-{cycle['cycle']}.prom")
|
||||
before = killed["samples"]
|
||||
after = read_metrics(path / f"metrics-restart-{cycle['cycle']}.prom")
|
||||
once = after["samples"].get("recovery_seconds_count") == 1
|
||||
cycle |= traffic(killed, start, cycle.get("load_seconds"), wal)
|
||||
cycle |= {"created_before_kill": before.get("created_objects_total"),
|
||||
"indexed_before_kill": before.get("indexed_objects"),
|
||||
"recovery_seconds": after["samples"].get("recovery_seconds_sum") if once else None,
|
||||
"recovered_objects": after["samples"].get("recovered_objects_total", 0 if once else None),
|
||||
"restart_operations": after["operations"],
|
||||
"replay": replay(path / f"server-restart-{cycle['cycle']}.log", after)}
|
||||
return cycles
|
||||
|
||||
|
||||
def load_cell(path: Path, spec: dict | None = None) -> dict:
|
||||
"""One cell's row; a planned cell that never recorded a result has no values, so no ratio uses its round."""
|
||||
recorded = path / "cell.json"
|
||||
cell = json.loads(recorded.read_text()) if recorded.exists() else spec | {"exit_code": None, "missing": True}
|
||||
result_path = path / "greptimedb-1" / "load-result.json"
|
||||
result = json.loads(result_path.read_text()) if result_path.exists() else {}
|
||||
metrics_path = path / "metrics.prom"
|
||||
metrics = parse_metrics(metrics_path.read_text() if metrics_path.exists() else "")
|
||||
ack = metrics["buckets"].get("append_ack_seconds", [])
|
||||
final, restarted = read_metrics(path / "metrics.prom"), read_metrics(path / "metrics-restart.prom")
|
||||
# With an idle window the scrape taken when the load ended describes the load.
|
||||
metrics = read_metrics(path / "metrics-idle.prom") if cell.get("idle_seconds") else final
|
||||
load_seconds = result.get("load_seconds")
|
||||
wall, cpu = cell.get("load_wall_seconds"), cell.get("server_cpu_seconds")
|
||||
row = {
|
||||
"id": cell["id"], "round": cell["round"], "concurrency": cell["concurrency"],
|
||||
"id": cell["id"], "group": cell.get("group", ""), "mode": cell.get("mode", "load"),
|
||||
"image": cell.get("image", ""), "warmup": cell.get("warmup", False),
|
||||
"round": cell["round"], "concurrency": cell["concurrency"],
|
||||
"config": cell["config"], "exit_code": cell["exit_code"], "wal_device": cell.get("wal_device"),
|
||||
"missing": cell.get("missing", False),
|
||||
"version": cell.get("version"), "load_wall_seconds": cell.get("load_wall_seconds"),
|
||||
"missing": cell.get("missing", False), "env": cell.get("env", {}),
|
||||
"version": cell.get("version"), "load_wall_seconds": wall,
|
||||
"rows": result.get("count_after"), "load_seconds": load_seconds,
|
||||
"rows_per_second": result.get("rows_per_second") if cell["exit_code"] == 0 else None,
|
||||
"append_ack_p50_seconds": quantile(0.5, ack), "append_ack_p99_seconds": quantile(0.99, ack),
|
||||
"server_cpu_cores": cpu / wall if cpu is not None and wall else None,
|
||||
"host_cpu_busy": cell.get("host_cpu_busy"), "host_cpus": cell.get("host_cpus"),
|
||||
"host_load_start": cell.get("host_load_start"), "host_load_end": cell.get("host_load_end"),
|
||||
}
|
||||
for name, key in (("append_ack_seconds", "append_ack"), ("seal_to_durable_seconds", "seal_to_durable"),
|
||||
("object_bytes", "object_bytes")):
|
||||
buckets = metrics["buckets"].get(name, [])
|
||||
row[f"{key}_p50"], row[f"{key}_p99"] = quantile(0.5, buckets), quantile(0.99, buckets)
|
||||
row["client_p50"], row["client_p99"] = client_latency(path / "greptimedb-1" / "load-transcript.jsonl")
|
||||
# An entry is one region's write; an append, counted only in the durable mode, carries one or more.
|
||||
samples = metrics["samples"]
|
||||
row["entries_per_object"] = ratio(samples.get("object_entries_sum"), samples.get("object_entries_count"))
|
||||
row["appends_per_object"] = ratio(samples.get("append_ack_seconds_count"), samples.get("object_bytes_count"))
|
||||
row["partitions"] = cell.get("partitions", 1)
|
||||
row["region_rows"] = region_rows(path / "regions.json")
|
||||
if cell.get("restart"):
|
||||
# A restarted process observed exactly one recovery, so the sum is its duration.
|
||||
count = restarted["samples"].get("recovery_seconds_count")
|
||||
row["restart"] = {
|
||||
"wall_seconds": cell.get("restart_seconds"), "query_seconds": cell.get("restart_query_seconds"),
|
||||
"rows": cell.get("restart_count"),
|
||||
"recovery_seconds": restarted["samples"].get("recovery_seconds_sum") if count == 1 else None,
|
||||
"recovered_objects": restarted["samples"].get("recovered_objects_total")
|
||||
if cell.get("restart_scraped") else None,
|
||||
"indexed_before_kill": cell.get("indexed_before_kill"), "min_objects": cell.get("min_objects", 0),
|
||||
"replay": replay(path / "server-restart.log", restarted),
|
||||
}
|
||||
# The counters register on their first event, so a scrape of an object store WAL
|
||||
# that created objects omits the ones that never moved: those are zero, not unknown.
|
||||
active = (cell.get("metrics_scraped") and cell.get("wal", {}).get("PROVIDER") == "experimental_object_store"
|
||||
@@ -91,49 +234,262 @@ def load_cell(path: Path, spec: dict | None = None) -> dict:
|
||||
row[counter] = metrics["samples"].get(counter, 0 if active else None)
|
||||
created = row["created_objects_total"]
|
||||
row["creates_per_minute"] = created * 60 / load_seconds if created is not None and load_seconds else None
|
||||
row |= traffic(metrics, {"samples": {}, "operations": {}}, load_seconds, cell.get("wal", {}))
|
||||
row["restart_operations"] = restarted["operations"]
|
||||
if cell.get("idle_seconds"):
|
||||
idle = traffic(final, metrics, cell["idle_seconds"], cell.get("wal", {}))
|
||||
end = final["samples"].get("created_objects_total")
|
||||
row["idle"] = {"seconds": cell["idle_seconds"], "operations": idle["operations"],
|
||||
"creates": None if end is None else end - metrics["samples"].get("created_objects_total", 0)}
|
||||
if row["mode"] == "drop":
|
||||
written = read_metrics(path / "metrics-written.prom")["samples"]
|
||||
restarted_samples = restarted["samples"]
|
||||
samples = [line.split() for line in (path / "collected.tsv").read_text().splitlines()
|
||||
if line.strip()] if (path / "collected.tsv").exists() else []
|
||||
row["drop"] = {
|
||||
"acked_batches": sum(len((p.read_text()).split()) for p in sorted((path / "dropped").glob("acked-*.log"))),
|
||||
"indexed_before_drop": written.get("indexed_objects"),
|
||||
"restart_seconds": cell.get("restart_seconds"),
|
||||
"recovered_objects": restarted_samples.get("recovered_objects_total"),
|
||||
"indexed_at_end": int(float(samples[-1][1])) if samples else None,
|
||||
"deleted_after_restart": restarted_samples.get("deleted_objects_total"),
|
||||
"seconds_to_collect": int(samples[-1][0]) - int(samples[0][0]) if samples else None,
|
||||
}
|
||||
if row["mode"] == "crash":
|
||||
row["cycles"] = crash_cycles(path, cell.get("wal", {}))
|
||||
if (path / "objects.tsv").exists():
|
||||
# A sample is time, indexed, deleted, created; a failed scrape is time and "missing".
|
||||
row["objects_samples"] = [[int(float(line.split()[0]))] + (
|
||||
[None] * 3 if line.split()[1:] == ["missing"] else [int(float(v)) for v in line.split()[1:]])
|
||||
for line in (path / "objects.tsv").read_text().splitlines() if line.strip()]
|
||||
return row
|
||||
|
||||
|
||||
def ratios(rows: list) -> list:
|
||||
"""D/R and E/R per round and their geometric mean over the rounds, per concurrency."""
|
||||
speed = {(r["round"], r["concurrency"], r["config"]): r["rows_per_second"] for r in rows}
|
||||
"""Each object store WAL cell over each Raft Engine control of the same image per round, and their
|
||||
geometric mean over the rounds, per concurrency; warm-up cells are left out."""
|
||||
rows = [r for r in rows if r["mode"] == "load" and not r["warmup"]]
|
||||
speed = {(r["group"], r["image"], r["round"], r["concurrency"], r["config"]): r["rows_per_second"]
|
||||
for r in rows}
|
||||
result = []
|
||||
for concurrency in sorted({r["concurrency"] for r in rows}):
|
||||
rounds = sorted({r["round"] for r in rows if r["concurrency"] == concurrency})
|
||||
for config in ("D", "E"):
|
||||
per_round = []
|
||||
for round_number in rounds:
|
||||
base, value = speed.get((round_number, concurrency, "R")), speed.get((round_number, concurrency, config))
|
||||
per_round.append(value / base if base and value else None)
|
||||
complete = [value for value in per_round if value is not None]
|
||||
result.append({"concurrency": concurrency, "ratio": f"{config}/R", "per_round": per_round,
|
||||
"geomean": geomean(complete) if len(complete) == len(per_round) else None})
|
||||
for group, image in dict.fromkeys((r["group"], r["image"]) for r in rows):
|
||||
members = [r for r in rows if (r["group"], r["image"]) == (group, image)]
|
||||
series = sorted({r["config"] for r in members if r["config"] not in BASELINES})
|
||||
baselines = [b for b in BASELINES if any(r["config"] == b for r in members)]
|
||||
for concurrency in sorted({r["concurrency"] for r in members}):
|
||||
rounds = sorted({r["round"] for r in members if r["concurrency"] == concurrency})
|
||||
for config, baseline in ((s, b) for s in series for b in baselines):
|
||||
per_round = []
|
||||
for round_number in rounds:
|
||||
base = speed.get((group, image, round_number, concurrency, baseline))
|
||||
measured = speed.get((group, image, round_number, concurrency, config))
|
||||
per_round.append(measured / base if base and measured else None)
|
||||
complete = [ratio for ratio in per_round if ratio is not None]
|
||||
result.append({"group": group, "image": image, "concurrency": concurrency,
|
||||
"ratio": f"{config}/{baseline}", "per_round": per_round,
|
||||
"geomean": geomean(complete) if len(complete) == len(per_round) else None})
|
||||
return result
|
||||
|
||||
|
||||
def image_ratios(rows: list, images: list) -> list:
|
||||
"""Every load cell of every image per round, and each image's rows/s over the first image's in the
|
||||
same round, with their geometric mean, minimum and maximum over the rounds; warm-up cells are left out."""
|
||||
rows = [r for r in rows if r["mode"] == "load" and not r["warmup"]]
|
||||
cells = {(r["group"], r["image"], r["round"], r["concurrency"], r["config"]): r for r in rows}
|
||||
result = []
|
||||
for group, concurrency, config in dict.fromkeys((r["group"], r["concurrency"], r["config"]) for r in rows):
|
||||
rounds = sorted({r["round"] for r in rows if (r["group"], r["concurrency"], r["config"]) == (group, concurrency, config)})
|
||||
for image in images:
|
||||
measured = [cells.get((group, image, n, concurrency, config)) or {} for n in rounds]
|
||||
reference = [cells.get((group, images[0], n, concurrency, config)) or {} for n in rounds]
|
||||
per_round = [ratio(m.get("rows_per_second"), r.get("rows_per_second")) for m, r in zip(measured, reference)]
|
||||
complete = image != images[0] and None not in per_round
|
||||
result.append({
|
||||
"group": group, "concurrency": concurrency, "config": config, "image": image,
|
||||
"rounds": [{"round": n, "id": m.get("id"), "rows_per_second": m.get("rows_per_second"),
|
||||
"client_p50": m.get("client_p50"), "client_p99": m.get("client_p99"),
|
||||
"host_load_start": m.get("host_load_start"), "host_load_end": m.get("host_load_end")}
|
||||
for n, m in zip(rounds, measured)],
|
||||
"ratio_to": None if image == images[0] else images[0],
|
||||
"per_round": None if image == images[0] else per_round,
|
||||
"geomean": geomean(per_round) if complete else None,
|
||||
"min": min(per_round) if complete else None, "max": max(per_round) if complete else None})
|
||||
return result
|
||||
|
||||
|
||||
def operations(counts: dict) -> str:
|
||||
return ", ".join(f"{op} {n:,.0f}" for op, n in sorted(counts.items())) or "—"
|
||||
|
||||
|
||||
def overrides(env: dict) -> str:
|
||||
return " ".join(f"{k}={v}" for k, v in sorted(env.items())) or "—"
|
||||
|
||||
|
||||
def pair(a, b, digits=0, scale=1.0) -> str:
|
||||
return f"{fmt(a, digits, scale)} / {fmt(b, digits, scale)}"
|
||||
|
||||
|
||||
def fmt(value, digits=0, scale=1.0) -> str:
|
||||
return "—" if value is None else f"{value * scale:,.{digits}f}"
|
||||
|
||||
|
||||
def render(run: dict, rows: list, pairs: list) -> str:
|
||||
def load_average(value) -> str:
|
||||
"""The 1-minute load average of a recorded [1, 5, 15] minute triple."""
|
||||
return "—" if not value else f"{value[0]:.2f}"
|
||||
|
||||
|
||||
def render(run: dict, rows: list, pairs: list, compared: list = ()) -> str:
|
||||
lines = [f"## Object store WAL against S3 ({'smoke, MinIO' if run.get('smoke') else run.get('instance_type')})", ""]
|
||||
lines += [f"GreptimeDB `{run.get('greptimedb_image')}` ({next((r['version'] for r in rows if r['version']), '—')}); "
|
||||
f"{fmt(run.get('rows'))} rows in {run.get('batch_rows')}-row batches; bucket region {run.get('region')}.", ""]
|
||||
lines += ["| Cell | Rows/s | Load s | Wall s | Rows | Ack p50 ms | Ack p99 ms | Creates | Creates/min "
|
||||
"| Stalled | Poisoned | Create failures | Conflicts | WAL on |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |"]
|
||||
for r in rows:
|
||||
cell = f"{r['id']} (not run)" if r["missing"] else r["id"] if r["exit_code"] == 0 \
|
||||
else f"{r['id']} (failed, exit {r['exit_code']})"
|
||||
images = run.get("greptimedb_images") or []
|
||||
if len(images) > 1:
|
||||
lines += [f"{fmt(run.get('rows'))} rows in {run.get('batch_rows')}-row batches; bucket region {run.get('region')}; "
|
||||
f"each image is compared with the first, `{images[0]['label']}`.", "",
|
||||
"| Image | Reference | Version | Commit |", "| --- | --- | --- | --- |"]
|
||||
lines += [f"| {i['label']} | `{i['image']}` | {i['version']} | `{i['commit']}` |" for i in images] + [""]
|
||||
else:
|
||||
lines += [f"GreptimeDB `{run.get('greptimedb_image')}` ({next((r['version'] for r in rows if r['version']), '—')}); "
|
||||
f"{fmt(run.get('rows'))} rows in {run.get('batch_rows')}-row batches; bucket region {run.get('region')}.", ""]
|
||||
lines += ["| Cell | Overrides | Rows/s | Load s | Wall s | Ack p50 / p99 ms | Seal to durable p50 / p99 ms "
|
||||
"| Object p50 / p99 MiB | Creates | Creates/min | Stalled | Poisoned | Create failures | Conflicts "
|
||||
"| Server CPUs | Host busy | WAL on | Load average at start / end |",
|
||||
"| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: "
|
||||
"| ---: | ---: | --- | ---: |"]
|
||||
def label(r):
|
||||
cell = f"{r['id']} (warm-up)" if r["warmup"] else r["id"]
|
||||
return f"{cell} (not run)" if r["missing"] else cell if r["exit_code"] == 0 \
|
||||
else f"{cell} (failed, exit {r['exit_code']})"
|
||||
loads = [r for r in rows if r["mode"] == "load"]
|
||||
for r in loads:
|
||||
cell = label(r)
|
||||
lines.append(" | ".join([
|
||||
f"| {cell}", fmt(r["rows_per_second"]), fmt(r["load_seconds"], 1), fmt(r["load_wall_seconds"], 1),
|
||||
fmt(r["rows"]), fmt(r["append_ack_p50_seconds"], 1, 1000), fmt(r["append_ack_p99_seconds"], 1, 1000),
|
||||
f"| {cell}", overrides(r["env"]), fmt(r["rows_per_second"]), fmt(r["load_seconds"], 1),
|
||||
fmt(r["load_wall_seconds"], 1), pair(r["append_ack_p50"], r["append_ack_p99"], 1, 1000),
|
||||
pair(r["seal_to_durable_p50"], r["seal_to_durable_p99"], 1, 1000),
|
||||
pair(r["object_bytes_p50"], r["object_bytes_p99"], 1, 1 / 2**20),
|
||||
*(fmt(r[c]) for c in COUNTERS[:1]), fmt(r["creates_per_minute"], 1),
|
||||
*(fmt(r[c]) for c in COUNTERS[1:]), f"`{r['wal_device'] or '—'}` |"]))
|
||||
lines += ["", "| Concurrency | Ratio | Per round | Geometric mean |", "| ---: | --- | --- | ---: |"]
|
||||
*(fmt(r[c]) for c in COUNTERS[1:]), fmt(r["server_cpu_cores"], 2),
|
||||
"—" if r["host_cpu_busy"] is None else f"{r['host_cpu_busy']:.0%} of {r['host_cpus'] or '?'}",
|
||||
f"`{r['wal_device'] or '—'}`",
|
||||
f"{load_average(r['host_load_start'])} / {load_average(r['host_load_end'])} |"]))
|
||||
lines += ["", "| Cell | WAL MB/s | Objects/s | Mean object MiB | Sealed on size | OpenDAL S3 operations during the load "
|
||||
"| OpenDAL S3 operations of the restart | Idle s | Idle creates | Idle OpenDAL S3 operations |",
|
||||
"| --- | ---: | ---: | ---: | --- | --- | --- | ---: | ---: | --- |"]
|
||||
for r in loads:
|
||||
idle = r.get("idle") or {}
|
||||
lines.append(" | ".join([
|
||||
f"| {label(r)}", fmt(r["wal_mb_per_second"], 1), fmt(r["objects_per_second"], 1),
|
||||
fmt(r["mean_object_bytes"], 2, 1 / 2**20),
|
||||
"—" if r["sealed_on_size"] is None else ("yes" if r["sealed_on_size"] else "no"),
|
||||
operations(r["operations"]), operations(r["restart_operations"]) if "restart" in r else "—",
|
||||
fmt(idle.get("seconds")), fmt(idle.get("creates")), operations(idle["operations"]) if idle else "—"]) + " |")
|
||||
crashes = [r for r in rows if r["mode"] == "crash"]
|
||||
if crashes:
|
||||
lines += ["", "| Cell | Cycle | Kill after s | Acked | Present | Lost | Lost rows | Extra | In flight "
|
||||
"| All found | All acked | All lost | Indexed at kill | Restart to /health s | Recovery s "
|
||||
"| Recovered objects | Pin rows | Verdict |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: "
|
||||
"| ---: | ---: | ---: | --- |"]
|
||||
for r in crashes:
|
||||
for c in r["cycles"] or [{}]:
|
||||
verdict = "—" if not c else "FAIL: " + "; ".join(c.get("kill_errors", []) + c.get("verify_errors", [])) \
|
||||
if not c.get("passed") else "pass" if c.get("metrics_complete", True) else "pass, scrape missing"
|
||||
lines.append(" | ".join([
|
||||
f"| {label(r)}", fmt(c.get("cycle")), fmt(c.get("kill_after_seconds"), 1),
|
||||
fmt(c.get("cycle_acked_batches")), fmt(c.get("cycle_present_batches")),
|
||||
fmt(c.get("cycle_lost_batches")), fmt(c.get("cycle_lost_rows")), fmt(c.get("cycle_extra_batches")),
|
||||
fmt(c.get("in_flight")), fmt(c.get("total_found_batches")), fmt(c.get("total_acked_batches")),
|
||||
fmt(c.get("total_lost_batches")), fmt(c.get("indexed_before_kill")),
|
||||
fmt(c.get("restart_seconds"), 2), fmt(c.get("recovery_seconds"), 3),
|
||||
fmt(c.get("recovered_objects")), fmt(c.get("pin_rows")), verdict]) + " |")
|
||||
lines += ["", "| Cell | Cycle | Load s | WAL MB/s | Objects/s | Mean object MiB | Sealed on size "
|
||||
"| OpenDAL S3 operations until the kill | OpenDAL S3 operations of the restart |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | --- | --- | --- |"]
|
||||
for r in crashes:
|
||||
for c in r["cycles"]:
|
||||
lines.append(" | ".join([
|
||||
f"| {r['id']}", fmt(c["cycle"]), fmt(c.get("load_seconds"), 1), fmt(c["wal_mb_per_second"], 1),
|
||||
fmt(c["objects_per_second"], 1), fmt(c["mean_object_bytes"], 2, 1 / 2**20),
|
||||
"—" if c["sealed_on_size"] is None else ("yes" if c["sealed_on_size"] else "no"),
|
||||
operations(c["operations"]), operations(c["restart_operations"])]) + " |")
|
||||
pinned = [r for r in crashes if r.get("objects_samples")]
|
||||
if pinned:
|
||||
lines += ["", "| Cell | Indexed objects every 30 s | Missing samples | Deleted objects at the end "
|
||||
"| Created objects at the end |", "| --- | --- | ---: | ---: | ---: |"]
|
||||
for r in pinned:
|
||||
samples = r["objects_samples"]
|
||||
last = next((s for s in reversed(samples) if s[1] is not None), [None] * 4)
|
||||
lines.append(f"| {r['id']} | {', '.join('missing' if s[1] is None else str(s[1]) for s in samples)} "
|
||||
f"| {sum(s[1] is None for s in samples)} | {fmt(last[2])} | {fmt(last[3])} |")
|
||||
details = [r for r in loads if any(r[k] is not None for k in ("client_p50", "entries_per_object", "region_rows"))]
|
||||
if details:
|
||||
lines += ["", "| Cell | Client p50 / p99 ms | Entries per object | Appends per object | Regions "
|
||||
"| Rows per region min / max |", "| --- | ---: | ---: | ---: | ---: | ---: |"]
|
||||
for r in details:
|
||||
spread = r["region_rows"] or []
|
||||
lines.append(" | ".join([
|
||||
f"| {label(r)}", pair(r["client_p50"], r["client_p99"], 1, 1000), fmt(r["entries_per_object"], 1),
|
||||
fmt(r["appends_per_object"], 2), fmt(len(spread) if spread else r["partitions"]),
|
||||
pair(min(spread), max(spread)) if spread else "—"]) + " |")
|
||||
replays = [(r["id"], "—", r["restart"]) for r in loads if r.get("restart", {}).get("replay")]
|
||||
replays += [(r["id"], fmt(c["cycle"]), c | {"wall_seconds": c.get("restart_seconds")})
|
||||
for r in crashes for c in r["cycles"] if c.get("replay")]
|
||||
if replays:
|
||||
lines += ["", "| Cell | Cycle | Restart to /health s | Recovery s | Recovered objects | Regions opened "
|
||||
"| Rows replayed | Object span | Slowest region replay s | Read seconds (reads) "
|
||||
"| Regions with rows |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |"]
|
||||
for cell, cycle, x in replays:
|
||||
y = x["replay"]
|
||||
lines.append(" | ".join([
|
||||
f"| {cell}", cycle, fmt(x.get("wall_seconds"), 2), fmt(x.get("recovery_seconds"), 3),
|
||||
fmt(x.get("recovered_objects")), fmt(len(y["regions"])), fmt(y["rows"]), fmt(y["object_span"]),
|
||||
fmt(y["seconds"], 2), f"{fmt(y['read_seconds'], 2)} ({fmt(y['reads'])})",
|
||||
replayed(y["regions"])]) + " |")
|
||||
drops = [r for r in rows if r.get("drop")]
|
||||
if drops:
|
||||
lines += ["", "| Cell | Acked batches before the drop | Indexed before the drop | Restart to /health s "
|
||||
"| Recovered objects | Indexed at the end | Deleted after the restart | Seconds to collect | Verdict |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |"]
|
||||
for r in drops:
|
||||
x = r["drop"]
|
||||
# Objects the release already collected before the kill leave nothing to show.
|
||||
verdict = "—" if x["indexed_at_end"] is None else \
|
||||
"nothing left at the kill" if (x["recovered_objects"] or 0) <= 3 else \
|
||||
"collected" if r["exit_code"] == 0 else "still indexed"
|
||||
lines.append(" | ".join([
|
||||
f"| {label(r)}", fmt(x["acked_batches"]), fmt(x["indexed_before_drop"]), fmt(x["restart_seconds"], 2),
|
||||
fmt(x["recovered_objects"]), fmt(x["indexed_at_end"]),
|
||||
fmt(x["deleted_after_restart"]), fmt(x["seconds_to_collect"]), verdict]) + " |")
|
||||
restarts = [r for r in rows if "restart" in r]
|
||||
if restarts:
|
||||
lines += ["", "| Cell | Overrides | Objects created | Indexed at kill | Restart to /health s | Restart to count s "
|
||||
"| Recovery s | Recovered objects | Rows after restart |",
|
||||
"| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"]
|
||||
for r in restarts:
|
||||
x = r["restart"]
|
||||
short = x["min_objects"] and (x["indexed_before_kill"] or 0) < x["min_objects"]
|
||||
lines.append(" | ".join([
|
||||
f"| {label(r)}", overrides(r["env"]), fmt(r["created_objects_total"]),
|
||||
fmt(x["indexed_before_kill"]) + (f" (under {x['min_objects']:,})" if short else ""),
|
||||
fmt(x["wall_seconds"], 2),
|
||||
fmt(x["query_seconds"], 2), fmt(x["recovery_seconds"], 3), fmt(x["recovered_objects"]),
|
||||
f"{fmt(x['rows'])} |"]))
|
||||
lines += ["", "| Group | Image | Concurrency | Ratio | Per round | Geometric mean |",
|
||||
"| --- | --- | ---: | --- | --- | ---: |"]
|
||||
for p in pairs:
|
||||
lines.append(f"| {p['concurrency']} | {p['ratio']} | {', '.join(fmt(v, 3) for v in p['per_round'])} "
|
||||
f"| {fmt(p['geomean'], 3)} |")
|
||||
lines.append(f"| {p['group'] or '—'} | {p['image'] or '—'} | {p['concurrency']} | {p['ratio']} "
|
||||
f"| {', '.join(fmt(v, 3) for v in p['per_round'])} | {fmt(p['geomean'], 3)} |")
|
||||
if compared:
|
||||
lines += ["", f"| Group | Concurrency | Config | Image | Rows/s per round | Client p50 / p99 ms per round "
|
||||
f"| Ratio to {compared[0]['image']} per round | Geometric mean | Min | Max |",
|
||||
"| --- | ---: | --- | --- | --- | --- | --- | ---: | ---: | ---: |"]
|
||||
for c in compared:
|
||||
lines.append(" | ".join([
|
||||
f"| {c['group'] or '—'}", str(c["concurrency"]), c["config"], c["image"],
|
||||
"; ".join(fmt(r["rows_per_second"]) for r in c["rounds"]),
|
||||
"; ".join(pair(r["client_p50"], r["client_p99"], 1, 1000) for r in c["rounds"]),
|
||||
"—" if c["per_round"] is None else ", ".join(fmt(v, 3) for v in c["per_round"]),
|
||||
fmt(c["geomean"], 3), fmt(c["min"], 3), fmt(c["max"], 3)]) + " |")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
@@ -145,17 +501,20 @@ def main() -> None:
|
||||
run_path = args.root / "run.json"
|
||||
run = json.loads(run_path.read_text()) if run_path.exists() else {}
|
||||
plan_path = args.root / "plan.json"
|
||||
planned = json.loads(plan_path.read_text())["cells"] if plan_path.exists() else []
|
||||
plan = json.loads(plan_path.read_text()) if plan_path.exists() else {}
|
||||
planned = plan.get("cells", [])
|
||||
# The plan, not what happened to run, defines the expected cells and rounds.
|
||||
rows = [load_cell(args.root / "cells" / spec["id"], spec) for spec in planned]
|
||||
rows += [load_cell(path) for path in sorted((args.root / "cells").glob("*"))
|
||||
if path.name not in {spec["id"] for spec in planned} and (path / "cell.json").exists()]
|
||||
pairs = ratios(rows)
|
||||
images = plan.get("images", [""])
|
||||
compared = image_ratios(rows, images) if len(images) > 1 else []
|
||||
missing = [r["id"] for r in rows if r["missing"]]
|
||||
(args.root / "summary.json").write_text(json.dumps(
|
||||
{"run": run, "planned_cells": len(planned), "missing_cells": missing, "cells": rows, "ratios": pairs},
|
||||
indent=2) + "\n")
|
||||
print(render(run, rows, pairs) if rows else "No benchmark cell completed.\n")
|
||||
{"run": run, "planned_cells": len(planned), "missing_cells": missing, "cells": rows, "ratios": pairs,
|
||||
"image_ratios": compared}, indent=2) + "\n")
|
||||
print(render(run, rows, pairs, compared) if rows else "No benchmark cell completed.\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -24,26 +24,84 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
SMOKE_ROWS = 50_000
|
||||
# The o11ybench data profiles; a matching row count generates the canonical dataset.
|
||||
DATA_PROFILES = {5_000: "S", 2_000_000: "D", 10_000_000: "P", 100_000_000: "M"}
|
||||
# Identical [storage] for every configuration; only [wal] differs.
|
||||
# Identical [storage] for every configuration; only [wal] differs. R0 is the standalone default.
|
||||
WAL_CONFIGS = {
|
||||
"R": {"PROVIDER": "raft_engine", "SYNC_WRITE": "true", "DIR": "/wal"},
|
||||
"R0": {"PROVIDER": "raft_engine", "SYNC_WRITE": "false", "DIR": "/wal"},
|
||||
"D": {"PROVIDER": "experimental_object_store", "ACK_MODE": "durable"},
|
||||
"E": {"PROVIDER": "experimental_object_store", "ACK_MODE": "enqueued"},
|
||||
}
|
||||
ORDERS = (("R", "D", "E"), ("E", "D", "R"))
|
||||
BASELINES = ("R", "R0")
|
||||
# Restart cells seal ten times as often as the default so recovery has thousands of objects.
|
||||
RESTART_FLUSH_INTERVAL = "10ms"
|
||||
# The loader's CREATE TABLE for the agent observability workload at the pinned runtime image.
|
||||
AGENT_TABLE = """CREATE TABLE `agent_observations` (
|
||||
`event_time` TIMESTAMP(3) NOT NULL,
|
||||
`biz_date` DATE NOT NULL,
|
||||
`trace_id` STRING NOT NULL SKIPPING INDEX,
|
||||
`session_id` STRING NOT NULL,
|
||||
`observation_id` STRING NOT NULL SKIPPING INDEX,
|
||||
`parent_observation_id` STRING NULL SKIPPING INDEX,
|
||||
`seq_no` BIGINT NOT NULL,
|
||||
`type` STRING NOT NULL INVERTED INDEX,
|
||||
`status` STRING NOT NULL INVERTED INDEX,
|
||||
`tenant` STRING NOT NULL,
|
||||
`app` STRING NOT NULL,
|
||||
`environment` STRING NOT NULL,
|
||||
`task_category` STRING NULL,
|
||||
`trace_archetype` STRING NULL,
|
||||
`model` STRING NULL,
|
||||
`tool_name` STRING NULL,
|
||||
`input` STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false', granularity = '10240', false_positive_rate = '0.01'),
|
||||
`output` STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false', granularity = '10240', false_positive_rate = '0.01'),
|
||||
`input_tokens` BIGINT NULL,
|
||||
`output_tokens` BIGINT NULL,
|
||||
`total_cost` DOUBLE NULL,
|
||||
`latency_ms` BIGINT NULL,
|
||||
`payload` JSON2 NOT NULL,
|
||||
TIME INDEX (`event_time`),
|
||||
PRIMARY KEY (`tenant`, `app`, `environment`)
|
||||
)
|
||||
{partitions}ENGINE=mito
|
||||
WITH(
|
||||
append_mode = 'true',
|
||||
'compaction.type' = 'twcs',
|
||||
'compaction.twcs.time_window' = '2h'
|
||||
)"""
|
||||
GROUP_NAME = re.compile(r"[a-z][a-z0-9]{0,11}")
|
||||
IMAGE_LABEL = re.compile(r"[a-z][a-z0-9]{0,7}")
|
||||
RESERVED_ENV = ("GREPTIMEDB_STANDALONE__STORAGE__", "GREPTIMEDB_STANDALONE__WAL__")
|
||||
ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
||||
# Docker's reference grammar: path components separated by ".", "_", "__" or runs of "-".
|
||||
IMAGE = re.compile(r"[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?::[0-9]+)?(?:/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)+"
|
||||
r"(?::[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?(?:@sha256:[0-9a-f]{64})?")
|
||||
NIGHTLY = re.compile(r"v\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?-nightly-(\d{8})")
|
||||
TAG = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}")
|
||||
DIGEST = re.compile(r".+@sha256:[0-9a-f]{64}")
|
||||
# Rows/s of a load cell in 1000-row batches on c6id.2xlarge at the concurrency levels earlier runs
|
||||
# measured, interpolated on a log scale between them; durable writes slow down with several regions.
|
||||
# Smaller batches are taken to keep the request rate, so their rows/s shrink with the batch.
|
||||
MEASURED_BATCH_ROWS = 1000
|
||||
LOAD_RATES = {"durable": [(1, 9_000), (8, 34_000), (32, 55_000)],
|
||||
"durable, partitioned": [(1, 7_000), (8, 26_000), (32, 55_000)],
|
||||
"other": [(1, 28_000), (8, 60_000)]}
|
||||
# Starting, scraping and stopping a server, and the restart of a restart cell or a crash cycle.
|
||||
CELL_SECONDS, RESTART_SECONDS, CYCLE_SECONDS = 5, 60, 15
|
||||
# Checkout, dependencies, image pulls and the bucket probe; corpus generation per million rows; and the
|
||||
# deletion of the run's objects one at a time, which took 12-17 % of the cells' time in earlier runs.
|
||||
JOB_SECONDS, GENERATE_SECONDS_PER_MILLION, CLEANUP_SHARE = 180, 30, 0.2
|
||||
|
||||
|
||||
def append_github(variable: str, name: str, value: str) -> None:
|
||||
@@ -71,42 +129,228 @@ def check_regions(bucket: str, runner: str) -> None:
|
||||
"a cross-region bucket measures the WAN, not S3")
|
||||
|
||||
|
||||
def build_manifest(plan: dict, greptimedb_inspect: list, runtime_image: str,
|
||||
def build_manifest(plan: dict, images: list, runtime_image: str,
|
||||
instance_type: str, region: str, prefix: str) -> dict:
|
||||
image = next((d for d in greptimedb_inspect[0].get("RepoDigests") or [] if DIGEST.fullmatch(d)), None)
|
||||
if image is None:
|
||||
raise ValueError("the GreptimeDB image has no sha256 repository digest")
|
||||
"""The run's record; `images` are the checked images, each pinned to its digest, the first the reference."""
|
||||
if [image["label"] for image in images] != plan["images"]:
|
||||
raise ValueError(f"the checked images {[i['label'] for i in images]} are not the plan's {plan['images']}")
|
||||
if not all(DIGEST.fullmatch(image["image"]) for image in images):
|
||||
raise ValueError("every GreptimeDB image must be pinned to a sha256 digest")
|
||||
return {"smoke": plan["smoke"], "rows": plan["rows"], "rounds": plan["rounds"],
|
||||
"concurrency": plan["concurrency"], "batch_rows": plan["batch_rows"],
|
||||
"greptimedb_image": image, "runtime_image": runtime_image,
|
||||
"configs": plan["configs"], "extra_env": plan["extra_env"],
|
||||
"restart": plan["restart"], "groups": plan["groups"], "warmup": plan["warmup"],
|
||||
"greptimedb_image": images[0]["image"], "greptimedb_images": images, "runtime_image": runtime_image,
|
||||
"instance_type": instance_type, "region": region, "prefix": prefix}
|
||||
|
||||
|
||||
def build_plan(concurrency: str, rows: str, batch_rows: str, rounds: str, smoke: bool) -> dict:
|
||||
def parse_extra_env(text: str) -> dict:
|
||||
"""A JSON object or K=V lines; the storage and WAL settings that define a cell stay the benchmark's own."""
|
||||
if text.strip().startswith("{"):
|
||||
parsed = json.loads(text)
|
||||
if not all(isinstance(v, (str, int)) and not isinstance(v, bool) for v in parsed.values()):
|
||||
raise ValueError("extra_env values must be strings or integers")
|
||||
env = {k: str(v) for k, v in parsed.items()}
|
||||
else:
|
||||
env = {}
|
||||
for line in filter(None, (line.strip() for line in text.splitlines())):
|
||||
name, sep, value = line.partition("=")
|
||||
if not sep or name.strip() in env:
|
||||
raise ValueError(f"extra_env line is not a new K=V: {line!r}")
|
||||
env[name.strip()] = value.strip()
|
||||
for name, value in env.items():
|
||||
# GreptimeDB matches its environment keys without regard to case.
|
||||
if not ENV_NAME.fullmatch(name) or name.upper().startswith(RESERVED_ENV) or "\n" in value:
|
||||
raise ValueError(f"extra_env may not set {name!r}")
|
||||
return env
|
||||
|
||||
|
||||
def partitioned_table(summary: dict, partitions: int) -> str:
|
||||
"""The loader's table split into equal event_time ranges over the corpus window. The
|
||||
generator draws event times uniformly from that window, so every range gets an equal
|
||||
share of the rows and every batch writes to every region."""
|
||||
def millis(text):
|
||||
return int(datetime.fromisoformat(text.replace("Z", "+00:00")).timestamp() * 1000)
|
||||
start, end = millis(summary["start"]), millis(summary["end"])
|
||||
bounds = [start + (end - start) * i // partitions for i in range(1, partitions)]
|
||||
if len(set(bounds)) != len(bounds) or not start < bounds[0]:
|
||||
raise ValueError(f"the corpus window {summary['start']}..{summary['end']} is too short for {partitions} ranges")
|
||||
ranges = [f"`event_time` < {bounds[0]}"]
|
||||
ranges += [f"`event_time` >= {low} AND `event_time` < {high}" for low, high in zip(bounds, bounds[1:])]
|
||||
ranges.append(f"`event_time` >= {bounds[-1]}")
|
||||
clause = "PARTITION ON COLUMNS (`event_time`) (\n " + ",\n ".join(ranges) + "\n)\n"
|
||||
return AGENT_TABLE.format(partitions=clause)
|
||||
|
||||
|
||||
def parse_images(text: str) -> list:
|
||||
"""The images to compare: a JSON list of 2..4 {label, image}, the first the reference. Empty
|
||||
means the one image of greptimedb_tag or greptimedb_image, with an empty label."""
|
||||
if not text.strip():
|
||||
return [{"label": "", "image": ""}]
|
||||
images = json.loads(text)
|
||||
if not isinstance(images, list) or not 2 <= len(images) <= 4 \
|
||||
or not all(isinstance(i, dict) and set(i) == {"label", "image"} for i in images):
|
||||
raise ValueError("images must be a JSON list of 2..4 objects, each with only a label and an image")
|
||||
labels = [image["label"] for image in images]
|
||||
for image in images:
|
||||
if not isinstance(image["label"], str) or not IMAGE_LABEL.fullmatch(image["label"]):
|
||||
raise ValueError(f"an image label is up to 8 lowercase letters and digits, got {image['label']!r}")
|
||||
full_reference("an image", image["image"])
|
||||
if len(set(labels)) != len(labels):
|
||||
raise ValueError(f"image labels repeat: {labels}")
|
||||
return images
|
||||
|
||||
|
||||
def cell_id(group: str, round_number: int, level: int, config: str, label: str) -> str:
|
||||
return (f"{group}-" if group else "") + f"{round_number}-{level}-{config}" + (f"-{label}" if label else "")
|
||||
|
||||
|
||||
def group_cells(group: dict, smoke: bool, labels: list) -> list:
|
||||
"""The cells of one group, in run order; a group without a name is the top-level inputs.
|
||||
Each cell runs once per image label, back to back."""
|
||||
name, mode = group.get("name", ""), group.get("mode", "load")
|
||||
if mode not in ("load", "crash", "drop"):
|
||||
raise ValueError(f"group {name!r}: mode must be load, crash or drop, got {mode!r}")
|
||||
concurrency = str(group["concurrency"])
|
||||
levels = [positive("concurrency", level.strip(), 256) for level in concurrency.split(",")]
|
||||
if len(set(levels)) != len(levels):
|
||||
raise ValueError(f"concurrency repeats a level: {concurrency!r}")
|
||||
row_count = SMOKE_ROWS if smoke else positive("rows", rows, 100_000_000)
|
||||
round_count = 1 if smoke else positive("rounds", rounds, 10)
|
||||
profile = DATA_PROFILES.get(row_count)
|
||||
chosen = [config.strip() for config in str(group["configs"]).split(",")]
|
||||
allowed = set(WAL_CONFIGS) - (set(BASELINES) if mode != "load" else set())
|
||||
if not chosen or len(set(chosen)) != len(chosen) or not set(chosen) <= allowed:
|
||||
raise ValueError(f"group {name!r}: configs must be distinct names among {', '.join(sorted(allowed))}, "
|
||||
f"got {group['configs']!r}")
|
||||
extra = group.get("extra_env", "")
|
||||
env = parse_extra_env(json.dumps(extra) if isinstance(extra, dict) else str(extra))
|
||||
restart = group.get("restart", False) is True
|
||||
rows = SMOKE_ROWS if smoke else positive("rows", str(group["rows"]), 100_000_000)
|
||||
rounds = 1 if smoke else positive("rounds", str(group["rounds"]), 10)
|
||||
idle = int(group.get("idle_seconds", 0))
|
||||
common = {"group": name, "mode": mode, "restart": restart, "idle_seconds": idle}
|
||||
partitions = int(group.get("partitions", 1))
|
||||
if partitions != 1 and (mode != "load" or not 2 <= partitions <= 64):
|
||||
raise ValueError(f"group {name!r}: partitions must be in 2..64 and needs mode load")
|
||||
if mode == "load":
|
||||
if not 0 <= idle <= 600:
|
||||
raise ValueError(f"group {name!r}: idle_seconds must be in 0..600")
|
||||
# The objects a restart cell must leave indexed at its kill; a smoke run is too small for any.
|
||||
min_objects = int(group.get("min_objects", 0))
|
||||
if min_objects < 0 or (min_objects and not restart):
|
||||
raise ValueError(f"group {name!r}: min_objects needs restart and must not be negative")
|
||||
common |= {"rows": rows, "data_profile": DATA_PROFILES.get(rows, "custom"),
|
||||
"min_objects": 0 if smoke else min_objects, "partitions": partitions}
|
||||
elif mode == "drop":
|
||||
# Writes one table for write_seconds, drops it, kills and restarts the server, then
|
||||
# writes and flushes another table and watches the dropped table's objects go.
|
||||
write_seconds = int(group.get("write_seconds", 30))
|
||||
if not 1 <= write_seconds <= 600 or restart or idle:
|
||||
raise ValueError(f"group {name!r}: write_seconds must be in 1..600, without restart or idle")
|
||||
common |= {"write_seconds": min(write_seconds, 10) if smoke else write_seconds}
|
||||
else:
|
||||
cycles = int(group.get("cycles", 5))
|
||||
kill_after = [int(v) for v in group.get("kill_after", [3, 10])]
|
||||
if not 1 <= cycles <= 20 or len(kill_after) != 2 or not 1 <= kill_after[0] <= kill_after[1] <= 3600:
|
||||
raise ValueError(f"group {name!r}: cycles must be in 1..20 and kill_after a [min, max] in 1..3600")
|
||||
if restart or idle:
|
||||
raise ValueError(f"group {name!r}: a crash group restarts after every kill and has no idle window")
|
||||
if smoke:
|
||||
cycles, kill_after = min(cycles, 2), [min(v, 60) for v in kill_after]
|
||||
common |= {"cycles": cycles, "kill_after": kill_after, "pin": group.get("pin", False) is True}
|
||||
series = [config for config in WAL_CONFIGS if config in chosen]
|
||||
cells = []
|
||||
for round_number in range(1, round_count + 1):
|
||||
# Alternate the order between rounds so no configuration always runs first.
|
||||
for round_number in range(1, rounds + 1):
|
||||
# Reverse the order every other round so no configuration or image always runs first.
|
||||
forward = round_number % 2
|
||||
for level in levels:
|
||||
for config in ORDERS[(round_number - 1) % 2]:
|
||||
cells.append({
|
||||
"id": f"{round_number}-{level}-{config}", "round": round_number,
|
||||
"concurrency": level, "config": config, "wal": WAL_CONFIGS[config],
|
||||
})
|
||||
for config in series if forward else reversed(series):
|
||||
wal = WAL_CONFIGS[config] | ({"FLUSH_INTERVAL": RESTART_FLUSH_INTERVAL} if restart and config not in BASELINES else {})
|
||||
for label in labels if forward else reversed(labels):
|
||||
cells.append(common | {
|
||||
"id": cell_id(name, round_number, level, config, label), "round": round_number,
|
||||
"concurrency": level, "config": config, "image": label, "wal": wal, "env": env,
|
||||
})
|
||||
return cells
|
||||
|
||||
|
||||
def parse_groups(text: str, defaults: dict) -> list:
|
||||
"""A JSON list of groups; every key a group leaves out takes the top-level input's value."""
|
||||
if not text.strip():
|
||||
return [defaults]
|
||||
groups = json.loads(text)
|
||||
if not isinstance(groups, list) or not groups or not all(isinstance(g, dict) for g in groups):
|
||||
raise ValueError("groups must be a non-empty JSON list of objects")
|
||||
known = set(defaults) | {"name", "mode", "idle_seconds", "cycles", "kill_after", "pin", "min_objects",
|
||||
"partitions", "write_seconds"}
|
||||
names = [g.get("name") for g in groups]
|
||||
for group, name in zip(groups, names):
|
||||
if not isinstance(name, str) or not GROUP_NAME.fullmatch(name):
|
||||
raise ValueError(f"every group needs a name of up to 12 lowercase letters and digits, got {name!r}")
|
||||
if set(group) - known:
|
||||
raise ValueError(f"group {name!r} has unknown keys {sorted(set(group) - known)}")
|
||||
if len(set(names)) != len(names):
|
||||
raise ValueError(f"group names repeat: {names}")
|
||||
return [defaults | group for group in groups]
|
||||
|
||||
|
||||
def build_plan(concurrency: str, rows: str, batch_rows: str, rounds: str, smoke: bool,
|
||||
configs: str = "R,D,E", extra_env: str = "", restart: bool = False, groups: str = "",
|
||||
images: str = "", warmup: bool = False) -> dict:
|
||||
defaults = {"concurrency": concurrency, "rows": rows, "rounds": rounds, "configs": configs,
|
||||
"extra_env": extra_env, "restart": restart}
|
||||
resolved = parse_groups(groups, defaults)
|
||||
labels = [image["label"] for image in parse_images(images)]
|
||||
cells = [cell for group in resolved for cell in group_cells(group, smoke, labels)]
|
||||
if warmup:
|
||||
# Round 0 runs the first load cell once per image before anything else; no statistic uses it.
|
||||
first = next((cell for cell in cells if cell["mode"] == "load"), None)
|
||||
if first is None:
|
||||
raise ValueError("warmup needs a load group")
|
||||
cells = [first | {"id": cell_id(first["group"], 0, first["concurrency"], first["config"], label),
|
||||
"round": 0, "image": label, "warmup": True} for label in labels] + cells
|
||||
if len({cell["id"] for cell in cells}) != len(cells):
|
||||
raise ValueError("two cells share an id")
|
||||
corpora = []
|
||||
for count in dict.fromkeys(cell["rows"] for cell in cells if cell["mode"] == "load"):
|
||||
profile = DATA_PROFILES.get(count)
|
||||
corpora.append({"rows": count, "dir": f"corpus-{count}", "data_profile": profile or "custom",
|
||||
"generator_args": ["--data-profile", profile] if profile else ["--line-count", str(count)]})
|
||||
first = cells[0]
|
||||
return {
|
||||
"smoke": smoke, "rows": row_count, "rounds": round_count,
|
||||
"batch_rows": positive("batch_rows", batch_rows, 100_000), "concurrency": levels,
|
||||
"data_profile": profile or "custom",
|
||||
"generator_args": ["--data-profile", profile] if profile else ["--line-count", str(row_count)],
|
||||
"cells": cells,
|
||||
"smoke": smoke, "rows": first.get("rows"), "rounds": max(cell["round"] for cell in cells),
|
||||
"batch_rows": positive("batch_rows", batch_rows, 100_000),
|
||||
"concurrency": sorted({cell["concurrency"] for cell in cells}),
|
||||
"configs": list(dict.fromkeys(cell["config"] for cell in cells)),
|
||||
"extra_env": parse_extra_env(extra_env), "groups": [g.get("name", "") for g in resolved],
|
||||
"restart": restart, "images": labels, "warmup": warmup, "corpora": corpora, "cells": cells,
|
||||
}
|
||||
|
||||
|
||||
def load_rate(cell: dict) -> float:
|
||||
durable = cell["wal"].get("ACK_MODE") == "durable"
|
||||
points = LOAD_RATES["other" if not durable else "durable, partitioned" if cell["partitions"] > 1 else "durable"]
|
||||
level = cell["concurrency"]
|
||||
for (low, low_rate), (high, high_rate) in zip(points, points[1:]):
|
||||
if level < high:
|
||||
return low_rate + (high_rate - low_rate) * math.log(max(level, low) / low) / math.log(high / low)
|
||||
return points[-1][1]
|
||||
|
||||
|
||||
def cell_seconds(cell: dict, batch_rows: int) -> float:
|
||||
if cell["mode"] == "crash":
|
||||
return CELL_SECONDS + cell["cycles"] * (sum(cell["kill_after"]) / 2 + CYCLE_SECONDS)
|
||||
if cell["mode"] == "drop":
|
||||
return CELL_SECONDS + cell["write_seconds"] + RESTART_SECONDS / 2
|
||||
rate = load_rate(cell) * min(1.0, batch_rows / MEASURED_BATCH_ROWS)
|
||||
return CELL_SECONDS + cell["rows"] / rate + cell["idle_seconds"] + (RESTART_SECONDS if cell["restart"] else 0)
|
||||
|
||||
|
||||
def estimate_minutes(plan: dict) -> int:
|
||||
"""The benchmark job's minutes on c6id.2xlarge, from the times of earlier runs."""
|
||||
generate = GENERATE_SECONDS_PER_MILLION * sum(corpus["rows"] for corpus in plan["corpora"]) / 1e6
|
||||
cells = sum(cell_seconds(cell, plan["batch_rows"]) for cell in plan["cells"])
|
||||
return math.ceil((JOB_SECONDS + generate + cells * (1 + CLEANUP_SHARE)) / 60)
|
||||
|
||||
|
||||
def newest_nightly(tags: list[str]) -> str:
|
||||
dated = [(match.group(1), tag) for tag in tags if (match := NIGHTLY.fullmatch(tag))]
|
||||
if not dated:
|
||||
@@ -114,6 +358,30 @@ def newest_nightly(tags: list[str]) -> str:
|
||||
return max(dated)[1]
|
||||
|
||||
|
||||
def full_reference(name: str, image: str) -> str:
|
||||
if not isinstance(image, str) or not IMAGE.fullmatch(image) \
|
||||
or not re.search(r"(:[^/]+|@sha256:[0-9a-f]{64})$", image):
|
||||
raise ValueError(f"{name} must be a full reference with a tag or digest: {image!r}")
|
||||
return image
|
||||
|
||||
|
||||
def resolve_image(image: str, tag: str) -> str:
|
||||
"""The full image reference when given, else greptime/greptimedb at the resolved tag."""
|
||||
if image:
|
||||
return full_reference("greptimedb_image", image)
|
||||
return f"greptime/greptimedb:{resolve_tag(tag)}"
|
||||
|
||||
|
||||
def resolve_images(images: str, image: str, tag: str) -> list:
|
||||
"""The images to pull and check: the list to compare, or the one image of the other two inputs."""
|
||||
entries = parse_images(images)
|
||||
if entries[0]["label"]:
|
||||
if image:
|
||||
raise ValueError("set either images or greptimedb_image, not both")
|
||||
return entries
|
||||
return [{"label": "", "image": resolve_image(image, tag)}]
|
||||
|
||||
|
||||
def resolve_tag(requested: str) -> str:
|
||||
if requested != "nightly":
|
||||
if not TAG.fullmatch(requested):
|
||||
@@ -169,14 +437,25 @@ def main() -> int:
|
||||
for name in ("concurrency", "rows", "batch-rows", "rounds"):
|
||||
plan.add_argument(f"--{name}", required=True)
|
||||
plan.add_argument("--smoke", choices=("true", "false"), required=True)
|
||||
plan.add_argument("--restart", choices=("true", "false"), default="false")
|
||||
plan.add_argument("--configs", default="R,D,E")
|
||||
plan.add_argument("--warmup", choices=("true", "false"), default="false")
|
||||
for name in ("extra-env", "groups", "images"):
|
||||
plan.add_argument(f"--{name}", default="")
|
||||
for name in ("timeout-minutes", "ttl-hours", "output"):
|
||||
plan.add_argument(f"--{name}")
|
||||
commands.add_parser("tag").add_argument("requested")
|
||||
image = commands.add_parser("image")
|
||||
image.add_argument("--tag", required=True)
|
||||
image.add_argument("--image", default="")
|
||||
image.add_argument("--images", default="")
|
||||
regions = commands.add_parser("regions")
|
||||
for name in ("bucket", "runner"):
|
||||
regions.add_argument(f"--{name}", required=True)
|
||||
table = commands.add_parser("table")
|
||||
table.add_argument("--summary", type=Path, required=True)
|
||||
table.add_argument("--partitions", type=int, required=True)
|
||||
manifest = commands.add_parser("manifest")
|
||||
for name in ("results", "runtime-image", "instance-type", "region", "prefix"):
|
||||
for name in ("results", "images", "runtime-image", "instance-type", "region", "prefix"):
|
||||
manifest.add_argument(f"--{name}", required=True)
|
||||
for name in ("probe", "cleanup"):
|
||||
bucket = commands.add_parser(name)
|
||||
@@ -187,26 +466,33 @@ def main() -> int:
|
||||
if args.command == "plan":
|
||||
if args.timeout_minutes or args.ttl_hours:
|
||||
validate_budgets(args.timeout_minutes or "", args.ttl_hours or "")
|
||||
result = build_plan(args.concurrency, args.rows, args.batch_rows, args.rounds, args.smoke == "true")
|
||||
result = build_plan(args.concurrency, args.rows, args.batch_rows, args.rounds, args.smoke == "true",
|
||||
args.configs, args.extra_env, args.restart == "true", args.groups,
|
||||
args.images, args.warmup == "true")
|
||||
minutes = estimate_minutes(result)
|
||||
if args.timeout_minutes and minutes > int(args.timeout_minutes):
|
||||
raise ValueError(f"the plan needs about {minutes} minutes on c6id.2xlarge, "
|
||||
f"more than the {args.timeout_minutes}-minute benchmark timeout")
|
||||
if args.output:
|
||||
with open(args.output, "w") as output:
|
||||
json.dump(result, output, indent=2)
|
||||
print(f"{len(result['cells'])} cells, {result['rows']} rows, profile {result['data_profile']}")
|
||||
elif args.command == "tag":
|
||||
tag = resolve_tag(args.requested)
|
||||
append_github("GITHUB_OUTPUT", "greptimedb_tag", tag)
|
||||
print(f"GreptimeDB image tag: {tag}")
|
||||
print(f"{len(result['cells'])} cells; corpora {[c['rows'] for c in result['corpora']]}; "
|
||||
f"about {minutes} minutes on c6id.2xlarge")
|
||||
elif args.command == "image":
|
||||
images = resolve_images(args.images, args.image, args.tag)
|
||||
append_github("GITHUB_OUTPUT", "requested_images", json.dumps(images))
|
||||
print("GreptimeDB images: " + ", ".join(f"{i['label']} {i['image']}".strip() for i in images))
|
||||
elif args.command == "regions":
|
||||
check_regions(args.bucket, args.runner)
|
||||
print(f"bucket and runner both in {args.runner}")
|
||||
elif args.command == "table":
|
||||
print(partitioned_table(json.loads(args.summary.read_text()), args.partitions))
|
||||
elif args.command == "manifest":
|
||||
results = Path(args.results)
|
||||
manifest = build_manifest(json.loads((results / "plan.json").read_text()),
|
||||
json.loads((results / "images" / "greptimedb.json").read_text()),
|
||||
manifest = build_manifest(json.loads((results / "plan.json").read_text()), json.loads(args.images),
|
||||
args.runtime_image, args.instance_type, args.region, args.prefix)
|
||||
(results / "run.json").write_text(json.dumps(manifest, indent=2) + "\n")
|
||||
append_github("GITHUB_ENV", "GREPTIMEDB_IMAGE", manifest["greptimedb_image"])
|
||||
print(f"GreptimeDB image: {manifest['greptimedb_image']}")
|
||||
print("GreptimeDB images: " + ", ".join(i["image"] for i in manifest["greptimedb_images"]))
|
||||
else:
|
||||
if not re.fullmatch(r"oswal-bench/[0-9]+-[0-9]+", args.prefix):
|
||||
raise ValueError(f"refusing a prefix outside oswal-bench/<run>: {args.prefix!r}")
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import importlib.util
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_module():
|
||||
path = Path(__file__).resolve().parent / "previous-release-tag.py"
|
||||
spec = importlib.util.spec_from_file_location("previous_release_tag", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
TAGS = [
|
||||
"v1.1.0",
|
||||
"v1.2.0",
|
||||
"v1.2.1",
|
||||
"v1.2.1^{}",
|
||||
"v1.3.0-alpha.1",
|
||||
"v1.3.0-beta.1",
|
||||
"v1.3.0-beta.2",
|
||||
"v1.2.0-nightly-20260706",
|
||||
]
|
||||
|
||||
|
||||
class PreviousReleaseTagTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.mod = load_module()
|
||||
|
||||
def test_beta_uses_latest_older_stable_not_an_ancestor(self) -> None:
|
||||
self.assertEqual(self.mod.previous_stable_tag("v1.3.0-beta.2", TAGS), "v1.2.1")
|
||||
|
||||
def test_stable_release_skips_itself(self) -> None:
|
||||
self.assertEqual(self.mod.previous_stable_tag("v1.2.1", TAGS), "v1.2.0")
|
||||
|
||||
def test_same_line_stable_is_not_baseline_for_its_prerelease(self) -> None:
|
||||
tags = TAGS + ["v1.3.0"]
|
||||
self.assertEqual(self.mod.previous_stable_tag("v1.3.0-beta.2", tags), "v1.2.1")
|
||||
self.assertEqual(self.mod.previous_stable_tag("v1.3.0", tags), "v1.2.1")
|
||||
|
||||
def test_nightly_of_unreleased_version_uses_previous_stable(self) -> None:
|
||||
self.assertEqual(
|
||||
self.mod.previous_stable_tag("v1.3.0-nightly-20261013", TAGS),
|
||||
"v1.2.1",
|
||||
)
|
||||
|
||||
def test_no_older_stable_tag(self) -> None:
|
||||
self.assertIsNone(self.mod.previous_stable_tag("v0.1.0", ["v0.1.0", "v1.2.1"]))
|
||||
|
||||
def test_invalid_version(self) -> None:
|
||||
with self.assertRaises(ValueError):
|
||||
self.mod.previous_stable_tag("nightly-20261013", TAGS)
|
||||
|
||||
def test_unordered_tags_use_numeric_order(self) -> None:
|
||||
forward = ["v1.2.1", "v1.9.0", "v1.10.0", "v0.10.0", "v1.9.9"]
|
||||
backward = list(reversed(forward))
|
||||
for tags in (forward, backward):
|
||||
self.assertEqual(self.mod.previous_stable_tag("v1.10.0", tags), "v1.9.9")
|
||||
self.assertEqual(self.mod.previous_stable_tag("v1.10.0-rc.1", tags), "v1.9.9")
|
||||
|
||||
def test_multi_digit_components_compare_as_integers(self) -> None:
|
||||
tags = ["v1.9.0", "v1.10.0", "v0.9.10", "v0.10.0", "v2.9.15", "v2.10.0", "v10.2.0"]
|
||||
self.assertEqual(self.mod.previous_stable_tag("v1.10.0", tags), "v1.9.0")
|
||||
self.assertEqual(self.mod.previous_stable_tag("v0.10.0", tags), "v0.9.10")
|
||||
self.assertEqual(self.mod.previous_stable_tag("v2.10.0", tags), "v2.9.15")
|
||||
self.assertEqual(self.mod.previous_stable_tag("v10.2.0", ["v10.0.9", "v9.20.0"]), "v10.0.9")
|
||||
|
||||
|
||||
class PreviousReleaseTagCliTest(unittest.TestCase):
|
||||
SCRIPT = Path(__file__).resolve().parent / "previous-release-tag.py"
|
||||
|
||||
def run_cli(
|
||||
self, version: str, tags: list[str] | None = None, stdin: str | None = None
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
command = [sys.executable, str(self.SCRIPT), "--version", version]
|
||||
if tags is not None:
|
||||
command.extend(tags)
|
||||
return subprocess.run(
|
||||
command,
|
||||
input=stdin,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def test_stdin_selects_baseline_and_exits_zero(self) -> None:
|
||||
completed = self.run_cli(
|
||||
"v1.10.0",
|
||||
stdin="v1.2.1\nv1.9.0\n\nv1.10.0\nv1.9.0^{}\n",
|
||||
)
|
||||
self.assertEqual(completed.returncode, 0)
|
||||
self.assertEqual(completed.stdout, "v1.9.0\n")
|
||||
self.assertEqual(completed.stderr, "")
|
||||
|
||||
def test_positional_tags_ignore_stdin(self) -> None:
|
||||
completed = self.run_cli("v1.10.0", tags=["v1.9.0"], stdin="v1.9.1\n")
|
||||
self.assertEqual(completed.returncode, 0)
|
||||
self.assertEqual(completed.stdout, "v1.9.0\n")
|
||||
|
||||
def test_no_older_tag_exits_zero_with_empty_stdout(self) -> None:
|
||||
completed = self.run_cli("v0.1.0", stdin="v0.1.0\nv1.2.1\n")
|
||||
self.assertEqual(completed.returncode, 0)
|
||||
self.assertEqual(completed.stdout, "")
|
||||
self.assertEqual(completed.stderr, "")
|
||||
|
||||
def test_invalid_version_exits_one(self) -> None:
|
||||
completed = self.run_cli("nightly", stdin="v1.2.1\n")
|
||||
self.assertEqual(completed.returncode, 1)
|
||||
self.assertEqual(completed.stdout, "")
|
||||
self.assertIn("not a release version: nightly", completed.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Pick the query-regression baseline tag for a release.
|
||||
|
||||
Compatibility tests do not use this tag. They read ``tests/compatibility/ci.toml``.
|
||||
|
||||
Release branches diverge, so a tag on ``release/v1.3`` is not a descendant of
|
||||
``v1.2.1``. Ancestry (``git tag --merged``) therefore skips the real previous
|
||||
release. Compare version numbers instead: the baseline is the greatest
|
||||
``vX.Y.Z`` tag whose version is strictly older than the release being built.
|
||||
Prerelease and nightly tags are not baselines.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
|
||||
STABLE_TAG = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
|
||||
RELEASE_VERSION = re.compile(r"^v(\d+)\.(\d+)\.(\d+)(?:-.+)?$")
|
||||
|
||||
|
||||
def _triple(match: re.Match[str]) -> tuple[int, int, int]:
|
||||
return int(match.group(1)), int(match.group(2)), int(match.group(3))
|
||||
|
||||
|
||||
def release_version(version: str) -> tuple[int, int, int]:
|
||||
match = RELEASE_VERSION.fullmatch(version.strip())
|
||||
if match is None:
|
||||
raise ValueError(f"not a release version: {version}")
|
||||
return _triple(match)
|
||||
|
||||
|
||||
def previous_stable_tag(version: str, tags: list[str]) -> str | None:
|
||||
"""Return the newest stable tag older than ``version``, or None."""
|
||||
current = release_version(version)
|
||||
best: tuple[int, int, int] | None = None
|
||||
best_tag: str | None = None
|
||||
for raw in tags:
|
||||
tag = raw.strip()
|
||||
if tag.endswith("^{}"):
|
||||
tag = tag[: -len("^{}")]
|
||||
match = STABLE_TAG.fullmatch(tag)
|
||||
if match is None:
|
||||
continue
|
||||
triple = _triple(match)
|
||||
if triple < current and (best is None or triple > best):
|
||||
best = triple
|
||||
best_tag = tag
|
||||
return best_tag
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--version", required=True, help="Release version, e.g. v1.3.0-beta.2")
|
||||
parser.add_argument(
|
||||
"tags",
|
||||
nargs="*",
|
||||
help="Candidate tags. Read stdin when omitted.",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
tags = list(args.tags)
|
||||
if not tags and not sys.stdin.isatty():
|
||||
tags = sys.stdin.read().splitlines()
|
||||
try:
|
||||
tag = previous_stable_tag(args.version, tags)
|
||||
except ValueError as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
return 1
|
||||
if tag is None:
|
||||
return 0
|
||||
print(tag)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import itertools
|
||||
import os
|
||||
import subprocess
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).with_name("release-linux-architectures.sh")
|
||||
ARCHITECTURES = ("amd64", "arm64", "riscv64")
|
||||
|
||||
|
||||
class ReleaseLinuxArchitecturesTest(unittest.TestCase):
|
||||
def resolve(self, selected, event="workflow_dispatch", publish=False, images=False):
|
||||
env = dict(
|
||||
os.environ,
|
||||
GITHUB_EVENT_NAME=event,
|
||||
PUBLISH_GITHUB_RELEASE=str(publish).lower(),
|
||||
RELEASE_IMAGES=str(images).lower(),
|
||||
)
|
||||
for arch in ARCHITECTURES:
|
||||
key = f"BUILD_LINUX_{arch.upper()}_ARTIFACTS"
|
||||
env.pop(key, None)
|
||||
if selected is not None:
|
||||
env[key] = str(arch in selected).lower()
|
||||
return subprocess.run(
|
||||
["bash", str(SCRIPT)], env=env, text=True, capture_output=True, check=False
|
||||
)
|
||||
|
||||
def assert_targets(self, result, targets):
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stderr, "")
|
||||
self.assertEqual(
|
||||
result.stdout,
|
||||
"".join(f"{arch}={str(arch in targets).lower()}\n" for arch in ARCHITECTURES),
|
||||
)
|
||||
|
||||
def test_checkbox_combinations(self):
|
||||
for flags in itertools.product((False, True), repeat=3):
|
||||
selected = {arch for arch, enabled in zip(ARCHITECTURES, flags) if enabled}
|
||||
for event, publish, images in itertools.product(
|
||||
("workflow_dispatch", "push", "schedule"), (False, True), (False, True)
|
||||
):
|
||||
with self.subTest(selected=selected, event=event, publish=publish, images=images):
|
||||
expected = (
|
||||
set(ARCHITECTURES)
|
||||
if event != "workflow_dispatch" or publish or images
|
||||
else selected
|
||||
)
|
||||
self.assert_targets(self.resolve(selected, event, publish, images), expected)
|
||||
|
||||
def test_default_and_omitted_inputs_preserve_previous_behavior(self):
|
||||
for selected in (None, set()):
|
||||
for event, publish, images in itertools.product(
|
||||
("workflow_dispatch", "push", "schedule"), (False, True), (False, True)
|
||||
):
|
||||
with self.subTest(selected=selected, event=event, publish=publish, images=images):
|
||||
# The old build_linux_artifacts switch defaulted to false.
|
||||
old_build_linux = publish or images or event in ("push", "schedule")
|
||||
self.assert_targets(
|
||||
self.resolve(selected, event, publish, images),
|
||||
set(ARCHITECTURES) if old_build_linux else set(),
|
||||
)
|
||||
|
||||
def test_unsupported_event(self):
|
||||
result = self.resolve(set(), event="pull_request")
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("Unsupported release event", result.stderr)
|
||||
self.assertEqual(result.stdout, "")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Publishing keeps the existing full-build behavior, regardless of checkboxes.
|
||||
build_all=false
|
||||
if [[ "${PUBLISH_GITHUB_RELEASE:-false}" == true || "${RELEASE_IMAGES:-false}" == true ]]; then
|
||||
build_all=true
|
||||
fi
|
||||
case "${GITHUB_EVENT_NAME}" in
|
||||
push|schedule) build_all=true ;;
|
||||
workflow_dispatch) ;;
|
||||
*) echo "Unsupported release event: ${GITHUB_EVENT_NAME}" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
for arch in amd64 arm64 riscv64; do
|
||||
input="BUILD_LINUX_${arch^^}_ARTIFACTS"
|
||||
selected=false
|
||||
if [[ "$build_all" == true || "${!input:-false}" == true ]]; then
|
||||
selected=true
|
||||
fi
|
||||
printf '%s=%s\n' "$arch" "$selected"
|
||||
done
|
||||
@@ -0,0 +1,101 @@
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"fixture", Path(__file__).resolve().parents[1] / "metric-snapshot-fixture.py"
|
||||
)
|
||||
fixture = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(fixture)
|
||||
|
||||
|
||||
class ValidatorTests(unittest.TestCase):
|
||||
def records(self):
|
||||
return {
|
||||
"schema": {"column_schemas": [
|
||||
{"name": "ts", "data_type": "TimestampMillisecond"},
|
||||
*[{"name": f"tag_{i:03d}", "data_type": "String"} for i in range(10)],
|
||||
{"name": "val", "data_type": "Float64"},
|
||||
]},
|
||||
"rows": [fixture.row(0, j, 50) for j in range(32)],
|
||||
}
|
||||
|
||||
def test_full_tag_coverage_and_heterogeneous_schemas(self):
|
||||
for per_group in (50, 10000):
|
||||
variants = {tuple(fixture.tags(i, per_group)) for i in range(per_group)}
|
||||
self.assertEqual(set().union(*map(set, variants)), set(range(298)))
|
||||
self.assertGreater(len(variants), 1)
|
||||
self.assertTrue(all(len(tags) == 10 for tags in variants))
|
||||
|
||||
def test_accepts_engine_column_order_with_matching_values(self):
|
||||
records = self.records()
|
||||
order = [*range(1, 11), 0, 11] # The engine returns tags, timestamp, value.
|
||||
records["schema"]["column_schemas"] = [
|
||||
records["schema"]["column_schemas"][index] for index in order
|
||||
]
|
||||
records["rows"] = [[values[index] for index in order] for values in records["rows"]]
|
||||
fixture.check_records(0, 50, records)
|
||||
records["rows"][1][10] = 2
|
||||
with self.assertRaisesRegex(RuntimeError, "typed values"):
|
||||
fixture.check_records(0, 50, records)
|
||||
|
||||
def test_rejects_column_type_and_membership_changes(self):
|
||||
fixture.check_records(0, 50, self.records())
|
||||
for mutation in ("type", "extra", "missing"):
|
||||
records = self.records()
|
||||
columns = records["schema"]["column_schemas"]
|
||||
if mutation == "type":
|
||||
columns[0]["data_type"] = "TimestampMicrosecond"
|
||||
elif mutation == "extra":
|
||||
columns.append({"name": "unexpected", "data_type": "String"})
|
||||
else:
|
||||
del columns[1]
|
||||
with self.subTest(mutation=mutation), self.assertRaisesRegex(RuntimeError, "schema"):
|
||||
fixture.check_records(0, 50, records)
|
||||
|
||||
def test_rejects_null_and_numeric_type_changes(self):
|
||||
for value in ("", 0, False):
|
||||
records = self.records()
|
||||
records["rows"][0][-1] = value # Expected SQL NULL.
|
||||
with self.subTest(value=value), self.assertRaisesRegex(RuntimeError, "typed values"):
|
||||
fixture.check_records(0, 50, records)
|
||||
records = self.records()
|
||||
records["rows"][4][-1] = 1 # Numeric equality to 1.0 must not hide a type change.
|
||||
with self.assertRaisesRegex(RuntimeError, "typed values"):
|
||||
fixture.check_records(0, 50, records)
|
||||
|
||||
def test_source_comparison_requires_same_ddl_and_remapped_ids(self):
|
||||
source = {"fixture": {"tables_per_group": 50}, "schema_sha256": "schema",
|
||||
"data_sha256": "data", "ddl_sha256": "ddl",
|
||||
"table_ids": {"m000000": 1024, "phy_00": 1025}}
|
||||
target = copy.deepcopy(source)
|
||||
target["table_ids"] = {"m000000": 1041, "phy_00": 1042}
|
||||
fixture.compare_source(target, source)
|
||||
self.assertEqual(target["remapped_table_ids"], 2)
|
||||
for field in ("schema_sha256", "data_sha256", "ddl_sha256"):
|
||||
changed = copy.deepcopy(target)
|
||||
changed[field] = "changed"
|
||||
with self.subTest(field=field), self.assertRaisesRegex(RuntimeError, field):
|
||||
fixture.compare_source(changed, source)
|
||||
target["table_ids"]["phy_00"] = 1025
|
||||
with self.assertRaisesRegex(RuntimeError, "not all remapped"):
|
||||
fixture.compare_source(target, source)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Failure accounting for the metric snapshot runner."""
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location(
|
||||
"metric_snapshot_bench", Path(__file__).resolve().parents[1] / "metric-snapshot-s3-bench.py")
|
||||
bench = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(bench)
|
||||
|
||||
|
||||
class PhaseTests(unittest.TestCase):
|
||||
def test_docker_nanosecond_timestamps(self):
|
||||
start = bench.docker_timestamp("2026-10-14T06:06:44.72000818Z")
|
||||
end = bench.docker_timestamp("2026-10-14T06:06:46.123456789Z")
|
||||
self.assertAlmostEqual((end - start).total_seconds(), 1.403448)
|
||||
|
||||
def test_resource_interference_invalidates_success_and_preserves_fixture(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
runner = bench.Benchmark.__new__(bench.Benchmark)
|
||||
runner.round_dir = Path(directory)
|
||||
runner.server_pid = 0
|
||||
runner.phases = []
|
||||
runner.metrics = lambda: ""
|
||||
runner.summary = lambda: None
|
||||
fixture = runner.round_dir / "source-validation.json"
|
||||
fixture.write_text('{"data_sha256":"fixture"}')
|
||||
before = {"available": True, "pswpout_pages": 0, "oom_kill": 0}
|
||||
after = {"available": True, "pswpout_pages": 1, "oom_kill": 0}
|
||||
with patch.object(bench, "host_memory", side_effect=[before, after]):
|
||||
with self.assertRaisesRegex(RuntimeError, "Resource interference"):
|
||||
runner.phase("source-validation", lambda: None)
|
||||
self.assertEqual(json.loads(fixture.read_text()), {"data_sha256": "fixture"})
|
||||
evidence = json.loads((runner.round_dir / "source-validation-resources.json").read_text())
|
||||
self.assertEqual(evidence["outcome"], "resource-invalid")
|
||||
self.assertEqual(evidence["new_swapout_pages"], 1)
|
||||
|
||||
def test_action_failure_keeps_failed_resource_evidence(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
runner = bench.Benchmark.__new__(bench.Benchmark)
|
||||
runner.round_dir = Path(directory)
|
||||
runner.server_pid = 0
|
||||
runner.phases = []
|
||||
runner.metrics = lambda: ""
|
||||
runner.summary = lambda: None
|
||||
def fail():
|
||||
raise RuntimeError("export failed")
|
||||
with patch.object(bench, "host_memory", return_value={"available": False}):
|
||||
with self.assertRaisesRegex(RuntimeError, "export failed"):
|
||||
runner.phase("export", fail)
|
||||
evidence = json.loads((runner.round_dir / "export-resources.json").read_text())
|
||||
self.assertEqual(evidence["outcome"], "failed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Offline safety checks for the metric snapshot workflow's budget and cleanup."""
|
||||
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location("snapshot_ops", Path(__file__).parents[1] / "metric-snapshot-s3-ops.py")
|
||||
ops = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(ops)
|
||||
PREFIX = "metric-snapshot-bench/123-1/"
|
||||
|
||||
|
||||
class FakeS3:
|
||||
def __init__(self):
|
||||
self.objects = [{"Key": PREFIX + "current", "Size": 11}, {"Key": PREFIX + "data", "Size": 7}]
|
||||
self.deleted = []
|
||||
|
||||
def get_paginator(self, operation):
|
||||
assert operation == "list_objects_v2"
|
||||
parent = self
|
||||
|
||||
class Paginator:
|
||||
def paginate(self, **kwargs):
|
||||
assert kwargs["Bucket"] == "bucket" and kwargs["Prefix"].startswith(PREFIX)
|
||||
# Keys outside the run prefix model a misbehaving listing and are always returned.
|
||||
return iter([{"Contents": [item for item in parent.objects if not item["Key"].startswith(PREFIX)
|
||||
or item["Key"].startswith(kwargs["Prefix"])]}])
|
||||
|
||||
return Paginator()
|
||||
|
||||
def put_object(self, **kwargs):
|
||||
self.objects.append({"Key": kwargs["Key"], "Size": len(kwargs["Body"])})
|
||||
return {}
|
||||
|
||||
def get_object(self, **kwargs):
|
||||
return {"Body": io.BytesIO(b"metric snapshot benchmark")}
|
||||
|
||||
def delete_object(self, **kwargs):
|
||||
self.deleted.append(kwargs)
|
||||
self.objects = [item for item in self.objects if item["Key"] != kwargs["Key"]]
|
||||
|
||||
|
||||
class OperationsTest(unittest.TestCase):
|
||||
def env(self, **changes):
|
||||
return dict(GREPTIMEDB_IMAGE="greptime/greptimedb-nightly@sha256:" + "a" * 64,
|
||||
EXPECTED_COMMIT="b" * 40, LOGICAL_TABLES_PER_GROUP="10000",
|
||||
BENCHMARK_TIMEOUT_MINUTES="240", PRIOR_SPEND_USD="0",
|
||||
S3_REGION="ap-southeast-1", RUNNER_REGION="ap-southeast-1", S3_BUCKET="bucket",
|
||||
RUN_PREFIX=PREFIX) | changes
|
||||
|
||||
def test_budget_includes_provision_cleanup_and_other_cost_reserve(self):
|
||||
plan = ops.plan(self.env())
|
||||
self.assertEqual(plan["attempt_reserve_usd"], "6.4392")
|
||||
ops.plan(self.env(PRIOR_SPEND_USD="13.5608"))
|
||||
for changes in ({"PRIOR_SPEND_USD": "13.5609"}, {"PRIOR_SPEND_USD": "15"},
|
||||
{"PRIOR_SPEND_USD": "NaN"}, {"PRIOR_SPEND_USD": "-1"},
|
||||
{"BENCHMARK_TIMEOUT_MINUTES": "241"}, {"LOGICAL_TABLES_PER_GROUP": "49"},
|
||||
{"LOGICAL_TABLES_PER_GROUP": "10001"}, {"S3_REGION": "us-east-1"},
|
||||
{"GREPTIMEDB_IMAGE": "greptime/greptimedb:latest"},
|
||||
{"GREPTIMEDB_IMAGE": "docker.io/greptime/greptimedb@sha256:" + "a" * 64},
|
||||
{"GREPTIMEDB_IMAGE": "greptime/greptimedb:nightly@sha256:" + "a" * 64},
|
||||
{"EXPECTED_COMMIT": "abcdef"}):
|
||||
with self.subTest(changes=changes), self.assertRaises(ValueError):
|
||||
ops.plan(self.env(**changes))
|
||||
|
||||
def watcher_env(self):
|
||||
return self.env(GITHUB_REPOSITORY="GreptimeTeam/greptimedb", GITHUB_RUN_ID="123",
|
||||
GITHUB_RUN_ATTEMPT="2", GH_TOKEN="test-token", BENCHMARK_TIMEOUT_MINUTES="30")
|
||||
|
||||
def test_watcher_completed_job_uses_original_attempt_and_does_not_wait(self):
|
||||
for conclusion in ("success", "failure", "cancelled"):
|
||||
response = io.BytesIO(json.dumps({"jobs": [{"name": "benchmark", "status": "completed",
|
||||
"conclusion": conclusion}]}).encode())
|
||||
with self.subTest(conclusion=conclusion), \
|
||||
mock.patch.object(ops.urllib.request, "urlopen", return_value=response) as fetch, \
|
||||
mock.patch.object(ops.time, "time", return_value=1005), \
|
||||
mock.patch.object(ops.time, "sleep") as sleep:
|
||||
self.assertEqual(ops.wait_for_benchmark(self.watcher_env(), 1000),
|
||||
{"benchmark_conclusion": conclusion, "owned_attempt": "1"})
|
||||
self.assertIn("/runs/123/attempts/1/jobs?", fetch.call_args.args[0].full_url)
|
||||
sleep.assert_not_called()
|
||||
|
||||
def test_watcher_queue_reaches_durable_deadline(self):
|
||||
deadline = 1000 + (45 + 30 + 10) * 60
|
||||
clock = [deadline - 31]
|
||||
|
||||
def advance(seconds):
|
||||
self.assertLessEqual(seconds, 30)
|
||||
clock[0] += seconds
|
||||
|
||||
def queued(*args, **kwargs):
|
||||
return io.BytesIO(b'{"jobs": [{"name": "benchmark", "status": "queued"}]}')
|
||||
|
||||
with mock.patch.object(ops.urllib.request, "urlopen", side_effect=queued) as fetch, \
|
||||
mock.patch.object(ops.time, "time", side_effect=lambda: clock[0]), \
|
||||
mock.patch.object(ops.time, "sleep", side_effect=advance), \
|
||||
self.assertRaisesRegex(TimeoutError, "proceeding to resource cleanup"):
|
||||
ops.wait_for_benchmark(self.watcher_env(), 1000)
|
||||
self.assertEqual(clock[0], deadline)
|
||||
self.assertEqual(fetch.call_count, 2)
|
||||
|
||||
def test_watcher_api_error_fails_immediately_for_cleanup(self):
|
||||
with mock.patch.object(ops.urllib.request, "urlopen", side_effect=OSError("API unavailable")), \
|
||||
mock.patch.object(ops.time, "time", return_value=1005), \
|
||||
mock.patch.object(ops.time, "sleep") as sleep, \
|
||||
self.assertRaises(OSError):
|
||||
ops.wait_for_benchmark(self.watcher_env(), 1000)
|
||||
sleep.assert_not_called()
|
||||
|
||||
def test_cleanup_deletes_every_listed_object(self):
|
||||
s3 = FakeS3()
|
||||
result = ops.cleanup(s3, "bucket", PREFIX)
|
||||
self.assertEqual(result["objects"], 0)
|
||||
self.assertEqual(s3.deleted, [{"Bucket": "bucket", "Key": PREFIX + "current"},
|
||||
{"Bucket": "bucket", "Key": PREFIX + "data"}])
|
||||
|
||||
def test_probe_refuses_occupied_prefix_and_removes_its_object(self):
|
||||
with self.assertRaisesRegex(ValueError, "already occupied"):
|
||||
ops.probe(FakeS3(), "bucket", PREFIX)
|
||||
s3 = FakeS3()
|
||||
s3.objects = []
|
||||
self.assertEqual(ops.probe(s3, "bucket", PREFIX)["objects"], 0)
|
||||
self.assertEqual(s3.deleted, [{"Bucket": "bucket", "Key": PREFIX + "_probe"}])
|
||||
|
||||
def test_failed_probe_delete_still_has_hosted_cleanup_path(self):
|
||||
s3 = FakeS3()
|
||||
s3.objects = []
|
||||
with mock.patch.object(s3, "delete_object", side_effect=OSError("temporary delete failure")), \
|
||||
self.assertRaises(OSError):
|
||||
ops.probe(s3, "bucket", PREFIX)
|
||||
self.assertEqual([item["Key"] for item in s3.objects], [PREFIX + "_probe"])
|
||||
workflow = (Path(__file__).parents[2] / "workflows/metric-snapshot-s3-bench.yml").read_text()
|
||||
teardown = workflow.split(" teardown:\n", 1)[1]
|
||||
job_if = next(line.strip() for line in teardown.splitlines() if line.startswith(" if:"))
|
||||
self.assertEqual(job_if, "if: ${{ always() && needs.validate.outputs.run_id != '' }}")
|
||||
ec2 = teardown.split(" - name: Delete EC2, its temporary disks and runner registration\n", 1)[1]
|
||||
self.assertEqual(ec2.splitlines()[0].strip(), "if: ${{ always() && needs.provision.result != 'skipped' }}")
|
||||
self.assertIn(" RUN_PREFIX: ${{ needs.validate.outputs.run_prefix }}\n", teardown.split(" steps:", 1)[0])
|
||||
s3_cleanup = teardown.split(" - name: Remove and verify only this run's S3 prefix\n", 1)[1].split(" - ", 1)[0]
|
||||
self.assertEqual(s3_cleanup.splitlines()[0].strip(), "if: always()")
|
||||
self.assertIn("run: uv run greptimedb/.github/scripts/metric-snapshot-s3-ops.py cleanup\n", s3_cleanup)
|
||||
self.assertEqual(ops.cleanup(s3, "bucket", PREFIX)["objects"], 0)
|
||||
|
||||
def test_round_inventory_excludes_other_rounds_while_cleanup_covers_the_run(self):
|
||||
s3 = FakeS3()
|
||||
s3.objects = [{"Key": PREFIX + "smoke/pack-000000.bin", "Size": 3},
|
||||
{"Key": PREFIX + "full/pack-000000.bin", "Size": 5},
|
||||
{"Key": PREFIX + "full/index.json", "Size": 2}]
|
||||
full = ops.inventory(s3, "bucket", PREFIX, "full")
|
||||
self.assertEqual((full["prefix"], full["objects"], full["bytes"]), (PREFIX + "full/", 2, 7))
|
||||
self.assertEqual(ops.inventory(s3, "bucket", PREFIX, "smoke")["bytes"], 3)
|
||||
self.assertEqual(ops.inventory(s3, "bucket", PREFIX)["objects"], 3)
|
||||
for round in ("", "../full", "full/"):
|
||||
with self.subTest(round=round), self.assertRaises(ValueError):
|
||||
ops.inventory(s3, "bucket", PREFIX, round)
|
||||
self.assertEqual(ops.cleanup(s3, "bucket", PREFIX)["objects"], 0)
|
||||
self.assertEqual(len(s3.deleted), 3)
|
||||
|
||||
def test_bad_prefix_and_out_of_scope_response_cannot_delete(self):
|
||||
for prefix in ("", "metric-snapshot-bench/", "metric-snapshot-bench/123-1", PREFIX + "../", "oswal-bench/123-1/"):
|
||||
s3 = FakeS3()
|
||||
with self.subTest(prefix=prefix), self.assertRaises(ValueError):
|
||||
ops.cleanup(s3, "bucket", prefix)
|
||||
self.assertEqual(s3.deleted, [])
|
||||
s3 = FakeS3()
|
||||
s3.objects[0]["Key"] = "other-run/current"
|
||||
with self.assertRaises(ValueError):
|
||||
ops.inventory(s3, "bucket", PREFIX, "full")
|
||||
with self.assertRaises(ValueError):
|
||||
ops.cleanup(s3, "bucket", PREFIX)
|
||||
self.assertEqual(s3.deleted, [])
|
||||
|
||||
def test_credential_crossing_read_boundary_blocks_artifact_upload(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
logfile = Path(directory) / "server.log"
|
||||
logfile.write_bytes(b"a" * (1024 * 1024 - 4) + b"private-key" + b"tail")
|
||||
with self.assertRaisesRegex(ValueError, "upload is blocked"):
|
||||
ops.scrub(directory, {"AWS_SECRET_ACCESS_KEY": "private-key"})
|
||||
logfile.write_text("safe log\n")
|
||||
ops.scrub(directory, {"AWS_SECRET_ACCESS_KEY": "private-key"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -19,9 +19,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
- run: cargo doc --workspace --no-deps --document-private-items
|
||||
- run: |
|
||||
|
||||
@@ -10,6 +10,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
|
||||
@@ -82,12 +82,18 @@ jobs:
|
||||
version: "latest"
|
||||
- name: Run check-version script tests
|
||||
run: uv run --no-project python .github/scripts/check-version-test.py
|
||||
- name: Run release Linux architecture tests
|
||||
run: python3 .github/scripts/release-linux-architectures-test.py
|
||||
- name: Run previous-release-tag script tests
|
||||
run: python3 .github/scripts/previous-release-tag-test.py
|
||||
- name: Run enterprise license check tests
|
||||
run: python3 scripts/check-enterprise-license-test.py
|
||||
- name: Run fuzz orchestration script tests
|
||||
run: .github/scripts/run-fuzz-targets-test.sh
|
||||
- name: Run CI comment-command tests
|
||||
run: python3 .github/scripts/tests/test_ci_slash.py
|
||||
- name: Run metric snapshot benchmark tests
|
||||
run: python3 -m unittest discover -s .github/scripts/tests -p 'test_metric_snapshot*.py'
|
||||
- name: Run query-regression Python tooling tests
|
||||
run: |
|
||||
python3 tests/perf/test_query_regression_runner_compaction_toctou.py
|
||||
@@ -113,9 +119,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
- name: Rust Cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
|
||||
@@ -54,9 +54,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
- name: Build Binaries Cache
|
||||
id: build-binaries-cache
|
||||
@@ -193,9 +193,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
- name: Rust Cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
@@ -327,9 +327,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
cache: false
|
||||
@@ -364,13 +364,11 @@ jobs:
|
||||
env:
|
||||
RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
|
||||
run: |
|
||||
sudo apt-get install -y libfuzzer-14-dev
|
||||
cargo install cargo-fuzz --version "${CARGO_FUZZ_VERSION}" --locked
|
||||
- name: Build fuzz target binaries
|
||||
if: steps.fuzz-binaries.outputs.cache-hit != 'true'
|
||||
shell: bash
|
||||
env:
|
||||
CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
|
||||
RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
|
||||
FUZZ_TARGETS: |
|
||||
fuzz_create_database
|
||||
@@ -587,9 +585,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
- name: Build Profile-CI Cache
|
||||
id: build-greptime-ci-cache
|
||||
@@ -705,9 +703,9 @@ jobs:
|
||||
uses: ./.github/actions/setup-etcd-cluster
|
||||
# Only the Kafka WAL helper needs the Rust toolchain.
|
||||
- if: matrix.mode.kafkaWalHelper
|
||||
uses: arduino/setup-protoc@v3
|
||||
uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- if: matrix.mode.kafkaWalHelper
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
name: Metric Snapshot S3 Benchmark
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
greptimedb_image:
|
||||
description: 'Immutable greptime/greptimedb or greptimedb-nightly image (repository@sha256:digest)'
|
||||
required: true
|
||||
type: string
|
||||
expected_commit:
|
||||
description: 'Full source commit SHA reported by the image'
|
||||
required: true
|
||||
type: string
|
||||
logical_tables_per_group:
|
||||
description: 'Logical tables per physical table, 50..10000; 10 physical tables, 32 rows per logical table'
|
||||
type: number
|
||||
default: 10000
|
||||
benchmark_timeout_minutes:
|
||||
description: 'Setup, 500-table smoke and full run timeout, 30..240 minutes'
|
||||
type: number
|
||||
default: 240
|
||||
prior_spend_usd:
|
||||
description: 'Cumulative estimated cloud spend for all earlier attempts in this $20 campaign'
|
||||
required: true
|
||||
type: string
|
||||
default: '0'
|
||||
permissions:
|
||||
contents: read
|
||||
concurrency:
|
||||
group: metric-snapshot-s3-bench
|
||||
cancel-in-progress: false
|
||||
env:
|
||||
GREPTIMEDB_IMAGE: ${{ inputs.greptimedb_image }}
|
||||
EXPECTED_COMMIT: ${{ inputs.expected_commit }}
|
||||
LOGICAL_TABLES_PER_GROUP: ${{ inputs.logical_tables_per_group }}
|
||||
BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }}
|
||||
PRIOR_SPEND_USD: ${{ inputs.prior_spend_usd }}
|
||||
RUN_PREFIX: metric-snapshot-bench/${{ github.run_id }}-${{ github.run_attempt }}/
|
||||
S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
|
||||
S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
|
||||
RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
jobs:
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
outputs:
|
||||
instance_type: ${{ steps.runner.outputs.instance_type }}
|
||||
run_id: ${{ steps.identity.outputs.run_id }}
|
||||
run_prefix: ${{ steps.identity.outputs.run_prefix }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Test benchmark boundaries
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/tests/test_metric_snapshot_s3_ops.py
|
||||
python3 greptimedb/.github/scripts/tests/test_metric_snapshot_fixture.py
|
||||
python3 greptimedb/.github/scripts/tests/test_metric_snapshot_s3_bench.py
|
||||
- name: Validate pinned version, scale, region and cumulative budget
|
||||
run: python3 greptimedb/.github/scripts/metric-snapshot-s3-ops.py plan
|
||||
- name: Bind resource ownership to this validated attempt
|
||||
id: identity
|
||||
run: |
|
||||
echo "run_id=$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT"
|
||||
echo "run_prefix=$RUN_PREFIX" >> "$GITHUB_OUTPUT"
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- name: Check S3 access before allocating EC2
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
|
||||
run: uv run greptimedb/.github/scripts/metric-snapshot-s3-ops.py probe
|
||||
- name: Resolve the fixed benchmark runner
|
||||
id: runner
|
||||
run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider AWS --instance-type c6id.4xlarge
|
||||
provision:
|
||||
needs: validate
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
outputs:
|
||||
label: ${{ steps.aws.outputs.label }}
|
||||
instance_id: ${{ steps.aws.outputs.instance_id }}
|
||||
runner_name: ${{ steps.aws.outputs.runner_name }}
|
||||
started_at: ${{ steps.start.outputs.started_at }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- uses: aws-actions/configure-aws-credentials@v4
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ env.RUNNER_REGION }}
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- name: Record allocation start
|
||||
id: start
|
||||
run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT"
|
||||
- name: Create ephemeral EC2 runner
|
||||
id: aws
|
||||
env:
|
||||
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
AWS_REGION: ${{ env.RUNNER_REGION }}
|
||||
AWS_EC2_IMAGE_ID: ${{ vars.BENCHMARK_EC2_IMAGE_ID || 'auto' }}
|
||||
AWS_EC2_INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }}
|
||||
AWS_EC2_SUBNET_ID: ${{ vars.EC2_RUNNER_SUBNET_ID }}
|
||||
AWS_EC2_SECURITY_GROUP_ID: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
|
||||
AWS_EC2_SYSTEM_DISK_GIB: '60'
|
||||
AWS_EC2_TTL_HOURS: '6'
|
||||
AWS_EC2_RUN_ID: ${{ needs.validate.outputs.run_id }}
|
||||
run: uv run greptimedb/.github/scripts/aws-ec2-runner-provision.py
|
||||
benchmark:
|
||||
needs:
|
||||
- validate
|
||||
- provision
|
||||
runs-on: ${{ needs.provision.outputs.label }}
|
||||
timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }}
|
||||
env:
|
||||
RUN_PREFIX: ${{ needs.validate.outputs.run_prefix }}
|
||||
RESULTS: ${{ github.workspace }}/metric-snapshot-results
|
||||
DISK_ROOT: /mnt/metric-snapshot-disk
|
||||
DATA_ROOT: /mnt/metric-snapshot-disk/data
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Set up benchmark dependencies
|
||||
uses: ./greptimedb/.github/actions/setup-benchmark
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- name: Mount the temporary local File data disk
|
||||
run: |
|
||||
device=$(lsblk -dnpo NAME,MODEL | awk '/Instance Storage/ {print $1; exit}')
|
||||
test -n "$device"
|
||||
if [[ "$(lsblk -nro TYPE "$device" | wc -l)" -ne 1 ]] || findmnt --source "$device" >/dev/null; then
|
||||
echo 'Refusing partitioned or mounted instance storage' >&2
|
||||
exit 1
|
||||
fi
|
||||
probe_status=0
|
||||
sudo -n blkid -p "$device" || probe_status=$?
|
||||
if ((probe_status != 2)); then
|
||||
echo 'Refusing a disk with an existing signature or an unsuccessful signature probe' >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo -n mkfs.ext4 -q "$device"
|
||||
sudo -n mkdir -p "$DISK_ROOT"
|
||||
sudo -n mount -o noatime "$device" "$DISK_ROOT"
|
||||
sudo -n chown "$(id -u):$(id -g)" "$DISK_ROOT"
|
||||
findmnt -no SOURCE,FSTYPE -T "$DISK_ROOT"
|
||||
- name: Probe the real same-region S3 bucket
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
|
||||
run: uv run greptimedb/.github/scripts/metric-snapshot-s3-ops.py probe
|
||||
- name: Run the 500-table smoke, then one full packed export and restore
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
|
||||
run: python3 greptimedb/.github/scripts/metric-snapshot-s3-bench.py run
|
||||
- name: Check results for credentials
|
||||
id: scrub
|
||||
if: always()
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
|
||||
run: python3 greptimedb/.github/scripts/metric-snapshot-s3-ops.py scrub
|
||||
- name: Publish the readable result
|
||||
if: ${{ always() && steps.scrub.outcome == 'success' }}
|
||||
run: |
|
||||
if test -f "$RESULTS/summary.md"; then
|
||||
cat "$RESULTS/summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
- name: Upload bounded-retention evidence
|
||||
if: ${{ always() && steps.scrub.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: metric-snapshot-s3-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ env.RESULTS }}
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
teardown:
|
||||
needs:
|
||||
- validate
|
||||
- provision
|
||||
# Start alongside the benchmark so an offline runner cannot hold cleanup in its queue.
|
||||
if: ${{ always() && needs.validate.outputs.run_id != '' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 330
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
RUN_PREFIX: ${{ needs.validate.outputs.run_prefix }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- name: Wait for completion within the allocation deadline
|
||||
if: ${{ !cancelled() && needs.provision.result == 'success' && needs.provision.outputs.started_at != '' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
STARTED_AT: ${{ needs.provision.outputs.started_at }}
|
||||
run: python3 greptimedb/.github/scripts/metric-snapshot-s3-ops.py wait-for-benchmark --started-at "$STARTED_AT"
|
||||
- uses: aws-actions/configure-aws-credentials@v4
|
||||
if: ${{ always() && needs.provision.result != 'skipped' }}
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ env.RUNNER_REGION }}
|
||||
- name: Delete EC2, its temporary disks and runner registration
|
||||
if: ${{ always() && needs.provision.result != 'skipped' }}
|
||||
env:
|
||||
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
AWS_REGION: ${{ env.RUNNER_REGION }}
|
||||
AWS_EC2_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }}
|
||||
AWS_EC2_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }}
|
||||
AWS_EC2_RUN_ID: ${{ needs.validate.outputs.run_id }}
|
||||
run: uv run greptimedb/.github/scripts/aws-ec2-runner-teardown.py
|
||||
- name: Remove and verify only this run's S3 prefix
|
||||
if: always()
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
|
||||
run: uv run greptimedb/.github/scripts/metric-snapshot-s3-ops.py cleanup
|
||||
- name: Estimate the cumulative cloud cost
|
||||
if: ${{ always() && needs.provision.outputs.started_at != '' }}
|
||||
env:
|
||||
STARTED_AT: ${{ needs.provision.outputs.started_at }}
|
||||
run: python3 greptimedb/.github/scripts/metric-snapshot-s3-ops.py cost --started-at "$STARTED_AT" >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -53,9 +53,9 @@ jobs:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/setup-cyborg
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
- name: Rust Cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
@@ -82,9 +82,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: rui314/setup-mold@v1
|
||||
- name: Install toolchain
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
@@ -134,9 +134,9 @@ jobs:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/setup-cyborg
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: KyleMayes/install-llvm-action@v1
|
||||
with:
|
||||
version: "14.0"
|
||||
@@ -157,6 +157,7 @@ jobs:
|
||||
env:
|
||||
CARGO_BUILD_RUSTFLAGS: "-C linker=lld-link"
|
||||
RUST_BACKTRACE: 1
|
||||
RUST_MIN_STACK: 8388608 # 8 MiB, aligned with Rust CI and release tests.
|
||||
CARGO_INCREMENTAL: 0
|
||||
GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
|
||||
GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
|
||||
|
||||
@@ -10,6 +10,34 @@ on:
|
||||
description: 'Docker Hub greptime/greptimedb tag; nightly = the newest v*-nightly-* tag (latest is a release without the object store WAL)'
|
||||
type: string
|
||||
default: nightly
|
||||
greptimedb_image:
|
||||
description: 'Full image reference with a tag or digest; used instead of greptimedb_tag when set'
|
||||
type: string
|
||||
default: ''
|
||||
images:
|
||||
description: 'JSON list of 2..4 {"label", "image"} to compare on one instance instead of greptimedb_tag and greptimedb_image: every cell runs once per image, in an order that reverses every other round; a label is up to 8 lowercase letters and digits, an image a full reference with a tag or digest, the first the reference of the ratios'
|
||||
type: string
|
||||
default: ''
|
||||
warmup:
|
||||
description: Before round 1, run the first load cell once per image and leave it out of every statistic
|
||||
type: boolean
|
||||
default: false
|
||||
configs:
|
||||
description: 'WAL configurations: R (Raft Engine, sync_write), R0 (Raft Engine, standalone default), D (durable), E (enqueued)'
|
||||
type: string
|
||||
default: R,D,E
|
||||
extra_env:
|
||||
description: 'Environment for every GreptimeDB container: a JSON object or K=V lines'
|
||||
type: string
|
||||
default: ''
|
||||
restart:
|
||||
description: 'After each load, SIGKILL the server, restart it on the same prefix and data home, and time recovery; object store cells flush every 10ms'
|
||||
type: boolean
|
||||
default: false
|
||||
groups:
|
||||
description: 'JSON list of cell groups, each overriding configs, extra_env, concurrency, rounds, rows, restart, plus idle_seconds and partitions (regions of the loaded table), or mode crash with cycles, kill_after [min, max] and pin, or mode drop with write_seconds; empty = one group from the inputs above'
|
||||
type: string
|
||||
default: ''
|
||||
instance_type:
|
||||
description: EC2 instance type in the bucket region; the d of c6id gives a local NVMe instance store for the Raft Engine WAL
|
||||
type: string
|
||||
@@ -27,7 +55,7 @@ on:
|
||||
type: number
|
||||
default: 1000
|
||||
rounds:
|
||||
description: Rounds; the configuration order alternates between rounds (R, D, E then E, D, R)
|
||||
description: Rounds; the configuration and image order alternates between rounds (R, D, E then E, D, R)
|
||||
type: number
|
||||
default: 2
|
||||
runtime_image:
|
||||
@@ -47,6 +75,13 @@ permissions:
|
||||
env:
|
||||
SMOKE: ${{ inputs.smoke }}
|
||||
GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
|
||||
GREPTIMEDB_IMAGE_INPUT: ${{ inputs.greptimedb_image }}
|
||||
OSWAL_IMAGES: ${{ inputs.images }}
|
||||
WARMUP: ${{ inputs.warmup }}
|
||||
CONFIGS: ${{ inputs.configs }}
|
||||
EXTRA_ENV: ${{ inputs.extra_env }}
|
||||
RESTART: ${{ inputs.restart }}
|
||||
OSWAL_GROUPS: ${{ inputs.groups }}
|
||||
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
|
||||
CONCURRENCY: ${{ inputs.concurrency }}
|
||||
ROWS: ${{ inputs.rows }}
|
||||
@@ -61,7 +96,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
outputs:
|
||||
instance_type: ${{ steps.runner.outputs.instance_type }}
|
||||
greptimedb_tag: ${{ steps.plan.outputs.greptimedb_tag }}
|
||||
greptimedb_images: ${{ steps.images.outputs.greptimedb_images }}
|
||||
runtime_image: ${{ steps.runtime.outputs.runtime_image }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -77,8 +112,16 @@ jobs:
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/oswal-s3-bench.py plan --smoke "$SMOKE" \
|
||||
--concurrency "$CONCURRENCY" --rows "$ROWS" --batch-rows "$BATCH_ROWS" --rounds "$ROUNDS" \
|
||||
--configs "$CONFIGS" --extra-env "$EXTRA_ENV" --restart "$RESTART" --groups "$OSWAL_GROUPS" \
|
||||
--images "$OSWAL_IMAGES" --warmup "$WARMUP" \
|
||||
--timeout-minutes "$BENCHMARK_TIMEOUT_MINUTES" --ttl-hours "$JANITOR_TTL_HOURS"
|
||||
python3 greptimedb/.github/scripts/oswal-s3-bench.py tag "$GREPTIMEDB_TAG"
|
||||
python3 greptimedb/.github/scripts/oswal-s3-bench.py image --tag "$GREPTIMEDB_TAG" --image "$GREPTIMEDB_IMAGE_INPUT" \
|
||||
--images "$OSWAL_IMAGES"
|
||||
- name: Pull every GreptimeDB image and record its version
|
||||
id: images
|
||||
env:
|
||||
REQUESTED_IMAGES: ${{ steps.plan.outputs.requested_images }}
|
||||
run: bash greptimedb/.github/scripts/oswal-s3-bench-run.sh check-images "$REQUESTED_IMAGES"
|
||||
- name: Resolve CI runner
|
||||
id: runner
|
||||
run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider AWS --instance-type "$REQUESTED_INSTANCE_TYPE"
|
||||
@@ -87,7 +130,7 @@ jobs:
|
||||
BUCKET_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
|
||||
RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
run: python3 greptimedb/.github/scripts/oswal-s3-bench.py regions --bucket "$BUCKET_REGION" --runner "$RUNNER_REGION"
|
||||
- name: Check the runtime image
|
||||
- name: Pin the runtime image digest
|
||||
id: runtime
|
||||
run: bash greptimedb/.github/scripts/oswal-s3-bench-run.sh check-runtime "$RUNTIME_IMAGE"
|
||||
provision:
|
||||
@@ -132,7 +175,7 @@ jobs:
|
||||
timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }}
|
||||
env:
|
||||
INSTANCE_TYPE: ${{ inputs.smoke && 'ubuntu-latest' || needs.validate.outputs.instance_type }}
|
||||
GREPTIMEDB_TAG: ${{ needs.validate.outputs.greptimedb_tag }}
|
||||
GREPTIMEDB_IMAGES: ${{ needs.validate.outputs.greptimedb_images }}
|
||||
RUNTIME_IMAGE: ${{ needs.validate.outputs.runtime_image }}
|
||||
RUN_PREFIX: oswal-bench/${{ github.run_id }}-${{ github.run_attempt }}
|
||||
OWNER: o11ybench-oswal-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
@@ -157,12 +200,14 @@ jobs:
|
||||
run: |
|
||||
bash greptimedb/.github/scripts/pull-benchmark-image.sh greptime/minio:2024
|
||||
docker compose -f greptimedb/tests-integration/fixtures/docker-compose.yml up -d --wait minio
|
||||
- name: Pull images and write the plan
|
||||
- name: Pull and check images, write the plan
|
||||
run: |
|
||||
bash greptimedb/.github/scripts/oswal-s3-bench-run.sh pull
|
||||
python3 greptimedb/.github/scripts/oswal-s3-bench.py plan --smoke "$SMOKE" --output "$PLAN" \
|
||||
--concurrency "$CONCURRENCY" --rows "$ROWS" --batch-rows "$BATCH_ROWS" --rounds "$ROUNDS"
|
||||
python3 greptimedb/.github/scripts/oswal-s3-bench.py manifest --results "$RESULTS" \
|
||||
--concurrency "$CONCURRENCY" --rows "$ROWS" --batch-rows "$BATCH_ROWS" --rounds "$ROUNDS" \
|
||||
--configs "$CONFIGS" --extra-env "$EXTRA_ENV" --restart "$RESTART" --groups "$OSWAL_GROUPS" \
|
||||
--images "$OSWAL_IMAGES" --warmup "$WARMUP"
|
||||
python3 greptimedb/.github/scripts/oswal-s3-bench.py manifest --results "$RESULTS" --images "$GREPTIMEDB_IMAGES" \
|
||||
--runtime-image "$RUNTIME_IMAGE" --instance-type "$INSTANCE_TYPE" --region "$S3_REGION" --prefix "$RUN_PREFIX"
|
||||
- name: Probe the bucket
|
||||
env:
|
||||
|
||||
@@ -14,6 +14,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
|
||||
@@ -19,7 +19,6 @@ on:
|
||||
# NIGHTLY_RELEASE_MAX_AGE_DAYS days, see the 'Check nightly release necessity' step.
|
||||
- cron: '0 0 * * 1-5'
|
||||
workflow_dispatch: # Allows you to run this workflow manually.
|
||||
# Notes: The GitHub Actions ONLY support 10 inputs; this final input is the tenth.
|
||||
inputs:
|
||||
linux_amd64_runner:
|
||||
type: choice
|
||||
@@ -57,9 +56,19 @@ on:
|
||||
description: Do not run integration tests during the build
|
||||
type: boolean
|
||||
default: true
|
||||
build_linux_artifacts:
|
||||
build_linux_amd64_artifacts:
|
||||
type: boolean
|
||||
description: Build linux artifacts (amd64, arm64, and riscv64)
|
||||
description: Build linux-amd64 artifacts (publishing always builds all Linux architectures)
|
||||
required: false
|
||||
default: false
|
||||
build_linux_arm64_artifacts:
|
||||
type: boolean
|
||||
description: Build linux-arm64 artifacts (publishing always builds all Linux architectures)
|
||||
required: false
|
||||
default: false
|
||||
build_linux_riscv64_artifacts:
|
||||
type: boolean
|
||||
description: Build linux-riscv64 artifacts (publishing always builds all Linux architectures)
|
||||
required: false
|
||||
default: false
|
||||
build_macos_artifacts:
|
||||
@@ -115,6 +124,9 @@ jobs:
|
||||
if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
build-linux-amd64: ${{ steps.linux-architectures.outputs.amd64 }}
|
||||
build-linux-arm64: ${{ steps.linux-architectures.outputs.arm64 }}
|
||||
build-linux-riscv64: ${{ steps.linux-architectures.outputs.riscv64 }}
|
||||
linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
|
||||
linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
|
||||
macos-runner: ${{ inputs.macos_runner || vars.DEFAULT_MACOS_RUNNER }}
|
||||
@@ -143,6 +155,17 @@ jobs:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
# Publishing and automatic releases retain the full Linux build.
|
||||
- name: Resolve Linux architectures
|
||||
id: linux-architectures
|
||||
env:
|
||||
BUILD_LINUX_AMD64_ARTIFACTS: ${{ inputs.build_linux_amd64_artifacts }}
|
||||
BUILD_LINUX_ARM64_ARTIFACTS: ${{ inputs.build_linux_arm64_artifacts }}
|
||||
BUILD_LINUX_RISCV64_ARTIFACTS: ${{ inputs.build_linux_riscv64_artifacts }}
|
||||
PUBLISH_GITHUB_RELEASE: ${{ inputs.publish_github_release }}
|
||||
RELEASE_IMAGES: ${{ inputs.release_images }}
|
||||
run: bash .github/scripts/release-linux-architectures.sh >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Check Rust toolchain version
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -182,7 +205,7 @@ jobs:
|
||||
./.github/scripts/check-version.sh "${{ steps.create-version.outputs.version }}"
|
||||
|
||||
- name: Allocate linux-amd64 runner
|
||||
if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && steps.check-nightly.outputs.nightly-required == 'true' }}
|
||||
if: ${{ (steps.linux-architectures.outputs.amd64 == 'true' || steps.linux-architectures.outputs.riscv64 == 'true') && steps.check-nightly.outputs.nightly-required == 'true' }}
|
||||
uses: ./.github/actions/start-runner
|
||||
id: start-linux-amd64-runner
|
||||
with:
|
||||
@@ -196,7 +219,7 @@ jobs:
|
||||
subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
|
||||
|
||||
- name: Allocate linux-arm64 runner
|
||||
if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && steps.check-nightly.outputs.nightly-required == 'true' }}
|
||||
if: ${{ steps.linux-architectures.outputs.arm64 == 'true' && steps.check-nightly.outputs.nightly-required == 'true' }}
|
||||
uses: ./.github/actions/start-runner
|
||||
id: start-linux-arm64-runner
|
||||
with:
|
||||
@@ -211,7 +234,7 @@ jobs:
|
||||
|
||||
build-linux-amd64-artifacts:
|
||||
name: Build linux-amd64 artifacts
|
||||
if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
|
||||
if: ${{ needs.allocate-runners.outputs.build-linux-amd64 == 'true' && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
|
||||
needs: [
|
||||
allocate-runners,
|
||||
]
|
||||
@@ -237,7 +260,7 @@ jobs:
|
||||
|
||||
build-linux-arm64-artifacts:
|
||||
name: Build linux-arm64 artifacts
|
||||
if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
|
||||
if: ${{ needs.allocate-runners.outputs.build-linux-arm64 == 'true' && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
|
||||
needs: [
|
||||
allocate-runners,
|
||||
]
|
||||
@@ -264,7 +287,7 @@ jobs:
|
||||
|
||||
build-linux-riscv64-artifacts:
|
||||
name: Build linux-riscv64 artifacts
|
||||
if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
|
||||
if: ${{ needs.allocate-runners.outputs.build-linux-riscv64 == 'true' && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
|
||||
needs: [
|
||||
allocate-runners,
|
||||
]
|
||||
@@ -415,11 +438,15 @@ jobs:
|
||||
REQUESTED_VALIDATION_POLICY: ${{ inputs.release_validation }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
base_commit="${GITHUB_SHA}"
|
||||
if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then
|
||||
base_commit="${GITHUB_SHA}^"
|
||||
fi
|
||||
previous_release_tag="$(git tag --merged "${base_commit}" --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -n 1 || true)"
|
||||
# Stable tags on older release branches are not ancestors of this
|
||||
# commit. Pick the newest vX.Y.Z older than the version being released.
|
||||
release_version="${{ needs.allocate-runners.outputs.version }}"
|
||||
previous_release_tag="$(
|
||||
git ls-remote --tags origin 'refs/tags/v*' \
|
||||
| awk '{print $2}' \
|
||||
| sed -e 's#^refs/tags/##' \
|
||||
| python3 .github/scripts/previous-release-tag.py --version "${release_version}"
|
||||
)"
|
||||
if [[ -z "${previous_release_tag}" ]]; then
|
||||
previous_release_tag="main"
|
||||
fi
|
||||
@@ -489,9 +516,9 @@ jobs:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
|
||||
@@ -83,10 +83,10 @@ jobs:
|
||||
java-version: '17'
|
||||
cache: 'maven'
|
||||
|
||||
- name: Setup Python 3.8
|
||||
- name: Setup Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.8'
|
||||
python-version: '3.13'
|
||||
|
||||
- name: Setup uv
|
||||
uses: astral-sh/setup-uv@v3
|
||||
|
||||
+15
-14
@@ -40,9 +40,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
components: rustfmt
|
||||
@@ -58,9 +58,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
components: clippy
|
||||
@@ -102,9 +102,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- name: Install riscv64 cross toolchain
|
||||
run: sudo apt-get update && sudo apt-get install -y gcc-riscv64-linux-gnu g++-riscv64-linux-gnu
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
@@ -130,9 +130,9 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/setup-cyborg
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: KyleMayes/install-llvm-action@v1
|
||||
with:
|
||||
version: "14.0"
|
||||
@@ -166,9 +166,9 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/setup-cyborg
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: KyleMayes/install-llvm-action@v1
|
||||
with:
|
||||
version: "14.0"
|
||||
@@ -210,9 +210,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: rui314/setup-mold@v1
|
||||
- name: Install toolchain
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
@@ -265,9 +265,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: arduino/setup-protoc@v3
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tool: protoc@3.23.4
|
||||
- uses: rui314/setup-mold@v1
|
||||
- name: Install toolchain
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
@@ -294,6 +294,7 @@ jobs:
|
||||
env:
|
||||
CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
||||
RUST_BACKTRACE: 1
|
||||
RUST_MIN_STACK: 8388608 # 8MB, aligned with the test job
|
||||
CARGO_INCREMENTAL: 0
|
||||
# Runs the object-store matrix against the local MinIO from
|
||||
# tests-integration/fixtures. Real AWS S3 is covered by the nightly
|
||||
|
||||
@@ -262,6 +262,7 @@ run-it-in-container: start-etcd ## Run integration tests in dev-builder.
|
||||
docker run --network=host \
|
||||
-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v ${CARGO_GIT_CACHE}:/root/.cargo/git -v /tmp:/tmp \
|
||||
-e GT_S3_BUCKET -e GT_S3_ACCESS_KEY_ID -e GT_S3_ACCESS_KEY -e GT_S3_REGION \
|
||||
-e RUST_MIN_STACK \
|
||||
-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:${DEV_BUILDER_IMAGE_TAG} \
|
||||
make test sqlness-test BUILD_JOBS=${BUILD_JOBS}
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# Metric snapshot benchmark on AWS S3
|
||||
|
||||
The manual `Metric Snapshot S3 Benchmark` workflow tests packed Metric export and
|
||||
restore against real S3 in `ap-southeast-1`. It must first exist on the repository's
|
||||
default branch for GitHub to register `workflow_dispatch`. It was validated on AWS
|
||||
by run [38544464731](https://github.com/GreptimeTeam/greptimedb/actions/runs/38544464731);
|
||||
[#9573](https://github.com/GreptimeTeam/greptimedb/pull/9573) records its conditions and results.
|
||||
|
||||
## Run
|
||||
|
||||
Use the repository's existing EC2 runner configuration and CI S3 credentials.
|
||||
The workflow passes credentials only through CI; no local AWS credentials are
|
||||
needed. Source, target and CLI use the same immutable image. This image's source
|
||||
SHA and required CLI capabilities have been checked:
|
||||
|
||||
```bash
|
||||
gh workflow run metric-snapshot-s3-bench.yml \
|
||||
--repo GreptimeTeam/greptimedb --ref main \
|
||||
-f greptimedb_image=greptime/greptimedb-nightly@sha256:20a1182bed0f452e6d573ca110b313e0b5cb76de7aedfa41a0c339963d4683d3 \
|
||||
-f expected_commit=f690bbb2040865c0d22b0b57d51612b60ae27cd0 \
|
||||
-f logical_tables_per_group=10000 \
|
||||
-f benchmark_timeout_minutes=240 \
|
||||
-f prior_spend_usd=0
|
||||
```
|
||||
|
||||
Start a new dispatch for each attempt and update `prior_spend_usd` to include all
|
||||
earlier cloud attempts in the campaign, including failed setup and cleanup.
|
||||
Do not reset the estimate to zero when retrying. The workflow accepts 50–10,000
|
||||
logical tables per group and a 30–240 minute benchmark job timeout.
|
||||
|
||||
The runner is fixed at `c6id.4xlarge` (16 vCPU, 32 GiB RAM). At the recorded
|
||||
Singapore Linux on-demand estimate of $0.9408/hour, the default attempt reserves
|
||||
$6.4392: four benchmark hours, 45 minutes of provisioning, 10 minutes for queueing,
|
||||
20 minutes of cleanup, and $1.50 for S3, disk and IPv4 costs. Preflight rejects prior spend at or above
|
||||
$15, or an attempt whose reserve would exceed the cumulative $20 budget.
|
||||
These estimates and checks are not an AWS hard spending cap or an AWS bill.
|
||||
|
||||
## Workload and validation
|
||||
|
||||
- A 10-physical-table × 50-logical-table smoke runs first. Its 500 logical tables
|
||||
cover all 298 string tag columns before the larger workload starts.
|
||||
- The default full run uses 10 physical tables and 100,000 logical tables, with
|
||||
32 rows per logical table: 3.2 million rows. Each logical schema has 10
|
||||
overlapping tags, a timestamp and a DOUBLE; schemas vary between tables.
|
||||
- Source and target use local File storage on the temporary NVMe disk. Only
|
||||
snapshots use S3. Source validation finishes before the source process stops;
|
||||
restoration then starts a separate target process.
|
||||
- Export uses `--pack-metric-data`, parallelism 4, chunk parallelism 1, and one
|
||||
complete time chunk. Import uses task parallelism 1 and automatic batch DDL.
|
||||
- Every source and restored logical table is checked for membership, schema,
|
||||
DDL and typed values including NULLs. Target placeholders force table ID
|
||||
remapping; successful batch DDL and removal of import resume state are checked.
|
||||
|
||||
## Results and cleanup
|
||||
|
||||
The job summary reports complete export, structural verify and import durations;
|
||||
data generation and full data validation are timed separately. Evidence artifacts
|
||||
are credential-scanned and retained for 14 days. Keep conclusions limited to
|
||||
this version's absolute AWS results; this workflow does not compare old versions.
|
||||
|
||||
Artifacts include sampled CPU/RSS, host memory, swap/OOM counters, server metrics,
|
||||
CLI logs and S3 object/byte inventories. Configured swap is not RAM. A phase with
|
||||
new swap-out or OOM fails and retains its evidence. OpenDAL operation counters
|
||||
are not complete AWS HTTP request counts: multipart calls, retries and CLI traffic
|
||||
are not fully covered. Object inventory bytes are not transferred bytes.
|
||||
|
||||
A separate GitHub-hosted teardown job starts when provisioning finishes and polls
|
||||
the benchmark job. Its deadline includes queue time: allocation start plus 45
|
||||
minutes, the benchmark timeout and 10 minutes. Completion, cancellation, an API
|
||||
error or the deadline leads to cleanup. Failed provisioning skips the wait.
|
||||
It terminates the EC2 instance and its disks, unregisters the runner, deletes every
|
||||
object listed under the run's unique prefix and verifies the listing is empty. Like
|
||||
the object store WAL benchmark, it needs only list, put, get and delete on the CI
|
||||
bucket. It does not abort incomplete multipart uploads or remove noncurrent object
|
||||
versions; those depend on the bucket's lifecycle configuration. Check teardown completion before another
|
||||
attempt. A deadline may leave the GitHub benchmark job queued after its resources
|
||||
are removed; cancel that workflow manually. If cleanup fails, rerun the teardown
|
||||
job only; it keeps the original attempt's resource identity and deadline. A new benchmark needs a new dispatch because teardown
|
||||
removes its runner. The daily EC2 janitor is only a fallback and does not clean S3.
|
||||
+494
-23
@@ -27,11 +27,14 @@ pub use state::{
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::hash_map::Entry;
|
||||
use std::collections::{BTreeMap, BinaryHeap, HashMap};
|
||||
use std::mem::size_of;
|
||||
use std::sync::Arc;
|
||||
|
||||
use datafusion::error::{DataFusionError, Result};
|
||||
use datatypes::arrow::array::Float64Array;
|
||||
use datatypes::series::MetricName;
|
||||
use datatypes::series::samples::{SampleRef, Series, SeriesSchema};
|
||||
use datatypes::series::samples::{SampleRef, Samples, Series, SeriesSchema};
|
||||
use datatypes::series::time::{GridSpan, TimeAxis};
|
||||
|
||||
use crate::go_format::format_float_f;
|
||||
use crate::grid_series::{GridSeriesBuilder, external};
|
||||
@@ -97,6 +100,7 @@ pub struct FloatAggregator {
|
||||
labels: Vec<Arc<str>>,
|
||||
group_index: HashMap<LabelKey, usize>,
|
||||
groups: Vec<Group>,
|
||||
retained: usize,
|
||||
}
|
||||
|
||||
impl FloatAggregator {
|
||||
@@ -104,25 +108,78 @@ impl FloatAggregator {
|
||||
schema: &Arc<SeriesSchema>,
|
||||
op: AggregateOp,
|
||||
grouping: Grouping<'_>,
|
||||
) -> Result<Self> {
|
||||
Self::try_new_budgeted(schema, op, grouping, |_| Ok(()))
|
||||
}
|
||||
|
||||
pub(crate) fn try_new_budgeted(
|
||||
schema: &Arc<SeriesSchema>,
|
||||
op: AggregateOp,
|
||||
grouping: Grouping<'_>,
|
||||
mut budget: impl FnMut(usize) -> Result<()>,
|
||||
) -> Result<Self> {
|
||||
validate_input(schema, &[], grouping)?;
|
||||
let (by, labels) = match grouping {
|
||||
Grouping::By(labels) => (true, labels),
|
||||
Grouping::Without(labels) => (false, labels),
|
||||
};
|
||||
let label_storage =
|
||||
grouping_bytes(grouping).saturating_add(labels_capacity_bytes(grouping));
|
||||
budget(label_storage)?;
|
||||
let mut owned_labels: Vec<Arc<str>> = Vec::new();
|
||||
owned_labels
|
||||
.try_reserve_exact(labels.len())
|
||||
.map_err(|error| {
|
||||
DataFusionError::Execution(format!(
|
||||
"cannot reserve aggregate grouping labels: {error}"
|
||||
))
|
||||
})?;
|
||||
owned_labels.extend(labels.iter().map(|label| Arc::from(*label)));
|
||||
let retained = owned_labels
|
||||
.capacity()
|
||||
.saturating_mul(size_of::<Arc<str>>())
|
||||
.saturating_add(owned_labels.iter().fold(0usize, |bytes, label| {
|
||||
bytes.saturating_add(label.len() + 2 * size_of::<usize>())
|
||||
}));
|
||||
Ok(Self {
|
||||
schema: schema.clone(),
|
||||
op,
|
||||
by,
|
||||
labels: labels.iter().map(|label| Arc::from(*label)).collect(),
|
||||
labels: owned_labels,
|
||||
group_index: HashMap::new(),
|
||||
groups: Vec::new(),
|
||||
retained,
|
||||
})
|
||||
}
|
||||
|
||||
/// Folds `series` into its group. A series that fails validation leaves the state unchanged.
|
||||
///
|
||||
/// A budget refusal is not a validation failure: the fold may have opened a group or filled
|
||||
/// blocks before the allocation it could not admit, so a failed [`Self::update_budgeted`]
|
||||
/// leaves the state half folded and its retained figure stale. The runtime stops the execution
|
||||
/// and drops the aggregator on that error, which is why no rollback is kept; a caller holding
|
||||
/// one must drop it rather than fold into it again.
|
||||
pub fn update(&mut self, series: &Series) -> Result<()> {
|
||||
self.update_budgeted(series, |_| Ok(()))
|
||||
}
|
||||
|
||||
pub(crate) fn update_budgeted(
|
||||
&mut self,
|
||||
series: &Series,
|
||||
mut budget: impl FnMut(usize) -> Result<()>,
|
||||
) -> Result<()> {
|
||||
validate_float_series(&self.schema, series)?;
|
||||
let existing_bytes = self.state_bytes();
|
||||
let grouping_workspace = grouping_workspace_bytes(&series.metric);
|
||||
let mut workspace_bytes =
|
||||
grouping_workspace.saturating_add(pending_bytes(self.schema.fields.len()));
|
||||
budget(existing_bytes.saturating_add(workspace_bytes))?;
|
||||
workspace_bytes = workspace_bytes.saturating_add(
|
||||
self.labels
|
||||
.len()
|
||||
.saturating_mul(std::mem::size_of::<&str>()),
|
||||
);
|
||||
budget(existing_bytes.saturating_add(workspace_bytes))?;
|
||||
let labels: Vec<&str> = self.labels.iter().map(|label| &**label).collect();
|
||||
let grouping = if self.by {
|
||||
Grouping::By(&labels)
|
||||
@@ -130,39 +187,210 @@ impl FloatAggregator {
|
||||
Grouping::Without(&labels)
|
||||
};
|
||||
let metric = grouping.apply(&series.metric)?;
|
||||
let index = match self.group_index.entry(label_key(&metric)) {
|
||||
let key_bytes = metric
|
||||
.tags()
|
||||
.len()
|
||||
.saturating_add(1)
|
||||
.saturating_mul(std::mem::size_of::<(Arc<str>, Arc<str>)>());
|
||||
workspace_bytes = workspace_bytes
|
||||
.saturating_add(grouping_workspace_bytes(&metric))
|
||||
.saturating_add(key_bytes);
|
||||
budget(existing_bytes.saturating_add(workspace_bytes))?;
|
||||
let key = label_key(&metric);
|
||||
let key_retained = key
|
||||
.capacity()
|
||||
.saturating_mul(size_of::<(Arc<str>, Arc<str>)>());
|
||||
let new_group = !self.group_index.contains_key(&key);
|
||||
let group_capacity = self.groups.capacity();
|
||||
let map_capacity = self.group_index.capacity();
|
||||
let old_map_backing = hash_map_backing_bytes(map_capacity);
|
||||
if new_group {
|
||||
let next_group_capacity = if self.groups.len() == group_capacity {
|
||||
group_capacity.saturating_mul(2).max(4)
|
||||
} else {
|
||||
group_capacity
|
||||
};
|
||||
let group_realloc_bytes = if next_group_capacity > group_capacity {
|
||||
next_group_capacity.saturating_mul(size_of::<Group>())
|
||||
} else {
|
||||
0
|
||||
};
|
||||
budget(
|
||||
existing_bytes
|
||||
.saturating_add(metric_bytes(&metric))
|
||||
.saturating_add(key_bytes)
|
||||
// A realloc that cannot grow in place keeps both buffers alive while it
|
||||
// copies, so a growing step admits the old buffer beside the new one --
|
||||
// the same reason the map below admits both tables. The old buffer alone
|
||||
// is already in `existing_bytes`; what is added here is the new one.
|
||||
.saturating_add(group_realloc_bytes)
|
||||
.saturating_add(if self.group_index.len() == map_capacity {
|
||||
hash_map_backing_bytes(next_hash_capacity(map_capacity))
|
||||
} else {
|
||||
0
|
||||
})
|
||||
.saturating_add(workspace_bytes),
|
||||
)?;
|
||||
}
|
||||
let group_capacity_growth = if new_group && self.groups.len() == group_capacity {
|
||||
group_capacity.saturating_mul(2).max(4)
|
||||
} else {
|
||||
group_capacity
|
||||
};
|
||||
if new_group && group_capacity_growth > group_capacity {
|
||||
self.groups.reserve(1);
|
||||
}
|
||||
if new_group && self.group_index.len() == self.group_index.capacity() {
|
||||
self.group_index.reserve(1);
|
||||
}
|
||||
let index = match self.group_index.entry(key) {
|
||||
Entry::Occupied(entry) => *entry.get(),
|
||||
Entry::Vacant(entry) => {
|
||||
self.groups.push(Group {
|
||||
metric,
|
||||
metric: metric.clone(),
|
||||
blocks: BTreeMap::new(),
|
||||
});
|
||||
*entry.insert(self.groups.len() - 1)
|
||||
let index = self.groups.len() - 1;
|
||||
entry.insert(index);
|
||||
index
|
||||
}
|
||||
};
|
||||
self.groups[index].accumulate(series, self.op);
|
||||
if new_group {
|
||||
self.retained = self
|
||||
.retained
|
||||
.saturating_add(metric_bytes(&metric))
|
||||
.saturating_add(key_retained)
|
||||
.saturating_add(
|
||||
self.groups
|
||||
.capacity()
|
||||
.saturating_sub(group_capacity)
|
||||
.saturating_mul(size_of::<Group>()),
|
||||
)
|
||||
.saturating_add(
|
||||
hash_map_backing_bytes(self.group_index.capacity())
|
||||
.saturating_sub(old_map_backing),
|
||||
);
|
||||
}
|
||||
drop(labels);
|
||||
drop(metric);
|
||||
self.retained = self.groups[index].accumulate(
|
||||
series,
|
||||
self.op,
|
||||
&mut budget,
|
||||
self.retained,
|
||||
self.schema.fields.len(),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Number of groups currently retained by this fold.
|
||||
pub(crate) fn groups(&self) -> usize {
|
||||
self.groups.len()
|
||||
}
|
||||
|
||||
/// Bytes reserved for this fold, using bounded allowances for its maps and group payloads.
|
||||
pub(crate) fn state_bytes(&self) -> usize {
|
||||
self.retained
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn recounted_state_bytes(&self) -> usize {
|
||||
let groups = self.groups.iter().fold(0usize, |total, group| {
|
||||
total
|
||||
.saturating_add(metric_bytes(&group.metric))
|
||||
.saturating_add(
|
||||
group
|
||||
.blocks
|
||||
.len()
|
||||
.saturating_mul(BLOCK_POINTS)
|
||||
.saturating_mul(self.schema.fields.len())
|
||||
.saturating_mul(std::mem::size_of::<FloatState>()),
|
||||
)
|
||||
.saturating_add(btree_backing_bytes(group.blocks.len()))
|
||||
});
|
||||
groups
|
||||
.saturating_add(
|
||||
self.labels
|
||||
.capacity()
|
||||
.saturating_mul(std::mem::size_of::<Arc<str>>()),
|
||||
)
|
||||
.saturating_add(hash_map_backing_bytes(self.group_index.capacity()))
|
||||
.saturating_add(self.group_index.keys().fold(0usize, |bytes, key| {
|
||||
bytes.saturating_add(
|
||||
key.capacity()
|
||||
.saturating_mul(size_of::<(Arc<str>, Arc<str>)>()),
|
||||
)
|
||||
}))
|
||||
.saturating_add(
|
||||
self.labels
|
||||
.iter()
|
||||
.map(|label| label.len() + 2 * std::mem::size_of::<usize>())
|
||||
.sum::<usize>(),
|
||||
)
|
||||
.saturating_add(
|
||||
self.groups
|
||||
.capacity()
|
||||
.saturating_mul(std::mem::size_of::<Group>()),
|
||||
)
|
||||
}
|
||||
|
||||
/// The result of every group, in the order the groups first appeared.
|
||||
pub fn finish(self) -> Result<Vec<Series>> {
|
||||
finish_groups(self.schema, self.op, self.groups)
|
||||
self.finish_budgeted(|_| Ok(()))
|
||||
}
|
||||
|
||||
pub(crate) fn finish_budgeted(
|
||||
self,
|
||||
budget: impl FnMut(usize) -> Result<()>,
|
||||
) -> Result<Vec<Series>> {
|
||||
let state = self.state_bytes();
|
||||
finish_groups_budgeted(self.schema, self.op, self.groups, state, budget)
|
||||
}
|
||||
}
|
||||
|
||||
/// Turns the state of `groups` into result series, in group order.
|
||||
fn finish_groups(
|
||||
/// Turns the state of `groups` into result series, in group order, admitting as it builds.
|
||||
fn finish_groups_budgeted(
|
||||
schema: Arc<SeriesSchema>,
|
||||
op: AggregateOp,
|
||||
groups: Vec<Group>,
|
||||
state: usize,
|
||||
mut budget: impl FnMut(usize) -> Result<()>,
|
||||
) -> Result<Vec<Series>> {
|
||||
let width = schema.fields.len();
|
||||
let group_count = groups.len();
|
||||
if group_count == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let groups = groups.into_iter();
|
||||
let mut builder = GridSeriesBuilder::try_new(schema)?;
|
||||
let mut output = Vec::with_capacity(groups.len());
|
||||
let mut row = Vec::with_capacity(width);
|
||||
let output_capacity_bytes = group_count.saturating_mul(std::mem::size_of::<Series>());
|
||||
let row_capacity_bytes = width.saturating_mul(std::mem::size_of::<Option<f64>>());
|
||||
let base = state
|
||||
.saturating_add(output_capacity_bytes)
|
||||
.saturating_add(row_capacity_bytes);
|
||||
budget(base)?;
|
||||
let mut output = Vec::new();
|
||||
output.try_reserve_exact(group_count).map_err(|error| {
|
||||
DataFusionError::Execution(format!("cannot reserve aggregate outputs: {error}"))
|
||||
})?;
|
||||
let mut row = Vec::new();
|
||||
row.try_reserve_exact(width).map_err(|error| {
|
||||
DataFusionError::Execution(format!("cannot reserve aggregate row: {error}"))
|
||||
})?;
|
||||
let mut completed_output_bytes = 0usize;
|
||||
for group in groups {
|
||||
// One builder serves every group, so each result series starts explicitly.
|
||||
builder.begin_series();
|
||||
let (rows, runs) = group_rows_runs(&group, width);
|
||||
let finish_bytes = finish_working_bytes(rows, runs, width);
|
||||
budget(
|
||||
state
|
||||
.saturating_add(output_capacity_bytes)
|
||||
.saturating_add(row_capacity_bytes)
|
||||
.saturating_add(completed_output_bytes)
|
||||
.saturating_add(finish_bytes),
|
||||
)?;
|
||||
builder.reserve_rows(rows)?;
|
||||
for (&block, states) in &group.blocks {
|
||||
// The last block can extend past the grid; only points with a sample are pushed.
|
||||
for (offset, point) in states.chunks_exact(width).enumerate() {
|
||||
@@ -174,11 +402,89 @@ fn finish_groups(
|
||||
builder.push(block * BLOCK_POINTS + offset, &row)?;
|
||||
}
|
||||
}
|
||||
output.extend(builder.finish(group.metric)?);
|
||||
let finished = builder.finish(group.metric)?;
|
||||
completed_output_bytes = completed_output_bytes.saturating_add(
|
||||
finished
|
||||
.iter()
|
||||
.map(crate::grid_series::series_bytes)
|
||||
.sum::<usize>(),
|
||||
);
|
||||
output.extend(finished);
|
||||
budget(
|
||||
state
|
||||
.saturating_add(output_capacity_bytes)
|
||||
.saturating_add(row_capacity_bytes)
|
||||
.saturating_add(completed_output_bytes),
|
||||
)?;
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
/// The result of every group, without a budget: the entry the partial-state merger takes, which
|
||||
/// does not hold a query pool to admit against.
|
||||
fn finish_groups(
|
||||
schema: Arc<SeriesSchema>,
|
||||
op: AggregateOp,
|
||||
groups: Vec<Group>,
|
||||
) -> Result<Vec<Series>> {
|
||||
finish_groups_budgeted(schema, op, groups, 0, |_| Ok(()))
|
||||
}
|
||||
|
||||
fn group_rows_runs(group: &Group, width: usize) -> (usize, usize) {
|
||||
if width == 0 {
|
||||
return (0, 0);
|
||||
}
|
||||
let mut rows = 0usize;
|
||||
let mut runs = 0usize;
|
||||
let mut previous: Option<usize> = None;
|
||||
for (&block, states) in &group.blocks {
|
||||
for (offset, point) in states.chunks_exact(width).enumerate() {
|
||||
if point.iter().any(|state| state.seen) {
|
||||
let index = block.saturating_mul(BLOCK_POINTS).saturating_add(offset);
|
||||
rows = rows.saturating_add(1);
|
||||
if previous.is_none_or(|previous| previous.checked_add(1) != Some(index)) {
|
||||
runs = runs.saturating_add(1);
|
||||
}
|
||||
previous = Some(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
(rows, runs)
|
||||
}
|
||||
|
||||
fn finish_working_bytes(rows: usize, runs: usize, width: usize) -> usize {
|
||||
let values = rows
|
||||
.saturating_mul(width)
|
||||
.saturating_mul(std::mem::size_of::<f64>());
|
||||
let validity_bytes = rows
|
||||
.saturating_add(7)
|
||||
.checked_div(8)
|
||||
.unwrap_or(0)
|
||||
.div_ceil(64)
|
||||
.saturating_mul(64);
|
||||
let fields = crate::grid_series::field_builder_bytes(width).saturating_add(
|
||||
width.saturating_mul(
|
||||
std::mem::size_of::<Float64Array>()
|
||||
.saturating_add(std::mem::size_of::<Samples>())
|
||||
.saturating_add(std::mem::size_of::<usize>()),
|
||||
),
|
||||
);
|
||||
let indices = rows.saturating_mul(std::mem::size_of::<usize>());
|
||||
let spans = runs.saturating_mul(std::mem::size_of::<GridSpan>());
|
||||
let axis_conversion = runs
|
||||
.saturating_mul(
|
||||
2 * std::mem::size_of::<GridSpan>() + std::mem::size_of::<Arc<[GridSpan]>>(),
|
||||
)
|
||||
.saturating_add(rows.saturating_mul(std::mem::size_of::<usize>()));
|
||||
values
|
||||
.saturating_add(width.saturating_mul(validity_bytes))
|
||||
.saturating_add(fields)
|
||||
.saturating_add(indices)
|
||||
.saturating_add(spans)
|
||||
.saturating_add(axis_conversion)
|
||||
.saturating_add(std::mem::size_of::<Series>() + std::mem::size_of::<TimeAxis>())
|
||||
}
|
||||
|
||||
/// `quantile(q, v)`, with `q` read at each grid point from an unlabelled single-field scalar on
|
||||
/// the same grid, as Prometheus evaluates the parameter at every step. A point where `q` has no
|
||||
/// value has no result; `q` outside `[0, 1]` or NaN gives what Prometheus' `quantile` gives.
|
||||
@@ -438,6 +744,105 @@ impl Grouping<'_> {
|
||||
|
||||
type LabelKey = Vec<(Arc<str>, Arc<str>)>;
|
||||
|
||||
fn metric_bytes(metric: &MetricName) -> usize {
|
||||
metric
|
||||
.name()
|
||||
.map_or(0, str::len)
|
||||
.saturating_add(
|
||||
metric
|
||||
.tags()
|
||||
.iter()
|
||||
.map(|tag| tag.name.len() + tag.value.len())
|
||||
.sum::<usize>(),
|
||||
)
|
||||
.saturating_add(
|
||||
metric
|
||||
.tags()
|
||||
.len()
|
||||
.saturating_mul(std::mem::size_of::<(Arc<str>, Arc<str>)>()),
|
||||
)
|
||||
.saturating_add(std::mem::size_of::<MetricName>())
|
||||
}
|
||||
|
||||
fn grouping_workspace_bytes(metric: &MetricName) -> usize {
|
||||
metric_bytes(metric)
|
||||
.saturating_add(
|
||||
metric
|
||||
.tags()
|
||||
.len()
|
||||
.saturating_add(1)
|
||||
.saturating_mul(std::mem::size_of::<(Arc<str>, Arc<str>)>()),
|
||||
)
|
||||
.saturating_add(std::mem::size_of::<LabelKey>())
|
||||
}
|
||||
|
||||
/// Conservative backing estimate for Rust 1.98.1's insert-only `HashMap` (hashbrown 0.17.1).
|
||||
/// Bucket count follows its 7/8 load factor; entries, one control byte per bucket and the
|
||||
/// 16-byte control-group tail are counted. This models requested table storage, not allocator slack.
|
||||
/// An upper bound on the bytes the group index holds for `capacity` entries.
|
||||
///
|
||||
/// The table keeps its entries in a power-of-two number of buckets below `16/7` of the usable
|
||||
/// capacity, with one control byte beside each bucket. Three buckets per entry therefore covers the
|
||||
/// table whatever exact growth the map uses, which is what keeps this a bound rather than a model of
|
||||
/// one implementation: a toolchain that changes how the map grows cannot make it under-count.
|
||||
fn hash_map_backing_bytes(capacity: usize) -> usize {
|
||||
capacity
|
||||
.saturating_mul(3)
|
||||
.saturating_mul(size_of::<(LabelKey, usize)>() + 1)
|
||||
}
|
||||
|
||||
/// An upper bound on the capacity a map holding `capacity` entries grows to.
|
||||
fn next_hash_capacity(capacity: usize) -> usize {
|
||||
capacity.saturating_mul(2).max(7)
|
||||
}
|
||||
|
||||
/// An upper bound on the bytes one B-tree node holds.
|
||||
///
|
||||
/// A node keeps at most eleven key/value pairs and twelve child pointers, plus a header of a few
|
||||
/// words; the leaf form is smaller than that. The header allowance is generous on purpose, so the
|
||||
/// bound does not depend on how a toolchain lays the node out.
|
||||
fn btree_node_bytes() -> usize {
|
||||
11 * size_of::<(usize, Box<[FloatState]>)>() + 16 * size_of::<usize>()
|
||||
}
|
||||
|
||||
/// An upper bound on the bytes a B-tree of `entries` holds.
|
||||
///
|
||||
/// Every node but the root keeps at least five entries, so `entries / 5` nodes is enough to hold
|
||||
/// them; a tree of one entry is one node.
|
||||
fn btree_backing_bytes(entries: usize) -> usize {
|
||||
if entries == 0 {
|
||||
return 0;
|
||||
}
|
||||
entries.div_ceil(5).saturating_mul(btree_node_bytes())
|
||||
}
|
||||
|
||||
fn pending_bytes(fields: usize) -> usize {
|
||||
working_set_bytes(fields, BLOCK_POINTS)
|
||||
}
|
||||
|
||||
fn working_set_bytes(fields: usize, pending_capacity: usize) -> usize {
|
||||
pending_capacity
|
||||
.saturating_mul(std::mem::size_of::<(usize, f64)>())
|
||||
.saturating_add(fields.saturating_mul(std::mem::size_of::<Option<f64>>() + 1))
|
||||
}
|
||||
|
||||
fn labels_capacity_bytes(grouping: Grouping<'_>) -> usize {
|
||||
let labels = match grouping {
|
||||
Grouping::By(labels) | Grouping::Without(labels) => labels,
|
||||
};
|
||||
labels.len().saturating_mul(std::mem::size_of::<Arc<str>>())
|
||||
}
|
||||
|
||||
fn grouping_bytes(grouping: Grouping<'_>) -> usize {
|
||||
let labels = match grouping {
|
||||
Grouping::By(labels) | Grouping::Without(labels) => labels,
|
||||
};
|
||||
labels
|
||||
.iter()
|
||||
.map(|label| label.len() + std::mem::size_of::<Arc<str>>())
|
||||
.sum::<usize>()
|
||||
}
|
||||
|
||||
fn label_key(metric: &MetricName) -> LabelKey {
|
||||
let mut labels = Vec::with_capacity(metric.tags().len() + 1);
|
||||
if let Some(name) = metric.name() {
|
||||
@@ -463,9 +868,29 @@ struct Group {
|
||||
blocks: BTreeMap<usize, Box<[FloatState]>>,
|
||||
}
|
||||
|
||||
/// Which field of which layout a fold is writing, and how it folds it.
|
||||
///
|
||||
/// One value carried through the per-block work instead of three separate arguments: the three
|
||||
/// never vary independently within one series, and the block loop is the only place that reads
|
||||
/// them.
|
||||
#[derive(Clone, Copy)]
|
||||
struct FieldFold {
|
||||
width: usize,
|
||||
field_index: usize,
|
||||
op: AggregateOp,
|
||||
}
|
||||
|
||||
impl Group {
|
||||
fn accumulate(&mut self, series: &Series, op: AggregateOp) {
|
||||
let width = series.fields.len();
|
||||
fn accumulate(
|
||||
&mut self,
|
||||
series: &Series,
|
||||
op: AggregateOp,
|
||||
budget: &mut impl FnMut(usize) -> Result<()>,
|
||||
mut retained: usize,
|
||||
width: usize,
|
||||
) -> Result<usize> {
|
||||
let pending_capacity_bytes = BLOCK_POINTS.saturating_mul(size_of::<(usize, f64)>());
|
||||
budget(retained.saturating_add(pending_capacity_bytes))?;
|
||||
let mut pending = Vec::with_capacity(BLOCK_POINTS);
|
||||
for (field_index, field) in series.fields.iter().enumerate() {
|
||||
// The axis says which absolute positions the field holds and the field iterator
|
||||
@@ -475,35 +900,81 @@ impl Group {
|
||||
let mut block = usize::MAX;
|
||||
for ((index, _), sample) in series.time.positions().zip(field.iter()) {
|
||||
if index / BLOCK_POINTS != block {
|
||||
self.update_block(block, width, field_index, op, &mut pending);
|
||||
retained = self.update_block(
|
||||
block,
|
||||
FieldFold {
|
||||
width,
|
||||
field_index,
|
||||
op,
|
||||
},
|
||||
&mut pending,
|
||||
budget,
|
||||
retained,
|
||||
)?;
|
||||
block = index / BLOCK_POINTS;
|
||||
}
|
||||
if let SampleRef::Float(sample) = sample {
|
||||
pending.push((index % BLOCK_POINTS, sample));
|
||||
}
|
||||
}
|
||||
self.update_block(block, width, field_index, op, &mut pending);
|
||||
retained = self.update_block(
|
||||
block,
|
||||
FieldFold {
|
||||
width,
|
||||
field_index,
|
||||
op,
|
||||
},
|
||||
&mut pending,
|
||||
budget,
|
||||
retained,
|
||||
)?;
|
||||
}
|
||||
Ok(retained)
|
||||
}
|
||||
|
||||
/// Applies one field's samples within a block, so a block costs one lookup per field run.
|
||||
fn update_block(
|
||||
&mut self,
|
||||
block: usize,
|
||||
width: usize,
|
||||
field_index: usize,
|
||||
op: AggregateOp,
|
||||
fold: FieldFold,
|
||||
pending: &mut Vec<(usize, f64)>,
|
||||
) {
|
||||
budget: &mut impl FnMut(usize) -> Result<()>,
|
||||
retained: usize,
|
||||
) -> Result<usize> {
|
||||
if pending.is_empty() {
|
||||
return;
|
||||
return Ok(retained);
|
||||
}
|
||||
let new_block = !self.blocks.contains_key(&block);
|
||||
let block_bytes = if new_block {
|
||||
BLOCK_POINTS
|
||||
.saturating_mul(fold.width)
|
||||
.saturating_mul(std::mem::size_of::<FloatState>())
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let backing_delta = if new_block {
|
||||
btree_backing_bytes(self.blocks.len() + 1)
|
||||
.saturating_sub(btree_backing_bytes(self.blocks.len()))
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if new_block {
|
||||
budget(
|
||||
retained
|
||||
.saturating_add(backing_delta)
|
||||
.saturating_add(block_bytes)
|
||||
.saturating_add(pending.capacity().saturating_mul(size_of::<(usize, f64)>())),
|
||||
)?;
|
||||
}
|
||||
let states = self.blocks.entry(block).or_insert_with(|| {
|
||||
vec![FloatState::default(); BLOCK_POINTS * width].into_boxed_slice()
|
||||
vec![FloatState::default(); BLOCK_POINTS * fold.width].into_boxed_slice()
|
||||
});
|
||||
for (offset, sample) in pending.drain(..) {
|
||||
states[offset * width + field_index].update(op, sample);
|
||||
states[offset * fold.width + fold.field_index].update(fold.op, sample);
|
||||
}
|
||||
Ok(retained
|
||||
.saturating_add(backing_delta)
|
||||
.saturating_add(block_bytes))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use datafusion::error::DataFusionError;
|
||||
use datatypes::arrow::array::{Float64Array, UInt32Array};
|
||||
use datatypes::series::Label;
|
||||
use datatypes::series::histogram::HistogramBatchBuilder;
|
||||
@@ -345,6 +346,27 @@ fn avg_compensation_matches_prometheus_bits() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_field_missing_and_another_nan_keep_row_nullability() {
|
||||
let schema = schema(&["missing", "value"]);
|
||||
let series = input(
|
||||
&schema,
|
||||
metric("m", &[]),
|
||||
0,
|
||||
&[
|
||||
vec![None, None, None],
|
||||
vec![Some(1.0), Some(f64::NAN), Some(2.0)],
|
||||
],
|
||||
);
|
||||
let output = aggregate_float(&schema, &[series], AggregateOp::Sum, Grouping::By(&[])).unwrap();
|
||||
assert_eq!(output.len(), 1);
|
||||
assert_eq!(values(&output[0], 0).1, vec![None, None, None]);
|
||||
let values = values(&output[0], 1).1;
|
||||
assert_eq!(values[0], Some(1.0));
|
||||
assert!(values[1].unwrap().is_nan());
|
||||
assert_eq!(values[2], Some(2.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fields_are_aggregated_independently() {
|
||||
// A field without input values at a row is null there, even for count and group; the row
|
||||
@@ -476,6 +498,301 @@ fn fields_are_aggregated_independently() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incremental_retained_bytes_match_full_recount_after_updates() {
|
||||
let schema = schema_with_grid_len(&["value", "other"], 256);
|
||||
let mut aggregator =
|
||||
FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&["zone", "rack"]))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
aggregator.state_bytes(),
|
||||
aggregator.recounted_state_bytes(),
|
||||
"initial state"
|
||||
);
|
||||
let updates = [
|
||||
input(
|
||||
&schema,
|
||||
metric("m", &[("zone", "a"), ("rack", "1")]),
|
||||
0,
|
||||
&[vec![Some(1.0)], vec![Some(2.0)]],
|
||||
),
|
||||
input(
|
||||
&schema,
|
||||
metric("m", &[("zone", "a"), ("rack", "1")]),
|
||||
64,
|
||||
&[vec![Some(3.0)], vec![Some(4.0)]],
|
||||
),
|
||||
input(
|
||||
&schema,
|
||||
metric("m", &[("zone", "a"), ("rack", "1")]),
|
||||
0,
|
||||
&[vec![Some(5.0)], vec![Some(6.0)]],
|
||||
),
|
||||
input(
|
||||
&schema,
|
||||
metric("m", &[("zone", "b"), ("rack", "2")]),
|
||||
128,
|
||||
&[vec![Some(7.0)], vec![Some(8.0)]],
|
||||
),
|
||||
];
|
||||
for (index, series) in updates.iter().enumerate() {
|
||||
let previous = aggregator.state_bytes();
|
||||
aggregator.update(series).unwrap();
|
||||
assert_eq!(
|
||||
aggregator.state_bytes(),
|
||||
aggregator.recounted_state_bytes(),
|
||||
"update {index}"
|
||||
);
|
||||
if index == 2 {
|
||||
assert_eq!(aggregator.state_bytes(), previous);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incremental_retained_bytes_match_full_recount_for_many_groups() {
|
||||
let schema = schema(&["value"]);
|
||||
let mut aggregator =
|
||||
FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&["id"])).unwrap();
|
||||
for id in 0..512 {
|
||||
let id = id.to_string();
|
||||
let series = input(
|
||||
&schema,
|
||||
metric("m", &[("id", &id)]),
|
||||
id.parse::<usize>().unwrap() % 64,
|
||||
&[vec![Some(1.0)]],
|
||||
);
|
||||
aggregator.update(&series).unwrap();
|
||||
assert_eq!(
|
||||
aggregator.state_bytes(),
|
||||
aggregator.recounted_state_bytes(),
|
||||
"group {id}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The unbudgeted entry point is the budgeted one with a no-op budget, so this compares one path
|
||||
/// with itself: it is a smoke check that the wrapper still runs, not evidence that the kernels are
|
||||
/// right. The numbers are checked against Prometheus in `aggregates_match_prometheus` and
|
||||
/// `stat_aggregates_match_prometheus`.
|
||||
#[test]
|
||||
fn every_supported_kind_runs_through_the_unbudgeted_entry_point() {
|
||||
let schema = schema(&["value"]);
|
||||
let series = input(&schema, metric("m", &[]), 0, &[vec![Some(1.0), Some(3.0)]]);
|
||||
for op in [
|
||||
AggregateOp::Sum,
|
||||
AggregateOp::Count,
|
||||
AggregateOp::Avg,
|
||||
AggregateOp::Min,
|
||||
AggregateOp::Max,
|
||||
AggregateOp::Group,
|
||||
AggregateOp::Stddev,
|
||||
AggregateOp::Stdvar,
|
||||
] {
|
||||
let mut plain = FloatAggregator::try_new(&schema, op, Grouping::By(&[])).unwrap();
|
||||
plain.update(&series).unwrap();
|
||||
let plain = plain.finish().unwrap();
|
||||
let mut budgeted =
|
||||
FloatAggregator::try_new_budgeted(&schema, op, Grouping::By(&[]), |_| Ok(())).unwrap();
|
||||
budgeted.update_budgeted(&series, |_| Ok(())).unwrap();
|
||||
let budgeted = budgeted.finish_budgeted(|_| Ok(())).unwrap();
|
||||
assert_eq!(
|
||||
values_of(&plain, 0, 128),
|
||||
values_of(&budgeted, 0, 128),
|
||||
"{op:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_updates_in_one_group_and_block_keep_one_block_charge() {
|
||||
let schema = schema(&["value"]);
|
||||
let first = input(&schema, metric("m", &[]), 0, &[vec![Some(1.0)]]);
|
||||
let second = input(&schema, metric("m", &[]), 1, &[vec![Some(3.0)]]);
|
||||
let mut aggregator =
|
||||
FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&[])).unwrap();
|
||||
aggregator.update(&first).unwrap();
|
||||
let retained = aggregator.state_bytes();
|
||||
aggregator.update(&second).unwrap();
|
||||
assert_eq!(aggregator.state_bytes(), retained);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adjacent_blocks_are_admitted_and_accounted_separately() {
|
||||
let schema = schema(&["value"]);
|
||||
let series = input(&schema, metric("m", &[]), 63, &[vec![Some(1.0), Some(2.0)]]);
|
||||
let mut aggregator =
|
||||
FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&[])).unwrap();
|
||||
let mut admissions = Vec::new();
|
||||
aggregator
|
||||
.update_budgeted(&series, |bytes| {
|
||||
admissions.push(bytes);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(aggregator.groups(), 1);
|
||||
assert_eq!(aggregator.groups[0].blocks.len(), 2);
|
||||
assert_eq!(
|
||||
admissions.last().copied(),
|
||||
Some(
|
||||
aggregator
|
||||
.state_bytes()
|
||||
.saturating_add(std::mem::size_of::<(usize, f64)>() * BLOCK_POINTS,)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rejected_block_allocation_does_not_insert_a_block() {
|
||||
let schema = schema(&["value"]);
|
||||
let series = input(&schema, metric("m", &[]), 0, &[vec![Some(1.0)]]);
|
||||
// The figure a new block's admission adds: the block's states and its pending buffer, on
|
||||
// top of whatever the fold already holds. Located by the value, not by which call reaches it,
|
||||
// so reordering the earlier admissions cannot make this test pass without testing anything.
|
||||
let block_states = BLOCK_POINTS
|
||||
.saturating_mul(schema.fields.len())
|
||||
.saturating_mul(std::mem::size_of::<FloatState>());
|
||||
let pending = BLOCK_POINTS * std::mem::size_of::<(usize, f64)>();
|
||||
let mut aggregator =
|
||||
FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&[])).unwrap();
|
||||
let mut refused = None;
|
||||
let result = aggregator.update_budgeted(&series, |bytes| {
|
||||
// Everything up to the block's own admission is admitted; only the block is refused.
|
||||
if bytes >= block_states.saturating_add(pending) {
|
||||
refused = Some(bytes);
|
||||
return Err(DataFusionError::Execution("block allocation budget".into()));
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"the block was admitted with no budget left"
|
||||
);
|
||||
let refused = refused.expect("the refusal did not come from a block admission");
|
||||
assert!(
|
||||
refused >= block_states.saturating_add(pending),
|
||||
"the refusal came from an admission smaller than a block"
|
||||
);
|
||||
assert_eq!(aggregator.groups(), 1);
|
||||
assert!(
|
||||
aggregator.groups[0].blocks.is_empty(),
|
||||
"a refused block was inserted anyway"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_map_growth_rejection_does_not_insert_the_new_group() {
|
||||
let schema = schema(&["value"]);
|
||||
let mut aggregator =
|
||||
FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&["i"])).unwrap();
|
||||
let first = input(&schema, metric("m", &[("i", "0")]), 0, &[vec![Some(1.0)]]);
|
||||
aggregator.update(&first).unwrap();
|
||||
let mut index = 1;
|
||||
while aggregator.group_index.len() < aggregator.group_index.capacity() {
|
||||
let label = index.to_string();
|
||||
let series = input(
|
||||
&schema,
|
||||
metric("m", &[("i", &label)]),
|
||||
0,
|
||||
&[vec![Some(1.0)]],
|
||||
);
|
||||
aggregator.update(&series).unwrap();
|
||||
index += 1;
|
||||
}
|
||||
let group_count = aggregator.groups();
|
||||
let old_state = aggregator.state_bytes();
|
||||
assert_eq!(aggregator.group_index.capacity(), group_count);
|
||||
|
||||
// A resize keeps the old table while it fills the new one, so the figure the growth has to
|
||||
// admit is the whole new table on top of the old state -- not the difference between them.
|
||||
// That is the property asserted here: the refusal threshold is the state plus a whole new
|
||||
// table, so the test fails if the admission ever drops to the delta.
|
||||
let new_table = hash_map_backing_bytes(next_hash_capacity(group_count));
|
||||
let threshold = old_state.saturating_add(new_table);
|
||||
let label = index.to_string();
|
||||
let series = input(
|
||||
&schema,
|
||||
metric("m", &[("i", &label)]),
|
||||
0,
|
||||
&[vec![Some(1.0)]],
|
||||
);
|
||||
let mut peak = 0;
|
||||
let result = aggregator.update_budgeted(&series, |bytes| {
|
||||
peak = peak.max(bytes);
|
||||
if bytes >= threshold {
|
||||
Err(DataFusionError::Execution("old+new map budget".into()))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
});
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"the map grew with no budget for both tables"
|
||||
);
|
||||
assert!(
|
||||
peak >= threshold,
|
||||
"the growth admitted {peak}, less than the old state plus a new table ({threshold})"
|
||||
);
|
||||
assert_eq!(aggregator.group_index.capacity(), group_count);
|
||||
assert_eq!(aggregator.groups(), group_count);
|
||||
assert_eq!(
|
||||
aggregator
|
||||
.groups
|
||||
.iter()
|
||||
.map(|group| group.blocks.len())
|
||||
.sum::<usize>(),
|
||||
group_count
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_rejection_after_first_group_is_budgeted_before_the_next() {
|
||||
let schema = schema(&["value"]);
|
||||
let mut aggregator =
|
||||
FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&["i"])).unwrap();
|
||||
for (label, value) in [("a", 1.0), ("b", 2.0)] {
|
||||
let series = input(
|
||||
&schema,
|
||||
metric("m", &[("i", label)]),
|
||||
0,
|
||||
&[vec![Some(value)]],
|
||||
);
|
||||
aggregator.update(&series).unwrap();
|
||||
}
|
||||
let mut calls = Vec::new();
|
||||
let result = aggregator.finish_budgeted(|bytes| {
|
||||
calls.push(bytes);
|
||||
if calls.len() == 4 {
|
||||
Err(DataFusionError::Execution("budget".into()))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
});
|
||||
assert!(result.is_err());
|
||||
assert_eq!(calls.len(), 4);
|
||||
assert!(calls[1] > calls[0], "first builder peak");
|
||||
assert!(calls[2] < calls[1], "first group settle");
|
||||
assert!(calls[3] > calls[2], "second builder peak");
|
||||
|
||||
// The base already includes the output Vec backing; later callbacks add only the
|
||||
// builder workspace or completed Series allocation.
|
||||
let builder_workspace = finish_working_bytes(1, 1, 1);
|
||||
let expected_output = aggregate_float(
|
||||
&schema,
|
||||
&[
|
||||
input(&schema, metric("m", &[("i", "a")]), 0, &[vec![Some(1.0)]]),
|
||||
input(&schema, metric("m", &[("i", "b")]), 0, &[vec![Some(2.0)]]),
|
||||
],
|
||||
AggregateOp::Sum,
|
||||
Grouping::By(&["i"]),
|
||||
)
|
||||
.unwrap();
|
||||
let completed_series = crate::grid_series::series_bytes(&expected_output[0]);
|
||||
assert_eq!(calls[1] - calls[0], builder_workspace);
|
||||
assert_eq!(calls[2] - calls[0], completed_series);
|
||||
assert_eq!(calls[3] - calls[2], builder_workspace);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_blocks_keep_absolute_indices_across_boundaries_and_gaps() {
|
||||
// A 512-point grid spans eight state blocks; inputs straddle a block edge, leave whole
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
mod absent;
|
||||
mod empty_metric;
|
||||
mod fused_series;
|
||||
mod histogram_fold;
|
||||
mod instant_manipulate;
|
||||
mod normalize;
|
||||
@@ -37,6 +38,7 @@ use datafusion::error::{DataFusionError, Result as DataFusionResult};
|
||||
use datafusion::logical_expr::{Expr, Extension, LogicalPlan};
|
||||
use datatypes::data_type::DataType as _;
|
||||
pub use empty_metric::{EmptyMetric, EmptyMetricExec, EmptyMetricStream, build_special_time_expr};
|
||||
pub use fused_series::{FusedSeries, FusedSeriesBinding, FusedSeriesExec, FusedSeriesStream};
|
||||
pub use histogram_fold::{
|
||||
HistogramFold, HistogramFoldExec, HistogramFoldOperation, HistogramFoldStream,
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+130
-20
@@ -19,12 +19,12 @@ use std::collections::hash_map::Entry;
|
||||
use std::sync::Arc;
|
||||
|
||||
use datafusion::arrow::array::builder::NullBufferBuilder;
|
||||
use datafusion::arrow::array::{Float64Array, UInt32Array};
|
||||
use datafusion::arrow::array::{Array, Float64Array, UInt32Array};
|
||||
use datafusion::error::{DataFusionError, Result};
|
||||
use datatypes::series::MetricName;
|
||||
use datatypes::series::histogram::HistogramBatchBuilder;
|
||||
use datatypes::series::histogram::{HistogramBatch, HistogramBatchBuilder};
|
||||
use datatypes::series::samples::{
|
||||
HistogramSamples, SampleRef, Samples, Series, SeriesFields, SeriesSchema,
|
||||
DenseSamplesRef, HistogramSamples, SampleRef, Samples, Series, SeriesFields, SeriesSchema,
|
||||
};
|
||||
use datatypes::series::time::{EvalGrid, GridSpan, TimeAxis};
|
||||
|
||||
@@ -35,6 +35,70 @@ pub(crate) mod reference;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub(crate) fn series_bytes(series: &Series) -> usize {
|
||||
let runs = series.time.runs().count();
|
||||
let segments: usize = series
|
||||
.fields
|
||||
.iter()
|
||||
.map(|field| field.segments().len())
|
||||
.sum();
|
||||
std::mem::size_of::<Series>()
|
||||
.saturating_add(std::mem::size_of::<TimeAxis>())
|
||||
.saturating_add(runs.saturating_mul(std::mem::size_of::<(usize, usize)>()))
|
||||
.saturating_add(
|
||||
segments.saturating_mul(std::mem::size_of::<Samples>() + std::mem::size_of::<usize>()),
|
||||
)
|
||||
.saturating_add(fields_bytes(&series.fields))
|
||||
.saturating_add(series.metric.name().map_or(0, str::len))
|
||||
.saturating_add(std::mem::size_of_val(series.metric.tags()))
|
||||
.saturating_add(
|
||||
series
|
||||
.metric
|
||||
.tags()
|
||||
.iter()
|
||||
.map(|tag| tag.name.len() + tag.value.len())
|
||||
.sum::<usize>(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn fields_bytes(fields: &SeriesFields) -> usize {
|
||||
fields
|
||||
.iter()
|
||||
.map(|field| {
|
||||
field.segments().iter().fold(0usize, |total, segment| {
|
||||
total.saturating_add(segment_bytes(segment))
|
||||
})
|
||||
})
|
||||
.fold(0usize, usize::saturating_add)
|
||||
}
|
||||
|
||||
/// The bytes one dense segment of samples holds.
|
||||
pub(crate) fn segment_bytes(segment: &Samples) -> usize {
|
||||
match segment.dense() {
|
||||
Some(DenseSamplesRef::Float(values)) => values.get_array_memory_size(),
|
||||
Some(DenseSamplesRef::Histogram(histograms)) => histograms
|
||||
.indices()
|
||||
.get_array_memory_size()
|
||||
.saturating_add(table_bytes(histograms.batch())),
|
||||
Some(DenseSamplesRef::Mixed(mixed)) => mixed
|
||||
.floats()
|
||||
.get_array_memory_size()
|
||||
.saturating_add(mixed.histograms().indices().get_array_memory_size())
|
||||
.saturating_add(table_bytes(mixed.histograms().batch())),
|
||||
// `segments` hands out dense segments only, so a nested chunk never reaches here.
|
||||
None => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// The bytes one histogram table holds: the batch struct and the arrays its columns view.
|
||||
pub(crate) fn table_bytes(table: &Arc<HistogramBatch>) -> usize {
|
||||
std::mem::size_of::<HistogramBatch>().saturating_add(table.as_arrow().get_array_memory_size())
|
||||
}
|
||||
|
||||
pub(crate) fn field_builder_bytes(width: usize) -> usize {
|
||||
width.saturating_mul(std::mem::size_of::<FieldBuilder>())
|
||||
}
|
||||
|
||||
/// Builds one grid series from rows pushed at absolute grid indices.
|
||||
///
|
||||
/// A push names one position and the value each field holds there. Rows are accumulated
|
||||
@@ -114,6 +178,36 @@ impl GridSeriesBuilder {
|
||||
})
|
||||
}
|
||||
|
||||
/// Reserves capacity for the exact number of valued rows in the next series.
|
||||
///
|
||||
/// Call after `begin_series` and before its first push. Existing public callers may continue
|
||||
/// to rely on normal growth; callers with a budget can reserve the concrete row capacity first.
|
||||
pub(crate) fn reserve_rows(&mut self, rows: usize) -> Result<()> {
|
||||
self.indices
|
||||
.try_reserve_exact(rows.saturating_sub(self.indices.len()))
|
||||
.map_err(|error| {
|
||||
DataFusionError::Execution(format!("cannot reserve grid row indices: {error}"))
|
||||
})?;
|
||||
if rows == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
if self.fields.is_empty() {
|
||||
self.fields = (0..self.schema.fields.len())
|
||||
.map(|_| FieldBuilder::new(rows))
|
||||
.collect();
|
||||
}
|
||||
for field in &mut self.fields {
|
||||
field
|
||||
.values
|
||||
.try_reserve_exact(rows.saturating_sub(field.values.len()))
|
||||
.map_err(|error| {
|
||||
DataFusionError::Execution(format!("cannot reserve grid values: {error}"))
|
||||
})?;
|
||||
field.nulls = NullBufferBuilder::new(rows);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Pushes the field values at absolute grid `index`; `None` is a missing value.
|
||||
///
|
||||
/// Indices must be strictly increasing; skipped indices are positions that hold no value.
|
||||
@@ -176,7 +270,7 @@ impl GridSeriesBuilder {
|
||||
return Ok(false);
|
||||
}
|
||||
if self.fields.is_empty() {
|
||||
self.fields = (0..len).map(|_| FieldBuilder::new()).collect();
|
||||
self.fields = (0..len).map(|_| FieldBuilder::new(0)).collect();
|
||||
}
|
||||
self.indices.push(index);
|
||||
Ok(true)
|
||||
@@ -239,24 +333,28 @@ impl GridSeriesBuilder {
|
||||
}
|
||||
let fields = std::mem::take(&mut self.fields);
|
||||
let indices = std::mem::take(&mut self.indices);
|
||||
let samples = fields
|
||||
.into_iter()
|
||||
.map(|mut field| {
|
||||
let floats = Float64Array::new(field.values.into(), field.nulls.finish());
|
||||
let Some(column) = field.histograms else {
|
||||
return Ok(Samples::from_float(floats));
|
||||
};
|
||||
let mut samples = Vec::new();
|
||||
samples.try_reserve_exact(fields.len()).map_err(|error| {
|
||||
DataFusionError::Execution(format!("cannot reserve grid field descriptions: {error}"))
|
||||
})?;
|
||||
for mut field in fields {
|
||||
let floats = Float64Array::new(field.values.into(), field.nulls.finish());
|
||||
let sample = if let Some(column) = field.histograms {
|
||||
let batch = column.batch.finish().map_err(external)?;
|
||||
let histograms =
|
||||
HistogramSamples::try_new(UInt32Array::from(column.indices), batch)
|
||||
.map_err(external)?;
|
||||
Samples::try_from_mixed(floats, histograms).map_err(external)
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
Samples::try_from_mixed(floats, histograms).map_err(external)?
|
||||
} else {
|
||||
Samples::from_float(floats)
|
||||
};
|
||||
samples.push(sample);
|
||||
}
|
||||
let fields = SeriesFields::try_from_vec(samples).map_err(external)?;
|
||||
// The runs are all the axis has to remember about the gaps, and the rows are already
|
||||
// compact, so no position holding no value is visited or stored.
|
||||
let spans = spans_of(&indices);
|
||||
let run_count = count_runs(&indices);
|
||||
let spans = spans_of(&indices, run_count)?;
|
||||
let time = TimeAxis::from_grid_spans(self.grid.clone(), spans).map_err(external)?;
|
||||
Series::try_new(&self.schema, metric, time, fields)
|
||||
.map(Some)
|
||||
@@ -265,8 +363,19 @@ impl GridSeriesBuilder {
|
||||
}
|
||||
|
||||
/// Cuts the retained indices into maximal runs of consecutive positions.
|
||||
fn spans_of(indices: &[usize]) -> Vec<GridSpan> {
|
||||
fn count_runs(indices: &[usize]) -> usize {
|
||||
indices
|
||||
.windows(2)
|
||||
.filter(|pair| pair[1] != pair[0] + 1)
|
||||
.count()
|
||||
.saturating_add(usize::from(!indices.is_empty()))
|
||||
}
|
||||
|
||||
fn spans_of(indices: &[usize], runs: usize) -> Result<Vec<GridSpan>> {
|
||||
let mut spans: Vec<GridSpan> = Vec::new();
|
||||
spans.try_reserve_exact(runs).map_err(|error| {
|
||||
DataFusionError::Execution(format!("cannot reserve grid spans: {error}"))
|
||||
})?;
|
||||
let mut start = 0;
|
||||
while let Some(&position) = indices.get(start) {
|
||||
let mut len = 1;
|
||||
@@ -276,14 +385,14 @@ fn spans_of(indices: &[usize]) -> Vec<GridSpan> {
|
||||
spans.push(GridSpan::new(position, len));
|
||||
start += len;
|
||||
}
|
||||
spans
|
||||
Ok(spans)
|
||||
}
|
||||
|
||||
impl FieldBuilder {
|
||||
fn new() -> Self {
|
||||
fn new(rows: usize) -> Self {
|
||||
Self {
|
||||
values: Vec::new(),
|
||||
nulls: NullBufferBuilder::new(0),
|
||||
nulls: NullBufferBuilder::new(rows),
|
||||
histograms: None,
|
||||
}
|
||||
}
|
||||
@@ -454,7 +563,8 @@ pub fn merge_same_labelset(schema: &Arc<SeriesSchema>, series: Vec<Series>) -> R
|
||||
// The runs are all the axis remembers about the holes, so a sparse merge stores one span
|
||||
// per run and no row for the positions between them.
|
||||
let time =
|
||||
TimeAxis::from_grid_spans(grid.clone(), spans_of(&positions)).map_err(external)?;
|
||||
TimeAxis::from_grid_spans(grid.clone(), spans_of(&positions, count_runs(&positions))?)
|
||||
.map_err(external)?;
|
||||
let fields = SeriesFields::try_from_vec(fields).map_err(external)?;
|
||||
// The first fragment names the output, as it always has: a label set ignores empty values,
|
||||
// so the metric of the first member is the one the merged series keeps.
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The run-time state of an aggregate node.
|
||||
//!
|
||||
//! An aggregate folds a whole vector into one series per group, so its node is the one that cannot
|
||||
//! answer as its input arrives: a group's value at a grid point is the fold of every value every
|
||||
//! member of the group has there, and the last member to arrive is the one that decides it. What the
|
||||
//! node keeps between chunks is therefore the fold's states and nothing else -- a chunk is folded as
|
||||
//! it arrives and dropped -- and it produces once, when the input ends.
|
||||
//!
|
||||
//! That is also why this runtime cannot tell an incomplete group from a complete one: it folds what
|
||||
//! it is handed. A [`AggregateStage::Complete`] aggregate is correct only if every contribution to
|
||||
//! each group lies within the streams this execution reads, which is the physical plan's job -- the
|
||||
//! caller that schedules scans and exchanges guarantees it, and a plan that ran a `Complete`
|
||||
//! aggregate once per partition and then combined the answers would be combining values that were
|
||||
//! each missing the other partitions' contributions. The stages that exist for that shape are
|
||||
//! [`AggregateStage::Partial`] and [`AggregateStage::Final`], which this runtime does not execute
|
||||
//! yet.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use datafusion::error::{DataFusionError, Result};
|
||||
use datatypes::series::samples::{Series, SeriesSchema};
|
||||
|
||||
use crate::aggregate::{AggregateOp, FloatAggregator, Grouping};
|
||||
use crate::evaluated::EvaluatedChunk;
|
||||
use crate::grid_series::series_bytes;
|
||||
use crate::map::validate_float_series;
|
||||
use crate::native::execute::{NodeBuffer, evaluated_bytes};
|
||||
use crate::native::metrics::NodePath;
|
||||
use crate::native::plan::{
|
||||
AggregateKind, AggregateStage, BoundAggregate, Grouping as BoundGrouping,
|
||||
};
|
||||
|
||||
/// One aggregate node's states, from the first chunk it is handed to the series it finishes with.
|
||||
pub(crate) struct AggregateRuntime {
|
||||
/// What the node computes, as the tree bound it: the kind it folds by and the stage it reports.
|
||||
op: BoundAggregate,
|
||||
/// The layout its input and output share: the fields every chunk has and the grid they are
|
||||
/// addressed on. A `Complete` aggregate neither adds nor drops a field, and it evaluates on its
|
||||
/// input's grid, so one schema describes both sides.
|
||||
schema: Arc<SeriesSchema>,
|
||||
/// The fold. `None` once [`Self::compute`] has taken it, which is also what says the node has
|
||||
/// nothing left to fold into.
|
||||
accumulator: Option<FloatAggregator>,
|
||||
/// Whether the fold has been finished and its series handed over. A node computes once: a second
|
||||
/// pull finds the series it queued rather than a fold that ran again.
|
||||
///
|
||||
/// The driver reads this to decide whether the node settled: an aggregate answers only once its
|
||||
/// fold has run, so a child with nothing left to hand over is not an answer.
|
||||
pub(crate) computed: bool,
|
||||
}
|
||||
|
||||
impl AggregateRuntime {
|
||||
/// What this node computes, for a rejection that has to say what it refused.
|
||||
///
|
||||
/// It names the kind this node computes, which distinguishes it from another node in the tree.
|
||||
fn describe(&self) -> String {
|
||||
format!("{:?}", self.op.kind)
|
||||
}
|
||||
|
||||
/// The states of a `Complete` aggregate over `schema`.
|
||||
///
|
||||
/// A stage other than `Complete` and a kind without a supported kernel are refused here, before a single
|
||||
/// chunk is folded: a node this runtime cannot compute must not start, because a partial answer
|
||||
/// would be one the caller never asked for. The rejection names what was found, so a reader can
|
||||
/// tell a tree this runtime cannot run from one it cannot run correctly.
|
||||
pub(crate) fn new(
|
||||
op: BoundAggregate,
|
||||
schema: Arc<SeriesSchema>,
|
||||
buffer: &mut NodeBuffer,
|
||||
) -> Result<Self> {
|
||||
if op.stage != AggregateStage::Complete {
|
||||
return Err(DataFusionError::NotImplemented(format!(
|
||||
"the native runtime does not execute a `{:?}` aggregate yet: a `Partial` aggregate \
|
||||
reports the state a later stage merges and a `Final` aggregate merges that state, \
|
||||
and only `Complete` -- every contribution inside one execution -- is computed here",
|
||||
op.stage
|
||||
)));
|
||||
}
|
||||
let kind = match op.kind {
|
||||
AggregateKind::Sum => AggregateOp::Sum,
|
||||
AggregateKind::Count => AggregateOp::Count,
|
||||
AggregateKind::Avg => AggregateOp::Avg,
|
||||
AggregateKind::Min => AggregateOp::Min,
|
||||
AggregateKind::Max => AggregateOp::Max,
|
||||
AggregateKind::Group => AggregateOp::Group,
|
||||
AggregateKind::Stddev => AggregateOp::Stddev,
|
||||
AggregateKind::Stdvar => AggregateOp::Stdvar,
|
||||
other => {
|
||||
return Err(DataFusionError::NotImplemented(format!(
|
||||
"the native runtime does not execute the `{other:?}` aggregate yet: only the supported `Complete` aggregate kinds are computed here"
|
||||
)));
|
||||
}
|
||||
};
|
||||
let name_count = match &op.group {
|
||||
BoundGrouping::By(labels) | BoundGrouping::Without(labels) => labels.len(),
|
||||
};
|
||||
let name_bytes = name_count.saturating_mul(std::mem::size_of::<&str>());
|
||||
buffer.grow(name_bytes)?;
|
||||
let names: Vec<&str> = match &op.group {
|
||||
BoundGrouping::By(labels) | BoundGrouping::Without(labels) => {
|
||||
labels.iter().map(|label| label.as_ref()).collect()
|
||||
}
|
||||
};
|
||||
let grouping = match &op.group {
|
||||
BoundGrouping::By(_) => Grouping::By(&names),
|
||||
BoundGrouping::Without(_) => Grouping::Without(&names),
|
||||
};
|
||||
let name_bytes = names.len().saturating_mul(std::mem::size_of::<&str>());
|
||||
let initial = FloatAggregator::try_new_budgeted(&schema, kind, grouping, |bytes| {
|
||||
buffer.resize(bytes.saturating_add(name_bytes))
|
||||
})?;
|
||||
drop(names);
|
||||
buffer.resize(initial.state_bytes())?;
|
||||
Ok(Self {
|
||||
op,
|
||||
schema,
|
||||
accumulator: Some(initial),
|
||||
computed: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// What folding `chunk` reserves while it is folded.
|
||||
///
|
||||
/// The chunk is not retained: it is folded into the states and dropped at the end of the call,
|
||||
/// so what this node holds afterwards is the fold and none of the input. The charge is therefore
|
||||
/// the chunk's own bytes -- the figure [`evaluated_bytes`] reports, which is one pass over the
|
||||
/// chunk and no state between chunks -- and not the `operand_chunk_bytes` figure a binary
|
||||
/// operand keeps, which exists because such an operand holds a series for the whole execution
|
||||
/// and so must count the slices of one buffer instead of the buffer once per chunk.
|
||||
pub(crate) fn pending(&self, chunk: &EvaluatedChunk) -> usize {
|
||||
evaluated_bytes(chunk)
|
||||
}
|
||||
|
||||
/// Folds one chunk into its group.
|
||||
///
|
||||
/// A chunk that closes its series closes nothing here: a group is not a series, so the last
|
||||
/// contribution of one member says nothing about the group. The fold keeps its states, which is
|
||||
/// what makes a group's answer its whole group's answer once the input ends.
|
||||
pub(crate) fn accumulate(
|
||||
&mut self,
|
||||
path: &NodePath,
|
||||
chunk: EvaluatedChunk,
|
||||
buffer: &mut NodeBuffer,
|
||||
pending: usize,
|
||||
) -> Result<()> {
|
||||
// `Series` keeps its fields public, so a chunk can be built by struct literal against
|
||||
// another schema; the states are indexed by absolute position on this node's grid, so a
|
||||
// chunk of another field count or another grid would be folded into the wrong places.
|
||||
validate_float_series(&self.schema, &chunk.series)
|
||||
.map_err(|error| folded_chunk(path, &self.describe(), error))?;
|
||||
let accumulator = self.accumulator.as_mut().ok_or_else(|| no_fold(path))?;
|
||||
// Keep the chunk covered while the fold decides whether it needs a new group or block.
|
||||
accumulator.update_budgeted(&chunk.series, |fold_bytes| {
|
||||
buffer.resize(pending.saturating_add(fold_bytes))
|
||||
})?;
|
||||
drop(chunk);
|
||||
buffer.resize(accumulator.state_bytes())
|
||||
}
|
||||
|
||||
/// The number of retained groups, for budgeting the output queue backing storage.
|
||||
pub(crate) fn groups(&self) -> usize {
|
||||
self.accumulator.as_ref().map_or(0, FloatAggregator::groups)
|
||||
}
|
||||
|
||||
/// Finishes the fold and returns one series per group, or `None` if the fold already ran.
|
||||
///
|
||||
/// `None` is what a node reports after it has queued its series: the caller reads them from the
|
||||
/// queue, exactly as it reads the chunks a binary node queued, so a second pull does not finish
|
||||
/// a fold whose states are gone.
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
path: &NodePath,
|
||||
buffer: &mut NodeBuffer,
|
||||
queue_bytes: usize,
|
||||
) -> Result<Option<Vec<Series>>> {
|
||||
if self.computed {
|
||||
return Ok(None);
|
||||
}
|
||||
if self.accumulator.is_none() {
|
||||
return Err(no_fold(path));
|
||||
}
|
||||
self.computed = true;
|
||||
let output = self
|
||||
.accumulator
|
||||
.take()
|
||||
.ok_or_else(|| no_fold(path))?
|
||||
.finish_budgeted(|total| buffer.resize(total.saturating_add(queue_bytes)))?;
|
||||
let output_bytes = output.iter().fold(0usize, |bytes, series| {
|
||||
bytes.saturating_add(series_bytes(series))
|
||||
});
|
||||
let output_vec_bytes = output
|
||||
.capacity()
|
||||
.saturating_mul(std::mem::size_of::<Series>());
|
||||
buffer.resize(
|
||||
output_bytes
|
||||
.saturating_add(output_vec_bytes)
|
||||
.saturating_add(queue_bytes),
|
||||
)?;
|
||||
Ok(Some(output))
|
||||
}
|
||||
}
|
||||
|
||||
/// The rejection of a chunk that is not the layout this node folds.
|
||||
///
|
||||
/// The fold indexes its states by absolute grid position and by field, so a chunk of another field
|
||||
/// count or another grid would be folded into states that mean something else. `error` is what the
|
||||
/// series check reported, which says which of the two disagreed.
|
||||
/// Wraps the error of a chunk this node cannot fold.
|
||||
///
|
||||
/// Only a chunk that does not fit the schema is the node's own refusal; anything else is the
|
||||
/// kernel saying it does not compute this shape at all -- a native histogram in a float aggregate,
|
||||
/// for instance -- and a caller that falls back to another planner on `NotImplemented` still can.
|
||||
/// Wrapping that one in an `Execution` would deny it.
|
||||
fn folded_chunk(path: &NodePath, aggregate: &str, error: DataFusionError) -> DataFusionError {
|
||||
match error {
|
||||
DataFusionError::NotImplemented(_) => error,
|
||||
error => DataFusionError::Execution(format!(
|
||||
"the `{aggregate}` aggregate at {path} folds chunks bound to its own schema, and this \
|
||||
one is not: {error}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// The error for a fold that has already been finished.
|
||||
fn no_fold(path: &NodePath) -> DataFusionError {
|
||||
DataFusionError::Execution(format!(
|
||||
"the aggregate at {path} already finished its fold, so it takes no more input"
|
||||
))
|
||||
}
|
||||
@@ -127,6 +127,9 @@ fn describe(node: &NativeNode) -> String {
|
||||
BoundVectorOp::Histogram(HistogramFunction::Fraction { lower, upper }) => {
|
||||
format!("histogram_fraction {lower} {upper}")
|
||||
}
|
||||
BoundVectorOp::CountValues { label, group } => {
|
||||
format!("count_values {label:?} {}", grouping(group))
|
||||
}
|
||||
},
|
||||
NativeExpr::Aggregate { op, .. } => format!(
|
||||
"{} {} [{}] {}",
|
||||
|
||||
@@ -446,6 +446,13 @@ fn a_vector_operation_names_its_binding() {
|
||||
},
|
||||
r#" root Vector absent {job="x"}"#,
|
||||
),
|
||||
(
|
||||
BoundVectorOp::CountValues {
|
||||
label: "value".into(),
|
||||
group: Grouping::Without(Arc::new(["job".into()])),
|
||||
},
|
||||
r#" root Vector count_values "value" without [job]"#,
|
||||
),
|
||||
];
|
||||
for (op, line) in lines {
|
||||
let tree = NativeTree::try_new(NativeNode {
|
||||
|
||||
+269
-217
@@ -91,9 +91,9 @@ use std::sync::Arc;
|
||||
use datafusion::error::{DataFusionError, Result};
|
||||
use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation};
|
||||
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
|
||||
use datatypes::arrow::array::{Array, Float64Array};
|
||||
use datatypes::arrow::array::Float64Array;
|
||||
use datatypes::arrow::buffer::{Buffer, ScalarBuffer};
|
||||
use datatypes::series::histogram::{HistogramBatch, HistogramView};
|
||||
use datatypes::series::histogram::HistogramView;
|
||||
use datatypes::series::raw::{RawSeriesChunk, RawTimestamps};
|
||||
use datatypes::series::samples::{
|
||||
DenseSamplesRef, RowCursor, SampleRef, Samples, Series, SeriesFields, SeriesSchema,
|
||||
@@ -101,8 +101,10 @@ use datatypes::series::samples::{
|
||||
use datatypes::series::time::{EvalGrid, GridRange, GridSpan, GridSpec, TimeAxis};
|
||||
use datatypes::series::{MetricName, SeriesOrdering};
|
||||
|
||||
use crate::aggregate;
|
||||
use crate::grid_series::{
|
||||
GridSample, GridSeriesBuilder, LabelSet, label_set, merge_same_labelset, owned,
|
||||
GridSample, GridSeriesBuilder, LabelSet, fields_bytes, label_set, merge_same_labelset, owned,
|
||||
series_bytes, table_bytes,
|
||||
};
|
||||
use crate::histogram_eval::evaluate_histograms;
|
||||
use crate::instant_eval::select_instants;
|
||||
@@ -110,12 +112,16 @@ use crate::map::{
|
||||
ArithmeticOp, Comparison, ComparisonOp, MathOp, ScalarOperand, drop_metric_name,
|
||||
map_arithmetic, map_compare, map_math,
|
||||
};
|
||||
use crate::native::aggregate::AggregateRuntime;
|
||||
use crate::native::batch::EvaluatedChunk;
|
||||
use crate::native::metrics::{NodeMetrics, NodePath};
|
||||
use crate::native::plan::{
|
||||
BoundInstant, BoundScalar, BoundVectorOp, Grouping, NativeExpr, NativeNode, NativeTree,
|
||||
NativeType,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::native::runs::absorb_rebuilds;
|
||||
use crate::native::runs::{Run, absorb};
|
||||
use crate::range_eval::RangeEvaluator;
|
||||
use crate::select::{self, SortOrder};
|
||||
use crate::vector_match::{
|
||||
@@ -189,34 +195,11 @@ fn raw_bytes(chunk: &RawSeriesChunk) -> usize {
|
||||
/// The bytes an evaluated chunk holds: its struct and the Arrow buffers behind its fields.
|
||||
///
|
||||
/// The grid is shared by every chunk addressed on it, so it is not counted per chunk.
|
||||
fn evaluated_bytes(chunk: &EvaluatedChunk) -> usize {
|
||||
std::mem::size_of::<EvaluatedChunk>().saturating_add(fields_bytes(&chunk.series.fields))
|
||||
}
|
||||
|
||||
/// The bytes one evaluated series holds: its header, the axis that addresses it, the identity it
|
||||
/// carries and the sample buffers behind its fields.
|
||||
///
|
||||
/// It is the figure a fragment of an affected window is charged with as it arrives. The axis's runs
|
||||
/// and the segments of its fields are counted in one pass over the series, so the estimate costs
|
||||
/// the fragment rather than a walk over everything the node holds. A payload a fragment shares with
|
||||
/// another — a histogram table behind two slices of it, the grid behind every axis — is counted
|
||||
/// once per fragment: that over-count is accepted here rather than a deduplication that would have
|
||||
/// to keep state across fragments.
|
||||
fn series_bytes(series: &Series) -> usize {
|
||||
let runs = series.time.runs().count();
|
||||
let segments: usize = series
|
||||
.fields
|
||||
.iter()
|
||||
.map(|field| field.segments().len())
|
||||
.sum();
|
||||
std::mem::size_of::<Series>()
|
||||
.saturating_add(std::mem::size_of::<TimeAxis>())
|
||||
.saturating_add(runs.saturating_mul(std::mem::size_of::<(usize, usize)>()))
|
||||
.saturating_add(
|
||||
segments.saturating_mul(std::mem::size_of::<Samples>() + std::mem::size_of::<usize>()),
|
||||
)
|
||||
.saturating_add(fields_bytes(&series.fields))
|
||||
.saturating_add(metric_bytes(&series.metric))
|
||||
/// It is what an aggregate node charges for a chunk it is folding: that node hands the chunk over
|
||||
/// to its fold and drops it, so the chunk alone is what it holds for as long as it holds anything.
|
||||
pub(super) fn evaluated_bytes(chunk: &EvaluatedChunk) -> usize {
|
||||
std::mem::size_of::<EvaluatedChunk>().saturating_add(fields_bytes(&chunk.series.fields))
|
||||
}
|
||||
|
||||
/// The bytes one chunk a binary operand retains holds on its own.
|
||||
@@ -342,42 +325,6 @@ fn merge_workspace_bytes(rows: usize, fragments: usize, fields: usize) -> usize
|
||||
.saturating_add(output)
|
||||
}
|
||||
|
||||
/// The bytes the fields of a chunk hold: the sample buffers of every segment, and the histogram
|
||||
/// tables those segments reference.
|
||||
fn fields_bytes(fields: &SeriesFields) -> usize {
|
||||
fields
|
||||
.iter()
|
||||
.map(|field| {
|
||||
field.segments().iter().fold(0usize, |total, segment| {
|
||||
total.saturating_add(segment_bytes(segment))
|
||||
})
|
||||
})
|
||||
.fold(0usize, usize::saturating_add)
|
||||
}
|
||||
|
||||
/// The bytes one dense segment of samples holds.
|
||||
fn segment_bytes(segment: &Samples) -> usize {
|
||||
match segment.dense() {
|
||||
Some(DenseSamplesRef::Float(values)) => values.get_array_memory_size(),
|
||||
Some(DenseSamplesRef::Histogram(histograms)) => histograms
|
||||
.indices()
|
||||
.get_array_memory_size()
|
||||
.saturating_add(table_bytes(histograms.batch())),
|
||||
Some(DenseSamplesRef::Mixed(mixed)) => mixed
|
||||
.floats()
|
||||
.get_array_memory_size()
|
||||
.saturating_add(mixed.histograms().indices().get_array_memory_size())
|
||||
.saturating_add(table_bytes(mixed.histograms().batch())),
|
||||
// `segments` hands out dense segments only, so a nested chunk never reaches here.
|
||||
None => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// The bytes one histogram table holds: the batch struct and the arrays its columns view.
|
||||
fn table_bytes(table: &Arc<HistogramBatch>) -> usize {
|
||||
std::mem::size_of::<HistogramBatch>().saturating_add(table.as_arrow().get_array_memory_size())
|
||||
}
|
||||
|
||||
/// The bytes the allocation `buffer` is a slice of.
|
||||
///
|
||||
/// The capacity is the whole allocation, which is what a slice keeps alive: a key of a few bytes
|
||||
@@ -408,6 +355,8 @@ enum NodeTag {
|
||||
/// Matches two evaluated vectors and applies one operation to every matched pair of series, or
|
||||
/// broadcasts one constant over the chunks of one evaluated vector.
|
||||
Binary,
|
||||
/// Folds every series of the input into one series per group, when the input ends.
|
||||
Aggregate,
|
||||
/// Keeps some samples of its whole input vector at each grid point.
|
||||
Vector,
|
||||
}
|
||||
@@ -422,6 +371,7 @@ impl NodeTag {
|
||||
Self::Window => "window",
|
||||
Self::Map => "map",
|
||||
Self::Binary => "binary",
|
||||
Self::Aggregate => "aggregate",
|
||||
Self::Vector => "vector operation",
|
||||
}
|
||||
}
|
||||
@@ -576,7 +526,9 @@ impl MergeInput {
|
||||
/// in the query; the metrics report the same figure so a slow node can be told apart from a wide
|
||||
/// one. The reservation is also the figure itself: a node holds nothing beside what it reserved, so
|
||||
/// a gauge reads its size rather than a copy that could drift from it.
|
||||
struct NodeBuffer {
|
||||
// Visible to the aggregate node so that its fold can reserve for what it is about to allocate; the
|
||||
// reservation belongs to a node, not to a module.
|
||||
pub(super) struct NodeBuffer {
|
||||
metrics: Arc<NodeMetrics>,
|
||||
reservation: MemoryReservation,
|
||||
}
|
||||
@@ -592,7 +544,7 @@ impl NodeBuffer {
|
||||
}
|
||||
|
||||
/// Reserves `bytes` beside what this node already holds.
|
||||
fn grow(&mut self, bytes: usize) -> Result<()> {
|
||||
pub(super) fn grow(&mut self, bytes: usize) -> Result<()> {
|
||||
self.reservation.try_grow(bytes)?;
|
||||
self.record();
|
||||
Ok(())
|
||||
@@ -603,7 +555,7 @@ impl NodeBuffer {
|
||||
/// What a node that consumes its input chunk by chunk holds afterwards is not the sum of what
|
||||
/// it consumed, so the reservation is set from the state that is left rather than grown per
|
||||
/// chunk.
|
||||
fn resize(&mut self, bytes: usize) -> Result<()> {
|
||||
pub(super) fn resize(&mut self, bytes: usize) -> Result<()> {
|
||||
self.reservation.try_resize(bytes)?;
|
||||
self.record();
|
||||
Ok(())
|
||||
@@ -628,20 +580,6 @@ impl NodeBuffer {
|
||||
}
|
||||
}
|
||||
|
||||
/// One run of grid positions, half-open, that hold a value in one field.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct Run {
|
||||
start: usize,
|
||||
end: usize,
|
||||
}
|
||||
|
||||
impl Run {
|
||||
/// Whether two runs share a position.
|
||||
fn overlaps(self, other: Self) -> bool {
|
||||
self.start < other.end && other.start < self.end
|
||||
}
|
||||
}
|
||||
|
||||
/// The positions one node has handed out, per output label set and field.
|
||||
///
|
||||
/// A node that rewrites labels — a `Map` dropping the metric name, a `Window` dropping the name
|
||||
@@ -876,109 +814,6 @@ fn misaligned_field(
|
||||
))
|
||||
}
|
||||
|
||||
// How many times `absorb` rebuilt the stored runs instead of appending to them.
|
||||
//
|
||||
// Test-only, and per thread: the runs stored afterwards are the same either way, so a rebuild shows
|
||||
// only in what it costs, and a test that reads its own thread's figure is not disturbed by another
|
||||
// test's chunks.
|
||||
#[cfg(test)]
|
||||
thread_local! {
|
||||
static ABSORB_REBUILDS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
|
||||
}
|
||||
|
||||
/// How many times this thread's [`absorb`] rebuilt the stored runs.
|
||||
#[cfg(test)]
|
||||
fn absorb_rebuilds() -> usize {
|
||||
ABSORB_REBUILDS.with(std::cell::Cell::get)
|
||||
}
|
||||
|
||||
/// Merges `incoming` into `stored`, keeping the result in position order and joining the runs that
|
||||
/// touch.
|
||||
///
|
||||
/// Within a chunk one pass over both sides is what keeps this linear, since each side is sorted and
|
||||
/// disjoint. Adding the runs one at a time and rebuilding the array after each would cost the square
|
||||
/// of the runs a chunk yields: a chunk of one hundred thousand positions alternating between values
|
||||
/// and nulls yields fifty thousand runs, and re-merging after each is over a billion element moves
|
||||
/// for a single series.
|
||||
///
|
||||
/// One pass per chunk is still a pass over everything the series has stored so far, so it is not
|
||||
/// free once the *history* grows across chunks: a series split into `K` chunks of one position each,
|
||||
/// a hole apart, stores one more run per chunk and moves `K(K+1)/2` of them, with no label collision
|
||||
/// and no illegal input anywhere. A chunk that starts no earlier than the end of the last stored run
|
||||
/// only ever adds to the end — the stored runs are sorted and disjoint, so every one of them ends
|
||||
/// before anything this chunk holds — and it is appended instead, which is amortised constant per
|
||||
/// run. That is the shape a scan hands over as its positions move forward chunk after chunk. A chunk
|
||||
/// that starts before the last stored run can split a run or reach back past it, so it still costs
|
||||
/// the full merge.
|
||||
///
|
||||
/// Either way the result is sorted, disjoint, and has the runs that touch joined, which is what
|
||||
/// [`Emitted::occupy`] looks a position up by.
|
||||
fn absorb(stored: &mut Vec<Run>, incoming: &[Run]) {
|
||||
if incoming.is_empty() {
|
||||
return;
|
||||
}
|
||||
if stored
|
||||
.last()
|
||||
.is_none_or(|last| incoming[0].start >= last.end)
|
||||
{
|
||||
append_runs(stored, incoming);
|
||||
} else {
|
||||
merge_runs(stored, incoming);
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends `incoming` to `stored`, all of which lies past the last stored run.
|
||||
///
|
||||
/// Nothing already stored has to be placed again: the two sides are each in position order and
|
||||
/// nothing `incoming` holds can reach back into what is stored, so its runs are pushed on the end and
|
||||
/// the vector's own growth is the only move — amortised over the runs appended. Where its first run
|
||||
/// touches the end of the last stored run the two are one run, as they are after a merge, so it is
|
||||
/// joined rather than left beside it.
|
||||
fn append_runs(stored: &mut Vec<Run>, incoming: &[Run]) {
|
||||
let mut incoming = incoming;
|
||||
if let Some(last) = stored.last_mut()
|
||||
&& let Some(first) = incoming.first()
|
||||
&& first.start <= last.end
|
||||
{
|
||||
last.end = last.end.max(first.end);
|
||||
incoming = &incoming[1..];
|
||||
}
|
||||
stored.extend_from_slice(incoming);
|
||||
}
|
||||
|
||||
/// Merges `incoming` into `stored`, where the two can interleave.
|
||||
///
|
||||
/// One pass is enough because each side is sorted and disjoint: whichever run starts first is
|
||||
/// placed, and it joins the run before it where the two touch.
|
||||
fn merge_runs(stored: &mut Vec<Run>, incoming: &[Run]) {
|
||||
#[cfg(test)]
|
||||
ABSORB_REBUILDS.with(|rebuilds| rebuilds.set(rebuilds.get() + 1));
|
||||
let mut merged: Vec<Run> = Vec::with_capacity(stored.len() + incoming.len());
|
||||
let (mut a, mut b) = (0, 0);
|
||||
while a < stored.len() || b < incoming.len() {
|
||||
let larger = match (stored.get(a), incoming.get(b)) {
|
||||
(Some(left), Some(right)) if right.start < left.start => {
|
||||
b += 1;
|
||||
*right
|
||||
}
|
||||
(Some(left), _) => {
|
||||
a += 1;
|
||||
*left
|
||||
}
|
||||
(None, Some(right)) => {
|
||||
b += 1;
|
||||
*right
|
||||
}
|
||||
(None, None) => break,
|
||||
};
|
||||
match merged.last_mut() {
|
||||
Some(last) if larger.start <= last.end => last.end = last.end.max(larger.end),
|
||||
_ => merged.push(larger),
|
||||
}
|
||||
}
|
||||
*stored = merged;
|
||||
}
|
||||
|
||||
/// What a map node applies.
|
||||
struct MapRuntime {
|
||||
ops: Arc<[MathOp]>,
|
||||
@@ -1915,6 +1750,19 @@ fn vector_series(
|
||||
BoundVectorOp::Histogram(function) => {
|
||||
evaluate_histograms(*function, schema, &merge_same_labelset(schema, series)?)
|
||||
}
|
||||
BoundVectorOp::CountValues { label, group } => {
|
||||
let (by, labels) = match group {
|
||||
Grouping::By(labels) => (true, labels),
|
||||
Grouping::Without(labels) => (false, labels),
|
||||
};
|
||||
let labels: Vec<&str> = labels.iter().map(|label| &**label).collect();
|
||||
let grouping = if by {
|
||||
aggregate::Grouping::By(&labels)
|
||||
} else {
|
||||
aggregate::Grouping::Without(&labels)
|
||||
};
|
||||
aggregate::count_values(schema, &series, label, grouping)
|
||||
}
|
||||
BoundVectorOp::Absent { labels } => Ok(select::absent(schema, &series, labels.clone())?
|
||||
.into_iter()
|
||||
.collect()),
|
||||
@@ -1933,6 +1781,10 @@ fn vector_series(
|
||||
/// - A sort: the value each series is keyed by, and the floats `sort` and `sort_desc` rebuild a
|
||||
/// series into when it holds histograms; the label sorts move the series they were given.
|
||||
/// - `absent`: a presence flag and a result value per grid point and field.
|
||||
/// - `count_values`, per field sample, since every field's distinct values make groups of their
|
||||
/// own and a gauge with distinct values makes about one group per sample: the group's label set
|
||||
/// twice (its metric and its key) with the value written into it, its count per field, and the
|
||||
/// result series it becomes.
|
||||
/// - `histogram_quantile` and `histogram_fraction`: the merge of label-set pieces before the kernel
|
||||
/// ([`merge_workspace_bytes`]); per series, its group entry and label pairs and a cursor per
|
||||
/// field; per row, the bucket it contributes and a result value per field.
|
||||
@@ -1979,6 +1831,21 @@ fn vector_workspace_bytes(op: &BoundVectorOp, schema: &SeriesSchema, series: &[S
|
||||
},
|
||||
);
|
||||
}
|
||||
BoundVectorOp::CountValues { .. } => {
|
||||
let label = std::mem::size_of::<(Arc<str>, Arc<str>)>();
|
||||
// The value label at the length of a 17-digit value with sign, point and exponent;
|
||||
// the `'f'` form writes a large magnitude out in full, which this does not follow.
|
||||
let value = 24;
|
||||
let group = std::mem::size_of::<Vec<f64>>()
|
||||
+ fields * std::mem::size_of::<f64>()
|
||||
+ std::mem::size_of::<Series>()
|
||||
+ fields * std::mem::size_of::<Samples>();
|
||||
return series.iter().fold(0usize, |total, item| {
|
||||
let labels = 2 * (item.metric.tags().len() + 2) * label + value;
|
||||
let samples = item.time.len().saturating_mul(fields);
|
||||
total.saturating_add(samples.saturating_mul(group + labels))
|
||||
});
|
||||
}
|
||||
BoundVectorOp::TopK { .. }
|
||||
| BoundVectorOp::LimitK { .. }
|
||||
| BoundVectorOp::LimitRatio { .. } => {}
|
||||
@@ -2260,6 +2127,14 @@ struct NodeRuntime {
|
||||
/// `None` for every other node, and for a binary node that broadcasts a constant instead: that
|
||||
/// one holds no operand at all and is described by [`NodeRuntime::broadcast`].
|
||||
binary: Option<BinaryRuntime>,
|
||||
/// What an aggregate node holds: the fold over what it was handed.
|
||||
///
|
||||
/// Unlike a binary node's operands, nothing of the input survives a chunk: the fold keeps its
|
||||
/// states and the chunk it read is dropped, so this is the one node whose reservation does not
|
||||
/// follow what it was handed but what it accumulated. That is also why this node cannot report
|
||||
/// its input complete from what it holds -- there is nothing to hold -- and reads the driver's
|
||||
/// [`NativeExecutor::inputs_finished`] and its child's [`NativeExecutionState::settled`] instead.
|
||||
aggregate: Option<AggregateRuntime>,
|
||||
/// What a vector node holds.
|
||||
vector: Option<VectorRuntime>,
|
||||
/// What a binary node holds when one operand is a constant: the operation and the layout of the
|
||||
@@ -2669,6 +2544,102 @@ impl NodeRuntime {
|
||||
self.buffer.resize(queued)
|
||||
}
|
||||
|
||||
/// Folds one chunk into this node's aggregate.
|
||||
///
|
||||
/// The chunk is reserved for before it is folded, by what the node reports it is worth, and the
|
||||
/// reservation settles afterwards on the fold that is left. Unlike [`Self::retain_binary`], what
|
||||
/// stops being held here is the chunk itself: the fold keeps its states, so a chunk that closes
|
||||
/// its series leaves nothing behind and the node is empty of input again.
|
||||
fn retain_aggregate(&mut self, path: &NodePath, chunk: NativeChunk) -> Result<()> {
|
||||
let NativeChunk::Evaluated(chunk) = chunk else {
|
||||
return Err(DataFusionError::Execution(format!(
|
||||
"the aggregate at {path} consumes evaluated chunks, not raw samples"
|
||||
)));
|
||||
};
|
||||
// The figure is read before the chunk is folded, because reserving after the fact would be
|
||||
// reserving for memory the fold already read.
|
||||
let pending = self
|
||||
.aggregate
|
||||
.as_ref()
|
||||
.ok_or_else(|| no_aggregate(path))?
|
||||
.pending(&chunk);
|
||||
let positions = chunk.series.time.len();
|
||||
self.buffer.grow(pending)?;
|
||||
let metrics = Arc::clone(&self.buffer.metrics);
|
||||
{
|
||||
let _timer = metrics.timer();
|
||||
let (aggregate, buffer) = (&mut self.aggregate, &mut self.buffer);
|
||||
aggregate
|
||||
.as_mut()
|
||||
.ok_or_else(|| no_aggregate(path))?
|
||||
.accumulate(path, chunk, buffer, pending)?;
|
||||
}
|
||||
metrics.record_input(1, positions);
|
||||
// A chunk that closed its series closed it for the node below: a group is not a series, so
|
||||
// the fold completing nothing on it is why nothing is counted here. The series this node
|
||||
// closes are the ones it hands out, counted when it queues them.
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Finishes this node's fold and queues one chunk per group it produced.
|
||||
///
|
||||
/// Each group is one series, decided in full by the fold, so the one chunk it is handed over as
|
||||
/// closes it: a consumer above reads "nothing more here" from that chunk rather than from another
|
||||
/// one arriving.
|
||||
fn compute_aggregate(&mut self, path: &NodePath) -> Result<()> {
|
||||
let metrics = Arc::clone(&self.buffer.metrics);
|
||||
let _timer = metrics.timer();
|
||||
let groups = self
|
||||
.aggregate
|
||||
.as_ref()
|
||||
.map_or(0, |aggregate| aggregate.groups());
|
||||
let queue_capacity = self.queued.capacity();
|
||||
let queue_target = self.queued.len().saturating_add(groups);
|
||||
let queue_growth = queue_target
|
||||
.saturating_sub(queue_capacity)
|
||||
.saturating_mul(std::mem::size_of::<(NativeChunk, usize)>());
|
||||
self.buffer.grow(queue_growth)?;
|
||||
if groups > 0 {
|
||||
self.queued.try_reserve_exact(groups).map_err(|error| {
|
||||
DataFusionError::Execution(format!(
|
||||
"cannot reserve aggregate output queue: {error}"
|
||||
))
|
||||
})?;
|
||||
}
|
||||
let queue_bytes = queue_target
|
||||
.max(queue_capacity)
|
||||
.saturating_mul(std::mem::size_of::<(NativeChunk, usize)>());
|
||||
let output = {
|
||||
let aggregate = self.aggregate.as_mut().ok_or_else(|| no_aggregate(path))?;
|
||||
aggregate.compute(path, &mut self.buffer, queue_bytes)?
|
||||
};
|
||||
// The fold has returned its series; reserve their handover before appending them to the queue.
|
||||
let Some(output) = output else {
|
||||
return Ok(());
|
||||
};
|
||||
let mut closed = 0usize;
|
||||
for series in output {
|
||||
let bytes = series_bytes(&series);
|
||||
let chunk = NativeChunk::Evaluated(EvaluatedChunk::try_new(series, true)?);
|
||||
self.queued.push_back((chunk, bytes));
|
||||
closed += 1;
|
||||
}
|
||||
// A group is one series and this node closes each of them exactly once, on the one chunk it
|
||||
// hands it over as.
|
||||
self.buffer.metrics.record_completed_series(closed);
|
||||
// The output Vec has been consumed; the node now holds only queue backing and chunks.
|
||||
let queued_bytes = self
|
||||
.queued
|
||||
.iter()
|
||||
.fold(0usize, |total, (_, bytes)| total.saturating_add(*bytes));
|
||||
let queue_backing = self
|
||||
.queued
|
||||
.capacity()
|
||||
.saturating_mul(std::mem::size_of::<(NativeChunk, usize)>());
|
||||
self.buffer
|
||||
.resize(queued_bytes.saturating_add(queue_backing))
|
||||
}
|
||||
|
||||
/// Feeds one chunk to this node's instant selection.
|
||||
///
|
||||
/// `None` means the series this chunk belongs to has not ended: the selection reads one whole
|
||||
@@ -2871,6 +2842,8 @@ impl NodeRuntime {
|
||||
self.window = None;
|
||||
self.instant = None;
|
||||
self.binary = None;
|
||||
// The fold is what an aggregate node holds, and dropping it is what gives its states back.
|
||||
self.aggregate = None;
|
||||
self.vector = None;
|
||||
self.broadcast = None;
|
||||
self.source_schema = None;
|
||||
@@ -2963,15 +2936,14 @@ impl NativeExecutionState {
|
||||
// two vectors, or a constant broadcast over one -- is decided where the operands'
|
||||
// layouts are read.
|
||||
NativeExpr::Binary { .. } => NodeTag::Binary,
|
||||
// An aggregate is built here and narrowed below, for the same reason: `validate`
|
||||
// admits every kind and every stage as a legal tree, and the runtime computes one
|
||||
// kind at one stage.
|
||||
NativeExpr::Aggregate { .. } => NodeTag::Aggregate,
|
||||
NativeExpr::Vector { .. } => NodeTag::Vector,
|
||||
// A scalar is a value node: it produces the constant it was bound to, hands over no
|
||||
// chunk and holds no state, so it is built like any other node and pulled like none.
|
||||
NativeExpr::Scalar(_) => NodeTag::Scalar,
|
||||
// An aggregate is legal in a bound tree, and `validate` stays the authority on that;
|
||||
// this runtime simply does not execute it yet. Rejected before a single chunk is
|
||||
// read: a tree the runtime cannot execute must not start, because a partial answer
|
||||
// would be one the caller never asked for.
|
||||
NativeExpr::Aggregate { .. } => return Err(not_implemented("Aggregate", &path)),
|
||||
};
|
||||
let buffer = NodeBuffer::new(&path, handles, &pool);
|
||||
let mut runtime = NodeRuntime {
|
||||
@@ -2982,6 +2954,7 @@ impl NativeExecutionState {
|
||||
window: None,
|
||||
map: None,
|
||||
binary: None,
|
||||
aggregate: None,
|
||||
vector: None,
|
||||
broadcast: None,
|
||||
source_schema: None,
|
||||
@@ -3081,6 +3054,29 @@ impl NativeExecutionState {
|
||||
// out. A broadcast that keeps the labels it reads is not where two series
|
||||
// become one either, so it keeps none; the one that drops them does, above.
|
||||
}
|
||||
NativeExpr::Aggregate { input, op } => {
|
||||
// A `Complete` aggregate neither adds nor drops a field and evaluates on its
|
||||
// input's grid, so the node's own output is the layout it folds. What it does
|
||||
// not execute -- another kind, or another stage -- is refused here, before the
|
||||
// pass reaches the input that would feed it, and naming what it found is that
|
||||
// rejection's job: a `Partial` aggregate's output is state and names no samples
|
||||
// at all, so the input's layout stands in for it and the stage decides.
|
||||
let schema = node
|
||||
.output
|
||||
.samples()
|
||||
.or_else(|| input.output.samples())
|
||||
.cloned()
|
||||
.ok_or_else(|| {
|
||||
DataFusionError::Execution(format!(
|
||||
"the aggregate at {path} does not produce samples"
|
||||
))
|
||||
})?;
|
||||
runtime.aggregate = Some(AggregateRuntime::new(
|
||||
op.clone(),
|
||||
schema,
|
||||
&mut runtime.buffer,
|
||||
)?);
|
||||
}
|
||||
NativeExpr::Vector { op, .. } => {
|
||||
let schema = node.output.samples().cloned().ok_or_else(|| {
|
||||
DataFusionError::Execution(format!(
|
||||
@@ -3103,10 +3099,9 @@ impl NativeExecutionState {
|
||||
// A vector operation keeps the labels of the series it reads, or answers under
|
||||
// label sets of its own, so it keeps no ledger of what it handed out.
|
||||
}
|
||||
NativeExpr::Scalar(_) | NativeExpr::Aggregate { .. } => {
|
||||
NativeExpr::Scalar(_) => {
|
||||
// A scalar holds nothing: what its consumer reads is the constant the node was
|
||||
// bound to, which is on the tree rather than in this state. The aggregate is
|
||||
// rejected above, where the pass has the position to name.
|
||||
// bound to, which is on the tree rather than in this state.
|
||||
}
|
||||
}
|
||||
// The inputs this node reads, in the order the tree declares them: a unary node's one
|
||||
@@ -3114,12 +3109,11 @@ impl NativeExecutionState {
|
||||
// pushed back to front, so the node the spec names is the one the pass reaches next and
|
||||
// nothing but its own subtree lies between the two.
|
||||
let reads_input = match &node.expr {
|
||||
NativeExpr::Input(_) | NativeExpr::Scalar(_) | NativeExpr::Aggregate { .. } => {
|
||||
false
|
||||
}
|
||||
NativeExpr::Input(_) | NativeExpr::Scalar(_) => false,
|
||||
NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Window { input, .. }
|
||||
| NativeExpr::Map { input, .. }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. } => {
|
||||
pending.push((input.as_ref(), path.input()));
|
||||
true
|
||||
@@ -3189,6 +3183,14 @@ impl NativeExecutionState {
|
||||
holds_shared[index] = input_holds_shared;
|
||||
}
|
||||
NodeTag::Map => holds_shared[index] = true,
|
||||
NodeTag::Aggregate => {
|
||||
// An aggregate hands over one series per group, each under the label set its
|
||||
// group was keyed by and each closed on its own chunk: two groups are two
|
||||
// label sets, so nothing above has to join pieces of a series. Its own input
|
||||
// is read whole -- the fold holds what it accumulated, not the chunks -- so
|
||||
// nothing about its input's flag reaches it either.
|
||||
holds_shared[index] = false;
|
||||
}
|
||||
NodeTag::Scalar => {
|
||||
// A scalar produces one value and no labels, so it is not a stream of series
|
||||
// and nothing above it can read two of them under one label set. The flag
|
||||
@@ -3229,8 +3231,9 @@ impl NativeExecutionState {
|
||||
// what ranking the merged series would, and what is above treats them as
|
||||
// it treats a map's. `absent` answers with one series under the labels it
|
||||
// was bound to, the histogram functions merge the label sets they answer
|
||||
// with, and `limitk` counts series in input order, so it merges the pieces
|
||||
// first; none of them hands over pieces.
|
||||
// with, `count_values` answers with one series per label set it makes, and
|
||||
// `limitk` counts series in input order, so it merges the pieces first;
|
||||
// none of them hands over pieces.
|
||||
let op = match expr {
|
||||
NativeExpr::Vector { op, .. } => Some(op),
|
||||
_ => None,
|
||||
@@ -3245,7 +3248,11 @@ impl NativeExecutionState {
|
||||
let whole = limitk
|
||||
|| matches!(
|
||||
op,
|
||||
Some(BoundVectorOp::Absent { .. } | BoundVectorOp::Histogram(_))
|
||||
Some(
|
||||
BoundVectorOp::Absent { .. }
|
||||
| BoundVectorOp::Histogram(_)
|
||||
| BoundVectorOp::CountValues { .. }
|
||||
)
|
||||
);
|
||||
holds_shared[index] = input_holds_shared && !whole;
|
||||
}
|
||||
@@ -3742,9 +3749,24 @@ impl NativeExecutor {
|
||||
// A scalar produces its value without reading anything, so it is settled from the
|
||||
// start: there is no chunk it owes its consumer.
|
||||
NodeTag::Scalar => true,
|
||||
// These report as they are handed chunks, so a child with nothing left is all it takes
|
||||
// for them to have nothing left either.
|
||||
NodeTag::Map | NodeTag::Window | NodeTag::Instant => self.state.nodes[index]
|
||||
.input
|
||||
.is_none_or(|child| self.settled(child)),
|
||||
// An aggregate answers one series per group and only once its input ended, so its own
|
||||
// fold has to have run as well: a child that has handed everything up is not enough,
|
||||
// because every one of those chunks was folded and none of them answered yet. This is
|
||||
// what keeps it out of the line above, which asks only about the child.
|
||||
NodeTag::Aggregate => {
|
||||
runtime
|
||||
.aggregate
|
||||
.as_ref()
|
||||
.is_some_and(|aggregate| aggregate.computed)
|
||||
&& self.state.nodes[index]
|
||||
.input
|
||||
.is_none_or(|child| self.settled(child))
|
||||
}
|
||||
NodeTag::Vector => {
|
||||
runtime
|
||||
.vector
|
||||
@@ -3942,6 +3964,7 @@ impl NativeExecutor {
|
||||
// does, so its values are counted rather than read as its positions.
|
||||
runtime.pop_queued(evaluated_values)
|
||||
}
|
||||
NodeTag::Aggregate => self.aggregate_node(index),
|
||||
NodeTag::Vector => {
|
||||
let child = input.ok_or_else(|| missing_input(tag))?;
|
||||
// Which samples are kept at a point is not decided until the whole input has
|
||||
@@ -3961,6 +3984,35 @@ impl NativeExecutor {
|
||||
}
|
||||
}
|
||||
|
||||
/// Folds everything the aggregate at `index` can read and hands over one group's series.
|
||||
///
|
||||
/// What it reads is taken to the end of what its child has, because nothing the fold is handed
|
||||
/// is retained: a chunk is folded as it arrives and dropped, so reading the child dry costs the
|
||||
/// fold's states and not the input's bytes. Nothing is answered until the input has ended --
|
||||
/// before that, a group's value at a point is still missing the members that have not arrived.
|
||||
///
|
||||
/// That is the difference from a binary node. A binary node retains both operands and so can ask
|
||||
/// whether they hold anything still open; an aggregate keeps none of them and asks the driver
|
||||
/// instead: [`Self::inputs_finished`] says the caller will hand over nothing more, and
|
||||
/// [`NativeExecutionState::settled`] says the child has none left to produce.
|
||||
fn aggregate_node(&mut self, index: usize) -> Result<Option<NativeChunk>> {
|
||||
let child = self.state.nodes[index]
|
||||
.input
|
||||
.ok_or_else(|| missing_input(NodeTag::Aggregate))?;
|
||||
while let Some(chunk) = self.pull(child)? {
|
||||
let path = &self.state.paths[index];
|
||||
self.state.runtimes[index].retain_aggregate(path, chunk)?;
|
||||
}
|
||||
if !self.inputs_finished || !self.settled(child) {
|
||||
return Ok(None);
|
||||
}
|
||||
let path = &self.state.paths[index];
|
||||
self.state.runtimes[index].compute_aggregate(path)?;
|
||||
// A group's series holds one value per field and point it decided, so its values are counted
|
||||
// rather than read as its positions, as a binary result's are.
|
||||
self.state.runtimes[index].pop_queued(evaluated_values)
|
||||
}
|
||||
|
||||
/// Pulls the vector side of the broadcast at `index` until it hands over a chunk, and returns
|
||||
/// that chunk transformed.
|
||||
///
|
||||
@@ -4188,11 +4240,7 @@ fn valid_values(chunk: &EvaluatedChunk) -> usize {
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// The values a chunk a binary node hands over holds.
|
||||
///
|
||||
/// A binary result is an evaluated chunk, so this is [`valid_values`]: one value per field sample
|
||||
/// that holds one. A raw chunk cannot reach a binary node's queue, and is answered as its positions
|
||||
/// so an impossible queue cannot under-count.
|
||||
/// The values an evaluated chunk hands over, or the timestamps a raw source hands over.
|
||||
fn evaluated_values(chunk: &NativeChunk) -> usize {
|
||||
match chunk {
|
||||
NativeChunk::Evaluated(chunk) => valid_values(chunk),
|
||||
@@ -4200,18 +4248,6 @@ fn evaluated_values(chunk: &NativeChunk) -> usize {
|
||||
}
|
||||
}
|
||||
|
||||
/// The rejection of a node kind this runtime does not execute yet.
|
||||
///
|
||||
/// It names the kind and the node's position: the position is what the metrics and `EXPLAIN` call
|
||||
/// the same node, so a reader can find it in the tree. An aggregate is the one kind left here: a
|
||||
/// scalar executes as a value node, and a binary node executes both as a vector match and as a
|
||||
/// constant broadcast.
|
||||
fn not_implemented(kind: &str, path: &NodePath) -> DataFusionError {
|
||||
DataFusionError::NotImplemented(format!(
|
||||
"the native runtime does not execute {kind} nodes yet: {kind} at {path}"
|
||||
))
|
||||
}
|
||||
|
||||
/// The error for a window whose evaluator this execution already closed.
|
||||
fn closed(path: &NodePath) -> DataFusionError {
|
||||
DataFusionError::Execution(format!("the window at {path} is already closed"))
|
||||
@@ -4293,11 +4329,27 @@ fn truncated_input(path: &NodePath) -> DataFusionError {
|
||||
))
|
||||
}
|
||||
|
||||
/// The rejection of a node kind this runtime does not execute yet.
|
||||
///
|
||||
/// It names the kind and the node's position: the position is what the metrics and `EXPLAIN` call
|
||||
/// the same node, so a reader can find it in the tree.
|
||||
fn not_implemented(kind: &str, path: &NodePath) -> DataFusionError {
|
||||
DataFusionError::NotImplemented(format!(
|
||||
"the native runtime does not execute {kind} nodes yet: {kind} at {path}"
|
||||
))
|
||||
}
|
||||
|
||||
/// The error for a node asked to compute without the runtime its kind holds.
|
||||
fn no_binary(path: &NodePath) -> DataFusionError {
|
||||
DataFusionError::Execution(format!("the node at {path} holds no binary operation"))
|
||||
}
|
||||
|
||||
/// The error for an aggregate node whose fold is missing.
|
||||
fn no_aggregate(path: &NodePath) -> DataFusionError {
|
||||
DataFusionError::Execution(format!("the node at {path} holds no aggregate"))
|
||||
}
|
||||
|
||||
/// The error for a node asked to run a vector operation without one.
|
||||
fn no_vector(path: &NodePath) -> DataFusionError {
|
||||
DataFusionError::Execution(format!("the node at {path} holds no vector operation"))
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ use datatypes::series::{Label, MetricName, SeriesOrdering};
|
||||
use super::{
|
||||
BinaryRuntime, Emitted, NativeChunk, NativeExecutionState, NativeExecutor, NodeBuffer,
|
||||
NodeRuntime, absorb_rebuilds, evaluated_bytes, grid_end_ms, map_chunk, operand_chunk_bytes,
|
||||
raw_bytes,
|
||||
raw_bytes, vector_workspace_bytes,
|
||||
};
|
||||
use crate::evaluated::EvaluatedChunk;
|
||||
use crate::grid_series::merge_same_labelset;
|
||||
@@ -45,8 +45,9 @@ use crate::instant_eval::select_instants;
|
||||
use crate::map::MathOp;
|
||||
use crate::native::metrics::{NodeMetric, NodeMetrics, NodePath};
|
||||
use crate::native::plan::{
|
||||
AggregateKind, AggregateStage, BoundAggregate, BoundInput, BoundInstant, BoundMap, BoundScalar,
|
||||
BoundVectorOp, BoundWindow, Grouping, NativeExpr, NativeNode, NativeTree, NativeType,
|
||||
AggregateKind, AggregateStage, AggregateStateSchema, BoundAggregate, BoundInput, BoundInstant,
|
||||
BoundMap, BoundScalar, BoundVectorOp, BoundWindow, Grouping, NativeExpr, NativeNode,
|
||||
NativeTree, NativeType,
|
||||
};
|
||||
use crate::range_eval::{RangeEvaluator, RangeFunction, RangeWindow};
|
||||
use crate::select::SortOrder;
|
||||
@@ -487,6 +488,7 @@ fn source_runtime(pool: &Arc<dyn MemoryPool>) -> NodeRuntime {
|
||||
window: None,
|
||||
map: None,
|
||||
binary: None,
|
||||
aggregate: None,
|
||||
broadcast: None,
|
||||
vector: None,
|
||||
source_schema: None,
|
||||
@@ -509,6 +511,7 @@ fn binary_runtime(pool: &Arc<dyn MemoryPool>, schema: &Arc<SeriesSchema>) -> Nod
|
||||
Arc::new(binary_matching(BinaryOp::Add)),
|
||||
[Arc::clone(schema), Arc::clone(schema)],
|
||||
)),
|
||||
aggregate: None,
|
||||
broadcast: None,
|
||||
vector: None,
|
||||
source_schema: None,
|
||||
@@ -1088,30 +1091,17 @@ fn a_downstream_window_emits_before_its_input_series_ends() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Every node kind the runtime does not execute is rejected, naming the kind and its position.
|
||||
/// Every binary shape the runtime does not execute is rejected, naming the kind and its position.
|
||||
///
|
||||
/// A scalar is not one of them any more: it is a value node, and a binary node over a constant is
|
||||
/// executed as a broadcast. The binary shapes that stay refused are the ones no kernel here takes,
|
||||
/// and the aggregate is still a kind this runtime does not run at all.
|
||||
/// executed as a broadcast. The binary shapes that stay refused are the ones no kernel here takes;
|
||||
/// the aggregate kinds and stages this runtime does not execute are refused by
|
||||
/// `an_aggregate_kind_or_stage_the_runtime_does_not_compute_is_named`.
|
||||
#[test]
|
||||
fn an_unimplemented_node_is_reported_not_panicked() {
|
||||
// One grid shared by the nodes that validate by allocation identity.
|
||||
let statement = || grid(0, 120_000, 2);
|
||||
|
||||
let aggregate_tree = {
|
||||
let out = statement();
|
||||
NativeNode {
|
||||
expr: NativeExpr::Aggregate {
|
||||
input: Box::new(window(input(), Arc::clone(&out))),
|
||||
op: BoundAggregate {
|
||||
kind: AggregateKind::Sum,
|
||||
group: Grouping::By(Arc::new(["job".into()])),
|
||||
stage: AggregateStage::Complete,
|
||||
},
|
||||
},
|
||||
output: NativeType::Samples(evaluated_schema(out)),
|
||||
}
|
||||
};
|
||||
// Two constants make a scalar, which is a legal node and a shape no binary here broadcasts
|
||||
// over: there is no vector to pair either of them with.
|
||||
let two_scalars_tree = {
|
||||
@@ -1143,12 +1133,6 @@ fn an_unimplemented_node_is_reported_not_panicked() {
|
||||
};
|
||||
|
||||
let cases = [
|
||||
(
|
||||
"Aggregate",
|
||||
"root",
|
||||
"does not execute Aggregate",
|
||||
aggregate_tree,
|
||||
),
|
||||
("Binary", "root", "two constants", two_scalars_tree),
|
||||
("Binary", "root", "matching modifiers", matched_scalar_tree),
|
||||
];
|
||||
@@ -6563,6 +6547,158 @@ fn a_constant_on_the_right_multiplies_every_value() {
|
||||
assert_eq!(execution.state().reserved_bytes(), 0);
|
||||
}
|
||||
|
||||
/// A `Complete` aggregate over `inner`, carrying `inner`'s fields and evaluated on its grid.
|
||||
///
|
||||
/// A `Complete` aggregate neither adds nor drops a field and lands on its input's grid, so the node's
|
||||
/// own output is the layout its input produces.
|
||||
fn aggregate(inner: NativeNode, op: BoundAggregate) -> NativeNode {
|
||||
let fields = inner
|
||||
.output
|
||||
.samples()
|
||||
.expect("an aggregate reads samples")
|
||||
.fields
|
||||
.clone();
|
||||
let grid = inner
|
||||
.output
|
||||
.grid()
|
||||
.expect("a complete aggregate reads an evaluated vector")
|
||||
.clone();
|
||||
let output = NativeType::Samples(Arc::new(SeriesSchema {
|
||||
fields,
|
||||
eval_grid: Some(grid),
|
||||
}));
|
||||
NativeNode {
|
||||
expr: NativeExpr::Aggregate {
|
||||
input: Box::new(inner),
|
||||
op,
|
||||
},
|
||||
output,
|
||||
}
|
||||
}
|
||||
|
||||
/// A `Complete` `sum` grouped as `group`.
|
||||
fn sum_grouping(group: Grouping) -> BoundAggregate {
|
||||
BoundAggregate {
|
||||
kind: AggregateKind::Sum,
|
||||
group,
|
||||
stage: AggregateStage::Complete,
|
||||
}
|
||||
}
|
||||
|
||||
/// `name{job=...,instance=...}`, the two labels these tests group by.
|
||||
fn job_instance_metric(name: &str, job: &str, instance: &str) -> MetricName {
|
||||
MetricName::try_new(
|
||||
Some(name.into()),
|
||||
vec![
|
||||
Label {
|
||||
name: "job".into(),
|
||||
value: job.into(),
|
||||
},
|
||||
Label {
|
||||
name: "instance".into(),
|
||||
value: instance.into(),
|
||||
},
|
||||
],
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// What one aggregate result reports: its name, its tags and the value at each point it decided.
|
||||
type GroupResult = (
|
||||
Option<String>,
|
||||
Vec<(String, String)>,
|
||||
Vec<(usize, Option<f64>)>,
|
||||
);
|
||||
|
||||
/// The groups a run produced, each with the absolute positions it decided, sorted by identity.
|
||||
///
|
||||
/// Sorting is the test's, not the runtime's: a group's place in the answer is the order its first
|
||||
/// input appeared, which the tests that check it assert on the stream rather than here.
|
||||
fn groups_of(chunks: &[EvaluatedChunk]) -> Vec<GroupResult> {
|
||||
let mut results: Vec<GroupResult> = chunks
|
||||
.iter()
|
||||
.map(|chunk| {
|
||||
let mut tags: Vec<(String, String)> = chunk
|
||||
.series
|
||||
.metric
|
||||
.tags()
|
||||
.iter()
|
||||
.map(|tag| (tag.name.to_string(), tag.value.to_string()))
|
||||
.collect();
|
||||
tags.sort();
|
||||
(
|
||||
chunk.series.metric.name().map(str::to_string),
|
||||
tags,
|
||||
decided(std::slice::from_ref(chunk)),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
results.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
results
|
||||
}
|
||||
|
||||
/// One group's answer under the name and tags `tags` give it, ordered by label name.
|
||||
fn grouped(tags: &[(&str, &str)], points: Vec<(usize, Option<f64>)>) -> GroupResult {
|
||||
let mut tags: Vec<(String, String)> = tags
|
||||
.iter()
|
||||
.map(|(name, value)| ((*name).to_string(), (*value).to_string()))
|
||||
.collect();
|
||||
tags.sort();
|
||||
(None, tags, points)
|
||||
}
|
||||
|
||||
/// One group's answer when every label of its input was dropped.
|
||||
fn unlabelled(points: Vec<(usize, Option<f64>)>) -> GroupResult {
|
||||
grouped(&[], points)
|
||||
}
|
||||
|
||||
/// `sum` with no grouping drops every label, name and tags alike, and adds the members per point.
|
||||
#[test]
|
||||
fn an_aggregate_drops_every_label_and_adds_its_group_per_point() {
|
||||
let grid = grid(0, 60_000, 4);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(aggregate(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
sum_grouping(Grouping::By(Arc::new([]))),
|
||||
))
|
||||
.unwrap();
|
||||
let set = ExecutionPlanMetricsSet::new();
|
||||
let mut execution = executor(&tree, pool(), &set);
|
||||
for (metric, values) in [
|
||||
(
|
||||
job_instance_metric("requests", "api", "a"),
|
||||
vec![Some(1.0), Some(2.0), Some(3.0), Some(4.0)],
|
||||
),
|
||||
(
|
||||
job_instance_metric("requests", "db", "b"),
|
||||
vec![Some(10.0), Some(20.0), Some(30.0), Some(40.0)],
|
||||
),
|
||||
] {
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(&schema, metric, 0, values, true))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let output = execution.finish().unwrap();
|
||||
assert_eq!(
|
||||
groups_of(&output),
|
||||
vec![unlabelled(vec![
|
||||
(0, Some(11.0)),
|
||||
(1, Some(22.0)),
|
||||
(2, Some(33.0)),
|
||||
(3, Some(44.0)),
|
||||
])],
|
||||
"the two series were not folded into one unlabelled group"
|
||||
);
|
||||
// One group is one series, so it closes exactly once and its chunk says so.
|
||||
assert!(output.len() == 1 && output[0].is_last);
|
||||
assert_eq!(
|
||||
count(&set, &NodePath::root(), NodeMetric::COMPLETED_SERIES),
|
||||
1
|
||||
);
|
||||
assert_eq!(execution.state().reserved_bytes(), 0);
|
||||
}
|
||||
|
||||
/// The side a constant is on decides which operand the vector's value is.
|
||||
///
|
||||
/// Division is not commutative, so `x / 100` and `100 / x` answering different values is what says
|
||||
@@ -7036,6 +7172,208 @@ fn a_window_above_a_broadcast_merges_the_series_it_collapses() {
|
||||
assert_eq!(execution.state().reserved_bytes(), 0);
|
||||
}
|
||||
|
||||
/// Every supported `Complete` kind reaches its matching float kernel through the native runtime.
|
||||
#[test]
|
||||
fn complete_aggregate_kinds_fold_two_values() {
|
||||
let cases = [
|
||||
(AggregateKind::Sum, 4.0),
|
||||
(AggregateKind::Count, 2.0),
|
||||
(AggregateKind::Avg, 2.0),
|
||||
(AggregateKind::Min, 1.0),
|
||||
(AggregateKind::Max, 3.0),
|
||||
(AggregateKind::Group, 1.0),
|
||||
(AggregateKind::Stddev, 1.0),
|
||||
(AggregateKind::Stdvar, 1.0),
|
||||
];
|
||||
|
||||
for (kind, expected) in cases {
|
||||
let grid = grid(0, 60_000, 1);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(aggregate(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
BoundAggregate {
|
||||
kind,
|
||||
group: Grouping::By(Arc::new([])),
|
||||
stage: AggregateStage::Complete,
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new());
|
||||
for value in [1.0, 3.0] {
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
metric(),
|
||||
0,
|
||||
vec![Some(value)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let output = execution.finish().unwrap();
|
||||
assert_eq!(
|
||||
groups_of(&output),
|
||||
vec![unlabelled(vec![(0, Some(expected))])],
|
||||
"{kind:?} did not compute its native complete result"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The same series, delivered in two chunks, fold to what the whole series folds to.
|
||||
#[test]
|
||||
fn an_aggregate_folds_a_series_split_across_chunks_like_a_whole_one() {
|
||||
let bound = grid(0, 60_000, 4);
|
||||
let schema = evaluated_schema(Arc::clone(&bound));
|
||||
// A fresh tree per execution: the node's state is the execution's, not the tree's.
|
||||
let tree = || {
|
||||
NativeTree::try_new(aggregate(
|
||||
evaluated_input(grid(0, 60_000, 4)),
|
||||
sum_grouping(Grouping::By(Arc::new(["job".into()]))),
|
||||
))
|
||||
.unwrap()
|
||||
};
|
||||
let (first, second) = (
|
||||
job_instance_metric("requests", "api", "a"),
|
||||
job_instance_metric("requests", "api", "b"),
|
||||
);
|
||||
|
||||
// One chunk per series.
|
||||
let mut whole = executor(&tree(), pool(), &ExecutionPlanMetricsSet::new());
|
||||
for (metric, values) in [
|
||||
(
|
||||
first.clone(),
|
||||
vec![Some(1.0), Some(2.0), Some(3.0), Some(4.0)],
|
||||
),
|
||||
(
|
||||
second.clone(),
|
||||
vec![Some(10.0), Some(20.0), Some(30.0), Some(40.0)],
|
||||
),
|
||||
] {
|
||||
whole
|
||||
.push_evaluated(evaluated_chunk_of(&schema, metric, 0, values, true))
|
||||
.unwrap();
|
||||
}
|
||||
let whole = whole.finish().unwrap();
|
||||
|
||||
// The first series split in two, the second whole: the group is the same group, folded from
|
||||
// positions the chunks name absolutely.
|
||||
let mut split = executor(&tree(), pool(), &ExecutionPlanMetricsSet::new());
|
||||
split
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
first.clone(),
|
||||
0,
|
||||
vec![Some(1.0), Some(2.0)],
|
||||
false,
|
||||
))
|
||||
.unwrap();
|
||||
assert!(
|
||||
split.next_chunk().unwrap().is_none(),
|
||||
"a chunk that does not close its series answered for the group"
|
||||
);
|
||||
split
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
first,
|
||||
2,
|
||||
vec![Some(3.0), Some(4.0)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
split
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
second,
|
||||
0,
|
||||
vec![Some(10.0), Some(20.0), Some(30.0), Some(40.0)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
let split = split.finish().unwrap();
|
||||
assert_eq!(groups_of(&split), groups_of(&whole));
|
||||
assert_eq!(
|
||||
groups_of(&split),
|
||||
vec![grouped(
|
||||
&[("job", "api")],
|
||||
vec![
|
||||
(0, Some(11.0)),
|
||||
(1, Some(22.0)),
|
||||
(2, Some(33.0)),
|
||||
(3, Some(44.0)),
|
||||
],
|
||||
)],
|
||||
"the split series was folded as two groups or at the wrong positions"
|
||||
);
|
||||
}
|
||||
|
||||
/// A point no member of a group has a value at is not a point of the group's result.
|
||||
///
|
||||
/// Sparse input is what says the fold indexes its states by absolute position: a chunk holding
|
||||
/// positions 0 and 1 and another holding 1 and 2 leave position 3 to nothing, and the hole between
|
||||
/// the rows is never filled with a value.
|
||||
#[test]
|
||||
fn an_aggregate_reports_only_the_points_a_group_holds_a_value_at() {
|
||||
let grid = grid(0, 60_000, 4);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(aggregate(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
sum_grouping(Grouping::By(Arc::new([]))),
|
||||
))
|
||||
.unwrap();
|
||||
let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new());
|
||||
// Positions 0 and 1 of the first series, 1 and 2 of the second, and nothing at 3.
|
||||
execution
|
||||
.push_evaluated(sparse_chunk(
|
||||
&schema,
|
||||
job_instance_metric("requests", "api", "a"),
|
||||
&[(0, 2)],
|
||||
vec![1.0, 2.0],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
execution
|
||||
.push_evaluated(sparse_chunk(
|
||||
&schema,
|
||||
job_instance_metric("requests", "db", "b"),
|
||||
&[(1, 2)],
|
||||
vec![20.0, 300.0],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
let output = execution.finish().unwrap();
|
||||
assert_eq!(
|
||||
groups_of(&output),
|
||||
vec![
|
||||
// Position 1 is the only one both members hold, and it is their sum.
|
||||
unlabelled(vec![(0, Some(1.0)), (1, Some(22.0)), (2, Some(300.0))]),
|
||||
],
|
||||
"the group was answered at a point no member holds, or summed the wrong members"
|
||||
);
|
||||
}
|
||||
|
||||
/// An aggregate handed no series produces nothing at all: no group, so no empty series either.
|
||||
#[test]
|
||||
fn an_aggregate_over_no_series_produces_nothing() {
|
||||
let grid = grid(0, 60_000, 4);
|
||||
let tree = NativeTree::try_new(aggregate(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
sum_grouping(Grouping::By(Arc::new(["job".into()]))),
|
||||
))
|
||||
.unwrap();
|
||||
let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new());
|
||||
let output = execution.finish().unwrap();
|
||||
assert!(
|
||||
output.is_empty(),
|
||||
"an aggregate with no input answered {} series",
|
||||
output.len()
|
||||
);
|
||||
assert_eq!(execution.state().reserved_bytes(), 0);
|
||||
}
|
||||
|
||||
/// A broadcast hands one chunk over for every chunk it reads, and closes the series once.
|
||||
///
|
||||
/// The metrics are what says the chunk-per-chunk behaviour held: one input batch per chunk read, one
|
||||
@@ -7251,6 +7589,411 @@ fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reserv
|
||||
);
|
||||
}
|
||||
|
||||
/// A series that ended is one member of a group, not the group: nothing is answered until the input
|
||||
/// ends.
|
||||
///
|
||||
/// The observable state between the two is what this pins: the fold has read the closing chunk of
|
||||
/// the first series and answered nothing, and `computed` still says so.
|
||||
#[test]
|
||||
fn an_aggregate_answers_nothing_until_its_whole_input_ends() {
|
||||
let grid = grid(0, 60_000, 4);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(aggregate(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
sum_grouping(Grouping::By(Arc::new([]))),
|
||||
))
|
||||
.unwrap();
|
||||
let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new());
|
||||
let computed = |execution: &NativeExecutor| {
|
||||
execution.state().runtimes[0]
|
||||
.aggregate
|
||||
.as_ref()
|
||||
.expect("the root holds an aggregate")
|
||||
.computed
|
||||
};
|
||||
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
job_instance_metric("requests", "api", "a"),
|
||||
0,
|
||||
vec![Some(1.0), Some(2.0), Some(3.0), Some(4.0)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
assert!(execution.next_chunk().unwrap().is_none());
|
||||
assert!(!computed(&execution), "the fold finished on one series");
|
||||
assert!(
|
||||
execution.next_chunk().unwrap().is_none(),
|
||||
"the closing chunk of one series answered for its group"
|
||||
);
|
||||
// The second series closed too, and the input still has not ended.
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
job_instance_metric("requests", "db", "b"),
|
||||
0,
|
||||
vec![Some(10.0), Some(20.0), Some(30.0), Some(40.0)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
assert!(
|
||||
!computed(&execution),
|
||||
"the fold finished before the input ended"
|
||||
);
|
||||
assert!(
|
||||
execution.next_chunk().unwrap().is_none(),
|
||||
"every series closed, and the group was answered before the input ended"
|
||||
);
|
||||
|
||||
execution.end_of_input().unwrap();
|
||||
assert!(
|
||||
!computed(&execution),
|
||||
"ending the input finished the fold before anything pulled for it"
|
||||
);
|
||||
let mut output = Vec::new();
|
||||
while let Some(chunk) = execution.next_chunk().unwrap() {
|
||||
output.push(chunk);
|
||||
}
|
||||
assert_eq!(
|
||||
groups_of(&output),
|
||||
vec![unlabelled(vec![
|
||||
(0, Some(11.0)),
|
||||
(1, Some(22.0)),
|
||||
(2, Some(33.0)),
|
||||
(3, Some(44.0)),
|
||||
])],
|
||||
"the group was not answered once the input ended"
|
||||
);
|
||||
}
|
||||
|
||||
/// A chunk of another layout is refused, naming the aggregate and its position, and the execution
|
||||
/// stops with nothing reserved.
|
||||
///
|
||||
/// The fold indexes its states by absolute position and by field, so a chunk of another field count
|
||||
/// would be folded into states that mean something else. It is refused rather than folded, and what
|
||||
/// the refusal leaves behind is a stopped execution whose reservation went back to the pool.
|
||||
#[test]
|
||||
fn an_aggregate_refuses_a_chunk_of_another_layout_and_returns_its_reservation() {
|
||||
let grid = grid(0, 60_000, 4);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(aggregate(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
sum_grouping(Grouping::By(Arc::new([]))),
|
||||
))
|
||||
.unwrap();
|
||||
let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new());
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
metric(),
|
||||
0,
|
||||
vec![Some(1.0), Some(2.0), Some(3.0), Some(4.0)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
// Two fields where the tree bound one: the source node accepts it, the fold refuses it.
|
||||
let wider = Arc::new(SeriesSchema {
|
||||
fields: field_count(2),
|
||||
eval_grid: Some(Arc::clone(&grid)),
|
||||
});
|
||||
execution
|
||||
.push_evaluated(evaluated_columns(
|
||||
&wider,
|
||||
metric(),
|
||||
0,
|
||||
vec![vec![Some(1.0); 4], vec![Some(2.0); 4]],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
let error = execution
|
||||
.next_chunk()
|
||||
.expect_err("a chunk of another field count was folded");
|
||||
assert!(
|
||||
matches!(error, DataFusionError::Execution(_)),
|
||||
"the refusal is not an execution error: {error}"
|
||||
);
|
||||
let message = error.to_string();
|
||||
assert!(
|
||||
message.contains("aggregate at root"),
|
||||
"the refusal does not name the aggregate: {message}"
|
||||
);
|
||||
assert!(
|
||||
message.contains("field count"),
|
||||
"the refusal does not say what disagreed: {message}"
|
||||
);
|
||||
assert_eq!(execution.state().reserved_bytes(), 0);
|
||||
// A stopped execution is not resumed: the fold must not be asked to continue after this.
|
||||
assert!(execution.next_chunk().is_err());
|
||||
}
|
||||
|
||||
/// A pool rejection while the first aggregate block is admitted releases the source and fold.
|
||||
#[test]
|
||||
fn aggregate_budget_rejects_first_block_and_releases_everything() {
|
||||
let grid = grid(0, 1, 4);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(aggregate(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
sum_grouping(Grouping::By(Arc::new([]))),
|
||||
))
|
||||
.unwrap();
|
||||
let pool = bounded_pool(2_000);
|
||||
let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new());
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
metric(),
|
||||
0,
|
||||
vec![Some(2.0)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
assert!(execution.next_chunk().is_err());
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
assert_eq!(execution.state().reserved_bytes(), 0);
|
||||
}
|
||||
|
||||
/// The aggregate runtime charges its output queue, releases each output as it is pulled, and keeps
|
||||
/// queue backing until the runtime itself is released.
|
||||
#[test]
|
||||
fn aggregate_runtime_queue_keeps_only_backing_after_pops_until_release() {
|
||||
let grid = grid(0, 1, 4);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(aggregate(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
sum_grouping(Grouping::By(Arc::new(["job".into()]))),
|
||||
))
|
||||
.unwrap();
|
||||
let pool = bounded_pool(1 << 20);
|
||||
let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new());
|
||||
for (job, value) in [("api", 1.0), ("db", 2.0)] {
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
job_instance_metric("requests", job, "a"),
|
||||
0,
|
||||
vec![Some(value)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
execution.end_of_input().unwrap();
|
||||
let first = execution.next_chunk().unwrap().expect("first group output");
|
||||
assert_eq!(first.series.time.positions().count(), 1);
|
||||
let runtime = &execution.state().runtimes[0];
|
||||
let entry = std::mem::size_of::<(NativeChunk, usize)>();
|
||||
assert_eq!(runtime.queued.len(), 1);
|
||||
let remaining = runtime.queued[0].1;
|
||||
let expected = runtime.queued.capacity() * entry + remaining;
|
||||
assert_eq!(runtime.buffer.reservation.size(), expected);
|
||||
|
||||
let second = execution
|
||||
.next_chunk()
|
||||
.unwrap()
|
||||
.expect("second group output");
|
||||
assert_eq!(second.series.time.positions().count(), 1);
|
||||
let runtime = &execution.state().runtimes[0];
|
||||
assert!(runtime.queued.is_empty());
|
||||
let backing = runtime.queued.capacity() * entry;
|
||||
assert!(backing > 0);
|
||||
assert_eq!(runtime.buffer.reservation.size(), backing);
|
||||
assert!(execution.next_chunk().unwrap().is_none());
|
||||
// Pulling the terminal None releases the queue backing; only explicit close checks the
|
||||
// runtime's final cleanup below.
|
||||
assert_eq!(execution.state().runtimes[0].buffer.reservation.size(), 0);
|
||||
|
||||
execution.state.close().unwrap();
|
||||
assert_eq!(execution.state().runtimes[0].queued.capacity(), 0);
|
||||
assert_eq!(execution.state().runtimes[0].buffer.reservation.size(), 0);
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
/// An aggregate whose sparse output has a few points on a large grid can finish in a small pool.
|
||||
#[test]
|
||||
fn aggregate_sparse_output_budget_uses_rows_not_grid_length() {
|
||||
let grid = grid(0, 1, 100_000);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(aggregate(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
sum_grouping(Grouping::By(Arc::new([]))),
|
||||
))
|
||||
.unwrap();
|
||||
let pool = bounded_pool(64 * 1024);
|
||||
let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new());
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
metric(),
|
||||
10,
|
||||
vec![Some(2.0)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
let output = execution.finish().unwrap();
|
||||
assert_eq!(groups_of(&output), vec![unlabelled(vec![(10, Some(2.0))])]);
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
/// An aggregate retains its fold before finish and release returns its reservation.
|
||||
#[test]
|
||||
fn aggregate_fold_state_is_released_after_input() {
|
||||
let grid = grid(0, 1, 4);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(aggregate(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
sum_grouping(Grouping::By(Arc::new([]))),
|
||||
))
|
||||
.unwrap();
|
||||
let pool = bounded_pool(1 << 20);
|
||||
let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new());
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
metric(),
|
||||
0,
|
||||
vec![Some(2.0)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
assert!(execution.next_chunk().unwrap().is_none());
|
||||
let runtime = &execution.state.runtimes[0];
|
||||
let aggregate = runtime.aggregate.as_ref().unwrap();
|
||||
assert_eq!(aggregate.groups(), 1);
|
||||
assert!(!aggregate.computed);
|
||||
assert!(pool.reserved() > 0);
|
||||
execution.state.close().unwrap();
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
/// An aggregate kind or stage this runtime does not compute is refused, naming what was found.
|
||||
#[test]
|
||||
fn an_aggregate_kind_or_stage_the_runtime_does_not_compute_is_named() {
|
||||
let grid = || grid(0, 60_000, 4);
|
||||
let kind_tree = NativeTree::try_new(aggregate(
|
||||
evaluated_input(grid()),
|
||||
BoundAggregate {
|
||||
kind: AggregateKind::Quantile { quantile: 0.5 },
|
||||
group: Grouping::By(Arc::new([])),
|
||||
stage: AggregateStage::Complete,
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
let partial_tree = {
|
||||
let out = grid();
|
||||
let group = Grouping::By(Arc::new(["job".into()]));
|
||||
let state = Arc::new(AggregateStateSchema {
|
||||
group: group.clone(),
|
||||
accumulators: fields(),
|
||||
grid: Arc::clone(&out),
|
||||
});
|
||||
NativeTree::try_new(NativeNode {
|
||||
expr: NativeExpr::Aggregate {
|
||||
input: Box::new(evaluated_input(Arc::clone(&out))),
|
||||
op: BoundAggregate {
|
||||
kind: AggregateKind::Sum,
|
||||
group,
|
||||
stage: AggregateStage::Partial,
|
||||
},
|
||||
},
|
||||
output: NativeType::AggregateState(state),
|
||||
})
|
||||
.unwrap()
|
||||
};
|
||||
|
||||
for (found, tree) in [("Quantile", kind_tree), ("Partial", partial_tree)] {
|
||||
let error = match NativeExecutionState::try_new(
|
||||
&tree,
|
||||
pool(),
|
||||
&ExecutionPlanMetricsSet::new(),
|
||||
0,
|
||||
) {
|
||||
Ok(_) => panic!("the runtime accepted a {found} aggregate"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(
|
||||
matches!(error, DataFusionError::NotImplemented(_)),
|
||||
"{found}: {error}"
|
||||
);
|
||||
let message = error.to_string();
|
||||
assert!(message.contains(found), "{found}: {message}");
|
||||
// A `Partial` rejection has to say why it is not computed: the stages it would need.
|
||||
if found == "Partial" {
|
||||
assert!(message.contains("Final"), "{message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An aggregate above a window that collects and merges its input waits for that merge.
|
||||
///
|
||||
/// The multiplication drops the metric name, so the two input series land on one label set and the
|
||||
/// `last_over_time` above them has to collect and merge before it evaluates. The aggregate above the
|
||||
/// window must wait for that merge: its guard is `inputs_finished && settled(child)`, and with only
|
||||
/// the first half it finalizes on the pull that follows the caller's last chunk, before the window
|
||||
/// has answered.
|
||||
#[test]
|
||||
fn an_aggregate_above_a_collecting_window_waits_for_the_merge() {
|
||||
let grid = grid(0, 60_000, 4);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = || {
|
||||
NativeTree::try_new(aggregate(
|
||||
window_with(
|
||||
broadcast_right(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
2.0,
|
||||
BinaryOp::Mul,
|
||||
false,
|
||||
),
|
||||
RangeFunction::LastOverTime,
|
||||
60_000,
|
||||
Some(Arc::clone(&grid)),
|
||||
Arc::clone(&grid),
|
||||
),
|
||||
sum_grouping(Grouping::By(Arc::new([]))),
|
||||
))
|
||||
.unwrap()
|
||||
};
|
||||
|
||||
let mut execution = executor(&tree(), pool(), &ExecutionPlanMetricsSet::new());
|
||||
// Two series that only differ by name, each holding half of the grid the other leaves empty:
|
||||
// after the name is dropped they are one series only once merged.
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
named_metric("requests", "x"),
|
||||
0,
|
||||
vec![Some(1.0), Some(2.0)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
assert!(
|
||||
execution.next_chunk().unwrap().is_none(),
|
||||
"the aggregate answered before the window above it had merged"
|
||||
);
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
named_metric("requests_total", "x"),
|
||||
2,
|
||||
vec![Some(3.0), Some(4.0)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
let out = run(&mut execution);
|
||||
assert_eq!(
|
||||
decided(&out),
|
||||
vec![
|
||||
(0, Some(2.0)),
|
||||
(1, Some(4.0)),
|
||||
(2, Some(6.0)),
|
||||
(3, Some(8.0)),
|
||||
],
|
||||
"the aggregate did not fold the merged window result"
|
||||
);
|
||||
assert_eq!(execution.state().reserved_bytes(), 0);
|
||||
}
|
||||
|
||||
fn vector_op(inner: NativeNode, op: BoundVectorOp) -> NativeNode {
|
||||
let output = inner.output.clone();
|
||||
NativeNode {
|
||||
@@ -7702,3 +8445,83 @@ fn histogram_quantile_reads_the_pieces_of_one_label_set_as_one_series() {
|
||||
.collect();
|
||||
assert_eq!(positions, [0, 1]);
|
||||
}
|
||||
|
||||
/// `count_values` counts the samples that share a value across the whole vector at a point, under
|
||||
/// a label set per value; `a` arrives in two chunks and counts at both points.
|
||||
#[test]
|
||||
fn count_values_counts_the_samples_sharing_a_value() {
|
||||
let grid = grid(0, 60_000, 2);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(vector_op(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
BoundVectorOp::CountValues {
|
||||
label: "v".into(),
|
||||
group: Grouping::By(Arc::from([])),
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new());
|
||||
for (job, start, value, is_last) in [
|
||||
("a", 0, 1.0, false),
|
||||
("a", 1, 1.0, true),
|
||||
("b", 0, 2.0, true),
|
||||
("c", 0, 1.0, true),
|
||||
] {
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
named_metric("m", job),
|
||||
start,
|
||||
vec![Some(value)],
|
||||
is_last,
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
let mut counts: Vec<(String, Decided)> = run(&mut execution)
|
||||
.iter()
|
||||
.map(|chunk| {
|
||||
(
|
||||
chunk.series.metric.get("v").unwrap().to_string(),
|
||||
decided(std::slice::from_ref(chunk)),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
counts.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
assert_eq!(
|
||||
counts,
|
||||
[
|
||||
("1".to_string(), vec![(0, Some(2.0)), (1, Some(1.0))]),
|
||||
("2".to_string(), vec![(0, Some(1.0))]),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// Every field's distinct values make `count_values` groups of their own, so its estimate grows
|
||||
/// with the field count, not only with the rows: a row of four fields reserves at least four
|
||||
/// times what a row of one field does.
|
||||
#[test]
|
||||
fn count_values_reserves_a_group_per_field_sample() {
|
||||
let grid = grid(0, 60_000, 1);
|
||||
let op = BoundVectorOp::CountValues {
|
||||
label: "v".into(),
|
||||
group: Grouping::By(Arc::from([])),
|
||||
};
|
||||
let estimate = |fields: usize| {
|
||||
let schema = SeriesSchema {
|
||||
fields: field_count(fields),
|
||||
eval_grid: Some(Arc::clone(&grid)),
|
||||
};
|
||||
let samples = (0..fields)
|
||||
.map(|field| Samples::from_float(Float64Array::from(vec![field as f64])))
|
||||
.collect();
|
||||
let series = Series::try_new(
|
||||
&schema,
|
||||
named_metric("m", "a"),
|
||||
TimeAxis::Grid(GridRange::try_new(Arc::clone(&grid), 0, 1).unwrap()),
|
||||
SeriesFields::try_from_vec(samples).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
vector_workspace_bytes(&op, &schema, &[series])
|
||||
};
|
||||
assert!(estimate(4) >= 4 * estimate(1));
|
||||
}
|
||||
|
||||
@@ -27,8 +27,10 @@
|
||||
//! - A series closes exactly once, on the chunk that says so, so a consumer can tell "nothing more
|
||||
//! here" from "no more input".
|
||||
|
||||
mod aggregate;
|
||||
pub mod batch;
|
||||
mod display;
|
||||
pub mod execute;
|
||||
pub mod metrics;
|
||||
pub mod plan;
|
||||
pub(crate) mod runs;
|
||||
|
||||
@@ -259,7 +259,7 @@ pub struct BoundAggregate {
|
||||
|
||||
/// An operator over the whole input vector at each grid point: one that keeps some of the input's
|
||||
/// samples with their series' labels and values, orders the series, reports where none is present,
|
||||
/// or reads a classic histogram from its bucket series.
|
||||
/// reads a classic histogram from its bucket series, or counts the samples sharing a value.
|
||||
///
|
||||
/// Its constants are bound here, as a window's are: a `k` or a ratio only known at run time has no
|
||||
/// field.
|
||||
@@ -282,6 +282,9 @@ pub enum BoundVectorOp {
|
||||
/// `histogram_quantile` or `histogram_fraction`: each classic histogram is read from the bucket
|
||||
/// series that share its labels, so this too reads the whole vector at a point.
|
||||
Histogram(HistogramFunction),
|
||||
/// `count_values(label, v)`: how many samples of each group share a value, under a label set
|
||||
/// per value. It writes a label per distinct value, which an aggregate's fixed groups cannot.
|
||||
CountValues { label: Arc<str>, group: Grouping },
|
||||
}
|
||||
|
||||
/// What a node computes. Every variant holds a bound operation, never a name to resolve.
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The grid positions a field occupies, as sorted, disjoint runs.
|
||||
//!
|
||||
//! A node that rewrites labels — a `Map` dropping the metric name, a `Window` dropping the name and
|
||||
//! the empty-valued labels — can turn two input series into one label set, and every layer above
|
||||
//! then reads the two chunks as one series. What decides whether that pair is legal is which
|
||||
//! positions of each field already hold a value: two values of one field at one position cannot be
|
||||
//! merged away, while a chunk that fills positions the set left empty merges cleanly, as
|
||||
//! [`crate::grid_series::merge_same_labelset`] does.
|
||||
//!
|
||||
//! The state is kept per label set, per field, as runs, so a chunk of many positions costs one run
|
||||
//! rather than one entry per position. Keeping it in one place is what keeps the two layers that
|
||||
//! check it — the native runtime's own [`crate::native::execute::Emitted`] table and the
|
||||
//! cross-partition [`crate::extension_plan::fused_series::LabelSets`] — from drifting apart: both
|
||||
//! ask the same functions for the same answers, so the fast path one of them earned is the fast path
|
||||
//! the other gets.
|
||||
|
||||
/// One run of grid positions, half-open, that hold a value in one field.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) struct Run {
|
||||
pub(crate) start: usize,
|
||||
pub(crate) end: usize,
|
||||
}
|
||||
|
||||
impl Run {
|
||||
/// Whether two runs share a position.
|
||||
pub(crate) fn overlaps(self, other: Self) -> bool {
|
||||
self.start < other.end && other.start < self.end
|
||||
}
|
||||
}
|
||||
|
||||
// How many times `absorb` rebuilt the stored runs instead of appending to them.
|
||||
//
|
||||
// Test-only, and per thread: the runs stored afterwards are the same either way, so a rebuild shows
|
||||
// only in what it costs, and a test that reads its own thread's figure is not disturbed by another
|
||||
// test's chunks.
|
||||
#[cfg(test)]
|
||||
thread_local! {
|
||||
static ABSORB_REBUILDS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
|
||||
}
|
||||
|
||||
/// How many times this thread's [`absorb`] rebuilt the stored runs.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn absorb_rebuilds() -> usize {
|
||||
ABSORB_REBUILDS.with(std::cell::Cell::get)
|
||||
}
|
||||
|
||||
/// Merges `incoming` into `stored`, keeping the result in position order and joining the runs that
|
||||
/// touch.
|
||||
///
|
||||
/// Within a chunk one pass over both sides is what keeps this linear, since each side is sorted and
|
||||
/// disjoint. Adding the runs one at a time and rebuilding the array after each would cost the square
|
||||
/// of the runs a chunk yields: a chunk of one hundred thousand positions alternating between values
|
||||
/// and nulls yields fifty thousand runs, and re-merging after each is over a billion element moves
|
||||
/// for a single series.
|
||||
///
|
||||
/// One pass per chunk is still a pass over everything the series has stored so far, so it is not
|
||||
/// free once the *history* grows across chunks: a series split into `K` chunks of one position each,
|
||||
/// a hole apart, stores one more run per chunk and moves `K(K+1)/2` of them, with no label collision
|
||||
/// and no illegal input anywhere. A chunk that starts no earlier than the end of the last stored run
|
||||
/// only ever adds to the end — the stored runs are sorted and disjoint, so every one of them ends
|
||||
/// before anything this chunk holds — and it is appended instead, which is amortised constant per
|
||||
/// run. That is the shape a scan hands over as its positions move forward chunk after chunk. A chunk
|
||||
/// that starts before the last stored run can split a run or reach back past it, so it still costs
|
||||
/// the full merge.
|
||||
///
|
||||
/// Either way the result is sorted, disjoint, and has the runs that touch joined, which is what a
|
||||
/// look-up by [`Run::overlaps`] on the runs stored here relies on.
|
||||
pub(crate) fn absorb(stored: &mut Vec<Run>, incoming: &[Run]) {
|
||||
if incoming.is_empty() {
|
||||
return;
|
||||
}
|
||||
if stored
|
||||
.last()
|
||||
.is_none_or(|last| incoming[0].start >= last.end)
|
||||
{
|
||||
append_runs(stored, incoming);
|
||||
} else {
|
||||
merge_runs(stored, incoming);
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends `incoming` to `stored`, all of which lies past the last stored run.
|
||||
///
|
||||
/// Nothing already stored has to be placed again: the two sides are each in position order and
|
||||
/// nothing `incoming` holds can reach back into what is stored, so its runs are pushed on the end and
|
||||
/// the vector's own growth is the only move — amortised over the runs appended. Where its first run
|
||||
/// touches the end of the last stored run the two are one run, as they are after a merge, so it is
|
||||
/// joined rather than left beside it.
|
||||
fn append_runs(stored: &mut Vec<Run>, incoming: &[Run]) {
|
||||
let mut incoming = incoming;
|
||||
if let Some(last) = stored.last_mut()
|
||||
&& let Some(first) = incoming.first()
|
||||
&& first.start <= last.end
|
||||
{
|
||||
last.end = last.end.max(first.end);
|
||||
incoming = &incoming[1..];
|
||||
}
|
||||
stored.extend_from_slice(incoming);
|
||||
}
|
||||
|
||||
/// Merges `incoming` into `stored`, where the two can interleave.
|
||||
///
|
||||
/// One pass is enough because each side is sorted and disjoint: whichever run starts first is
|
||||
/// placed, and it joins the run before it where the two touch.
|
||||
fn merge_runs(stored: &mut Vec<Run>, incoming: &[Run]) {
|
||||
#[cfg(test)]
|
||||
ABSORB_REBUILDS.with(|rebuilds| rebuilds.set(rebuilds.get() + 1));
|
||||
let mut merged: Vec<Run> = Vec::with_capacity(stored.len() + incoming.len());
|
||||
let (mut a, mut b) = (0, 0);
|
||||
while a < stored.len() || b < incoming.len() {
|
||||
let larger = match (stored.get(a), incoming.get(b)) {
|
||||
(Some(left), Some(right)) if right.start < left.start => {
|
||||
b += 1;
|
||||
*right
|
||||
}
|
||||
(Some(left), _) => {
|
||||
a += 1;
|
||||
*left
|
||||
}
|
||||
(None, Some(right)) => {
|
||||
b += 1;
|
||||
*right
|
||||
}
|
||||
(None, None) => break,
|
||||
};
|
||||
match merged.last_mut() {
|
||||
Some(last) if larger.start <= last.end => last.end = last.end.max(larger.end),
|
||||
_ => merged.push(larger),
|
||||
}
|
||||
}
|
||||
*stored = merged;
|
||||
}
|
||||
@@ -13,17 +13,17 @@
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, UNIX_EPOCH};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use datafusion::error::DataFusionError;
|
||||
use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool};
|
||||
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
|
||||
use datatypes::arrow::array::Float64Array;
|
||||
use datatypes::arrow::buffer::{Buffer, ScalarBuffer};
|
||||
use datatypes::series::MetricName;
|
||||
use datatypes::series::SeriesOrdering;
|
||||
use datatypes::series::raw::{RawSeriesChunk, RawTimestamps};
|
||||
use datatypes::series::samples::{FieldSpec, SampleRef, Samples, SeriesFields, SeriesSchema};
|
||||
use datatypes::series::{Label, MetricName};
|
||||
use promql::evaluated::EvaluatedChunk;
|
||||
use promql::map::MathOp;
|
||||
use promql::native::execute::{NativeExecutionState, NativeExecutor};
|
||||
@@ -3138,11 +3138,25 @@ fn large_subquery_child_grids_compile_deferred() {
|
||||
assert_eq!(grid.2, range_ms);
|
||||
}
|
||||
|
||||
/// The largest millisecond endpoint the platform can represent as a `SystemTime`.
|
||||
///
|
||||
/// The tests below need an endpoint close enough to `i64::MAX` milliseconds that a one-second
|
||||
/// rounding or a one-millisecond offset crosses the representable range. Unix-like platforms can
|
||||
/// hold that instant; Windows represents `SystemTime` as a 64-bit count of 100-nanosecond
|
||||
/// intervals since 1601, so the same instant does not fit and `UNIX_EPOCH + duration` panics. The
|
||||
/// behaviour under test is about arithmetic on the millisecond value, not about the platform's
|
||||
/// clock range, so a platform that cannot represent the instant skips instead of failing.
|
||||
fn extreme_millisecond_endpoint() -> Option<SystemTime> {
|
||||
UNIX_EPOCH.checked_add(Duration::from_millis(i64::MAX as u64))
|
||||
}
|
||||
|
||||
/// A sampled child endpoint can fit even when the unrounded read-window endpoint does not.
|
||||
#[test]
|
||||
fn a_subquery_accepts_an_in_range_sampled_endpoint() {
|
||||
let Some(endpoint) = extreme_millisecond_endpoint() else {
|
||||
return;
|
||||
};
|
||||
let mut stmt = eval_stmt("rate(x[1s:1s] offset -1ms)");
|
||||
let endpoint = UNIX_EPOCH + Duration::from_millis(i64::MAX as u64);
|
||||
stmt.start = endpoint;
|
||||
stmt.end = endpoint;
|
||||
stmt.interval = Duration::from_millis(1);
|
||||
@@ -3161,6 +3175,9 @@ fn a_subquery_accepts_an_in_range_sampled_endpoint() {
|
||||
/// An anchored fold offset that cannot be represented as i64 is a plan error.
|
||||
#[test]
|
||||
fn a_subquery_with_an_overflowing_fold_offset_is_refused() {
|
||||
let Some(endpoint) = extreme_millisecond_endpoint() else {
|
||||
return;
|
||||
};
|
||||
let mut stmt = eval_stmt("rate(x[5m:1m] @ 0)");
|
||||
let PromExpr::Call(call) = &mut stmt.expr else {
|
||||
unreachable!("query parses to a call");
|
||||
@@ -3168,9 +3185,7 @@ fn a_subquery_with_an_overflowing_fold_offset_is_refused() {
|
||||
let PromExpr::Subquery(subquery) = &mut *call.args.args[0] else {
|
||||
unreachable!("call argument parses to a subquery");
|
||||
};
|
||||
subquery.at = Some(AtModifier::At(
|
||||
UNIX_EPOCH + Duration::from_millis(i64::MAX as u64),
|
||||
));
|
||||
subquery.at = Some(AtModifier::At(endpoint));
|
||||
subquery.offset = Some(Offset::Neg(Duration::from_millis(1)));
|
||||
|
||||
let error = compile(&stmt, raw_schema(), SeriesOrdering::KeyThenTime)
|
||||
@@ -3723,3 +3738,526 @@ fn whole_vector_functions_bind_a_vector_operation() {
|
||||
assert_eq!(root, format!(" root Vector {op}"), "`{query}`");
|
||||
}
|
||||
}
|
||||
|
||||
/// One raw chunk of `metric`, keyed by `key`, closing the series.
|
||||
///
|
||||
/// `timestamps` are milliseconds and `values` one per timestamp, which is what a scan of the series
|
||||
/// hands the tree.
|
||||
fn agg_raw_chunk(
|
||||
schema: &SeriesSchema,
|
||||
key: &[u8],
|
||||
metric: MetricName,
|
||||
timestamps: &[i64],
|
||||
values: &[f64],
|
||||
) -> RawSeriesChunk {
|
||||
agg_raw_chunk_with_last(schema, key, metric, timestamps, values, true)
|
||||
}
|
||||
|
||||
/// One raw chunk of `metric`, keyed by `key`, with an explicit series-ending flag.
|
||||
fn agg_raw_chunk_with_last(
|
||||
schema: &SeriesSchema,
|
||||
key: &[u8],
|
||||
metric: MetricName,
|
||||
timestamps: &[i64],
|
||||
values: &[f64],
|
||||
is_last: bool,
|
||||
) -> RawSeriesChunk {
|
||||
RawSeriesChunk::try_new(
|
||||
schema,
|
||||
Buffer::from_slice_ref(key),
|
||||
Some(metric),
|
||||
RawTimestamps::try_new(ScalarBuffer::from(timestamps.to_vec())).unwrap(),
|
||||
SeriesFields::one(Samples::from_float(Float64Array::from(values.to_vec()))),
|
||||
is_last,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// `name` with the labels in `tags`, in the order given.
|
||||
fn agg_metric(name: &str, tags: &[(&str, &str)]) -> MetricName {
|
||||
MetricName::try_new(
|
||||
Some(name.into()),
|
||||
tags.iter()
|
||||
.map(|(name, value)| Label {
|
||||
name: (*name).into(),
|
||||
value: (*value).into(),
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// What one result series of a native execution reports.
|
||||
type AggSeries = (
|
||||
Option<String>,
|
||||
Vec<(String, String)>,
|
||||
Vec<(usize, Option<f64>)>,
|
||||
);
|
||||
|
||||
/// Reads the outputs a native execution produced.
|
||||
fn aggregate_results(output: &[EvaluatedChunk]) -> Vec<AggSeries> {
|
||||
let mut results: Vec<AggSeries> = output
|
||||
.iter()
|
||||
.map(|chunk| {
|
||||
let mut tags: Vec<(String, String)> = chunk
|
||||
.series
|
||||
.metric
|
||||
.tags()
|
||||
.iter()
|
||||
.map(|tag| (tag.name.to_string(), tag.value.to_string()))
|
||||
.collect();
|
||||
tags.sort();
|
||||
let points = chunk
|
||||
.series
|
||||
.time
|
||||
.positions()
|
||||
.map(|(position, row)| {
|
||||
let sample = chunk.series.fields.as_slice()[0]
|
||||
.iter()
|
||||
.nth(row)
|
||||
.expect("the axis and the field are aligned");
|
||||
(
|
||||
position,
|
||||
match sample {
|
||||
SampleRef::Float(value) => Some(value),
|
||||
_ => None,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
(chunk.series.metric.name().map(str::to_string), tags, points)
|
||||
})
|
||||
.collect();
|
||||
results.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
results
|
||||
}
|
||||
|
||||
/// Runs `tree` over `chunks`, which enter at the tree's only input, and reads the answer.
|
||||
///
|
||||
/// The answer is read the way a caller reads it: the whole input is pushed, the input is ended, and
|
||||
/// what the root reports is collected. Each series is reported by its identity and the value it
|
||||
/// decided at each absolute grid position.
|
||||
fn run_aggregate_tree(tree: &NativeTree, chunks: Vec<RawSeriesChunk>) -> Vec<AggSeries> {
|
||||
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1 << 26));
|
||||
let state =
|
||||
NativeExecutionState::try_new(tree, pool, &ExecutionPlanMetricsSet::new(), 0).unwrap();
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
for chunk in chunks {
|
||||
execution
|
||||
.push_raw(chunk)
|
||||
.unwrap_or_else(|error| panic!("a source chunk was refused: {error}"));
|
||||
}
|
||||
let output = execution
|
||||
.finish()
|
||||
.unwrap_or_else(|error| panic!("the tree did not answer: {error}"));
|
||||
aggregate_results(&output)
|
||||
}
|
||||
|
||||
/// The 21 grid points of a statement over 100 seconds in 5-second steps.
|
||||
fn agg_points_0_to_100() -> Vec<usize> {
|
||||
(0..21).collect()
|
||||
}
|
||||
|
||||
/// The timestamps of the same 21 points, in milliseconds.
|
||||
fn agg_timestamps_0_to_100() -> Vec<i64> {
|
||||
(0..21).map(|index| index * 5_000).collect()
|
||||
}
|
||||
|
||||
/// `sum(x)` over two series: one unlabelled group, and every label of both inputs dropped.
|
||||
#[test]
|
||||
fn a_compiled_sum_drops_every_label_and_adds_its_group() {
|
||||
let tree = compile(
|
||||
&eval_stmt("sum(x)"),
|
||||
raw_schema(),
|
||||
SeriesOrdering::KeyThenTime,
|
||||
)
|
||||
.unwrap();
|
||||
let schema = raw_schema();
|
||||
let timestamps = agg_timestamps_0_to_100();
|
||||
let chunks = vec![
|
||||
agg_raw_chunk(
|
||||
&schema,
|
||||
b"a",
|
||||
agg_metric("x", &[("job", "api"), ("instance", "a")]),
|
||||
×tamps,
|
||||
&(1..=21).map(f64::from).collect::<Vec<_>>(),
|
||||
),
|
||||
agg_raw_chunk(
|
||||
&schema,
|
||||
b"b",
|
||||
agg_metric("x", &[("job", "db"), ("instance", "b")]),
|
||||
×tamps,
|
||||
&(1..=21).map(|value| (2 * value) as f64).collect::<Vec<_>>(),
|
||||
),
|
||||
];
|
||||
|
||||
let output = run_aggregate_tree(&tree, chunks);
|
||||
// `3` at the first point, three more at each of the twenty after it: the two members summed.
|
||||
let values: Vec<(usize, Option<f64>)> = agg_points_0_to_100()
|
||||
.into_iter()
|
||||
.map(|position| (position, Some(3.0 * (position as f64 + 1.0))))
|
||||
.collect();
|
||||
assert_eq!(output, vec![(None, Vec::new(), values)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_compiled_sum_aggregates_series_chunks_after_input_ends() {
|
||||
let schema = raw_schema();
|
||||
let timestamps = agg_timestamps_0_to_100();
|
||||
let values: Vec<f64> = (1..=21).map(f64::from).collect();
|
||||
let cases = [
|
||||
("sum(x)", None, Vec::new()),
|
||||
(
|
||||
"sum by(job)(x)",
|
||||
None,
|
||||
vec![("job".to_string(), "api".to_string())],
|
||||
),
|
||||
(
|
||||
"sum without(instance)(x)",
|
||||
None,
|
||||
vec![("job".to_string(), "api".to_string())],
|
||||
),
|
||||
];
|
||||
|
||||
for (query, name, tags) in cases {
|
||||
let tree = compile(&eval_stmt(query), raw_schema(), SeriesOrdering::KeyThenTime).unwrap();
|
||||
let metric = agg_metric("x", &[("job", "api"), ("instance", "a")]);
|
||||
let chunks = vec![
|
||||
agg_raw_chunk_with_last(
|
||||
&schema,
|
||||
b"a",
|
||||
metric.clone(),
|
||||
×tamps[..10],
|
||||
&values[..10],
|
||||
false,
|
||||
),
|
||||
agg_raw_chunk_with_last(
|
||||
&schema,
|
||||
b"a",
|
||||
metric,
|
||||
×tamps[10..],
|
||||
&values[10..],
|
||||
true,
|
||||
),
|
||||
];
|
||||
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1 << 26));
|
||||
let state =
|
||||
NativeExecutionState::try_new(&tree, pool, &ExecutionPlanMetricsSet::new(), 0).unwrap();
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
for chunk in chunks {
|
||||
execution.push_raw(chunk).unwrap();
|
||||
assert!(
|
||||
execution.next_chunk().unwrap().is_none(),
|
||||
"`{query}` produced output before the whole input ended"
|
||||
);
|
||||
}
|
||||
let output = execution.finish().unwrap();
|
||||
let results = aggregate_results(&output);
|
||||
let expected_points = values
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(position, value)| (position, Some(*value)))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
results,
|
||||
vec![(name.map(str::to_string), tags, expected_points)],
|
||||
"`{query}`"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Empty source input is not a member of an aggregate's implicit group.
|
||||
#[test]
|
||||
fn a_compiled_aggregate_over_empty_input_produces_no_groups() {
|
||||
for query in ["sum(x)", "sum by(job)(x)", "sum without(instance)(x)"] {
|
||||
let tree = compile(&eval_stmt(query), raw_schema(), SeriesOrdering::KeyThenTime).unwrap();
|
||||
assert_eq!(
|
||||
run_aggregate_tree(&tree, Vec::new()),
|
||||
Vec::new(),
|
||||
"`{query}` invented a group for empty input"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// `sum by(job)(x)` over five series: two groups, each the sum of its own members.
|
||||
///
|
||||
/// The members cover different points, so each group is a sum over a subset that changes from point
|
||||
/// to point: one series holds the whole grid, one holds a single sample between two grid points, and
|
||||
/// one stops a whole lookback before the grid does. No last sample sits exactly one lookback behind
|
||||
/// a grid point, so which points each member covers does not depend on how the boundary is read.
|
||||
#[test]
|
||||
fn a_compiled_sum_by_groups_its_members_and_sums_each_group() {
|
||||
let tree = compile(
|
||||
&eval_stmt("sum by(job)(x)"),
|
||||
raw_schema(),
|
||||
SeriesOrdering::KeyThenTime,
|
||||
)
|
||||
.unwrap();
|
||||
let schema = raw_schema();
|
||||
let all = agg_timestamps_0_to_100();
|
||||
let chunks = vec![
|
||||
// The whole grid, holding one.
|
||||
agg_raw_chunk(
|
||||
&schema,
|
||||
b"a",
|
||||
agg_metric("x", &[("job", "api"), ("instance", "a")]),
|
||||
&all,
|
||||
&[1.0; 21],
|
||||
),
|
||||
// One sample halfway between grid points 0 and 1, so only grid point 1 selects it.
|
||||
agg_raw_chunk(
|
||||
&schema,
|
||||
b"b",
|
||||
agg_metric("x", &[("job", "api"), ("instance", "b")]),
|
||||
&[2_500],
|
||||
&[2.0],
|
||||
),
|
||||
// Four samples between grid points, covering grid points 1 to 4.
|
||||
agg_raw_chunk(
|
||||
&schema,
|
||||
b"e",
|
||||
agg_metric("x", &[("job", "api"), ("instance", "e")]),
|
||||
&[2_400, 7_400, 12_400, 17_400],
|
||||
&[5.0; 4],
|
||||
),
|
||||
// The whole grid, holding ten.
|
||||
agg_raw_chunk(
|
||||
&schema,
|
||||
b"c",
|
||||
agg_metric("x", &[("job", "db"), ("instance", "c")]),
|
||||
&all,
|
||||
&[10.0; 21],
|
||||
),
|
||||
// One sample halfway between grid points 0 and 1, holding twenty.
|
||||
agg_raw_chunk(
|
||||
&schema,
|
||||
b"d",
|
||||
agg_metric("x", &[("job", "db"), ("instance", "d")]),
|
||||
&[2_500],
|
||||
&[20.0],
|
||||
),
|
||||
];
|
||||
|
||||
let output = run_aggregate_tree(&tree, chunks);
|
||||
// `api` is `a` alone at points 0 and 5 to 20, and every member at the points they cover:
|
||||
// point 1 is `1 + 2 + 5`, points 2 to 4 are `1 + 5`.
|
||||
let api: Vec<(usize, Option<f64>)> = agg_points_0_to_100()
|
||||
.into_iter()
|
||||
.map(|position| {
|
||||
let value = match position {
|
||||
1 => 8.0,
|
||||
2..=4 => 6.0,
|
||||
_ => 1.0,
|
||||
};
|
||||
(position, Some(value))
|
||||
})
|
||||
.collect();
|
||||
// `db` is the same shape over its own members: point 1 is `10 + 20`, the rest is `c` alone.
|
||||
let db: Vec<(usize, Option<f64>)> = agg_points_0_to_100()
|
||||
.into_iter()
|
||||
.map(|position| (position, Some(if position == 1 { 30.0 } else { 10.0 })))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
output,
|
||||
vec![
|
||||
(None, vec![("job".to_string(), "api".to_string())], api),
|
||||
(None, vec![("job".to_string(), "db".to_string())], db),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// `sum without(instance)(x)`: `instance` goes, `job` and `zone` stay, and so does `__name__` going.
|
||||
#[test]
|
||||
fn a_compiled_sum_without_drops_the_named_label_and_the_name() {
|
||||
let tree = compile(
|
||||
&eval_stmt("sum without(instance)(x)"),
|
||||
raw_schema(),
|
||||
SeriesOrdering::KeyThenTime,
|
||||
)
|
||||
.unwrap();
|
||||
let schema = raw_schema();
|
||||
let timestamps = agg_timestamps_0_to_100();
|
||||
let chunks = vec![
|
||||
agg_raw_chunk(
|
||||
&schema,
|
||||
b"a",
|
||||
agg_metric("x", &[("job", "api"), ("instance", "a"), ("zone", "west")]),
|
||||
×tamps,
|
||||
&[1.0; 21],
|
||||
),
|
||||
agg_raw_chunk(
|
||||
&schema,
|
||||
b"b",
|
||||
agg_metric("x", &[("job", "api"), ("instance", "b"), ("zone", "east")]),
|
||||
×tamps,
|
||||
&[2.0; 21],
|
||||
),
|
||||
];
|
||||
|
||||
let output = run_aggregate_tree(&tree, chunks);
|
||||
let values_of = |value: f64| -> Vec<(usize, Option<f64>)> {
|
||||
agg_points_0_to_100()
|
||||
.into_iter()
|
||||
.map(|position| (position, Some(value)))
|
||||
.collect()
|
||||
};
|
||||
// Two groups, because `zone` is not the dropped label: the two series differ by more than the
|
||||
// label the grouping removes.
|
||||
assert_eq!(
|
||||
output,
|
||||
vec![
|
||||
(
|
||||
None,
|
||||
vec![
|
||||
("job".to_string(), "api".to_string()),
|
||||
("zone".to_string(), "east".to_string()),
|
||||
],
|
||||
values_of(2.0),
|
||||
),
|
||||
(
|
||||
None,
|
||||
vec![
|
||||
("job".to_string(), "api".to_string()),
|
||||
("zone".to_string(), "west".to_string()),
|
||||
],
|
||||
values_of(1.0),
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// `sum by(job)(rate(x[5m]))`: an aggregate over a window, mid-chain.
|
||||
///
|
||||
/// Every series is a counter rising by one per five seconds, and every grid point's window ends on a
|
||||
/// sample and begins five seconds before one, so the extrapolated rate is the same at every point:
|
||||
/// `59 / 295 = 0.2` per second, which the double arithmetic lands on the literals below.
|
||||
#[test]
|
||||
fn a_compiled_sum_over_a_rate_groups_the_window_it_reads() {
|
||||
let statement = EvalStmt {
|
||||
expr: promql_parser::parser::parse("sum by(job)(rate(x[5m]))").unwrap(),
|
||||
start: UNIX_EPOCH + Duration::from_secs(300),
|
||||
end: UNIX_EPOCH + Duration::from_secs(400),
|
||||
interval: Duration::from_secs(5),
|
||||
lookback_delta: Duration::from_secs(5),
|
||||
};
|
||||
let tree = compile(&statement, raw_schema(), SeriesOrdering::KeyThenTime).unwrap();
|
||||
// The window reads 300 seconds, so the samples start a full window before the first grid point.
|
||||
let schema = raw_schema();
|
||||
let timestamps: Vec<i64> = (0..=80).map(|index| index * 5_000).collect();
|
||||
let values: Vec<f64> = (0..=80).map(|index| f64::from(index) + 1.0).collect();
|
||||
let chunks = vec![
|
||||
agg_raw_chunk(
|
||||
&schema,
|
||||
b"a",
|
||||
agg_metric("x", &[("job", "api"), ("instance", "a")]),
|
||||
×tamps,
|
||||
&values,
|
||||
),
|
||||
agg_raw_chunk(
|
||||
&schema,
|
||||
b"b",
|
||||
agg_metric("x", &[("job", "api"), ("instance", "b")]),
|
||||
×tamps,
|
||||
&values,
|
||||
),
|
||||
agg_raw_chunk(
|
||||
&schema,
|
||||
b"c",
|
||||
agg_metric("x", &[("job", "db"), ("instance", "c")]),
|
||||
×tamps,
|
||||
&values,
|
||||
),
|
||||
];
|
||||
|
||||
let output = run_aggregate_tree(&tree, chunks);
|
||||
let at_every_point = |value: f64| -> Vec<(usize, Option<f64>)> {
|
||||
(0..21).map(|position| (position, Some(value))).collect()
|
||||
};
|
||||
// One `rate` is `0.19999999999999998`, and the pair of them add to `0.39999999999999997`.
|
||||
assert_eq!(
|
||||
output,
|
||||
vec![
|
||||
(
|
||||
None,
|
||||
vec![("job".to_string(), "api".to_string())],
|
||||
at_every_point(0.39999999999999997),
|
||||
),
|
||||
(
|
||||
None,
|
||||
vec![("job".to_string(), "db".to_string())],
|
||||
at_every_point(0.19999999999999998),
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// Splitting one counter's samples does not change the rate aggregate or make it answer early.
|
||||
#[test]
|
||||
fn a_compiled_sum_over_a_rate_aggregates_series_chunks_after_input_ends() {
|
||||
let statement = EvalStmt {
|
||||
expr: promql_parser::parser::parse("sum by(job)(rate(x[5m]))").unwrap(),
|
||||
start: UNIX_EPOCH + Duration::from_secs(300),
|
||||
end: UNIX_EPOCH + Duration::from_secs(400),
|
||||
interval: Duration::from_secs(5),
|
||||
lookback_delta: Duration::from_secs(5),
|
||||
};
|
||||
let tree = compile(&statement, raw_schema(), SeriesOrdering::KeyThenTime).unwrap();
|
||||
let schema = raw_schema();
|
||||
let timestamps: Vec<i64> = (0..=80).map(|index| index * 5_000).collect();
|
||||
let values: Vec<f64> = (0..=80).map(|index| f64::from(index) + 1.0).collect();
|
||||
let metric = agg_metric("x", &[("job", "api"), ("instance", "a")]);
|
||||
let chunks = [
|
||||
agg_raw_chunk_with_last(
|
||||
&schema,
|
||||
b"a",
|
||||
metric.clone(),
|
||||
×tamps[..41],
|
||||
&values[..41],
|
||||
false,
|
||||
),
|
||||
agg_raw_chunk_with_last(
|
||||
&schema,
|
||||
b"a",
|
||||
metric,
|
||||
×tamps[41..],
|
||||
&values[41..],
|
||||
true,
|
||||
),
|
||||
];
|
||||
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1 << 26));
|
||||
let state =
|
||||
NativeExecutionState::try_new(&tree, pool, &ExecutionPlanMetricsSet::new(), 0).unwrap();
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
for chunk in chunks {
|
||||
execution.push_raw(chunk).unwrap();
|
||||
assert!(
|
||||
execution.next_chunk().unwrap().is_none(),
|
||||
"the rate aggregate produced output before the whole input ended"
|
||||
);
|
||||
}
|
||||
let output = execution.finish().unwrap();
|
||||
assert_eq!(
|
||||
aggregate_results(&output),
|
||||
vec![(
|
||||
None,
|
||||
vec![("job".to_string(), "api".to_string())],
|
||||
(0..21)
|
||||
.map(|position| (position, Some(0.19999999999999998)))
|
||||
.collect(),
|
||||
)]
|
||||
);
|
||||
}
|
||||
|
||||
/// Empty source input is not a member of this aggregate's implicit group, with or without a window.
|
||||
#[test]
|
||||
fn a_compiled_sum_over_a_rate_with_empty_input_produces_no_groups() {
|
||||
let statement = EvalStmt {
|
||||
expr: promql_parser::parser::parse("sum by(job)(rate(x[5m]))").unwrap(),
|
||||
start: UNIX_EPOCH + Duration::from_secs(300),
|
||||
end: UNIX_EPOCH + Duration::from_secs(400),
|
||||
interval: Duration::from_secs(5),
|
||||
lookback_delta: Duration::from_secs(5),
|
||||
};
|
||||
let tree = compile(&statement, raw_schema(), SeriesOrdering::KeyThenTime).unwrap();
|
||||
assert_eq!(run_aggregate_tree(&tree, Vec::new()), Vec::new());
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user