diff --git a/.cargo/config.toml b/.cargo/config.toml index fd372c95b84..8291bf467e4 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -21,5 +21,6 @@ checkout = true list_files = true internal_use_git2 = false -[env] -CARGO_WORKSPACE_DIR = { value = "", relative = true } +# Avoid globally injecting a checkout-specific workspace path. The version build script +# resolves the absolute workspace path from layout at build time; set CARGO_WORKSPACE_DIR +# explicitly when this repo is built as a submodule. diff --git a/.github/actions/start-runner/action.yml b/.github/actions/start-runner/action.yml index 78568e26d76..44a5ff3feb3 100644 --- a/.github/actions/start-runner/action.yml +++ b/.github/actions/start-runner/action.yml @@ -56,7 +56,7 @@ runs: - name: Start EC2 runner if: startsWith(inputs.runner, 'ec2') - uses: machulav/ec2-github-runner@v2.3.8 + uses: machulav/ec2-github-runner@v2.6.1 id: start-linux-arm64-ec2-runner with: mode: start diff --git a/.github/actions/stop-runner/action.yml b/.github/actions/stop-runner/action.yml index e25a1d8dad7..fb84f2b7181 100644 --- a/.github/actions/stop-runner/action.yml +++ b/.github/actions/stop-runner/action.yml @@ -33,7 +33,7 @@ runs: - name: Stop EC2 runner if: ${{ inputs.label && inputs.ec2-instance-id }} - uses: machulav/ec2-github-runner@v2.3.8 + uses: machulav/ec2-github-runner@v2.6.1 with: mode: stop label: ${{ inputs.label }} diff --git a/.github/scripts/agent-observability-summary.py b/.github/scripts/agent-observability-summary.py index 38ba4e9dbdd..33ba3ad7e6e 100644 --- a/.github/scripts/agent-observability-summary.py +++ b/.github/scripts/agent-observability-summary.py @@ -18,6 +18,8 @@ import json import math from pathlib import Path +from benchmark_resource_summary import render_resources + TARGETS = ('greptimedb', 'clickhouse', 'victorialogs') @@ -201,6 +203,7 @@ def lifecycle(root, targets): rows.append([context, entry.get('phase'), duration(entry.get('elapsed_ms')) if valid else 'invalid timestamp units (legacy artifact)', entry.get('exit_code')]) lines += ['## Phase timings', '', 'Phase duration includes orchestration overhead. Container cleanup does not prove ECS teardown.', '', table(['DB / scope', 'Phase', 'Duration', 'Exit code'], rows), ''] + lines.append(render_resources([('dataset', root)] + [(t, root / f'{t}-1') for t in targets], table)) return '\n'.join(lines) diff --git a/.github/scripts/benchmark-version-comparison.py b/.github/scripts/benchmark-version-comparison.py new file mode 100644 index 00000000000..1aa744df9ad --- /dev/null +++ b/.github/scripts/benchmark-version-comparison.py @@ -0,0 +1,191 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compare validated GreptimeDB query results; never gate CI on noisy latency deltas.""" +import argparse +import html +import json +import math +import os +from pathlib import Path +import re +import statistics + + +def read(path): + return json.loads(path.read_text()) if path.is_file() else {} + + +def validate(env): + enabled = env.get('COMPARE_GREPTIMEDB', 'false') + if enabled not in ('true', 'false'): + raise ValueError('COMPARE_GREPTIMEDB must be true or false') + if enabled == 'false': + return + if env.get('RUN_GREPTIMEDB') != 'true': + raise ValueError('Version comparison requires Run GreptimeDB') + for key in ('BASELINE_GREPTIMEDB_TAG', 'GREPTIMEDB_TAG'): + if not re.fullmatch(r'[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}(@sha256:[0-9a-f]{64})?', env.get(key, '')): + raise ValueError(f'Invalid {key}') + if env['BASELINE_GREPTIMEDB_TAG'] == env['GREPTIMEDB_TAG']: + raise ValueError('Baseline and candidate must have different image tags') + + +def same(a, b, fields): + return all(a.get(k) is not None and a.get(k) == b.get(k) for k in fields) + + +def finite(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0 + + +def improvement(old, new, throughput=False): + if not finite(old) or not finite(new) or old <= 0: + return None + return (new / old - 1 if throughput else 1 - new / old) * 100 + + +def row(query, metric, old, new, compatible, reason): + delta = improvement(old, new, metric == 'QPS') if compatible else None + return dict(query=query, metric=metric, baseline=old, candidate=new, + improvement_percent=delta, status='comparable' if delta is not None else reason) + + +def agent(root): + def load(r): + return read(r / 'greptimedb-1/greptimedb-1/measured-summary.json') + a, b = load(root / 'baseline'), load(root) + finished = [r / 'greptimedb-1/exit-code.txt' for r in (root / 'baseline', root)] + compatible = (all(p.is_file() and p.read_text().strip() == '0' for p in finished) + and a.get('passed') is True and b.get('passed') is True + and same(a, b, ('contract_sha256', 'data_profile', 'query_window', 'warmup_runs', 'measured_runs'))) + qa, qb = ({q['query_id']: q for q in x.get('per_query', [])} for x in (a, b)) + rows = [] + for q in sorted(qa.keys() | qb.keys()): + old, new = qa.get(q, {}), qb.get(q, {}) + ok = compatible and same(old, new, ('result_fingerprint', 'success')) and old.get('success', 0) > 0 + ok = ok and all(v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new)) + for key in ('p50_ms', 'p95_ms', 'p99_ms'): + rows.append(row(q, key, old.get(key), new.get(key), ok, 'failed, missing or incompatible result')) + return rows + + +def traces(root): + a, b = (read(r / 'runs/greptimedb/run.json') for r in (root / 'baseline', root)) + compatible = (a.get('success') is True and b.get('success') is True + and same(a, b, ('expected', 'protocol', 'audit_expected', 'corpus')) + and same(a.get('execution', {}), b.get('execution', {}), + ('db_cpus', 'db_memory', 'load_workers_requested', 'max_in_flight'))) + rows = [] + for q in (f'T{i:02}' for i in range(1, 9)): + values, identities, counts = [], [], [] + ok = compatible + for run in (a, b): + correctness = [s for s in run.get('correctness', []) if s.get('query_id') == q] + all_samples = [s for s in run.get('samples', []) if s.get('query_id') == q] + measured = [s for s in all_samples if s.get('phase') == 'measured'] + passed = bool(correctness and measured) and all(s.get('outcome') == 'passed' for s in correctness + all_samples) + expected = run.get('expected', {}).get('contracts', {}).get(q, {}) + passed = passed and all(s.get('fingerprint') == expected.get('fingerprint') and s.get('fingerprint') is not None + and s.get('row_count') == expected.get('row_count') for s in correctness + all_samples) + elapsed = [s.get('elapsed_ms') for s in measured] + passed = passed and all(finite(v) for v in elapsed) + ok = ok and passed + values.append(statistics.median(elapsed) if passed else None) + identities.append(expected) + counts.append((len(correctness), len(all_samples), len(measured))) + ok = ok and identities[0] == identities[1] and counts[0] == counts[1] + rows.append(row(q, 'median_ms', *values, ok, 'failed, skipped, missing or incompatible result')) + return rows + + +def long_range(root): + rows = [] + manifest = read(root / 'run-manifest.json') + healthy = all( + all(field in (manifest.get(phase, {}).get(target, {}).get('docker', {}) if phase == 'post_load_results' + else manifest.get(phase, {}).get(target, {})).get('state', '').split() + for field in ('Running=true', 'ExitCode=0', 'OOMKilled=false')) + for phase in ('post_load_results', 'post_query_results') + for target in ('greptimedb-baseline', 'greptimedb')) + for mode in ('serial', 'concurrent-duration-c1', 'concurrent-duration-c4'): + a, b = (read(root / name / 'results-vmbench-long-range' / (mode + '-summary.json')) + for name in ('greptimedb-baseline', 'greptimedb')) + fields = ('workload', 'query_file_sha256', 'start', 'end', 'step', 'profile', 'data_duration', + 'scrape_interval', 'cache_policy_effective', 'mode', 'connection_mode', 'request_timeout_seconds') + fields += ('tries', 'warmup_runs') if mode == 'serial' else ('concurrency', 'duration_seconds', 'warmup_seconds') + compatible = healthy and same(a, b, fields) and all(x.get('errors') == 0 and x.get('timeouts') == 0 for x in (a, b)) + qa, qb = ({q['query_index']: q for q in x.get('per_query', [])} for x in (a, b)) + for q in sorted(qa.keys() | qb.keys()): + old, new = qa.get(q, {}), qb.get(q, {}) + ok = compatible and same(old, new, ('query', 'series', 'points')) + ok = ok and all(v.get('success', 0) > 0 and v.get('series', 0) > 0 and v.get('points', 0) > 0 + and v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new)) + for key in ('p50_ms', 'qps') if mode != 'serial' else ('p50_ms',): + rows.append(row(f'{mode}/Q{q}', 'QPS' if key == 'qps' else key, + old.get(key), new.get(key), ok, 'failed, missing or incompatible result')) + return rows + + +def render(document): + def cell(value): + return html.escape(str(value)).replace('|', '|').replace('\n', ' ') + def value(v, metric): + if not finite(v): + return '—' + if metric == 'QPS': + return f'{v:.3f}' + return f'{v / 1000:.2f} s' if v >= 1000 else f'{v:.2f} ms' + lines = ['# GreptimeDB version comparison', '', + f'Baseline: `{cell(document["baseline_tag"])}` → candidate: `{cell(document["candidate_tag"])}`.', '', + 'Same-host sequential runs with fresh databases and shared input data. Positive = improvement; ' + 'latency: (baseline − candidate) / baseline; QPS: (candidate − baseline) / baseline. ' + 'Single runs are diagnostic, not a statistically established regression. No performance threshold gates CI.', '', + '| Query | Metric | Baseline | Candidate | Improvement | Status |', + '| --- | --- | --- | --- | --- | --- |'] + for r in document['rows']: + delta = r['improvement_percent'] + lines.append('| ' + ' | '.join(map(cell, [r['query'], r['metric'], value(r['baseline'], r['metric']), + value(r['candidate'], r['metric']), f'{delta:+.1f}%' if delta is not None else '—', r['status']])) + ' |') + if document.get('kind') == 'long-range': + lines += ['', 'Long-range retains the existing non-empty series/point-count checks; these do not establish equality of every returned value.'] + if not document['rows']: + lines += ['', 'Results unavailable; comparison not performed.'] + return '\n'.join(lines) + '\n' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', choices=('validate', 'report')) + parser.add_argument('--kind', choices=('agent', 'traces', 'long-range')) + parser.add_argument('--root', type=Path) + args = parser.parse_args() + if args.command == 'validate': + validate(os.environ) + return + if args.root is None or args.kind is None: + parser.error('report requires --root and --kind') + document = {'kind': args.kind, 'baseline_tag': os.environ.get('BASELINE_GREPTIMEDB_TAG', ''), + 'candidate_tag': os.environ.get('GREPTIMEDB_TAG', ''), + 'rows': {'agent': agent, 'traces': traces, 'long-range': long_range}[args.kind](args.root)} + text = render(document) + # Preserve partial comparisons even if a target failed before creating output. + args.root.mkdir(parents=True, exist_ok=True) + (args.root / 'version-comparison.json').write_text(json.dumps(document, indent=2, allow_nan=False) + '\n') + (args.root / 'version-comparison.md').write_text(text) + print(text, end='') + + +if __name__ == '__main__': + main() diff --git a/.github/scripts/benchmark_resource_summary.py b/.github/scripts/benchmark_resource_summary.py new file mode 100644 index 00000000000..3a0a896c46e --- /dev/null +++ b/.github/scripts/benchmark_resource_summary.py @@ -0,0 +1,99 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Render shared O11yBench sampled resource artifacts for lifecycle summaries.""" +import json +import math + + +def number(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0 + + +def metric(values, key, statistic): + entry = values.get(key, {}) + value = entry.get(statistic) + return value if number(entry.get('samples')) and entry['samples'] > 0 and number(value) else None + + +def decimal(value, suffix=''): + return f'{value:.2f}{suffix}' if number(value) else '—' + + +def memory(value): + if not number(value): + return '—' + for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'): + if value < 1024 or unit == 'TiB': + return decimal(value, f' {unit}') + value /= 1024 + + +def capacities(path): + # Quotas are metadata, not recomputed CPU statistics. Bound reads on long runs. + result = {} + try: + with path.open() as source: + for index, line in enumerate(source): + if index == 256: + break + record = json.loads(line) + cores = record.get('host', {}).get('cpu_count') + if number(cores) and cores > 0: + result.setdefault('host', cores) + for role, values in record.get('containers', {}).items(): + quota = values.get('configured_cpu_limit_cores') + if number(quota): + result.setdefault(role, quota) + if all(role in result for role in ('host', 'runtime', 'db')): + break + except (OSError, ValueError, TypeError, AttributeError): + pass + return result + + +def render_resources(scopes, table): + rows = [] + for scope, directory in scopes: + try: + summary = json.loads((directory / 'resources.summary.json').read_text()) + phases = summary.get('phases', {}) if summary.get('schema') == 'o11ybench-resources-v1' else {} + except (OSError, ValueError, TypeError, AttributeError): + phases = {} + if not isinstance(phases, dict) or not phases: + rows.append([scope, 'unavailable', *['—'] * 9]) + continue + limits = capacities(directory / 'resources.jsonl') + for phase, data in phases.items(): + for role in ('host', 'runtime', 'db'): + values = data.get('host', {}) if role == 'host' else data.get('containers', {}).get(role, {}) + if role == 'db' and scope == 'dataset': + continue + quota = limits.get(role) + cpu_samples = values.get('cpu_used_cores', {}).get('samples') + rows.append([scope, phase, role, cpu_samples, + decimal(metric(values, 'cpu_used_cores', 'mean')), + decimal(metric(values, 'cpu_used_cores', 'max')), + 'unlimited' if quota == 0 else decimal(quota), + memory(metric(values, 'memory_used_bytes' if role == 'host' else 'memory_usage_bytes', 'max')), + decimal(metric(values, 'io_wait_percent', 'mean'), '%'), + decimal(metric(values, 'io_wait_percent', 'max'), '%'), + memory(metric(values, 'swap_used_bytes', 'max'))]) + return '\n'.join(['## Sampled resources', '', + 'CPU is measured in cores. Mean/max summarize available samples, not continuous peaks; ' + 'short phases may have no CPU sample. CPU capacity is host cores or the configured container quota. ' + 'Host memory is total minus available; Docker memory excludes cache where supported. ' + 'Swap is host-wide usage, not swap activity. Missing evidence stays —.', '', + table(['Scope', 'Phase', 'Component', 'CPU samples', 'Mean CPU cores', 'Max CPU cores', + 'CPU capacity', 'Max memory', 'Mean I/O wait', 'Max I/O wait', 'Max swap'], rows), '']) diff --git a/.github/scripts/check-nightly-release.sh b/.github/scripts/check-nightly-release.sh new file mode 100755 index 00000000000..82eb910d6fd --- /dev/null +++ b/.github/scripts/check-nightly-release.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Returns whether the current release run should proceed, 'true' or 'false'. +# +# The nightly schedule runs every weekday, but a scheduled release only proceeds +# when the latest published nightly release is older than +# NIGHTLY_RELEASE_MAX_AGE_DAYS days. This way, a failed nightly run is retried +# by the next weekday schedule, instead of leaving a whole week without nightly +# builds. Non-scheduled runs (tag pushes and manual dispatches) always proceed. +# +# Only the result is written to stdout, so that it can be used in GitHub Actions +# outputs; all diagnostics go to stderr. +# +# You can run as following examples: +# GITHUB_EVENT_NAME=push ./check-nightly-release.sh +# GITHUB_EVENT_NAME=schedule GITHUB_REPOSITORY=GreptimeTeam/greptimedb GH_TOKEN=ghp_xxx ./check-nightly-release.sh +# GITHUB_EVENT_NAME=schedule NIGHTLY_RELEASE_MAX_AGE_DAYS=5 GITHUB_REPOSITORY=GreptimeTeam/greptimedb GH_TOKEN=ghp_xxx ./check-nightly-release.sh +if [ -z "$GITHUB_EVENT_NAME" ]; then + echo "GITHUB_EVENT_NAME is empty" >&2 + exit 1 +fi + +if [ "$GITHUB_EVENT_NAME" != "schedule" ]; then + echo "Not a scheduled run, the nightly check is skipped." >&2 + echo "true" + exit 0 +fi + +MAX_AGE_DAYS="${NIGHTLY_RELEASE_MAX_AGE_DAYS:-5}" +if ! [[ "$MAX_AGE_DAYS" =~ ^[0-9]+$ ]]; then + echo "Invalid NIGHTLY_RELEASE_MAX_AGE_DAYS: ${MAX_AGE_DAYS}" >&2 + exit 1 +fi + +SECONDS_PER_DAY=86400 + +# The publish time of the latest published nightly release, like the tag +# 'v0.2.0-nightly-20230313'. Fail open on GitHub API errors: proceeding with +# the release is safer than skipping it. +if ! latest_nightly_published_at="$(gh api "repos/${GITHUB_REPOSITORY}/releases?per_page=100" \ + --jq '[.[] | select(.draft == false and (.tag_name | contains("nightly")))][0].published_at // empty')"; then + echo "::warning::Failed to query the latest nightly release from the GitHub API, will proceed with the nightly release." >&2 + echo "true" + exit 0 +fi + +if [ -z "$latest_nightly_published_at" ]; then + echo "No published nightly release found, will proceed with the nightly release." >&2 + echo "true" + exit 0 +fi + +published_at_epoch="$(date --date="$latest_nightly_published_at" +%s)" +now_epoch="$(date +%s)" +age_seconds=$(( now_epoch - published_at_epoch )) +age_days=$(( age_seconds / SECONDS_PER_DAY )) + +echo "The latest nightly release was published at ${latest_nightly_published_at} (${age_days} days ago)." >&2 + +if [ "$age_seconds" -gt $(( MAX_AGE_DAYS * SECONDS_PER_DAY )) ]; then + echo "The latest nightly release is older than ${MAX_AGE_DAYS} days, will proceed with the nightly release." >&2 + echo "true" +else + echo "The latest nightly release is not older than ${MAX_AGE_DAYS} days, will skip the nightly release." >&2 + echo "false" +fi diff --git a/.github/scripts/oswal-s3-bench-run.sh b/.github/scripts/oswal-s3-bench-run.sh new file mode 100644 index 00000000000..a9835c1e5f1 --- /dev/null +++ b/.github/scripts/oswal-s3-bench-run.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Host side of oswal-s3-bench.yml: one subcommand per step; each cell of $PLAN gets a fresh GreptimeDB. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Same isolation as the o11ybench runtime's own docker run. +runtime() { + docker run --rm --user "$(id -u):$(id -g)" --label "o11ybench.owner=$OWNER" \ + --security-opt no-new-privileges --cap-drop ALL "$@" +} + +check_runtime() { + local image=$1 resolved + [[ "$image" == public.ecr.aws/* ]] || { echo 'The runtime image must come from AWS ECR Public' >&2; return 1; } + bash "$here/pull-benchmark-image.sh" "$image" + resolved=$(docker image inspect "$image" | jq -er '.[0].RepoDigests[0]') + docker run --rm --entrypoint sh "$resolved" -c \ + 'test -f scripts/agent-observability/greptimedb.sh && test -f scripts/generate_agent_observations.py' \ + || { echo 'The runtime image lacks the agent observability loader or generator' >&2; return 1; } + printf 'runtime_image=%s\n' "$resolved" >> "${GITHUB_OUTPUT:-/dev/stdout}" +} + +pull_images() { + : "${RESULTS:?}" "${RUNTIME_IMAGE:?}" "${GREPTIMEDB_TAG:?}" + mkdir -p "$RESULTS/images" + bash "$here/pull-benchmark-image.sh" "$RUNTIME_IMAGE" + bash "$here/pull-benchmark-image.sh" "greptime/greptimedb:$GREPTIMEDB_TAG" + docker image inspect "$RUNTIME_IMAGE" > "$RESULTS/images/runtime.json" + docker image inspect "greptime/greptimedb:$GREPTIMEDB_TAG" > "$RESULTS/images/greptimedb.json" +} + +wal_disk() { + : "${RESULTS:?}" + local root=$1 device + device=$(lsblk -dnpo NAME,MODEL | awk '/Instance Storage/ {print $1; exit}') + # Only a smoke run may keep the Raft Engine WAL on the root volume. + [[ -n "$device" || "${SMOKE:-}" == true ]] || { echo 'No local NVMe instance store; choose an instance type with one' >&2; return 1; } + sudo -n mkdir -p "$root" + if [[ -n "$device" ]]; then + sudo -n umount "$device" 2>/dev/null || true + sudo -n mkfs.ext4 -q -F "$device" + sudo -n mount -o noatime "$device" "$root" + fi + sudo -n chown "$(id -u):$(id -g)" "$root" + findmnt -no SOURCE,FSTYPE -T "$root" | tee "$RESULTS/wal-disk.txt" +} + +generate() { + : "${PLAN:?}" "${RESULTS:?}" "${CORPUS_DIR:?}" "${OWNER:?}" "${RUNTIME_IMAGE:?}" + local args=() arg + while IFS= read -r arg; do args+=("$arg"); done < <(jq -r '.generator_args[]' "$PLAN") + mkdir -p "$CORPUS_DIR" + runtime --name "$OWNER-generator" --mount "type=bind,src=$CORPUS_DIR,dst=/work/corpus" \ + "$RUNTIME_IMAGE" scripts/generate_agent_observations.py "${args[@]}" \ + --output /work/corpus/agent_observations.jsonl \ + --summary-output /work/corpus/summary.json --evidence-output /work/corpus/evidence.json \ + > "$RESULTS/generator.log" 2>&1 + jq -e --argjson rows "$(jq .rows "$PLAN")" '.line_count == $rows' "$CORPUS_DIR/summary.json" >/dev/null +} + +# Scrape once, after the load succeeded or failed, while the server still runs. +scrape_metrics() { + curl --noproxy '*' -fsS --max-time 30 http://127.0.0.1:4000/metrics > "$dir/metrics.all" || { rm -f "$dir/metrics.all"; return 1; } + grep -E '^(# (HELP|TYPE) )?greptime_logstore_object_store_wal_' "$dir/metrics.all" > "$dir/metrics.prom" || true + rm -f "$dir/metrics.all" + metrics_scraped=true +} + +finish() { + local rc=$? + trap - EXIT + set +e + if [[ -n "$cid" ]]; then + # A load that succeeded without its metrics is not a measurement. + scrape_metrics || { echo "$id: metrics scrape failed" >&2; [[ $rc != 0 ]] || rc=4; } + docker stop -t 120 "$cid" >/dev/null + docker logs "$cid" > "$dir/server.log" 2>&1 + docker rm -fv "$cid" >/dev/null + fi + [[ -z "$wal_dir" ]] || sudo -n rm -rf "$wal_dir" + jq -n --argjson spec "$spec" --arg device "$wal_device" --arg version "$version" \ + --arg load_seconds "$load_seconds" --argjson exit_code "$rc" --argjson scraped "$metrics_scraped" \ + '$spec + {wal_device: $device, version: $version, exit_code: $exit_code, metrics_scraped: $scraped, + load_wall_seconds: ($load_seconds | tonumber? // null)}' > "$dir/cell.json" + exit "$rc" +} + +# One cell in its own process, so errexit holds inside it. The cell's state is +# global because the EXIT trap runs after this function returns. +cell() { + : "${PLAN:?}" "${RESULTS:?}" "${CORPUS_DIR:?}" "${OWNER:?}" "${RUNTIME_IMAGE:?}" "${GREPTIMEDB_IMAGE:?}" + : "${RUN_PREFIX:?}" "${WAL_ROOT:?}" "${S3_BUCKET:?}" "${S3_REGION:?}" "${AWS_ACCESS_KEY_ID:?}" "${AWS_SECRET_ACCESS_KEY:?}" + spec=$1 cid= wal_dir= version= load_seconds= metrics_scraped=false id= + local concurrency name setting started ended + id=$(jq -r .id <<< "$spec"); concurrency=$(jq -r .concurrency <<< "$spec") + dir="$RESULTS/cells/$id"; name="$OWNER-$id" + mkdir -p "$dir" + # The root holds data/ and the WAL's default prefix wal/ side by side. + local settings=(STORAGE__TYPE=S3 "STORAGE__BUCKET=$S3_BUCKET" "STORAGE__REGION=$S3_REGION" + "STORAGE__ROOT=$RUN_PREFIX/$id") + [[ -z "${S3_CONTAINER_ENDPOINT:-}" ]] || settings+=("STORAGE__ENDPOINT=$S3_CONTAINER_ENDPOINT") + while IFS= read -r setting; do settings+=("$setting"); done \ + < <(jq -r '.wal | to_entries[] | "WAL__\(.key)=\(.value)"' <<< "$spec") + local docker_env=() mounts=() + for setting in "${settings[@]}"; do + docker_env+=(-e "GREPTIMEDB_STANDALONE__$setting") + printf 'GREPTIMEDB_STANDALONE__%s\n' "$setting" >> "$dir/config.env" + done + # Pass the two keys by name: their values never reach argv, a file or the artifact. + export GREPTIMEDB_STANDALONE__STORAGE__ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID + export GREPTIMEDB_STANDALONE__STORAGE__SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY + docker_env+=(-e GREPTIMEDB_STANDALONE__STORAGE__ACCESS_KEY_ID -e GREPTIMEDB_STANDALONE__STORAGE__SECRET_ACCESS_KEY) + printf 'GREPTIMEDB_STANDALONE__STORAGE__%s=\n' ACCESS_KEY_ID SECRET_ACCESS_KEY >> "$dir/config.env" + if jq -e '.wal.DIR' <<< "$spec" >/dev/null; then + wal_dir="$WAL_ROOT/$id"; mkdir -p "$wal_dir" + mounts=(--mount "type=bind,src=$wal_dir,dst=$(jq -r .wal.DIR <<< "$spec")") + wal_device=$(findmnt -no SOURCE -T "$wal_dir") + else + wal_device="s3://$S3_BUCKET/$RUN_PREFIX/$id/wal" + fi + trap finish EXIT + cid=$(docker run -d --init --name "$name" --label "o11ybench.owner=$OWNER" \ + --add-host host.docker.internal:host-gateway \ + -p 127.0.0.1:4000:4000 -p 127.0.0.1:4001:4001 "${docker_env[@]}" ${mounts[@]+"${mounts[@]}"} \ + "$GREPTIMEDB_IMAGE" standalone start --http-addr=0.0.0.0:4000 --grpc-bind-addr=0.0.0.0:4001) + local deadline=$((SECONDS + 300)) + until curl --noproxy '*' -fsS --max-time 5 http://127.0.0.1:4000/health >/dev/null 2>&1; do + [[ $(docker inspect --format '{{.State.Running}}' "$cid") == true ]] || { echo "$id: server exited" >&2; exit 1; } + ((SECONDS < deadline)) || { echo "$id: server not healthy in 300s" >&2; exit 1; } + sleep 2 + done + version=$(docker exec "$cid" greptime --version | sed -n 's/^version: //p') + [[ -n "$version" ]] + started=$(date +%s.%N) + # Only the load phase, run exactly as run-agent-observability.sh runs it. + runtime --name "$name-load" --network host \ + --mount "type=bind,src=$CORPUS_DIR,dst=/work/corpus,readonly" --mount "type=bind,src=$dir,dst=/results" \ + -e GRPC_ENDPOINT=127.0.0.1:4001 -e "DATA_PROFILE=$(jq -r .data_profile "$PLAN")" -e RESULT_NAME=greptimedb-1 \ + -e "ENDPOINT=http://127.0.0.1:4000/v1/sql?db=public&format=greptimedb_v1" \ + -e "TARGET_IMAGE=$GREPTIMEDB_IMAGE" -e "TARGET_VERSION=$version" -e "TARGET_DIGEST=${GREPTIMEDB_IMAGE##*@}" \ + -e "TARGET_ID=$name" -e TARGET_HOST=127.0.0.1 -e PREPARE_WORKERS= -e "MAX_IN_FLIGHT=$concurrency" \ + -e "GREPTIMEDB_BATCH_SIZE=$(jq -r .batch_rows "$PLAN")" --entrypoint /bin/bash \ + "$RUNTIME_IMAGE" scripts/agent-observability/greptimedb.sh load > "$dir/load.log" 2>&1 + ended=$(date +%s.%N) + load_seconds=$(awk -v a="$started" -v b="$ended" 'BEGIN {printf "%.3f", b - a}') + jq -e --argjson rows "$(jq .rows "$PLAN")" '.count_before == 0 and .count_after == $rows' \ + "$dir/greptimedb-1/load-result.json" >/dev/null +} + +scrub() { + : "${RESULTS:?}" + [[ -n "${AWS_ACCESS_KEY_ID:-}" && -n "${AWS_SECRET_ACCESS_KEY:-}" && -d "$RESULTS" ]] || return 0 + if grep -rqF -e "$AWS_ACCESS_KEY_ID" -e "$AWS_SECRET_ACCESS_KEY" "$RESULTS"; then + echo '::error::A credential value appears in the results; removing them instead of uploading' >&2 + rm -rf "$RESULTS" + return 1 + fi +} + +case "${1:-}" in + check-runtime) check_runtime "${2:?}" ;; + pull) pull_images ;; + wal-disk) wal_disk "${2:?}" ;; + generate) generate ;; + cell) cell "${2:?}" ;; + cells) + : "${PLAN:?}" + failed=0 + while IFS= read -r spec; do + id=$(jq -r .id <<< "$spec") + echo "::group::cell $id" + bash "${BASH_SOURCE[0]}" cell "$spec" < /dev/null || { echo "::error::cell $id failed"; failed=1; } + echo '::endgroup::' + done < <(jq -c '.cells[]' "$PLAN") + exit "$failed" ;; + scrub) scrub ;; + *) echo "usage: $0 check-runtime IMAGE | pull | wal-disk DIR | generate | cells | scrub" >&2; exit 2 ;; +esac diff --git a/.github/scripts/oswal-s3-bench-summary.py b/.github/scripts/oswal-s3-bench-summary.py new file mode 100644 index 00000000000..67186d422da --- /dev/null +++ b/.github/scripts/oswal-s3-bench-summary.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Summarize the object store WAL S3 benchmark as Markdown and summary.json.""" + +from __future__ import annotations + +import argparse +import json +import math +import re +from pathlib import Path + +PREFIX = "greptime_logstore_object_store_wal_" +COUNTERS = ("created_objects_total", "stalled_appends_total", "poisoned_total", + "create_failures_total", "create_conflicts_total") +SAMPLE = re.compile(r'([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)') + + +def parse_metrics(text: str) -> dict: + """Map each unlabeled sample to its value and each histogram to its buckets.""" + samples, buckets = {}, {} + for line in text.splitlines(): + match = SAMPLE.fullmatch(line.strip()) + if not match or not match.group(1).startswith(PREFIX): + continue + name, labels, value = match.group(1)[len(PREFIX):], match.group(2) or "", float(match.group(3)) + le = re.search(r'le="([^"]+)"', labels) + if name.endswith("_bucket") and le: + buckets.setdefault(name[:-len("_bucket")], []).append((float(le.group(1)), value)) + else: + samples[name] = value + return {"samples": samples, "buckets": {name: sorted(items) for name, items in buckets.items()}} + + +def quantile(q: float, buckets: list) -> float | None: + """Prometheus histogram_quantile over cumulative (upper bound, count) buckets.""" + if not buckets or buckets[-1][1] == 0: + return None + rank = q * buckets[-1][1] + lower, below = 0.0, 0.0 + for upper, count in buckets: + if count >= rank: + if math.isinf(upper): + return lower + return lower + (upper - lower) * (rank - below) / (count - below) if count > below else upper + lower, below = upper, count + return lower + + +def geomean(values: list) -> float | None: + return math.exp(sum(map(math.log, values)) / len(values)) if values and all(v > 0 for v in values) else None + + +def load_cell(path: Path, spec: dict | None = None) -> dict: + """One cell's row; a planned cell that never recorded a result has no values, so no ratio uses its round.""" + recorded = path / "cell.json" + cell = json.loads(recorded.read_text()) if recorded.exists() else spec | {"exit_code": None, "missing": True} + result_path = path / "greptimedb-1" / "load-result.json" + result = json.loads(result_path.read_text()) if result_path.exists() else {} + metrics_path = path / "metrics.prom" + metrics = parse_metrics(metrics_path.read_text() if metrics_path.exists() else "") + ack = metrics["buckets"].get("append_ack_seconds", []) + load_seconds = result.get("load_seconds") + row = { + "id": cell["id"], "round": cell["round"], "concurrency": cell["concurrency"], + "config": cell["config"], "exit_code": cell["exit_code"], "wal_device": cell.get("wal_device"), + "missing": cell.get("missing", False), + "version": cell.get("version"), "load_wall_seconds": cell.get("load_wall_seconds"), + "rows": result.get("count_after"), "load_seconds": load_seconds, + "rows_per_second": result.get("rows_per_second") if cell["exit_code"] == 0 else None, + "append_ack_p50_seconds": quantile(0.5, ack), "append_ack_p99_seconds": quantile(0.99, ack), + } + # The counters register on their first event, so a scrape of an object store WAL + # that created objects omits the ones that never moved: those are zero, not unknown. + active = (cell.get("metrics_scraped") and cell.get("wal", {}).get("PROVIDER") == "experimental_object_store" + and metrics["samples"].get("created_objects_total")) + for counter in COUNTERS: + row[counter] = metrics["samples"].get(counter, 0 if active else None) + created = row["created_objects_total"] + row["creates_per_minute"] = created * 60 / load_seconds if created is not None and load_seconds else None + return row + + +def ratios(rows: list) -> list: + """D/R and E/R per round and their geometric mean over the rounds, per concurrency.""" + speed = {(r["round"], r["concurrency"], r["config"]): r["rows_per_second"] for r in rows} + result = [] + for concurrency in sorted({r["concurrency"] for r in rows}): + rounds = sorted({r["round"] for r in rows if r["concurrency"] == concurrency}) + for config in ("D", "E"): + per_round = [] + for round_number in rounds: + base, value = speed.get((round_number, concurrency, "R")), speed.get((round_number, concurrency, config)) + per_round.append(value / base if base and value else None) + complete = [value for value in per_round if value is not None] + result.append({"concurrency": concurrency, "ratio": f"{config}/R", "per_round": per_round, + "geomean": geomean(complete) if len(complete) == len(per_round) else None}) + return result + + +def fmt(value, digits=0, scale=1.0) -> str: + return "—" if value is None else f"{value * scale:,.{digits}f}" + + +def render(run: dict, rows: list, pairs: list) -> str: + lines = [f"## Object store WAL against S3 ({'smoke, MinIO' if run.get('smoke') else run.get('instance_type')})", ""] + lines += [f"GreptimeDB `{run.get('greptimedb_image')}` ({next((r['version'] for r in rows if r['version']), '—')}); " + f"{fmt(run.get('rows'))} rows in {run.get('batch_rows')}-row batches; bucket region {run.get('region')}.", ""] + lines += ["| Cell | Rows/s | Load s | Wall s | Rows | Ack p50 ms | Ack p99 ms | Creates | Creates/min " + "| Stalled | Poisoned | Create failures | Conflicts | WAL on |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |"] + for r in rows: + cell = f"{r['id']} (not run)" if r["missing"] else r["id"] if r["exit_code"] == 0 \ + else f"{r['id']} (failed, exit {r['exit_code']})" + lines.append(" | ".join([ + f"| {cell}", fmt(r["rows_per_second"]), fmt(r["load_seconds"], 1), fmt(r["load_wall_seconds"], 1), + fmt(r["rows"]), fmt(r["append_ack_p50_seconds"], 1, 1000), fmt(r["append_ack_p99_seconds"], 1, 1000), + *(fmt(r[c]) for c in COUNTERS[:1]), fmt(r["creates_per_minute"], 1), + *(fmt(r[c]) for c in COUNTERS[1:]), f"`{r['wal_device'] or '—'}` |"])) + lines += ["", "| Concurrency | Ratio | Per round | Geometric mean |", "| ---: | --- | --- | ---: |"] + for p in pairs: + lines.append(f"| {p['concurrency']} | {p['ratio']} | {', '.join(fmt(v, 3) for v in p['per_round'])} " + f"| {fmt(p['geomean'], 3)} |") + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + args = parser.parse_args() + args.root.mkdir(parents=True, exist_ok=True) + run_path = args.root / "run.json" + run = json.loads(run_path.read_text()) if run_path.exists() else {} + plan_path = args.root / "plan.json" + planned = json.loads(plan_path.read_text())["cells"] if plan_path.exists() else [] + # The plan, not what happened to run, defines the expected cells and rounds. + rows = [load_cell(args.root / "cells" / spec["id"], spec) for spec in planned] + rows += [load_cell(path) for path in sorted((args.root / "cells").glob("*")) + if path.name not in {spec["id"] for spec in planned} and (path / "cell.json").exists()] + pairs = ratios(rows) + missing = [r["id"] for r in rows if r["missing"]] + (args.root / "summary.json").write_text(json.dumps( + {"run": run, "planned_cells": len(planned), "missing_cells": missing, "cells": rows, "ratios": pairs}, + indent=2) + "\n") + print(render(run, rows, pairs) if rows else "No benchmark cell completed.\n") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/oswal-s3-bench.py b/.github/scripts/oswal-s3-bench.py new file mode 100644 index 00000000000..faad8ace674 --- /dev/null +++ b/.github/scripts/oswal-s3-bench.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# /// script +# requires-python = ">=3.10" +# dependencies = ["boto3>=1.35,<2"] +# /// + +"""Helpers of the object store WAL S3 benchmark; only `probe` and `cleanup` need boto3 (`uv run`).""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import time +import urllib.request +from pathlib import Path + +SMOKE_ROWS = 50_000 +# The o11ybench data profiles; a matching row count generates the canonical dataset. +DATA_PROFILES = {5_000: "S", 2_000_000: "D", 10_000_000: "P", 100_000_000: "M"} +# Identical [storage] for every configuration; only [wal] differs. +WAL_CONFIGS = { + "R": {"PROVIDER": "raft_engine", "SYNC_WRITE": "true", "DIR": "/wal"}, + "D": {"PROVIDER": "experimental_object_store", "ACK_MODE": "durable"}, + "E": {"PROVIDER": "experimental_object_store", "ACK_MODE": "enqueued"}, +} +ORDERS = (("R", "D", "E"), ("E", "D", "R")) +NIGHTLY = re.compile(r"v\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?-nightly-(\d{8})") +TAG = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}") +DIGEST = re.compile(r".+@sha256:[0-9a-f]{64}") + + +def append_github(variable: str, name: str, value: str) -> None: + if path := os.environ.get(variable): + with open(path, "a") as output: + output.write(f"{name}={value}\n") + + +def positive(name: str, value: str, upper: int) -> int: + if not re.fullmatch(r"[1-9][0-9]*", value) or int(value) > upper: + raise ValueError(f"{name} must be an integer in 1..{upper}, got {value!r}") + return int(value) + + +def validate_budgets(timeout_minutes: str, ttl_hours: str) -> None: + timeout, ttl = positive("timeout", timeout_minutes, 360), positive("ttl", ttl_hours, 168) + # Provisioning, teardown and queueing need two hours beyond the benchmark job. + if ttl * 60 < timeout + 120: + raise ValueError(f"janitor TTL {ttl}h must cover the {timeout}-minute timeout plus 2 hours") + + +def check_regions(bucket: str, runner: str) -> None: + if not bucket or bucket != runner: + raise ValueError(f"bucket region {bucket!r} differs from runner region {runner!r}; " + "a cross-region bucket measures the WAN, not S3") + + +def build_manifest(plan: dict, greptimedb_inspect: list, runtime_image: str, + instance_type: str, region: str, prefix: str) -> dict: + image = next((d for d in greptimedb_inspect[0].get("RepoDigests") or [] if DIGEST.fullmatch(d)), None) + if image is None: + raise ValueError("the GreptimeDB image has no sha256 repository digest") + return {"smoke": plan["smoke"], "rows": plan["rows"], "rounds": plan["rounds"], + "concurrency": plan["concurrency"], "batch_rows": plan["batch_rows"], + "greptimedb_image": image, "runtime_image": runtime_image, + "instance_type": instance_type, "region": region, "prefix": prefix} + + +def build_plan(concurrency: str, rows: str, batch_rows: str, rounds: str, smoke: bool) -> dict: + levels = [positive("concurrency", level.strip(), 256) for level in concurrency.split(",")] + if len(set(levels)) != len(levels): + raise ValueError(f"concurrency repeats a level: {concurrency!r}") + row_count = SMOKE_ROWS if smoke else positive("rows", rows, 100_000_000) + round_count = 1 if smoke else positive("rounds", rounds, 10) + profile = DATA_PROFILES.get(row_count) + cells = [] + for round_number in range(1, round_count + 1): + # Alternate the order between rounds so no configuration always runs first. + for level in levels: + for config in ORDERS[(round_number - 1) % 2]: + cells.append({ + "id": f"{round_number}-{level}-{config}", "round": round_number, + "concurrency": level, "config": config, "wal": WAL_CONFIGS[config], + }) + return { + "smoke": smoke, "rows": row_count, "rounds": round_count, + "batch_rows": positive("batch_rows", batch_rows, 100_000), "concurrency": levels, + "data_profile": profile or "custom", + "generator_args": ["--data-profile", profile] if profile else ["--line-count", str(row_count)], + "cells": cells, + } + + +def newest_nightly(tags: list[str]) -> str: + dated = [(match.group(1), tag) for tag in tags if (match := NIGHTLY.fullmatch(tag))] + if not dated: + raise ValueError("no v*-nightly-* tag found") + return max(dated)[1] + + +def resolve_tag(requested: str) -> str: + if requested != "nightly": + if not TAG.fullmatch(requested): + raise ValueError(f"invalid image tag: {requested!r}") + return requested + url = ("https://hub.docker.com/v2/repositories/greptime/greptimedb/tags" + "?page_size=100&name=nightly&ordering=last_updated") + with urllib.request.urlopen(url, timeout=30) as response: + return newest_nightly([tag["name"] for tag in json.load(response)["results"]]) + + +def s3_client(args): + import boto3 + from botocore.config import Config + + return boto3.client("s3", region_name=args.region, endpoint_url=args.endpoint or None, + config=Config(retries={"mode": "standard", "max_attempts": 5})) + + +def probe(args) -> None: + client, key = s3_client(args), f"{args.prefix}/probe/object" + deadline = time.monotonic() + 60 + while True: + try: + client.put_object(Bucket=args.bucket, Key=key, Body=b"probe") + break + except Exception: + # A fresh MinIO creates its bucket after the container starts. + if time.monotonic() > deadline: + raise + time.sleep(3) + listed = client.list_objects_v2(Bucket=args.bucket, Prefix=f"{args.prefix}/probe/") + if [item["Key"] for item in listed.get("Contents", [])] != [key]: + raise RuntimeError(f"listing {args.prefix}/probe/ did not return exactly {key}") + client.delete_object(Bucket=args.bucket, Key=key) + print(f"put, list and delete succeeded under s3://{args.bucket}/{args.prefix}/probe/") + + +def cleanup(args) -> None: + # One DeleteObject per key: DeleteObjects needs a checksum header the MinIO fixture rejects. + client, deleted = s3_client(args), 0 + for page in client.get_paginator("list_objects_v2").paginate(Bucket=args.bucket, Prefix=f"{args.prefix}/"): + for item in page.get("Contents", []): + client.delete_object(Bucket=args.bucket, Key=item["Key"]) + deleted += 1 + print(f"deleted {deleted} objects under s3://{args.bucket}/{args.prefix}/") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + plan = commands.add_parser("plan") + for name in ("concurrency", "rows", "batch-rows", "rounds"): + plan.add_argument(f"--{name}", required=True) + plan.add_argument("--smoke", choices=("true", "false"), required=True) + for name in ("timeout-minutes", "ttl-hours", "output"): + plan.add_argument(f"--{name}") + commands.add_parser("tag").add_argument("requested") + regions = commands.add_parser("regions") + for name in ("bucket", "runner"): + regions.add_argument(f"--{name}", required=True) + manifest = commands.add_parser("manifest") + for name in ("results", "runtime-image", "instance-type", "region", "prefix"): + manifest.add_argument(f"--{name}", required=True) + for name in ("probe", "cleanup"): + bucket = commands.add_parser(name) + bucket.add_argument("--endpoint", default="") + for option in ("bucket", "region", "prefix"): + bucket.add_argument(f"--{option}", required=True) + args = parser.parse_args() + if args.command == "plan": + if args.timeout_minutes or args.ttl_hours: + validate_budgets(args.timeout_minutes or "", args.ttl_hours or "") + result = build_plan(args.concurrency, args.rows, args.batch_rows, args.rounds, args.smoke == "true") + if args.output: + with open(args.output, "w") as output: + json.dump(result, output, indent=2) + print(f"{len(result['cells'])} cells, {result['rows']} rows, profile {result['data_profile']}") + elif args.command == "tag": + tag = resolve_tag(args.requested) + append_github("GITHUB_OUTPUT", "greptimedb_tag", tag) + print(f"GreptimeDB image tag: {tag}") + elif args.command == "regions": + check_regions(args.bucket, args.runner) + print(f"bucket and runner both in {args.runner}") + elif args.command == "manifest": + results = Path(args.results) + manifest = build_manifest(json.loads((results / "plan.json").read_text()), + json.loads((results / "images" / "greptimedb.json").read_text()), + args.runtime_image, args.instance_type, args.region, args.prefix) + (results / "run.json").write_text(json.dumps(manifest, indent=2) + "\n") + append_github("GITHUB_ENV", "GREPTIMEDB_IMAGE", manifest["greptimedb_image"]) + print(f"GreptimeDB image: {manifest['greptimedb_image']}") + else: + if not re.fullmatch(r"oswal-bench/[0-9]+-[0-9]+", args.prefix): + raise ValueError(f"refusing a prefix outside oswal-bench/: {args.prefix!r}") + (probe if args.command == "probe" else cleanup)(args) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ValueError as error: + print(f"::error::{error}", file=sys.stderr) + sys.exit(1) diff --git a/.github/scripts/pull-benchmark-image.sh b/.github/scripts/pull-benchmark-image.sh new file mode 100644 index 00000000000..1a26a4aeaee --- /dev/null +++ b/.github/scripts/pull-benchmark-image.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Pull only: never retry database writes, benchmarks, or cloud provisioning. +set -euo pipefail + +if [[ $# != 1 || -z "$1" || "$1" == -* ]]; then + echo "Usage: $0 IMAGE" >&2 + exit 2 +fi +image=$1 +attempts=3 +for ((attempt=1; attempt<=attempts; attempt++)); do + echo "Pulling $image (attempt $attempt/$attempts)" >&2 + if timeout --kill-after=10s 180s docker pull "$image"; then + exit 0 + else + status=$? + fi + if ((attempt == attempts)); then + echo "Failed to pull $image after $attempts attempts (exit $status)" >&2 + exit "$status" + fi + delay=$((attempt * 10)) + echo "Pull failed (exit $status); retrying in ${delay}s" >&2 + sleep "$delay" +done diff --git a/.github/scripts/tracesbench-summary.py b/.github/scripts/tracesbench-summary.py new file mode 100644 index 00000000000..087a67498f4 --- /dev/null +++ b/.github/scripts/tracesbench-summary.py @@ -0,0 +1,169 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import html +import math +from pathlib import Path + +from benchmark_resource_summary import render_resources + +TARGETS = ('greptimedb', 'victoriatraces', 'tempo') + + +def duration(ms): + if not isinstance(ms, (int, float)) or not math.isfinite(ms) or ms < 0: + return '—' + if ms < 1: + return f'{ms * 1000:.0f} µs' + if ms < 1000: + return f'{ms:.1f} ms' + if ms < 60_000: + return f'{ms / 1000:.2f} s' + minutes, seconds = divmod(round(ms / 1000), 60) + hours, minutes = divmod(minutes, 60) + return f'{hours}h {minutes}m {seconds}s' if hours else f'{minutes}m {seconds}s' + + +def cell(value): + return html.escape(str(value if value is not None else '—')).replace('|', '|').replace('\n', ' ') + + +def read(path): + if not path.exists(): + return {} + return json.loads(path.read_text()) + + +def table(headers, rows): + return '\n'.join(['| ' + ' | '.join(map(cell, headers)) + ' |', + '| ' + ' | '.join('---' for _ in headers) + ' |'] + + ['| ' + ' | '.join(map(cell, row)) + ' |' for row in rows]) + + +def delta(value, baseline): + if not all(isinstance(v, (int, float)) and math.isfinite(v) for v in (value, baseline)): + return '—' + return f'{(value / baseline - 1) * 100:+.1f}%' if baseline > 0 and value >= 0 else '—' + + + +def load(root): + report = read(root / 'report.json') + runs = [read(p) for p in sorted((root / 'runs').glob('*/run.json'))] + return report, runs + + +def benchmark(report, runs, targets=TARGETS): + lines = ['# Tracesbench benchmark summary', '', + 'Median Δ = (target / GreptimeDB − 1) × 100%; negative is faster. ' + 'Min/median/max come from the validated report, not recalculated samples. ' + 'Counts exclude warmup. Missing or excluded results have no latency comparison.', + '', f'Classification: {cell(report.get("classification"))}. ' + 'See the uploaded HTML for query semantics and client-side compensation.', ''] + statistics = {} + for corpus in report.get('per_corpus', []): + for stat in corpus.get('statistics', []): + if stat['target'] not in targets: + continue + statistics[(corpus['corpus'], stat['round'], stat['target'], stat['query_id'])] = stat + rows, details = [], [] + for target in targets: + selected = [run for run in runs if run.get('target') == target] + if not selected: + lines.append(f'- **{target}**: unavailable / not run.') + for run in selected: + lines.append(f'- **{target}**: lifecycle ' + f'{"completed" if run.get("success") is True else "failed / incomplete"}; ' + f'query failures: {len(run.get("query_failures", []))}.') + for qid in (f'T{i:02}' for i in range(1, 9)): + correct = [q for q in run.get('correctness', []) if q.get('query_id') == qid] + measured = [q for q in run.get('samples', []) + if q.get('query_id') == qid and q.get('phase') == 'measured'] + errors = [q for q in correct + run.get('samples', []) + if q.get('query_id') == qid and q.get('outcome') not in ('passed', 'not_executed')] + reason = (run.get('exclusions', {}).get(qid) or + run.get('execution', {}).get('query_skips', {}).get(qid)) + stat = statistics.get((run.get('corpus'), run.get('round'), target, qid), {}) + base = statistics.get((run.get('corpus'), run.get('round'), 'greptimedb', qid), {}) + status = ('failed' if errors else 'excluded' if reason else + 'passed' if stat else + 'correctness passed; report unavailable' if correct and + all(q.get('outcome') == 'passed' for q in correct) else 'unavailable / incomplete') + rows.append([run.get('corpus'), run.get('round'), qid, target, status, + duration(stat.get('min_ms')), duration(stat.get('median_ms')), + duration(stat.get('max_ms')), + delta(stat.get('median_ms'), base.get('median_ms')) if target != 'greptimedb' else '—', + sum(q.get('outcome') == 'passed' for q in measured), + sum(q.get('outcome') != 'passed' for q in measured)]) + if reason or errors: + messages = [str(q.get('error') or q.get('outcome')) for q in errors] + details.append(f'- **{target} / {qid}**: {cell(str(reason or "") + " " + "; ".join(messages))[:4000]}') + lines += ['', table(['Dataset', 'Round', 'Query', 'DB', 'Status', 'Min', 'Median', 'Max', + 'Median Δ', 'Measured success', 'Measured errors'], rows), '', + '
', 'Query failures and exclusions', '', *details, '', '
'] + if not report: + lines += ['', 'Validated report unavailable; raw outcomes are shown without latency statistics.'] + return '\n'.join(lines) + + +def lifecycle(report, runs, root, targets=TARGETS): + rows = [] + for target in targets: + selected = [r for r in runs if r.get('target') == target] or [{}] + for run in selected: + load = run.get('load', {}) + ingest = load.get('ingest', {}) + if target == 'tempo' and 'ingest' not in load: + ingest = load.get('transport', {}).get('ingest', {}) + health = run.get('post_load_health', {}) + execution = run.get('execution', {}) + start, finish = run.get('started_unix_ns'), run.get('finished_unix_ns') + elapsed = (finish - start) / 1_000_000 if isinstance(start, int) and isinstance(finish, int) else None + seconds = ingest.get('wall_seconds') + rows.append([target, run.get('corpus'), run.get('load_summary', {}).get('after'), + duration(elapsed), duration(seconds * 1000 if isinstance(seconds, (int, float)) else None), + ingest.get('successful_requests'), ingest.get('failed_requests'), + execution.get('load_workers_requested'), execution.get('max_in_flight'), + execution.get('db_cpus'), execution.get('db_memory'), + health.get('running'), health.get('oom_killed'), + run.get('cleanup', {}).get('verified_absent')]) + return '\n'.join(['# Tracesbench load / lifecycle summary', '', + 'Total is the recorded target lifecycle; ingest is the sender wall time, ' + 'not the entire load/readiness phase. Health is the post-load snapshot. ' + 'Missing evidence remains blank.', '', + table(['DB', 'Dataset', 'Loaded spans', 'Total', 'Ingest', + 'Write success', 'Write errors', 'Load workers', 'Max in flight', + 'DB CPU', 'DB memory', 'Running after load', 'OOM after load', + 'Container removed'], rows), '', + render_resources([('dataset', root / 'runs' / 'generate')] + + [(t, root / 'runs' / t) for t in targets], table)]) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--root', type=Path, required=True) + parser.add_argument('--section', choices=('benchmark', 'lifecycle'), required=True) + parser.add_argument('--targets', default=','.join(TARGETS), help='Comma-separated selected engines') + args = parser.parse_args() + targets = args.targets.split(',') + if not targets or any(t not in TARGETS for t in targets) or len(set(targets)) != len(targets): + parser.error('targets must be a non-empty comma-separated subset of greptimedb,victoriatraces,tempo') + report, runs = load(args.root) + print(benchmark(report, runs, targets) if args.section == 'benchmark' else lifecycle(report, runs, args.root, targets)) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/agent-observability.yml b/.github/workflows/agent-observability.yml index 86713c911ed..d265801d4a0 100644 --- a/.github/workflows/agent-observability.yml +++ b/.github/workflows/agent-observability.yml @@ -26,8 +26,18 @@ on: - M default: S required: true + compare_greptimedb: + description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run) + type: boolean + default: false + required: false + baseline_greptimedb_tag: + description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled + type: string + default: v1.2.1 + required: false greptimedb_tag: - description: Docker Hub greptime/greptimedb tag + description: Candidate Docker Hub greptime/greptimedb tag type: string default: latest required: true @@ -46,9 +56,9 @@ on: type: string required: false runtime_image: - description: Aliyun runtime image (tag or digest reference) + description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides' type: string - default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260917-8111034e@sha256:ce9dee2abb1e70f907a09ea01854904a91cc97748b0c4ef6f33d9ec34044bec0 + default: auto required: true provider: description: Cloud provider for the benchmark runner @@ -89,12 +99,14 @@ on: permissions: contents: read env: + COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} + BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }} PROFILE: ${{ inputs.dataset }} GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }} VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }} O11YBENCH_REF: ${{ inputs.o11ybench_ref }} - RUNTIME_IMAGE: ${{ inputs.runtime_image }} + RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' }} PROVIDER: ${{ inputs.provider }} REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }} @@ -118,6 +130,12 @@ jobs: with: path: greptimedb persist-credentials: false + - name: Validate version comparison + env: + RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} + run: | + python3 greptimedb/tests/perf/test_benchmark_version_comparison.py + python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate - name: Resolve CI runner id: runner run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE" @@ -126,6 +144,16 @@ jobs: python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py python3 greptimedb/tests/perf/test_ci_runner_scripts.py python3 greptimedb/tests/perf/test_agent_observability_summary.py + - name: Check runtime supports shared version-comparison data + if: inputs.compare_greptimedb + run: | + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE" + docker run --rm --entrypoint grep "$RUNTIME_IMAGE" -q -- '--data-root)' /opt/o11ybench/scripts/run-agent-observability.sh \ + || { echo 'Comparison requires a runtime with --data-root support'; exit 1; } + - name: Validate dataset admission in runtime + run: | + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE" + docker run --rm "$RUNTIME_IMAGE" scripts/logbench_agent_observability_measured.py --preflight-profile "$PROFILE" - name: Validate requested benchmark id: inputs shell: bash @@ -262,11 +290,11 @@ jobs: run: | set -euo pipefail actual= - [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]] + [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]] images='{}' resolve_image() { local target=$1 reference=$2 variable=$3 metadata resolved revision - docker pull "$reference" + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$reference" metadata=$(docker image inspect "$reference") resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata") if [[ "$target" == runtime ]]; then @@ -289,6 +317,9 @@ jobs: victorialogs) resolve_image "$target" "victoriametrics/victoria-logs:$VICTORIALOGS_TAG" VICTORIALOGS_IMAGE ;; esac done + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + resolve_image greptimedb-baseline "greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG" BASELINE_GREPTIMEDB_IMAGE + fi jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \ --arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \ --arg provider "$PROVIDER" --arg instance_type "$INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \ @@ -312,6 +343,15 @@ jobs: --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \ --root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json" + - name: Load and benchmark baseline GreptimeDB + id: baseline_greptimedb + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }} + run: |- + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ + --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline" --data-root "$ARTIFACT_ROOT" \ + --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + --target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \ + --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT" - name: Load and benchmark greptimedb id: greptimedb if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }} @@ -357,6 +397,13 @@ jobs: GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \ --owner "$OWNER" --root "$ARTIFACT_ROOT" fi + - name: Clean up baseline containers + if: always() && inputs.compare_greptimedb + run: |- + if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \ + --owner "$OWNER-base" --root "$ARTIFACT_ROOT/baseline" + fi - name: Benchmark summary if: always() run: | @@ -367,6 +414,16 @@ jobs: run: | python3 greptimedb/.github/scripts/agent-observability-summary.py \ --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY" + - name: Baseline load / lifecycle summary + if: always() && inputs.compare_greptimedb + run: | + echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/agent-observability-summary.py \ + --root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY" + - name: GreptimeDB version comparison + if: always() && inputs.compare_greptimedb + run: | + python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" >> "$GITHUB_STEP_SUMMARY" - name: Upload results (no dataset) if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d3bad06f41d..deba5010c80 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -99,6 +99,7 @@ jobs: python3 tests/perf/test_aliyun_ecs_runner_scripts.py python3 tests/perf/test_ci_runner_scripts.py python3 tests/perf/test_agent_observability_summary.py + python3 tests/perf/test_tracesbench_summary.py check: if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }} diff --git a/.github/workflows/oswal-s3-bench.yml b/.github/workflows/oswal-s3-bench.yml new file mode 100644 index 00000000000..c31dfe8f1c3 --- /dev/null +++ b/.github/workflows/oswal-s3-bench.yml @@ -0,0 +1,229 @@ +name: Object Store WAL S3 Benchmark +on: + workflow_dispatch: + inputs: + smoke: + description: 'Smoke run: ubuntu-latest and the MinIO fixture instead of EC2 and S3, 50000 rows, one round; its numbers mean nothing' + type: boolean + default: false + greptimedb_tag: + description: 'Docker Hub greptime/greptimedb tag; nightly = the newest v*-nightly-* tag (latest is a release without the object store WAL)' + type: string + default: nightly + instance_type: + description: EC2 instance type in the bucket region; the d of c6id gives a local NVMe instance store for the Raft Engine WAL + type: string + default: c6id.2xlarge + concurrency: + description: Comma-separated loader concurrency levels + type: string + default: '1,8' + rows: + description: Dataset rows (2000000 = o11ybench profile D) + type: number + default: 2000000 + batch_rows: + description: Rows per write request + type: number + default: 1000 + rounds: + description: Rounds; the configuration order alternates between rounds (R, D, E then E, D, R) + type: number + default: 2 + runtime_image: + description: 'o11ybench runtime image: auto = AWS ECR Public; explicit tag/digest overrides' + type: string + default: auto + benchmark_timeout_minutes: + description: Benchmark job timeout in minutes (1..360) + type: number + default: 240 + janitor_ttl_hours: + description: Instance lifetime before janitor eligibility (1..168 hours; at least timeout + 2h) + type: number + default: 8 +permissions: + contents: read +env: + SMOKE: ${{ inputs.smoke }} + GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} + REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} + CONCURRENCY: ${{ inputs.concurrency }} + ROWS: ${{ inputs.rows }} + BATCH_ROWS: ${{ inputs.batch_rows }} + ROUNDS: ${{ inputs.rounds }} + RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' }} + BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }} + JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }} +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + instance_type: ${{ steps.runner.outputs.instance_type }} + greptimedb_tag: ${{ steps.plan.outputs.greptimedb_tag }} + runtime_image: ${{ steps.runtime.outputs.runtime_image }} + steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - name: Test benchmark scripts + run: | + python3 greptimedb/tests/perf/test_oswal_s3_bench_scripts.py + python3 greptimedb/tests/perf/test_ci_runner_scripts.py + - name: Validate plan, budgets and image tag + id: plan + run: | + python3 greptimedb/.github/scripts/oswal-s3-bench.py plan --smoke "$SMOKE" \ + --concurrency "$CONCURRENCY" --rows "$ROWS" --batch-rows "$BATCH_ROWS" --rounds "$ROUNDS" \ + --timeout-minutes "$BENCHMARK_TIMEOUT_MINUTES" --ttl-hours "$JANITOR_TTL_HOURS" + python3 greptimedb/.github/scripts/oswal-s3-bench.py tag "$GREPTIMEDB_TAG" + - name: Resolve CI runner + id: runner + run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider AWS --instance-type "$REQUESTED_INSTANCE_TYPE" + - name: Check bucket and runner regions + env: + BUCKET_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }} + RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }} + run: python3 greptimedb/.github/scripts/oswal-s3-bench.py regions --bucket "$BUCKET_REGION" --runner "$RUNNER_REGION" + - name: Check the runtime image + id: runtime + run: bash greptimedb/.github/scripts/oswal-s3-bench-run.sh check-runtime "$RUNTIME_IMAGE" + provision: + needs: validate + if: ${{ !inputs.smoke }} + runs-on: ubuntu-latest + timeout-minutes: 45 + outputs: + label: ${{ steps.aws.outputs.label }} + instance_id: ${{ steps.aws.outputs.instance_id }} + runner_name: ${{ steps.aws.outputs.runner_name }} + steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + - uses: astral-sh/setup-uv@v6 + - name: Create ephemeral EC2 runner + id: aws + env: + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + AWS_REGION: ${{ vars.EC2_RUNNER_REGION }} + AWS_EC2_IMAGE_ID: ${{ vars.BENCHMARK_EC2_IMAGE_ID || 'auto' }} + AWS_EC2_INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }} + AWS_EC2_SUBNET_ID: ${{ vars.EC2_RUNNER_SUBNET_ID }} + AWS_EC2_SECURITY_GROUP_ID: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }} + AWS_EC2_SYSTEM_DISK_GIB: '60' + AWS_EC2_TTL_HOURS: ${{ inputs.janitor_ttl_hours }} + AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} + run: uv run greptimedb/.github/scripts/aws-ec2-runner-provision.py + benchmark: + needs: + - validate + - provision + if: ${{ !cancelled() && needs.validate.result == 'success' && (inputs.smoke || needs.provision.result == 'success') }} + runs-on: ${{ inputs.smoke && 'ubuntu-latest' || needs.provision.outputs.label }} + timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }} + env: + INSTANCE_TYPE: ${{ inputs.smoke && 'ubuntu-latest' || needs.validate.outputs.instance_type }} + GREPTIMEDB_TAG: ${{ needs.validate.outputs.greptimedb_tag }} + RUNTIME_IMAGE: ${{ needs.validate.outputs.runtime_image }} + RUN_PREFIX: oswal-bench/${{ github.run_id }}-${{ github.run_attempt }} + OWNER: o11ybench-oswal-${{ github.run_id }}-${{ github.run_attempt }} + RESULTS: ${{ github.workspace }}/oswal-results + CORPUS_DIR: ${{ github.workspace }}/oswal-corpus + PLAN: ${{ github.workspace }}/oswal-results/plan.json + WAL_ROOT: /mnt/oswal-wal + S3_BUCKET: ${{ inputs.smoke && 'greptime' || vars.AWS_CI_TEST_BUCKET }} + S3_REGION: ${{ inputs.smoke && 'us-west-2' || vars.AWS_CI_TEST_BUCKET_REGION }} + S3_ENDPOINT: ${{ inputs.smoke && 'http://127.0.0.1:9000' || '' }} + S3_CONTAINER_ENDPOINT: ${{ inputs.smoke && 'http://host.docker.internal:9000' || '' }} + steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - name: Set up benchmark dependencies + uses: ./greptimedb/.github/actions/setup-benchmark + - uses: astral-sh/setup-uv@v6 + - name: Start the MinIO fixture + if: inputs.smoke + run: | + bash greptimedb/.github/scripts/pull-benchmark-image.sh greptime/minio:2024 + docker compose -f greptimedb/tests-integration/fixtures/docker-compose.yml up -d --wait minio + - name: Pull images and write the plan + run: | + bash greptimedb/.github/scripts/oswal-s3-bench-run.sh pull + python3 greptimedb/.github/scripts/oswal-s3-bench.py plan --smoke "$SMOKE" --output "$PLAN" \ + --concurrency "$CONCURRENCY" --rows "$ROWS" --batch-rows "$BATCH_ROWS" --rounds "$ROUNDS" + python3 greptimedb/.github/scripts/oswal-s3-bench.py manifest --results "$RESULTS" \ + --runtime-image "$RUNTIME_IMAGE" --instance-type "$INSTANCE_TYPE" --region "$S3_REGION" --prefix "$RUN_PREFIX" + - name: Probe the bucket + env: + AWS_ACCESS_KEY_ID: ${{ inputs.smoke && 'superpower_ci_user' || secrets.AWS_CI_TEST_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.smoke && 'superpower_password' || secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }} + run: uv run greptimedb/.github/scripts/oswal-s3-bench.py probe --bucket "$S3_BUCKET" --region "$S3_REGION" --endpoint "$S3_ENDPOINT" --prefix "$RUN_PREFIX" + - name: Put the Raft Engine WAL on the instance store + run: bash greptimedb/.github/scripts/oswal-s3-bench-run.sh wal-disk "$WAL_ROOT" + - name: Generate the dataset once + run: bash greptimedb/.github/scripts/oswal-s3-bench-run.sh generate + - name: Load every cell + env: + AWS_ACCESS_KEY_ID: ${{ inputs.smoke && 'superpower_ci_user' || secrets.AWS_CI_TEST_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.smoke && 'superpower_password' || secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }} + run: bash greptimedb/.github/scripts/oswal-s3-bench-run.sh cells + - name: Summary + if: always() + run: python3 greptimedb/.github/scripts/oswal-s3-bench-summary.py --root "$RESULTS" >> "$GITHUB_STEP_SUMMARY" + - name: Delete the run's S3 prefix + if: always() + env: + AWS_ACCESS_KEY_ID: ${{ inputs.smoke && 'superpower_ci_user' || secrets.AWS_CI_TEST_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.smoke && 'superpower_password' || secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }} + run: uv run greptimedb/.github/scripts/oswal-s3-bench.py cleanup --bucket "$S3_BUCKET" --region "$S3_REGION" --endpoint "$S3_ENDPOINT" --prefix "$RUN_PREFIX" + - name: Check the results hold no credentials + if: always() + env: + AWS_ACCESS_KEY_ID: ${{ inputs.smoke && 'superpower_ci_user' || secrets.AWS_CI_TEST_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.smoke && 'superpower_password' || secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }} + run: bash greptimedb/.github/scripts/oswal-s3-bench-run.sh scrub + - name: Upload results (no corpus) + if: always() + uses: actions/upload-artifact@v4 + with: + name: oswal-s3-bench-${{ inputs.smoke && 'smoke' || 'ec2' }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.RESULTS }} + if-no-files-found: warn + retention-days: 14 + teardown: + needs: + - provision + - benchmark + if: ${{ always() && !inputs.smoke && (needs.provision.outputs.instance_id != '' || needs.provision.result != 'skipped') }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + - uses: astral-sh/setup-uv@v6 + - name: Delete EC2 and unregister runner + env: + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + AWS_REGION: ${{ vars.EC2_RUNNER_REGION }} + AWS_EC2_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }} + AWS_EC2_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }} + AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} + run: uv run greptimedb/.github/scripts/aws-ec2-runner-teardown.py diff --git a/.github/workflows/query-regression.yml b/.github/workflows/query-regression.yml index 47daa3f0435..6a368c455d6 100644 --- a/.github/workflows/query-regression.yml +++ b/.github/workflows/query-regression.yml @@ -549,7 +549,7 @@ jobs: readonly EXPECTED_SCCACHE_DIR="/home/runner/.cache/sccache" readonly EXPECTED_RUSTC_WRAPPER="/usr/local/bin/sccache" readonly RUNNER_IMAGE_DIGEST="sha256:e713b294e23b7e15184e558866c90025e59930033e72c97650dbc7f1ca022d11" - readonly RUNNER_IMAGE_EPOCH="6" + readonly RUNNER_IMAGE_EPOCH="7" require_expected_root() { local name="$1" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6c0c36c895e..0fb22f23e61 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,15 +4,20 @@ name: Release # 1. The tag('v*.*.*') push release: the release workflow will be triggered by the tag push event. # 2. The scheduled release(the version will be '-nightly-YYYYMMDD', any pre-release # extension in the next release version like '-alpha.1' is stripped, so 'nightly' is the extension): -# the release workflow will be triggered by the schedule event. +# the release workflow will be triggered by the schedule event. The schedule runs every weekday, +# but the release only proceeds when the latest published nightly release is older than +# NIGHTLY_RELEASE_MAX_AGE_DAYS days, so a failed nightly run is retried by the next weekday +# schedule instead of leaving a whole week without nightly builds. # A failed tag push release can be manually dispatched on the same tag to recover it without changing the formal release version. on: push: tags: - "v*.*.*" schedule: - # At 00:00 on Monday. - - cron: '0 0 * * 1' + # At 00:00 on every weekday (Monday to Friday). The scheduled nightly release only + # proceeds when the latest published nightly release is older than + # NIGHTLY_RELEASE_MAX_AGE_DAYS days, see the 'Check nightly release necessity' step. + - cron: '0 0 * * 1-5' workflow_dispatch: # Allows you to run this workflow manually. # Notes: The GitHub Actions ONLY support 10 inputs; this final input is the tenth. inputs: @@ -79,7 +84,7 @@ on: default: false release_validation: type: choice - description: Manual policy for accepted perf regressions or runner failures; automatic releases always run all validation + description: Manual policy for accepted perf regressions or runner failures; automatic releases always run all validation (scheduled nightlies run query regression non-blocking) required: true default: all options: @@ -100,6 +105,10 @@ env: # the pre-release extension of the next release version is stripped, e.g. v0.2.0-alpha.1 -> v0.2.0-nightly-20230313. NIGHTLY_RELEASE_PREFIX: nightly + # The maximum age (in days) of the latest published nightly release before a scheduled + # nightly release may run again; see the 'Check nightly release necessity' step. + NIGHTLY_RELEASE_MAX_AGE_DAYS: 5 + jobs: allocate-runners: name: Allocate runners @@ -123,6 +132,10 @@ jobs: # The 'is-current-version-latest' determines whether to update 'latest' Docker tags and downstream repositories. is-current-version-latest: ${{ steps.check-version.outputs.is-current-version-latest }} is-current-version-stable: ${{ steps.check-version.outputs.is-current-version-stable }} + + # Whether a scheduled nightly release should run: it only runs when the latest published + # nightly release is older than NIGHTLY_RELEASE_MAX_AGE_DAYS days. Always 'true' for other events. + nightly-required: ${{ steps.check-nightly.outputs.nightly-required }} steps: - name: Checkout uses: actions/checkout@v4 @@ -135,6 +148,19 @@ jobs: run: | ./scripts/check-builder-rust-version.sh + # A scheduled run only proceeds when the latest published nightly release is older than + # NIGHTLY_RELEASE_MAX_AGE_DAYS days, so a failed nightly run is retried by the next weekday + # schedule. The check runs before the runners are allocated to avoid wasting EC2 resources. + - name: Check nightly release necessity + id: check-nightly + run: | + echo "nightly-required=$(./.github/scripts/check-nightly-release.sh)" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_REPOSITORY: ${{ github.repository }} + NIGHTLY_RELEASE_MAX_AGE_DAYS: ${{ env.NIGHTLY_RELEASE_MAX_AGE_DAYS }} + # The create-version will create a global variable named 'version' in the global workflows. # - If it's a tag push release or manual dispatch on a tag with REUSE_EXISTING_RELEASE_TAG=true, the version is the tag name(${{ github.ref_name }}); # - If it's a scheduled release, the version is '-nightly-$buildTime', like v0.2.0-nightly-20230313 (pre-release extensions like '-alpha.1' are stripped); @@ -156,7 +182,7 @@ jobs: ./.github/scripts/check-version.sh "${{ steps.create-version.outputs.version }}" - name: Allocate linux-amd64 runner - if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && steps.check-nightly.outputs.nightly-required == 'true' }} uses: ./.github/actions/start-runner id: start-linux-amd64-runner with: @@ -170,7 +196,7 @@ jobs: subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }} - name: Allocate linux-arm64 runner - if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && steps.check-nightly.outputs.nightly-required == 'true' }} uses: ./.github/actions/start-runner id: start-linux-arm64-runner with: @@ -185,7 +211,7 @@ jobs: build-linux-amd64-artifacts: name: Build linux-amd64 artifacts - if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} needs: [ allocate-runners, ] @@ -211,7 +237,7 @@ jobs: build-linux-arm64-artifacts: name: Build linux-arm64 artifacts - if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} needs: [ allocate-runners, ] @@ -238,7 +264,7 @@ jobs: build-linux-riscv64-artifacts: name: Build linux-riscv64 artifacts - if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} needs: [ allocate-runners, ] @@ -302,7 +328,7 @@ jobs: needs: [ allocate-runners, ] - if: ${{ inputs.build_macos_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_macos_artifacts || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} steps: - uses: actions/checkout@v4 with: @@ -340,7 +366,7 @@ jobs: needs: [ allocate-runners, ] - if: ${{ inputs.build_windows_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_windows_artifacts || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} steps: - run: git config --global core.autocrlf false @@ -365,7 +391,7 @@ jobs: prepare-release-validation: name: Prepare release validation - if: ${{ inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} needs: [ allocate-runners, ] @@ -442,6 +468,9 @@ jobs: if [[ "${validation_policy}" != "all" ]]; then echo "- ⚠️ Validation bypassed: compatibility=${run_compat}, query-regression=${run_query_regression}" fi + if [[ "${GITHUB_EVENT_NAME}" == "schedule" ]]; then + echo "- ℹ️ Query-regression failure or cancellation is non-blocking for scheduled nightly releases" + fi } >> "${GITHUB_STEP_SUMMARY}" compat-release: @@ -525,9 +554,13 @@ jobs: release-images-to-dockerhub: name: Build and push images to DockerHub + # Scheduled nightly releases do not gate on query regression: the test + # still runs and reports, but a failure or cancellation does not block + # the nightly release. if: | always() && (inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && + (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') && needs.allocate-runners.result == 'success' && needs.build-linux-amd64-artifacts.result == 'success' && needs.build-linux-arm64-artifacts.result == 'success' && @@ -535,7 +568,8 @@ jobs: (needs.compat-release.result == 'success' || (needs.prepare-release-validation.outputs.run-compat == 'false' && needs.compat-release.result == 'skipped')) && (needs.query-regression-release.result == 'success' || - (needs.prepare-release-validation.outputs.run-query-regression == 'false' && needs.query-regression-release.result == 'skipped')) + (needs.prepare-release-validation.outputs.run-query-regression == 'false' && needs.query-regression-release.result == 'skipped') || + github.event_name == 'schedule') needs: [ allocate-runners, build-linux-amd64-artifacts, @@ -571,7 +605,18 @@ jobs: release-cn-artifacts: name: Release artifacts to CN region - if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + # A non-blocking query regression failure must not skip downstream releases. + if: | + always() && + (inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && + (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') && + needs.allocate-runners.result == 'success' && + needs.build-linux-amd64-artifacts.result == 'success' && + needs.build-linux-arm64-artifacts.result == 'success' && + needs.build-linux-riscv64-artifacts.result == 'success' && + needs.build-macos-artifacts.result == 'success' && + needs.build-windows-artifacts.result == 'success' && + needs.release-images-to-dockerhub.result == 'success' needs: [ # The job have to wait for all the artifacts are built. allocate-runners, build-linux-amd64-artifacts, @@ -615,15 +660,19 @@ jobs: name: Create GitHub release and upload artifacts # Use always() to run even when optional jobs (macos, windows) are skipped. # Then check that required jobs succeeded and optional jobs didn't fail. + # Scheduled nightly releases do not gate on query regression either + # (see release-images-to-dockerhub); the test still runs and reports. if: | always() && (inputs.publish_github_release || github.event_name == 'push' || github.event_name == 'schedule') && + (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') && needs.allocate-runners.result == 'success' && needs.prepare-release-validation.result == 'success' && (needs.compat-release.result == 'success' || (needs.prepare-release-validation.outputs.run-compat == 'false' && needs.compat-release.result == 'skipped')) && (needs.query-regression-release.result == 'success' || - (needs.prepare-release-validation.outputs.run-query-regression == 'false' && needs.query-regression-release.result == 'skipped')) && + (needs.prepare-release-validation.outputs.run-query-regression == 'false' && needs.query-regression-release.result == 'skipped') || + github.event_name == 'schedule') && needs.build-linux-amd64-artifacts.result == 'success' && needs.build-linux-arm64-artifacts.result == 'success' && needs.build-linux-riscv64-artifacts.result == 'success' && @@ -726,7 +775,21 @@ jobs: bump-downstream-repo-versions: name: Bump downstream repo versions - if: ${{ github.event_name == 'schedule' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_type == 'tag' && !contains(github.ref_name, 'nightly')) && (needs.allocate-runners.outputs.is-current-version-stable == 'true' && needs.allocate-runners.outputs.is-current-version-latest == 'true') }} + # A non-blocking query regression failure must not skip downstream updates. + if: | + always() && + needs.allocate-runners.result == 'success' && + needs.publish-github-release.result == 'success' && + ( + (github.event_name == 'schedule' && needs.allocate-runners.outputs.nightly-required == 'true') || + ( + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && + github.ref_type == 'tag' && + !contains(github.ref_name, 'nightly') && + needs.allocate-runners.outputs.is-current-version-stable == 'true' && + needs.allocate-runners.outputs.is-current-version-latest == 'true' + ) + ) needs: [allocate-runners, publish-github-release] runs-on: ubuntu-latest # Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs @@ -793,9 +856,10 @@ jobs: ./.github/scripts/update-homebrew-greptme-version.sh notification: - if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'push' || github.event_name == 'schedule') && always() }} + if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'push' || (github.event_name == 'schedule' && needs.allocate-runners.outputs.nightly-required == 'true')) && always() }} name: Send notification to Greptime team needs: [ + allocate-runners, release-images-to-dockerhub, build-macos-artifacts, build-windows-artifacts, diff --git a/.github/workflows/tracesbench.yml b/.github/workflows/tracesbench.yml index 60081d69ad9..049faaef1a7 100644 --- a/.github/workflows/tracesbench.yml +++ b/.github/workflows/tracesbench.yml @@ -2,6 +2,36 @@ name: Tracesbench on: workflow_dispatch: inputs: + run_greptimedb: + description: Run GreptimeDB + type: boolean + default: true + required: false + run_victoriatraces: + description: Run VictoriaTraces + type: boolean + default: false + required: false + run_tempo: + description: Run Tempo + type: boolean + default: false + required: false + compare_greptimedb: + description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run) + type: boolean + default: false + required: false + baseline_greptimedb_tag: + description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled + type: string + default: v1.2.1 + required: false + greptimedb_tag: + description: Candidate Docker Hub greptime/greptimedb tag + type: string + default: latest + required: true dataset: description: Dataset profile (main is experimental; native queries may exceed memory limits) type: choice @@ -9,9 +39,9 @@ on: default: sanity required: true runtime_image: - description: Aliyun runtime image (tag or digest reference) + description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides' type: string - default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260928-4e52d0b0@sha256:39d2d87593788e0c47a54768f6323679b8e03d7dc72a8158a9b22cf29f883baa + default: auto required: true provider: description: Cloud provider for the benchmark runner @@ -20,7 +50,7 @@ on: default: Aliyun required: true instance_type: - description: 'Instance type; auto = Aliyun: ecs.c9i.2xlarge, AWS: c7i.2xlarge (both 8 vCPU / 16 GiB). Enter a specific type to override.' + description: 'Instance type; auto = Aliyun: ecs.g9i.2xlarge, AWS: m7i.2xlarge (both 8 vCPU / 32 GiB). Enter a specific type to override.' type: string default: auto required: true @@ -45,33 +75,46 @@ on: default: 8 required: true db_cpus: - description: CPU limit per DB container + description: CPU limit per DB container (default 8 vCPU) type: string default: '8' required: true db_memory: - description: Memory limit per DB container + description: Memory limit per DB container (default 32 GiB; shared host also runs the runtime and runner) type: string - default: 16g + default: 32g + required: true + load_workers: + description: CPU workers for parallel import reading/parsing across all engines (separate from write concurrency) + type: number + default: 4 + required: true + query_timeout_seconds: + description: Timeout per query request in seconds (correctness, warmup and measurement; 1..3600) + type: number + default: 300 required: true runtime_cpus: - description: CPU limit per runtime container + description: CPU limit per runtime container (0 = unrestricted) type: string - default: '2' + default: '0' required: true runtime_memory: - description: Memory limit per runtime container (separate from DB memory) + description: Memory limit per runtime container (0 = unrestricted; separate from DB memory) type: string - default: 4g + default: '0' required: true permissions: contents: read env: + COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} + BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }} DATASET: ${{ inputs.dataset }} + GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} RUNTIME_CPUS: ${{ inputs.runtime_cpus }} RUNTIME_MEMORY: ${{ inputs.runtime_memory }} BENCHMARK_ROOT: ${{ inputs.benchmark_root }} - RUNTIME_IMAGE: ${{ inputs.runtime_image }} + RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' }} PROVIDER: ${{ inputs.provider }} REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }} @@ -79,12 +122,18 @@ env: JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }} DB_CPUS: ${{ inputs.db_cpus }} DB_MEMORY: ${{ inputs.db_memory }} + QUERY_TIMEOUT_SECONDS: ${{ inputs.query_timeout_seconds }} + LOAD_WORKERS: ${{ inputs.load_workers }} jobs: validate: runs-on: ubuntu-latest timeout-minutes: 15 outputs: instance_type: ${{ steps.runner.outputs.instance_type }} + targets: ${{ steps.inputs.outputs.targets }} + greptimedb: ${{ steps.inputs.outputs.greptimedb }} + victoriatraces: ${{ steps.inputs.outputs.victoriatraces }} + tempo: ${{ steps.inputs.outputs.tempo }} system_disk_gib: ${{ steps.runtime.outputs.system_disk_gib }} runtime_image: ${{ steps.runtime.outputs.runtime_image }} o11ybench_ref: ${{ steps.runtime.outputs.o11ybench_ref }} @@ -93,9 +142,44 @@ jobs: with: path: greptimedb persist-credentials: false + - name: Validate selected targets + id: inputs + env: + greptimedb: ${{ inputs.run_greptimedb }} + victoriatraces: ${{ inputs.run_victoriatraces }} + tempo: ${{ inputs.run_tempo }} + run: | + set -euo pipefail + selected= + for target in greptimedb victoriatraces tempo; do + enabled=${!target} + [[ "$enabled" == true || "$enabled" == false ]] + printf '%s=%s\n' "$target" "$enabled" >> "$GITHUB_OUTPUT" + if [[ "$enabled" == true ]]; then selected="${selected:+$selected,}$target"; fi + done + [[ -n "$selected" ]] || { echo 'Select at least one database' >&2; exit 1; } + if [[ "$greptimedb" == true ]]; then + [[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; } + fi + printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT" + - name: Validate version comparison + env: + RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} + run: | + python3 greptimedb/tests/perf/test_benchmark_version_comparison.py + python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate - name: Resolve CI runner id: runner - run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE" + run: | + set -euo pipefail + instance_type="$REQUESTED_INSTANCE_TYPE" + if [[ "$instance_type" == auto ]]; then + case "$PROVIDER" in + Aliyun) instance_type=ecs.g9i.2xlarge ;; + AWS) instance_type=m7i.2xlarge ;; + esac + fi + python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$instance_type" - name: Validate budgets and shared runtime id: runtime run: | @@ -115,19 +199,38 @@ jobs: ((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168)) ((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120)) [[ "$DB_CPUS" =~ ^[1-9][0-9]*$ && "$DB_MEMORY" =~ ^[1-9][0-9]*[mMgG]$ ]] - [[ "$RUNTIME_CPUS" =~ ^[1-9][0-9]*$ && "$RUNTIME_MEMORY" =~ ^[1-9][0-9]*[mMgG]$ ]] - [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]] - docker pull "$RUNTIME_IMAGE" + [[ "$RUNTIME_CPUS" =~ ^(0|[1-9][0-9]*)$ && "$RUNTIME_MEMORY" =~ ^(0|[1-9][0-9]*[mMgG])$ ]] + [[ "$QUERY_TIMEOUT_SECONDS" =~ ^[1-9][0-9]{0,3}$ ]] + ((QUERY_TIMEOUT_SECONDS <= 3600)) + [[ "$LOAD_WORKERS" =~ ^[1-9][0-9]{0,2}$ ]] + ((LOAD_WORKERS <= 256)) + runtime_limits=() + [[ "$RUNTIME_CPUS" == 0 ]] || runtime_limits+=(--cpus "$RUNTIME_CPUS") + [[ "$RUNTIME_MEMORY" == 0 ]] || runtime_limits+=(--memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY") + [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]] + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE" metadata=$(docker image inspect "$RUNTIME_IMAGE") resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata") + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + docker run --rm "$resolved" scripts/run_tracesbench_docker.py --help | grep -q -- '--data-root' \ + || { echo 'Comparison requires a runtime with --data-root support'; exit 1; } + fi revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata") [[ "$revision" =~ ^[0-9a-f]{40}$ ]] # Refuse runtimes without the trace runner and its validated dataset/report contract. - help=$(docker run --rm --cpus "$RUNTIME_CPUS" --memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY" "$resolved" scripts/run_tracesbench_docker.py --help) + help=$(docker run --rm "${runtime_limits[@]}" "$resolved" scripts/run_tracesbench_docker.py --help) + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + grep -F -- '--baseline-run' <<< "$help" \ + || { echo 'Comparison requires a runtime with --baseline-run report support'; exit 1; } + fi grep -F -- '--dataset {sanity,preflight,main}' <<< "$help" grep -F -- '--runtime-cpus' <<< "$help" grep -F -- '--runtime-memory' <<< "$help" - docker run --rm --cpus "$RUNTIME_CPUS" --memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY" "$resolved" scripts/check_run_tracesbench_docker.py + grep -F -- '--query-timeout-seconds' <<< "$help" + grep -F -- '--load-workers' <<< "$help" + grep -F -- '--targets' <<< "$help" + grep -F -- '--greptimedb-image' <<< "$help" + docker run --rm "${runtime_limits[@]}" "$resolved" scripts/check_run_tracesbench_docker.py printf 'runtime_image=%s\no11ybench_ref=%s\nsystem_disk_gib=%s\n' "$resolved" "$revision" "$SYSTEM_DISK_GIB" >> "$GITHUB_OUTPUT" provision: needs: validate @@ -190,6 +293,7 @@ jobs: runs-on: ${{ needs.provision.outputs.label }} timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }} env: + TARGETS: ${{ needs.validate.outputs.targets }} RESOLVED_RUNTIME_IMAGE: ${{ needs.validate.outputs.runtime_image }} RESOLVED_O11YBENCH_REF: ${{ needs.validate.outputs.o11ybench_ref }} RUN_ROOT: ${{ inputs.benchmark_root }}/tracesbench-${{ github.run_id }}-${{ github.run_attempt }} @@ -224,36 +328,83 @@ jobs: available=$(df -Pk "$path" | awk 'NR == 2 {print $4}') ((available >= minimum_gib * 1024 * 1024)) || { echo "Need at least $minimum_gib GiB free on $path"; exit 1; } done - docker pull "$RESOLVED_RUNTIME_IMAGE" + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE" docker image inspect "$RESOLVED_RUNTIME_IMAGE" > "$METADATA_ROOT/runtime-image.json" test "$(jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]' "$METADATA_ROOT/runtime-image.json")" = "$RESOLVED_O11YBENCH_REF" - # Preserve the pinned catalog images and require all three targets in the report. + - name: Pull database images before generating data + run: | + set -euo pipefail + IFS=, read -ra targets <<< "$TARGETS" + for target in "${targets[@]}"; do + if [[ "$target" == greptimedb ]]; then + image="greptime/greptimedb:$GREPTIMEDB_TAG" + else + image=$(jq -er '.target_identity.image | select(test("@sha256:[a-f0-9]{64}$"))' \ + "o11ybench/workloads/tracesbench/catalogs/$target.json") + fi + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image" + docker image inspect "$image" > "$METADATA_ROOT/$target-image.json" + if [[ "$target" == greptimedb ]]; then + resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/$target-image.json") + printf 'GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV" + fi + done + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + image="greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG" + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image" + docker image inspect "$image" > "$METADATA_ROOT/greptimedb-baseline-image.json" + resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/greptimedb-baseline-image.json") + printf 'BASELINE_GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV" + fi - name: Generate selected dataset once + id: generate run: | python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - - name: GreptimeDB load, correctness and benchmark + - name: Baseline GreptimeDB load, correctness and benchmark + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }} run: | - python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 \ - --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \ + --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ + --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ + --root "$RUN_ROOT/baseline" --data-root "$RUN_ROOT" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/runs/greptimedb" + - name: GreptimeDB load, correctness and benchmark + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }} + run: | + python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \ + --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/runs/greptimedb" - name: VictoriaTraces load, correctness and benchmark + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victoriatraces == 'true' }} run: | python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target victoriatraces --port 19428 \ - --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Tempo load, correctness and benchmark + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.tempo == 'true' }} run: | python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target tempo --port 13200 \ - --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - - name: Validate and render report + - name: GreptimeDB version comparison + if: always() && inputs.compare_greptimedb run: | - python3 o11ybench/scripts/run_tracesbench_docker.py --stage report \ + python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" >> "$GITHUB_STEP_SUMMARY" + - name: Validate and render report + env: + COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} + run: | + comparison_args=() + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + comparison_args+=(--baseline-run "$RUN_ROOT/baseline/runs/greptimedb/run.json") + fi + python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "$TARGETS" "${comparison_args[@]}" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Clean up owned containers @@ -262,6 +413,23 @@ jobs: python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Clean up baseline containers + if: always() && steps.runtime.outcome == 'success' && inputs.compare_greptimedb + run: | + python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \ + --root "$RUN_ROOT/baseline" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Benchmark summary + if: always() + run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY" + - name: Load / lifecycle summary + if: always() + run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY" + - name: Baseline load / lifecycle summary + if: always() && inputs.compare_greptimedb + run: | + echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/tracesbench-summary.py \ + --root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY" - name: Stage benchmark artifacts if: always() run: | @@ -269,11 +437,15 @@ jobs: if [[ -d "$METADATA_ROOT" ]]; then rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/tracesbench-artifacts/" fi - for entry in runs report report.json report.md; do + for entry in runs report report.json report.md version-comparison.json version-comparison.md; do if [[ -e "$RUN_ROOT/$entry" ]]; then rsync -a --no-owner --no-group "$RUN_ROOT/$entry" "$RUNNER_TEMP/tracesbench-artifacts/" fi done + if [[ -d "$RUN_ROOT/baseline/runs" ]]; then + mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/baseline" + rsync -a --no-owner --no-group "$RUN_ROOT/baseline/runs" "$RUNNER_TEMP/tracesbench-artifacts/baseline/" + fi - name: Upload HTML and results (no corpus) if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/vmbench-long-range.yml b/.github/workflows/vmbench-long-range.yml index 357d2b21c69..aaf2dd7f437 100644 --- a/.github/workflows/vmbench-long-range.yml +++ b/.github/workflows/vmbench-long-range.yml @@ -20,8 +20,18 @@ on: - 1mo default: 1w required: true + compare_greptimedb: + description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run) + type: boolean + default: false + required: false + baseline_greptimedb_tag: + description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled + type: string + default: v1.2.1 + required: false greptimedb_tag: - description: Docker Hub greptime/greptimedb tag + description: Candidate Docker Hub greptime/greptimedb tag type: string default: latest required: true @@ -31,9 +41,9 @@ on: default: v1.114.0@sha256:227ad53fd57a4b4430624898a1be0cac39431e1235fa0dc92f567cce38969344 required: true runtime_image: - description: Aliyun runtime image (tag or digest reference) + description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides' type: string - default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260918-545dc570@sha256:0ff03e7e690575e875bb85b3054a03486dd9359073546dc715da177153cf7a29 + default: auto required: true provider: description: Cloud provider for the benchmark runner @@ -79,11 +89,13 @@ on: permissions: contents: read env: + COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} + BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }} BENCHMARK_ROOT: ${{ inputs.benchmark_root }} RANGE_PRESET: ${{ inputs.range_preset }} GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }} - RUNTIME_IMAGE: ${{ inputs.runtime_image }} + RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' }} PROVIDER: ${{ inputs.provider }} REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }} @@ -107,6 +119,12 @@ jobs: with: path: greptimedb persist-credentials: false + - name: Validate version comparison + env: + RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} + run: | + python3 greptimedb/tests/perf/test_benchmark_version_comparison.py + python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate - name: Resolve CI runner id: runner run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE" @@ -135,8 +153,8 @@ jobs: ((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168)) # Match the existing ECS workflow's provision, teardown and queue headroom. ((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120)) - [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]] - docker pull "$RUNTIME_IMAGE" + [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]] + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE" metadata=$(docker image inspect "$RUNTIME_IMAGE") resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata") revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata") @@ -146,6 +164,9 @@ jobs: args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --run-classification controlled_server_benchmark --no-open-browser) + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + args+=(--target "greptimedb:name=greptimedb-baseline,image=greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG,url=http://127.0.0.1:14000/v1/prometheus/api/v1/query_range") + fi # Tags become target-spec fields; reject separators before passing them to the planner. for target in greptimedb victoriametrics; do case "$target" in @@ -273,13 +294,13 @@ jobs: available=$(df -Pk "$path" | awk 'NR == 2 {print $4}') ((available >= 10 * 1024 * 1024)) || { echo "Need at least 10 GiB free on $path"; exit 1; } done - docker pull "$RESOLVED_RUNTIME_IMAGE" + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE" docker image inspect "$RESOLVED_RUNTIME_IMAGE" > "$METADATA_ROOT/runtime-image.json" test "$(jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]' "$METADATA_ROOT/runtime-image.json")" = "$RESOLVED_O11YBENCH_REF" plan="$METADATA_ROOT/warm.json" while IFS=$'\t' read -r target reference; do echo "Resolving $target: $reference" - docker pull "$reference" + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$reference" docker image inspect "$reference" > "$METADATA_ROOT/$target-image.json" digest=$(jq -er '.[0].RepoDigests[0]' "$METADATA_ROOT/$target-image.json") jq --arg target "$target" --arg image "$digest" '.targets |= map(if .name == $target then .image = $image else . end)' "$plan" > "$plan.tmp" @@ -303,6 +324,10 @@ jobs: if: always() run: | docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv + - name: GreptimeDB version comparison + if: always() && inputs.compare_greptimedb + run: | + python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" >> "$GITHUB_STEP_SUMMARY" - name: Stage benchmark artifacts if: always() run: | diff --git a/Cargo.lock b/Cargo.lock index 51a2a905a40..cbd87363918 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -230,7 +230,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" [[package]] name = "api" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "arrow-schema 59.2.0", "common-base", @@ -959,7 +959,7 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "auth" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "async-trait", @@ -1618,7 +1618,7 @@ dependencies = [ [[package]] name = "cache" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "catalog", "common-error", @@ -1684,7 +1684,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "catalog" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "arrow 59.2.0", @@ -2040,7 +2040,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "cli" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "async-stream", "async-trait", @@ -2095,7 +2095,7 @@ dependencies = [ [[package]] name = "client" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "arc-swap", @@ -2127,10 +2127,11 @@ dependencies = [ "serde_json", "snafu 0.8.6", "store-api", - "substrait 1.3.0-beta.1", + "substrait 1.3.0-beta.2", "tokio", "tokio-stream", "tonic 0.14.2", + "tonic-prost", ] [[package]] @@ -2175,7 +2176,7 @@ dependencies = [ [[package]] name = "cmd" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "async-trait", @@ -2311,7 +2312,7 @@ dependencies = [ [[package]] name = "common-base" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "ahash 0.8.12", "anymap2", @@ -2320,6 +2321,7 @@ dependencies = [ "bytes", "common-test-util", "futures", + "icu_properties", "lazy_static", "pin-project", "regex", @@ -2332,7 +2334,7 @@ dependencies = [ [[package]] name = "common-batcher" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "dashmap", "futures", @@ -2341,14 +2343,14 @@ dependencies = [ [[package]] name = "common-catalog" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "const_format", ] [[package]] name = "common-config" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "common-base", "common-error", @@ -2371,7 +2373,7 @@ dependencies = [ [[package]] name = "common-datasource" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -2408,7 +2410,7 @@ dependencies = [ [[package]] name = "common-decimal" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "bigdecimal 0.4.8", "common-error", @@ -2421,7 +2423,7 @@ dependencies = [ [[package]] name = "common-error" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "common-macro", "http 1.5.0", @@ -2433,7 +2435,7 @@ dependencies = [ [[package]] name = "common-event-recorder" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "async-trait", @@ -2458,7 +2460,7 @@ dependencies = [ [[package]] name = "common-frontend" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "async-trait", @@ -2477,7 +2479,7 @@ dependencies = [ [[package]] name = "common-function" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "ahash 0.8.12", "api", @@ -2513,7 +2515,6 @@ dependencies = [ "geohash", "h3o", "hyperloglogplus", - "icu_properties", "jsonb", "memchr", "mito-codec", @@ -2539,7 +2540,7 @@ dependencies = [ [[package]] name = "common-greptimedb-telemetry" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "async-trait", "common-runtime", @@ -2556,7 +2557,7 @@ dependencies = [ [[package]] name = "common-grpc" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "arrow-flight", @@ -2591,7 +2592,7 @@ dependencies = [ [[package]] name = "common-grpc-expr" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "common-catalog", @@ -2609,7 +2610,7 @@ dependencies = [ [[package]] name = "common-macro" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "greptime-proto", "once_cell", @@ -2620,7 +2621,7 @@ dependencies = [ [[package]] name = "common-mem-prof" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "anyhow", "common-error", @@ -2636,7 +2637,7 @@ dependencies = [ [[package]] name = "common-memory-manager" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "common-error", "common-macro", @@ -2648,7 +2649,7 @@ dependencies = [ [[package]] name = "common-meta" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "anymap2", "api", @@ -2719,7 +2720,7 @@ dependencies = [ [[package]] name = "common-options" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "common-base", "common-grpc", @@ -2730,11 +2731,11 @@ dependencies = [ [[package]] name = "common-plugins" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" [[package]] name = "common-pprof" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "common-error", "common-macro", @@ -2745,7 +2746,7 @@ dependencies = [ [[package]] name = "common-procedure" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "async-stream", @@ -2774,7 +2775,7 @@ dependencies = [ [[package]] name = "common-procedure-test" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "async-trait", "common-procedure", @@ -2783,7 +2784,7 @@ dependencies = [ [[package]] name = "common-query" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "async-trait", @@ -2813,7 +2814,7 @@ dependencies = [ [[package]] name = "common-recordbatch" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "arc-swap", "common-base", @@ -2838,7 +2839,7 @@ dependencies = [ [[package]] name = "common-runtime" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "async-trait", "catio", @@ -2866,7 +2867,7 @@ dependencies = [ [[package]] name = "common-session" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "serde", "strum 0.27.1", @@ -2874,7 +2875,7 @@ dependencies = [ [[package]] name = "common-sql" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "arrow-schema 59.2.0", "common-base", @@ -2893,7 +2894,7 @@ dependencies = [ [[package]] name = "common-stat" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "common-base", "common-runtime", @@ -2907,7 +2908,7 @@ dependencies = [ [[package]] name = "common-telemetry" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "backtrace", "common-base", @@ -2937,7 +2938,7 @@ dependencies = [ [[package]] name = "common-test-util" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "common-grpc", "common-query", @@ -2949,7 +2950,7 @@ dependencies = [ [[package]] name = "common-time" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "arrow 59.2.0", "chrono", @@ -2967,7 +2968,7 @@ dependencies = [ [[package]] name = "common-version" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "cargo-manifest", "const_format", @@ -2977,7 +2978,7 @@ dependencies = [ [[package]] name = "common-wal" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "common-base", "common-error", @@ -3000,7 +3001,7 @@ dependencies = [ [[package]] name = "common-workload" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "common-telemetry", "serde", @@ -4473,7 +4474,7 @@ dependencies = [ [[package]] name = "datanode" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "arrow-flight", @@ -4546,7 +4547,7 @@ checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" [[package]] name = "datatypes" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "arrow 59.2.0", "arrow-array 59.2.0", @@ -5278,7 +5279,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-engine" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "async-trait", @@ -5419,7 +5420,7 @@ checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8" [[package]] name = "flow" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "arrow 59.2.0", @@ -5471,7 +5472,7 @@ dependencies = [ "snafu 0.8.6", "sql", "store-api", - "substrait 1.3.0-beta.1", + "substrait 1.3.0-beta.2", "table", "tokio", "tokio-stream", @@ -5568,7 +5569,7 @@ dependencies = [ [[package]] name = "frontend" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "arrow-flight", @@ -6946,7 +6947,7 @@ dependencies = [ [[package]] name = "index" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "ahash 0.8.12", "async-trait", @@ -6965,8 +6966,6 @@ dependencies = [ "futures", "greptime-proto", "itertools 0.14.0", - "jieba-rs", - "lazy_static", "mockall", "pin-project", "prost 0.14.1", @@ -8097,7 +8096,7 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "log-query" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "chrono", "common-error", @@ -8109,7 +8108,7 @@ dependencies = [ [[package]] name = "log-store" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "async-stream", "async-trait", @@ -8461,7 +8460,7 @@ dependencies = [ [[package]] name = "meta-client" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "async-trait", @@ -8490,7 +8489,7 @@ dependencies = [ [[package]] name = "meta-srv" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "async-trait", @@ -8581,7 +8580,7 @@ dependencies = [ [[package]] name = "metric-engine" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "aquamarine", @@ -8681,7 +8680,7 @@ dependencies = [ [[package]] name = "mito-codec" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "bytes", @@ -8706,7 +8705,7 @@ dependencies = [ [[package]] name = "mito2" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "aquamarine", @@ -9549,7 +9548,7 @@ dependencies = [ [[package]] name = "object-store" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "anyhow", "bytes", @@ -9695,9 +9694,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendal" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fe43e16d96bed57937eb7c4a28559bf4c5fc4a3951656505d1fdd4f856349b4" +checksum = "3cbd193aec6d62efaff87611dfae52c132aed650bb3cb0c684c0a9394f09a42a" dependencies = [ "ctor", "opendal-core", @@ -9720,9 +9719,9 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de4566a412776e3f65d53dd9fceced5b432a0a9c9a1e8d50ccf58823795e5c5b" +checksum = "54b5555d753dd6a6383dc0891254b72f3cdc40c011ba0372b560e0ae5aa29f5c" dependencies = [ "anyhow", "asyncband", @@ -9746,9 +9745,9 @@ dependencies = [ [[package]] name = "opendal-http-transport-reqwest" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f772ce6c137ab43647726116d505a649a88d41928fcd813438eb30b02312bcbb" +checksum = "4d6588678f3040a3eab1819364dcbc6a746a8286d1ceb23a70a6614faa01dc6c" dependencies = [ "bytes", "futures", @@ -9760,9 +9759,9 @@ dependencies = [ [[package]] name = "opendal-layer-concurrent-limit" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3734c648617e5ad724d49175dde8323550624f9614f50e9e81b7aafae30b1734" +checksum = "db8574309e68bf6d49c5378eaef9d0b3f6786971650b061c69e808254d71cb05" dependencies = [ "asyncband", "futures", @@ -9772,9 +9771,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a8daab5a91eac76f94516e7c71e677195c9221840ce8f5e71815e3fc8207742" +checksum = "4ad43246d2cb7b7ad32b10e22bfa7b6ced8a6677cb6d6f005ae7e1d94cb19c6a" dependencies = [ "log", "opendal-core", @@ -9782,9 +9781,9 @@ dependencies = [ [[package]] name = "opendal-layer-observe-metrics-common" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fbe664be117c293cfa3980a0d2fc238bbb38eae5f6c191f376e88ccb830d26b" +checksum = "3b8bc97d5a00eb41d50f1f6077c4bd5396c1eefea495614084ee9a37202ae06d" dependencies = [ "futures", "http 1.5.0", @@ -9793,9 +9792,9 @@ dependencies = [ [[package]] name = "opendal-layer-prometheus" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32078d8248555ee6271332ec480b78306a44f0d902ea4721ccac00d9b866a21" +checksum = "404e766cc2e4e4c9225ce08605ae151f768a05a41a448a235e016b1aea9ff98f" dependencies = [ "opendal-core", "opendal-layer-observe-metrics-common", @@ -9804,9 +9803,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb8e22cb1f86ef9af771a8947d3ccf380c8f5b76236f628314dd41c9ba73331a" +checksum = "4aeb54be4940c68609483591934a813f7bdd97d747826783a393914d3139e363" dependencies = [ "backon", "log", @@ -9815,9 +9814,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ac6c77fe0d1b18e0064a02c19b98dbebcab78bf5e7753ad1f3abbc18850ff4" +checksum = "1d11d53e67473e8914eacb2e14d586779ae57e1a84759ad25232ef339db77d8a" dependencies = [ "opendal-core", "tokio", @@ -9825,9 +9824,9 @@ dependencies = [ [[package]] name = "opendal-layer-tracing" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "603807927a799ea981f62a3b0012e17181e39f468793b78ca8bedda66d19813c" +checksum = "bbe2ebfc795bf08da77d20bed13d149b478cc8e5002df44d237e460134f00be4" dependencies = [ "futures", "http 1.5.0", @@ -9837,9 +9836,9 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6cf31b02e7b44191d97c07254eea109b5da9c17210a29e478c8d0dc26dffb1f" +checksum = "c32c3fc2e507c854ce53615b28fc2208f5e00cbe456c4d344ffdfbf4c9cb976d" dependencies = [ "base64 0.23.1", "bytes", @@ -9858,9 +9857,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8107324c3f3cb9970fe20526c2c341b7c5769acbd4699bab73bd525f5fd63d62" +checksum = "cc93c30bd347a36d93fd46d47f3e2bb6494dcf9458938aa501fe046ff8f7562f" dependencies = [ "http 1.5.0", "opendal-core", @@ -9868,9 +9867,9 @@ dependencies = [ [[package]] name = "opendal-service-fs" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0e7d8974aebd33be899200c9d0d50b2141fd5fa5c462b29e20653e3ddbfe339" +checksum = "ec5c16fe91b77c0364f66546a1c32b2999e9c3c3c75c2e908a282268e0890bac" dependencies = [ "bytes", "log", @@ -9882,9 +9881,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d80eca6f9227bdf65d43e5b8d5fe75dca8cfab1da07571442a0420400847bea" +checksum = "e1b5e42c6606ea1e37fa9cdedf07dc998073a45eb5044317c0fa6f3153e98d6e" dependencies = [ "async-trait", "bytes", @@ -9904,9 +9903,9 @@ dependencies = [ [[package]] name = "opendal-service-hdfs-native" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b84340e11a3bcd7f1f0b55df032d040b2907483c546c3021485aba749c3072c" +checksum = "20fb97b2ac06a87e6720f0f93d5107ac046117deb8ac957030b0962ba4ecc3c5" dependencies = [ "bytes", "futures", @@ -9918,9 +9917,9 @@ dependencies = [ [[package]] name = "opendal-service-http" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac1553d6b57ca79e200e5c7ded9ab708f2016dadb28ede0f2da71e02fd0b960" +checksum = "65fdde60d1abf14e47eaf19360c08ea1b89c14f03769db448e016781c2e0e32b" dependencies = [ "http 1.5.0", "log", @@ -9930,9 +9929,9 @@ dependencies = [ [[package]] name = "opendal-service-mysql" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e15592be32b105cd152ab398b3db847e22c8c5f230d9b34ff095ea3d0dc6fd97" +checksum = "9ba30e2a0b38aa55579b69bf57f607df7aa613f4d964496407b7998f2dbaefcc" dependencies = [ "asyncband", "opendal-core", @@ -9942,9 +9941,9 @@ dependencies = [ [[package]] name = "opendal-service-oss" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ebfa355b1d4edb0e5b2a5b86d8de9f811fae1d7baca6d65246b230b2afc0bd" +checksum = "d64d2909b444008e42f00137a50df6b55ba599eeb6c8b46c71a2a84948f91880" dependencies = [ "bytes", "http 1.5.0", @@ -9959,9 +9958,9 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2942f2d8d3d4953c0e8d07cd1244879628d084948e48ce6b6d808c5938f7858d" +checksum = "1d83e716c25a81fe2ca701894b8e14b9966acca3812af250e50132accb19da02" dependencies = [ "base64 0.23.1", "bytes", @@ -9981,7 +9980,7 @@ dependencies = [ [[package]] name = "opensrv-mysql" version = "0.8.0" -source = "git+https://github.com/GreptimeTeam/opensrv?rev=f3135e5f00401e06075cf241c0cc26489e45b20a#f3135e5f00401e06075cf241c0cc26489e45b20a" +source = "git+https://github.com/GreptimeTeam/opensrv?rev=1f3d88ccaafb5721be6cfcc3ccbca48642426119#1f3d88ccaafb5721be6cfcc3ccbca48642426119" dependencies = [ "async-trait", "byteorder", @@ -10094,7 +10093,7 @@ dependencies = [ [[package]] name = "operator" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "ahash 0.8.12", "api", @@ -10102,7 +10101,9 @@ dependencies = [ "async-stream", "async-trait", "axum 0.8.4", + "backon", "bytes", + "cache", "catalog", "chrono", "client", @@ -10133,12 +10134,14 @@ dependencies = [ "futures", "futures-util", "humantime", + "humantime-serde", "itertools 0.14.0", "jsonb", "lazy_static", "meta-client", "meter-core", "meter-macros", + "mito-codec", "moka", "object-store", "partition", @@ -10156,9 +10159,10 @@ dependencies = [ "sql", "sqlparser", "store-api", - "substrait 1.3.0-beta.1", + "substrait 1.3.0-beta.2", "table", "tokio", + "tokio-stream", "tokio-util", "tonic 0.14.2", "tracing", @@ -10490,7 +10494,7 @@ checksum = "e3c406c9e2aa74554e662d2c2ee11cd3e73756988800be7e6f5eddb16fed4699" [[package]] name = "partition" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "common-base", @@ -10884,7 +10888,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pipeline" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "ahash 0.8.12", "api", @@ -11039,7 +11043,7 @@ dependencies = [ [[package]] name = "plugins" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "auth", "catalog", @@ -11382,7 +11386,7 @@ dependencies = [ [[package]] name = "promql" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "ahash 0.8.12", "async-trait", @@ -11799,7 +11803,7 @@ dependencies = [ [[package]] name = "puffin" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "async-compression", "async-trait", @@ -11851,7 +11855,7 @@ dependencies = [ [[package]] name = "query" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "ahash 0.8.12", "api", @@ -11918,7 +11922,7 @@ dependencies = [ "sql", "sqlparser", "store-api", - "substrait 1.3.0-beta.1", + "substrait 1.3.0-beta.2", "table", "tokio", "toml 0.8.23", @@ -13533,7 +13537,7 @@ dependencies = [ [[package]] name = "servers" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "ahash 0.8.12", "api", @@ -13676,7 +13680,7 @@ dependencies = [ [[package]] name = "session" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "ahash 0.8.12", "api", @@ -14036,7 +14040,7 @@ dependencies = [ [[package]] name = "sql" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "arrow-buffer 59.2.0", @@ -14091,7 +14095,7 @@ dependencies = [ [[package]] name = "sqlness-runner" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "async-trait", "clap", @@ -14370,7 +14374,7 @@ dependencies = [ [[package]] name = "standalone" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "async-trait", @@ -14420,7 +14424,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "store-api" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "aquamarine", @@ -14612,7 +14616,7 @@ dependencies = [ [[package]] name = "substrait" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "async-trait", "bytes", @@ -14748,7 +14752,7 @@ dependencies = [ [[package]] name = "table" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "arc-swap", @@ -15028,7 +15032,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "tests-fuzz" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "arbitrary", "async-trait", @@ -15076,7 +15080,7 @@ dependencies = [ [[package]] name = "tests-integration" -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" dependencies = [ "api", "arrow-flight", @@ -15155,7 +15159,7 @@ dependencies = [ "sqlx", "standalone", "store-api", - "substrait 1.3.0-beta.1", + "substrait 1.3.0-beta.2", "table", "tempfile", "time", diff --git a/Cargo.toml b/Cargo.toml index 1901ae52767..c002d2c7a18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.3.0-beta.1" +version = "1.3.0-beta.2" edition = "2024" license = "Apache-2.0" @@ -271,6 +271,7 @@ url = "2.3" # Pin below 0.14.5 to avoid the `connection_timeout_future` "async fn resumed after # completion" panic (greptimedb#8265 / grpc-rust#2522). Remove once tonic ships a fix. tonic = { version = "0.14, <0.14.5", features = ["tls-aws-lc", "gzip", "zstd"] } +tonic-prost = "0.14" tower = "0.5" tracing = "0.1" tracing-appender = "0.2" diff --git a/Makefile b/Makefile index 9144432b66d..7f92098262f 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ CARGO_BUILD_OPTS := --locked IMAGE_REGISTRY ?= docker.io IMAGE_NAMESPACE ?= greptime IMAGE_TAG ?= latest -DEV_BUILDER_IMAGE_TAG ?= 1.96.1-822042c7-20260924091315 +DEV_BUILDER_IMAGE_TAG ?= 1.98.1-7f0f592b-20261008091949 DEV_BUILDER_RISCV64_IMAGE_TAG ?= $(DEV_BUILDER_IMAGE_TAG) DEV_BUILDER_RISCV64_BASE_IMAGE ?= ubuntu:22.04 RISCV64_TARGET ?= riscv64gc-unknown-linux-gnu diff --git a/config/config.md b/config/config.md index 29f2d6d2c56..dbd72b5fb1c 100644 --- a/config/config.md +++ b/config/config.md @@ -16,17 +16,16 @@ | `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, and native histogram columns.
Legacy OTLP summary columns keep their historical `greptime_` prefix. | | `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.
When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. | | `experimental_metric_export` | Bool | `false` | Enables experimental Parquet COPY DATABASE using shared Metric physical scans.
Resume requires the previous export and storage writes to have ended; HTTP timeout is not confirmation. | -| `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
Password verifier formats: "plain:", "pbkdf2_sha256:::",
"mysql_native_password:",
"pg_scram_sha256::::"
"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but cannot authenticate over MySQL's
native password handshake; a MySQL client must send the password in cleartext for such users.
"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are
observable in the SCRAM server-first message, and non-default values weaken resistance to username
enumeration. | -| `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
Set to 0 to disable the limit. Default: "0" (unlimited) | -| `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" | +| `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
Password verifier formats: "plain:", "pbkdf2_sha256:::",
"mysql_native_password:",
"pg_scram_sha256::::"
"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:
`static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.
"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are
observable in the SCRAM server-first message, and non-default values weaken resistance to username
enumeration. | +| `max_in_flight_write_bytes` | String | 0 | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
Set to 0 to disable the limit. Default: "0" (unlimited) | +| `write_bytes_exhausted_policy` | String | `wait` | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" | | `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.
By default, it provides services after all regions have been initialized. | | `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. | -| `max_concurrent_queries` | Integer | `0` | The maximum concurrent queries allowed to be executed. Zero means unlimited. | | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. Enabled by default. | | `runtime` | -- | -- | The runtime options. | -| `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. | -| `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute compact operations. | -| `runtime.compact_rt_max_blocking_threads` | Integer | `4` | The maximum number of blocking threads for compact operations.
Defaults to max(num_cpus / 2, 2). | +| `runtime.global_rt_size` | Integer | Auto | The number of threads to execute the runtime for global read operations.
Defaults to max(num_cpus, 2). | +| `runtime.compact_rt_size` | Integer | Auto | The number of threads to execute compact operations.
Defaults to max(num_cpus / 2, 2). | +| `runtime.compact_rt_max_blocking_threads` | Integer | Auto | The maximum number of blocking threads for compact operations.
Defaults to max(num_cpus / 2, 2). | | `runtime.experimental_workload_scheduler` | -- | -- | Experimental weighted, work-conserving query/write task scheduler. | | `runtime.experimental_workload_scheduler.enable` | Bool | `false` | Enable when concurrent queries and writes interfere with each other—for example, when long-running queries increase ingestion latency.
The weights set their relative runtime shares while both are backlogged. Disabled by default. | | `runtime.experimental_workload_scheduler.query_weight` | Integer | `2` | Relative query share while both query and write workloads are backlogged. | @@ -110,7 +109,7 @@ | `wal.enable_log_recycle` | Bool | `true` | Whether to reuse logically truncated log files.
**It's only used when the provider is `raft_engine`**. | | `wal.prefill_log_files` | Bool | `false` | Whether to pre-create log files on start up.
**It's only used when the provider is `raft_engine`**. | | `wal.sync_period` | String | `5s` | Duration for fsyncing log files.
**It's only used when the provider is `raft_engine`**. | -| `wal.recovery_parallelism` | Integer | `2` | Parallelism during WAL recovery. | +| `wal.recovery_parallelism` | Integer | Auto | Parallelism during WAL recovery. Defaults to the number of CPU cores; this example sets it to 2. | | `wal.broker_endpoints` | Array | -- | The Kafka broker endpoints.
**It's only used when the provider is `kafka`**. | | `wal.connect_timeout` | String | `3s` | The connect timeout for kafka client.
**It's only used when the provider is `kafka`**. | | `wal.timeout` | String | `5s` | The total request timeout for kafka client.
**It's only used when the provider is `kafka`**. | @@ -140,7 +139,7 @@ | `procedure.max_running_procedures` | Integer | `128` | Max running procedures.
The maximum number of procedures that can be running at the same time.
If the number of running procedures exceeds this limit, the procedure will be rejected. | | `query` | -- | -- | The query engine options. | | `query.parallelism` | Integer | `0` | Parallelism of the query engine.
Default to 0, which means the number of CPU cores. | -| `query.memory_pool_size` | String | `50%` | Memory pool size for query execution operators (aggregation, sorting, join).
Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%").
Setting it to 0 disables the limit (unbounded, default behavior).
When this limit is reached, queries will fail with ResourceExhausted error.
NOTE: This does NOT limit memory used by table scans. | +| `query.memory_pool_size` | String | 0 | Memory pool size for query execution operators (aggregation, sorting, join).
Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%").
Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.
When this limit is reached, queries will fail with ResourceExhausted error.
NOTE: This does NOT limit memory used by table scans. | | `query.experimental_memory_pool_policy` | String | `greedy` | Experimental memory pool allocation policy:
- "greedy" (default): first-come-first-served allocation; preserves current behavior.
- "fair": divides available memory among spillable operators and may spill earlier.
Only effective when `memory_pool_size` is bounded (>0). | | `query.experimental_spill_mode` | String | `default` | Spill mode:
- "default": preserve DataFusion built-in OS temp directory (default).
- "custom": explicitly configure spill path, quota, and compression.
- "disabled": explicitly disable disk spilling.
Set this to "custom" before using the path/quota/compression keys below. | | `query.experimental_spill_path` | String | Unset | Spill directory path. Ignored unless mode is "custom". | @@ -174,7 +173,7 @@ | `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification
**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. | | `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines.
Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail. | | `region_engine.mito` | -- | -- | The Mito engine options. | -| `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. | +| `region_engine.mito.num_workers` | Integer | Auto | Number of region workers. Defaults to half of the available CPU cores, rounded up. | | `region_engine.mito.worker_channel_size` | Integer | `128` | Request channel size of each worker. | | `region_engine.mito.worker_request_batch_size` | Integer | `64` | Max batch size for a worker to handle requests. | | `region_engine.mito.manifest_checkpoint_distance` | Integer | `10` | Number of meta action updated to trigger a new checkpoint for the manifest. | @@ -258,17 +257,18 @@ | `logging.tracing_sample_ratio` | -- | Unset | The percentage of tracing will be sampled and exported.
Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
ratio > 1 are treated as 1. Fractions < 0 are treated as 0 | | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- | | `slow_query` | -- | -- | The slow query log options. | -| `slow_query.enable` | Bool | `false` | Whether to enable slow query log. | -| `slow_query.record_type` | String | Unset | The record type of slow queries. It can be `system_table` or `log`. | -| `slow_query.threshold` | String | Unset | The threshold of slow query. | -| `slow_query.sample_ratio` | Float | Unset | The sampling ratio of slow query log. The value should be in the range of (0, 1]. | +| `slow_query.enable` | Bool | `true` | Whether to enable slow query log. | +| `slow_query.record_type` | String | `system_table` | The record type of slow queries. It can be `system_table` or `log`.
If `system_table` is selected, the slow queries will be recorded in a system table `greptime_private.slow_queries`.
If `log` is selected, the slow queries will be logged in a log file `greptimedb-slow-queries.*`. | +| `slow_query.threshold` | String | `30s` | The threshold of slow query. It can be human readable time string, for example: `10s`, `100ms`, `1s`. | +| `slow_query.sample_ratio` | Float | `1.0` | The sampling ratio of slow query log. The value should be in the range of (0, 1]. For example, `0.1` means 10% of the slow queries will be logged and `1.0` means all slow queries will be logged. | +| `slow_query.ttl` | String | `90d` | The TTL of the `slow_queries` system table. Default is `90d` when `record_type` is `system_table`. | | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. | | `tracing.tokio_console_addr` | String | Unset | The tokio console address. | | `pipeline` | -- | -- | The pipeline options. | | `pipeline.cache_ttl` | String | `10s` | Time to live of the local pipeline cache. Default is `10s`. | | `event_recorder` | -- | -- | Configuration options for the event recorder. | | `event_recorder.ttl` | String | `90d` | TTL for the events table that will be used to store the events. Default is `90d`. | -| `event_recorder.event_types` | Array | -- | Event types to record. Current available event types: `create_database`,
`alter_database`, `drop_database`, `create_flow`, `drop_flow`,
`create_table`, `create_logical_tables`, `alter_table`, `alter_logical_tables`,
`drop_table`, `undrop_table`, `purge_dropped_table`, `truncate_table`,
`create_view`, `drop_view`, `admin_function`, `reconcile_catalog`,
`reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.
When omitted, all current and future event types are recorded.
Set to an empty array to disable event recording. | +| `event_recorder.event_types` | Array | All | Event types to record. Current available event types: `create_database`,
`alter_database`, `drop_database`, `create_flow`, `drop_flow`,
`create_table`, `create_logical_tables`, `alter_table`, `alter_logical_tables`,
`drop_table`, `undrop_table`, `purge_dropped_table`, `truncate_table`,
`create_view`, `drop_view`, `admin_function`, `reconcile_catalog`,
`reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.
When omitted, all current and future event types are recorded.
Set to an empty array to disable event recording. | | `memory` | -- | -- | The memory options. | | `memory.enable_heap_profiling` | Bool | `true` | Whether to enable heap profiling activation during startup.
When enabled, heap profiling will be activated if the `MALLOC_CONF` environment variable
is set to "prof:true,prof_active:false". The official image adds this env variable.
Default is true. | @@ -283,13 +283,13 @@ | `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, and native histogram columns.
Legacy OTLP summary columns keep their historical `greptime_` prefix. | | `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.
When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. | | `experimental_metric_export` | Bool | `false` | Enables experimental Parquet COPY DATABASE using shared Metric physical scans.
Resume requires the previous export and storage writes to have ended; HTTP timeout is not confirmation. | -| `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
Password verifier formats: "plain:", "pbkdf2_sha256:::",
"mysql_native_password:",
"pg_scram_sha256::::"
"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but cannot authenticate over MySQL's
native password handshake; a MySQL client must send the password in cleartext for such users.
"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are
observable in the SCRAM server-first message, and non-default values weaken resistance to username
enumeration. | -| `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
Set to 0 to disable the limit. Default: "0" (unlimited) | -| `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" | +| `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
Password verifier formats: "plain:", "pbkdf2_sha256:::",
"mysql_native_password:",
"pg_scram_sha256::::"
"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:
`static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.
"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are
observable in the SCRAM server-first message, and non-default values weaken resistance to username
enumeration. | +| `max_in_flight_write_bytes` | String | 0 | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
Set to 0 to disable the limit. Default: "0" (unlimited) | +| `write_bytes_exhausted_policy` | String | `wait` | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" | | `runtime` | -- | -- | The runtime options. | -| `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. | -| `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute compact operations. | -| `runtime.compact_rt_max_blocking_threads` | Integer | `4` | The maximum number of blocking threads for compact operations.
Defaults to max(num_cpus / 2, 2). | +| `runtime.global_rt_size` | Integer | Auto | The number of threads to execute the runtime for global read operations.
Defaults to max(num_cpus, 2). | +| `runtime.compact_rt_size` | Integer | Auto | The number of threads to execute compact operations.
Defaults to max(num_cpus / 2, 2). | +| `runtime.compact_rt_max_blocking_threads` | Integer | Auto | The maximum number of blocking threads for compact operations.
Defaults to max(num_cpus / 2, 2). | | `http` | -- | -- | The HTTP server options. | | `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. | | `http.timeout` | String | `0s` | HTTP request timeout. Set to 0 to disable timeout.
When synchronous Prometheus, OTLP metrics, or ordinary-table batching is enabled, a nonzero timeout is
raised to at least the largest active flush interval plus 1 second. The intervals come from
`pending_rows_batcher.logical_table.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`. | @@ -300,20 +300,20 @@ | `http.api_server_addr` | String | `127.0.0.1:4006` | The address to bind the dedicated HTTP API server, in the same form as `addr`.
Defaults to `127.0.0.1:4006`. | | `grpc` | -- | -- | The gRPC server options. | | `grpc.bind_addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. | -| `grpc.server_addr` | String | `127.0.0.1:4001` | The address advertised to the metasrv, and used for connections from outside the host.
If left empty or unset, the server will automatically use the IP address of the first network interface
on the host, with the same port number as the one specified in `grpc.bind_addr`. | +| `grpc.server_addr` | String | Unset | The address advertised to the metasrv, and used for connections from outside the host.
If left empty or unset, the server will automatically use the IP address of the first network interface
on the host, with the same port number as the one specified in `grpc.bind_addr`. | | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. | -| `grpc.flight_compression` | String | `arrow_ipc` | Compression mode for frontend side Arrow IPC service. Available options:
- `none`: disable all compression
- `transport`: only enable gRPC transport compression (zstd)
- `arrow_ipc`: only enable Arrow IPC compression (lz4)
- `all`: enable all compression.
Default to `none` | +| `grpc.flight_compression` | String | `arrow_ipc` | Compression mode for frontend side Arrow IPC service. Available options:
- `none`: disable all compression
- `transport`: only enable gRPC transport compression (zstd)
- `arrow_ipc`: only enable Arrow IPC compression (lz4)
- `all`: enable all compression.
Default to `arrow_ipc`. | | `grpc.max_connection_age` | String | Unset | The maximum connection age for gRPC connection.
The value can be a human-readable time string. For example: `10m` for ten minutes or `1h` for one hour.
Refer to https://grpc.io/docs/guides/keepalive/ for more details. | | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. | | `grpc.tls.mode` | String | `disable` | TLS mode. | | `grpc.tls.cert_path` | String | Unset | Certificate file path. | | `grpc.tls.key_path` | String | Unset | Private key file path. | | `grpc.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload.
For now, gRPC tls config does not support auto reload. | -| `internal_grpc` | -- | -- | The internal gRPC server options. Internal gRPC port for nodes inside cluster to access frontend. | -| `internal_grpc.bind_addr` | String | `127.0.0.1:4010` | The address to bind the gRPC server. | -| `internal_grpc.server_addr` | String | `127.0.0.1:4010` | The address advertised to the metasrv, and used for connections from outside the host.
If left empty or unset, the server will automatically use the IP address of the first network interface
on the host, with the same port number as the one specified in `internal_grpc.bind_addr`. | +| `internal_grpc` | -- | -- | The internal gRPC server options. Internal gRPC port for nodes inside cluster to access frontend.
The internal gRPC server starts only when this section is set. | +| `internal_grpc.bind_addr` | String | Unset | The address to bind the gRPC server.
If unset, it falls back to `127.0.0.1:4001`, the same address as `grpc.bind_addr`. | +| `internal_grpc.server_addr` | String | Unset | The address advertised to the metasrv, and used for connections from outside the host.
Unlike `grpc.server_addr`, it is not detected automatically; set it explicitly. | | `internal_grpc.runtime_size` | Integer | `8` | The number of server worker threads. | -| `internal_grpc.flight_compression` | String | `arrow_ipc` | Compression mode for frontend side Arrow IPC service. Available options:
- `none`: disable all compression
- `transport`: only enable gRPC transport compression (zstd)
- `arrow_ipc`: only enable Arrow IPC compression (lz4)
- `all`: enable all compression.
Default to `none` | +| `internal_grpc.flight_compression` | String | `arrow_ipc` | Compression mode for frontend side Arrow IPC service. Available options:
- `none`: disable all compression
- `transport`: only enable gRPC transport compression (zstd)
- `arrow_ipc`: only enable Arrow IPC compression (lz4)
- `all`: enable all compression.
Default to `arrow_ipc`. | | `internal_grpc.tls` | -- | -- | internal gRPC server TLS options, see `mysql.tls` section. | | `internal_grpc.tls.mode` | String | `disable` | TLS mode. | | `internal_grpc.tls.cert_path` | String | Unset | Certificate file path. | @@ -381,15 +381,20 @@ | `query` | -- | -- | The query engine options. | | `query.parallelism` | Integer | `0` | Parallelism of the query engine.
Default to 0, which means the number of CPU cores. | | `query.allow_query_fallback` | Bool | `false` | Whether to allow query fallback when push down optimize fails.
Default to false, meaning when push down optimize failed, return error msg | -| `query.memory_pool_size` | String | `50%` | Memory pool size for query execution operators (aggregation, sorting, join).
Supports absolute size (e.g., "4GB", "8GB") or percentage of system memory (e.g., "30%").
Setting it to 0 disables the limit (unbounded, default behavior).
When this limit is reached, queries will fail with ResourceExhausted error.
NOTE: This does NOT limit memory used by table scans (only applies to datanodes). | +| `query.memory_pool_size` | String | 0 | Memory pool size for query execution operators (aggregation, sorting, join).
Supports absolute size (e.g., "4GB", "8GB") or percentage of system memory (e.g., "30%").
Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.
When this limit is reached, queries will fail with ResourceExhausted error.
NOTE: This does NOT limit memory used by table scans (only applies to datanodes). | | `query.experimental_memory_pool_policy` | String | `greedy` | Experimental memory pool allocation policy:
- "greedy" (default): first-come-first-served allocation; preserves current behavior.
- "fair": divides available memory among spillable operators and may spill earlier.
Only effective when `memory_pool_size` is bounded (>0). | | `query.experimental_spill_mode` | String | `default` | Spill mode:
- "default": preserve DataFusion built-in OS temp directory (default).
- "custom": explicitly configure spill path, quota, and compression.
- "disabled": explicitly disable disk spilling.
Set this to "custom" before using the path/quota/compression keys below. | | `query.experimental_spill_path` | String | Unset | Spill directory path. Ignored unless mode is "custom". | | `query.experimental_spill_max_temp_directory_size` | String | `1GiB` | Maximum total size of spill directory (default: "1GiB").
Ignored unless mode is "custom". | | `query.experimental_spill_compression` | String | `uncompressed` | Compression for spilled data files: "uncompressed" (default), "lz4_frame", "zstd".
Ignored unless mode is "custom". | +| `write_retry` | -- | -- | Retry retryable distributed write failures.
Backoff grows exponentially, with jitter to spread concurrent retries. | +| `write_retry.max_retries` | Integer | `3` | Maximum retries after the initial write attempt. Set to 0 to disable retries. | +| `write_retry.initial_backoff` | String | `1500ms` | Initial backoff before jitter is applied. Must be positive. | +| `write_retry.max_backoff` | String | `10s` | Maximum backoff before jitter is applied. Must be at least initial_backoff. | +| `write_retry.backoff_multiplier` | Float | `2.0` | Backoff multiplier. Must be finite and at least 1. | | `datanode` | -- | -- | Datanode options. | | `datanode.client` | -- | -- | Datanode client options. | -| `datanode.client.connect_timeout` | String | `10s` | -- | +| `datanode.client.connect_timeout` | String | 1s | Connect server timeout. Defaults to `1s`; this example sets it to `10s`. | | `datanode.client.tcp_nodelay` | Bool | `true` | -- | | `datanode.client.max_recv_message_size` | String | `512MB` | The maximum receive message size for the gRPC client. | | `datanode.client.max_send_message_size` | String | `512MB` | The maximum send message size for the gRPC client. | @@ -422,7 +427,7 @@ | `pipeline.cache_ttl` | String | `10s` | Time to live of the frontend-local pipeline cache. A pipeline created or deleted on
another frontend takes effect on this one after at most this duration. | | `event_recorder` | -- | -- | Configuration options for the event recorder. | | `event_recorder.ttl` | String | `90d` | TTL for the events table that will be used to store the events. Default is `90d`. | -| `event_recorder.event_types` | Array | -- | Event types to record. Current available event type: `admin_function`.
When omitted, all current and future event types are recorded.
Set to an empty array to disable event recording. | +| `event_recorder.event_types` | Array | All | Event types to record. Current available event type: `admin_function`.
When omitted, all current and future event types are recorded.
Set to an empty array to disable event recording. | ### Metasrv @@ -434,7 +439,7 @@ | `store_key_prefix` | String | `""` | If it's not empty, the metasrv will store all data with this key prefix. | | `backend` | String | `etcd_store` | The datastore for meta server.
Available values:
- `etcd_store` (default value)
- `memory_store`
- `postgres_store`
- `mysql_store` | | `meta_table_name` | String | `greptime_metakv` | Table name in RDS to store metadata. Effect when using a RDS kvbackend.
**Only used when backend is `postgres_store`.** | -| `meta_schema_name` | String | `greptime_schema` | Optional PostgreSQL schema for metadata table and election table name qualification.
When PostgreSQL public schema is not writable (e.g., PostgreSQL 15+ with restricted public),
set this to a writable schema. GreptimeDB will use `meta_schema_name`.`meta_table_name`.
**Only used when backend is `postgres_store`.** | +| `meta_schema_name` | String | Unset | Optional PostgreSQL schema for metadata table and election table name qualification.
When PostgreSQL public schema is not writable (e.g., PostgreSQL 15+ with restricted public),
set this to a writable schema. GreptimeDB will use `meta_schema_name`.`meta_table_name`.
**Only used when backend is `postgres_store`.**
Unset by default; this example sets it to `greptime_schema`. | | `auto_create_schema` | Bool | `true` | Automatically create PostgreSQL schema if it doesn't exist.
When enabled, the system will execute `CREATE SCHEMA IF NOT EXISTS `
before creating metadata tables. This is useful in production environments where
manual schema creation may be restricted.
Default is true.
Note: The PostgreSQL user must have CREATE SCHEMA permission for this to work.
**Only used when backend is `postgres_store`.** | | `meta_election_lock_id` | Integer | `1` | Advisory lock id in PostgreSQL for election. Effect when using PostgreSQL as kvbackend
Only used when backend is `postgres_store`. | | `selector` | String | `round_robin` | Datanode selector type.
- `round_robin` (default value)
- `lease_based`
- `load_based`
For details, please see "https://docs.greptime.com/developer-guide/metasrv/selector". | @@ -445,9 +450,9 @@ | `heartbeat_interval` | String | `3s` | Base heartbeat interval for calculating distributed time constants.
The frontend heartbeat interval is 6 times of the base heartbeat interval.
The flownode/datanode heartbeat interval is 1 times of the base heartbeat interval.
e.g., If the base heartbeat interval is 3s, the frontend heartbeat interval is 18s, the flownode/datanode heartbeat interval is 3s.
Heartbeat intervals are negotiated from metasrv during handshake; local node configs do not override this. | | `enable_telemetry` | Bool | `true` | Whether to enable greptimedb telemetry. Enabled by default. | | `runtime` | -- | -- | The runtime options. | -| `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. | -| `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute compact operations. | -| `runtime.compact_rt_max_blocking_threads` | Integer | `4` | The maximum number of blocking threads for compact operations.
Defaults to max(num_cpus / 2, 2). | +| `runtime.global_rt_size` | Integer | Auto | The number of threads to execute the runtime for global read operations.
Defaults to max(num_cpus, 2). | +| `runtime.compact_rt_size` | Integer | Auto | The number of threads to execute compact operations.
Defaults to max(num_cpus / 2, 2). | +| `runtime.compact_rt_max_blocking_threads` | Integer | Auto | The maximum number of blocking threads for compact operations.
Defaults to max(num_cpus / 2, 2). | | `backend_tls` | -- | -- | TLS configuration for kv store backend (applicable for etcd, PostgreSQL, and MySQL backends)
When using etcd, PostgreSQL, or MySQL as metadata store, you can configure TLS here

Note: if TLS is configured in both this section and the `store_addrs` connection string, the
settings here will override the TLS settings in `store_addrs`. | | `backend_tls.mode` | String | `prefer` | TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html
- "disable" - No TLS
- "prefer" (default) - Try TLS, fallback to plain
- "require" - Require TLS
- "verify_ca" - Require TLS and verify CA
- "verify_full" - Require TLS and verify hostname | | `backend_tls.cert_path` | String | `""` | Path to client certificate file (for client authentication)
Like "/path/to/client.crt" | @@ -459,7 +464,7 @@ | `backend_client.connect_timeout` | String | `3s` | The connect timeout for backend client. | | `grpc` | -- | -- | The gRPC server options. | | `grpc.bind_addr` | String | `127.0.0.1:3002` | The address to bind the gRPC server. | -| `grpc.server_addr` | String | `127.0.0.1:3002` | The communication server address for the frontend and datanode to connect to metasrv.
If left empty or unset, the server will automatically use the IP address of the first network interface
on the host, with the same port number as the one specified in `grpc.bind_addr`. | +| `grpc.server_addr` | String | Unset | The communication server address for the frontend and datanode to connect to metasrv.
If left empty or unset, the server will automatically use the IP address of the first network interface
on the host, with the same port number as the one specified in `grpc.bind_addr`. | | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. | | `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. | | `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. | @@ -481,7 +486,7 @@ | `datanode` | -- | -- | Datanode options. | | `datanode.client` | -- | -- | Datanode client options. | | `datanode.client.timeout` | String | `10s` | Operation timeout. | -| `datanode.client.connect_timeout` | String | `10s` | Connect server timeout. | +| `datanode.client.connect_timeout` | String | 1s | Connect server timeout. Defaults to `1s`; this example sets it to `10s`. | | `datanode.client.tcp_nodelay` | Bool | `true` | `TCP_NODELAY` option for accepted connections. | | `datanode.client.max_recv_message_size` | String | `512MB` | The maximum receive message size for the gRPC client. | | `datanode.client.max_send_message_size` | String | `512MB` | The maximum send message size for the gRPC client. | @@ -503,7 +508,7 @@ | `wal.create_topic_timeout` | String | `30s` | The timeout for creating a Kafka topic.
**It's only used when the provider is `kafka`**. | | `event_recorder` | -- | -- | Configuration options for the event recorder. | | `event_recorder.ttl` | String | `90d` | TTL for the events table that will be used to store the events. Default is `90d`. | -| `event_recorder.event_types` | Array | -- | Event types to record. Current available event types: `region_migration`,
`create_database`, `alter_database`, `drop_database`, `create_flow`,
`drop_flow`, `create_table`, `create_logical_tables`, `alter_table`,
`alter_logical_tables`, `drop_table`, `undrop_table`, `purge_dropped_table`,
`truncate_table`, `create_view`, `drop_view`, `repartition`,
`repartition_group`, `wal_prune`, `batch_gc`, `reconcile_catalog`,
`reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.
When omitted, all current and future event types are recorded.
Set to an empty array to disable event recording. | +| `event_recorder.event_types` | Array | All | Event types to record. Current available event types: `region_migration`,
`create_database`, `alter_database`, `drop_database`, `create_flow`,
`drop_flow`, `create_table`, `create_logical_tables`, `alter_table`,
`alter_logical_tables`, `drop_table`, `undrop_table`, `purge_dropped_table`,
`truncate_table`, `create_view`, `drop_view`, `repartition`,
`repartition_group`, `wal_prune`, `batch_gc`, `reconcile_catalog`,
`reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.
When omitted, all current and future event types are recorded.
Set to an empty array to disable event recording. | | `stats_persistence` | -- | -- | Configuration options for the stats persistence. | | `stats_persistence.ttl` | String | `0s` | TTL for the stats table that will be used to store the stats.
Set to `0s` to disable stats persistence.
Default is `0s`.
If you want to enable stats persistence, set the TTL to a value greater than 0.
It is recommended to set a small value, e.g., `3h`. | | `stats_persistence.interval` | String | `10m` | The interval to persist the stats. Default is `10m`.
The minimum value is `10m`, if the value is less than `10m`, it will be overridden to `10m`. | @@ -548,22 +553,22 @@ | `http.body_limit` | String | `64MB` | HTTP request body limit.
The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.
Set to 0 to disable limit. | | `grpc` | -- | -- | The gRPC server options. | | `grpc.bind_addr` | String | `127.0.0.1:3001` | The address to bind the gRPC server. | -| `grpc.server_addr` | String | `127.0.0.1:3001` | The address advertised to the metasrv, and used for connections from outside the host.
If left empty or unset, the server will automatically use the IP address of the first network interface
on the host, with the same port number as the one specified in `grpc.bind_addr`. | +| `grpc.server_addr` | String | Unset | The address advertised to the metasrv, and used for connections from outside the host.
If left empty or unset, the server will automatically use the IP address of the first network interface
on the host, with the same port number as the one specified in `grpc.bind_addr`. | | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. | | `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. | | `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. | -| `grpc.flight_compression` | String | `arrow_ipc` | Compression mode for datanode side Arrow IPC service. Available options:
- `none`: disable all compression
- `transport`: only enable gRPC transport compression (zstd)
- `arrow_ipc`: only enable Arrow IPC compression (lz4)
- `all`: enable all compression.
Default to `none` | +| `grpc.flight_compression` | String | `arrow_ipc` | Compression mode for datanode side Arrow IPC service. Available options:
- `none`: disable all compression
- `transport`: only enable gRPC transport compression (zstd)
- `arrow_ipc`: only enable Arrow IPC compression (lz4)
- `all`: enable all compression.
Default to `arrow_ipc`. | | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. | | `grpc.tls.mode` | String | `disable` | TLS mode. | | `grpc.tls.cert_path` | String | Unset | Certificate file path. | | `grpc.tls.key_path` | String | Unset | Private key file path. | | `grpc.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload.
For now, gRPC tls config does not support auto reload. | | `runtime` | -- | -- | The runtime options. | -| `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. | -| `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute compact operations. | -| `runtime.compact_rt_max_blocking_threads` | Integer | `4` | The maximum number of blocking threads for compact operations.
Defaults to max(num_cpus / 2, 2). | -| `runtime.query_rt_size` | Integer | `7` | The number of threads to execute datanode query operations.
Defaults to max(num_cpus - 1, 2). | -| `runtime.ingest_rt_size` | Integer | `8` | The number of threads to execute datanode ingestion operations. | +| `runtime.global_rt_size` | Integer | Auto | The number of threads to execute the runtime for global read operations.
Defaults to max(num_cpus, 2). | +| `runtime.compact_rt_size` | Integer | Auto | The number of threads to execute compact operations.
Defaults to max(num_cpus / 2, 2). | +| `runtime.compact_rt_max_blocking_threads` | Integer | Auto | The maximum number of blocking threads for compact operations.
Defaults to max(num_cpus / 2, 2). | +| `runtime.query_rt_size` | Integer | Auto | The number of threads to execute datanode query operations.
Defaults to max(num_cpus - 1, 2). | +| `runtime.ingest_rt_size` | Integer | Auto | The number of threads to execute datanode ingestion operations.
Defaults to max(num_cpus, 2). | | `runtime.experimental_workload_scheduler` | -- | -- | Experimental weighted, work-conserving query/write task scheduler. | | `runtime.experimental_workload_scheduler.enable` | Bool | `false` | Enable when concurrent queries and writes interfere with each other—for example, when long-running queries increase ingestion latency.
The weights set their relative runtime shares while both are backlogged. Disabled by default. | | `runtime.experimental_workload_scheduler.query_weight` | Integer | `2` | Relative query share while both query and write workloads are backlogged. | @@ -589,7 +594,7 @@ | `wal.enable_log_recycle` | Bool | `true` | Whether to reuse logically truncated log files.
**It's only used when the provider is `raft_engine`**. | | `wal.prefill_log_files` | Bool | `false` | Whether to pre-create log files on start up.
**It's only used when the provider is `raft_engine`**. | | `wal.sync_period` | String | `5s` | Duration for fsyncing log files.
**It's only used when the provider is `raft_engine`**. | -| `wal.recovery_parallelism` | Integer | `2` | Parallelism during WAL recovery. | +| `wal.recovery_parallelism` | Integer | Auto | Parallelism during WAL recovery. Defaults to the number of CPU cores; this example sets it to 2. | | `wal.broker_endpoints` | Array | -- | The Kafka broker endpoints.
**It's only used when the provider is `kafka`**. | | `wal.connect_timeout` | String | `3s` | The connect timeout for kafka client.
**It's only used when the provider is `kafka`**. | | `wal.timeout` | String | `5s` | The total request timeout for kafka client.
**It's only used when the provider is `kafka`**. | @@ -607,7 +612,7 @@ | `wal.on_corrupted_segment` | String | `skip` | What a read does with a segment that still does not decode after a second fetch, because its checksum does not match or its content disagrees with its footer entry.
- `skip`: the segment is skipped and recorded as a WAL hole of its region, a metric is incremented and a warning is logged; the other regions of the object are unaffected (the default).
- `fail`: the read fails, so the region does not open.
**It's only used when the provider is `experimental_object_store`**. | | `query` | -- | -- | The query engine options. | | `query.parallelism` | Integer | `0` | Parallelism of the query engine.
Default to 0, which means the number of CPU cores. | -| `query.memory_pool_size` | String | `50%` | Memory pool size for query execution operators (aggregation, sorting, join).
Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%").
Setting it to 0 disables the limit (unbounded, default behavior).
When this limit is reached, queries will fail with ResourceExhausted error.
NOTE: This does NOT limit memory used by table scans. | +| `query.memory_pool_size` | String | 0 | Memory pool size for query execution operators (aggregation, sorting, join).
Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%").
Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.
When this limit is reached, queries will fail with ResourceExhausted error.
NOTE: This does NOT limit memory used by table scans. | | `query.experimental_memory_pool_policy` | String | `greedy` | Experimental memory pool allocation policy:
- "greedy" (default): first-come-first-served allocation; preserves current behavior.
- "fair": divides available memory among spillable operators and may spill earlier.
Only effective when `memory_pool_size` is bounded (>0). | | `query.experimental_spill_mode` | String | `default` | Spill mode:
- "default": preserve DataFusion built-in OS temp directory (default).
- "custom": explicitly configure spill path, quota, and compression.
- "disabled": explicitly disable disk spilling.
Set this to "custom" before using the path/quota/compression keys below. | | `query.experimental_spill_path` | String | Unset | Spill directory path. Ignored unless mode is "custom". | @@ -640,7 +645,7 @@ | `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification
**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. | | `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines.
Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail. | | `region_engine.mito` | -- | -- | The Mito engine options. | -| `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. | +| `region_engine.mito.num_workers` | Integer | Auto | Number of region workers. Defaults to half of the available CPU cores, rounded up. | | `region_engine.mito.worker_channel_size` | Integer | `128` | Request channel size of each worker. | | `region_engine.mito.worker_request_batch_size` | Integer | `64` | Max batch size for a worker to handle requests. | | `region_engine.mito.manifest_checkpoint_distance` | Integer | `10` | Number of meta action updated to trigger a new checkpoint for the manifest. | @@ -757,9 +762,9 @@ | `flow.batching_mode.frontend_tls.client_cert_path` | String | Unset | Client Certificate file path. | | `flow.batching_mode.frontend_tls.client_key_path` | String | Unset | Client Private key file path. | | `grpc` | -- | -- | The gRPC server options. | -| `grpc.bind_addr` | String | `127.0.0.1:6800` | The address to bind the gRPC server. | -| `grpc.server_addr` | String | `127.0.0.1:6800` | The address advertised to the metasrv,
and used for connections from outside the host | -| `grpc.runtime_size` | Integer | `2` | The number of server worker threads. | +| `grpc.bind_addr` | String | 127.0.0.1:3004 | The address to bind the gRPC server. Defaults to `127.0.0.1:3004`; this example sets it to `127.0.0.1:6800`. | +| `grpc.server_addr` | String | Unset | The address advertised to the metasrv, and used for connections from outside the host.
If left empty or unset, the server will automatically use the IP address of the first network interface
on the host, with the same port number as the one specified in `grpc.bind_addr`. | +| `grpc.runtime_size` | Integer | 8 | The number of server worker threads. Defaults to 8; this example sets it to 2. | | `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. | | `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. | | `http` | -- | -- | The HTTP server options. | @@ -793,7 +798,7 @@ | `tracing.tokio_console_addr` | String | Unset | The tokio console address. | | `query` | -- | -- | -- | | `query.parallelism` | Integer | `1` | Parallelism of the query engine for query sent by flownode.
Default to 1, so it won't use too much cpu or memory | -| `query.memory_pool_size` | String | `50%` | Memory pool size for query execution operators (aggregation, sorting, join).
Supports absolute size (e.g., "1GB", "2GB") or percentage of system memory (e.g., "20%").
Setting it to 0 disables the limit (unbounded, default behavior).
When this limit is reached, queries will fail with ResourceExhausted error.
NOTE: This does NOT limit memory used by table scans. | +| `query.memory_pool_size` | String | 0 | Memory pool size for query execution operators (aggregation, sorting, join).
Supports absolute size (e.g., "1GB", "2GB") or percentage of system memory (e.g., "20%").
Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.
When this limit is reached, queries will fail with ResourceExhausted error.
NOTE: This does NOT limit memory used by table scans. | | `query.experimental_memory_pool_policy` | String | `greedy` | Experimental memory pool allocation policy:
- "greedy" (default): first-come-first-served allocation; preserves current behavior.
- "fair": divides available memory among spillable operators and may spill earlier.
Only effective when `memory_pool_size` is bounded (>0). | | `query.experimental_spill_mode` | String | `default` | Spill mode:
- "default": preserve DataFusion built-in OS temp directory (default).
- "custom": explicitly configure spill path, quota, and compression.
- "disabled": explicitly disable disk spilling.
Set this to "custom" before using the path/quota/compression keys below. | | `query.experimental_spill_path` | String | Unset | Spill directory path. Ignored unless mode is "custom". | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index e61ef3fbf22..d0f7da36acd 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -45,6 +45,7 @@ bind_addr = "127.0.0.1:3001" ## The address advertised to the metasrv, and used for connections from outside the host. ## If left empty or unset, the server will automatically use the IP address of the first network interface ## on the host, with the same port number as the one specified in `grpc.bind_addr`. +## @toml2docs:none-default server_addr = "127.0.0.1:3001" ## The number of server worker threads. runtime_size = 8 @@ -57,7 +58,7 @@ max_send_message_size = "512MB" ## - `transport`: only enable gRPC transport compression (zstd) ## - `arrow_ipc`: only enable Arrow IPC compression (lz4) ## - `all`: enable all compression. -## Default to `none` +## Default to `arrow_ipc`. flight_compression = "arrow_ipc" ## gRPC server TLS options, see `mysql.tls` section. @@ -80,16 +81,24 @@ watch = false ## The runtime options. #+ [runtime] ## The number of threads to execute the runtime for global read operations. +## Defaults to max(num_cpus, 2). +## @toml2docs:none-default="Auto" #+ global_rt_size = 8 ## The number of threads to execute compact operations. +## Defaults to max(num_cpus / 2, 2). +## @toml2docs:none-default="Auto" #+ compact_rt_size = 4 ## The maximum number of blocking threads for compact operations. ## Defaults to max(num_cpus / 2, 2). +## @toml2docs:none-default="Auto" #+ compact_rt_max_blocking_threads = 4 ## The number of threads to execute datanode query operations. ## Defaults to max(num_cpus - 1, 2). +## @toml2docs:none-default="Auto" #+ query_rt_size = 7 ## The number of threads to execute datanode ingestion operations. +## Defaults to max(num_cpus, 2). +## @toml2docs:none-default="Auto" #+ ingest_rt_size = 8 ## Experimental weighted, work-conserving query/write task scheduler. @@ -176,7 +185,8 @@ prefill_log_files = false ## **It's only used when the provider is `raft_engine`**. sync_period = "5s" -## Parallelism during WAL recovery. +## Parallelism during WAL recovery. Defaults to the number of CPU cores; this example sets it to 2. +## @toml2docs:none-default="Auto" recovery_parallelism = 2 ## The Kafka broker endpoints. @@ -330,9 +340,10 @@ parallelism = 0 ## Memory pool size for query execution operators (aggregation, sorting, join). ## Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%"). -## Setting it to 0 disables the limit (unbounded, default behavior). +## Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`. ## When this limit is reached, queries will fail with ResourceExhausted error. ## NOTE: This does NOT limit memory used by table scans. +## @toml2docs:none-default="0" memory_pool_size = "50%" ## Experimental memory pool allocation policy: @@ -503,7 +514,8 @@ skip_ssl_validation = false ## The Mito engine options. [region_engine.mito] -## Number of region workers. +## Number of region workers. Defaults to half of the available CPU cores, rounded up. +## @toml2docs:none-default="Auto" #+ num_workers = 8 ## Request channel size of each worker. diff --git a/config/flownode.example.toml b/config/flownode.example.toml index b0a53bee54a..fab31fed510 100644 --- a/config/flownode.example.toml +++ b/config/flownode.example.toml @@ -44,12 +44,16 @@ node_id = 14 ## The gRPC server options. [grpc] -## The address to bind the gRPC server. +## The address to bind the gRPC server. Defaults to `127.0.0.1:3004`; this example sets it to `127.0.0.1:6800`. +## @toml2docs:none-default="127.0.0.1:3004" bind_addr = "127.0.0.1:6800" -## The address advertised to the metasrv, -## and used for connections from outside the host +## The address advertised to the metasrv, and used for connections from outside the host. +## If left empty or unset, the server will automatically use the IP address of the first network interface +## on the host, with the same port number as the one specified in `grpc.bind_addr`. +## @toml2docs:none-default server_addr = "127.0.0.1:6800" -## The number of server worker threads. +## The number of server worker threads. Defaults to 8; this example sets it to 2. +## @toml2docs:none-default="8" runtime_size = 2 ## The maximum receive message size for gRPC server. max_recv_message_size = "512MB" @@ -153,9 +157,10 @@ parallelism = 1 ## Memory pool size for query execution operators (aggregation, sorting, join). ## Supports absolute size (e.g., "1GB", "2GB") or percentage of system memory (e.g., "20%"). -## Setting it to 0 disables the limit (unbounded, default behavior). +## Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`. ## When this limit is reached, queries will fail with ResourceExhausted error. ## NOTE: This does NOT limit memory used by table scans. +## @toml2docs:none-default="0" memory_pool_size = "50%" ## Experimental memory pool allocation policy: diff --git a/config/frontend.example.toml b/config/frontend.example.toml index b528da0304f..0e7c0482fa3 100644 --- a/config/frontend.example.toml +++ b/config/frontend.example.toml @@ -20,8 +20,8 @@ default_column_prefix = "greptime" ## Password verifier formats: "plain:", "pbkdf2_sha256:::", ## "mysql_native_password:", ## "pg_scram_sha256::::" -## "pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but cannot authenticate over MySQL's -## native password handshake; a MySQL client must send the password in cleartext for such users. +## "pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL: +## `static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats. ## "mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all. ## PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or ## "mysql_native_password", PostgreSQL falls back to cleartext password auth for every user. @@ -33,22 +33,26 @@ default_column_prefix = "greptime" ## Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight). ## Set to 0 to disable the limit. Default: "0" (unlimited) -## @toml2docs:none-default +## @toml2docs:none-default="0" #+ max_in_flight_write_bytes = "1GB" ## Policy when write bytes quota is exhausted. ## Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" -## @toml2docs:none-default #+ write_bytes_exhausted_policy = "wait" ## The runtime options. #+ [runtime] ## The number of threads to execute the runtime for global read operations. +## Defaults to max(num_cpus, 2). +## @toml2docs:none-default="Auto" #+ global_rt_size = 8 ## The number of threads to execute compact operations. +## Defaults to max(num_cpus / 2, 2). +## @toml2docs:none-default="Auto" #+ compact_rt_size = 4 ## The maximum number of blocking threads for compact operations. ## Defaults to max(num_cpus / 2, 2). +## @toml2docs:none-default="Auto" #+ compact_rt_max_blocking_threads = 4 ## The HTTP server options. @@ -86,6 +90,7 @@ bind_addr = "127.0.0.1:4001" ## The address advertised to the metasrv, and used for connections from outside the host. ## If left empty or unset, the server will automatically use the IP address of the first network interface ## on the host, with the same port number as the one specified in `grpc.bind_addr`. +## @toml2docs:none-default server_addr = "127.0.0.1:4001" ## The number of server worker threads. runtime_size = 8 @@ -94,7 +99,7 @@ runtime_size = 8 ## - `transport`: only enable gRPC transport compression (zstd) ## - `arrow_ipc`: only enable Arrow IPC compression (lz4) ## - `all`: enable all compression. -## Default to `none` +## Default to `arrow_ipc`. flight_compression = "arrow_ipc" ## The maximum connection age for gRPC connection. ## The value can be a human-readable time string. For example: `10m` for ten minutes or `1h` for one hour. @@ -120,12 +125,15 @@ key_path = "" watch = false ## The internal gRPC server options. Internal gRPC port for nodes inside cluster to access frontend. +## The internal gRPC server starts only when this section is set. [internal_grpc] ## The address to bind the gRPC server. +## If unset, it falls back to `127.0.0.1:4001`, the same address as `grpc.bind_addr`. +## @toml2docs:none-default bind_addr = "127.0.0.1:4010" ## The address advertised to the metasrv, and used for connections from outside the host. -## If left empty or unset, the server will automatically use the IP address of the first network interface -## on the host, with the same port number as the one specified in `internal_grpc.bind_addr`. +## Unlike `grpc.server_addr`, it is not detected automatically; set it explicitly. +## @toml2docs:none-default server_addr = "127.0.0.1:4010" ## The number of server worker threads. runtime_size = 8 @@ -134,7 +142,7 @@ runtime_size = 8 ## - `transport`: only enable gRPC transport compression (zstd) ## - `arrow_ipc`: only enable Arrow IPC compression (lz4) ## - `all`: enable all compression. -## Default to `none` +## Default to `arrow_ipc`. flight_compression = "arrow_ipc" ## internal gRPC server TLS options, see `mysql.tls` section. @@ -352,9 +360,10 @@ allow_query_fallback = false ## Memory pool size for query execution operators (aggregation, sorting, join). ## Supports absolute size (e.g., "4GB", "8GB") or percentage of system memory (e.g., "30%"). -## Setting it to 0 disables the limit (unbounded, default behavior). +## Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`. ## When this limit is reached, queries will fail with ResourceExhausted error. ## NOTE: This does NOT limit memory used by table scans (only applies to datanodes). +## @toml2docs:none-default="0" memory_pool_size = "50%" ## Experimental memory pool allocation policy: @@ -381,10 +390,24 @@ memory_pool_size = "50%" #+ experimental_spill_compression = "uncompressed" # --- End spill-to-disk --- +## Retry retryable distributed write failures. +## Backoff grows exponentially, with jitter to spread concurrent retries. +[write_retry] +## Maximum retries after the initial write attempt. Set to 0 to disable retries. +max_retries = 3 +## Initial backoff before jitter is applied. Must be positive. +initial_backoff = "1500ms" +## Maximum backoff before jitter is applied. Must be at least initial_backoff. +max_backoff = "10s" +## Backoff multiplier. Must be finite and at least 1. +backoff_multiplier = 2.0 + ## Datanode options. [datanode] ## Datanode client options. [datanode.client] +## Connect server timeout. Defaults to `1s`; this example sets it to `10s`. +## @toml2docs:none-default="1s" connect_timeout = "10s" tcp_nodelay = true ## The maximum receive message size for the gRPC client. @@ -489,4 +512,5 @@ ttl = "90d" ## Event types to record. Current available event type: `admin_function`. ## When omitted, all current and future event types are recorded. ## Set to an empty array to disable event recording. +## @toml2docs:none-default="All" #+ event_types = ["admin_function"] diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml index b923c5a549c..fe4c38d02f9 100644 --- a/config/metasrv.example.toml +++ b/config/metasrv.example.toml @@ -35,6 +35,8 @@ meta_table_name = "greptime_metakv" ## When PostgreSQL public schema is not writable (e.g., PostgreSQL 15+ with restricted public), ## set this to a writable schema. GreptimeDB will use `meta_schema_name`.`meta_table_name`. ## **Only used when backend is `postgres_store`.** +## Unset by default; this example sets it to `greptime_schema`. +## @toml2docs:none-default meta_schema_name = "greptime_schema" ## Automatically create PostgreSQL schema if it doesn't exist. @@ -88,11 +90,16 @@ node_max_idle_time = "24hours" ## The runtime options. #+ [runtime] ## The number of threads to execute the runtime for global read operations. +## Defaults to max(num_cpus, 2). +## @toml2docs:none-default="Auto" #+ global_rt_size = 8 ## The number of threads to execute compact operations. +## Defaults to max(num_cpus / 2, 2). +## @toml2docs:none-default="Auto" #+ compact_rt_size = 4 ## The maximum number of blocking threads for compact operations. ## Defaults to max(num_cpus / 2, 2). +## @toml2docs:none-default="Auto" #+ compact_rt_max_blocking_threads = 4 ## TLS configuration for kv store backend (applicable for etcd, PostgreSQL, and MySQL backends) @@ -140,6 +147,7 @@ bind_addr = "127.0.0.1:3002" ## The communication server address for the frontend and datanode to connect to metasrv. ## If left empty or unset, the server will automatically use the IP address of the first network interface ## on the host, with the same port number as the one specified in `grpc.bind_addr`. +## @toml2docs:none-default server_addr = "127.0.0.1:3002" ## The number of server worker threads. runtime_size = 8 @@ -206,7 +214,8 @@ acceptable_heartbeat_pause = "10000ms" ## Operation timeout. timeout = "10s" -## Connect server timeout. +## Connect server timeout. Defaults to `1s`; this example sets it to `10s`. +## @toml2docs:none-default="1s" connect_timeout = "10s" ## `TCP_NODELAY` option for accepted connections. @@ -332,6 +341,7 @@ ttl = "90d" ## `reconcile_database`, `reconcile_table`, `reconcile_logical_tables`. ## When omitted, all current and future event types are recorded. ## Set to an empty array to disable event recording. +## @toml2docs:none-default="All" #+ event_types = ["region_migration"] ## Configuration options for the stats persistence. diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 417a3da6220..cebbf8ac265 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -20,8 +20,8 @@ default_column_prefix = "greptime" ## Password verifier formats: "plain:", "pbkdf2_sha256:::", ## "mysql_native_password:", ## "pg_scram_sha256::::" -## "pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but cannot authenticate over MySQL's -## native password handshake; a MySQL client must send the password in cleartext for such users. +## "pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL: +## `static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats. ## "mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all. ## PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or ## "mysql_native_password", PostgreSQL falls back to cleartext password auth for every user. @@ -33,12 +33,11 @@ default_column_prefix = "greptime" ## Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight). ## Set to 0 to disable the limit. Default: "0" (unlimited) -## @toml2docs:none-default +## @toml2docs:none-default="0" #+ max_in_flight_write_bytes = "1GB" ## Policy when write bytes quota is exhausted. ## Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" -## @toml2docs:none-default #+ write_bytes_exhausted_policy = "wait" ## Initialize all regions in the background during the startup. @@ -48,20 +47,22 @@ init_regions_in_background = false ## Parallelism of initializing regions. init_regions_parallelism = 16 -## The maximum concurrent queries allowed to be executed. Zero means unlimited. -max_concurrent_queries = 0 - ## Enable telemetry to collect anonymous usage data. Enabled by default. #+ enable_telemetry = true ## The runtime options. #+ [runtime] ## The number of threads to execute the runtime for global read operations. +## Defaults to max(num_cpus, 2). +## @toml2docs:none-default="Auto" #+ global_rt_size = 8 ## The number of threads to execute compact operations. +## Defaults to max(num_cpus / 2, 2). +## @toml2docs:none-default="Auto" #+ compact_rt_size = 4 ## The maximum number of blocking threads for compact operations. ## Defaults to max(num_cpus / 2, 2). +## @toml2docs:none-default="Auto" #+ compact_rt_max_blocking_threads = 4 ## Experimental weighted, work-conserving query/write task scheduler. @@ -339,7 +340,8 @@ prefill_log_files = false ## **It's only used when the provider is `raft_engine`**. sync_period = "5s" -## Parallelism during WAL recovery. +## Parallelism during WAL recovery. Defaults to the number of CPU cores; this example sets it to 2. +## @toml2docs:none-default="Auto" recovery_parallelism = 2 ## The Kafka broker endpoints. @@ -533,9 +535,10 @@ parallelism = 0 ## Memory pool size for query execution operators (aggregation, sorting, join). ## Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%"). -## Setting it to 0 disables the limit (unbounded, default behavior). +## Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`. ## When this limit is reached, queries will fail with ResourceExhausted error. ## NOTE: This does NOT limit memory used by table scans. +## @toml2docs:none-default="0" memory_pool_size = "50%" ## Experimental memory pool allocation policy: @@ -715,7 +718,8 @@ skip_ssl_validation = false ## The Mito engine options. [region_engine.mito] -## Number of region workers. +## Number of region workers. Defaults to half of the available CPU cores, rounded up. +## @toml2docs:none-default="Auto" #+ num_workers = 8 ## Request channel size of each worker. @@ -1057,19 +1061,21 @@ default_ratio = 1.0 ## The slow query log options. [slow_query] ## Whether to enable slow query log. -#+ enable = false +enable = true ## The record type of slow queries. It can be `system_table` or `log`. -## @toml2docs:none-default -#+ record_type = "system_table" +## If `system_table` is selected, the slow queries will be recorded in a system table `greptime_private.slow_queries`. +## If `log` is selected, the slow queries will be logged in a log file `greptimedb-slow-queries.*`. +record_type = "system_table" -## The threshold of slow query. -## @toml2docs:none-default -#+ threshold = "10s" +## The threshold of slow query. It can be human readable time string, for example: `10s`, `100ms`, `1s`. +threshold = "30s" -## The sampling ratio of slow query log. The value should be in the range of (0, 1]. -## @toml2docs:none-default -#+ sample_ratio = 1.0 +## The sampling ratio of slow query log. The value should be in the range of (0, 1]. For example, `0.1` means 10% of the slow queries will be logged and `1.0` means all slow queries will be logged. +sample_ratio = 1.0 + +## The TTL of the `slow_queries` system table. Default is `90d` when `record_type` is `system_table`. +ttl = "90d" ## The tracing options. Only effect when compiled with `tokio-console` feature. #+ [tracing] @@ -1094,6 +1100,7 @@ ttl = "90d" ## `reconcile_database`, `reconcile_table`, `reconcile_logical_tables`. ## When omitted, all current and future event types are recorded. ## Set to an empty array to disable event recording. +## @toml2docs:none-default="All" #+ event_types = ["create_database"] ## The memory options. diff --git a/flake.lock b/flake.lock index 27c1797f10b..621f1e2f3a0 100644 --- a/flake.lock +++ b/flake.lock @@ -8,11 +8,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1785314864, - "narHash": "sha256-imz9J5iMNersDeQ1ymenFg35N6C2R3MCaGg2i0XCIrE=", + "lastModified": 1790642863, + "narHash": "sha256-JIwE6EnebvxW7QFFEitWHuZDXwN6Qcf6ZC8DIvYW3y8=", "owner": "nix-community", "repo": "fenix", - "rev": "594418b2c9ea0731bee6988236ef5bfd97e81dca", + "rev": "a8f386c482e78e9a48be5abecdc17a9b098a95a1", "type": "github" }, "original": { @@ -41,11 +41,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1785133411, - "narHash": "sha256-Yjv0WEg39KRYS0rBdTbu6Fc/or/ihAKk13W9sQ6VWd0=", + "lastModified": 1790533781, + "narHash": "sha256-F+b8u43BVF5DlM0pz7JQSbtbwKy5PrHIT5zQUekXsD0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2f5a153c270b70cb0f8c11f46d96d6d3bc39f4e3", + "rev": "cf5e76507c6e23b59f7e0ffcc7baa2a39ddd8442", "type": "github" }, "original": { @@ -65,11 +65,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1785261141, - "narHash": "sha256-sA+DHPejWD68mLA7gtTiHGd5T41F6W+NKf0g+ibFCW4=", + "lastModified": 1790532271, + "narHash": "sha256-j8y/AO3AMDF9p+11aIDhnA21JnNj5PjvcejTI43vI8Q=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "bec66814323579659ffd77c909b3d963af118ece", + "rev": "03fcb77246f2568adb0e9b2fa60d19c6cc1686f4", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 6e4151fb3e8..df71403b24f 100644 --- a/flake.nix +++ b/flake.nix @@ -22,7 +22,7 @@ lib = nixpkgs.lib; rustToolchain = fenix.packages.${system}.fromToolchainName { name = (lib.importTOML ./rust-toolchain.toml).toolchain.channel; - sha256 = "sha256-h+t2xTBz5yt2YIO+1VMIIGlCU7gyp2LYOFvaV1nwOXU="; + sha256 = "sha256-p8h3Sl/YRByZfZTAKXdsvF6xEenXKrXSVvpphmZENH4="; }; in { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 2d45363a5be..642220914d8 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.96.1" +channel = "1.98.1" diff --git a/src/catalog/src/kvbackend/client.rs b/src/catalog/src/kvbackend/client.rs index 69b31d62322..402af1d9865 100644 --- a/src/catalog/src/kvbackend/client.rs +++ b/src/catalog/src/kvbackend/client.rs @@ -192,6 +192,10 @@ impl KvBackend for CachedKvBackend { } } + if miss_keys.is_empty() { + return Ok(BatchGetResponse { kvs }); + } + let batch_get_req = BatchGetRequest::new().with_keys(miss_keys.clone()); let pre_version = self.version(); @@ -503,6 +507,7 @@ mod tests { pub struct SimpleKvBackend { inner_map: DashMap, Vec>, get_execute_times: Arc, + batch_get_execute_times: Arc, } impl TxnService for SimpleKvBackend { @@ -520,6 +525,7 @@ mod tests { } async fn batch_get(&self, req: BatchGetRequest) -> Result { + self.batch_get_execute_times.fetch_add(1, Ordering::SeqCst); let mut kvs = Vec::with_capacity(req.keys.len()); for key in req.keys.iter() { if let Some(kv) = self.get(key).await? { @@ -571,6 +577,7 @@ mod tests { async fn test_cached_kv_backend() { let simple_kv = Arc::new(SimpleKvBackend::default()); let get_execute_times = simple_kv.get_execute_times.clone(); + let batch_get_execute_times = simple_kv.batch_get_execute_times.clone(); let cached_kv = CachedKvBackend::wrap(simple_kv); add_some_vals(&cached_kv).await; @@ -582,9 +589,11 @@ mod tests { assert_eq!(get_execute_times.load(Ordering::SeqCst), 0); for _ in 0..10 { - let _batch_get_resp = cached_kv.batch_get(batch_get_req.clone()).await.unwrap(); + let batch_get_resp = cached_kv.batch_get(batch_get_req.clone()).await.unwrap(); + assert_eq!(batch_get_resp.kvs.len(), 2); assert_eq!(get_execute_times.load(Ordering::SeqCst), 2); + assert_eq!(batch_get_execute_times.load(Ordering::SeqCst), 1); } let batch_get_req = BatchGetRequest { @@ -594,14 +603,29 @@ mod tests { let _batch_get_resp = cached_kv.batch_get(batch_get_req.clone()).await.unwrap(); assert_eq!(get_execute_times.load(Ordering::SeqCst), 3); + assert_eq!(batch_get_execute_times.load(Ordering::SeqCst), 2); for _ in 0..10 { - let _batch_get_resp = cached_kv.batch_get(batch_get_req.clone()).await.unwrap(); + let batch_get_resp = cached_kv.batch_get(batch_get_req.clone()).await.unwrap(); + assert_eq!(batch_get_resp.kvs.len(), 3); assert_eq!(get_execute_times.load(Ordering::SeqCst), 3); + assert_eq!(batch_get_execute_times.load(Ordering::SeqCst), 2); } } + #[tokio::test] + async fn test_cached_kv_backend_empty_batch_get() { + let simple_kv = Arc::new(SimpleKvBackend::default()); + let cached_kv = CachedKvBackend::wrap(simple_kv.clone()); + + let response = cached_kv.batch_get(BatchGetRequest::new()).await.unwrap(); + + assert!(response.kvs.is_empty()); + assert_eq!(simple_kv.batch_get_execute_times.load(Ordering::SeqCst), 0); + assert_eq!(simple_kv.get_execute_times.load(Ordering::SeqCst), 0); + } + #[tokio::test] async fn test_cached_kv_backend_rejects_writes_with_read_only_inner() { let inner = Arc::new(MemoryKvBackend::::new()); diff --git a/src/catalog/src/system_schema/information_schema/partitions.rs b/src/catalog/src/system_schema/information_schema/partitions.rs index 4b00d367129..3bb3c29b704 100644 --- a/src/catalog/src/system_schema/information_schema/partitions.rs +++ b/src/catalog/src/system_schema/information_schema/partitions.rs @@ -263,6 +263,51 @@ impl InformationSchemaPartitionsBuilder { let predicates = Predicates::from_scan_request(&request); + // Fast path: when the predicates pin both the schema and the table name + // with equality, resolve the table directly instead of enumerating all + // tables in all schemas and fetching partition metadata for each of them. + if let (Some(schema_name), Some(table_name)) = ( + predicates + .find_eq_value(TABLE_SCHEMA) + .and_then(Value::as_string), + predicates + .find_eq_value(TABLE_NAME) + .and_then(Value::as_string), + ) { + if let Some(table) = catalog_manager + .table(&catalog_name, &schema_name, &table_name, None) + .await? + { + let table_info = table.table_info(); + + // The enumeration path below skips temporary tables, keep the + // behavior identical. + if table_info.table_type != TableType::Temporary { + let mut table_partitions = partition_manager + .batch_find_table_partitions(&[table_info.table_id()]) + .await + .context(FindPartitionsSnafu)?; + + let partitions = table_partitions + .remove(&table_info.table_id()) + .unwrap_or_default(); + + self.add_partitions( + &predicates, + &table_info, + &catalog_name, + &schema_name, + &table_info.name, + &partitions, + ); + } + } + + // The pinned table fully determines the result: the table above was + // found (or not) unambiguously, so no other table can contribute rows. + return self.finish(); + } + for schema_name in catalog_manager.schema_names(&catalog_name, None).await? { let table_info_stream = catalog_manager .tables(&catalog_name, &schema_name, None) @@ -430,3 +475,253 @@ impl DfPartitionStream for InformationSchemaPartitions { )) } } + +#[cfg(test)] +mod tests { + use api::v1::meta::Peer; + use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry}; + use common_catalog::consts::{DEFAULT_CATALOG_NAME, INFORMATION_SCHEMA_NAME}; + use common_meta::cache::{CacheRegistryBuilder, LayeredCacheRegistryBuilder}; + use common_meta::key::schema_name::SchemaNameKey; + use common_meta::key::table_route::TableRouteValue; + use common_meta::kv_backend::memory::MemoryKvBackend; + use common_meta::rpc::router::{Region, RegionRoute}; + use common_meta::wal_provider::RegionWalOptions; + use datafusion::logical_expr::{BinaryExpr, Expr, Operator, col, lit}; + use futures::TryStreamExt; + use store_api::storage::RegionId; + + use super::*; + use crate::information_schema::NoopInformationExtension; + use crate::kvbackend::KvBackendCatalogManagerBuilder; + + const SCHEMA_1: &str = "partitions_schema_1"; + const SCHEMA_2: &str = "partitions_schema_2"; + const PARTITIONED_TABLE: &str = "partitioned_table"; + const MISSING_TABLE: &str = "missing_table"; + const TABLE_ID_1: TableId = 1; + const TABLE_ID_2: TableId = 2; + const REGION_NUMBER_1: u32 = 1; + const REGION_NUMBER_2: u32 = 2; + const REGION_NUMBER_7: u32 = 7; + + fn kv_catalog_manager() -> Arc { + let backend = Arc::new(MemoryKvBackend::default()); + let layered_cache_builder = LayeredCacheRegistryBuilder::default() + .add_cache_registry(CacheRegistryBuilder::default().build()); + let fundamental_cache_registry = build_fundamental_cache_registry(backend.clone()); + let layered_cache_registry = Arc::new( + with_default_composite_cache_registry( + layered_cache_builder.add_cache_registry(fundamental_cache_registry), + ) + .unwrap() + .build(), + ); + + KvBackendCatalogManagerBuilder::new( + Arc::new(NoopInformationExtension), + backend, + layered_cache_registry, + ) + .build() + } + + fn region_route(table_id: TableId, region_number: u32) -> RegionRoute { + RegionRoute { + region: Region { + id: RegionId::new(table_id, region_number), + ..Default::default() + }, + leader_peer: Some(Peer { + id: 1, + addr: "127.0.0.1:3001".to_string(), + }), + ..Default::default() + } + } + + fn table_info(schema: &str, table: &str, table_id: TableId) -> TableInfo { + // The catalog defaults to the default catalog name. + let mut table_info = + common_meta::key::test_utils::new_test_table_info_with_name(table_id, table); + table_info.schema_name = schema.to_string(); + table_info + } + + /// Creates a catalog with two partitioned tables named `PARTITIONED_TABLE` + /// in two different schemas: + /// + /// - `SCHEMA_1.PARTITIONED_TABLE` has two regions. + /// - `SCHEMA_2.PARTITIONED_TABLE` has one region. + async fn setup_catalog() -> Arc { + let manager = kv_catalog_manager(); + let metadata = manager.table_metadata_manager_ref().clone(); + + for schema in [SCHEMA_1, SCHEMA_2] { + metadata + .schema_manager() + .create(SchemaNameKey::new(DEFAULT_CATALOG_NAME, schema), None, true) + .await + .unwrap(); + } + + for (table_id, schema, region_numbers) in [ + (TABLE_ID_1, SCHEMA_1, vec![REGION_NUMBER_1, REGION_NUMBER_2]), + (TABLE_ID_2, SCHEMA_2, vec![REGION_NUMBER_7]), + ] { + let routes = region_numbers + .into_iter() + .map(|region_number| region_route(table_id, region_number)) + .collect(); + metadata + .create_table_metadata( + table_info(schema, PARTITIONED_TABLE, table_id), + TableRouteValue::physical(routes), + RegionWalOptions::default(), + ) + .await + .unwrap(); + } + + manager + } + + /// Scans `information_schema.partitions` and returns the total row count + /// and the pretty printed batches. + async fn scan_partitions( + manager: &Arc, + filters: Vec, + ) -> (usize, String) { + let table = manager + .table( + DEFAULT_CATALOG_NAME, + INFORMATION_SCHEMA_NAME, + PARTITIONS, + None, + ) + .await + .unwrap() + .unwrap(); + let stream = table + .scan_to_stream(ScanRequest { + filters, + ..Default::default() + }) + .await + .unwrap(); + let batches = stream.try_collect::>().await.unwrap(); + let rows = batches.iter().map(|batch| batch.num_rows()).sum(); + let output = batches + .iter() + .map(|batch| batch.pretty_print()) + .collect::>() + .join("\n"); + (rows, output) + } + + fn eq_filter(column: &str, value: &str) -> Expr { + Expr::BinaryExpr(BinaryExpr::new( + Box::new(col(column)), + Operator::Eq, + Box::new(lit(value)), + )) + } + + fn not_eq_filter(column: &str, value: &str) -> Expr { + Expr::BinaryExpr(BinaryExpr::new( + Box::new(col(column)), + Operator::NotEq, + Box::new(lit(value)), + )) + } + + #[tokio::test] + async fn test_scan_pinned_table_skips_catalog_enumeration() { + let manager: Arc = setup_catalog().await; + + // No predicates: every partitioned table is listed. + let (rows, output) = scan_partitions(&manager, vec![]).await; + assert_eq!(3, rows, "{output}"); + + // Pins both the schema and the table name, so the scan resolves the + // table directly instead of enumerating all catalog entries. + let (rows, output) = scan_partitions( + &manager, + vec![ + eq_filter(TABLE_SCHEMA, SCHEMA_1), + eq_filter(TABLE_NAME, PARTITIONED_TABLE), + ], + ) + .await; + assert_eq!(2, rows, "{output}"); + assert!(output.contains(SCHEMA_1), "{output}"); + assert!(!output.contains(SCHEMA_2), "{output}"); + assert!( + output.contains( + &RegionId::new(TABLE_ID_1, REGION_NUMBER_1) + .as_u64() + .to_string() + ), + "{output}" + ); + assert!( + output.contains( + &RegionId::new(TABLE_ID_1, REGION_NUMBER_2) + .as_u64() + .to_string() + ), + "{output}" + ); + + // The fallback enumeration path must return the identical result. The + // inequality on `table_name` doesn't pin a single name, so no direct + // lookup happens here. + let (fallback_rows, fallback_output) = scan_partitions( + &manager, + vec![ + eq_filter(TABLE_SCHEMA, SCHEMA_1), + not_eq_filter(TABLE_NAME, MISSING_TABLE), + ], + ) + .await; + assert_eq!(rows, fallback_rows, "{fallback_output}"); + assert_eq!(output, fallback_output); + + // Only the table name is pinned: the schema can't be resolved and both + // tables of that name are kept. + let (rows, output) = + scan_partitions(&manager, vec![eq_filter(TABLE_NAME, PARTITIONED_TABLE)]).await; + assert_eq!(3, rows, "{output}"); + assert!(output.contains(SCHEMA_1), "{output}"); + assert!(output.contains(SCHEMA_2), "{output}"); + + // An unknown table in a pinned schema yields no rows. + let (rows, output) = scan_partitions( + &manager, + vec![ + eq_filter(TABLE_SCHEMA, SCHEMA_1), + eq_filter(TABLE_NAME, MISSING_TABLE), + ], + ) + .await; + assert_eq!(0, rows, "{output}"); + } + + #[tokio::test] + async fn test_scan_equality_under_disjunction_falls_back_to_enumeration() { + let manager: Arc = setup_catalog().await; + + // `table_schema = SCHEMA_1 OR table_name = PARTITIONED_TABLE` matches + // the tables in both schemas: equality nested under a disjunction must + // not pin a single table. + let filter = Expr::BinaryExpr(BinaryExpr::new( + Box::new(eq_filter(TABLE_SCHEMA, SCHEMA_1)), + Operator::Or, + Box::new(eq_filter(TABLE_NAME, PARTITIONED_TABLE)), + )); + let (rows, output) = scan_partitions(&manager, vec![filter]).await; + assert_eq!(3, rows, "{output}"); + assert!(output.contains(SCHEMA_1), "{output}"); + assert!(output.contains(SCHEMA_2), "{output}"); + } +} diff --git a/src/catalog/src/system_schema/predicate.rs b/src/catalog/src/system_schema/predicate.rs index 7957ba454d4..cb71e51ed0e 100644 --- a/src/catalog/src/system_schema/predicate.rs +++ b/src/catalog/src/system_schema/predicate.rs @@ -214,6 +214,22 @@ impl Predicate { _ => None, } } + + /// Collects the values that constrain `column` to be equal to them at this + /// predicate's top-level conjunctions. + /// + /// Constraints under `Or`/`Not` are ignored because they don't guarantee + /// that `column` equals a single value. + fn collect_eq_values<'a>(&'a self, column: &str, values: &mut Vec<&'a Value>) { + match self { + Predicate::Eq(c, v) if c == column => values.push(v), + Predicate::And(left, right) => { + left.collect_eq_values(column, values); + right.collect_eq_values(column, values); + } + _ => {} + } + } } /// Perform SQL left LIKE right, return `None` if fail to evaluate. @@ -269,6 +285,25 @@ impl Predicates { } } + /// Returns the single equality value that constrains `column` in the + /// top-level conjunctions of the predicates. + /// + /// Returns `None` if there is no equality constraint on `column`, or if + /// the predicates constrain it with multiple distinct values. + pub fn find_eq_value(&self, column: &str) -> Option<&Value> { + let mut values = Vec::new(); + for predicate in &self.predicates { + predicate.collect_eq_values(column, &mut values); + } + + let (first, rest) = values.split_first()?; + if rest.iter().all(|v| v == first) { + Some(*first) + } else { + None + } + } + /// Evaluate the predicates with the row. /// returns true when all the predicates are satisfied or can't be evaluated. pub fn eval(&self, row: &[(&str, &Value)]) -> bool { @@ -556,6 +591,71 @@ mod tests { ); } + #[test] + fn test_find_eq_value() { + fn eq(name: &str, value: &str) -> Predicate { + Predicate::Eq(name.to_string(), Value::from(value)) + } + + fn and(left: Predicate, right: Predicate) -> Predicate { + Predicate::And(Box::new(left), Box::new(right)) + } + + fn or(left: Predicate, right: Predicate) -> Predicate { + Predicate::Or(Box::new(left), Box::new(right)) + } + + fn predicates(predicates: Vec) -> Predicates { + Predicates { predicates } + } + + // No predicates, no equality value. + assert!(predicates(vec![]).find_eq_value("a").is_none()); + + // No equality on the column. + let p = predicates(vec![Predicate::NotEq("a".to_string(), Value::from("x"))]); + assert!(p.find_eq_value("a").is_none()); + assert!(p.find_eq_value("b").is_none()); + + // Simple equality. + let p = predicates(vec![eq("a", "x")]); + let v = p.find_eq_value("a").unwrap(); + assert!(match_string_value(v, "x")); + assert!(p.find_eq_value("b").is_none()); + + // Duplicated equality on the same value. + let p = predicates(vec![and(eq("a", "x"), eq("a", "x"))]); + let v = p.find_eq_value("a").unwrap(); + assert!(match_string_value(v, "x")); + + // Conflicting equalities reject the lookup. + let p = predicates(vec![and(eq("a", "x"), eq("a", "y"))]); + assert!(p.find_eq_value("a").is_none()); + + // Conflicting equalities in separate predicates reject the lookup too. + let p = predicates(vec![eq("a", "x"), eq("a", "y")]); + assert!(p.find_eq_value("a").is_none()); + + // Equality nested in deeper conjunctions. + let p = predicates(vec![and(and(eq("b", "y"), eq("a", "x")), eq("a", "x"))]); + let v = p.find_eq_value("a").unwrap(); + assert!(match_string_value(v, "x")); + + // Equality under a disjunction or a negation doesn't constrain the column. + let p = predicates(vec![or(eq("a", "x"), eq("a", "x"))]); + assert!(p.find_eq_value("a").is_none()); + let p = predicates(vec![Predicate::Not(Box::new(eq("a", "x")))]); + assert!(p.find_eq_value("a").is_none()); + + // Equality value in a top-level conjunction composed of separate predicates. + let p = predicates(vec![ + eq("a", "x"), + Predicate::Like("b".to_string(), "%y".to_string(), false), + ]); + let v = p.find_eq_value("a").unwrap(); + assert!(match_string_value(v, "x")); + } + #[test] fn test_predicates_eval_row() { let wrong_row = [ diff --git a/src/cli/src/common/store.rs b/src/cli/src/common/store.rs index e974037ea4d..c3b8e477ed1 100644 --- a/src/cli/src/common/store.rs +++ b/src/cli/src/common/store.rs @@ -86,7 +86,13 @@ pub struct StoreConfig { /// Automatically create PostgreSQL schema if it doesn't exist (default: true). #[cfg(feature = "pg_kvbackend")] - #[clap(long, default_value_t = true)] + #[clap( + long, + default_value_t = true, + default_missing_value = "true", + num_args = 0..=1, + action = clap::ArgAction::Set + )] pub auto_create_schema: bool, /// TLS mode for backend store connections (etcd, PostgreSQL, MySQL) @@ -223,3 +229,27 @@ impl StoreConfig { } } } + +#[cfg(all(test, feature = "pg_kvbackend"))] +mod tests { + use clap::Parser; + + use super::StoreConfig; + + #[test] + fn test_auto_create_schema_accepts_explicit_value() { + let config = + StoreConfig::try_parse_from(["store", "--auto-create-schema", "false"]).unwrap(); + assert!(!config.auto_create_schema); + + let config = + StoreConfig::try_parse_from(["store", "--auto-create-schema", "true"]).unwrap(); + assert!(config.auto_create_schema); + } + + #[test] + fn test_auto_create_schema_defaults_to_true() { + let config = StoreConfig::try_parse_from(["store"]).unwrap(); + assert!(config.auto_create_schema); + } +} diff --git a/src/client/Cargo.toml b/src/client/Cargo.toml index 36c3f60a9a4..e5bd479cc7d 100644 --- a/src/client/Cargo.toml +++ b/src/client/Cargo.toml @@ -45,3 +45,4 @@ substrait.workspace = true tokio.workspace = true tokio-stream = { workspace = true, features = ["net"] } tonic.workspace = true +tonic-prost.workspace = true diff --git a/src/client/src/client.rs b/src/client/src/client.rs index dfc0e44fad5..1746854cd26 100644 --- a/src/client/src/client.rs +++ b/src/client/src/client.rs @@ -20,7 +20,6 @@ use api::v1::HealthCheckRequest; use api::v1::flow::flow_client::FlowClient as PbFlowClient; use api::v1::health_check_client::HealthCheckClient; use api::v1::prometheus_gateway_client::PrometheusGatewayClient; -use api::v1::region::region_client::RegionClient as PbRegionClient; use arrow_flight::flight_service_client::FlightServiceClient; use common_grpc::channel_manager::{ ChannelConfig, ChannelManager, ClientTlsOption, load_client_tls_config, @@ -526,23 +525,12 @@ impl Client { Ok(FlightClient { addr, client }) } - pub(crate) fn raw_region_client(&self) -> Result<(String, PbRegionClient)> { + /// Creates the region gRPC transport with the configured message limits. + pub(crate) fn raw_region_client(&self) -> Result<(String, tonic::client::Grpc)> { let (addr, channel) = self.find_channel()?; - let client = PbRegionClient::new(channel) - .max_decoding_message_size( - self.inner - .control_channel_manager - .config() - .max_recv_message_size - .as_bytes() as usize, - ) - .max_encoding_message_size( - self.inner - .control_channel_manager - .config() - .max_send_message_size - .as_bytes() as usize, - ); + let client = tonic::client::Grpc::new(channel) + .max_decoding_message_size(self.max_grpc_recv_message_size()) + .max_encoding_message_size(self.max_grpc_send_message_size()); Ok((addr, client)) } diff --git a/src/client/src/database.rs b/src/client/src/database.rs index e97fff6db00..de3c722a86a 100644 --- a/src/client/src/database.rs +++ b/src/client/src/database.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; use std::pin::Pin; use std::str::FromStr; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, Mutex, RwLock}; use std::task::{Context, Poll}; use std::time::Duration; @@ -55,6 +55,7 @@ use futures::future; use futures_util::{Stream, StreamExt, TryStreamExt}; use prost::Message; use snafu::{IntoError, ResultExt}; +use tokio::sync::Notify; use tonic::metadata::{AsciiMetadataKey, AsciiMetadataValue, MetadataMap, MetadataValue}; use tonic::transport::Channel; @@ -70,20 +71,47 @@ type FlightDataStream = Pin + Send>>; type DoPutResponseStream = Pin>>>; const HINTS_METADATA_KEY: &str = "x-greptime-hints"; +/// Maximum time to wait for the optional trailing metrics message after +/// affected rows have already been delivered. +const FLIGHT_TRAILING_METRICS_TIMEOUT: Duration = Duration::from_secs(5); /// Terminal metrics associated with a query output. /// /// For streaming outputs, metrics are only final after the stream is fully -/// drained and [`Self::is_ready`] returns `true`. +/// drained and [`Self::is_ready`] returns `true`. Affected-row outputs may +/// briefly await a compatibility trailing metrics message. #[derive(Debug, Clone, Default)] pub struct OutputMetrics { inner: Arc, } -#[derive(Debug, Default)] +#[derive(Debug)] struct OutputMetricsInner { metrics: RwLock>, + completion_error: RwLock>, ready: AtomicBool, + ready_notify: Notify, + compatibility_task: Mutex>, +} + +impl Default for OutputMetricsInner { + fn default() -> Self { + Self { + metrics: RwLock::new(None), + completion_error: RwLock::new(None), + ready: AtomicBool::new(false), + ready_notify: Notify::new(), + compatibility_task: Mutex::new(None), + } + } +} + +impl Drop for OutputMetricsInner { + fn drop(&mut self) { + if let Some(handle) = self.compatibility_task.get_mut().unwrap().take() { + handle.abort(); + } + } } impl OutputMetrics { @@ -98,10 +126,45 @@ impl OutputMetrics { /// Marks the terminal metrics as final for this output. pub fn mark_ready(&self) { - let _ = self + if self .inner .ready - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire); + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + self.inner.ready_notify.notify_waiters(); + } + } + + /// Waits until terminal metrics are final. + pub async fn wait_ready(&self) { + loop { + let notified = self.inner.ready_notify.notified(); + if self.is_ready() { + return; + } + notified.await; + } + } + + /// Returns an error encountered while completing the output, if any. + pub fn completion_error(&self) -> Option { + self.inner.completion_error.read().unwrap().clone() + } + + fn set_completion_error(&self, error: impl Into) { + *self.inner.completion_error.write().unwrap() = Some(error.into()); + } + + fn set_compatibility_task(&self, handle: tokio::task::AbortHandle) { + let mut task = self.inner.compatibility_task.lock().unwrap(); + if !self.is_ready() { + *task = Some(handle); + } + } + + fn take_compatibility_task(&self) -> Option { + self.inner.compatibility_task.lock().unwrap().take() } /// Returns whether terminal metrics are final. @@ -148,7 +211,8 @@ impl OutputMetrics { /// /// The contained [`OutputMetrics`] lets callers read stream terminal metrics /// after consuming `output`. For non-stream outputs, metrics are ready -/// immediately. +/// immediately. Flight affected-row outputs without inline metrics require +/// [`OutputMetrics::wait_ready`] before compatibility trailing metrics can be read. #[derive(Debug)] pub struct OutputWithMetrics { pub output: Output, @@ -194,6 +258,55 @@ fn parse_terminal_metrics(metrics_json: &str) -> Result { }) } +fn spawn_affected_rows_trailing_metrics_task( + terminal_metrics: &OutputMetrics, + mut reader: FlightMessageReader, +) where + S: Stream> + Send + Unpin + 'static, +{ + let metrics_ref = Arc::downgrade(&terminal_metrics.inner); + let remote_addr = reader.remote_addr().to_string(); + let task = common_runtime::spawn_global(async move { + let result = + tokio::time::timeout(FLIGHT_TRAILING_METRICS_TIMEOUT, reader.read_next()).await; + let Some(inner) = metrics_ref.upgrade() else { + return; + }; + let metrics = OutputMetrics { inner }; + match result { + Ok(Ok(Some(FlightMessage::Metrics(s)))) => match parse_terminal_metrics(&s) { + Ok(metrics_json) => metrics.update(Some(metrics_json)), + Err(error) => { + metrics.set_completion_error(error.to_string()); + warn!( + "Failed to decode trailing Flight metrics from {}: {}", + remote_addr, error + ); + } + }, + Ok(Ok(None)) => {} + Ok(Ok(Some(other))) => { + let error = format!("Unexpected trailing Flight message: {other:?}"); + metrics.set_completion_error(error.clone()); + warn!("{} from {}", error, remote_addr); + } + Ok(Err(error)) => { + let error = flight_stream_error(&remote_addr, error); + metrics.set_completion_error(error.to_string()); + warn!("{}", error); + } + Err(_) => { + let error = "Timed out waiting for trailing Flight metrics"; + metrics.set_completion_error(error); + warn!("{} from {}", error, remote_addr); + } + } + metrics.mark_ready(); + metrics.take_compatibility_task(); + }); + terminal_metrics.set_compatibility_task(task.abort_handle()); +} + struct StreamWithMetrics { stream: common_recordbatch::SendableRecordBatchStream, metrics: OutputMetrics, @@ -233,6 +346,9 @@ impl Stream for StreamWithMetrics { fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let polled = Pin::new(&mut self.stream).poll_next(cx); + if let Poll::Ready(Some(Err(error))) = &polled { + self.metrics.set_completion_error(error.to_string()); + } if let Poll::Ready(None) = &polled { self.sync_terminal_metrics(); self.metrics.mark_ready(); @@ -280,32 +396,12 @@ where FlightMessage::AffectedRows { rows, metrics } => { let terminal_metrics = OutputMetrics::new(); if let Some(metrics) = metrics { + // Inline metrics are authoritative. They complete the output + // without touching the rest of the Flight stream. terminal_metrics.update(Some(parse_terminal_metrics(&metrics)?)); - } - let next_message = reader - .read_next() - .await - .map_err(|error| flight_stream_error(reader.remote_addr(), error))?; - match next_message { - None => terminal_metrics.mark_ready(), - Some(FlightMessage::Metrics(s)) if terminal_metrics.get().is_none() => { - terminal_metrics.update(Some(parse_terminal_metrics(&s)?)); - terminal_metrics.mark_ready(); - } - Some(FlightMessage::Metrics(_)) => { - return IllegalFlightMessagesSnafu { - reason: "'AffectedRows' Flight metadata already carries Metrics and cannot be followed by another Metrics message", - } - .fail(); - } - Some(other) => { - return IllegalFlightMessagesSnafu { - reason: format!( - "'AffectedRows' Flight message can only be followed by a Metrics message, got {other:?}" - ), - } - .fail(); - } + terminal_metrics.mark_ready(); + } else { + spawn_affected_rows_trailing_metrics_task(&terminal_metrics, reader); } Ok(OutputWithMetrics { output: Output::new_with_affected_rows(rows), @@ -749,7 +845,9 @@ impl Database { /// Executes a SQL query and returns the output with terminal metrics. /// /// For stream outputs, callers must consume the stream before reading final - /// terminal metrics from [`OutputWithMetrics::metrics`]. + /// terminal metrics from [`OutputWithMetrics::metrics`]. For affected-row + /// outputs without inline metrics, call [`OutputMetrics::wait_ready`] when + /// compatibility trailing metrics are required. pub async fn sql_with_terminal_metrics( &self, sql: S, @@ -986,6 +1084,7 @@ mod tests { use datatypes::schema::{ColumnSchema, Schema}; use datatypes::vectors::Int32Vector; use futures_util::StreamExt; + use tokio::sync::oneshot; use tonic::codegen::http::{HeaderMap, HeaderValue}; use tonic::metadata::MetadataMap; use tonic::{Code, Status}; @@ -1296,11 +1395,6 @@ mod tests { ); } - #[test] - fn test_parse_terminal_metrics_rejects_invalid_json() { - assert!(parse_terminal_metrics("{not-json}").is_err()); - } - #[tokio::test] async fn test_affected_rows_inline_metrics_are_parsed() { let output = output_from_flight_message_stream( @@ -1322,27 +1416,191 @@ mod tests { } #[tokio::test] - async fn test_affected_rows_inline_metrics_rejects_trailing_metrics() { + async fn test_affected_rows_inline_metrics_do_not_poll_trailer() { let metrics_json = terminal_metrics_json(); - let err = output_from_flight_message_stream( + let output = output_from_flight_message_stream( "test-peer".to_string(), futures_util::stream::iter(vec![ Ok(FlightMessage::AffectedRows { rows: 3, - metrics: Some(metrics_json.clone()), + metrics: Some(metrics_json), }), - Ok(FlightMessage::Metrics(metrics_json)), + Ok(FlightMessage::Metrics(terminal_metrics_json_with_seq(99))), ] as Vec>), ) .await - .unwrap_err(); + .unwrap(); - assert!( - err.to_string().contains("already carries Metrics"), - "unexpected error: {err:?}" + assert!(output.metrics.is_ready()); + assert_eq!( + output.metrics.region_watermark_map(), + Some(std::collections::HashMap::from([(7, 42)])) ); } + #[tokio::test] + async fn test_affected_rows_without_inline_metrics_becomes_ready_after_trailer() { + // Hold the trailer until the not-ready state has been observed. + let (trailer_tx, trailer_rx) = oneshot::channel(); + let trailer = futures_util::stream::once(trailer_rx).map(|ready| { + ready.unwrap(); + Ok(FlightMessage::Metrics(terminal_metrics_json())) + }); + let output = output_from_flight_message_stream( + "test-peer".to_string(), + futures_util::stream::iter(vec![Ok(FlightMessage::AffectedRows { + rows: 3, + metrics: None, + })]) + .chain(trailer), + ) + .await + .unwrap(); + + assert!(!output.metrics.is_ready()); + trailer_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(1), output.metrics.wait_ready()) + .await + .expect("terminal metrics must become ready once the trailer arrives"); + assert!(output.metrics.completion_error().is_none()); + assert_eq!( + output.metrics.region_watermark_map(), + Some(std::collections::HashMap::from([(7, 42)])) + ); + } + + #[tokio::test] + async fn test_affected_rows_malformed_trailer_sets_completion_error_and_ready() { + let output = output_from_flight_message_stream( + "test-peer".to_string(), + futures_util::stream::iter(vec![ + Ok(FlightMessage::AffectedRows { + rows: 3, + metrics: None, + }), + Ok(FlightMessage::Metrics("{not-json}".to_string())), + ] as Vec>), + ) + .await + .unwrap(); + + output.metrics.wait_ready().await; + let error = output.metrics.completion_error().unwrap(); + assert!(error.contains("Invalid terminal metrics message")); + assert!(output.metrics.is_ready()); + } + + #[tokio::test] + async fn test_affected_rows_transport_trailer_error_sets_completion_error_and_ready() { + let output = output_from_flight_message_stream( + "test-peer".to_string(), + futures_util::stream::iter(vec![ + Ok(FlightMessage::AffectedRows { + rows: 3, + metrics: None, + }), + Err(Status::unavailable("trailer read failed").into()), + ] as Vec>), + ) + .await + .unwrap(); + + output.metrics.wait_ready().await; + let error = output.metrics.completion_error().unwrap(); + assert!(error.contains("trailer read failed")); + assert!(output.metrics.is_ready()); + } + + #[tokio::test] + async fn test_affected_rows_compatibility_reader_is_cancelled_after_second_poll_begins() { + struct DropProbe { + first: Option>, + polled: Option>, + dropped: Option>, + } + + impl Stream for DropProbe { + type Item = Result; + + fn poll_next( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll> { + if let Some(message) = self.first.take() { + return Poll::Ready(Some(message)); + } + if let Some(polled) = self.polled.take() { + let _ = polled.send(()); + } + Poll::Pending + } + } + + impl Drop for DropProbe { + fn drop(&mut self) { + if let Some(dropped) = self.dropped.take() { + let _ = dropped.send(()); + } + } + } + + let (polled_tx, polled_rx) = oneshot::channel(); + let (dropped_tx, dropped_rx) = oneshot::channel(); + let output = output_from_flight_message_stream( + "test-peer".to_string(), + DropProbe { + first: Some(Ok(FlightMessage::AffectedRows { + rows: 3, + metrics: None, + })), + polled: Some(polled_tx), + dropped: Some(dropped_tx), + }, + ) + .await + .unwrap(); + polled_rx.await.unwrap(); + drop(output); + // Require cancellation before the five-second compatibility timeout. + tokio::time::timeout(Duration::from_secs(1), dropped_rx) + .await + .expect("the compatibility reader must be dropped once the output is dropped") + .unwrap(); + } + + #[tokio::test] + async fn test_schema_record_batch_yields_before_pending_message_stream() { + let schema = Arc::new(Schema::new(vec![ColumnSchema::new( + "v", + ConcreteDataType::int32_datatype(), + false, + )])); + let batch = RecordBatch::new( + schema.clone(), + vec![Arc::new(Int32Vector::from_slice([1])) as VectorRef], + ) + .unwrap(); + let messages = futures_util::stream::iter(vec![ + Ok(FlightMessage::Schema(schema.arrow_schema().clone())), + Ok(FlightMessage::RecordBatch(batch.into_df_record_batch())), + ] as Vec>) + .chain(futures_util::stream::pending()); + let output = output_from_flight_message_stream("test-peer".to_string(), messages) + .await + .unwrap(); + let OutputData::Stream(mut stream) = output.output.data else { + panic!("expected stream output"); + }; + + let batch = tokio::time::timeout(Duration::from_secs(1), stream.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(batch.num_rows(), 1); + assert!(!output.metrics.is_ready()); + } + #[tokio::test] async fn test_invalid_terminal_metrics_after_record_batch_yields_batch_then_error() { let schema = Arc::new(Schema::new(vec![ColumnSchema::new( diff --git a/src/client/src/error.rs b/src/client/src/error.rs index 168ba422715..fdf0df9582f 100644 --- a/src/client/src/error.rs +++ b/src/client/src/error.rs @@ -18,7 +18,7 @@ use common_error::define_from_tonic_status; use common_error::ext::{BoxedError, ErrorExt, RetryHint}; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; -use snafu::{Location, Snafu}; +use snafu::{Location, Snafu, location}; use tonic::Code; use tonic::metadata::errors::InvalidMetadataValue; @@ -191,9 +191,22 @@ impl ErrorExt for Error { Error::Tonic { retry_hint, .. } => *retry_hint, Error::FlightGet { source, .. } | Error::FlightStream { source, .. } - | Error::RegionServer { source, .. } | Error::FlowServer { source, .. } | Error::External { source, .. } => source.retry_hint(), + Error::RegionServer { code, source, .. } => { + let hint = source.retry_hint(); + // Preserve the transport retry policy previously applied by should_retry(). + if !hint.is_retryable() + && matches!( + code, + Code::Unavailable | Code::Cancelled | Code::DeadlineExceeded + ) + { + RetryHint::Retryable + } else { + hint + } + } Error::ConvertFlightData { source, .. } | Error::CreateChannel { source, .. } | Error::CreateTlsChannel { source, .. } => source.retry_hint(), @@ -205,6 +218,15 @@ impl ErrorExt for Error { define_from_tonic_status!(Error, Tonic); +impl From for common_meta::error::Error { + fn from(error: Error) -> Self { + Self::External { + source: BoxedError::new(error), + location: location!(), + } + } +} + impl Error { /// Returns the gRPC status code if this error is caused by a gRPC request failure. pub fn tonic_code(&self) -> Option { diff --git a/src/client/src/flight.rs b/src/client/src/flight.rs index 1753fdd5246..3a21e6daa4e 100644 --- a/src/client/src/flight.rs +++ b/src/client/src/flight.rs @@ -12,40 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::pin::Pin; - use arrow_flight::FlightData; use common_grpc::flight::{FlightDecoder, FlightMessage}; -use futures_util::stream::Peekable; use futures_util::{Stream, StreamExt}; use snafu::{OptionExt, ResultExt}; use crate::Result; use crate::error::{ConvertFlightDataSnafu, Error, IllegalFlightMessagesSnafu}; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum FlightMessageKind { - Schema, - RecordBatch, - AffectedRows, - Metrics, -} - -impl From<&FlightMessage> for FlightMessageKind { - fn from(message: &FlightMessage) -> Self { - match message { - FlightMessage::Schema(_) => Self::Schema, - FlightMessage::RecordBatch(_) => Self::RecordBatch, - FlightMessage::AffectedRows { .. } => Self::AffectedRows, - FlightMessage::Metrics(_) => Self::Metrics, - } - } -} - pub(crate) struct FlightMessageReader { /// Remote Flight peer associated with this response stream. remote_addr: String, - messages: Peekable, + messages: S, } impl FlightMessageReader @@ -55,7 +33,7 @@ where pub(crate) fn new(remote_addr: impl Into, messages: S) -> Self { Self { remote_addr: remote_addr.into(), - messages: messages.peekable(), + messages, } } @@ -72,21 +50,6 @@ where pub(crate) async fn read_next(&mut self) -> Result> { self.messages.next().await.transpose() } - - pub(crate) async fn peek_next_message_kind(&mut self) -> Result> { - match Pin::new(&mut self.messages).peek().await { - Some(Ok(message)) => Ok(Some(message.into())), - None => Ok(None), - Some(Err(_)) => match self.read_next().await { - // `peek` only borrows the error; consume it to preserve the source error. - Err(error) => Err(error), - Ok(_) => IllegalFlightMessagesSnafu { - reason: "Flight stream changed after peek".to_string(), - } - .fail(), - }, - } - } } pub(crate) fn decode_flight_data( diff --git a/src/client/src/region.rs b/src/client/src/region.rs index f0c683ec155..f9ebbb0d0bf 100644 --- a/src/client/src/region.rs +++ b/src/client/src/region.rs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod codec; + +use std::borrow::Borrow; use std::sync::Arc; use std::time::Duration; @@ -48,11 +51,13 @@ use crate::error::{ self, FlightGetSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu, MissingFieldSnafu, Result, ServerSnafu, }; -use crate::flight::{FlightMessageKind, FlightMessageReader, decode_flight_data}; +use crate::flight::{FlightMessageReader, decode_flight_data}; +use crate::region::codec::RegionCodec; use crate::{Client, metrics}; const FLIGHT_DO_GET_TIMEOUT: Duration = Duration::from_secs(10); +/// Sends region operations through a configured gRPC client. #[derive(Debug)] pub struct RegionRequester { client: Client, @@ -62,20 +67,8 @@ pub struct RegionRequester { #[async_trait] impl Datanode for RegionRequester { - async fn handle(&self, request: RegionRequest) -> MetaResult { - self.handle_inner(request).await.map_err(|err| { - if err.should_retry() { - meta_error::Error::RetryLater { - source: BoxedError::new(err), - clean_poisons: false, - } - } else { - meta_error::Error::External { - source: BoxedError::new(err), - location: location!(), - } - } - }) + async fn handle_request(&self, request: Arc) -> MetaResult { + self.handle_inner(request).await.map_err(Into::into) } async fn handle_query(&self, request: QueryRequest) -> MetaResult { @@ -144,11 +137,49 @@ impl RegionRequester { recordbatches_from_flight_message_stream(addr, flight_message_stream).await } - async fn handle_inner(&self, request: RegionRequest) -> Result { + /// Sends an owned or shared request without deep-cloning its protobuf payload. + /// + /// Accepts both [`RegionRequest`] and [`Arc`]. Callers can retain a + /// shared request for another attempt; this method does not retry or determine + /// whether replaying the request is safe. + pub async fn handle(&self, request: R) -> Result + where + R: Borrow + Send + Sync + 'static, + { + self.handle_inner(request).await + } + + /// Sends a remote dynamic filter update to the region service. + pub async fn handle_remote_dyn_filter_update( + &self, + query_id: impl Into, + update: RemoteDynFilterUpdate, + ) -> Result { + self.handle_inner(build_remote_dyn_filter_update_request(query_id, update)) + .await + } + + /// Unregisters a remote dynamic filter in the region service. + pub async fn handle_remote_dyn_filter_unregister( + &self, + query_id: impl Into, + unregister: RemoteDynFilterUnregister, + ) -> Result { + self.handle_inner(build_remote_dyn_filter_unregister_request( + query_id, unregister, + )) + .await + } + + async fn handle_inner(&self, request: R) -> Result + where + R: Borrow + Send + Sync + 'static, + { let request_body = request + .borrow() .body .as_ref() - .with_context(|| MissingFieldSnafu { field: "body" })?; + .context(MissingFieldSnafu { field: "body" })?; let is_insert = matches!( request_body, region_request::Body::Inserts(_) | region_request::Body::BulkInsert(_) @@ -168,48 +199,42 @@ impl RegionRequester { } } - let response = client - .handle(request) - .await - .map_err(|e| { - let code = e.code(); - // Uses `Error::RegionServer` instead of `Error::Server` - error::Error::RegionServer { - addr, - code, - source: BoxedError::new(error::Error::from(e)), - location: location!(), - } - })? - .into_inner(); + let response = async { + client + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {e}")))?; + let mut request = tonic::Request::new(request); + request.extensions_mut().insert(tonic::GrpcMethod::new( + "greptime.v1.region.Region", + "Handle", + )); + client + .unary( + request, + tonic::codegen::http::uri::PathAndQuery::from_static( + "/greptime.v1.region.Region/Handle", + ), + RegionCodec::default(), + ) + .await + } + .await + .map_err(|status| map_region_error(status, addr))? + .into_inner(); check_response_header(&response.header)?; Ok(RegionResponse::from_region_response(response)) } +} - pub async fn handle(&self, request: RegionRequest) -> Result { - self.handle_inner(request).await - } - - pub async fn handle_remote_dyn_filter_update( - &self, - query_id: impl Into, - update: RemoteDynFilterUpdate, - ) -> Result { - self.handle_inner(build_remote_dyn_filter_update_request(query_id, update)) - .await - } - - pub async fn handle_remote_dyn_filter_unregister( - &self, - query_id: impl Into, - unregister: RemoteDynFilterUnregister, - ) -> Result { - self.handle_inner(build_remote_dyn_filter_unregister_request( - query_id, unregister, - )) - .await +fn map_region_error(status: tonic::Status, addr: String) -> error::Error { + error::Error::RegionServer { + addr, + code: status.code(), + source: BoxedError::new(error::Error::from(status)), + location: location!(), } } @@ -247,9 +272,7 @@ where "poll_flight_data_stream" )); - let mut stream_ended = false; - - while !stream_ended { + loop { let flight_message = match reader.read_next().await { Ok(Some(message)) => message, Ok(None) => break, @@ -262,59 +285,27 @@ where match flight_message { FlightMessage::RecordBatch(record_batch) => { - let result_to_yield = - RecordBatch::from_df_record_batch(schema_cloned.clone(), record_batch); - - // Metrics follow a batch so MergeScan can observe them before yielding it. - match reader.peek_next_message_kind().await { - Ok(Some(FlightMessageKind::Metrics)) => { - let metrics_message = match reader.read_next().await { - Ok(Some(FlightMessage::Metrics(metrics))) => metrics, - Ok(Some(_) | None) => { - yield IllegalFlightMessagesSnafu { - reason: "Flight stream changed after peek", - } - .fail() - .map_err(BoxedError::new) - .context(ExternalSnafu); - break; - } - Err(error) => { - yield Err(BoxedError::new(flight_stream_error( - &stream_addr, - error, - ))) - .context(ExternalSnafu); - break; - } - }; - let metrics = serde_json::from_str(&metrics_message).ok().map(Arc::new); - metrics_ref.swap(metrics); - } - Ok(Some(FlightMessageKind::RecordBatch)) => {} - Ok(Some(FlightMessageKind::Schema | FlightMessageKind::AffectedRows)) => { - yield IllegalFlightMessagesSnafu { - reason: "A RecordBatch message can only be succeeded by a Metrics message or another RecordBatch message" - } - .fail() - .map_err(BoxedError::new) - .context(ExternalSnafu); - break; - } - Ok(None) => stream_ended = true, - Err(error) => { - yield Err(BoxedError::new(flight_stream_error(&stream_addr, error))) - .context(ExternalSnafu); - break; - } - } - - yield Ok(result_to_yield); + // Deliver each batch immediately. In particular, do not + // wait for a possible following Metrics message; it is + // consumed on the next poll of this stream. + yield Ok(RecordBatch::from_df_record_batch( + schema_cloned.clone(), + record_batch, + )); } FlightMessage::Metrics(s) => { // Metrics may arrive before the next RecordBatch. - let m = serde_json::from_str(&s).ok().map(Arc::new); - metrics_ref.swap(m); + match serde_json::from_str(&s) { + Ok(metrics) => { + metrics_ref.swap(Some(Arc::new(metrics))); + } + Err(error) => { + common_telemetry::warn!( + "Failed to decode region Flight metrics: {}", + error + ); + } + } continue; } _ => { @@ -419,13 +410,22 @@ pub fn check_response_header(header: &Option) -> Result<()> { #[cfg(test)] #[allow(deprecated)] -mod test { - use api::v1::Status as PbStatus; +mod tests { + use std::collections::VecDeque; + use std::sync::Mutex; + use api::v1::region::region_server::{Region, RegionServer}; use api::v1::region::{ - BulkInsertRequest, RegionResponse as PbRegionResponse, RemoteDynFilterUnregister, - RemoteDynFilterUpdate, region_request, remote_dyn_filter_request, + BulkInsertRequest, InsertRequest, InsertRequests, RegionResponse as PbRegionResponse, + RemoteDynFilterUnregister, RemoteDynFilterUpdate, region_request, + remote_dyn_filter_request, }; + use api::v1::value::ValueData; + use api::v1::{ + ColumnDataType, ColumnSchema as PbColumnSchema, Row, Rows, Status as PbStatus, Value, + }; + use common_error::ext::RetryHint; + use common_grpc::channel_manager::{ChannelConfig, ChannelManager}; use common_recordbatch::adapter::RecordBatchMetrics; use datatypes::arrow::array::Int32Array; use datatypes::prelude::{ConcreteDataType, VectorRef}; @@ -433,74 +433,526 @@ mod test { use datatypes::vectors::Int32Vector; use futures_util::stream; use tokio::net::TcpListener; + use tokio::task::JoinHandle; use tokio_stream::wrappers::TcpListenerStream; use tonic::codec::CompressionEncoding; use tonic::{Request, Response, Status}; use super::*; - use crate::Error::{self, IllegalDatabaseResponse, Server}; + use crate::Error; - #[derive(Clone)] - struct CompressionRecordingRegionService { - zstd_headers: Arc>>, + #[derive(Clone, Default)] + struct RecordingRegionService { + requests: Arc>>, + responses: Arc>>>, } #[tonic::async_trait] - impl Region for CompressionRecordingRegionService { + impl Region for RecordingRegionService { async fn handle( &self, request: Request, ) -> std::result::Result, Status> { let metadata = request.metadata(); - let sends_zstd = metadata - .get("grpc-encoding") - .and_then(|v| v.to_str().ok()) - .is_some_and(|value| value == "zstd"); + let sends_zstd = metadata.get("grpc-encoding").is_some_and(|v| v == "zstd"); let accepts_zstd = metadata .get("grpc-accept-encoding") - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.split(',').any(|encoding| encoding == "zstd")); - self.zstd_headers + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.split(',').any(|encoding| encoding == "zstd")); + self.requests .lock() .unwrap() - .push((sends_zstd, accepts_zstd)); + .push((request.into_inner(), sends_zstd, accepts_zstd)); + self.responses + .lock() + .unwrap() + .pop_front() + .unwrap_or_else(|| Ok(build_response(StatusCode::Success))) + .map(Response::new) + } + } - Ok(Response::new(PbRegionResponse { - header: Some(ResponseHeader { - status: Some(PbStatus { - status_code: StatusCode::Success as u32, - ..Default::default() - }), + struct RegionTestServer { + addr: String, + service: RecordingRegionService, + task: JoinHandle<()>, + } + + impl RegionTestServer { + async fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + let service = RecordingRegionService::default(); + let server_service = service.clone(); + let task = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service( + RegionServer::new(server_service) + .accept_compressed(CompressionEncoding::Zstd) + .send_compressed(CompressionEncoding::Zstd), + ) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .unwrap(); + }); + Self { + addr, + service, + task, + } + } + + fn create_requester( + &self, + config: ChannelConfig, + send: bool, + accept: bool, + ) -> RegionRequester { + RegionRequester::new( + Client::with_manager_and_urls( + ChannelManager::with_config(config, None), + [self.addr.clone()], + ), + send, + accept, + ) + } + } + + impl Drop for RegionTestServer { + fn drop(&mut self) { + self.task.abort(); + } + } + + fn build_response(code: StatusCode) -> PbRegionResponse { + PbRegionResponse { + header: Some(ResponseHeader { + status: Some(PbStatus { + status_code: code as u32, + ..Default::default() }), + }), + affected_rows: 1, + ..Default::default() + } + } + + fn build_insert_request() -> RegionRequest { + RegionRequest { + header: Some(RegionRequestHeader { + dbname: "shared-request-test".to_string(), ..Default::default() - })) + }), + body: Some(region_request::Body::Inserts(InsertRequests { + requests: vec![InsertRequest { + region_id: 42, + rows: Some(Rows { + schema: vec![ + PbColumnSchema { + column_name: "message".to_string(), + datatype: ColumnDataType::String as i32, + ..Default::default() + }, + PbColumnSchema { + column_name: "payload".to_string(), + datatype: ColumnDataType::Binary as i32, + ..Default::default() + }, + ], + rows: vec![Row { + values: vec![ + Value { + value_data: Some(ValueData::StringValue( + "hello 世界".to_string(), + )), + }, + Value { + value_data: Some(ValueData::BinaryValue(vec![0, 1, 127, 255])), + }, + ], + }], + }), + ..Default::default() + }], + })), } } #[tokio::test] - async fn test_inserts_and_bulk_insert_use_transport_compression() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - let zstd_headers = Arc::new(std::sync::Mutex::new(Vec::new())); - let service = CompressionRecordingRegionService { - zstd_headers: zstd_headers.clone(), - }; - let server = tokio::spawn(async move { - tonic::transport::Server::builder() - .add_service( - RegionServer::new(service) - .accept_compressed(CompressionEncoding::Zstd) - .send_compressed(CompressionEncoding::Zstd), - ) - .serve_with_incoming(TcpListenerStream::new(listener)) + async fn test_owned_and_shared_requests_preserve_payload_and_compression() { + let server = RegionTestServer::start().await; + let shared = Arc::new(build_insert_request()); + for (send, accept) in [(false, false), (true, false), (false, true), (true, true)] { + let requester = server.create_requester(ChannelConfig::default(), send, accept); + let channel = + tonic::transport::Endpoint::from_shared(format!("http://{}", server.addr)) + .unwrap() + .connect() + .await + .unwrap(); + let mut baseline = api::v1::region::region_client::RegionClient::new(channel); + if send { + baseline = baseline.send_compressed(CompressionEncoding::Zstd); + } + if accept { + baseline = baseline.accept_compressed(CompressionEncoding::Zstd); + } + assert_eq!( + baseline + .handle(build_insert_request()) + .await + .unwrap() + .into_inner() + .affected_rows, + 1 + ); + assert_eq!( + requester + .handle(build_insert_request()) + .await + .unwrap() + .affected_rows, + 1 + ); + assert_eq!( + requester + .handle(Arc::clone(&shared)) + .await + .unwrap() + .affected_rows, + 1 + ); + } + let requests = server.service.requests.lock().unwrap(); + assert_eq!(requests.len(), 12); + for ((request, sends, accepts), (send, accept)) in requests.iter().zip( + [(false, false), (true, false), (false, true), (true, true)] + .into_iter() + .flat_map(|settings| [settings, settings, settings]), + ) { + assert_eq!(request, shared.as_ref()); + assert_eq!((*sends, *accepts), (send, accept)); + } + } + + #[tokio::test] + async fn test_shared_request_can_be_resent_after_transport_error() { + let server = RegionTestServer::start().await; + let mut status = Status::unavailable("migrating"); + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT, + RetryHint::Retryable.as_str().parse().unwrap(), + ); + server + .service + .responses + .lock() + .unwrap() + .push_back(Err(status)); + let requester = server.create_requester(ChannelConfig::default(), false, false); + let shared = Arc::new(build_insert_request()); + let error = requester.handle(Arc::clone(&shared)).await.unwrap_err(); + assert_eq!(error.retry_hint(), RetryHint::Retryable); + assert!( + matches!(error, Error::RegionServer { addr, code: tonic::Code::Unavailable, .. } if addr == server.addr) + ); + requester.handle(Arc::clone(&shared)).await.unwrap(); + let requests = server.service.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(&requests[0].0, shared.as_ref()); + assert_eq!(&requests[1].0, shared.as_ref()); + } + + #[test] + fn test_retry_hint_wire_compatibility() { + for (name, wire_hint, expected_hint) in [ + ("legacy_missing", None, RetryHint::NonRetryable), + ("legacy_retryable", Some("retryable"), RetryHint::Retryable), + ( + "legacy_non_retryable", + Some("non_retryable"), + RetryHint::NonRetryable, + ), + ( + "route_refresh", + Some("retry_after_route_refresh"), + RetryHint::RetryAfterRouteRefresh, + ), + ( + "unknown_future_hint", + Some("future_retry_hint"), + RetryHint::NonRetryable, + ), + ] { + let mut status = Status::failed_precondition("region unavailable"); + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_CODE, + (StatusCode::RegionNotReady as u32) + .to_string() + .parse() + .unwrap(), + ); + if let Some(hint) = wire_hint { + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT, + hint.parse().unwrap(), + ); + } + let error: Error = status.into(); + assert_eq!(error.retry_hint(), expected_hint, "{name}"); + assert_eq!(error.status_code(), StatusCode::RegionNotReady, "{name}"); + assert_eq!( + error.tonic_code(), + Some(tonic::Code::FailedPrecondition), + "{name}" + ); + } + } + + #[tokio::test] + async fn test_transport_error_parity_and_datanode_mapping() { + let server = RegionTestServer::start().await; + let requester = server.create_requester(ChannelConfig::default(), false, false); + for hint in [ + RetryHint::Retryable, + RetryHint::RetryAfterRouteRefresh, + RetryHint::NonRetryable, + ] { + let mut status = Status::failed_precondition("region unavailable"); + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT, + hint.as_str().parse().unwrap(), + ); + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_CODE, + (StatusCode::RegionNotFound as u32) + .to_string() + .parse() + .unwrap(), + ); + server + .service + .responses + .lock() + .unwrap() + .extend((0..4).map(|_| Err(status.clone()))); + let mut baseline = api::v1::region::region_client::RegionClient::connect(format!( + "http://{}", + server.addr + )) + .await + .unwrap(); + let baseline: Error = baseline + .handle(build_insert_request()) .await - .unwrap(); + .unwrap_err() + .into(); + assert_eq!(baseline.status_code(), StatusCode::RegionNotFound); + assert_eq!(baseline.retry_hint(), hint); + assert_eq!(baseline.tonic_code(), Some(tonic::Code::FailedPrecondition)); + let owned = requester.handle(build_insert_request()).await.unwrap_err(); + let shared = requester + .handle(Arc::new(build_insert_request())) + .await + .unwrap_err(); + for error in [owned, shared] { + assert_eq!(error.status_code(), baseline.status_code()); + assert_eq!(error.retry_hint(), baseline.retry_hint()); + assert_eq!(error.tonic_code(), baseline.tonic_code()); + } + let datanode: &dyn Datanode = &requester; + let error = datanode.handle(build_insert_request()).await.unwrap_err(); + assert_eq!(error.status_code(), StatusCode::RegionNotFound); + assert_eq!(error.retry_hint(), hint); + assert!(matches!(error, meta_error::Error::External { .. })); + } + } + + #[tokio::test] + async fn test_datanode_transport_retry_hints() { + let server = RegionTestServer::start().await; + let requester = server.create_requester(ChannelConfig::default(), false, false); + let datanode: &dyn Datanode = &requester; + let statuses = [ + tonic::Code::Unavailable, + tonic::Code::Cancelled, + tonic::Code::DeadlineExceeded, + tonic::Code::FailedPrecondition, + ] + .into_iter() + .flat_map(|code| { + [ + None, + Some(RetryHint::NonRetryable), + Some(RetryHint::Retryable), + Some(RetryHint::RetryAfterRouteRefresh), + ] + .into_iter() + .map(move |hint| (code, hint)) }); - let client = Client::with_urls([addr.to_string()]); - let requester = RegionRequester::new(client.clone(), true, true); - let send_only_requester = RegionRequester::new(client.clone(), true, false); - let accept_only_requester = RegionRequester::new(client.clone(), false, true); - let disabled_requester = RegionRequester::new(client, false, false); + for (code, hint) in statuses { + for body in [ + region_request::Body::Inserts(Default::default()), + region_request::Body::Deletes(Default::default()), + region_request::Body::BulkInsert(Default::default()), + region_request::Body::Create(Default::default()), + region_request::Body::RemoteDynFilter(Default::default()), + ] { + let request_type = body.as_ref().to_string(); + let mut status = Status::new(code, "request failed"); + // Metadata status is deliberately different from the transport status. + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_CODE, + (StatusCode::RegionNotReady as u32) + .to_string() + .parse() + .unwrap(), + ); + if let Some(hint) = hint { + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT, + hint.as_str().parse().unwrap(), + ); + } + let original = Error::from(status.clone()); + let expected = if original.should_retry() && !original.retry_hint().is_retryable() { + RetryHint::Retryable + } else { + original.retry_hint() + }; + server + .service + .responses + .lock() + .unwrap() + .extend([Err(status.clone()), Err(status)]); + let request = RegionRequest { + header: None, + body: Some(body), + }; + let shared = Arc::new(request); + let shared_error = datanode.handle(Arc::clone(&shared)).await.unwrap_err(); + assert_eq!(Arc::strong_count(&shared), 1); + let request = Arc::try_unwrap(shared).unwrap(); + let owned_error = datanode.handle(request).await.unwrap_err(); + for error in [owned_error, shared_error] { + assert_eq!( + error.retry_hint(), + expected, + "{code:?}, {hint:?}, {request_type}" + ); + assert_eq!(error.retry_hint().is_retryable(), original.should_retry()); + assert_eq!(error.status_code(), StatusCode::RegionNotReady); + let meta_error::Error::External { source, .. } = error else { + panic!("expected an external error"); + }; + assert_eq!(source.retry_hint(), expected); + let Error::RegionServer { + source, + code: actual_code, + .. + } = source.as_any().downcast_ref::().unwrap() + else { + panic!("expected a region server error"); + }; + assert_eq!(*actual_code, code); + assert_eq!(source.retry_hint(), original.retry_hint()); + assert_eq!(source.to_string(), original.to_string()); + } + } + } + } + + #[tokio::test] + async fn test_shared_request_preserves_response_header_errors() { + let server = RegionTestServer::start().await; + server.service.responses.lock().unwrap().extend([ + Ok(PbRegionResponse::default()), + Ok(build_response(StatusCode::RegionNotFound)), + ]); + let requester = server.create_requester(ChannelConfig::default(), false, false); + let shared = Arc::new(build_insert_request()); + assert!(matches!( + requester.handle(Arc::clone(&shared)).await.unwrap_err(), + Error::IllegalDatabaseResponse { .. } + )); + assert!(matches!( + requester.handle(shared).await.unwrap_err(), + Error::Server { + code: StatusCode::RegionNotFound, + .. + } + )); + } + + #[tokio::test] + async fn test_shared_request_respects_configured_message_limits() { + let server = RegionTestServer::start().await; + let shared = Arc::new(build_insert_request()); + for send_limit in [true, false] { + let mut config = ChannelConfig::default(); + if send_limit { + config.max_send_message_size = "1B".parse().unwrap(); + } else { + config.max_recv_message_size = "1B".parse().unwrap(); + } + let mut baseline = api::v1::region::region_client::RegionClient::connect(format!( + "http://{}", + server.addr + )) + .await + .unwrap() + .max_encoding_message_size(config.max_send_message_size.as_bytes() as usize) + .max_decoding_message_size(config.max_recv_message_size.as_bytes() as usize); + let baseline: Error = baseline + .handle(build_insert_request()) + .await + .unwrap_err() + .into(); + if !send_limit { + assert_eq!( + baseline.tonic_code(), + Some(tonic::Code::OutOfRange), + "{baseline:?}" + ); + } + let requester = server.create_requester(config, false, false); + let error = requester.handle(Arc::clone(&shared)).await.unwrap_err(); + assert_eq!( + error.tonic_code(), + baseline.tonic_code(), + "shared: {error:?}; generated: {baseline:?}" + ); + assert_eq!( + error.status_code(), + baseline.status_code(), + "shared: {error:?}; generated: {baseline:?}" + ); + let Error::RegionServer { source, .. } = &error else { + panic!("expected a region server error: {error:?}"); + }; + assert_eq!( + source.retry_hint(), + baseline.retry_hint(), + "shared: {error:?}; generated: {baseline:?}" + ); + assert_eq!( + server.service.requests.lock().unwrap().len(), + if send_limit { 0 } else { 2 } + ); + } + let requester = server.create_requester(ChannelConfig::default(), false, false); + requester.handle(shared).await.unwrap(); + } + + #[tokio::test] + async fn test_inserts_and_bulk_insert_use_transport_compression() { + let server = RegionTestServer::start().await; + let requester = server.create_requester(ChannelConfig::default(), true, true); + let send_only_requester = server.create_requester(ChannelConfig::default(), true, false); + let accept_only_requester = server.create_requester(ChannelConfig::default(), false, true); + let disabled_requester = server.create_requester(ChannelConfig::default(), false, false); let inserts = || RegionRequest { body: Some(region_request::Body::Inserts(Default::default())), ..Default::default() @@ -536,9 +988,15 @@ mod test { (false, false), (false, false) ], - *zstd_headers.lock().unwrap() + server + .service + .requests + .lock() + .unwrap() + .iter() + .map(|(_, send, accept)| (*send, *accept)) + .collect::>() ); - server.abort(); } #[test] @@ -600,13 +1058,13 @@ mod test { let result = check_response_header(&None); assert!(matches!( result.unwrap_err(), - IllegalDatabaseResponse { .. } + Error::IllegalDatabaseResponse { .. } )); let result = check_response_header(&Some(ResponseHeader { status: None })); assert!(matches!( result.unwrap_err(), - IllegalDatabaseResponse { .. } + Error::IllegalDatabaseResponse { .. } )); let result = check_response_header(&Some(ResponseHeader { @@ -625,7 +1083,7 @@ mod test { })); assert!(matches!( result.unwrap_err(), - IllegalDatabaseResponse { .. } + Error::IllegalDatabaseResponse { .. } )); let result = check_response_header(&Some(ResponseHeader { @@ -634,7 +1092,7 @@ mod test { err_msg: "blabla".to_string(), }), })); - let Server { code, msg, .. } = result.unwrap_err() else { + let Error::Server { code, msg, .. } = result.unwrap_err() else { unreachable!() }; assert_eq!(code, StatusCode::Internal); @@ -719,7 +1177,34 @@ mod test { } #[tokio::test] - async fn test_record_batch_stream_updates_following_metrics_before_yielding_batch() { + async fn test_record_batch_is_yielded_without_waiting_for_next_message() { + let schema = test_schema(); + let batch = RecordBatch::new( + schema.clone(), + vec![Arc::new(Int32Vector::from_slice([1])) as VectorRef], + ) + .unwrap(); + + let messages = stream::iter(vec![ + Ok(FlightMessage::Schema(schema.arrow_schema().clone())), + Ok(FlightMessage::RecordBatch(batch.into_df_record_batch())), + ]) + .chain(stream::pending::>()); + let mut recordbatches = + recordbatches_from_flight_message_stream("test-peer".to_string(), messages) + .await + .unwrap(); + + let batch = tokio::time::timeout(Duration::from_secs(1), recordbatches.next()) + .await + .expect("the first batch must not wait for lookahead") + .unwrap() + .unwrap(); + assert_eq!(batch.num_rows(), 1); + } + + #[tokio::test] + async fn test_malformed_region_metrics_are_non_fatal() { let schema = test_schema(); let batch = RecordBatch::new( schema.clone(), @@ -730,23 +1215,19 @@ mod test { "test-peer".to_string(), stream::iter(vec![ Ok(FlightMessage::Schema(schema.arrow_schema().clone())), + Ok(FlightMessage::Metrics("{not-json}".to_string())), Ok(FlightMessage::RecordBatch(batch.into_df_record_batch())), - Ok(FlightMessage::Metrics(test_metrics_json())), ]), ) .await .unwrap(); - let batch = recordbatches.next().await.unwrap().unwrap(); - assert_eq!(batch.num_rows(), 1); - - let metrics = recordbatches.metrics().unwrap(); - assert_eq!(metrics.elapsed_compute, 7); + assert_eq!(recordbatches.next().await.unwrap().unwrap().num_rows(), 1); assert!(recordbatches.next().await.is_none()); } #[tokio::test] - async fn test_record_batch_stream_preserves_peeked_record_batch() { + async fn test_record_batch_stream_preserves_following_record_batch() { let schema = test_schema(); let first_batch = RecordBatch::new( schema.clone(), diff --git a/src/client/src/region/codec.rs b/src/client/src/region/codec.rs new file mode 100644 index 00000000000..9476385df36 --- /dev/null +++ b/src/client/src/region/codec.rs @@ -0,0 +1,63 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::borrow::Borrow; +use std::marker::PhantomData; + +use api::v1::region::{RegionRequest, RegionResponse}; +use prost::Message; +use tonic::Status; +use tonic::codec::{Codec, EncodeBuf, Encoder}; +use tonic_prost::ProstDecoder; + +/// Encodes owned or shared region requests using the existing protobuf wire format. +pub(crate) struct RegionCodec(PhantomData); + +impl Default for RegionCodec { + fn default() -> Self { + Self(PhantomData) + } +} + +impl Codec for RegionCodec +where + R: Borrow + Send + 'static, +{ + type Encode = R; + type Decode = RegionResponse; + type Encoder = RegionEncoder; + type Decoder = ProstDecoder; + + fn encoder(&mut self) -> Self::Encoder { + RegionEncoder(PhantomData) + } + + fn decoder(&mut self) -> Self::Decoder { + ProstDecoder::default() + } +} + +/// Borrows the protobuf payload while consuming its owned or shared container. +pub(crate) struct RegionEncoder(PhantomData); + +impl> Encoder for RegionEncoder { + type Item = R; + type Error = Status; + + fn encode(&mut self, item: R, buf: &mut EncodeBuf<'_>) -> Result<(), Status> { + item.borrow() + .encode(buf) + .map_err(|error| Status::internal(error.to_string())) + } +} diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs index cc095c5c92e..ba38793b8cd 100644 --- a/src/cmd/tests/load_config_test.rs +++ b/src/cmd/tests/load_config_test.rs @@ -475,6 +475,261 @@ fn test_load_standalone_example_config() { similar_asserts::assert_eq!(options, expected); } +/// Renders an example config with its `#+` options enabled, as `config.md` +/// lists them. With `drop_none_default`, options annotated with +/// `@toml2docs:none-default` are removed, because `config.md` does not present +/// their example value as the default. +fn render_example(example: &str, drop_none_default: bool) -> String { + let mut rendered = String::new(); + let mut none_default = false; + for line in example.lines() { + let line = line.trim_start(); + if line.starts_with("##") { + none_default |= line.contains("@toml2docs:none-default"); + continue; + } + let line = line.strip_prefix("#+").map_or(line, str::trim_start); + let is_option = !line.starts_with('[') && line.contains('='); + if !(is_option && drop_none_default && none_default) { + rendered.push_str(line); + rendered.push('\n'); + } + none_default = false; + } + rendered +} + +/// Flattens leaf values into dotted paths. Arrays of tables such as +/// `region_engine` are merged, since each element is keyed by its engine name. +fn leaf_values(value: &toml::Value, prefix: &str, leaves: &mut Vec<(String, toml::Value)>) { + match value { + toml::Value::Table(table) => { + for (key, value) in table { + let path = if prefix.is_empty() { + key.clone() + } else { + format!("{prefix}.{key}") + }; + leaf_values(value, &path, leaves); + } + } + toml::Value::Array(array) + if array.iter().all(toml::Value::is_table) && !array.is_empty() => + { + for element in array { + leaf_values(element, prefix, leaves); + } + } + _ => leaves.push((prefix.to_string(), value.clone())), + } +} + +/// Removes the option at a dotted path produced by [`leaf_values`]. +fn remove_leaf(value: &mut toml::Value, path: &str) -> bool { + match value { + toml::Value::Array(array) => array.iter_mut().any(|element| remove_leaf(element, path)), + toml::Value::Table(table) => match path.split_once('.') { + None => table.remove(path).is_some(), + Some((head, rest)) => table + .get_mut(head) + .is_some_and(|value| remove_leaf(value, rest)), + }, + _ => false, + } +} + +fn load_as_leaves( + config: Option<&str>, +) -> common_config::error::Result> +where + T: Configurable + serde::Serialize, +{ + let file = config.map(|config| { + let mut file = create_named_temp_file(); + file.write_all(config.as_bytes()).unwrap(); + file + }); + let options = GreptimeOptions::::load_layered_options( + file.as_ref().map(|file| file.path().to_str().unwrap()), + "", + )?; + let mut leaves = Vec::new(); + leaf_values(&toml::Value::try_from(&options).unwrap(), "", &mut leaves); + Ok(leaves.into_iter().collect()) +} + +/// Returns the default of an option inside the example's other settings, so +/// that options in sections absent by default still have one to compare with. +/// Removes the closest enclosing table that still loads without the option. +fn default_within_example(full: &toml::Value, path: &str) -> Option +where + T: Configurable + serde::Serialize, +{ + let mut removed = path; + loop { + let mut without = full.clone(); + assert!(remove_leaf(&mut without, removed)); + if let Ok(defaults) = load_as_leaves::(Some(&toml::to_string(&without).unwrap())) { + return defaults.get(path).cloned(); + } + removed = removed + .rsplit_once('.') + .unwrap_or_else(|| panic!("the example does not load without `{path}`")) + .0; + } +} + +/// Checks that every option in an example config exists, and that every value +/// `config.md` presents as a default is the actual default. +/// +/// `variants` lists tagged sections, e.g. `("wal", "provider", ["kafka"])`, +/// whose options only deserialize under a specific tag value. +fn check_example_defaults(name: &str, variants: &[(&str, &str, &[&str])]) -> Vec +where + T: Configurable + serde::Serialize, +{ + let example = std::fs::read_to_string(common_test_util::find_workspace_path(&format!( + "config/{name}" + ))) + .unwrap(); + let full: toml::Value = toml::from_str(&render_example(&example, false)).unwrap(); + let documented: toml::Value = toml::from_str(&render_example(&example, true)).unwrap(); + + // (options loaded from the example, options loaded without config) per variant + let mut loaded = vec![( + load_as_leaves::(Some(&toml::to_string(&full).unwrap())).unwrap(), + load_as_leaves::(None).unwrap(), + )]; + for (section, tag, values) in variants { + for value in *values { + let mut variant = full.clone(); + variant[section][tag] = toml::Value::String(value.to_string()); + loaded.push(( + load_as_leaves::(Some(&toml::to_string(&variant).unwrap())).unwrap(), + load_as_leaves::(Some(&format!("[{section}]\n{tag} = \"{value}\"\n"))).unwrap(), + )); + } + } + + // Deserialize-only fields, invisible in the serialized options. + const SKIP_SERIALIZING: &[&str] = &[ + "storage.access_key_id", + "storage.secret_access_key", + "storage.access_key_secret", + "storage.account_name", + "storage.account_key", + "storage.credential_path", + "storage.credential", + "region_engine.mito.inverted_index.intermediate_path", + ]; + + let mut errors = Vec::new(); + let mut full_leaves = Vec::new(); + leaf_values(&full, "", &mut full_leaves); + for (path, _) in full_leaves { + if SKIP_SERIALIZING.contains(&path.as_str()) + || (path == "tracing.tokio_console_addr" && !cfg!(feature = "tokio-console")) + { + continue; + } + if !loaded + .iter() + .any(|(example, _)| example.contains_key(&path)) + { + errors.push(format!("{name}: `{path}` is not a known option")); + } + } + + // Options left unset by default and resolved where they are used. + let resolved_at_runtime: [(&str, toml::Value); 7] = [ + ( + "logging.dir", + format!("{DEFAULT_DATA_HOME}/{DEFAULT_LOGGING_DIR}").into(), + ), + ("logging.otlp_endpoint", DEFAULT_OTLP_HTTP_ENDPOINT.into()), + ("logging.otlp_export_protocol", "http".into()), + ("logging.tracing_sample_ratio.default_ratio", 1.0.into()), + ("wal.sync_period", "5s".into()), + ( + "storage.copy_root", + format!("{DEFAULT_DATA_HOME}/copy").into(), + ), + ("flow.batching_mode.frontend_tls.enabled", false.into()), + ]; + + let mut documented_leaves = Vec::new(); + leaf_values(&documented, "", &mut documented_leaves); + for (path, documented_value) in documented_leaves { + if SKIP_SERIALIZING.contains(&path.as_str()) { + continue; + } + if let Some((_, resolved)) = resolved_at_runtime.iter().find(|(p, _)| *p == path) { + if &documented_value != resolved { + errors.push(format!( + "{name}: `{path}` is documented as default `{documented_value}`, but resolves to `{resolved}`" + )); + } + continue; + } + let mut pairs = loaded + .iter() + .filter_map(|(example, default)| Some((example.get(&path)?, default.get(&path)?))) + .map(|(example, default)| (example.clone(), default.clone())) + .collect::>(); + if pairs.is_empty() { + // The option sits in a section that is absent without config, or + // is itself optional. + match default_within_example::(&full, &path) { + Some(default) => pairs.push((loaded[0].0[&path].clone(), default)), + None => { + errors.push(format!( + "{name}: `{path}` has no default to compare with; annotate it with \ + `@toml2docs:none-default` or list its runtime default in this test" + )); + continue; + } + } + } + if pairs.iter().all(|(example, default)| example != default) { + let (example, default) = &pairs[0]; + errors.push(format!( + "{name}: `{path}` is documented as default `{example}`, but the default is `{default}`; \ + fix the value or annotate it with `@toml2docs:none-default`" + )); + } + } + errors +} + +// The examples document HDFS storage options. +#[cfg(feature = "hdfs-object-store")] +#[test] +fn test_example_configs_document_actual_defaults() { + const WAL: (&str, &str, &[&str]) = ("wal", "provider", &["kafka", "experimental_object_store"]); + const STORAGE: (&str, &str, &[&str]) = + ("storage", "type", &["S3", "Oss", "Azblob", "Gcs", "Hdfs"]); + + let mut errors = + check_example_defaults::("standalone.example.toml", &[WAL, STORAGE]); + errors.extend(check_example_defaults::( + "frontend.example.toml", + &[], + )); + errors.extend(check_example_defaults::( + "datanode.example.toml", + &[WAL, STORAGE], + )); + errors.extend(check_example_defaults::( + "metasrv.example.toml", + &[("wal", "provider", &["kafka"])], + )); + errors.extend(check_example_defaults::( + "flownode.example.toml", + &[], + )); + assert!(errors.is_empty(), "{}", errors.join("\n")); +} + #[test] fn test_load_removed_histogram_options() { for enabled in [false, true] { diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml index 702171e9d67..6df2283c2cf 100644 --- a/src/common/base/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -14,6 +14,7 @@ async-trait.workspace = true bitvec = "1.0" bytes.workspace = true futures.workspace = true +icu_properties.workspace = true lazy_static.workspace = true pin-project.workspace = true regex.workspace = true diff --git a/src/common/base/src/lib.rs b/src/common/base/src/lib.rs index 04fc7038018..2d5a6fbd894 100644 --- a/src/common/base/src/lib.rs +++ b/src/common/base/src/lib.rs @@ -25,6 +25,7 @@ pub mod readable_size; pub mod regex_pattern; pub mod secrets; pub mod serde; +pub mod term_token; pub type AffectedRows = usize; diff --git a/src/common/base/src/term_token.rs b/src/common/base/src/term_token.rs new file mode 100644 index 00000000000..a6121559a67 --- /dev/null +++ b/src/common/base/src/term_token.rs @@ -0,0 +1,280 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Word boundaries shared by `matches_term` and the fulltext bloom index. +//! +//! Bloom pruning is only sound if the index splits text exactly where +//! `matches_term` sees word boundaries, so both use [`classify_char`], and the +//! query side derives its probes with [`extract_term_probes`] and [`extract_like_probes`]. + +use std::ops::Range; + +use icu_properties::props::Script; +use icu_properties::{CodePointMapData, CodePointMapDataBorrowed}; + +/// Classes of characters that decide where `matches_term` sees a word boundary. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CharClass { + /// `[0-9A-Za-z]`. + AsciiWord, + /// Characters of the Han script, alphanumeric or not. + Han, + /// Non-ASCII alphanumerics outside the Han script, such as `é` or `д`. + UnicodeWord, + /// Everything else, including `_`. + Other, +} + +/// Returns the [`CharClass`] of `c`. [`tokenize`] splits text wherever the class +/// changes and at every [`CharClass::Other`] character. +pub fn classify_char(c: char) -> CharClass { + if c.is_ascii_alphanumeric() { + CharClass::AsciiWord + } else if is_han(c) { + CharClass::Han + } else if c.is_alphanumeric() { + CharClass::UnicodeWord + } else { + CharClass::Other + } +} + +static HAN_SCRIPT_DATA: CodePointMapDataBorrowed<'static, Script> = + CodePointMapData::