diff --git a/.cargo/config.toml b/.cargo/config.toml index fd372c95b84..8291bf467e4 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -21,5 +21,6 @@ checkout = true list_files = true internal_use_git2 = false -[env] -CARGO_WORKSPACE_DIR = { value = "", relative = true } +# Avoid globally injecting a checkout-specific workspace path. The version build script +# resolves the absolute workspace path from layout at build time; set CARGO_WORKSPACE_DIR +# explicitly when this repo is built as a submodule. diff --git a/.github/scripts/oswal-s3-bench-run.sh b/.github/scripts/oswal-s3-bench-run.sh new file mode 100644 index 00000000000..a9835c1e5f1 --- /dev/null +++ b/.github/scripts/oswal-s3-bench-run.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Host side of oswal-s3-bench.yml: one subcommand per step; each cell of $PLAN gets a fresh GreptimeDB. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Same isolation as the o11ybench runtime's own docker run. +runtime() { + docker run --rm --user "$(id -u):$(id -g)" --label "o11ybench.owner=$OWNER" \ + --security-opt no-new-privileges --cap-drop ALL "$@" +} + +check_runtime() { + local image=$1 resolved + [[ "$image" == public.ecr.aws/* ]] || { echo 'The runtime image must come from AWS ECR Public' >&2; return 1; } + bash "$here/pull-benchmark-image.sh" "$image" + resolved=$(docker image inspect "$image" | jq -er '.[0].RepoDigests[0]') + docker run --rm --entrypoint sh "$resolved" -c \ + 'test -f scripts/agent-observability/greptimedb.sh && test -f scripts/generate_agent_observations.py' \ + || { echo 'The runtime image lacks the agent observability loader or generator' >&2; return 1; } + printf 'runtime_image=%s\n' "$resolved" >> "${GITHUB_OUTPUT:-/dev/stdout}" +} + +pull_images() { + : "${RESULTS:?}" "${RUNTIME_IMAGE:?}" "${GREPTIMEDB_TAG:?}" + mkdir -p "$RESULTS/images" + bash "$here/pull-benchmark-image.sh" "$RUNTIME_IMAGE" + bash "$here/pull-benchmark-image.sh" "greptime/greptimedb:$GREPTIMEDB_TAG" + docker image inspect "$RUNTIME_IMAGE" > "$RESULTS/images/runtime.json" + docker image inspect "greptime/greptimedb:$GREPTIMEDB_TAG" > "$RESULTS/images/greptimedb.json" +} + +wal_disk() { + : "${RESULTS:?}" + local root=$1 device + device=$(lsblk -dnpo NAME,MODEL | awk '/Instance Storage/ {print $1; exit}') + # Only a smoke run may keep the Raft Engine WAL on the root volume. + [[ -n "$device" || "${SMOKE:-}" == true ]] || { echo 'No local NVMe instance store; choose an instance type with one' >&2; return 1; } + sudo -n mkdir -p "$root" + if [[ -n "$device" ]]; then + sudo -n umount "$device" 2>/dev/null || true + sudo -n mkfs.ext4 -q -F "$device" + sudo -n mount -o noatime "$device" "$root" + fi + sudo -n chown "$(id -u):$(id -g)" "$root" + findmnt -no SOURCE,FSTYPE -T "$root" | tee "$RESULTS/wal-disk.txt" +} + +generate() { + : "${PLAN:?}" "${RESULTS:?}" "${CORPUS_DIR:?}" "${OWNER:?}" "${RUNTIME_IMAGE:?}" + local args=() arg + while IFS= read -r arg; do args+=("$arg"); done < <(jq -r '.generator_args[]' "$PLAN") + mkdir -p "$CORPUS_DIR" + runtime --name "$OWNER-generator" --mount "type=bind,src=$CORPUS_DIR,dst=/work/corpus" \ + "$RUNTIME_IMAGE" scripts/generate_agent_observations.py "${args[@]}" \ + --output /work/corpus/agent_observations.jsonl \ + --summary-output /work/corpus/summary.json --evidence-output /work/corpus/evidence.json \ + > "$RESULTS/generator.log" 2>&1 + jq -e --argjson rows "$(jq .rows "$PLAN")" '.line_count == $rows' "$CORPUS_DIR/summary.json" >/dev/null +} + +# Scrape once, after the load succeeded or failed, while the server still runs. +scrape_metrics() { + curl --noproxy '*' -fsS --max-time 30 http://127.0.0.1:4000/metrics > "$dir/metrics.all" || { rm -f "$dir/metrics.all"; return 1; } + grep -E '^(# (HELP|TYPE) )?greptime_logstore_object_store_wal_' "$dir/metrics.all" > "$dir/metrics.prom" || true + rm -f "$dir/metrics.all" + metrics_scraped=true +} + +finish() { + local rc=$? + trap - EXIT + set +e + if [[ -n "$cid" ]]; then + # A load that succeeded without its metrics is not a measurement. + scrape_metrics || { echo "$id: metrics scrape failed" >&2; [[ $rc != 0 ]] || rc=4; } + docker stop -t 120 "$cid" >/dev/null + docker logs "$cid" > "$dir/server.log" 2>&1 + docker rm -fv "$cid" >/dev/null + fi + [[ -z "$wal_dir" ]] || sudo -n rm -rf "$wal_dir" + jq -n --argjson spec "$spec" --arg device "$wal_device" --arg version "$version" \ + --arg load_seconds "$load_seconds" --argjson exit_code "$rc" --argjson scraped "$metrics_scraped" \ + '$spec + {wal_device: $device, version: $version, exit_code: $exit_code, metrics_scraped: $scraped, + load_wall_seconds: ($load_seconds | tonumber? // null)}' > "$dir/cell.json" + exit "$rc" +} + +# One cell in its own process, so errexit holds inside it. The cell's state is +# global because the EXIT trap runs after this function returns. +cell() { + : "${PLAN:?}" "${RESULTS:?}" "${CORPUS_DIR:?}" "${OWNER:?}" "${RUNTIME_IMAGE:?}" "${GREPTIMEDB_IMAGE:?}" + : "${RUN_PREFIX:?}" "${WAL_ROOT:?}" "${S3_BUCKET:?}" "${S3_REGION:?}" "${AWS_ACCESS_KEY_ID:?}" "${AWS_SECRET_ACCESS_KEY:?}" + spec=$1 cid= wal_dir= version= load_seconds= metrics_scraped=false id= + local concurrency name setting started ended + id=$(jq -r .id <<< "$spec"); concurrency=$(jq -r .concurrency <<< "$spec") + dir="$RESULTS/cells/$id"; name="$OWNER-$id" + mkdir -p "$dir" + # The root holds data/ and the WAL's default prefix wal/ side by side. + local settings=(STORAGE__TYPE=S3 "STORAGE__BUCKET=$S3_BUCKET" "STORAGE__REGION=$S3_REGION" + "STORAGE__ROOT=$RUN_PREFIX/$id") + [[ -z "${S3_CONTAINER_ENDPOINT:-}" ]] || settings+=("STORAGE__ENDPOINT=$S3_CONTAINER_ENDPOINT") + while IFS= read -r setting; do settings+=("$setting"); done \ + < <(jq -r '.wal | to_entries[] | "WAL__\(.key)=\(.value)"' <<< "$spec") + local docker_env=() mounts=() + for setting in "${settings[@]}"; do + docker_env+=(-e "GREPTIMEDB_STANDALONE__$setting") + printf 'GREPTIMEDB_STANDALONE__%s\n' "$setting" >> "$dir/config.env" + done + # Pass the two keys by name: their values never reach argv, a file or the artifact. + export GREPTIMEDB_STANDALONE__STORAGE__ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID + export GREPTIMEDB_STANDALONE__STORAGE__SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY + docker_env+=(-e GREPTIMEDB_STANDALONE__STORAGE__ACCESS_KEY_ID -e GREPTIMEDB_STANDALONE__STORAGE__SECRET_ACCESS_KEY) + printf 'GREPTIMEDB_STANDALONE__STORAGE__%s=\n' ACCESS_KEY_ID SECRET_ACCESS_KEY >> "$dir/config.env" + if jq -e '.wal.DIR' <<< "$spec" >/dev/null; then + wal_dir="$WAL_ROOT/$id"; mkdir -p "$wal_dir" + mounts=(--mount "type=bind,src=$wal_dir,dst=$(jq -r .wal.DIR <<< "$spec")") + wal_device=$(findmnt -no SOURCE -T "$wal_dir") + else + wal_device="s3://$S3_BUCKET/$RUN_PREFIX/$id/wal" + fi + trap finish EXIT + cid=$(docker run -d --init --name "$name" --label "o11ybench.owner=$OWNER" \ + --add-host host.docker.internal:host-gateway \ + -p 127.0.0.1:4000:4000 -p 127.0.0.1:4001:4001 "${docker_env[@]}" ${mounts[@]+"${mounts[@]}"} \ + "$GREPTIMEDB_IMAGE" standalone start --http-addr=0.0.0.0:4000 --grpc-bind-addr=0.0.0.0:4001) + local deadline=$((SECONDS + 300)) + until curl --noproxy '*' -fsS --max-time 5 http://127.0.0.1:4000/health >/dev/null 2>&1; do + [[ $(docker inspect --format '{{.State.Running}}' "$cid") == true ]] || { echo "$id: server exited" >&2; exit 1; } + ((SECONDS < deadline)) || { echo "$id: server not healthy in 300s" >&2; exit 1; } + sleep 2 + done + version=$(docker exec "$cid" greptime --version | sed -n 's/^version: //p') + [[ -n "$version" ]] + started=$(date +%s.%N) + # Only the load phase, run exactly as run-agent-observability.sh runs it. + runtime --name "$name-load" --network host \ + --mount "type=bind,src=$CORPUS_DIR,dst=/work/corpus,readonly" --mount "type=bind,src=$dir,dst=/results" \ + -e GRPC_ENDPOINT=127.0.0.1:4001 -e "DATA_PROFILE=$(jq -r .data_profile "$PLAN")" -e RESULT_NAME=greptimedb-1 \ + -e "ENDPOINT=http://127.0.0.1:4000/v1/sql?db=public&format=greptimedb_v1" \ + -e "TARGET_IMAGE=$GREPTIMEDB_IMAGE" -e "TARGET_VERSION=$version" -e "TARGET_DIGEST=${GREPTIMEDB_IMAGE##*@}" \ + -e "TARGET_ID=$name" -e TARGET_HOST=127.0.0.1 -e PREPARE_WORKERS= -e "MAX_IN_FLIGHT=$concurrency" \ + -e "GREPTIMEDB_BATCH_SIZE=$(jq -r .batch_rows "$PLAN")" --entrypoint /bin/bash \ + "$RUNTIME_IMAGE" scripts/agent-observability/greptimedb.sh load > "$dir/load.log" 2>&1 + ended=$(date +%s.%N) + load_seconds=$(awk -v a="$started" -v b="$ended" 'BEGIN {printf "%.3f", b - a}') + jq -e --argjson rows "$(jq .rows "$PLAN")" '.count_before == 0 and .count_after == $rows' \ + "$dir/greptimedb-1/load-result.json" >/dev/null +} + +scrub() { + : "${RESULTS:?}" + [[ -n "${AWS_ACCESS_KEY_ID:-}" && -n "${AWS_SECRET_ACCESS_KEY:-}" && -d "$RESULTS" ]] || return 0 + if grep -rqF -e "$AWS_ACCESS_KEY_ID" -e "$AWS_SECRET_ACCESS_KEY" "$RESULTS"; then + echo '::error::A credential value appears in the results; removing them instead of uploading' >&2 + rm -rf "$RESULTS" + return 1 + fi +} + +case "${1:-}" in + check-runtime) check_runtime "${2:?}" ;; + pull) pull_images ;; + wal-disk) wal_disk "${2:?}" ;; + generate) generate ;; + cell) cell "${2:?}" ;; + cells) + : "${PLAN:?}" + failed=0 + while IFS= read -r spec; do + id=$(jq -r .id <<< "$spec") + echo "::group::cell $id" + bash "${BASH_SOURCE[0]}" cell "$spec" < /dev/null || { echo "::error::cell $id failed"; failed=1; } + echo '::endgroup::' + done < <(jq -c '.cells[]' "$PLAN") + exit "$failed" ;; + scrub) scrub ;; + *) echo "usage: $0 check-runtime IMAGE | pull | wal-disk DIR | generate | cells | scrub" >&2; exit 2 ;; +esac diff --git a/.github/scripts/oswal-s3-bench-summary.py b/.github/scripts/oswal-s3-bench-summary.py new file mode 100644 index 00000000000..67186d422da --- /dev/null +++ b/.github/scripts/oswal-s3-bench-summary.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Summarize the object store WAL S3 benchmark as Markdown and summary.json.""" + +from __future__ import annotations + +import argparse +import json +import math +import re +from pathlib import Path + +PREFIX = "greptime_logstore_object_store_wal_" +COUNTERS = ("created_objects_total", "stalled_appends_total", "poisoned_total", + "create_failures_total", "create_conflicts_total") +SAMPLE = re.compile(r'([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)') + + +def parse_metrics(text: str) -> dict: + """Map each unlabeled sample to its value and each histogram to its buckets.""" + samples, buckets = {}, {} + for line in text.splitlines(): + match = SAMPLE.fullmatch(line.strip()) + if not match or not match.group(1).startswith(PREFIX): + continue + name, labels, value = match.group(1)[len(PREFIX):], match.group(2) or "", float(match.group(3)) + le = re.search(r'le="([^"]+)"', labels) + if name.endswith("_bucket") and le: + buckets.setdefault(name[:-len("_bucket")], []).append((float(le.group(1)), value)) + else: + samples[name] = value + return {"samples": samples, "buckets": {name: sorted(items) for name, items in buckets.items()}} + + +def quantile(q: float, buckets: list) -> float | None: + """Prometheus histogram_quantile over cumulative (upper bound, count) buckets.""" + if not buckets or buckets[-1][1] == 0: + return None + rank = q * buckets[-1][1] + lower, below = 0.0, 0.0 + for upper, count in buckets: + if count >= rank: + if math.isinf(upper): + return lower + return lower + (upper - lower) * (rank - below) / (count - below) if count > below else upper + lower, below = upper, count + return lower + + +def geomean(values: list) -> float | None: + return math.exp(sum(map(math.log, values)) / len(values)) if values and all(v > 0 for v in values) else None + + +def load_cell(path: Path, spec: dict | None = None) -> dict: + """One cell's row; a planned cell that never recorded a result has no values, so no ratio uses its round.""" + recorded = path / "cell.json" + cell = json.loads(recorded.read_text()) if recorded.exists() else spec | {"exit_code": None, "missing": True} + result_path = path / "greptimedb-1" / "load-result.json" + result = json.loads(result_path.read_text()) if result_path.exists() else {} + metrics_path = path / "metrics.prom" + metrics = parse_metrics(metrics_path.read_text() if metrics_path.exists() else "") + ack = metrics["buckets"].get("append_ack_seconds", []) + load_seconds = result.get("load_seconds") + row = { + "id": cell["id"], "round": cell["round"], "concurrency": cell["concurrency"], + "config": cell["config"], "exit_code": cell["exit_code"], "wal_device": cell.get("wal_device"), + "missing": cell.get("missing", False), + "version": cell.get("version"), "load_wall_seconds": cell.get("load_wall_seconds"), + "rows": result.get("count_after"), "load_seconds": load_seconds, + "rows_per_second": result.get("rows_per_second") if cell["exit_code"] == 0 else None, + "append_ack_p50_seconds": quantile(0.5, ack), "append_ack_p99_seconds": quantile(0.99, ack), + } + # The counters register on their first event, so a scrape of an object store WAL + # that created objects omits the ones that never moved: those are zero, not unknown. + active = (cell.get("metrics_scraped") and cell.get("wal", {}).get("PROVIDER") == "experimental_object_store" + and metrics["samples"].get("created_objects_total")) + for counter in COUNTERS: + row[counter] = metrics["samples"].get(counter, 0 if active else None) + created = row["created_objects_total"] + row["creates_per_minute"] = created * 60 / load_seconds if created is not None and load_seconds else None + return row + + +def ratios(rows: list) -> list: + """D/R and E/R per round and their geometric mean over the rounds, per concurrency.""" + speed = {(r["round"], r["concurrency"], r["config"]): r["rows_per_second"] for r in rows} + result = [] + for concurrency in sorted({r["concurrency"] for r in rows}): + rounds = sorted({r["round"] for r in rows if r["concurrency"] == concurrency}) + for config in ("D", "E"): + per_round = [] + for round_number in rounds: + base, value = speed.get((round_number, concurrency, "R")), speed.get((round_number, concurrency, config)) + per_round.append(value / base if base and value else None) + complete = [value for value in per_round if value is not None] + result.append({"concurrency": concurrency, "ratio": f"{config}/R", "per_round": per_round, + "geomean": geomean(complete) if len(complete) == len(per_round) else None}) + return result + + +def fmt(value, digits=0, scale=1.0) -> str: + return "—" if value is None else f"{value * scale:,.{digits}f}" + + +def render(run: dict, rows: list, pairs: list) -> str: + lines = [f"## Object store WAL against S3 ({'smoke, MinIO' if run.get('smoke') else run.get('instance_type')})", ""] + lines += [f"GreptimeDB `{run.get('greptimedb_image')}` ({next((r['version'] for r in rows if r['version']), '—')}); " + f"{fmt(run.get('rows'))} rows in {run.get('batch_rows')}-row batches; bucket region {run.get('region')}.", ""] + lines += ["| Cell | Rows/s | Load s | Wall s | Rows | Ack p50 ms | Ack p99 ms | Creates | Creates/min " + "| Stalled | Poisoned | Create failures | Conflicts | WAL on |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |"] + for r in rows: + cell = f"{r['id']} (not run)" if r["missing"] else r["id"] if r["exit_code"] == 0 \ + else f"{r['id']} (failed, exit {r['exit_code']})" + lines.append(" | ".join([ + f"| {cell}", fmt(r["rows_per_second"]), fmt(r["load_seconds"], 1), fmt(r["load_wall_seconds"], 1), + fmt(r["rows"]), fmt(r["append_ack_p50_seconds"], 1, 1000), fmt(r["append_ack_p99_seconds"], 1, 1000), + *(fmt(r[c]) for c in COUNTERS[:1]), fmt(r["creates_per_minute"], 1), + *(fmt(r[c]) for c in COUNTERS[1:]), f"`{r['wal_device'] or '—'}` |"])) + lines += ["", "| Concurrency | Ratio | Per round | Geometric mean |", "| ---: | --- | --- | ---: |"] + for p in pairs: + lines.append(f"| {p['concurrency']} | {p['ratio']} | {', '.join(fmt(v, 3) for v in p['per_round'])} " + f"| {fmt(p['geomean'], 3)} |") + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + args = parser.parse_args() + args.root.mkdir(parents=True, exist_ok=True) + run_path = args.root / "run.json" + run = json.loads(run_path.read_text()) if run_path.exists() else {} + plan_path = args.root / "plan.json" + planned = json.loads(plan_path.read_text())["cells"] if plan_path.exists() else [] + # The plan, not what happened to run, defines the expected cells and rounds. + rows = [load_cell(args.root / "cells" / spec["id"], spec) for spec in planned] + rows += [load_cell(path) for path in sorted((args.root / "cells").glob("*")) + if path.name not in {spec["id"] for spec in planned} and (path / "cell.json").exists()] + pairs = ratios(rows) + missing = [r["id"] for r in rows if r["missing"]] + (args.root / "summary.json").write_text(json.dumps( + {"run": run, "planned_cells": len(planned), "missing_cells": missing, "cells": rows, "ratios": pairs}, + indent=2) + "\n") + print(render(run, rows, pairs) if rows else "No benchmark cell completed.\n") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/oswal-s3-bench.py b/.github/scripts/oswal-s3-bench.py new file mode 100644 index 00000000000..faad8ace674 --- /dev/null +++ b/.github/scripts/oswal-s3-bench.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# /// script +# requires-python = ">=3.10" +# dependencies = ["boto3>=1.35,<2"] +# /// + +"""Helpers of the object store WAL S3 benchmark; only `probe` and `cleanup` need boto3 (`uv run`).""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import time +import urllib.request +from pathlib import Path + +SMOKE_ROWS = 50_000 +# The o11ybench data profiles; a matching row count generates the canonical dataset. +DATA_PROFILES = {5_000: "S", 2_000_000: "D", 10_000_000: "P", 100_000_000: "M"} +# Identical [storage] for every configuration; only [wal] differs. +WAL_CONFIGS = { + "R": {"PROVIDER": "raft_engine", "SYNC_WRITE": "true", "DIR": "/wal"}, + "D": {"PROVIDER": "experimental_object_store", "ACK_MODE": "durable"}, + "E": {"PROVIDER": "experimental_object_store", "ACK_MODE": "enqueued"}, +} +ORDERS = (("R", "D", "E"), ("E", "D", "R")) +NIGHTLY = re.compile(r"v\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?-nightly-(\d{8})") +TAG = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}") +DIGEST = re.compile(r".+@sha256:[0-9a-f]{64}") + + +def append_github(variable: str, name: str, value: str) -> None: + if path := os.environ.get(variable): + with open(path, "a") as output: + output.write(f"{name}={value}\n") + + +def positive(name: str, value: str, upper: int) -> int: + if not re.fullmatch(r"[1-9][0-9]*", value) or int(value) > upper: + raise ValueError(f"{name} must be an integer in 1..{upper}, got {value!r}") + return int(value) + + +def validate_budgets(timeout_minutes: str, ttl_hours: str) -> None: + timeout, ttl = positive("timeout", timeout_minutes, 360), positive("ttl", ttl_hours, 168) + # Provisioning, teardown and queueing need two hours beyond the benchmark job. + if ttl * 60 < timeout + 120: + raise ValueError(f"janitor TTL {ttl}h must cover the {timeout}-minute timeout plus 2 hours") + + +def check_regions(bucket: str, runner: str) -> None: + if not bucket or bucket != runner: + raise ValueError(f"bucket region {bucket!r} differs from runner region {runner!r}; " + "a cross-region bucket measures the WAN, not S3") + + +def build_manifest(plan: dict, greptimedb_inspect: list, runtime_image: str, + instance_type: str, region: str, prefix: str) -> dict: + image = next((d for d in greptimedb_inspect[0].get("RepoDigests") or [] if DIGEST.fullmatch(d)), None) + if image is None: + raise ValueError("the GreptimeDB image has no sha256 repository digest") + return {"smoke": plan["smoke"], "rows": plan["rows"], "rounds": plan["rounds"], + "concurrency": plan["concurrency"], "batch_rows": plan["batch_rows"], + "greptimedb_image": image, "runtime_image": runtime_image, + "instance_type": instance_type, "region": region, "prefix": prefix} + + +def build_plan(concurrency: str, rows: str, batch_rows: str, rounds: str, smoke: bool) -> dict: + levels = [positive("concurrency", level.strip(), 256) for level in concurrency.split(",")] + if len(set(levels)) != len(levels): + raise ValueError(f"concurrency repeats a level: {concurrency!r}") + row_count = SMOKE_ROWS if smoke else positive("rows", rows, 100_000_000) + round_count = 1 if smoke else positive("rounds", rounds, 10) + profile = DATA_PROFILES.get(row_count) + cells = [] + for round_number in range(1, round_count + 1): + # Alternate the order between rounds so no configuration always runs first. + for level in levels: + for config in ORDERS[(round_number - 1) % 2]: + cells.append({ + "id": f"{round_number}-{level}-{config}", "round": round_number, + "concurrency": level, "config": config, "wal": WAL_CONFIGS[config], + }) + return { + "smoke": smoke, "rows": row_count, "rounds": round_count, + "batch_rows": positive("batch_rows", batch_rows, 100_000), "concurrency": levels, + "data_profile": profile or "custom", + "generator_args": ["--data-profile", profile] if profile else ["--line-count", str(row_count)], + "cells": cells, + } + + +def newest_nightly(tags: list[str]) -> str: + dated = [(match.group(1), tag) for tag in tags if (match := NIGHTLY.fullmatch(tag))] + if not dated: + raise ValueError("no v*-nightly-* tag found") + return max(dated)[1] + + +def resolve_tag(requested: str) -> str: + if requested != "nightly": + if not TAG.fullmatch(requested): + raise ValueError(f"invalid image tag: {requested!r}") + return requested + url = ("https://hub.docker.com/v2/repositories/greptime/greptimedb/tags" + "?page_size=100&name=nightly&ordering=last_updated") + with urllib.request.urlopen(url, timeout=30) as response: + return newest_nightly([tag["name"] for tag in json.load(response)["results"]]) + + +def s3_client(args): + import boto3 + from botocore.config import Config + + return boto3.client("s3", region_name=args.region, endpoint_url=args.endpoint or None, + config=Config(retries={"mode": "standard", "max_attempts": 5})) + + +def probe(args) -> None: + client, key = s3_client(args), f"{args.prefix}/probe/object" + deadline = time.monotonic() + 60 + while True: + try: + client.put_object(Bucket=args.bucket, Key=key, Body=b"probe") + break + except Exception: + # A fresh MinIO creates its bucket after the container starts. + if time.monotonic() > deadline: + raise + time.sleep(3) + listed = client.list_objects_v2(Bucket=args.bucket, Prefix=f"{args.prefix}/probe/") + if [item["Key"] for item in listed.get("Contents", [])] != [key]: + raise RuntimeError(f"listing {args.prefix}/probe/ did not return exactly {key}") + client.delete_object(Bucket=args.bucket, Key=key) + print(f"put, list and delete succeeded under s3://{args.bucket}/{args.prefix}/probe/") + + +def cleanup(args) -> None: + # One DeleteObject per key: DeleteObjects needs a checksum header the MinIO fixture rejects. + client, deleted = s3_client(args), 0 + for page in client.get_paginator("list_objects_v2").paginate(Bucket=args.bucket, Prefix=f"{args.prefix}/"): + for item in page.get("Contents", []): + client.delete_object(Bucket=args.bucket, Key=item["Key"]) + deleted += 1 + print(f"deleted {deleted} objects under s3://{args.bucket}/{args.prefix}/") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + plan = commands.add_parser("plan") + for name in ("concurrency", "rows", "batch-rows", "rounds"): + plan.add_argument(f"--{name}", required=True) + plan.add_argument("--smoke", choices=("true", "false"), required=True) + for name in ("timeout-minutes", "ttl-hours", "output"): + plan.add_argument(f"--{name}") + commands.add_parser("tag").add_argument("requested") + regions = commands.add_parser("regions") + for name in ("bucket", "runner"): + regions.add_argument(f"--{name}", required=True) + manifest = commands.add_parser("manifest") + for name in ("results", "runtime-image", "instance-type", "region", "prefix"): + manifest.add_argument(f"--{name}", required=True) + for name in ("probe", "cleanup"): + bucket = commands.add_parser(name) + bucket.add_argument("--endpoint", default="") + for option in ("bucket", "region", "prefix"): + bucket.add_argument(f"--{option}", required=True) + args = parser.parse_args() + if args.command == "plan": + if args.timeout_minutes or args.ttl_hours: + validate_budgets(args.timeout_minutes or "", args.ttl_hours or "") + result = build_plan(args.concurrency, args.rows, args.batch_rows, args.rounds, args.smoke == "true") + if args.output: + with open(args.output, "w") as output: + json.dump(result, output, indent=2) + print(f"{len(result['cells'])} cells, {result['rows']} rows, profile {result['data_profile']}") + elif args.command == "tag": + tag = resolve_tag(args.requested) + append_github("GITHUB_OUTPUT", "greptimedb_tag", tag) + print(f"GreptimeDB image tag: {tag}") + elif args.command == "regions": + check_regions(args.bucket, args.runner) + print(f"bucket and runner both in {args.runner}") + elif args.command == "manifest": + results = Path(args.results) + manifest = build_manifest(json.loads((results / "plan.json").read_text()), + json.loads((results / "images" / "greptimedb.json").read_text()), + args.runtime_image, args.instance_type, args.region, args.prefix) + (results / "run.json").write_text(json.dumps(manifest, indent=2) + "\n") + append_github("GITHUB_ENV", "GREPTIMEDB_IMAGE", manifest["greptimedb_image"]) + print(f"GreptimeDB image: {manifest['greptimedb_image']}") + else: + if not re.fullmatch(r"oswal-bench/[0-9]+-[0-9]+", args.prefix): + raise ValueError(f"refusing a prefix outside oswal-bench/: {args.prefix!r}") + (probe if args.command == "probe" else cleanup)(args) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ValueError as error: + print(f"::error::{error}", file=sys.stderr) + sys.exit(1) diff --git a/.github/workflows/oswal-s3-bench.yml b/.github/workflows/oswal-s3-bench.yml new file mode 100644 index 00000000000..c31dfe8f1c3 --- /dev/null +++ b/.github/workflows/oswal-s3-bench.yml @@ -0,0 +1,229 @@ +name: Object Store WAL S3 Benchmark +on: + workflow_dispatch: + inputs: + smoke: + description: 'Smoke run: ubuntu-latest and the MinIO fixture instead of EC2 and S3, 50000 rows, one round; its numbers mean nothing' + type: boolean + default: false + greptimedb_tag: + description: 'Docker Hub greptime/greptimedb tag; nightly = the newest v*-nightly-* tag (latest is a release without the object store WAL)' + type: string + default: nightly + instance_type: + description: EC2 instance type in the bucket region; the d of c6id gives a local NVMe instance store for the Raft Engine WAL + type: string + default: c6id.2xlarge + concurrency: + description: Comma-separated loader concurrency levels + type: string + default: '1,8' + rows: + description: Dataset rows (2000000 = o11ybench profile D) + type: number + default: 2000000 + batch_rows: + description: Rows per write request + type: number + default: 1000 + rounds: + description: Rounds; the configuration order alternates between rounds (R, D, E then E, D, R) + type: number + default: 2 + runtime_image: + description: 'o11ybench runtime image: auto = AWS ECR Public; explicit tag/digest overrides' + type: string + default: auto + benchmark_timeout_minutes: + description: Benchmark job timeout in minutes (1..360) + type: number + default: 240 + janitor_ttl_hours: + description: Instance lifetime before janitor eligibility (1..168 hours; at least timeout + 2h) + type: number + default: 8 +permissions: + contents: read +env: + SMOKE: ${{ inputs.smoke }} + GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} + REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} + CONCURRENCY: ${{ inputs.concurrency }} + ROWS: ${{ inputs.rows }} + BATCH_ROWS: ${{ inputs.batch_rows }} + ROUNDS: ${{ inputs.rounds }} + RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' }} + BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }} + JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }} +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + instance_type: ${{ steps.runner.outputs.instance_type }} + greptimedb_tag: ${{ steps.plan.outputs.greptimedb_tag }} + runtime_image: ${{ steps.runtime.outputs.runtime_image }} + steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - name: Test benchmark scripts + run: | + python3 greptimedb/tests/perf/test_oswal_s3_bench_scripts.py + python3 greptimedb/tests/perf/test_ci_runner_scripts.py + - name: Validate plan, budgets and image tag + id: plan + run: | + python3 greptimedb/.github/scripts/oswal-s3-bench.py plan --smoke "$SMOKE" \ + --concurrency "$CONCURRENCY" --rows "$ROWS" --batch-rows "$BATCH_ROWS" --rounds "$ROUNDS" \ + --timeout-minutes "$BENCHMARK_TIMEOUT_MINUTES" --ttl-hours "$JANITOR_TTL_HOURS" + python3 greptimedb/.github/scripts/oswal-s3-bench.py tag "$GREPTIMEDB_TAG" + - name: Resolve CI runner + id: runner + run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider AWS --instance-type "$REQUESTED_INSTANCE_TYPE" + - name: Check bucket and runner regions + env: + BUCKET_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }} + RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }} + run: python3 greptimedb/.github/scripts/oswal-s3-bench.py regions --bucket "$BUCKET_REGION" --runner "$RUNNER_REGION" + - name: Check the runtime image + id: runtime + run: bash greptimedb/.github/scripts/oswal-s3-bench-run.sh check-runtime "$RUNTIME_IMAGE" + provision: + needs: validate + if: ${{ !inputs.smoke }} + runs-on: ubuntu-latest + timeout-minutes: 45 + outputs: + label: ${{ steps.aws.outputs.label }} + instance_id: ${{ steps.aws.outputs.instance_id }} + runner_name: ${{ steps.aws.outputs.runner_name }} + steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + - uses: astral-sh/setup-uv@v6 + - name: Create ephemeral EC2 runner + id: aws + env: + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + AWS_REGION: ${{ vars.EC2_RUNNER_REGION }} + AWS_EC2_IMAGE_ID: ${{ vars.BENCHMARK_EC2_IMAGE_ID || 'auto' }} + AWS_EC2_INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }} + AWS_EC2_SUBNET_ID: ${{ vars.EC2_RUNNER_SUBNET_ID }} + AWS_EC2_SECURITY_GROUP_ID: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }} + AWS_EC2_SYSTEM_DISK_GIB: '60' + AWS_EC2_TTL_HOURS: ${{ inputs.janitor_ttl_hours }} + AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} + run: uv run greptimedb/.github/scripts/aws-ec2-runner-provision.py + benchmark: + needs: + - validate + - provision + if: ${{ !cancelled() && needs.validate.result == 'success' && (inputs.smoke || needs.provision.result == 'success') }} + runs-on: ${{ inputs.smoke && 'ubuntu-latest' || needs.provision.outputs.label }} + timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }} + env: + INSTANCE_TYPE: ${{ inputs.smoke && 'ubuntu-latest' || needs.validate.outputs.instance_type }} + GREPTIMEDB_TAG: ${{ needs.validate.outputs.greptimedb_tag }} + RUNTIME_IMAGE: ${{ needs.validate.outputs.runtime_image }} + RUN_PREFIX: oswal-bench/${{ github.run_id }}-${{ github.run_attempt }} + OWNER: o11ybench-oswal-${{ github.run_id }}-${{ github.run_attempt }} + RESULTS: ${{ github.workspace }}/oswal-results + CORPUS_DIR: ${{ github.workspace }}/oswal-corpus + PLAN: ${{ github.workspace }}/oswal-results/plan.json + WAL_ROOT: /mnt/oswal-wal + S3_BUCKET: ${{ inputs.smoke && 'greptime' || vars.AWS_CI_TEST_BUCKET }} + S3_REGION: ${{ inputs.smoke && 'us-west-2' || vars.AWS_CI_TEST_BUCKET_REGION }} + S3_ENDPOINT: ${{ inputs.smoke && 'http://127.0.0.1:9000' || '' }} + S3_CONTAINER_ENDPOINT: ${{ inputs.smoke && 'http://host.docker.internal:9000' || '' }} + steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - name: Set up benchmark dependencies + uses: ./greptimedb/.github/actions/setup-benchmark + - uses: astral-sh/setup-uv@v6 + - name: Start the MinIO fixture + if: inputs.smoke + run: | + bash greptimedb/.github/scripts/pull-benchmark-image.sh greptime/minio:2024 + docker compose -f greptimedb/tests-integration/fixtures/docker-compose.yml up -d --wait minio + - name: Pull images and write the plan + run: | + bash greptimedb/.github/scripts/oswal-s3-bench-run.sh pull + python3 greptimedb/.github/scripts/oswal-s3-bench.py plan --smoke "$SMOKE" --output "$PLAN" \ + --concurrency "$CONCURRENCY" --rows "$ROWS" --batch-rows "$BATCH_ROWS" --rounds "$ROUNDS" + python3 greptimedb/.github/scripts/oswal-s3-bench.py manifest --results "$RESULTS" \ + --runtime-image "$RUNTIME_IMAGE" --instance-type "$INSTANCE_TYPE" --region "$S3_REGION" --prefix "$RUN_PREFIX" + - name: Probe the bucket + env: + AWS_ACCESS_KEY_ID: ${{ inputs.smoke && 'superpower_ci_user' || secrets.AWS_CI_TEST_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.smoke && 'superpower_password' || secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }} + run: uv run greptimedb/.github/scripts/oswal-s3-bench.py probe --bucket "$S3_BUCKET" --region "$S3_REGION" --endpoint "$S3_ENDPOINT" --prefix "$RUN_PREFIX" + - name: Put the Raft Engine WAL on the instance store + run: bash greptimedb/.github/scripts/oswal-s3-bench-run.sh wal-disk "$WAL_ROOT" + - name: Generate the dataset once + run: bash greptimedb/.github/scripts/oswal-s3-bench-run.sh generate + - name: Load every cell + env: + AWS_ACCESS_KEY_ID: ${{ inputs.smoke && 'superpower_ci_user' || secrets.AWS_CI_TEST_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.smoke && 'superpower_password' || secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }} + run: bash greptimedb/.github/scripts/oswal-s3-bench-run.sh cells + - name: Summary + if: always() + run: python3 greptimedb/.github/scripts/oswal-s3-bench-summary.py --root "$RESULTS" >> "$GITHUB_STEP_SUMMARY" + - name: Delete the run's S3 prefix + if: always() + env: + AWS_ACCESS_KEY_ID: ${{ inputs.smoke && 'superpower_ci_user' || secrets.AWS_CI_TEST_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.smoke && 'superpower_password' || secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }} + run: uv run greptimedb/.github/scripts/oswal-s3-bench.py cleanup --bucket "$S3_BUCKET" --region "$S3_REGION" --endpoint "$S3_ENDPOINT" --prefix "$RUN_PREFIX" + - name: Check the results hold no credentials + if: always() + env: + AWS_ACCESS_KEY_ID: ${{ inputs.smoke && 'superpower_ci_user' || secrets.AWS_CI_TEST_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.smoke && 'superpower_password' || secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }} + run: bash greptimedb/.github/scripts/oswal-s3-bench-run.sh scrub + - name: Upload results (no corpus) + if: always() + uses: actions/upload-artifact@v4 + with: + name: oswal-s3-bench-${{ inputs.smoke && 'smoke' || 'ec2' }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.RESULTS }} + if-no-files-found: warn + retention-days: 14 + teardown: + needs: + - provision + - benchmark + if: ${{ always() && !inputs.smoke && (needs.provision.outputs.instance_id != '' || needs.provision.result != 'skipped') }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + - uses: astral-sh/setup-uv@v6 + - name: Delete EC2 and unregister runner + env: + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + AWS_REGION: ${{ vars.EC2_RUNNER_REGION }} + AWS_EC2_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }} + AWS_EC2_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }} + AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} + run: uv run greptimedb/.github/scripts/aws-ec2-runner-teardown.py diff --git a/Cargo.lock b/Cargo.lock index 055bf712394..b0c66cc8b5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2122,6 +2122,7 @@ dependencies = [ "tokio", "tokio-stream", "tonic 0.14.2", + "tonic-prost", ] [[package]] @@ -2311,6 +2312,7 @@ dependencies = [ "bytes", "common-test-util", "futures", + "icu_properties", "lazy_static", "pin-project", "regex", @@ -2506,7 +2508,6 @@ dependencies = [ "geohash", "h3o", "hyperloglogplus", - "icu_properties", "jsonb", "memchr", "mito-codec", @@ -3722,7 +3723,7 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "datafusion" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -3774,7 +3775,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3798,7 +3799,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3821,7 +3822,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-ipc 59.2.0", @@ -3847,7 +3848,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "futures", "log", @@ -3857,7 +3858,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "async-compression", @@ -3893,7 +3894,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-ipc 59.2.0", @@ -3917,7 +3918,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3940,7 +3941,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3963,7 +3964,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -3995,12 +3996,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" [[package]] name = "datafusion-execution" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-buffer 59.2.0", @@ -4025,7 +4026,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -4049,7 +4050,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4060,7 +4061,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-buffer 59.2.0", @@ -4091,7 +4092,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4111,7 +4112,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4122,7 +4123,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-ord 59.2.0", @@ -4146,7 +4147,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "async-trait", @@ -4161,7 +4162,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4177,7 +4178,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -4186,7 +4187,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "datafusion-doc", "quote", @@ -4196,7 +4197,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "chrono", @@ -4244,7 +4245,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -4267,7 +4268,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4281,7 +4282,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "chrono", @@ -4298,7 +4299,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4317,7 +4318,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-data 59.2.0", @@ -4354,7 +4355,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-catalog", @@ -4380,7 +4381,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4390,7 +4391,7 @@ dependencies = [ [[package]] name = "datafusion-proto-models" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "datafusion-common", "datafusion-proto-common", @@ -4400,7 +4401,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4415,7 +4416,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow-schema 59.2.0", "async-trait", @@ -4429,7 +4430,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "bigdecimal 0.4.8", @@ -4448,7 +4449,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "async-recursion", "async-trait", @@ -4623,15 +4624,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "delta-encoding" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cabe72c7de5725dcd4597e7d2bc8a4b7a4698879481607385fbc50a23203eeb9" -dependencies = [ - "num-traits", -] - [[package]] name = "der" version = "0.7.10" @@ -6962,8 +6954,6 @@ dependencies = [ "futures", "greptime-proto", "itertools 0.14.0", - "jieba-rs", - "lazy_static", "mockall", "pin-project", "prost 0.14.1", @@ -8118,7 +8108,6 @@ dependencies = [ "common-wal", "crc32fast", "dashmap", - "delta-encoding", "derive_builder 0.20.2", "futures", "futures-util", @@ -8137,6 +8126,7 @@ dependencies = [ "snafu 0.8.6", "store-api", "tokio", + "tokio-util", "uuid", ] @@ -8702,6 +8692,7 @@ dependencies = [ "api", "aquamarine", "arrow-schema 59.2.0", + "arrow-select 59.2.0", "async-channel 1.9.0", "async-stream", "async-trait", @@ -9686,9 +9677,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendal" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fe43e16d96bed57937eb7c4a28559bf4c5fc4a3951656505d1fdd4f856349b4" +checksum = "3cbd193aec6d62efaff87611dfae52c132aed650bb3cb0c684c0a9394f09a42a" dependencies = [ "ctor", "opendal-core", @@ -9711,9 +9702,9 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de4566a412776e3f65d53dd9fceced5b432a0a9c9a1e8d50ccf58823795e5c5b" +checksum = "54b5555d753dd6a6383dc0891254b72f3cdc40c011ba0372b560e0ae5aa29f5c" dependencies = [ "anyhow", "asyncband", @@ -9737,9 +9728,9 @@ dependencies = [ [[package]] name = "opendal-http-transport-reqwest" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f772ce6c137ab43647726116d505a649a88d41928fcd813438eb30b02312bcbb" +checksum = "4d6588678f3040a3eab1819364dcbc6a746a8286d1ceb23a70a6614faa01dc6c" dependencies = [ "bytes", "futures", @@ -9751,9 +9742,9 @@ dependencies = [ [[package]] name = "opendal-layer-concurrent-limit" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3734c648617e5ad724d49175dde8323550624f9614f50e9e81b7aafae30b1734" +checksum = "db8574309e68bf6d49c5378eaef9d0b3f6786971650b061c69e808254d71cb05" dependencies = [ "asyncband", "futures", @@ -9763,9 +9754,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a8daab5a91eac76f94516e7c71e677195c9221840ce8f5e71815e3fc8207742" +checksum = "4ad43246d2cb7b7ad32b10e22bfa7b6ced8a6677cb6d6f005ae7e1d94cb19c6a" dependencies = [ "log", "opendal-core", @@ -9773,9 +9764,9 @@ dependencies = [ [[package]] name = "opendal-layer-observe-metrics-common" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fbe664be117c293cfa3980a0d2fc238bbb38eae5f6c191f376e88ccb830d26b" +checksum = "3b8bc97d5a00eb41d50f1f6077c4bd5396c1eefea495614084ee9a37202ae06d" dependencies = [ "futures", "http 1.5.0", @@ -9784,9 +9775,9 @@ dependencies = [ [[package]] name = "opendal-layer-prometheus" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32078d8248555ee6271332ec480b78306a44f0d902ea4721ccac00d9b866a21" +checksum = "404e766cc2e4e4c9225ce08605ae151f768a05a41a448a235e016b1aea9ff98f" dependencies = [ "opendal-core", "opendal-layer-observe-metrics-common", @@ -9795,9 +9786,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb8e22cb1f86ef9af771a8947d3ccf380c8f5b76236f628314dd41c9ba73331a" +checksum = "4aeb54be4940c68609483591934a813f7bdd97d747826783a393914d3139e363" dependencies = [ "backon", "log", @@ -9806,9 +9797,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ac6c77fe0d1b18e0064a02c19b98dbebcab78bf5e7753ad1f3abbc18850ff4" +checksum = "1d11d53e67473e8914eacb2e14d586779ae57e1a84759ad25232ef339db77d8a" dependencies = [ "opendal-core", "tokio", @@ -9816,9 +9807,9 @@ dependencies = [ [[package]] name = "opendal-layer-tracing" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "603807927a799ea981f62a3b0012e17181e39f468793b78ca8bedda66d19813c" +checksum = "bbe2ebfc795bf08da77d20bed13d149b478cc8e5002df44d237e460134f00be4" dependencies = [ "futures", "http 1.5.0", @@ -9828,9 +9819,9 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6cf31b02e7b44191d97c07254eea109b5da9c17210a29e478c8d0dc26dffb1f" +checksum = "c32c3fc2e507c854ce53615b28fc2208f5e00cbe456c4d344ffdfbf4c9cb976d" dependencies = [ "base64 0.23.1", "bytes", @@ -9849,9 +9840,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8107324c3f3cb9970fe20526c2c341b7c5769acbd4699bab73bd525f5fd63d62" +checksum = "cc93c30bd347a36d93fd46d47f3e2bb6494dcf9458938aa501fe046ff8f7562f" dependencies = [ "http 1.5.0", "opendal-core", @@ -9859,9 +9850,9 @@ dependencies = [ [[package]] name = "opendal-service-fs" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0e7d8974aebd33be899200c9d0d50b2141fd5fa5c462b29e20653e3ddbfe339" +checksum = "ec5c16fe91b77c0364f66546a1c32b2999e9c3c3c75c2e908a282268e0890bac" dependencies = [ "bytes", "log", @@ -9873,9 +9864,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d80eca6f9227bdf65d43e5b8d5fe75dca8cfab1da07571442a0420400847bea" +checksum = "e1b5e42c6606ea1e37fa9cdedf07dc998073a45eb5044317c0fa6f3153e98d6e" dependencies = [ "async-trait", "bytes", @@ -9895,9 +9886,9 @@ dependencies = [ [[package]] name = "opendal-service-hdfs-native" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b84340e11a3bcd7f1f0b55df032d040b2907483c546c3021485aba749c3072c" +checksum = "20fb97b2ac06a87e6720f0f93d5107ac046117deb8ac957030b0962ba4ecc3c5" dependencies = [ "bytes", "futures", @@ -9909,9 +9900,9 @@ dependencies = [ [[package]] name = "opendal-service-http" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac1553d6b57ca79e200e5c7ded9ab708f2016dadb28ede0f2da71e02fd0b960" +checksum = "65fdde60d1abf14e47eaf19360c08ea1b89c14f03769db448e016781c2e0e32b" dependencies = [ "http 1.5.0", "log", @@ -9921,9 +9912,9 @@ dependencies = [ [[package]] name = "opendal-service-mysql" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e15592be32b105cd152ab398b3db847e22c8c5f230d9b34ff095ea3d0dc6fd97" +checksum = "9ba30e2a0b38aa55579b69bf57f607df7aa613f4d964496407b7998f2dbaefcc" dependencies = [ "asyncband", "opendal-core", @@ -9933,9 +9924,9 @@ dependencies = [ [[package]] name = "opendal-service-oss" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ebfa355b1d4edb0e5b2a5b86d8de9f811fae1d7baca6d65246b230b2afc0bd" +checksum = "d64d2909b444008e42f00137a50df6b55ba599eeb6c8b46c71a2a84948f91880" dependencies = [ "bytes", "http 1.5.0", @@ -9950,9 +9941,9 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2942f2d8d3d4953c0e8d07cd1244879628d084948e48ce6b6d808c5938f7858d" +checksum = "1d83e716c25a81fe2ca701894b8e14b9966acca3812af250e50132accb19da02" dependencies = [ "base64 0.23.1", "bytes", @@ -10093,7 +10084,9 @@ dependencies = [ "async-stream", "async-trait", "axum 0.8.4", + "backon", "bytes", + "cache", "catalog", "chrono", "client", @@ -10124,12 +10117,14 @@ dependencies = [ "futures", "futures-util", "humantime", + "humantime-serde", "itertools 0.14.0", "jsonb", "lazy_static", "meta-client", "meter-core", "meter-macros", + "mito-codec", "moka", "object-store", "partition", @@ -10150,6 +10145,7 @@ dependencies = [ "substrait 1.4.0-alpha.0", "table", "tokio", + "tokio-stream", "tokio-util", "tonic 0.14.2", "tracing", @@ -15483,6 +15479,7 @@ dependencies = [ "futures-core", "futures-io", "futures-sink", + "futures-util", "pin-project-lite", "slab", "tokio", diff --git a/Cargo.toml b/Cargo.toml index cce9b502928..702498ee5e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,7 @@ arrow-buffer = "=59.2.0" arrow-flight = "=59.2.0" arrow-ipc = { version = "=59.2.0", default-features = false, features = ["lz4", "zstd"] } arrow-schema = { version = "=59.2.0", features = ["serde"] } +arrow-select = "=59.2.0" async-stream = "0.3" async-trait = "0.1" # Remember to update axum-extra, axum-macros when updating axum @@ -271,6 +272,7 @@ url = "2.3" # Pin below 0.14.5 to avoid the `connection_timeout_future` "async fn resumed after # completion" panic (greptimedb#8265 / grpc-rust#2522). Remove once tonic ships a fix. tonic = { version = "0.14, <0.14.5", features = ["tls-aws-lc", "gzip", "zstd"] } +tonic-prost = "0.14" tower = "0.5" tracing = "0.1" tracing-appender = "0.2" @@ -363,21 +365,21 @@ rev = "f743c3a5fe8c1f57363eac5866c356b7d91f9b39" [patch.crates-io] substrait = { git = "https://github.com/GreptimeTeam/substrait-rs.git", rev = "91ec978b0649417ad3da8390e7a515baec723b1b" } -datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } datafusion-pg-catalog = { git = "https://github.com/datafusion-contrib/datafusion-postgres.git", rev = "ffb14a52b3d7c7489559812e1ec38096f225ce93" } arrow-pg = { git = "https://github.com/datafusion-contrib/datafusion-postgres.git", rev = "ffb14a52b3d7c7489559812e1ec38096f225ce93" } sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "9e9019bb1c7040ed956f654e39378dd42ab17884" } diff --git a/config/config.md b/config/config.md index 612a5cd43c0..6faae09c347 100644 --- a/config/config.md +++ b/config/config.md @@ -15,7 +15,6 @@ | `default_timezone` | String | Unset | The default timezone of the server. | | `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, and native histogram columns.
Legacy OTLP summary columns keep their historical `greptime_` prefix. | | `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.
When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. | -| `experimental_metric_export` | Bool | `false` | Enables experimental Parquet COPY DATABASE using shared Metric physical scans.
Resume requires the previous export and storage writes to have ended; HTTP timeout is not confirmation. | | `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
Password verifier formats: "plain:", "pbkdf2_sha256:::",
"mysql_native_password:",
"pg_scram_sha256::::"
"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:
`static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.
"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are
observable in the SCRAM server-first message, and non-default values weaken resistance to username
enumeration. | | `max_in_flight_write_bytes` | String | 0 | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
Set to 0 to disable the limit. Default: "0" (unlimited) | | `write_bytes_exhausted_policy` | String | `wait` | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" | @@ -127,7 +126,7 @@ | `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.
**It's only used when the provider is `kafka`**.

This option ensures that when Kafka messages are deleted, the system
can still successfully replay memtable data without throwing an
out-of-range error.
However, enabling this option might lead to unexpected data loss,
as the system will skip over missing entries instead of treating
them as critical errors. | | `wal.storage_provider` | String | `""` | The name of the storage provider that holds the WAL objects, an empty name selects the default object store.
**It's only used when the provider is `experimental_object_store`**. | | `wal.prefix` | String | `wal` | The path prefix of the WAL objects inside the storage provider.
The objects are written under `/datanodes//epochs/`, which is derived from this prefix.
**It's only used when the provider is `experimental_object_store`**. | -| `wal.flush_interval` | String | `100ms` | The interval of flushing buffered entries to the object store, at least `10ms`, defaults to `100ms`.
Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so under sustained load the batch seals on size and the interval no longer matters.
When writes are sparse, a shorter interval lowers the acknowledgement latency of `durable` appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node, and in `enqueued` mode the backlog thresholds can seal earlier. An `enqueued` append does not wait for its batch to seal once it is admitted, but the backlog thresholds can delay admission.
**It's only used when the provider is `experimental_object_store`**. | +| `wal.flush_interval` | String | `100ms` | The interval of flushing buffered entries to the object store, at least `10ms`, defaults to `100ms`.
Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so the batch seals on size only when the admitted bytes reach `max_batch_bytes` before the next timer tick; otherwise it seals on the timer, whatever the load.
When writes are sparse, a shorter interval lowers the acknowledgement latency of `durable` appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node, and in `enqueued` mode the backlog thresholds can seal earlier. An `enqueued` append does not wait for its batch to seal once it is admitted, but the backlog thresholds can delay admission.
**It's only used when the provider is `experimental_object_store`**. | | `wal.ack_mode` | String | `durable` | When an append to the object store WAL returns.
- `durable`: an append returns after the object holding its entries is durable (the default).
- `enqueued`: an append returns once its entries are admitted and their ids are assigned, and the object is created in the background. A crash loses the unpersisted backlog.
**It's only used when the provider is `experimental_object_store`**. | | `wal.max_unpersisted_bytes` | String | `64MB` | The size of the unpersisted backlog at which new appends stall until an upload completes, in `enqueued` mode.
Nothing is dropped and nothing is rejected; the threshold does not bound what a crash during an outage can lose.
**It's only used when the provider is `experimental_object_store` and `ack_mode` is `enqueued`**. | | `wal.max_unpersisted_age` | String | `8s` | The age of the oldest unpersisted entry at which new appends stall until an upload completes, in `enqueued` mode.
**It's only used when the provider is `experimental_object_store` and `ack_mode` is `enqueued`**. | @@ -285,7 +284,6 @@ | `default_timezone` | String | Unset | The default timezone of the server. | | `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, and native histogram columns.
Legacy OTLP summary columns keep their historical `greptime_` prefix. | | `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.
When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. | -| `experimental_metric_export` | Bool | `false` | Enables experimental Parquet COPY DATABASE using shared Metric physical scans.
Resume requires the previous export and storage writes to have ended; HTTP timeout is not confirmation. | | `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
Password verifier formats: "plain:", "pbkdf2_sha256:::",
"mysql_native_password:",
"pg_scram_sha256::::"
"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:
`static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.
"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are
observable in the SCRAM server-first message, and non-default values weaken resistance to username
enumeration. | | `max_in_flight_write_bytes` | String | 0 | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
Set to 0 to disable the limit. Default: "0" (unlimited) | | `write_bytes_exhausted_policy` | String | `wait` | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" | @@ -393,6 +391,11 @@ | `query.experimental_spill_path` | String | Unset | Spill directory path. Ignored unless mode is "custom". | | `query.experimental_spill_max_temp_directory_size` | String | `1GiB` | Maximum total size of spill directory (default: "1GiB").
Ignored unless mode is "custom". | | `query.experimental_spill_compression` | String | `uncompressed` | Compression for spilled data files: "uncompressed" (default), "lz4_frame", "zstd".
Ignored unless mode is "custom". | +| `write_retry` | -- | -- | Retry retryable distributed write failures.
Backoff grows exponentially, with jitter to spread concurrent retries. | +| `write_retry.max_retries` | Integer | `3` | Maximum retries after the initial write attempt. Set to 0 to disable retries. | +| `write_retry.initial_backoff` | String | `1500ms` | Initial backoff before jitter is applied. Must be positive. | +| `write_retry.max_backoff` | String | `10s` | Maximum backoff before jitter is applied. Must be at least initial_backoff. | +| `write_retry.backoff_multiplier` | Float | `2.0` | Backoff multiplier. Must be finite and at least 1. | | `datanode` | -- | -- | Datanode options. | | `datanode.client` | -- | -- | Datanode client options. | | `datanode.client.connect_timeout` | String | 1s | Connect server timeout. Defaults to `1s`; this example sets it to `10s`. | @@ -601,12 +604,10 @@ | `wal.timeout` | String | `5s` | The total request timeout for kafka client.
**It's only used when the provider is `kafka`**. | | `wal.max_batch_bytes` | String | `1MB` | The max size of a single producer batch.
Warning: Kafka has a default limit of 1MB per message in a topic.
Defaults to `8MB` when the provider is `experimental_object_store`.
**It's only used when the provider is `kafka` or `experimental_object_store`**. | | `wal.consumer_wait_timeout` | String | `100ms` | The consumer wait timeout.
**It's only used when the provider is `kafka`**. | -| `wal.create_index` | Bool | `false` | Whether to enable WAL index creation.
**It's only used when the provider is `kafka`**. | -| `wal.dump_index_interval` | String | `60s` | The interval for dumping WAL indexes.
**It's only used when the provider is `kafka`**. | | `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.
**It's only used when the provider is `kafka`**.

This option ensures that when Kafka messages are deleted, the system
can still successfully replay memtable data without throwing an
out-of-range error.
However, enabling this option might lead to unexpected data loss,
as the system will skip over missing entries instead of treating
them as critical errors. | | `wal.storage_provider` | String | `""` | The name of the storage provider that holds the WAL objects, an empty name selects the default object store.
**It's only used when the provider is `experimental_object_store`**. | | `wal.prefix` | String | `wal` | The path prefix of the WAL objects inside the storage provider.
The objects are written under `/datanodes//epochs/`, which is derived from this prefix.
**It's only used when the provider is `experimental_object_store`**. | -| `wal.flush_interval` | String | `100ms` | The interval of flushing buffered entries to the object store, at least `10ms`, defaults to `100ms`.
Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so under sustained load the batch seals on size and the interval no longer matters.
When writes are sparse, a shorter interval lowers the acknowledgement latency of `durable` appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node, and in `enqueued` mode the backlog thresholds can seal earlier. An `enqueued` append does not wait for its batch to seal once it is admitted, but the backlog thresholds can delay admission.
**It's only used when the provider is `experimental_object_store`**. | +| `wal.flush_interval` | String | `100ms` | The interval of flushing buffered entries to the object store, at least `10ms`, defaults to `100ms`.
Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so the batch seals on size only when the admitted bytes reach `max_batch_bytes` before the next timer tick; otherwise it seals on the timer, whatever the load.
When writes are sparse, a shorter interval lowers the acknowledgement latency of `durable` appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node, and in `enqueued` mode the backlog thresholds can seal earlier. An `enqueued` append does not wait for its batch to seal once it is admitted, but the backlog thresholds can delay admission.
**It's only used when the provider is `experimental_object_store`**. | | `wal.ack_mode` | String | `durable` | When an append to the object store WAL returns.
- `durable`: an append returns after the object holding its entries is durable (the default).
- `enqueued`: an append returns once its entries are admitted and their ids are assigned, and the object is created in the background. A crash loses the unpersisted backlog.
**It's only used when the provider is `experimental_object_store`**. | | `wal.max_unpersisted_bytes` | String | `64MB` | The size of the unpersisted backlog at which new appends stall until an upload completes, in `enqueued` mode.
Nothing is dropped and nothing is rejected; the threshold does not bound what a crash during an outage can lose.
**It's only used when the provider is `experimental_object_store` and `ack_mode` is `enqueued`**. | | `wal.max_unpersisted_age` | String | `8s` | The age of the oldest unpersisted entry at which new appends stall until an upload completes, in `enqueued` mode.
**It's only used when the provider is `experimental_object_store` and `ack_mode` is `enqueued`**. | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 413afb609c6..2d72f795189 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -211,14 +211,6 @@ max_batch_bytes = "1MB" ## **It's only used when the provider is `kafka`**. consumer_wait_timeout = "100ms" -## Whether to enable WAL index creation. -## **It's only used when the provider is `kafka`**. -create_index = false - -## The interval for dumping WAL indexes. -## **It's only used when the provider is `kafka`**. -dump_index_interval = "60s" - ## Ignore missing entries during read WAL. ## **It's only used when the provider is `kafka`**. ## @@ -240,7 +232,7 @@ overwrite_entry_start_id = false #+ prefix = "wal" ## The interval of flushing buffered entries to the object store, at least `10ms`, defaults to `100ms`. -## Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so under sustained load the batch seals on size and the interval no longer matters. +## Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so the batch seals on size only when the admitted bytes reach `max_batch_bytes` before the next timer tick; otherwise it seals on the timer, whatever the load. ## When writes are sparse, a shorter interval lowers the acknowledgement latency of `durable` appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node, and in `enqueued` mode the backlog thresholds can seal earlier. An `enqueued` append does not wait for its batch to seal once it is admitted, but the backlog thresholds can delay admission. ## **It's only used when the provider is `experimental_object_store`**. #+ flush_interval = "100ms" diff --git a/config/frontend.example.toml b/config/frontend.example.toml index e4ef6e36935..c42d254671c 100644 --- a/config/frontend.example.toml +++ b/config/frontend.example.toml @@ -11,10 +11,6 @@ default_column_prefix = "greptime" ## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. #+ auto_create_table = true -## Enables experimental Parquet COPY DATABASE using shared Metric physical scans. -## Resume requires the previous export and storage writes to have ended; HTTP timeout is not confirmation. -#+ experimental_metric_export = false - ## The user provider for authentication. ## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" ## Password verifier formats: "plain:", "pbkdf2_sha256:::", @@ -402,6 +398,18 @@ memory_pool_size = "50%" #+ experimental_spill_compression = "uncompressed" # --- End spill-to-disk --- +## Retry retryable distributed write failures. +## Backoff grows exponentially, with jitter to spread concurrent retries. +[write_retry] +## Maximum retries after the initial write attempt. Set to 0 to disable retries. +max_retries = 3 +## Initial backoff before jitter is applied. Must be positive. +initial_backoff = "1500ms" +## Maximum backoff before jitter is applied. Must be at least initial_backoff. +max_backoff = "10s" +## Backoff multiplier. Must be finite and at least 1. +backoff_multiplier = 2.0 + ## Datanode options. [datanode] ## Datanode client options. diff --git a/config/standalone.example.toml b/config/standalone.example.toml index c656049c5d2..82fb3d2c489 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -11,10 +11,6 @@ default_column_prefix = "greptime" ## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. #+ auto_create_table = true -## Enables experimental Parquet COPY DATABASE using shared Metric physical scans. -## Resume requires the previous export and storage writes to have ended; HTTP timeout is not confirmation. -#+ experimental_metric_export = false - ## The user provider for authentication. ## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" ## Password verifier formats: "plain:", "pbkdf2_sha256:::", @@ -427,7 +423,7 @@ overwrite_entry_start_id = false #+ prefix = "wal" ## The interval of flushing buffered entries to the object store, at least `10ms`, defaults to `100ms`. -## Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so under sustained load the batch seals on size and the interval no longer matters. +## Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so the batch seals on size only when the admitted bytes reach `max_batch_bytes` before the next timer tick; otherwise it seals on the timer, whatever the load. ## When writes are sparse, a shorter interval lowers the acknowledgement latency of `durable` appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node, and in `enqueued` mode the backlog thresholds can seal earlier. An `enqueued` append does not wait for its batch to seal once it is admitted, but the backlog thresholds can delay admission. ## **It's only used when the provider is `experimental_object_store`**. #+ flush_interval = "100ms" diff --git a/docs/rfcs/2026-05-28-table-semantic-layer.md b/docs/rfcs/2026-05-28-table-semantic-layer.md index cb5f6e84dc4..79764ad80d2 100644 --- a/docs/rfcs/2026-05-28-table-semantic-layer.md +++ b/docs/rfcs/2026-05-28-table-semantic-layer.md @@ -70,6 +70,13 @@ concern, not query-time semantics). | `greptime.semantic.source_version` | Source protocol version, e.g. Prometheus remote write `1.0` / `2.0` | | `greptime.semantic.pipeline` | `greptime_trace_v1` (the signal-agnostic successor to `table_data_model`) | +**Information-schema tables** carry `greptime.semantic.table_scope`: `catalog` +means the provider filters every query to its construction catalog; `global` +means catalog-independent or shared data, including compatibility stubs. The +scope is declared at registration and appears as `table_scope` in `semantic_options`. +Enterprise authorization trusts only registered system-table metadata. Setting +this option on a user table does not change its permissions. + **Trace**: `greptime.semantic.trace.conventions` (the OTel `schema_url` the rows conform to, or `mixed` / `unknown` when not single-valued). **Metric** — v1 assumes one metric type per table, which is how both Prom RW and the post-v0.16 OTel ingestion path land data today; mixed-type tables are a follow-up. These are stamped for OTLP (which declares them on the wire and then discards them); Prometheus carries its type/unit in the name and gets identity only. diff --git a/docs/rfcs/2026-09-06-object-store-wal.md b/docs/rfcs/2026-09-06-object-store-wal.md new file mode 100644 index 00000000000..b49f97c330c --- /dev/null +++ b/docs/rfcs/2026-09-06-object-store-wal.md @@ -0,0 +1,357 @@ +--- +Feature Name: Object Store WAL +Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/9197 +Date: 2026-09-06 +Author: jeremyhi +--- + +# Summary + +Add an object store backed write-ahead log as a first-class WAL provider, alongside Raft Engine and Kafka. A datanode batches the entries of all its regions into immutable objects under one prefix, creates each object conditionally under a monotonically increasing sequence number, and by default acknowledges a write only after the object holding it is durable; an opt-in acknowledgement mode returns on admission with a stated loss bound. Recovery lists the objects, selects the chain of objects that ends at the latest complete one, rebuilds an in-memory catalog from their footers, and replays each region from its own segments. The provider is standalone-only and marked experimental in this RFC; the persisted metadata and object format are designed so that lifting either restriction later does not require a migration. + +A reference implementation of everything this document describes as implemented exists and was validated as reported under *Evidence*; it is submitted in a series of pull requests that follow this RFC, each covering one part of the design in the order the *Details* sections give. The document keeps two things apart: what that implementation does, and what is proposed on top of it. Every passage that describes proposed behaviour is introduced with the word *Proposed*; everything else describes the reference implementation. The *Future work* and *Unresolved questions* sections are proposed by definition and carry no marker. + +# Motivation + +GreptimeDB already stores data files in object storage. The WAL is the last component that still needs either local disk with durability guarantees (Raft Engine) or an external cluster (Kafka). Both are awkward for a cloud-native standalone deployment: + +- Raft Engine ties the node to a persistent volume. Losing the volume loses unflushed writes; moving the node means moving the volume. +- Kafka is an operational dependency of its own, with topics, retention and a broker fleet to size, and it is disproportionate for a single node. + +Object storage is already provisioned, already durable across zones, and already the place the rest of the data lives. An object store WAL takes the WAL out of the local state that must survive a restart: unflushed writes are no longer tied to a volume, and a node restarted against the same bucket and prefix replays them. It does not yet make a standalone node independent of local disk altogether, because standalone keeps its metadata in a local key-value store under the data home regardless of the WAL provider; moving that metadata off local disk is a separate change. The cost is write latency under the default durable acknowledgement: a write waits for its batch to seal, for every earlier object to be durable, and for one PUT. That is acceptable for the ingestion patterns GreptimeDB targets when writes are batched; *Write path measurements* under *Evidence* shows where the cost lies. + +# Goals + +- A WAL provider that needs nothing but an object store and a prefix. +- Exactly-once replay of acknowledged writes across graceful and ungraceful restarts under the default acknowledgement mode. The `enqueued` mode is an explicit choice whose loss is bounded to the unpersisted backlog and reported. +- Region isolation: many regions share objects, but a region only ever replays its own entries. +- Recovery that never silently loses data. Structural corruption fails recovery. A corrupted segment is skipped at segment granularity by default, and the skip is recorded for its region, counted and logged rather than hidden; `on_corrupted_segment = "fail"` fails the read instead. +- Default behaviour of the existing providers unchanged; the new provider is opt-in through configuration. + +# Non-goals + +- Distributed mode. The provider is rejected when a datanode is controlled by a metasrv; the metasrv side has no allocation logic for it yet. +- Persistence of per-region obsolete watermarks. Mito re-establishes every watermark when it opens a region, which is what garbage collection reasons from; persisting them is a follow-up. +- Matching Raft Engine's write latency. The provider trades latency for the absence of local durable WAL state. +- Making standalone metadata independent of local disk. The metadata store is unchanged by this RFC. + +# Details + +## Identity + +The provider is added to the four parallel WAL enums so that every resolution path handles it explicitly: + +| Enum | Variant | Notes | +| --- | --- | --- | +| `DatanodeWalConfig` | `ObjectStore(ObjectStoreWalConfig)` | serde tag `experimental_object_store`; fields `storage_provider`, `prefix`, `flush_interval`, `max_batch_bytes`, `ack_mode`, `max_unpersisted_bytes`, `max_unpersisted_age`, `on_corrupted_segment` | +| `WalOptions` | `ObjectStore(ObjectStoreWalOptions { prefix })` | persisted per region with the stable tag `object_store`; flat key `wal.object_store.prefix`; the value is the derived node prefix | +| `WalProvider` | `ObjectStore { prefix }` | allocates the per-region options in standalone under the derived node prefix; `start` is a no-op | +| `Provider` | `ObjectStore(Arc)` | `is_remote_wal()` is true | + +Two choices are deliberate: + +- The persisted `WalOptions` carry the derived node prefix `/datanodes//epochs/` described under *Object format*, not the configured root. On reopen the log store compares the persisted prefix with the one it runs under and fails fast on a mismatch, so an operator who changes the prefix or the node id cannot silently point a region at an empty WAL. Changing the prefix of an existing region requires an explicit migration, which is out of scope here. +- `Provider::ObjectStore` is classified as a remote WAL. Mito's remote-WAL branches (initial flushed entry id from `latest_entry_id`, `topic_latest_entry_id` after replay, region-scoped reads, no topic grouping on batch open) apply unchanged. This is a classification of replay semantics, not of physical location. + +A datanode that is configured with a metasrv client rejects the provider before any store is built: `DatanodeBuilder` returns an error stating that the object store WAL is standalone-only. Separately, `DatanodeWalConfig::ObjectStore` cannot be converted into `MetasrvWalConfig`, so a metasrv cannot be configured with it either. The standalone bootstrap matches the datanode WAL configuration before that conversion and constructs `WalProvider::ObjectStore` directly. + +## Object format + +Every object holds one sealed batch: + +```text +header | segment(region 1) | ... | segment(region N) | footer | trailer +``` + +- The header carries the magic `GTWALOBJ`, format version 1, the object sequence number, the epoch of the store instance that wrote the object, a link to the object this one extends, as that object's sequence number and epoch or none for the first object of a chain, the reclaim boundary described under *Entry ids*, and a CRC32 of the preceding header bytes; the header is 54 bytes. A reclaim boundary above the object's own sequence fails the header check. A sequence number and an epoch together identify an object: an instance never writes two objects under one sequence, and instances have distinct epochs, as described under *Log store*. Recovery selects the history it replays by the links and the epochs, as described under *Log store*. +- An object may hold no segment. A store writes such an object when it opens, as described under *Log store*; it is a chain object like any other, and its segments are trivially all flushed. +- Each segment holds the entries of exactly one region, ordered by entry id; segments are ordered by region id. Ordering makes the encoding deterministic, which the identical-retry rule below depends on. +- The footer records, per segment, the region id, the entry id range, the entry count, the byte range and a CRC32 of the segment. +- The fixed-size trailer carries the magic `GTWALTRL`, the footer location, a CRC32 of the footer and a CRC32 of the whole object. A reader locates the footer by reading the trailer at the end of the object and can decode a single region's segment without touching the others. + +Corruption is handled at two levels; nothing else is checked in production. Structural damage fails recovery: recovery reads only the header, the trailer and the footer of every object, as described under *Log store*, and verifies the header and trailer magic, the format version, the header CRC, the header sequence against the key, the footer CRC and that the segment byte ranges tile the object body exactly; the first object that fails any of these checks fails construction of the store, since nothing can be trusted about such an object. A segment is verified by the read that decodes it, and a segment that does not decode, because its CRC does not match or its content disagrees with its footer entry, is a data-level failure confined to one region. The read fetches the segment once more, since the download rather than the object may be damaged, and decodes it again; if it still does not decode, the behaviour follows `on_corrupted_segment`. `skip`, the default, skips the segment, records a WAL hole for the region with the object key, the object sequence and the entry id range the footer holds for the segment, increments the counter `greptime_logstore_object_store_wal_skipped_segments_total` and logs a warning with those fields, and the read continues with the next object; every other region in the object is unaffected. The holes are kept in memory per region and can be listed from the store; nothing is persisted, so a store opened later on the same prefix records a hole again when a read meets the segment. `fail` fails the read with the decode error, so a region whose replay meets the segment does not open. Skipping loses only the unflushed entries of that region in that object: nothing already flushed is affected, and the loss is reported rather than hidden. A failed fetch is not corruption and fails the read in both modes, unless garbage collection deleted the object, as described under *Entry ids*, in which case the object is skipped; an object the catalog still holds once a delete of it has settled fails the read whether it is missing or unreadable, and a segment of such an object that does not decode follows `on_corrupted_segment` as it would without any collection. The whole-object CRC is not a recovery requirement; it stays in the trailer as a fixture-level check. Because recovery never checks it, the header carries its own CRC, so a damaged link fails recovery instead of silently cutting acknowledged objects off the chain. The format is frozen at version 1. + +The operator configures only the prefix root. The store runs under a node prefix derived from it, `/datanodes//epochs/`, and object keys are `/objects/.wal`, where the sequence is zero-padded to 20 decimal digits. Standalone uses its datanode id, zero unless configured, and generation zero, so with prefix `wal` the first object is `wal/datanodes/0/epochs/0/objects/00000000000000000000.wal`. Every place that names the prefix, whether to open the store or to allocate the WAL options of a region, derives it from the same function, so the prefix a region persists matches the one its store runs under. Listing accepts only well-formed keys under `/objects/` and returns them in sequence order. This is the standalone form of the key layout under *Alignment with cluster mode*; the cluster assignment of node id and generation is proposed there. + +## Catalog + +The catalog is an in-memory index rebuilt at recovery from the objects on the replayed chain: objects by sequence number, and per region the objects that hold its entries with their entry id ranges. Its invariants are enforced on insertion: a region's entry id ranges are strictly increasing along the object sequence, an object holds at most one segment per region, and an already indexed sequence number is rejected, whether or not the footer matches; an object without segments is indexed like any other and takes its sequence. The next sequence number is the larger of the largest indexed one plus one and the floor that any region's largest entry id sets, as described under *Entry ids*, checked against the sequence limit; construction raises it above every listed object, as described under *Log store*, and gaps in the sequence are tolerated. + +## Conditional create + +Objects are created with an if-not-exists put. Three outcomes matter: + +| Outcome | Meaning | +| --- | --- | +| created | the batch is durable | +| exists with identical content | a retry of a batch that was durable before the client observed it; accepted | +| exists with different content | another object owns this sequence; its epoch decides, as described under *Log store* | + +Deterministic encoding is what makes the middle row decidable. + +## Entry ids + +Entry ids are object-sequence-major. The high bits are the sequence number of the object that holds the entry, the low bits are the entry's position among that region's entries inside the object: + +```text +entry_id = (object_seq << 20) | position (1 <= position < 2^20, object_seq < 2^44) +``` + +Positions start at one, so entry id zero is never assigned: Mito treats zero as the watermark of a region that has no durable entry and replays from the next id, and the first object has sequence zero. Ids are assigned at admission, from the sequence the open batch takes when it is sealed. A store never hands out a sequence twice, so a batch that fails is never written again under its sequence, and a retry of its entries is a new batch with new ids. A batch that would take a region past the position range is sealed first and the entries open the next batch; an append that alone exceeds the range is rejected. This keeps two properties at once. `entry_id >> 20` names the object that holds an entry without consulting any index, which is what a future cross-node takeover needs to describe a WAL position as a range of object sequences. And every entry has a unique, strictly increasing id within its region, so a flush that lands between two appends of the same region that ended up in the same object records exactly which entries it covered; replay from `flushed_entry_id` can never skip an unflushed entry or repeat a flushed one. A region's ids have gaps wherever other regions or other positions took the sequence, exactly as Kafka offsets do, and Mito already tolerates gaps; the recovered history of a region may also lack whole objects where a batch was never acknowledged, as described under *Writing*. The id values are encoded in two places that must agree: in each record inside its segment and in the footer's entry id range for that segment. + +`latest_entry_id(provider)` is region-scoped: the highest entry id durable under the prefix for the provider's region, or zero when the region has none. A freshly created region therefore starts at zero even if other regions under the prefix have entries. This differs from Kafka, whose high watermark is per topic; it is sufficient here because ids are per region, and Mito consumes the value only for the region that asked. + +Two rules keep every new id of a region above every id it already has, on a prefix that already holds objects and across the watermark Mito hands the store. On open, the next object sequence is the larger of the largest listed sequence plus one, orphans included, and, over every region, the object named by the region's largest existing entry id plus one, that is `(max_entry_id >> 20) + 1`, so that the catalog's strictly increasing ranges hold; sequence gaps are tolerated, and a sequence at or above `2^44` is rejected as exhausted. Objects written under an earlier contiguous scheme are not rewritten: they remain readable, their ids carry no object information, and the rule places them by their high bits just the same. + +The second rule is the sequence floor of `obsolete`. Mito calls `obsolete` with the manifest's `flushed_entry_id` when it opens a region, and the actor applies the watermark and a floor derived from that entry id together: the next object sequence is raised to `(entry_id >> 20) + 1` unless it is there already, while zero, the watermark of a region without entries, names no object and needs no floor, so that a watermark assigned under another prefix, or one whose object this prefix no longer holds, is never passed by a new id. The floor moves only the next sequence, never the chain: the next batch still links to its predecessor, so a skipped sequence leaves no gap in the history recovery replays. While the open batch holds ids under the next sequence, the call is refused with a retryable error and records neither the watermark nor the floor, because those ids would fall below the floor. In the `enqueued` mode an id the store itself handed out needs no floor, even before it is durable: it is never handed out again while the store runs, so every later id of the region is greater. In the current single-prefix layout the floor never needs to move, since every watermark Mito hands the store names an object below the next sequence; the rule exists for the cross-prefix cases under *Alignment with cluster mode*. No additional persisted state is introduced. + +Garbage collection deletes the objects below a reclaim boundary that the store persists in every header, so that recovery can tell a collected predecessor from one that never landed. The boundary is computed from the catalog and the watermarks the store already keeps: each region with a recorded watermark contributes the sequence of its first indexed object holding a segment whose largest entry id is above the watermark; a region without a recorded watermark contributes its first indexed object; a region whose watermark is the maximum entry id, after `obsolete_all`, contributes nothing; the boundary is the smallest contribution, or the last indexed sequence plus one when nothing contributes, and it never decreases. Whether a segment is flushed is decided per segment from the footer, on the full entry id, so a watermark that stops in the middle of an object keeps the object. The boundary is recomputed after every successful `obsolete`, after `obsolete_all` and after every object is indexed; Mito calls `obsolete` after a flush, after a truncate, when it drops a region and when it opens one, so a restart re-establishes every watermark and the boundary advances on the same rule. Every sealed batch carries the boundary current at sealing and the start object carries the boundary recovery found, so a boundary is always at or below the sequence of the object carrying it. The boundary that permits deletion is the one carried by the last indexed object; the boundary of a batch whose create failed or is in flight permits nothing. + +Every present object with a sequence below the durable boundary is deletable, on the chain or not: the objects the catalog indexes, the objects recovery listed but did not index, the objects of an earlier epoch that the start object moved past, and the sequences of batches that failed in this run, whose objects may or may not exist. A collection starts after a successful `obsolete`, after `obsolete_all` and after an indexing that raised the durable boundary. Deletes run in the background, at most four at a time and in no particular order, and never hold up admission, sealing, uploads or acknowledgements; an indexed object is unindexed only after its delete succeeded, and the WAL holes recorded for it are dropped with it. Every deleted object is counted in `greptime_logstore_object_store_wal_deleted_objects_total`; a failed delete is logged, counted in `greptime_logstore_object_store_wal_failed_deletes_total` and retried at the next collection, and the object stays indexed meanwhile. An object deleted but not yet unindexed when the process crashes is simply absent at the next recovery, which lists the prefix. Nothing is collected once `stop` has begun or while the store is poisoned; a delete in flight when `stop` begins runs to completion before `stop` returns, and the actor settles every attempt it abandons when it exits. + +A read never needs a deleted object, since entries at or below a region's watermark are hidden anyway, but a read that listed an object before it was collected skips the object when it gets there. A read that meets an error on such an object makes two observations, in the reverse of the order the actor writes them: it looks at the set of objects whose delete is in flight, and then at the catalog. The actor registers an object in that set before its delete starts, unindexes it if the delete succeeded, and only then takes it out, so an object absent from the set was either never collected, in which case the catalog still holds it, or is already unindexed by the time the catalog is read. Observing the catalog first would let a collection complete between the two observations and leave the read with an object that still looked indexed and no longer looked deleting. An object that is still in the set has an undecided delete, so the read waits for that one delete, a single object store request, rather than assume it succeeds: a delete that fails leaves the object present and indexed, and the error the read met is then reported as any other, `on_corrupted_segment` included. What the read waits for is the delete attempt its observation found, taken together with the lookup, not the object: a retry of the same object is a separate attempt, so the attempt the read holds releases it even while that retry is in flight, and a read is never handed from one attempt to the next. A read stream outlives the store it came from, so the actor settles every attempt it abandons when it exits, whether the store was stopped or dropped; the objects of those attempts stay present and indexed, so the reads waiting for them report the error they met. A collected object is skipped silently: it is not recorded as a hole and does not count as a skipped segment. An object the catalog still holds after those observations fails the read like any other fetch failure, `on_corrupted_segment` included. Within one run the catalog keeps the largest entry id of a region after the objects holding it were collected, so `latest_entry_id`, the durability wait and the `enqueued` clamp keep their reference point; a store opened later learns only what is listed, so after a restart `latest_entry_id` of a region whose later objects were collected is the largest id among the objects left, which only the pruning hint of Mito consumes. No state is persisted outside the headers and no configuration option is added. + +## Log store + +`ObjectStoreLogStore` implements the `LogStore` trait for `Provider::ObjectStore`. + +**Construction and recovery.** The prefix, flush interval, batch size and backlog thresholds are validated, then the store lists the prefix and fetches, for up to eight objects at a time, the header and a window at the end of each object with range reads; a short object is read whole, and the footer is read separately only when it starts before the window. Each header and footer is verified as described under *Object format*; the first corrupted object fails construction; a fetch that finds the object gone is judged once the history is selected, as described below, and any other failed fetch abandons the rest. Segments are not read, so recovery costs a few small reads per object however large the objects are. + +The store then selects the history it replays from the headers. An object's link holds when the object has no predecessor, when its predecessor is present and carries the epoch the link records, or when the predecessor's sequence is below the reclaim boundary the object carries, in which case the chain ends there whether or not the predecessor is present; above that boundary a missing predecessor breaks the link. An object is complete when every link on its chain holds, each judged with the boundary of the object carrying the link. The tip is the complete object with the largest epoch and, among those, the largest sequence. Only the objects on the tip's chain are indexed and replayed, in sequence order whatever the order the fetches completed in, so the catalog checks the entry ranges of every object against its predecessors like a sequential replay, and a conflicting object fails construction. Every other present object is an orphan: it is neither indexed nor read, but its sequence counts, so the next sequence is above every listed object, and the objects below the tip's boundary are collected. A listed object whose header or footer is gone by the time it is fetched is ignored when it lies below the tip's boundary and fails construction when it lies at or above it, since collection never deletes one there. Construction also fails when a listed object carries an epoch above the next sequence, which no instance can have produced, since an epoch is a start sequence plus one. When objects are present but none is complete, construction fails rather than start an empty history. Construction then records the largest durable entry id per region. + +Before it admits any append, the store writes a start object: an object without segments at the next sequence, which is above every listed object, orphans included, linked to the tip, or to nothing on an empty prefix. The instance's epoch is the sequence of its start object plus one, and every object the instance writes carries it. The conditional create assigns each sequence to exactly one object, so no two instances share an epoch; and the start sequence is above every object present at open, including the start objects of every earlier instance, so the epoch of every later instance is greater than that of any object an earlier instance ever writes, including one whose create lands after that instance stopped or crashed, and such an object never becomes the tip. Only once the start object is created does the store accept appends, and the first batch links to it. If the sequence turns out to be taken, the store reads the object there: an object of an earlier epoch is stale, and the start object moves to the next sequence, taking the epoch that sequence gives; an object of the same or a later epoch means another writer, and construction fails with a conflict error that is not retried; an object with identical content, which another open of the same prefix has just written, fails construction with a retryable error, so that the next attempt lists it and starts above it. A create whose outcome is unknown fails construction with a retryable error too: an object it may have left is listed by the next attempt, which therefore starts above it. Two assumptions remain. Listing the prefix is strongly consistent, so an open sees every start object that was durable before it listed; this holds for S3, GCS, Azure Blob Storage and MinIO, and an S3-compatible store without it is not supported. And one process at a time opens a prefix: two processes opening it concurrently either contend for the same start sequence, where the conditional create lets one through and fails the other's construction, or, if one of them listed a late object the other did not, start under different sequences and epochs and both proceed until their sequences meet, where the writer that meets the other's object follows the conflict rules under *Writing* and the *Failure matrix*: in the `durable` mode an object of an earlier epoch fails its batch and one of a later epoch poisons it, and in the `enqueued` mode any conflict poisons it, so which writer is poisoned depends on who observes the collision, not on the epochs alone; the store does not fence concurrent writers beyond that, and in cluster mode the generation in the key prefix does. + +**Writing.** One background actor per store admits appended entries into a single open batch and assigns entry ids as described under *Entry ids* above. The batch is sealed when its estimated size reaches `max_batch_bytes` or when `flush_interval` elapses with entries in it; a timer tick on an empty batch creates nothing. Sealing encodes the batch under the next sequence number and hands it to a pipelined uploader: the actor keeps admitting entries into the next batch while uploads are in flight, and up to four conditional creates run at a time, started in sequence order. At most eight sealed batches wait to be indexed: once seven do, which leaves no room for the two one admission can seal, the next append is held back in both acknowledgement modes until a batch is indexed, and later appends wait behind it. Every sealed batch is encoded with the store's epoch and a link to its pipeline predecessor: the last batch sealed before it that is not indexed yet, or, when there is none, the last indexed object, which after opening is the start object. A link to the last indexed object while an earlier batch is still uploading would let a later chain skip that batch once it is acknowledged. Batches are indexed in sequence order in both acknowledgement modes: a created object is not indexed until every earlier batch is. The rest of this paragraph is `durable` mode behaviour; the `enqueued` mode, whose appends have already returned, is described under *Acknowledgement*. In the `durable` mode a batch is acknowledged when it is indexed, so a batch whose object is durable is not acknowledged until every earlier batch is durable too, and the acknowledged prefix of a region's history never has a missing predecessor. When a create fails transiently, every batch that is not indexed fails at once, including batches whose objects were already created and batches whose creates are still in flight. Their sequence numbers are never handed out again, the next batch links to the last indexed object, and the store keeps serving. A create still in flight completes later and finds no batch to acknowledge, so whatever it stored is an orphan. The next object acknowledged links past every failed batch, and from then on recovery replays none of them, whether their objects exist, land later or never do. A create that meets a different object reads that object's header: an object of an earlier epoch is stale and can never be the tip, so the batch fails as for a transient error and the store keeps serving; an object of the same or a later epoch belongs to another writer on the prefix, and the store poisons itself. + +The guarantee is therefore about acknowledged history. An acknowledged entry is always replayed; an entry whose append failed is replayed only when nothing was acknowledged after it, the same outcome as a crash between an object's creation and its acknowledgement, which the engine already tolerates. The distinction matters because Mito does not advance a region's committed sequence when an append fails and reuses those row sequences in its next write, so replaying a failed batch next to a later acknowledged one could let the unacknowledged rows win deduplication. + +**Acknowledgement.** `ack_mode` selects when `append_batch` returns. `durable`, the default, returns only after the object holding the entries is durable and indexed, so callers never see an entry id for an entry that is not durable; a write-ahead log is expected to mean durability on return, which is why this stays the default. `enqueued` returns as soon as the entries are admitted, with their entry ids already assigned, and creates the object in the background. In that mode the recovery point objective is the unpersisted backlog. `max_unpersisted_bytes` and `max_unpersisted_age` are admission thresholds: when the open batch plus the sealed batches that are not durable yet reach either, new appends are held back in arrival order until an upload completes, and the store seals the open batch if no create is in flight so that there is an upload to wait for; nothing is dropped and the backlog cannot grow without limit. They are not a hard bound on what a crash can lose: during an outage the writes already acknowledged stay in memory for as long as the outage lasts, however old they get, and a crash in that window loses them. + +The failure contract of `enqueued` differs from the durable one because no caller remains to retry: a create the object store did not confirm is repeated under the same sequence with the same bytes instead of failing the batch. A transient failure is repeated after 100 ms while appends are still admitted. A failure that requires intervention, such as denied access or a missing bucket, makes the store failing. The store tells the two apart by the kind of the object store error, not by its status, because OpenDAL's retry layer under the store marks every error it returns as persistent. + +A failing store keeps its acknowledged backlog instead of poisoning itself, since poisoning would drop those entries and need a restart for a cause that can be fixed in place. It attempts the lowest pending batch alone, with exponential backoff, and rejects every new append at once with a retryable error that carries the last object store error, so no append is admitted or waits behind the backlog. A durability wait for an entry in the backlog keeps waiting. The first create that succeeds ends the state: the backlog is uploaded and appends are admitted again. + +A conflicting object or an encoding or catalog error poisons the store. Once `stop` has begun a failed create is no longer repeated: the store attempts its whole backlog at once, failing or not, and any object store failure drops the backlog and `stop` reports the loss as its error, since those entries were acknowledged but never became durable. Otherwise `stop` seals and uploads the remaining backlog before it returns. + +The mode also needs a barrier that the durable mode gets for free. An entry id that Mito has been handed may not yet be durable, and a flush that persisted such an id as the manifest's `flushed_entry_id` would, after a crash before the object was created, leave a watermark that refers to an object which never existed, so that the sequence is reused on restart and later entries land below the watermark and are skipped by replay. The `LogStore` trait therefore has `wait_durable(provider, entry_id)`, with a default that returns at once, so that every existing provider keeps its own acknowledgement and persistence semantics, whatever they are; the durability guarantee this section describes belongs to the object store provider alone. The object store WAL answers it once the region is durable and indexed through `entry_id`; an id the store never handed out has nothing to wait for. Mito waits before it publishes a manifest watermark: a flush waits after its SSTs are written and before the manifest edit, a full truncate waits before it records the truncated entry id as the replay frontier, and discarding unflushed data waits before it records the discarded entry id as flushed. The wait in a flush observes the flush's cancellation, so a cancelled flush does not hold up the drop or truncate queued behind it; a full truncate and a discard of unflushed data still run their own wait before publishing their frontier. After a backlog was lost, a durability wait for an entry that is not durable fails rather than certifying it, whether the entry was in that backlog or not. In `enqueued` mode `obsolete` also clamps the watermark to the region's durable entry id, since an entry that is not durable yet is replayed after a crash and hiding it would skip it. A manifest watermark can then only ever name a durable entry, and the restart allocation rule under *Entry ids* keeps every new id above it. + +**Failure matrix.** + +| Situation | Sequence | Waiters of the batch | Store | +| --- | --- | --- | --- | +| create succeeds, or identical retry | advances | acknowledged in sequence order | healthy | +| transient I/O error, `durable` mode | consumed and never reused, nor are the sequences of the other failed batches | this and every batch that is not indexed fail, including those whose objects were created; a create still in flight completes as an orphan | healthy: the next batch links to the last indexed object | +| an object of an earlier epoch holds the sequence, `durable` mode | consumed and never reused | as for a transient I/O error | healthy | +| transient I/O error, `enqueued` mode | unchanged | already acknowledged | healthy: the create is repeated after a short delay | +| I/O error that requires intervention, `enqueued` mode | unchanged | already acknowledged; later appends are rejected with a retryable error until a create succeeds | failing: the lowest pending create is repeated alone with backoff; the first create that succeeds ends it | +| any I/O error, `enqueued` mode after `stop` began | unchanged | already acknowledged | the backlog is dropped and `stop` reports the loss | +| an object of the same or a later epoch holds the sequence, or any conflicting object in the `enqueued` mode | unchanged | every unacknowledged waiter fails, including those of batches whose objects were created but not yet indexed; an `enqueued` acknowledgement already returned cannot be revoked | poisoned | +| encoding or catalog error | unchanged | every unacknowledged waiter fails, including those of batches whose objects were created but not yet indexed; an `enqueued` acknowledgement already returned cannot be revoked | poisoned | +| create succeeds at the last representable sequence | cannot advance | acknowledged | poisoned: no later batch can be allocated a sequence, every later append fails | +| the start object meets an object of an earlier epoch | moves to the next sequence | none admitted yet | construction continues | +| the start object meets an object of the same or a later epoch, or its create has an unknown outcome | unchanged | none admitted yet | construction fails; an unknown outcome is retryable | + +A store cannot be constructed on a prefix whose largest object already carries the maximum sequence. A poisoned store fails every `LogStore` operation except `stop` with the same terminal error; in particular `obsolete` can no longer move a watermark. In the `durable` mode transient errors are not retried inside the store, but a failed create is reconciled once: the store immediately reads the key back, accepts the batch if the object exists with identical bytes, treats it as a conflict if the bytes differ, and reports the transient error only when the key is absent or that read fails too. So a lost response alone does not fail the append. When the append does fail, neither the store nor Mito keeps the batch: it surfaces through Mito to the caller of the write, and a retry is a new write by the caller under a new sequence. If the create had in fact succeeded and both its response and the read-back were lost, or its request lands only later, the object is an orphan as soon as a later object is acknowledged. Retrying the batch itself inside the store in the `durable` mode would only hide the latency and would hold writes up for as long as an outage lasts; the `enqueued` mode repeats the create with the same bytes, so a create that succeeded without a response is accepted as an identical retry. + +**Stopping.** `stop` is idempotent and awaits the actor. Once stop has begun, queued appends are not admitted, no timer or size triggered seal starts, and appends after stop, including empty ones, fail with the stopped error. In the `durable` mode the open batch and every sealed batch whose create has not started are dropped: their waiters receive the stopped error. A create that is already in flight runs to completion; if it succeeds while its batch is still pending, the batch is indexed in order and acknowledged. A transient failure after stop began fails every batch that is not indexed, as it does while the store runs, and those batches receive the stopped error; a create of such a batch that completes later is an orphan and acknowledges nothing. A create that finds a conflicting object of the same or a later epoch after stop began poisons the store as it would while the store runs, but the waiters of the batches that are not indexed receive the stopped error, the conflict is kept only as the terminal error, and `stop` returns successfully once the creates in flight have completed. In the `enqueued` mode the held-back appends receive the stopped error, the open batch is sealed and the whole backlog is uploaded before `stop` returns, by a failing store too; any object store failure in that phase drops the backlog and `stop` returns the error, as described under *Acknowledgement*. A durability wait still pending when the actor exits receives the stopped error, and a delete already in flight runs to completion before `stop` returns. + +**Reading.** `read(provider, entry_id)` returns the region's entries with id at least `entry_id`, in order, fetching only that region's segments with range reads and decoding nothing else, and hides entries at or below the region's obsolete watermark. `obsolete` records the watermark in memory, clamped to the durable entry id in the `enqueued` mode; the watermark is re-established after a restart because Mito calls `obsolete` with the manifest's `flushed_entry_id` when it opens a region, so no WAL-owned watermark state is persisted. `obsolete_all` hides the whole region. `latest_entry_id` is not affected by the watermark. Every method taking a provider requires `Provider::ObjectStore` with the store's prefix and the caller's region. + +**Metrics.** The store exports metrics under `greptime_logstore_object_store_wal_`. The write path reports `seal_to_durable_seconds`, the time from sealing a batch until its object is durable, `append_ack_seconds`, the time an append waits for its acknowledgement in the `durable` mode, `object_bytes` and `object_entries`, the size and the entry count of every object sealed, and `created_objects_total`, the creates that succeeded, an identical retry included; a create is counted and timed whether or not its batch is still there to be indexed, since an object a poisoned store gave up on is as durable as any other. `indexed_objects` and `indexed_bytes` are the objects and the bytes the catalog holds, set by a recovery that succeeded and moved by every object indexed or unindexed afterwards, so a collection takes the object it deleted off both, and a recovery that is abandoned publishes neither. A recovery that succeeds reports `recovery_seconds` and the objects it read footers from in `recovered_objects_total`, both of the same recoveries, so one that is abandoned at a fetch or rejected by the catalog contributes to neither; a read stream reports `read_seconds` once it has been read to its end, and one that fails or is dropped part way reports nothing. The failure paths are counted: `create_conflicts_total` for a create that found a conflicting object, `create_failures_total` for one the object store did not confirm, `poisoned_total` for a store that recorded a terminal error, `skipped_segments_total` for a segment a read skipped, and `stalled_appends_total` with `stalled_append_seconds` for an append held back because the sealed batches were full, in either mode, or by the `enqueued` backlog thresholds, timed until it is admitted or refused, a stop, a terminal error, a failing store or the teardown of the actor releasing it included. The gauge `failing` counts the stores whose backlog the object store refuses with an error that requires intervention, and `rejected_appends_total` the appends such a store rejected; the attempts of a failing store are counted in `create_failures_total`. The delete counters of a collection are described under *Entry ids*. The histogram buckets are chosen for the default flush interval of 100 ms and the default `max_batch_bytes` of 8 MiB. + +A `testing` cargo feature exposes hooks to wait for admitted appends, to seal the open batch, to hold and release creates and to make creates fail, so engine tests can flush deterministically and observe entries that are admitted but not durable. + +## Mito integration + +Mito maps `WalOptions::ObjectStore { prefix }` to `Provider::object_store_provider(region_id, prefix)`; resolving object store options against a Raft Engine or Kafka log store reports the existing incompatible-provider error. Reads go through the region-scoped reader used for Kafka, so a namespace shared by many regions never leaks entries across regions even if a log store returned them. Object store regions are opened individually rather than grouped by topic. After replay, `topic_latest_entry_id` follows the existing remote-WAL rule: it is taken from the store's latest entry id only when the region's memtables are empty, otherwise it stays at the replay start, because that value feeds pruning and must never advance past unflushed data. Before a flush, a full truncate or a discard of unflushed data publishes a manifest watermark, Mito waits on the durability barrier described under *Acknowledgement*; with the `durable` mode and with every other provider the wait returns at once. + +## Datanode wiring and lifecycle + +The scalar options are validated before any store or engine exists: the prefix must be non-empty, relative and free of empty or `..` components; `flush_interval` at least 10 ms; `max_batch_bytes`, `max_unpersisted_bytes` and `max_unpersisted_age` positive. `storage_provider` is resolved later, once the object store manager has been built and before the log store and the Mito engine are: empty selects the default store, otherwise the name must match a configured provider. Failures in both stages are `InvalidArguments` naming the field, and both happen before any WAL object is touched. The datanode requires a node id to derive the node prefix it opens the store under, and the standalone bootstrap derives the same prefix for the WAL options it allocates. + +`DatanodeBuilder` keeps the store handle from the moment it is created and stops it if a later build step fails. On success the handle moves into `Datanode`, whose shutdown stops the store after the region server. Shutdown of the datanode and of the standalone instance is best-effort and ordered: every step runs, the first error is returned, so a failing region engine does not leave the WAL actor alive. + +## Configuration + +The options: + +```toml +[wal] +provider = "experimental_object_store" +# Name of a configured storage provider; empty selects the default store. +storage_provider = "" +prefix = "wal" +# How long a non-empty batch may stay open before it is sealed; at +# least 10ms. A shorter interval lowers the acknowledgement latency of +# sparse `durable` writes and raises the request count: timer-triggered +# sealing creates at most one object per interval per node, and in the +# `enqueued` mode the backlog thresholds can seal earlier. The batch +# seals on size only when the admitted bytes reach `max_batch_bytes` +# before the next timer tick. +flush_interval = "100ms" +# Estimated batch size at which the open batch is sealed. It is a +# threshold, not an upper bound: a single append larger than this is +# admitted whole, and object framing adds bytes on top. Object stores +# bill per request, and the cost-effective request size is 8 to 16 MiB. +max_batch_bytes = "8MB" +# "durable": append returns after the object is durable (the default). +# "enqueued": append returns on admission. New appends stall once the +# unpersisted backlog reaches either threshold below; the thresholds do +# not bound what a crash during an outage can lose. +ack_mode = "durable" +max_unpersisted_bytes = "64MB" +max_unpersisted_age = "8s" +# What a read does with a segment that still does not decode after a +# second fetch, because its checksum does not match or its content +# disagrees with its footer entry. "skip": the segment is skipped and +# recorded as a WAL hole of its region, counted and logged; the other +# regions of the object are unaffected (the default). "fail": the read +# fails, so the region does not open. +on_corrupted_segment = "skip" +``` + +`config/config.md` is generated from the example files. + +## Compatibility + +- `WalOptions::ObjectStore` is a new persisted variant. Existing `raft_engine`, `kafka` and `noop` options decode unchanged; a region's options never change provider implicitly. +- The object format is version 1 and frozen once the provider is released; a version bump after that is a new decoder, not an in-place change. The version 1 header layout was revised before release to carry the epoch, the link, the reclaim boundary and the header CRC. No object in an earlier layout exists, since the provider has not been enabled anywhere. +- The configuration tag carries `experimental_`; the persisted tag does not, so promoting the provider to stable is a configuration rename with an alias, not a metadata migration. +- `LogStore::wait_durable` has a default implementation that returns at once, so Raft Engine, Kafka and the noop provider keep their own acknowledgement and persistence semantics; the durability barrier binds only the object store provider. +- The object-sequence-major id scheme needs no migration of existing objects. A prefix written under the contiguous scheme keeps its objects, and the sequence allocation rule under *Entry ids* keeps new ids above the old ones. Garbage collection reasons from footers and from the reclaim boundary, not from the id encoding, so it applies to objects of either scheme. +- `on_corrupted_segment` is a new configuration option with a default, so existing configurations decode unchanged; it changes what a read does with a damaged segment, not what is written. + +## Evidence + +In summary, the evidence supports going ahead with the experimental standalone implementation. Targeted tests and a process-level crash gate cover the recovery paths and the durability barrier; a local MinIO run confirms recovery and garbage collection. In same-session comparisons on a laptop the default `durable` mode reaches about 23% to 28% less throughput than Raft Engine with synchronous writes; on S3 from an instance in the bucket's region, against Raft Engine on a local NVMe store, it reaches 31% of Raft Engine's throughput with one writer and 54% with eight, bound by the seal interval rather than by S3, while the `enqueued` mode matches Raft Engine and never stalls. Under sustained load the default interval costs about 600 object creates per minute per node. The pipelined uploader has shown no consistent throughput benefit; in a multi-region experiment with injected object store latency it lowered write latency at the cost of more object requests. Long-running stability and recovery at production scale remain unverified. + +The reference implementation was validated on the following evidence; the tests are part of the pull requests that submit it. The first table maps each guarantee of this RFC to what was verified, the list after it names the failures that were driven through the store, and the last list states what the evidence does not cover. + +### Guarantees verified + +| Guarantee | Verified by | Result | +| --- | --- | --- | +| Acknowledged writes replay exactly once across graceful and ungraceful restarts under `durable` | Deterministic recovery tests on a real store over an in-memory object store: two regions sharing a prefix with only one flushed, an abrupt drop after an object is durable but before any flush, two consecutive restarts with writes and a flush between them. A process-level crash gate: concurrent writers, SIGKILL at a random point of a window, restart on the same bucket and data home, repeated cycles with replay accumulating. A standalone round trip against MinIO with a restart before and one after a flush. | The concrete flushed, replayed, manifest and latest entry ids and row-level scan equality after every restart. In the crash gate every acknowledged row is present exactly once after the restart, and every present row that was not acknowledged was in flight at the kill. | +| The loss of `enqueued` is bounded to the unpersisted backlog and reported | The same recovery tests and the same crash gate run in `enqueued` mode; unit tests of both admission thresholds stalling until an upload completes, of `stop` uploading the backlog, and of `stop` reporting a backlog lost before it was called. | No row is present twice and no present row was never attempted; the crash gate reports the acknowledged rows lost per cycle. | +| A manifest watermark never names an entry that is not durable | Durability barrier tests: a flush held until its entry is durable; a crash in `enqueued` mode after an acknowledgement and a flush attempt but before the object exists, followed by a restart, new writes and a second crash; a drop cancelling a flush that waits; a truncate that waits; a flush after a backlog was lost. | Every durable entry replays; the flush after the lost backlog publishes no frontier; `obsolete` never passes the durable id. | +| A region replays only its own entries | The two-region recovery test above. | Per-region row-level scan equality with the other region unflushed. | +| Recovery never silently loses data | Unit coverage of every corruption class at recovery and on read; footer-only recovery compared with a full decode; a footer longer than the tail window. | Structural damage fails construction of the store; a corrupted segment is skipped, recorded as a hole of its region and counted, or fails the read under `fail`; the footer-only catalog equals the full decode. | +| A region cannot be silently pointed at another WAL | Reopen on a mismatched prefix; reopen on an empty prefix. | The mismatch is rejected with both prefixes named and the region is not opened; the empty prefix opens cleanly with the expected row counts. | +| Garbage collection deletes everything below the persisted reclaim boundary, and recovery stays correct while objects disappear | Unit tests of the boundary computation (a region without a watermark holds it at its first object, a watermark inside an object keeps the object, `obsolete_all` releases it, last indexed plus one when everything is flushed), of stamping and durability (headers carry the boundary current at sealing, a failed create's boundary permits nothing, the start object carries the recovered boundary), of deletion (chain objects, listed orphans and failed batches below the durable boundary, nothing at or above it, a failed delete retried, `stop` waiting for deletes in flight, holes dropped), of recovery (two restarts with a collection between them, a late object of an earlier epoch below the boundary collected, a missing predecessor above the boundary still failing) and of a read that listed an object before it was collected; the Mito flush-then-collect test in both acknowledgement modes; the standalone integration test's assertion after a flush. | Objects below the durable boundary are gone after a flush, the start object included; a restart replays everything above the boundary; exactly one WAL object remains after a flush and one more insert in the integration test. | +| Every row of the *Failure matrix* holds, and every metric moves as documented | A table-driven fault matrix, `test_store_fault_matrix`, that drives every failure the object store access can produce through the store and asserts, per case, which waiters fail and with what error, whether the store is poisoned, whether the sequence is consumed, the indexed gauges at the failure itself, the exact movement of every metric, and what a read of the store sees afterwards; every timer is observed twice around two forced waits whose intervals cannot overlap. | 36 cases: 19 for the eleven rows of the matrix and 17 for the failures listed under *Failures driven through the store*. The test fails if a row of the matrix is left uncovered. | +| Recovery cost does not grow with object size | Footer-only recovery over 2,000 objects; recovery concurrency bounded and ordered by sequence. | Construction of the store fell from 3.6 s to 0.9 s. | + +### Failures driven through the store + +- Conditional create outcomes: success, identical retry, conflicting content from an earlier epoch and from the same or a later one, lost response, transient failure in `durable` mode with later objects already created and with creates still in flight (every batch that is not indexed fails, no sequence is reused, late completions are orphans), transient failure in `enqueued` mode and after `stop` began, a failure that requires intervention in `enqueued` mode and a `stop` while it lasts, completion after a conflict had already poisoned the store, a create at the last representable sequence. +- Object store access: an unwritable store; a listing, a trailer fetch, a footer fetch, a segment fetch and a delete that fail; a delete that fails and is retried by the next collection; an object collected between the listing and its fetch, below and at the tip's reclaim boundary; an object store that answers slowly. +- Store state: an object the catalog rejects, per-region entry id exhaustion, the catalog refusing a next sequence past the maximum, every stop path with several creates in flight that succeed, fail transiently or conflict after stop began, a stop with a delete in flight, an append held back by full sealed batches, and the appends the backlog thresholds were holding back released by admission, by a stop, by a terminal error and by the teardown of the actor. +- Pipelined uploader: concurrent creates acknowledged in sequence order, every sealed batch linked to its pipeline predecessor, the open batch sealing at the threshold while creates are in flight, a transient failure failing every batch that is not indexed, created ones included, a permanent failure poisoning the store. +- Chain recovery: a linear chain, orphans below and above the tip, a broken tip falling back to the latest complete object, a late object of an earlier epoch next to the chain of a later one, a missing predecessor, objects present without a complete chain, orphans raising the next sequence, and the start object on an empty prefix and meeting objects of an earlier and of a later epoch. + +### What the evidence does not cover + +- The cloud measurement covers one instance type, one region and one bucket, with the Raft Engine control on the instance's local NVMe store; no crash gate ran on the cloud host, and the request cost is the list price, not a bill. +- Loss during writes is measured only by the crash gate, in single samples. The Raft Engine comparison below killed the server after every row was counted, so its replay checks confirm completeness and say nothing about loss. +- The store's own work, encoding, indexing and the actor, is not quantified separately from the object store round trip. +- A create whose response and immediate read-back are both lost is covered only at recovery, by placing such objects next to a chain; no test loses a response and its read-back at the object store. +- The benefit of the pipelined uploader was measured only with injected latency on a local object store. + +### Write path measurements + +A log workload benchmark was run against the write path on a laptop with a local MinIO server. The numbers calibrate the defaults; the same workload on S3 is reported under *Cloud write path measurements* below. The profile writes 2,000,000 rows of about 900 bytes in 1000-row batches into one region at client concurrency 1, 8 and 32. Raft Engine with `sync_write = true` on the same build and the same storage is the control. Rows per second: + +| Write path | 1 | 8 | 32 | +| --- | --- | --- | --- | +| Raft Engine, `sync_write = true` | 3,500 to 4,000 | 19,000 to 25,000 | 20,000 to 28,000 | +| one create in flight, `durable`, 1 s | 998 | 4,001 | 19,029 | +| one create in flight, `durable`, 100 ms | 3,287 | 13,662 | 17,204 | +| one create in flight, `durable`, 10 ms | 3,632 | 19,835 | 19,309 | +| pipelined uploader, `durable`, 1 s | 1,000 | 3,999 | 22,043 | +| pipelined uploader, `durable`, 100 ms | 3,304 | 17,207 | 23,185 | +| pipelined uploader, `enqueued`, 1 s | 3,955 | 24,074 | 22,168 | +| pipelined uploader, `enqueued`, 100 ms | 3,999 | 23,595 | 22,662 | + +The rows come from separate runs, and the same build with the same configuration, on an idle host each time, measured 1.4 times apart in runs days apart, so a difference between two rows of the table is not a measurement of the difference between those write paths. The Raft Engine range covers several runs. Both write paths converge on the host's CPU ceiling at concurrency 32, 6 to 7.5 of 10 cores busy with one region, so the last column measures the host rather than the WAL. A serial writer under the `durable` mode is phase-locked to the seal timer: its extra latency over Raft Engine is the interval minus the base insert cost modulo the interval, which is 747 ms at 1 s, 52 ms at 100 ms, and a floor of about 14 ms, one seal plus one PUT, at 10 ms. The pipelined uploader alone was measured by counterbalanced A/B runs of the builds before and after it, run in both orders within one session: at 1 s and concurrency 32 the two builds were within about 1.4% and the two pairs disagreed on which was faster, and at 100 ms and concurrency 8, where both builds sealed about ten objects per second, they were 0.25% apart on the geometric mean of two pairs, while whichever build ran first in a pair was 3.5% to 4% faster, and their object counts and sizes were within 2%. No throughput benefit was measured at either point. Every restart under the `durable` mode replayed exactly the acknowledged rows. In the `enqueued` mode a SIGKILL lost one to two 1000-row batches at 1 s and none at 100 ms, in single samples, always whole batches and never a duplicate. Footer-only recovery cut log store construction from 3.6 s to 0.9 s at 2,000 objects. + +The conditions under which the pipelined uploader does help were then examined, again comparing the builds before and after it within one session in both orders at concurrency 8 and 100 ms, with a proxy in front of the object store that delays every request of the WAL while the data files keep a direct connection. In the build before it the actor awaits each create before it admits more entries, and Mito's region worker in turn waits for one append to become durable before it sends the next, so what the uploader can run side by side depends on how many appends wait at once, not only on how slow a create is. With one region, adding about 40 ms or about 160 ms to each create left the two builds within the order effect of these runs, since one region never has a second sealed batch while the first uploads. With a table split into three regions, each served by a different region worker, and about 160 ms added, the pipelined uploader was faster by 13.6% and 2.1%, 7.7% on the geometric mean, still within that margin, and it alone let the regions fall out of step: 27% to 46% of its sampled objects held fewer than three regions, against 0% to 4% before it, because a region whose append became durable while another create was still in flight started the next batch. The clients of those runs wait for each response, which ties every region back to the seal. With the three regions driven at a fixed request rate of 0.7 times that closed-loop throughput instead, the pipelined uploader lowered the median insert latency by 24.0% and 12.2% in the two pairs, 18.3% on the geometric mean, and the 99th percentile in both pairs; at 1.0 times the rate one valid pair showed 23.8%, while the second pair was not completed because the load generator repeatedly fell behind its send schedule. The lower latency is paid for in requests: with about 160 ms added, the pipelined uploader created 19% to 28% more objects with one region, 51% to 67% more with three regions in the closed loop, and 35% to 63% more at 0.7 times the rate in the open loop. These measurements used a local object store with added latency and do not measure a cloud object store. + +The comparison with Raft Engine was measured again within one session, with one binary and only the `[wal]` section differing, each comparison run in both orders to reduce the bias of running first or second. Runs on this host still varied by 4% to 8% with their position alone, which limits how small a difference these runs can resolve. Both runs of a pair killed the server only after every row was written and counted, so the replay checks confirm a complete replay and say nothing of what a crash during writes loses. Rows per second: + +| Write path | Concurrency 1 | Concurrency 8 | +| --- | --- | --- | +| Raft Engine, `sync_write = true` | 3,817 and 4,503 | 25,124 to 26,880, four runs | +| object store WAL, `durable`, 100 ms | 3,184 and 3,198 | 17,779 to 19,174, four runs | +| object store WAL slower by, geometric mean of the pairs | about 23% | about 28% | + +The gap does not close under concurrency; it is slightly wider at 8 than at 1. At concurrency 1 the object store WAL measured the same in both runs, 301 ms at the median, while Raft Engine moved between a median of 237 ms and 221 ms, both of which round up to the same 300 ms seal, so the single-writer figure depends on how fast Raft Engine happens to be. Raft Engine exceeded the upper end of the range in the table above in every one of the four runs at concurrency 8. + +Two further configurations of the same binary, run in both orders in two rounds at concurrency 8, locate the gap. With the WAL objects written to local disk through a file object store instead of MinIO, still in the `durable` mode, MinIO was 4.5% slower. Switching the MinIO configuration to the `enqueued` mode was 35% faster, and the `enqueued` configuration was 4.4% slower than Raft Engine. The first and third of these differences are within the run-to-run variation of the host, while the second is far larger, which points to the `durable` mode waiting for the seal and the create before it acknowledges as the main contributor to the gap in this workload. These are differences between whole configurations: they do not measure the MinIO round trip or the store's own work, encoding, indexing and the actor, individually, and the runs leave those costs unquantified. The file object store is not equivalent to a remote one: its conditional create checks, writes a temporary file and renames it rather than creating atomically, and it does not sync the directory after the rename. The `enqueued` mode does not provide the durability of Raft Engine with `sync_write = true`, so this comparison separates costs and is not a comparison of equivalent write paths; against Raft Engine with its default `sync_write = false`, also run in both orders within one session, the `enqueued` mode was 5.1% slower, again within the noise. These local runs do not measure the latency or throughput of a cloud object store. + +The default interval is 100 ms because it removes most of the seal-timer latency of a sparse `durable` writer, 52 ms of residual wait against 747 ms at 1 s, at a request cost that stays bounded: timer-triggered sealing creates at most one object per interval per node, ten per second at the default, the `enqueued` mode can seal earlier only when the unpersisted backlog reaches a threshold, and the batch seals on size only when the admitted bytes reach `max_batch_bytes` before the next timer tick, which the measured S3 workload did not reach at about 60 MB/s: every object store cell sealed on the timer. Going to 10 ms buys another 40 ms for ten times the request ceiling. On S3 at the default configuration the seal timer, not the PUT, bounded a serial `durable` writer to one batch per interval, see *Cloud write path measurements*; a shorter interval removes part of that wait but not the create's latency, and was not measured on S3. The measured way to Raft Engine's throughput from one writer is the `enqueued` mode, at the cost of its loss bound. The default acknowledgement mode stays `durable` because that is what a write-ahead log means: an acknowledged write survives a crash. `enqueued` is an explicit choice with a documented loss bound, the unpersisted backlog, and the crash gate reports what it loses. + +### Cloud write path measurements + +The same workload was run against S3 from an EC2 instance in the bucket's region (`c6id.2xlarge`, 8 vCPU, `ap-southeast-1`, the nightly image of `main` of 2026-10-12) by the workflow of pull request #9536 (https://github.com/GreptimeTeam/greptimedb/pull/9536), which starts GreptimeDB with its data files on S3 and differs between cells only in the `[wal]` section: Raft Engine with `sync_write = true` on the instance's local NVMe store, and the object store WAL in the `durable` and in the `enqueued` mode, both with the defaults. 2,000,000 rows of about 900 bytes in 1000-row batches at concurrency 1 and 8, two rounds with the order of the configurations reversed between them, every cell on a fresh container and a fresh prefix. Rows per second, geometric mean of the two rounds of run 38232084219 (https://github.com/GreptimeTeam/greptimedb/actions/runs/38232084219): + +| Write path | 1 | 8 | +| --- | --- | --- | +| Raft Engine, `sync_write = true`, local NVMe | 31,700 | 67,800 | +| object store WAL, `durable`, 100 ms | 9,670 | 36,900 | +| object store WAL, `enqueued`, 100 ms | 32,900 | 67,900 | + +A second run of one round at concurrency 8, 38234710697 (https://github.com/GreptimeTeam/greptimedb/actions/runs/38234710697), reproduced the ratios within 1%: the `durable` mode at 0.54 of Raft Engine, the `enqueued` mode at 1.00. The latencies, object counts and histograms below are read from the metrics scraped after every cell, kept in the runs' artifacts. + +A conditional create on S3 took about 50 ms at the median and 110 to 130 ms at the 99th percentile from seal to durable, against single-digit milliseconds on the local server, and still a serial `durable` writer is bound by the seal timer, not by S3: every 1000-row batch took about 103 ms, one object per batch, with its acknowledgement at 91 ms at the median, so at the default configuration its throughput was the batch size over the interval, about ten batches per second: the measured 103 ms per batch is one interval plus the admission of the next batch, since the create, the acknowledgement and the next append all complete before the following tick. No shorter interval was measured on S3. Raft Engine on the instance store took 32 ms per batch, hence 0.31. The local ratio of 0.77 was higher only because the local Raft Engine batch took over 220 ms, which rounds up to the next seal with little loss: the faster the control, the lower the ratio at low concurrency under the default interval. At concurrency 8 the `durable` mode still seals on the timer, about four batches per object, and reaches 0.54. The `enqueued` mode matched Raft Engine at both concurrencies and never stalled: at about 60 MB/s of WAL the backlog thresholds were not approached, since one create per interval of 3.5 to 8 MiB drains it. Under sustained load the timer ceiling sets the request count: 550 to 600 creates per minute per node in every cell, which at the S3 standard list price of $0.005 per thousand PUTs is about $0.18 per node-hour of continuous writing, plus about 30 deletes per cell for garbage collection. No create failed, found a conflicting object or poisoned the store in 7,609 creates, the 6,809 of the first run's eight object store cells and the 800 of the second run's two. + +# Alternatives + +## A generic external provider injected from outside the repository + +The backend could stay outside the main crates and be injected through an opaque provider variant plus a resolution hook on the `LogStore` trait. That avoids touching the four enums, but every resolution path then has to special-case an opaque value, distributed rejection can only be enforced by convention, and nothing in the repository can test the backend. Making the provider an explicit variant keeps every match exhaustive and lets the engine tests run against the real store, at the cost of one more arm in each enum. + +## Reusing the Kafka provider with an object store "topic" + +The Kafka code path assumes a broker assigns offsets and that a topic is shared across datanodes. Neither holds here: entry ids are assigned by the datanode, and one prefix belongs to one datanode. Reusing the Kafka provider would have inherited the topic tickers, the failover preconditions and the metasrv allocation logic, all of which are wrong for this design. + +## One object per region per flush + +Simpler catalog, but every region then produces its own timer-driven object, so the request rate scales with the number of regions even when they write little, which is the cost trap object stores punish. Sharing one batch across regions removes that per-region floor: at low volume timer-triggered sealing creates one object per interval regardless of region count, with the `enqueued` backlog thresholds able to seal earlier, and at high volume the count is governed by aggregate bytes and `max_batch_bytes`, which is the same request volume a single busy region would produce. + +# Drawbacks + +- Write latency under the `durable` mode is the time until the batch seals (up to `flush_interval`, 100 ms by default), plus waiting for every earlier object, plus one PUT. A serial writer waits for the next seal, so its extra latency over Raft Engine is the interval minus the base insert cost modulo the interval, and concurrency does not close the gap: measured within one session on a laptop it was about 23% of throughput at concurrency 1 and about 28% at concurrency 8, and on S3 against Raft Engine on a local NVMe store the `durable` mode reached 31% of Raft Engine's throughput at concurrency 1 and 54% at concurrency 8, a serial writer being bound to one batch per interval, while the `enqueued` mode matched it, see *Write path measurements* and *Cloud write path measurements*. +- Every object costs a request. Timer-triggered sealing creates at most one object per interval per node, ten per second at the default; the `enqueued` mode can seal earlier when the unpersisted backlog reaches a threshold, and the batch seals on size only when the admitted bytes reach `max_batch_bytes` before the next tick. Under sustained load on S3 the timer ceiling was reached, about 600 creates per minute per node, about $0.18 per node-hour at the S3 standard list price. +- Garbage collection keeps every object at or above the reclaim boundary, so a region that is written but rarely flushed, or never flushed, holds the boundary at its first object and keeps every later object the prefix shares with other regions; recovery time grows with the number of retained objects. A prefix that receives no appends cannot persist a higher boundary, so an idle prefix is collected only after its next object. +- Every open of the store writes one start object without segments, one extra PUT per open, and every object carries a 54-byte header. Orphans, the objects of failed batches that were stored anyway, hold storage together with the rows Mito wrote again after those failures until garbage collection deletes them. +- Object-sequence-major entry ids consume 20 bits per object for positions, which caps a region at about a million entries per object; the batch seals when the cap is reached. Sequence numbers are 44 bits wide as a result, enough for one object per second for half a million years. + +# Alignment with cluster mode + +Distributed mode is out of scope, but several decisions were taken so that the cluster design can build on this backend without a metadata migration. Except where a paragraph says otherwise, all of them are *Proposed*. + +**Key layout.** The layout is implemented in its standalone form, see *Object format*: the operator configures only the prefix root, the store derives the prefix it writes under as `/datanodes//epochs/`, so objects live at `/datanodes//epochs//objects/.wal`, standalone uses its datanode id and generation zero, and the persisted `WalOptions` of a region carry the full derived prefix, so a region's WAL location is self-describing. *Proposed*: in cluster mode the metasrv assigns both the node id and the generation, and issues a new generation every time it hands a region's write ownership to a node; the self-describing location then lets a node other than the writer open a region's WAL read-only, which today is rejected because a datanode builds its store only under its own derived prefix and the store refuses providers with another prefix. Putting the generation into the key rather than only into the object header means that a writer which lost its lease and keeps writing lands in its old epoch directory, where the new writer never creates objects, so the two can never contend for a sequence number; the generation in the key is the fence. The version 1 header carries no generation; adding one as a read-side check would be a format evolution, a header version bump whose reader keeps decoding version 1 objects, and is not part of this proposal. The header does carry an epoch, which is implemented: the epoch orders the store instances that wrote under one prefix, while the generation separates prefixes, so the two are orthogonal. The generation could later seed the epoch, but the epoch does not depend on it. Despite the name, the `epochs/` component of the key holds the generation, not the header epoch. + +**Planned migration.** A region that moves while its source node is alive follows the existing migration flow, and the cheap path applies only when that flow proves the handoff: the source stops admitting writes, flushes, and the resulting manifest watermark covers the last acknowledged entry. Then the target opens the region from that watermark and starts writing under its own prefix, and no cross-node read is needed. If the final flush fails or times out, the fact that the source process is alive proves nothing about its unflushed tail; the migration then either takes the failover path below, with the source's prefix in the chain, or is aborted. + +**Failover.** When the source is gone, the target must replay the region's unflushed tail from the source's prefix. The region's WAL position becomes a chain of segments whose last element is the target's own prefix. Each earlier segment is `(prefix, first sequence, cutover sequence, holes)`: the metasrv appends it to the region's `WalOptions` when it reassigns the region, after the source's lease has expired, and it selects the history under the source prefix at that moment by the rule the store applies at open: the complete object with the largest epoch and, among those, the largest sequence, and every object it links to. It records that chain as the segment's replay history, the cutover sequence and the sequences of the chain's objects, and the record is sealed: a present object outside the chain, whether the orphan of a create the source reported as failed or a stale object of an earlier epoch, is excluded, an object that becomes visible later, because the old writer's upload was still in flight, stays excluded, and an object above the cutover is ignored, so every open of the region replays the same source history. On open the target replays the earlier segments in order, reading only footers and its own region's segments through range reads, and never creates an object under a foreign prefix, so there is nothing to conflict with. + +Sealing may only exclude objects whose writes were never acknowledged, which takes two fences. Admission is fenced by the lease mechanism Mito already has: when a region's lease expires the datanode demotes it to a follower, which refuses writes, and the metasrv waits for that deadline before it upgrades the candidate. Completion is fenced by the store: every conditional create runs under an upload deadline, a create that has not completed by its deadline is reported failed to its waiters and is never acknowledged afterwards even if the object turns out to exist, and when a region's lease expires the store fails every waiter of that region that is not yet acknowledged, so that a create still in flight at that moment cannot be acknowledged later. The metasrv seals the segment only after the lease deadline plus the upload deadline have both passed, so an object that appears after sealing belongs to a create the old writer had already reported as failed. Under the durable acknowledgement mode this means a sealed set never excludes an acknowledged write; under the `enqueued` mode the entries in the old writer's unpersisted backlog at that moment are lost, which is that mode's stated recovery point objective. The late-completing create is part of the acceptance criteria for takeover, stated in terms of the sealing instant: an object that was absent at sealing, whether it fills a hole later or lies above the cutover, is excluded on every open of the region, and so is a visible object outside the selected chain; an object on the sealed chain replays on every open even if its write was never acknowledged, which is the same outcome as a crash between an object's creation and its acknowledgement; and in the durable mode every caller whose create had not completed when the lease expired must have been failed, so no acknowledged write is among the excluded objects. The generation in the key is an additional guarantee that such a late object lands in a directory the new writer never reads beyond the sealed set. Object-sequence-major entry ids are what make this cheap: `entry_id >> 20` locates the object under the right prefix without the source node's in-memory catalog. + +**Entry id floor across prefixes.** A region that arrives under a new prefix, by migration or by failover, brings a watermark and replayed ids that were assigned under another prefix's sequence. Before the target assigns the region its first new id, the target store raises its next object sequence above `max_id >> 20`, where `max_id` is the largest of the region's inherited watermark and every id it replayed, so that every new id of that region is greater than every id it already has; sequence gaps are tolerated and the sequence space is wide enough for this. The rule is applied on every open from the region's persisted state, in the same place Mito already hands the store the manifest watermark, so it needs no persisted state of its own and survives a crash before the target's first flush: the restart re-derives the floor and the region's acknowledged-but-unflushed entries under the target prefix are above the inherited watermark and replay. + +**Acknowledgement modes.** Both are implemented. Cluster mode is expected to run `enqueued` for latency-sensitive workloads; the object layout, the flush durability barrier and the recovery path are identical in both modes. + +**Corruption.** Segment-level skipping with the hole recorded per region is implemented, see *Object format*. *Proposed*: cluster mode routes the WAL hole event to the metasrv so that a follower or a takeover knows the region needs a fresh replica rather than a replay. + +**Garbage collection across nodes.** Within one prefix, garbage collection is implemented as described under *Entry ids*: watermark from each region's manifest, per-segment footer comparison, a reclaim boundary persisted in every header. A source prefix that still holds segments of regions which have moved away cannot be trimmed by the source alone, because it no longer learns those regions' watermarks. Either the target reports to the metasrv, after each flush, the source sequence below which it no longer needs the region, and the metasrv drives deletion, or a cluster-level janitor collects the watermarks; which of the two is the last of the *Unresolved questions*. + +# Future work + +1. Probe the next sequence with a GET from the last known sequence instead of listing the whole prefix at recovery; on a prefix with many objects the probe is cheaper than the listing. +2. Distributed mode, which needs metasrv-side allocation of node ids and generations, the takeover chain in `WalOptions`, read-only replay of a foreign prefix, and cross-node garbage collection, as sketched under *Alignment with cluster mode*. + +# Unresolved questions + +- Whether concurrent writers on one prefix need a fence beyond the epoch conflict. Conditional creates already prevent overwrites, and a concurrent writer is noticed when its epoch meets another at a sequence, which can be after both acknowledged writes. +- Whether garbage collection of a source prefix after a failover is driven by the target reporting its watermark to the metasrv, or by a cluster-level janitor that collects watermarks; the same question covers regions that are closed on a node but still own objects under its prefix. diff --git a/docs/rfcs/2026-09-11-metric-export-import.md b/docs/rfcs/2026-09-11-metric-export-import.md index 8ddc31f40b0..4b504065f6c 100644 --- a/docs/rfcs/2026-09-11-metric-export-import.md +++ b/docs/rfcs/2026-09-11-metric-export-import.md @@ -31,13 +31,21 @@ includes reports of exports taking days for roughly 100,000 logical tables with only several GiB of data. Table count, physical schema width and data volume must be measured separately; total database bytes do not describe the transfer cost. -The [experiment conclusions](2026-09-11-metric-export-import/experiment-results.md) -support shared scans, batch DDL, explicit-file COPY and packed-object transfer. -Packing with a bounded range reader substantially reduces storage requests and -improves full export and restore for many small logical tables. Benefits depend -on request latency, and memory use increases. Remaining CREATE work makes batch -DDL part of the first release. These local experiments establish direction; -production V2 recovery and release performance still require acceptance tests. +Local standalone experiments with development builds, local files and Docker +MinIO support shared scans, batch DDL, explicit-file COPY and packed-object +transfer. Packing with a bounded range reader reduces storage requests and +improves full export and restore for many small logical tables; low-latency +export gains are smaller. CPU and request overhead fell while peak process +memory increased. Remaining CREATE work makes batch DDL part of the first +release. Explicit-file COPY removes repeated directory traversal while retaining +ordinary per-table insertion, independently of merged database writes. + +The exercised fixtures preserved schemas, DDL and typed values. Standalone-file +experiments used the existing importer; packing used an independent driver and +prototype range reader. These experiments establish design direction, not +production packed-format compatibility, V2 recovery safety or release performance. +The 8 MiB write/part limit was a design choice, not a measured optimum. Current +acceptance status is maintained in the [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120). # Terminology @@ -185,11 +193,20 @@ Packed Metric exports write manifest version 2 with a required data-layout marke exports can keep version 1. A version-2 snapshot indexes all data in its schema chunks, including ordinary tables stored as standalone Parquet objects. -Before creating or replacing a snapshot, the CLI checks the source server's -explicit packed-export capability. Before target DDL or data writes, the importer +`export-v2 create --pack-metric-data` selects shared physical scans and +packed output. Omitting the flag keeps the default version-1 export path; +schema-only snapshots also retain version 1. Default frontend and standalone +configurations accept explicit packed export requests. Resume derives the layout +from the manifest, so packed snapshots do not need the flag repeated. + +Before creating or replacing a packed data snapshot, the CLI checks the source +server's explicit packed-export capability. Schema-only exports do not require +packed-data capability. Before target DDL or data writes, the importer checks the manifest version/layout and the target's packed-import capability. An unsupported packed layout fails; it must not fall through to listing `.parquet` files and silently omit packed tables. Batch-DDL capability is checked separately. +General batch-DDL and schema metadata optimizations apply independently of packed +export selection. | Reader and snapshot | Behavior | | --- | --- | @@ -424,8 +441,8 @@ The CLI retains one `COPY DATABASE FROM` operation per `(chunk, schema)` task. Add an explicit `metric_data_layout = 'packed'` COPY option, paired with `FORMAT = 'parquet'`, for the advertised capability. The server validates the index and selected destination tables, then schedules table streams by object -and offset. Use the same explicit layout option for packed export. The existing -Metric feature gate continues to control the experimental export path. +and offset. Use the same explicit layout option for packed export, with +per-table authorization and storage access checks. A reader owned by that request translates each Parquet-relative read into a bounded object range. Start with two shared 8 MiB read windows per request; @@ -504,16 +521,15 @@ the existing durable state update. - Pin supported old-reader binaries for version-2 rejection and version-1 compatibility tests. The first-release storage targets are local files and S3-compatible storage exercised through MinIO. -- Decide when release acceptance permits removing the existing experimental - gate. Packed export/restore capability is required while mixed versions exist. +- Packed export/restore capability is required while mixed versions exist. # Integration and release acceptance Use existing COPY DATABASE orchestration to form Metric export units. Retain ordinary COPY for non-Metric tables and other formats, and direct COPY TABLE -behavior. Gate the unfinished Metric fast path behind an `experimental_` -configuration option; disabling it selects the existing path. Add packed-import -support before enabling packed snapshot creation. The writer and reader form one +behavior. Explicit packed requests select the Metric fast path; ordinary exports +keep the existing path. Packed snapshot creation requires packed-export capability, +and restore requires packed-import capability. The writer and reader form one first-release feature, even though they land in separate PRs. The [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120) owns @@ -526,7 +542,7 @@ must verify its own correctness and failure behavior. Release acceptance require | Restore | Authenticated remote batch DDL, per-table authorization, fresh target IDs and reconstructed metadata, dependency ordering, older-server behavior and batch failure/retry. | | V2 lifecycle | New-reader version-1/2 coverage, old-reader version-2 rejection, target capability preflight; local/object-store interruption, multipart abort, owned-output cleanup, index/manifest failure and import replay. | | Scale | Repeated release-build measurements varying table count up to approximately 100,000, rows/file, physical union width, regions and backend. Report median/spread, control execution order/cache, and include schema export and metadata discovery in end-to-end timing. | -| Open decisions | Agreed batch-DDL interface, resource limits, compatibility targets and experimental gate. | +| Open decisions | Agreed batch-DDL interface, resource limits, compatibility targets. | ## Snapshot format acceptance @@ -552,18 +568,39 @@ batching, still writing standalone table objects; (C) B plus packing and its reader, with identical DDL handling; (D) C plus batch DDL. If general restore request deduplication is available, include it in B as well. -The proposed release target is at least 3x faster full export and full restore -for D versus A on 10,000 and 100,000 small logical tables with MinIO request -latency increased by 5 ms. Fix this target during RFC review, before running -release measurements. The independent local C-versus-B experiment establishes -neither this comparison nor proof of that target. +The release target is at least 3x faster full export and full restore for D +versus A on both 10,000 and 100,000 small logical tables at both +50 ms and ++100 ms added MinIO request latency. Populate every selected table with the same +deterministic data across controls, initially 32 rows per table. Include schema +export, metadata discovery and CLI startup/shutdown in complete wall duration. +The independent local C-versus-B experiment establishes neither this comparison +nor proof of that target. Keep production improvements already shared by all +controls in A, and identify which increments can actually be isolated. -Use Linux release builds and at least three alternating rounds for primary -comparisons. Report medians/spread, schema/data/full durations, CPU, RSS, bytes, -objects, request counts and queue/cache peaks. Include zero-added-latency, -local-file, mixed small/large-table and wide-schema controls. Investigate and -remeasure full-duration regressions exceeding 5% versus A; do not generalize the -latency-injected speedup to these cases. Check values and schemas outside timing. +Start with native optimized release builds on macOS. Record hardware, OS, +MinIO placement, container/VM CPU and memory limits, and connection/storage +settings. Linux deployment-side performance confirmation can follow separately; +Linux root/non-root and Windows correctness checks remain distinct requirements. +Earlier development-build diagnostics, runs with paging interference and sparse +fixtures with only one populated table do not establish this acceptance. + +Calibrate one added delay per object-store HTTP request, including HEAD, GET +(and range GET), PUT, LIST, DELETE and each multipart operation. Do not delay +individual packets, streaming body chunks or database SQL/DDL. Route CLI and +server object access through the same controlled endpoint and report measured +per-method latency distributions. Synthetic delay is not a cloud provider profile. + +Use a small calibration pilot before each scale and at least three alternating +A/B/C/D rounds for primary comparisons, with fresh restore namespaces, equal +concurrency and comparable cache/compaction conditions. Report medians/spread, +schema/data/full durations, CPU, RSS, swap activity, bytes, objects, requests and +queue/cache peaks. Check RAM, disk and MinIO health before large runs; do not +compile concurrently. Fix interfering paging/load or report the scale unverified. +Retain local-file and MinIO 0/+5 ms controls, mixed small/large tables, wide +physical column unions and multiple physical groups. Add +20 ms only if needed +to explain sensitivity. Investigate and remeasure full-duration regressions +exceeding 5% versus A. Check table membership, schemas and typed values outside +timing, and report B/A, C/B and D/C independently where comparable. Validate actual write/part sizes at or below 8 MiB, aggregate concurrency across groups/chunks, and resource release after success, cancellation and failure. diff --git a/docs/rfcs/2026-09-11-metric-export-import/experiment-results.md b/docs/rfcs/2026-09-11-metric-export-import/experiment-results.md deleted file mode 100644 index 9cc7b46464d..00000000000 --- a/docs/rfcs/2026-09-11-metric-export-import/experiment-results.md +++ /dev/null @@ -1,34 +0,0 @@ -# Metric export/import experiment conclusions - -The experiments support shared physical scans, explicit-file COPY, batched -logical DDL, and packing small logical-table Parquet streams into shared objects. - -- **Shared scans reduce repeated query work.** Projecting each logical schema - before dictionary expansion preserves its columns and types, but does not - remove the memory cost of a wide physical schema or distributed ordering. -- **Explicit-file COPY removes repeated directory traversal.** This improvement - uses ordinary per-table insertion; it is independent of database merged writes. -- **Packing and its range reader reduce small-object overhead.** They improve - complete export and restore and sharply reduce object-store requests. Benefits - grow with table count and request latency; low-latency export gains are smaller. - Some reader request savings can also benefit standalone files. -- **Batch DDL remains necessary.** Once data restore is faster, logical-table - creation becomes the main remaining restore cost. The batch-DDL experiment used - an internal interface; production transport and retry integration remain work. -- **Bounded writes allow larger objects.** The packing experiment kept writes - and upload parts within 8 MiB without imposing that limit on whole objects or - splitting logical tables. This was the chosen size, not a measured optimum. -- **Packing has a memory trade-off.** CPU and request overhead fell, while peak - process memory increased. Encoding buffers, upload concurrency, range windows - and table/index metadata need separate accounting. - -The exercised fixtures preserved schemas, DDL and typed values after restore. -Earlier standalone-file experiments used the existing importer; the packing -experiment used an independent driver and prototype range reader. The latter -does not establish production packed-format compatibility or V2 resume safety. - -These were local standalone experiments using development builds, local files -and Docker MinIO. Release acceptance still needs the actual packed V2 path, -request-owned reader isolation, old-reader rejection, large-table throughput, -distributed ordering, cancellation and recovery. See the [RFC](../2026-09-11-metric-export-import.md) -and [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120). diff --git a/grafana/dashboards/metrics/cluster/dashboard.json b/grafana/dashboards/metrics/cluster/dashboard.json index daa0f018836..2fffd27b25f 100644 --- a/grafana/dashboards/metrics/cluster/dashboard.json +++ b/grafana/dashboards/metrics/cluster/dashboard.json @@ -10249,7 +10249,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Compaction OPS per Instance.", + "description": "Published compaction units per second per Instance. A local unit is one input dependency group; a remote job counts as one unit.", "fieldConfig": { "defaults": { "color": { @@ -10335,9 +10335,9 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "expr": "sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", "instant": false, - "legendFormat": "[{{ instance }}]-[{{pod}}]", + "legendFormat": "[{{ instance }}]-[{{pod}}]-[{{type}}]", "range": true, "refId": "A" } @@ -10350,7 +10350,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Ongoing compaction task count", + "description": "Ongoing compaction count: local units and remote jobs", "fieldConfig": { "defaults": { "color": { @@ -10451,7 +10451,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Compaction P99 and average per Instance.", + "description": "Compaction unit P99 and average elapsed time per Instance. Local units are timed from task start through worker apply; remote jobs from request start.", "fieldConfig": { "defaults": { "color": { @@ -10537,9 +10537,9 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(instance, pod, type, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p99", "range": true, "refId": "A" }, @@ -10549,10 +10549,10 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "expr": "sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-avg", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-avg", "range": true, "refId": "B" } diff --git a/grafana/dashboards/metrics/cluster/dashboard.md b/grafana/dashboards/metrics/cluster/dashboard.md index 3278db52bfd..455268cdd43 100644 --- a/grafana/dashboards/metrics/cluster/dashboard.md +++ b/grafana/dashboards/metrics/cluster/dashboard.md @@ -117,9 +117,9 @@ | Flush Elapsed Time | `histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | | Flush Throughput | `sum by (instance, pod) (rate(greptime_mito_flush_bytes_total{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_flush_file_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-bytes` | | Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | -| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` | -| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | -| Compaction P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | +| Compaction OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Published compaction units per second per Instance. A local unit is one input dependency group; a remote job counts as one unit. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]-[{{type}}]` | +| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction count: local units and remote jobs | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | +| Compaction P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, type, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction unit P99 and average elapsed time per Instance. Local units are timed from task start through worker apply; remote jobs from request start. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p99` | | Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` | | Compaction Input/Output Bytes | `sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))`
`sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))` | `timeseries` | Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-input` | # Index diff --git a/grafana/dashboards/metrics/cluster/dashboard.yaml b/grafana/dashboards/metrics/cluster/dashboard.yaml index bc659b401b5..7513c5e50e1 100644 --- a/grafana/dashboards/metrics/cluster/dashboard.yaml +++ b/grafana/dashboards/metrics/cluster/dashboard.yaml @@ -1417,17 +1417,17 @@ groups: legendFormat: '[{{instance}}]-[{{pod}}]' - title: Compaction OPS per Instance type: timeseries - description: Compaction OPS per Instance. + description: Published compaction units per second per Instance. A local unit is one input dependency group; a remote job counts as one unit. unit: ops queries: - - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval])) + - expr: sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{ instance }}]-[{{pod}}]' + legendFormat: '[{{ instance }}]-[{{pod}}]-[{{type}}]' - title: Inflight Compaction type: timeseries - description: Ongoing compaction task count + description: 'Ongoing compaction count: local units and remote jobs' unit: none queries: - expr: greptime_mito_inflight_compaction_count @@ -1437,19 +1437,19 @@ groups: legendFormat: '[{{instance}}]-[{{pod}}]' - title: Compaction P99 and Avg per Instance type: timeseries - description: Compaction P99 and average per Instance. + description: Compaction unit P99 and average elapsed time per Instance. Local units are timed from task start through worker apply; remote jobs from request start. unit: s queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) + - expr: histogram_quantile(0.99, sum by(instance, pod, type, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-p99' - - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval])) + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99' + - expr: sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-avg' + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-avg' - title: Compaction Elapsed Time per Instance by Stage type: timeseries description: Compaction latency by stage diff --git a/grafana/dashboards/metrics/standalone/dashboard.json b/grafana/dashboards/metrics/standalone/dashboard.json index c1fda955d90..f17d40b4f96 100644 --- a/grafana/dashboards/metrics/standalone/dashboard.json +++ b/grafana/dashboards/metrics/standalone/dashboard.json @@ -10249,7 +10249,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Compaction OPS per Instance.", + "description": "Published compaction units per second per Instance. A local unit is one input dependency group; a remote job counts as one unit.", "fieldConfig": { "defaults": { "color": { @@ -10335,9 +10335,9 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))", + "expr": "sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))", "instant": false, - "legendFormat": "[{{ instance }}]-[{{pod}}]", + "legendFormat": "[{{ instance }}]-[{{pod}}]-[{{type}}]", "range": true, "refId": "A" } @@ -10350,7 +10350,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Ongoing compaction task count", + "description": "Ongoing compaction count: local units and remote jobs", "fieldConfig": { "defaults": { "color": { @@ -10451,7 +10451,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Compaction P99 and average per Instance.", + "description": "Compaction unit P99 and average elapsed time per Instance. Local units are timed from task start through worker apply; remote jobs from request start.", "fieldConfig": { "defaults": { "color": { @@ -10537,9 +10537,9 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(instance, pod, type, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval])))", "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p99", "range": true, "refId": "A" }, @@ -10549,10 +10549,10 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))", + "expr": "sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-avg", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-avg", "range": true, "refId": "B" } diff --git a/grafana/dashboards/metrics/standalone/dashboard.md b/grafana/dashboards/metrics/standalone/dashboard.md index 5dd6f1600e1..5c7a4042444 100644 --- a/grafana/dashboards/metrics/standalone/dashboard.md +++ b/grafana/dashboards/metrics/standalone/dashboard.md @@ -117,9 +117,9 @@ | Flush Elapsed Time | `histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))`
`sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count[$__rate_interval]))` | `timeseries` | Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | | Flush Throughput | `sum by (instance, pod) (rate(greptime_mito_flush_bytes_total[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_flush_file_total[$__rate_interval]))` | `timeseries` | Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-bytes` | | Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | -| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` | -| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | -| Compaction P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | +| Compaction OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))` | `timeseries` | Published compaction units per second per Instance. A local unit is one input dependency group; a remote job counts as one unit. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]-[{{type}}]` | +| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction count: local units and remote jobs | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | +| Compaction P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, type, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction unit P99 and average elapsed time per Instance. Local units are timed from task start through worker apply; remote jobs from request start. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p99` | | Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` | | Compaction Input/Output Bytes | `sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))`
`sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))` | `timeseries` | Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-input` | # Index diff --git a/grafana/dashboards/metrics/standalone/dashboard.yaml b/grafana/dashboards/metrics/standalone/dashboard.yaml index beadb9c3215..627cd8b5d63 100644 --- a/grafana/dashboards/metrics/standalone/dashboard.yaml +++ b/grafana/dashboards/metrics/standalone/dashboard.yaml @@ -1417,17 +1417,17 @@ groups: legendFormat: '[{{instance}}]-[{{pod}}]' - title: Compaction OPS per Instance type: timeseries - description: Compaction OPS per Instance. + description: Published compaction units per second per Instance. A local unit is one input dependency group; a remote job counts as one unit. unit: ops queries: - - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval])) + - expr: sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{ instance }}]-[{{pod}}]' + legendFormat: '[{{ instance }}]-[{{pod}}]-[{{type}}]' - title: Inflight Compaction type: timeseries - description: Ongoing compaction task count + description: 'Ongoing compaction count: local units and remote jobs' unit: none queries: - expr: greptime_mito_inflight_compaction_count @@ -1437,19 +1437,19 @@ groups: legendFormat: '[{{instance}}]-[{{pod}}]' - title: Compaction P99 and Avg per Instance type: timeseries - description: Compaction P99 and average per Instance. + description: Compaction unit P99 and average elapsed time per Instance. Local units are timed from task start through worker apply; remote jobs from request start. unit: s queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval]))) + - expr: histogram_quantile(0.99, sum by(instance, pod, type, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-p99' - - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval])) + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99' + - expr: sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-avg' + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-avg' - title: Compaction Elapsed Time per Instance by Stage type: timeseries description: Compaction latency by stage diff --git a/src/api/src/error.rs b/src/api/src/error.rs index 0b35bbc934f..f6694c68679 100644 --- a/src/api/src/error.rs +++ b/src/api/src/error.rs @@ -60,6 +60,14 @@ pub enum Error { source: datatypes::error::Error, }, + #[snafu(display("Invalid compression options for column {column}"))] + InvalidColumnCompression { + column: String, + #[snafu(implicit)] + location: Location, + source: datatypes::error::Error, + }, + #[snafu(display("Failed to serialize JSON"))] SerializeJson { #[snafu(source)] @@ -93,7 +101,8 @@ impl ErrorExt for Error { StatusCode::Unexpected } Error::ConvertColumnDefaultConstraint { source, .. } - | Error::InvalidColumnDefaultConstraint { source, .. } => source.status_code(), + | Error::InvalidColumnDefaultConstraint { source, .. } + | Error::InvalidColumnCompression { source, .. } => source.status_code(), } } diff --git a/src/api/src/v1/column_def.rs b/src/api/src/v1/column_def.rs index b02b06feafd..af62ed716e5 100644 --- a/src/api/src/v1/column_def.rs +++ b/src/api/src/v1/column_def.rs @@ -16,9 +16,9 @@ use std::collections::HashMap; use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY}; use datatypes::schema::{ - COMMENT_KEY, ColumnDefaultConstraint, ColumnSchema, FULLTEXT_KEY, FulltextAnalyzer, - FulltextBackend, FulltextOptions, INVERTED_INDEX_KEY, Metadata, SKIPPING_INDEX_KEY, - SkippingIndexOptions, SkippingIndexType, + COMMENT_KEY, COMPRESSION_KEY, ColumnDefaultConstraint, ColumnSchema, FULLTEXT_KEY, + FulltextAnalyzer, FulltextBackend, FulltextOptions, INVERTED_INDEX_KEY, Metadata, + SKIPPING_INDEX_KEY, SkippingIndexOptions, SkippingIndexType, }; use greptime_proto::v1::{ Analyzer, FulltextBackend as PbFulltextBackend, SkippingIndexType as PbSkippingIndexType, @@ -35,11 +35,14 @@ const FULLTEXT_GRPC_KEY: &str = "fulltext"; const INVERTED_INDEX_GRPC_KEY: &str = "inverted_index"; /// Key used to store skip index options in gRPC column options. const SKIPPING_INDEX_GRPC_KEY: &str = "skipping_index"; +/// Key used to store SST compression options in gRPC column options. +const COMPRESSION_GRPC_KEY: &str = "compression"; -const COLUMN_OPTION_MAPPINGS: [(&str, &str); 5] = [ +const COLUMN_OPTION_MAPPINGS: [(&str, &str); 6] = [ (FULLTEXT_GRPC_KEY, FULLTEXT_KEY), (INVERTED_INDEX_GRPC_KEY, INVERTED_INDEX_KEY), (SKIPPING_INDEX_GRPC_KEY, SKIPPING_INDEX_KEY), + (COMPRESSION_GRPC_KEY, COMPRESSION_KEY), (EXTENSION_TYPE_NAME_KEY, EXTENSION_TYPE_NAME_KEY), (EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_METADATA_KEY), ]; @@ -77,6 +80,9 @@ pub fn try_as_column_schema(column_def: &ColumnDef) -> Result { if let Some(skipping_index) = options.options.get(SKIPPING_INDEX_GRPC_KEY) { metadata.insert(SKIPPING_INDEX_KEY.to_string(), skipping_index.to_owned()); } + if let Some(compression) = options.options.get(COMPRESSION_GRPC_KEY) { + metadata.insert(COMPRESSION_KEY.to_string(), compression.to_owned()); + } if let Some(extension_name) = options.options.get(EXTENSION_TYPE_NAME_KEY) { metadata.insert(EXTENSION_TYPE_NAME_KEY.to_string(), extension_name.clone()); } @@ -88,8 +94,14 @@ pub fn try_as_column_schema(column_def: &ColumnDef) -> Result { } } - ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable) - .with_metadata(metadata) + let schema = ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable) + .with_metadata(metadata); + schema + .compression_options() + .context(error::InvalidColumnCompressionSnafu { + column: &column_def.name, + })?; + schema .with_time_index(column_def.semantic_type() == SemanticType::Timestamp) .with_default_constraint(constraint) .context(error::InvalidColumnDefaultConstraintSnafu { @@ -172,6 +184,11 @@ pub fn options_from_column_schema(column_schema: &ColumnSchema) -> Option Skipp #[cfg(test)] mod tests { + use common_base::readable_size::ReadableSize; use datatypes::data_type::ConcreteDataType; - use datatypes::schema::{FulltextAnalyzer, FulltextBackend}; + use datatypes::schema::{CompressionOptions, FulltextAnalyzer, FulltextBackend}; use super::*; use crate::v1::ColumnDataType; @@ -307,6 +325,54 @@ mod tests { assert!(schema.is_inverted_indexed()); } + #[test] + fn test_compression_options_round_trip() { + let options = CompressionOptions { + level: Some(3), + page_size: Some(ReadableSize::mb(8)), + ..Default::default() + }; + let schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true) + .with_compression_options(options) + .unwrap(); + + let column_def = try_as_column_def(&schema, false).unwrap(); + let schema = try_as_column_schema(&column_def).unwrap(); + assert_eq!(Some(options), schema.compression_options().unwrap()); + + let metadata = collect_column_options(column_def.options.as_ref()); + assert!(metadata.contains_key(COMPRESSION_KEY)); + } + + #[test] + fn test_reject_invalid_compression_metadata() { + use common_error::ext::ErrorExt; + use common_error::status_code::StatusCode; + + for (data_type, json) in [ + (ConcreteDataType::string_datatype(), r#"{"level":23}"#), + (ConcreteDataType::string_datatype(), r#"{"level":0}"#), + (ConcreteDataType::string_datatype(), r#"{"page_size":"0B"}"#), + (ConcreteDataType::string_datatype(), r#"{"type":"lz4"}"#), + (ConcreteDataType::string_datatype(), "not json"), + ( + ConcreteDataType::list_datatype(std::sync::Arc::new( + ConcreteDataType::string_datatype(), + )), + r#"{"level":3}"#, + ), + ] { + let schema = + ColumnSchema::new("test", data_type, true).with_metadata(HashMap::from([( + COMPRESSION_KEY.to_string(), + json.to_string(), + )])); + let column_def = try_as_column_def(&schema, false).unwrap(); + let err = try_as_column_schema(&column_def).unwrap_err(); + assert_eq!(StatusCode::InvalidArguments, err.status_code(), "{json}"); + } + } + #[test] fn test_options_from_column_schema() { let schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true); diff --git a/src/catalog/src/kvbackend/manager.rs b/src/catalog/src/kvbackend/manager.rs index 2afc77c0d23..0beb2c0e1f5 100644 --- a/src/catalog/src/kvbackend/manager.rs +++ b/src/catalog/src/kvbackend/manager.rs @@ -94,6 +94,11 @@ pub struct KvBackendCatalogManager { pub(super) const CATALOG_CACHE_MAX_CAPACITY: u64 = 128; impl KvBackendCatalogManager { + /// Returns the registered system tables, excluding user-created metadata. + pub fn information_schema_provider(&self) -> &InformationSchemaProvider { + &self.system_catalog.information_schema_provider + } + pub fn view_info_cache(&self) -> Result { self.cache_registry.get().context(CacheNotFoundSnafu { name: "view_info_cache", diff --git a/src/catalog/src/system_schema.rs b/src/catalog/src/system_schema.rs index add91cc75f4..099b8832f40 100644 --- a/src/catalog/src/system_schema.rs +++ b/src/catalog/src/system_schema.rs @@ -33,9 +33,8 @@ use snafu::ResultExt; use store_api::data_source::DataSource; use store_api::storage::ScanRequest; use table::error::{SchemaConversionSnafu, TablesRecordBatchSnafu}; -use table::metadata::{ - FilterPushDownType, TableId, TableInfoBuilder, TableInfoRef, TableMetaBuilder, TableType, -}; +use table::metadata::{FilterPushDownType, TableId, TableInfoBuilder, TableMetaBuilder, TableType}; +use table::requests::TableOptions; use table::{Table, TableRef}; use crate::error::Result; @@ -71,33 +70,44 @@ trait SystemSchemaProviderInner { fn schema_name() -> &'static str; fn build_table(&self, name: &str) -> Option { self.system_table(name).map(|table| { - let table_info = Self::table_info(self.catalog_name().to_string(), &table); - let filter_pushdown = FilterPushDownType::Inexact; - let data_source = Arc::new(SystemTableDataSource::new(table)); - let table = Table::new(table_info, filter_pushdown, data_source); - Arc::new(table) + build_system_table( + self.catalog_name().to_string(), + Self::schema_name(), + table, + TableOptions::default(), + ) }) } fn system_table(&self, name: &str) -> Option; +} - fn table_info(catalog_name: String, table: &SystemTableRef) -> TableInfoRef { - let table_meta = TableMetaBuilder::empty() - .schema(table.schema()) - .primary_key_indices(vec![]) - .next_column_id(0) - .build() - .unwrap(); - let table_info = TableInfoBuilder::default() - .table_id(table.table_id()) - .name(table.table_name().to_string()) - .catalog_name(catalog_name) - .schema_name(Self::schema_name().to_string()) - .meta(table_meta) - .table_type(table.table_type()) - .build() - .unwrap(); - Arc::new(table_info) - } +fn build_system_table( + catalog_name: String, + schema_name: &str, + table: SystemTableRef, + options: TableOptions, +) -> TableRef { + let table_meta = TableMetaBuilder::empty() + .schema(table.schema()) + .primary_key_indices(vec![]) + .next_column_id(0) + .options(options) + .build() + .unwrap(); + let table_info = TableInfoBuilder::default() + .table_id(table.table_id()) + .name(table.table_name().to_string()) + .catalog_name(catalog_name) + .schema_name(schema_name.to_string()) + .meta(table_meta) + .table_type(table.table_type()) + .build() + .unwrap(); + Arc::new(Table::new( + Arc::new(table_info), + FilterPushDownType::Inexact, + Arc::new(SystemTableDataSource::new(table)), + )) } pub trait SystemTable { diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs index 8c3f4bf0f5f..b2145df9cd3 100644 --- a/src/catalog/src/system_schema/information_schema.rs +++ b/src/catalog/src/system_schema/information_schema.rs @@ -38,7 +38,7 @@ mod views; #[cfg(all(test, feature = "enterprise"))] mod recycle_bin_test; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Weak}; use common_catalog::consts::{self, DEFAULT_CATALOG_NAME, INFORMATION_SCHEMA_NAME}; @@ -63,15 +63,15 @@ use store_api::metric_engine_consts::{ }; use store_api::region_info::RegionInfoEntry; use store_api::sst_entry::{ManifestSstEntry, PuffinIndexMetaEntry, StorageSstEntry}; -use store_api::storage::{ScanRequest, TableId}; +use store_api::storage::ScanRequest; use table::TableRef; -use table::metadata::TableType; +use table::requests::{SEMANTIC_TABLE_SCOPE, TableOptions}; pub use table_names::*; use views::InformationSchemaViews; use self::columns::InformationSchemaColumns; use crate::CatalogManager; -use crate::error::{Error, Result}; +use crate::error::Error; use crate::process_manager::ProcessManagerRef; use crate::system_schema::information_schema::cluster_info::InformationSchemaClusterInfo; use crate::system_schema::information_schema::flow_statistics::InformationSchemaFlowStatistics; @@ -92,9 +92,7 @@ use crate::system_schema::information_schema::table_semantics::InformationSchema use crate::system_schema::information_schema::tables::InformationSchemaTables; use crate::system_schema::memory_table::MemoryTable; pub(crate) use crate::system_schema::predicate::Predicates; -use crate::system_schema::{ - SystemSchemaProvider, SystemSchemaProviderInner, SystemTable, SystemTableRef, -}; +use crate::system_schema::{SystemSchemaProvider, SystemTable, build_system_table}; const DENSE_PRIMARY_KEY_ENCODING: &str = "dense"; const SPARSE_PRIMARY_KEY_ENCODING: &str = "sparse"; @@ -134,7 +132,6 @@ lazy_static! { TABLE_PRIVILEGES, GLOBAL_STATUS, SESSION_STATUS, - PARTITIONS, PLUGINS, USER_PRIVILEGES, PROCESSLIST, @@ -163,12 +160,31 @@ pub struct MakeInformationTableRequest { pub kv_backend: KvBackendRef, } +/// The visibility guaranteed by an information-schema table provider. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InformationSchemaTableScope { + /// Every query is filtered to the catalog supplied when constructing the provider. + Catalog, + /// The table contains catalog-independent or shared data, including compatibility stubs. + Global, +} + +impl InformationSchemaTableScope { + pub const fn as_str(self) -> &'static str { + match self { + Self::Catalog => "catalog", + Self::Global => "global", + } + } +} + /// A factory trait for making information schema tables. /// /// This trait allows for extensibility of the information schema by providing /// a way to dynamically create custom information schema tables. pub trait InformationSchemaTableFactory { - fn make_information_table(&self, req: MakeInformationTableRequest) -> SystemTableRef; + /// Creates a table whose scope is declared by its [`InformationTable`] implementation. + fn make_information_table(&self, req: MakeInformationTableRequest) -> InformationTableRef; } pub type InformationSchemaTableFactoryRef = Arc; @@ -192,15 +208,8 @@ impl SystemSchemaProvider for InformationSchemaProvider { } } -impl SystemSchemaProviderInner for InformationSchemaProvider { - fn catalog_name(&self) -> &str { - &self.catalog_name - } - fn schema_name() -> &'static str { - INFORMATION_SCHEMA_NAME - } - - fn system_table(&self, name: &str) -> Option { +impl InformationSchemaProvider { + fn information_table(&self, name: &str) -> Option { if let Some(factory) = self.extra_table_factories.get(name) { let req = MakeInformationTableRequest { catalog_name: self.catalog_name.clone(), @@ -357,142 +366,76 @@ impl InformationSchemaProvider { } fn build_tables(&mut self) { - let mut tables = HashMap::new(); + let mut table_names = HashSet::new(); // SECURITY NOTE: // Carefully consider the tables that may expose sensitive cluster configurations, // authentication details, and other critical information. // Only put these tables under `greptime` catalog to prevent info leak. if self.catalog_name == DEFAULT_CATALOG_NAME { - tables.insert( - BUILD_INFO.to_string(), - self.build_table(BUILD_INFO).unwrap(), - ); - tables.insert( - REGION_PEERS.to_string(), - self.build_table(REGION_PEERS).unwrap(), - ); - tables.insert( - CLUSTER_INFO.to_string(), - self.build_table(CLUSTER_INFO).unwrap(), - ); - tables.insert( - PROCEDURE_INFO.to_string(), - self.build_table(PROCEDURE_INFO).unwrap(), - ); - tables.insert( - REGION_STATISTICS.to_string(), - self.build_table(REGION_STATISTICS).unwrap(), - ); - tables.insert( - REGION_INFO.to_string(), - self.build_table(REGION_INFO).unwrap(), - ); - tables.insert( - SSTS_MANIFEST.to_string(), - self.build_table(SSTS_MANIFEST).unwrap(), - ); - tables.insert( - SSTS_STORAGE.to_string(), - self.build_table(SSTS_STORAGE).unwrap(), - ); - tables.insert( - SSTS_INDEX_META.to_string(), - self.build_table(SSTS_INDEX_META).unwrap(), - ); + table_names.extend([ + BUILD_INFO, + REGION_PEERS, + CLUSTER_INFO, + PROCEDURE_INFO, + REGION_STATISTICS, + REGION_INFO, + SSTS_MANIFEST, + SSTS_STORAGE, + SSTS_INDEX_META, + ]); } - tables.insert(TABLES.to_string(), self.build_table(TABLES).unwrap()); - tables.insert(VIEWS.to_string(), self.build_table(VIEWS).unwrap()); - tables.insert(SCHEMATA.to_string(), self.build_table(SCHEMATA).unwrap()); - tables.insert(COLUMNS.to_string(), self.build_table(COLUMNS).unwrap()); - tables.insert( - KEY_COLUMN_USAGE.to_string(), - self.build_table(KEY_COLUMN_USAGE).unwrap(), - ); - tables.insert( - TABLE_CONSTRAINTS.to_string(), - self.build_table(TABLE_CONSTRAINTS).unwrap(), - ); - tables.insert( - STATISTICS.to_string(), - self.build_table(STATISTICS).unwrap(), - ); - tables.insert(FLOWS.to_string(), self.build_table(FLOWS).unwrap()); - tables.insert( - FLOW_STATISTICS.to_string(), - self.build_table(FLOW_STATISTICS).unwrap(), - ); - #[cfg(feature = "enterprise")] - tables.insert( - RECYCLE_BIN.to_string(), - self.build_table(RECYCLE_BIN).unwrap(), - ); - tables.insert( - TABLE_SEMANTICS.to_string(), - self.build_table(TABLE_SEMANTICS).unwrap(), - ); - if let Some(process_list) = self.build_table(PROCESS_LIST) { - tables.insert(PROCESS_LIST.to_string(), process_list); + table_names.extend([ + TABLES, + VIEWS, + SCHEMATA, + COLUMNS, + KEY_COLUMN_USAGE, + TABLE_CONSTRAINTS, + STATISTICS, + FLOWS, + FLOW_STATISTICS, + #[cfg(feature = "enterprise")] + RECYCLE_BIN, + TABLE_SEMANTICS, + PARTITIONS, + ]); + if self.process_manager.is_some() { + table_names.insert(PROCESS_LIST); } - for name in self.extra_table_factories.keys() { - tables.insert(name.clone(), self.build_table(name).expect(name)); - } - // Add memory tables - for name in MEMORY_TABLES.iter() { - tables.insert((*name).to_string(), self.build_table(name).expect(name)); - } - self.tables = tables; + table_names.extend(MEMORY_TABLES.iter().copied()); + table_names.extend(self.extra_table_factories.keys().map(String::as_str)); + self.tables = table_names + .into_iter() + .map(|name| { + let table = self.information_table(name).expect(name); + let options = TableOptions { + extra_options: HashMap::from([( + SEMANTIC_TABLE_SCOPE.to_string(), + table.scope().as_str().to_string(), + )]), + ..Default::default() + }; + let table = build_system_table( + self.catalog_name.clone(), + INFORMATION_SCHEMA_NAME, + table, + options, + ); + (name.to_string(), table) + }) + .collect(); } } -pub trait InformationTable { - fn table_id(&self) -> TableId; - - fn table_name(&self) -> &'static str; - - fn schema(&self) -> SchemaRef; - - fn to_stream(&self, request: ScanRequest) -> Result; - - fn scan_plan(&self, _request: ScanRequest) -> Result>> { - Ok(None) - } - - fn table_type(&self) -> TableType { - TableType::Temporary - } +/// An information-schema table with an explicit visibility guarantee. +pub trait InformationTable: SystemTable { + /// Returns the visibility guaranteed by this table's query implementation. + fn scope(&self) -> InformationSchemaTableScope; } -// Provide compatibility for legacy `information_schema` code. -impl SystemTable for T -where - T: InformationTable, -{ - fn table_id(&self) -> TableId { - InformationTable::table_id(self) - } - - fn table_name(&self) -> &'static str { - InformationTable::table_name(self) - } - - fn schema(&self) -> SchemaRef { - InformationTable::schema(self) - } - - fn table_type(&self) -> TableType { - InformationTable::table_type(self) - } - - fn to_stream(&self, request: ScanRequest) -> Result { - InformationTable::to_stream(self, request) - } - - fn scan_plan(&self, request: ScanRequest) -> Result>> { - InformationTable::scan_plan(self, request) - } -} +pub type InformationTableRef = Arc; pub type InformationExtensionRef = Arc + Send + Sync>; @@ -597,9 +540,252 @@ impl InformationExtension for NoopInformationExtension { #[cfg(test)] mod tests { + use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry}; + use common_meta::cache::{CacheRegistryBuilder, LayeredCacheRegistryBuilder}; + use common_meta::ddl::test_util::create_table::test_create_table_task; + use common_meta::key::schema_name::SchemaNameKey; + use common_meta::key::table_route::TableRouteValue; + use common_meta::kv_backend::memory::MemoryKvBackend; + use common_meta::kv_backend::txn::TxnService; + use common_meta::wal_provider::RegionWalOptions; + use datatypes::schema::Schema; + use futures_util::TryStreamExt; use store_api::region_info::RegionInfoEntry; + use store_api::storage::TableId; use super::*; + use crate::error::Result; + use crate::information_schema::NoopInformationExtension; + use crate::kvbackend::KvBackendCatalogManagerBuilder; + use crate::memory::MemoryCatalogManager; + use crate::process_manager::ProcessManager; + + struct ScopedFactory { + name: &'static str, + scope: InformationSchemaTableScope, + } + + impl InformationSchemaTableFactory for ScopedFactory { + fn make_information_table(&self, _req: MakeInformationTableRequest) -> InformationTableRef { + Arc::new(ScopedTable { + name: self.name, + scope: self.scope, + }) + } + } + + struct ScopedTable { + name: &'static str, + scope: InformationSchemaTableScope, + } + + impl InformationTable for ScopedTable { + fn scope(&self) -> InformationSchemaTableScope { + self.scope + } + } + + impl SystemTable for ScopedTable { + fn table_id(&self) -> TableId { + 999 + } + + fn table_name(&self) -> &'static str { + self.name + } + + fn schema(&self) -> SchemaRef { + Arc::new(Schema::new(vec![])) + } + + fn to_stream(&self, _request: ScanRequest) -> Result { + Ok(common_recordbatch::RecordBatches::empty().as_stream()) + } + } + + #[test] + fn every_registered_information_schema_table_has_scope() { + use InformationSchemaTableScope::{Catalog, Global}; + + let manager: Arc = MemoryCatalogManager::new(); + let backend = Arc::new(MemoryKvBackend::default()); + let factories: HashMap<_, InformationSchemaTableFactoryRef> = [ + ("future_catalog_table", Catalog), + ("future_global_table", Global), + (ENGINES, Catalog), + ] + .into_iter() + .map(|(name, scope)| { + ( + name.to_string(), + Arc::new(ScopedFactory { name, scope }) as _, + ) + }) + .collect(); + let catalog_tables = [ + TABLES, + VIEWS, + SCHEMATA, + COLUMNS, + KEY_COLUMN_USAGE, + TABLE_CONSTRAINTS, + STATISTICS, + FLOWS, + FLOW_STATISTICS, + TABLE_SEMANTICS, + PARTITIONS, + REGION_PEERS, + #[cfg(feature = "enterprise")] + RECYCLE_BIN, + "future_catalog_table", + ENGINES, + ]; + + for (catalog, with_process_manager) in [ + (DEFAULT_CATALOG_NAME, true), + (DEFAULT_CATALOG_NAME, false), + ("tenant", true), + ("tenant", false), + ] { + let provider = InformationSchemaProvider::new( + catalog.to_string(), + Arc::downgrade(&manager), + Arc::new(FlowMetadataManager::new(backend.clone())), + with_process_manager.then(|| Arc::new(ProcessManager::new(String::new(), None))), + backend.clone(), + ); + assert_eq!( + provider + .table(ENGINES) + .unwrap() + .table_info() + .meta + .options + .extra_options + .get(SEMANTIC_TABLE_SCOPE) + .map(String::as_str), + Some(Global.as_str()), + ); + // An extension replacing a built-in supplies the replacement table's scope. + let provider = provider.with_extra_table_factories(factories.clone()); + let expected_count = + 43 + usize::from(with_process_manager) + usize::from(cfg!(feature = "enterprise")) + - if catalog == DEFAULT_CATALOG_NAME { + 0 + } else { + 9 + }; + assert_eq!(provider.tables().len(), expected_count); + for (name, table) in provider.tables() { + let scope = if catalog_tables.contains(&name.as_str()) { + Catalog + } else { + Global + }; + assert_eq!( + table + .table_info() + .meta + .options + .extra_options + .get(SEMANTIC_TABLE_SCOPE) + .map(String::as_str), + Some(scope.as_str()), + "{catalog}.{name}" + ); + } + assert_eq!( + provider.table(REGION_PEERS).is_some(), + catalog == DEFAULT_CATALOG_NAME + ); + assert_eq!(provider.table(PROCESS_LIST).is_some(), with_process_manager); + if let Some(table) = provider.table(PROCESS_LIST) { + assert!(table.schema().column_schema_by_name("catalog").is_some()); + } + } + } + + #[tokio::test] + async fn newly_accessible_information_schema_tables_filter_catalogs() { + let backend = Arc::new(MemoryKvBackend::default()); + let caches = LayeredCacheRegistryBuilder::default() + .add_cache_registry(CacheRegistryBuilder::default().build()) + .add_cache_registry(build_fundamental_cache_registry(backend.clone())); + let manager = KvBackendCatalogManagerBuilder::new( + Arc::new(NoopInformationExtension), + backend.clone(), + Arc::new( + with_default_composite_cache_registry(caches) + .unwrap() + .build(), + ), + ) + .build(); + let metadata = manager.table_metadata_manager_ref(); + let flow_metadata = FlowMetadataManager::new(backend.clone()); + for (catalog, name, id) in [ + ("tenant_a", "visible_a", 1024), + ("tenant_b", "visible_b", 1025), + ] { + metadata + .schema_manager() + .create(SchemaNameKey::new(catalog, "public"), None, false) + .await + .unwrap(); + let mut info = test_create_table_task(name, id).table_info; + info.catalog_name = catalog.to_string(); + info.meta + .options + .extra_options + .insert(SEMANTIC_TABLE_SCOPE.to_string(), "catalog".to_string()); + metadata + .create_table_metadata( + info, + TableRouteValue::physical(vec![]), + RegionWalOptions::default(), + ) + .await + .unwrap(); + let (txn, _) = flow_metadata + .flow_name_manager() + .build_create_txn(catalog, name, id) + .unwrap(); + backend.txn(txn).await.unwrap(); + } + + for (catalog, visible, hidden) in [ + ("tenant_a", "visible_a", "visible_b"), + ("tenant_b", "visible_b", "visible_a"), + ] { + for name in [TABLE_SEMANTICS, STATISTICS, FLOW_STATISTICS] { + let table = manager + .table(catalog, INFORMATION_SCHEMA_NAME, name, None) + .await + .unwrap() + .unwrap(); + let batches = table + .scan_to_stream(ScanRequest::default()) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let output = batches + .iter() + .map(|batch| batch.pretty_print()) + .collect::>() + .join("\n"); + assert!(output.contains(visible), "{catalog}.{name}: {output}"); + assert!(!output.contains(hidden), "{catalog}.{name}: {output}"); + } + } + assert!( + !manager + .information_schema_provider() + .tables() + .contains_key("visible_a") + ); + } #[test] fn test_datanode_inspect_region_info_build_plan() { diff --git a/src/catalog/src/system_schema/information_schema/cluster_info.rs b/src/catalog/src/system_schema/information_schema/cluster_info.rs index 1ba1a55fb63..20ecba07d87 100644 --- a/src/catalog/src/system_schema/information_schema/cluster_info.rs +++ b/src/catalog/src/system_schema/information_schema/cluster_info.rs @@ -40,7 +40,10 @@ use store_api::storage::{ScanRequest, TableId}; use crate::CatalogManager; use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result}; -use crate::system_schema::information_schema::{CLUSTER_INFO, InformationTable, Predicates}; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{ + CLUSTER_INFO, InformationSchemaTableScope, InformationTable, Predicates, +}; use crate::system_schema::utils; const PEER_TYPE_FRONTEND: &str = "FRONTEND"; @@ -139,6 +142,12 @@ impl InformationSchemaClusterInfo { } impl InformationTable for InformationSchemaClusterInfo { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Global + } +} + +impl SystemTable for InformationSchemaClusterInfo { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_CLUSTER_INFO_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/columns.rs b/src/catalog/src/system_schema/information_schema/columns.rs index e92efd3bebf..6e0524390ee 100644 --- a/src/catalog/src/system_schema/information_schema/columns.rs +++ b/src/catalog/src/system_schema/information_schema/columns.rs @@ -43,7 +43,10 @@ use crate::error::{ CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu, }; use crate::information_schema::Predicates; -use crate::system_schema::information_schema::{COLUMNS, InformationTable}; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{ + COLUMNS, InformationSchemaTableScope, InformationTable, +}; #[derive(Debug)] pub(super) struct InformationSchemaColumns { @@ -160,6 +163,12 @@ impl InformationSchemaColumns { } impl InformationTable for InformationSchemaColumns { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaColumns { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_COLUMNS_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/flow_statistics.rs b/src/catalog/src/system_schema/information_schema/flow_statistics.rs index 6163461eda7..218104765fd 100644 --- a/src/catalog/src/system_schema/information_schema/flow_statistics.rs +++ b/src/catalog/src/system_schema/information_schema/flow_statistics.rs @@ -41,7 +41,8 @@ use store_api::storage::{ScanRequest, TableId}; use crate::CatalogManager; use crate::error::{CreateRecordBatchSnafu, InternalSnafu, ListFlowsSnafu, Result}; use crate::information_schema::{FLOW_STATISTICS, Predicates}; -use crate::system_schema::information_schema::InformationTable; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{InformationSchemaTableScope, InformationTable}; use crate::system_schema::utils; const INIT_CAPACITY: usize = 42; @@ -108,6 +109,12 @@ impl InformationSchemaFlowStatistics { } impl InformationTable for InformationSchemaFlowStatistics { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaFlowStatistics { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_FLOW_STATISTICS_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/flows.rs b/src/catalog/src/system_schema/information_schema/flows.rs index 12a8783b58e..5530d67bb8e 100644 --- a/src/catalog/src/system_schema/information_schema/flows.rs +++ b/src/catalog/src/system_schema/information_schema/flows.rs @@ -50,7 +50,8 @@ use crate::error::{ Result, UpgradeWeakCatalogManagerRefSnafu, }; use crate::information_schema::{FLOWS, Predicates}; -use crate::system_schema::information_schema::InformationTable; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{InformationSchemaTableScope, InformationTable}; use crate::system_schema::utils; const INIT_CAPACITY: usize = 42; @@ -195,6 +196,12 @@ impl InformationSchemaFlows { } impl InformationTable for InformationSchemaFlows { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaFlows { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_FLOW_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/key_column_usage.rs b/src/catalog/src/system_schema/information_schema/key_column_usage.rs index 4f8f116afaf..46d10829758 100644 --- a/src/catalog/src/system_schema/information_schema/key_column_usage.rs +++ b/src/catalog/src/system_schema/information_schema/key_column_usage.rs @@ -35,8 +35,10 @@ use crate::CatalogManager; use crate::error::{ CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu, }; +use crate::system_schema::SystemTable; use crate::system_schema::information_schema::{ - InformationTable, KEY_COLUMN_USAGE, Predicates, primary_key_encoding_index_type, + InformationSchemaTableScope, InformationTable, KEY_COLUMN_USAGE, Predicates, + primary_key_encoding_index_type, }; pub const CONSTRAINT_SCHEMA: &str = "constraint_schema"; @@ -148,6 +150,12 @@ impl InformationSchemaKeyColumnUsage { } impl InformationTable for InformationSchemaKeyColumnUsage { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaKeyColumnUsage { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_KEY_COLUMN_USAGE_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/partitions.rs b/src/catalog/src/system_schema/information_schema/partitions.rs index 1bfe3f634fd..3bb3c29b704 100644 --- a/src/catalog/src/system_schema/information_schema/partitions.rs +++ b/src/catalog/src/system_schema/information_schema/partitions.rs @@ -44,7 +44,10 @@ use crate::error::{ Result, UpgradeWeakCatalogManagerRefSnafu, }; use crate::kvbackend::KvBackendCatalogManager; -use crate::system_schema::information_schema::{InformationTable, PARTITIONS, Predicates}; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{ + InformationSchemaTableScope, InformationTable, PARTITIONS, Predicates, +}; const TABLE_CATALOG: &str = "table_catalog"; const TABLE_SCHEMA: &str = "table_schema"; @@ -167,6 +170,12 @@ impl InformationSchemaPartitions { } impl InformationTable for InformationSchemaPartitions { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaPartitions { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_PARTITIONS_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/procedure_info.rs b/src/catalog/src/system_schema/information_schema/procedure_info.rs index b9a892d2ac8..e9f3490a592 100644 --- a/src/catalog/src/system_schema/information_schema/procedure_info.rs +++ b/src/catalog/src/system_schema/information_schema/procedure_info.rs @@ -35,7 +35,10 @@ use store_api::storage::{ScanRequest, TableId}; use crate::CatalogManager; use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result}; -use crate::system_schema::information_schema::{InformationTable, PROCEDURE_INFO, Predicates}; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{ + InformationSchemaTableScope, InformationTable, PROCEDURE_INFO, Predicates, +}; use crate::system_schema::utils; const PROCEDURE_ID: &str = "procedure_id"; @@ -97,6 +100,12 @@ impl InformationSchemaProcedureInfo { } impl InformationTable for InformationSchemaProcedureInfo { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Global + } +} + +impl SystemTable for InformationSchemaProcedureInfo { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_PROCEDURE_INFO_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index fb453c8c333..02d3148197b 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -36,7 +36,8 @@ use store_api::storage::{ScanRequest, TableId}; use crate::error::{self, InternalSnafu}; use crate::information_schema::Predicates; use crate::process_manager::ProcessManagerRef; -use crate::system_schema::information_schema::InformationTable; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{InformationSchemaTableScope, InformationTable}; /// Column names of `information_schema.process_list` pub const ID: &str = "id"; @@ -82,6 +83,12 @@ impl InformationSchemaProcessList { } impl InformationTable for InformationSchemaProcessList { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Global + } +} + +impl SystemTable for InformationSchemaProcessList { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_PROCESS_LIST_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/recycle_bin.rs b/src/catalog/src/system_schema/information_schema/recycle_bin.rs index c2022cf23da..7019b840507 100644 --- a/src/catalog/src/system_schema/information_schema/recycle_bin.rs +++ b/src/catalog/src/system_schema/information_schema/recycle_bin.rs @@ -31,7 +31,10 @@ use store_api::storage::{ScanRequest, TableId}; use crate::CatalogManager; use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result, TableMetadataManagerSnafu}; -use crate::system_schema::information_schema::{InformationTable, Predicates, RECYCLE_BIN}; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{ + InformationSchemaTableScope, InformationTable, Predicates, RECYCLE_BIN, +}; use crate::system_schema::utils; const OBJECT_ID: &str = "object_id"; @@ -118,6 +121,12 @@ impl InformationSchemaRecycleBin { } impl InformationTable for InformationSchemaRecycleBin { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaRecycleBin { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_RECYCLE_BIN_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/region_info.rs b/src/catalog/src/system_schema/information_schema/region_info.rs index 3fad1c84405..defb32df904 100644 --- a/src/catalog/src/system_schema/information_schema/region_info.rs +++ b/src/catalog/src/system_schema/information_schema/region_info.rs @@ -27,8 +27,10 @@ use store_api::storage::{ScanRequest, TableId}; use crate::CatalogManager; use crate::error::{ProjectSchemaSnafu, Result}; use crate::information_schema::{ - DatanodeInspectKind, DatanodeInspectRequest, InformationTable, REGION_INFO, + DatanodeInspectKind, DatanodeInspectRequest, InformationSchemaTableScope, InformationTable, + REGION_INFO, }; +use crate::system_schema::SystemTable; use crate::system_schema::utils; /// Information schema table for region info. @@ -47,6 +49,12 @@ impl InformationSchemaRegionInfo { } impl InformationTable for InformationSchemaRegionInfo { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Global + } +} + +impl SystemTable for InformationSchemaRegionInfo { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_REGION_INFO_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/region_peers.rs b/src/catalog/src/system_schema/information_schema/region_peers.rs index 9eddb061f80..40ed16f1dc4 100644 --- a/src/catalog/src/system_schema/information_schema/region_peers.rs +++ b/src/catalog/src/system_schema/information_schema/region_peers.rs @@ -41,7 +41,10 @@ use crate::error::{ UpgradeWeakCatalogManagerRefSnafu, }; use crate::kvbackend::KvBackendCatalogManager; -use crate::system_schema::information_schema::{InformationTable, Predicates, REGION_PEERS}; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{ + InformationSchemaTableScope, InformationTable, Predicates, REGION_PEERS, +}; pub const TABLE_CATALOG: &str = "table_catalog"; pub const TABLE_SCHEMA: &str = "table_schema"; @@ -105,6 +108,12 @@ impl InformationSchemaRegionPeers { } impl InformationTable for InformationSchemaRegionPeers { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaRegionPeers { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_REGION_PEERS_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/region_statistics.rs b/src/catalog/src/system_schema/information_schema/region_statistics.rs index 8d1a15ddcc3..8a7c19adc00 100644 --- a/src/catalog/src/system_schema/information_schema/region_statistics.rs +++ b/src/catalog/src/system_schema/information_schema/region_statistics.rs @@ -38,7 +38,10 @@ use store_api::storage::{ScanRequest, TableId}; use crate::CatalogManager; use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result}; use crate::information_schema::Predicates; -use crate::system_schema::information_schema::{InformationTable, REGION_STATISTICS}; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{ + InformationSchemaTableScope, InformationTable, REGION_STATISTICS, +}; use crate::system_schema::utils; const REGION_ID: &str = "region_id"; @@ -138,6 +141,12 @@ impl InformationSchemaRegionStatistics { } impl InformationTable for InformationSchemaRegionStatistics { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Global + } +} + +impl SystemTable for InformationSchemaRegionStatistics { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_REGION_STATISTICS_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/schemata.rs b/src/catalog/src/system_schema/information_schema/schemata.rs index c8756057cdf..d9654a48bd5 100644 --- a/src/catalog/src/system_schema/information_schema/schemata.rs +++ b/src/catalog/src/system_schema/information_schema/schemata.rs @@ -36,7 +36,10 @@ use crate::error::{ CreateRecordBatchSnafu, InternalSnafu, Result, TableMetadataManagerSnafu, UpgradeWeakCatalogManagerRefSnafu, }; -use crate::system_schema::information_schema::{InformationTable, Predicates, SCHEMATA}; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{ + InformationSchemaTableScope, InformationTable, Predicates, SCHEMATA, +}; use crate::system_schema::utils; pub const CATALOG_NAME: &str = "catalog_name"; @@ -93,6 +96,12 @@ impl InformationSchemaSchemata { } impl InformationTable for InformationSchemaSchemata { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaSchemata { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_SCHEMATA_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/ssts.rs b/src/catalog/src/system_schema/information_schema/ssts.rs index e364bddfd3d..9006ab6fed9 100644 --- a/src/catalog/src/system_schema/information_schema/ssts.rs +++ b/src/catalog/src/system_schema/information_schema/ssts.rs @@ -30,9 +30,10 @@ use store_api::storage::{ScanRequest, TableId}; use crate::CatalogManager; use crate::error::{ProjectSchemaSnafu, Result}; use crate::information_schema::{ - DatanodeInspectKind, DatanodeInspectRequest, InformationTable, SSTS_INDEX_META, SSTS_MANIFEST, - SSTS_STORAGE, + DatanodeInspectKind, DatanodeInspectRequest, InformationSchemaTableScope, InformationTable, + SSTS_INDEX_META, SSTS_MANIFEST, SSTS_STORAGE, }; +use crate::system_schema::SystemTable; use crate::system_schema::utils; fn projected_schema(schema: &SchemaRef, request: &ScanRequest) -> Result { @@ -75,6 +76,12 @@ impl InformationSchemaSstsManifest { } impl InformationTable for InformationSchemaSstsManifest { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Global + } +} + +impl SystemTable for InformationSchemaSstsManifest { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_SSTS_MANIFEST_TABLE_ID } @@ -134,6 +141,12 @@ impl InformationSchemaSstsStorage { } impl InformationTable for InformationSchemaSstsStorage { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Global + } +} + +impl SystemTable for InformationSchemaSstsStorage { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_SSTS_STORAGE_TABLE_ID } @@ -194,6 +207,12 @@ impl InformationSchemaSstsIndexMeta { } impl InformationTable for InformationSchemaSstsIndexMeta { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Global + } +} + +impl SystemTable for InformationSchemaSstsIndexMeta { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_SSTS_INDEX_META_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/statistics.rs b/src/catalog/src/system_schema/information_schema/statistics.rs index 70312350759..4aeacd21ebd 100644 --- a/src/catalog/src/system_schema/information_schema/statistics.rs +++ b/src/catalog/src/system_schema/information_schema/statistics.rs @@ -39,12 +39,14 @@ use crate::CatalogManager; use crate::error::{ CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu, }; +use crate::system_schema::SystemTable; use crate::system_schema::information_schema::key_column_usage::{ CONSTRAINT_NAME_FULLTEXT_INDEX, CONSTRAINT_NAME_INVERTED_INDEX, CONSTRAINT_NAME_PRI, CONSTRAINT_NAME_SKIPPING_INDEX, CONSTRAINT_NAME_TIME_INDEX, }; use crate::system_schema::information_schema::{ - InformationTable, Predicates, STATISTICS, primary_key_encoding_index_type, + InformationSchemaTableScope, InformationTable, Predicates, STATISTICS, + primary_key_encoding_index_type, }; pub const TABLE_CATALOG: &str = "table_catalog"; @@ -135,6 +137,12 @@ impl InformationSchemaStatistics { } impl InformationTable for InformationSchemaStatistics { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaStatistics { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_STATISTICS_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/table_constraints.rs b/src/catalog/src/system_schema/information_schema/table_constraints.rs index 2208d303c2e..aedbe251935 100644 --- a/src/catalog/src/system_schema/information_schema/table_constraints.rs +++ b/src/catalog/src/system_schema/information_schema/table_constraints.rs @@ -40,7 +40,10 @@ use crate::information_schema::Predicates; use crate::information_schema::key_column_usage::{ CONSTRAINT_NAME_PRI, CONSTRAINT_NAME_TIME_INDEX, }; -use crate::system_schema::information_schema::{InformationTable, TABLE_CONSTRAINTS}; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{ + InformationSchemaTableScope, InformationTable, TABLE_CONSTRAINTS, +}; /// The `TABLE_CONSTRAINTS` table describes which tables have constraints. #[derive(Debug)] @@ -102,6 +105,12 @@ impl InformationSchemaTableConstraints { } impl InformationTable for InformationSchemaTableConstraints { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaTableConstraints { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_TABLE_CONSTRAINTS_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/table_semantics.rs b/src/catalog/src/system_schema/information_schema/table_semantics.rs index 07f1a844f9e..b571b03d40f 100644 --- a/src/catalog/src/system_schema/information_schema/table_semantics.rs +++ b/src/catalog/src/system_schema/information_schema/table_semantics.rs @@ -59,7 +59,10 @@ use crate::CatalogManager; use crate::error::{ CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu, }; -use crate::system_schema::information_schema::{InformationTable, Predicates, TABLE_SEMANTICS}; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{ + InformationSchemaTableScope, InformationTable, Predicates, TABLE_SEMANTICS, +}; use crate::system_schema::semantic_graph::{EntityGraphProviderRef, TableEntityDeclaration}; use crate::system_schema::utils; @@ -235,6 +238,12 @@ impl InformationSchemaTableSemantics { } impl InformationTable for InformationSchemaTableSemantics { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaTableSemantics { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_TABLE_SEMANTICS_TABLE_ID } @@ -514,6 +523,17 @@ mod tests { ); } + #[test] + fn extract_system_table_scope() { + let info = table_info(&[(table::requests::SEMANTIC_TABLE_SCOPE, "catalog")]); + let row = SemanticRow::extract(&info).unwrap(); + assert_eq!( + row.options_json.as_deref(), + Some(r#"{"table_scope":"catalog"}"#) + ); + assert!(row.signal_type.is_none()); + } + #[test] fn extract_skips_untagged_table() { let info = table_info(&[("ttl", "7d")]); diff --git a/src/catalog/src/system_schema/information_schema/tables.rs b/src/catalog/src/system_schema/information_schema/tables.rs index 6175c17d39b..6c9c0167da0 100644 --- a/src/catalog/src/system_schema/information_schema/tables.rs +++ b/src/catalog/src/system_schema/information_schema/tables.rs @@ -43,7 +43,10 @@ use crate::error::{ UpgradeWeakCatalogManagerRefSnafu, }; use crate::kvbackend::KvBackendCatalogManager; -use crate::system_schema::information_schema::{InformationTable, Predicates, TABLES}; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{ + InformationSchemaTableScope, InformationTable, Predicates, TABLES, +}; use crate::system_schema::utils; pub const TABLE_CATALOG: &str = "table_catalog"; @@ -139,6 +142,12 @@ impl InformationSchemaTables { } impl InformationTable for InformationSchemaTables { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaTables { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_TABLES_TABLE_ID } diff --git a/src/catalog/src/system_schema/information_schema/views.rs b/src/catalog/src/system_schema/information_schema/views.rs index d1eb3b80ac0..65e16dfe120 100644 --- a/src/catalog/src/system_schema/information_schema/views.rs +++ b/src/catalog/src/system_schema/information_schema/views.rs @@ -38,7 +38,10 @@ use crate::error::{ UpgradeWeakCatalogManagerRefSnafu, ViewInfoNotFoundSnafu, }; use crate::kvbackend::KvBackendCatalogManager; -use crate::system_schema::information_schema::{InformationTable, Predicates, VIEWS}; +use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{ + InformationSchemaTableScope, InformationTable, Predicates, VIEWS, +}; const INIT_CAPACITY: usize = 42; pub const TABLE_CATALOG: &str = "table_catalog"; @@ -102,6 +105,12 @@ impl InformationSchemaViews { } impl InformationTable for InformationSchemaViews { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Catalog + } +} + +impl SystemTable for InformationSchemaViews { fn table_id(&self) -> TableId { INFORMATION_SCHEMA_VIEW_TABLE_ID } diff --git a/src/catalog/src/system_schema/memory_table.rs b/src/catalog/src/system_schema/memory_table.rs index de5ff120b2f..c7246939c23 100644 --- a/src/catalog/src/system_schema/memory_table.rs +++ b/src/catalog/src/system_schema/memory_table.rs @@ -31,6 +31,7 @@ use store_api::storage::{ScanRequest, TableId}; use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result}; use crate::system_schema::SystemTable; +use crate::system_schema::information_schema::{InformationSchemaTableScope, InformationTable}; /// A memory table with specified schema and columns. #[derive(Debug)] @@ -104,6 +105,12 @@ impl DfPartitionStream for MemoryTable { } } +impl InformationTable for MemoryTable { + fn scope(&self) -> InformationSchemaTableScope { + InformationSchemaTableScope::Global + } +} + impl SystemTable for MemoryTable { fn table_id(&self) -> TableId { self.table_id diff --git a/src/cli/src/data/export_v2.rs b/src/cli/src/data/export_v2.rs index 9887e525030..08652d28c6d 100644 --- a/src/cli/src/data/export_v2.rs +++ b/src/cli/src/data/export_v2.rs @@ -38,10 +38,10 @@ //! --end-time 2025-01-31T23:59:59Z //! ``` //! -//! `--experimental-metric-export` enables shared Metric physical scans for Parquet. -//! Enable `experimental_metric_export = true` on the frontend or standalone server -//! and use an endpoint whose frontends all support and enable this option. -//! Add `--metric-data-layout packed` to write version-2 packed snapshots. +//! `--pack-metric-data` enables shared Metric physical scans for Parquet +//! and writes version-2 packed snapshots. +//! Use an endpoint whose frontends all support packed export. +//! Resume reads the layout from the existing manifest. //! This requires the server packed-export capability; schema-only exports remain version 1. //! The command checks the server capability before modifying the snapshot. //! Each snapshot path belongs to one export task. Resume requires stable source diff --git a/src/cli/src/data/export_v2/command.rs b/src/cli/src/data/export_v2/command.rs index 48779f3c435..ee10091e5ae 100644 --- a/src/cli/src/data/export_v2/command.rs +++ b/src/cli/src/data/export_v2/command.rs @@ -20,11 +20,9 @@ use std::time::Duration; use async_trait::async_trait; use clap::{Parser, Subcommand}; -use common_catalog::consts::DEFAULT_SCHEMA_NAME; use common_error::ext::BoxedError; use common_telemetry::info; use serde_json::Value; -use servers::http::{ColumnSchema, GreptimeQueryOutput, OutputSchema}; use snafu::ResultExt; use crate::Tool; @@ -32,8 +30,9 @@ use crate::common::ObjectStoreConfig; use crate::data::export_v2::coordinator::{ExportDataOptions, export_data}; use crate::data::export_v2::error::{ ChunkTimeWindowRequiresBoundsSnafu, DatabaseSnafu, IoSnafu, ManifestVersionMismatchSnafu, - Result, ResumeConfigMismatchSnafu, SchemaOnlyArgsNotAllowedSnafu, SchemaOnlyModeMismatchSnafu, - SnapshotVerifyFailedSnafu, UnexpectedValueTypeSnafu, + MetricExportRequiresParquetSnafu, Result, ResumeConfigMismatchSnafu, + SchemaOnlyArgsNotAllowedSnafu, SchemaOnlyModeMismatchSnafu, SnapshotVerifyFailedSnafu, + UnexpectedValueTypeSnafu, }; use crate::data::export_v2::extractor::SchemaExtractor; use crate::data::export_v2::manifest::{ @@ -292,14 +291,11 @@ pub struct ExportCreateCommand { #[clap(long, value_enum, default_value = "parquet")] format: DataFormat, - /// Use shared Metric physical scans (Parquet only). Resume requires that the previous - /// export and storage writes have ended; an HTTP timeout does not establish this. + /// Use shared Metric physical scans and packed objects (Parquet only). + /// Resume uses the manifest layout and requires that previous export and storage + /// writes have ended; an HTTP timeout does not establish this. #[clap(long)] - experimental_metric_export: bool, - - /// Store independent Metric Parquet streams in shared packed objects. - #[clap(long, value_parser = ["packed"], requires = "experimental_metric_export")] - metric_data_layout: Option, + pack_metric_data: bool, /// Delete existing snapshot and recreate. #[clap(long)] @@ -350,10 +346,7 @@ impl ExportCreateCommand { let time_range = TimeRange::parse(self.start_time.as_deref(), self.end_time.as_deref()) .map_err(BoxedError::new)?; - if self.metric_data_layout.is_some() - && time_range.is_bounded() - && time_range.start == time_range.end - { + if self.pack_metric_data && time_range.is_bounded() && time_range.start == time_range.end { return crate::error::InvalidArgumentsSnafu { msg: "Packed export requires --start-time to be earlier than --end-time", } @@ -413,39 +406,15 @@ impl ExportCreateCommand { ); // The filesystem storage constructor can create the snapshot root. - if self.experimental_metric_export { - if self.format != DataFormat::Parquet { - return crate::data::export_v2::error::MetricExportUnavailableSnafu - .fail() - .map_err(BoxedError::new); + if self.pack_metric_data { + require_metric_export_format(self.format).map_err(BoxedError::new)?; + if !self.schema_only { + database_client + .require_packed_export() + .await + .context(DatabaseSnafu) + .map_err(BoxedError::new)?; } - let capability = database_client - .sql_response( - "SHOW VARIABLES experimental_metric_export", - DEFAULT_SCHEMA_NAME, - ) - .await - .context(DatabaseSnafu) - .map_err(BoxedError::new)?; - let expected_schema = OutputSchema::new(vec![ColumnSchema::new( - "EXPERIMENTAL_METRIC_EXPORT".to_string(), - "String".to_string(), - )]); - if !matches!(capability.output(), [GreptimeQueryOutput::Records(records)] - if records.schema() == &expected_schema - && records.rows() == &vec![vec![Value::String("true".to_string())]]) - { - return crate::data::export_v2::error::MetricExportUnavailableSnafu - .fail() - .map_err(BoxedError::new); - } - } - if self.metric_data_layout.is_some() && !self.schema_only { - database_client - .require_packed_export() - .await - .context(DatabaseSnafu) - .map_err(BoxedError::new)?; } let storage = OpenDalStorage::from_uri(&self.to, &self.storage).map_err(BoxedError::new)?; @@ -455,8 +424,7 @@ impl ExportCreateCommand { schemas, schema_only: self.schema_only, format: self.format, - experimental_metric_export: self.experimental_metric_export, - packed: self.metric_data_layout.is_some(), + packed: self.pack_metric_data, force: self.force, time_range, chunk_time_window: self.chunk_time_window, @@ -484,7 +452,6 @@ struct ExportConfig { schemas: Option>, schema_only: bool, format: DataFormat, - experimental_metric_export: bool, packed: bool, force: bool, time_range: TimeRange, @@ -538,13 +505,11 @@ impl ExportCreate { validate_resume_config(&manifest, &self.config)?; if manifest.is_packed() { + require_metric_export_format(manifest.format)?; self.database_client .require_packed_export() .await .context(DatabaseSnafu)?; - if !self.config.experimental_metric_export { - return crate::data::export_v2::error::MetricExportUnavailableSnafu.fail(); - } } info!( @@ -573,7 +538,6 @@ impl ExportCreate { storage_config: &self.config.storage_config, parallelism: self.config.parallelism, chunk_parallelism: self.config.chunk_parallelism, - experimental_metric_export: self.config.experimental_metric_export, resume: true, }, progress.as_ref(), @@ -608,7 +572,7 @@ impl ExportCreate { manifest.version = 2; manifest.data_layout = Some(common_datasource::packed_snapshot::PACKED_LAYOUT.into()); } - if self.config.experimental_metric_export { + if manifest.is_packed() { for chunk in &manifest.chunks { self.storage .prepare_export_chunk(&schema_names, chunk.id, false, manifest.is_packed()) @@ -651,7 +615,6 @@ impl ExportCreate { storage_config: &self.config.storage_config, parallelism: self.config.parallelism, chunk_parallelism: self.config.chunk_parallelism, - experimental_metric_export: self.config.experimental_metric_export, resume: false, }, progress.as_ref(), @@ -761,10 +724,17 @@ impl ExportCreate { } } +fn require_metric_export_format(format: DataFormat) -> Result<()> { + if format != DataFormat::Parquet { + return MetricExportRequiresParquetSnafu.fail(); + } + Ok(()) +} + fn validate_resume_config(manifest: &Manifest, config: &ExportConfig) -> Result<()> { if config.packed && !manifest.schema_only && !manifest.is_packed() { return ResumeConfigMismatchSnafu { - field: "metric_data_layout", + field: "data_layout", existing: "standalone".to_string(), requested: "packed".to_string(), } @@ -1683,6 +1653,44 @@ mod tests { ); } + #[tokio::test] + async fn test_pack_metric_data_arguments() { + let directory = tempfile::tempdir().unwrap(); + let snapshot = directory.path().join("snapshot"); + let uri = url::Url::from_file_path(&snapshot).unwrap(); + let args = [ + "export-v2-create", + "--addr", + "127.0.0.1:1", + "--to", + uri.as_str(), + ]; + assert!(!ExportCreateCommand::parse_from(args).pack_metric_data); + let packed = + ExportCreateCommand::parse_from(args.into_iter().chain(["--pack-metric-data"])); + assert!(packed.pack_metric_data); + for removed in [ + &["--experimental-metric-export"][..], + &["--metric-data-layout", "packed"][..], + ] { + let error = ExportCreateCommand::try_parse_from( + args.into_iter().chain(removed.iter().copied()), + ) + .unwrap_err(); + assert_eq!(error.kind(), clap::error::ErrorKind::UnknownArgument); + } + for format in ["csv", "json"] { + let command = ExportCreateCommand::parse_from(args.into_iter().chain([ + "--pack-metric-data", + "--format", + format, + ])); + let error = command.build().await.err().unwrap(); + assert!(error.to_string().contains("Metric export requires Parquet")); + assert!(!snapshot.exists()); + } + } + #[test] fn test_chunk_parallelism_parses_valid_value() { let cmd = ExportCreateCommand::parse_from([ @@ -1728,7 +1736,7 @@ mod tests { #[test] fn test_schema_only_mode_mismatch_error_message() { - let error = crate::data::export_v2::error::SchemaOnlyModeMismatchSnafu { + let error = SchemaOnlyModeMismatchSnafu { existing_schema_only: false, requested_schema_only: true, } @@ -1755,7 +1763,6 @@ mod tests { schemas: None, schema_only: false, format: DataFormat::Parquet, - experimental_metric_export: false, packed: false, force: false, time_range: TimeRange::unbounded(), @@ -1794,7 +1801,6 @@ mod tests { ]), schema_only: false, format: DataFormat::Parquet, - experimental_metric_export: false, packed: false, force: false, time_range: TimeRange::unbounded(), @@ -1812,7 +1818,7 @@ mod tests { validate_resume_config(&manifest, &config) .unwrap_err() .to_string() - .contains("metric_data_layout") + .contains("data_layout") ); manifest.version = 2; manifest.data_layout = Some(common_datasource::packed_snapshot::PACKED_LAYOUT.into()); @@ -1840,7 +1846,6 @@ mod tests { schemas: None, schema_only: false, format: DataFormat::Parquet, - experimental_metric_export: false, packed: false, force: false, time_range, @@ -1875,7 +1880,6 @@ mod tests { schemas: None, schema_only: false, format: DataFormat::Csv, - experimental_metric_export: false, packed: false, force: false, time_range: TimeRange::unbounded(), @@ -1912,7 +1916,6 @@ mod tests { schemas: None, schema_only: false, format: DataFormat::Parquet, - experimental_metric_export: false, packed: false, force: false, time_range: TimeRange::new(Some(start), Some(start)), diff --git a/src/cli/src/data/export_v2/coordinator.rs b/src/cli/src/data/export_v2/coordinator.rs index c6181e51211..e18014af32f 100644 --- a/src/cli/src/data/export_v2/coordinator.rs +++ b/src/cli/src/data/export_v2/coordinator.rs @@ -39,7 +39,6 @@ struct ExportContext<'a> { schemas: Vec, format: DataFormat, parallelism: usize, - experimental_metric_export: bool, packed: bool, resume: bool, } @@ -49,7 +48,6 @@ pub struct ExportDataOptions<'a> { pub storage_config: &'a ObjectStoreConfig, pub parallelism: usize, pub chunk_parallelism: usize, - pub experimental_metric_export: bool, pub resume: bool, } @@ -74,7 +72,6 @@ pub async fn export_data( format: manifest.format, packed: manifest.is_packed(), parallelism: options.parallelism, - experimental_metric_export: options.experimental_metric_export, resume: options.resume, }; @@ -278,7 +275,7 @@ async fn export_chunk( chunk_id: u32, time_range: TimeRange, ) -> Result> { - if context.experimental_metric_export { + if context.packed { context .storage .prepare_export_chunk(&context.schemas, chunk_id, context.resume, context.packed) @@ -290,7 +287,6 @@ async fn export_chunk( format: context.format, time_range, parallelism: context.parallelism, - experimental_metric_export: context.experimental_metric_export, packed: context.packed, }; diff --git a/src/cli/src/data/export_v2/data.rs b/src/cli/src/data/export_v2/data.rs index 53c3899f59e..dac094c0dda 100644 --- a/src/cli/src/data/export_v2/data.rs +++ b/src/cli/src/data/export_v2/data.rs @@ -30,7 +30,6 @@ pub(super) struct CopyOptions { pub(super) format: DataFormat, pub(super) time_range: TimeRange, pub(super) parallelism: usize, - pub(crate) experimental_metric_export: bool, pub(super) packed: bool, } @@ -218,10 +217,8 @@ pub(crate) async fn execute_copy_database_from( fn build_with_options(options: &CopyOptions) -> String { let mut parts = vec![format!("FORMAT='{}'", options.format)]; - if options.experimental_metric_export { - parts.push("experimental_metric_export='true'".to_string()); - } if options.packed { + parts.push("experimental_metric_export='true'".to_string()); parts.push("metric_data_layout='packed'".to_string()); } if let Some(start) = options.time_range.start { @@ -430,6 +427,25 @@ mod tests { use super::*; use crate::common::{PrefixedAzblobConnection, PrefixedGcsConnection, PrefixedOssConnection}; + #[test] + fn test_packed_copy_options() { + let mut options = CopyOptions { + format: DataFormat::Parquet, + time_range: TimeRange::unbounded(), + parallelism: 2, + packed: false, + }; + assert_eq!( + build_with_options(&options), + "FORMAT='parquet', PARALLELISM=2" + ); + options.packed = true; + assert_eq!( + build_with_options(&options), + "FORMAT='parquet', experimental_metric_export='true', metric_data_layout='packed', PARALLELISM=2" + ); + } + #[test] fn test_build_oss_connection_includes_endpoint() { let storage = ObjectStoreConfig { diff --git a/src/cli/src/data/export_v2/error.rs b/src/cli/src/data/export_v2/error.rs index dbb4a88ae46..a64ff93865b 100644 --- a/src/cli/src/data/export_v2/error.rs +++ b/src/cli/src/data/export_v2/error.rs @@ -24,10 +24,8 @@ use snafu::{Location, Snafu}; #[snafu(visibility(pub))] #[stack_trace_debug] pub enum Error { - #[snafu(display( - "Experimental Metric export requires Parquet and a frontend that explicitly reports experimental_metric_export=true. Use an upgraded, consistently configured frontend entry point." - ))] - MetricExportUnavailable { + #[snafu(display("Packed Metric export requires Parquet"))] + MetricExportRequiresParquet { #[snafu(implicit)] location: Location, }, @@ -219,7 +217,7 @@ pub type Result = std::result::Result; impl ErrorExt for Error { fn status_code(&self) -> StatusCode { match self { - Error::MetricExportUnavailable { .. } + Error::MetricExportRequiresParquet { .. } | Error::InvalidUri { .. } | Error::UnsupportedScheme { .. } | Error::SchemaOnlyModeMismatch { .. } diff --git a/src/cli/src/data/snapshot_storage.rs b/src/cli/src/data/snapshot_storage.rs index 17008706066..f8dd22f0307 100644 --- a/src/cli/src/data/snapshot_storage.rs +++ b/src/cli/src/data/snapshot_storage.rs @@ -21,6 +21,7 @@ use std::collections::BTreeSet; use std::path::Component; use async_trait::async_trait; +use common_datasource::packed_writer::WRITE_BYTES; use futures::TryStreamExt; use object_store::services::{Azblob, Fs, Gcs, Oss, S3}; use object_store::util::{with_instrument_layers, with_retry_layers}; @@ -570,13 +571,31 @@ impl OpenDalStorage { /// Writes bytes to a file. async fn write_file(&self, path: &str, data: Vec) -> Result<()> { - self.object_store - .write(path, data) + let mut writer = self + .object_store + .writer_with(path) + .chunk(WRITE_BYTES) + .concurrent(1) .await - .map(|_| ()) - .context(StorageOperationSnafu { - operation: format!("write {}", path), - }) + .with_context(|_| StorageOperationSnafu { + operation: format!("open writer {}", path), + })?; + let data = object_store::Buffer::from(data); + let result = async { + for chunk in data.chunks(WRITE_BYTES) { + writer.write(chunk).await?; + } + writer.close().await.map(|_| ()) + } + .await; + if result.is_err() + && let Err(error) = writer.abort().await + { + common_telemetry::warn!(error; "Failed to abort snapshot metadata write"); + } + result.with_context(|_| StorageOperationSnafu { + operation: format!("write {}", path), + }) } /// Checks if a file exists using stat. @@ -837,6 +856,96 @@ mod tests { ) } + #[tokio::test] + async fn snapshot_metadata_submissions_are_bounded_and_invoke_abort_on_failure() { + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use object_store::layers::mock::{Metadata, ObservedWriter, WriterObservations, oio}; + + struct FailWriter { + inner: oio::Writer, + writes: usize, + fail: Arc, + } + impl oio::Write for FailWriter { + async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> { + self.writes += 1; + if self.fail.load(Ordering::SeqCst) == 1 && self.writes == 2 { + return Err(object_store::Error::new( + ErrorKind::Unexpected, + "injected metadata write failure", + )); + } + self.inner.write(bytes).await + } + async fn close(&mut self) -> object_store::Result { + if self.fail.load(Ordering::SeqCst) == 2 { + return Err(object_store::Error::new( + ErrorKind::Unexpected, + "injected metadata close failure", + )); + } + self.inner.close().await + } + async fn abort(&mut self) -> object_store::Result<()> { + self.inner.abort().await + } + } + + let dir = tempdir().unwrap(); + let mut storage = make_storage_with_rooted_fs(dir.path()); + let observations = Arc::new(Mutex::new(WriterObservations::default())); + let fail = Arc::new(AtomicUsize::new(0)); + storage.object_store = storage.object_store.layer( + MockLayerBuilder::default() + .writer_factory(Arc::new({ + let observations = observations.clone(); + let fail = fail.clone(); + move |_, _, inner| { + Box::new(ObservedWriter::new( + Box::new(FailWriter { + inner, + writes: 0, + fail: fail.clone(), + }), + observations.clone(), + )) + } + })) + .build() + .unwrap(), + ); + for length in [2 * WRITE_BYTES + 17, 1, 0] { + *observations.lock().unwrap() = WriterObservations::default(); + let data = vec![42; length]; + storage.write_file("metadata", data.clone()).await.unwrap(); + assert_eq!(storage.read_file("metadata").await.unwrap(), data); + let observed = observations.lock().unwrap(); + assert_eq!(observed.write_sizes.iter().sum::(), length); + assert!(observed.write_sizes.iter().all(|size| *size <= WRITE_BYTES)); + assert_eq!(observed.close_calls, 1); + assert_eq!(observed.abort_calls, 0); + } + for failure in [1, 2] { + *observations.lock().unwrap() = WriterObservations::default(); + fail.store(failure, Ordering::SeqCst); + assert!( + storage + .write_file("failed", vec![42; 2 * WRITE_BYTES + 17]) + .await + .is_err() + ); + assert_eq!( + observations.lock().unwrap().write_sizes.len(), + if failure == 1 { 2 } else { 3 } + ); + let observed = observations.lock().unwrap(); + assert_eq!(observed.close_calls, usize::from(failure == 2)); + assert_eq!(observed.abort_calls, 1); + } + } + #[test] fn test_validate_uri_valid() { assert_eq!(validate_uri("s3://bucket/path").unwrap(), StorageScheme::S3); diff --git a/src/client/Cargo.toml b/src/client/Cargo.toml index 36c3f60a9a4..e5bd479cc7d 100644 --- a/src/client/Cargo.toml +++ b/src/client/Cargo.toml @@ -45,3 +45,4 @@ substrait.workspace = true tokio.workspace = true tokio-stream = { workspace = true, features = ["net"] } tonic.workspace = true +tonic-prost.workspace = true diff --git a/src/client/src/client.rs b/src/client/src/client.rs index dfc0e44fad5..1746854cd26 100644 --- a/src/client/src/client.rs +++ b/src/client/src/client.rs @@ -20,7 +20,6 @@ use api::v1::HealthCheckRequest; use api::v1::flow::flow_client::FlowClient as PbFlowClient; use api::v1::health_check_client::HealthCheckClient; use api::v1::prometheus_gateway_client::PrometheusGatewayClient; -use api::v1::region::region_client::RegionClient as PbRegionClient; use arrow_flight::flight_service_client::FlightServiceClient; use common_grpc::channel_manager::{ ChannelConfig, ChannelManager, ClientTlsOption, load_client_tls_config, @@ -526,23 +525,12 @@ impl Client { Ok(FlightClient { addr, client }) } - pub(crate) fn raw_region_client(&self) -> Result<(String, PbRegionClient)> { + /// Creates the region gRPC transport with the configured message limits. + pub(crate) fn raw_region_client(&self) -> Result<(String, tonic::client::Grpc)> { let (addr, channel) = self.find_channel()?; - let client = PbRegionClient::new(channel) - .max_decoding_message_size( - self.inner - .control_channel_manager - .config() - .max_recv_message_size - .as_bytes() as usize, - ) - .max_encoding_message_size( - self.inner - .control_channel_manager - .config() - .max_send_message_size - .as_bytes() as usize, - ); + let client = tonic::client::Grpc::new(channel) + .max_decoding_message_size(self.max_grpc_recv_message_size()) + .max_encoding_message_size(self.max_grpc_send_message_size()); Ok((addr, client)) } diff --git a/src/client/src/error.rs b/src/client/src/error.rs index 168ba422715..fdf0df9582f 100644 --- a/src/client/src/error.rs +++ b/src/client/src/error.rs @@ -18,7 +18,7 @@ use common_error::define_from_tonic_status; use common_error::ext::{BoxedError, ErrorExt, RetryHint}; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; -use snafu::{Location, Snafu}; +use snafu::{Location, Snafu, location}; use tonic::Code; use tonic::metadata::errors::InvalidMetadataValue; @@ -191,9 +191,22 @@ impl ErrorExt for Error { Error::Tonic { retry_hint, .. } => *retry_hint, Error::FlightGet { source, .. } | Error::FlightStream { source, .. } - | Error::RegionServer { source, .. } | Error::FlowServer { source, .. } | Error::External { source, .. } => source.retry_hint(), + Error::RegionServer { code, source, .. } => { + let hint = source.retry_hint(); + // Preserve the transport retry policy previously applied by should_retry(). + if !hint.is_retryable() + && matches!( + code, + Code::Unavailable | Code::Cancelled | Code::DeadlineExceeded + ) + { + RetryHint::Retryable + } else { + hint + } + } Error::ConvertFlightData { source, .. } | Error::CreateChannel { source, .. } | Error::CreateTlsChannel { source, .. } => source.retry_hint(), @@ -205,6 +218,15 @@ impl ErrorExt for Error { define_from_tonic_status!(Error, Tonic); +impl From for common_meta::error::Error { + fn from(error: Error) -> Self { + Self::External { + source: BoxedError::new(error), + location: location!(), + } + } +} + impl Error { /// Returns the gRPC status code if this error is caused by a gRPC request failure. pub fn tonic_code(&self) -> Option { diff --git a/src/client/src/region.rs b/src/client/src/region.rs index fb2e4d10ca1..f9ebbb0d0bf 100644 --- a/src/client/src/region.rs +++ b/src/client/src/region.rs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod codec; + +use std::borrow::Borrow; use std::sync::Arc; use std::time::Duration; @@ -49,10 +52,12 @@ use crate::error::{ MissingFieldSnafu, Result, ServerSnafu, }; use crate::flight::{FlightMessageReader, decode_flight_data}; +use crate::region::codec::RegionCodec; use crate::{Client, metrics}; const FLIGHT_DO_GET_TIMEOUT: Duration = Duration::from_secs(10); +/// Sends region operations through a configured gRPC client. #[derive(Debug)] pub struct RegionRequester { client: Client, @@ -62,20 +67,8 @@ pub struct RegionRequester { #[async_trait] impl Datanode for RegionRequester { - async fn handle(&self, request: RegionRequest) -> MetaResult { - self.handle_inner(request).await.map_err(|err| { - if err.should_retry() { - meta_error::Error::RetryLater { - source: BoxedError::new(err), - clean_poisons: false, - } - } else { - meta_error::Error::External { - source: BoxedError::new(err), - location: location!(), - } - } - }) + async fn handle_request(&self, request: Arc) -> MetaResult { + self.handle_inner(request).await.map_err(Into::into) } async fn handle_query(&self, request: QueryRequest) -> MetaResult { @@ -144,11 +137,49 @@ impl RegionRequester { recordbatches_from_flight_message_stream(addr, flight_message_stream).await } - async fn handle_inner(&self, request: RegionRequest) -> Result { + /// Sends an owned or shared request without deep-cloning its protobuf payload. + /// + /// Accepts both [`RegionRequest`] and [`Arc`]. Callers can retain a + /// shared request for another attempt; this method does not retry or determine + /// whether replaying the request is safe. + pub async fn handle(&self, request: R) -> Result + where + R: Borrow + Send + Sync + 'static, + { + self.handle_inner(request).await + } + + /// Sends a remote dynamic filter update to the region service. + pub async fn handle_remote_dyn_filter_update( + &self, + query_id: impl Into, + update: RemoteDynFilterUpdate, + ) -> Result { + self.handle_inner(build_remote_dyn_filter_update_request(query_id, update)) + .await + } + + /// Unregisters a remote dynamic filter in the region service. + pub async fn handle_remote_dyn_filter_unregister( + &self, + query_id: impl Into, + unregister: RemoteDynFilterUnregister, + ) -> Result { + self.handle_inner(build_remote_dyn_filter_unregister_request( + query_id, unregister, + )) + .await + } + + async fn handle_inner(&self, request: R) -> Result + where + R: Borrow + Send + Sync + 'static, + { let request_body = request + .borrow() .body .as_ref() - .with_context(|| MissingFieldSnafu { field: "body" })?; + .context(MissingFieldSnafu { field: "body" })?; let is_insert = matches!( request_body, region_request::Body::Inserts(_) | region_request::Body::BulkInsert(_) @@ -168,48 +199,42 @@ impl RegionRequester { } } - let response = client - .handle(request) - .await - .map_err(|e| { - let code = e.code(); - // Uses `Error::RegionServer` instead of `Error::Server` - error::Error::RegionServer { - addr, - code, - source: BoxedError::new(error::Error::from(e)), - location: location!(), - } - })? - .into_inner(); + let response = async { + client + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {e}")))?; + let mut request = tonic::Request::new(request); + request.extensions_mut().insert(tonic::GrpcMethod::new( + "greptime.v1.region.Region", + "Handle", + )); + client + .unary( + request, + tonic::codegen::http::uri::PathAndQuery::from_static( + "/greptime.v1.region.Region/Handle", + ), + RegionCodec::default(), + ) + .await + } + .await + .map_err(|status| map_region_error(status, addr))? + .into_inner(); check_response_header(&response.header)?; Ok(RegionResponse::from_region_response(response)) } +} - pub async fn handle(&self, request: RegionRequest) -> Result { - self.handle_inner(request).await - } - - pub async fn handle_remote_dyn_filter_update( - &self, - query_id: impl Into, - update: RemoteDynFilterUpdate, - ) -> Result { - self.handle_inner(build_remote_dyn_filter_update_request(query_id, update)) - .await - } - - pub async fn handle_remote_dyn_filter_unregister( - &self, - query_id: impl Into, - unregister: RemoteDynFilterUnregister, - ) -> Result { - self.handle_inner(build_remote_dyn_filter_unregister_request( - query_id, unregister, - )) - .await +fn map_region_error(status: tonic::Status, addr: String) -> error::Error { + error::Error::RegionServer { + addr, + code: status.code(), + source: BoxedError::new(error::Error::from(status)), + location: location!(), } } @@ -385,13 +410,22 @@ pub fn check_response_header(header: &Option) -> Result<()> { #[cfg(test)] #[allow(deprecated)] -mod test { - use api::v1::Status as PbStatus; +mod tests { + use std::collections::VecDeque; + use std::sync::Mutex; + use api::v1::region::region_server::{Region, RegionServer}; use api::v1::region::{ - BulkInsertRequest, RegionResponse as PbRegionResponse, RemoteDynFilterUnregister, - RemoteDynFilterUpdate, region_request, remote_dyn_filter_request, + BulkInsertRequest, InsertRequest, InsertRequests, RegionResponse as PbRegionResponse, + RemoteDynFilterUnregister, RemoteDynFilterUpdate, region_request, + remote_dyn_filter_request, }; + use api::v1::value::ValueData; + use api::v1::{ + ColumnDataType, ColumnSchema as PbColumnSchema, Row, Rows, Status as PbStatus, Value, + }; + use common_error::ext::RetryHint; + use common_grpc::channel_manager::{ChannelConfig, ChannelManager}; use common_recordbatch::adapter::RecordBatchMetrics; use datatypes::arrow::array::Int32Array; use datatypes::prelude::{ConcreteDataType, VectorRef}; @@ -399,74 +433,526 @@ mod test { use datatypes::vectors::Int32Vector; use futures_util::stream; use tokio::net::TcpListener; + use tokio::task::JoinHandle; use tokio_stream::wrappers::TcpListenerStream; use tonic::codec::CompressionEncoding; use tonic::{Request, Response, Status}; use super::*; - use crate::Error::{self, IllegalDatabaseResponse, Server}; + use crate::Error; - #[derive(Clone)] - struct CompressionRecordingRegionService { - zstd_headers: Arc>>, + #[derive(Clone, Default)] + struct RecordingRegionService { + requests: Arc>>, + responses: Arc>>>, } #[tonic::async_trait] - impl Region for CompressionRecordingRegionService { + impl Region for RecordingRegionService { async fn handle( &self, request: Request, ) -> std::result::Result, Status> { let metadata = request.metadata(); - let sends_zstd = metadata - .get("grpc-encoding") - .and_then(|v| v.to_str().ok()) - .is_some_and(|value| value == "zstd"); + let sends_zstd = metadata.get("grpc-encoding").is_some_and(|v| v == "zstd"); let accepts_zstd = metadata .get("grpc-accept-encoding") - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.split(',').any(|encoding| encoding == "zstd")); - self.zstd_headers + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.split(',').any(|encoding| encoding == "zstd")); + self.requests .lock() .unwrap() - .push((sends_zstd, accepts_zstd)); + .push((request.into_inner(), sends_zstd, accepts_zstd)); + self.responses + .lock() + .unwrap() + .pop_front() + .unwrap_or_else(|| Ok(build_response(StatusCode::Success))) + .map(Response::new) + } + } - Ok(Response::new(PbRegionResponse { - header: Some(ResponseHeader { - status: Some(PbStatus { - status_code: StatusCode::Success as u32, - ..Default::default() - }), + struct RegionTestServer { + addr: String, + service: RecordingRegionService, + task: JoinHandle<()>, + } + + impl RegionTestServer { + async fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + let service = RecordingRegionService::default(); + let server_service = service.clone(); + let task = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service( + RegionServer::new(server_service) + .accept_compressed(CompressionEncoding::Zstd) + .send_compressed(CompressionEncoding::Zstd), + ) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .unwrap(); + }); + Self { + addr, + service, + task, + } + } + + fn create_requester( + &self, + config: ChannelConfig, + send: bool, + accept: bool, + ) -> RegionRequester { + RegionRequester::new( + Client::with_manager_and_urls( + ChannelManager::with_config(config, None), + [self.addr.clone()], + ), + send, + accept, + ) + } + } + + impl Drop for RegionTestServer { + fn drop(&mut self) { + self.task.abort(); + } + } + + fn build_response(code: StatusCode) -> PbRegionResponse { + PbRegionResponse { + header: Some(ResponseHeader { + status: Some(PbStatus { + status_code: code as u32, + ..Default::default() }), + }), + affected_rows: 1, + ..Default::default() + } + } + + fn build_insert_request() -> RegionRequest { + RegionRequest { + header: Some(RegionRequestHeader { + dbname: "shared-request-test".to_string(), ..Default::default() - })) + }), + body: Some(region_request::Body::Inserts(InsertRequests { + requests: vec![InsertRequest { + region_id: 42, + rows: Some(Rows { + schema: vec![ + PbColumnSchema { + column_name: "message".to_string(), + datatype: ColumnDataType::String as i32, + ..Default::default() + }, + PbColumnSchema { + column_name: "payload".to_string(), + datatype: ColumnDataType::Binary as i32, + ..Default::default() + }, + ], + rows: vec![Row { + values: vec![ + Value { + value_data: Some(ValueData::StringValue( + "hello 世界".to_string(), + )), + }, + Value { + value_data: Some(ValueData::BinaryValue(vec![0, 1, 127, 255])), + }, + ], + }], + }), + ..Default::default() + }], + })), } } #[tokio::test] - async fn test_inserts_and_bulk_insert_use_transport_compression() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - let zstd_headers = Arc::new(std::sync::Mutex::new(Vec::new())); - let service = CompressionRecordingRegionService { - zstd_headers: zstd_headers.clone(), - }; - let server = tokio::spawn(async move { - tonic::transport::Server::builder() - .add_service( - RegionServer::new(service) - .accept_compressed(CompressionEncoding::Zstd) - .send_compressed(CompressionEncoding::Zstd), - ) - .serve_with_incoming(TcpListenerStream::new(listener)) + async fn test_owned_and_shared_requests_preserve_payload_and_compression() { + let server = RegionTestServer::start().await; + let shared = Arc::new(build_insert_request()); + for (send, accept) in [(false, false), (true, false), (false, true), (true, true)] { + let requester = server.create_requester(ChannelConfig::default(), send, accept); + let channel = + tonic::transport::Endpoint::from_shared(format!("http://{}", server.addr)) + .unwrap() + .connect() + .await + .unwrap(); + let mut baseline = api::v1::region::region_client::RegionClient::new(channel); + if send { + baseline = baseline.send_compressed(CompressionEncoding::Zstd); + } + if accept { + baseline = baseline.accept_compressed(CompressionEncoding::Zstd); + } + assert_eq!( + baseline + .handle(build_insert_request()) + .await + .unwrap() + .into_inner() + .affected_rows, + 1 + ); + assert_eq!( + requester + .handle(build_insert_request()) + .await + .unwrap() + .affected_rows, + 1 + ); + assert_eq!( + requester + .handle(Arc::clone(&shared)) + .await + .unwrap() + .affected_rows, + 1 + ); + } + let requests = server.service.requests.lock().unwrap(); + assert_eq!(requests.len(), 12); + for ((request, sends, accepts), (send, accept)) in requests.iter().zip( + [(false, false), (true, false), (false, true), (true, true)] + .into_iter() + .flat_map(|settings| [settings, settings, settings]), + ) { + assert_eq!(request, shared.as_ref()); + assert_eq!((*sends, *accepts), (send, accept)); + } + } + + #[tokio::test] + async fn test_shared_request_can_be_resent_after_transport_error() { + let server = RegionTestServer::start().await; + let mut status = Status::unavailable("migrating"); + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT, + RetryHint::Retryable.as_str().parse().unwrap(), + ); + server + .service + .responses + .lock() + .unwrap() + .push_back(Err(status)); + let requester = server.create_requester(ChannelConfig::default(), false, false); + let shared = Arc::new(build_insert_request()); + let error = requester.handle(Arc::clone(&shared)).await.unwrap_err(); + assert_eq!(error.retry_hint(), RetryHint::Retryable); + assert!( + matches!(error, Error::RegionServer { addr, code: tonic::Code::Unavailable, .. } if addr == server.addr) + ); + requester.handle(Arc::clone(&shared)).await.unwrap(); + let requests = server.service.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(&requests[0].0, shared.as_ref()); + assert_eq!(&requests[1].0, shared.as_ref()); + } + + #[test] + fn test_retry_hint_wire_compatibility() { + for (name, wire_hint, expected_hint) in [ + ("legacy_missing", None, RetryHint::NonRetryable), + ("legacy_retryable", Some("retryable"), RetryHint::Retryable), + ( + "legacy_non_retryable", + Some("non_retryable"), + RetryHint::NonRetryable, + ), + ( + "route_refresh", + Some("retry_after_route_refresh"), + RetryHint::RetryAfterRouteRefresh, + ), + ( + "unknown_future_hint", + Some("future_retry_hint"), + RetryHint::NonRetryable, + ), + ] { + let mut status = Status::failed_precondition("region unavailable"); + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_CODE, + (StatusCode::RegionNotReady as u32) + .to_string() + .parse() + .unwrap(), + ); + if let Some(hint) = wire_hint { + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT, + hint.parse().unwrap(), + ); + } + let error: Error = status.into(); + assert_eq!(error.retry_hint(), expected_hint, "{name}"); + assert_eq!(error.status_code(), StatusCode::RegionNotReady, "{name}"); + assert_eq!( + error.tonic_code(), + Some(tonic::Code::FailedPrecondition), + "{name}" + ); + } + } + + #[tokio::test] + async fn test_transport_error_parity_and_datanode_mapping() { + let server = RegionTestServer::start().await; + let requester = server.create_requester(ChannelConfig::default(), false, false); + for hint in [ + RetryHint::Retryable, + RetryHint::RetryAfterRouteRefresh, + RetryHint::NonRetryable, + ] { + let mut status = Status::failed_precondition("region unavailable"); + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT, + hint.as_str().parse().unwrap(), + ); + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_CODE, + (StatusCode::RegionNotFound as u32) + .to_string() + .parse() + .unwrap(), + ); + server + .service + .responses + .lock() + .unwrap() + .extend((0..4).map(|_| Err(status.clone()))); + let mut baseline = api::v1::region::region_client::RegionClient::connect(format!( + "http://{}", + server.addr + )) + .await + .unwrap(); + let baseline: Error = baseline + .handle(build_insert_request()) .await - .unwrap(); + .unwrap_err() + .into(); + assert_eq!(baseline.status_code(), StatusCode::RegionNotFound); + assert_eq!(baseline.retry_hint(), hint); + assert_eq!(baseline.tonic_code(), Some(tonic::Code::FailedPrecondition)); + let owned = requester.handle(build_insert_request()).await.unwrap_err(); + let shared = requester + .handle(Arc::new(build_insert_request())) + .await + .unwrap_err(); + for error in [owned, shared] { + assert_eq!(error.status_code(), baseline.status_code()); + assert_eq!(error.retry_hint(), baseline.retry_hint()); + assert_eq!(error.tonic_code(), baseline.tonic_code()); + } + let datanode: &dyn Datanode = &requester; + let error = datanode.handle(build_insert_request()).await.unwrap_err(); + assert_eq!(error.status_code(), StatusCode::RegionNotFound); + assert_eq!(error.retry_hint(), hint); + assert!(matches!(error, meta_error::Error::External { .. })); + } + } + + #[tokio::test] + async fn test_datanode_transport_retry_hints() { + let server = RegionTestServer::start().await; + let requester = server.create_requester(ChannelConfig::default(), false, false); + let datanode: &dyn Datanode = &requester; + let statuses = [ + tonic::Code::Unavailable, + tonic::Code::Cancelled, + tonic::Code::DeadlineExceeded, + tonic::Code::FailedPrecondition, + ] + .into_iter() + .flat_map(|code| { + [ + None, + Some(RetryHint::NonRetryable), + Some(RetryHint::Retryable), + Some(RetryHint::RetryAfterRouteRefresh), + ] + .into_iter() + .map(move |hint| (code, hint)) }); - let client = Client::with_urls([addr.to_string()]); - let requester = RegionRequester::new(client.clone(), true, true); - let send_only_requester = RegionRequester::new(client.clone(), true, false); - let accept_only_requester = RegionRequester::new(client.clone(), false, true); - let disabled_requester = RegionRequester::new(client, false, false); + for (code, hint) in statuses { + for body in [ + region_request::Body::Inserts(Default::default()), + region_request::Body::Deletes(Default::default()), + region_request::Body::BulkInsert(Default::default()), + region_request::Body::Create(Default::default()), + region_request::Body::RemoteDynFilter(Default::default()), + ] { + let request_type = body.as_ref().to_string(); + let mut status = Status::new(code, "request failed"); + // Metadata status is deliberately different from the transport status. + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_CODE, + (StatusCode::RegionNotReady as u32) + .to_string() + .parse() + .unwrap(), + ); + if let Some(hint) = hint { + status.metadata_mut().insert( + common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT, + hint.as_str().parse().unwrap(), + ); + } + let original = Error::from(status.clone()); + let expected = if original.should_retry() && !original.retry_hint().is_retryable() { + RetryHint::Retryable + } else { + original.retry_hint() + }; + server + .service + .responses + .lock() + .unwrap() + .extend([Err(status.clone()), Err(status)]); + let request = RegionRequest { + header: None, + body: Some(body), + }; + let shared = Arc::new(request); + let shared_error = datanode.handle(Arc::clone(&shared)).await.unwrap_err(); + assert_eq!(Arc::strong_count(&shared), 1); + let request = Arc::try_unwrap(shared).unwrap(); + let owned_error = datanode.handle(request).await.unwrap_err(); + for error in [owned_error, shared_error] { + assert_eq!( + error.retry_hint(), + expected, + "{code:?}, {hint:?}, {request_type}" + ); + assert_eq!(error.retry_hint().is_retryable(), original.should_retry()); + assert_eq!(error.status_code(), StatusCode::RegionNotReady); + let meta_error::Error::External { source, .. } = error else { + panic!("expected an external error"); + }; + assert_eq!(source.retry_hint(), expected); + let Error::RegionServer { + source, + code: actual_code, + .. + } = source.as_any().downcast_ref::().unwrap() + else { + panic!("expected a region server error"); + }; + assert_eq!(*actual_code, code); + assert_eq!(source.retry_hint(), original.retry_hint()); + assert_eq!(source.to_string(), original.to_string()); + } + } + } + } + + #[tokio::test] + async fn test_shared_request_preserves_response_header_errors() { + let server = RegionTestServer::start().await; + server.service.responses.lock().unwrap().extend([ + Ok(PbRegionResponse::default()), + Ok(build_response(StatusCode::RegionNotFound)), + ]); + let requester = server.create_requester(ChannelConfig::default(), false, false); + let shared = Arc::new(build_insert_request()); + assert!(matches!( + requester.handle(Arc::clone(&shared)).await.unwrap_err(), + Error::IllegalDatabaseResponse { .. } + )); + assert!(matches!( + requester.handle(shared).await.unwrap_err(), + Error::Server { + code: StatusCode::RegionNotFound, + .. + } + )); + } + + #[tokio::test] + async fn test_shared_request_respects_configured_message_limits() { + let server = RegionTestServer::start().await; + let shared = Arc::new(build_insert_request()); + for send_limit in [true, false] { + let mut config = ChannelConfig::default(); + if send_limit { + config.max_send_message_size = "1B".parse().unwrap(); + } else { + config.max_recv_message_size = "1B".parse().unwrap(); + } + let mut baseline = api::v1::region::region_client::RegionClient::connect(format!( + "http://{}", + server.addr + )) + .await + .unwrap() + .max_encoding_message_size(config.max_send_message_size.as_bytes() as usize) + .max_decoding_message_size(config.max_recv_message_size.as_bytes() as usize); + let baseline: Error = baseline + .handle(build_insert_request()) + .await + .unwrap_err() + .into(); + if !send_limit { + assert_eq!( + baseline.tonic_code(), + Some(tonic::Code::OutOfRange), + "{baseline:?}" + ); + } + let requester = server.create_requester(config, false, false); + let error = requester.handle(Arc::clone(&shared)).await.unwrap_err(); + assert_eq!( + error.tonic_code(), + baseline.tonic_code(), + "shared: {error:?}; generated: {baseline:?}" + ); + assert_eq!( + error.status_code(), + baseline.status_code(), + "shared: {error:?}; generated: {baseline:?}" + ); + let Error::RegionServer { source, .. } = &error else { + panic!("expected a region server error: {error:?}"); + }; + assert_eq!( + source.retry_hint(), + baseline.retry_hint(), + "shared: {error:?}; generated: {baseline:?}" + ); + assert_eq!( + server.service.requests.lock().unwrap().len(), + if send_limit { 0 } else { 2 } + ); + } + let requester = server.create_requester(ChannelConfig::default(), false, false); + requester.handle(shared).await.unwrap(); + } + + #[tokio::test] + async fn test_inserts_and_bulk_insert_use_transport_compression() { + let server = RegionTestServer::start().await; + let requester = server.create_requester(ChannelConfig::default(), true, true); + let send_only_requester = server.create_requester(ChannelConfig::default(), true, false); + let accept_only_requester = server.create_requester(ChannelConfig::default(), false, true); + let disabled_requester = server.create_requester(ChannelConfig::default(), false, false); let inserts = || RegionRequest { body: Some(region_request::Body::Inserts(Default::default())), ..Default::default() @@ -502,9 +988,15 @@ mod test { (false, false), (false, false) ], - *zstd_headers.lock().unwrap() + server + .service + .requests + .lock() + .unwrap() + .iter() + .map(|(_, send, accept)| (*send, *accept)) + .collect::>() ); - server.abort(); } #[test] @@ -566,13 +1058,13 @@ mod test { let result = check_response_header(&None); assert!(matches!( result.unwrap_err(), - IllegalDatabaseResponse { .. } + Error::IllegalDatabaseResponse { .. } )); let result = check_response_header(&Some(ResponseHeader { status: None })); assert!(matches!( result.unwrap_err(), - IllegalDatabaseResponse { .. } + Error::IllegalDatabaseResponse { .. } )); let result = check_response_header(&Some(ResponseHeader { @@ -591,7 +1083,7 @@ mod test { })); assert!(matches!( result.unwrap_err(), - IllegalDatabaseResponse { .. } + Error::IllegalDatabaseResponse { .. } )); let result = check_response_header(&Some(ResponseHeader { @@ -600,7 +1092,7 @@ mod test { err_msg: "blabla".to_string(), }), })); - let Server { code, msg, .. } = result.unwrap_err() else { + let Error::Server { code, msg, .. } = result.unwrap_err() else { unreachable!() }; assert_eq!(code, StatusCode::Internal); diff --git a/src/client/src/region/codec.rs b/src/client/src/region/codec.rs new file mode 100644 index 00000000000..9476385df36 --- /dev/null +++ b/src/client/src/region/codec.rs @@ -0,0 +1,63 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::borrow::Borrow; +use std::marker::PhantomData; + +use api::v1::region::{RegionRequest, RegionResponse}; +use prost::Message; +use tonic::Status; +use tonic::codec::{Codec, EncodeBuf, Encoder}; +use tonic_prost::ProstDecoder; + +/// Encodes owned or shared region requests using the existing protobuf wire format. +pub(crate) struct RegionCodec(PhantomData); + +impl Default for RegionCodec { + fn default() -> Self { + Self(PhantomData) + } +} + +impl Codec for RegionCodec +where + R: Borrow + Send + 'static, +{ + type Encode = R; + type Decode = RegionResponse; + type Encoder = RegionEncoder; + type Decoder = ProstDecoder; + + fn encoder(&mut self) -> Self::Encoder { + RegionEncoder(PhantomData) + } + + fn decoder(&mut self) -> Self::Decoder { + ProstDecoder::default() + } +} + +/// Borrows the protobuf payload while consuming its owned or shared container. +pub(crate) struct RegionEncoder(PhantomData); + +impl> Encoder for RegionEncoder { + type Item = R; + type Error = Status; + + fn encode(&mut self, item: R, buf: &mut EncodeBuf<'_>) -> Result<(), Status> { + item.borrow() + .encode(buf) + .map_err(|error| Status::internal(error.to_string())) + } +} diff --git a/src/cmd/src/datanode/scanbench.rs b/src/cmd/src/datanode/scanbench.rs index 42eac50ec45..9fbde16fb7b 100644 --- a/src/cmd/src/datanode/scanbench.rs +++ b/src/cmd/src/datanode/scanbench.rs @@ -836,7 +836,7 @@ impl ScanbenchCommand { } ScanbenchLogStore::Kafka(kafka_config) => { let log_store = Arc::new( - KafkaLogStore::try_new(kafka_config, None) + KafkaLogStore::try_new(kafka_config) .await .map_err(BoxedError::new) .context(error::BuildCliSnafu)?, diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index b000f7e1f02..83c16ffc2dc 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -212,6 +212,10 @@ pub struct StartCommand { internal_grpc_server_addr: Option, #[clap(long)] http_addr: Option, + /// The address to bind the dedicated public HTTP API server. Setting this + /// flag also enables the API server. + #[clap(long)] + http_api_server_addr: Option, #[clap(long)] http_timeout: Option, #[clap(long)] @@ -301,6 +305,11 @@ impl StartCommand { opts.http.disable_dashboard = disable_dashboard; } + if let Some(addr) = &self.http_api_server_addr { + opts.http.api_server_addr.clone_from(addr); + opts.http.enable_api_server = true; + } + if let Some(addr) = &self.grpc_bind_addr { opts.grpc.bind_addr.clone_from(addr); opts.grpc.tls = merge_tls_option(&opts.grpc.tls, tls_opts.clone()); diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 76c95c26d32..ca0f3b74057 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -328,6 +328,10 @@ impl App for Instance { pub struct StartCommand { #[clap(long)] http_addr: Option, + /// The address to bind the dedicated public HTTP API server. Setting this + /// flag also enables the API server. + #[clap(long)] + http_api_server_addr: Option, #[clap(long = "grpc-bind-addr", alias = "rpc-bind-addr", alias = "rpc-addr")] grpc_bind_addr: Option, #[clap(long)] @@ -418,6 +422,11 @@ impl StartCommand { opts.http.addr.clone_from(addr); } + if let Some(addr) = &self.http_api_server_addr { + opts.http.api_server_addr.clone_from(addr); + opts.http.enable_api_server = true; + } + if let Some(data_home) = &self.data_home { opts.storage.data_home.clone_from(data_home); } diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml index 702171e9d67..6df2283c2cf 100644 --- a/src/common/base/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -14,6 +14,7 @@ async-trait.workspace = true bitvec = "1.0" bytes.workspace = true futures.workspace = true +icu_properties.workspace = true lazy_static.workspace = true pin-project.workspace = true regex.workspace = true diff --git a/src/common/base/src/lib.rs b/src/common/base/src/lib.rs index 04fc7038018..2d5a6fbd894 100644 --- a/src/common/base/src/lib.rs +++ b/src/common/base/src/lib.rs @@ -25,6 +25,7 @@ pub mod readable_size; pub mod regex_pattern; pub mod secrets; pub mod serde; +pub mod term_token; pub type AffectedRows = usize; diff --git a/src/common/base/src/term_token.rs b/src/common/base/src/term_token.rs new file mode 100644 index 00000000000..a6121559a67 --- /dev/null +++ b/src/common/base/src/term_token.rs @@ -0,0 +1,280 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Word boundaries shared by `matches_term` and the fulltext bloom index. +//! +//! Bloom pruning is only sound if the index splits text exactly where +//! `matches_term` sees word boundaries, so both use [`classify_char`], and the +//! query side derives its probes with [`extract_term_probes`] and [`extract_like_probes`]. + +use std::ops::Range; + +use icu_properties::props::Script; +use icu_properties::{CodePointMapData, CodePointMapDataBorrowed}; + +/// Classes of characters that decide where `matches_term` sees a word boundary. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CharClass { + /// `[0-9A-Za-z]`. + AsciiWord, + /// Characters of the Han script, alphanumeric or not. + Han, + /// Non-ASCII alphanumerics outside the Han script, such as `é` or `д`. + UnicodeWord, + /// Everything else, including `_`. + Other, +} + +/// Returns the [`CharClass`] of `c`. [`tokenize`] splits text wherever the class +/// changes and at every [`CharClass::Other`] character. +pub fn classify_char(c: char) -> CharClass { + if c.is_ascii_alphanumeric() { + CharClass::AsciiWord + } else if is_han(c) { + CharClass::Han + } else if c.is_alphanumeric() { + CharClass::UnicodeWord + } else { + CharClass::Other + } +} + +static HAN_SCRIPT_DATA: CodePointMapDataBorrowed<'static, Script> = + CodePointMapData::