fix: adapt serializer tests to fallible query context

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

.cargo/config.toml

@@ -7,6 +7,11 @@ linker = "riscv64-linux-gnu-gcc"
 [alias]
 sqlness = "run --bin sqlness-runner --target-dir target/sqlness --"
 
+[registry]
+# Requires cargo 1.100+; older cargo ignores this key. Only registry crates
+# carry a publish time, so git dependencies (our GreptimeTeam forks) are exempt.
+global-min-publish-age = "7 days"
+
 [unstable.git]
 shallow_index = true
 shallow_deps = true

.github/actions/build-linux-artifacts/action.yml

@@ -31,6 +31,22 @@ inputs:
     description: Build GreptimeDB with large page size (65536).
     required: false
     default: 'false'
+  aws-ci-test-bucket:
+    description: Bucket used by the object-store integration tests
+    required: false
+    default: ''
+  aws-region:
+    description: Region of the test bucket
+    required: false
+    default: ''
+  aws-access-key-id:
+    description: Access key id for the test bucket
+    required: false
+    default: ''
+  aws-secret-access-key:
+    description: Secret access key for the test bucket
+    required: false
+    default: ''
 runs:
   using: composite
   steps:
@@ -38,6 +54,13 @@ runs:
       if: ${{ inputs.disable-run-tests == 'false' }}
       shell: bash
       # NOTE: If the BUILD_JOBS > 4, it's always OOM in EC2 instance.
+      # Without GT_S3_*, every remote object-store case is skipped and the
+      # release only ever exercises the local file backend.
+      env:
+        GT_S3_BUCKET: ${{ inputs.aws-ci-test-bucket }}
+        GT_S3_REGION: ${{ inputs.aws-region }}
+        GT_S3_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
+        GT_S3_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
       run: |
         cd ${{ inputs.working-dir }} && \
         make run-it-in-container BUILD_JOBS=4 \

.github/actions/setup-benchmark/action.yml

@@ -0,0 +1,38 @@
+name: Set up benchmark dependencies
+description: Reuse installed tools and install missing dependencies on an Ubuntu self-hosted runner
+runs:
+  using: composite
+  steps:
+    - name: Check host and install missing tools
+      shell: bash
+      run: |
+        set -euo pipefail
+        [[ "$(uname -m)" == x86_64 ]]
+        command -v git
+        missing=()
+        for tool in curl jq python3 rsync; do
+          command -v "$tool" >/dev/null || missing+=("$tool")
+        done
+        if ((${#missing[@]})); then
+          sudo -n apt-get update
+          sudo -n env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y --no-install-recommends "${missing[@]}"
+        fi
+    - name: Install Docker if missing and verify access
+      shell: bash
+      run: |
+        set -euo pipefail
+        if ! command -v docker >/dev/null; then
+          . /etc/os-release
+          [[ "$ID" == ubuntu ]]
+          sudo -n install -m 0755 -d /etc/apt/keyrings
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo -n tee /etc/apt/keyrings/docker.asc >/dev/null
+          sudo -n chmod a+r /etc/apt/keyrings/docker.asc
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $VERSION_CODENAME stable" | sudo -n tee /etc/apt/sources.list.d/docker.list >/dev/null
+          sudo -n apt-get update
+          sudo -n env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io docker-buildx-plugin
+        fi
+        if ! docker info >/dev/null 2>&1; then
+          sudo -n systemctl enable --now docker
+        fi
+        # Membership must be configured before the runner starts, not in this job.
+        docker info

.github/actions/setup-minio/action.yml

@@ -19,6 +19,8 @@ runs:
       --set rootUser=rootuser,rootPassword=rootpass123 \
       --set buckets[0].name=default \
       --set service.port=80,service.targetPort=9000 \
+      --set image.repository=pgsty/silo,image.tag=RELEASE.2026-09-16T00-00-00Z \
+      --set mcImage.repository=pgsty/mc,mcImage.tag=RELEASE.2026-09-16T00-00-00Z \
       minio/minio \
       --create-namespace \
       -n minio

.github/actions/start-runner/action.yml

@@ -56,7 +56,7 @@ runs:
 
     - name: Start EC2 runner
       if: startsWith(inputs.runner, 'ec2')
-      uses: machulav/ec2-github-runner@v2.3.8
+      uses: machulav/ec2-github-runner@v2.6.1
       id: start-linux-arm64-ec2-runner
       with:
         mode: start

.github/actions/stop-runner/action.yml

@@ -33,7 +33,7 @@ runs:
 
     - name: Stop EC2 runner
       if: ${{ inputs.label && inputs.ec2-instance-id }}
-      uses: machulav/ec2-github-runner@v2.3.8
+      uses: machulav/ec2-github-runner@v2.6.1
       with:
         mode: stop
         label: ${{ inputs.label }}

.github/runner-scale-sets/query-regression/Dockerfile

@@ -13,7 +13,7 @@ RUN git init /tmp/otelgen \
         -o /usr/local/bin/otelgen ./cmd/otelgen
 
 FROM ghcr.io/astral-sh/uv@sha256:3d868e555f8f1dbc324afa005066cd11e1053fc4743b9808ca8025283e65efa5 AS uv
-FROM ghcr.io/actions/actions-runner@sha256:08c30b0a7105f64bddfc485d2487a22aa03932a791402393352fdf674bda2c29
+FROM ghcr.io/actions/actions-runner:2.337.0@sha256:e5496277be5d09bc968b3d64911b74e219ac4a3f2edce956a3ecf9271bea1ef4
 
 USER root
 
@@ -22,6 +22,9 @@ ARG SCCACHE_SHA256=aec995a83ad3dff3d14b6314e08858b7b73d35ca85a5bcf3d3a9ec07dee35
 ARG RUSTUP_INIT_VERSION=1.29.0
 ARG RUSTUP_INIT_TARGET=x86_64-unknown-linux-gnu
 ARG RUSTUP_INIT_SHA256=4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10
+# Pinned to a nightly deliberately: the benchmark compiles historical base
+# checkouts (previous nightly builds) that still use #![feature] gates, so
+# the runner cannot follow the workspace's rust-toolchain.toml (now stable).
 ARG RUST_TOOLCHAIN=nightly-2026-03-21
 
 ENV RUSTUP_HOME=/opt/rustup \

.github/runner-scale-sets/query-regression/README.md

@@ -86,15 +86,35 @@ sweep.
 ### Building and updating the ECS image
 
 The runner `Dockerfile` in the parent directory stays the single source of the
-tool contract. Build a new ECS image from it:
+tool contract. The image is rebuilt **automatically** by the
+`rebuild-query-regression-runner-image` job in
+`.github/workflows/release-dev-builder-images.yaml` whenever anything under
+this directory changes on main (or via manual dispatch): it runs the ops tool
+below, opens an epoch-bump PR for `RUNNER_IMAGE_EPOCH` in
+`query-regression.yml`, and points the `QUERY_REGRESSION_ECS_IMAGE_ID` repo
+variable at the new image.
+
+The runner toolchain is **pinned inside the Dockerfile** to
+`nightly-2026-03-21` and deliberately does NOT follow the workspace
+`rust-toolchain.toml`: the benchmark also compiles the BASE checkout (the
+previous nightly build), which may predate the stable-toolchain migration and
+still require a nightly compiler.
+
+The manual fallback (also what the workflow runs):
 
 ```bash
 ALIBABA_CLOUD_ACCESS_KEY_ID=... ALIBABA_CLOUD_ACCESS_KEY_SECRET=... \
 uv run .github/runner-scale-sets/query-regression/ecs-image/build-ecs-image.py \
   --region-id <region> --vswitch-id <vsw-...> --security-group-id <sg-...> \
-  --base-image-id <ubuntu-24.04-image-id>
+  [--base-image-id <ubuntu-24.04-image-id>]
 ```
 
+`--base-image-id` is optional: the script defaults to the latest public
+Ubuntu 26.04 image in the region (26.04 matches the tool-version pins the
+Verify step asserts, e.g. python3 3.14; keep the two in sync); pass it — or
+set the `ALIYUN_ECS_BASE_IMAGE_ID` repo variable consumed by the automated
+job — to pin a specific base image.
+
 The script boots a temporary builder instance, `docker build`s the runner
 image, materializes `/opt/rustup`, `/opt/cargo`, `/usr/local/bin` tools, and
 `/home/runner` (actions-runner) onto the host, installs the ephemeral-runner
@@ -158,10 +178,16 @@ overridable via `QUERY_REGRESSION_RUNNER_UID`/`QUERY_REGRESSION_RUNNER_GID`)
 and exact tool versions: `libprotoc 3.21.12`, `uv 0.11.26`, `mold 2.40.4`,
 `Python 3.14.4`, `sccache 0.16.0`, `otelgen` commit
 `863a3f395d062c7322cc1de08a38774b7fdaa6c8`, root-owned `rustup 1.29.0`, and
-the image-baked `nightly-2026-03-21` Rust toolchain. `mold` and `python3`
-come from apt at image-build time (not Ubuntu 24.04's default 3.12); bump
-the Verify pins together with `QUERY_REGRESSION_ECS_IMAGE_ID` when the
-image is rebuilt. Rustup, Cargo, and Rustc
+the image-baked Rust toolchain `nightly-2026-03-21` pinned in the runner
+Dockerfile (deliberately independent of `rust-toolchain.toml` — see above).
+`mold` and `python3`
+come from apt at image-build time; if the
+Ubuntu archive ships a newer package revision between rebuilds, the Verify
+step fails with the observed version — bump those pins in `query-regression.yml`
+when that happens. Everything else (toolchain, uv, sccache, otelgen, rustup,
+the runner base) is pinned by digest/sha/commit in the Dockerfile, and the
+image id + `RUNNER_IMAGE_EPOCH` updates are automated by the rebuild job, so
+a routine rebuild needs no manual file updates. Rustup, Cargo, and Rustc
 must resolve from `/opt/cargo/bin`; the runner cannot write `/opt/rustup` or
 `/opt/cargo/bin`. Protobuf well-known includes, including
 `google/protobuf/any.proto` and `google/protobuf/empty.proto`, are an image

.github/runner-scale-sets/query-regression/ecs-image/build-ecs-image.py

@@ -25,7 +25,7 @@
 
 """Build the query-regression ECS custom image (manual ops tool).
 
-Boots a temporary pay-as-you-go ECS instance from a public Ubuntu 24.04 image,
+Boots a temporary pay-as-you-go ECS instance from a public Ubuntu 26.04 image,
 builds the existing runner container image (the Dockerfile in the parent
 directory remains the single source of the tool contract), materializes the
 tool directories onto the host filesystem so the workflow's "Verify runner
@@ -58,9 +58,9 @@ POLL_INTERVAL_SECONDS = 15
 CONSOLE_POLL_INTERVAL_SECONDS = 30
 BUILD_TIMEOUT_SECONDS = 60 * 60
 
-# Same apt package contract as the runner Dockerfile; the base
-# actions-runner image is Ubuntu 24.04, so an Ubuntu 24.04 host resolves the
-# same tool versions (protoc 3.21.12, mold 2.40.4, Python 3.14.4).
+# Same apt package contract as the runner Dockerfile; the base is an
+# Ubuntu 26.04 image, so an Ubuntu 26.04 host resolves the same tool
+# versions (protoc 3.21.12, mold 2.40.4, Python 3.14.4).
 # Docker itself comes from Docker's official repository (docker-ce), not the
 # distribution-packaged docker.io.
 APT_PACKAGES = [
@@ -229,6 +229,64 @@ def call_api_with_retry(fn, description: str, attempts: int = 5):
     raise RuntimeError("unreachable: retry loop exited without returning")
 
 
+def resolve_base_image_id(client, region_id: str) -> str:
+    """Resolve the latest public Ubuntu 26.04 x86_64 system image.
+
+    Used as the default for --base-image-id: the runner Dockerfile pins
+    every tool version itself, so a current stock Ubuntu LTS base is all
+    the builder needs. Pass --base-image-id (or ALIYUN_ECS_BASE_IMAGE_ID)
+    to pin a specific base image deterministically.
+    """
+    from alibabacloud_ecs20140526 import models as ecs_models
+
+    def _is_target_ubuntu(image) -> bool:
+        # osname is localized (e.g. "Ubuntu 26.04 64位"), osname_en the
+        # English form; accept either. Keep the Ubuntu version in sync with
+        # the tool-version pins asserted by the Verify step in
+        # query-regression.yml (e.g. python3 3.14 comes from 26.04).
+        for os_name in (image.osname_en, image.osname):
+            if os_name and "Ubuntu" in os_name and "26.04" in os_name:
+                return True
+        return False
+
+    page_size = 100
+    images: list = []
+    page_number = 1
+    while True:
+        page = call_api_with_retry(
+            lambda: client.describe_images(
+                ecs_models.DescribeImagesRequest(
+                    region_id=region_id,
+                    image_owner_alias="system",
+                    ostype="linux",
+                    architecture="x86_64",
+                    page_number=page_number,
+                    page_size=page_size,
+                )
+            ),
+            f"DescribeImages(system base, page {page_number})",
+        ).body.images.image or []
+        images.extend(page)
+        if len(page) < page_size:
+            break
+        page_number += 1
+
+    candidates = [image for image in images if _is_target_ubuntu(image)]
+    if not candidates:
+        raise SystemExit(
+            "No public Ubuntu 26.04 x86_64 system image found in region "
+            f"{region_id}; pass --base-image-id explicitly"
+        )
+    candidates.sort(key=lambda image: image.creation_time or "", reverse=True)
+    picked = candidates[0]
+    print(
+        f"Resolved base image: {picked.image_id} ({picked.osname_en or picked.osname}, "
+        f"created {picked.creation_time}) from {len(candidates)} candidates",
+        flush=True,
+    )
+    return picked.image_id
+
+
 def read_console_output(client, region_id: str, instance_id: str) -> str:
     """Fetch the instance's serial console output; no in-guest agent needed."""
     from alibabacloud_ecs20140526 import models as ecs_models
@@ -253,13 +311,19 @@ def main() -> int:
     parser.add_argument("--image-name", default=None, help="Defaults to a timestamped name.")
     args = parser.parse_args()
 
-    for name in ("region_id", "vswitch_id", "security_group_id", "base_image_id"):
+    for name in ("region_id", "vswitch_id", "security_group_id"):
         if not getattr(args, name):
             raise SystemExit(f"Missing required configuration: --{name.replace('_', '-')}")
 
     from alibabacloud_ecs20140526 import models as ecs_models
 
     client = make_ecs_client(args.region_id)
+    # --base-image-id is optional: default to the latest public Ubuntu 26.04
+    # image in the region (the Dockerfile pins every tool version itself, so
+    # base drift is low-risk; pass --base-image-id or set
+    # ALIYUN_ECS_BASE_IMAGE_ID to pin deterministically).
+    if not args.base_image_id:
+        args.base_image_id = resolve_base_image_id(client, args.region_id)
     image_name = args.image_name or time.strftime(
         "greptimedb-query-regression-runner-%Y%m%d%H%M%S", time.gmtime()
     )

.github/scripts/agent-observability-summary.py

@@ -18,6 +18,8 @@ import json
 import math
 from pathlib import Path
 
+from benchmark_resource_summary import render_resources
+
 TARGETS = ('greptimedb', 'clickhouse', 'victorialogs')
 
 
@@ -201,6 +203,7 @@ def lifecycle(root, targets):
             rows.append([context, entry.get('phase'), duration(entry.get('elapsed_ms')) if valid else
                          'invalid timestamp units (legacy artifact)', entry.get('exit_code')])
     lines += ['## Phase timings', '', 'Phase duration includes orchestration overhead. Container cleanup does not prove ECS teardown.', '', table(['DB / scope', 'Phase', 'Duration', 'Exit code'], rows), '']
+    lines.append(render_resources([('dataset', root)] + [(t, root / f'{t}-1') for t in targets], table))
     return '\n'.join(lines)
 
 

.github/scripts/aliyun-ecs-runner-provision.py

@@ -49,10 +49,10 @@ import base64
 import json
 import os
 import time
-import urllib.error
-import urllib.request
 from dataclasses import dataclass
 
+from runner_utils import append_github_output, create_registration_token, find_runner_by_name, github_api
+
 RUNNER_NAME_PREFIX = "qreg-ecs"
 RUNNER_LABEL_PREFIX = "query-regression-ecs"
 MANAGED_BY_TAG_KEY = "managed-by"
@@ -243,63 +243,6 @@ def encode_user_data(script: str) -> str:
     return base64.b64encode(script.encode("utf-8")).decode("ascii")
 
 
-def github_api(token: str, method: str, path: str, body: dict | None = None) -> dict:
-    data = json.dumps(body).encode("utf-8") if body is not None else None
-    request = urllib.request.Request(
-        f"https://api.github.com{path}",
-        data=data,
-        method=method,
-        headers={
-            "Accept": "application/vnd.github+json",
-            "Authorization": f"Bearer {token}",
-            "X-GitHub-Api-Version": "2022-11-28",
-        },
-    )
-    try:
-        with urllib.request.urlopen(request, timeout=30) as response:
-            if response.status == 204:
-                return {}
-            return json.loads(response.read().decode("utf-8"))
-    except urllib.error.HTTPError as error:
-        # GitHub's error body says exactly why (e.g. "Must have admin rights to
-        # Repository" for a PAT without the required scope); surface it instead
-        # of a bare "HTTP Error 403".
-        body = error.read().decode("utf-8", "replace")
-        raise SystemExit(
-            f"GitHub API {method} {path} failed: HTTP {error.code}: {body}\n"
-            "The token comes from the GH_PERSONAL_ACCESS_TOKEN secret; it needs "
-            "'repo' scope (classic PAT) or 'Administration: write' on the "
-            "repository (fine-grained PAT)."
-        ) from error
-
-
-def create_registration_token(github_token: str, repo: str) -> str:
-    response = github_api(
-        github_token,
-        "POST",
-        f"/repos/{repo}/actions/runners/registration-token",
-        body={},
-    )
-    return response["token"]
-
-
-def find_runner_by_name(github_token: str, repo: str, name: str) -> dict | None:
-    page = 1
-    while True:
-        response = github_api(
-            github_token,
-            "GET",
-            f"/repos/{repo}/actions/runners?per_page=100&page={page}",
-        )
-        runners = response.get("runners", [])
-        for runner in runners:
-            if runner.get("name") == name:
-                return runner
-        if len(runners) < 100:
-            return None
-        page += 1
-
-
 def make_ecs_client(config: ProvisionConfig):
     from alibabacloud_ecs20140526.client import Client as EcsClient
     from alibabacloud_tea_openapi.models import Config as OpenApiConfig
@@ -388,13 +331,6 @@ def dump_console_output(client, region_id: str, instance_id: str) -> None:
             summary.write("\n```\n</details>\n")
 
 
-def append_github_output(name: str, value: str) -> None:
-    output_path = os.environ.get("GITHUB_OUTPUT")
-    if output_path:
-        with open(output_path, "a", encoding="utf-8") as output:
-            output.write(f"{name}={value}\n")
-
-
 def run_instance(client, config: ProvisionConfig, user_data: str) -> str:
     from alibabacloud_ecs20140526 import models as ecs_models
 

.github/scripts/aliyun-ecs-runner-teardown.py

@@ -47,6 +47,8 @@ import sys
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 
+from runner_utils import deregister_runner
+
 # Reuse the GitHub API client, runner lookup, and tag constants from the
 # provision script.
 _PROVISION_SPEC = importlib.util.spec_from_file_location(
@@ -159,20 +161,6 @@ def delete_instance(client, instance_id: str, region_id: str | None = None) -> b
             return False
 
 
-def deregister_runner(token: str, repo: str, runner_name: str) -> bool:
-    runner = provision.find_runner_by_name(token, repo, runner_name)
-    if runner is None:
-        print(f"Runner {runner_name} is not registered", flush=True)
-        return True
-    try:
-        provision.github_api(token, "DELETE", f"/repos/{repo}/actions/runners/{runner['id']}")
-        print(f"Deregistered runner {runner_name} (id {runner['id']})", flush=True)
-        return True
-    except Exception as error:  # noqa: BLE001
-        print(f"Failed to deregister runner {runner_name}: {error}", flush=True)
-        return False
-
-
 def list_managed_instances(
     client, region_id: str
 ) -> list[tuple[str, str, str, str | None]]:

.github/scripts/aws-ec2-runner-provision.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# /// script
+# requires-python = ">=3.10"
+# dependencies = ["boto3>=1.35,<2"]
+# ///
+
+"""Provision an ephemeral EC2 CI runner from an Ubuntu x86_64 AMI."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import shlex
+import time
+from dataclasses import dataclass
+
+from runner_utils import append_github_output, create_registration_token, find_runner_by_name, github_api
+
+MANAGED_BY = "query-regression-ci"
+REPO_TAG = "github-repository"
+RUN_TAG = "query-regression-run-id"
+RUNNER_TAG = "github-runner-name"
+TTL_TAG_KEY = "runner-ttl-hours"
+
+
+def make_client(region: str):
+    import boto3
+    from botocore.config import Config
+
+    return boto3.client("ec2", region_name=region, config=Config(
+        connect_timeout=10, read_timeout=30,
+        retries={"mode": "standard", "max_attempts": 3},
+    ))
+
+
+@dataclass(frozen=True)
+class ProvisionConfig:
+    region: str
+    image_id: str
+    instance_type: str
+    subnet_id: str
+    security_group_id: str
+    repo: str
+    run_id: str
+    github_token: str
+    system_disk_gib: int = 500
+    ttl_hours: int = 8
+
+    def __post_init__(self):
+        if not re.fullmatch(r"[0-9]+-[0-9]+", self.run_id):
+            raise ValueError("run-id must be GitHub run_id-run_attempt")
+        if not 20 <= self.system_disk_gib <= 2048:
+            raise ValueError("system-disk-gib must be between 20 and 2048")
+        if not 1 <= self.ttl_hours <= 168:
+            raise ValueError("ttl-hours must be between 1 and 168")
+
+    @property
+    def runner_name(self) -> str:
+        return f"ci-ec2-{self.run_id}"
+
+
+def resolve_image(client, image_id: str) -> str:
+    if image_id != "auto":
+        return image_id
+    images = client.describe_images(
+        Owners=["099720109477"],
+        Filters=[
+            {"Name": "name", "Values": ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"]},
+            {"Name": "state", "Values": ["available"]},
+            {"Name": "architecture", "Values": ["x86_64"]},
+        ],
+    )["Images"]
+    if not images:
+        raise ValueError("No Canonical Ubuntu 24.04 amd64 AMI found in this region")
+    image = max(images, key=lambda image: image["CreationDate"])
+    print(f"Using AMI {image['ImageId']}: {image['Name']}")
+    return image["ImageId"]
+
+
+def root_disk(client, image_id: str, size: int) -> dict:
+    images = client.describe_images(ImageIds=[image_id])["Images"]
+    if len(images) != 1:
+        raise ValueError(f"AMI not found: {image_id}")
+    image = images[0]
+    if image["State"] != "available" or image["Architecture"] != "x86_64":
+        raise ValueError("Runner AMI must be available and x86_64")
+    root = next((d for d in image["BlockDeviceMappings"]
+                 if d["DeviceName"] == image["RootDeviceName"] and "Ebs" in d), None)
+    if root is None:
+        raise ValueError("Runner AMI must have an EBS root disk")
+    minimum = root["Ebs"]["VolumeSize"]
+    if size < minimum:
+        raise ValueError(f"Requested root disk {size} GiB is smaller than AMI minimum {minimum} GiB")
+    return {"DeviceName": image["RootDeviceName"], "Ebs": {
+        "VolumeSize": size, "VolumeType": "gp3", "Iops": 3000,
+        "Throughput": 125, "DeleteOnTermination": True,
+    }}
+
+
+def runner_download(token: str, repo: str) -> str:
+    runners = github_api(token, "GET", f"/repos/{repo}/actions/runners/downloads")
+    return next(r["download_url"] for r in runners if r["os"] == "linux" and r["architecture"] == "x64")
+
+
+def render_user_data(name: str, token: str, repo: str, download_url: str) -> str:
+    # Quote every external value before embedding it in the root shell script.
+    name, token, repo, download_url = map(shlex.quote, (name, token, repo, download_url))
+    return f"""#!/bin/bash
+set -euo pipefail
+for tool in git curl tar sudo; do command -v "$tool"; done
+# Mask systemd-oomd before enabling swap: Ubuntu 24.04 kills the whole
+# service cgroup on PSI pressure, and swap thrashing looks like pressure.
+systemctl disable --now systemd-oomd.socket systemd-oomd.service || true
+systemctl mask systemd-oomd.socket systemd-oomd.service || true
+# apt-daily-upgrade / unattended-upgrades can `systemctl restart` services
+# whose libraries were updated. That SIGTERM-s the runner mid-job; GitHub
+# records UserCancelled (job still valid, child exit 143). These VMs live
+# ~1h; security updates belong in the image, not at job time.
+systemctl disable --now unattended-upgrades.service apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service || true
+systemctl mask unattended-upgrades.service apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service || true
+systemctl stop unattended-upgrades.service || true
+killall -9 unattended-upgr || true
+cat > /etc/apt/apt.conf.d/99disable-auto-updates <<'APTEOF'
+APT::Periodic::Update-Package-Lists "0";
+APT::Periodic::Unattended-Upgrade "0";
+APT::Periodic::Download-Upgradeable-Packages "0";
+APTEOF
+if [[ ! -f "/swapfile" ]]; then
+  fallocate --length 16G "/swapfile"
+  chmod 600 "/swapfile"
+  mkswap "/swapfile"
+fi
+swapon "/swapfile"
+sysctl --write vm.swappiness=10
+swapon --show
+free --human
+
+id runner >/dev/null 2>&1 || useradd --create-home --shell /bin/bash runner
+# Establish membership before the runner service starts; setup installs Docker later.
+getent group docker >/dev/null || groupadd --system docker
+usermod -aG docker runner
+printf 'runner ALL=(ALL) NOPASSWD: ALL\\n' > /etc/sudoers.d/ci-runner
+chmod 0440 /etc/sudoers.d/ci-runner
+visudo -cf /etc/sudoers.d/ci-runner
+install -d -o runner -g "$(id -gn runner)" /home/runner/benchmark-data /opt/ci-github-runner
+cd /opt/ci-github-runner
+curl --fail --location --retry 3 {download_url} | tar xz
+./bin/installdependencies.sh
+chown -R runner:"$(id -gn runner)" /opt/ci-github-runner
+runuser -u runner -- ./config.sh --unattended --ephemeral --disableupdate --url https://github.com/{repo} --token {token} --name {name} --labels {name}
+./svc.sh install runner
+# Match the ECS runner's behavior when a child process is OOM-killed.
+service_name=$(cat .service)
+mkdir -p "/etc/systemd/system/$service_name.d"
+cat > "/etc/systemd/system/$service_name.d/oom.conf" <<'EOF'
+[Service]
+OOMPolicy=continue
+EOF
+systemctl daemon-reload
+# Package installation in a job must not restart its runner service.
+install -d /etc/needrestart/conf.d
+cat > /etc/needrestart/conf.d/actions_runner_services.conf <<'EOF'
+$nrconf{{override_rc}}{{qr(^actions\\.runner\\..+\\.service$)}} = 0;
+EOF
+./svc.sh start
+"""
+
+
+def run_instance(client, config: ProvisionConfig) -> str:
+    image_id = resolve_image(client, config.image_id)
+    disk = root_disk(client, image_id, config.system_disk_gib)
+    instance = client.describe_instance_types(InstanceTypes=[config.instance_type])["InstanceTypes"][0]
+    if "x86_64" not in instance["ProcessorInfo"]["SupportedArchitectures"]:
+        raise ValueError("Runner instance type must support x86_64")
+    token = create_registration_token(config.github_token, config.repo)
+    print(f"::add-mask::{token}")
+    script = render_user_data(config.runner_name, token, config.repo, runner_download(config.github_token, config.repo))
+    tags = [
+        {"Key": "Name", "Value": config.runner_name},
+        {"Key": "managed-by", "Value": MANAGED_BY},
+        {"Key": REPO_TAG, "Value": config.repo},
+        {"Key": RUN_TAG, "Value": config.run_id},
+        {"Key": RUNNER_TAG, "Value": config.runner_name},
+        {"Key": TTL_TAG_KEY, "Value": str(config.ttl_hours)},
+    ]
+    response = client.run_instances(
+        ImageId=image_id, InstanceType=config.instance_type,
+        MinCount=1, MaxCount=1, SubnetId=config.subnet_id,
+        SecurityGroupIds=[config.security_group_id],
+        BlockDeviceMappings=[disk], UserData=script,
+        ClientToken=config.runner_name,
+        MetadataOptions={"HttpTokens": "required", "HttpEndpoint": "enabled"},
+        TagSpecifications=[{"ResourceType": kind, "Tags": tags} for kind in ("instance", "volume")],
+    )
+    return response["Instances"][0]["InstanceId"]
+
+
+def dump_console(client, instance_id: str) -> None:
+    try:
+        # Boto3 decodes the API's base64 console output.
+        output = client.get_console_output(InstanceId=instance_id, Latest=True).get("Output", "")
+        print(f"::group::EC2 console {instance_id}")
+        print("\n".join(output.splitlines()[-80:]))
+        print("::endgroup::")
+    except Exception as error:
+        print(f"Unable to read console for {instance_id}: {error}")
+
+
+def provision(client, config: ProvisionConfig) -> int:
+    instance_id = run_instance(client, config)
+    # Publish before waiting: failed registration must still reach teardown.
+    for key, value in (("instance_id", instance_id),
+                       ("label", config.runner_name), ("runner_name", config.runner_name)):
+        append_github_output(key, value)
+    try:
+        client.get_waiter("instance_running").wait(
+            InstanceIds=[instance_id], WaiterConfig={"Delay": 5, "MaxAttempts": 60})
+        deadline = time.monotonic() + 10 * 60
+        while time.monotonic() < deadline:
+            runner = find_runner_by_name(config.github_token, config.repo, config.runner_name)
+            if runner and runner.get("status") == "online":
+                print(f"Runner {config.runner_name} online: {instance_id}")
+                return 0
+            time.sleep(5)
+        raise TimeoutError(f"Runner {config.runner_name} did not come online")
+    finally:
+        dump_console(client, instance_id)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    for name, env in (("region", "AWS_REGION"), ("repo", "GITHUB_REPOSITORY"),
+                      ("github-token", "GH_PERSONAL_ACCESS_TOKEN"),
+                      ("run-id", "AWS_EC2_RUN_ID"), ("image-id", "AWS_EC2_IMAGE_ID"),
+                      ("instance-type", "AWS_EC2_INSTANCE_TYPE"),
+                      ("subnet-id", "AWS_EC2_SUBNET_ID"),
+                      ("security-group-id", "AWS_EC2_SECURITY_GROUP_ID")):
+        parser.add_argument(f"--{name}", default=os.environ.get(env) or ("auto" if name == "image-id" else None))
+    parser.add_argument("--system-disk-gib", type=int, default=os.environ.get("AWS_EC2_SYSTEM_DISK_GIB", "500"))
+    parser.add_argument("--ttl-hours", type=int, default=os.environ.get("AWS_EC2_TTL_HOURS", "8"))
+    args = parser.parse_args()
+    for name, value in vars(args).items():
+        if value is None or value == "":
+            parser.error(f"Missing {name.replace('_', '-')}")
+    config = ProvisionConfig(**vars(args))
+    return provision(make_client(args.region), config)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/aws-ec2-runner-teardown.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# /// script
+# requires-python = ">=3.10"
+# dependencies = ["boto3>=1.35,<2"]
+# ///
+
+"""Delete managed EC2 CI runners, or sweep expired instances by ownership tags."""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import os
+import sys
+import time
+from pathlib import Path
+from datetime import datetime, timedelta, timezone
+
+from runner_utils import deregister_runner
+
+_SPEC = importlib.util.spec_from_file_location("aws_ec2_runner_provision", Path(__file__).with_name("aws-ec2-runner-provision.py"))
+assert _SPEC is not None and _SPEC.loader is not None
+provision = importlib.util.module_from_spec(_SPEC)
+sys.modules[_SPEC.name] = provision
+_SPEC.loader.exec_module(provision)
+make_client = provision.make_client
+dump_console = provision.dump_console
+MANAGED_BY = provision.MANAGED_BY
+REPO_TAG = provision.REPO_TAG
+RUN_TAG = provision.RUN_TAG
+RUNNER_TAG = provision.RUNNER_TAG
+TTL_TAG_KEY = provision.TTL_TAG_KEY
+SWEEP_TTL_HOURS = 4
+
+
+def managed_instances(client, repo: str, run_id: str | None = None,
+                      instance_id: str | None = None) -> list[dict]:
+    filters = [{"Name": "tag:managed-by", "Values": [MANAGED_BY]},
+               {"Name": f"tag:{REPO_TAG}", "Values": [repo]}]
+    if instance_id:
+        # A retried teardown may have a newer workflow attempt than creation.
+        filters.append({"Name": "instance-id", "Values": [instance_id]})
+    elif run_id:
+        filters.append({"Name": f"tag:{RUN_TAG}", "Values": [run_id]})
+    return [instance
+            for page in client.get_paginator("describe_instances").paginate(Filters=filters)
+            for reservation in page["Reservations"] for instance in reservation["Instances"]]
+
+
+def delete_instance(client, instance_id: str) -> bool:
+    dump_console(client, instance_id)
+    for attempt in range(12):
+        try:
+            client.terminate_instances(InstanceIds=[instance_id])
+            client.get_waiter("instance_terminated").wait(
+                InstanceIds=[instance_id], WaiterConfig={"Delay": 5, "MaxAttempts": 60})
+            return True
+        except Exception as error:
+            code = getattr(error, "response", {}).get("Error", {}).get("Code")
+            if code == "InvalidInstanceID.NotFound":
+                if attempt < 11:
+                    time.sleep(5)
+                    continue
+                # The ownership lookup observed this instance. Treat sustained
+                # NotFound after bounded retries as an already-deleted VM.
+                return True
+            print(f"Failed to terminate {instance_id}: {error}")
+            return False
+    return False
+
+
+def teardown(client, repo: str, token: str, run_id: str | None = None,
+             instance_id: str | None = None, now: datetime | None = None,
+             runner_name: str | None = None) -> int:
+    # Lookup by ownership/run tags also recovers a create whose response or job
+    # outputs were lost. Sweep never touches legacy build/JSONBench instances.
+    instances = managed_instances(client, repo, run_id, instance_id)
+    # Tags can lag RunInstances. Retry before concluding a targeted VM is gone.
+    if run_id and not instances:
+        for _ in range(11):
+            time.sleep(5)
+            instances = managed_instances(client, repo, run_id, instance_id)
+            if instances:
+                break
+    ok = True
+    found = False
+    for instance in instances:
+        identity = instance["InstanceId"]
+        if instance_id and identity != instance_id:
+            continue
+        found = True
+        tags = {t["Key"]: t["Value"] for t in instance.get("Tags", [])}
+        name = tags.get(RUNNER_TAG, "")
+        if not name.startswith("ci-ec2-"):
+            print(f"Skipping {identity}: missing managed runner name")
+            ok = False
+            continue
+        if not run_id:
+            ttl = tags.get(TTL_TAG_KEY)
+            try:
+                hours = SWEEP_TTL_HOURS if ttl is None else int(ttl)
+                if not 1 <= hours <= 168:
+                    raise ValueError("TTL outside 1..168 hours")
+            except (TypeError, ValueError):
+                print(f"Skipping {identity}: invalid TTL {ttl!r}")
+                continue
+            if (now or datetime.now(timezone.utc)) - instance["LaunchTime"] < timedelta(hours=hours):
+                continue
+        # A GitHub outage must not prevent EC2 deletion, or vice versa.
+        ok &= delete_instance(client, identity)
+        try:
+            ok &= deregister_runner(token, repo, name)
+        except (Exception, SystemExit) as error:
+            print(f"Failed to unregister {name}: {error}")
+            ok = False
+    if run_id and not found:
+        # EC2 may already be gone while its GitHub registration remains.
+        name = runner_name or f"ci-ec2-{run_id}"
+        if not name.startswith("ci-ec2-"):
+            raise ValueError("Invalid CI runner name")
+        ok &= deregister_runner(token, repo, name)
+    return 0 if ok else 1
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    for name, env in (("region", "AWS_REGION"), ("repo", "GITHUB_REPOSITORY"),
+                      ("github-token", "GH_PERSONAL_ACCESS_TOKEN"),
+                      ("run-id", "AWS_EC2_RUN_ID"), ("instance-id", "AWS_EC2_INSTANCE_ID"),
+                      ("runner-name", "AWS_EC2_RUNNER_NAME")):
+        parser.add_argument(f"--{name}", default=os.environ.get(env))
+    parser.add_argument("--sweep", action="store_true")
+    args = parser.parse_args()
+    for name in ("region", "repo", "github_token"):
+        if not getattr(args, name):
+            parser.error(f"Missing {name.replace('_', '-')}")
+    if not args.sweep and not args.run_id:
+        parser.error("Missing run-id for targeted teardown")
+    return teardown(make_client(args.region), args.repo, args.github_token,
+                    None if args.sweep else args.run_id,
+                    None if args.sweep else args.instance_id,
+                    runner_name=None if args.sweep else args.runner_name)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/benchmark-version-comparison.py

@@ -0,0 +1,191 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compare validated GreptimeDB query results; never gate CI on noisy latency deltas."""
+import argparse
+import html
+import json
+import math
+import os
+from pathlib import Path
+import re
+import statistics
+
+
+def read(path):
+    return json.loads(path.read_text()) if path.is_file() else {}
+
+
+def validate(env):
+    enabled = env.get('COMPARE_GREPTIMEDB', 'false')
+    if enabled not in ('true', 'false'):
+        raise ValueError('COMPARE_GREPTIMEDB must be true or false')
+    if enabled == 'false':
+        return
+    if env.get('RUN_GREPTIMEDB') != 'true':
+        raise ValueError('Version comparison requires Run GreptimeDB')
+    for key in ('BASELINE_GREPTIMEDB_TAG', 'GREPTIMEDB_TAG'):
+        if not re.fullmatch(r'[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}(@sha256:[0-9a-f]{64})?', env.get(key, '')):
+            raise ValueError(f'Invalid {key}')
+    if env['BASELINE_GREPTIMEDB_TAG'] == env['GREPTIMEDB_TAG']:
+        raise ValueError('Baseline and candidate must have different image tags')
+
+
+def same(a, b, fields):
+    return all(a.get(k) is not None and a.get(k) == b.get(k) for k in fields)
+
+
+def finite(value):
+    return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0
+
+
+def improvement(old, new, throughput=False):
+    if not finite(old) or not finite(new) or old <= 0:
+        return None
+    return (new / old - 1 if throughput else 1 - new / old) * 100
+
+
+def row(query, metric, old, new, compatible, reason):
+    delta = improvement(old, new, metric == 'QPS') if compatible else None
+    return dict(query=query, metric=metric, baseline=old, candidate=new,
+                improvement_percent=delta, status='comparable' if delta is not None else reason)
+
+
+def agent(root):
+    def load(r):
+        return read(r / 'greptimedb-1/greptimedb-1/measured-summary.json')
+    a, b = load(root / 'baseline'), load(root)
+    finished = [r / 'greptimedb-1/exit-code.txt' for r in (root / 'baseline', root)]
+    compatible = (all(p.is_file() and p.read_text().strip() == '0' for p in finished)
+                  and a.get('passed') is True and b.get('passed') is True
+                  and same(a, b, ('contract_sha256', 'data_profile', 'query_window', 'warmup_runs', 'measured_runs')))
+    qa, qb = ({q['query_id']: q for q in x.get('per_query', [])} for x in (a, b))
+    rows = []
+    for q in sorted(qa.keys() | qb.keys()):
+        old, new = qa.get(q, {}), qb.get(q, {})
+        ok = compatible and same(old, new, ('result_fingerprint', 'success')) and old.get('success', 0) > 0
+        ok = ok and all(v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new))
+        for key in ('p50_ms', 'p95_ms', 'p99_ms'):
+            rows.append(row(q, key, old.get(key), new.get(key), ok, 'failed, missing or incompatible result'))
+    return rows
+
+
+def traces(root):
+    a, b = (read(r / 'runs/greptimedb/run.json') for r in (root / 'baseline', root))
+    compatible = (a.get('success') is True and b.get('success') is True
+                  and same(a, b, ('expected', 'protocol', 'audit_expected', 'corpus'))
+                  and same(a.get('execution', {}), b.get('execution', {}),
+                           ('db_cpus', 'db_memory', 'load_workers_requested', 'max_in_flight')))
+    rows = []
+    for q in (f'T{i:02}' for i in range(1, 9)):
+        values, identities, counts = [], [], []
+        ok = compatible
+        for run in (a, b):
+            correctness = [s for s in run.get('correctness', []) if s.get('query_id') == q]
+            all_samples = [s for s in run.get('samples', []) if s.get('query_id') == q]
+            measured = [s for s in all_samples if s.get('phase') == 'measured']
+            passed = bool(correctness and measured) and all(s.get('outcome') == 'passed' for s in correctness + all_samples)
+            expected = run.get('expected', {}).get('contracts', {}).get(q, {})
+            passed = passed and all(s.get('fingerprint') == expected.get('fingerprint') and s.get('fingerprint') is not None
+                                    and s.get('row_count') == expected.get('row_count') for s in correctness + all_samples)
+            elapsed = [s.get('elapsed_ms') for s in measured]
+            passed = passed and all(finite(v) for v in elapsed)
+            ok = ok and passed
+            values.append(statistics.median(elapsed) if passed else None)
+            identities.append(expected)
+            counts.append((len(correctness), len(all_samples), len(measured)))
+        ok = ok and identities[0] == identities[1] and counts[0] == counts[1]
+        rows.append(row(q, 'median_ms', *values, ok, 'failed, skipped, missing or incompatible result'))
+    return rows
+
+
+def long_range(root):
+    rows = []
+    manifest = read(root / 'run-manifest.json')
+    healthy = all(
+        all(field in (manifest.get(phase, {}).get(target, {}).get('docker', {}) if phase == 'post_load_results'
+                      else manifest.get(phase, {}).get(target, {})).get('state', '').split()
+            for field in ('Running=true', 'ExitCode=0', 'OOMKilled=false'))
+        for phase in ('post_load_results', 'post_query_results')
+        for target in ('greptimedb-baseline', 'greptimedb'))
+    for mode in ('serial', 'concurrent-duration-c1', 'concurrent-duration-c4'):
+        a, b = (read(root / name / 'results-vmbench-long-range' / (mode + '-summary.json'))
+                for name in ('greptimedb-baseline', 'greptimedb'))
+        fields = ('workload', 'query_file_sha256', 'start', 'end', 'step', 'profile', 'data_duration',
+                  'scrape_interval', 'cache_policy_effective', 'mode', 'connection_mode', 'request_timeout_seconds')
+        fields += ('tries', 'warmup_runs') if mode == 'serial' else ('concurrency', 'duration_seconds', 'warmup_seconds')
+        compatible = healthy and same(a, b, fields) and all(x.get('errors') == 0 and x.get('timeouts') == 0 for x in (a, b))
+        qa, qb = ({q['query_index']: q for q in x.get('per_query', [])} for x in (a, b))
+        for q in sorted(qa.keys() | qb.keys()):
+            old, new = qa.get(q, {}), qb.get(q, {})
+            ok = compatible and same(old, new, ('query', 'series', 'points'))
+            ok = ok and all(v.get('success', 0) > 0 and v.get('series', 0) > 0 and v.get('points', 0) > 0
+                            and v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new))
+            for key in ('p50_ms', 'qps') if mode != 'serial' else ('p50_ms',):
+                rows.append(row(f'{mode}/Q{q}', 'QPS' if key == 'qps' else key,
+                                old.get(key), new.get(key), ok, 'failed, missing or incompatible result'))
+    return rows
+
+
+def render(document):
+    def cell(value):
+        return html.escape(str(value)).replace('|', '&#124;').replace('\n', ' ')
+    def value(v, metric):
+        if not finite(v):
+            return '—'
+        if metric == 'QPS':
+            return f'{v:.3f}'
+        return f'{v / 1000:.2f} s' if v >= 1000 else f'{v:.2f} ms'
+    lines = ['# GreptimeDB version comparison', '',
+             f'Baseline: `{cell(document["baseline_tag"])}` → candidate: `{cell(document["candidate_tag"])}`.', '',
+             'Same-host sequential runs with fresh databases and shared input data. Positive = improvement; '
+             'latency: (baseline − candidate) / baseline; QPS: (candidate − baseline) / baseline. '
+             'Single runs are diagnostic, not a statistically established regression. No performance threshold gates CI.', '',
+             '| Query | Metric | Baseline | Candidate | Improvement | Status |',
+             '| --- | --- | --- | --- | --- | --- |']
+    for r in document['rows']:
+        delta = r['improvement_percent']
+        lines.append('| ' + ' | '.join(map(cell, [r['query'], r['metric'], value(r['baseline'], r['metric']),
+                     value(r['candidate'], r['metric']), f'{delta:+.1f}%' if delta is not None else '—', r['status']])) + ' |')
+    if document.get('kind') == 'long-range':
+        lines += ['', 'Long-range retains the existing non-empty series/point-count checks; these do not establish equality of every returned value.']
+    if not document['rows']:
+        lines += ['', 'Results unavailable; comparison not performed.']
+    return '\n'.join(lines) + '\n'
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('command', choices=('validate', 'report'))
+    parser.add_argument('--kind', choices=('agent', 'traces', 'long-range'))
+    parser.add_argument('--root', type=Path)
+    args = parser.parse_args()
+    if args.command == 'validate':
+        validate(os.environ)
+        return
+    if args.root is None or args.kind is None:
+        parser.error('report requires --root and --kind')
+    document = {'kind': args.kind, 'baseline_tag': os.environ.get('BASELINE_GREPTIMEDB_TAG', ''),
+                'candidate_tag': os.environ.get('GREPTIMEDB_TAG', ''),
+                'rows': {'agent': agent, 'traces': traces, 'long-range': long_range}[args.kind](args.root)}
+    text = render(document)
+    # Preserve partial comparisons even if a target failed before creating output.
+    args.root.mkdir(parents=True, exist_ok=True)
+    (args.root / 'version-comparison.json').write_text(json.dumps(document, indent=2, allow_nan=False) + '\n')
+    (args.root / 'version-comparison.md').write_text(text)
+    print(text, end='')
+
+
+if __name__ == '__main__':
+    main()

.github/scripts/benchmark_resource_summary.py

@@ -0,0 +1,99 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Render shared O11yBench sampled resource artifacts for lifecycle summaries."""
+import json
+import math
+
+
+def number(value):
+    return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0
+
+
+def metric(values, key, statistic):
+    entry = values.get(key, {})
+    value = entry.get(statistic)
+    return value if number(entry.get('samples')) and entry['samples'] > 0 and number(value) else None
+
+
+def decimal(value, suffix=''):
+    return f'{value:.2f}{suffix}' if number(value) else '—'
+
+
+def memory(value):
+    if not number(value):
+        return '—'
+    for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'):
+        if value < 1024 or unit == 'TiB':
+            return decimal(value, f' {unit}')
+        value /= 1024
+
+
+def capacities(path):
+    # Quotas are metadata, not recomputed CPU statistics. Bound reads on long runs.
+    result = {}
+    try:
+        with path.open() as source:
+            for index, line in enumerate(source):
+                if index == 256:
+                    break
+                record = json.loads(line)
+                cores = record.get('host', {}).get('cpu_count')
+                if number(cores) and cores > 0:
+                    result.setdefault('host', cores)
+                for role, values in record.get('containers', {}).items():
+                    quota = values.get('configured_cpu_limit_cores')
+                    if number(quota):
+                        result.setdefault(role, quota)
+                if all(role in result for role in ('host', 'runtime', 'db')):
+                    break
+    except (OSError, ValueError, TypeError, AttributeError):
+        pass
+    return result
+
+
+def render_resources(scopes, table):
+    rows = []
+    for scope, directory in scopes:
+        try:
+            summary = json.loads((directory / 'resources.summary.json').read_text())
+            phases = summary.get('phases', {}) if summary.get('schema') == 'o11ybench-resources-v1' else {}
+        except (OSError, ValueError, TypeError, AttributeError):
+            phases = {}
+        if not isinstance(phases, dict) or not phases:
+            rows.append([scope, 'unavailable', *['—'] * 9])
+            continue
+        limits = capacities(directory / 'resources.jsonl')
+        for phase, data in phases.items():
+            for role in ('host', 'runtime', 'db'):
+                values = data.get('host', {}) if role == 'host' else data.get('containers', {}).get(role, {})
+                if role == 'db' and scope == 'dataset':
+                    continue
+                quota = limits.get(role)
+                cpu_samples = values.get('cpu_used_cores', {}).get('samples')
+                rows.append([scope, phase, role, cpu_samples,
+                             decimal(metric(values, 'cpu_used_cores', 'mean')),
+                             decimal(metric(values, 'cpu_used_cores', 'max')),
+                             'unlimited' if quota == 0 else decimal(quota),
+                             memory(metric(values, 'memory_used_bytes' if role == 'host' else 'memory_usage_bytes', 'max')),
+                             decimal(metric(values, 'io_wait_percent', 'mean'), '%'),
+                             decimal(metric(values, 'io_wait_percent', 'max'), '%'),
+                             memory(metric(values, 'swap_used_bytes', 'max'))])
+    return '\n'.join(['## Sampled resources', '',
+                      'CPU is measured in cores. Mean/max summarize available samples, not continuous peaks; '
+                      'short phases may have no CPU sample. CPU capacity is host cores or the configured container quota. '
+                      'Host memory is total minus available; Docker memory excludes cache where supported. '
+                      'Swap is host-wide usage, not swap activity. Missing evidence stays —.', '',
+                      table(['Scope', 'Phase', 'Component', 'CPU samples', 'Mean CPU cores', 'Max CPU cores',
+                             'CPU capacity', 'Max memory', 'Mean I/O wait', 'Max I/O wait', 'Max swap'], rows), ''])

.github/scripts/check-nightly-release.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Returns whether the current release run should proceed, 'true' or 'false'.
+#
+# The nightly schedule runs every weekday, but a scheduled release only proceeds
+# when the latest published nightly release is older than
+# NIGHTLY_RELEASE_MAX_AGE_DAYS days. This way, a failed nightly run is retried
+# by the next weekday schedule, instead of leaving a whole week without nightly
+# builds. Non-scheduled runs (tag pushes and manual dispatches) always proceed.
+#
+# Only the result is written to stdout, so that it can be used in GitHub Actions
+# outputs; all diagnostics go to stderr.
+#
+# You can run as following examples:
+#   GITHUB_EVENT_NAME=push ./check-nightly-release.sh
+#   GITHUB_EVENT_NAME=schedule GITHUB_REPOSITORY=GreptimeTeam/greptimedb GH_TOKEN=ghp_xxx ./check-nightly-release.sh
+#   GITHUB_EVENT_NAME=schedule NIGHTLY_RELEASE_MAX_AGE_DAYS=5 GITHUB_REPOSITORY=GreptimeTeam/greptimedb GH_TOKEN=ghp_xxx ./check-nightly-release.sh
+if [ -z "$GITHUB_EVENT_NAME" ]; then
+    echo "GITHUB_EVENT_NAME is empty" >&2
+    exit 1
+fi
+
+if [ "$GITHUB_EVENT_NAME" != "schedule" ]; then
+    echo "Not a scheduled run, the nightly check is skipped." >&2
+    echo "true"
+    exit 0
+fi
+
+MAX_AGE_DAYS="${NIGHTLY_RELEASE_MAX_AGE_DAYS:-5}"
+if ! [[ "$MAX_AGE_DAYS" =~ ^[0-9]+$ ]]; then
+    echo "Invalid NIGHTLY_RELEASE_MAX_AGE_DAYS: ${MAX_AGE_DAYS}" >&2
+    exit 1
+fi
+
+SECONDS_PER_DAY=86400
+
+# The publish time of the latest published nightly release, like the tag
+# 'v0.2.0-nightly-20230313'. Fail open on GitHub API errors: proceeding with
+# the release is safer than skipping it.
+if ! latest_nightly_published_at="$(gh api "repos/${GITHUB_REPOSITORY}/releases?per_page=100" \
+    --jq '[.[] | select(.draft == false and (.tag_name | contains("nightly")))][0].published_at // empty')"; then
+    echo "::warning::Failed to query the latest nightly release from the GitHub API, will proceed with the nightly release." >&2
+    echo "true"
+    exit 0
+fi
+
+if [ -z "$latest_nightly_published_at" ]; then
+    echo "No published nightly release found, will proceed with the nightly release." >&2
+    echo "true"
+    exit 0
+fi
+
+published_at_epoch="$(date --date="$latest_nightly_published_at" +%s)"
+now_epoch="$(date +%s)"
+age_seconds=$(( now_epoch - published_at_epoch ))
+age_days=$(( age_seconds / SECONDS_PER_DAY ))
+
+echo "The latest nightly release was published at ${latest_nightly_published_at} (${age_days} days ago)." >&2
+
+if [ "$age_seconds" -gt $(( MAX_AGE_DAYS * SECONDS_PER_DAY )) ]; then
+    echo "The latest nightly release is older than ${MAX_AGE_DAYS} days, will proceed with the nightly release." >&2
+    echo "true"
+else
+    echo "The latest nightly release is not older than ${MAX_AGE_DAYS} days, will skip the nightly release." >&2
+    echo "false"
+fi

.github/scripts/ci-runner-config.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Resolve dispatch provider and instance type before allocating a runner."""
+
+import argparse
+import re
+
+from runner_utils import append_github_output
+
+DEFAULT_INSTANCE_TYPES = {"Aliyun": "ecs.c9i.2xlarge", "AWS": "c7i.2xlarge"}
+
+
+def resolve_instance_type(provider: str, requested: str) -> str:
+    if provider not in DEFAULT_INSTANCE_TYPES:
+        raise ValueError("provider must be Aliyun or AWS")
+    instance_type = DEFAULT_INSTANCE_TYPES[provider] if requested == "auto" else requested
+    pattern = r"ecs\.[a-z0-9-]+\.[a-z0-9-]+" if provider == "Aliyun" else r"[a-z0-9-]+\.[a-z0-9-]+"
+    if not re.fullmatch(pattern, instance_type):
+        raise ValueError(f"Invalid {provider} instance type: {instance_type!r}")
+    return instance_type
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--provider", required=True)
+    parser.add_argument("--instance-type", default="auto")
+    args = parser.parse_args()
+    instance_type = resolve_instance_type(args.provider, args.instance_type)
+    append_github_output("instance_type", instance_type)
+    print(f"CI runner: {args.provider} / {instance_type}")
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/ci-runners.md

@@ -0,0 +1,73 @@
+# CI runners
+
+The agent observability and long-range benchmark workflows select `Aliyun` or
+`AWS` through the `provider` dispatch input. `instance_type: auto` selects
+`ecs.c9i.2xlarge` or `c7i.2xlarge`, respectively (8 vCPU, 16 GiB).
+Existing build/release and JSONBench workflows keep their current runner actions.
+
+Aliyun uses the existing ECS actions and image. Its provision and teardown
+scripts share only GitHub API and output helpers from `runner_utils.py` with
+the new AWS scripts.
+
+AWS uses `aws-ec2-runner-provision.py` and `aws-ec2-runner-teardown.py` directly.
+Repository variables supply `EC2_RUNNER_REGION`, `EC2_RUNNER_SUBNET_ID`, and
+`EC2_RUNNER_SECURITY_GROUP_ID`; credentials use `AWS_ACCESS_KEY_ID`,
+`AWS_SECRET_ACCESS_KEY`, and `GH_PERSONAL_ACCESS_TOKEN` secrets.
+The current region is Singapore (`ap-southeast-1`). By default the script selects
+the newest available Canonical Ubuntu 24.04 amd64 server AMI in that region
+(owner `099720109477`) and logs its ID. Set `BENCHMARK_EC2_IMAGE_ID` to pin a
+prepared Ubuntu AMI. Existing build CI's `EC2_RUNNER_LINUX_AMD64_IMAGE_ID` is
+unaffected. Both workflows default `system_disk_gib` to `auto` (also accepting
+`0`): agent observability uses S=80, P=100, M=500 GiB, and long-range uses 80 GiB.
+These defaults are the same for both providers. Explicit values from 20 to 2048
+GiB override them and must be at least the selected AMI's root snapshot size.
+The official 24.04 image does not have the old custom AMI's 500 GiB minimum.
+
+AWS user data registers a fresh ephemeral runner. Git, curl, tar, and sudo are
+expected in the Ubuntu server image. Docker group membership is set before the
+runner service starts, and this dedicated VM grants the runner passwordless sudo
+so its job can install dependencies. Like ECS, the host creates 16 GiB swap with
+swappiness 10, disables systemd-oomd and background automatic updates, and sets
+`OOMPolicy=continue` on the runner service. Explicit setup package installation
+remains enabled.
+
+Both benchmark jobs call `.github/actions/setup-benchmark` after checkout. It
+reuses existing tools, installs missing curl/jq/python3/rsync, installs Docker CE
+if absent, and checks Docker access. Missing packages require passwordless sudo;
+Aliyun's prepared image normally provides the tools and Docker already. Existing
+Docker installations are preserved, so exact versions need not match between
+providers. Provisioning must establish Docker permissions before starting the
+runner; adding a group inside the running job would not update its groups.
+
+The subnet must provide outbound access to Ubuntu and Docker packages, GitHub,
+and the configured container registries. The IAM identity needs EC2
+image/instance-type discovery, instance and volume tagging, launch, describe,
+console output, and termination permissions. GitHub credentials need repository
+runner administration access.
+
+Creation publishes instance identity before waiting for registration. Teardown
+can recover missing outputs using repository and workflow-attempt tags. The
+daily AWS janitor (03:23 UTC) removes expired instances tagged `managed-by=query-regression-ci`
+and unregisters their runners. Both providers use the existing
+`managed-by=query-regression-ci`, `query-regression-run-id`, and
+`runner-ttl-hours` tags. AWS retains a run-attempt suffix in its run identity to
+isolate retries, plus repository and runner-name tags to restrict cleanup.
+Legacy build instances do not match these ownership filters.
+Both janitors use a 4-hour fallback when the TTL tag is absent and skip invalid
+TTL tags. Benchmark jobs explicitly tag their configured TTL (default 8 hours).
+Cleanup does not check whether the GitHub job is still running. Expired instances
+are removed on the next daily sweep; normal workflow teardown runs immediately.
+TTL is measured from instance launch, so it must cover provisioning and the
+benchmark. The scheduled janitor becomes active after landing on the default
+branch.
+
+Local validation:
+
+```sh
+python3 tests/perf/test_ci_runner_scripts.py
+python3 tests/perf/test_aliyun_ecs_runner_scripts.py
+actionlint .github/workflows/agent-observability.yml .github/workflows/vmbench-long-range.yml .github/workflows/aws-ec2-runner-janitor.yml
+```
+
+A real AWS dispatch is still needed to verify AMI boot, package
+installation, registry connectivity, and end-to-end benchmark execution.

.github/scripts/ci-slash.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Admit `/ci` comments and dispatch a pinned draft-PR CI workflow."""
+"""Admit `/ci` comments and dispatch a draft-PR CI workflow."""
 import json, os, re, sys, urllib.request
 
 COMMAND = re.compile(r'^/ci(?:\s+(.+))?$')
@@ -29,7 +29,7 @@ OPTIONS = {
     'fuzz chaos': ('integration.yml', 'chaos', 'chaos fuzz'),
     'fuzz all': ('integration.yml', 'all', 'all fuzz'),
 }
-HELP = '''Available draft-PR CI commands:\n\n- `/ci` or `/ci rust` — Rust CI\n- `/ci integration` — integration CI without fuzz\n- `/ci checks` or `/ci docs`\n- `/ci fuzz standalone|distributed|chaos`\n- `/ci fuzz all` — all fuzz suites (admin only)\n\nCommands require repository admin permission and a same-repository open draft PR. CI is pinned to the current head SHA; comment again after a push.'''
+HELP = '''Available draft-PR CI commands:\n\n- `/ci` — standard CI\n- `/ci rust` — Rust CI\n- `/ci integration` — integration CI without fuzz\n- `/ci checks` or `/ci docs`\n- `/ci fuzz standalone|distributed|chaos`\n- `/ci fuzz all` — all fuzz suites (admin only)\n\nCommands require the PR author or repository write/maintain/admin permission and an open draft PR. Fork PRs require repository write permission and rerun existing pull-request CI; fuzz commands are same-repository only. `/ci fuzz all` requires admin permission. Same-repository CI runs the branch head at dispatch time. Fork CI reruns the original pull-request revision (within GitHub rerun limits); sync your branch with the updated CI configuration first.'''
 
 def api(path):
     req=urllib.request.Request(os.environ['GITHUB_API_URL']+path, headers={'Authorization':'Bearer '+os.environ['GITHUB_TOKEN'],'Accept':'application/vnd.github+json'})
@@ -46,7 +46,7 @@ def reject(number, text): out(skip='true',pr_number=number,reply='CI command ign
 def main():
     if os.environ.get('DISPATCH_SENDER') != 'github-actions[bot]': return reject('', 'invalid dispatch sender.')
     comment=api('/repos/'+os.environ['GITHUB_REPOSITORY']+'/issues/comments/'+os.environ['COMMENT_ID'])
-    body=comment.get('body','').splitlines()[0].strip(); m=COMMAND.fullmatch(body)
+    body=(comment.get('body') or '').partition('\n')[0].strip(); m=COMMAND.fullmatch(body)
     number=str(comment.get('issue_url','').rstrip('/').split('/')[-1])
     if not m: return reject(number,'comment is not a `/ci` command.')
     arg=(m.group(1) or '').strip().lower()
@@ -54,13 +54,38 @@ def main():
     if arg=='help': out(skip='true',pr_number=number,reply=HELP); return 0
     pr=api('/repos/'+os.environ['GITHUB_REPOSITORY']+'/pulls/'+number)
     if pr.get('state')!='open' or not pr.get('draft'): return reject(number,'PR must be open and draft.')
-    if pr.get('head',{}).get('repo',{}).get('full_name') != os.environ['GITHUB_REPOSITORY']: return reject(number,'fork PRs are not admitted.')
-    head=pr.get('head',{}).get('sha','')
-    if head != os.environ.get('DISPATCH_HEAD_SHA') or not re.fullmatch('[0-9a-f]{40}',head): return reject(number,'PR head changed; comment again.')
+    fork=pr.get('head',{}).get('repo',{}).get('full_name') != os.environ['GITHUB_REPOSITORY']
+    if fork and arg.startswith('fuzz '): return reject(number,'fuzz commands require a same-repository PR.')
+    head=pr.get('head',{})
+    head_sha=head.get('sha','')
+    head_ref=head.get('ref','')
+    if head_sha != os.environ.get('DISPATCH_HEAD_SHA') or not re.fullmatch('[0-9a-f]{40}',head_sha) or not head_ref: return reject(number,'PR head changed; comment again.')
     actor=comment.get('user',{}).get('login','')
-    permission=api('/repos/'+os.environ['GITHUB_REPOSITORY']+'/collaborators/'+actor+'/permission').get('user',{}).get('permission')
-    if permission!='admin': return reject(number,'repository admin permission is required.')
+    if not actor: return reject(number,'comment author is missing.')
+    is_author=actor==pr.get('user',{}).get('login')
+    if fork or arg=='fuzz all' or not is_author:
+        permission=api('/repos/'+os.environ['GITHUB_REPOSITORY']+'/collaborators/'+actor+'/permission').get('permission')
+        if arg=='fuzz all':
+            if permission!='admin': return reject(number,'repository admin permission is required for `/ci fuzz all`.')
+        elif permission not in ('write','maintain','admin'):
+            return reject(number,'PR author or repository write permission is required.')
     workflow, profile, label=OPTIONS[arg]
-    out(skip='false',pr_number=number,head_sha=head,workflow=workflow,fuzz_profile=profile,reply=f'Dispatched {label} for `{head}`.')
+    if fork:
+        runs=api('/repos/'+os.environ['GITHUB_REPOSITORY']+'/actions/runs?event=pull_request&head_sha='+head_sha+'&per_page=100')['workflow_runs']
+        selected={}
+        for run in runs:
+            name=run.get('path','').split('@',1)[0].removeprefix('.github/workflows/')
+            if name not in workflow.split(',') or name in selected: continue
+            if run.get('head_sha') != head_sha or run.get('head_repository',{}).get('full_name') != head['repo']['full_name'] or run.get('head_branch') != head_ref: continue
+            selected[name]=run
+        if not selected: return reject(number,'no pull-request CI runs found for this head; push a new commit first.')
+        if arg and workflow not in selected: return reject(number,'selected CI has no pull-request run for this head.')
+        if any(run['status'] != 'completed' for run in selected.values()): return reject(number,'pull-request CI is still pending; approve or wait for it, then comment again.')
+        for run in selected.values():
+            original=run if run.get('run_attempt',1)==1 else api('/repos/'+os.environ['GITHUB_REPOSITORY']+'/actions/runs/'+str(run['id'])+'/attempts/1')
+            if original.get('conclusion') != 'skipped': return reject(number,'only originally skipped draft CI runs can be requested; push a new commit while draft.')
+        out(skip='false',pr_number=number,run_ids=','.join(str(run['id']) for run in selected.values()),reply=f'Requested rerun of {", ".join(selected)} for `{head_sha}`.')
+        return 0
+    out(skip='false',pr_number=number,head_sha=head_sha,head_ref=head_ref,workflow=workflow,fuzz_profile=profile,reply=f'Dispatched {label} for branch `{head_ref}`.')
     return 0
 if __name__ == '__main__': sys.exit(main())

.github/scripts/pull-benchmark-image.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Pull only: never retry database writes, benchmarks, or cloud provisioning.
+set -euo pipefail
+
+if [[ $# != 1 || -z "$1" || "$1" == -* ]]; then
+    echo "Usage: $0 IMAGE" >&2
+    exit 2
+fi
+image=$1
+attempts=3
+for ((attempt=1; attempt<=attempts; attempt++)); do
+    echo "Pulling $image (attempt $attempt/$attempts)" >&2
+    if timeout --kill-after=10s 180s docker pull "$image"; then
+        exit 0
+    else
+        status=$?
+    fi
+    if ((attempt == attempts)); then
+        echo "Failed to pull $image after $attempts attempts (exit $status)" >&2
+        exit "$status"
+    fi
+    delay=$((attempt * 10))
+    echo "Pull failed (exit $status); retrying in ${delay}s" >&2
+    sleep "$delay"
+done

.github/scripts/run-fuzz-targets-test.sh

@@ -31,7 +31,7 @@ new_fixture() {
 #!/usr/bin/env bash
 set -euo pipefail
 printf '%s\t%s\t%s\n' "${GT_FUZZ_DUMP_DIR}" "$*" "${MOCK_CARGO_MARKER:-}" >>"${MOCK_CARGO_LOG}"
-target="$3"
+target="$4"
 case " ${MOCK_FAIL_TARGETS:-} " in
   *" ${target} "*) exit 17 ;;
 esac
@@ -117,6 +117,20 @@ EOF
   set -e
 }
 
+test_rustup_toolchain_env_is_honored() {
+  new_fixture
+  export RUSTUP_TOOLCHAIN=nightly-2026-03-21
+  run_fixture $'fuzz_create_table' true true "fuzz_create_table"
+  unset RUSTUP_TOOLCHAIN
+
+  assert_eq 1 "${fixture_status}" "pinned toolchain run status"
+  grep -q -- '+nightly-2026-03-21 fuzz run fuzz_create_table' "${fixture}/cargo.log" || \
+    fail "pinned toolchain missing from cargo invocation"
+  grep -q 'cargo +nightly-2026-03-21 fuzz run fuzz_create_table' "${fixture}/artifacts/summary.md" || \
+    fail "pinned toolchain missing from reproduce command"
+  cleanup_fixture
+}
+
 test_successful_targets_run_in_order() {
   new_fixture
   run_fixture $'fuzz_create_table\nfuzz_insert' false true ""
@@ -125,7 +139,7 @@ test_successful_targets_run_in_order() {
   assert_eq 2 "$(wc -l <"${fixture}/cargo.log" | tr -d ' ')" "cargo invocation count"
   assert_eq \
     $'fuzz_create_table\nfuzz_insert' \
-    "$(awk -F '\t' '{print $2}' "${fixture}/cargo.log" | sed -E 's/^fuzz run ([^ ]+).*/\1/')" \
+    "$(awk -F '\t' '{print $2}' "${fixture}/cargo.log" | sed -E 's/^\+nightly fuzz run ([^ ]+).*/\1/')" \
     "target order"
   grep -q -- '--features=unstable' "${fixture}/cargo.log" || fail "unstable feature missing"
   grep -q -- '-max_total_time=120' "${fixture}/cargo.log" || fail "fuzz time missing"
@@ -175,7 +189,7 @@ test_fail_fast_stops_after_first_failure() {
     fail "skipped target annotation missing"
   grep -q '### Reproduce failed targets' "${fixture}/artifacts/summary.md" || \
     fail "reproduction section missing"
-  grep -q 'cargo fuzz run fuzz_insert' "${fixture}/artifacts/summary.md" || \
+  grep -q 'cargo +nightly fuzz run fuzz_insert' "${fixture}/artifacts/summary.md" || \
     fail "reproduction command missing"
   cleanup_fixture
 }
@@ -367,5 +381,6 @@ test_collector_keeps_target_scopes_separate
 test_cluster_collector_honors_target_scope_and_namespace
 test_setup_failure_writes_artifact_contract
 test_setup_failure_keeps_manifest_when_collection_fails
+test_rustup_toolchain_env_is_honored
 
 printf 'All fuzz orchestration script tests passed.\n'

.github/scripts/run-fuzz-targets.sh

@@ -104,7 +104,7 @@ write_summary() {
             reproduce_args+=("GT_FUZZ_INSTANCE_ROOT_DIR=${GT_FUZZ_INSTANCE_ROOT_DIR}")
           fi
         fi
-        reproduce_args+=(cargo fuzz run "${target}" --fuzz-dir tests-fuzz -D -s none)
+        reproduce_args+=(cargo +"${RUSTUP_TOOLCHAIN:-nightly}" fuzz run "${target}" --fuzz-dir tests-fuzz -D -s none)
         if [[ "${fuzz_unstable}" == true ]]; then
           reproduce_args+=(--features=unstable)
         fi
@@ -195,7 +195,7 @@ for ((index = 0; index < ${#targets[@]}; index++)); do
     fi
     run_args=("${fuzz_binary}" "-max_total_time=${FUZZ_MAX_TOTAL_TIME}" "-artifact_prefix=${target_dir}/libfuzzer/")
   else
-    run_args=(cargo fuzz run "${target}" --fuzz-dir tests-fuzz -D -s none)
+    run_args=(cargo +"${RUSTUP_TOOLCHAIN:-nightly}" fuzz run "${target}" --fuzz-dir tests-fuzz -D -s none)
     if [[ "${fuzz_unstable}" == true ]]; then
       run_args+=(--features=unstable)
     fi

.github/scripts/runner_utils.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Small shared GitHub runner API and workflow-output helpers."""
+
+from __future__ import annotations
+
+import json
+import os
+import urllib.error
+import urllib.request
+
+
+def github_api(token: str, method: str, path: str, body: dict | None = None) -> dict:
+    data = json.dumps(body).encode("utf-8") if body is not None else None
+    request = urllib.request.Request(
+        f"https://api.github.com{path}",
+        data=data,
+        method=method,
+        headers={
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=30) as response:
+            if response.status == 204:
+                return {}
+            return json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as error:
+        # GitHub's error body says exactly why (e.g. "Must have admin rights to
+        # Repository" for a PAT without the required scope); surface it instead
+        # of a bare "HTTP Error 403".
+        body = error.read().decode("utf-8", "replace")
+        raise SystemExit(
+            f"GitHub API {method} {path} failed: HTTP {error.code}: {body}\n"
+            "The token comes from the GH_PERSONAL_ACCESS_TOKEN secret; it needs "
+            "'repo' scope (classic PAT) or 'Administration: write' on the "
+            "repository (fine-grained PAT)."
+        ) from error
+
+
+def create_registration_token(github_token: str, repo: str) -> str:
+    response = github_api(
+        github_token,
+        "POST",
+        f"/repos/{repo}/actions/runners/registration-token",
+        body={},
+    )
+    return response["token"]
+
+
+def find_runner_by_name(github_token: str, repo: str, name: str) -> dict | None:
+    page = 1
+    while True:
+        response = github_api(
+            github_token,
+            "GET",
+            f"/repos/{repo}/actions/runners?per_page=100&page={page}",
+        )
+        runners = response.get("runners", [])
+        for runner in runners:
+            if runner.get("name") == name:
+                return runner
+        if len(runners) < 100:
+            return None
+        page += 1
+
+
+def append_github_output(name: str, value: str) -> None:
+    output_path = os.environ.get("GITHUB_OUTPUT")
+    if output_path:
+        with open(output_path, "a", encoding="utf-8") as output:
+            output.write(f"{name}={value}\n")
+
+
+def deregister_runner(token: str, repo: str, runner_name: str) -> bool:
+    runner = find_runner_by_name(token, repo, runner_name)
+    if runner is None:
+        print(f"Runner {runner_name} is not registered", flush=True)
+        return True
+    try:
+        github_api(token, "DELETE", f"/repos/{repo}/actions/runners/{runner['id']}")
+        print(f"Deregistered runner {runner_name} (id {runner['id']})", flush=True)
+        return True
+    except Exception as error:  # noqa: BLE001
+        print(f"Failed to deregister runner {runner_name}: {error}", flush=True)
+        return False

.github/scripts/tests/test_ci_slash.py

@@ -14,8 +14,11 @@
 # limitations under the License.
 
 import importlib.util
+import json
 import os
 import tempfile
+import subprocess
+import textwrap
 import unittest
 from pathlib import Path
 from unittest.mock import patch
@@ -27,6 +30,9 @@ ci = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(ci)
 
 
+OPTIONS_TO_TEST = ["/ci", "/ci rust", "/ci fuzz chaos", "/ci fuzz all"]
+
+
 class CiSlashTest(unittest.TestCase):
     def setUp(self):
         self.output = tempfile.NamedTemporaryFile(delete=False)
@@ -54,23 +60,145 @@ class CiSlashTest(unittest.TestCase):
         self.assertIn("skip=true", output)
         self.assertIn("Available draft-PR CI commands", output)
 
-    def test_dispatches_full_suite_at_verified_head(self):
+    def test_dispatches_full_suite_with_top_level_admin_permission(self):
         output = self.run_main([
             {"body": "/ci", "issue_url": "https://api.github.test/repos/GreptimeTeam/greptimedb/issues/42", "user": {"login": "admin"}},
-            {"state": "open", "draft": True, "head": {"sha": "a" * 40, "repo": {"full_name": "GreptimeTeam/greptimedb"}}},
-            {"user": {"permission": "admin"}},
+            {"state": "open", "draft": True, "head": {"sha": "a" * 40, "ref": "fix/draft-ci", "repo": {"full_name": "GreptimeTeam/greptimedb"}}},
+            {"permission": "admin", "user": {"login": "admin"}},
         ])
         self.assertIn("skip=false", output)
+        self.assertIn("head_ref=fix/draft-ci", output)
         self.assertIn("workflow=rust.yml,integration.yml,checks.yml,docs.yml", output)
 
     def test_rejects_non_draft_before_permission_lookup(self):
         output = self.run_main([
             {"body": "/ci fuzz chaos", "issue_url": "https://api.github.test/repos/GreptimeTeam/greptimedb/issues/42", "user": {"login": "admin"}},
-            {"state": "open", "draft": False, "head": {"sha": "a" * 40, "repo": {"full_name": "GreptimeTeam/greptimedb"}}},
+            {"state": "open", "draft": False, "head": {"sha": "a" * 40, "ref": "fix/draft-ci", "repo": {"full_name": "GreptimeTeam/greptimedb"}}},
         ])
         self.assertIn("skip=true", output)
         self.assertIn("PR must be open and draft", output)
 
+    def test_rejects_missing_head_ref_before_permission_lookup(self):
+        output = self.run_main([
+            {"body": "/ci", "issue_url": "https://api.github.test/repos/GreptimeTeam/greptimedb/issues/42", "user": {"login": "admin"}},
+            {"state": "open", "draft": True, "head": {"sha": "a" * 40, "repo": {"full_name": "GreptimeTeam/greptimedb"}}},
+        ])
+        self.assertIn("skip=true", output)
+        self.assertIn("PR head changed; comment again.", output)
+
+    def test_permission_matrix(self):
+        for command in OPTIONS_TO_TEST:
+            for author in (True, False):
+                for permission in ("admin", "maintain", "write", "triage", "read", "none", None):
+                    with self.subTest(command=command, author=author, permission=permission):
+                        Path(self.output.name).write_text("")
+                        responses = [
+                            {"body": command, "issue_url": "https://api.github.test/repos/GreptimeTeam/greptimedb/issues/42", "user": {"login": "actor"}},
+                            {"state": "open", "draft": True, "user": {"login": "actor" if author else "other"}, "head": {"sha": "a" * 40, "ref": "fix/draft-ci", "repo": {"full_name": "GreptimeTeam/greptimedb"}}},
+                        ]
+                        if command == "/ci fuzz all" or not author:
+                            responses.append({"permission": permission})
+                        output = self.run_main(responses)
+                        allowed = permission == "admin" if command == "/ci fuzz all" else author or permission in ("admin", "maintain", "write")
+                        self.assertIn("skip=false" if allowed else "skip=true", output)
+
+    def test_reply_uses_issue_comment_endpoint(self):
+        workflow = SCRIPT.parents[1] / "workflows" / "ci-slash.yml"
+        step = workflow.read_text().split("      - name: Reply with command result\n", 1)[1]
+        command = textwrap.dedent(step.split("        run: |\n", 1)[1])
+        reply = "Denied: `example`\nSecond line with $variables and 'quotes'"
+        with tempfile.TemporaryDirectory() as directory:
+            gh = Path(directory) / "gh"
+            gh.write_text("#!/usr/bin/env python3\nimport json, sys\nprint(json.dumps(sys.argv[1:]))\n")
+            gh.chmod(0o755)
+            env = {**os.environ, "PATH": directory + os.pathsep + os.environ["PATH"], "GITHUB_REPOSITORY": "GreptimeTeam/greptimedb", "PR_NUMBER": "42", "REPLY": reply, "ADMIT_OUTCOME": "success", "DISPATCH_OUTCOME": "success"}
+            result = subprocess.run(["bash", "-eu", "-c", command], env=env, check=True, capture_output=True, text=True)
+        self.assertEqual(json.loads(result.stdout), ["api", "--method", "POST", "/repos/GreptimeTeam/greptimedb/issues/42/comments", "-f", "body=" + reply])
+
+    def test_failure_reply_and_partial_dispatch(self):
+        workflow = (SCRIPT.parents[1] / "workflows" / "ci-slash.yml").read_text()
+        dispatch = textwrap.dedent(workflow.split("      - name: Dispatch selected CI workflow\n", 1)[1].split("        run: |\n", 1)[1].split("      - name: Reply", 1)[0])
+        reply = textwrap.dedent(workflow.split("      - name: Reply with command result\n", 1)[1].split("        run: |\n", 1)[1])
+        with tempfile.TemporaryDirectory() as directory:
+            gh = Path(directory) / "gh"
+            log = Path(directory) / "calls"
+            gh.write_text("#!/usr/bin/env python3\nimport json, os, sys\nwith open(os.environ['CALLS'], 'a') as f: f.write(json.dumps(sys.argv[1:]) + '\\n')\nsys.exit(1 if '/runs/8/rerun' in sys.argv[-1] else 0)\n")
+            gh.chmod(0o755)
+            env = {**os.environ, "PATH": directory + os.pathsep + os.environ["PATH"], "CALLS": str(log), "GITHUB_REPOSITORY": "GreptimeTeam/greptimedb", "RUN_IDS": "7,8,9", "PR_NUMBER": "42", "REPLY": "Success", "ADMIT_OUTCOME": "success", "DISPATCH_OUTCOME": "failure", "GITHUB_SERVER_URL": "https://github.com", "GITHUB_RUN_ID": "123"}
+            result = subprocess.run(["bash", "-eu", "-c", dispatch], env=env, capture_output=True)
+            self.assertNotEqual(result.returncode, 0)
+            subprocess.run(["bash", "-eu", "-c", reply], env=env, check=True, capture_output=True)
+            calls = [json.loads(line) for line in log.read_text().splitlines()]
+        self.assertEqual([call[-1] for call in calls[:2]], ["/repos/GreptimeTeam/greptimedb/actions/runs/7/rerun", "/repos/GreptimeTeam/greptimedb/actions/runs/8/rerun"])
+        self.assertIn("CI trigger failed; some workflows may already have been requested", calls[2][-1])
+        self.assertIn("/actions/runs/123", calls[2][-1])
+        self.assertIn("!cancelled()", workflow.split("      - name: Reply with command result", 1)[1])
+
+    def test_fork_rerun_gates_cover_standard_jobs(self):
+        import re
+        for name, count in [("rust", 7), ("integration", 5), ("checks", 5), ("docs", 3)]:
+            workflow = (SCRIPT.parents[1] / "workflows" / (name + ".yml")).read_text()
+            gates = re.findall(r"^    if:.*github.run_attempt > 1.*$", workflow, re.MULTILINE)
+            self.assertEqual(len(gates), count, name)
+            self.assertTrue(all("always()" not in gate for gate in gates))
+
+    def test_command_permissions(self):
+        workflow = SCRIPT.parents[1] / "workflows" / "slash-command-dispatch.yml"
+        config = workflow.read_text().split("config: >-", 1)[1].split("      - name:", 1)[0]
+        self.assertEqual(json.loads(config), [
+            {"command": "query-regression", "permission": "admin", "issue_type": "pull-request"},
+            {"command": "ci", "permission": "none", "issue_type": "pull-request"},
+        ])
+
+    def test_fork_requires_writer_even_for_author(self):
+        for permission in ("read", "write", "maintain", "admin"):
+            with self.subTest(permission=permission):
+                Path(self.output.name).write_text("")
+                responses = [
+                    {"body": "/ci", "issue_url": "https://api.github.test/issues/42", "user": {"login": "actor"}},
+                    {"state": "open", "draft": True, "user": {"login": "actor"}, "head": {"sha": "a" * 40, "ref": "fork-branch", "repo": {"full_name": "actor/greptimedb"}}},
+                    {"permission": permission},
+                ]
+                if permission != "read":
+                    responses.append({"workflow_runs": [
+                        {"id": 7, "path": ".github/workflows/rust.yml", "head_sha": "a" * 40, "head_branch": "fork-branch", "head_repository": {"full_name": "actor/greptimedb"}, "status": "completed", "conclusion": "skipped"},
+                        {"id": 8, "path": ".github/workflows/checks.yml", "head_sha": "a" * 40, "head_branch": "fork-branch", "head_repository": {"full_name": "someone/greptimedb"}, "status": "completed", "conclusion": "skipped"},
+                    ]})
+                output = self.run_main(responses)
+                if permission == "read":
+                    self.assertIn("skip=true", output)
+                else:
+                    self.assertIn("run_ids=7\n", output)
+                    self.assertNotIn("head_ref=", output)
+
+    def test_fork_rejects_non_draft_original_run(self):
+        output = self.run_main([
+            {"body": "/ci rust", "issue_url": "https://api.github.test/issues/42", "user": {"login": "writer"}},
+            {"state": "open", "draft": True, "head": {"sha": "a" * 40, "ref": "fork-branch", "repo": {"full_name": "actor/greptimedb"}}},
+            {"permission": "write"},
+            {"workflow_runs": [{"id": 7, "path": ".github/workflows/rust.yml@main", "head_sha": "a" * 40, "head_branch": "fork-branch", "head_repository": {"full_name": "actor/greptimedb"}, "status": "completed", "conclusion": "success", "run_attempt": 2}]},
+            {"conclusion": "success"},
+        ])
+        self.assertIn("only originally skipped draft CI runs", output)
+        self.assertIn("skip=true", output)
+
+    def test_author_cannot_bypass_pr_guards(self):
+        for change, reason in [
+            ({"state": "closed"}, "PR must be open and draft"),
+            ({"draft": False}, "PR must be open and draft"),
+            ({"head": {"sha": "b" * 40, "ref": "fix/draft-ci", "repo": {"full_name": "GreptimeTeam/greptimedb"}}}, "PR head changed"),
+        ]:
+            with self.subTest(change=change):
+                Path(self.output.name).write_text("")
+                pr = {"state": "open", "draft": True, "user": {"login": "actor"}, "head": {"sha": "a" * 40, "ref": "fix/draft-ci", "repo": {"full_name": "GreptimeTeam/greptimedb"}}}
+                pr.update(change)
+                output = self.run_main([
+                    {"body": "/ci rust", "issue_url": "https://api.github.test/repos/GreptimeTeam/greptimedb/issues/42", "user": {"login": "actor"}},
+                    pr,
+                ])
+                self.assertIn("skip=true", output)
+                self.assertIn(reason, output)
+
 
 if __name__ == "__main__":
     unittest.main()

.github/scripts/tracesbench-summary.py

@@ -0,0 +1,169 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import html
+import math
+from pathlib import Path
+
+from benchmark_resource_summary import render_resources
+
+TARGETS = ('greptimedb', 'victoriatraces', 'tempo')
+
+
+def duration(ms):
+    if not isinstance(ms, (int, float)) or not math.isfinite(ms) or ms < 0:
+        return '—'
+    if ms < 1:
+        return f'{ms * 1000:.0f} µs'
+    if ms < 1000:
+        return f'{ms:.1f} ms'
+    if ms < 60_000:
+        return f'{ms / 1000:.2f} s'
+    minutes, seconds = divmod(round(ms / 1000), 60)
+    hours, minutes = divmod(minutes, 60)
+    return f'{hours}h {minutes}m {seconds}s' if hours else f'{minutes}m {seconds}s'
+
+
+def cell(value):
+    return html.escape(str(value if value is not None else '—')).replace('|', '&#124;').replace('\n', ' ')
+
+
+def read(path):
+    if not path.exists():
+        return {}
+    return json.loads(path.read_text())
+
+
+def table(headers, rows):
+    return '\n'.join(['| ' + ' | '.join(map(cell, headers)) + ' |',
+                      '| ' + ' | '.join('---' for _ in headers) + ' |'] +
+                     ['| ' + ' | '.join(map(cell, row)) + ' |' for row in rows])
+
+
+def delta(value, baseline):
+    if not all(isinstance(v, (int, float)) and math.isfinite(v) for v in (value, baseline)):
+        return '—'
+    return f'{(value / baseline - 1) * 100:+.1f}%' if baseline > 0 and value >= 0 else '—'
+
+
+
+def load(root):
+    report = read(root / 'report.json')
+    runs = [read(p) for p in sorted((root / 'runs').glob('*/run.json'))]
+    return report, runs
+
+
+def benchmark(report, runs, targets=TARGETS):
+    lines = ['# Tracesbench benchmark summary', '',
+             'Median Δ = (target / GreptimeDB − 1) × 100%; negative is faster. '
+             'Min/median/max come from the validated report, not recalculated samples. '
+             'Counts exclude warmup. Missing or excluded results have no latency comparison.',
+             '', f'Classification: {cell(report.get("classification"))}. '
+             'See the uploaded HTML for query semantics and client-side compensation.', '']
+    statistics = {}
+    for corpus in report.get('per_corpus', []):
+        for stat in corpus.get('statistics', []):
+            if stat['target'] not in targets:
+                continue
+            statistics[(corpus['corpus'], stat['round'], stat['target'], stat['query_id'])] = stat
+    rows, details = [], []
+    for target in targets:
+        selected = [run for run in runs if run.get('target') == target]
+        if not selected:
+            lines.append(f'- **{target}**: unavailable / not run.')
+        for run in selected:
+            lines.append(f'- **{target}**: lifecycle '
+                         f'{"completed" if run.get("success") is True else "failed / incomplete"}; '
+                         f'query failures: {len(run.get("query_failures", []))}.')
+            for qid in (f'T{i:02}' for i in range(1, 9)):
+                correct = [q for q in run.get('correctness', []) if q.get('query_id') == qid]
+                measured = [q for q in run.get('samples', [])
+                            if q.get('query_id') == qid and q.get('phase') == 'measured']
+                errors = [q for q in correct + run.get('samples', [])
+                          if q.get('query_id') == qid and q.get('outcome') not in ('passed', 'not_executed')]
+                reason = (run.get('exclusions', {}).get(qid) or
+                          run.get('execution', {}).get('query_skips', {}).get(qid))
+                stat = statistics.get((run.get('corpus'), run.get('round'), target, qid), {})
+                base = statistics.get((run.get('corpus'), run.get('round'), 'greptimedb', qid), {})
+                status = ('failed' if errors else 'excluded' if reason else
+                          'passed' if stat else
+                          'correctness passed; report unavailable' if correct and
+                          all(q.get('outcome') == 'passed' for q in correct) else 'unavailable / incomplete')
+                rows.append([run.get('corpus'), run.get('round'), qid, target, status,
+                             duration(stat.get('min_ms')), duration(stat.get('median_ms')),
+                             duration(stat.get('max_ms')),
+                             delta(stat.get('median_ms'), base.get('median_ms')) if target != 'greptimedb' else '—',
+                             sum(q.get('outcome') == 'passed' for q in measured),
+                             sum(q.get('outcome') != 'passed' for q in measured)])
+                if reason or errors:
+                    messages = [str(q.get('error') or q.get('outcome')) for q in errors]
+                    details.append(f'- **{target} / {qid}**: {cell(str(reason or "") + " " + "; ".join(messages))[:4000]}')
+    lines += ['', table(['Dataset', 'Round', 'Query', 'DB', 'Status', 'Min', 'Median', 'Max',
+                         'Median Δ', 'Measured success', 'Measured errors'], rows), '',
+              '<details>', '<summary>Query failures and exclusions</summary>', '', *details, '', '</details>']
+    if not report:
+        lines += ['', 'Validated report unavailable; raw outcomes are shown without latency statistics.']
+    return '\n'.join(lines)
+
+
+def lifecycle(report, runs, root, targets=TARGETS):
+    rows = []
+    for target in targets:
+        selected = [r for r in runs if r.get('target') == target] or [{}]
+        for run in selected:
+            load = run.get('load', {})
+            ingest = load.get('ingest', {})
+            if target == 'tempo' and 'ingest' not in load:
+                ingest = load.get('transport', {}).get('ingest', {})
+            health = run.get('post_load_health', {})
+            execution = run.get('execution', {})
+            start, finish = run.get('started_unix_ns'), run.get('finished_unix_ns')
+            elapsed = (finish - start) / 1_000_000 if isinstance(start, int) and isinstance(finish, int) else None
+            seconds = ingest.get('wall_seconds')
+            rows.append([target, run.get('corpus'), run.get('load_summary', {}).get('after'),
+                         duration(elapsed), duration(seconds * 1000 if isinstance(seconds, (int, float)) else None),
+                         ingest.get('successful_requests'), ingest.get('failed_requests'),
+                         execution.get('load_workers_requested'), execution.get('max_in_flight'),
+                         execution.get('db_cpus'), execution.get('db_memory'),
+                         health.get('running'), health.get('oom_killed'),
+                         run.get('cleanup', {}).get('verified_absent')])
+    return '\n'.join(['# Tracesbench load / lifecycle summary', '',
+                       'Total is the recorded target lifecycle; ingest is the sender wall time, '
+                       'not the entire load/readiness phase. Health is the post-load snapshot. '
+                       'Missing evidence remains blank.', '',
+                       table(['DB', 'Dataset', 'Loaded spans', 'Total', 'Ingest',
+                              'Write success', 'Write errors', 'Load workers', 'Max in flight',
+                              'DB CPU', 'DB memory', 'Running after load', 'OOM after load',
+                              'Container removed'], rows), '',
+                       render_resources([('dataset', root / 'runs' / 'generate')] +
+                                        [(t, root / 'runs' / t) for t in targets], table)])
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--root', type=Path, required=True)
+    parser.add_argument('--section', choices=('benchmark', 'lifecycle'), required=True)
+    parser.add_argument('--targets', default=','.join(TARGETS), help='Comma-separated selected engines')
+    args = parser.parse_args()
+    targets = args.targets.split(',')
+    if not targets or any(t not in TARGETS for t in targets) or len(set(targets)) != len(targets):
+        parser.error('targets must be a non-empty comma-separated subset of greptimedb,victoriatraces,tempo')
+    report, runs = load(args.root)
+    print(benchmark(report, runs, targets) if args.section == 'benchmark' else lifecycle(report, runs, args.root, targets))
+
+
+if __name__ == '__main__':
+    main()

.github/workflows/agent-observability.yml

@@ -26,8 +26,18 @@ on:
           - M
         default: S
         required: true
+      compare_greptimedb:
+        description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+        type: boolean
+        default: false
+        required: false
+      baseline_greptimedb_tag:
+        description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+        type: string
+        default: v1.2.1
+        required: false
       greptimedb_tag:
-        description: Docker Hub greptime/greptimedb tag
+        description: Candidate Docker Hub greptime/greptimedb tag
         type: string
         default: latest
         required: true
@@ -46,19 +56,25 @@ on:
         type: string
         required: false
       runtime_image:
-        description: Aliyun runtime image (tag or digest reference)
+        description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
         type: string
-        default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260917-8111034e@sha256:ce9dee2abb1e70f907a09ea01854904a91cc97748b0c4ef6f33d9ec34044bec0
+        default: auto
         required: true
-      ecs_instance_type:
-        description: ECS instance type (independent of Query Regression)
+      provider:
+        description: Cloud provider for the benchmark runner
+        type: choice
+        options: [Aliyun, AWS]
+        default: Aliyun
+        required: true
+      instance_type:
+        description: 'Instance type; auto = Aliyun: ecs.c9i.2xlarge, AWS: c7i.2xlarge (both 8 vCPU / 16 GiB). Enter a specific type to override.'
         type: string
-        default: ecs.c9i.2xlarge
+        default: auto
         required: true
       system_disk_gib:
-        description: System disk GiB; 0 = auto (S 80, P 100, M 500), or 20..2048
-        type: number
-        default: 0
+        description: 'System disk GiB: auto (or 0) = S 80, P 100, M 500; enter 20..2048 to override'
+        type: string
+        default: auto
         required: true
       benchmark_timeout_minutes:
         description: Benchmark job timeout in minutes (1..360)
@@ -83,13 +99,16 @@ on:
 permissions:
   contents: read
 env:
+  COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+  BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
   PROFILE: ${{ inputs.dataset }}
   GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
   CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }}
   VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }}
   O11YBENCH_REF: ${{ inputs.o11ybench_ref }}
-  RUNTIME_IMAGE: ${{ inputs.runtime_image }}
-  ECS_INSTANCE_TYPE: ${{ inputs.ecs_instance_type }}
+  RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' }}
+  PROVIDER: ${{ inputs.provider }}
+  REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
   SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
   BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }}
   JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
@@ -100,6 +119,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     outputs:
+      instance_type: ${{ steps.runner.outputs.instance_type }}
       targets: ${{ steps.inputs.outputs.targets }}
       system_disk_gib: ${{ steps.inputs.outputs.system_disk_gib }}
       greptimedb: ${{ steps.inputs.outputs.greptimedb }}
@@ -110,10 +130,30 @@ jobs:
         with:
           path: greptimedb
           persist-credentials: false
-      - name: Test ECS lifecycle helpers
+      - name: Validate version comparison
+        env:
+          RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+        run: |
+          python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+          python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
+      - name: Resolve CI runner
+        id: runner
+        run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
+      - name: Test CI runner helpers
         run: |
           python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
+          python3 greptimedb/tests/perf/test_ci_runner_scripts.py
           python3 greptimedb/tests/perf/test_agent_observability_summary.py
+      - name: Check runtime supports shared version-comparison data
+        if: inputs.compare_greptimedb
+        run: |
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
+          docker run --rm --entrypoint grep "$RUNTIME_IMAGE" -q -- '--data-root)' /opt/o11ybench/scripts/run-agent-observability.sh \
+            || { echo 'Comparison requires a runtime with --data-root support'; exit 1; }
+      - name: Validate dataset admission in runtime
+        run: |
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
+          docker run --rm "$RUNTIME_IMAGE" scripts/logbench_agent_observability_measured.py --preflight-profile "$PROFILE"
       - name: Validate requested benchmark
         id: inputs
         shell: bash
@@ -124,14 +164,15 @@ jobs:
         run: |
           set -euo pipefail
           # Validate cloud lifecycle budgets before provisioning; runtime owns workload validation.
-          [[ -n "$ECS_INSTANCE_TYPE" ]]
           case "$PROFILE" in
             S) auto_disk_gib=80 ;;
             P) auto_disk_gib=100 ;;
             M) auto_disk_gib=500 ;;
             *) echo "Unknown dataset: $PROFILE" >&2; exit 1 ;;
           esac
-          [[ "$SYSTEM_DISK_GIB" != 0 ]] || SYSTEM_DISK_GIB=$auto_disk_gib
+          if [[ "$SYSTEM_DISK_GIB" == auto || "$SYSTEM_DISK_GIB" == 0 ]]; then
+            SYSTEM_DISK_GIB=$auto_disk_gib
+          fi
           for value in "$SYSTEM_DISK_GIB" "$BENCHMARK_TIMEOUT_MINUTES" "$JANITOR_TTL_HOURS"; do
             [[ "$value" =~ ^[1-9][0-9]{0,3}$ ]] || { echo 'Resource budgets must be positive integers' >&2; exit 1; }
           done
@@ -160,15 +201,16 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 45
     outputs:
-      label: ${{ steps.ecs.outputs.label }}
-      instance_id: ${{ steps.ecs.outputs.instance_id }}
-      runner_name: ${{ steps.ecs.outputs.runner_name }}
+      label: ${{ steps.ecs.outputs.label || steps.aws.outputs.label }}
+      instance_id: ${{ steps.ecs.outputs.instance_id || steps.aws.outputs.instance_id }}
+      runner_name: ${{ steps.ecs.outputs.runner_name || steps.aws.outputs.runner_name }}
     steps:
       - uses: actions/checkout@v4
         with:
           path: greptimedb
           persist-credentials: false
       - name: Create ephemeral ECS runner
+        if: inputs.provider == 'Aliyun'
         id: ecs
         uses: ./greptimedb/.github/actions/aliyun-ecs-create
         with:
@@ -179,13 +221,35 @@ jobs:
           vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
           security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
           image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
-          instance-type: ${{ env.ECS_INSTANCE_TYPE }}
+          instance-type: ${{ needs.validate.outputs.instance_type }}
           enable-docker: 'true'
           system-disk-gib: ${{ needs.validate.outputs.system_disk_gib }}
           ttl-hours: ${{ inputs.janitor_ttl_hours }}
           resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
           runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
           runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
+      - uses: aws-actions/configure-aws-credentials@v4
+        if: inputs.provider == 'AWS'
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+      - uses: astral-sh/setup-uv@v6
+        if: inputs.provider == 'AWS'
+      - name: Create ephemeral EC2 runner
+        if: inputs.provider == 'AWS'
+        id: aws
+        env:
+          GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
+          AWS_EC2_IMAGE_ID: ${{ vars.BENCHMARK_EC2_IMAGE_ID || 'auto' }}
+          AWS_EC2_INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }}
+          AWS_EC2_SUBNET_ID: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+          AWS_EC2_SECURITY_GROUP_ID: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          AWS_EC2_SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }}
+          AWS_EC2_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
+          AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+        run: uv run greptimedb/.github/scripts/aws-ec2-runner-provision.py
   benchmark:
     needs:
       - validate
@@ -193,6 +257,7 @@ jobs:
     runs-on: ${{ needs.provision.outputs.label }}
     timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }}
     env:
+      INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }}
       TARGETS: ${{ needs.validate.outputs.targets }}
       SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }}
       OWNER: o11ybench-${{ github.run_id }}-${{ github.run_attempt }}
@@ -209,6 +274,8 @@ jobs:
         with:
           path: greptimedb
           persist-credentials: false
+      - name: Set up benchmark dependencies
+        uses: ./greptimedb/.github/actions/setup-benchmark
       - name: Check Docker on ephemeral Linux host
         shell: bash
         run: |
@@ -223,11 +290,11 @@ jobs:
         run: |
           set -euo pipefail
           actual=
-          [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]]
+          [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]]
           images='{}'
           resolve_image() {
             local target=$1 reference=$2 variable=$3 metadata resolved revision
-            docker pull "$reference"
+            bash greptimedb/.github/scripts/pull-benchmark-image.sh "$reference"
             metadata=$(docker image inspect "$reference")
             resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata")
             if [[ "$target" == runtime ]]; then
@@ -250,11 +317,14 @@ jobs:
               victorialogs) resolve_image "$target" "victoriametrics/victoria-logs:$VICTORIALOGS_TAG" VICTORIALOGS_IMAGE ;;
             esac
           done
+          if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+            resolve_image greptimedb-baseline "greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG" BASELINE_GREPTIMEDB_IMAGE
+          fi
           jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \
             --arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \
-            --arg instance_type "$ECS_INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \
+            --arg provider "$PROVIDER" --arg instance_type "$INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \
             --argjson timeout_minutes "$BENCHMARK_TIMEOUT_MINUTES" --argjson ttl_hours "$JANITOR_TTL_HOURS" \
-            '{o11ybench_sha:$sha, targets:$targets, profile:$profile, db_cpus:$cpus, db_memory:$memory, images:$images, ecs:{instance_type:$instance_type, system_disk_gib:$disk_gib, benchmark_timeout_minutes:$timeout_minutes, janitor_ttl_hours:$ttl_hours}}' > "$MANIFEST_PATH"
+            '{o11ybench_sha:$sha, targets:$targets, profile:$profile, db_cpus:$cpus, db_memory:$memory, images:$images, ($provider | if . == "AWS" then "ec2" else "ecs" end):{instance_type:$instance_type, system_disk_gib:$disk_gib, benchmark_timeout_minutes:$timeout_minutes, janitor_ttl_hours:$ttl_hours}}' > "$MANIFEST_PATH"
       - name: Checkout benchmark tools
         uses: actions/checkout@v4
         with:
@@ -273,6 +343,15 @@ jobs:
             --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
             --root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
           cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json"
+      - name: Load and benchmark baseline GreptimeDB
+        id: baseline_greptimedb
+        if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
+        run: |-
+          GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
+            --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline" --data-root "$ARTIFACT_ROOT" \
+            --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+            --target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \
+            --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
       - name: Load and benchmark greptimedb
         id: greptimedb
         if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
@@ -318,6 +397,13 @@ jobs:
             GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
               --owner "$OWNER" --root "$ARTIFACT_ROOT"
           fi
+      - name: Clean up baseline containers
+        if: always() && inputs.compare_greptimedb
+        run: |-
+          if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
+            GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
+              --owner "$OWNER-base" --root "$ARTIFACT_ROOT/baseline"
+          fi
       - name: Benchmark summary
         if: always()
         run: |
@@ -328,6 +414,16 @@ jobs:
         run: |
           python3 greptimedb/.github/scripts/agent-observability-summary.py \
             --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+      - name: Baseline load / lifecycle summary
+        if: always() && inputs.compare_greptimedb
+        run: |
+          echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
+          python3 greptimedb/.github/scripts/agent-observability-summary.py \
+            --root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+      - name: GreptimeDB version comparison
+        if: always() && inputs.compare_greptimedb
+        run: |
+          python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" >> "$GITHUB_STEP_SUMMARY"
       - name: Upload results (no dataset)
         if: always()
         uses: actions/upload-artifact@v4
@@ -343,7 +439,7 @@ jobs:
     needs:
       - provision
       - benchmark
-    if: ${{ always() && needs.provision.outputs.instance_id != '' }}
+    if: ${{ always() && (needs.provision.outputs.instance_id != '' || (inputs.provider == 'AWS' && needs.provision.result != 'skipped')) }}
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -352,6 +448,7 @@ jobs:
           path: greptimedb
           persist-credentials: false
       - name: Delete ECS and unregister runner
+        if: inputs.provider == 'Aliyun'
         uses: ./greptimedb/.github/actions/aliyun-ecs-delete
         with:
           access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
@@ -360,3 +457,20 @@ jobs:
           region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
           instance-id: ${{ needs.provision.outputs.instance_id }}
           runner-name: ${{ needs.provision.outputs.runner_name }}
+      - uses: aws-actions/configure-aws-credentials@v4
+        if: inputs.provider == 'AWS'
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+      - uses: astral-sh/setup-uv@v6
+        if: inputs.provider == 'AWS'
+      - name: Delete EC2 and unregister runner
+        if: inputs.provider == 'AWS'
+        env:
+          GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
+          AWS_EC2_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }}
+          AWS_EC2_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }}
+          AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+        run: uv run greptimedb/.github/scripts/aws-ec2-runner-teardown.py

.github/workflows/aws-ec2-runner-janitor.yml

@@ -0,0 +1,29 @@
+name: AWS EC2 Runner Janitor
+
+on:
+  schedule:
+    - cron: '23 3 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  sweep:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+      - uses: astral-sh/setup-uv@v6
+      - name: Sweep expired CI instances and unregister runners
+        env:
+          AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
+          GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+        run: uv run .github/scripts/aws-ec2-runner-teardown.py --sweep

.github/workflows/cargo-lock-check.yml

@@ -5,6 +5,7 @@ name: Cargo.lock Diff Check
 # posts/updates a PR comment when the threshold is exceeded.
 on:
     pull_request:
+        types: [opened, synchronize, reopened, ready_for_review]
         paths:
             - "Cargo.lock"
 

.github/workflows/check-git-deps.yml

@@ -2,6 +2,7 @@ name: Check Git Dependencies on Main Branch
 
 on:
     pull_request:
+        types: [opened, synchronize, reopened, ready_for_review]
         branches: [main]
         paths:
             - 'Cargo.toml'

.github/workflows/checks.yml

@@ -32,7 +32,7 @@ concurrency:
 
 jobs:
   check-typos-and-docs:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Check typos and docs
     runs-on: ubuntu-latest
     steps:
@@ -47,7 +47,7 @@ jobs:
           || (echo "'config/config.md' is not up-to-date, please run 'make config-docs'." && exit 1)
 
   license-header-check:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     runs-on: ubuntu-latest
     name: Check License Header
     steps:
@@ -63,7 +63,7 @@ jobs:
           config: licenserc-enterprise.toml
 
   github-script-tests:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: GitHub Script Tests
     runs-on: ubuntu-latest
     timeout-minutes: 5
@@ -97,10 +97,12 @@ jobs:
           python3 tests/perf/test_query_regression_nightly_refs.py
           python3 tests/perf/test_query_regression_slash.py
           python3 tests/perf/test_aliyun_ecs_runner_scripts.py
+          python3 tests/perf/test_ci_runner_scripts.py
           python3 tests/perf/test_agent_observability_summary.py
+          python3 tests/perf/test_tracesbench_summary.py
 
   check:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Check
     runs-on: ${{ matrix.os }}
     strategy:
@@ -129,7 +131,7 @@ jobs:
         run: cargo check --locked --workspace --all-targets --all-features
 
   toml:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Toml Check
     runs-on: ubuntu-latest
     timeout-minutes: 60

.github/workflows/ci-slash.yml

@@ -8,7 +8,7 @@ permissions:
   actions: write
   contents: read
   issues: write
-  pull-requests: read
+  pull-requests: write
 
 jobs:
   admit:
@@ -30,23 +30,46 @@ jobs:
           DISPATCH_SENDER: ${{ github.event.sender.login }}
           DISPATCH_HEAD_SHA: ${{ github.event.client_payload.pull_request.head.sha }}
         run: python3 .github/scripts/ci-slash.py
-      - name: Reply with command result
-        if: ${{ steps.admit.outputs.reply != '' }}
-        env:
-          GH_TOKEN: ${{ github.token }}
-          REPLY: ${{ steps.admit.outputs.reply }}
-          PR_NUMBER: ${{ steps.admit.outputs.pr_number }}
-        run: printf '%s\n' "${REPLY}" | gh pr comment "${PR_NUMBER}" --body-file -
       - name: Dispatch selected CI workflow
+        id: dispatch
         if: ${{ steps.admit.outputs.skip == 'false' }}
         env:
           GH_TOKEN: ${{ github.token }}
+          RUN_IDS: ${{ steps.admit.outputs.run_ids }}
           WORKFLOWS: ${{ steps.admit.outputs.workflow }}
-          HEAD_SHA: ${{ steps.admit.outputs.head_sha }}
+          HEAD_REF: ${{ steps.admit.outputs.head_ref }}
           FUZZ_PROFILE: ${{ steps.admit.outputs.fuzz_profile }}
         run: |
+          if [[ -n "${RUN_IDS}" ]]; then
+            IFS=, read -ra runs <<<"${RUN_IDS}"
+            for run in "${runs[@]}"; do
+              gh api --method POST "/repos/${GITHUB_REPOSITORY}/actions/runs/${run}/rerun"
+            done
+            exit 0
+          fi
           IFS=, read -ra workflows <<<"${WORKFLOWS}"
           for workflow in "${workflows[@]}"; do
+            inputs=(-F 'inputs[ci_command]=true')
+            if [[ "${workflow}" == 'integration.yml' ]]; then
+              inputs+=(-f "inputs[fuzz_profile]=${FUZZ_PROFILE}")
+            fi
             gh api --method POST "/repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow}/dispatches" \
-              -f ref="${HEAD_SHA}" -F 'inputs[ci_command]=true' -f "inputs[fuzz_profile]=${FUZZ_PROFILE}"
+              -f ref="${HEAD_REF}" "${inputs[@]}"
           done
+      - name: Reply with command result
+        if: ${{ !cancelled() && github.event.sender.login == 'github-actions[bot]' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPLY: ${{ steps.admit.outputs.reply }}
+          PR_NUMBER: ${{ steps.admit.outputs.pr_number || github.event.client_payload.github.payload.issue.number }}
+          ADMIT_OUTCOME: ${{ steps.admit.outcome }}
+          DISPATCH_OUTCOME: ${{ steps.dispatch.outcome }}
+        # PR replies require pull-requests: write even through the issue-comment endpoint.
+        run: |
+          [[ "${PR_NUMBER}" =~ ^[0-9]+$ ]] || exit 1
+          if [[ "${ADMIT_OUTCOME}" == failure || "${DISPATCH_OUTCOME}" == failure ]]; then
+            REPLY="CI trigger failed; some workflows may already have been requested. See ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} for details."
+          fi
+          [[ -n "${REPLY}" ]] || exit 1
+          gh api --method POST "/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" \
+            -f "body=${REPLY}"

.github/workflows/dependency-check.yml

@@ -2,6 +2,7 @@ name: Check Dependencies
 
 on:
   pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
     branches:
       - main
 

.github/workflows/dev-build.yml

@@ -183,6 +183,10 @@ jobs:
           image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
           image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}
           large-page-size: ${{ inputs.large-page-size }}
+          aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
+          aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
 
   build-linux-arm64-artifacts:
     name: Build linux-arm64 artifacts
@@ -217,6 +221,10 @@ jobs:
           image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
           image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}
           large-page-size: ${{ inputs.large-page-size }}
+          aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
+          aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
 
   release-images-to-dockerhub:
     name: Build and push images to DockerHub

.github/workflows/docbot.yml

@@ -1,10 +1,10 @@
 name: Follow Up Docs
 on:
   pull_request_target:
-    types: [opened, edited]
+    types: [opened, edited, closed]
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.event.action }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
@@ -13,6 +13,7 @@ jobs:
     permissions:
       pull-requests: write
       contents: read
+      actions: read
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4

.github/workflows/docs.yml

@@ -38,7 +38,7 @@ name: Docs CI
 
 jobs:
   typos:
-    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
+    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1 }}
     name: Spell Check with Typos
     runs-on: ubuntu-latest
     steps:
@@ -48,7 +48,7 @@ jobs:
       - uses: crate-ci/typos@v1.50.2
 
   license-header-check:
-    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
+    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1 }}
     runs-on: ubuntu-latest
     name: Check License Header
     steps:
@@ -58,7 +58,7 @@ jobs:
       - uses: korandoru/hawkeye@v5
 
   required-checks:
-    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
+    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1 }}
     name: ${{ matrix.check }}
     runs-on: ubuntu-slim
     strategy:

.github/workflows/grafana.yml

@@ -2,6 +2,7 @@ name: Check Grafana Panels
 
 on:
   pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
     branches:
       - main
     paths:

.github/workflows/integration.yml

@@ -31,6 +31,11 @@ name: Integration CI
 
 env:
   CARGO_FUZZ_VERSION: "0.13.2"
+  # Fuzz targets require a nightly toolchain (cargo-fuzz needs
+  # `-Zsanitizer=fuzzer`). This pin decouples the fuzz workflow from the
+  # workspace default in rust-toolchain.toml, so it keeps working when the
+  # workspace moves to a stable toolchain.
+  FUZZ_RUST_TOOLCHAIN: "nightly-2026-03-21"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -38,7 +43,7 @@ concurrency:
 
 jobs:
   build:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Build GreptimeDB binaries
     runs-on: ${{ matrix.os }}
     strategy:
@@ -81,7 +86,7 @@ jobs:
         # `cargo gc` will invoke `cargo build` with specified args
         run: |
           started_epoch="$(date +%s)"
-          cargo gc -- --bin greptime --bin sqlness-runner --features "pg_kvbackend,mysql_kvbackend,vector_index,ai_functions"
+          cargo gc -- --bin greptime --bin sqlness-runner --features "pg_kvbackend,mysql_kvbackend,ai_functions"
           elapsed_secs=$(( $(date +%s) - started_epoch ))
           printf -- '- Binary build: **%ss**\n' "${elapsed_secs}" >>"${GITHUB_STEP_SUMMARY}"
       - name: Pack greptime binaries
@@ -99,7 +104,7 @@ jobs:
           version: current
 
   sqlness:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Sqlness Test (${{ matrix.mode.name }})
     needs: build
     runs-on: ${{ matrix.os }}
@@ -179,7 +184,7 @@ jobs:
           retention-days: 3
 
   export-import-v2-e2e:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Export/Import V2 E2E Test
     needs: build
     runs-on: ubuntu-latest
@@ -249,7 +254,7 @@ jobs:
           retention-days: 3
 
   run-multi-lang-tests:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Run Multi-language SDK Tests
     needs: build
     uses: ./.github/workflows/run-multi-lang-tests.yml
@@ -257,7 +262,7 @@ jobs:
       artifact-name: bins
 
   compat-updater-check:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) && (github.event_name == 'merge_group' || github.event_name == 'pull_request') }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) && (github.event_name == 'merge_group' || github.event_name == 'pull_request') }}
     name: Check compat version updater
     runs-on: ubuntu-latest
     timeout-minutes: 10
@@ -328,6 +333,7 @@ jobs:
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           cache: false
+          toolchain: ${{ env.FUZZ_RUST_TOOLCHAIN }}
       - name: Fuzz Dependency Cache
         id: fuzz-dependencies-cache
         uses: Swatinem/rust-cache@v2
@@ -343,7 +349,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ${{ runner.temp }}/greptime-fuzz-binaries
-          key: v1-fuzz-binaries-${{ env.CARGO_FUZZ_VERSION }}-all-${{ hashFiles('Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo/**', 'tests-fuzz/**', 'src/**') }}
+          key: v1-fuzz-binaries-${{ env.CARGO_FUZZ_VERSION }}-${{ env.FUZZ_RUST_TOOLCHAIN }}-all-${{ hashFiles('Cargo.toml', 'Cargo.lock', '.cargo/**', 'tests-fuzz/**', 'src/**') }}
       - name: Report fuzz cache lookup
         shell: bash
         run: |
@@ -355,6 +361,8 @@ jobs:
       - name: Set Rust Fuzz
         if: steps.fuzz-binaries.outputs.cache-hit != 'true'
         shell: bash
+        env:
+          RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
         run: |
           sudo apt-get install -y libfuzzer-14-dev
           cargo install cargo-fuzz --version "${CARGO_FUZZ_VERSION}" --locked
@@ -363,6 +371,7 @@ jobs:
         shell: bash
         env:
           CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
+          RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
           FUZZ_TARGETS: |
             fuzz_create_database
             fuzz_create_table
@@ -410,7 +419,7 @@ jobs:
         uses: actions/cache/save@v4
         with:
           path: ${{ runner.temp }}/greptime-fuzz-binaries
-          key: v1-fuzz-binaries-${{ env.CARGO_FUZZ_VERSION }}-all-${{ hashFiles('Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo/**', 'tests-fuzz/**', 'src/**') }}
+          key: v1-fuzz-binaries-${{ env.CARGO_FUZZ_VERSION }}-${{ env.FUZZ_RUST_TOOLCHAIN }}-all-${{ hashFiles('Cargo.toml', 'Cargo.lock', '.cargo/**', 'tests-fuzz/**', 'src/**') }}
       - name: Pack prebuilt fuzz binaries
         shell: bash
         run: |
@@ -479,6 +488,7 @@ jobs:
       - name: Fuzz Test
         uses: ./.github/actions/fuzz-test
         env:
+          RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
           GT_MYSQL_ADDR: 127.0.0.1:4002
           FUZZ_SERVICE_LOG: /tmp/greptime-fuzz-runtime/greptime.log
         with:
@@ -537,6 +547,7 @@ jobs:
       - name: Run Fuzz Test
         uses: ./.github/actions/fuzz-test
         env:
+          RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
           GT_MYSQL_ADDR: 127.0.0.1:4002
           GT_FUZZ_BINARY_PATH: ./bin/greptime
           GT_FUZZ_INSTANCE_ROOT_DIR: /tmp/greptime-fuzz-artifacts/targets/unstable_fuzz_create_table_standalone/service/instance/
@@ -820,6 +831,7 @@ jobs:
       - name: Fuzz Test
         uses: ./.github/actions/fuzz-test
         env:
+          RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
           GT_MYSQL_ADDR: 127.0.0.1:4002
           GT_KAFKA_WAL_HELPER_URL: http://127.0.0.1:8080
           GT_KAFKA_ADDR: kafka.kafka-cluster.svc.cluster.local:9092
@@ -994,6 +1006,7 @@ jobs:
       - name: Fuzz Test
         uses: ./.github/actions/fuzz-test
         env:
+          RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
           GT_MYSQL_ADDR: 127.0.0.1:4002
         with:
           targets: ${{ matrix.group.targets }}

.github/workflows/nightly-build.yml

@@ -151,6 +151,10 @@ jobs:
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
           image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
           image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}
+          aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
+          aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
 
   build-linux-arm64-artifacts:
     name: Build linux-arm64 artifacts
@@ -173,6 +177,10 @@ jobs:
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
           image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
           image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}
+          aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
+          aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
 
   release-images-to-dockerhub:
     name: Build and push images to DockerHub

.github/workflows/nightly-ci.yml

@@ -71,6 +71,57 @@ jobs:
           path: C:\Users\RUNNER~1\AppData\Local\Temp\sqlness*
           retention-days: 3
 
+  # PR CI runs the object-store matrix against a local MinIO. This job is where
+  # the same matrix meets real AWS S3.
+  test-on-linux-real-s3:
+    name: Run tests on Linux against real S3
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+    runs-on: ubuntu-22.04-arm
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: rui314/setup-mold@v1
+      - name: Install toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache: false
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: "coverage-test"
+          cache-all-crates: "true"
+          save-if: false
+      - name: Install latest nextest release
+        uses: taiki-e/install-action@nextest
+      - name: Setup external services
+        working-directory: tests-integration/fixtures
+        run: ../../.github/scripts/pull-test-deps-images.sh && docker compose up -d --wait
+      - name: Run nextest cases
+        run: cargo nextest run --workspace -F dashboard -F pg_kvbackend -F mysql_kvbackend -F ai_functions -F hdfs-object-store
+        env:
+          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
+          RUST_BACKTRACE: 1
+          RUST_MIN_STACK: 8388608 # 8MB
+          CARGO_INCREMENTAL: 0
+          GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
+          GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
+          GT_ETCD_TLS_ENDPOINTS: https://127.0.0.1:2378
+          GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
+          GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
+          GT_POSTGRES15_ENDPOINTS: postgres://test_user:test_password@127.0.0.1:5433/postgres
+          GT_POSTGRES15_SCHEMA: test_schema
+          GT_MYSQL_ENDPOINTS: mysql://greptimedb:admin@127.0.0.1:3306/mysql
+          GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
+          GT_KAFKA_SASL_ENDPOINTS: 127.0.0.1:9093
+          UNITTEST_LOG_DIR: "__unittest_logs"
+
   test-on-windows:
     name: Run tests on Windows
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
@@ -130,7 +181,7 @@ jobs:
 
   check-status:
     name: Check status
-    needs: [sqlness-test, sqlness-windows, test-on-windows]
+    needs: [sqlness-test, sqlness-windows, test-on-windows, test-on-linux-real-s3]
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     runs-on: ubuntu-latest
     outputs:

.github/workflows/query-regression.yml

@@ -190,6 +190,12 @@ jobs:
       CARGO_HOME: /home/runner/.cargo
       UV_CACHE_DIR: /home/runner/.cargo/uv-cache
       RUSTUP_HOME: /opt/rustup
+      # Deliberately NOT derived from rust-toolchain.toml: the benchmark
+      # also compiles the BASE checkout (the previous nightly build, which
+      # may predate the stable-toolchain migration and still uses
+      # #![feature] gates), so the runner must keep a nightly toolchain
+      # that can build historical revisions. This pin moves only via a
+      # deliberate runner-image rebuild.
       RUSTUP_TOOLCHAIN: nightly-2026-03-21
       RUSTUP_AUTO_INSTALL: "0"
       CARGO_TARGET_DIR: /home/runner/query-regression-target
@@ -504,6 +510,9 @@ jobs:
           require_eq cargo_path "$(command -v cargo || true)" "/opt/cargo/bin/cargo"
           require_eq rustc_path "$(command -v rustc || true)" "/opt/cargo/bin/rustc"
           require_match rustup "$(capture rustup --version)" '^rustup[[:space:]]1\.29\.0([[:space:]]|$)'
+          # The runner toolchain is pinned to nightly-2026-03-21 (see the
+          # RUSTUP_TOOLCHAIN comment): it must compile historical base
+          # checkouts, so it intentionally does not follow rust-toolchain.toml.
           require_match cargo "$(capture cargo --version)" \
             '^cargo[[:space:]]1\.96\.0-nightly[[:space:]]\(cbb9bb8bd[[:space:]][0-9]{4}-[0-9]{2}-[0-9]{2}\)$'
           require_match rustc "$(capture rustc --version)" \
@@ -540,7 +549,7 @@ jobs:
           readonly EXPECTED_SCCACHE_DIR="/home/runner/.cache/sccache"
           readonly EXPECTED_RUSTC_WRAPPER="/usr/local/bin/sccache"
           readonly RUNNER_IMAGE_DIGEST="sha256:e713b294e23b7e15184e558866c90025e59930033e72c97650dbc7f1ca022d11"
-          readonly RUNNER_IMAGE_EPOCH="5"
+          readonly RUNNER_IMAGE_EPOCH="7"
 
           require_expected_root() {
             local name="$1"

.github/workflows/release-dev-builder-images.yaml

@@ -1,4 +1,4 @@
-name: Release dev-builder images
+name: Release builder images
 
 on:
   push:
@@ -7,6 +7,7 @@ on:
     paths:
       - rust-toolchain.toml
       - 'docker/dev-builder/**'
+      - '.github/runner-scale-sets/query-regression/**'
   workflow_dispatch: # Allows you to run this workflow manually.
     inputs:
       release_dev_builder_ubuntu_image:
@@ -29,6 +30,11 @@ on:
         description: Release dev-builder-riscv64 image
         required: false
         default: false
+      release_query_regression_runner_image:
+        type: boolean
+        description: Rebuild the query-regression ECS runner image (and update QUERY_REGRESSION_ECS_IMAGE_ID + RUNNER_IMAGE_EPOCH)
+        required: false
+        default: false
       update_dev_builder_image_tag:
         type: boolean
         description: Update the DEV_BUILDER_IMAGE_TAG in Makefile and create a PR
@@ -36,12 +42,59 @@ on:
         default: false
 
 jobs:
+  changes:
+    name: Detect changed components
+    runs-on: ubuntu-latest
+    outputs:
+      dev-builder: ${{ steps.filter.outputs.dev-builder }}
+      query-regression-runner: ${{ steps.filter.outputs.query-regression-runner }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Filter changes
+        id: filter
+        shell: bash
+        run: |
+          set -euo pipefail
+          base="${{ github.event.before }}"
+          head="${{ github.sha }}"
+          if [[ "${{ github.event_name }}" != "push" ]] || [[ -z "${base}" ]] || ! git cat-file -e "${base}^{commit}" 2>/dev/null; then
+            # Dispatch, or an unknown base (new branch/force-push): treat
+            # everything as changed so gating falls back to job inputs.
+            echo "dev-builder=true" >> "$GITHUB_OUTPUT"
+            echo "query-regression-runner=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          files="$(git diff --name-only "${base}" "${head}")"
+          dev_builder=false
+          query_regression_runner=false
+          # rust-toolchain.toml only gates the dev-builder images: they bake
+          # the workspace toolchain. The query-regression runner toolchain is
+          # pinned inside its Dockerfile (nightly-2026-03-21) and must NOT
+          # follow the workspace pin, so its rebuild triggers only on changes
+          # under its own directory.
+          if grep -qx 'rust-toolchain.toml' <<<"${files}"; then
+            dev_builder=true
+          fi
+          if grep -q '^docker/dev-builder/' <<<"${files}"; then
+            dev_builder=true
+          fi
+          if grep -q '^\.github/runner-scale-sets/query-regression/' <<<"${files}"; then
+            query_regression_runner=true
+          fi
+          echo "dev-builder=${dev_builder}" >> "$GITHUB_OUTPUT"
+          echo "query-regression-runner=${query_regression_runner}" >> "$GITHUB_OUTPUT"
+
   release-dev-builder-images:
     name: Release dev builder images
     # The jobs are triggered by the following events:
     # 1. Manually triggered workflow_dispatch event
     # 2. Push event when the PR that modifies the `rust-toolchain.toml` or `docker/dev-builder/**` is merged to main
-    if: ${{ github.event_name == 'push' || inputs.release_dev_builder_ubuntu_image || inputs.release_dev_builder_centos_image || inputs.release_dev_builder_android_image || inputs.release_dev_builder_riscv64_image }}
+    needs: [changes]
+    if: ${{ (github.event_name == 'push' && needs.changes.outputs.dev-builder == 'true') || inputs.release_dev_builder_ubuntu_image || inputs.release_dev_builder_centos_image || inputs.release_dev_builder_android_image || inputs.release_dev_builder_riscv64_image }}
     runs-on: ubuntu-latest
     outputs:
       version: ${{ steps.set-version.outputs.version }}
@@ -59,7 +112,10 @@ jobs:
           commitShortSHA=`echo ${{ github.sha }} | cut -c1-8`
           buildTime=`date +%Y%m%d%H%M%S`
           BUILD_VERSION="$commitShortSHA-$buildTime"
-          RUST_TOOLCHAIN_VERSION=$(cat rust-toolchain.toml | grep -Eo '[0-9]{4}-[0-9]{2}-[0-9]{2}')
+          # Extract the full channel string (e.g. "1.96.1" or
+          # "nightly-2026-03-21") so the image version always carries the
+          # toolchain it was built with.
+          RUST_TOOLCHAIN_VERSION=$(grep -E '^channel' rust-toolchain.toml | cut -d'"' -f2)
           IMAGE_VERSION="${RUST_TOOLCHAIN_VERSION}-${BUILD_VERSION}"
           echo "VERSION=${IMAGE_VERSION}" >> $GITHUB_ENV
           echo "version=$IMAGE_VERSION" >> $GITHUB_OUTPUT
@@ -278,3 +334,120 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           ./.github/scripts/update-dev-builder-version.sh ${{ needs.release-dev-builder-images.outputs.version }}
+
+  rebuild-query-regression-runner-image:
+    # Rebuilds the query-regression ECS runner image via the ops tool in
+    # .github/runner-scale-sets/query-regression/ecs-image/: boots a
+    # temporary pay-as-you-go ECS builder, snapshots a new custom image,
+    # then completes the documented lockstep updates:
+    #   1. bumps RUNNER_IMAGE_EPOCH in query-regression.yml (target-cache
+    #      invalidation) via a PR from a timestamped ci/ branch (main is
+    #      branch-protected; mirrors update-dev-builder-version.sh), and
+    #   2. points the repo variable QUERY_REGRESSION_ECS_IMAGE_ID at the
+    #      new image. The runner toolchain is pinned inside the Dockerfile
+    #      (nightly-2026-03-21, independent of rust-toolchain.toml — the
+    #      benchmark must compile historical base checkouts), so image
+    #      rebuilds do not change the cache ABI.
+    #
+    # Required repository configuration (same names the provisioning job
+    # uses): vars ALIYUN_ECS_REGION_ID, ALIYUN_ECS_VSWITCH_ID,
+    # ALIYUN_ECS_SECURITY_GROUP_ID, optionally ALIYUN_ECS_RESOURCE_GROUP_ID
+    # and ALIYUN_ECS_BASE_IMAGE_ID (deterministic base-image pin; otherwise
+    # the rebuild auto-resolves the latest public Ubuntu 26.04 image).
+    # Secrets: ALICLOUD_ECS_ACCESS_KEY_ID, ALICLOUD_ECS_ACCESS_KEY_SECRET,
+    # GH_PERSONAL_ACCESS_TOKEN (repo push + actions-variable write).
+    name: Rebuild query-regression runner image
+    needs: [changes]
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && ((github.event_name == 'push' && needs.changes.outputs.query-regression-runner == 'true') || inputs.release_query_regression_runner_image) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 150
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          # Keep the PAT credential for the epoch-bump push below.
+          persist-credentials: true
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Rebuild the runner image
+        id: rebuild
+        env:
+          ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
+          ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
+        run: |
+          set -euo pipefail
+          args=(
+            --region-id "${{ vars.ALIYUN_ECS_REGION_ID }}"
+            --vswitch-id "${{ vars.ALIYUN_ECS_VSWITCH_ID }}"
+            --security-group-id "${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}"
+            --base-image-id "${{ vars.ALIYUN_ECS_BASE_IMAGE_ID }}"
+          )
+          if [[ -n "${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}" ]]; then
+            args+=(--resource-group-id "${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}")
+          fi
+          uv run .github/runner-scale-sets/query-regression/ecs-image/build-ecs-image.py \
+            "${args[@]}" 2>&1 | tee rebuild.log
+          image_id="$(sed -n 's/^Custom image ready: \([^ ]*\) .*/\1/p' rebuild.log | tail -n1)"
+          if [[ -z "${image_id}" ]]; then
+            echo "Error: no image id found in build output" >&2
+            exit 1
+          fi
+          echo "image_id=${image_id}" >> "$GITHUB_OUTPUT"
+          echo "## Rebuilt runner image" >> "$GITHUB_STEP_SUMMARY"
+          echo "New image: \`${image_id}\`" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Open the epoch-bump PR and update the image id variable
+        env:
+          GH_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+        run: |
+          set -euo pipefail
+          image_id="${{ steps.rebuild.outputs.image_id }}"
+          old_epoch="$(grep -oE 'readonly RUNNER_IMAGE_EPOCH="[0-9]+"' \
+            .github/workflows/query-regression.yml | grep -oE '[0-9]+' | head -n1)"
+          if [[ -z "${old_epoch}" ]]; then
+            echo "Error: RUNNER_IMAGE_EPOCH not found in query-regression.yml" >&2
+            exit 1
+          fi
+          new_epoch=$((old_epoch + 1))
+
+          # main is branch-protected (required reviews + status checks), so
+          # the epoch bump cannot be pushed there directly. Mirror
+          # .github/scripts/update-dev-builder-version.sh: timestamped bot
+          # branch, plain push, and a PR with reviewers. Do NOT use [skip ci]
+          # on the commit: the required checks must be able to run for the PR
+          # to become mergeable.
+          BRANCH="ci/update-query-regression-epoch-$(date +%Y%m%d%H%M%S)"
+          git config user.name "greptimedb-ci"
+          git config user.email "greptimedb-ci@greptime.com"
+          git fetch origin main
+          git checkout -b "${BRANCH}" origin/main
+          sed -i "s/readonly RUNNER_IMAGE_EPOCH=\"${old_epoch}\"/readonly RUNNER_IMAGE_EPOCH=\"${new_epoch}\"/" \
+            .github/workflows/query-regression.yml
+          git add .github/workflows/query-regression.yml
+          git commit -s -m "ci(query-regression): bump RUNNER_IMAGE_EPOCH to ${new_epoch} for image ${image_id}"
+          git push origin "${BRANCH}"
+
+          gh pr create \
+            --title "ci(query-regression): bump RUNNER_IMAGE_EPOCH to ${new_epoch}" \
+            --body "Auto-generated by the \`Rebuild query-regression runner image\` job in ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} for image \`${image_id}\`. Merging this invalidates the query-regression target cache (RUNNER_IMAGE_EPOCH ${old_epoch} → ${new_epoch}). The \`QUERY_REGRESSION_ECS_IMAGE_ID\` repo variable already points at the new image; the epoch bump is belt-and-suspenders for image-content drift." \
+            --base main \
+            --head "${BRANCH}" \
+            --reviewer sunng87 \
+            --reviewer daviderli614 \
+            --reviewer killme2008 \
+            --reviewer evenyag \
+            --reviewer fengjiachun \
+            --reviewer WenyXu
+
+          # Point the variable at the new image once the PR exists. Image
+          # rebuilds keep the same pinned toolchain, so the cache ABI is
+          # unchanged and not waiting for the PR merge is fine.
+          gh api -X PATCH "repos/${{ github.repository }}/actions/variables/QUERY_REGRESSION_ECS_IMAGE_ID" \
+            -f value="${image_id}"
+
+          echo "## Lockstep updates" >> "$GITHUB_STEP_SUMMARY"
+          echo "- QUERY_REGRESSION_ECS_IMAGE_ID → \`${image_id}\`" >> "$GITHUB_STEP_SUMMARY"
+          echo "- RUNNER_IMAGE_EPOCH → \`${new_epoch}\` (PR on \`${BRANCH}\`, pending review/merge)" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/release.yml

@@ -4,15 +4,20 @@ name: Release
 # 1. The tag('v*.*.*') push release: the release workflow will be triggered by the tag push event.
 # 2. The scheduled release(the version will be '<base-version>-nightly-YYYYMMDD', any pre-release
 #    extension in the next release version like '-alpha.1' is stripped, so 'nightly' is the extension):
-#    the release workflow will be triggered by the schedule event.
+#    the release workflow will be triggered by the schedule event. The schedule runs every weekday,
+#    but the release only proceeds when the latest published nightly release is older than
+#    NIGHTLY_RELEASE_MAX_AGE_DAYS days, so a failed nightly run is retried by the next weekday
+#    schedule instead of leaving a whole week without nightly builds.
 # A failed tag push release can be manually dispatched on the same tag to recover it without changing the formal release version.
 on:
   push:
     tags:
       - "v*.*.*"
   schedule:
-    # At 00:00 on Monday.
-    - cron: '0 0 * * 1'
+    # At 00:00 on every weekday (Monday to Friday). The scheduled nightly release only
+    # proceeds when the latest published nightly release is older than
+    # NIGHTLY_RELEASE_MAX_AGE_DAYS days, see the 'Check nightly release necessity' step.
+    - cron: '0 0 * * 1-5'
   workflow_dispatch: # Allows you to run this workflow manually.
     # Notes: The GitHub Actions ONLY support 10 inputs; this final input is the tenth.
     inputs:
@@ -79,7 +84,7 @@ on:
         default: false
       release_validation:
         type: choice
-        description: Manual policy for accepted perf regressions or runner failures; automatic releases always run all validation
+        description: Manual policy for accepted perf regressions or runner failures; automatic releases always run all validation (scheduled nightlies run query regression non-blocking)
         required: true
         default: all
         options:
@@ -100,6 +105,10 @@ env:
   # the pre-release extension of the next release version is stripped, e.g. v0.2.0-alpha.1 -> v0.2.0-nightly-20230313.
   NIGHTLY_RELEASE_PREFIX: nightly
 
+  # The maximum age (in days) of the latest published nightly release before a scheduled
+  # nightly release may run again; see the 'Check nightly release necessity' step.
+  NIGHTLY_RELEASE_MAX_AGE_DAYS: 5
+
 jobs:
   allocate-runners:
     name: Allocate runners
@@ -123,6 +132,10 @@ jobs:
       # The 'is-current-version-latest' determines whether to update 'latest' Docker tags and downstream repositories.
       is-current-version-latest: ${{ steps.check-version.outputs.is-current-version-latest }}
       is-current-version-stable: ${{ steps.check-version.outputs.is-current-version-stable }}
+
+      # Whether a scheduled nightly release should run: it only runs when the latest published
+      # nightly release is older than NIGHTLY_RELEASE_MAX_AGE_DAYS days. Always 'true' for other events.
+      nightly-required: ${{ steps.check-nightly.outputs.nightly-required }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -135,6 +148,19 @@ jobs:
         run: |
           ./scripts/check-builder-rust-version.sh
 
+      # A scheduled run only proceeds when the latest published nightly release is older than
+      # NIGHTLY_RELEASE_MAX_AGE_DAYS days, so a failed nightly run is retried by the next weekday
+      # schedule. The check runs before the runners are allocated to avoid wasting EC2 resources.
+      - name: Check nightly release necessity
+        id: check-nightly
+        run: |
+          echo "nightly-required=$(./.github/scripts/check-nightly-release.sh)" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          NIGHTLY_RELEASE_MAX_AGE_DAYS: ${{ env.NIGHTLY_RELEASE_MAX_AGE_DAYS }}
+
       # The create-version will create a global variable named 'version' in the global workflows.
       # - If it's a tag push release or manual dispatch on a tag with REUSE_EXISTING_RELEASE_TAG=true, the version is the tag name(${{ github.ref_name }});
       # - If it's a scheduled release, the version is '<base-version>-nightly-$buildTime', like v0.2.0-nightly-20230313 (pre-release extensions like '-alpha.1' are stripped);
@@ -156,7 +182,7 @@ jobs:
           ./.github/scripts/check-version.sh "${{ steps.create-version.outputs.version }}"
 
       - name: Allocate linux-amd64 runner
-        if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+        if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && steps.check-nightly.outputs.nightly-required == 'true' }}
         uses: ./.github/actions/start-runner
         id: start-linux-amd64-runner
         with:
@@ -170,7 +196,7 @@ jobs:
           subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
 
       - name: Allocate linux-arm64 runner
-        if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+        if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && steps.check-nightly.outputs.nightly-required == 'true' }}
         uses: ./.github/actions/start-runner
         id: start-linux-arm64-runner
         with:
@@ -185,7 +211,7 @@ jobs:
 
   build-linux-amd64-artifacts:
     name: Build linux-amd64 artifacts
-    if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     needs: [
       allocate-runners,
     ]
@@ -204,10 +230,14 @@ jobs:
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
           image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
           image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}
+          aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
+          aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
 
   build-linux-arm64-artifacts:
     name: Build linux-arm64 artifacts
-    if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     needs: [
       allocate-runners,
     ]
@@ -227,10 +257,14 @@ jobs:
           image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
           image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}
           large-page-size: true
+          aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
+          aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
 
   build-linux-riscv64-artifacts:
     name: Build linux-riscv64 artifacts
-    if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     needs: [
       allocate-runners,
     ]
@@ -294,7 +328,7 @@ jobs:
     needs: [
       allocate-runners,
     ]
-    if: ${{ inputs.build_macos_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.build_macos_artifacts || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -332,7 +366,7 @@ jobs:
     needs: [
       allocate-runners,
     ]
-    if: ${{ inputs.build_windows_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.build_windows_artifacts || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     steps:
       - run: git config --global core.autocrlf false
 
@@ -357,7 +391,7 @@ jobs:
 
   prepare-release-validation:
     name: Prepare release validation
-    if: ${{ inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     needs: [
       allocate-runners,
     ]
@@ -434,6 +468,9 @@ jobs:
             if [[ "${validation_policy}" != "all" ]]; then
               echo "- ⚠️ Validation bypassed: compatibility=${run_compat}, query-regression=${run_query_regression}"
             fi
+            if [[ "${GITHUB_EVENT_NAME}" == "schedule" ]]; then
+              echo "- ℹ️ Query-regression failure or cancellation is non-blocking for scheduled nightly releases"
+            fi
           } >> "${GITHUB_STEP_SUMMARY}"
 
   compat-release:
@@ -517,9 +554,13 @@ jobs:
 
   release-images-to-dockerhub:
     name: Build and push images to DockerHub
+    # Scheduled nightly releases do not gate on query regression: the test
+    # still runs and reports, but a failure or cancellation does not block
+    # the nightly release.
     if: |
       always() &&
       (inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') &&
+      (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') &&
       needs.allocate-runners.result == 'success' &&
       needs.build-linux-amd64-artifacts.result == 'success' &&
       needs.build-linux-arm64-artifacts.result == 'success' &&
@@ -527,7 +568,8 @@ jobs:
       (needs.compat-release.result == 'success' ||
        (needs.prepare-release-validation.outputs.run-compat == 'false' && needs.compat-release.result == 'skipped')) &&
       (needs.query-regression-release.result == 'success' ||
-       (needs.prepare-release-validation.outputs.run-query-regression == 'false' && needs.query-regression-release.result == 'skipped'))
+       (needs.prepare-release-validation.outputs.run-query-regression == 'false' && needs.query-regression-release.result == 'skipped') ||
+       github.event_name == 'schedule')
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -563,7 +605,18 @@ jobs:
 
   release-cn-artifacts:
     name: Release artifacts to CN region
-    if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    # A non-blocking query regression failure must not skip downstream releases.
+    if: |
+      always() &&
+      (inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') &&
+      (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') &&
+      needs.allocate-runners.result == 'success' &&
+      needs.build-linux-amd64-artifacts.result == 'success' &&
+      needs.build-linux-arm64-artifacts.result == 'success' &&
+      needs.build-linux-riscv64-artifacts.result == 'success' &&
+      needs.build-macos-artifacts.result == 'success' &&
+      needs.build-windows-artifacts.result == 'success' &&
+      needs.release-images-to-dockerhub.result == 'success'
     needs: [ # The job have to wait for all the artifacts are built.
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -607,15 +660,19 @@ jobs:
     name: Create GitHub release and upload artifacts
     # Use always() to run even when optional jobs (macos, windows) are skipped.
     # Then check that required jobs succeeded and optional jobs didn't fail.
+    # Scheduled nightly releases do not gate on query regression either
+    # (see release-images-to-dockerhub); the test still runs and reports.
     if: |
       always() &&
       (inputs.publish_github_release || github.event_name == 'push' || github.event_name == 'schedule') &&
+      (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') &&
       needs.allocate-runners.result == 'success' &&
       needs.prepare-release-validation.result == 'success' &&
       (needs.compat-release.result == 'success' ||
        (needs.prepare-release-validation.outputs.run-compat == 'false' && needs.compat-release.result == 'skipped')) &&
       (needs.query-regression-release.result == 'success' ||
-       (needs.prepare-release-validation.outputs.run-query-regression == 'false' && needs.query-regression-release.result == 'skipped')) &&
+       (needs.prepare-release-validation.outputs.run-query-regression == 'false' && needs.query-regression-release.result == 'skipped') ||
+       github.event_name == 'schedule') &&
       needs.build-linux-amd64-artifacts.result == 'success' &&
       needs.build-linux-arm64-artifacts.result == 'success' &&
       needs.build-linux-riscv64-artifacts.result == 'success' &&
@@ -718,7 +775,21 @@ jobs:
 
   bump-downstream-repo-versions:
     name: Bump downstream repo versions
-    if: ${{ github.event_name == 'schedule' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_type == 'tag' && !contains(github.ref_name, 'nightly')) && (needs.allocate-runners.outputs.is-current-version-stable == 'true' && needs.allocate-runners.outputs.is-current-version-latest == 'true') }}
+    # A non-blocking query regression failure must not skip downstream updates.
+    if: |
+      always() &&
+      needs.allocate-runners.result == 'success' &&
+      needs.publish-github-release.result == 'success' &&
+      (
+        (github.event_name == 'schedule' && needs.allocate-runners.outputs.nightly-required == 'true') ||
+        (
+          (github.event_name == 'push' || github.event_name == 'workflow_dispatch') &&
+          github.ref_type == 'tag' &&
+          !contains(github.ref_name, 'nightly') &&
+          needs.allocate-runners.outputs.is-current-version-stable == 'true' &&
+          needs.allocate-runners.outputs.is-current-version-latest == 'true'
+        )
+      )
     needs: [allocate-runners, publish-github-release]
     runs-on: ubuntu-latest
     # Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
@@ -785,9 +856,10 @@ jobs:
           ./.github/scripts/update-homebrew-greptme-version.sh
 
   notification:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'push' || github.event_name == 'schedule') && always() }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'push' || (github.event_name == 'schedule' && needs.allocate-runners.outputs.nightly-required == 'true')) && always() }}
     name: Send notification to Greptime team
     needs: [
+      allocate-runners,
       release-images-to-dockerhub,
       build-macos-artifacts,
       build-windows-artifacts,

.github/workflows/rust.yml

@@ -24,7 +24,7 @@ on:
 name: Rust CI
 
 env:
-  CARGO_UDEPS_VERSION: "0.1.61"
+  CARGO_SHEAR_VERSION: "1.13.4"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -32,7 +32,7 @@ concurrency:
 
 jobs:
   fmt:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Rustfmt
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -50,7 +50,7 @@ jobs:
         run: make fmt-check
 
   clippy:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Clippy
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -76,7 +76,7 @@ jobs:
         run: make clippy
 
   check-udeps:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Check Unused Dependencies
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -84,26 +84,17 @@ jobs:
       - uses: actions/checkout@v4
         with:
           persist-credentials: false
-      - uses: arduino/setup-protoc@v3
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          shared-key: "check-udeps"
-          cache-all-crates: "true"
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-      - name: Install cargo-udeps
+      - name: Install cargo-shear
         uses: taiki-e/install-action@v2
         with:
-          tool: cargo-udeps@${{ env.CARGO_UDEPS_VERSION }}
+          tool: cargo-shear@${{ env.CARGO_SHEAR_VERSION }}
           fallback: none
       - name: Check unused dependencies
         run: make check-udeps
 
   check-riscv64:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Check riscv64 build
     runs-on: ubuntu-latest
     timeout-minutes: 90
@@ -129,7 +120,7 @@ jobs:
         run: cargo check --locked --workspace --all-targets --features servers/dashboard --target riscv64gc-unknown-linux-gnu
 
   check-windows:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Check Windows build
     runs-on: windows-2022
     timeout-minutes: 60
@@ -157,8 +148,50 @@ jobs:
         env:
           RUSTFLAGS: "-D warnings"
 
-  conflict-check:
+  test-windows:
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    name: Test Windows
+    runs-on: windows-2022
+    timeout-minutes: 90
+    env:
+      CARGO_INCREMENTAL: 0
+      CARGO_PROFILE_DEV_DEBUG: 0
+      CARGO_PROFILE_TEST_DEBUG: 0
+      RUSTFLAGS: "-D warnings -C linker=lld-link"
+      RUST_BACKTRACE: 1
+      RUST_MIN_STACK: 8388608 # 8MB
+    steps:
+      - run: git config --global core.autocrlf false
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/setup-cyborg
+      - uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: KyleMayes/install-llvm-action@v1
+        with:
+          version: "14.0"
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache: false
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: "test-windows"
+          cache-all-crates: "true"
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      - name: Install Cargo Nextest
+        uses: taiki-e/install-action@nextest
+      - name: Install WSL distribution
+        uses: Vampire/setup-wsl@v2
+        with:
+          distribution: Ubuntu-22.04
+      - name: Run Windows tests
+        run: cargo nextest run --locked --workspace -F dashboard --no-fail-fast
+
+  conflict-check:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     name: Check for conflict
     runs-on: ubuntu-latest
     steps:
@@ -169,7 +202,7 @@ jobs:
         uses: olivernybroe/action-conflict-finder@v4.0
 
   test:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'merge_group' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'merge_group' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
     runs-on: ubuntu-22.04-arm
     timeout-minutes: 60
     needs: [conflict-check, clippy, fmt, check-udeps]
@@ -200,21 +233,20 @@ jobs:
         run: ../../.github/scripts/pull-test-deps-images.sh && docker compose up -d --wait
 
       - name: Run nextest cases
-        run: cargo nextest run --workspace -F dashboard -F pg_kvbackend -F mysql_kvbackend -F vector_index -F ai_functions
+        run: cargo nextest run --workspace -F dashboard -F pg_kvbackend -F mysql_kvbackend -F ai_functions -F hdfs-object-store
         env:
           CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
           RUST_BACKTRACE: 1
           RUST_MIN_STACK: 8388608 # 8MB
           CARGO_INCREMENTAL: 0
-          GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
-          GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
-          GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
-          GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
-          GT_MINIO_BUCKET: greptime
-          GT_MINIO_ACCESS_KEY_ID: superpower_ci_user
-          GT_MINIO_ACCESS_KEY: superpower_password
-          GT_MINIO_REGION: us-west-2
-          GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
+          # Runs the object-store matrix against the local MinIO from
+          # tests-integration/fixtures. Real AWS S3 is covered by the nightly
+          # jobs and by the release build.
+          GT_S3_BUCKET: greptime
+          GT_S3_ACCESS_KEY_ID: superpower_ci_user
+          GT_S3_ACCESS_KEY: superpower_password
+          GT_S3_REGION: us-west-2
+          GT_S3_ENDPOINT_URL: http://127.0.0.1:9000
           GT_ETCD_TLS_ENDPOINTS: https://127.0.0.1:2378
           GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
           GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
@@ -258,20 +290,19 @@ jobs:
         run: ../../.github/scripts/pull-test-deps-images.sh && docker compose up -d --wait
 
       - name: Run nextest cases
-        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F dashboard -F pg_kvbackend -F mysql_kvbackend -F vector_index -F ai_functions
+        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F dashboard -F pg_kvbackend -F mysql_kvbackend -F ai_functions -F hdfs-object-store
         env:
           CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
           RUST_BACKTRACE: 1
           CARGO_INCREMENTAL: 0
-          GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
-          GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
-          GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
-          GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
-          GT_MINIO_BUCKET: greptime
-          GT_MINIO_ACCESS_KEY_ID: superpower_ci_user
-          GT_MINIO_ACCESS_KEY: superpower_password
-          GT_MINIO_REGION: us-west-2
-          GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
+          # Runs the object-store matrix against the local MinIO from
+          # tests-integration/fixtures. Real AWS S3 is covered by the nightly
+          # jobs and by the release build.
+          GT_S3_BUCKET: greptime
+          GT_S3_ACCESS_KEY_ID: superpower_ci_user
+          GT_S3_ACCESS_KEY: superpower_password
+          GT_S3_REGION: us-west-2
+          GT_S3_ENDPOINT_URL: http://127.0.0.1:9000
           GT_ETCD_TLS_ENDPOINTS: https://127.0.0.1:2378
           GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
           GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres

.github/workflows/slash-command-dispatch.yml

@@ -1,13 +1,13 @@
 name: Slash Command Dispatch
 
-# ChatOps front door: parse `/command` on PR comments, check admin
-# permission, and repository_dispatch to a per-command handler. Handlers
+# ChatOps front door: parse `/command` on PR comments, apply command-specific
+# permissions, and repository_dispatch to a per-command handler. Handlers
 # own allowlists, SHA admission, and the actual work. Same-repo dispatch
 # uses github.token with contents: write; GitHub starts the handler run
 # for GITHUB_TOKEN-created repository_dispatch events. Do not pass the
 # long-lived GH_PERSONAL_ACCESS_TOKEN into this third-party action.
 #
-# To add a command: list it under `commands` and add a workflow with
+# To add a command: add it to `config` and add a workflow with
 # `on.repository_dispatch.types: ["<command>-command"]`.
 
 on:
@@ -30,8 +30,17 @@ jobs:
         uses: peter-evans/slash-command-dispatch@9bdcd7914ec1b75590b790b844aa3b8eee7c683a # v5.0.2
         with:
           token: ${{ github.token }}
-          permission: admin
-          issue-type: pull-request
-          commands: |
-            query-regression
-            ci
+          # CI admission re-fetches the comment and checks author/member permissions.
+          config: >-
+            [
+              {"command": "query-regression", "permission": "admin", "issue_type": "pull-request"},
+              {"command": "ci", "permission": "none", "issue_type": "pull-request"}
+            ]
+      - name: Report CI dispatch failure
+        if: ${{ failure() && (github.event.comment.body == '/ci' || startsWith(github.event.comment.body, '/ci ')) }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.issue.number }}
+        run: |
+          gh api --method POST "/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" \
+            -f "body=CI command dispatch failed. See ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} for details."

.github/workflows/tracesbench.yml

@@ -0,0 +1,495 @@
+name: Tracesbench
+on:
+  workflow_dispatch:
+    inputs:
+      run_greptimedb:
+        description: Run GreptimeDB
+        type: boolean
+        default: true
+        required: false
+      run_victoriatraces:
+        description: Run VictoriaTraces
+        type: boolean
+        default: false
+        required: false
+      run_tempo:
+        description: Run Tempo
+        type: boolean
+        default: false
+        required: false
+      compare_greptimedb:
+        description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+        type: boolean
+        default: false
+        required: false
+      baseline_greptimedb_tag:
+        description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+        type: string
+        default: v1.2.1
+        required: false
+      greptimedb_tag:
+        description: Candidate Docker Hub greptime/greptimedb tag
+        type: string
+        default: latest
+        required: true
+      dataset:
+        description: Dataset profile (main is experimental; native queries may exceed memory limits)
+        type: choice
+        options: [sanity, preflight, main]
+        default: sanity
+        required: true
+      runtime_image:
+        description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
+        type: string
+        default: auto
+        required: true
+      provider:
+        description: Cloud provider for the benchmark runner
+        type: choice
+        options: [Aliyun, AWS]
+        default: Aliyun
+        required: true
+      instance_type:
+        description: 'Instance type; auto = Aliyun: ecs.g9i.2xlarge, AWS: m7i.2xlarge (both 8 vCPU / 32 GiB). Enter a specific type to override.'
+        type: string
+        default: auto
+        required: true
+      benchmark_root:
+        description: Absolute disk-backed work directory (outside the o11ybench checkout)
+        type: string
+        default: /home/runner/benchmark-data
+        required: true
+      system_disk_gib:
+        description: 'System disk GiB: auto (or 0) = 40 for sanity/preflight, 200 for main; enter 20..2048 to override'
+        type: string
+        default: auto
+        required: true
+      benchmark_timeout_minutes:
+        description: Benchmark job timeout in minutes (1..360)
+        type: number
+        default: 360
+        required: true
+      janitor_ttl_hours:
+        description: Instance lifetime before janitor eligibility (1..168 hours; at least timeout + 2h)
+        type: number
+        default: 8
+        required: true
+      db_cpus:
+        description: CPU limit per DB container (default 8 vCPU)
+        type: string
+        default: '8'
+        required: true
+      db_memory:
+        description: Memory limit per DB container (default 32 GiB; shared host also runs the runtime and runner)
+        type: string
+        default: 32g
+        required: true
+      load_workers:
+        description: CPU workers for parallel import reading/parsing across all engines (separate from write concurrency)
+        type: number
+        default: 4
+        required: true
+      query_timeout_seconds:
+        description: Timeout per query request in seconds (correctness, warmup and measurement; 1..3600)
+        type: number
+        default: 300
+        required: true
+      runtime_cpus:
+        description: CPU limit per runtime container (0 = unrestricted)
+        type: string
+        default: '0'
+        required: true
+      runtime_memory:
+        description: Memory limit per runtime container (0 = unrestricted; separate from DB memory)
+        type: string
+        default: '0'
+        required: true
+permissions:
+  contents: read
+env:
+  COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+  BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
+  DATASET: ${{ inputs.dataset }}
+  GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
+  RUNTIME_CPUS: ${{ inputs.runtime_cpus }}
+  RUNTIME_MEMORY: ${{ inputs.runtime_memory }}
+  BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
+  RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' }}
+  PROVIDER: ${{ inputs.provider }}
+  REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
+  SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
+  BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }}
+  JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
+  DB_CPUS: ${{ inputs.db_cpus }}
+  DB_MEMORY: ${{ inputs.db_memory }}
+  QUERY_TIMEOUT_SECONDS: ${{ inputs.query_timeout_seconds }}
+  LOAD_WORKERS: ${{ inputs.load_workers }}
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    outputs:
+      instance_type: ${{ steps.runner.outputs.instance_type }}
+      targets: ${{ steps.inputs.outputs.targets }}
+      greptimedb: ${{ steps.inputs.outputs.greptimedb }}
+      victoriatraces: ${{ steps.inputs.outputs.victoriatraces }}
+      tempo: ${{ steps.inputs.outputs.tempo }}
+      system_disk_gib: ${{ steps.runtime.outputs.system_disk_gib }}
+      runtime_image: ${{ steps.runtime.outputs.runtime_image }}
+      o11ybench_ref: ${{ steps.runtime.outputs.o11ybench_ref }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        path: greptimedb
+        persist-credentials: false
+    - name: Validate selected targets
+      id: inputs
+      env:
+        greptimedb: ${{ inputs.run_greptimedb }}
+        victoriatraces: ${{ inputs.run_victoriatraces }}
+        tempo: ${{ inputs.run_tempo }}
+      run: |
+        set -euo pipefail
+        selected=
+        for target in greptimedb victoriatraces tempo; do
+          enabled=${!target}
+          [[ "$enabled" == true || "$enabled" == false ]]
+          printf '%s=%s\n' "$target" "$enabled" >> "$GITHUB_OUTPUT"
+          if [[ "$enabled" == true ]]; then selected="${selected:+$selected,}$target"; fi
+        done
+        [[ -n "$selected" ]] || { echo 'Select at least one database' >&2; exit 1; }
+        if [[ "$greptimedb" == true ]]; then
+          [[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; }
+        fi
+        printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT"
+    - name: Validate version comparison
+      env:
+        RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+      run: |
+        python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
+    - name: Resolve CI runner
+      id: runner
+      run: |
+        set -euo pipefail
+        instance_type="$REQUESTED_INSTANCE_TYPE"
+        if [[ "$instance_type" == auto ]]; then
+          case "$PROVIDER" in
+            Aliyun) instance_type=ecs.g9i.2xlarge ;;
+            AWS) instance_type=m7i.2xlarge ;;
+          esac
+        fi
+        python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$instance_type"
+    - name: Validate budgets and shared runtime
+      id: runtime
+      run: |
+        set -euo pipefail
+        [[ "$BENCHMARK_ROOT" == /* && "$BENCHMARK_ROOT" != / && "$BENCHMARK_ROOT" != */ ]]
+        [[ "$BENCHMARK_ROOT" != /tmp && "$BENCHMARK_ROOT" != /tmp/* && "$BENCHMARK_ROOT" != *,* ]]
+        [[ "$BENCHMARK_ROOT" != *$'\n'* && "$BENCHMARK_ROOT" != *$'\r'* ]]
+        [[ "/${BENCHMARK_ROOT#/}/" != */../* && "/${BENCHMARK_ROOT#/}/" != */./* ]]
+        case "$DATASET" in sanity|preflight|main) ;; *) echo "Unknown dataset"; exit 1;; esac
+        if [[ "$SYSTEM_DISK_GIB" == auto || "$SYSTEM_DISK_GIB" == 0 ]]; then
+          if [[ "$DATASET" == main ]]; then SYSTEM_DISK_GIB=200; else SYSTEM_DISK_GIB=40; fi
+        fi
+        for value in "$SYSTEM_DISK_GIB" "$BENCHMARK_TIMEOUT_MINUTES" "$JANITOR_TTL_HOURS"; do
+          [[ "$value" =~ ^[1-9][0-9]{0,3}$ ]] || { echo 'Resource budgets must be positive integers'; exit 1; }
+        done
+        ((SYSTEM_DISK_GIB >= 20 && SYSTEM_DISK_GIB <= 2048))
+        ((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168))
+        ((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120))
+        [[ "$DB_CPUS" =~ ^[1-9][0-9]*$ && "$DB_MEMORY" =~ ^[1-9][0-9]*[mMgG]$ ]]
+        [[ "$RUNTIME_CPUS" =~ ^(0|[1-9][0-9]*)$ && "$RUNTIME_MEMORY" =~ ^(0|[1-9][0-9]*[mMgG])$ ]]
+        [[ "$QUERY_TIMEOUT_SECONDS" =~ ^[1-9][0-9]{0,3}$ ]]
+        ((QUERY_TIMEOUT_SECONDS <= 3600))
+        [[ "$LOAD_WORKERS" =~ ^[1-9][0-9]{0,2}$ ]]
+        ((LOAD_WORKERS <= 256))
+        runtime_limits=()
+        [[ "$RUNTIME_CPUS" == 0 ]] || runtime_limits+=(--cpus "$RUNTIME_CPUS")
+        [[ "$RUNTIME_MEMORY" == 0 ]] || runtime_limits+=(--memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY")
+        [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]]
+        bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
+        metadata=$(docker image inspect "$RUNTIME_IMAGE")
+        resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata")
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          docker run --rm "$resolved" scripts/run_tracesbench_docker.py --help | grep -q -- '--data-root' \
+            || { echo 'Comparison requires a runtime with --data-root support'; exit 1; }
+        fi
+        revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata")
+        [[ "$revision" =~ ^[0-9a-f]{40}$ ]]
+        # Refuse runtimes without the trace runner and its validated dataset/report contract.
+        help=$(docker run --rm "${runtime_limits[@]}" "$resolved" scripts/run_tracesbench_docker.py --help)
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          grep -F -- '--baseline-run' <<< "$help" \
+            || { echo 'Comparison requires a runtime with --baseline-run report support'; exit 1; }
+        fi
+        grep -F -- '--dataset {sanity,preflight,main}' <<< "$help"
+        grep -F -- '--runtime-cpus' <<< "$help"
+        grep -F -- '--runtime-memory' <<< "$help"
+        grep -F -- '--query-timeout-seconds' <<< "$help"
+        grep -F -- '--load-workers' <<< "$help"
+        grep -F -- '--targets' <<< "$help"
+        grep -F -- '--greptimedb-image' <<< "$help"
+        docker run --rm "${runtime_limits[@]}" "$resolved" scripts/check_run_tracesbench_docker.py
+        printf 'runtime_image=%s\no11ybench_ref=%s\nsystem_disk_gib=%s\n' "$resolved" "$revision" "$SYSTEM_DISK_GIB" >> "$GITHUB_OUTPUT"
+  provision:
+    needs: validate
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    outputs:
+      label: ${{ steps.ecs.outputs.label || steps.aws.outputs.label }}
+      instance_id: ${{ steps.ecs.outputs.instance_id || steps.aws.outputs.instance_id }}
+      runner_name: ${{ steps.ecs.outputs.runner_name || steps.aws.outputs.runner_name }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        path: greptimedb
+        persist-credentials: false
+    - name: Create ephemeral ECS runner
+      if: inputs.provider == 'Aliyun'
+      id: ecs
+      uses: ./greptimedb/.github/actions/aliyun-ecs-create
+      with:
+        access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
+        access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
+        github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+        region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
+        vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
+        security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
+        image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
+        instance-type: ${{ needs.validate.outputs.instance_type }}
+        enable-docker: 'true'
+        system-disk-gib: ${{ needs.validate.outputs.system_disk_gib }}
+        ttl-hours: ${{ inputs.janitor_ttl_hours }}
+        resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
+        runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
+        runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
+    - uses: aws-actions/configure-aws-credentials@v4
+      if: inputs.provider == 'AWS'
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ vars.EC2_RUNNER_REGION }}
+    - uses: astral-sh/setup-uv@v6
+      if: inputs.provider == 'AWS'
+    - name: Create ephemeral EC2 runner
+      if: inputs.provider == 'AWS'
+      id: aws
+      env:
+        GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+        AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
+        AWS_EC2_IMAGE_ID: ${{ vars.BENCHMARK_EC2_IMAGE_ID || 'auto' }}
+        AWS_EC2_INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }}
+        AWS_EC2_SUBNET_ID: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+        AWS_EC2_SECURITY_GROUP_ID: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+        AWS_EC2_SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }}
+        AWS_EC2_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
+        AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+      run: uv run greptimedb/.github/scripts/aws-ec2-runner-provision.py
+  benchmark:
+    needs:
+    - validate
+    - provision
+    runs-on: ${{ needs.provision.outputs.label }}
+    timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }}
+    env:
+      TARGETS: ${{ needs.validate.outputs.targets }}
+      RESOLVED_RUNTIME_IMAGE: ${{ needs.validate.outputs.runtime_image }}
+      RESOLVED_O11YBENCH_REF: ${{ needs.validate.outputs.o11ybench_ref }}
+      RUN_ROOT: ${{ inputs.benchmark_root }}/tracesbench-${{ github.run_id }}-${{ github.run_attempt }}
+      OWNER: o11ybench-traces-${{ github.run_id }}-${{ github.run_attempt }}
+      METADATA_ROOT: ${{ github.workspace }}/tracesbench-metadata
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        path: greptimedb
+        persist-credentials: false
+    - name: Set up benchmark dependencies
+      uses: ./greptimedb/.github/actions/setup-benchmark
+    - name: Checkout matching benchmark tools
+      uses: actions/checkout@v4
+      with:
+        repository: GreptimeTeam/o11ybench
+        token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+        ref: ${{ env.RESOLVED_O11YBENCH_REF }}
+        path: o11ybench
+        persist-credentials: false
+    - name: Check runner and runtime identity
+      id: runtime
+      run: |
+        set -euo pipefail
+        test "$(git -C o11ybench rev-parse HEAD)" = "$RESOLVED_O11YBENCH_REF"
+        mkdir -p "$BENCHMARK_ROOT" "$METADATA_ROOT"
+        test "$(stat -f -c %T "$BENCHMARK_ROOT")" != tmpfs || { echo 'Benchmark root must be disk-backed'; exit 1; }
+        df -h "$BENCHMARK_ROOT" /var/lib/docker
+        minimum_gib=10
+        if [[ "$DATASET" == main ]]; then minimum_gib=100; fi
+        for path in "$BENCHMARK_ROOT" /var/lib/docker; do
+          available=$(df -Pk "$path" | awk 'NR == 2 {print $4}')
+          ((available >= minimum_gib * 1024 * 1024)) || { echo "Need at least $minimum_gib GiB free on $path"; exit 1; }
+        done
+        bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE"
+        docker image inspect "$RESOLVED_RUNTIME_IMAGE" > "$METADATA_ROOT/runtime-image.json"
+        test "$(jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]' "$METADATA_ROOT/runtime-image.json")" = "$RESOLVED_O11YBENCH_REF"
+    - name: Pull database images before generating data
+      run: |
+        set -euo pipefail
+        IFS=, read -ra targets <<< "$TARGETS"
+        for target in "${targets[@]}"; do
+          if [[ "$target" == greptimedb ]]; then
+            image="greptime/greptimedb:$GREPTIMEDB_TAG"
+          else
+            image=$(jq -er '.target_identity.image | select(test("@sha256:[a-f0-9]{64}$"))' \
+              "o11ybench/workloads/tracesbench/catalogs/$target.json")
+          fi
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image"
+          docker image inspect "$image" > "$METADATA_ROOT/$target-image.json"
+          if [[ "$target" == greptimedb ]]; then
+            resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/$target-image.json")
+            printf 'GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV"
+          fi
+        done
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          image="greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG"
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image"
+          docker image inspect "$image" > "$METADATA_ROOT/greptimedb-baseline-image.json"
+          resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/greptimedb-baseline-image.json")
+          printf 'BASELINE_GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV"
+        fi
+    - name: Generate selected dataset once
+      id: generate
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \
+          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
+          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+    - name: Baseline GreptimeDB load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
+          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
+          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
+          --root "$RUN_ROOT/baseline" --data-root "$RUN_ROOT" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+        python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/runs/greptimedb"
+    - name: GreptimeDB load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
+          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
+          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
+          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+        python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/runs/greptimedb"
+    - name: VictoriaTraces load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victoriatraces == 'true' }}
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target victoriatraces --port 19428 \
+          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
+          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
+          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+    - name: Tempo load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.tempo == 'true' }}
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target tempo --port 13200 \
+          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
+          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
+          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+    - name: GreptimeDB version comparison
+      if: always() && inputs.compare_greptimedb
+      run: |
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" >> "$GITHUB_STEP_SUMMARY"
+    - name: Validate and render report
+      env:
+        COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+      run: |
+        comparison_args=()
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          comparison_args+=(--baseline-run "$RUN_ROOT/baseline/runs/greptimedb/run.json")
+        fi
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "$TARGETS" "${comparison_args[@]}" \
+          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
+          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+    - name: Clean up owned containers
+      if: always() && steps.runtime.outcome == 'success'
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
+          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
+          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+    - name: Clean up baseline containers
+      if: always() && steps.runtime.outcome == 'success' && inputs.compare_greptimedb
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
+          --root "$RUN_ROOT/baseline" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+    - name: Benchmark summary
+      if: always()
+      run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY"
+    - name: Load / lifecycle summary
+      if: always()
+      run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+    - name: Baseline load / lifecycle summary
+      if: always() && inputs.compare_greptimedb
+      run: |
+        echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
+        python3 greptimedb/.github/scripts/tracesbench-summary.py \
+          --root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+    - name: Stage benchmark artifacts
+      if: always()
+      run: |
+        mkdir -p "$RUNNER_TEMP/tracesbench-artifacts"
+        if [[ -d "$METADATA_ROOT" ]]; then
+          rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/tracesbench-artifacts/"
+        fi
+        for entry in runs report report.json report.md version-comparison.json version-comparison.md; do
+          if [[ -e "$RUN_ROOT/$entry" ]]; then
+            rsync -a --no-owner --no-group "$RUN_ROOT/$entry" "$RUNNER_TEMP/tracesbench-artifacts/"
+          fi
+        done
+        if [[ -d "$RUN_ROOT/baseline/runs" ]]; then
+          mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/baseline"
+          rsync -a --no-owner --no-group "$RUN_ROOT/baseline/runs" "$RUNNER_TEMP/tracesbench-artifacts/baseline/"
+        fi
+    - name: Upload HTML and results (no corpus)
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: tracesbench-${{ inputs.dataset }}-${{ github.run_id }}-${{ github.run_attempt }}
+        path: ${{ runner.temp }}/tracesbench-artifacts
+        if-no-files-found: warn
+        retention-days: 14
+  teardown:
+    needs:
+    - provision
+    - benchmark
+    if: ${{ always() && (needs.provision.outputs.instance_id != '' || (inputs.provider == 'AWS' && needs.provision.result != 'skipped')) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        path: greptimedb
+        persist-credentials: false
+    - name: Delete ECS and unregister runner
+      if: inputs.provider == 'Aliyun'
+      uses: ./greptimedb/.github/actions/aliyun-ecs-delete
+      with:
+        access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
+        access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
+        github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+        region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
+        instance-id: ${{ needs.provision.outputs.instance_id }}
+        runner-name: ${{ needs.provision.outputs.runner_name }}
+    - uses: aws-actions/configure-aws-credentials@v4
+      if: inputs.provider == 'AWS'
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ vars.EC2_RUNNER_REGION }}
+    - uses: astral-sh/setup-uv@v6
+      if: inputs.provider == 'AWS'
+    - name: Delete EC2 and unregister runner
+      if: inputs.provider == 'AWS'
+      env:
+        GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+        AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
+        AWS_EC2_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }}
+        AWS_EC2_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }}
+        AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+      run: uv run greptimedb/.github/scripts/aws-ec2-runner-teardown.py

.github/workflows/vmbench-long-range.yml

@@ -20,8 +20,18 @@ on:
         - 1mo
         default: 1w
         required: true
+      compare_greptimedb:
+        description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+        type: boolean
+        default: false
+        required: false
+      baseline_greptimedb_tag:
+        description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+        type: string
+        default: v1.2.1
+        required: false
       greptimedb_tag:
-        description: Docker Hub greptime/greptimedb tag
+        description: Candidate Docker Hub greptime/greptimedb tag
         type: string
         default: latest
         required: true
@@ -31,14 +41,20 @@ on:
         default: v1.114.0@sha256:227ad53fd57a4b4430624898a1be0cac39431e1235fa0dc92f567cce38969344
         required: true
       runtime_image:
-        description: Aliyun runtime image (tag or digest reference)
+        description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
         type: string
-        default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260918-545dc570@sha256:0ff03e7e690575e875bb85b3054a03486dd9359073546dc715da177153cf7a29
+        default: auto
         required: true
-      ecs_instance_type:
-        description: ECS instance type (independent of Query Regression)
+      provider:
+        description: Cloud provider for the benchmark runner
+        type: choice
+        options: [Aliyun, AWS]
+        default: Aliyun
+        required: true
+      instance_type:
+        description: 'Instance type; auto = Aliyun: ecs.c9i.2xlarge, AWS: c7i.2xlarge (both 8 vCPU / 16 GiB). Enter a specific type to override.'
         type: string
-        default: ecs.c9i.2xlarge
+        default: auto
         required: true
       benchmark_root:
         description: Absolute disk-backed work directory (outside the o11ybench checkout)
@@ -46,9 +62,9 @@ on:
         default: /home/runner/benchmark-data
         required: true
       system_disk_gib:
-        description: System disk GiB (20..2048)
-        type: number
-        default: 80
+        description: 'System disk GiB: auto (or 0) = 80; enter 20..2048 to override'
+        type: string
+        default: auto
         required: true
       benchmark_timeout_minutes:
         description: Benchmark job timeout in minutes (1..360)
@@ -73,12 +89,15 @@ on:
 permissions:
   contents: read
 env:
+  COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+  BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
   BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
   RANGE_PRESET: ${{ inputs.range_preset }}
   GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
   VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }}
-  RUNTIME_IMAGE: ${{ inputs.runtime_image }}
-  ECS_INSTANCE_TYPE: ${{ inputs.ecs_instance_type }}
+  RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' }}
+  PROVIDER: ${{ inputs.provider }}
+  REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
   SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
   BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }}
   JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
@@ -89,6 +108,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     outputs:
+      instance_type: ${{ steps.runner.outputs.instance_type }}
       system_disk_gib: ${{ steps.plan.outputs.system_disk_gib }}
       runtime_image: ${{ steps.plan.outputs.runtime_image }}
       o11ybench_ref: ${{ steps.plan.outputs.o11ybench_ref }}
@@ -99,8 +119,19 @@ jobs:
       with:
         path: greptimedb
         persist-credentials: false
-    - name: Test existing ECS lifecycle helpers
-      run: python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
+    - name: Validate version comparison
+      env:
+        RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+      run: |
+        python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
+    - name: Resolve CI runner
+      id: runner
+      run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
+    - name: Test CI runner helpers
+      run: |
+        python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
+        python3 greptimedb/tests/perf/test_ci_runner_scripts.py
     - name: Validate budgets and build warm plan
       id: plan
       env:
@@ -108,11 +139,13 @@ jobs:
         RUN_VICTORIAMETRICS: ${{ inputs.run_victoriametrics }}
       run: |
         set -euo pipefail
-        [[ -n "$ECS_INSTANCE_TYPE" ]]
         [[ "$BENCHMARK_ROOT" == /* && "$BENCHMARK_ROOT" != / && "$BENCHMARK_ROOT" != */ ]]
         [[ "$BENCHMARK_ROOT" != /tmp && "$BENCHMARK_ROOT" != /tmp/* ]]
         [[ "$BENCHMARK_ROOT" != *$'\n'* && "$BENCHMARK_ROOT" != *$'\r'* ]]
         [[ "/${BENCHMARK_ROOT#/}/" != */../* && "/${BENCHMARK_ROOT#/}/" != */./* ]]
+        if [[ "$SYSTEM_DISK_GIB" == auto || "$SYSTEM_DISK_GIB" == 0 ]]; then
+          SYSTEM_DISK_GIB=80
+        fi
         for value in "$SYSTEM_DISK_GIB" "$BENCHMARK_TIMEOUT_MINUTES" "$JANITOR_TTL_HOURS"; do
           [[ "$value" =~ ^[1-9][0-9]{0,3}$ ]] || { echo 'Resource budgets must be positive integers'; exit 1; }
         done
@@ -120,8 +153,8 @@ jobs:
         ((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168))
         # Match the existing ECS workflow's provision, teardown and queue headroom.
         ((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120))
-        [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]]
-        docker pull "$RUNTIME_IMAGE"
+        [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]]
+        bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
         metadata=$(docker image inspect "$RUNTIME_IMAGE")
         resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata")
         revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata")
@@ -131,6 +164,9 @@ jobs:
         args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET"
           --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
           --run-classification controlled_server_benchmark --no-open-browser)
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          args+=(--target "greptimedb:name=greptimedb-baseline,image=greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG,url=http://127.0.0.1:14000/v1/prometheus/api/v1/query_range")
+        fi
         # Tags become target-spec fields; reject separators before passing them to the planner.
         for target in greptimedb victoriametrics; do
           case "$target" in
@@ -159,15 +195,16 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 45
     outputs:
-      label: ${{ steps.ecs.outputs.label }}
-      instance_id: ${{ steps.ecs.outputs.instance_id }}
-      runner_name: ${{ steps.ecs.outputs.runner_name }}
+      label: ${{ steps.ecs.outputs.label || steps.aws.outputs.label }}
+      instance_id: ${{ steps.ecs.outputs.instance_id || steps.aws.outputs.instance_id }}
+      runner_name: ${{ steps.ecs.outputs.runner_name || steps.aws.outputs.runner_name }}
     steps:
     - uses: actions/checkout@v4
       with:
         path: greptimedb
         persist-credentials: false
     - name: Create ephemeral ECS runner
+      if: inputs.provider == 'Aliyun'
       id: ecs
       uses: ./greptimedb/.github/actions/aliyun-ecs-create
       with:
@@ -178,13 +215,35 @@ jobs:
         vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
         security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
         image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
-        instance-type: ${{ env.ECS_INSTANCE_TYPE }}
+        instance-type: ${{ needs.validate.outputs.instance_type }}
         enable-docker: 'true'
         system-disk-gib: ${{ needs.validate.outputs.system_disk_gib }}
         ttl-hours: ${{ inputs.janitor_ttl_hours }}
         resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
         runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
         runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
+    - uses: aws-actions/configure-aws-credentials@v4
+      if: inputs.provider == 'AWS'
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ vars.EC2_RUNNER_REGION }}
+    - uses: astral-sh/setup-uv@v6
+      if: inputs.provider == 'AWS'
+    - name: Create ephemeral EC2 runner
+      if: inputs.provider == 'AWS'
+      id: aws
+      env:
+        GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+        AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
+        AWS_EC2_IMAGE_ID: ${{ vars.BENCHMARK_EC2_IMAGE_ID || 'auto' }}
+        AWS_EC2_INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }}
+        AWS_EC2_SUBNET_ID: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+        AWS_EC2_SECURITY_GROUP_ID: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+        AWS_EC2_SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }}
+        AWS_EC2_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
+        AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+      run: uv run greptimedb/.github/scripts/aws-ec2-runner-provision.py
   benchmark:
     needs:
     - validate
@@ -202,6 +261,12 @@ jobs:
       TSDG_BATCH_SIZE: '100000'
       O11YBENCH_GENERATOR_CPUS: '4'
     steps:
+    - uses: actions/checkout@v4
+      with:
+        path: greptimedb
+        persist-credentials: false
+    - name: Set up benchmark dependencies
+      uses: ./greptimedb/.github/actions/setup-benchmark
     - name: Checkout matching benchmark tools
       uses: actions/checkout@v4
       with:
@@ -229,13 +294,13 @@ jobs:
           available=$(df -Pk "$path" | awk 'NR == 2 {print $4}')
           ((available >= 10 * 1024 * 1024)) || { echo "Need at least 10 GiB free on $path"; exit 1; }
         done
-        docker pull "$RESOLVED_RUNTIME_IMAGE"
+        bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE"
         docker image inspect "$RESOLVED_RUNTIME_IMAGE" > "$METADATA_ROOT/runtime-image.json"
         test "$(jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]' "$METADATA_ROOT/runtime-image.json")" = "$RESOLVED_O11YBENCH_REF"
         plan="$METADATA_ROOT/warm.json"
         while IFS=$'\t' read -r target reference; do
           echo "Resolving $target: $reference"
-          docker pull "$reference"
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$reference"
           docker image inspect "$reference" > "$METADATA_ROOT/$target-image.json"
           digest=$(jq -er '.[0].RepoDigests[0]' "$METADATA_ROOT/$target-image.json")
           jq --arg target "$target" --arg image "$digest" '.targets |= map(if .name == $target then .image = $image else . end)' "$plan" > "$plan.tmp"
@@ -259,6 +324,10 @@ jobs:
       if: always()
       run: |
         docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv
+    - name: GreptimeDB version comparison
+      if: always() && inputs.compare_greptimedb
+      run: |
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" >> "$GITHUB_STEP_SUMMARY"
     - name: Stage benchmark artifacts
       if: always()
       run: |
@@ -283,7 +352,7 @@ jobs:
     needs:
     - provision
     - benchmark
-    if: ${{ always() && needs.provision.outputs.instance_id != '' }}
+    if: ${{ always() && (needs.provision.outputs.instance_id != '' || (inputs.provider == 'AWS' && needs.provision.result != 'skipped')) }}
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -292,6 +361,7 @@ jobs:
         path: greptimedb
         persist-credentials: false
     - name: Delete ECS and unregister runner
+      if: inputs.provider == 'Aliyun'
       uses: ./greptimedb/.github/actions/aliyun-ecs-delete
       with:
         access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
@@ -300,3 +370,20 @@ jobs:
         region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
         instance-id: ${{ needs.provision.outputs.instance_id }}
         runner-name: ${{ needs.provision.outputs.runner_name }}
+    - uses: aws-actions/configure-aws-credentials@v4
+      if: inputs.provider == 'AWS'
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ vars.EC2_RUNNER_REGION }}
+    - uses: astral-sh/setup-uv@v6
+      if: inputs.provider == 'AWS'
+    - name: Delete EC2 and unregister runner
+      if: inputs.provider == 'AWS'
+      env:
+        GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+        AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
+        AWS_EC2_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }}
+        AWS_EC2_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }}
+        AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+      run: uv run greptimedb/.github/scripts/aws-ec2-runner-teardown.py

.gitignore

@@ -52,8 +52,6 @@ venv/
 tests-fuzz/artifacts/
 tests-fuzz/corpus/
 
-# cargo-udeps reports
-udeps-report.json
 
 # Nix
 .direnv

AGENTS.md

@@ -25,7 +25,7 @@ provides sub-second querying at PB scale with high cost efficiency.
 | Lint | `make clippy` (= `cargo clippy --workspace --all-targets --all-features -- -D warnings`) |
 | Type check | `make check` |
 
-Toolchain: Rust nightly, Protobuf compiler (>= 3.15), C/C++ build essentials.
+Toolchain: Rust (stable, pinned by `rust-toolchain.toml`), Protobuf compiler (>= 3.15), C/C++ build essentials.
 Install the test runner with `cargo install cargo-nextest --locked`.
 
 ## Repo map

Cargo.toml

@@ -76,7 +76,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "1.3.0-alpha.1"
+version = "1.4.0-alpha.0"
 edition = "2024"
 license = "Apache-2.0"
 
@@ -104,7 +104,6 @@ aquamarine = "0.6"
 arrow = { version = "=59.2.0", features = ["prettyprint"] }
 arrow-array = { version = "=59.2.0", default-features = false, features = ["chrono-tz"] }
 arrow-buffer = "=59.2.0"
-arrow-cast = "=59.2.0"
 arrow-flight = "=59.2.0"
 arrow-ipc = { version = "=59.2.0", default-features = false, features = ["lz4", "zstd"] }
 arrow-schema = { version = "=59.2.0", features = ["serde"] }
@@ -161,7 +160,8 @@ fs2 = "0.4"
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "0c4beffaa2b3968034915867593a717c6ab0c6b4" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "9533078d6fd4bdaddbca0750cb9bf7ad64b2e568" }
+hdfs-native = "0.14.6"
 hex = "0.4"
 hostname = "0.4.0"
 http = "1"
@@ -236,7 +236,6 @@ rskafka = { git = "https://github.com/influxdata/rskafka.git", rev = "80f42eb15e
 rstest = "0.25"
 rstest_reuse = "0.7"
 rust_decimal = "1.33"
-rustc-hash = "2.0"
 rustls = { version = "0.23.25", default-features = false }
 ryu = "1.0"
 sea-query = "0.32"
@@ -273,7 +272,6 @@ url = "2.3"
 # completion" panic (greptimedb#8265 / grpc-rust#2522). Remove once tonic ships a fix.
 tonic = { version = "0.14, <0.14.5", features = ["tls-aws-lc", "gzip", "zstd"] }
 tower = "0.5"
-tower-http = "0.6"
 tracing = "0.1"
 tracing-appender = "0.2"
 tracing-opentelemetry = "0.33.0"
@@ -365,21 +363,21 @@ rev = "f743c3a5fe8c1f57363eac5866c356b7d91f9b39"
 
 [patch.crates-io]
 substrait = { git = "https://github.com/GreptimeTeam/substrait-rs.git", rev = "91ec978b0649417ad3da8390e7a515baec723b1b" }
-datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
-datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
+datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
+datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" }
 datafusion-pg-catalog = { git = "https://github.com/datafusion-contrib/datafusion-postgres.git", rev = "ffb14a52b3d7c7489559812e1ec38096f225ce93" }
 arrow-pg = { git = "https://github.com/datafusion-contrib/datafusion-postgres.git", rev = "ffb14a52b3d7c7489559812e1ec38096f225ce93" }
 sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "9e9019bb1c7040ed956f654e39378dd42ab17884" }

Makefile

@@ -10,7 +10,7 @@ CARGO_BUILD_OPTS := --locked
 IMAGE_REGISTRY ?= docker.io
 IMAGE_NAMESPACE ?= greptime
 IMAGE_TAG ?= latest
-DEV_BUILDER_IMAGE_TAG ?= 2026-03-21-04f07a91-20260921031255
+DEV_BUILDER_IMAGE_TAG ?= 1.98.1-7f0f592b-20261008091949
 DEV_BUILDER_RISCV64_IMAGE_TAG ?= $(DEV_BUILDER_IMAGE_TAG)
 DEV_BUILDER_RISCV64_BASE_IMAGE ?= ubuntu:22.04
 RISCV64_TARGET ?= riscv64gc-unknown-linux-gnu
@@ -212,12 +212,12 @@ sqlness-test: ## Run sqlness test.
 RUNS ?= 1
 FUZZ_TARGET ?= fuzz_alter_table
 .PHONY: fuzz
-fuzz: ## Run fuzz test ${FUZZ_TARGET}.
-	cargo fuzz run ${FUZZ_TARGET} --fuzz-dir tests-fuzz -D -s none -- -runs=${RUNS}
+fuzz: ## Run fuzz test ${FUZZ_TARGET} (requires a nightly toolchain).
+	cargo +nightly fuzz run ${FUZZ_TARGET} --fuzz-dir tests-fuzz -D -s none -- -runs=${RUNS}
 
 .PHONY: fuzz-ls
-fuzz-ls: ## List all fuzz targets.
-	cargo fuzz list --fuzz-dir tests-fuzz
+fuzz-ls: ## List all fuzz targets (requires a nightly toolchain).
+	cargo +nightly fuzz list --fuzz-dir tests-fuzz
 
 .PHONY: check
 check: ## Cargo check all the targets.
@@ -233,14 +233,11 @@ fix-clippy: ## Fix clippy violations.
 
 .PHONY: check-udeps
 check-udeps: ## Check unused dependencies.
-	cargo udeps --workspace --all-targets
+	cargo shear
 
 .PHONY: fix-udeps
 fix-udeps: ## Remove unused dependencies automatically.
-	@echo "Running cargo-udeps to find unused dependencies..."
-	@cargo udeps --workspace --all-targets --output json > udeps-report.json || true
-	@echo "Removing unused dependencies..."
-	@python3 scripts/fix-udeps.py udeps-report.json
+	cargo shear --fix
 
 .PHONY: fmt-check
 fmt-check: ## Check code format.
@@ -264,6 +261,7 @@ stop-etcd: ## Stop single node etcd for testing purpose.
 run-it-in-container: start-etcd ## Run integration tests in dev-builder.
 	docker run --network=host \
 	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v ${CARGO_GIT_CACHE}:/root/.cargo/git -v /tmp:/tmp \
+	-e GT_S3_BUCKET -e GT_S3_ACCESS_KEY_ID -e GT_S3_ACCESS_KEY -e GT_S3_REGION \
 	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:${DEV_BUILDER_IMAGE_TAG} \
 	make test sqlness-test BUILD_JOBS=${BUILD_JOBS}
 

README.md

@@ -168,7 +168,7 @@ Read more in the [full Install Guide](https://docs.greptime.com/getting-started/
 ## Build From Source
 
 **Prerequisites:**
-* [Rust toolchain](https://www.rust-lang.org/tools/install) — nightly, pinned by [`rust-toolchain.toml`](https://github.com/GreptimeTeam/greptimedb/blob/main/rust-toolchain.toml)
+* [Rust toolchain](https://www.rust-lang.org/tools/install) — stable, pinned by [`rust-toolchain.toml`](https://github.com/GreptimeTeam/greptimedb/blob/main/rust-toolchain.toml)
 * [Protobuf compiler](https://grpc.io/docs/protoc-installation/) (>= 3.15)
 * C/C++ building essentials: `gcc` / `g++` / `autoconf` and the glibc dev package (`libc6-dev` on Ubuntu, `glibc-devel` on Fedora)
 * Python toolchain (optional, only for some test scripts)

SECURITY.md

@@ -14,6 +14,9 @@ and cloud platform. If you come across a security vulnerability in GreptimeDB,
 we kindly request that you inform us immediately. We will thoroughly investigate
 all valid reports and make every effort to resolve the issue promptly.
 
-To report any issues or vulnerabilities, please email us at info@greptime.com, rather than 
-posting publicly on GitHub. Be sure to provide us with the version identifier as well as details
+Please do not report security vulnerabilities through public GitHub issues, discussions, or pull requests.
+
+Report them privately through GitHub's [private vulnerability reporting](https://github.com/GreptimeTeam/greptimedb/security/advisories/new)
+(the "Report a vulnerability" button on the repository's Security tab). If you cannot use GitHub,
+email info@greptime.com instead. Be sure to provide us with the version identifier as well as details
 on how the vulnerability can be exploited.

config/config.md

@@ -16,17 +16,16 @@
 | `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, and native histogram columns.<br/>Legacy OTLP summary columns keep their historical `greptime_` prefix. |
 | `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.<br/>When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. |
 | `experimental_metric_export` | Bool | `false` | Enables experimental Parquet COPY DATABASE using shared Metric physical scans.<br/>Resume requires the previous export and storage writes to have ended; HTTP timeout is not confirmation. |
-| `user_provider` | String | Unset | The user provider for authentication.<br/>Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"<br/>Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",<br/>"mysql_native_password:<hex_sha1_sha1_password>",<br/>"pg_scram_sha256:<iterations>:<hex_salt>:<hex_stored_key>:<hex_server_key>"<br/>"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but cannot authenticate over MySQL's<br/>native password handshake; a MySQL client must send the password in cleartext for such users.<br/>"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.<br/>PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or<br/>"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.<br/>For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are<br/>observable in the SCRAM server-first message, and non-default values weaken resistance to username<br/>enumeration. |
-| `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).<br/>Set to 0 to disable the limit. Default: "0" (unlimited) |
-| `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.<br/>Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail" |
+| `user_provider` | String | Unset | The user provider for authentication.<br/>Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"<br/>Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",<br/>"mysql_native_password:<hex_sha1_sha1_password>",<br/>"pg_scram_sha256:<iterations>:<hex_salt>:<hex_stored_key>:<hex_server_key>"<br/>"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:<br/>`static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.<br/>"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.<br/>PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or<br/>"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.<br/>For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are<br/>observable in the SCRAM server-first message, and non-default values weaken resistance to username<br/>enumeration. |
+| `max_in_flight_write_bytes` | String | 0 | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).<br/>Set to 0 to disable the limit. Default: "0" (unlimited) |
+| `write_bytes_exhausted_policy` | String | `wait` | Policy when write bytes quota is exhausted.<br/>Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail" |
 | `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.<br/>By default, it provides services after all regions have been initialized. |
 | `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
-| `max_concurrent_queries` | Integer | `0` | The maximum concurrent queries allowed to be executed. Zero means unlimited. |
 | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. Enabled by default. |
 | `runtime` | -- | -- | The runtime options. |
-| `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
-| `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute compact operations. |
-| `runtime.compact_rt_max_blocking_threads` | Integer | `4` | The maximum number of blocking threads for compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
+| `runtime.global_rt_size` | Integer | Auto | The number of threads to execute the runtime for global read operations.<br/>Defaults to max(num_cpus, 2). |
+| `runtime.compact_rt_size` | Integer | Auto | The number of threads to execute compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
+| `runtime.compact_rt_max_blocking_threads` | Integer | Auto | The maximum number of blocking threads for compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
 | `runtime.experimental_workload_scheduler` | -- | -- | Experimental weighted, work-conserving query/write task scheduler. |
 | `runtime.experimental_workload_scheduler.enable` | Bool | `false` | Enable when concurrent queries and writes interfere with each other—for example, when long-running queries increase ingestion latency.<br/>The weights set their relative runtime shares while both are backlogged. Disabled by default. |
 | `runtime.experimental_workload_scheduler.query_weight` | Integer | `2` | Relative query share while both query and write workloads are backlogged. |
@@ -34,7 +33,7 @@
 | `runtime.experimental_workload_scheduler.sample_every_polls` | Integer | `16` | Number of polls between scheduler fairness samples. Must be greater than zero. |
 | `http` | -- | -- | The HTTP server options. |
 | `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
-| `http.timeout` | String | `0s` | HTTP request timeout. Set to 0 to disable timeout.<br/>When synchronous Prometheus or shared table batching is enabled, a nonzero timeout is<br/>raised to at least the largest active flush interval plus 1 second. The intervals come from<br/>`prom_store.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`. |
+| `http.timeout` | String | `0s` | HTTP request timeout. Set to 0 to disable timeout.<br/>When synchronous Prometheus, OTLP metrics, or ordinary-table batching is enabled, a nonzero timeout is<br/>raised to at least the largest active flush interval plus 1 second. The intervals come from<br/>`pending_rows_batcher.logical_table.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`. |
 | `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
 | `http.enable_cors` | Bool | `true` | HTTP CORS support, it's turned on by default<br/>This allows browser to access http APIs without CORS restrictions |
 | `http.cors_allowed_origins` | Array | Unset | Customize allowed origins for HTTP CORS. |
@@ -43,6 +42,8 @@
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.bind_addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
+| `grpc.enable_cors` | Bool | `false` | Enable CORS for gRPC-Web clients in browsers. Disabled by default. |
+| `grpc.cors_allowed_origins` | Array | Unset | Origins allowed by gRPC CORS. An empty list allows any origin. |
 | `grpc.max_connection_age` | String | Unset | The maximum connection age for gRPC connection.<br/>The value can be a human-readable time string. For example: `10m` for ten minutes or `1h` for one hour.<br/>Refer to https://grpc.io/docs/guides/keepalive/ for more details. |
 | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
 | `grpc.tls.mode` | String | `disable` | TLS mode. |
@@ -75,33 +76,33 @@
 | `influxdb` | -- | -- | InfluxDB protocol options. |
 | `influxdb.enable` | Bool | `true` | Whether to enable InfluxDB protocol in HTTP API. |
 | `influxdb.default_merge_mode` | String | `last_non_null` | Default merge mode for tables automatically created by InfluxDB protocol.<br/>Available values: "last_non_null", "last_row". |
-| `pending_rows_batcher` | -- | -- | Shared experimental ordinary-table batching for opted-in ingestion protocols.<br/>Legacy Prometheus batching settings under prom_store remain supported.<br/>HTTP write protocols sharing this batcher. Omitted or empty disables all entrances.<br/>Supported: influxdb, opentsdb, otlp, logs, loki, splunk, elasticsearch, http_sql, prom.<br/>Prom uses ordinary-table batching without metric engine, otherwise its dedicated batcher.<br/>Effective shared Prom settings take precedence; existing prom_store settings remain compatible. |
+| `pending_rows_batcher` | -- | -- | Ordinary-table batching for opted-in ingestion protocols.<br/>PENDING_ROWS_BATCH_SYNC defaults to true for both batchers. Set it to false to acknowledge<br/>queue admission without waiting for storage; later failures cannot be returned to the client.<br/>Omitted or empty protocols disables batching. Prom without metric engine uses this batcher.<br/>OTLP logs, traces and ordinary metrics use this batcher.<br/>MySQL and PostgreSQL use the same acknowledgement policy. Protocol timeouts are unchanged.<br/>Single-connection writes and INSERT SELECT may incur additional flush waits. |
 | `pending_rows_batcher.pending_rows_flush_interval` | String | `0s` | Flush interval measured from the first pending submission. Zero disables batching. |
 | `pending_rows_batcher.max_batch_rows` | Integer | `100000` | Flush after a complete submission reaches this row threshold. |
 | `pending_rows_batcher.max_concurrent_flushes` | Integer | `256` | Maximum concurrent flushes shared by the frontend batcher. |
-| `pending_rows_batcher.worker_channel_capacity` | Integer | `65526` | Maximum queued submissions per table worker. |
+| `pending_rows_batcher.worker_channel_capacity` | Integer | `65536` | Maximum queued submissions per table worker. |
 | `pending_rows_batcher.max_inflight_requests` | Integer | `3000` | Maximum admitted original requests awaiting completion. |
 | `pending_rows_batcher.flow_notification_queue_capacity` | Integer | `1024` | Maximum number of queued table Flow notifications. |
+| `pending_rows_batcher.logical_table` | -- | -- | Metric-engine logical-table batching for Prom remote write and non-legacy OTLP metrics.<br/>Requires prom_store.with_metric_engine. Logs, traces and legacy metrics are not eligible.<br/>Enable independently with protocols and a nonzero flush interval.<br/>Omitted fields use independent defaults, not parent settings.<br/>Empty protocols or a zero interval disables logical batching without fallback.<br/>Omitting this entire section preserves legacy Prom batching; it does not enable OTLP batching. |
+| `pending_rows_batcher.logical_table.pending_rows_flush_interval` | String | `0s` | Flush interval measured from the first pending submission. Zero disables batching. |
+| `pending_rows_batcher.logical_table.max_batch_rows` | Integer | `100000` | Flush after a complete submission reaches this row threshold. |
+| `pending_rows_batcher.logical_table.max_concurrent_flushes` | Integer | `256` | Maximum concurrent flushes shared by Prom and OTLP metrics. |
+| `pending_rows_batcher.logical_table.worker_channel_capacity` | Integer | `65536` | Maximum queued submissions per physical-table worker. |
+| `pending_rows_batcher.logical_table.max_inflight_requests` | Integer | `3000` | Maximum admitted original requests awaiting completion. |
+| `pending_rows_batcher.logical_table.flow_notification_queue_capacity` | Integer | `1024` | Maximum number of queued logical-table Flow notifications. |
 | `jaeger` | -- | -- | Jaeger protocol options. |
 | `jaeger.enable` | Bool | `true` | Whether to enable Jaeger protocol in HTTP API. |
 | `otlp` | -- | -- | OpenTelemetry protocol options. |
 | `otlp.enable` | Bool | `true` | Whether to enable OpenTelemetry protocol in HTTP API. |
-| `otlp.experimental_enable_exponential_histogram` | Bool | `false` | Experimental: enable cumulative OTLP exponential histogram ingestion. |
 | `otlp.trace_ingest_chunk_size` | Integer | `512` | Maximum spans per trace ingest chunk. Set to 0 to disable splitting. |
+| `otlp.trace_aux_cache_size` | String | Auto | Estimated memory budget for cached trace service/operation keys per frontend,<br/>shared across all catalogs, schemas, and trace tables. Set to 0 to disable caching.<br/>Defaults to 1/128 of the host or pod memory limit, with a minimum of 32MiB.<br/>Uses 32MiB if memory detection is unavailable.<br/>Excludes cache and allocator overhead; shared table names are charged per entry. |
 | `otlp.experimental_enable_resource_info` | Bool | `false` | Whether to synthesize the `greptime_otel_resource_info` table from OTLP metric<br/>resource attributes, so metrics-only services reach the semantic graph. |
 | `prom_store` | -- | -- | Prometheus remote storage options |
 | `prom_store.enable` | Bool | `true` | Whether to enable Prometheus remote write and read in HTTP API. |
 | `prom_store.with_metric_engine` | Bool | `true` | Whether to store the data from Prometheus remote write in metric engine. |
 | `prom_store.prom_validation_mode` | String | `strict` | Whether to enable validation for Prometheus remote write requests.<br/>Available options:<br/>- strict: deny invalid UTF-8 strings (default).<br/>- lossy: allow invalid UTF-8 strings, replace invalid characters with REPLACEMENT_CHARACTER(U+FFFD).<br/>- unchecked: do not valid strings. |
-| `prom_store.experimental_enable_prometheus_native_histogram` | Bool | `false` | Experimental: enable Prometheus remote write v2 native histogram ingestion. |
-| `prom_store.pending_rows_flush_interval` | String | `0s` | Interval to flush pending rows batcher.<br/>Set to "0s" to disable batching mode in Prometheus Remote Write endpoint |
-| `prom_store.max_batch_rows` | Integer | `100000` | Max rows per pending batch before triggering a flush. |
-| `prom_store.max_concurrent_flushes` | Integer | `256` | Max number of concurrent batch flushes. |
-| `prom_store.worker_channel_capacity` | Integer | `65526` | Capacity of the pending batch worker channel. |
-| `prom_store.max_inflight_requests` | Integer | `3000` | Max inflight write requests before backpressure. |
-| `prom_store.flow_notification_queue_capacity` | Integer | `1024` | Maximum number of logical-table flow notifications waiting in the shared queue. |
 | `wal` | -- | -- | The WAL options. |
-| `wal.provider` | String | `raft_engine` | The provider of the WAL.<br/>- `raft_engine`: the wal is stored in the local file system by raft-engine.<br/>- `kafka`: it's remote wal that data is stored in Kafka.<br/>- `experimental_object_store`: the wal is stored as objects in an object store.<br/>**Notes: experimental and not supported yet.** |
+| `wal.provider` | String | `raft_engine` | The provider of the WAL.<br/>- `raft_engine`: the wal is stored in the local file system by raft-engine.<br/>- `kafka`: it's remote wal that data is stored in Kafka.<br/>- `experimental_object_store`: the wal is stored as objects in an object store.<br/>**Notes: experimental.** |
 | `wal.dir` | String | Unset | The directory to store the WAL files.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.file_size` | String | `128MB` | The size of the WAL segment file.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.purge_threshold` | String | `1GB` | The threshold of the WAL size to trigger a purge.<br/>**It's only used when the provider is `raft_engine`**. |
@@ -111,7 +112,7 @@
 | `wal.enable_log_recycle` | Bool | `true` | Whether to reuse logically truncated log files.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.prefill_log_files` | Bool | `false` | Whether to pre-create log files on start up.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.sync_period` | String | `5s` | Duration for fsyncing log files.<br/>**It's only used when the provider is `raft_engine`**. |
-| `wal.recovery_parallelism` | Integer | `2` | Parallelism during WAL recovery. |
+| `wal.recovery_parallelism` | Integer | Auto | Parallelism during WAL recovery. Defaults to the number of CPU cores; this example sets it to 2. |
 | `wal.broker_endpoints` | Array | -- | The Kafka broker endpoints.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.connect_timeout` | String | `3s` | The connect timeout for kafka client.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.timeout` | String | `5s` | The total request timeout for kafka client.<br/>**It's only used when the provider is `kafka`**. |
@@ -126,7 +127,10 @@
 | `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.<br/>**It's only used when the provider is `kafka`**.<br/><br/>This option ensures that when Kafka messages are deleted, the system<br/>can still successfully replay memtable data without throwing an<br/>out-of-range error.<br/>However, enabling this option might lead to unexpected data loss,<br/>as the system will skip over missing entries instead of treating<br/>them as critical errors. |
 | `wal.storage_provider` | String | `""` | The name of the storage provider that holds the WAL objects, an empty name selects the default object store.<br/>**It's only used when the provider is `experimental_object_store`**. |
 | `wal.prefix` | String | `wal` | The path prefix of the WAL objects inside the storage provider.<br/>The objects are written under `<prefix>/datanodes/<node_id>/epochs/<generation>`, which is derived from this prefix.<br/>**It's only used when the provider is `experimental_object_store`**. |
-| `wal.flush_interval` | String | `100ms` | The interval of flushing buffered entries to the object store, at least `10ms`, defaults to `100ms`.<br/>Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so under sustained load the batch seals on size and the interval no longer matters.<br/>When writes are sparse, a shorter interval lowers the acknowledgement latency of appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node.<br/>**It's only used when the provider is `experimental_object_store`**. |
+| `wal.flush_interval` | String | `100ms` | The interval of flushing buffered entries to the object store, at least `10ms`, defaults to `100ms`.<br/>Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so under sustained load the batch seals on size and the interval no longer matters.<br/>When writes are sparse, a shorter interval lowers the acknowledgement latency of `durable` appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node, and in `enqueued` mode the backlog thresholds can seal earlier. An `enqueued` append does not wait for its batch to seal once it is admitted, but the backlog thresholds can delay admission.<br/>**It's only used when the provider is `experimental_object_store`**. |
+| `wal.ack_mode` | String | `durable` | When an append to the object store WAL returns.<br/>- `durable`: an append returns after the object holding its entries is durable (the default).<br/>- `enqueued`: an append returns once its entries are admitted and their ids are assigned, and the object is created in the background. A crash loses the unpersisted backlog.<br/>**It's only used when the provider is `experimental_object_store`**. |
+| `wal.max_unpersisted_bytes` | String | `64MB` | The size of the unpersisted backlog at which new appends stall until an upload completes, in `enqueued` mode.<br/>Nothing is dropped and nothing is rejected; the threshold does not bound what a crash during an outage can lose.<br/>**It's only used when the provider is `experimental_object_store` and `ack_mode` is `enqueued`**. |
+| `wal.max_unpersisted_age` | String | `8s` | The age of the oldest unpersisted entry at which new appends stall until an upload completes, in `enqueued` mode.<br/>**It's only used when the provider is `experimental_object_store` and `ack_mode` is `enqueued`**. |
 | `wal.on_corrupted_segment` | String | `skip` | What a read does with a segment that still does not decode after a second fetch, because its checksum does not match or its content disagrees with its footer entry.<br/>- `skip`: the segment is skipped and recorded as a WAL hole of its region, a metric is incremented and a warning is logged; the other regions of the object are unaffected (the default).<br/>- `fail`: the read fails, so the region does not open.<br/>**It's only used when the provider is `experimental_object_store`**. |
 | `metadata_store` | -- | -- | Metadata storage options. |
 | `metadata_store.file_size` | String | `64MB` | The size of the metadata store log file. |
@@ -136,11 +140,9 @@
 | `procedure.max_retry_times` | Integer | `3` | Procedure max retry time. |
 | `procedure.retry_delay` | String | `500ms` | Initial retry delay of procedures, increases exponentially |
 | `procedure.max_running_procedures` | Integer | `128` | Max running procedures.<br/>The maximum number of procedures that can be running at the same time.<br/>If the number of running procedures exceeds this limit, the procedure will be rejected. |
-| `flow` | -- | -- | flow engine options. |
-| `flow.num_workers` | Integer | `0` | The number of flow worker in flownode.<br/>Not setting(or set to 0) this value will use the number of CPU cores divided by 2. |
 | `query` | -- | -- | The query engine options. |
 | `query.parallelism` | Integer | `0` | Parallelism of the query engine.<br/>Default to 0, which means the number of CPU cores. |
-| `query.memory_pool_size` | String | `50%` | Memory pool size for query execution operators (aggregation, sorting, join).<br/>Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%").<br/>Setting it to 0 disables the limit (unbounded, default behavior).<br/>When this limit is reached, queries will fail with ResourceExhausted error.<br/>NOTE: This does NOT limit memory used by table scans. |
+| `query.memory_pool_size` | String | 0 | Memory pool size for query execution operators (aggregation, sorting, join).<br/>Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%").<br/>Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.<br/>When this limit is reached, queries will fail with ResourceExhausted error.<br/>NOTE: This does NOT limit memory used by table scans. |
 | `query.experimental_memory_pool_policy` | String | `greedy` | Experimental memory pool allocation policy:<br/>- "greedy" (default): first-come-first-served allocation; preserves current behavior.<br/>- "fair": divides available memory among spillable operators and may spill earlier.<br/>Only effective when `memory_pool_size` is bounded (>0). |
 | `query.experimental_spill_mode` | String | `default` | Spill mode:<br/>- "default": preserve DataFusion built-in OS temp directory (default).<br/>- "custom": explicitly configure spill path, quota, and compression.<br/>- "disabled": explicitly disable disk spilling.<br/>Set this to "custom" before using the path/quota/compression keys below. |
 | `query.experimental_spill_path` | String | Unset | Spill directory path. Ignored unless mode is "custom". |
@@ -149,9 +151,11 @@
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `./greptimedb_data` | The working home directory. |
 | `storage.copy_root` | String | `./greptimedb_data/copy` | Root directory for standalone SQL access to local files.<br/>Relative SQL paths are resolved below this directory. Absolute paths are accepted only when<br/>they are inside this directory. Defaults to `<data_home>/copy`.<br/>Distributed deployments always reject SQL access to local files.<br/>Upgrade note: COPY commands and existing external tables that reference paths outside this<br/>directory will fail. Move those files below the copy root, set this option to a dedicated<br/>directory containing them, or migrate the files to object storage before upgrading. |
-| `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
+| `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS.<br/>- `Hdfs`: the data is stored in the Hadoop Distributed File System. |
 | `storage.bucket` | String | Unset | The S3 bucket name.<br/>**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. |
-| `storage.root` | String | Unset | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.<br/>**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. |
+| `storage.root` | String | Unset | The directory or object prefix under which data is stored.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs`, `Azblob` and `Hdfs`**. |
+| `storage.name_node` | String | Unset | The HDFS NameNode URI, for example, `hdfs://127.0.0.1:9000`.<br/>**It's only used when the storage type is `Hdfs`**. |
+| `storage.options` | InlineTable | Unset | Additional options passed to the native HDFS client.<br/>**It's only used when the storage type is `Hdfs`**. |
 | `storage.access_key_id` | String | Unset | The access key id of the aws account.<br/>It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.<br/>**It's only used when the storage type is `S3` and `Oss`**. |
 | `storage.secret_access_key` | String | Unset | The secret access key of the aws account.<br/>It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.<br/>**It's only used when the storage type is `S3`**. |
 | `storage.access_key_secret` | String | Unset | The secret access key of the aliyun account.<br/>**It's only used when the storage type is `Oss`**. |
@@ -172,12 +176,13 @@
 | `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification<br/>**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. |
 | `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines.<br/>Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail. |
 | `region_engine.mito` | -- | -- | The Mito engine options. |
-| `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. |
+| `region_engine.mito.num_workers` | Integer | Auto | Number of region workers. Defaults to half of the available CPU cores, rounded up. |
 | `region_engine.mito.worker_channel_size` | Integer | `128` | Request channel size of each worker. |
 | `region_engine.mito.worker_request_batch_size` | Integer | `64` | Max batch size for a worker to handle requests. |
 | `region_engine.mito.manifest_checkpoint_distance` | Integer | `10` | Number of meta action updated to trigger a new checkpoint for the manifest. |
 | `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
 | `region_engine.mito.experimental_enable_series_index` | Bool | `false` | Under development; do not enable. Whether to enable series indexes.<br/>Indexes are stored on the local filesystem under `{data_home}/series_index`. |
+| `region_engine.mito.experimental_series_index_max_size` | String | `5GiB` | Approximate series and range index size limit in open regions, shared across workers.<br/>Workers periodically refresh usage and skip maintenance when full. In-flight reconciliation<br/>can exceed the limit. Closed-region files, temporary output, catalogs, and old snapshots<br/>retained by readers are not counted.<br/>Minimum: 1KiB. Takes effect on restart. |
 | `region_engine.mito.experimental_enable_range_index` | Bool | `false` | Whether to build and query range indexes when series indexes are enabled.<br/>Obsolete range-index metadata and files are still cleaned up when disabled. |
 | `region_engine.mito.experimental_series_index_maintenance_interval` | String | `5m` | Interval between series-index maintenance runs. Zero uses the default of 5 min. |
 | `region_engine.mito.experimental_series_index_bucket_width` | String | `5days` | Requested minimum series-index bucket width (default: 5 days), rounded up to<br/>an exact multiple of each region's compaction time window. |
@@ -212,7 +217,7 @@
 | `region_engine.mito.min_compaction_interval` | String | `0m` | Minimum time interval between two compactions.<br/>To align with the old behavior, the default value is 0 (no restrictions). |
 | `region_engine.mito.schedule_compaction_after_edit` | Bool | `true` | Whether to allow to schedule a compaction after a successful region edit.<br/><br/>Setting this to "true" is a necessary but not sufficient condition for scheduling compaction after a region edit.<br/>Other constraints, such as "min_compaction_interval", may still prevent compaction from being scheduled.<br/>Setting this to "false", however, guarantees that compaction will not be scheduled after a region edit. |
 | `region_engine.mito.default_flat_format` | Bool | `true` | Whether to enable flat format as the default SST format. |
-| `region_engine.mito.experimental_series_scan_v2` | Bool | `true` | Whether to enable the experimental two-phase mode for eligible metric series scans. |
+| `region_engine.mito.experimental_series_scan_v2` | Bool | `false` | Whether to enable the experimental two-phase mode for eligible metric series scans. |
 | `region_engine.mito.index` | -- | -- | The options for index in Mito engine. |
 | `region_engine.mito.index.aux_path` | String | `""` | Auxiliary directory path for the index in filesystem, used to store intermediate files for<br/>creating the index and staging files for searching the index, defaults to `{data_home}/index_intermediate`.<br/>The default name for this directory is `index_intermediate` for backward compatibility.<br/><br/>This path contains two subdirectories:<br/>- `__intm`: for storing intermediate files used during creating index.<br/>- `staging`: for storing staging files used during searching index. |
 | `region_engine.mito.index.staging_size` | String | `2GB` | The max capacity of the staging directory. |
@@ -255,17 +260,18 @@
 | `logging.tracing_sample_ratio` | -- | Unset | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
 | `slow_query` | -- | -- | The slow query log options. |
-| `slow_query.enable` | Bool | `false` | Whether to enable slow query log. |
-| `slow_query.record_type` | String | Unset | The record type of slow queries. It can be `system_table` or `log`. |
-| `slow_query.threshold` | String | Unset | The threshold of slow query. |
-| `slow_query.sample_ratio` | Float | Unset | The sampling ratio of slow query log. The value should be in the range of (0, 1]. |
+| `slow_query.enable` | Bool | `true` | Whether to enable slow query log. |
+| `slow_query.record_type` | String | `system_table` | The record type of slow queries. It can be `system_table` or `log`.<br/>If `system_table` is selected, the slow queries will be recorded in a system table `greptime_private.slow_queries`.<br/>If `log` is selected, the slow queries will be logged in a log file `greptimedb-slow-queries.*`. |
+| `slow_query.threshold` | String | `30s` | The threshold of slow query. It can be human readable time string, for example: `10s`, `100ms`, `1s`. |
+| `slow_query.sample_ratio` | Float | `1.0` | The sampling ratio of slow query log. The value should be in the range of (0, 1]. For example, `0.1` means 10% of the slow queries will be logged and `1.0` means all slow queries will be logged. |
+| `slow_query.ttl` | String | `90d` | The TTL of the `slow_queries` system table. Default is `90d` when `record_type` is `system_table`. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
 | `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
 | `pipeline` | -- | -- | The pipeline options. |
 | `pipeline.cache_ttl` | String | `10s` | Time to live of the local pipeline cache. Default is `10s`. |
 | `event_recorder` | -- | -- | Configuration options for the event recorder. |
 | `event_recorder.ttl` | String | `90d` | TTL for the events table that will be used to store the events. Default is `90d`. |
-| `event_recorder.event_types` | Array | -- | Event types to record. Current available event types: `create_database`,<br/>`alter_database`, `drop_database`, `create_flow`, `drop_flow`,<br/>`create_table`, `create_logical_tables`, `alter_table`, `alter_logical_tables`,<br/>`drop_table`, `undrop_table`, `purge_dropped_table`, `truncate_table`,<br/>`create_view`, `drop_view`, `admin_function`, `reconcile_catalog`,<br/>`reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.<br/>When omitted, all current and future event types are recorded.<br/>Set to an empty array to disable event recording. |
+| `event_recorder.event_types` | Array | All | Event types to record. Current available event types: `create_database`,<br/>`alter_database`, `drop_database`, `create_flow`, `drop_flow`,<br/>`create_table`, `create_logical_tables`, `alter_table`, `alter_logical_tables`,<br/>`drop_table`, `undrop_table`, `purge_dropped_table`, `truncate_table`,<br/>`create_view`, `drop_view`, `admin_function`, `reconcile_catalog`,<br/>`reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.<br/>When omitted, all current and future event types are recorded.<br/>Set to an empty array to disable event recording. |
 | `memory` | -- | -- | The memory options. |
 | `memory.enable_heap_profiling` | Bool | `true` | Whether to enable heap profiling activation during startup.<br/>When enabled, heap profiling will be activated if the `MALLOC_CONF` environment variable<br/>is set to "prof:true,prof_active:false". The official image adds this env variable.<br/>Default is true. |
 
@@ -280,16 +286,16 @@
 | `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, and native histogram columns.<br/>Legacy OTLP summary columns keep their historical `greptime_` prefix. |
 | `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.<br/>When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. |
 | `experimental_metric_export` | Bool | `false` | Enables experimental Parquet COPY DATABASE using shared Metric physical scans.<br/>Resume requires the previous export and storage writes to have ended; HTTP timeout is not confirmation. |
-| `user_provider` | String | Unset | The user provider for authentication.<br/>Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"<br/>Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",<br/>"mysql_native_password:<hex_sha1_sha1_password>",<br/>"pg_scram_sha256:<iterations>:<hex_salt>:<hex_stored_key>:<hex_server_key>"<br/>"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but cannot authenticate over MySQL's<br/>native password handshake; a MySQL client must send the password in cleartext for such users.<br/>"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.<br/>PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or<br/>"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.<br/>For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are<br/>observable in the SCRAM server-first message, and non-default values weaken resistance to username<br/>enumeration. |
-| `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).<br/>Set to 0 to disable the limit. Default: "0" (unlimited) |
-| `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.<br/>Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail" |
+| `user_provider` | String | Unset | The user provider for authentication.<br/>Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"<br/>Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",<br/>"mysql_native_password:<hex_sha1_sha1_password>",<br/>"pg_scram_sha256:<iterations>:<hex_salt>:<hex_stored_key>:<hex_server_key>"<br/>"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:<br/>`static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.<br/>"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.<br/>PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or<br/>"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.<br/>For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are<br/>observable in the SCRAM server-first message, and non-default values weaken resistance to username<br/>enumeration. |
+| `max_in_flight_write_bytes` | String | 0 | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).<br/>Set to 0 to disable the limit. Default: "0" (unlimited) |
+| `write_bytes_exhausted_policy` | String | `wait` | Policy when write bytes quota is exhausted.<br/>Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail" |
 | `runtime` | -- | -- | The runtime options. |
-| `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
-| `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute compact operations. |
-| `runtime.compact_rt_max_blocking_threads` | Integer | `4` | The maximum number of blocking threads for compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
+| `runtime.global_rt_size` | Integer | Auto | The number of threads to execute the runtime for global read operations.<br/>Defaults to max(num_cpus, 2). |
+| `runtime.compact_rt_size` | Integer | Auto | The number of threads to execute compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
+| `runtime.compact_rt_max_blocking_threads` | Integer | Auto | The maximum number of blocking threads for compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
 | `http` | -- | -- | The HTTP server options. |
 | `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
-| `http.timeout` | String | `0s` | HTTP request timeout. Set to 0 to disable timeout.<br/>When synchronous Prometheus or shared table batching is enabled, a nonzero timeout is<br/>raised to at least the largest active flush interval plus 1 second. The intervals come from<br/>`prom_store.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`. |
+| `http.timeout` | String | `0s` | HTTP request timeout. Set to 0 to disable timeout.<br/>When synchronous Prometheus, OTLP metrics, or ordinary-table batching is enabled, a nonzero timeout is<br/>raised to at least the largest active flush interval plus 1 second. The intervals come from<br/>`pending_rows_batcher.logical_table.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`. |
 | `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
 | `http.enable_cors` | Bool | `true` | HTTP CORS support, it's turned on by default<br/>This allows browser to access http APIs without CORS restrictions |
 | `http.cors_allowed_origins` | Array | Unset | Customize allowed origins for HTTP CORS. |
@@ -297,20 +303,22 @@
 | `http.api_server_addr` | String | `127.0.0.1:4006` | The address to bind the dedicated HTTP API server, in the same form as `addr`.<br/>Defaults to `127.0.0.1:4006`. |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.bind_addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
-| `grpc.server_addr` | String | `127.0.0.1:4001` | The address advertised to the metasrv, and used for connections from outside the host.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `grpc.bind_addr`. |
+| `grpc.server_addr` | String | Unset | The address advertised to the metasrv, and used for connections from outside the host.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `grpc.bind_addr`. |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
-| `grpc.flight_compression` | String | `arrow_ipc` | Compression mode for frontend side Arrow IPC service. Available options:<br/>- `none`: disable all compression<br/>- `transport`: only enable gRPC transport compression (zstd)<br/>- `arrow_ipc`: only enable Arrow IPC compression (lz4)<br/>- `all`: enable all compression.<br/>Default to `none` |
+| `grpc.flight_compression` | String | `arrow_ipc` | Compression mode for frontend side Arrow IPC service. Available options:<br/>- `none`: disable all compression<br/>- `transport`: only enable gRPC transport compression (zstd)<br/>- `arrow_ipc`: only enable Arrow IPC compression (lz4)<br/>- `all`: enable all compression.<br/>Default to `arrow_ipc`. |
+| `grpc.enable_cors` | Bool | `false` | Enable CORS for gRPC-Web clients in browsers. Disabled by default. |
+| `grpc.cors_allowed_origins` | Array | Unset | Origins allowed by gRPC CORS. An empty list allows any origin. |
 | `grpc.max_connection_age` | String | Unset | The maximum connection age for gRPC connection.<br/>The value can be a human-readable time string. For example: `10m` for ten minutes or `1h` for one hour.<br/>Refer to https://grpc.io/docs/guides/keepalive/ for more details. |
 | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
 | `grpc.tls.mode` | String | `disable` | TLS mode. |
 | `grpc.tls.cert_path` | String | Unset | Certificate file path. |
 | `grpc.tls.key_path` | String | Unset | Private key file path. |
 | `grpc.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload.<br/>For now, gRPC tls config does not support auto reload. |
-| `internal_grpc` | -- | -- | The internal gRPC server options. Internal gRPC port for nodes inside cluster to access frontend. |
-| `internal_grpc.bind_addr` | String | `127.0.0.1:4010` | The address to bind the gRPC server. |
-| `internal_grpc.server_addr` | String | `127.0.0.1:4010` | The address advertised to the metasrv, and used for connections from outside the host.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `internal_grpc.bind_addr`. |
+| `internal_grpc` | -- | -- | The internal gRPC server options. Internal gRPC port for nodes inside cluster to access frontend.<br/>The internal gRPC server starts only when this section is set. |
+| `internal_grpc.bind_addr` | String | Unset | The address to bind the gRPC server.<br/>If unset, it falls back to `127.0.0.1:4001`, the same address as `grpc.bind_addr`. |
+| `internal_grpc.server_addr` | String | Unset | The address advertised to the metasrv, and used for connections from outside the host.<br/>Unlike `grpc.server_addr`, it is not detected automatically; set it explicitly. |
 | `internal_grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
-| `internal_grpc.flight_compression` | String | `arrow_ipc` | Compression mode for frontend side Arrow IPC service. Available options:<br/>- `none`: disable all compression<br/>- `transport`: only enable gRPC transport compression (zstd)<br/>- `arrow_ipc`: only enable Arrow IPC compression (lz4)<br/>- `all`: enable all compression.<br/>Default to `none` |
+| `internal_grpc.flight_compression` | String | `arrow_ipc` | Compression mode for frontend side Arrow IPC service. Available options:<br/>- `none`: disable all compression<br/>- `transport`: only enable gRPC transport compression (zstd)<br/>- `arrow_ipc`: only enable Arrow IPC compression (lz4)<br/>- `all`: enable all compression.<br/>Default to `arrow_ipc`. |
 | `internal_grpc.tls` | -- | -- | internal gRPC server TLS options, see `mysql.tls` section. |
 | `internal_grpc.tls.mode` | String | `disable` | TLS mode. |
 | `internal_grpc.tls.cert_path` | String | Unset | Certificate file path. |
@@ -342,31 +350,31 @@
 | `influxdb` | -- | -- | InfluxDB protocol options. |
 | `influxdb.enable` | Bool | `true` | Whether to enable InfluxDB protocol in HTTP API. |
 | `influxdb.default_merge_mode` | String | `last_non_null` | Default merge mode for tables automatically created by InfluxDB protocol.<br/>Available values: "last_non_null", "last_row". |
-| `pending_rows_batcher` | -- | -- | Shared experimental ordinary-table batching for opted-in ingestion protocols.<br/>Legacy Prometheus batching settings under prom_store remain supported.<br/>HTTP write protocols sharing this batcher. Omitted or empty disables all entrances.<br/>Supported: influxdb, opentsdb, otlp, logs, loki, splunk, elasticsearch, http_sql, prom.<br/>Prom uses ordinary-table batching without metric engine, otherwise its dedicated batcher.<br/>Effective shared Prom settings take precedence; existing prom_store settings remain compatible. |
+| `pending_rows_batcher` | -- | -- | Ordinary-table batching for opted-in ingestion protocols.<br/>PENDING_ROWS_BATCH_SYNC defaults to true for both batchers. Set it to false to acknowledge<br/>queue admission without waiting for storage; later failures cannot be returned to the client.<br/>Omitted or empty protocols disables batching. Prom without metric engine uses this batcher.<br/>OTLP logs, traces and ordinary metrics use this batcher.<br/>MySQL and PostgreSQL use the same acknowledgement policy. Protocol timeouts are unchanged.<br/>Single-connection writes and INSERT SELECT may incur additional flush waits. |
 | `pending_rows_batcher.pending_rows_flush_interval` | String | `0s` | Flush interval measured from the first pending submission. Zero disables batching. |
 | `pending_rows_batcher.max_batch_rows` | Integer | `100000` | Flush after a complete submission reaches this row threshold. |
 | `pending_rows_batcher.max_concurrent_flushes` | Integer | `256` | Maximum concurrent flushes shared by the frontend batcher. |
-| `pending_rows_batcher.worker_channel_capacity` | Integer | `65526` | Maximum queued submissions per table worker. |
+| `pending_rows_batcher.worker_channel_capacity` | Integer | `65536` | Maximum queued submissions per table worker. |
 | `pending_rows_batcher.max_inflight_requests` | Integer | `3000` | Maximum admitted original requests awaiting completion. |
 | `pending_rows_batcher.flow_notification_queue_capacity` | Integer | `1024` | Maximum number of queued table Flow notifications. |
+| `pending_rows_batcher.logical_table` | -- | -- | Metric-engine logical-table batching for Prom remote write and non-legacy OTLP metrics.<br/>Requires prom_store.with_metric_engine. Logs, traces and legacy metrics are not eligible.<br/>Enable independently with protocols and a nonzero flush interval.<br/>Omitted fields use independent defaults, not parent settings.<br/>Empty protocols or a zero interval disables logical batching without fallback.<br/>Omitting this entire section preserves legacy Prom batching; it does not enable OTLP batching. |
+| `pending_rows_batcher.logical_table.pending_rows_flush_interval` | String | `0s` | Flush interval measured from the first pending submission. Zero disables batching. |
+| `pending_rows_batcher.logical_table.max_batch_rows` | Integer | `100000` | Flush after a complete submission reaches this row threshold. |
+| `pending_rows_batcher.logical_table.max_concurrent_flushes` | Integer | `256` | Maximum concurrent flushes shared by Prom and OTLP metrics. |
+| `pending_rows_batcher.logical_table.worker_channel_capacity` | Integer | `65536` | Maximum queued submissions per physical-table worker. |
+| `pending_rows_batcher.logical_table.max_inflight_requests` | Integer | `3000` | Maximum admitted original requests awaiting completion. |
+| `pending_rows_batcher.logical_table.flow_notification_queue_capacity` | Integer | `1024` | Maximum number of queued logical-table Flow notifications. |
 | `jaeger` | -- | -- | Jaeger protocol options. |
 | `jaeger.enable` | Bool | `true` | Whether to enable Jaeger protocol in HTTP API. |
 | `otlp` | -- | -- | OpenTelemetry protocol options. |
 | `otlp.enable` | Bool | `true` | Whether to enable OpenTelemetry protocol in HTTP API. |
-| `otlp.experimental_enable_exponential_histogram` | Bool | `false` | Experimental: enable cumulative OTLP exponential histogram ingestion. |
 | `otlp.trace_ingest_chunk_size` | Integer | `512` | Maximum spans per trace ingest chunk. Set to 0 to disable splitting. |
+| `otlp.trace_aux_cache_size` | String | Auto | Estimated memory budget for cached trace service/operation keys per frontend,<br/>shared across all catalogs, schemas, and trace tables. Set to 0 to disable caching.<br/>Defaults to 1/128 of the host or pod memory limit, with a minimum of 32MiB.<br/>Uses 32MiB if memory detection is unavailable.<br/>Excludes cache and allocator overhead; shared table names are charged per entry. |
 | `otlp.experimental_enable_resource_info` | Bool | `false` | Whether to synthesize the `greptime_otel_resource_info` table from OTLP metric<br/>resource attributes, so metrics-only services reach the semantic graph. |
 | `prom_store` | -- | -- | Prometheus remote storage options |
 | `prom_store.enable` | Bool | `true` | Whether to enable Prometheus remote write and read in HTTP API. |
 | `prom_store.with_metric_engine` | Bool | `true` | Whether to store the data from Prometheus remote write in metric engine. |
 | `prom_store.prom_validation_mode` | String | `strict` | Whether to enable validation for Prometheus remote write requests.<br/>Available options:<br/>- strict: deny invalid UTF-8 strings (default).<br/>- lossy: allow invalid UTF-8 strings, replace invalid characters with REPLACEMENT_CHARACTER(U+FFFD).<br/>- unchecked: do not valid strings. |
-| `prom_store.experimental_enable_prometheus_native_histogram` | Bool | `false` | Experimental: enable Prometheus remote write v2 native histogram ingestion. |
-| `prom_store.pending_rows_flush_interval` | String | `0s` | Interval to flush pending rows batcher.<br/>Set to "0s" to disable batching mode in Prometheus Remote Write endpoint |
-| `prom_store.max_batch_rows` | Integer | `100000` | Max rows per pending batch before triggering a flush. |
-| `prom_store.max_concurrent_flushes` | Integer | `256` | Max number of concurrent batch flushes. |
-| `prom_store.worker_channel_capacity` | Integer | `65526` | Capacity of the pending batch worker channel. |
-| `prom_store.max_inflight_requests` | Integer | `3000` | Max inflight write requests before backpressure. |
-| `prom_store.flow_notification_queue_capacity` | Integer | `1024` | Maximum number of logical-table flow notifications waiting in the shared queue. |
 | `meta_client` | -- | -- | The metasrv client options. |
 | `meta_client.metasrv_addrs` | Array | -- | The addresses of the metasrv. |
 | `meta_client.timeout` | String | `3s` | Operation timeout. |
@@ -379,7 +387,7 @@
 | `query` | -- | -- | The query engine options. |
 | `query.parallelism` | Integer | `0` | Parallelism of the query engine.<br/>Default to 0, which means the number of CPU cores. |
 | `query.allow_query_fallback` | Bool | `false` | Whether to allow query fallback when push down optimize fails.<br/>Default to false, meaning when push down optimize failed, return error msg |
-| `query.memory_pool_size` | String | `50%` | Memory pool size for query execution operators (aggregation, sorting, join).<br/>Supports absolute size (e.g., "4GB", "8GB") or percentage of system memory (e.g., "30%").<br/>Setting it to 0 disables the limit (unbounded, default behavior).<br/>When this limit is reached, queries will fail with ResourceExhausted error.<br/>NOTE: This does NOT limit memory used by table scans (only applies to datanodes). |
+| `query.memory_pool_size` | String | 0 | Memory pool size for query execution operators (aggregation, sorting, join).<br/>Supports absolute size (e.g., "4GB", "8GB") or percentage of system memory (e.g., "30%").<br/>Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.<br/>When this limit is reached, queries will fail with ResourceExhausted error.<br/>NOTE: This does NOT limit memory used by table scans (only applies to datanodes). |
 | `query.experimental_memory_pool_policy` | String | `greedy` | Experimental memory pool allocation policy:<br/>- "greedy" (default): first-come-first-served allocation; preserves current behavior.<br/>- "fair": divides available memory among spillable operators and may spill earlier.<br/>Only effective when `memory_pool_size` is bounded (>0). |
 | `query.experimental_spill_mode` | String | `default` | Spill mode:<br/>- "default": preserve DataFusion built-in OS temp directory (default).<br/>- "custom": explicitly configure spill path, quota, and compression.<br/>- "disabled": explicitly disable disk spilling.<br/>Set this to "custom" before using the path/quota/compression keys below. |
 | `query.experimental_spill_path` | String | Unset | Spill directory path. Ignored unless mode is "custom". |
@@ -387,7 +395,7 @@
 | `query.experimental_spill_compression` | String | `uncompressed` | Compression for spilled data files: "uncompressed" (default), "lz4_frame", "zstd".<br/>Ignored unless mode is "custom". |
 | `datanode` | -- | -- | Datanode options. |
 | `datanode.client` | -- | -- | Datanode client options. |
-| `datanode.client.connect_timeout` | String | `10s` | -- |
+| `datanode.client.connect_timeout` | String | 1s | Connect server timeout. Defaults to `1s`; this example sets it to `10s`. |
 | `datanode.client.tcp_nodelay` | Bool | `true` | -- |
 | `datanode.client.max_recv_message_size` | String | `512MB` | The maximum receive message size for the gRPC client. |
 | `datanode.client.max_send_message_size` | String | `512MB` | The maximum send message size for the gRPC client. |
@@ -420,7 +428,7 @@
 | `pipeline.cache_ttl` | String | `10s` | Time to live of the frontend-local pipeline cache. A pipeline created or deleted on<br/>another frontend takes effect on this one after at most this duration. |
 | `event_recorder` | -- | -- | Configuration options for the event recorder. |
 | `event_recorder.ttl` | String | `90d` | TTL for the events table that will be used to store the events. Default is `90d`. |
-| `event_recorder.event_types` | Array | -- | Event types to record. Current available event type: `admin_function`.<br/>When omitted, all current and future event types are recorded.<br/>Set to an empty array to disable event recording. |
+| `event_recorder.event_types` | Array | All | Event types to record. Current available event type: `admin_function`.<br/>When omitted, all current and future event types are recorded.<br/>Set to an empty array to disable event recording. |
 
 
 ### Metasrv
@@ -432,7 +440,7 @@
 | `store_key_prefix` | String | `""` | If it's not empty, the metasrv will store all data with this key prefix. |
 | `backend` | String | `etcd_store` | The datastore for meta server.<br/>Available values:<br/>- `etcd_store` (default value)<br/>- `memory_store`<br/>- `postgres_store`<br/>- `mysql_store` |
 | `meta_table_name` | String | `greptime_metakv` | Table name in RDS to store metadata. Effect when using a RDS kvbackend.<br/>**Only used when backend is `postgres_store`.** |
-| `meta_schema_name` | String | `greptime_schema` | Optional PostgreSQL schema for metadata table and election table name qualification.<br/>When PostgreSQL public schema is not writable (e.g., PostgreSQL 15+ with restricted public),<br/>set this to a writable schema. GreptimeDB will use `meta_schema_name`.`meta_table_name`.<br/>**Only used when backend is `postgres_store`.** |
+| `meta_schema_name` | String | Unset | Optional PostgreSQL schema for metadata table and election table name qualification.<br/>When PostgreSQL public schema is not writable (e.g., PostgreSQL 15+ with restricted public),<br/>set this to a writable schema. GreptimeDB will use `meta_schema_name`.`meta_table_name`.<br/>**Only used when backend is `postgres_store`.**<br/>Unset by default; this example sets it to `greptime_schema`. |
 | `auto_create_schema` | Bool | `true` | Automatically create PostgreSQL schema if it doesn't exist.<br/>When enabled, the system will execute `CREATE SCHEMA IF NOT EXISTS <schema_name>`<br/>before creating metadata tables. This is useful in production environments where<br/>manual schema creation may be restricted.<br/>Default is true.<br/>Note: The PostgreSQL user must have CREATE SCHEMA permission for this to work.<br/>**Only used when backend is `postgres_store`.** |
 | `meta_election_lock_id` | Integer | `1` | Advisory lock id in PostgreSQL for election. Effect when using PostgreSQL as kvbackend<br/>Only used when backend is `postgres_store`. |
 | `selector` | String | `round_robin` | Datanode selector type.<br/>- `round_robin` (default value)<br/>- `lease_based`<br/>- `load_based`<br/>For details, please see "https://docs.greptime.com/developer-guide/metasrv/selector". |
@@ -443,9 +451,9 @@
 | `heartbeat_interval` | String | `3s` | Base heartbeat interval for calculating distributed time constants.<br/>The frontend heartbeat interval is 6 times of the base heartbeat interval.<br/>The flownode/datanode heartbeat interval is 1 times of the base heartbeat interval.<br/>e.g., If the base heartbeat interval is 3s, the frontend heartbeat interval is 18s, the flownode/datanode heartbeat interval is 3s.<br/>Heartbeat intervals are negotiated from metasrv during handshake; local node configs do not override this. |
 | `enable_telemetry` | Bool | `true` | Whether to enable greptimedb telemetry. Enabled by default. |
 | `runtime` | -- | -- | The runtime options. |
-| `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
-| `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute compact operations. |
-| `runtime.compact_rt_max_blocking_threads` | Integer | `4` | The maximum number of blocking threads for compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
+| `runtime.global_rt_size` | Integer | Auto | The number of threads to execute the runtime for global read operations.<br/>Defaults to max(num_cpus, 2). |
+| `runtime.compact_rt_size` | Integer | Auto | The number of threads to execute compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
+| `runtime.compact_rt_max_blocking_threads` | Integer | Auto | The maximum number of blocking threads for compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
 | `backend_tls` | -- | -- | TLS configuration for kv store backend (applicable for etcd, PostgreSQL, and MySQL backends)<br/>When using etcd, PostgreSQL, or MySQL as metadata store, you can configure TLS here<br/><br/>Note: if TLS is configured in both this section and the `store_addrs` connection string, the<br/>settings here will override the TLS settings in `store_addrs`. |
 | `backend_tls.mode` | String | `prefer` | TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html<br/>- "disable" - No TLS<br/>- "prefer" (default) - Try TLS, fallback to plain<br/>- "require" - Require TLS<br/>- "verify_ca" - Require TLS and verify CA<br/>- "verify_full" - Require TLS and verify hostname |
 | `backend_tls.cert_path` | String | `""` | Path to client certificate file (for client authentication)<br/>Like "/path/to/client.crt" |
@@ -457,7 +465,7 @@
 | `backend_client.connect_timeout` | String | `3s` | The connect timeout for backend client. |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.bind_addr` | String | `127.0.0.1:3002` | The address to bind the gRPC server. |
-| `grpc.server_addr` | String | `127.0.0.1:3002` | The communication server address for the frontend and datanode to connect to metasrv.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `grpc.bind_addr`. |
+| `grpc.server_addr` | String | Unset | The communication server address for the frontend and datanode to connect to metasrv.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `grpc.bind_addr`. |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
 | `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
 | `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
@@ -479,7 +487,7 @@
 | `datanode` | -- | -- | Datanode options. |
 | `datanode.client` | -- | -- | Datanode client options. |
 | `datanode.client.timeout` | String | `10s` | Operation timeout. |
-| `datanode.client.connect_timeout` | String | `10s` | Connect server timeout. |
+| `datanode.client.connect_timeout` | String | 1s | Connect server timeout. Defaults to `1s`; this example sets it to `10s`. |
 | `datanode.client.tcp_nodelay` | Bool | `true` | `TCP_NODELAY` option for accepted connections. |
 | `datanode.client.max_recv_message_size` | String | `512MB` | The maximum receive message size for the gRPC client. |
 | `datanode.client.max_send_message_size` | String | `512MB` | The maximum send message size for the gRPC client. |
@@ -501,7 +509,7 @@
 | `wal.create_topic_timeout` | String | `30s` | The timeout for creating a Kafka topic.<br/>**It's only used when the provider is `kafka`**. |
 | `event_recorder` | -- | -- | Configuration options for the event recorder. |
 | `event_recorder.ttl` | String | `90d` | TTL for the events table that will be used to store the events. Default is `90d`. |
-| `event_recorder.event_types` | Array | -- | Event types to record. Current available event types: `region_migration`,<br/>`create_database`, `alter_database`, `drop_database`, `create_flow`,<br/>`drop_flow`, `create_table`, `create_logical_tables`, `alter_table`,<br/>`alter_logical_tables`, `drop_table`, `undrop_table`, `purge_dropped_table`,<br/>`truncate_table`, `create_view`, `drop_view`, `repartition`,<br/>`repartition_group`, `wal_prune`, `batch_gc`, `reconcile_catalog`,<br/>`reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.<br/>When omitted, all current and future event types are recorded.<br/>Set to an empty array to disable event recording. |
+| `event_recorder.event_types` | Array | All | Event types to record. Current available event types: `region_migration`,<br/>`create_database`, `alter_database`, `drop_database`, `create_flow`,<br/>`drop_flow`, `create_table`, `create_logical_tables`, `alter_table`,<br/>`alter_logical_tables`, `drop_table`, `undrop_table`, `purge_dropped_table`,<br/>`truncate_table`, `create_view`, `drop_view`, `repartition`,<br/>`repartition_group`, `wal_prune`, `batch_gc`, `reconcile_catalog`,<br/>`reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.<br/>When omitted, all current and future event types are recorded.<br/>Set to an empty array to disable event recording. |
 | `stats_persistence` | -- | -- | Configuration options for the stats persistence. |
 | `stats_persistence.ttl` | String | `0s` | TTL for the stats table that will be used to store the stats.<br/>Set to `0s` to disable stats persistence.<br/>Default is `0s`.<br/>If you want to enable stats persistence, set the TTL to a value greater than 0.<br/>It is recommended to set a small value, e.g., `3h`. |
 | `stats_persistence.interval` | String | `10m` | The interval to persist the stats. Default is `10m`.<br/>The minimum value is `10m`, if the value is less than `10m`, it will be overridden to `10m`. |
@@ -546,22 +554,22 @@
 | `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.bind_addr` | String | `127.0.0.1:3001` | The address to bind the gRPC server. |
-| `grpc.server_addr` | String | `127.0.0.1:3001` | The address advertised to the metasrv, and used for connections from outside the host.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `grpc.bind_addr`. |
+| `grpc.server_addr` | String | Unset | The address advertised to the metasrv, and used for connections from outside the host.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `grpc.bind_addr`. |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
 | `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
 | `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
-| `grpc.flight_compression` | String | `arrow_ipc` | Compression mode for datanode side Arrow IPC service. Available options:<br/>- `none`: disable all compression<br/>- `transport`: only enable gRPC transport compression (zstd)<br/>- `arrow_ipc`: only enable Arrow IPC compression (lz4)<br/>- `all`: enable all compression.<br/>Default to `none` |
+| `grpc.flight_compression` | String | `arrow_ipc` | Compression mode for datanode side Arrow IPC service. Available options:<br/>- `none`: disable all compression<br/>- `transport`: only enable gRPC transport compression (zstd)<br/>- `arrow_ipc`: only enable Arrow IPC compression (lz4)<br/>- `all`: enable all compression.<br/>Default to `arrow_ipc`. |
 | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
 | `grpc.tls.mode` | String | `disable` | TLS mode. |
 | `grpc.tls.cert_path` | String | Unset | Certificate file path. |
 | `grpc.tls.key_path` | String | Unset | Private key file path. |
 | `grpc.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload.<br/>For now, gRPC tls config does not support auto reload. |
 | `runtime` | -- | -- | The runtime options. |
-| `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
-| `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute compact operations. |
-| `runtime.compact_rt_max_blocking_threads` | Integer | `4` | The maximum number of blocking threads for compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
-| `runtime.query_rt_size` | Integer | `7` | The number of threads to execute datanode query operations.<br/>Defaults to max(num_cpus - 1, 2). |
-| `runtime.ingest_rt_size` | Integer | `8` | The number of threads to execute datanode ingestion operations. |
+| `runtime.global_rt_size` | Integer | Auto | The number of threads to execute the runtime for global read operations.<br/>Defaults to max(num_cpus, 2). |
+| `runtime.compact_rt_size` | Integer | Auto | The number of threads to execute compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
+| `runtime.compact_rt_max_blocking_threads` | Integer | Auto | The maximum number of blocking threads for compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
+| `runtime.query_rt_size` | Integer | Auto | The number of threads to execute datanode query operations.<br/>Defaults to max(num_cpus - 1, 2). |
+| `runtime.ingest_rt_size` | Integer | Auto | The number of threads to execute datanode ingestion operations.<br/>Defaults to max(num_cpus, 2). |
 | `runtime.experimental_workload_scheduler` | -- | -- | Experimental weighted, work-conserving query/write task scheduler. |
 | `runtime.experimental_workload_scheduler.enable` | Bool | `false` | Enable when concurrent queries and writes interfere with each other—for example, when long-running queries increase ingestion latency.<br/>The weights set their relative runtime shares while both are backlogged. Disabled by default. |
 | `runtime.experimental_workload_scheduler.query_weight` | Integer | `2` | Relative query share while both query and write workloads are backlogged. |
@@ -577,7 +585,7 @@
 | `meta_client.metadata_cache_ttl` | String | `10m` | TTL of the metadata cache. |
 | `meta_client.metadata_cache_tti` | String | `5m` | -- |
 | `wal` | -- | -- | The WAL options. |
-| `wal.provider` | String | `raft_engine` | The provider of the WAL.<br/>- `raft_engine`: the wal is stored in the local file system by raft-engine.<br/>- `kafka`: it's remote wal that data is stored in Kafka.<br/>- `noop`: it's a no-op WAL provider that does not store any WAL data.<br/>**Notes: any unflushed data will be lost when the datanode is shutdown.**<br/>- `experimental_object_store`: the wal is stored as objects in an object store.<br/>**Notes: experimental and not supported yet.** |
+| `wal.provider` | String | `raft_engine` | The provider of the WAL.<br/>- `raft_engine`: the wal is stored in the local file system by raft-engine.<br/>- `kafka`: it's remote wal that data is stored in Kafka.<br/>- `noop`: it's a no-op WAL provider that does not store any WAL data.<br/>**Notes: any unflushed data will be lost when the datanode is shutdown.**<br/>- `experimental_object_store`: the wal is stored as objects in an object store.<br/>**Notes: experimental.** |
 | `wal.dir` | String | Unset | The directory to store the WAL files.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.file_size` | String | `128MB` | The size of the WAL segment file.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.purge_threshold` | String | `1GB` | The threshold of the WAL size to trigger a purge.<br/>**It's only used when the provider is `raft_engine`**. |
@@ -587,7 +595,7 @@
 | `wal.enable_log_recycle` | Bool | `true` | Whether to reuse logically truncated log files.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.prefill_log_files` | Bool | `false` | Whether to pre-create log files on start up.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.sync_period` | String | `5s` | Duration for fsyncing log files.<br/>**It's only used when the provider is `raft_engine`**. |
-| `wal.recovery_parallelism` | Integer | `2` | Parallelism during WAL recovery. |
+| `wal.recovery_parallelism` | Integer | Auto | Parallelism during WAL recovery. Defaults to the number of CPU cores; this example sets it to 2. |
 | `wal.broker_endpoints` | Array | -- | The Kafka broker endpoints.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.connect_timeout` | String | `3s` | The connect timeout for kafka client.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.timeout` | String | `5s` | The total request timeout for kafka client.<br/>**It's only used when the provider is `kafka`**. |
@@ -598,11 +606,14 @@
 | `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.<br/>**It's only used when the provider is `kafka`**.<br/><br/>This option ensures that when Kafka messages are deleted, the system<br/>can still successfully replay memtable data without throwing an<br/>out-of-range error.<br/>However, enabling this option might lead to unexpected data loss,<br/>as the system will skip over missing entries instead of treating<br/>them as critical errors. |
 | `wal.storage_provider` | String | `""` | The name of the storage provider that holds the WAL objects, an empty name selects the default object store.<br/>**It's only used when the provider is `experimental_object_store`**. |
 | `wal.prefix` | String | `wal` | The path prefix of the WAL objects inside the storage provider.<br/>The objects are written under `<prefix>/datanodes/<node_id>/epochs/<generation>`, which is derived from this prefix.<br/>**It's only used when the provider is `experimental_object_store`**. |
-| `wal.flush_interval` | String | `100ms` | The interval of flushing buffered entries to the object store, at least `10ms`, defaults to `100ms`.<br/>Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so under sustained load the batch seals on size and the interval no longer matters.<br/>When writes are sparse, a shorter interval lowers the acknowledgement latency of appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node.<br/>**It's only used when the provider is `experimental_object_store`**. |
+| `wal.flush_interval` | String | `100ms` | The interval of flushing buffered entries to the object store, at least `10ms`, defaults to `100ms`.<br/>Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so under sustained load the batch seals on size and the interval no longer matters.<br/>When writes are sparse, a shorter interval lowers the acknowledgement latency of `durable` appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node, and in `enqueued` mode the backlog thresholds can seal earlier. An `enqueued` append does not wait for its batch to seal once it is admitted, but the backlog thresholds can delay admission.<br/>**It's only used when the provider is `experimental_object_store`**. |
+| `wal.ack_mode` | String | `durable` | When an append to the object store WAL returns.<br/>- `durable`: an append returns after the object holding its entries is durable (the default).<br/>- `enqueued`: an append returns once its entries are admitted and their ids are assigned, and the object is created in the background. A crash loses the unpersisted backlog.<br/>**It's only used when the provider is `experimental_object_store`**. |
+| `wal.max_unpersisted_bytes` | String | `64MB` | The size of the unpersisted backlog at which new appends stall until an upload completes, in `enqueued` mode.<br/>Nothing is dropped and nothing is rejected; the threshold does not bound what a crash during an outage can lose.<br/>**It's only used when the provider is `experimental_object_store` and `ack_mode` is `enqueued`**. |
+| `wal.max_unpersisted_age` | String | `8s` | The age of the oldest unpersisted entry at which new appends stall until an upload completes, in `enqueued` mode.<br/>**It's only used when the provider is `experimental_object_store` and `ack_mode` is `enqueued`**. |
 | `wal.on_corrupted_segment` | String | `skip` | What a read does with a segment that still does not decode after a second fetch, because its checksum does not match or its content disagrees with its footer entry.<br/>- `skip`: the segment is skipped and recorded as a WAL hole of its region, a metric is incremented and a warning is logged; the other regions of the object are unaffected (the default).<br/>- `fail`: the read fails, so the region does not open.<br/>**It's only used when the provider is `experimental_object_store`**. |
 | `query` | -- | -- | The query engine options. |
 | `query.parallelism` | Integer | `0` | Parallelism of the query engine.<br/>Default to 0, which means the number of CPU cores. |
-| `query.memory_pool_size` | String | `50%` | Memory pool size for query execution operators (aggregation, sorting, join).<br/>Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%").<br/>Setting it to 0 disables the limit (unbounded, default behavior).<br/>When this limit is reached, queries will fail with ResourceExhausted error.<br/>NOTE: This does NOT limit memory used by table scans. |
+| `query.memory_pool_size` | String | 0 | Memory pool size for query execution operators (aggregation, sorting, join).<br/>Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%").<br/>Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.<br/>When this limit is reached, queries will fail with ResourceExhausted error.<br/>NOTE: This does NOT limit memory used by table scans. |
 | `query.experimental_memory_pool_policy` | String | `greedy` | Experimental memory pool allocation policy:<br/>- "greedy" (default): first-come-first-served allocation; preserves current behavior.<br/>- "fair": divides available memory among spillable operators and may spill earlier.<br/>Only effective when `memory_pool_size` is bounded (>0). |
 | `query.experimental_spill_mode` | String | `default` | Spill mode:<br/>- "default": preserve DataFusion built-in OS temp directory (default).<br/>- "custom": explicitly configure spill path, quota, and compression.<br/>- "disabled": explicitly disable disk spilling.<br/>Set this to "custom" before using the path/quota/compression keys below. |
 | `query.experimental_spill_path` | String | Unset | Spill directory path. Ignored unless mode is "custom". |
@@ -610,9 +621,11 @@
 | `query.experimental_spill_compression` | String | `uncompressed` | Compression for spilled data files: "uncompressed" (default), "lz4_frame", "zstd".<br/>Ignored unless mode is "custom". |
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `./greptimedb_data` | The working home directory. |
-| `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
+| `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS.<br/>- `Hdfs`: the data is stored in the Hadoop Distributed File System. |
 | `storage.bucket` | String | Unset | The S3 bucket name.<br/>**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. |
-| `storage.root` | String | Unset | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.<br/>**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. |
+| `storage.root` | String | Unset | The directory or object prefix under which data is stored.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs`, `Azblob` and `Hdfs`**. |
+| `storage.name_node` | String | Unset | The HDFS NameNode URI, for example, `hdfs://127.0.0.1:9000`.<br/>**It's only used when the storage type is `Hdfs`**. |
+| `storage.options` | InlineTable | Unset | Additional options passed to the native HDFS client.<br/>**It's only used when the storage type is `Hdfs`**. |
 | `storage.access_key_id` | String | Unset | The access key id of the aws account.<br/>It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.<br/>**It's only used when the storage type is `S3` and `Oss`**. |
 | `storage.secret_access_key` | String | Unset | The secret access key of the aws account.<br/>It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.<br/>**It's only used when the storage type is `S3`**. |
 | `storage.access_key_secret` | String | Unset | The secret access key of the aliyun account.<br/>**It's only used when the storage type is `Oss`**. |
@@ -633,7 +646,7 @@
 | `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification<br/>**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. |
 | `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines.<br/>Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail. |
 | `region_engine.mito` | -- | -- | The Mito engine options. |
-| `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. |
+| `region_engine.mito.num_workers` | Integer | Auto | Number of region workers. Defaults to half of the available CPU cores, rounded up. |
 | `region_engine.mito.worker_channel_size` | Integer | `128` | Request channel size of each worker. |
 | `region_engine.mito.worker_request_batch_size` | Integer | `64` | Max batch size for a worker to handle requests. |
 | `region_engine.mito.manifest_checkpoint_distance` | Integer | `10` | Number of meta action updated to trigger a new checkpoint for the manifest. |
@@ -641,6 +654,7 @@
 | `region_engine.mito.experimental_manifest_keep_removed_file_ttl` | String | `1h` | How long to keep removed files in the `removed_files` field of manifest<br/>after they are removed from manifest.<br/>files will only be removed from `removed_files` field<br/>if both `keep_removed_file_count` and `keep_removed_file_ttl` is reached. |
 | `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
 | `region_engine.mito.experimental_enable_series_index` | Bool | `false` | Under development; do not enable. Whether to enable series indexes.<br/>Indexes are stored on the local filesystem under `{data_home}/series_index`. |
+| `region_engine.mito.experimental_series_index_max_size` | String | `5GiB` | Approximate series and range index size limit in open regions, shared across workers.<br/>Workers periodically refresh usage and skip maintenance when full. In-flight reconciliation<br/>can exceed the limit. Closed-region files, temporary output, catalogs, and old snapshots<br/>retained by readers are not counted.<br/>Minimum: 1KiB. Takes effect on restart. |
 | `region_engine.mito.experimental_enable_range_index` | Bool | `false` | Whether to build and query range indexes when series indexes are enabled.<br/>Obsolete range-index metadata and files are still cleaned up when disabled. |
 | `region_engine.mito.experimental_series_index_maintenance_interval` | String | `5m` | Interval between series-index maintenance runs. Zero uses the default of 5 min. |
 | `region_engine.mito.experimental_series_index_bucket_width` | String | `5days` | Requested minimum series-index bucket width (default: 5 days), rounded up to<br/>an exact multiple of each region's compaction time window. |
@@ -675,7 +689,7 @@
 | `region_engine.mito.min_compaction_interval` | String | `0m` | Minimum time interval between two compactions.<br/>To align with the old behavior, the default value is 0 (no restrictions). |
 | `region_engine.mito.schedule_compaction_after_edit` | Bool | `true` | Whether to allow to schedule a compaction after a successful region edit.<br/><br/>Setting this to "true" is a necessary but not sufficient condition for scheduling compaction after a region edit.<br/>Other constraints, such as "min_compaction_interval", may still prevent compaction from being scheduled.<br/>Setting this to "false", however, guarantees that compaction will not be scheduled after a region edit. |
 | `region_engine.mito.default_flat_format` | Bool | `true` | Whether to enable flat format as the default SST format. |
-| `region_engine.mito.experimental_series_scan_v2` | Bool | `true` | Whether to enable the experimental two-phase mode for eligible metric series scans. |
+| `region_engine.mito.experimental_series_scan_v2` | Bool | `false` | Whether to enable the experimental two-phase mode for eligible metric series scans. |
 | `region_engine.mito.index` | -- | -- | The options for index in Mito engine. |
 | `region_engine.mito.index.aux_path` | String | `""` | Auxiliary directory path for the index in filesystem, used to store intermediate files for<br/>creating the index and staging files for searching the index, defaults to `{data_home}/index_intermediate`.<br/>The default name for this directory is `index_intermediate` for backward compatibility.<br/><br/>This path contains two subdirectories:<br/>- `__intm`: for storing intermediate files used during creating index.<br/>- `staging`: for storing staging files used during searching index. |
 | `region_engine.mito.index.staging_size` | String | `2GB` | The max capacity of the staging directory. |
@@ -732,7 +746,6 @@
 | --- | -----| ------- | ----------- |
 | `node_id` | Integer | Unset | The flownode identifier and should be unique in the cluster. |
 | `flow` | -- | -- | flow engine options. |
-| `flow.num_workers` | Integer | `0` | The number of flow worker in flownode.<br/>Not setting(or set to 0) this value will use the number of CPU cores divided by 2. |
 | `flow.batching_mode` | -- | -- | -- |
 | `flow.batching_mode.query_timeout` | String | `600s` | The default batching engine query timeout is 10 minutes. |
 | `flow.batching_mode.slow_query_threshold` | String | `60s` | will output a warn log for any query that runs for more that this threshold |
@@ -750,9 +763,9 @@
 | `flow.batching_mode.frontend_tls.client_cert_path` | String | Unset | Client Certificate file path. |
 | `flow.batching_mode.frontend_tls.client_key_path` | String | Unset | Client Private key file path. |
 | `grpc` | -- | -- | The gRPC server options. |
-| `grpc.bind_addr` | String | `127.0.0.1:6800` | The address to bind the gRPC server. |
-| `grpc.server_addr` | String | `127.0.0.1:6800` | The address advertised to the metasrv,<br/>and used for connections from outside the host |
-| `grpc.runtime_size` | Integer | `2` | The number of server worker threads. |
+| `grpc.bind_addr` | String | 127.0.0.1:3004 | The address to bind the gRPC server. Defaults to `127.0.0.1:3004`; this example sets it to `127.0.0.1:6800`. |
+| `grpc.server_addr` | String | Unset | The address advertised to the metasrv, and used for connections from outside the host.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `grpc.bind_addr`. |
+| `grpc.runtime_size` | Integer | 8 | The number of server worker threads. Defaults to 8; this example sets it to 2. |
 | `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
 | `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
 | `http` | -- | -- | The HTTP server options. |
@@ -786,7 +799,7 @@
 | `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
 | `query` | -- | -- | -- |
 | `query.parallelism` | Integer | `1` | Parallelism of the query engine for query sent by flownode.<br/>Default to 1, so it won't use too much cpu or memory |
-| `query.memory_pool_size` | String | `50%` | Memory pool size for query execution operators (aggregation, sorting, join).<br/>Supports absolute size (e.g., "1GB", "2GB") or percentage of system memory (e.g., "20%").<br/>Setting it to 0 disables the limit (unbounded, default behavior).<br/>When this limit is reached, queries will fail with ResourceExhausted error.<br/>NOTE: This does NOT limit memory used by table scans. |
+| `query.memory_pool_size` | String | 0 | Memory pool size for query execution operators (aggregation, sorting, join).<br/>Supports absolute size (e.g., "1GB", "2GB") or percentage of system memory (e.g., "20%").<br/>Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.<br/>When this limit is reached, queries will fail with ResourceExhausted error.<br/>NOTE: This does NOT limit memory used by table scans. |
 | `query.experimental_memory_pool_policy` | String | `greedy` | Experimental memory pool allocation policy:<br/>- "greedy" (default): first-come-first-served allocation; preserves current behavior.<br/>- "fair": divides available memory among spillable operators and may spill earlier.<br/>Only effective when `memory_pool_size` is bounded (>0). |
 | `query.experimental_spill_mode` | String | `default` | Spill mode:<br/>- "default": preserve DataFusion built-in OS temp directory (default).<br/>- "custom": explicitly configure spill path, quota, and compression.<br/>- "disabled": explicitly disable disk spilling.<br/>Set this to "custom" before using the path/quota/compression keys below. |
 | `query.experimental_spill_path` | String | Unset | Spill directory path. Ignored unless mode is "custom". |

config/datanode.example.toml

@@ -45,6 +45,7 @@ bind_addr = "127.0.0.1:3001"
 ## The address advertised to the metasrv, and used for connections from outside the host.
 ## If left empty or unset, the server will automatically use the IP address of the first network interface
 ## on the host, with the same port number as the one specified in `grpc.bind_addr`.
+## @toml2docs:none-default
 server_addr = "127.0.0.1:3001"
 ## The number of server worker threads.
 runtime_size = 8
@@ -57,7 +58,7 @@ max_send_message_size = "512MB"
 ## - `transport`: only enable gRPC transport compression (zstd)
 ## - `arrow_ipc`: only enable Arrow IPC compression (lz4)
 ## - `all`: enable all compression.
-## Default to `none`
+## Default to `arrow_ipc`.
 flight_compression = "arrow_ipc"
 
 ## gRPC server TLS options, see `mysql.tls` section.
@@ -80,16 +81,24 @@ watch = false
 ## The runtime options.
 #+ [runtime]
 ## The number of threads to execute the runtime for global read operations.
+## Defaults to max(num_cpus, 2).
+## @toml2docs:none-default="Auto"
 #+ global_rt_size = 8
 ## The number of threads to execute compact operations.
+## Defaults to max(num_cpus / 2, 2).
+## @toml2docs:none-default="Auto"
 #+ compact_rt_size = 4
 ## The maximum number of blocking threads for compact operations.
 ## Defaults to max(num_cpus / 2, 2).
+## @toml2docs:none-default="Auto"
 #+ compact_rt_max_blocking_threads = 4
 ## The number of threads to execute datanode query operations.
 ## Defaults to max(num_cpus - 1, 2).
+## @toml2docs:none-default="Auto"
 #+ query_rt_size = 7
 ## The number of threads to execute datanode ingestion operations.
+## Defaults to max(num_cpus, 2).
+## @toml2docs:none-default="Auto"
 #+ ingest_rt_size = 8
 
 ## Experimental weighted, work-conserving query/write task scheduler.
@@ -136,7 +145,7 @@ metadata_cache_tti = "5m"
 ## - `raft_engine`: the wal is stored in the local file system by raft-engine.
 ## - `kafka`: it's remote wal that data is stored in Kafka.
 ## - `noop`: it's a no-op WAL provider that does not store any WAL data.<br/>**Notes: any unflushed data will be lost when the datanode is shutdown.**
-## - `experimental_object_store`: the wal is stored as objects in an object store.<br/>**Notes: experimental and not supported yet.**
+## - `experimental_object_store`: the wal is stored as objects in an object store.<br/>**Notes: experimental.**
 provider = "raft_engine"
 
 ## The directory to store the WAL files.
@@ -176,7 +185,8 @@ prefill_log_files = false
 ## **It's only used when the provider is `raft_engine`**.
 sync_period = "5s"
 
-## Parallelism during WAL recovery.
+## Parallelism during WAL recovery. Defaults to the number of CPU cores; this example sets it to 2.
+## @toml2docs:none-default="Auto"
 recovery_parallelism = 2
 
 ## The Kafka broker endpoints.
@@ -231,10 +241,25 @@ overwrite_entry_start_id = false
 
 ## The interval of flushing buffered entries to the object store, at least `10ms`, defaults to `100ms`.
 ## Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so under sustained load the batch seals on size and the interval no longer matters.
-## When writes are sparse, a shorter interval lowers the acknowledgement latency of appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node.
+## When writes are sparse, a shorter interval lowers the acknowledgement latency of `durable` appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node, and in `enqueued` mode the backlog thresholds can seal earlier. An `enqueued` append does not wait for its batch to seal once it is admitted, but the backlog thresholds can delay admission.
 ## **It's only used when the provider is `experimental_object_store`**.
 #+ flush_interval = "100ms"
 
+## When an append to the object store WAL returns.
+## - `durable`: an append returns after the object holding its entries is durable (the default).
+## - `enqueued`: an append returns once its entries are admitted and their ids are assigned, and the object is created in the background. A crash loses the unpersisted backlog.
+## **It's only used when the provider is `experimental_object_store`**.
+#+ ack_mode = "durable"
+
+## The size of the unpersisted backlog at which new appends stall until an upload completes, in `enqueued` mode.
+## Nothing is dropped and nothing is rejected; the threshold does not bound what a crash during an outage can lose.
+## **It's only used when the provider is `experimental_object_store` and `ack_mode` is `enqueued`**.
+#+ max_unpersisted_bytes = "64MB"
+
+## The age of the oldest unpersisted entry at which new appends stall until an upload completes, in `enqueued` mode.
+## **It's only used when the provider is `experimental_object_store` and `ack_mode` is `enqueued`**.
+#+ max_unpersisted_age = "8s"
+
 ## What a read does with a segment that still does not decode after a second fetch, because its checksum does not match or its content disagrees with its footer entry.
 ## - `skip`: the segment is skipped and recorded as a WAL hole of its region, a metric is incremented and a warning is logged; the other regions of the object are unaffected (the default).
 ## - `fail`: the read fails, so the region does not open.
@@ -300,6 +325,13 @@ overwrite_entry_start_id = false
 # credential = "base64-credential"
 # endpoint = "https://storage.googleapis.com"
 
+# Example of using HDFS as the storage with the native Rust client.
+# [storage]
+# type = "Hdfs"
+# root = "/greptimedb"
+# name_node = "hdfs://127.0.0.1:9000"
+# options = { "dfs.client.block.write.replace-datanode-on-failure.enable" = "true" }
+
 ## The query engine options.
 [query]
 ## Parallelism of the query engine.
@@ -308,9 +340,10 @@ parallelism = 0
 
 ## Memory pool size for query execution operators (aggregation, sorting, join).
 ## Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%").
-## Setting it to 0 disables the limit (unbounded, default behavior).
+## Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.
 ## When this limit is reached, queries will fail with ResourceExhausted error.
 ## NOTE: This does NOT limit memory used by table scans.
+## @toml2docs:none-default="0"
 memory_pool_size = "50%"
 
 ## Experimental memory pool allocation policy:
@@ -348,6 +381,7 @@ data_home = "./greptimedb_data"
 ## - `Gcs`: the data is stored in the Google Cloud Storage.
 ## - `Azblob`: the data is stored in the Azure Blob Storage.
 ## - `Oss`: the data is stored in the Aliyun OSS.
+## - `Hdfs`: the data is stored in the Hadoop Distributed File System.
 type = "File"
 
 ## The S3 bucket name.
@@ -355,11 +389,21 @@ type = "File"
 ## @toml2docs:none-default
 bucket = "greptimedb"
 
-## The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.
-## **It's only used when the storage type is `S3`, `Oss` and `Azblob`**.
+## The directory or object prefix under which data is stored.
+## **It's only used when the storage type is `S3`, `Oss`, `Gcs`, `Azblob` and `Hdfs`**.
 ## @toml2docs:none-default
 root = "greptimedb"
 
+## The HDFS NameNode URI, for example, `hdfs://127.0.0.1:9000`.
+## **It's only used when the storage type is `Hdfs`**.
+## @toml2docs:none-default
+name_node = "hdfs://127.0.0.1:9000"
+
+## Additional options passed to the native HDFS client.
+## **It's only used when the storage type is `Hdfs`**.
+## @toml2docs:none-default
+options = {}
+
 ## The access key id of the aws account.
 ## It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.
 ## **It's only used when the storage type is `S3` and `Oss`**.
@@ -470,7 +514,8 @@ skip_ssl_validation = false
 ## The Mito engine options.
 [region_engine.mito]
 
-## Number of region workers.
+## Number of region workers. Defaults to half of the available CPU cores, rounded up.
+## @toml2docs:none-default="Auto"
 #+ num_workers = 8
 
 ## Request channel size of each worker.
@@ -502,6 +547,13 @@ compress_manifest = false
 ## Indexes are stored on the local filesystem under `{data_home}/series_index`.
 #+ experimental_enable_series_index = false
 
+## Approximate series and range index size limit in open regions, shared across workers.
+## Workers periodically refresh usage and skip maintenance when full. In-flight reconciliation
+## can exceed the limit. Closed-region files, temporary output, catalogs, and old snapshots
+## retained by readers are not counted.
+## Minimum: 1KiB. Takes effect on restart.
+#+ experimental_series_index_max_size = "5GiB"
+
 ## Whether to build and query range indexes when series indexes are enabled.
 ## Obsolete range-index metadata and files are still cleaned up when disabled.
 #+ experimental_enable_range_index = false
@@ -647,7 +699,7 @@ schedule_compaction_after_edit = true
 default_flat_format = true
 
 ## Whether to enable the experimental two-phase mode for eligible metric series scans.
-experimental_series_scan_v2 = true
+experimental_series_scan_v2 = false
 
 ## The options for index in Mito engine.
 [region_engine.mito.index]

config/flownode.example.toml

@@ -4,9 +4,6 @@ node_id = 14
 
 ## flow engine options.
 [flow]
-## The number of flow worker in flownode.
-## Not setting(or set to 0) this value will use the number of CPU cores divided by 2.
-#+num_workers=0
 [flow.batching_mode]
 ## The default batching engine query timeout is 10 minutes.
 #+query_timeout="600s"
@@ -47,12 +44,16 @@ node_id = 14
 
 ## The gRPC server options.
 [grpc]
-## The address to bind the gRPC server.
+## The address to bind the gRPC server. Defaults to `127.0.0.1:3004`; this example sets it to `127.0.0.1:6800`.
+## @toml2docs:none-default="127.0.0.1:3004"
 bind_addr = "127.0.0.1:6800"
-## The address advertised to the metasrv,
-## and used for connections from outside the host
+## The address advertised to the metasrv, and used for connections from outside the host.
+## If left empty or unset, the server will automatically use the IP address of the first network interface
+## on the host, with the same port number as the one specified in `grpc.bind_addr`.
+## @toml2docs:none-default
 server_addr = "127.0.0.1:6800"
-## The number of server worker threads.
+## The number of server worker threads. Defaults to 8; this example sets it to 2.
+## @toml2docs:none-default="8"
 runtime_size = 2
 ## The maximum receive message size for gRPC server.
 max_recv_message_size = "512MB"
@@ -156,9 +157,10 @@ parallelism = 1
 
 ## Memory pool size for query execution operators (aggregation, sorting, join).
 ## Supports absolute size (e.g., "1GB", "2GB") or percentage of system memory (e.g., "20%").
-## Setting it to 0 disables the limit (unbounded, default behavior).
+## Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.
 ## When this limit is reached, queries will fail with ResourceExhausted error.
 ## NOTE: This does NOT limit memory used by table scans.
+## @toml2docs:none-default="0"
 memory_pool_size = "50%"
 
 ## Experimental memory pool allocation policy:

config/frontend.example.toml

@@ -20,8 +20,8 @@ default_column_prefix = "greptime"
 ## Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",
 ## "mysql_native_password:<hex_sha1_sha1_password>",
 ## "pg_scram_sha256:<iterations>:<hex_salt>:<hex_stored_key>:<hex_server_key>"
-## "pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but cannot authenticate over MySQL's
-## native password handshake; a MySQL client must send the password in cleartext for such users.
+## "pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:
+## `static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.
 ## "mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
 ## PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
 ## "mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
@@ -33,22 +33,26 @@ default_column_prefix = "greptime"
 
 ## Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
 ## Set to 0 to disable the limit. Default: "0" (unlimited)
-## @toml2docs:none-default
+## @toml2docs:none-default="0"
 #+ max_in_flight_write_bytes = "1GB"
 
 ## Policy when write bytes quota is exhausted.
 ## Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail"
-## @toml2docs:none-default
 #+ write_bytes_exhausted_policy = "wait"
 
 ## The runtime options.
 #+ [runtime]
 ## The number of threads to execute the runtime for global read operations.
+## Defaults to max(num_cpus, 2).
+## @toml2docs:none-default="Auto"
 #+ global_rt_size = 8
 ## The number of threads to execute compact operations.
+## Defaults to max(num_cpus / 2, 2).
+## @toml2docs:none-default="Auto"
 #+ compact_rt_size = 4
 ## The maximum number of blocking threads for compact operations.
 ## Defaults to max(num_cpus / 2, 2).
+## @toml2docs:none-default="Auto"
 #+ compact_rt_max_blocking_threads = 4
 
 ## The HTTP server options.
@@ -56,9 +60,9 @@ default_column_prefix = "greptime"
 ## The address to bind the HTTP server.
 addr = "127.0.0.1:4000"
 ## HTTP request timeout. Set to 0 to disable timeout.
-## When synchronous Prometheus or shared table batching is enabled, a nonzero timeout is
+## When synchronous Prometheus, OTLP metrics, or ordinary-table batching is enabled, a nonzero timeout is
 ## raised to at least the largest active flush interval plus 1 second. The intervals come from
-## `prom_store.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`.
+## `pending_rows_batcher.logical_table.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`.
 timeout = "0s"
 ## HTTP request body limit.
 ## The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.
@@ -86,6 +90,7 @@ bind_addr = "127.0.0.1:4001"
 ## The address advertised to the metasrv, and used for connections from outside the host.
 ## If left empty or unset, the server will automatically use the IP address of the first network interface
 ## on the host, with the same port number as the one specified in `grpc.bind_addr`.
+## @toml2docs:none-default
 server_addr = "127.0.0.1:4001"
 ## The number of server worker threads.
 runtime_size = 8
@@ -94,8 +99,13 @@ runtime_size = 8
 ## - `transport`: only enable gRPC transport compression (zstd)
 ## - `arrow_ipc`: only enable Arrow IPC compression (lz4)
 ## - `all`: enable all compression.
-## Default to `none`
+## Default to `arrow_ipc`.
 flight_compression = "arrow_ipc"
+## Enable CORS for gRPC-Web clients in browsers. Disabled by default.
+#+ enable_cors = false
+## Origins allowed by gRPC CORS. An empty list allows any origin.
+## @toml2docs:none-default
+#+ cors_allowed_origins = ["https://example.com"]
 ## The maximum connection age for gRPC connection.
 ## The value can be a human-readable time string. For example: `10m` for ten minutes or `1h` for one hour.
 ## Refer to https://grpc.io/docs/guides/keepalive/ for more details.
@@ -120,12 +130,15 @@ key_path = ""
 watch = false
 
 ## The internal gRPC server options. Internal gRPC port for nodes inside cluster to access frontend.
+## The internal gRPC server starts only when this section is set.
 [internal_grpc]
 ## The address to bind the gRPC server.
+## If unset, it falls back to `127.0.0.1:4001`, the same address as `grpc.bind_addr`.
+## @toml2docs:none-default
 bind_addr = "127.0.0.1:4010"
 ## The address advertised to the metasrv, and used for connections from outside the host.
-## If left empty or unset, the server will automatically use the IP address of the first network interface
-## on the host, with the same port number as the one specified in `internal_grpc.bind_addr`.
+## Unlike `grpc.server_addr`, it is not detected automatically; set it explicitly.
+## @toml2docs:none-default
 server_addr = "127.0.0.1:4010"
 ## The number of server worker threads.
 runtime_size = 8
@@ -134,7 +147,7 @@ runtime_size = 8
 ## - `transport`: only enable gRPC transport compression (zstd)
 ## - `arrow_ipc`: only enable Arrow IPC compression (lz4)
 ## - `all`: enable all compression.
-## Default to `none`
+## Default to `arrow_ipc`.
 flight_compression = "arrow_ipc"
 
 ## internal gRPC server TLS options, see `mysql.tls` section.
@@ -231,12 +244,13 @@ enable = true
 ## Available values: "last_non_null", "last_row".
 default_merge_mode = "last_non_null"
 
-## Shared experimental ordinary-table batching for opted-in ingestion protocols.
-## Legacy Prometheus batching settings under prom_store remain supported.
-## HTTP write protocols sharing this batcher. Omitted or empty disables all entrances.
-## Supported: influxdb, opentsdb, otlp, logs, loki, splunk, elasticsearch, http_sql, prom.
-## Prom uses ordinary-table batching without metric engine, otherwise its dedicated batcher.
-## Effective shared Prom settings take precedence; existing prom_store settings remain compatible.
+## Ordinary-table batching for opted-in ingestion protocols.
+## PENDING_ROWS_BATCH_SYNC defaults to true for both batchers. Set it to false to acknowledge
+## queue admission without waiting for storage; later failures cannot be returned to the client.
+## Omitted or empty protocols disables batching. Prom without metric engine uses this batcher.
+## OTLP logs, traces and ordinary metrics use this batcher.
+## MySQL and PostgreSQL use the same acknowledgement policy. Protocol timeouts are unchanged.
+## Single-connection writes and INSERT SELECT may incur additional flush waits.
 [pending_rows_batcher]
 # protocols = [
 #     "influxdb",
@@ -247,6 +261,8 @@ default_merge_mode = "last_non_null"
 #     "splunk",
 #     "elasticsearch",
 #     "http_sql",
+#     "mysql",
+#     "postgres",
 #     "prom",
 # ]
 ## Flush interval measured from the first pending submission. Zero disables batching.
@@ -256,12 +272,33 @@ max_batch_rows = 100000
 ## Maximum concurrent flushes shared by the frontend batcher.
 max_concurrent_flushes = 256
 ## Maximum queued submissions per table worker.
-worker_channel_capacity = 65526
+worker_channel_capacity = 65536
 ## Maximum admitted original requests awaiting completion.
 max_inflight_requests = 3000
 ## Maximum number of queued table Flow notifications.
 flow_notification_queue_capacity = 1024
 
+## Metric-engine logical-table batching for Prom remote write and non-legacy OTLP metrics.
+## Requires prom_store.with_metric_engine. Logs, traces and legacy metrics are not eligible.
+## Enable independently with protocols and a nonzero flush interval.
+## Omitted fields use independent defaults, not parent settings.
+## Empty protocols or a zero interval disables logical batching without fallback.
+## Omitting this entire section preserves legacy Prom batching; it does not enable OTLP batching.
+[pending_rows_batcher.logical_table]
+# protocols = ["prom", "otlp"]
+## Flush interval measured from the first pending submission. Zero disables batching.
+pending_rows_flush_interval = "0s"
+## Flush after a complete submission reaches this row threshold.
+max_batch_rows = 100000
+## Maximum concurrent flushes shared by Prom and OTLP metrics.
+max_concurrent_flushes = 256
+## Maximum queued submissions per physical-table worker.
+worker_channel_capacity = 65536
+## Maximum admitted original requests awaiting completion.
+max_inflight_requests = 3000
+## Maximum number of queued logical-table Flow notifications.
+flow_notification_queue_capacity = 1024
+
 ## Jaeger protocol options.
 [jaeger]
 ## Whether to enable Jaeger protocol in HTTP API.
@@ -271,10 +308,15 @@ enable = true
 [otlp]
 ## Whether to enable OpenTelemetry protocol in HTTP API.
 enable = true
-## Experimental: enable cumulative OTLP exponential histogram ingestion.
-experimental_enable_exponential_histogram = false
 ## Maximum spans per trace ingest chunk. Set to 0 to disable splitting.
 trace_ingest_chunk_size = 512
+## Estimated memory budget for cached trace service/operation keys per frontend,
+## shared across all catalogs, schemas, and trace tables. Set to 0 to disable caching.
+## Defaults to 1/128 of the host or pod memory limit, with a minimum of 32MiB.
+## Uses 32MiB if memory detection is unavailable.
+## Excludes cache and allocator overhead; shared table names are charged per entry.
+## @toml2docs:none-default="Auto"
+#+ trace_aux_cache_size = "32MiB"
 ## Whether to synthesize the `greptime_otel_resource_info` table from OTLP metric
 ## resource attributes, so metrics-only services reach the semantic graph.
 experimental_enable_resource_info = false
@@ -291,21 +333,7 @@ with_metric_engine = true
 ## - lossy: allow invalid UTF-8 strings, replace invalid characters with REPLACEMENT_CHARACTER(U+FFFD).
 ## - unchecked: do not valid strings.
 prom_validation_mode = "strict"
-## Experimental: enable Prometheus remote write v2 native histogram ingestion.
-experimental_enable_prometheus_native_histogram = false
-## Interval to flush pending rows batcher.
-## Set to "0s" to disable batching mode in Prometheus Remote Write endpoint
-#+pending_rows_flush_interval = "0s"
-## Max rows per pending batch before triggering a flush.
-#+max_batch_rows = 100000
-## Max number of concurrent batch flushes.
-#+max_concurrent_flushes = 256
-## Capacity of the pending batch worker channel.
-#+worker_channel_capacity = 65526
-## Max inflight write requests before backpressure.
-#+max_inflight_requests = 3000
-## Maximum number of logical-table flow notifications waiting in the shared queue.
-#+flow_notification_queue_capacity = 1024
+
 
 ## The metasrv client options.
 [meta_client]
@@ -344,9 +372,10 @@ allow_query_fallback = false
 
 ## Memory pool size for query execution operators (aggregation, sorting, join).
 ## Supports absolute size (e.g., "4GB", "8GB") or percentage of system memory (e.g., "30%").
-## Setting it to 0 disables the limit (unbounded, default behavior).
+## Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.
 ## When this limit is reached, queries will fail with ResourceExhausted error.
 ## NOTE: This does NOT limit memory used by table scans (only applies to datanodes).
+## @toml2docs:none-default="0"
 memory_pool_size = "50%"
 
 ## Experimental memory pool allocation policy:
@@ -377,6 +406,8 @@ memory_pool_size = "50%"
 [datanode]
 ## Datanode client options.
 [datanode.client]
+## Connect server timeout. Defaults to `1s`; this example sets it to `10s`.
+## @toml2docs:none-default="1s"
 connect_timeout = "10s"
 tcp_nodelay = true
 ## The maximum receive message size for the gRPC client.
@@ -481,4 +512,5 @@ ttl = "90d"
 ## Event types to record. Current available event type: `admin_function`.
 ## When omitted, all current and future event types are recorded.
 ## Set to an empty array to disable event recording.
+## @toml2docs:none-default="All"
 #+ event_types = ["admin_function"]

config/metasrv.example.toml

@@ -35,6 +35,8 @@ meta_table_name = "greptime_metakv"
 ## When PostgreSQL public schema is not writable (e.g., PostgreSQL 15+ with restricted public),
 ## set this to a writable schema. GreptimeDB will use `meta_schema_name`.`meta_table_name`.
 ## **Only used when backend is `postgres_store`.**
+## Unset by default; this example sets it to `greptime_schema`.
+## @toml2docs:none-default
 meta_schema_name = "greptime_schema"
 
 ## Automatically create PostgreSQL schema if it doesn't exist.
@@ -88,11 +90,16 @@ node_max_idle_time = "24hours"
 ## The runtime options.
 #+ [runtime]
 ## The number of threads to execute the runtime for global read operations.
+## Defaults to max(num_cpus, 2).
+## @toml2docs:none-default="Auto"
 #+ global_rt_size = 8
 ## The number of threads to execute compact operations.
+## Defaults to max(num_cpus / 2, 2).
+## @toml2docs:none-default="Auto"
 #+ compact_rt_size = 4
 ## The maximum number of blocking threads for compact operations.
 ## Defaults to max(num_cpus / 2, 2).
+## @toml2docs:none-default="Auto"
 #+ compact_rt_max_blocking_threads = 4
 
 ## TLS configuration for kv store backend (applicable for etcd, PostgreSQL, and MySQL backends)
@@ -140,6 +147,7 @@ bind_addr = "127.0.0.1:3002"
 ## The communication server address for the frontend and datanode to connect to metasrv.
 ## If left empty or unset, the server will automatically use the IP address of the first network interface
 ## on the host, with the same port number as the one specified in `grpc.bind_addr`.
+## @toml2docs:none-default
 server_addr = "127.0.0.1:3002"
 ## The number of server worker threads.
 runtime_size = 8
@@ -206,7 +214,8 @@ acceptable_heartbeat_pause = "10000ms"
 ## Operation timeout.
 timeout = "10s"
 
-## Connect server timeout.
+## Connect server timeout. Defaults to `1s`; this example sets it to `10s`.
+## @toml2docs:none-default="1s"
 connect_timeout = "10s"
 
 ## `TCP_NODELAY` option for accepted connections.
@@ -332,6 +341,7 @@ ttl = "90d"
 ## `reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.
 ## When omitted, all current and future event types are recorded.
 ## Set to an empty array to disable event recording.
+## @toml2docs:none-default="All"
 #+ event_types = ["region_migration"]
 
 ## Configuration options for the stats persistence.

config/standalone.example.toml

@@ -20,8 +20,8 @@ default_column_prefix = "greptime"
 ## Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",
 ## "mysql_native_password:<hex_sha1_sha1_password>",
 ## "pg_scram_sha256:<iterations>:<hex_salt>:<hex_stored_key>:<hex_server_key>"
-## "pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but cannot authenticate over MySQL's
-## native password handshake; a MySQL client must send the password in cleartext for such users.
+## "pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:
+## `static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.
 ## "mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
 ## PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
 ## "mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
@@ -33,12 +33,11 @@ default_column_prefix = "greptime"
 
 ## Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
 ## Set to 0 to disable the limit. Default: "0" (unlimited)
-## @toml2docs:none-default
+## @toml2docs:none-default="0"
 #+ max_in_flight_write_bytes = "1GB"
 
 ## Policy when write bytes quota is exhausted.
 ## Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail"
-## @toml2docs:none-default
 #+ write_bytes_exhausted_policy = "wait"
 
 ## Initialize all regions in the background during the startup.
@@ -48,20 +47,22 @@ init_regions_in_background = false
 ## Parallelism of initializing regions.
 init_regions_parallelism = 16
 
-## The maximum concurrent queries allowed to be executed. Zero means unlimited.
-max_concurrent_queries = 0
-
 ## Enable telemetry to collect anonymous usage data. Enabled by default.
 #+ enable_telemetry = true
 
 ## The runtime options.
 #+ [runtime]
 ## The number of threads to execute the runtime for global read operations.
+## Defaults to max(num_cpus, 2).
+## @toml2docs:none-default="Auto"
 #+ global_rt_size = 8
 ## The number of threads to execute compact operations.
+## Defaults to max(num_cpus / 2, 2).
+## @toml2docs:none-default="Auto"
 #+ compact_rt_size = 4
 ## The maximum number of blocking threads for compact operations.
 ## Defaults to max(num_cpus / 2, 2).
+## @toml2docs:none-default="Auto"
 #+ compact_rt_max_blocking_threads = 4
 
 ## Experimental weighted, work-conserving query/write task scheduler.
@@ -81,9 +82,9 @@ max_concurrent_queries = 0
 ## The address to bind the HTTP server.
 addr = "127.0.0.1:4000"
 ## HTTP request timeout. Set to 0 to disable timeout.
-## When synchronous Prometheus or shared table batching is enabled, a nonzero timeout is
+## When synchronous Prometheus, OTLP metrics, or ordinary-table batching is enabled, a nonzero timeout is
 ## raised to at least the largest active flush interval plus 1 second. The intervals come from
-## `prom_store.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`.
+## `pending_rows_batcher.logical_table.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`.
 timeout = "0s"
 ## HTTP request body limit.
 ## The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.
@@ -111,6 +112,11 @@ api_server_addr = "127.0.0.1:4006"
 bind_addr = "127.0.0.1:4001"
 ## The number of server worker threads.
 runtime_size = 8
+## Enable CORS for gRPC-Web clients in browsers. Disabled by default.
+#+ enable_cors = false
+## Origins allowed by gRPC CORS. An empty list allows any origin.
+## @toml2docs:none-default
+#+ cors_allowed_origins = ["https://example.com"]
 ## The maximum connection age for gRPC connection.
 ## The value can be a human-readable time string. For example: `10m` for ten minutes or `1h` for one hour.
 ## Refer to https://grpc.io/docs/guides/keepalive/ for more details.
@@ -210,12 +216,13 @@ enable = true
 ## Available values: "last_non_null", "last_row".
 default_merge_mode = "last_non_null"
 
-## Shared experimental ordinary-table batching for opted-in ingestion protocols.
-## Legacy Prometheus batching settings under prom_store remain supported.
-## HTTP write protocols sharing this batcher. Omitted or empty disables all entrances.
-## Supported: influxdb, opentsdb, otlp, logs, loki, splunk, elasticsearch, http_sql, prom.
-## Prom uses ordinary-table batching without metric engine, otherwise its dedicated batcher.
-## Effective shared Prom settings take precedence; existing prom_store settings remain compatible.
+## Ordinary-table batching for opted-in ingestion protocols.
+## PENDING_ROWS_BATCH_SYNC defaults to true for both batchers. Set it to false to acknowledge
+## queue admission without waiting for storage; later failures cannot be returned to the client.
+## Omitted or empty protocols disables batching. Prom without metric engine uses this batcher.
+## OTLP logs, traces and ordinary metrics use this batcher.
+## MySQL and PostgreSQL use the same acknowledgement policy. Protocol timeouts are unchanged.
+## Single-connection writes and INSERT SELECT may incur additional flush waits.
 [pending_rows_batcher]
 # protocols = [
 #     "influxdb",
@@ -226,6 +233,8 @@ default_merge_mode = "last_non_null"
 #     "splunk",
 #     "elasticsearch",
 #     "http_sql",
+#     "mysql",
+#     "postgres",
 #     "prom",
 # ]
 ## Flush interval measured from the first pending submission. Zero disables batching.
@@ -235,12 +244,33 @@ max_batch_rows = 100000
 ## Maximum concurrent flushes shared by the frontend batcher.
 max_concurrent_flushes = 256
 ## Maximum queued submissions per table worker.
-worker_channel_capacity = 65526
+worker_channel_capacity = 65536
 ## Maximum admitted original requests awaiting completion.
 max_inflight_requests = 3000
 ## Maximum number of queued table Flow notifications.
 flow_notification_queue_capacity = 1024
 
+## Metric-engine logical-table batching for Prom remote write and non-legacy OTLP metrics.
+## Requires prom_store.with_metric_engine. Logs, traces and legacy metrics are not eligible.
+## Enable independently with protocols and a nonzero flush interval.
+## Omitted fields use independent defaults, not parent settings.
+## Empty protocols or a zero interval disables logical batching without fallback.
+## Omitting this entire section preserves legacy Prom batching; it does not enable OTLP batching.
+[pending_rows_batcher.logical_table]
+# protocols = ["prom", "otlp"]
+## Flush interval measured from the first pending submission. Zero disables batching.
+pending_rows_flush_interval = "0s"
+## Flush after a complete submission reaches this row threshold.
+max_batch_rows = 100000
+## Maximum concurrent flushes shared by Prom and OTLP metrics.
+max_concurrent_flushes = 256
+## Maximum queued submissions per physical-table worker.
+worker_channel_capacity = 65536
+## Maximum admitted original requests awaiting completion.
+max_inflight_requests = 3000
+## Maximum number of queued logical-table Flow notifications.
+flow_notification_queue_capacity = 1024
+
 ## Jaeger protocol options.
 [jaeger]
 ## Whether to enable Jaeger protocol in HTTP API.
@@ -250,10 +280,15 @@ enable = true
 [otlp]
 ## Whether to enable OpenTelemetry protocol in HTTP API.
 enable = true
-## Experimental: enable cumulative OTLP exponential histogram ingestion.
-experimental_enable_exponential_histogram = false
 ## Maximum spans per trace ingest chunk. Set to 0 to disable splitting.
 trace_ingest_chunk_size = 512
+## Estimated memory budget for cached trace service/operation keys per frontend,
+## shared across all catalogs, schemas, and trace tables. Set to 0 to disable caching.
+## Defaults to 1/128 of the host or pod memory limit, with a minimum of 32MiB.
+## Uses 32MiB if memory detection is unavailable.
+## Excludes cache and allocator overhead; shared table names are charged per entry.
+## @toml2docs:none-default="Auto"
+#+ trace_aux_cache_size = "32MiB"
 ## Whether to synthesize the `greptime_otel_resource_info` table from OTLP metric
 ## resource attributes, so metrics-only services reach the semantic graph.
 experimental_enable_resource_info = false
@@ -270,28 +305,14 @@ with_metric_engine = true
 ## - lossy: allow invalid UTF-8 strings, replace invalid characters with REPLACEMENT_CHARACTER(U+FFFD).
 ## - unchecked: do not valid strings.
 prom_validation_mode = "strict"
-## Experimental: enable Prometheus remote write v2 native histogram ingestion.
-experimental_enable_prometheus_native_histogram = false
-## Interval to flush pending rows batcher.
-## Set to "0s" to disable batching mode in Prometheus Remote Write endpoint
-#+pending_rows_flush_interval = "0s"
-## Max rows per pending batch before triggering a flush.
-#+max_batch_rows = 100000
-## Max number of concurrent batch flushes.
-#+max_concurrent_flushes = 256
-## Capacity of the pending batch worker channel.
-#+worker_channel_capacity = 65526
-## Max inflight write requests before backpressure.
-#+max_inflight_requests = 3000
-## Maximum number of logical-table flow notifications waiting in the shared queue.
-#+flow_notification_queue_capacity = 1024
+
 
 ## The WAL options.
 [wal]
 ## The provider of the WAL.
 ## - `raft_engine`: the wal is stored in the local file system by raft-engine.
 ## - `kafka`: it's remote wal that data is stored in Kafka.
-## - `experimental_object_store`: the wal is stored as objects in an object store.<br/>**Notes: experimental and not supported yet.**
+## - `experimental_object_store`: the wal is stored as objects in an object store.<br/>**Notes: experimental.**
 provider = "raft_engine"
 
 ## The directory to store the WAL files.
@@ -331,7 +352,8 @@ prefill_log_files = false
 ## **It's only used when the provider is `raft_engine`**.
 sync_period = "5s"
 
-## Parallelism during WAL recovery.
+## Parallelism during WAL recovery. Defaults to the number of CPU cores; this example sets it to 2.
+## @toml2docs:none-default="Auto"
 recovery_parallelism = 2
 
 ## The Kafka broker endpoints.
@@ -406,10 +428,25 @@ overwrite_entry_start_id = false
 
 ## The interval of flushing buffered entries to the object store, at least `10ms`, defaults to `100ms`.
 ## Each non-empty timer-triggered flush creates one object, and a batch that reaches `max_batch_bytes` is flushed immediately, so under sustained load the batch seals on size and the interval no longer matters.
-## When writes are sparse, a shorter interval lowers the acknowledgement latency of appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node.
+## When writes are sparse, a shorter interval lowers the acknowledgement latency of `durable` appends and raises the number of object requests: timer-triggered sealing creates at most one object per interval per node, and in `enqueued` mode the backlog thresholds can seal earlier. An `enqueued` append does not wait for its batch to seal once it is admitted, but the backlog thresholds can delay admission.
 ## **It's only used when the provider is `experimental_object_store`**.
 #+ flush_interval = "100ms"
 
+## When an append to the object store WAL returns.
+## - `durable`: an append returns after the object holding its entries is durable (the default).
+## - `enqueued`: an append returns once its entries are admitted and their ids are assigned, and the object is created in the background. A crash loses the unpersisted backlog.
+## **It's only used when the provider is `experimental_object_store`**.
+#+ ack_mode = "durable"
+
+## The size of the unpersisted backlog at which new appends stall until an upload completes, in `enqueued` mode.
+## Nothing is dropped and nothing is rejected; the threshold does not bound what a crash during an outage can lose.
+## **It's only used when the provider is `experimental_object_store` and `ack_mode` is `enqueued`**.
+#+ max_unpersisted_bytes = "64MB"
+
+## The age of the oldest unpersisted entry at which new appends stall until an upload completes, in `enqueued` mode.
+## **It's only used when the provider is `experimental_object_store` and `ack_mode` is `enqueued`**.
+#+ max_unpersisted_age = "8s"
+
 ## What a read does with a segment that still does not decode after a second fetch, because its checksum does not match or its content disagrees with its footer entry.
 ## - `skip`: the segment is skipped and recorded as a WAL hole of its region, a metric is incremented and a warning is logged; the other regions of the object are unaffected (the default).
 ## - `fail`: the read fails, so the region does not open.
@@ -454,12 +491,6 @@ retry_delay = "500ms"
 ## If the number of running procedures exceeds this limit, the procedure will be rejected.
 max_running_procedures = 128
 
-## flow engine options.
-[flow]
-## The number of flow worker in flownode.
-## Not setting(or set to 0) this value will use the number of CPU cores divided by 2.
-#+num_workers=0
-
 # Example of using S3 as the storage.
 # [storage]
 # type = "S3"
@@ -501,6 +532,13 @@ max_running_procedures = 128
 # credential = "base64-credential"
 # endpoint = "https://storage.googleapis.com"
 
+# Example of using HDFS as the storage with the native Rust client.
+# [storage]
+# type = "Hdfs"
+# root = "/greptimedb"
+# name_node = "hdfs://127.0.0.1:9000"
+# options = { "dfs.client.block.write.replace-datanode-on-failure.enable" = "true" }
+
 ## The query engine options.
 [query]
 ## Parallelism of the query engine.
@@ -509,9 +547,10 @@ parallelism = 0
 
 ## Memory pool size for query execution operators (aggregation, sorting, join).
 ## Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%").
-## Setting it to 0 disables the limit (unbounded, default behavior).
+## Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.
 ## When this limit is reached, queries will fail with ResourceExhausted error.
 ## NOTE: This does NOT limit memory used by table scans.
+## @toml2docs:none-default="0"
 memory_pool_size = "50%"
 
 ## Experimental memory pool allocation policy:
@@ -558,6 +597,7 @@ data_home = "./greptimedb_data"
 ## - `Gcs`: the data is stored in the Google Cloud Storage.
 ## - `Azblob`: the data is stored in the Azure Blob Storage.
 ## - `Oss`: the data is stored in the Aliyun OSS.
+## - `Hdfs`: the data is stored in the Hadoop Distributed File System.
 type = "File"
 
 ## The S3 bucket name.
@@ -565,11 +605,21 @@ type = "File"
 ## @toml2docs:none-default
 bucket = "greptimedb"
 
-## The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.
-## **It's only used when the storage type is `S3`, `Oss` and `Azblob`**.
+## The directory or object prefix under which data is stored.
+## **It's only used when the storage type is `S3`, `Oss`, `Gcs`, `Azblob` and `Hdfs`**.
 ## @toml2docs:none-default
 root = "greptimedb"
 
+## The HDFS NameNode URI, for example, `hdfs://127.0.0.1:9000`.
+## **It's only used when the storage type is `Hdfs`**.
+## @toml2docs:none-default
+name_node = "hdfs://127.0.0.1:9000"
+
+## Additional options passed to the native HDFS client.
+## **It's only used when the storage type is `Hdfs`**.
+## @toml2docs:none-default
+options = {}
+
 ## The access key id of the aws account.
 ## It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.
 ## **It's only used when the storage type is `S3` and `Oss`**.
@@ -680,7 +730,8 @@ skip_ssl_validation = false
 ## The Mito engine options.
 [region_engine.mito]
 
-## Number of region workers.
+## Number of region workers. Defaults to half of the available CPU cores, rounded up.
+## @toml2docs:none-default="Auto"
 #+ num_workers = 8
 
 ## Request channel size of each worker.
@@ -699,6 +750,13 @@ compress_manifest = false
 ## Indexes are stored on the local filesystem under `{data_home}/series_index`.
 #+ experimental_enable_series_index = false
 
+## Approximate series and range index size limit in open regions, shared across workers.
+## Workers periodically refresh usage and skip maintenance when full. In-flight reconciliation
+## can exceed the limit. Closed-region files, temporary output, catalogs, and old snapshots
+## retained by readers are not counted.
+## Minimum: 1KiB. Takes effect on restart.
+#+ experimental_series_index_max_size = "5GiB"
+
 ## Whether to build and query range indexes when series indexes are enabled.
 ## Obsolete range-index metadata and files are still cleaned up when disabled.
 #+ experimental_enable_range_index = false
@@ -844,7 +902,7 @@ schedule_compaction_after_edit = true
 default_flat_format = true
 
 ## Whether to enable the experimental two-phase mode for eligible metric series scans.
-experimental_series_scan_v2 = true
+experimental_series_scan_v2 = false
 
 ## The options for index in Mito engine.
 [region_engine.mito.index]
@@ -1015,19 +1073,21 @@ default_ratio = 1.0
 ## The slow query log options.
 [slow_query]
 ## Whether to enable slow query log.
-#+ enable = false
+enable = true
 
 ## The record type of slow queries. It can be `system_table` or `log`.
-## @toml2docs:none-default
-#+ record_type = "system_table"
+## If `system_table` is selected, the slow queries will be recorded in a system table `greptime_private.slow_queries`.
+## If `log` is selected, the slow queries will be logged in a log file `greptimedb-slow-queries.*`.
+record_type = "system_table"
 
-## The threshold of slow query.
-## @toml2docs:none-default
-#+ threshold = "10s"
+## The threshold of slow query. It can be human readable time string, for example: `10s`, `100ms`, `1s`.
+threshold = "30s"
 
-## The sampling ratio of slow query log. The value should be in the range of (0, 1].
-## @toml2docs:none-default
-#+ sample_ratio = 1.0
+## The sampling ratio of slow query log. The value should be in the range of (0, 1]. For example, `0.1` means 10% of the slow queries will be logged and `1.0` means all slow queries will be logged.
+sample_ratio = 1.0
+
+## The TTL of the `slow_queries` system table. Default is `90d` when `record_type` is `system_table`.
+ttl = "90d"
 
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
 #+ [tracing]
@@ -1052,6 +1112,7 @@ ttl = "90d"
 ## `reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.
 ## When omitted, all current and future event types are recorded.
 ## Set to an empty array to disable event recording.
+## @toml2docs:none-default="All"
 #+ event_types = ["create_database"]
 
 ## The memory options.

cyborg/bin/follow-up-docs-issue.ts

@@ -17,7 +17,12 @@
 import * as core from '@actions/core'
 import {handleError, obtainClient} from "@/common";
 import {context} from "@actions/github";
-import {PullRequestEditedEvent, PullRequestEvent, PullRequestOpenedEvent} from "@octokit/webhooks-types";
+import {
+    PullRequestClosedEvent,
+    PullRequestEditedEvent,
+    PullRequestEvent,
+    PullRequestOpenedEvent
+} from "@octokit/webhooks-types";
 // @ts-expect-error moduleResolution:nodenext issue 54523
 import {RequestError} from "@octokit/request-error";
 
@@ -30,18 +35,59 @@ async function main() {
         throw new Error(`Only pull request event supported. ${context.eventName} is unsupported.`)
     }
 
-    const client = obtainClient("GITHUB_TOKEN")
-    const docsClient = obtainClient("DOCS_REPO_TOKEN")
     const payload = context.payload as PullRequestEvent
-    const { owner, repo, number, actor, title, html_url } = {
+    const { owner, repo, number, actor } = {
         owner: payload.pull_request.base.user.login,
         repo: payload.pull_request.base.repo.name,
         number: payload.pull_request.number,
-        title: payload.pull_request.title,
-        html_url: payload.pull_request.html_url,
         actor: payload.pull_request.user.login,
     }
-    const followUpDocs = checkPullRequestEvent(payload)
+
+    switch (payload.action) {
+        case "opened": {
+            const client = obtainClient("GITHUB_TOKEN")
+            await updateDocsLabels(client, owner, repo, number, checkPullRequestOpenedEvent(payload as PullRequestOpenedEvent))
+            break
+        }
+        case "edited": {
+            const followUpDocs = checkPullRequestEditedEvent(payload as PullRequestEditedEvent)
+            if (followUpDocs === undefined) {
+                core.info("Docs checkbox state unchanged; leaving labels as-is.")
+                return
+            }
+            const client = obtainClient("GITHUB_TOKEN")
+            await updateDocsLabels(client, owner, repo, number, followUpDocs)
+            break
+        }
+        case "closed": {
+            const event = payload as PullRequestClosedEvent
+            if (!event.pull_request.merged) {
+                core.info("PR closed without merging; no docs issue needed.")
+                return
+            }
+            // The docs-required label is the single source of truth. Coordinate
+            // with pending checkbox-driven label updates before deciding: an edit
+            // run may still be applying or removing the label, and the event
+            // payload's label snapshot may be stale.
+            const client = obtainClient("GITHUB_TOKEN")
+            const { data: pr } = await client.rest.pulls.get({ owner, repo, pull_number: number })
+            await waitForPendingLabelUpdates(client, owner, repo, pr.head.sha)
+            const { data: freshPr } = await client.rest.pulls.get({ owner, repo, pull_number: number })
+            const hasDocsLabel = freshPr.labels.some((label) => label.name === labelDocsRequired)
+            if (!hasDocsLabel) {
+                core.info(`Label ${labelDocsRequired} not present; no docs issue needed.`)
+                return
+            }
+            const docsClient = obtainClient("DOCS_REPO_TOKEN")
+            await createDocsIssue(docsClient, freshPr.title, freshPr.html_url, actor)
+            break
+        }
+        default:
+            throw new Error(`${payload.action} is unsupported.`)
+    }
+}
+
+async function updateDocsLabels(client: ReturnType<typeof obtainClient>, owner: string, repo: string, number: number, followUpDocs: boolean) {
     if (followUpDocs) {
         core.info("Follow up docs.")
         await client.rest.issues.removeLabel({
@@ -55,28 +101,6 @@ async function main() {
         await client.rest.issues.addLabels({
             owner, repo, issue_number: number, labels: [labelDocsRequired],
         })
-
-        // Get available assignees for the docs repo
-        const assigneesResponse = await docsClient.rest.issues.listAssignees({
-            owner: 'GreptimeTeam',
-            repo: 'docs',
-        })
-        const validAssignees = assigneesResponse.data.map(assignee => assignee.login)
-        core.info(`Available assignees: ${validAssignees.join(', ')}`)
-
-        // Check if the actor is a valid assignee, otherwise fallback to fengjiachun
-        const assignee = validAssignees.includes(actor) ? actor : 'fengjiachun'
-        core.info(`Assigning issue to: ${assignee}`)
-
-        await docsClient.rest.issues.create({
-            owner: 'GreptimeTeam',
-            repo: 'docs',
-            title: `Update docs for ${title}`,
-            body: `A document change request is generated from ${html_url}`,
-            assignee: assignee,
-        }).then((res) => {
-            core.info(`Created issue ${res.data}`)
-        })
     } else {
         core.info("No need to follow up docs.")
         await client.rest.issues.removeLabel({
@@ -93,15 +117,52 @@ async function main() {
     }
 }
 
-function checkPullRequestEvent(payload: PullRequestEvent) {
-    switch (payload.action) {
-        case "opened":
-            return checkPullRequestOpenedEvent(payload as PullRequestOpenedEvent)
-        case "edited":
-            return checkPullRequestEditedEvent(payload as PullRequestEditedEvent)
-        default:
-            throw new Error(`${payload.action} is unsupported.`)
+async function createDocsIssue(docsClient: ReturnType<typeof obtainClient>, title: string, html_url: string, actor: string) {
+    core.info("Creating follow-up docs issue for merged PR.")
+
+    // Get available assignees for the docs repo
+    const assigneesResponse = await docsClient.rest.issues.listAssignees({
+        owner: 'GreptimeTeam',
+        repo: 'docs',
+    })
+    const validAssignees = assigneesResponse.data.map(assignee => assignee.login)
+    core.info(`Available assignees: ${validAssignees.join(', ')}`)
+
+    // Check if the actor is a valid assignee, otherwise fallback to fengjiachun
+    const assignee = validAssignees.includes(actor) ? actor : 'fengjiachun'
+    core.info(`Assigning issue to: ${assignee}`)
+
+    await docsClient.rest.issues.create({
+        owner: 'GreptimeTeam',
+        repo: 'docs',
+        title: `Update docs for ${title}`,
+        body: `A document change request is generated from ${html_url}`,
+        assignee: assignee,
+    }).then((res) => {
+        core.info(`Created issue ${res.data}`)
+    })
+}
+
+// Waits until no other runs of this workflow are pending for the PR head SHA,
+// so that any checkbox-driven label update has been applied before the caller
+// reads the labels. Bounded by a deadline; on timeout it proceeds with whatever
+// the current label state is.
+async function waitForPendingLabelUpdates(client: ReturnType<typeof obtainClient>, owner: string, repo: string, headSha: string) {
+    const deadlineMs = 5 * 60 * 1000
+    const pollIntervalMs = 5000
+    const deadline = Date.now() + deadlineMs
+    while (Date.now() < deadline) {
+        const { data } = await client.rest.actions.listWorkflowRuns({
+            owner, repo, workflow_id: "docbot.yml", head_sha: headSha, per_page: 20,
+        })
+        const pending = data.workflow_runs.filter((run) => run.id !== context.runId && run.status !== "completed")
+        if (pending.length === 0) {
+            return
+        }
+        core.info(`Waiting for ${pending.length} pending docbot run(s) to finish label updates...`)
+        await new Promise((resolve) => setTimeout(resolve, pollIntervalMs))
     }
+    core.warning("Timed out waiting for pending docbot runs; proceeding with the current label state.")
 }
 
 function checkPullRequestOpenedEvent(event: PullRequestOpenedEvent): boolean {
@@ -109,11 +170,19 @@ function checkPullRequestOpenedEvent(event: PullRequestOpenedEvent): boolean {
     return event.pull_request.body?.includes(needFollowUpDocs)
 }
 
-function checkPullRequestEditedEvent(event: PullRequestEditedEvent): boolean {
-    const previous = event.changes.body?.from.includes(needFollowUpDocs)
-    const current = event.pull_request.body?.includes(needFollowUpDocs)
-    // from docs-not-need to docs-required
-    return (!previous) && current
+// Returns undefined when the checkbox state did not change in this edit, so the
+// caller leaves the labels untouched (preserving manual label overrides).
+function checkPullRequestEditedEvent(event: PullRequestEditedEvent): boolean | undefined {
+    if (!event.changes.body) {
+        // The body was not part of this edit (e.g. title-only edit).
+        return undefined
+    }
+    const previous = event.changes.body.from.includes(needFollowUpDocs)
+    const current = event.pull_request.body?.includes(needFollowUpDocs) ?? false
+    if (previous === current) {
+        return undefined
+    }
+    return current
 }
 
 main().catch(handleError)

docs/how-to/ai-functions.md

@@ -0,0 +1,247 @@
+# AI functions (experimental MVP)
+
+Three asynchronous SQL scalar functions provide natural-language matching,
+classification, and rating. The current backend is TypeSafe's Jev model:
+
+| Function | Mode | Result |
+| --- | --- | --- |
+| `ai_match(text, prompt)` | Noul | `Float64` probability in **[0, 1]** that the prompt's statement is true |
+| `ai_choose(text, prompt, criteria)` | Choice | `String` containing the selected option's name |
+| `ai_score(text, prompt, criteria)` | Score | JSONB object containing `score`, `confidence`, and a `probabilities` array |
+
+Use scalar results or extracted JSON fields in `SELECT`, comparisons in `WHERE`,
+and `ORDER BY`.
+All arguments are SQL strings. Any SQL NULL argument produces SQL `NULL`
+without validating that row's criteria or calling the API.
+
+## Start GreptimeDB
+
+The functions are compiled and registered by the **default-enabled Cargo feature
+`ai_functions`**. Enable the Jev backend separately in the server process:
+
+```sh
+export GREPTIMEDB_EXPERIMENTAL_JEV=true
+export JEV_API_KEY='<your TypeSafe API key>'
+cargo run -p cmd -- standalone start
+```
+
+If your key is already exported in `~/.zshrc`, run `source ~/.zshrc` first.
+The optional `JEV_MODEL` defaults to `jev-latest`; `JEV_ENDPOINT` defaults to
+`https://api.typesafe.ai/v1/systemone` (the full evaluation endpoint URL).
+The MVP uses environment variables rather than TOML configuration.
+The Cargo feature includes all three SQL functions in the build; the runtime environment
+variables enable and configure its API calls. Default compilation does not turn
+on external API calls. To enable the feature explicitly, use
+`--features ai_functions`.
+
+## Query
+
+For a new example table, quote `service` and `message` in the DDL because the
+current parser treats them as keywords:
+
+```sql
+CREATE TABLE events (
+    occurred_at TIMESTAMP TIME INDEX,
+    "service" STRING,
+    "message" STRING,
+    PRIMARY KEY ("service")
+);
+```
+
+### Matching probability (Noul)
+
+```sql
+SELECT occurred_at, service, message
+FROM events
+WHERE occurred_at >= '2026-09-19T00:00:00Z'
+  AND occurred_at <  '2026-09-20T00:00:00Z'
+  AND service = 'payments'
+  AND ai_match(
+    (message),
+    'The event reports that a payment still failed after retries.'
+  ) >= 0.8
+ORDER BY occurred_at;
+```
+
+To return the scores and rank matching events:
+
+```sql
+SELECT occurred_at, message,
+       ai_match(message, 'The event reports that a payment still failed after retries.') AS score
+FROM events
+WHERE occurred_at >= '2026-09-19T00:00:00Z'
+  AND occurred_at <  '2026-09-20T00:00:00Z'
+  AND service = 'payments'
+ORDER BY score DESC NULLS LAST;
+```
+
+Both arguments are strings; larger scores indicate a stronger match.
+`(message)` is an ordinary parenthesized string expression. For several columns,
+combine them explicitly, for example `concat(service, ': ', message)`.
+Any null argument produces SQL `NULL`, which `WHERE` excludes, without an API call.
+
+### Classification (Choice)
+
+`criteria` is a JSON object with **1 to 255 options**. Each key is an option name;
+its value is a description (string, object, or array), or JSON `null` if the name
+is sufficient. The function returns the selected key, suitable for filtering
+or grouping:
+
+```sql
+SELECT occurred_at, message,
+       ai_choose(message, 'Which team should handle this event?',
+                  '{"billing":"Payments, invoices, refunds","technical":"Bugs and outages","other":null}') AS team
+FROM events
+WHERE occurred_at >= '2026-09-19T00:00:00Z'
+  AND occurred_at <  '2026-09-20T00:00:00Z';
+```
+
+### Rating (Score)
+
+`criteria` is a JSON array of **2 to 10 ordered level descriptions**, from low
+to high. Descriptions can be strings, objects, or arrays. Level numbers start
+at zero. The function returns a JSONB object from one model evaluation:
+
+```json
+{
+  "score": 1.05,
+  "confidence": 0.92,
+  "probabilities": [0.0, 0.95, 0.05]
+}
+```
+
+- `score` is the provider's probability-weighted mean of the level numbers, in
+  `[0, N − 1]` for N levels. It may fall between levels.
+- `confidence` is the provider's confidence in `[0, 1]`, reflecting how concentrated
+  the distribution is. It is not a guarantee that the answer is correct.
+- `probabilities[i]` is the probability of `criteria[i]`, with exactly N entries in
+  level order. These are individual level probabilities, not cumulative probabilities.
+
+Different distributions can have the same score. `[0, 1, 0]` and `[0.5, 0, 0.5]`
+both have score `1`: the first concentrates on the middle level, while the second
+is split between the extremes. Inspect confidence and the distribution before
+treating a mean score as a definite severity level.
+
+For example, compute the rating once, extract numeric fields, and rank only rows
+meeting a chosen confidence threshold:
+
+```sql
+SELECT occurred_at, message,
+       json_get_float(rating, 'score') AS severity,
+       json_get_float(rating, 'confidence') AS confidence
+FROM (
+    SELECT occurred_at, message,
+           ai_score(message, 'How severe is this event?',
+                    '["No impact to functionality","Degraded service with a workaround","Blocking issue with no workaround"]') AS rating
+    FROM events
+    WHERE occurred_at >= '2026-09-19T00:00:00Z'
+      AND occurred_at <  '2026-09-20T00:00:00Z'
+) AS rated
+WHERE json_get_float(rating, 'confidence') >= 0.8
+ORDER BY severity DESC NULLS LAST;
+```
+
+Use `json_to_string(rating)` to display the full object, or
+`json_get_float(rating, 'probabilities[2]')` to read the probability of level 2.
+The array positions correspond to the supplied criteria, which provide the level
+descriptions.
+
+## Reusing an evaluation
+
+AI functions are marked volatile to prevent HTTP calls during planning. Identical
+calls written separately in `SELECT` and `WHERE` are evaluated separately. If the
+filter evaluates N non-null rows and M rows pass to the projection, this can cost
+N + M requests, reaching 2N when all rows pass.
+
+To return a result and filter by it, compute it once in a subquery and reference
+its alias in the outer query:
+
+```sql
+SELECT occurred_at, message, score
+FROM (
+    SELECT occurred_at, message,
+           ai_match(message, 'The event reports that a payment still failed after retries.') AS score
+    FROM events
+    WHERE occurred_at >= '2026-09-19T00:00:00Z'
+      AND occurred_at <  '2026-09-20T00:00:00Z'
+      AND service = 'payments'
+) AS scored
+WHERE score >= 0.8
+ORDER BY score DESC;
+```
+
+This also applies to `ai_choose` and `ai_score`. Extract multiple JSON fields from
+the aliased `ai_score` result, as in the rating example above, rather than calling
+the model again for each field. Filter pushdown preserves the volatile projection
+instead of duplicating it into the outer predicate.
+
+## MVP behavior
+
+- Each non-null row makes one HTTP request per function invocation: the text is
+  `state`, and the prompt is the question's `instructions`. Choice and Score pass
+  the parsed JSON as `criteria`. Calls to different functions are separate requests.
+- Criteria for all non-null rows in a batch are validated before sending that
+  batch's requests. Malformed JSON, invalid description types, or invalid option/
+  level counts fail the query locally.
+- Answers must have the requested question type. Noul probabilities outside
+  `[0, 1]`, Choice labels not in the criteria, and Score values outside `[0, N − 1]`
+  fail the query rather than being clamped or replaced with NULL. Score responses
+  must also contain a numeric confidence in `[0, 1]` and probabilities for every
+  level. Each probability must be numeric and in `[0, 1]`; their sum must be within
+  `1e-6` of 1. Provider values are preserved without renormalizing the distribution
+  or recomputing the score.
+- Up to eight requests run concurrently per expression/batch invocation, not per
+  query or process. Concurrent partitions and queries can exceed eight requests
+  in total. Each request has a 30-second timeout. Errors (including rate limits
+  and invalid responses) fail the query. There is no automatic retry or
+  cross-query cache.
+- Start with small, time-bounded queries. Ordinary SQL predicates can reduce the
+  candidate set, but SQL does not guarantee left-to-right predicate evaluation or
+  that `LIMIT` bounds the number of API calls.
+- Text passed to the function is sent to the configured TypeSafe endpoint.
+  The API key remains in the server environment, not in SQL.
+- This MVP is intended for standalone use. A distributed deployment would need
+  the environment on every node evaluating the function and separate validation.
+
+API reference: <https://docs.typesafe.ai/api>
+
+## Before stabilizing
+
+The experimental MVP still needs the following controls before stabilization:
+
+- A process-wide concurrency limit shared by AI function invocations, such as a semaphore.
+- Bounded retries with exponential backoff for HTTP `429` and `529`, following
+  TypeSafe's rate-limit guidance.
+- Request budgets and metrics for API calls, latency, retries, and rate-limit errors.
+
+These are follow-up work, not guarantees provided by the current implementation.
+
+## Validation
+
+Check both the default build (where all three functions are registered) and an isolated
+`common-function` build without default features. The regular AI function tests use a local
+HTTP server and need no API key:
+
+```sh
+cargo nextest run -p common-function
+cargo nextest run -p common-function --no-default-features
+cargo nextest run -p common-function --no-default-features --features ai_functions
+```
+
+The sqlness runner explicitly includes `ai_functions` when building its test binary:
+
+```sh
+cargo sqlness bare -t ai_functions
+```
+
+If using `--bins-dir`, provide a binary built with `ai_functions` (included by
+default). CI covers the AI-enabled path through the regular unit tests and
+sqlness tests; the feature-off checks above can be run locally when needed.
+
+An opt-in test calls the real service on synthetic payment events:
+
+```sh
+source ~/.zshrc
+GREPTIMEDB_EXPERIMENTAL_JEV=true cargo nextest run -p common-function \
+  -E 'test(test_ai_match_live)' --run-ignored ignored-only
+```

docs/how-to/jev.md

@@ -1,119 +0,0 @@
-# Natural-language filtering with Jev (experimental MVP)
-
-`jev(text, statement, threshold)` returns whether Jev judges the statement true
-for the text with probability **greater than or equal to** the threshold.
-It is an asynchronous SQL scalar function usable in `WHERE` and `SELECT`.
-
-## Start GreptimeDB
-
-Jev is compiled and registered by the **default-enabled Cargo feature
-`ai_functions`**. Enable API evaluation separately in the server process:
-
-```sh
-export GREPTIMEDB_EXPERIMENTAL_JEV=true
-export JEV_API_KEY='<your TypeSafe API key>'
-cargo run -p cmd -- standalone start
-```
-
-If your key is already exported in `~/.zshrc`, run `source ~/.zshrc` first.
-The optional `JEV_MODEL` defaults to `jev-latest`; `JEV_ENDPOINT` defaults to
-`https://api.typesafe.ai/v1/systemone` (the full evaluation endpoint URL).
-The MVP uses environment variables rather than TOML configuration.
-The Cargo feature includes the SQL function in the build; the runtime environment
-variables enable and configure its API calls. Default compilation does not turn
-on external API calls. To enable the feature explicitly, use
-`--features ai_functions`.
-
-## Query
-
-For a new example table, quote `service` and `message` in the DDL because the
-current parser treats them as keywords:
-
-```sql
-CREATE TABLE events (
-    occurred_at TIMESTAMP TIME INDEX,
-    "service" STRING,
-    "message" STRING,
-    PRIMARY KEY ("service")
-);
-```
-
-```sql
-SELECT occurred_at, service, message
-FROM events
-WHERE occurred_at >= '2026-09-19T00:00:00Z'
-  AND occurred_at <  '2026-09-20T00:00:00Z'
-  AND service = 'payments'
-  AND jev(
-    (message),
-    'The event reports that a payment still failed after retries.',
-    0.8
-  )
-ORDER BY occurred_at;
-```
-
-The first two arguments are strings and the third is a number in `[0, 1]`.
-`(message)` is an ordinary parenthesized string expression. For several columns,
-combine them explicitly, for example `concat(service, ': ', message)`.
-Any null argument produces SQL `NULL`, which `WHERE` excludes, without an API call.
-
-## MVP behavior
-
-- Each non-null row makes one HTTP request: the text is `state`, and the statement
-  is a `noul` question's `instructions`. The returned `noul` probability is compared
-  with the threshold locally; this is not the separate Choice/Score confidence.
-- Up to eight requests run concurrently per expression/batch invocation, not per
-  query or process. Concurrent partitions and queries can exceed eight requests
-  in total. Each request has a 30-second timeout. Errors (including rate limits
-  and invalid responses) fail the query. There is no automatic retry or
-  cross-query cache.
-- Start with small, time-bounded queries. Ordinary SQL predicates can reduce the
-  candidate set, but SQL does not guarantee left-to-right predicate evaluation or
-  that `LIMIT` bounds the number of API calls.
-- Text passed to the function is sent to the configured TypeSafe endpoint.
-  The API key remains in the server environment, not in SQL.
-- This MVP is intended for standalone use. A distributed deployment would need
-  the environment on every node evaluating the function and separate validation.
-
-API reference: <https://docs.typesafe.ai/api>
-
-## Before stabilizing
-
-The experimental MVP still needs the following controls before stabilization:
-
-- A process-wide concurrency limit shared by Jev invocations, such as a semaphore.
-- Bounded retries with exponential backoff for HTTP `429` and `529`, following
-  TypeSafe's rate-limit guidance.
-- Request budgets and metrics for API calls, latency, retries, and rate-limit errors.
-
-These are follow-up work, not guarantees provided by the current implementation.
-
-## Validation
-
-Check both the default build (where `jev` is registered) and an isolated
-`common-function` build without default features. Jev's regular tests use a local
-HTTP server and need no API key:
-
-```sh
-cargo nextest run -p common-function
-cargo nextest run -p common-function --no-default-features
-cargo nextest run -p common-function --no-default-features --features ai_functions
-```
-
-The sqlness runner explicitly includes `ai_functions` when building its test binary:
-
-```sh
-cargo sqlness bare -t jev
-```
-
-If using `--bins-dir`, provide a binary built with `ai_functions` (included by
-default). CI covers the AI-enabled path through the regular unit tests and
-sqlness tests; the feature-off checks above can be run locally when needed.
-
-An opt-in test calls the real service on synthetic payment events:
-
-```sh
-source ~/.zshrc
-GREPTIMEDB_EXPERIMENTAL_JEV=true cargo nextest run -p common-function \
-  -E 'test(test_jev_live)' --run-ignored ignored-only
-```

docs/rfcs/2026-05-28-table-semantic-layer.md

@@ -93,13 +93,25 @@ Two design decisions worth pinning down up front, because they constrain everyth
 - **Conflict.** Some table-level keys (`trace.conventions` lifted from `schema_url`, `metric.temporality`, ...) cannot represent the truth when a long-lived table sees rows from multiple sources. v1 records `mixed` or `unknown` rather than a fictitious single value. Downstream consumers must treat any single-valued semantic key as best-effort, not strong evidence.
 - **Update.** Semantic options are stamped at table creation. v1 does not specify an update path; promoting `metadata_quality` from `inferred` to `declared`, refreshing `resource.attributes_preserved`, or revising `trace.conventions` on later writes is deferred. If real usage shows update is needed, it lands as a separate RFC.
 
+OTLP delta sums and explicit histograms store raw interval values without
+accumulating across timestamps. Delta-to-cumulative conversion and its
+per-series ingestion state are intentionally out of scope. Explicit histogram
+bucket counts are prefix-summed only within each point to produce the classic
+histogram representation. Delta exponential histograms are currently rejected;
+[planned raw-delta support](2026-08-04-native-histograms.md#planned-raw-delta-exponential-histograms)
+will store their interval values in the canonical native-histogram Struct using
+the same temporality tag and a gauge reset hint. The tag will select histogram
+summation for `increase()` and summation divided by range duration for `rate()`,
+without cumulative reset correction or extrapolation.
+
 OTLP delta sums and explicit histograms additionally store the query-visible
 String tag `otlp_aggregation_temporality="delta"` on each generated row. Its
 name is fixed and does not follow `default_column_prefix`. The tag is part of
 series identity and is authoritative for per-series float `rate()` and
 `increase()` behavior; the table option is never used as a row-level
-discriminator. Native histograms retain their native algorithms. Prometheus
-metadata reports `unknown` for counter, histogram, and up/down-counter tables
+discriminator. Native histograms currently retain their native algorithms;
+the planned delta extension will change only tagged delta histogram series.
+Prometheus metadata reports `unknown` for counter, histogram, and up/down-counter tables
 whose catalog temporality is `delta` or `mixed`. A same-request conflict can
 create `mixed`; a later write does not update an existing table option, so the
 catalog value can also be stale while the row tag remains authoritative.

docs/rfcs/2026-08-04-native-histograms.md

@@ -1,6 +1,6 @@
 ---
 Feature Name: Native Histogram Support and Compatibility Decisions
-Tracking Issue: TBD
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/8887
 Date: 2026-08-04
 Author: codex
 ---
@@ -13,8 +13,8 @@ field and evaluates them as first-class PromQL samples. Prometheus Remote Write
 ingestion transport: accepted cumulative points are normalized into the same
 Struct before persistence and are queried only as native histograms.
 
-Native histograms are experimental. Prometheus Remote Write 2.0 and cumulative
-OTLP exponential histograms are supported behind separate configuration gates.
+Prometheus Remote Write 2.0 native histograms and cumulative OTLP/HTTP
+exponential histograms are enabled by default.
 Remote Write 1.0 histogram payloads are rejected instead of being acknowledged and dropped.
 Native-histogram Remote Read is deferred; the existing Remote Read path
 continues to return scalar samples only.
@@ -34,6 +34,9 @@ continues to return scalar samples only.
 - Returning native histograms through Prometheus Remote Read.
 - Persisting Remote Write metric metadata.
 - Persisting exemplars.
+- Converting OTLP delta values to cumulative values during ingestion. Supported
+  delta metrics are stored as interval values without per-series accumulation
+  state; this also applies to the planned exponential-histogram support.
 - Providing an OTLP-specific histogram query surface or reconstructing and
   re-exporting the original OTLP point after persistence.
 - Removing mixed float/histogram handling from PromQL expressions.
@@ -95,8 +98,7 @@ that send native histograms must use Remote Write 2.0.
 
 ## Remote Write 2.0
 
-Remote Write 2.0 accepts integer and float native histograms while
-`prom_store.experimental_enable_prometheus_native_histogram` is enabled. Supported
+Remote Write 2.0 accepts integer and float native histograms by default. Supported
 exponential schemas are `-4` through `8`; schema `-53` represents native
 histograms with custom buckets.
 
@@ -121,15 +123,23 @@ explicit if sampled responses are implemented.
 
 ## OTLP
 
-OTLP exponential histograms are accepted when
-`otlp.experimental_enable_exponential_histogram` is enabled. The option defaults
-to false and applies to OTLP/HTTP. Disabled points are rejected rather than
-silently acknowledged. OTel Arrow exponential histograms are rejected because
+OTLP/HTTP exponential histograms are accepted by default.
+OTel Arrow exponential histograms are rejected because
 the current Arrow wire format omits `zero_threshold`; accepting them would
-silently change the distribution. Cumulative temporality is required; delta and
-unspecified exponential histograms are rejected before their points are
+silently change the distribution. Currently, cumulative temporality is required;
+delta and unspecified exponential histograms are rejected before their points are
 converted. Explicit OTLP histograms keep their existing `_bucket`, `_sum`, and
-`_count` representation, including their existing delta behavior.
+`_count` representation. OTLP delta sums and explicit histograms are stored as
+raw interval values with `otlp_aggregation_temporality="delta"`. Explicit
+histogram bucket counts are prefix-summed within each point for the classic
+histogram representation, but values are never accumulated across timestamps.
+For these tagged float series, PromQL `increase()` sums the interval values
+in the query range, and `rate()` divides that sum by the range duration.
+Delta-to-cumulative conversion is intentionally out of scope, not deferred work.
+Raw-delta exponential-histogram support is planned below; delta exponential
+points remain rejected until it is implemented. See
+[Table Semantic Layer](2026-05-28-table-semantic-layer.md#conflict-and-update-semantics)
+for the per-series temporality contract.
 
 OTLP scales `-4` through `8` map directly to Prometheus schemas. Higher scales
 are downscaled to schema `8`: dense counts that collide are merged before the
@@ -159,8 +169,36 @@ for mixed batches and `INVALID_ARGUMENT` when all points are rejected. Rejection
 details are bounded, and metric metadata is emitted only for a metric that
 produced an accepted row.
 
-Minimum, maximum, and exemplars are not persisted. Delta accumulation,
-zero-run span compaction, and a dedicated rejection metric remain deferred.
+Zero-count buckets are omitted after downscaling, splitting nonzero runs into
+sparse spans without changing integer counts. Rejected data points increment
+`greptime_servers_otlp_exponential_histogram_rejected_data_points_total`, labeled
+by `reason`: `delta_temporality`, `unspecified_temporality`, or
+`invalid_data_point`. Each rejected point is counted once, including when
+resource descriptors are enabled. Minimum, maximum, and exemplars are not persisted.
+
+### Planned raw-delta exponential histograms
+
+Accept OTLP delta exponential histograms and store each point's interval counts
+and sum in the existing canonical native-histogram Struct, without accumulating
+across timestamps. Reuse the integer-count encoding, scale conversion,
+zero-threshold preservation, stale-marker handling, and partial-success path.
+OTel Arrow support remains blocked independently by its missing `zero_threshold`.
+
+Store `otlp_aggregation_temporality="delta"` as part of series identity, as for
+delta sums and explicit histograms. Use the gauge reset hint for delta histogram
+samples so decreasing interval counts are not interpreted as counter resets.
+Prometheus uses the same gauge hint for
+[raw-delta exponential-histogram ingestion](https://github.com/prometheus/prometheus/blob/9f159f6df198280b757163ec555ffde0c8bfe8e0/storage/remote/otlptranslator/prometheusremotewrite/histograms.go#L125).
+The temporality tag, rather than the gauge hint alone or table metadata,
+determines raw-delta query behavior.
+
+Extend the per-series `increase()` and `rate()` selection to native histogram
+inputs: sum interval histograms over the query range, then divide by the range
+duration for `rate()`. Do not apply cumulative reset correction, synthetic-zero
+insertion, or counter extrapolation to these delta samples. Preserve cumulative
+histogram behavior and mixed float/histogram range warnings. Acceptance coverage
+must include multiple intervals whose counts decrease, exact integer ingestion,
+stale points, partial rejection, and pure-native and mixed-range query paths.
 
 # PromQL Compatibility Decisions
 
@@ -276,7 +314,8 @@ native-histogram data remains readable.
 
 - Native-histogram Remote Read, including exact integer round-trips and streamed
   chunks with start timestamps.
-- OTLP exponential-histogram delta accumulation.
+- Raw-delta OTLP exponential-histogram storage and temporality-aware native
+  histogram queries, as described above; no delta-to-cumulative conversion.
 - Persistent Remote Write metadata and accurate help/unit updates.
 - Native-histogram exemplars and exemplar query APIs.
 - Start-timestamp overlap annotations.
@@ -284,3 +323,10 @@ native-histogram data remains readable.
 - Versioned two-phase native-histogram aggregation state.
 - Exact Prometheus summation compensation if measured precision differences
   justify the extra aggregate state.
+
+## Configuration migration
+
+The `prom_store.experimental_enable_prometheus_native_histogram` and
+`otlp.experimental_enable_exponential_histogram` options have been removed.
+Remove them from existing configurations; both ingestion paths are always enabled
+when their protocol is enabled. Old option values are ignored, including `false`.

docs/rfcs/2026-09-11-metric-export-import.md

@@ -390,14 +390,34 @@ validation, schema-option inheritance and target metadata resolution. Bound the
 request size and the importer's DDL buffering separately.
 
 The production CLI needs an authenticated server operation that checks CREATE
-authorization for every member and reuses the ordinary DDL path. Transport,
-capability discovery and request limits must be agreed before accepting this
-interface. Initially bound a batch to 128 statements and 1 MiB of SQL, with an
-explicit oversized-statement error; separately bound the importer's parsed DDL
-buffer. A target without batch-DDL capability uses ordinary DDL if it supports
+authorization for every member and reuses the ordinary DDL path. Bound the
+importer's parsed DDL buffer separately from the server request limits below.
+A target without batch-DDL capability uses ordinary DDL if it supports
 the snapshot's data layout. Do not retry an ambiguously completed batch through
 another path.
 
+The server operation is `POST /v1/ddl/logical-tables` with an
+`application/x-www-form-urlencoded` body containing `sql` and optional `db`.
+`db` uses the same catalog/schema interpretation and defaults as `/v1/sql`.
+Authenticated `GET /v1/capabilities` advertises `metric_batch_ddl: 1` when the
+SQL handler supports the operation, independently of packed import/export.
+The form body is limited to 4 MiB before decoding, or the configured HTTP body
+limit when that limit is stricter. Decoded SQL is limited to 1 MiB and 1–128
+statements. Only explicit Metric logical CREATE TABLE statements in one
+catalog/schema with one literal `on_physical_table` value are accepted;
+duplicate normalized target names are rejected.
+
+All members pass SQL interception, authorization and ordinary CREATE preparation
+before one logical-table batch procedure is submitted. Success uses the normal
+GreptimeDB v1 response with one `affectedrows: 0` output per input statement in
+order; this is CREATE's output, not a count of newly created tables. A failure
+returns one error without a partial success list. Validation failures before
+submission have no DDL side effects. Once submitted, the existing procedure may
+have applied some or all changes even if the HTTP request times out or disconnects;
+there is no cross-table rollback guarantee. `IF NOT EXISTS` keeps ordinary CREATE
+semantics. The operation does not regroup requests or retry through single-table
+CREATE.
+
 ## Packed data COPY
 
 The CLI retains one `COPY DATABASE FROM` operation per `(chunk, schema)` task.

flake.lock

@@ -8,11 +8,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1785314864,
-        "narHash": "sha256-imz9J5iMNersDeQ1ymenFg35N6C2R3MCaGg2i0XCIrE=",
+        "lastModified": 1790642863,
+        "narHash": "sha256-JIwE6EnebvxW7QFFEitWHuZDXwN6Qcf6ZC8DIvYW3y8=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "594418b2c9ea0731bee6988236ef5bfd97e81dca",
+        "rev": "a8f386c482e78e9a48be5abecdc17a9b098a95a1",
         "type": "github"
       },
       "original": {
@@ -41,11 +41,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1785133411,
-        "narHash": "sha256-Yjv0WEg39KRYS0rBdTbu6Fc/or/ihAKk13W9sQ6VWd0=",
+        "lastModified": 1790533781,
+        "narHash": "sha256-F+b8u43BVF5DlM0pz7JQSbtbwKy5PrHIT5zQUekXsD0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "2f5a153c270b70cb0f8c11f46d96d6d3bc39f4e3",
+        "rev": "cf5e76507c6e23b59f7e0ffcc7baa2a39ddd8442",
         "type": "github"
       },
       "original": {
@@ -65,11 +65,11 @@
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1785261141,
-        "narHash": "sha256-sA+DHPejWD68mLA7gtTiHGd5T41F6W+NKf0g+ibFCW4=",
+        "lastModified": 1790532271,
+        "narHash": "sha256-j8y/AO3AMDF9p+11aIDhnA21JnNj5PjvcejTI43vI8Q=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "bec66814323579659ffd77c909b3d963af118ece",
+        "rev": "03fcb77246f2568adb0e9b2fa60d19c6cc1686f4",
         "type": "github"
       },
       "original": {

flake.nix

@@ -22,7 +22,7 @@
         lib = nixpkgs.lib;
         rustToolchain = fenix.packages.${system}.fromToolchainName {
           name = (lib.importTOML ./rust-toolchain.toml).toolchain.channel;
-          sha256 = "sha256-rboGKQLH4eDuiY01SINOqmXUFUNr9F4awoFZGzib17o=";
+          sha256 = "sha256-p8h3Sl/YRByZfZTAKXdsvF6xEenXKrXSVvpphmZENH4=";
         };
       in
       {
@@ -35,6 +35,7 @@
             protobuf
             gnumake
             mold
+            wild
             (rustToolchain.withComponents [
               "cargo"
               "clippy"
@@ -46,7 +47,7 @@
             ])
             cargo-nextest
             cargo-llvm-cov
-            cargo-udeps
+            cargo-shear
             taplo
             curl
             gnuplot ## for cargo bench

grafana/dashboards/metrics/cluster/dashboard.json

@@ -3585,19 +3585,6 @@
       },
       "pluginVersion": "11.6.0",
       "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${metrics}"
-          },
-          "editorMode": "code",
-          "expr": "sum by (code) (rate(greptime_flow_errors[$__rate_interval]))",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "flow-{{code}}",
-          "range": true,
-          "refId": "A"
-        },
         {
           "datasource": {
             "type": "prometheus",
@@ -11583,7 +11570,7 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (type) (rate(greptime_mito_cache_hit{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))",
+              "expr": "sum by (type) (rate(greptime_mito_cache_hit{type=~\"index.*|index_result\"}[$__rate_interval]))",
               "hide": false,
               "instant": false,
               "legendFormat": "hit-{{type}}",
@@ -11596,7 +11583,7 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (type) (rate(greptime_mito_cache_miss{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))",
+              "expr": "sum by (type) (rate(greptime_mito_cache_miss{type=~\"index.*|index_result\"}[$__rate_interval]))",
               "hide": false,
               "instant": false,
               "legendFormat": "miss-{{type}}",
@@ -11609,7 +11596,7 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))",
+              "expr": "sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~\"index.*|index_result\"}[$__rate_interval]))",
               "hide": false,
               "instant": false,
               "legendFormat": "evict-{{type}}-{{cause}}",
@@ -16622,127 +16609,6 @@
           "title": "Flow Ingest / Output Rate",
           "type": "timeseries"
         },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${metrics}"
-          },
-          "description": "Flow Ingest Latency.",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "points",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "auto",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              }
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 8,
-            "w": 8,
-            "x": 8,
-            "y": 170
-          },
-          "id": 521,
-          "options": {
-            "legend": {
-              "calcs": [],
-              "displayMode": "list",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
-            }
-          },
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "${metrics}"
-              },
-              "editorMode": "code",
-              "expr": "histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))",
-              "instant": false,
-              "legendFormat": "[{{instance}}]-[{{pod}}]-p95",
-              "range": true,
-              "refId": "A"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "${metrics}"
-              },
-              "editorMode": "code",
-              "expr": "histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))",
-              "hide": false,
-              "instant": false,
-              "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
-              "range": true,
-              "refId": "B"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "${metrics}"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))",
-              "hide": false,
-              "instant": false,
-              "legendFormat": "[{{instance}}]-[{{pod}}]-avg",
-              "range": true,
-              "refId": "C"
-            }
-          ],
-          "title": "Flow Ingest Latency",
-          "type": "timeseries"
-        },
         {
           "datasource": {
             "type": "prometheus",
@@ -16805,7 +16671,7 @@
           "gridPos": {
             "h": 8,
             "w": 8,
-            "x": 16,
+            "x": 8,
             "y": 170
           },
           "id": 522,
@@ -16851,196 +16717,6 @@
           "title": "Flow Operation Latency",
           "type": "timeseries"
         },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${metrics}"
-          },
-          "description": "Flow Buffer Size per Instance.",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "auto",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              }
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 8,
-            "w": 8,
-            "x": 0,
-            "y": 178
-          },
-          "id": 523,
-          "options": {
-            "legend": {
-              "calcs": [],
-              "displayMode": "list",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
-            }
-          },
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "${metrics}"
-              },
-              "editorMode": "code",
-              "expr": "greptime_flow_input_buf_size",
-              "instant": false,
-              "legendFormat": "[{{instance}}]-[{{pod}}]",
-              "range": true,
-              "refId": "A"
-            }
-          ],
-          "title": "Flow Buffer Size per Instance",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${metrics}"
-          },
-          "description": "Flow Processing Error per Instance.",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "auto",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              }
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 8,
-            "w": 8,
-            "x": 8,
-            "y": 178
-          },
-          "id": 524,
-          "options": {
-            "legend": {
-              "calcs": [],
-              "displayMode": "list",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
-            }
-          },
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "${metrics}"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))",
-              "instant": false,
-              "legendFormat": "[{{instance}}]-[{{pod}}]-[{{code}}]",
-              "range": true,
-              "refId": "A"
-            }
-          ],
-          "title": "Flow Processing Error per Instance",
-          "type": "timeseries"
-        },
         {
           "datasource": {
             "type": "prometheus",
@@ -17105,7 +16781,7 @@
             "h": 8,
             "w": 8,
             "x": 16,
-            "y": 178
+            "y": 170
           },
           "id": 558,
           "options": {
@@ -17229,7 +16905,7 @@
             "h": 8,
             "w": 8,
             "x": 0,
-            "y": 186
+            "y": 178
           },
           "id": 559,
           "options": {
@@ -17353,7 +17029,7 @@
             "h": 8,
             "w": 8,
             "x": 8,
-            "y": 186
+            "y": 178
           },
           "id": 560,
           "options": {
@@ -17464,7 +17140,7 @@
             "h": 8,
             "w": 8,
             "x": 16,
-            "y": 186
+            "y": 178
           },
           "id": 561,
           "options": {
@@ -17575,7 +17251,7 @@
             "h": 8,
             "w": 8,
             "x": 0,
-            "y": 194
+            "y": 186
           },
           "id": 562,
           "options": {
@@ -17673,7 +17349,7 @@
             "h": 8,
             "w": 8,
             "x": 8,
-            "y": 194
+            "y": 186
           },
           "id": 563,
           "options": {
@@ -17771,7 +17447,7 @@
             "h": 8,
             "w": 8,
             "x": 16,
-            "y": 194
+            "y": 186
           },
           "id": 564,
           "options": {

grafana/dashboards/metrics/cluster/dashboard.md

@@ -36,7 +36,7 @@
 | Scan and Compaction Memory Rejects | `sum(rate(greptime_mito_scan_requests_rejected_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_scan_memory_exhausted_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_compaction_memory_rejected_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Datanode scan and compaction memory rejection/exhaustion counters. | `prometheus` | `rps` | `scan-rejected` |
 | OpenDAL Errors | `sum by (scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode",error!="NotFound"}[$__rate_interval]))` | `timeseries` | Object-store errors by scheme, operation, and error, excluding NotFound noise. | `prometheus` | `eps` | `{{scheme}}-{{operation}}-{{error}}` |
 | Metasrv Failures | `sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))`<br/>`sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))` | `timeseries` | Region migration and reconciliation failures in metasrv. | `prometheus` | `eps` | `migration-fail` |
-| Flow and Trigger Failures | `sum by (code) (rate(greptime_flow_errors[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_flow_batching_error_count[$__rate_interval]))` | `timeseries` | Derived-data and alerting pipeline failures. | `prometheus` | `eps` | `flow-{{code}}` |
+| Flow and Trigger Failures | `sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_flow_batching_error_count[$__rate_interval]))` | `timeseries` | Derived-data and alerting pipeline failures. | `prometheus` | `eps` | `trigger-evaluate` |
 | Mito GC Failures | `sum(rate(greptime_mito_gc_errors_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_orphaned_index_files{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito garbage-collection errors and skipped/orphaned files on datanodes. | `prometheus` | `short` | `gc-errors` |
 # Capacity
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
@@ -131,7 +131,7 @@
 | Index Memory Usage | `greptime_index_apply_memory_usage`<br/>`sum by (type) (greptime_index_create_memory_usage)` | `timeseries` | Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work. | `prometheus` | `bytes` | `apply` |
 | Index IO Bytes | `sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))` | `timeseries` | Index read and write byte rates by operation and file type for puffin and intermediate files. | `prometheus` | `Bps` | `{{type}}-{{file_type}}` |
 | Index IO Operations | `sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))` | `timeseries` | Index IO operation rates by operation and file type, including read, write, seek, and flush operations. | `prometheus` | `ops` | `{{type}}-{{file_type}}` |
-| Index Cache | `sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`<br/>`sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`<br/>`sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))` | `timeseries` | Index-related cache hits, misses, and evictions from Mito caches. | `prometheus` | `ops` | `hit-{{type}}` |
+| Index Cache | `sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*\|index_result"}[$__rate_interval]))`<br/>`sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*\|index_result"}[$__rate_interval]))`<br/>`sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*\|index_result"}[$__rate_interval]))` | `timeseries` | Index-related cache hits, misses, and evictions from Mito caches. | `prometheus` | `ops` | `hit-{{type}}` |
 # Metasrv
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
 | --- | --- | --- | --- | --- | --- | --- |
@@ -278,10 +278,7 @@ ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
 | --- | --- | --- | --- | --- | --- | --- |
 | Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` |
-| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`<br/>`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`<br/>`sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` |
 | Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`<br/>`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
-| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
-| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` |
 | Flow Batching Attempt / Error Rate | `sum by(instance,pod,flow_id) (rate(greptime_flow_batching_start_query_count[$__rate_interval]))`<br/>`sum by(instance,pod,flow_id) (rate(greptime_flow_batching_error_count[$__rate_interval]))`<br/>`sum by(instance,pod,flow_id) (rate(greptime_flow_batching_engine_slow_query_secs_count[$__rate_interval]))` | `timeseries` | Flow batching mode task attempt, error, and slow-query rates by flow. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[flow={{flow_id}}]-attempt` |
 | Flow Batching Query Latency | `histogram_quantile(0.95, sum by(le,instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_time_secs_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by(le,instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_time_secs_bucket[$__rate_interval])))`<br/>`sum by(instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_time_secs_sum[$__rate_interval])) / sum by(instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_time_secs_count[$__rate_interval]))` | `timeseries` | Flow batching mode query latency by flow. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[flow={{flow_id}}]-p95` |
 | Flow Batching Query Window Count | `histogram_quantile(0.95, sum by(le,instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_window_cnt_bucket[$__rate_interval])))`<br/>`sum by(instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_window_cnt_sum[$__rate_interval])) / sum by(instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_window_cnt_count[$__rate_interval]))` | `timeseries` | Flow batching mode query window count by flow. | `prometheus` | `short` | `[{{instance}}]-[{{pod}}]-[flow={{flow_id}}]-p95` |

grafana/dashboards/metrics/cluster/dashboard.yaml

@@ -515,11 +515,6 @@ groups:
           description: Derived-data and alerting pipeline failures.
           unit: eps
           queries:
-            - expr: sum by (code) (rate(greptime_flow_errors[$__rate_interval]))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: flow-{{code}}
             - expr: sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))
               datasource:
                 type: prometheus
@@ -1572,17 +1567,17 @@ groups:
           description: Index-related cache hits, misses, and evictions from Mito caches.
           unit: ops
           queries:
-            - expr: sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*|vector_index|index_result"}[$__rate_interval]))
+            - expr: sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*|index_result"}[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: hit-{{type}}
-            - expr: sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*|vector_index|index_result"}[$__rate_interval]))
+            - expr: sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*|index_result"}[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: miss-{{type}}
-            - expr: sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*|vector_index|index_result"}[$__rate_interval]))
+            - expr: sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*|index_result"}[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
@@ -2199,25 +2194,6 @@ groups:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '[{{pod}}]-[{{instance}}]-[{{direction}}]'
-        - title: Flow Ingest Latency
-          type: timeseries
-          description: Flow Ingest Latency.
-          queries:
-            - expr: histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-p95'
-            - expr: histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-p99'
-            - expr: sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-avg'
         - title: Flow Operation Latency
           type: timeseries
           description: Flow Operation Latency.
@@ -2232,24 +2208,6 @@ groups:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99'
-        - title: Flow Buffer Size per Instance
-          type: timeseries
-          description: Flow Buffer Size per Instance.
-          queries:
-            - expr: greptime_flow_input_buf_size
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]'
-        - title: Flow Processing Error per Instance
-          type: timeseries
-          description: Flow Processing Error per Instance.
-          queries:
-            - expr: sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{code}}]'
         - title: Flow Batching Attempt / Error Rate
           type: timeseries
           description: Flow batching mode task attempt, error, and slow-query rates by flow.

grafana/dashboards/metrics/standalone/dashboard.json

@@ -3585,19 +3585,6 @@
       },
       "pluginVersion": "11.6.0",
       "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${metrics}"
-          },
-          "editorMode": "code",
-          "expr": "sum by (code) (rate(greptime_flow_errors[$__rate_interval]))",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "flow-{{code}}",
-          "range": true,
-          "refId": "A"
-        },
         {
           "datasource": {
             "type": "prometheus",
@@ -11583,7 +11570,7 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (type) (rate(greptime_mito_cache_hit{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))",
+              "expr": "sum by (type) (rate(greptime_mito_cache_hit{type=~\"index.*|index_result\"}[$__rate_interval]))",
               "hide": false,
               "instant": false,
               "legendFormat": "hit-{{type}}",
@@ -11596,7 +11583,7 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (type) (rate(greptime_mito_cache_miss{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))",
+              "expr": "sum by (type) (rate(greptime_mito_cache_miss{type=~\"index.*|index_result\"}[$__rate_interval]))",
               "hide": false,
               "instant": false,
               "legendFormat": "miss-{{type}}",
@@ -11609,7 +11596,7 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))",
+              "expr": "sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~\"index.*|index_result\"}[$__rate_interval]))",
               "hide": false,
               "instant": false,
               "legendFormat": "evict-{{type}}-{{cause}}",
@@ -16622,127 +16609,6 @@
           "title": "Flow Ingest / Output Rate",
           "type": "timeseries"
         },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${metrics}"
-          },
-          "description": "Flow Ingest Latency.",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "points",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "auto",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              }
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 8,
-            "w": 8,
-            "x": 8,
-            "y": 170
-          },
-          "id": 521,
-          "options": {
-            "legend": {
-              "calcs": [],
-              "displayMode": "list",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
-            }
-          },
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "${metrics}"
-              },
-              "editorMode": "code",
-              "expr": "histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))",
-              "instant": false,
-              "legendFormat": "[{{instance}}]-[{{pod}}]-p95",
-              "range": true,
-              "refId": "A"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "${metrics}"
-              },
-              "editorMode": "code",
-              "expr": "histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))",
-              "hide": false,
-              "instant": false,
-              "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
-              "range": true,
-              "refId": "B"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "${metrics}"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))",
-              "hide": false,
-              "instant": false,
-              "legendFormat": "[{{instance}}]-[{{pod}}]-avg",
-              "range": true,
-              "refId": "C"
-            }
-          ],
-          "title": "Flow Ingest Latency",
-          "type": "timeseries"
-        },
         {
           "datasource": {
             "type": "prometheus",
@@ -16805,7 +16671,7 @@
           "gridPos": {
             "h": 8,
             "w": 8,
-            "x": 16,
+            "x": 8,
             "y": 170
           },
           "id": 522,
@@ -16851,196 +16717,6 @@
           "title": "Flow Operation Latency",
           "type": "timeseries"
         },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${metrics}"
-          },
-          "description": "Flow Buffer Size per Instance.",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "auto",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              }
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 8,
-            "w": 8,
-            "x": 0,
-            "y": 178
-          },
-          "id": 523,
-          "options": {
-            "legend": {
-              "calcs": [],
-              "displayMode": "list",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
-            }
-          },
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "${metrics}"
-              },
-              "editorMode": "code",
-              "expr": "greptime_flow_input_buf_size",
-              "instant": false,
-              "legendFormat": "[{{instance}}]-[{{pod}}]",
-              "range": true,
-              "refId": "A"
-            }
-          ],
-          "title": "Flow Buffer Size per Instance",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${metrics}"
-          },
-          "description": "Flow Processing Error per Instance.",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "auto",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              }
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 8,
-            "w": 8,
-            "x": 8,
-            "y": 178
-          },
-          "id": 524,
-          "options": {
-            "legend": {
-              "calcs": [],
-              "displayMode": "list",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
-            }
-          },
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "${metrics}"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))",
-              "instant": false,
-              "legendFormat": "[{{instance}}]-[{{pod}}]-[{{code}}]",
-              "range": true,
-              "refId": "A"
-            }
-          ],
-          "title": "Flow Processing Error per Instance",
-          "type": "timeseries"
-        },
         {
           "datasource": {
             "type": "prometheus",
@@ -17105,7 +16781,7 @@
             "h": 8,
             "w": 8,
             "x": 16,
-            "y": 178
+            "y": 170
           },
           "id": 558,
           "options": {
@@ -17229,7 +16905,7 @@
             "h": 8,
             "w": 8,
             "x": 0,
-            "y": 186
+            "y": 178
           },
           "id": 559,
           "options": {
@@ -17353,7 +17029,7 @@
             "h": 8,
             "w": 8,
             "x": 8,
-            "y": 186
+            "y": 178
           },
           "id": 560,
           "options": {
@@ -17464,7 +17140,7 @@
             "h": 8,
             "w": 8,
             "x": 16,
-            "y": 186
+            "y": 178
           },
           "id": 561,
           "options": {
@@ -17575,7 +17251,7 @@
             "h": 8,
             "w": 8,
             "x": 0,
-            "y": 194
+            "y": 186
           },
           "id": 562,
           "options": {
@@ -17673,7 +17349,7 @@
             "h": 8,
             "w": 8,
             "x": 8,
-            "y": 194
+            "y": 186
           },
           "id": 563,
           "options": {
@@ -17771,7 +17447,7 @@
             "h": 8,
             "w": 8,
             "x": 16,
-            "y": 194
+            "y": 186
           },
           "id": 564,
           "options": {

grafana/dashboards/metrics/standalone/dashboard.md

@@ -36,7 +36,7 @@
 | Scan and Compaction Memory Rejects | `sum(rate(greptime_mito_scan_requests_rejected_total[$__rate_interval]))`<br/>`sum(rate(greptime_mito_scan_memory_exhausted_total[$__rate_interval]))`<br/>`sum(rate(greptime_mito_compaction_memory_rejected_total[$__rate_interval]))` | `timeseries` | Datanode scan and compaction memory rejection/exhaustion counters. | `prometheus` | `rps` | `scan-rejected` |
 | OpenDAL Errors | `sum by (scheme, operation, error) (rate(opendal_operation_errors_total{error!="NotFound"}[$__rate_interval]))` | `timeseries` | Object-store errors by scheme, operation, and error, excluding NotFound noise. | `prometheus` | `eps` | `{{scheme}}-{{operation}}-{{error}}` |
 | Metasrv Failures | `sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))`<br/>`sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))` | `timeseries` | Region migration and reconciliation failures in metasrv. | `prometheus` | `eps` | `migration-fail` |
-| Flow and Trigger Failures | `sum by (code) (rate(greptime_flow_errors[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_flow_batching_error_count[$__rate_interval]))` | `timeseries` | Derived-data and alerting pipeline failures. | `prometheus` | `eps` | `flow-{{code}}` |
+| Flow and Trigger Failures | `sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_flow_batching_error_count[$__rate_interval]))` | `timeseries` | Derived-data and alerting pipeline failures. | `prometheus` | `eps` | `trigger-evaluate` |
 | Mito GC Failures | `sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))` | `timeseries` | Mito garbage-collection errors and skipped/orphaned files on datanodes. | `prometheus` | `short` | `gc-errors` |
 # Capacity
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
@@ -131,7 +131,7 @@
 | Index Memory Usage | `greptime_index_apply_memory_usage`<br/>`sum by (type) (greptime_index_create_memory_usage)` | `timeseries` | Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work. | `prometheus` | `bytes` | `apply` |
 | Index IO Bytes | `sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))` | `timeseries` | Index read and write byte rates by operation and file type for puffin and intermediate files. | `prometheus` | `Bps` | `{{type}}-{{file_type}}` |
 | Index IO Operations | `sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))` | `timeseries` | Index IO operation rates by operation and file type, including read, write, seek, and flush operations. | `prometheus` | `ops` | `{{type}}-{{file_type}}` |
-| Index Cache | `sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`<br/>`sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`<br/>`sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))` | `timeseries` | Index-related cache hits, misses, and evictions from Mito caches. | `prometheus` | `ops` | `hit-{{type}}` |
+| Index Cache | `sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*\|index_result"}[$__rate_interval]))`<br/>`sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*\|index_result"}[$__rate_interval]))`<br/>`sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*\|index_result"}[$__rate_interval]))` | `timeseries` | Index-related cache hits, misses, and evictions from Mito caches. | `prometheus` | `ops` | `hit-{{type}}` |
 # Metasrv
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
 | --- | --- | --- | --- | --- | --- | --- |
@@ -278,10 +278,7 @@ ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
 | --- | --- | --- | --- | --- | --- | --- |
 | Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` |
-| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`<br/>`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`<br/>`sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` |
 | Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`<br/>`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
-| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
-| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` |
 | Flow Batching Attempt / Error Rate | `sum by(instance,pod,flow_id) (rate(greptime_flow_batching_start_query_count[$__rate_interval]))`<br/>`sum by(instance,pod,flow_id) (rate(greptime_flow_batching_error_count[$__rate_interval]))`<br/>`sum by(instance,pod,flow_id) (rate(greptime_flow_batching_engine_slow_query_secs_count[$__rate_interval]))` | `timeseries` | Flow batching mode task attempt, error, and slow-query rates by flow. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[flow={{flow_id}}]-attempt` |
 | Flow Batching Query Latency | `histogram_quantile(0.95, sum by(le,instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_time_secs_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by(le,instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_time_secs_bucket[$__rate_interval])))`<br/>`sum by(instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_time_secs_sum[$__rate_interval])) / sum by(instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_time_secs_count[$__rate_interval]))` | `timeseries` | Flow batching mode query latency by flow. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[flow={{flow_id}}]-p95` |
 | Flow Batching Query Window Count | `histogram_quantile(0.95, sum by(le,instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_window_cnt_bucket[$__rate_interval])))`<br/>`sum by(instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_window_cnt_sum[$__rate_interval])) / sum by(instance,pod,flow_id) (rate(greptime_flow_batching_engine_query_window_cnt_count[$__rate_interval]))` | `timeseries` | Flow batching mode query window count by flow. | `prometheus` | `short` | `[{{instance}}]-[{{pod}}]-[flow={{flow_id}}]-p95` |

grafana/dashboards/metrics/standalone/dashboard.yaml

@@ -515,11 +515,6 @@ groups:
           description: Derived-data and alerting pipeline failures.
           unit: eps
           queries:
-            - expr: sum by (code) (rate(greptime_flow_errors[$__rate_interval]))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: flow-{{code}}
             - expr: sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))
               datasource:
                 type: prometheus
@@ -1572,17 +1567,17 @@ groups:
           description: Index-related cache hits, misses, and evictions from Mito caches.
           unit: ops
           queries:
-            - expr: sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*|vector_index|index_result"}[$__rate_interval]))
+            - expr: sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*|index_result"}[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: hit-{{type}}
-            - expr: sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*|vector_index|index_result"}[$__rate_interval]))
+            - expr: sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*|index_result"}[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: miss-{{type}}
-            - expr: sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*|vector_index|index_result"}[$__rate_interval]))
+            - expr: sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*|index_result"}[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
@@ -2199,25 +2194,6 @@ groups:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '[{{pod}}]-[{{instance}}]-[{{direction}}]'
-        - title: Flow Ingest Latency
-          type: timeseries
-          description: Flow Ingest Latency.
-          queries:
-            - expr: histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-p95'
-            - expr: histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-p99'
-            - expr: sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-avg'
         - title: Flow Operation Latency
           type: timeseries
           description: Flow Operation Latency.
@@ -2232,24 +2208,6 @@ groups:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99'
-        - title: Flow Buffer Size per Instance
-          type: timeseries
-          description: Flow Buffer Size per Instance.
-          queries:
-            - expr: greptime_flow_input_buf_size
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]'
-        - title: Flow Processing Error per Instance
-          type: timeseries
-          description: Flow Processing Error per Instance.
-          queries:
-            - expr: sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{code}}]'
         - title: Flow Batching Attempt / Error Rate
           type: timeseries
           description: Flow batching mode task attempt, error, and slow-query rates by flow.

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2026-03-21"
+channel = "1.98.1"

rustfmt.toml

@@ -1,2 +1,3 @@
-group_imports = "StdExternalCrate"
-imports_granularity = "Module"
+# disable these features until they become stable
+#group_imports = "StdExternalCrate"
+#imports_granularity = "Module"

scripts/check-builder-rust-version.sh

@@ -16,26 +16,50 @@ function check_rust_toolchain_version() {
 
   DEV_BUILDER_UBUNTU_IMAGE="$DEV_BUILDER_UBUNTU_REGISTRY/$DEV_BUILDER_UBUNTU_NAMESPACE/$DEV_BUILDER_UBUNTU_NAME:$DEV_BUILDER_IMAGE_TAG"
 
-  CURRENT_VERSION=$(grep -Eo '[0-9]{4}-[0-9]{2}-[0-9]{2}' "$RUST_TOOLCHAIN_VERSION_FILE")
-  if [ -z "$CURRENT_VERSION" ]; then
-    echo "Error: No rust toolchain version found in $RUST_TOOLCHAIN_VERSION_FILE"
+  # The pinned channel, e.g. "1.96.1" or "nightly-2026-03-21".
+  RUST_TOOLCHAIN=$(grep -E '^channel' "$RUST_TOOLCHAIN_VERSION_FILE" | cut -d'"' -f2)
+  if [ -z "$RUST_TOOLCHAIN" ]; then
+    echo "Error: No rust toolchain channel found in $RUST_TOOLCHAIN_VERSION_FILE"
     exit 1
   fi
 
-  RUST_TOOLCHAIN_VERSION_IN_BUILDER=$(docker run "$DEV_BUILDER_UBUNTU_IMAGE" rustc --version | grep  -Eo '[0-9]{4}-[0-9]{2}-[0-9]{2}')
-  if [ -z "$RUST_TOOLCHAIN_VERSION_IN_BUILDER" ]; then
+  RUSTC_VERSION_OUTPUT=$(docker run "$DEV_BUILDER_UBUNTU_IMAGE" rustc --version)
+  RUSTC_VERSION=$(echo "$RUSTC_VERSION_OUTPUT" | awk '{print $2}')
+  if [ -z "$RUSTC_VERSION" ]; then
     echo "Error: No rustc version found in $DEV_BUILDER_UBUNTU_IMAGE"
     exit 1
   fi
 
-  # Compare the version and the difference should be less than 1 day.
-  current_rust_toolchain_seconds=$(date -d "$CURRENT_VERSION" +%s)
-  rust_toolchain_in_dev_builder_ubuntu_seconds=$(date -d "$RUST_TOOLCHAIN_VERSION_IN_BUILDER" +%s)
-  date_diff=$(( (current_rust_toolchain_seconds - rust_toolchain_in_dev_builder_ubuntu_seconds) / 86400 ))
+  if [[ "$RUST_TOOLCHAIN" =~ ^[0-9]+\.[0-9]+ ]]; then
+    # Stable channel: the builder image must ship exactly the pinned version.
+    if [ "$RUSTC_VERSION" != "$RUST_TOOLCHAIN" ]; then
+      echo "Error: The rust toolchain '$RUSTC_VERSION' in builder '$DEV_BUILDER_UBUNTU_IMAGE' doesn't match the pinned '$RUST_TOOLCHAIN', please rebuild the dev-builder image"
+      exit 1
+    fi
+  else
+    # Nightly channel: keep the legacy date-based check. The difference
+    # between the pinned nightly date and the rustc build date shipped in
+    # the builder should be less than 1 day.
+    CURRENT_VERSION=$(echo "$RUST_TOOLCHAIN" | grep -Eo '[0-9]{4}-[0-9]{2}-[0-9]{2}')
+    if [ -z "$CURRENT_VERSION" ]; then
+      echo "Error: Unsupported toolchain channel '$RUST_TOOLCHAIN' in $RUST_TOOLCHAIN_VERSION_FILE"
+      exit 1
+    fi
 
-  if [ $date_diff -gt 1 ]; then
-    echo "Error: The rust toolchain '$RUST_TOOLCHAIN_VERSION_IN_BUILDER' in builder '$DEV_BUILDER_UBUNTU_IMAGE' maybe outdated, please update it to '$CURRENT_VERSION'"
-    exit 1
+    RUST_TOOLCHAIN_VERSION_IN_BUILDER=$(echo "$RUSTC_VERSION_OUTPUT" | grep -Eo '[0-9]{4}-[0-9]{2}-[0-9]{2}')
+    if [ -z "$RUST_TOOLCHAIN_VERSION_IN_BUILDER" ]; then
+      echo "Error: No rustc version found in $DEV_BUILDER_UBUNTU_IMAGE"
+      exit 1
+    fi
+
+    current_rust_toolchain_seconds=$(date -d "$CURRENT_VERSION" +%s)
+    rust_toolchain_in_dev_builder_ubuntu_seconds=$(date -d "$RUST_TOOLCHAIN_VERSION_IN_BUILDER" +%s)
+    date_diff=$(( (current_rust_toolchain_seconds - rust_toolchain_in_dev_builder_ubuntu_seconds) / 86400 ))
+
+    if [ $date_diff -gt 1 ]; then
+      echo "Error: The rust toolchain '$RUST_TOOLCHAIN_VERSION_IN_BUILDER' in builder '$DEV_BUILDER_UBUNTU_IMAGE' maybe outdated, please update it to '$CURRENT_VERSION'"
+      exit 1
+    fi
   fi
 }
 

scripts/fix-udeps.py

@@ -1,265 +0,0 @@
-# Copyright 2023 Greptime Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import os
-import re
-import sys
-
-
-def load_udeps_report(report_path):
-    try:
-        with open(report_path, "r") as f:
-            return json.load(f)
-    except FileNotFoundError:
-        print(f"Error: Report file '{report_path}' not found.")
-        return None
-    except json.JSONDecodeError as e:
-        print(f"Error: Invalid JSON in report file: {e}")
-        return None
-
-
-def extract_unused_dependencies(report):
-    """
-    Extract and organize unused dependencies from the cargo-udeps JSON report.
-
-    The cargo-udeps report has this structure:
-    {
-        "unused_deps": {
-            "package_name v0.1.0 (/path/to/package)": {
-                "normal": ["dep1", "dep2"],
-                "development": ["dev_dep1"],
-                "build": ["build_dep1"],
-                "manifest_path": "/path/to/Cargo.toml"
-            }
-        }
-    }
-
-    Args:
-        report (dict): The parsed JSON report from cargo-udeps
-
-    Returns:
-        dict: Organized unused dependencies by package name:
-        {
-            "package_name": {
-                "dependencies": [("dep1", "normal"), ("dev_dep1", "dev")],
-                "manifest_path": "/path/to/Cargo.toml"
-            }
-        }
-    """
-    if not report or "unused_deps" not in report:
-        return {}
-
-    unused_deps = {}
-    for package_full_name, deps_info in report["unused_deps"].items():
-        package_name = package_full_name.split(" ")[0]
-
-        all_unused = []
-        if deps_info.get("normal"):
-            all_unused.extend([(dep, "normal") for dep in deps_info["normal"]])
-        if deps_info.get("development"):
-            all_unused.extend([(dep, "dev") for dep in deps_info["development"]])
-        if deps_info.get("build"):
-            all_unused.extend([(dep, "build") for dep in deps_info["build"]])
-
-        if all_unused:
-            unused_deps[package_name] = {
-                "dependencies": all_unused,
-                "manifest_path": deps_info.get("manifest_path", "unknown"),
-            }
-
-    return unused_deps
-
-
-def get_section_pattern(dep_type):
-    """
-    Get regex patterns to identify different dependency sections in Cargo.toml.
-
-    Args:
-        dep_type (str): Type of dependency ("normal", "dev", or "build")
-
-    Returns:
-        list: List of regex patterns to match the appropriate section headers
-
-    """
-    patterns = {
-        "normal": [r"\[dependencies\]", r"\[dependencies\..*?\]"],
-        "dev": [r"\[dev-dependencies\]", r"\[dev-dependencies\..*?\]"],
-        "build": [r"\[build-dependencies\]", r"\[build-dependencies\..*?\]"],
-    }
-    return patterns.get(dep_type, [])
-
-
-def remove_dependency_line(content, dep_name, section_start, section_end):
-    """
-    Remove a dependency line from a specific section of a Cargo.toml file.
-
-    Args:
-        content (str): The entire content of the Cargo.toml file
-        dep_name (str): Name of the dependency to remove (e.g., "serde", "tokio")
-        section_start (int): Starting position of the section in the content
-        section_end (int): Ending position of the section in the content
-
-    Returns:
-        tuple: (new_content, removed) where:
-            - new_content (str): The modified content with dependency removed
-            - removed (bool): True if dependency was found and removed, False otherwise
-
-    Example input content format:
-        content = '''
-        [package]
-        name = "my-crate"
-        version = "0.1.0"
-
-        [dependencies]
-        serde = "1.0"
-        tokio = { version = "1.0", features = ["full"] }
-        serde_json.workspace = true
-
-        [dev-dependencies]
-        tempfile = "3.0"
-        '''
-
-        # If dep_name = "serde", section_start = start of [dependencies],
-        # section_end = start of [dev-dependencies], this function will:
-        # 1. Extract the section: "serde = "1.0"\ntokio = { version = "1.0", features = ["full"] }\nserde_json.workspace = true\n"
-        # 2. Find and remove the line: "serde = "1.0""
-        # 3. Return the modified content with that line removed
-    """
-    section_content = content[section_start:section_end]
-
-    dep_patterns = [
-        rf"^{re.escape(dep_name)}\s*=.*$",  # e.g., "serde = "1.0""
-        rf"^{re.escape(dep_name)}\.workspace\s*=.*$",  # e.g., "serde_json.workspace = true"
-    ]
-
-    for pattern in dep_patterns:
-        match = re.search(pattern, section_content, re.MULTILINE)
-        if match:
-            line_start = section_start + match.start()  # Start of the matched line
-            line_end = section_start + match.end()  # End of the matched line
-
-            if line_end < len(content) and content[line_end] == "\n":
-                line_end += 1
-
-            return content[:line_start] + content[line_end:], True
-
-    return content, False
-
-
-def remove_dependency_from_toml(file_path, dep_name, dep_type):
-    """
-    Remove a specific dependency from a Cargo.toml file.
-
-    Args:
-        file_path (str): Path to the Cargo.toml file
-        dep_name (str): Name of the dependency to remove
-        dep_type (str): Type of dependency ("normal", "dev", or "build")
-
-    Returns:
-        bool: True if dependency was successfully removed, False otherwise
-    """
-    try:
-        with open(file_path, "r") as f:
-            content = f.read()
-
-        section_patterns = get_section_pattern(dep_type)
-        if not section_patterns:
-            return False
-
-        for pattern in section_patterns:
-            section_match = re.search(pattern, content, re.IGNORECASE)
-            if not section_match:
-                continue
-
-            section_start = section_match.end()
-            next_section = re.search(r"\n\s*\[", content[section_start:])
-            section_end = (
-                section_start + next_section.start() if next_section else len(content)
-            )
-
-            new_content, removed = remove_dependency_line(
-                content, dep_name, section_start, section_end
-            )
-            if removed:
-                with open(file_path, "w") as f:
-                    f.write(new_content)
-                return True
-
-        return False
-
-    except Exception as e:
-        print(f"Error processing {file_path}: {e}")
-        return False
-
-
-def process_unused_dependencies(unused_deps):
-    """
-    Process and remove all unused dependencies from their respective Cargo.toml files.
-
-    Args:
-        unused_deps (dict): Dictionary of unused dependencies organized by package:
-            {
-                "package_name": {
-                    "dependencies": [("dep1", "normal"), ("dev_dep1", "dev")],
-                    "manifest_path": "/path/to/Cargo.toml"
-                }
-            }
-
-    """
-    if not unused_deps:
-        print("No unused dependencies found.")
-        return
-
-    total_removed = 0
-    total_failed = 0
-
-    for package, info in unused_deps.items():
-        deps = info["dependencies"]
-        manifest_path = info["manifest_path"]
-
-        if not os.path.exists(manifest_path):
-            print(f"Manifest file not found: {manifest_path}")
-            total_failed += len(deps)
-            continue
-
-        for dep, dep_type in deps:
-            if remove_dependency_from_toml(manifest_path, dep, dep_type):
-                print(f"Removed {dep} from {package}")
-                total_removed += 1
-            else:
-                print(f"Failed to remove {dep} from {package}")
-                total_failed += 1
-
-    print(f"Removed {total_removed} dependencies")
-    if total_failed > 0:
-        print(f"Failed to remove {total_failed} dependencies")
-
-
-def main():
-    if len(sys.argv) > 1:
-        report_path = sys.argv[1]
-    else:
-        report_path = "udeps-report.json"
-
-    report = load_udeps_report(report_path)
-    if report is None:
-        sys.exit(1)
-
-    unused_deps = extract_unused_dependencies(report)
-    process_unused_dependencies(unused_deps)
-
-
-if __name__ == "__main__":
-    main()

src/api/src/helper.rs

@@ -87,6 +87,28 @@ impl ColumnDataTypeWrapper {
     }
 }
 
+/// Returns the time unit if `datatype` is a timestamp type.
+pub fn timestamp_unit(datatype: ColumnDataType) -> Option<TimeUnit> {
+    match datatype {
+        ColumnDataType::TimestampSecond => Some(TimeUnit::Second),
+        ColumnDataType::TimestampMillisecond => Some(TimeUnit::Millisecond),
+        ColumnDataType::TimestampMicrosecond => Some(TimeUnit::Microsecond),
+        ColumnDataType::TimestampNanosecond => Some(TimeUnit::Nanosecond),
+        _ => None,
+    }
+}
+
+/// Returns the timestamp [ColumnDataType] for the given time unit.
+/// This is the inverse of [timestamp_unit].
+pub fn timestamp_datatype(unit: TimeUnit) -> ColumnDataType {
+    match unit {
+        TimeUnit::Second => ColumnDataType::TimestampSecond,
+        TimeUnit::Millisecond => ColumnDataType::TimestampMillisecond,
+        TimeUnit::Microsecond => ColumnDataType::TimestampMicrosecond,
+        TimeUnit::Nanosecond => ColumnDataType::TimestampNanosecond,
+    }
+}
+
 impl From<ColumnDataTypeWrapper> for ConcreteDataType {
     fn from(datatype_wrapper: ColumnDataTypeWrapper) -> Self {
         match datatype_wrapper.datatype {
@@ -1215,6 +1237,21 @@ mod tests {
     use super::*;
     use crate::v1::Column;
 
+    #[test]
+    fn test_timestamp_unit_roundtrip() {
+        for unit in [
+            TimeUnit::Second,
+            TimeUnit::Millisecond,
+            TimeUnit::Microsecond,
+            TimeUnit::Nanosecond,
+        ] {
+            assert_eq!(timestamp_unit(timestamp_datatype(unit)), Some(unit));
+        }
+        // Non-timestamp types have no time unit.
+        assert_eq!(timestamp_unit(ColumnDataType::String), None);
+        assert_eq!(timestamp_unit(ColumnDataType::Datetime), None);
+    }
+
     #[test]
     fn test_values_with_capacity() {
         let values = values_with_capacity(ColumnDataType::Int8, 2);

src/api/src/v1/column_def.rs

@@ -18,7 +18,7 @@ use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_K
 use datatypes::schema::{
     COMMENT_KEY, ColumnDefaultConstraint, ColumnSchema, FULLTEXT_KEY, FulltextAnalyzer,
     FulltextBackend, FulltextOptions, INVERTED_INDEX_KEY, Metadata, SKIPPING_INDEX_KEY,
-    SkippingIndexOptions, SkippingIndexType, VECTOR_INDEX_KEY,
+    SkippingIndexOptions, SkippingIndexType,
 };
 use greptime_proto::v1::{
     Analyzer, FulltextBackend as PbFulltextBackend, SkippingIndexType as PbSkippingIndexType,
@@ -35,14 +35,11 @@ const FULLTEXT_GRPC_KEY: &str = "fulltext";
 const INVERTED_INDEX_GRPC_KEY: &str = "inverted_index";
 /// Key used to store skip index options in gRPC column options.
 const SKIPPING_INDEX_GRPC_KEY: &str = "skipping_index";
-/// Key used to store vector index options in gRPC column options.
-const VECTOR_INDEX_GRPC_KEY: &str = "vector_index";
 
-const COLUMN_OPTION_MAPPINGS: [(&str, &str); 6] = [
+const COLUMN_OPTION_MAPPINGS: [(&str, &str); 5] = [
     (FULLTEXT_GRPC_KEY, FULLTEXT_KEY),
     (INVERTED_INDEX_GRPC_KEY, INVERTED_INDEX_KEY),
     (SKIPPING_INDEX_GRPC_KEY, SKIPPING_INDEX_KEY),
-    (VECTOR_INDEX_GRPC_KEY, VECTOR_INDEX_KEY),
     (EXTENSION_TYPE_NAME_KEY, EXTENSION_TYPE_NAME_KEY),
     (EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_METADATA_KEY),
 ];
@@ -80,9 +77,6 @@ pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
         if let Some(skipping_index) = options.options.get(SKIPPING_INDEX_GRPC_KEY) {
             metadata.insert(SKIPPING_INDEX_KEY.to_string(), skipping_index.to_owned());
         }
-        if let Some(vector_index) = options.options.get(VECTOR_INDEX_GRPC_KEY) {
-            metadata.insert(VECTOR_INDEX_KEY.to_string(), vector_index.to_owned());
-        }
         if let Some(extension_name) = options.options.get(EXTENSION_TYPE_NAME_KEY) {
             metadata.insert(EXTENSION_TYPE_NAME_KEY.to_string(), extension_name.clone());
         }
@@ -178,11 +172,6 @@ pub fn options_from_column_schema(column_schema: &ColumnSchema) -> Option<Column
             .options
             .insert(SKIPPING_INDEX_GRPC_KEY.to_string(), skipping_index.clone());
     }
-    if let Some(vector_index) = column_schema.metadata().get(VECTOR_INDEX_KEY) {
-        options
-            .options
-            .insert(VECTOR_INDEX_GRPC_KEY.to_string(), vector_index.clone());
-    }
     if let Some(extension_name) = column_schema.metadata().get(EXTENSION_TYPE_NAME_KEY) {
         options
             .options
@@ -270,10 +259,7 @@ pub fn as_skipping_index_type(skipping_index_type: PbSkippingIndexType) -> Skipp
 mod tests {
 
     use datatypes::data_type::ConcreteDataType;
-    use datatypes::schema::{
-        FulltextAnalyzer, FulltextBackend, VectorDistanceMetric, VectorIndexOptions,
-    };
-    use serde_json::json;
+    use datatypes::schema::{FulltextAnalyzer, FulltextBackend};
 
     use super::*;
     use crate::v1::ColumnDataType;
@@ -297,10 +283,6 @@ mod tests {
                         "{\"enable\":true}".to_string(),
                     ),
                     (INVERTED_INDEX_GRPC_KEY.to_string(), "true".to_string()),
-                    (
-                        VECTOR_INDEX_GRPC_KEY.to_string(),
-                        "{\"engine\":\"usearch\",\"metric\":\"l2sq\",\"connectivity\":16,\"expansion-add\":128,\"expansion-search\":64}".to_string(),
-                    ),
                 ]),
             }),
         };
@@ -323,8 +305,6 @@ mod tests {
             }
         );
         assert!(schema.is_inverted_indexed());
-        let vector_options = schema.vector_index_options().unwrap().unwrap();
-        assert_eq!(vector_options.metric, VectorDistanceMetric::L2sq);
     }
 
     #[test]
@@ -355,29 +335,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_vector_index_options_roundtrip() {
-        let schema = ColumnSchema::new("test", ConcreteDataType::vector_datatype(4), true)
-            .with_vector_index_options(&VectorIndexOptions::default())
-            .unwrap();
-        let column_def = try_as_column_def(&schema, false).unwrap();
-        let roundtrip = try_as_column_schema(&column_def).unwrap();
-        let options = roundtrip.vector_index_options().unwrap().unwrap();
-        assert_eq!(options.metric, VectorDistanceMetric::L2sq);
-
-        let options = column_def.options.unwrap();
-        let raw = options.options.get(VECTOR_INDEX_GRPC_KEY).unwrap();
-        let json_value: serde_json::Value = serde_json::from_str(raw).unwrap();
-        let expected = json!({
-            "engine": "usearch",
-            "metric": "l2sq",
-            "connectivity": 16,
-            "expansion-add": 128,
-            "expansion-search": 64
-        });
-        assert_eq!(json_value, expected);
-    }
-
     #[test]
     fn test_options_with_fulltext() {
         let fulltext = FulltextOptions::new_unchecked(

src/auth/src/user_provider/watch_file_user_provider.rs

@@ -199,4 +199,46 @@ pub mod test {
         test_authenticate(&provider, "root", "654321", false, Some(timeout)).await;
         test_authenticate(&provider, "admin", "654321", false, Some(timeout)).await;
     }
+
+    /// Regression test for #9310: the users file is reached through a chain
+    /// of symlinks and a directory symlink in the chain gets swapped
+    /// atomically (like `ln -sfn`, as done by sops-nix).
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn test_file_provider_symlink_chain() {
+        use std::os::unix::fs::symlink;
+
+        common_telemetry::init_default_ut_logging();
+
+        let dir = create_temp_dir("test_file_provider_symlink_chain");
+        let base = dir.path();
+        let timeout = Duration::from_secs(10);
+
+        // secrets.1/users <- secrets (dir symlink) <- conf/users (file symlink)
+        std::fs::create_dir(base.join("secrets.1")).unwrap();
+        std::fs::write(base.join("secrets.1/users"), "root=123456\n").unwrap();
+        symlink(base.join("secrets.1"), base.join("secrets")).unwrap();
+        std::fs::create_dir(base.join("conf")).unwrap();
+        symlink(base.join("secrets/users"), base.join("conf/users")).unwrap();
+
+        let file_path = base.join("conf/users");
+        let provider = WatchFileUserProvider::new(file_path.to_str().unwrap()).unwrap();
+        test_authenticate(&provider, "root", "123456", true, None).await;
+
+        // Atomically swap the directory symlink to a new generation.
+        std::fs::create_dir(base.join("secrets.2")).unwrap();
+        std::fs::write(base.join("secrets.2/users"), "root=654321\n").unwrap();
+        symlink(base.join("secrets.2"), base.join("secrets.tmp")).unwrap();
+        std::fs::rename(base.join("secrets.tmp"), base.join("secrets")).unwrap();
+        std::fs::remove_dir_all(base.join("secrets.1")).unwrap();
+
+        test_authenticate(&provider, "root", "654321", true, Some(timeout)).await;
+        test_authenticate(&provider, "root", "123456", false, None).await;
+
+        // In-place edits of the new target are picked up too, which requires
+        // the watch to follow the chain to the new directory.
+        std::fs::write(base.join("secrets.2/users"), "root=abcdef\n").unwrap();
+        test_authenticate(&provider, "root", "abcdef", true, Some(timeout)).await;
+        test_authenticate(&provider, "root", "654321", false, None).await;
+    }
 }

src/cache/src/lib.rs

@@ -20,8 +20,8 @@ use std::time::Duration;
 
 use catalog::kvbackend::new_table_cache;
 use common_meta::cache::{
-    CacheRegistry, CacheRegistryBuilder, LayeredCacheRegistryBuilder, new_schema_cache,
-    new_table_flownode_set_cache, new_table_info_cache, new_table_name_cache,
+    CacheRegistry, CacheRegistryBuilder, LayeredCacheRegistry, LayeredCacheRegistryBuilder,
+    new_schema_cache, new_table_flownode_set_cache, new_table_info_cache, new_table_name_cache,
     new_table_route_cache, new_table_schema_cache, new_view_info_cache,
 };
 use common_meta::kv_backend::KvBackendRef;
@@ -53,11 +53,11 @@ pub const TABLE_FLOWNODE_SET_CACHE_NAME: &str = "table_flownode_set_cache";
 pub const TABLE_ROUTE_CACHE_NAME: &str = "table_route_cache";
 pub const PARTITION_INFO_CACHE_NAME: &str = "partition_info_cache";
 
-/// Builds cache registry for datanode, including:
-/// - Schema cache.
-/// - Table id to schema name cache.
-pub fn build_datanode_cache_registry(kv_backend: KvBackendRef) -> CacheRegistry {
-    // Builds table id schema name cache that never expires.
+/// Builds the datanode cache registry with base metadata caches before derived caches.
+///
+/// Invalidating the base layer first prevents a concurrent query from refilling a derived cache
+/// from stale metadata after that derived cache has been invalidated.
+pub fn build_datanode_layered_cache_registry(kv_backend: KvBackendRef) -> LayeredCacheRegistry {
     let cache = CacheBuilder::new(DEFAULT_CACHE_MAX_CAPACITY).build();
     let table_id_schema_cache = Arc::new(new_table_schema_cache(
         TABLE_SCHEMA_NAME_CACHE_NAME.to_string(),
@@ -65,7 +65,6 @@ pub fn build_datanode_cache_registry(kv_backend: KvBackendRef) -> CacheRegistry
         kv_backend.clone(),
     ));
 
-    // Builds schema cache
     let cache = default_cache();
     let schema_cache = Arc::new(new_schema_cache(
         SCHEMA_CACHE_NAME.to_string(),
@@ -73,9 +72,57 @@ pub fn build_datanode_cache_registry(kv_backend: KvBackendRef) -> CacheRegistry
         kv_backend.clone(),
     ));
 
-    CacheRegistryBuilder::default()
+    let cache = default_cache();
+    let table_info_cache = Arc::new(new_table_info_cache(
+        TABLE_INFO_CACHE_NAME.to_string(),
+        cache,
+        kv_backend.clone(),
+    ));
+
+    let cache = default_cache();
+    let table_name_cache = Arc::new(new_table_name_cache(
+        TABLE_NAME_CACHE_NAME.to_string(),
+        cache,
+        kv_backend.clone(),
+    ));
+
+    let cache = default_cache();
+    let table_route_cache = Arc::new(new_table_route_cache(
+        TABLE_ROUTE_CACHE_NAME.to_string(),
+        cache,
+        kv_backend.clone(),
+    ));
+
+    let cache = default_cache();
+    let table_cache = Arc::new(new_table_cache(
+        TABLE_CACHE_NAME.to_string(),
+        cache,
+        table_info_cache.clone(),
+        table_name_cache.clone(),
+    ));
+
+    let cache = default_cache();
+    let partition_info_cache = Arc::new(new_partition_info_cache(
+        PARTITION_INFO_CACHE_NAME.to_string(),
+        cache,
+        table_route_cache.clone(),
+    ));
+
+    let base_registry = CacheRegistryBuilder::default()
         .add_cache(table_id_schema_cache)
         .add_cache(schema_cache)
+        .add_cache(table_info_cache)
+        .add_cache(table_name_cache)
+        .add_cache(table_route_cache)
+        .build();
+    let derived_registry = CacheRegistryBuilder::default()
+        .add_cache(table_cache)
+        .add_cache(partition_info_cache)
+        .build();
+
+    LayeredCacheRegistryBuilder::default()
+        .add_cache_registry(base_registry)
+        .add_cache_registry(derived_registry)
         .build()
 }
 

src/catalog/Cargo.toml

@@ -61,9 +61,7 @@ tokio-stream.workspace = true
 
 [dev-dependencies]
 cache.workspace = true
-catalog = { workspace = true, features = ["testing"] }
 chrono.workspace = true
 common-meta = { workspace = true, features = ["testing"] }
 common-query = { workspace = true, features = ["testing"] }
-object-store.workspace = true
 tokio.workspace = true

src/catalog/src/kvbackend/client.rs

@@ -192,6 +192,10 @@ impl KvBackend for CachedKvBackend {
             }
         }
 
+        if miss_keys.is_empty() {
+            return Ok(BatchGetResponse { kvs });
+        }
+
         let batch_get_req = BatchGetRequest::new().with_keys(miss_keys.clone());
 
         let pre_version = self.version();
@@ -503,6 +507,7 @@ mod tests {
     pub struct SimpleKvBackend {
         inner_map: DashMap<Vec<u8>, Vec<u8>>,
         get_execute_times: Arc<AtomicU32>,
+        batch_get_execute_times: Arc<AtomicU32>,
     }
 
     impl TxnService for SimpleKvBackend {
@@ -520,6 +525,7 @@ mod tests {
         }
 
         async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse, Self::Error> {
+            self.batch_get_execute_times.fetch_add(1, Ordering::SeqCst);
             let mut kvs = Vec::with_capacity(req.keys.len());
             for key in req.keys.iter() {
                 if let Some(kv) = self.get(key).await? {
@@ -571,6 +577,7 @@ mod tests {
     async fn test_cached_kv_backend() {
         let simple_kv = Arc::new(SimpleKvBackend::default());
         let get_execute_times = simple_kv.get_execute_times.clone();
+        let batch_get_execute_times = simple_kv.batch_get_execute_times.clone();
         let cached_kv = CachedKvBackend::wrap(simple_kv);
 
         add_some_vals(&cached_kv).await;
@@ -582,9 +589,11 @@ mod tests {
         assert_eq!(get_execute_times.load(Ordering::SeqCst), 0);
 
         for _ in 0..10 {
-            let _batch_get_resp = cached_kv.batch_get(batch_get_req.clone()).await.unwrap();
+            let batch_get_resp = cached_kv.batch_get(batch_get_req.clone()).await.unwrap();
 
+            assert_eq!(batch_get_resp.kvs.len(), 2);
             assert_eq!(get_execute_times.load(Ordering::SeqCst), 2);
+            assert_eq!(batch_get_execute_times.load(Ordering::SeqCst), 1);
         }
 
         let batch_get_req = BatchGetRequest {
@@ -594,14 +603,29 @@ mod tests {
         let _batch_get_resp = cached_kv.batch_get(batch_get_req.clone()).await.unwrap();
 
         assert_eq!(get_execute_times.load(Ordering::SeqCst), 3);
+        assert_eq!(batch_get_execute_times.load(Ordering::SeqCst), 2);
 
         for _ in 0..10 {
-            let _batch_get_resp = cached_kv.batch_get(batch_get_req.clone()).await.unwrap();
+            let batch_get_resp = cached_kv.batch_get(batch_get_req.clone()).await.unwrap();
 
+            assert_eq!(batch_get_resp.kvs.len(), 3);
             assert_eq!(get_execute_times.load(Ordering::SeqCst), 3);
+            assert_eq!(batch_get_execute_times.load(Ordering::SeqCst), 2);
         }
     }
 
+    #[tokio::test]
+    async fn test_cached_kv_backend_empty_batch_get() {
+        let simple_kv = Arc::new(SimpleKvBackend::default());
+        let cached_kv = CachedKvBackend::wrap(simple_kv.clone());
+
+        let response = cached_kv.batch_get(BatchGetRequest::new()).await.unwrap();
+
+        assert!(response.kvs.is_empty());
+        assert_eq!(simple_kv.batch_get_execute_times.load(Ordering::SeqCst), 0);
+        assert_eq!(simple_kv.get_execute_times.load(Ordering::SeqCst), 0);
+    }
+
     #[tokio::test]
     async fn test_cached_kv_backend_rejects_writes_with_read_only_inner() {
         let inner = Arc::new(MemoryKvBackend::<common_meta::error::Error>::new());

src/catalog/src/system_schema/information_schema/partitions.rs

@@ -254,6 +254,51 @@ impl InformationSchemaPartitionsBuilder {
 
         let predicates = Predicates::from_scan_request(&request);
 
+        // Fast path: when the predicates pin both the schema and the table name
+        // with equality, resolve the table directly instead of enumerating all
+        // tables in all schemas and fetching partition metadata for each of them.
+        if let (Some(schema_name), Some(table_name)) = (
+            predicates
+                .find_eq_value(TABLE_SCHEMA)
+                .and_then(Value::as_string),
+            predicates
+                .find_eq_value(TABLE_NAME)
+                .and_then(Value::as_string),
+        ) {
+            if let Some(table) = catalog_manager
+                .table(&catalog_name, &schema_name, &table_name, None)
+                .await?
+            {
+                let table_info = table.table_info();
+
+                // The enumeration path below skips temporary tables, keep the
+                // behavior identical.
+                if table_info.table_type != TableType::Temporary {
+                    let mut table_partitions = partition_manager
+                        .batch_find_table_partitions(&[table_info.table_id()])
+                        .await
+                        .context(FindPartitionsSnafu)?;
+
+                    let partitions = table_partitions
+                        .remove(&table_info.table_id())
+                        .unwrap_or_default();
+
+                    self.add_partitions(
+                        &predicates,
+                        &table_info,
+                        &catalog_name,
+                        &schema_name,
+                        &table_info.name,
+                        &partitions,
+                    );
+                }
+            }
+
+            // The pinned table fully determines the result: the table above was
+            // found (or not) unambiguously, so no other table can contribute rows.
+            return self.finish();
+        }
+
         for schema_name in catalog_manager.schema_names(&catalog_name, None).await? {
             let table_info_stream = catalog_manager
                 .tables(&catalog_name, &schema_name, None)
@@ -421,3 +466,253 @@ impl DfPartitionStream for InformationSchemaPartitions {
         ))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use api::v1::meta::Peer;
+    use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry};
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, INFORMATION_SCHEMA_NAME};
+    use common_meta::cache::{CacheRegistryBuilder, LayeredCacheRegistryBuilder};
+    use common_meta::key::schema_name::SchemaNameKey;
+    use common_meta::key::table_route::TableRouteValue;
+    use common_meta::kv_backend::memory::MemoryKvBackend;
+    use common_meta::rpc::router::{Region, RegionRoute};
+    use common_meta::wal_provider::RegionWalOptions;
+    use datafusion::logical_expr::{BinaryExpr, Expr, Operator, col, lit};
+    use futures::TryStreamExt;
+    use store_api::storage::RegionId;
+
+    use super::*;
+    use crate::information_schema::NoopInformationExtension;
+    use crate::kvbackend::KvBackendCatalogManagerBuilder;
+
+    const SCHEMA_1: &str = "partitions_schema_1";
+    const SCHEMA_2: &str = "partitions_schema_2";
+    const PARTITIONED_TABLE: &str = "partitioned_table";
+    const MISSING_TABLE: &str = "missing_table";
+    const TABLE_ID_1: TableId = 1;
+    const TABLE_ID_2: TableId = 2;
+    const REGION_NUMBER_1: u32 = 1;
+    const REGION_NUMBER_2: u32 = 2;
+    const REGION_NUMBER_7: u32 = 7;
+
+    fn kv_catalog_manager() -> Arc<KvBackendCatalogManager> {
+        let backend = Arc::new(MemoryKvBackend::default());
+        let layered_cache_builder = LayeredCacheRegistryBuilder::default()
+            .add_cache_registry(CacheRegistryBuilder::default().build());
+        let fundamental_cache_registry = build_fundamental_cache_registry(backend.clone());
+        let layered_cache_registry = Arc::new(
+            with_default_composite_cache_registry(
+                layered_cache_builder.add_cache_registry(fundamental_cache_registry),
+            )
+            .unwrap()
+            .build(),
+        );
+
+        KvBackendCatalogManagerBuilder::new(
+            Arc::new(NoopInformationExtension),
+            backend,
+            layered_cache_registry,
+        )
+        .build()
+    }
+
+    fn region_route(table_id: TableId, region_number: u32) -> RegionRoute {
+        RegionRoute {
+            region: Region {
+                id: RegionId::new(table_id, region_number),
+                ..Default::default()
+            },
+            leader_peer: Some(Peer {
+                id: 1,
+                addr: "127.0.0.1:3001".to_string(),
+            }),
+            ..Default::default()
+        }
+    }
+
+    fn table_info(schema: &str, table: &str, table_id: TableId) -> TableInfo {
+        // The catalog defaults to the default catalog name.
+        let mut table_info =
+            common_meta::key::test_utils::new_test_table_info_with_name(table_id, table);
+        table_info.schema_name = schema.to_string();
+        table_info
+    }
+
+    /// Creates a catalog with two partitioned tables named `PARTITIONED_TABLE`
+    /// in two different schemas:
+    ///
+    /// - `SCHEMA_1.PARTITIONED_TABLE` has two regions.
+    /// - `SCHEMA_2.PARTITIONED_TABLE` has one region.
+    async fn setup_catalog() -> Arc<KvBackendCatalogManager> {
+        let manager = kv_catalog_manager();
+        let metadata = manager.table_metadata_manager_ref().clone();
+
+        for schema in [SCHEMA_1, SCHEMA_2] {
+            metadata
+                .schema_manager()
+                .create(SchemaNameKey::new(DEFAULT_CATALOG_NAME, schema), None, true)
+                .await
+                .unwrap();
+        }
+
+        for (table_id, schema, region_numbers) in [
+            (TABLE_ID_1, SCHEMA_1, vec![REGION_NUMBER_1, REGION_NUMBER_2]),
+            (TABLE_ID_2, SCHEMA_2, vec![REGION_NUMBER_7]),
+        ] {
+            let routes = region_numbers
+                .into_iter()
+                .map(|region_number| region_route(table_id, region_number))
+                .collect();
+            metadata
+                .create_table_metadata(
+                    table_info(schema, PARTITIONED_TABLE, table_id),
+                    TableRouteValue::physical(routes),
+                    RegionWalOptions::default(),
+                )
+                .await
+                .unwrap();
+        }
+
+        manager
+    }
+
+    /// Scans `information_schema.partitions` and returns the total row count
+    /// and the pretty printed batches.
+    async fn scan_partitions(
+        manager: &Arc<dyn CatalogManager>,
+        filters: Vec<Expr>,
+    ) -> (usize, String) {
+        let table = manager
+            .table(
+                DEFAULT_CATALOG_NAME,
+                INFORMATION_SCHEMA_NAME,
+                PARTITIONS,
+                None,
+            )
+            .await
+            .unwrap()
+            .unwrap();
+        let stream = table
+            .scan_to_stream(ScanRequest {
+                filters,
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        let batches = stream.try_collect::<Vec<_>>().await.unwrap();
+        let rows = batches.iter().map(|batch| batch.num_rows()).sum();
+        let output = batches
+            .iter()
+            .map(|batch| batch.pretty_print())
+            .collect::<Vec<_>>()
+            .join("\n");
+        (rows, output)
+    }
+
+    fn eq_filter(column: &str, value: &str) -> Expr {
+        Expr::BinaryExpr(BinaryExpr::new(
+            Box::new(col(column)),
+            Operator::Eq,
+            Box::new(lit(value)),
+        ))
+    }
+
+    fn not_eq_filter(column: &str, value: &str) -> Expr {
+        Expr::BinaryExpr(BinaryExpr::new(
+            Box::new(col(column)),
+            Operator::NotEq,
+            Box::new(lit(value)),
+        ))
+    }
+
+    #[tokio::test]
+    async fn test_scan_pinned_table_skips_catalog_enumeration() {
+        let manager: Arc<dyn CatalogManager> = setup_catalog().await;
+
+        // No predicates: every partitioned table is listed.
+        let (rows, output) = scan_partitions(&manager, vec![]).await;
+        assert_eq!(3, rows, "{output}");
+
+        // Pins both the schema and the table name, so the scan resolves the
+        // table directly instead of enumerating all catalog entries.
+        let (rows, output) = scan_partitions(
+            &manager,
+            vec![
+                eq_filter(TABLE_SCHEMA, SCHEMA_1),
+                eq_filter(TABLE_NAME, PARTITIONED_TABLE),
+            ],
+        )
+        .await;
+        assert_eq!(2, rows, "{output}");
+        assert!(output.contains(SCHEMA_1), "{output}");
+        assert!(!output.contains(SCHEMA_2), "{output}");
+        assert!(
+            output.contains(
+                &RegionId::new(TABLE_ID_1, REGION_NUMBER_1)
+                    .as_u64()
+                    .to_string()
+            ),
+            "{output}"
+        );
+        assert!(
+            output.contains(
+                &RegionId::new(TABLE_ID_1, REGION_NUMBER_2)
+                    .as_u64()
+                    .to_string()
+            ),
+            "{output}"
+        );
+
+        // The fallback enumeration path must return the identical result. The
+        // inequality on `table_name` doesn't pin a single name, so no direct
+        // lookup happens here.
+        let (fallback_rows, fallback_output) = scan_partitions(
+            &manager,
+            vec![
+                eq_filter(TABLE_SCHEMA, SCHEMA_1),
+                not_eq_filter(TABLE_NAME, MISSING_TABLE),
+            ],
+        )
+        .await;
+        assert_eq!(rows, fallback_rows, "{fallback_output}");
+        assert_eq!(output, fallback_output);
+
+        // Only the table name is pinned: the schema can't be resolved and both
+        // tables of that name are kept.
+        let (rows, output) =
+            scan_partitions(&manager, vec![eq_filter(TABLE_NAME, PARTITIONED_TABLE)]).await;
+        assert_eq!(3, rows, "{output}");
+        assert!(output.contains(SCHEMA_1), "{output}");
+        assert!(output.contains(SCHEMA_2), "{output}");
+
+        // An unknown table in a pinned schema yields no rows.
+        let (rows, output) = scan_partitions(
+            &manager,
+            vec![
+                eq_filter(TABLE_SCHEMA, SCHEMA_1),
+                eq_filter(TABLE_NAME, MISSING_TABLE),
+            ],
+        )
+        .await;
+        assert_eq!(0, rows, "{output}");
+    }
+
+    #[tokio::test]
+    async fn test_scan_equality_under_disjunction_falls_back_to_enumeration() {
+        let manager: Arc<dyn CatalogManager> = setup_catalog().await;
+
+        // `table_schema = SCHEMA_1 OR table_name = PARTITIONED_TABLE` matches
+        // the tables in both schemas: equality nested under a disjunction must
+        // not pin a single table.
+        let filter = Expr::BinaryExpr(BinaryExpr::new(
+            Box::new(eq_filter(TABLE_SCHEMA, SCHEMA_1)),
+            Operator::Or,
+            Box::new(eq_filter(TABLE_NAME, PARTITIONED_TABLE)),
+        ));
+        let (rows, output) = scan_partitions(&manager, vec![filter]).await;
+        assert_eq!(3, rows, "{output}");
+        assert!(output.contains(SCHEMA_1), "{output}");
+        assert!(output.contains(SCHEMA_2), "{output}");
+    }
+}

src/catalog/src/system_schema/predicate.rs

@@ -214,6 +214,22 @@ impl Predicate {
             _ => None,
         }
     }
+
+    /// Collects the values that constrain `column` to be equal to them at this
+    /// predicate's top-level conjunctions.
+    ///
+    /// Constraints under `Or`/`Not` are ignored because they don't guarantee
+    /// that `column` equals a single value.
+    fn collect_eq_values<'a>(&'a self, column: &str, values: &mut Vec<&'a Value>) {
+        match self {
+            Predicate::Eq(c, v) if c == column => values.push(v),
+            Predicate::And(left, right) => {
+                left.collect_eq_values(column, values);
+                right.collect_eq_values(column, values);
+            }
+            _ => {}
+        }
+    }
 }
 
 /// Perform SQL left LIKE right, return `None` if fail to evaluate.
@@ -269,6 +285,25 @@ impl Predicates {
         }
     }
 
+    /// Returns the single equality value that constrains `column` in the
+    /// top-level conjunctions of the predicates.
+    ///
+    /// Returns `None` if there is no equality constraint on `column`, or if
+    /// the predicates constrain it with multiple distinct values.
+    pub fn find_eq_value(&self, column: &str) -> Option<&Value> {
+        let mut values = Vec::new();
+        for predicate in &self.predicates {
+            predicate.collect_eq_values(column, &mut values);
+        }
+
+        let (first, rest) = values.split_first()?;
+        if rest.iter().all(|v| v == first) {
+            Some(*first)
+        } else {
+            None
+        }
+    }
+
     /// Evaluate the predicates with the row.
     /// returns true when all the predicates are satisfied or can't be evaluated.
     pub fn eval(&self, row: &[(&str, &Value)]) -> bool {
@@ -556,6 +591,71 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_find_eq_value() {
+        fn eq(name: &str, value: &str) -> Predicate {
+            Predicate::Eq(name.to_string(), Value::from(value))
+        }
+
+        fn and(left: Predicate, right: Predicate) -> Predicate {
+            Predicate::And(Box::new(left), Box::new(right))
+        }
+
+        fn or(left: Predicate, right: Predicate) -> Predicate {
+            Predicate::Or(Box::new(left), Box::new(right))
+        }
+
+        fn predicates(predicates: Vec<Predicate>) -> Predicates {
+            Predicates { predicates }
+        }
+
+        // No predicates, no equality value.
+        assert!(predicates(vec![]).find_eq_value("a").is_none());
+
+        // No equality on the column.
+        let p = predicates(vec![Predicate::NotEq("a".to_string(), Value::from("x"))]);
+        assert!(p.find_eq_value("a").is_none());
+        assert!(p.find_eq_value("b").is_none());
+
+        // Simple equality.
+        let p = predicates(vec![eq("a", "x")]);
+        let v = p.find_eq_value("a").unwrap();
+        assert!(match_string_value(v, "x"));
+        assert!(p.find_eq_value("b").is_none());
+
+        // Duplicated equality on the same value.
+        let p = predicates(vec![and(eq("a", "x"), eq("a", "x"))]);
+        let v = p.find_eq_value("a").unwrap();
+        assert!(match_string_value(v, "x"));
+
+        // Conflicting equalities reject the lookup.
+        let p = predicates(vec![and(eq("a", "x"), eq("a", "y"))]);
+        assert!(p.find_eq_value("a").is_none());
+
+        // Conflicting equalities in separate predicates reject the lookup too.
+        let p = predicates(vec![eq("a", "x"), eq("a", "y")]);
+        assert!(p.find_eq_value("a").is_none());
+
+        // Equality nested in deeper conjunctions.
+        let p = predicates(vec![and(and(eq("b", "y"), eq("a", "x")), eq("a", "x"))]);
+        let v = p.find_eq_value("a").unwrap();
+        assert!(match_string_value(v, "x"));
+
+        // Equality under a disjunction or a negation doesn't constrain the column.
+        let p = predicates(vec![or(eq("a", "x"), eq("a", "x"))]);
+        assert!(p.find_eq_value("a").is_none());
+        let p = predicates(vec![Predicate::Not(Box::new(eq("a", "x")))]);
+        assert!(p.find_eq_value("a").is_none());
+
+        // Equality value in a top-level conjunction composed of separate predicates.
+        let p = predicates(vec![
+            eq("a", "x"),
+            Predicate::Like("b".to_string(), "%y".to_string(), false),
+        ]);
+        let v = p.find_eq_value("a").unwrap();
+        assert!(match_string_value(v, "x"));
+    }
+
     #[test]
     fn test_predicates_eval_row() {
         let wrong_row = [

src/cli/Cargo.toml

@@ -18,10 +18,8 @@ workspace = true
 [dependencies]
 async-stream.workspace = true
 async-trait.workspace = true
-auth.workspace = true
 base64.workspace = true
 cache.workspace = true
-catalog.workspace = true
 chrono.workspace = true
 clap.workspace = true
 client = { workspace = true, features = ["testing"] }
@@ -34,14 +32,11 @@ common-grpc.workspace = true
 common-macro.workspace = true
 common-meta.workspace = true
 common-procedure.workspace = true
-common-query.workspace = true
-common-recordbatch.workspace = true
 common-runtime.workspace = true
 common-telemetry = { workspace = true, features = [
     "deadlock_detection",
 ] }
 common-time.workspace = true
-common-version.workspace = true
 common-wal.workspace = true
 datatypes.workspace = true
 etcd-client.workspace = true
@@ -61,21 +56,19 @@ reqwest.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 servers.workspace = true
-session.workspace = true
 snafu.workspace = true
 sql.workspace = true
 standalone.workspace = true
 store-api.workspace = true
 table.workspace = true
 tokio.workspace = true
-tracing-appender.workspace = true
 url.workspace = true
 uuid.workspace = true
 
 [dev-dependencies]
 common-meta = { workspace = true, features = ["testing"] }
 common-test-util.workspace = true
-common-version.workspace = true
 object-store = { workspace = true, features = ["testing"] }
 serde.workspace = true
 tempfile.workspace = true
+tokio-util.workspace = true

src/cli/src/common/store.rs

@@ -86,7 +86,13 @@ pub struct StoreConfig {
 
     /// Automatically create PostgreSQL schema if it doesn't exist (default: true).
     #[cfg(feature = "pg_kvbackend")]
-    #[clap(long, default_value_t = true)]
+    #[clap(
+        long,
+        default_value_t = true,
+        default_missing_value = "true",
+        num_args = 0..=1,
+        action = clap::ArgAction::Set
+    )]
     pub auto_create_schema: bool,
 
     /// TLS mode for backend store connections (etcd, PostgreSQL, MySQL)
@@ -223,3 +229,27 @@ impl StoreConfig {
         }
     }
 }
+
+#[cfg(all(test, feature = "pg_kvbackend"))]
+mod tests {
+    use clap::Parser;
+
+    use super::StoreConfig;
+
+    #[test]
+    fn test_auto_create_schema_accepts_explicit_value() {
+        let config =
+            StoreConfig::try_parse_from(["store", "--auto-create-schema", "false"]).unwrap();
+        assert!(!config.auto_create_schema);
+
+        let config =
+            StoreConfig::try_parse_from(["store", "--auto-create-schema", "true"]).unwrap();
+        assert!(config.auto_create_schema);
+    }
+
+    #[test]
+    fn test_auto_create_schema_defaults_to_true() {
+        let config = StoreConfig::try_parse_from(["store"]).unwrap();
+        assert!(config.auto_create_schema);
+    }
+}

src/cli/src/data.rs

@@ -18,6 +18,7 @@ mod import;
 pub mod import_v2;
 pub(crate) mod path;
 pub(crate) mod progress;
+mod schema_export;
 pub mod snapshot_storage;
 pub(crate) mod sql;
 mod storage_export;

src/cli/src/data/export.rs

@@ -27,6 +27,7 @@ use tokio::sync::Semaphore;
 use tokio::time::Instant;
 
 use crate::common::{ObjectStoreConfig, new_fs_object_store};
+use crate::data::schema_export::append_schema_ddl;
 use crate::data::storage_export::{
     AzblobBackend, FsBackend, GcsBackend, OssBackend, S3Backend, StorageExport, StorageType,
 };
@@ -335,32 +336,6 @@ impl Export {
         ))
     }
 
-    async fn show_create(
-        &self,
-        show_type: &str,
-        catalog: &str,
-        schema: &str,
-        table: Option<&str>,
-    ) -> Result<String> {
-        let sql = match table {
-            Some(table) => format!(
-                r#"SHOW CREATE {} "{}"."{}"."{}""#,
-                show_type, catalog, schema, table
-            ),
-            None => format!(r#"SHOW CREATE {} "{}"."{}""#, show_type, catalog, schema),
-        };
-        let records = self
-            .database_client
-            .sql_in_public(&sql)
-            .await?
-            .context(EmptyResultSnafu)?;
-        let Value::String(create) = &records[0][1] else {
-            unreachable!()
-        };
-
-        Ok(format!("{};\n", create))
-    }
-
     async fn export_create_database(&self) -> Result<()> {
         let timer = Instant::now();
         let db_names = self.get_db_names().await?;
@@ -368,9 +343,15 @@ impl Export {
         let operator = self.build_prefer_fs_operator().await?;
 
         for schema in db_names {
-            let create_database = self
-                .show_create("DATABASE", &self.catalog, &schema, None)
-                .await?;
+            let mut create_database = String::new();
+            append_schema_ddl(
+                &self.database_client,
+                &self.catalog,
+                &schema,
+                std::iter::once(("DATABASE", None)),
+                &mut create_database,
+            )
+            .await?;
 
             let file_path = self.get_file_path(&schema, "create_database.sql");
             self.write_to_storage(&operator, &file_path, create_database.into_bytes())
@@ -415,23 +396,28 @@ impl Export {
                 }
 
                 let file_path = export_self.get_file_path(&schema, "create_tables.sql");
-                let mut content = Vec::new();
-
-                // Add table creation SQL
-                for (c, s, t) in metric_physical_tables.iter().chain(&remaining_tables) {
-                    let create_table = export_self.show_create("TABLE", c, s, Some(t)).await?;
-                    content.extend_from_slice(create_table.as_bytes());
-                }
-
-                // Add view creation SQL
-                for (c, s, v) in &views {
-                    let create_view = export_self.show_create("VIEW", c, s, Some(v)).await?;
-                    content.extend_from_slice(create_view.as_bytes());
-                }
+                let mut content = String::new();
+                let objects = metric_physical_tables
+                    .iter()
+                    .chain(&remaining_tables)
+                    .map(|(_, _, table)| ("TABLE", Some(table.as_str())))
+                    .chain(
+                        views
+                            .iter()
+                            .map(|(_, _, view)| ("VIEW", Some(view.as_str()))),
+                    );
+                append_schema_ddl(
+                    &export_self.database_client,
+                    &export_self.catalog,
+                    &schema,
+                    objects,
+                    &mut content,
+                )
+                .await?;
 
                 // Write to storage
                 export_self
-                    .write_to_storage(&operator, &file_path, content)
+                    .write_to_storage(&operator, &file_path, content.into_bytes())
                     .await?;
 
                 info!(
@@ -445,9 +431,11 @@ impl Export {
             });
         }
 
-        let success = self.execute_tasks(tasks).await;
+        for result in futures::future::join_all(tasks).await {
+            result?;
+        }
         let elapsed = timer.elapsed();
-        info!("Success {success}/{db_count} jobs, cost: {elapsed:?}");
+        info!("Success {db_count}/{db_count} jobs, cost: {elapsed:?}");
 
         Ok(())
     }

src/cli/src/data/export_v2.rs

@@ -41,11 +41,13 @@
 //! `--experimental-metric-export` enables shared Metric physical scans for Parquet.
 //! Enable `experimental_metric_export = true` on the frontend or standalone server
 //! and use an endpoint whose frontends all support and enable this option.
+//! Add `--metric-data-layout packed` to write version-2 packed snapshots.
+//! This requires the server packed-export capability; schema-only exports remain version 1.
 //! The command checks the server capability before modifying the snapshot.
 //! Each snapshot path belongs to one export task. Resume requires stable source
 //! schemas/data and confirmation that the previous export and storage writes ended;
 //! an HTTP timeout does not establish that. Only unfinished chunks are cleaned and
-//! rerun. Completed chunks and the V2 snapshot/import format remain unchanged.
+//! rerun. Resume retains the snapshot version/layout and completed chunks.
 
 mod chunker;
 mod command;

src/cli/src/data/export_v2/command.rs

@@ -25,15 +25,15 @@ use common_error::ext::BoxedError;
 use common_telemetry::info;
 use serde_json::Value;
 use servers::http::{ColumnSchema, GreptimeQueryOutput, OutputSchema};
-use snafu::{OptionExt, ResultExt};
+use snafu::ResultExt;
 
 use crate::Tool;
 use crate::common::ObjectStoreConfig;
 use crate::data::export_v2::coordinator::{ExportDataOptions, export_data};
 use crate::data::export_v2::error::{
-    ChunkTimeWindowRequiresBoundsSnafu, DatabaseSnafu, EmptyResultSnafu, IoSnafu,
-    ManifestVersionMismatchSnafu, Result, ResumeConfigMismatchSnafu, SchemaOnlyArgsNotAllowedSnafu,
-    SchemaOnlyModeMismatchSnafu, SnapshotVerifyFailedSnafu, UnexpectedValueTypeSnafu,
+    ChunkTimeWindowRequiresBoundsSnafu, DatabaseSnafu, IoSnafu, ManifestVersionMismatchSnafu,
+    Result, ResumeConfigMismatchSnafu, SchemaOnlyArgsNotAllowedSnafu, SchemaOnlyModeMismatchSnafu,
+    SnapshotVerifyFailedSnafu, UnexpectedValueTypeSnafu,
 };
 use crate::data::export_v2::extractor::SchemaExtractor;
 use crate::data::export_v2::manifest::{
@@ -42,10 +42,11 @@ use crate::data::export_v2::manifest::{
 use crate::data::export_v2::schema::{DDL_DIR, SCHEMA_DIR, SCHEMAS_FILE};
 use crate::data::path::{data_dir_for_schema_chunk, ddl_path_for_schema};
 use crate::data::progress::{ProgressMode, build_progress_reporter};
+use crate::data::schema_export::append_schema_ddl;
 use crate::data::snapshot_storage::{
     OpenDalStorage, SnapshotStorage, validate_snapshot_uri, validate_uri,
 };
-use crate::data::sql::{escape_sql_identifier, escape_sql_literal};
+use crate::data::sql::escape_sql_literal;
 use crate::database::{DatabaseClient, parse_proxy_opts};
 
 /// Export V2 commands.
@@ -296,6 +297,10 @@ pub struct ExportCreateCommand {
     #[clap(long)]
     experimental_metric_export: bool,
 
+    /// Store independent Metric Parquet streams in shared packed objects.
+    #[clap(long, value_parser = ["packed"], requires = "experimental_metric_export")]
+    metric_data_layout: Option<String>,
+
     /// Delete existing snapshot and recreate.
     #[clap(long)]
     force: bool,
@@ -345,6 +350,16 @@ impl ExportCreateCommand {
 
         let time_range = TimeRange::parse(self.start_time.as_deref(), self.end_time.as_deref())
             .map_err(BoxedError::new)?;
+        if self.metric_data_layout.is_some()
+            && time_range.is_bounded()
+            && time_range.start == time_range.end
+        {
+            return crate::error::InvalidArgumentsSnafu {
+                msg: "Packed export requires --start-time to be earlier than --end-time",
+            }
+            .fail()
+            .map_err(BoxedError::new);
+        }
         if self.chunk_time_window.is_some() && !time_range.is_bounded() {
             return ChunkTimeWindowRequiresBoundsSnafu
                 .fail()
@@ -425,6 +440,13 @@ impl ExportCreateCommand {
                     .map_err(BoxedError::new);
             }
         }
+        if self.metric_data_layout.is_some() && !self.schema_only {
+            database_client
+                .require_packed_export()
+                .await
+                .context(DatabaseSnafu)
+                .map_err(BoxedError::new)?;
+        }
         let storage = OpenDalStorage::from_uri(&self.to, &self.storage).map_err(BoxedError::new)?;
 
         Ok(Box::new(ExportCreate {
@@ -434,6 +456,7 @@ impl ExportCreateCommand {
                 schema_only: self.schema_only,
                 format: self.format,
                 experimental_metric_export: self.experimental_metric_export,
+                packed: self.metric_data_layout.is_some(),
                 force: self.force,
                 time_range,
                 chunk_time_window: self.chunk_time_window,
@@ -462,6 +485,7 @@ struct ExportConfig {
     schema_only: bool,
     format: DataFormat,
     experimental_metric_export: bool,
+    packed: bool,
     force: bool,
     time_range: TimeRange,
     chunk_time_window: Option<Duration>,
@@ -504,7 +528,7 @@ impl ExportCreate {
                 let mut manifest = self.storage.read_manifest().await?;
 
                 // Check version compatibility
-                if manifest.version != MANIFEST_VERSION || manifest.data_layout.is_some() {
+                if manifest.validate_layout().is_err() {
                     return ManifestVersionMismatchSnafu {
                         expected: MANIFEST_VERSION,
                         found: manifest.version,
@@ -513,6 +537,15 @@ impl ExportCreate {
                 }
 
                 validate_resume_config(&manifest, &self.config)?;
+                if manifest.is_packed() {
+                    self.database_client
+                        .require_packed_export()
+                        .await
+                        .context(DatabaseSnafu)?;
+                    if !self.config.experimental_metric_export {
+                        return crate::data::export_v2::error::MetricExportUnavailableSnafu.fail();
+                    }
+                }
 
                 info!(
                     "Resuming existing snapshot: {} (completed: {}/{} chunks)",
@@ -571,10 +604,14 @@ impl ExportCreate {
             self.config.chunk_time_window,
         )?;
 
+        if self.config.packed && !self.config.schema_only {
+            manifest.version = 2;
+            manifest.data_layout = Some(common_datasource::packed_snapshot::PACKED_LAYOUT.into());
+        }
         if self.config.experimental_metric_export {
             for chunk in &manifest.chunks {
                 self.storage
-                    .prepare_export_chunk(&schema_names, chunk.id, false)
+                    .prepare_export_chunk(&schema_names, chunk.id, false, manifest.is_packed())
                     .await?;
             }
         }
@@ -584,8 +621,10 @@ impl ExportCreate {
         info!("Exported {} schemas", schema_snapshot.schemas.len());
 
         // 5. Export DDL files for import recovery.
-        let ddl_by_schema = self.build_ddl_by_schema(&schema_names).await?;
-        for (schema, ddl) in ddl_by_schema {
+        let mut sorted_schemas = schema_names;
+        sorted_schemas.sort();
+        for schema in sorted_schemas {
+            let ddl = self.build_schema_ddl(&schema).await?;
             let ddl_path = ddl_path_for_schema(&schema);
             self.storage.write_text(&ddl_path, &ddl).await?;
             info!("Exported DDL for schema {} to {}", schema, ddl_path);
@@ -623,45 +662,31 @@ impl ExportCreate {
         Ok(())
     }
 
-    async fn build_ddl_by_schema(&self, schema_names: &[String]) -> Result<Vec<(String, String)>> {
-        let mut schemas = schema_names.to_vec();
-        schemas.sort();
-
-        let mut ddl_by_schema = Vec::with_capacity(schemas.len());
-        for schema in schemas {
-            let create_database = self.show_create("DATABASE", &schema, None).await?;
-
-            let (mut physical_tables, mut tables, mut views) =
-                self.get_schema_objects(&schema).await?;
-            physical_tables.sort();
-            let mut physical_ddls = Vec::with_capacity(physical_tables.len());
-            for table in physical_tables {
-                physical_ddls.push(self.show_create("TABLE", &schema, Some(&table)).await?);
-            }
-
-            tables.sort();
-            let mut table_ddls = Vec::with_capacity(tables.len());
-            for table in tables {
-                table_ddls.push(self.show_create("TABLE", &schema, Some(&table)).await?);
-            }
-
-            views.sort();
-            let mut view_ddls = Vec::with_capacity(views.len());
-            for view in views {
-                view_ddls.push(self.show_create("VIEW", &schema, Some(&view)).await?);
-            }
-
-            let ddl = build_schema_ddl(
-                &schema,
-                create_database,
-                physical_ddls,
-                table_ddls,
-                view_ddls,
-            );
-            ddl_by_schema.push((schema, ddl));
-        }
-
-        Ok(ddl_by_schema)
+    async fn build_schema_ddl(&self, schema: &str) -> Result<String> {
+        let (mut physical_tables, mut tables, mut views) = self.get_schema_objects(schema).await?;
+        physical_tables.sort();
+        tables.sort();
+        views.sort();
+        let objects = std::iter::once(("DATABASE", None))
+            .chain(
+                physical_tables
+                    .iter()
+                    .chain(&tables)
+                    .map(|table| ("TABLE", Some(table.as_str()))),
+            )
+            .chain(views.iter().map(|view| ("VIEW", Some(view.as_str()))));
+        let mut ddl = format!("-- Schema: {schema}\n");
+        append_schema_ddl(
+            &self.database_client,
+            &self.config.catalog,
+            schema,
+            objects,
+            &mut ddl,
+        )
+        .await
+        .context(DatabaseSnafu)?;
+        ddl.push('\n');
+        Ok(ddl)
     }
 
     async fn get_schema_objects(
@@ -734,68 +759,17 @@ impl ExportCreate {
 
         Ok(tables.into_iter().collect())
     }
-
-    async fn show_create(
-        &self,
-        show_type: &str,
-        schema: &str,
-        table: Option<&str>,
-    ) -> Result<String> {
-        let sql = match table {
-            Some(table) => format!(
-                r#"SHOW CREATE {} "{}"."{}"."{}""#,
-                show_type,
-                escape_sql_identifier(&self.config.catalog),
-                escape_sql_identifier(schema),
-                escape_sql_identifier(table)
-            ),
-            None => format!(
-                r#"SHOW CREATE {} "{}"."{}""#,
-                show_type,
-                escape_sql_identifier(&self.config.catalog),
-                escape_sql_identifier(schema)
-            ),
-        };
-
-        let records: Option<Vec<Vec<Value>>> = self
-            .database_client
-            .sql_in_public(&sql)
-            .await
-            .context(DatabaseSnafu)?;
-        let rows = records.context(EmptyResultSnafu)?;
-        let row = rows.first().context(EmptyResultSnafu)?;
-        let Some(Value::String(create)) = row.get(1) else {
-            return UnexpectedValueTypeSnafu.fail();
-        };
-
-        Ok(format!("{};\n", create))
-    }
-}
-
-fn build_schema_ddl(
-    schema: &str,
-    create_database: String,
-    physical_tables: Vec<String>,
-    tables: Vec<String>,
-    views: Vec<String>,
-) -> String {
-    let mut ddl = String::new();
-    ddl.push_str(&format!("-- Schema: {}\n", schema));
-    ddl.push_str(&create_database);
-    for stmt in physical_tables {
-        ddl.push_str(&stmt);
-    }
-    for stmt in tables {
-        ddl.push_str(&stmt);
-    }
-    for stmt in views {
-        ddl.push_str(&stmt);
-    }
-    ddl.push('\n');
-    ddl
 }
 
 fn validate_resume_config(manifest: &Manifest, config: &ExportConfig) -> Result<()> {
+    if config.packed && !manifest.schema_only && !manifest.is_packed() {
+        return ResumeConfigMismatchSnafu {
+            field: "metric_data_layout",
+            existing: "standalone".to_string(),
+            requested: "packed".to_string(),
+        }
+        .fail();
+    }
     if manifest.schema_only != config.schema_only {
         return SchemaOnlyModeMismatchSnafu {
             existing_schema_only: manifest.schema_only,
@@ -1593,25 +1567,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_build_schema_ddl_order() {
-        let ddl = build_schema_ddl(
-            "public",
-            "CREATE DATABASE public;\n".to_string(),
-            vec!["PHYSICAL;\n".to_string()],
-            vec!["TABLE;\n".to_string()],
-            vec!["VIEW;\n".to_string()],
-        );
-
-        let db_pos = ddl.find("CREATE DATABASE").unwrap();
-        let physical_pos = ddl.find("PHYSICAL;").unwrap();
-        let table_pos = ddl.find("TABLE;").unwrap();
-        let view_pos = ddl.find("VIEW;").unwrap();
-        assert!(db_pos < physical_pos);
-        assert!(physical_pos < table_pos);
-        assert!(table_pos < view_pos);
-    }
-
     #[tokio::test]
     async fn test_build_rejects_chunk_window_without_bounds() {
         let cmd = ExportCreateCommand::parse_from([
@@ -1801,6 +1756,7 @@ mod tests {
             schema_only: false,
             format: DataFormat::Parquet,
             experimental_metric_export: false,
+            packed: false,
             force: false,
             time_range: TimeRange::unbounded(),
             chunk_time_window: None,
@@ -1820,7 +1776,7 @@ mod tests {
 
     #[test]
     fn test_validate_resume_config_accepts_schema_selection_with_different_case_and_order() {
-        let manifest = Manifest::new_for_export(
+        let mut manifest = Manifest::new_for_export(
             "greptime".to_string(),
             vec!["public".to_string(), "analytics".to_string()],
             false,
@@ -1829,7 +1785,7 @@ mod tests {
             None,
         )
         .unwrap();
-        let config = ExportConfig {
+        let mut config = ExportConfig {
             catalog: "greptime".to_string(),
             schemas: Some(vec![
                 "ANALYTICS".to_string(),
@@ -1839,6 +1795,7 @@ mod tests {
             schema_only: false,
             format: DataFormat::Parquet,
             experimental_metric_export: false,
+            packed: false,
             force: false,
             time_range: TimeRange::unbounded(),
             chunk_time_window: None,
@@ -1850,6 +1807,18 @@ mod tests {
         };
 
         assert!(validate_resume_config(&manifest, &config).is_ok());
+        config.packed = true;
+        assert!(
+            validate_resume_config(&manifest, &config)
+                .unwrap_err()
+                .to_string()
+                .contains("metric_data_layout")
+        );
+        manifest.version = 2;
+        manifest.data_layout = Some(common_datasource::packed_snapshot::PACKED_LAYOUT.into());
+        assert!(validate_resume_config(&manifest, &config).is_ok());
+        config.packed = false;
+        assert!(validate_resume_config(&manifest, &config).is_ok());
     }
 
     #[test]
@@ -1872,6 +1841,7 @@ mod tests {
             schema_only: false,
             format: DataFormat::Parquet,
             experimental_metric_export: false,
+            packed: false,
             force: false,
             time_range,
             chunk_time_window: Some(Duration::from_secs(3600)),
@@ -1906,6 +1876,7 @@ mod tests {
             schema_only: false,
             format: DataFormat::Csv,
             experimental_metric_export: false,
+            packed: false,
             force: false,
             time_range: TimeRange::unbounded(),
             chunk_time_window: None,
@@ -1942,6 +1913,7 @@ mod tests {
             schema_only: false,
             format: DataFormat::Parquet,
             experimental_metric_export: false,
+            packed: false,
             force: false,
             time_range: TimeRange::new(Some(start), Some(start)),
             chunk_time_window: None,

src/cli/src/data/export_v2/coordinator.rs

@@ -40,6 +40,7 @@ struct ExportContext<'a> {
     format: DataFormat,
     parallelism: usize,
     experimental_metric_export: bool,
+    packed: bool,
     resume: bool,
 }
 
@@ -71,6 +72,7 @@ pub async fn export_data(
         catalog: manifest.catalog.clone(),
         schemas: manifest.schemas.clone(),
         format: manifest.format,
+        packed: manifest.is_packed(),
         parallelism: options.parallelism,
         experimental_metric_export: options.experimental_metric_export,
         resume: options.resume,
@@ -279,7 +281,7 @@ async fn export_chunk(
     if context.experimental_metric_export {
         context
             .storage
-            .prepare_export_chunk(&context.schemas, chunk_id, context.resume)
+            .prepare_export_chunk(&context.schemas, chunk_id, context.resume, context.packed)
             .await?;
     }
     let scheme = StorageScheme::from_uri(context.snapshot_uri)?;
@@ -289,6 +291,7 @@ async fn export_chunk(
         time_range,
         parallelism: context.parallelism,
         experimental_metric_export: context.experimental_metric_export,
+        packed: context.packed,
     };
 
     for schema in &context.schemas {
@@ -314,6 +317,39 @@ async fn export_chunk(
     }
 
     let files = list_chunk_files(context.storage, &context.schemas, chunk_id).await?;
+    if context.packed {
+        use common_datasource::packed_snapshot::{PACK_INDEX_FILE, PackIndex};
+        let mut expected = Vec::new();
+        for schema in &context.schemas {
+            let prefix = data_dir_for_schema_chunk(schema, chunk_id);
+            let index_path = format!("{prefix}{PACK_INDEX_FILE}");
+            let text = context.storage.read_text(&index_path).await?;
+            let index: PackIndex = serde_json::from_str(&text).map_err(|e| {
+                crate::data::export_v2::error::InvalidUriSnafu {
+                    uri: context.snapshot_uri,
+                    reason: e.to_string(),
+                }
+                .build()
+            })?;
+            index.validate().map_err(|e| {
+                crate::data::export_v2::error::InvalidUriSnafu {
+                    uri: context.snapshot_uri,
+                    reason: e.to_string(),
+                }
+                .build()
+            })?;
+            expected.push(index_path);
+            expected.extend(index.objects.iter().map(|o| format!("{prefix}{}", o.path)));
+        }
+        expected.sort();
+        if files != expected {
+            return crate::data::export_v2::error::InvalidUriSnafu {
+                uri: context.snapshot_uri,
+                reason: "chunk inventory differs from packed indexes",
+            }
+            .fail();
+        }
+    }
     info!("Collected {} files for chunk {}", files.len(), chunk_id);
     Ok(files)
 }
@@ -462,6 +498,158 @@ mod tests {
         }
     }
 
+    struct PausedClose {
+        inner: Option<object_store::layers::mock::oio::Writer>,
+        started: std::sync::Arc<tokio::sync::Notify>,
+        release: std::sync::Arc<tokio::sync::Notify>,
+    }
+
+    impl object_store::layers::mock::oio::Write for PausedClose {
+        async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> {
+            self.inner.as_mut().unwrap().write(bytes).await
+        }
+
+        async fn close(&mut self) -> object_store::Result<object_store::layers::mock::Metadata> {
+            let mut inner = self.inner.take().unwrap();
+            let (started, release) = (self.started.clone(), self.release.clone());
+            // Model blocking storage I/O that survives dropping its caller.
+            let (inner, result) = tokio::spawn(async move {
+                started.notify_one();
+                release.notified().await;
+                let result = inner.close().await;
+                (inner, result)
+            })
+            .await
+            .unwrap();
+            self.inner = Some(inner);
+            result
+        }
+
+        async fn abort(&mut self) -> object_store::Result<()> {
+            self.inner.as_mut().unwrap().abort().await
+        }
+    }
+
+    #[tokio::test]
+    async fn packed_cancellation_drains_close_before_failing_chunk() {
+        use std::sync::Arc;
+
+        use common_datasource::packed_snapshot::PACK_INDEX_FILE;
+        use common_datasource::packed_writer::{PackedTableWriter, PackedWriter};
+        use common_datasource::parquet_writer::ParquetFileWriter;
+        use datatypes::arrow::datatypes::{DataType, Field, Schema};
+        use object_store::layers::mock::{MockLayerBuilder, MockWriterFactory};
+        use tokio_util::sync::CancellationToken;
+
+        use crate::data::snapshot_storage::OpenDalStorage;
+
+        for paused_path in ["pack-000000.bin", PACK_INDEX_FILE] {
+            let directory = tempfile::tempdir().unwrap();
+            let uri = url::Url::from_directory_path(directory.path()).unwrap();
+            let storage =
+                OpenDalStorage::from_uri(uri.as_str(), &ObjectStoreConfig::default()).unwrap();
+            let started = Arc::new(tokio::sync::Notify::new());
+            let release = Arc::new(tokio::sync::Notify::new());
+            let factory: MockWriterFactory = Arc::new({
+                let (started, release) = (started.clone(), release.clone());
+                move |path, _, inner| {
+                    if path.trim_start_matches('/') == paused_path {
+                        Box::new(PausedClose {
+                            inner: Some(inner),
+                            started: started.clone(),
+                            release: release.clone(),
+                        })
+                    } else {
+                        inner
+                    }
+                }
+            });
+            let store = object_store::secure_fs::SecureFsRoot::open(directory.path())
+                .unwrap()
+                .build_operator()
+                .layer(
+                    MockLayerBuilder::default()
+                        .writer_factory(factory)
+                        .build()
+                        .unwrap(),
+                );
+            let packed = PackedWriter::new(store.clone()).unwrap();
+            let schema = Arc::new(Schema::new(vec![Field::new(
+                "value",
+                DataType::Int64,
+                true,
+            )]));
+            let mut table = ParquetFileWriter::open_packed(
+                schema,
+                store.clone(),
+                "unused",
+                None,
+                PackedTableWriter::new(packed.clone(), "empty".into(), 1, false),
+            )
+            .unwrap();
+            table.finish(None).await.unwrap();
+            let token = CancellationToken::new();
+            let mut manifest = pending_manifest(1);
+            manifest.version = 2;
+            manifest.data_layout = Some(common_datasource::packed_snapshot::PACKED_LAYOUT.into());
+            let export = export_data_concurrent(
+                &storage,
+                &mut manifest,
+                2,
+                &crate::data::progress::NoopProgress,
+                |_, _| async {
+                    let mut packed = packed.lock().await;
+                    let result = packed.finish(&token).await;
+                    if result.is_err() {
+                        packed.abort().await.unwrap();
+                    }
+                    result.map_err(|error| {
+                        crate::data::export_v2::error::IoSnafu {
+                            operation: "exporting packed chunk",
+                            error: std::io::Error::other(error),
+                        }
+                        .build()
+                    })
+                },
+            );
+            tokio::pin!(export);
+            tokio::select! {
+                _ = started.notified() => {},
+                result = &mut export => panic!("completed while close paused: {result:?}"),
+            }
+            token.cancel();
+            assert!(futures::poll!(&mut export).is_pending());
+            assert_eq!(
+                storage.read_manifest().await.unwrap().chunks[0].status,
+                ChunkStatus::InProgress
+            );
+            release.notify_one();
+            assert!(export.await.unwrap_err().to_string().contains("cancelled"));
+            let persisted = storage.read_manifest().await.unwrap();
+            assert_eq!(persisted.chunks[0].status, ChunkStatus::Failed);
+            assert!(!persisted.is_complete());
+            assert!(persisted.chunks[0].files.is_empty());
+            assert!(
+                store
+                    .stat("pack-000000.bin")
+                    .await
+                    .unwrap()
+                    .content_length()
+                    > 0
+            );
+            assert_eq!(
+                store.exists(PACK_INDEX_FILE).await.unwrap(),
+                paused_path == PACK_INDEX_FILE
+            );
+            if paused_path == PACK_INDEX_FILE {
+                let index: common_datasource::packed_snapshot::PackIndex =
+                    serde_json::from_slice(&store.read(PACK_INDEX_FILE).await.unwrap().to_bytes())
+                        .unwrap();
+                index.validate_membership(["empty"]).unwrap();
+            }
+        }
+    }
+
     #[test]
     fn test_next_eligible_chunk_scans_in_order() {
         let manifest = pending_manifest(3);

src/cli/src/data/export_v2/data.rs

@@ -31,6 +31,7 @@ pub(super) struct CopyOptions {
     pub(super) time_range: TimeRange,
     pub(super) parallelism: usize,
     pub(crate) experimental_metric_export: bool,
+    pub(super) packed: bool,
 }
 
 pub(super) struct CopyTarget {
@@ -220,6 +221,9 @@ fn build_with_options(options: &CopyOptions) -> String {
     if options.experimental_metric_export {
         parts.push("experimental_metric_export='true'".to_string());
     }
+    if options.packed {
+        parts.push("metric_data_layout='packed'".to_string());
+    }
     if let Some(start) = options.time_range.start {
         parts.push(format!(
             "START_TIME='{}'",

src/cli/src/data/export_v2/manifest.rs

@@ -28,7 +28,7 @@ use crate::data::export_v2::error::{
     TimeParseInvalidFormatSnafu,
 };
 
-/// Manifest format version produced by the current exporter.
+/// Default standalone manifest version. Packed data snapshots use version 2.
 pub const MANIFEST_VERSION: u32 = 1;
 
 /// Manifest file name within snapshot directory.

src/cli/src/data/import_v2/command.rs

@@ -222,12 +222,19 @@ impl Import {
         manifest.validate_layout().map_err(|reason| {
             crate::data::import_v2::error::InvalidPackedSnapshotSnafu { reason }.build()
         })?;
-        if manifest.is_packed() {
-            self.database_client
-                .require_packed_import()
+        let capabilities = if manifest.is_packed() {
+            let capabilities = self
+                .database_client
+                .capabilities()
                 .await
                 .context(crate::data::import_v2::error::DatabaseSnafu)?;
-        }
+            capabilities
+                .require("metric_packed_import")
+                .context(crate::data::import_v2::error::DatabaseSnafu)?;
+            Some(capabilities)
+        } else {
+            None
+        };
 
         info!("Snapshot contains {} schema(s)", manifest.schemas.len());
 
@@ -279,6 +286,15 @@ impl Import {
             return Ok(());
         }
 
+        let capabilities = match capabilities {
+            Some(capabilities) => capabilities,
+            None => self
+                .database_client
+                .capabilities()
+                .await
+                .context(crate::data::import_v2::error::DatabaseSnafu)?,
+        };
+
         let mut resume_session = if !data_tasks.is_empty() {
             let state_path = resolve_state_path(
                 self.state_path.as_deref(),
@@ -316,7 +332,9 @@ impl Import {
             false
         } else {
             let executor = DdlExecutor::new(&self.database_client);
-            executor.execute_strict(&ddl_statements).await?;
+            executor
+                .execute_strict(&ddl_statements, capabilities.supports("metric_batch_ddl"))
+                .await?;
             if let Some(session) = resume_session.as_mut() {
                 session.mark_ddl_completed().await?;
             }
@@ -356,9 +374,7 @@ impl Import {
                 .await
                 .context(SnapshotStorageSnafu)?;
             statements.extend(
-                parse_ddl_statements(&content)
-                    .into_iter()
-                    .map(|sql| ddl_statement_for_schema(schema, sql)),
+                iter_ddl_statements(&content).map(|sql| ddl_statement_for_schema(schema, sql)),
             );
         }
 
@@ -402,90 +418,92 @@ impl ImportTaskExecutor for CopyDatabaseImportTaskExecutor<'_> {
     }
 }
 
+#[cfg(test)]
 fn parse_ddl_statements(content: &str) -> Vec<String> {
-    let mut statements = Vec::new();
-    let mut current = String::new();
+    iter_ddl_statements(content).collect()
+}
+
+pub(crate) fn iter_ddl_statements(content: &str) -> impl Iterator<Item = String> + '_ {
     let mut chars = content.chars().peekable();
     let mut in_single_quote = false;
     let mut in_double_quote = false;
     let mut in_line_comment = false;
     let mut in_block_comment = false;
 
-    while let Some(ch) = chars.next() {
-        if in_line_comment {
-            if ch == '\n' {
-                in_line_comment = false;
-                current.push('\n');
-            }
-            continue;
-        }
-
-        if in_block_comment {
-            if ch == '*' && chars.peek() == Some(&'/') {
-                chars.next();
-                in_block_comment = false;
-            }
-            continue;
-        }
-
-        if in_single_quote {
-            current.push(ch);
-            if ch == '\'' {
-                if chars.peek() == Some(&'\'') {
-                    current.push(chars.next().expect("peeked quote must exist"));
-                } else {
-                    in_single_quote = false;
+    std::iter::from_fn(move || {
+        let mut current = String::new();
+        while let Some(ch) = chars.next() {
+            if in_line_comment {
+                if ch == '\n' {
+                    in_line_comment = false;
+                    current.push('\n');
                 }
+                continue;
             }
-            continue;
-        }
 
-        if in_double_quote {
-            current.push(ch);
-            if ch == '"' {
-                if chars.peek() == Some(&'"') {
-                    current.push(chars.next().expect("peeked quote must exist"));
-                } else {
-                    in_double_quote = false;
+            if in_block_comment {
+                if ch == '*' && chars.peek() == Some(&'/') {
+                    chars.next();
+                    in_block_comment = false;
                 }
+                continue;
             }
-            continue;
-        }
 
-        match ch {
-            '-' if chars.peek() == Some(&'-') => {
-                chars.next();
-                in_line_comment = true;
-            }
-            '/' if chars.peek() == Some(&'*') => {
-                chars.next();
-                in_block_comment = true;
-            }
-            '\'' => {
-                in_single_quote = true;
+            if in_single_quote {
                 current.push(ch);
-            }
-            '"' => {
-                in_double_quote = true;
-                current.push(ch);
-            }
-            ';' => {
-                let statement = current.trim();
-                if !statement.is_empty() {
-                    statements.push(statement.to_string());
+                if ch == '\'' {
+                    if chars.peek() == Some(&'\'') {
+                        current.push(chars.next().expect("peeked quote must exist"));
+                    } else {
+                        in_single_quote = false;
+                    }
                 }
-                current.clear();
+                continue;
+            }
+
+            if in_double_quote {
+                current.push(ch);
+                if ch == '"' {
+                    if chars.peek() == Some(&'"') {
+                        current.push(chars.next().expect("peeked quote must exist"));
+                    } else {
+                        in_double_quote = false;
+                    }
+                }
+                continue;
+            }
+
+            match ch {
+                '-' if chars.peek() == Some(&'-') => {
+                    chars.next();
+                    in_line_comment = true;
+                }
+                '/' if chars.peek() == Some(&'*') => {
+                    chars.next();
+                    in_block_comment = true;
+                }
+                '\'' => {
+                    in_single_quote = true;
+                    current.push(ch);
+                }
+                '"' => {
+                    in_double_quote = true;
+                    current.push(ch);
+                }
+                ';' => {
+                    let statement = current.trim();
+                    if !statement.is_empty() {
+                        return Some(statement.to_string());
+                    }
+                    current.clear();
+                }
+                _ => current.push(ch),
             }
-            _ => current.push(ch),
         }
-    }
 
-    let statement = current.trim();
-    if !statement.is_empty() {
-        statements.push(statement.to_string());
-    }
-
-    statements
+        let statement = current.trim();
+        (!statement.is_empty()).then(|| statement.to_string())
+    })
 }
 
 fn ddl_statement_for_schema(schema: &str, sql: String) -> DdlStatement {

src/cli/src/data/import_v2/executor.rs

@@ -14,8 +14,18 @@
 
 //! DDL execution for import.
 
+use std::collections::HashSet;
+
+use common_catalog::consts::DEFAULT_SCHEMA_NAME;
 use common_telemetry::info;
+use servers::query_handler::sql::{MAX_LOGICAL_TABLE_DDL_BYTES, MAX_LOGICAL_TABLE_DDL_STATEMENTS};
 use snafu::ResultExt;
+use sql::dialect::GreptimeDbDialect;
+use sql::parser::{ParseOptions, ParserContext};
+use sql::statements::statement::Statement;
+use store_api::metric_engine_consts::{
+    LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY,
+};
 
 use crate::data::import_v2::error::{DatabaseSnafu, Result};
 use crate::database::DatabaseClient;
@@ -54,69 +64,379 @@ impl<'a> DdlExecutor<'a> {
         Self { client }
     }
 
-    /// Executes a list of DDL statements, stopping on first error.
-    pub async fn execute_strict(&self, statements: &[DdlStatement]) -> Result<()> {
-        let total = statements.len();
-
+    /// Executes DDL in order, stopping at the first failed request.
+    pub async fn execute_strict(
+        &self,
+        statements: &[DdlStatement],
+        batch_supported: bool,
+    ) -> Result<()> {
+        let mut batch = LogicalBatch::default();
         for (i, stmt) in statements.iter().enumerate() {
-            let preview = preview_sql(&stmt.sql);
-
-            info!("Executing DDL ({}/{}): {}", i + 1, total, preview);
-
-            if let Some(schema) = stmt.execution_schema.as_deref() {
+            info!("Executing DDL ({}/{})", i + 1, statements.len());
+            let schema = stmt
+                .execution_schema
+                .as_deref()
+                .unwrap_or(DEFAULT_SCHEMA_NAME);
+            let candidate = if batch_supported {
+                classify(stmt, self.client.catalog()).context(DatabaseSnafu)?
+            } else {
+                None
+            };
+            if let Some(candidate) = candidate {
+                if !batch.accepts(&candidate, &stmt.sql) {
+                    batch.flush(self.client).await?;
+                }
+                batch.push(candidate, &stmt.sql);
+            } else {
+                batch.flush(self.client).await?;
                 self.client
                     .sql(&stmt.sql, schema)
                     .await
                     .context(DatabaseSnafu)?;
-            } else {
-                self.client
-                    .sql_in_public(&stmt.sql)
-                    .await
-                    .context(DatabaseSnafu)?;
             }
         }
+        batch.flush(self.client).await
+    }
+}
 
+struct LogicalTable {
+    catalog: String,
+    execution_schema: String,
+    schema: String,
+    physical: String,
+    name: String,
+}
+
+fn classify(stmt: &DdlStatement, catalog: &str) -> crate::error::Result<Option<LogicalTable>> {
+    if stmt.sql.len() > MAX_LOGICAL_TABLE_DDL_BYTES {
+        return Ok(None);
+    }
+    let parsed = parse_ddl(&stmt.sql)?;
+    let [Statement::CreateTable(create)] = parsed.as_slice() else {
+        return Ok(None);
+    };
+    let Some(physical) = create.options.get(LOGICAL_TABLE_METADATA_KEY) else {
+        return Ok(None);
+    };
+    if create.engine != METRIC_ENGINE_NAME
+        || create.options.value(PHYSICAL_TABLE_METADATA_KEY).is_some()
+    {
+        return Ok(None);
+    }
+    let parts: Vec<_> = create
+        .name
+        .0
+        .iter()
+        .filter_map(|p| p.as_ident().map(|i| i.value.as_str()))
+        .collect();
+    let schema = stmt
+        .execution_schema
+        .as_deref()
+        .unwrap_or(DEFAULT_SCHEMA_NAME);
+    let (catalog, schema, name) = match parts.as_slice() {
+        [name] => (catalog, schema, *name),
+        [schema, name] => (catalog, *schema, *name),
+        [catalog, schema, name] => (*catalog, *schema, *name),
+        _ => return Ok(None),
+    };
+    Ok(Some(LogicalTable {
+        catalog: catalog.into(),
+        execution_schema: stmt
+            .execution_schema
+            .as_deref()
+            .unwrap_or(DEFAULT_SCHEMA_NAME)
+            .into(),
+        schema: schema.into(),
+        physical: physical.into(),
+        name: name.into(),
+    }))
+}
+
+pub(crate) fn parse_ddl(sql: &str) -> crate::error::Result<Vec<Statement>> {
+    ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()).map_err(
+        |_| {
+            crate::error::InvalidArgumentsSnafu {
+                msg: "cannot parse import DDL",
+            }
+            .build()
+        },
+    )
+}
+
+#[derive(Default)]
+struct LogicalBatch {
+    group: Option<LogicalTable>,
+    names: HashSet<String>,
+    sql: String,
+}
+
+impl LogicalBatch {
+    fn accepts(&self, table: &LogicalTable, sql: &str) -> bool {
+        self.group.as_ref().is_none_or(|g| {
+            g.catalog == table.catalog
+                && g.schema == table.schema
+                && g.physical == table.physical
+                && g.execution_schema == table.execution_schema
+        }) && !self.names.contains(&table.name)
+            && self.names.len() < MAX_LOGICAL_TABLE_DDL_STATEMENTS
+            && self.sql.len() + (if self.sql.is_empty() { 0 } else { 2 }) + sql.len()
+                <= MAX_LOGICAL_TABLE_DDL_BYTES
+    }
+
+    fn push(&mut self, table: LogicalTable, sql: &str) {
+        if !self.sql.is_empty() {
+            self.sql.push_str(";\n");
+        }
+        self.sql.push_str(sql);
+        self.names.insert(table.name.clone());
+        self.group = Some(table);
+    }
+
+    async fn flush(&mut self, client: &DatabaseClient) -> Result<()> {
+        if let Some(group) = self.group.as_ref() {
+            client
+                .logical_tables(&self.sql, &group.execution_schema, self.names.len())
+                .await
+                .context(DatabaseSnafu)?;
+            self.sql.clear();
+            self.names.clear();
+            self.group = None;
+        }
         Ok(())
     }
 }
 
-fn preview_sql(sql: &str) -> String {
-    let mut chars = sql.chars();
-    let preview: String = chars.by_ref().take(80).collect();
-    if chars.next().is_some() {
-        format!("{preview}...")
-    } else {
-        preview
-    }
-}
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    #[test]
-    fn test_statement_without_execution_schema_uses_public() {
-        let stmt = DdlStatement::new("CREATE DATABASE IF NOT EXISTS test_db".to_string());
-        assert_eq!(stmt.execution_schema, None);
+    fn logical(name: &str, physical: &str) -> String {
+        format!(
+            "CREATE TABLE IF NOT EXISTS {name} (ts TIMESTAMP TIME INDEX) ENGINE=metric WITH(on_physical_table='{physical}')"
+        )
     }
 
     #[test]
-    fn test_statement_with_execution_schema_preserves_context() {
-        let stmt = DdlStatement::with_execution_schema(
-            r#"CREATE TABLE IF NOT EXISTS "my""schema"."metrics" (ts TIMESTAMP TIME INDEX)"#
-                .to_string(),
-            r#"my"schema"#.to_string(),
-        );
-        assert_eq!(stmt.execution_schema.as_deref(), Some(r#"my"schema"#));
+    fn classification_and_batch_boundaries() {
+        let mut batch = LogicalBatch::default();
+        for (name, context, physical, expected) in [
+            ("A", "public", "p.q", true),
+            ("public.b", "other", "p.q", false),
+            ("greptime.public.c", "public", "p.q", true),
+            ("a", "public", "p.q", false),
+            ("\"A\"", "public", "p.q", true),
+            ("d", "public", "p", false),
+            ("d", "other", "p.q", false),
+            ("other.public.d", "public", "p.q", false),
+        ] {
+            let sql = logical(name, physical);
+            let stmt = DdlStatement::with_execution_schema(sql.clone(), context.into());
+            let table = classify(&stmt, "greptime").unwrap().unwrap();
+            assert_eq!(batch.accepts(&table, &sql), expected, "{name}");
+            if expected {
+                batch.push(table, &sql);
+            }
+        }
+        for sql in [
+            "CREATE DATABASE d",
+            "CREATE TABLE t (ts TIMESTAMP TIME INDEX)",
+            "CREATE TABLE p (ts TIMESTAMP TIME INDEX) ENGINE=metric WITH(physical_metric_table='')",
+            "CREATE VIEW v AS SELECT * FROM a",
+        ] {
+            assert!(
+                classify(&DdlStatement::new(sql.into()), "greptime")
+                    .unwrap()
+                    .is_none()
+            );
+        }
+        let mut batch = LogicalBatch::default();
+        for i in 0..128 {
+            let sql = logical(&format!("t{i}"), "p");
+            let table = classify(&DdlStatement::new(sql.clone()), "greptime")
+                .unwrap()
+                .unwrap();
+            assert!(batch.accepts(&table, &sql));
+            batch.push(table, &sql);
+        }
+        let table = classify(&DdlStatement::new(logical("next", "p")), "greptime")
+            .unwrap()
+            .unwrap();
+        assert!(!batch.accepts(&table, &logical("next", "p")));
+        batch.names.clear();
+        batch.sql = "测".repeat(MAX_LOGICAL_TABLE_DDL_BYTES / 3 - 1);
+        assert!(batch.accepts(&table, "ab"));
+        assert!(!batch.accepts(&table, "测"));
+        let oversized =
+            DdlStatement::new(logical("large", &"x".repeat(MAX_LOGICAL_TABLE_DDL_BYTES)));
+        assert!(classify(&oversized, "greptime").unwrap().is_none());
     }
 
-    #[test]
-    fn test_preview_sql_truncates_at_char_boundary() {
-        let sql = format!(
-            "CREATE TABLE {} (ts TIMESTAMP TIME INDEX)",
-            "测".repeat(100)
-        );
-        let preview = preview_sql(&sql);
-        assert!(preview.ends_with("..."));
-        assert!(preview.is_char_boundary(preview.len()));
+    #[tokio::test]
+    async fn oversized_ddl_uses_ordinary_sql_in_order() {
+        let large = "x".repeat(MAX_LOGICAL_TABLE_DDL_BYTES);
+        let statements = [
+            DdlStatement::new(logical("before", "p")),
+            DdlStatement::new(format!(
+                "CREATE TABLE ordinary (ts TIMESTAMP TIME INDEX, payload STRING DEFAULT '{large}')"
+            )),
+            DdlStatement::new(logical("between", "p")),
+            DdlStatement::new(format!("CREATE VIEW v AS SELECT '{large}' AS payload")),
+            DdlStatement::new(logical("oversized", &large)),
+            DdlStatement::new(logical("after", "p")),
+        ];
+        let (client, requests, server) = crate::database::tests::test_server(
+            200,
+            r#"{"execution_time_ms":0,"output":[{"affectedrows":0}]}"#,
+        )
+        .await;
+        DdlExecutor::new(&client)
+            .execute_strict(&statements, true)
+            .await
+            .unwrap();
+        {
+            let requests = requests.lock().unwrap();
+            assert_eq!(requests.len(), statements.len());
+            for (i, (request, statement)) in requests.iter().zip(&statements).enumerate() {
+                let endpoint = if [0, 2, 5].contains(&i) {
+                    "ddl/logical-tables"
+                } else {
+                    "sql"
+                };
+                assert!(request.starts_with(&format!("POST /v1/{endpoint} ")));
+                let body = request.split_once("\r\n\r\n").unwrap().1;
+                let sql = url::form_urlencoded::parse(body.as_bytes())
+                    .find(|(key, _)| key == "sql")
+                    .unwrap()
+                    .1;
+                assert_eq!(sql, statement.sql);
+            }
+        }
+        server.abort();
+        let _ = server.await;
+    }
+
+    #[tokio::test]
+    async fn qualified_targets_preserve_request_context() {
+        let (client, requests, server) = crate::database::tests::test_server(
+            200,
+            r#"{"execution_time_ms":0,"output":[{"affectedrows":0}]}"#,
+        )
+        .await;
+        let statements = [
+            DdlStatement::new(logical("other.metrics.a", "p")),
+            DdlStatement::with_execution_schema(
+                logical("other.metrics.b", "p"),
+                "existing_schema".into(),
+            ),
+        ];
+        DdlExecutor::new(&client)
+            .execute_strict(&statements, true)
+            .await
+            .unwrap();
+        {
+            let requests = requests.lock().unwrap();
+            assert_eq!(requests.len(), 2);
+            for (request, schema) in requests.iter().zip(["public", "existing_schema"]) {
+                assert!(request.starts_with("POST /v1/ddl/logical-tables "));
+                let body = request.split_once("\r\n\r\n").unwrap().1;
+                let db = url::form_urlencoded::parse(body.as_bytes())
+                    .find(|(k, _)| k == "db")
+                    .unwrap()
+                    .1;
+                assert_eq!(db, format!("greptime-{schema}"));
+            }
+        }
+        server.abort();
+        let _ = server.await;
+    }
+
+    #[tokio::test]
+    async fn transport_preserves_order_and_stops_without_fallback() {
+        use crate::database::tests::test_server;
+        let statements = [
+            DdlStatement::new("CREATE DATABASE d".into()),
+            DdlStatement::new(logical("a", "p").replace(
+                "on_physical_table='p'",
+                "on_physical_table='p', access_key_id='original-secret'",
+            )),
+            DdlStatement::new(logical("a", "p")),
+            DdlStatement::new("CREATE VIEW v AS SELECT * FROM a".into()),
+        ];
+        for supported in [true, false] {
+            let (client, requests, server) = test_server(
+                200,
+                r#"{"execution_time_ms":0,"output":[{"affectedrows":0}]}"#,
+            )
+            .await;
+            DdlExecutor::new(&client)
+                .execute_strict(&statements, supported)
+                .await
+                .unwrap();
+            let requests = requests.lock().unwrap();
+            assert_eq!(requests.len(), 4);
+            for (i, request) in requests.iter().enumerate() {
+                let path = if supported && (i == 1 || i == 2) {
+                    "ddl/logical-tables"
+                } else {
+                    "sql"
+                };
+                assert!(request.starts_with(&format!("POST /v1/{path} ")));
+                assert!(
+                    request
+                        .to_lowercase()
+                        .contains("authorization: basic dxnlcjpwyxnzd29yza==")
+                );
+                let body = request.split_once("\r\n\r\n").unwrap().1;
+                let form: std::collections::HashMap<_, _> =
+                    url::form_urlencoded::parse(body.as_bytes())
+                        .into_owned()
+                        .collect();
+                assert_eq!(form["sql"], statements[i].sql);
+                assert_eq!(form["db"], "greptime-public");
+            }
+            server.abort();
+        }
+        let statements = [
+            DdlStatement::new(logical("a", "p")),
+            DdlStatement::new(logical("b", "p")),
+            DdlStatement::new("CREATE VIEW v AS SELECT * FROM a".into()),
+        ];
+        for (status, body, success) in [
+            (
+                200,
+                r#"{"execution_time_ms":0,"output":[{"affectedrows":0},{"affectedrows":0}]}"#,
+                true,
+            ),
+            (
+                200,
+                r#"{"execution_time_ms":0,"output":[{"affectedrows":0}]}"#,
+                false,
+            ),
+            (
+                200,
+                r#"{"execution_time_ms":0,"output":[{"affectedrows":0},{"affectedrows":1}]}"#,
+                false,
+            ),
+            (200, r#"{"execution_time_ms":0,"output":[{},{}]}"#, false),
+            (
+                200,
+                r#"{"execution_time_ms":0,"output":[{"affectedrows":0},{"error":"secret"}]}"#,
+                false,
+            ),
+            (500, r#"{"error":"secret"}"#, false),
+            (200, "invalid", false),
+        ] {
+            let (client, requests, server) = test_server(status, body).await;
+            let result = DdlExecutor::new(&client)
+                .execute_strict(&statements, true)
+                .await;
+            assert_eq!(result.is_ok(), success);
+            assert_eq!(requests.lock().unwrap().len(), if success { 2 } else { 1 });
+            if let Err(error) = result {
+                assert!(!format!("{error:?}").contains("secret"));
+            }
+            server.abort();
+            let _ = server.await;
+        }
     }
 }

src/cli/src/data/import_v2/packed.rs

@@ -16,12 +16,11 @@ use std::collections::HashSet;
 
 use common_datasource::packed_snapshot::{PACK_INDEX_FILE, PackIndex};
 use snafu::ResultExt;
-use sql::dialect::GreptimeDbDialect;
-use sql::parser::{ParseOptions, ParserContext};
 use sql::statements::statement::Statement;
 
 use crate::data::export_v2::manifest::{ChunkStatus, Manifest};
 use crate::data::import_v2::error::{InvalidPackedSnapshotSnafu, Result, SnapshotStorageSnafu};
+use crate::data::import_v2::{command, executor};
 use crate::data::path::{data_dir_for_schema_chunk, ddl_path_for_schema};
 use crate::data::snapshot_storage::SnapshotStorage;
 
@@ -51,49 +50,46 @@ pub(crate) async fn validate_snapshot(
             .read_text(&ddl_path_for_schema(schema))
             .await
             .context(SnapshotStorageSnafu)?;
-        let statements = ParserContext::create_with_dialect(
-            &ddl,
-            &GreptimeDbDialect {},
-            ParseOptions::default(),
-        )
-        .map_err(|e| {
-            InvalidPackedSnapshotSnafu {
-                reason: e.to_string(),
-            }
-            .build()
-        })?;
         let mut names = HashSet::new();
-        for statement in statements {
-            if let Statement::CreateTable(create) = statement {
-                if create.engine.eq_ignore_ascii_case("metric")
-                    && create.options.get("on_physical_table").is_none()
-                {
-                    continue;
+        for sql in command::iter_ddl_statements(&ddl) {
+            let statements = executor::parse_ddl(&sql).map_err(|_| {
+                InvalidPackedSnapshotSnafu {
+                    reason: "invalid snapshot DDL",
                 }
-                let parts: Vec<_> = create
-                    .name
-                    .0
-                    .iter()
-                    .map(|p| p.as_ident().map(|i| i.value.as_str()))
-                    .collect();
-                let allowed = match parts.as_slice() {
-                    [Some(_)] => true,
-                    [Some(s), Some(_)] => *s == schema,
-                    [Some(c), Some(s), Some(_)] => *c == manifest.catalog && *s == schema,
-                    _ => false,
-                };
-                if !allowed {
-                    return InvalidPackedSnapshotSnafu {
-                        reason: "snapshot table DDL escapes its catalog/schema",
+                .build()
+            })?;
+            for statement in statements {
+                if let Statement::CreateTable(create) = statement {
+                    if create.engine.eq_ignore_ascii_case("metric")
+                        && create.options.get("on_physical_table").is_none()
+                    {
+                        continue;
                     }
-                    .fail();
-                }
-                let name = parts.last().and_then(|p| *p).unwrap_or_default();
-                if !names.insert(name.to_string()) {
-                    return InvalidPackedSnapshotSnafu {
-                        reason: "duplicate snapshot table DDL",
+                    let parts: Vec<_> = create
+                        .name
+                        .0
+                        .iter()
+                        .map(|p| p.as_ident().map(|i| i.value.as_str()))
+                        .collect();
+                    let allowed = match parts.as_slice() {
+                        [Some(_)] => true,
+                        [Some(s), Some(_)] => *s == schema,
+                        [Some(c), Some(s), Some(_)] => *c == manifest.catalog && *s == schema,
+                        _ => false,
+                    };
+                    if !allowed {
+                        return InvalidPackedSnapshotSnafu {
+                            reason: "snapshot table DDL escapes its catalog/schema",
+                        }
+                        .fail();
+                    }
+                    let name = parts.last().and_then(|p| *p).unwrap_or_default();
+                    if !names.insert(name.to_string()) {
+                        return InvalidPackedSnapshotSnafu {
+                            reason: "duplicate snapshot table DDL",
+                        }
+                        .fail();
                     }
-                    .fail();
                 }
             }
         }
@@ -180,13 +176,14 @@ mod tests {
             .unwrap();
         let uri = url::Url::from_file_path(dir.path()).unwrap();
         let storage = OpenDalStorage::from_uri(uri.as_str(), &Default::default()).unwrap();
-        storage.write_text("schema/ddl/public.sql", "CREATE TABLE p (ts TIMESTAMP TIME INDEX) ENGINE=metric WITH(physical_metric_table=''); CREATE TABLE \"logical.name\" (ts TIMESTAMP TIME INDEX) ENGINE=metric WITH(on_physical_table='p'); CREATE VIEW v AS SELECT * FROM \"logical.name\";").await.unwrap();
+        let large = "x".repeat(servers::query_handler::sql::MAX_LOGICAL_TABLE_DDL_BYTES);
+        storage.write_text("schema/ddl/public.sql", &format!("CREATE TABLE p (ts TIMESTAMP TIME INDEX) ENGINE=metric WITH(physical_metric_table=''); CREATE TABLE \"logical.name\" (ts TIMESTAMP TIME INDEX) ENGINE=metric WITH(on_physical_table='p'); CREATE VIEW v AS SELECT * FROM \"logical.name\"; CREATE TABLE ordinary (ts TIMESTAMP TIME INDEX, payload STRING DEFAULT '{large}'); CREATE VIEW large_view AS SELECT '{large}' AS payload;")).await.unwrap();
         let path = "data/public/1/pack-index.json";
-        let index = serde_json::json!({"version":1,"objects":[{"path":"pack-0.bin","kind":"pack","length":12}],"tables":[{"table_name":"logical.name","object":"pack-0.bin","offset":0,"length":12,"row_count":0}]});
+        let index = serde_json::json!({"version":1,"objects":[{"path":"pack-0.bin","kind":"pack","length":24}],"tables":[{"table_name":"logical.name","object":"pack-0.bin","offset":0,"length":12,"row_count":0},{"table_name":"ordinary","object":"pack-0.bin","offset":12,"length":12,"row_count":0}]});
         storage.write_text(path, &index.to_string()).await.unwrap();
         assert!(dir.path().join(path).is_file());
         storage
-            .write_text("data/public/1/pack-0.bin", "abcdefghijkl")
+            .write_text("data/public/1/pack-0.bin", "abcdefghijklmnopqrstuvwx")
             .await
             .unwrap();
         let mut manifest = Manifest::new_schema_only("greptime".into(), vec!["public".into()]);

src/cli/src/data/schema_export.rs

@@ -0,0 +1,508 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Bounded SHOW CREATE requests shared by both export commands.
+
+use common_catalog::consts::DEFAULT_SCHEMA_NAME;
+use servers::http::GreptimeQueryOutput;
+use servers::http::result::greptime_result_v1::GreptimedbV1Response;
+
+use crate::data::sql::escape_sql_identifier;
+use crate::database::DatabaseClient;
+use crate::error::{InvalidArgumentsSnafu, Result, UnexpectedSnafu};
+
+const MAX_STATEMENTS: usize = 128;
+const MAX_SQL_BYTES: usize = 128 * 1024;
+
+/// Appends DDL in dependency order supplied by the caller using bounded requests.
+pub(crate) fn append_schema_ddl<'a>(
+    client: &'a DatabaseClient,
+    catalog: &'a str,
+    schema: &'a str,
+    objects: impl Iterator<Item = (&'static str, Option<&'a str>)> + Send + 'a,
+    ddl: &'a mut String,
+) -> impl std::future::Future<Output = Result<()>> + Send + 'a {
+    let mut statements = objects
+        .map(move |(kind, table)| show_create_sql(kind, catalog, schema, table))
+        .peekable();
+    async move {
+        loop {
+            let batch = next_batch(&mut statements)?;
+            if batch.is_empty() {
+                return Ok(());
+            }
+            let response = client
+                .sql_response(&batch.concat(), DEFAULT_SCHEMA_NAME)
+                .await
+                .map_err(|error| {
+                    UnexpectedSnafu {
+                        msg: format!("SHOW CREATE batch failed for {}: {error}", batch.concat()),
+                    }
+                    .build()
+                })?;
+            append_results(&batch, &response, ddl)?;
+        }
+    }
+}
+
+fn show_create_sql(kind: &str, catalog: &str, schema: &str, table: Option<&str>) -> String {
+    let mut sql = format!(
+        "SHOW CREATE {kind} \"{}\".\"{}\"",
+        escape_sql_identifier(catalog),
+        escape_sql_identifier(schema),
+    );
+    if let Some(table) = table {
+        sql.push_str(".\"");
+        sql.push_str(&escape_sql_identifier(table));
+        sql.push('"');
+    }
+    sql.push_str(";\n");
+    sql
+}
+
+fn next_batch(
+    statements: &mut std::iter::Peekable<impl Iterator<Item = String>>,
+) -> Result<Vec<String>> {
+    let mut batch = Vec::new();
+    let mut bytes = 0;
+    while let Some(sql) = statements.peek() {
+        if sql.len() > MAX_SQL_BYTES {
+            return InvalidArgumentsSnafu {
+                msg: format!(
+                    "SHOW CREATE exceeds {MAX_SQL_BYTES} SQL bytes ({}): {sql}",
+                    sql.len()
+                ),
+            }
+            .fail();
+        }
+        if batch.len() == MAX_STATEMENTS || bytes + sql.len() > MAX_SQL_BYTES {
+            break;
+        }
+        bytes += sql.len();
+        if let Some(sql) = statements.next() {
+            batch.push(sql);
+        }
+    }
+    Ok(batch)
+}
+
+fn append_results(
+    batch: &[String],
+    response: &GreptimedbV1Response,
+    ddl: &mut String,
+) -> Result<()> {
+    let outputs = response.output();
+    if outputs.len() != batch.len() {
+        return UnexpectedSnafu {
+            msg: format!(
+                "SHOW CREATE expected {} results, received {} for {}",
+                batch.len(),
+                outputs.len(),
+                batch.concat()
+            ),
+        }
+        .fail();
+    }
+    for (sql, output) in batch.iter().zip(outputs) {
+        let GreptimeQueryOutput::Records(records) = output else {
+            return UnexpectedSnafu {
+                msg: format!("expected SHOW CREATE records for {sql}"),
+            }
+            .fail();
+        };
+        if records.num_cols() != 2 || records.rows().len() != 1 || records.rows()[0].len() != 2 {
+            return UnexpectedSnafu {
+                msg: format!("expected one two-column SHOW CREATE row for {sql}"),
+            }
+            .fail();
+        }
+        let row = &records.rows()[0];
+        let Some(create) = row[1].as_str().filter(|value| !value.is_empty()) else {
+            return UnexpectedSnafu {
+                msg: format!("expected SHOW CREATE DDL string for {sql}"),
+            }
+            .fail();
+        };
+        if !row[0].is_string() {
+            return UnexpectedSnafu {
+                msg: format!("expected SHOW CREATE object name for {sql}"),
+            }
+            .fail();
+        }
+        ddl.push_str(create);
+        ddl.push_str(";\n");
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use serde_json::{Value, json};
+    use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
+
+    use super::*;
+
+    fn records(name: &str, ddl: &str) -> Value {
+        json!({"records": {"schema": {"column_schemas": [
+            {"name": "Table", "data_type": "String"},
+            {"name": "Create Table", "data_type": "String"}
+        ]}, "rows": [[name, ddl]], "total_rows": 1}})
+    }
+
+    #[test]
+    fn batch_limits_include_quoted_utf8_and_separators() {
+        let special = show_create_sql("TABLE", "cat.name", "s\";库", Some("ta.\";表"));
+        assert_eq!(
+            special,
+            "SHOW CREATE TABLE \"cat.name\".\"s\"\";库\".\"ta.\"\";表\";\n"
+        );
+        let mut statements = std::iter::repeat_n(special.clone(), 129).peekable();
+        assert_eq!(
+            next_batch(&mut statements).unwrap(),
+            vec![special.clone(); 128]
+        );
+        assert_eq!(next_batch(&mut statements).unwrap(), vec![special]);
+        assert!(next_batch(&mut statements).unwrap().is_empty());
+
+        let prefix = show_create_sql("TABLE", "c", "s", Some(""));
+        // Every quote expands to two bytes; this also exercises a multibyte name.
+        let name = format!("表{}", "\"".repeat((MAX_SQL_BYTES - prefix.len() - 3) / 2));
+        let mut exact = show_create_sql("TABLE", "c", "s", Some(&name));
+        let padding = MAX_SQL_BYTES - exact.len();
+        exact = show_create_sql(
+            "TABLE",
+            "c",
+            "s",
+            Some(&format!("{name}{}", "a".repeat(padding))),
+        );
+        assert_eq!(exact.len(), MAX_SQL_BYTES);
+        let small = show_create_sql("TABLE", "c", "s", Some("next"));
+        let mut statements = [exact.clone(), small.clone()].into_iter().peekable();
+        assert_eq!(next_batch(&mut statements).unwrap(), vec![exact.clone()]);
+        assert_eq!(next_batch(&mut statements).unwrap(), vec![small]);
+        let mut oversized = std::iter::once(format!("{exact} ")).peekable();
+        assert!(
+            next_batch(&mut oversized)
+                .unwrap_err()
+                .to_string()
+                .contains("exceeds 131072")
+        );
+
+        let half = show_create_sql(
+            "TABLE",
+            "c",
+            "s",
+            Some(&"a".repeat(MAX_SQL_BYTES / 2 - prefix.len())),
+        );
+        let mut statements = [half.clone(), half.clone(), prefix.clone()]
+            .into_iter()
+            .peekable();
+        assert_eq!(
+            next_batch(&mut statements).unwrap(),
+            vec![half.clone(), half]
+        );
+        assert_eq!(next_batch(&mut statements).unwrap(), vec![prefix]);
+    }
+
+    #[test]
+    fn show_create_requires_exact_record_shape_and_count() {
+        let batch = vec![
+            "SHOW CREATE TABLE first".into(),
+            "SHOW CREATE TABLE middle".into(),
+        ];
+        let valid = records("first", "CREATE TABLE first");
+        for output in [
+            vec![],
+            vec![valid.clone()],
+            vec![valid.clone(); 3],
+            vec![valid.clone(), json!({"affectedrows": 1})],
+            vec![valid.clone(), records("middle", "")],
+            vec![valid.clone(), {
+                let mut r = valid.clone();
+                r["records"]["rows"] = json!([]);
+                r
+            }],
+            vec![valid.clone(), {
+                let mut r = valid.clone();
+                r["records"]["rows"] = json!([["middle", 4]]);
+                r
+            }],
+            vec![valid.clone(), {
+                let mut r = valid.clone();
+                r["records"]["rows"] = json!([["middle", "ddl", "extra"]]);
+                r
+            }],
+        ] {
+            let response =
+                serde_json::from_value(json!({"output": output, "execution_time_ms": 0})).unwrap();
+            let error = append_results(&batch, &response, &mut String::new()).unwrap_err();
+            assert!(error.to_string().contains("middle"), "{error}");
+        }
+    }
+
+    async fn start_mock_sql_server(
+        bodies: Vec<(u16, String)>,
+    ) -> (DatabaseClient, tokio::task::JoinHandle<Vec<String>>) {
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let server = tokio::spawn(async move {
+            tokio::time::timeout(std::time::Duration::from_secs(5), async move {
+                let (socket, _) = listener.accept().await.unwrap();
+                let mut socket = BufReader::new(socket);
+                let mut queries = Vec::new();
+                for (status, body) in bodies {
+                    let mut headers = String::new();
+                    loop {
+                        let mut line = String::new();
+                        if socket.read_line(&mut line).await.unwrap() == 0 {
+                            assert!(headers.is_empty());
+                            return queries;
+                        }
+                        headers.push_str(&line);
+                        if line == "\r\n" {
+                            break;
+                        }
+                    }
+                    let headers = headers.to_lowercase();
+                    assert!(headers.starts_with("post /v1/sql "));
+                    assert!(headers.contains("authorization: basic dxnlcjpwyxnzd29yza=="));
+                    assert!(headers.contains("x-greptime-timeout: 7s"));
+                    let length: usize = headers
+                        .lines()
+                        .find_map(|line| line.strip_prefix("content-length: "))
+                        .unwrap()
+                        .parse()
+                        .unwrap();
+                    let mut form = vec![0; length];
+                    socket.read_exact(&mut form).await.unwrap();
+                    let params: std::collections::HashMap<_, _> =
+                        url::form_urlencoded::parse(&form).into_owned().collect();
+                    assert_eq!(params["db"], "catalog-public");
+                    queries.push(params["sql"].clone());
+                    socket
+                        .get_mut()
+                        .write_all(
+                            format!(
+                                "HTTP/1.1 {status} Response\r\nContent-Length: {}\r\n\r\n{body}",
+                                body.len()
+                            )
+                            .as_bytes(),
+                        )
+                        .await
+                        .unwrap();
+                }
+                queries
+            })
+            .await
+            .expect("mock SQL exchange timed out")
+        });
+        let client = DatabaseClient::new(
+            address.to_string(),
+            "catalog".into(),
+            Some("user:password".into()),
+            std::time::Duration::from_secs(7),
+            None,
+            true,
+        );
+        (client, server)
+    }
+
+    #[tokio::test]
+    async fn batches_preserve_order_context_and_reuse_connection() {
+        let names: Vec<_> = (0..129).map(|i| format!("表.{i}\";x")).collect();
+        let outputs: Vec<_> = names
+            .iter()
+            .map(|name| records(name, &format!("DDL {name}")))
+            .collect();
+        let bodies = outputs
+            .chunks(128)
+            .map(|chunk| {
+                (
+                    200,
+                    json!({"output": chunk, "execution_time_ms": 0}).to_string(),
+                )
+            })
+            .collect();
+        let (client, server) = start_mock_sql_server(bodies).await;
+        let mut ddl = String::new();
+        append_schema_ddl(
+            &client.clone(),
+            "catalog",
+            "schema",
+            names.iter().map(|name| ("TABLE", Some(name.as_str()))),
+            &mut ddl,
+        )
+        .await
+        .unwrap();
+        assert_eq!(
+            ddl,
+            names
+                .iter()
+                .map(|name| format!("DDL {name};\n"))
+                .collect::<String>()
+        );
+        let queries = server.await.unwrap();
+        let expected: Vec<_> = (0..129)
+            .map(|i| format!("SHOW CREATE TABLE \"catalog\".\"schema\".\"表.{i}\"\";x\";\n"))
+            .collect();
+        assert_eq!(
+            queries,
+            vec![expected[..128].concat(), expected[128..].concat()]
+        );
+    }
+
+    #[tokio::test]
+    async fn legacy_command_stops_before_data_after_schema_failure() {
+        use clap::Parser;
+
+        use crate::data::export::ExportCommand;
+
+        let output = |rows: Value, columns: usize| {
+            json!({"records": {"schema": {"column_schemas": (0..columns)
+                .map(|i| json!({"name": i.to_string(), "data_type": "String"}))
+                .collect::<Vec<_>>()}, "rows": rows, "total_rows": rows.as_array().unwrap().len()}})
+        };
+        let databases = output(json!([["public"]]), 1);
+        let mut malformed = records("middle", "CREATE TABLE middle");
+        malformed["records"]["rows"][0][1] = json!(42);
+        let bodies = vec![
+            vec![databases.clone()],
+            vec![records("public", "CREATE DATABASE public")],
+            vec![databases.clone()],
+            vec![output(json!([]), 3)],
+            vec![output(
+                json!([
+                    ["catalog", "public", "first", "BASE TABLE"],
+                    ["catalog", "public", "middle", "BASE TABLE"],
+                    ["catalog", "public", "last", "BASE TABLE"]
+                ]),
+                4,
+            )],
+            vec![
+                records("first", "CREATE TABLE first"),
+                malformed,
+                records("last", "CREATE TABLE last"),
+            ],
+            vec![databases],
+            vec![json!({"affectedrows": 0})],
+        ]
+        .into_iter()
+        .map(|output| {
+            (
+                200,
+                json!({"output": output, "execution_time_ms": 0}).to_string(),
+            )
+        })
+        .collect::<Vec<_>>();
+        for target in ["schema", "all"] {
+            let (client, server) = start_mock_sql_server(bodies.clone()).await;
+            let dir = tempfile::tempdir().unwrap();
+            let command = ExportCommand::parse_from([
+                "export",
+                "--addr",
+                client.addr(),
+                "--database",
+                "catalog-public",
+                "--output-dir",
+                dir.path().to_str().unwrap(),
+                "--target",
+                target,
+                "--auth-basic",
+                "user:password",
+                "--timeout",
+                "7s",
+                "--no-proxy",
+            ]);
+            let tool = command.build().await.unwrap();
+            let result = tool.do_work().await;
+            drop(tool);
+            let queries = server.await.unwrap();
+            assert!(result.unwrap_err().to_string().contains("middle"));
+            assert!(
+                dir.path()
+                    .join("catalog/public/create_database.sql")
+                    .is_file()
+            );
+            assert!(!dir.path().join("catalog/public/create_tables.sql").exists());
+            assert_eq!(queries.len(), 6);
+            assert!(queries[5].contains("SHOW CREATE TABLE \"catalog\".\"public\".\"middle\""));
+            assert!(!queries.iter().any(|sql| sql.starts_with("COPY")));
+        }
+    }
+
+    #[tokio::test]
+    async fn sql_response_errors_do_not_expose_copy_credentials() {
+        let secret = "sentinel-connection-secret";
+        let echoed = "sentinel-response-secret";
+        let sql = format!(
+            "COPY DATABASE public TO 's3://bucket/' CONNECTION (SECRET_ACCESS_KEY='{secret}')"
+        );
+        let bodies = vec![
+            (200, format!("not json {echoed}")),
+            (200, json!({"output": [{"records": echoed}], "execution_time_ms": 0}).to_string()),
+            (400, json!({"code": 3001, "error": echoed}).to_string()),
+            (200, json!({"error": echoed}).to_string()),
+            (200, json!({"code": 3001, "output": [records("first", echoed)], "execution_time_ms": 0}).to_string()),
+            (200, json!({"output": [{"error": echoed}]}).to_string()),
+            (200, json!({"output": [{"code": 3001, "records": records("first", echoed)["records"]}], "execution_time_ms": 0}).to_string()),
+        ];
+        let count = bodies.len();
+        let (client, server) = start_mock_sql_server(bodies).await;
+        for _ in 0..count {
+            let error = client
+                .sql_response(&sql, "public")
+                .await
+                .unwrap_err()
+                .to_string();
+            assert!(error.contains("SQL"), "{error}");
+            assert!(!error.contains(secret), "{error}");
+            assert!(!error.contains(echoed), "{error}");
+            assert!(!error.contains("COPY DATABASE"), "{error}");
+        }
+        assert_eq!(server.await.unwrap().len(), count);
+    }
+
+    #[tokio::test]
+    async fn rejects_http_and_statement_errors_without_retry() {
+        let valid = records("first", "CREATE TABLE first");
+        let bodies = vec![
+            (400, json!({"code": 3001, "error": "middle not found", "execution_time_ms": 0}).to_string()),
+            (200, json!({"code": 3001, "output": [valid.clone(), valid.clone(), valid.clone()], "execution_time_ms": 0}).to_string()),
+            (200, json!({"output": [valid.clone(), {"error": "middle denied"}, valid.clone()], "execution_time_ms": 0}).to_string()),
+            (200, json!({"output": [valid.clone(), {"error": "middle denied", "records": valid["records"]}, valid], "execution_time_ms": 0}).to_string()),
+            (200, json!({"output": [records("first", "DDL"), {"code": 3001, "records": records("middle", "DDL")["records"]}, records("last", "DDL")], "execution_time_ms": 0}).to_string()),
+            (200, r#"{"execution_time_ms":0}"#.into()),
+            (200, "not json".into()),
+        ];
+        let count = bodies.len();
+        let (client, server) = start_mock_sql_server(bodies).await;
+        for _ in 0..count {
+            let mut ddl = String::new();
+            let error = append_schema_ddl(
+                &client,
+                "catalog",
+                "schema",
+                ["first", "middle", "last"]
+                    .into_iter()
+                    .map(|name| ("TABLE", Some(name))),
+                &mut ddl,
+            )
+            .await
+            .unwrap_err();
+            assert!(error.to_string().contains("middle"), "{error}");
+            assert!(ddl.is_empty());
+        }
+        assert_eq!(server.await.unwrap().len(), count);
+    }
+}

src/cli/src/data/snapshot_storage.rs

@@ -295,8 +295,9 @@ pub trait SnapshotStorage: Send + Sync {
         schemas: &[String],
         chunk_id: u32,
         resume: bool,
+        packed: bool,
     ) -> Result<()> {
-        let _ = (schemas, chunk_id, resume);
+        let _ = (schemas, chunk_id, resume, packed);
         InvalidUriSnafu {
             uri: "snapshot",
             reason: "storage does not support preparing export chunks",
@@ -730,6 +731,7 @@ impl SnapshotStorage for OpenDalStorage {
         schemas: &[String],
         chunk_id: u32,
         resume: bool,
+        packed: bool,
     ) -> Result<()> {
         let mut files = Vec::new();
         for schema in schemas {
@@ -751,10 +753,17 @@ impl SnapshotStorage for OpenDalStorage {
                     continue;
                 }
                 let name = path.strip_prefix(&prefix).unwrap_or("");
-                if !resume || entry.metadata().is_dir() || !valid_chunk_filename(name) {
+                if !resume
+                    || entry.metadata().is_dir()
+                    || !(if packed {
+                        valid_packed_chunk_filename(name)
+                    } else {
+                        valid_chunk_filename(name)
+                    })
+                {
                     return InvalidUriSnafu {
                         uri: path,
-                        reason: "expected an empty new chunk or direct Parquet files in an owned unfinished chunk",
+                        reason: "expected an empty new chunk or recognized files in an owned unfinished chunk",
                     }.fail();
                 }
                 files.push(path.to_string());
@@ -786,6 +795,17 @@ impl SnapshotStorage for OpenDalStorage {
     }
 }
 
+fn valid_packed_chunk_filename(name: &str) -> bool {
+    name == common_datasource::packed_snapshot::PACK_INDEX_FILE
+        || [("pack-", ".bin"), ("table-", ".parquet")]
+            .iter()
+            .any(|(prefix, suffix)| {
+                name.strip_prefix(prefix)
+                    .and_then(|n| n.strip_suffix(suffix))
+                    .is_some_and(|id| !id.is_empty() && id.bytes().all(|b| b.is_ascii_digit()))
+            })
+}
+
 fn valid_chunk_filename(name: &str) -> bool {
     name.strip_suffix(".parquet")
         .is_some_and(|stem| !stem.is_empty())
@@ -1074,7 +1094,7 @@ mod tests {
         let schemas = vec!["public".to_string(), "other".to_string()];
         assert!(
             storage
-                .prepare_export_chunk(&schemas, 2, false)
+                .prepare_export_chunk(&schemas, 2, false, false)
                 .await
                 .is_err()
         );
@@ -1085,11 +1105,11 @@ mod tests {
                 .unwrap()
         );
         storage
-            .prepare_export_chunk(&schemas, 2, true)
+            .prepare_export_chunk(&schemas, 2, true, false)
             .await
             .unwrap();
         storage
-            .prepare_export_chunk(&schemas, 2, true)
+            .prepare_export_chunk(&schemas, 2, true, false)
             .await
             .unwrap();
         assert!(
@@ -1129,7 +1149,7 @@ mod tests {
             }
             assert!(
                 storage
-                    .prepare_export_chunk(&["public".into(), "other".into()], 2, true)
+                    .prepare_export_chunk(&["public".into(), "other".into()], 2, true, false)
                     .await
                     .is_err()
             );
@@ -1143,6 +1163,70 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn packed_retry_preserves_foreign_files_and_completed_chunks() {
+        let dir = tempdir().unwrap();
+        let storage = make_storage_with_rooted_fs(dir.path());
+        let owned = ["pack-000000.bin", "table-42.parquet", "pack-index.json"];
+        for name in owned {
+            storage
+                .write_text(&format!("data/public/2/{name}"), "partial")
+                .await
+                .unwrap();
+        }
+        for path in [
+            "data/public/1/pack-000000.bin",
+            "data/public/2/foreign.parquet",
+        ] {
+            storage.write_text(path, "keep").await.unwrap();
+        }
+        let schemas = ["public".into()];
+        assert!(
+            storage
+                .prepare_export_chunk(&schemas, 2, true, true)
+                .await
+                .is_err()
+        );
+        for name in owned {
+            assert!(
+                storage
+                    .file_exists(&format!("data/public/2/{name}"))
+                    .await
+                    .unwrap()
+            );
+        }
+        assert_eq!(
+            storage
+                .read_text("data/public/2/foreign.parquet")
+                .await
+                .unwrap(),
+            "keep"
+        );
+        storage
+            .object_store
+            .delete("data/public/2/foreign.parquet")
+            .await
+            .unwrap();
+        storage
+            .prepare_export_chunk(&schemas, 2, true, true)
+            .await
+            .unwrap();
+        assert!(
+            storage
+                .list_files_recursive("data/public/2/")
+                .await
+                .unwrap()
+                .is_empty()
+        );
+        assert_eq!(
+            storage
+                .read_text("data/public/1/pack-000000.bin")
+                .await
+                .unwrap(),
+            "keep"
+        );
+    }
+
     #[tokio::test]
     async fn test_prepare_export_chunk_reports_delete_failure() {
         let dir = tempdir().unwrap();
@@ -1159,7 +1243,7 @@ mod tests {
             .await
             .unwrap();
         let error = storage
-            .prepare_export_chunk(&["public".into()], 2, true)
+            .prepare_export_chunk(&["public".into()], 2, true, false)
             .await
             .unwrap_err();
         let Error::StorageOperation {
@@ -1191,7 +1275,7 @@ mod tests {
             .await
             .unwrap();
         storage
-            .prepare_export_chunk(&[schema], 2, true)
+            .prepare_export_chunk(&[schema], 2, true, false)
             .await
             .unwrap();
         assert!(!storage.file_exists(&owned).await.unwrap());