ci: add sqlness compat smoke (#8370)

* ci: add sqlness compat smoke

Signed-off-by: discord9 <discord9@163.com>

* ci: limit compat smoke pull request trigger

Signed-off-by: discord9 <discord9@163.com>

* ci: run compat smoke in merge queue

Signed-off-by: discord9 <discord9@163.com>

* ci: configure compat version window

Signed-off-by: discord9 <discord9@163.com>

* ci: move compat smoke logic into script

Signed-off-by: discord9 <discord9@163.com>

* ci: expand compat version window

Signed-off-by: discord9 <discord9@163.com>

* ci: rename compat job for recent releases

Signed-off-by: discord9 <discord9@163.com>

* ci: report compat test failures

Signed-off-by: discord9 <discord9@163.com>

---------

Signed-off-by: discord9 <discord9@163.com>

.github/scripts/run-compat.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Run sqlness compatibility checks for a small release window.
+
+The workflow intentionally keeps YAML small and delegates the maintainable parts
+here:
+
+- read `tests/compatibility/ci.toml`
+- validate the checked-in recent-release window
+- preview selected cases with `compat --dry-run`
+- run the real compat check for each sampled `from` version
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import os
+import re
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+
+
+VERSION_RE = re.compile(r"v[0-9]+\.[0-9]+\.[0-9]+")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--config",
+        default="tests/compatibility/ci.toml",
+        help="Path to the compatibility CI window config.",
+    )
+    parser.add_argument(
+        "--runner",
+        default="./bins/sqlness-runner",
+        help="Path to the sqlness-runner binary built by CI.",
+    )
+    parser.add_argument(
+        "--to-bins-dir",
+        default="./bins",
+        help="Directory containing the PR-built greptime binary.",
+    )
+    parser.add_argument(
+        "--preserve-state",
+        action="store_true",
+        help="Pass --preserve-state to the real compat run for artifact upload.",
+    )
+    return parser.parse_args()
+
+
+def load_from_versions(config_path: Path) -> list[str]:
+    if not config_path.is_file():
+        raise SystemExit(f"Compatibility CI config not found: {config_path}")
+
+    # Parse the simple TOML string array without depending on Python 3.11+
+    # tomllib. This intentionally supports only the checked-in shape used by
+    # the CI job: from_versions = ["vX.Y.Z", ...].
+    content = config_path.read_text(encoding="utf-8")
+    content_without_comments = "\n".join(
+        line.split("#", 1)[0] for line in content.splitlines()
+    )
+    match = re.search(
+        r"(?ms)^\s*from_versions\s*=\s*(\[[^\]]*\])",
+        content_without_comments,
+    )
+    if match is None:
+        raise SystemExit(f"{config_path} must define from_versions")
+
+    try:
+        versions = ast.literal_eval(match.group(1))
+    except (SyntaxError, ValueError) as err:
+        raise SystemExit(f"Invalid from_versions in {config_path}: {err}") from err
+
+    if not isinstance(versions, list) or not versions:
+        raise SystemExit(f"{config_path} must define a non-empty from_versions list")
+
+    seen: set[str] = set()
+    validated: list[str] = []
+    for version in versions:
+        if not isinstance(version, str) or VERSION_RE.fullmatch(version) is None:
+            raise SystemExit(f"Invalid compat from version: {version!r}")
+        if version in seen:
+            raise SystemExit(f"Duplicate compat from version: {version}")
+        seen.add(version)
+        validated.append(version)
+
+    return validated
+
+
+def check_inputs(runner: Path, to_bins_dir: Path) -> None:
+    if not runner.is_file():
+        raise SystemExit(f"sqlness-runner binary not found: {runner}")
+    if not to_bins_dir.is_dir():
+        raise SystemExit(f"to-bins directory not found: {to_bins_dir}")
+    if not to_bins_dir.joinpath("greptime").is_file():
+        raise SystemExit(f"greptime binary not found in to-bins directory: {to_bins_dir}")
+
+
+def github_group(title: str):
+    class Group:
+        def __enter__(self) -> None:
+            print(f"::group::{title}", flush=True)
+
+        def __exit__(self, exc_type, exc, traceback) -> None:
+            print("::endgroup::", flush=True)
+
+    return Group()
+
+
+def run_command(command: list[str], *, env: dict[str, str] | None = None) -> None:
+    print(f"$ {shlex.join(command)}", flush=True)
+    subprocess.run(command, check=True, env=env)
+
+
+def run_for_version(
+    *,
+    runner: Path,
+    to_bins_dir: Path,
+    from_version: str,
+    preserve_state: bool,
+) -> None:
+    base_command = [
+        str(runner),
+        "compat",
+        "--from-version",
+        from_version,
+        "--to-bins-dir",
+        str(to_bins_dir),
+    ]
+
+    with github_group(f"Preview {from_version} -> current"):
+        run_command([*base_command, "--dry-run"])
+
+    real_command = [*base_command]
+    if preserve_state:
+        real_command.append("--preserve-state")
+
+    env = os.environ.copy()
+    env.setdefault("RUST_BACKTRACE", "1")
+    with github_group(f"Compatibility {from_version} -> current"):
+        run_command(real_command, env=env)
+
+
+def main() -> int:
+    args = parse_args()
+    config_path = Path(args.config)
+    runner = Path(args.runner)
+    to_bins_dir = Path(args.to_bins_dir)
+
+    check_inputs(runner, to_bins_dir)
+    from_versions = load_from_versions(config_path)
+
+    print("Compatibility from-version window:", flush=True)
+    for version in from_versions:
+        print(f"  - {version}", flush=True)
+
+    for from_version in from_versions:
+        run_for_version(
+            runner=runner,
+            to_bins_dir=to_bins_dir,
+            from_version=from_version,
+            preserve_state=args.preserve_state,
+        )
+
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except subprocess.CalledProcessError as err:
+        raise SystemExit(err.returncode) from err

.github/workflows/develop.yml

@@ -1019,22 +1019,29 @@ jobs:
     with:
       artifact-name: bins
 
-
-  # compat:
-  #   if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-  #   name: Compatibility Test
-  #   needs: build
-  #   runs-on: ubuntu-22.04
-  #   timeout-minutes: 60
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Download pre-built binaries
-  #       uses: actions/download-artifact@v4
-  #       with:
-  #         name: bins
-  #         path: .
-  #     - name: Unzip binaries
-  #       run: |
-  #         mkdir -p ./bins/current
-  #         tar -xvf ./bins.tar.gz --strip-components=1 -C ./bins/current
-  #     - run: ./tests/compat/test-compat.sh 0.6.0
+  compat:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && github.event.action == 'ready_for_review')) }}
+    name: Compatibility Test (recent releases)
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Download pre-built binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: bins
+          path: .
+      - name: Unzip binaries
+        run: tar -xvf ./bins.tar.gz
+      - name: Run compatibility test
+        run: python3 .github/scripts/run-compat.py --preserve-state
+      - name: Upload compatibility logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: compatibility-logs
+          path: /tmp/sqlness-compat*
+          retention-days: 3

tests/compatibility/AGENTS.md

@@ -51,3 +51,16 @@ The dry-run performs full discovery and filtering (name, topology, metadata
 validation, namespace dedup, version-range matching) but starts no services,
 creates no temp dirs, and mutates no files. Use it to check which cases
 would be selected before a real run.
+
+## CI Version Window
+
+- `tests/compatibility/ci.toml` controls the small sliding window of recent
+  released versions used by the CI job. Do not hard-code old versions
+  directly in workflow YAML.
+- Keep the PR/merge-queue window short; wider compatibility coverage belongs in
+  nightly or release-validation workflows.
+- Case `from_range` / `to_range` metadata still controls whether each case runs
+  for a sampled version pair.
+- `.github/scripts/run-compat.py` owns the CI-side window parsing and compat
+  invocation. Keep workflow YAML thin; update the script instead of embedding
+  parsing or loops in `.github/workflows/develop.yml`.

tests/compatibility/README.md

@@ -89,6 +89,27 @@ to_range = [">=v1.1.1"]
 ```
 This case only runs when the old binary is <= v1.1.0 and the new binary is >= v1.1.1.
 
+### CI Version Window
+
+The CI job uses `tests/compatibility/ci.toml` to choose the small sliding
+window of recent released `from` versions to test against the PR-built `to`
+binary:
+
+```toml
+from_versions = ["v1.0.0", "v1.1.0"]
+```
+
+Keep this window small for PR and merge-queue latency: the goal is to catch
+upgrade compatibility issues from recent releases to the latest build, not to
+retest every historical version on every PR. Case-level `from_range`/`to_range`
+still decides which cases run for each version pair; the CI window only decides
+which old binaries are sampled. Broader historical windows belong in nightly or
+release-validation workflows.
+
+The GitHub Actions workflow delegates the window loading and compat invocation
+to `.github/scripts/run-compat.py`; the workflow YAML should stay as a thin
+wrapper around artifact download/extraction and this script.
+
 ### `setup.sql` — Setup Phase (Old Version)
 
 SQL statements executed on the **old** version cluster. These must succeed (any error fails the case). Setup output is NOT compared against any result file.

tests/compatibility/ci.toml

@@ -0,0 +1,6 @@
+# Compatibility CI version window.
+#
+# The CI job tests this small sliding window of recent released `from`
+# versions against the PR-built `to` binary. Broader historical windows should
+# run in nightly or release-validation workflows.
+from_versions = ["v1.0.0", "v1.1.0"]