From 16b60fe8470d4ba20c494317cc2d15a48480ac30 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 29 Jun 2026 16:18:28 +0800 Subject: [PATCH] ci: add sqlness compat smoke (#8370) * ci: add sqlness compat smoke Signed-off-by: discord9 * ci: limit compat smoke pull request trigger Signed-off-by: discord9 * ci: run compat smoke in merge queue Signed-off-by: discord9 * ci: configure compat version window Signed-off-by: discord9 * ci: move compat smoke logic into script Signed-off-by: discord9 * ci: expand compat version window Signed-off-by: discord9 * ci: rename compat job for recent releases Signed-off-by: discord9 * ci: report compat test failures Signed-off-by: discord9 --------- Signed-off-by: discord9 --- .github/scripts/run-compat.py | 188 ++++++++++++++++++++++++++++++++++ .github/workflows/develop.yml | 45 ++++---- tests/compatibility/AGENTS.md | 13 +++ tests/compatibility/README.md | 21 ++++ tests/compatibility/ci.toml | 6 ++ 5 files changed, 254 insertions(+), 19 deletions(-) create mode 100644 .github/scripts/run-compat.py create mode 100644 tests/compatibility/ci.toml diff --git a/.github/scripts/run-compat.py b/.github/scripts/run-compat.py new file mode 100644 index 0000000000..1521b70c14 --- /dev/null +++ b/.github/scripts/run-compat.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run sqlness compatibility checks for a small release window. + +The workflow intentionally keeps YAML small and delegates the maintainable parts +here: + +- read `tests/compatibility/ci.toml` +- validate the checked-in recent-release window +- preview selected cases with `compat --dry-run` +- run the real compat check for each sampled `from` version +""" + +from __future__ import annotations + +import argparse +import ast +import os +import re +import shlex +import subprocess +import sys +from pathlib import Path + + +VERSION_RE = re.compile(r"v[0-9]+\.[0-9]+\.[0-9]+") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config", + default="tests/compatibility/ci.toml", + help="Path to the compatibility CI window config.", + ) + parser.add_argument( + "--runner", + default="./bins/sqlness-runner", + help="Path to the sqlness-runner binary built by CI.", + ) + parser.add_argument( + "--to-bins-dir", + default="./bins", + help="Directory containing the PR-built greptime binary.", + ) + parser.add_argument( + "--preserve-state", + action="store_true", + help="Pass --preserve-state to the real compat run for artifact upload.", + ) + return parser.parse_args() + + +def load_from_versions(config_path: Path) -> list[str]: + if not config_path.is_file(): + raise SystemExit(f"Compatibility CI config not found: {config_path}") + + # Parse the simple TOML string array without depending on Python 3.11+ + # tomllib. This intentionally supports only the checked-in shape used by + # the CI job: from_versions = ["vX.Y.Z", ...]. + content = config_path.read_text(encoding="utf-8") + content_without_comments = "\n".join( + line.split("#", 1)[0] for line in content.splitlines() + ) + match = re.search( + r"(?ms)^\s*from_versions\s*=\s*(\[[^\]]*\])", + content_without_comments, + ) + if match is None: + raise SystemExit(f"{config_path} must define from_versions") + + try: + versions = ast.literal_eval(match.group(1)) + except (SyntaxError, ValueError) as err: + raise SystemExit(f"Invalid from_versions in {config_path}: {err}") from err + + if not isinstance(versions, list) or not versions: + raise SystemExit(f"{config_path} must define a non-empty from_versions list") + + seen: set[str] = set() + validated: list[str] = [] + for version in versions: + if not isinstance(version, str) or VERSION_RE.fullmatch(version) is None: + raise SystemExit(f"Invalid compat from version: {version!r}") + if version in seen: + raise SystemExit(f"Duplicate compat from version: {version}") + seen.add(version) + validated.append(version) + + return validated + + +def check_inputs(runner: Path, to_bins_dir: Path) -> None: + if not runner.is_file(): + raise SystemExit(f"sqlness-runner binary not found: {runner}") + if not to_bins_dir.is_dir(): + raise SystemExit(f"to-bins directory not found: {to_bins_dir}") + if not to_bins_dir.joinpath("greptime").is_file(): + raise SystemExit(f"greptime binary not found in to-bins directory: {to_bins_dir}") + + +def github_group(title: str): + class Group: + def __enter__(self) -> None: + print(f"::group::{title}", flush=True) + + def __exit__(self, exc_type, exc, traceback) -> None: + print("::endgroup::", flush=True) + + return Group() + + +def run_command(command: list[str], *, env: dict[str, str] | None = None) -> None: + print(f"$ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True, env=env) + + +def run_for_version( + *, + runner: Path, + to_bins_dir: Path, + from_version: str, + preserve_state: bool, +) -> None: + base_command = [ + str(runner), + "compat", + "--from-version", + from_version, + "--to-bins-dir", + str(to_bins_dir), + ] + + with github_group(f"Preview {from_version} -> current"): + run_command([*base_command, "--dry-run"]) + + real_command = [*base_command] + if preserve_state: + real_command.append("--preserve-state") + + env = os.environ.copy() + env.setdefault("RUST_BACKTRACE", "1") + with github_group(f"Compatibility {from_version} -> current"): + run_command(real_command, env=env) + + +def main() -> int: + args = parse_args() + config_path = Path(args.config) + runner = Path(args.runner) + to_bins_dir = Path(args.to_bins_dir) + + check_inputs(runner, to_bins_dir) + from_versions = load_from_versions(config_path) + + print("Compatibility from-version window:", flush=True) + for version in from_versions: + print(f" - {version}", flush=True) + + for from_version in from_versions: + run_for_version( + runner=runner, + to_bins_dir=to_bins_dir, + from_version=from_version, + preserve_state=args.preserve_state, + ) + + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except subprocess.CalledProcessError as err: + raise SystemExit(err.returncode) from err diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index a79dad1ab7..bba15c9300 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -1019,22 +1019,29 @@ jobs: with: artifact-name: bins - - # compat: - # if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} - # name: Compatibility Test - # needs: build - # runs-on: ubuntu-22.04 - # timeout-minutes: 60 - # steps: - # - uses: actions/checkout@v4 - # - name: Download pre-built binaries - # uses: actions/download-artifact@v4 - # with: - # name: bins - # path: . - # - name: Unzip binaries - # run: | - # mkdir -p ./bins/current - # tar -xvf ./bins.tar.gz --strip-components=1 -C ./bins/current - # - run: ./tests/compat/test-compat.sh 0.6.0 + compat: + if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && github.event.action == 'ready_for_review')) }} + name: Compatibility Test (recent releases) + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Download pre-built binaries + uses: actions/download-artifact@v4 + with: + name: bins + path: . + - name: Unzip binaries + run: tar -xvf ./bins.tar.gz + - name: Run compatibility test + run: python3 .github/scripts/run-compat.py --preserve-state + - name: Upload compatibility logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: compatibility-logs + path: /tmp/sqlness-compat* + retention-days: 3 diff --git a/tests/compatibility/AGENTS.md b/tests/compatibility/AGENTS.md index edce1edf1a..2d2e9952b2 100644 --- a/tests/compatibility/AGENTS.md +++ b/tests/compatibility/AGENTS.md @@ -51,3 +51,16 @@ The dry-run performs full discovery and filtering (name, topology, metadata validation, namespace dedup, version-range matching) but starts no services, creates no temp dirs, and mutates no files. Use it to check which cases would be selected before a real run. + +## CI Version Window + +- `tests/compatibility/ci.toml` controls the small sliding window of recent + released versions used by the CI job. Do not hard-code old versions + directly in workflow YAML. +- Keep the PR/merge-queue window short; wider compatibility coverage belongs in + nightly or release-validation workflows. +- Case `from_range` / `to_range` metadata still controls whether each case runs + for a sampled version pair. +- `.github/scripts/run-compat.py` owns the CI-side window parsing and compat + invocation. Keep workflow YAML thin; update the script instead of embedding + parsing or loops in `.github/workflows/develop.yml`. diff --git a/tests/compatibility/README.md b/tests/compatibility/README.md index 1b16fdb470..c4a20a4139 100644 --- a/tests/compatibility/README.md +++ b/tests/compatibility/README.md @@ -89,6 +89,27 @@ to_range = [">=v1.1.1"] ``` This case only runs when the old binary is <= v1.1.0 and the new binary is >= v1.1.1. +### CI Version Window + +The CI job uses `tests/compatibility/ci.toml` to choose the small sliding +window of recent released `from` versions to test against the PR-built `to` +binary: + +```toml +from_versions = ["v1.0.0", "v1.1.0"] +``` + +Keep this window small for PR and merge-queue latency: the goal is to catch +upgrade compatibility issues from recent releases to the latest build, not to +retest every historical version on every PR. Case-level `from_range`/`to_range` +still decides which cases run for each version pair; the CI window only decides +which old binaries are sampled. Broader historical windows belong in nightly or +release-validation workflows. + +The GitHub Actions workflow delegates the window loading and compat invocation +to `.github/scripts/run-compat.py`; the workflow YAML should stay as a thin +wrapper around artifact download/extraction and this script. + ### `setup.sql` — Setup Phase (Old Version) SQL statements executed on the **old** version cluster. These must succeed (any error fails the case). Setup output is NOT compared against any result file. diff --git a/tests/compatibility/ci.toml b/tests/compatibility/ci.toml new file mode 100644 index 0000000000..587f822bb8 --- /dev/null +++ b/tests/compatibility/ci.toml @@ -0,0 +1,6 @@ +# Compatibility CI version window. +# +# The CI job tests this small sliding window of recent released `from` +# versions against the PR-built `to` binary. Broader historical windows should +# run in nightly or release-validation workflows. +from_versions = ["v1.0.0", "v1.1.0"]