ci: optimize fuzz and split workflows (#8710)

* ci: batch fuzz targets in GitHub Actions Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: improve fuzz test observability Signed-off-by: WenyXu <wenymedia@gmail.com> * fix(ci): preserve fuzz setup failure artifacts Signed-off-by: WenyXu <wenymedia@gmail.com> * test(ci): keep fuzz mock output in logs Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: optimize fuzz worker cache Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: warm fuzz target binaries Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: isolate fuzz workflow Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: centralize fuzz target preparation Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: split general workflows Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: streamline docs required checks Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: transfer fuzz targets as artifacts Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: preserve fuzz binary permissions Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: streamline fuzz workers Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: cache PR build dependencies Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: retain main build cache policy Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: address fuzz review feedback Signed-off-by: WenyXu <wenymedia@gmail.com> --------- Signed-off-by: WenyXu <wenymedia@gmail.com>
2026-08-25 07:28:39 +00:00 · 2026-08-03 15:58:29 +08:00
parent a7590f8174
commit a6107fbe3d
14 changed files with 2082 additions and 990 deletions
@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+if [[ "$#" -ne 2 ]]; then
+  echo "Usage: $0 <target> <target-artifact-dir>" >&2
+  exit 2
+fi
+
+target="$1"
+target_dir="$2"
+namespace="${GT_FUZZ_NS:-my-greptimedb}"
+collection_log="${target_dir}/artifact-collection.log"
+monitor_collector="${FUZZ_MONITOR_COLLECTOR:-.github/scripts/collect-fuzz-monitor-artifacts.sh}"
+failed=false
+
+mkdir -p "${target_dir}"
+exec > >(tee -a "${collection_log}") 2>&1
+
+log() {
+  printf '[collect-fuzz-target-artifacts] %s\n' "$*"
+}
+
+collect() {
+  local name="$1"
+  shift
+
+  log "collect ${name}"
+  if "$@"; then
+    return
+  fi
+
+  log "failed to collect ${name}"
+  failed=true
+}
+
+write_setup_failure_manifest() {
+  local artifact_root
+  local result_path="${target_dir}/result.json"
+  local manifest_path
+  local summary_path
+  local collection_status
+  local completed_at
+
+  artifact_root="$(dirname "$(dirname "${target_dir}")")"
+  manifest_path="${artifact_root}/manifest.json"
+  summary_path="${artifact_root}/summary.md"
+  completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  if [[ "${failed}" == true ]]; then
+    collection_status="failed"
+  else
+    collection_status="success"
+  fi
+
+  jq -n \
+    --arg target "${target}" \
+    --arg group "${FUZZ_GROUP:-setup-failure}" \
+    --arg git_sha "${GITHUB_SHA:-}" \
+    --arg completed_at "${completed_at}" \
+    --arg artifact_collection "${collection_status}" \
+    '{
+      target: $target,
+      group: $group,
+      git_sha: $git_sha,
+      status: "failure",
+      phase: "setup",
+      started_at: null,
+      completed_at: $completed_at,
+      duration_secs: null,
+      exit_code: null,
+      max_total_time_secs: null,
+      after_prior_failure: false,
+      artifact_collection: $artifact_collection
+    }' >"${result_path}"
+  jq -s '.' "${result_path}" >"${manifest_path}"
+
+  {
+    printf '## Fuzz target results: `%s`\n\n' "${FUZZ_GROUP:-setup-failure}"
+    printf -- '- Failure phase: `setup`\n'
+    printf -- '- Artifact collection: `%s`\n\n' "${collection_status}"
+    printf '| Target | Status | Exit code | Artifacts |\n'
+    printf '| --- | --- | ---: | --- |\n'
+    printf '| `%s` | failure | - | %s |\n' "${target}" "${collection_status}"
+  } >"${summary_path}"
+
+  if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+    cat "${summary_path}" >>"${GITHUB_STEP_SUMMARY}"
+  fi
+}
+
+log "collect artifacts for target ${target} under ${target_dir}"
+
+if [[ -n "${FUZZ_SERVICE_LOG:-}" && -f "${FUZZ_SERVICE_LOG}" ]]; then
+  mkdir -p "${target_dir}/service"
+  collect "service log" cp "${FUZZ_SERVICE_LOG}" "${target_dir}/service/greptime.log"
+fi
+
+case "${FUZZ_COLLECT_CLUSTER_ARTIFACTS:-false}" in
+  false)
+    ;;
+  true)
+    mkdir -p "${target_dir}/kind" "${target_dir}/kubernetes" "${target_dir}/monitor"
+    collect "Kubernetes nodes" \
+      bash -c 'kubectl describe nodes >"$1" 2>&1' _ \
+      "${target_dir}/kubernetes/nodes.txt"
+    collect "Kubernetes pods" \
+      bash -c 'kubectl get pods -A -o wide >"$1" 2>&1 && kubectl describe pod -n "$2" >>"$1" 2>&1' _ \
+      "${target_dir}/kubernetes/pods.txt" "${namespace}"
+    collect "Kubernetes events" \
+      bash -c 'kubectl get events -A --sort-by=.lastTimestamp >"$1" 2>&1' _ \
+      "${target_dir}/kubernetes/events.txt"
+    collect "Kind logs" kind export logs "${target_dir}/kind"
+    collect "monitor dumps" \
+      env \
+        GT_FUZZ_NS="${namespace}" \
+        GT_FUZZ_CLUSTER="${GT_FUZZ_CLUSTER:-my-greptimedb}" \
+        GT_MONITOR_HTTP_LOCAL_PORT="${GT_MONITOR_HTTP_LOCAL_PORT:-14000}" \
+        GT_MONITOR_ARTIFACT_DIR="${target_dir}/monitor" \
+        GT_MONITOR_SERVER_EXPORT_DIR="${GT_MONITOR_SERVER_EXPORT_DIR:-/tmp/gt-monitor-dump/${target}}" \
+        bash "${monitor_collector}"
+    ;;
+  *)
+    log "FUZZ_COLLECT_CLUSTER_ARTIFACTS must be true or false"
+    exit 2
+    ;;
+esac
+
+if [[ "${target}" == setup ]]; then
+  write_setup_failure_manifest
+fi
+
+if [[ "${failed}" == true ]]; then
+  exit 1
+fi
+
+log "artifact collection completed"
@@ -0,0 +1,371 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+RUNNER="${SCRIPT_DIR}/run-fuzz-targets.sh"
+COLLECTOR="${SCRIPT_DIR}/collect-fuzz-target-artifacts.sh"
+
+fail() {
+  printf 'FAIL: %s\n' "$*" >&2
+  exit 1
+}
+
+assert_eq() {
+  local expected="$1"
+  local actual="$2"
+  local message="$3"
+  [[ "${expected}" == "${actual}" ]] || \
+    fail "${message}: expected=${expected@Q}, actual=${actual@Q}"
+}
+
+assert_file() {
+  [[ -f "$1" ]] || fail "expected file: $1"
+}
+
+new_fixture() {
+  fixture="$(mktemp -d -t fuzz-runner-test.XXXXXX)"
+  mkdir -p "${fixture}/bin"
+
+  cat >"${fixture}/bin/cargo" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+printf '%s\t%s\t%s\n' "${GT_FUZZ_DUMP_DIR}" "$*" "${MOCK_CARGO_MARKER:-}" >>"${MOCK_CARGO_LOG}"
+target="$3"
+case " ${MOCK_FAIL_TARGETS:-} " in
+  *" ${target} "*) exit 17 ;;
+esac
+EOF
+
+  cat >"${fixture}/collector" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+printf '%s\t%s\t%s\n' "$1" "$2" "${FUZZ_COLLECT_CLUSTER_ARTIFACTS}" >>"${MOCK_COLLECTOR_LOG}"
+mkdir -p "$2/mock-artifacts"
+printf 'collected\n' >"$2/mock-artifacts/state.txt"
+EOF
+
+  chmod +x "${fixture}/bin/cargo" "${fixture}/collector"
+}
+
+cleanup_fixture() {
+  rm -rf "${fixture}"
+}
+
+run_fixture() {
+  local targets="$1"
+  local fail_fast="$2"
+  local unstable="$3"
+  local fail_targets="$4"
+  local artifact_root="${fixture}/artifacts"
+  local -a runner_env=(
+    "PATH=${fixture}/bin:${PATH}"
+    "MOCK_CARGO_LOG=${fixture}/cargo.log"
+    "MOCK_COLLECTOR_LOG=${fixture}/collector.log"
+    "MOCK_FAIL_TARGETS=${fail_targets}"
+    "MOCK_CARGO_MARKER=fixture"
+    "FUZZ_TARGETS=${targets}"
+    "FUZZ_GROUP=test-group"
+    "FUZZ_MAX_TOTAL_TIME=120"
+    "FUZZ_UNSTABLE=${unstable}"
+    "FUZZ_COLLECT_CLUSTER_ARTIFACTS=true"
+    "FUZZ_ARTIFACT_ROOT=${artifact_root}"
+    "FUZZ_ARTIFACT_COLLECTOR=${fixture}/collector"
+    "GT_FUZZ_BINARY_PATH=${GT_FUZZ_BINARY_PATH:-}"
+    "GT_FUZZ_INSTANCE_ROOT_DIR=${GT_FUZZ_INSTANCE_ROOT_DIR:-}"
+    "GITHUB_SHA=deadbeef"
+    "GITHUB_STEP_SUMMARY=${fixture}/github-step-summary.md"
+  )
+  if [[ "${fail_fast}" != unset ]]; then
+    runner_env+=("FUZZ_FAIL_FAST=${fail_fast}")
+  fi
+
+  set +e
+  env -u FUZZ_FAIL_FAST "${runner_env[@]}" \
+    "${RUNNER}" >"${fixture}/stdout.log" 2>&1
+  fixture_status=$?
+  set -e
+}
+
+run_prebuilt_fixture() {
+  new_fixture
+  mkdir -p "${fixture}/prebuilt"
+  cat >"${fixture}/prebuilt/fuzz_create_table" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+printf '%s\t%s\t%s\n' "${GT_FUZZ_DUMP_DIR}" "$*" "${MOCK_BINARY_MARKER:-}" >>"${MOCK_BINARY_LOG}"
+EOF
+  chmod +x "${fixture}/prebuilt/fuzz_create_table"
+
+  set +e
+  env \
+    "MOCK_BINARY_LOG=${fixture}/binary.log" \
+    "MOCK_BINARY_MARKER=fixture" \
+    "FUZZ_TARGETS=fuzz_create_table" \
+    "FUZZ_GROUP=test-group" \
+    "FUZZ_MAX_TOTAL_TIME=120" \
+    "FUZZ_UNSTABLE=false" \
+    "FUZZ_FAIL_FAST=true" \
+    "FUZZ_COLLECT_CLUSTER_ARTIFACTS=false" \
+    "FUZZ_ARTIFACT_ROOT=${fixture}/artifacts" \
+    "FUZZ_ARTIFACT_COLLECTOR=${fixture}/collector" \
+    "FUZZ_BIN_DIR=${fixture}/prebuilt" \
+    "GITHUB_SHA=deadbeef" \
+    "GITHUB_STEP_SUMMARY=${fixture}/github-step-summary.md" \
+    "${RUNNER}" >"${fixture}/stdout.log" 2>&1
+  fixture_status=$?
+  set -e
+}
+
+test_successful_targets_run_in_order() {
+  new_fixture
+  run_fixture $'fuzz_create_table\nfuzz_insert' false true ""
+
+  assert_eq 0 "${fixture_status}" "successful run status"
+  assert_eq 2 "$(wc -l <"${fixture}/cargo.log" | tr -d ' ')" "cargo invocation count"
+  assert_eq \
+    $'fuzz_create_table\nfuzz_insert' \
+    "$(awk -F '\t' '{print $2}' "${fixture}/cargo.log" | sed -E 's/^fuzz run ([^ ]+).*/\1/')" \
+    "target order"
+  grep -q -- '--features=unstable' "${fixture}/cargo.log" || fail "unstable feature missing"
+  grep -q -- '-max_total_time=120' "${fixture}/cargo.log" || fail "fuzz time missing"
+  grep -q -- '-artifact_prefix=.*/targets/fuzz_create_table/libfuzzer/' "${fixture}/cargo.log" || \
+    fail "target-scoped libFuzzer prefix missing"
+  assert_eq success "$(jq -r '.[0].status' "${fixture}/artifacts/manifest.json")" "first status"
+  assert_eq success "$(jq -r '.[1].status' "${fixture}/artifacts/manifest.json")" "second status"
+  assert_file "${fixture}/artifacts/summary.md"
+  grep -q 'Policy: `continue-after-failure`' "${fixture}/artifacts/summary.md" || \
+    fail "group policy missing from summary"
+  grep -q 'Results: \*\*2 passed\*\*, \*\*0 failed\*\*, \*\*0 skipped\*\* / 2 total' \
+    "${fixture}/artifacts/summary.md" || fail "group counts missing from summary"
+  grep -q 'title=Fuzz target completed' "${fixture}/stdout.log" || \
+    fail "target completion notice missing"
+  grep -q 'title=Fuzz group completed' "${fixture}/stdout.log" || \
+    fail "group completion notice missing"
+  assert_file "${fixture}/github-step-summary.md"
+  cleanup_fixture
+}
+
+test_prebuilt_binary_runs_without_cargo() {
+  run_prebuilt_fixture
+
+  assert_eq 0 "${fixture_status}" "prebuilt run status"
+  [[ ! -e "${fixture}/cargo.log" ]] || fail "cargo ran for prebuilt binary"
+  assert_eq 1 "$(wc -l <"${fixture}/binary.log" | tr -d ' ')" "prebuilt invocation count"
+  grep -q -- '-max_total_time=120' "${fixture}/binary.log" || fail "prebuilt fuzz time missing"
+  grep -q -- '-artifact_prefix=.*/targets/fuzz_create_table/libfuzzer/' "${fixture}/binary.log" || \
+    fail "prebuilt target-scoped libFuzzer prefix missing"
+  cleanup_fixture
+}
+
+test_fail_fast_stops_after_first_failure() {
+  new_fixture
+  run_fixture $'fuzz_create_table\nfuzz_insert\nfuzz_alter_table' true false "fuzz_insert"
+
+  assert_eq 1 "${fixture_status}" "fail-fast run status"
+  assert_eq 2 "$(wc -l <"${fixture}/cargo.log" | tr -d ' ')" "fail-fast cargo count"
+  assert_eq 1 "$(wc -l <"${fixture}/collector.log" | tr -d ' ')" "collector count"
+  assert_eq failure "$(jq -r '.[1].status' "${fixture}/artifacts/manifest.json")" "failed status"
+  assert_eq skipped_after_failure "$(jq -r '.[2].status' "${fixture}/artifacts/manifest.json")" "skipped status"
+  assert_eq true "$(jq -r '.[2].after_prior_failure' "${fixture}/artifacts/manifest.json")" "skipped provenance"
+  assert_file "${fixture}/artifacts/targets/fuzz_insert/mock-artifacts/state.txt"
+  grep -q 'title=Fuzz target failed.*target=fuzz_insert' "${fixture}/stdout.log" || \
+    fail "target failure annotation missing"
+  grep -q 'title=Fuzz target skipped.*target=fuzz_alter_table' "${fixture}/stdout.log" || \
+    fail "skipped target annotation missing"
+  grep -q '### Reproduce failed targets' "${fixture}/artifacts/summary.md" || \
+    fail "reproduction section missing"
+  grep -q 'cargo fuzz run fuzz_insert' "${fixture}/artifacts/summary.md" || \
+    fail "reproduction command missing"
+  cleanup_fixture
+}
+
+test_fail_fast_defaults_to_true() {
+  new_fixture
+  GT_FUZZ_BINARY_PATH=./bin/greptime \
+    GT_FUZZ_INSTANCE_ROOT_DIR=/tmp/greptime-fuzz-artifacts/targets/fuzz_insert/service/instance/ \
+    run_fixture $'fuzz_create_table\nfuzz_insert\nfuzz_alter_table' unset true "fuzz_insert"
+
+  assert_eq 1 "${fixture_status}" "default fail-fast run status"
+  assert_eq 2 "$(wc -l <"${fixture}/cargo.log" | tr -d ' ')" "default fail-fast cargo count"
+  assert_eq skipped_after_failure \
+    "$(jq -r '.[2].status' "${fixture}/artifacts/manifest.json")" \
+    "default fail-fast skipped status"
+  grep -q -- '--features=unstable' "${fixture}/artifacts/summary.md" || \
+    fail "unstable reproduction argument missing"
+  grep -q -- 'GT_FUZZ_BINARY_PATH=./bin/greptime' \
+    "${fixture}/artifacts/summary.md" || fail "unstable binary path missing from reproduction"
+  grep -q -- 'GT_FUZZ_INSTANCE_ROOT_DIR=/tmp/greptime-fuzz-artifacts/targets/fuzz_insert/service/instance/' \
+    "${fixture}/artifacts/summary.md" || fail "unstable instance root missing from reproduction"
+  cleanup_fixture
+}
+
+test_non_fail_fast_marks_later_targets() {
+  new_fixture
+  run_fixture $'fuzz_create_table\nfuzz_insert' false false "fuzz_create_table"
+
+  assert_eq 1 "${fixture_status}" "non-fail-fast run status"
+  assert_eq 2 "$(wc -l <"${fixture}/cargo.log" | tr -d ' ')" "non-fail-fast cargo count"
+  assert_eq true "$(jq -r '.[1].after_prior_failure' "${fixture}/artifacts/manifest.json")" "later target provenance"
+  assert_eq success "$(jq -r '.[1].status' "${fixture}/artifacts/manifest.json")" "later target status"
+  cleanup_fixture
+}
+
+test_invalid_configuration_fails_before_cargo() {
+  new_fixture
+  run_fixture 'fuzz target' true false ""
+
+  assert_eq 2 "${fixture_status}" "invalid target status"
+  [[ ! -e "${fixture}/cargo.log" ]] || fail "cargo ran for invalid target"
+  cleanup_fixture
+
+  new_fixture
+  run_fixture 'fuzz_create_table' sometimes false ""
+
+  assert_eq 2 "${fixture_status}" "invalid fail-fast status"
+  [[ ! -e "${fixture}/cargo.log" ]] || fail "cargo ran for invalid fail-fast"
+  cleanup_fixture
+}
+
+test_collector_keeps_target_scopes_separate() {
+  new_fixture
+  service_log="${fixture}/greptime.log"
+  printf 'service output\n' >"${service_log}"
+
+  FUZZ_COLLECT_CLUSTER_ARTIFACTS=false \
+    FUZZ_SERVICE_LOG="${service_log}" \
+    "${COLLECTOR}" fuzz_insert "${fixture}/target-a"
+  FUZZ_COLLECT_CLUSTER_ARTIFACTS=false \
+    FUZZ_SERVICE_LOG="${service_log}" \
+    "${COLLECTOR}" fuzz_alter_table "${fixture}/target-b"
+
+  assert_file "${fixture}/target-a/service/greptime.log"
+  assert_file "${fixture}/target-b/service/greptime.log"
+  assert_file "${fixture}/target-a/artifact-collection.log"
+  assert_file "${fixture}/target-b/artifact-collection.log"
+  cleanup_fixture
+}
+
+test_cluster_collector_honors_target_scope_and_namespace() {
+  new_fixture
+
+  cat >"${fixture}/bin/kubectl" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+printf 'kubectl %s\n' "$*" >>"${MOCK_COMMAND_LOG}"
+printf 'mock kubectl output\n'
+EOF
+  cat >"${fixture}/bin/kind" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+printf 'kind %s\n' "$*" >>"${MOCK_COMMAND_LOG}"
+EOF
+  cat >"${fixture}/monitor-collector" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+printf '%s\t%s\t%s\t%s\n' \
+  "${GT_FUZZ_NS}" "${GT_FUZZ_CLUSTER}" "${GT_MONITOR_ARTIFACT_DIR}" \
+  "${GT_MONITOR_SERVER_EXPORT_DIR}" \
+  >>"${MOCK_MONITOR_LOG}"
+printf 'monitor output\n' >"${GT_MONITOR_ARTIFACT_DIR}/monitor.txt"
+EOF
+  chmod +x "${fixture}/bin/kubectl" "${fixture}/bin/kind"
+
+  PATH="${fixture}/bin:${PATH}" \
+    MOCK_COMMAND_LOG="${fixture}/commands.log" \
+    MOCK_MONITOR_LOG="${fixture}/monitor.log" \
+    FUZZ_COLLECT_CLUSTER_ARTIFACTS=true \
+    FUZZ_MONITOR_COLLECTOR="${fixture}/monitor-collector" \
+    GT_FUZZ_NS=test-namespace \
+    GT_FUZZ_CLUSTER=test-cluster \
+    "${COLLECTOR}" fuzz_insert "${fixture}/target"
+
+  grep -q 'kubectl describe pod -n test-namespace' "${fixture}/commands.log" || \
+    fail "collector ignored configured namespace"
+  grep -q "kind export logs ${fixture}/target/kind" "${fixture}/commands.log" || \
+    fail "kind logs are not target-scoped"
+  assert_eq \
+    "test-namespace"$'\t'"test-cluster"$'\t'"${fixture}/target/monitor"$'\t'"/tmp/gt-monitor-dump/fuzz_insert" \
+    "$(cat "${fixture}/monitor.log")" \
+    "monitor collector scope"
+  assert_file "${fixture}/target/kubernetes/nodes.txt"
+  assert_file "${fixture}/target/kubernetes/pods.txt"
+  assert_file "${fixture}/target/kubernetes/events.txt"
+  assert_file "${fixture}/target/monitor/monitor.txt"
+  cleanup_fixture
+}
+
+test_setup_failure_writes_artifact_contract() {
+  new_fixture
+  artifact_root="${fixture}/artifacts"
+  setup_summary="${fixture}/setup-step-summary.md"
+
+  FUZZ_COLLECT_CLUSTER_ARTIFACTS=false \
+    FUZZ_GROUP=setup-group \
+    GITHUB_SHA=deadbeef \
+    GITHUB_STEP_SUMMARY="${setup_summary}" \
+    "${COLLECTOR}" setup "${artifact_root}/targets/setup"
+
+  assert_file "${artifact_root}/targets/setup/result.json"
+  assert_file "${artifact_root}/manifest.json"
+  assert_file "${artifact_root}/summary.md"
+  assert_file "${setup_summary}"
+  assert_eq setup-group "$(jq -r '.[0].group' "${artifact_root}/manifest.json")" \
+    "setup manifest group"
+  assert_eq setup "$(jq -r '.[0].target' "${artifact_root}/manifest.json")" \
+    "setup manifest target"
+  assert_eq failure "$(jq -r '.[0].status' "${artifact_root}/manifest.json")" \
+    "setup manifest status"
+  assert_eq setup "$(jq -r '.[0].phase' "${artifact_root}/manifest.json")" \
+    "setup manifest phase"
+  grep -q 'Failure phase: `setup`' "${artifact_root}/summary.md" || \
+    fail "setup failure phase missing from summary"
+  cmp -s "${artifact_root}/summary.md" "${setup_summary}" || \
+    fail "setup summary was not appended to the job summary"
+  cleanup_fixture
+}
+
+test_setup_failure_keeps_manifest_when_collection_fails() {
+  new_fixture
+  artifact_root="${fixture}/artifacts"
+
+  cat >"${fixture}/bin/kubectl" <<'EOF'
+#!/usr/bin/env bash
+exit 1
+EOF
+  cat >"${fixture}/bin/kind" <<'EOF'
+#!/usr/bin/env bash
+exit 1
+EOF
+  chmod +x "${fixture}/bin/kubectl" "${fixture}/bin/kind"
+
+  set +e
+  PATH="${fixture}/bin:${PATH}" \
+    FUZZ_COLLECT_CLUSTER_ARTIFACTS=true \
+    FUZZ_GROUP=setup-group \
+    FUZZ_MONITOR_COLLECTOR="${fixture}/missing-monitor-collector" \
+    GITHUB_STEP_SUMMARY="${fixture}/setup-step-summary.md" \
+    "${COLLECTOR}" setup "${artifact_root}/targets/setup" \
+    >"${fixture}/collector-stdout.log" 2>&1
+  collector_status=$?
+  set -e
+
+  assert_eq 1 "${collector_status}" "failed setup collection status"
+  assert_file "${artifact_root}/manifest.json"
+  assert_eq failed "$(jq -r '.[0].artifact_collection' "${artifact_root}/manifest.json")" \
+    "failed setup collection marker"
+  cleanup_fixture
+}
+
+test_successful_targets_run_in_order
+test_prebuilt_binary_runs_without_cargo
+test_fail_fast_stops_after_first_failure
+test_fail_fast_defaults_to_true
+test_non_fail_fast_marks_later_targets
+test_invalid_configuration_fails_before_cargo
+test_collector_keeps_target_scopes_separate
+test_cluster_collector_honors_target_scope_and_namespace
+test_setup_failure_writes_artifact_contract
+test_setup_failure_keeps_manifest_when_collection_fails
+
+printf 'All fuzz orchestration script tests passed.\n'
@@ -0,0 +1,272 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+FUZZ_ARTIFACT_ROOT="${FUZZ_ARTIFACT_ROOT:-/tmp/greptime-fuzz-artifacts}"
+FUZZ_ARTIFACT_COLLECTOR="${FUZZ_ARTIFACT_COLLECTOR:-.github/scripts/collect-fuzz-target-artifacts.sh}"
+FUZZ_ARTIFACT_COLLECTION_TIMEOUT_SECS="${FUZZ_ARTIFACT_COLLECTION_TIMEOUT_SECS:-180}"
+
+log() {
+  printf '[run-fuzz-targets] %s\n' "$*"
+}
+
+parse_bool() {
+  case "$2" in
+    true | false)
+      printf -v "$1" '%s' "$2"
+      ;;
+    *)
+      log "$1 must be true or false, got: $2"
+      exit 2
+      ;;
+  esac
+}
+
+write_result() {
+  local path="$1"
+  local target="$2"
+  local status="$3"
+  local started_at="$4"
+  local completed_at="$5"
+  local duration_secs="$6"
+  local exit_code="$7"
+  local after_prior_failure="$8"
+  local artifact_collection="$9"
+
+  jq -n \
+    --arg target "${target}" \
+    --arg group "${FUZZ_GROUP}" \
+    --arg git_sha "${GITHUB_SHA:-}" \
+    --arg status "${status}" \
+    --arg started_at "${started_at}" \
+    --arg completed_at "${completed_at}" \
+    --arg duration_secs "${duration_secs}" \
+    --arg exit_code "${exit_code}" \
+    --arg max_total_time "${FUZZ_MAX_TOTAL_TIME}" \
+    --argjson after_prior_failure "${after_prior_failure}" \
+    --arg artifact_collection "${artifact_collection}" \
+    '{
+      target: $target,
+      group: $group,
+      git_sha: $git_sha,
+      status: $status,
+      started_at: (if $started_at == "" then null else $started_at end),
+      completed_at: (if $completed_at == "" then null else $completed_at end),
+      duration_secs: (if $duration_secs == "" then null else ($duration_secs | tonumber) end),
+      exit_code: (if $exit_code == "" then null else ($exit_code | tonumber) end),
+      max_total_time_secs: ($max_total_time | tonumber),
+      after_prior_failure: $after_prior_failure,
+      artifact_collection: $artifact_collection
+    }' >"${path}"
+}
+
+write_summary() {
+  local manifest="$1"
+  local summary="$2"
+  local elapsed_secs="$3"
+  local total_count="$4"
+  local success_count="$5"
+  local failure_count="$6"
+  local skipped_count="$7"
+  local policy
+  local target reproduce_command
+  local -a reproduce_args
+
+  if [[ "${fuzz_fail_fast}" == true ]]; then
+    policy="fail-fast"
+  else
+    policy="continue-after-failure"
+  fi
+
+  {
+    printf '## Fuzz target results: `%s`\n\n' "${FUZZ_GROUP}"
+    printf -- '- Policy: `%s`\n' "${policy}"
+    printf -- '- Results: **%s passed**, **%s failed**, **%s skipped** / %s total\n' \
+      "${success_count}" "${failure_count}" "${skipped_count}" "${total_count}"
+    printf -- '- Group elapsed time: **%ss**\n' "${elapsed_secs}"
+    if [[ -n "${GITHUB_SHA:-}" ]]; then
+      printf -- '- Commit: `%s`\n' "${GITHUB_SHA}"
+    fi
+    printf '\n'
+    printf '| Target | Status | Duration (s) | Exit code | After prior failure | Artifacts |\n'
+    printf '| --- | --- | ---: | ---: | --- | --- |\n'
+    jq -r '.[] | "| `\(.target)` | \(.status) | \(.duration_secs // "-") | \(.exit_code // "-") | \(.after_prior_failure) | \(.artifact_collection) |"' "${manifest}"
+
+    if [[ "${failure_count}" -gt 0 ]]; then
+      printf '\n### Reproduce failed targets\n\n'
+      while IFS= read -r target; do
+        reproduce_args=()
+        if [[ "${fuzz_unstable}" == true ]]; then
+          if [[ -n "${GT_FUZZ_BINARY_PATH:-}" ]]; then
+            reproduce_args+=("GT_FUZZ_BINARY_PATH=${GT_FUZZ_BINARY_PATH}")
+          fi
+          if [[ -n "${GT_FUZZ_INSTANCE_ROOT_DIR:-}" ]]; then
+            reproduce_args+=("GT_FUZZ_INSTANCE_ROOT_DIR=${GT_FUZZ_INSTANCE_ROOT_DIR}")
+          fi
+        fi
+        reproduce_args+=(cargo fuzz run "${target}" --fuzz-dir tests-fuzz -D -s none)
+        if [[ "${fuzz_unstable}" == true ]]; then
+          reproduce_args+=(--features=unstable)
+        fi
+        reproduce_args+=(-- "-max_total_time=${FUZZ_MAX_TOTAL_TIME}")
+        printf -v reproduce_command '%q ' "${reproduce_args[@]}"
+        printf -- '- `%s`\n' "${reproduce_command% }"
+      done < <(jq -r '.[] | select(.status == "failure") | .target' "${manifest}")
+    fi
+  } >"${summary}"
+
+  if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+    cat "${summary}" >>"${GITHUB_STEP_SUMMARY}"
+  fi
+}
+
+: "${FUZZ_TARGETS:?FUZZ_TARGETS is required}"
+: "${FUZZ_GROUP:?FUZZ_GROUP is required}"
+: "${FUZZ_MAX_TOTAL_TIME:?FUZZ_MAX_TOTAL_TIME is required}"
+
+if [[ ! "${FUZZ_GROUP}" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then
+  log "FUZZ_GROUP must be a lowercase slug, got: ${FUZZ_GROUP}"
+  exit 2
+fi
+if [[ ! "${FUZZ_MAX_TOTAL_TIME}" =~ ^[1-9][0-9]*$ ]]; then
+  log "FUZZ_MAX_TOTAL_TIME must be a positive integer, got: ${FUZZ_MAX_TOTAL_TIME}"
+  exit 2
+fi
+if [[ ! "${FUZZ_ARTIFACT_COLLECTION_TIMEOUT_SECS}" =~ ^[1-9][0-9]*$ ]]; then
+  log "FUZZ_ARTIFACT_COLLECTION_TIMEOUT_SECS must be a positive integer"
+  exit 2
+fi
+
+parse_bool fuzz_unstable "${FUZZ_UNSTABLE:-false}"
+parse_bool fuzz_fail_fast "${FUZZ_FAIL_FAST:-true}"
+parse_bool fuzz_collect_cluster_artifacts "${FUZZ_COLLECT_CLUSTER_ARTIFACTS:-false}"
+
+if [[ -n "${FUZZ_BIN_DIR:-}" && ! -d "${FUZZ_BIN_DIR}" ]]; then
+  log "FUZZ_BIN_DIR does not exist: ${FUZZ_BIN_DIR}"
+  exit 2
+fi
+
+targets=()
+while IFS= read -r target; do
+  target="${target%$'\r'}"
+  [[ -z "${target}" ]] && continue
+  if [[ ! "${target}" =~ ^[a-zA-Z0-9_]+$ ]]; then
+    log "invalid fuzz target: ${target}"
+    exit 2
+  fi
+  targets+=("${target}")
+done <<<"${FUZZ_TARGETS}"
+
+if [[ "${#targets[@]}" -eq 0 ]]; then
+  log "FUZZ_TARGETS contains no targets"
+  exit 2
+fi
+
+mkdir -p "${FUZZ_ARTIFACT_ROOT}/targets"
+result_paths=()
+failed_targets=()
+prior_failure=false
+group_started_epoch="$(date +%s)"
+
+for ((index = 0; index < ${#targets[@]}; index++)); do
+  target="${targets[index]}"
+  target_dir="${FUZZ_ARTIFACT_ROOT}/targets/${target}"
+  result_path="${target_dir}/result.json"
+  mkdir -p "${target_dir}/csv" "${target_dir}/libfuzzer"
+  result_paths+=("${result_path}")
+
+  if [[ "${prior_failure}" == true && "${fuzz_fail_fast}" == true ]]; then
+    write_result \
+      "${result_path}" "${target}" "skipped_after_failure" \
+      "" "" "" "" true "not_requested"
+    echo "::notice title=Fuzz target skipped::target=${target}, group=${FUZZ_GROUP}, reason=prior target failure"
+    continue
+  fi
+
+  started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  started_epoch="$(date +%s)"
+  echo "::group::Fuzz target: ${target}"
+  log "run target ${target}"
+  if [[ -n "${FUZZ_BIN_DIR:-}" ]]; then
+    fuzz_binary="${FUZZ_BIN_DIR}/${target}"
+    if [[ ! -x "${fuzz_binary}" ]]; then
+      log "prebuilt fuzz binary does not exist or is not executable: ${fuzz_binary}"
+      exit 2
+    fi
+    run_args=("${fuzz_binary}" "-max_total_time=${FUZZ_MAX_TOTAL_TIME}" "-artifact_prefix=${target_dir}/libfuzzer/")
+  else
+    run_args=(cargo fuzz run "${target}" --fuzz-dir tests-fuzz -D -s none)
+    if [[ "${fuzz_unstable}" == true ]]; then
+      run_args+=(--features=unstable)
+    fi
+    run_args+=(-- "-max_total_time=${FUZZ_MAX_TOTAL_TIME}" "-artifact_prefix=${target_dir}/libfuzzer/")
+  fi
+  if GT_FUZZ_DUMP_DIR="${target_dir}/csv" \
+    "${run_args[@]}" 2>&1 | tee "${target_dir}/fuzz.log"; then
+    exit_code=0
+    status="success"
+    artifact_collection="not_requested"
+  else
+    exit_code=$?
+    status="failure"
+    artifact_collection="success"
+  fi
+
+  completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  completed_epoch="$(date +%s)"
+  duration_secs=$((completed_epoch - started_epoch))
+
+  if [[ "${status}" == failure ]]; then
+    failed_targets+=("${target}")
+    echo "::error title=Fuzz target failed::target=${target}, group=${FUZZ_GROUP}, exit_code=${exit_code}, duration_secs=${duration_secs}; collecting diagnostics"
+    log "target ${target} failed with exit code ${exit_code}; collect diagnostics"
+    if timeout --signal=TERM --kill-after=10s \
+      "${FUZZ_ARTIFACT_COLLECTION_TIMEOUT_SECS}s" \
+      env \
+        FUZZ_COLLECT_CLUSTER_ARTIFACTS="${fuzz_collect_cluster_artifacts}" \
+        "${FUZZ_ARTIFACT_COLLECTOR}" "${target}" "${target_dir}"; then
+      :
+    else
+      collector_exit_code=$?
+      if [[ "${collector_exit_code}" -eq 124 ]]; then
+        artifact_collection="timeout"
+      else
+        artifact_collection="failed"
+      fi
+      echo "::warning title=Fuzz artifact collection failed::target=${target}, status=${artifact_collection}, exit_code=${collector_exit_code}"
+    fi
+  fi
+  echo "::endgroup::"
+
+  write_result \
+    "${result_path}" "${target}" "${status}" \
+    "${started_at}" "${completed_at}" "${duration_secs}" "${exit_code}" \
+    "${prior_failure}" "${artifact_collection}"
+
+  if [[ "${status}" == failure ]]; then
+    prior_failure=true
+  else
+    echo "::notice title=Fuzz target completed::target=${target}, group=${FUZZ_GROUP}, status=success, duration_secs=${duration_secs}, after_prior_failure=${prior_failure}"
+  fi
+done
+
+manifest="${FUZZ_ARTIFACT_ROOT}/manifest.json"
+summary="${FUZZ_ARTIFACT_ROOT}/summary.md"
+jq -s '.' "${result_paths[@]}" >"${manifest}"
+group_completed_epoch="$(date +%s)"
+group_elapsed_secs=$((group_completed_epoch - group_started_epoch))
+total_count="$(jq 'length' "${manifest}")"
+success_count="$(jq '[.[] | select(.status == "success")] | length' "${manifest}")"
+failure_count="$(jq '[.[] | select(.status == "failure")] | length' "${manifest}")"
+skipped_count="$(jq '[.[] | select(.status == "skipped_after_failure")] | length' "${manifest}")"
+write_summary \
+  "${manifest}" "${summary}" "${group_elapsed_secs}" \
+  "${total_count}" "${success_count}" "${failure_count}" "${skipped_count}"
+echo "::notice title=Fuzz group completed::group=${FUZZ_GROUP}, passed=${success_count}, failed=${failure_count}, skipped=${skipped_count}, elapsed_secs=${group_elapsed_secs}"
+
+if [[ "${#failed_targets[@]}" -gt 0 ]]; then
+  log "failed targets: ${failed_targets[*]}"
+  exit 1
+fi
+
+log "all targets passed: ${targets[*]}"