#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" RUNNER="${SCRIPT_DIR}/run-fuzz-targets.sh" COLLECTOR="${SCRIPT_DIR}/collect-fuzz-target-artifacts.sh" fail() { printf 'FAIL: %s\n' "$*" >&2 exit 1 } assert_eq() { local expected="$1" local actual="$2" local message="$3" [[ "${expected}" == "${actual}" ]] || \ fail "${message}: expected=${expected@Q}, actual=${actual@Q}" } assert_file() { [[ -f "$1" ]] || fail "expected file: $1" } new_fixture() { fixture="$(mktemp -d -t fuzz-runner-test.XXXXXX)" mkdir -p "${fixture}/bin" cat >"${fixture}/bin/cargo" <<'EOF' #!/usr/bin/env bash set -euo pipefail printf '%s\t%s\t%s\n' "${GT_FUZZ_DUMP_DIR}" "$*" "${MOCK_CARGO_MARKER:-}" >>"${MOCK_CARGO_LOG}" target="$3" case " ${MOCK_FAIL_TARGETS:-} " in *" ${target} "*) exit 17 ;; esac EOF cat >"${fixture}/collector" <<'EOF' #!/usr/bin/env bash set -euo pipefail printf '%s\t%s\t%s\n' "$1" "$2" "${FUZZ_COLLECT_CLUSTER_ARTIFACTS}" >>"${MOCK_COLLECTOR_LOG}" mkdir -p "$2/mock-artifacts" printf 'collected\n' >"$2/mock-artifacts/state.txt" EOF chmod +x "${fixture}/bin/cargo" "${fixture}/collector" } cleanup_fixture() { rm -rf "${fixture}" } run_fixture() { local targets="$1" local fail_fast="$2" local unstable="$3" local fail_targets="$4" local artifact_root="${fixture}/artifacts" local -a runner_env=( "PATH=${fixture}/bin:${PATH}" "MOCK_CARGO_LOG=${fixture}/cargo.log" "MOCK_COLLECTOR_LOG=${fixture}/collector.log" "MOCK_FAIL_TARGETS=${fail_targets}" "MOCK_CARGO_MARKER=fixture" "FUZZ_TARGETS=${targets}" "FUZZ_GROUP=test-group" "FUZZ_MAX_TOTAL_TIME=120" "FUZZ_UNSTABLE=${unstable}" "FUZZ_COLLECT_CLUSTER_ARTIFACTS=true" "FUZZ_ARTIFACT_ROOT=${artifact_root}" "FUZZ_ARTIFACT_COLLECTOR=${fixture}/collector" "GT_FUZZ_BINARY_PATH=${GT_FUZZ_BINARY_PATH:-}" "GT_FUZZ_INSTANCE_ROOT_DIR=${GT_FUZZ_INSTANCE_ROOT_DIR:-}" "GITHUB_SHA=deadbeef" "GITHUB_STEP_SUMMARY=${fixture}/github-step-summary.md" ) if [[ "${fail_fast}" != unset ]]; then runner_env+=("FUZZ_FAIL_FAST=${fail_fast}") fi set +e env -u FUZZ_FAIL_FAST "${runner_env[@]}" \ "${RUNNER}" >"${fixture}/stdout.log" 2>&1 fixture_status=$? set -e } run_prebuilt_fixture() { new_fixture mkdir -p "${fixture}/prebuilt" cat >"${fixture}/prebuilt/fuzz_create_table" <<'EOF' #!/usr/bin/env bash set -euo pipefail printf '%s\t%s\t%s\n' "${GT_FUZZ_DUMP_DIR}" "$*" "${MOCK_BINARY_MARKER:-}" >>"${MOCK_BINARY_LOG}" EOF chmod +x "${fixture}/prebuilt/fuzz_create_table" set +e env \ "MOCK_BINARY_LOG=${fixture}/binary.log" \ "MOCK_BINARY_MARKER=fixture" \ "FUZZ_TARGETS=fuzz_create_table" \ "FUZZ_GROUP=test-group" \ "FUZZ_MAX_TOTAL_TIME=120" \ "FUZZ_UNSTABLE=false" \ "FUZZ_FAIL_FAST=true" \ "FUZZ_COLLECT_CLUSTER_ARTIFACTS=false" \ "FUZZ_ARTIFACT_ROOT=${fixture}/artifacts" \ "FUZZ_ARTIFACT_COLLECTOR=${fixture}/collector" \ "FUZZ_BIN_DIR=${fixture}/prebuilt" \ "GITHUB_SHA=deadbeef" \ "GITHUB_STEP_SUMMARY=${fixture}/github-step-summary.md" \ "${RUNNER}" >"${fixture}/stdout.log" 2>&1 fixture_status=$? set -e } test_successful_targets_run_in_order() { new_fixture run_fixture $'fuzz_create_table\nfuzz_insert' false true "" assert_eq 0 "${fixture_status}" "successful run status" assert_eq 2 "$(wc -l <"${fixture}/cargo.log" | tr -d ' ')" "cargo invocation count" assert_eq \ $'fuzz_create_table\nfuzz_insert' \ "$(awk -F '\t' '{print $2}' "${fixture}/cargo.log" | sed -E 's/^fuzz run ([^ ]+).*/\1/')" \ "target order" grep -q -- '--features=unstable' "${fixture}/cargo.log" || fail "unstable feature missing" grep -q -- '-max_total_time=120' "${fixture}/cargo.log" || fail "fuzz time missing" grep -q -- '-artifact_prefix=.*/targets/fuzz_create_table/libfuzzer/' "${fixture}/cargo.log" || \ fail "target-scoped libFuzzer prefix missing" assert_eq success "$(jq -r '.[0].status' "${fixture}/artifacts/manifest.json")" "first status" assert_eq success "$(jq -r '.[1].status' "${fixture}/artifacts/manifest.json")" "second status" assert_file "${fixture}/artifacts/summary.md" grep -q 'Policy: `continue-after-failure`' "${fixture}/artifacts/summary.md" || \ fail "group policy missing from summary" grep -q 'Results: \*\*2 passed\*\*, \*\*0 failed\*\*, \*\*0 skipped\*\* / 2 total' \ "${fixture}/artifacts/summary.md" || fail "group counts missing from summary" grep -q 'title=Fuzz target completed' "${fixture}/stdout.log" || \ fail "target completion notice missing" grep -q 'title=Fuzz group completed' "${fixture}/stdout.log" || \ fail "group completion notice missing" assert_file "${fixture}/github-step-summary.md" cleanup_fixture } test_prebuilt_binary_runs_without_cargo() { run_prebuilt_fixture assert_eq 0 "${fixture_status}" "prebuilt run status" [[ ! -e "${fixture}/cargo.log" ]] || fail "cargo ran for prebuilt binary" assert_eq 1 "$(wc -l <"${fixture}/binary.log" | tr -d ' ')" "prebuilt invocation count" grep -q -- '-max_total_time=120' "${fixture}/binary.log" || fail "prebuilt fuzz time missing" grep -q -- '-artifact_prefix=.*/targets/fuzz_create_table/libfuzzer/' "${fixture}/binary.log" || \ fail "prebuilt target-scoped libFuzzer prefix missing" cleanup_fixture } test_fail_fast_stops_after_first_failure() { new_fixture run_fixture $'fuzz_create_table\nfuzz_insert\nfuzz_alter_table' true false "fuzz_insert" assert_eq 1 "${fixture_status}" "fail-fast run status" assert_eq 2 "$(wc -l <"${fixture}/cargo.log" | tr -d ' ')" "fail-fast cargo count" assert_eq 1 "$(wc -l <"${fixture}/collector.log" | tr -d ' ')" "collector count" assert_eq failure "$(jq -r '.[1].status' "${fixture}/artifacts/manifest.json")" "failed status" assert_eq skipped_after_failure "$(jq -r '.[2].status' "${fixture}/artifacts/manifest.json")" "skipped status" assert_eq true "$(jq -r '.[2].after_prior_failure' "${fixture}/artifacts/manifest.json")" "skipped provenance" assert_file "${fixture}/artifacts/targets/fuzz_insert/mock-artifacts/state.txt" grep -q 'title=Fuzz target failed.*target=fuzz_insert' "${fixture}/stdout.log" || \ fail "target failure annotation missing" grep -q 'title=Fuzz target skipped.*target=fuzz_alter_table' "${fixture}/stdout.log" || \ fail "skipped target annotation missing" grep -q '### Reproduce failed targets' "${fixture}/artifacts/summary.md" || \ fail "reproduction section missing" grep -q 'cargo fuzz run fuzz_insert' "${fixture}/artifacts/summary.md" || \ fail "reproduction command missing" cleanup_fixture } test_fail_fast_defaults_to_true() { new_fixture GT_FUZZ_BINARY_PATH=./bin/greptime \ GT_FUZZ_INSTANCE_ROOT_DIR=/tmp/greptime-fuzz-artifacts/targets/fuzz_insert/service/instance/ \ run_fixture $'fuzz_create_table\nfuzz_insert\nfuzz_alter_table' unset true "fuzz_insert" assert_eq 1 "${fixture_status}" "default fail-fast run status" assert_eq 2 "$(wc -l <"${fixture}/cargo.log" | tr -d ' ')" "default fail-fast cargo count" assert_eq skipped_after_failure \ "$(jq -r '.[2].status' "${fixture}/artifacts/manifest.json")" \ "default fail-fast skipped status" grep -q -- '--features=unstable' "${fixture}/artifacts/summary.md" || \ fail "unstable reproduction argument missing" grep -q -- 'GT_FUZZ_BINARY_PATH=./bin/greptime' \ "${fixture}/artifacts/summary.md" || fail "unstable binary path missing from reproduction" grep -q -- 'GT_FUZZ_INSTANCE_ROOT_DIR=/tmp/greptime-fuzz-artifacts/targets/fuzz_insert/service/instance/' \ "${fixture}/artifacts/summary.md" || fail "unstable instance root missing from reproduction" cleanup_fixture } test_non_fail_fast_marks_later_targets() { new_fixture run_fixture $'fuzz_create_table\nfuzz_insert' false false "fuzz_create_table" assert_eq 1 "${fixture_status}" "non-fail-fast run status" assert_eq 2 "$(wc -l <"${fixture}/cargo.log" | tr -d ' ')" "non-fail-fast cargo count" assert_eq true "$(jq -r '.[1].after_prior_failure' "${fixture}/artifacts/manifest.json")" "later target provenance" assert_eq success "$(jq -r '.[1].status' "${fixture}/artifacts/manifest.json")" "later target status" cleanup_fixture } test_invalid_configuration_fails_before_cargo() { new_fixture run_fixture 'fuzz target' true false "" assert_eq 2 "${fixture_status}" "invalid target status" [[ ! -e "${fixture}/cargo.log" ]] || fail "cargo ran for invalid target" cleanup_fixture new_fixture run_fixture 'fuzz_create_table' sometimes false "" assert_eq 2 "${fixture_status}" "invalid fail-fast status" [[ ! -e "${fixture}/cargo.log" ]] || fail "cargo ran for invalid fail-fast" cleanup_fixture } test_collector_keeps_target_scopes_separate() { new_fixture service_log="${fixture}/greptime.log" printf 'service output\n' >"${service_log}" FUZZ_COLLECT_CLUSTER_ARTIFACTS=false \ FUZZ_SERVICE_LOG="${service_log}" \ "${COLLECTOR}" fuzz_insert "${fixture}/target-a" FUZZ_COLLECT_CLUSTER_ARTIFACTS=false \ FUZZ_SERVICE_LOG="${service_log}" \ "${COLLECTOR}" fuzz_alter_table "${fixture}/target-b" assert_file "${fixture}/target-a/service/greptime.log" assert_file "${fixture}/target-b/service/greptime.log" assert_file "${fixture}/target-a/artifact-collection.log" assert_file "${fixture}/target-b/artifact-collection.log" cleanup_fixture } test_cluster_collector_honors_target_scope_and_namespace() { new_fixture cat >"${fixture}/bin/kubectl" <<'EOF' #!/usr/bin/env bash set -euo pipefail printf 'kubectl %s\n' "$*" >>"${MOCK_COMMAND_LOG}" printf 'mock kubectl output\n' EOF cat >"${fixture}/bin/kind" <<'EOF' #!/usr/bin/env bash set -euo pipefail printf 'kind %s\n' "$*" >>"${MOCK_COMMAND_LOG}" EOF cat >"${fixture}/monitor-collector" <<'EOF' #!/usr/bin/env bash set -euo pipefail printf '%s\t%s\t%s\t%s\n' \ "${GT_FUZZ_NS}" "${GT_FUZZ_CLUSTER}" "${GT_MONITOR_ARTIFACT_DIR}" \ "${GT_MONITOR_SERVER_EXPORT_DIR}" \ >>"${MOCK_MONITOR_LOG}" printf 'monitor output\n' >"${GT_MONITOR_ARTIFACT_DIR}/monitor.txt" EOF chmod +x "${fixture}/bin/kubectl" "${fixture}/bin/kind" PATH="${fixture}/bin:${PATH}" \ MOCK_COMMAND_LOG="${fixture}/commands.log" \ MOCK_MONITOR_LOG="${fixture}/monitor.log" \ FUZZ_COLLECT_CLUSTER_ARTIFACTS=true \ FUZZ_MONITOR_COLLECTOR="${fixture}/monitor-collector" \ GT_FUZZ_NS=test-namespace \ GT_FUZZ_CLUSTER=test-cluster \ "${COLLECTOR}" fuzz_insert "${fixture}/target" grep -q 'kubectl describe pod -n test-namespace' "${fixture}/commands.log" || \ fail "collector ignored configured namespace" grep -q "kind export logs ${fixture}/target/kind" "${fixture}/commands.log" || \ fail "kind logs are not target-scoped" assert_eq \ "test-namespace"$'\t'"test-cluster"$'\t'"${fixture}/target/monitor"$'\t'"/tmp/gt-monitor-dump/fuzz_insert" \ "$(cat "${fixture}/monitor.log")" \ "monitor collector scope" assert_file "${fixture}/target/kubernetes/nodes.txt" assert_file "${fixture}/target/kubernetes/pods.txt" assert_file "${fixture}/target/kubernetes/events.txt" assert_file "${fixture}/target/monitor/monitor.txt" cleanup_fixture } test_setup_failure_writes_artifact_contract() { new_fixture artifact_root="${fixture}/artifacts" setup_summary="${fixture}/setup-step-summary.md" FUZZ_COLLECT_CLUSTER_ARTIFACTS=false \ FUZZ_GROUP=setup-group \ GITHUB_SHA=deadbeef \ GITHUB_STEP_SUMMARY="${setup_summary}" \ "${COLLECTOR}" setup "${artifact_root}/targets/setup" assert_file "${artifact_root}/targets/setup/result.json" assert_file "${artifact_root}/manifest.json" assert_file "${artifact_root}/summary.md" assert_file "${setup_summary}" assert_eq setup-group "$(jq -r '.[0].group' "${artifact_root}/manifest.json")" \ "setup manifest group" assert_eq setup "$(jq -r '.[0].target' "${artifact_root}/manifest.json")" \ "setup manifest target" assert_eq failure "$(jq -r '.[0].status' "${artifact_root}/manifest.json")" \ "setup manifest status" assert_eq setup "$(jq -r '.[0].phase' "${artifact_root}/manifest.json")" \ "setup manifest phase" grep -q 'Failure phase: `setup`' "${artifact_root}/summary.md" || \ fail "setup failure phase missing from summary" cmp -s "${artifact_root}/summary.md" "${setup_summary}" || \ fail "setup summary was not appended to the job summary" cleanup_fixture } test_setup_failure_keeps_manifest_when_collection_fails() { new_fixture artifact_root="${fixture}/artifacts" cat >"${fixture}/bin/kubectl" <<'EOF' #!/usr/bin/env bash exit 1 EOF cat >"${fixture}/bin/kind" <<'EOF' #!/usr/bin/env bash exit 1 EOF chmod +x "${fixture}/bin/kubectl" "${fixture}/bin/kind" set +e PATH="${fixture}/bin:${PATH}" \ FUZZ_COLLECT_CLUSTER_ARTIFACTS=true \ FUZZ_GROUP=setup-group \ FUZZ_MONITOR_COLLECTOR="${fixture}/missing-monitor-collector" \ GITHUB_STEP_SUMMARY="${fixture}/setup-step-summary.md" \ "${COLLECTOR}" setup "${artifact_root}/targets/setup" \ >"${fixture}/collector-stdout.log" 2>&1 collector_status=$? set -e assert_eq 1 "${collector_status}" "failed setup collection status" assert_file "${artifact_root}/manifest.json" assert_eq failed "$(jq -r '.[0].artifact_collection' "${artifact_root}/manifest.json")" \ "failed setup collection marker" cleanup_fixture } test_successful_targets_run_in_order test_prebuilt_binary_runs_without_cargo test_fail_fast_stops_after_first_failure test_fail_fast_defaults_to_true test_non_fail_fast_marks_later_targets test_invalid_configuration_fails_before_cargo test_collector_keeps_target_scopes_separate test_cluster_collector_honors_target_scope_and_namespace test_setup_failure_writes_artifact_contract test_setup_failure_keeps_manifest_when_collection_fails printf 'All fuzz orchestration script tests passed.\n'