name: API Integration Test

on:
  # No `push: main` trigger: the merge_group (merge-queue) run already builds and
  # tests the exact commit that lands on main, so a post-merge push run would be a
  # full-suite duplicate. The merge_group run is now the trusted main-ward build and
  # also warms the shared Rust cache (see save-if on the rust-cache step below).
  pull_request:
    branches:
      - "**"
    # labeled/unlabeled so adding or removing 'ready-for-ci' starts or resets CI; drafts never run paid CI.
    types: [opened, reopened, synchronize, labeled, unlabeled, ready_for_review, converted_to_draft]
  merge_group:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.merge_group.head_ref || github.ref }}
  cancel-in-progress: true

env:
  COLUMNS: 150
  ZO_ROOT_USER_EMAIL: root@example.com
  ZO_ROOT_USER_PASSWORD: Complexpass#123
  ZO_BASE_URL: http://localhost:5080/
  ZO_BASE_URL_SC: http://localhost:5080/
  WS_ZO_BASE_URL: ws://localhost:5080/
  ZO_USAGE_PUBLISH_INTERVAL: 5
  ZO_ALERT_SCHEDULE_INTERVAL: 3
  ZO_SMTP_ENABLED: true
  ZO_CREATE_ORG_THROUGH_INGESTION: true
  ZO_INGEST_ALLOWED_UPTO: 48  # Allow backdated data ingestion for backfill tests
  ZO_SSRF_ALLOW_LOOPBACK: true  # Allow localhost destinations in API tests

jobs:
  check_changes:
    # Runs only on PRs carrying the 'ready-for-ci' label; the merge queue runs unit tests only (see docs/ci).
    if: ${{ github.event_name != 'merge_group' && (github.event_name != 'pull_request' || (contains(github.event.pull_request.labels.*.name, 'ready-for-ci') && github.event.pull_request.draft == false)) }}
    name: check_changes
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      has_changes: ${{ steps.check.outputs.has_changes }}
    steps:
      # Debounce: a push inside this window cancels the run (concurrency) before anything paid starts.
      - if: github.event_name == 'pull_request'
        run: sleep 60
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0

      # One shared classification for every suite (see .github/actions/classify-changes); a change to
      # this workflow file re-tests itself on top of it.
      - uses: ./.github/actions/classify-changes
        id: classify
      - id: check
        env:
          HAS_CHANGES: ${{ steps.classify.outputs.run_api }}
          CHANGED_WORKFLOWS: ${{ steps.classify.outputs.changed_workflows }}
        run: |
          if [ "$HAS_CHANGES" = "true" ] || echo " $CHANGED_WORKFLOWS " | grep -q ' api-testing.yml '; then
            echo "has_changes=true" >> "$GITHUB_OUTPUT"
          else
            echo "has_changes=false" >> "$GITHUB_OUTPUT"
          fi
  build_api_binary:
    name: build_api_binary
    needs: [check_changes]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubicloud-standard-4
    env:
      # CI executes a stripped binary; omitting dependency debuginfo keeps the
      # shared compile cache small while preserving dev-profile debug assertions.
      CARGO_PROFILE_DEV_DEBUG: "0"
    permissions:
      contents: read
    steps:
      - name: Remove unused tools
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"

      - name: Checkout git repo
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - uses: ./.github/actions/github-auth
      - name: Setup Rust Toolchain
        uses: dtolnay/rust-toolchain@7e38f4b43b4db5c8dd498af069a4f6196df1d067 # master
        with:
          toolchain: nightly-2026-05-20

      - name: Setup Rust Cache
        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
        with:
          # Share one dependency build cache with the DB main-branch writer. Do
          # not cache workspace crates: a measured warm cache still rebuilt all
          # 42 workspace crates while increasing the archive from ~1.1 to 3.1 GiB.
          shared-key: backend-debug0-mimalloc-deps-v1
          add-job-id-key: false
          # The DB workflow's successful push:main build is the only writer.
          save-if: false

      - name: Install Protoc
        run: bash ./.github/protoc.sh

      - name: Build OpenObserve debug binary
        run: cargo build --features mimalloc

      # The consumers only execute this binary, so remove debug symbols from the
      # distribution copy before compressing and uploading it once.
      - name: Package OpenObserve debug binary
        run: |
          strip target/debug/openobserve
          tar -C target/debug -czf openobserve-api-debug.tar.gz openobserve

      - name: Upload OpenObserve debug binary
        uses: actions/upload-artifact@v4
        with:
          name: openobserve-api-debug-${{ github.sha }}
          path: openobserve-api-debug.tar.gz
          if-no-files-found: error
          retention-days: 3
          # The payload is already gzip-compressed; avoid a second compression pass.
          compression-level: 0

  api_integration_tests:
    name: api_integration_tests
    needs: [check_changes, build_api_binary]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubicloud-standard-4
    permissions:
      actions: read
      contents: read  # only need to checkout the repo
    steps:
      - name: Checkout git repo
        uses: actions/checkout@v5

      - name: Download OpenObserve debug binary
        uses: actions/download-artifact@v5
        with:
          name: openobserve-api-debug-${{ github.sha }}
          path: api-binary-artifact

      - name: Extract OpenObserve debug binary
        run: |
          mkdir -p target/debug
          tar -xzf api-binary-artifact/openobserve-api-debug.tar.gz -C target/debug
          chmod +x target/debug/openobserve

      - name: Setup Python
        uses: actions/setup-python@v7
        id: setup-python
        with:
          python-version: "3.11.6"

      - name: Setup rye
        uses: eifinger/setup-rye@c694239a43768373e87d0103d7f547027a23f3c8 # v4.2.9
        env:
          # Reuse setup-python's interpreter so `rye self install` skips the flaky toolchain download.
          RYE_TOOLCHAIN: ${{ steps.setup-python.outputs.python-path }}
        with:
          enable-cache: true
          working-directory: "tests/api-testing/"
          version: "0.27.0"

      - name: Pin cpython
        run: rye pin cpython@3.11.6
        working-directory: tests/api-testing/

      - name: Rye sync
        run: rye sync
        working-directory: tests/api-testing/

      - name: Run default-set tests
        # Phase 6.b: regression matrix tests moved to their own
        # api_regression_tests job below — they now run in parallel
        # with this default-set job instead of sequentially within it.
        # Cuts ~8 min off CI wall time at the cost of more runner minutes.
        # NB: --force-sugar is intentionally omitted. pytest-sugar is already
        # auto-loaded via the entry point; passing --force-sugar registers it
        # a second time, which inflates the progress counter (CI displayed
        # ~3x the real test count). Letting the plugin load via discovery
        # alone keeps the counter honest.
        run: |
          ${{ github.workspace }}/target/debug/openobserve > o2_default.log 2>&1 &
          SERVER_PID_DEFAULT=$!
          echo "OpenObserve PID for default tests: $SERVER_PID_DEFAULT"
          for i in $(seq 1 60); do
            if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
              echo "Server ready after ${i}s"; break
            fi
            sleep 1
          done
          # test_pending_silence.py is carved into api_alerts_pending_tests
          # (runs -n 4 in parallel) — those 4 wall-clock tests were ~85% of the
          # alerts runtime when run serially here.
          rye run pytest --ignore=tests/regression --ignore=tests/query_agent --ignore=tests/vortex --ignore=tests/alerts/test_pending_silence.py --junitxml=junit-integration.xml
          kill $SERVER_PID_DEFAULT || true
        working-directory: tests/api-testing/

      # Test-count metrics sidecar: ship the junit so report_to_openobserve can
      # tally API active/skipped counts (suite=api). always() so counts emit even
      # when pytest fails (junit is written before pytest's non-zero exit).
      - name: Upload API junit (integration)
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-junit__api_integration_tests__main
          path: tests/api-testing/junit-integration.xml
          if-no-files-found: warn
          retention-days: 7

      - name: Upload OpenObserve server logs on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: openobserve-server-logs
          path: tests/api-testing/o2_*.log
          if-no-files-found: warn
          retention-days: 14

  # Dedicated query agent SQL regression suite. Runs in parallel with
  # the default-set and regression jobs so the 310-case (155 queries ×
  # 2 phases: memtable + parquet) suite doesn't extend CI wall time.
  api_query_agent_tests:
    name: "api_query_agent / ${{ matrix.config.label }}"
    needs: [check_changes, build_api_binary]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubicloud-standard-4
    # Run the full SQL regression suite with the single-node physical
    # optimizer both enabled (default) and disabled. The non-optimized
    # RemoteScanRewriter path triggers known upstream DataFusion bugs on
    # a handful of queries — those are tagged skip_without_single_node_opt
    # and filtered by load_all_queries() when SINGLE_NODE_OPT_MODE=false.
    strategy:
      fail-fast: false
      matrix:
        config:
          - label: single_node_opt_false
            env: "ZO_FEATURE_SINGLE_NODE_OPTIMIZE_ENABLED=false"
            mode: "false"
          - label: single_node_opt_true
            env: "ZO_FEATURE_SINGLE_NODE_OPTIMIZE_ENABLED=true"
            mode: "true"
          - label: remove_filter_with_index_false
            env: "ZO_FEATURE_QUERY_REMOVE_FILTER_WITH_INDEX=false"
            mode: "true"
    permissions:
      actions: read
      contents: read
    steps:
      - name: Checkout git repo
        uses: actions/checkout@v5

      - name: Download OpenObserve debug binary
        uses: actions/download-artifact@v5
        with:
          name: openobserve-api-debug-${{ github.sha }}
          path: api-binary-artifact

      - name: Extract OpenObserve debug binary
        run: |
          mkdir -p target/debug
          tar -xzf api-binary-artifact/openobserve-api-debug.tar.gz -C target/debug
          chmod +x target/debug/openobserve

      - name: Setup Python
        uses: actions/setup-python@v7
        id: setup-python
        with:
          python-version: "3.11.6"

      - name: Setup rye
        uses: eifinger/setup-rye@c694239a43768373e87d0103d7f547027a23f3c8 # v4.2.9
        env:
          # Reuse setup-python's interpreter so `rye self install` skips the flaky toolchain download.
          RYE_TOOLCHAIN: ${{ steps.setup-python.outputs.python-path }}
        with:
          enable-cache: true
          working-directory: "tests/api-testing/"
          version: "0.27.0"

      - name: Pin cpython
        run: rye pin cpython@3.11.6
        working-directory: tests/api-testing/

      - name: Rye sync
        run: rye sync
        working-directory: tests/api-testing/

      - name: Regenerate query-agent expected values
        run: |
          cd ${{ github.workspace }}/tests/test-data/query-agent
          ${{ github.workspace }}/tests/api-testing/.venv/bin/python compute_counts.py

      - name: Run query agent tests
        run: |
          ZO_MAX_FILE_RETENTION_TIME=1 ${{ matrix.config.env }} ${{ github.workspace }}/target/debug/openobserve > o2_query_agent_${{ matrix.config.label }}.log 2>&1 &
          SERVER_PID=$!
          for i in $(seq 1 60); do
            if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
              echo "Server ready after ${i}s"; break
            fi
            sleep 1
          done
          # SINGLE_NODE_OPT_MODE controls which queries load_all_queries() includes
          SINGLE_NODE_OPT_MODE=${{ matrix.config.mode }} rye run pytest tests/query_agent/ --ignore=tests/query_agent/test_3_vortex.py -v --junitxml=junit-queryagent-${{ matrix.config.label }}.xml
          kill $SERVER_PID || true
        working-directory: tests/api-testing/

      # Test-count metrics sidecar (see integration job). Parser dedups the two
      # opt-mode legs by test identity so the same suite isn't counted twice.
      - name: Upload API junit (query_agent)
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-junit__query_agent_tests__${{ matrix.config.label }}
          path: tests/api-testing/junit-queryagent-${{ matrix.config.label }}.xml
          if-no-files-found: warn
          retention-days: 7

      - name: Upload server log on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: openobserve-query-agent-${{ matrix.config.label }}-log
          path: tests/api-testing/o2_query_agent_${{ matrix.config.label }}.log
          if-no-files-found: warn
          retention-days: 14

  # Phase 6.b: regression matrix split. The 4 env-flag-sensitive test
  # configurations now run as 4 parallel jobs instead of 4 sequential
  # restarts in one job. Each job downloads the run-scoped OO binary and
  # runs its single configured pytest target.
  api_regression_tests:
    name: "api_regression / ${{ matrix.config.label }}"
    needs: [check_changes, build_api_binary]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubicloud-standard-4
    permissions:
      actions: read
      contents: read  # only need to checkout the repo
    strategy:
      fail-fast: false
      matrix:
        config:
          - label: utf8_view_false
            env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true ZO_UTF8_VIEW_ENABLED=false"
            target: tests/regression/test_subquery_join_match.py
          - label: utf8_view_true
            env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true ZO_UTF8_VIEW_ENABLED=true"
            target: tests/regression/test_subquery_join_match.py
          - label: join_match_one_true
            env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true"
            target: tests/regression/test_logs_to_logs_join.py
          - label: join_match_one_false
            env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=false"
            target: tests/regression/test_logs_to_logs_join.py
          # A free-space requirement no runner meets keeps the disk circuit breaker tripped.
          - label: otlp_grpc_backpressure
            env: "ZO_DISK_CIRCUIT_BREAKER_ENABLED=true ZO_DISK_CIRCUIT_BREAKER_THRESHOLD=900000000"
            target: tests/regression/test_otlp_grpc_backpressure.py
          # Spawns and restarts its own server from target/debug/openobserve.
          - label: sqlite_meta_invalid_utf8
            env: ""
            target: tests/regression/test_sqlite_invalid_utf8_meta_startup.py
    steps:
      - uses: actions/checkout@v5

      - name: Download OpenObserve debug binary
        uses: actions/download-artifact@v5
        with:
          name: openobserve-api-debug-${{ github.sha }}
          path: api-binary-artifact

      - name: Extract OpenObserve debug binary
        run: |
          mkdir -p target/debug
          tar -xzf api-binary-artifact/openobserve-api-debug.tar.gz -C target/debug
          chmod +x target/debug/openobserve

      - uses: actions/setup-python@v7
        id: setup-python
        with: { python-version: "3.11.6" }

      - name: Setup rye
        uses: eifinger/setup-rye@c694239a43768373e87d0103d7f547027a23f3c8 # v4.2.9
        env:
          # Reuse setup-python's interpreter so `rye self install` skips the flaky toolchain download.
          RYE_TOOLCHAIN: ${{ steps.setup-python.outputs.python-path }}
        with:
          enable-cache: true
          working-directory: "tests/api-testing/"
          version: "0.27.0"

      - name: Pin cpython
        run: rye pin cpython@3.11.6
        working-directory: tests/api-testing/

      - name: Rye sync
        run: rye sync
        working-directory: tests/api-testing/

      - name: "Run ${{ matrix.config.label }} regression test"
        # Server binary lives at the repo root (target/debug/openobserve);
        # this step uses working-directory: tests/api-testing/, so we must
        # invoke it via ${{ github.workspace }} or the shell will fail with
        # "No such file or directory". --force-sugar omitted on purpose —
        # pytest-sugar auto-loads via entry point, and passing the flag
        # registers it twice and inflates the progress counter ~3x.
        run: |
          ${{ matrix.config.env }} ${{ github.workspace }}/target/debug/openobserve > "o2_${{ matrix.config.label }}.log" 2>&1 &
          SERVER_PID=$!
          for i in $(seq 1 60); do
            if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
              echo "Server ready after ${i}s"; break
            fi
            sleep 1
          done
          rye run pytest ${{ matrix.config.target }} -v --junitxml=junit-regression-${{ matrix.config.label }}.xml
          kill $SERVER_PID || true
        working-directory: tests/api-testing/

      # Test-count metrics sidecar (see integration job). Parser dedups the
      # flag-combo legs that re-run the same target files.
      - name: Upload API junit (regression)
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-junit__api_integration_tests__regression_${{ matrix.config.label }}
          path: tests/api-testing/junit-regression-${{ matrix.config.label }}.xml
          if-no-files-found: warn
          retention-days: 7

      - name: Upload server log on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: "openobserve-regression-${{ matrix.config.label }}-log"
          path: "tests/api-testing/o2_${{ matrix.config.label }}.log"
          if-no-files-found: warn
          retention-days: 14

  # Carved out of the serial default-set job: the 4 pending/silence alert
  # tests are wall-clock bound (each needs 2-3 alert eval cycles at the 60s
  # frequency floor + a real pending window), so serially they were ~85% of
  # the alerts runtime (~10 min). Run here with -n 4 so their waits overlap
  # and this job lands at ~the single longest test (~3 min), in parallel with
  # the other API jobs.
  api_alerts_pending_tests:
    name: api_alerts_pending
    needs: [check_changes, build_api_binary]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubicloud-standard-4
    permissions:
      actions: read
      contents: read  # only need to checkout the repo
    steps:
      - uses: actions/checkout@v5

      - name: Download OpenObserve debug binary
        uses: actions/download-artifact@v5
        with:
          name: openobserve-api-debug-${{ github.sha }}
          path: api-binary-artifact

      - name: Extract OpenObserve debug binary
        run: |
          mkdir -p target/debug
          tar -xzf api-binary-artifact/openobserve-api-debug.tar.gz -C target/debug
          chmod +x target/debug/openobserve

      - uses: actions/setup-python@v7
        id: setup-python
        with: { python-version: "3.11.6" }

      - name: Setup rye
        uses: eifinger/setup-rye@c694239a43768373e87d0103d7f547027a23f3c8 # v4.2.9
        env:
          # Reuse setup-python's interpreter so `rye self install` skips the flaky toolchain download.
          RYE_TOOLCHAIN: ${{ steps.setup-python.outputs.python-path }}
        with:
          enable-cache: true
          working-directory: "tests/api-testing/"
          version: "0.27.0"

      - name: Pin cpython
        run: rye pin cpython@3.11.6
        working-directory: tests/api-testing/

      - name: Rye sync
        run: rye sync
        working-directory: tests/api-testing/

      - name: Run pending/silence alert tests (parallel)
        # Server binary lives at the repo root (target/debug/openobserve);
        # this step uses working-directory: tests/api-testing/, so invoke it
        # via ${{ github.workspace }}. The 4 tests use uniq() names/streams and
        # a non-destructive shared org, so -n 4 is safe. --force-sugar omitted
        # on purpose (see integration job — double-registers, ~3x count).
        run: |
          ${{ github.workspace }}/target/debug/openobserve > o2_alerts_pending.log 2>&1 &
          SERVER_PID=$!
          for i in $(seq 1 60); do
            if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
              echo "Server ready after ${i}s"; break
            fi
            sleep 1
          done
          rye run pytest tests/alerts/test_pending_silence.py -n 4 -v --junitxml=junit-alerts-pending.xml
          kill $SERVER_PID || true
        working-directory: tests/api-testing/

      # Test-count metrics sidecar (see integration job). Category dir name is
      # api_integration_tests so the parser folds it into the integration total.
      - name: Upload API junit (alerts pending)
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-junit__api_integration_tests__alerts_pending
          path: tests/api-testing/junit-alerts-pending.xml
          if-no-files-found: warn
          retention-days: 7

      - name: Upload server log on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: openobserve-alerts-pending-log
          path: tests/api-testing/o2_alerts_pending.log
          if-no-files-found: warn
          retention-days: 14

  # Phase 5: schemathesis fuzz job — informational (continue-on-error),
  # surfaces 5xx panics across the 150 OpenAPI endpoints. Decoupled from
  # api_integration_tests so its findings don't block merges yet; teams
  # can promote it to blocking after the noise floor is understood.
  api_fuzz_tests:
    name: api_fuzz_tests
    needs: [check_changes, build_api_binary]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubicloud-standard-4
    continue-on-error: true
    permissions:
      actions: read
      contents: read  # only need to checkout the repo
    steps:
      - name: Checkout git repo
        uses: actions/checkout@v5

      - name: Download OpenObserve debug binary
        uses: actions/download-artifact@v5
        with:
          name: openobserve-api-debug-${{ github.sha }}
          path: api-binary-artifact

      - name: Extract OpenObserve debug binary
        run: |
          mkdir -p target/debug
          tar -xzf api-binary-artifact/openobserve-api-debug.tar.gz -C target/debug
          chmod +x target/debug/openobserve

      - name: Setup Python
        uses: actions/setup-python@v7
        id: setup-python
        with:
          python-version: "3.11.6"

      - name: Setup rye
        uses: eifinger/setup-rye@c694239a43768373e87d0103d7f547027a23f3c8 # v4.2.9
        env:
          # Reuse setup-python's interpreter so `rye self install` skips the flaky toolchain download.
          RYE_TOOLCHAIN: ${{ steps.setup-python.outputs.python-path }}
        with:
          enable-cache: true
          working-directory: "tests/api-testing/"
          version: "0.27.0"

      - name: Pin cpython
        run: rye pin cpython@3.11.6
        working-directory: tests/api-testing/

      - name: Rye sync (schemathesis dep installed)
        run: rye sync
        working-directory: tests/api-testing/

      - name: Start OpenObserve
        run: target/debug/openobserve > o2_fuzz.log 2>&1 &

      - name: Wait for /healthz
        run: |
          for i in $(seq 1 60); do
            if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then exit 0; fi
            sleep 1
          done
          exit 1

      - name: Run schemathesis fuzz tests
        # Schemathesis 4.x CLI flag renames vs the 3.x originals:
        #   --base-url               -> --url / -u
        #   --auth-type basic        -> removed (USER:PASS implies basic)
        #   --hypothesis-max-examples -> --max-examples / -n
        #   --hypothesis-deadline    -> no equivalent; use --request-timeout
        #                               (seconds, not ms) to bound per-request
        #   --data-generation-method -> --mode / -m
        # The two slack/oauth paths are excluded because CI configures no Slack
        # credentials, so they correctly answer 503 "not configured" — which the
        # not_a_server_error check cannot tell apart from a real 5xx panic.
        # `--checks not_a_server_error` keeps focus on 5xx panics (actual
        # backend bugs, not spec-conformance nits). `-n 20` bounds runtime;
        # `--mode positive` keeps inputs realistic (negative inputs are a
        # separate exercise once the noise floor here is understood).
        # run_fuzz.py prevents Hypothesis' auto-loaded CI profile from
        # overriding --suppress-health-check. Keep CI's too_slow suppression
        # alongside filter_too_much; the 5xx response check is unchanged.
        run: |
          rye run python run_fuzz.py run \
            -u http://localhost:5080 \
            --auth "$ZO_ROOT_USER_EMAIL:$ZO_ROOT_USER_PASSWORD" \
            --checks not_a_server_error \
            -n 20 \
            --request-timeout 5 \
            -m positive \
            --suppress-health-check=filter_too_much,too_slow \
            --exclude-path "/api/{org_id}/v1/logs" \
            --exclude-path "/api/{org_id}/v1/metrics" \
            --exclude-path "/api/{org_id}/v1/traces" \
            --exclude-path "/api/{org_id}/loki/api/v1/push" \
            --exclude-path "/api/{org_id}/prometheus/api/v1/write" \
            --exclude-path "/api/{org_id}/dashboards" \
            --exclude-path "/api/{org_id}/dashboards/{dashboard_id}" \
            --exclude-path "/api/{org_id}/dashboards/{dashboard_id}/panels" \
            --exclude-path "/api/{org_id}/dashboards/{dashboard_id}/panels/{panel_id}" \
            --exclude-path "/api/{org_id}/pipelines/history" \
            --exclude-path "/api/{org_id}/organizations/assume_service_account" \
            --exclude-path "/api/{org_id}/alerts/destinations/slack/oauth/start" \
            --exclude-path "/api/{org_id}/alerts/destinations/slack/oauth/exchange" \
            http://localhost:5080/api-doc/openapi.json
        working-directory: tests/api-testing/

      - name: Upload OpenObserve server log on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: openobserve-fuzz-log
          path: o2_fuzz.log
          if-no-files-found: warn
          retention-days: 14

  api_tests_summary:
    runs-on: ubuntu-latest
    permissions: {}
    needs: [check_changes, build_api_binary, api_integration_tests, api_query_agent_tests, api_regression_tests, api_alerts_pending_tests]
    if: always() && github.event_name != 'merge_group' && (github.event_name != 'pull_request' || (contains(github.event.pull_request.labels.*.name, 'ready-for-ci') && github.event.pull_request.draft == false))
    steps:
      - name: Check test results
        run: |
          # 1. check_changes must have succeeded.
          if [ "${{ needs.check_changes.result }}" != "success" ]; then
            echo "check_changes job did not succeed: ${{ needs.check_changes.result }}"
            exit 1
          fi

          # 2. Nothing API-relevant changed → nothing ran → allow.
          if [ "${{ needs.check_changes.outputs.has_changes }}" != "true" ]; then
            echo "✅ No API-relevant changes — suite not required."
            exit 0
          fi

          # 3. Suite was expected to run — the run-scoped binary and all gating
          #    test jobs must have succeeded.
          if [ "${{ needs.build_api_binary.result }}" == "success" ] && \
             [ "${{ needs.api_integration_tests.result }}" == "success" ] && \
             [ "${{ needs.api_query_agent_tests.result }}" == "success" ] && \
             [ "${{ needs.api_regression_tests.result }}" == "success" ] && \
             [ "${{ needs.api_alerts_pending_tests.result }}" == "success" ]; then
            echo "All API test jobs passed"
            exit 0
          else
            echo "API tests failed: build=${{ needs.build_api_binary.result }}, integration=${{ needs.api_integration_tests.result }}, query_agent=${{ needs.api_query_agent_tests.result }}, regression=${{ needs.api_regression_tests.result }}, alerts_pending=${{ needs.api_alerts_pending_tests.result }}"
            exit 1
          fi

  # Non-fatal metrics sidecar: one retry-aware run-summary doc to OpenObserve (stream ci_test_runs,
  # suite=api). Re-runs append one doc per attempt; the dashboard keeps the latest attempt per run_id.
  report_to_openobserve:
    name: Report run metrics to OpenObserve
    needs: [check_changes, api_integration_tests, api_query_agent_tests, api_regression_tests, api_alerts_pending_tests]
    if: always() && github.event_name != 'merge_group' && (github.event_name != 'pull_request' || (contains(github.event.pull_request.labels.*.name, 'ready-for-ci') && github.event.pull_request.draft == false))
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
    steps:
      - name: Clone the current repo
        uses: actions/checkout@v5
      # Pull every per-shard junit. NO merge-multiple: each artifact lands in its
      # own subdir (api-junit__<category>__<leg>) so the parser can read the test
      # CATEGORY from the dir name. Missing artifacts are non-fatal.
      - name: Download API junit reports
        if: always()
        continue-on-error: true
        uses: actions/download-artifact@v4
        with:
          pattern: api-junit__*
          path: junit-xml
      - name: Report run metrics
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
          O2_REPORTING_INGEST_BASE: ${{ secrets.O2_REPORTING_INGEST_BASE }}
          O2_REPORTING_AUTH: ${{ secrets.O2_REPORTING_AUTH }}
          O2_REPORTING_INSECURE: ${{ vars.O2_REPORTING_INSECURE }}
          INTEG: ${{ needs.api_integration_tests.result }}
          AGENT: ${{ needs.api_query_agent_tests.result }}
          REG: ${{ needs.api_regression_tests.result }}
          PEND: ${{ needs.api_alerts_pending_tests.result }}
        run: |
          if [ "$INTEG" = "skipped" ] && [ "$AGENT" = "skipped" ] && [ "$REG" = "skipped" ] && [ "$PEND" = "skipped" ]; then
            echo "::notice::no API tests ran (no relevant changes) — skipping metrics"; exit 0
          fi
          if [ "$INTEG" = "failure" ] || [ "$AGENT" = "failure" ] || [ "$REG" = "failure" ] || [ "$PEND" = "failure" ]; then CONCLUSION=failure
          elif [ "$INTEG" = "cancelled" ] || [ "$AGENT" = "cancelled" ] || [ "$REG" = "cancelled" ] || [ "$PEND" = "cancelled" ]; then CONCLUSION=cancelled
          else CONCLUSION=success; fi
          export CONCLUSION STREAM=ci_test_runs SUITE=api
          # Parse junit into per-category + total counts. The TOTAL folds into the
          # run doc (EXTRA_JSON) so suite=api shows up in the same KPI/pie panels
          # as UI; the per-category rows go to ci_test_counts for the per-shard
          # breakdown (the API analog of UI's ci_test_modules). All best-effort.
          if [ -d junit-xml ]; then
            COUNTS=$(python3 .github/scripts/parse-api-junit.py junit-xml 2>/dev/null || echo "")
            if [ -n "$COUNTS" ] && printf '%s' "$COUNTS" | jq -e . >/dev/null 2>&1; then
              echo "API counts: $COUNTS"
              EXTRA_JSON=$(printf '%s' "$COUNTS" | jq -c .total); export EXTRA_JSON
              printf '%s' "$COUNTS" | jq -c --arg run_id "$GITHUB_RUN_ID" --arg repo "$GITHUB_REPOSITORY" \
                '[.by_category[] | {run_id:$run_id, repo:$repo, suite:"api", workflow:"test", ingest_source:"live",
                  module:.module, tests_total, tests_passed, tests_failed, tests_flaky, tests_skipped}]' > api_counts_rows.json
              if [ "$(jq 'length' api_counts_rows.json 2>/dev/null || echo 0)" -gt 0 ]; then
                bash .github/scripts/o2-report.sh ci_test_counts api_counts_rows.json
              fi
            fi
          fi
          bash .github/scripts/o2-run-report.sh