openobserve/.github/workflows/api-testing.yml

name: API Integration Test

on:
  push:
    branches:
      - "main"
  pull_request:
    branches:
      - "**"
  merge_group:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.merge_group.head_ref || github.ref }}
  cancel-in-progress: true

env:
  COLUMNS: 150
  ZO_ROOT_USER_EMAIL: root@example.com
  ZO_ROOT_USER_PASSWORD: Complexpass#123
  ZO_BASE_URL: http://localhost:5080/
  ZO_BASE_URL_SC: http://localhost:5080/
  WS_ZO_BASE_URL: ws://localhost:5080/
  ZO_USAGE_REPORTING_ENABLED: true
  ZO_USAGE_PUBLISH_INTERVAL: 5
  ZO_ALERT_SCHEDULE_INTERVAL: 3
  ZO_SMTP_ENABLED: true
  ZO_CREATE_ORG_THROUGH_INGESTION: true
  ZO_INGEST_ALLOWED_UPTO: 48  # Allow backdated data ingestion for backfill tests
  ZO_SSRF_ALLOW_LOOPBACK: true  # Allow localhost destinations in API tests

jobs:
  check_changes:
    name: check_changes
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      has_changes: ${{ steps.check.outputs.has_changes }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - id: check
        run: |
          CHANGED=$(git diff --name-only origin/${{ github.base_ref || 'main' }} ${{ github.sha }})
          if [ -z "$CHANGED" ]; then
            echo "has_changes=true" >> $GITHUB_OUTPUT
          elif echo "$CHANGED" | grep -qvE '(\.md$|^docs/|^web/|^tests/ui-testing/|^tests/playwright-tests/)'; then
            echo "has_changes=true" >> $GITHUB_OUTPUT
          else
            echo "has_changes=false" >> $GITHUB_OUTPUT
          fi

  api_integration_tests:
    name: api_integration_tests
    needs: [check_changes]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubicloud-standard-4
    permissions:
      contents: read  # only need to checkout the repo
    steps:
      - name: Remove unused tools
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"

      - name: Checkout git repo
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - name: Setup Rust Toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: nightly-2026-05-20

      - name: Setup Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true
          shared-key: backend-debug-mimalloc
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Install Protoc
        run: |
          bash ./.github/protoc.sh

      - name: Build OpenObserve debug binary
        run: cargo build --features mimalloc

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11.6"

      - name: Setup rye
        uses: eifinger/setup-rye@v4
        with:
          enable-cache: true
          working-directory: "tests/api-testing/"
          version: "0.27.0"

      - name: Pin cpython
        run: rye pin cpython@3.11.6
        working-directory: tests/api-testing/

      - name: Rye sync
        run: rye sync
        working-directory: tests/api-testing/

      - name: Run default-set tests
        # Phase 6.b: regression matrix tests moved to their own
        # api_regression_tests job below — they now run in parallel
        # with this default-set job instead of sequentially within it.
        # Cuts ~8 min off CI wall time at the cost of more runner minutes.
        # NB: --force-sugar is intentionally omitted. pytest-sugar is already
        # auto-loaded via the entry point; passing --force-sugar registers it
        # a second time, which inflates the progress counter (CI displayed
        # ~3x the real test count). Letting the plugin load via discovery
        # alone keeps the counter honest.
        run: |
          ${{ github.workspace }}/target/debug/openobserve > o2_default.log 2>&1 &
          SERVER_PID_DEFAULT=$!
          echo "OpenObserve PID for default tests: $SERVER_PID_DEFAULT"
          for i in $(seq 1 60); do
            if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
              echo "Server ready after ${i}s"; break
            fi
            sleep 1
          done
          rye run pytest --ignore=tests/regression --ignore=tests/query_agent --ignore=tests/vortex --junitxml=junit-integration.xml
          kill $SERVER_PID_DEFAULT || true
        working-directory: tests/api-testing/

      # Test-count metrics sidecar: ship the junit so report_to_openobserve can
      # tally API active/skipped counts (suite=api). always() so counts emit even
      # when pytest fails (junit is written before pytest's non-zero exit).
      - name: Upload API junit (integration)
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-junit__api_integration_tests__main
          path: tests/api-testing/junit-integration.xml
          if-no-files-found: warn
          retention-days: 7

      - name: Upload OpenObserve server logs on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: openobserve-server-logs
          path: tests/api-testing/o2_*.log
          if-no-files-found: warn
          retention-days: 14

  # Dedicated query agent SQL regression suite. Runs in parallel with
  # the default-set and regression jobs so the 310-case (155 queries ×
  # 2 phases: memtable + parquet) suite doesn't extend CI wall time.
  api_query_agent_tests:
    name: "api_query_agent / ${{ matrix.config.label }}"
    needs: [check_changes]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubicloud-standard-4
    # Run the full SQL regression suite with the single-node physical
    # optimizer both enabled (default) and disabled. The non-optimized
    # RemoteScanRewriter path triggers known upstream DataFusion bugs on
    # a handful of queries — those are tagged skip_without_single_node_opt
    # and filtered by load_all_queries() when SINGLE_NODE_OPT_MODE=false.
    strategy:
      fail-fast: false
      matrix:
        config:
          - label: single_node_opt_false
            env: "ZO_FEATURE_SINGLE_NODE_OPTIMIZE_ENABLED=false"
            mode: "false"
          - label: single_node_opt_true
            env: "ZO_FEATURE_SINGLE_NODE_OPTIMIZE_ENABLED=true"
            mode: "true"
    permissions:
      contents: read
    steps:
      - name: Remove unused tools
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"

      - name: Checkout git repo
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - name: Setup Rust Toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: nightly-2026-05-20

      - name: Setup Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true
          shared-key: backend-debug-mimalloc
          save-if: false  # main api_integration_tests job writes the cache

      - name: Install Protoc
        run: bash ./.github/protoc.sh

      - name: Build OpenObserve debug binary
        run: cargo build --features mimalloc

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11.6"

      - name: Setup rye
        uses: eifinger/setup-rye@v4
        with:
          enable-cache: true
          working-directory: "tests/api-testing/"
          version: "0.27.0"

      - name: Pin cpython
        run: rye pin cpython@3.11.6
        working-directory: tests/api-testing/

      - name: Rye sync
        run: rye sync
        working-directory: tests/api-testing/

      - name: Regenerate query-agent expected values
        run: |
          cd ${{ github.workspace }}/tests/test-data/query-agent
          ${{ github.workspace }}/tests/api-testing/.venv/bin/python compute_counts.py

      - name: Run query agent tests
        run: |
          ZO_MAX_FILE_RETENTION_TIME=1 ${{ matrix.config.env }} ${{ github.workspace }}/target/debug/openobserve > o2_query_agent_${{ matrix.config.label }}.log 2>&1 &
          SERVER_PID=$!
          for i in $(seq 1 60); do
            if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
              echo "Server ready after ${i}s"; break
            fi
            sleep 1
          done
          # SINGLE_NODE_OPT_MODE controls which queries load_all_queries() includes
          SINGLE_NODE_OPT_MODE=${{ matrix.config.mode }} rye run pytest tests/query_agent/ --ignore=tests/query_agent/test_3_vortex.py -v --junitxml=junit-queryagent-${{ matrix.config.label }}.xml
          kill $SERVER_PID || true
        working-directory: tests/api-testing/

      # Test-count metrics sidecar (see integration job). Parser dedups the two
      # opt-mode legs by test identity so the same suite isn't counted twice.
      - name: Upload API junit (query_agent)
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-junit__query_agent_tests__${{ matrix.config.label }}
          path: tests/api-testing/junit-queryagent-${{ matrix.config.label }}.xml
          if-no-files-found: warn
          retention-days: 7

      - name: Upload server log on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: openobserve-query-agent-${{ matrix.config.label }}-log
          path: tests/api-testing/o2_query_agent_${{ matrix.config.label }}.log
          if-no-files-found: warn
          retention-days: 14

  # Phase 6.b: regression matrix split. The 4 env-flag-sensitive test
  # configurations now run as 4 parallel jobs instead of 4 sequential
  # restarts in one job. Each job builds OO once and runs its single
  # configured pytest target.
  api_regression_tests:
    name: "api_regression / ${{ matrix.config.label }}"
    needs: [check_changes]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubicloud-standard-4
    permissions:
      contents: read  # only need to checkout the repo
    strategy:
      fail-fast: false
      matrix:
        config:
          - label: utf8_view_false
            env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true ZO_UTF8_VIEW_ENABLED=false"
            target: tests/regression/test_subquery_join_match.py
          - label: utf8_view_true
            env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true ZO_UTF8_VIEW_ENABLED=true"
            target: tests/regression/test_subquery_join_match.py
          - label: join_match_one_true
            env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true"
            target: tests/regression/test_logs_to_logs_join.py
          - label: join_match_one_false
            env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=false"
            target: tests/regression/test_logs_to_logs_join.py
    steps:
      - name: Remove unused tools
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"

      - uses: actions/checkout@v5
        with: { fetch-depth: 0 }

      - name: Setup Rust Toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: nightly-2026-05-20

      - name: Setup Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true
          shared-key: backend-debug-mimalloc
          save-if: false  # main api_integration_tests job writes the cache

      - name: Install Protoc
        run: bash ./.github/protoc.sh

      - name: Build OpenObserve debug binary
        run: cargo build --features mimalloc

      - uses: actions/setup-python@v5
        with: { python-version: "3.11.6" }

      - name: Setup rye
        uses: eifinger/setup-rye@v4
        with:
          enable-cache: true
          working-directory: "tests/api-testing/"
          version: "0.27.0"

      - name: Pin cpython
        run: rye pin cpython@3.11.6
        working-directory: tests/api-testing/

      - name: Rye sync
        run: rye sync
        working-directory: tests/api-testing/

      - name: "Run ${{ matrix.config.label }} regression test"
        # Server binary lives at the repo root (target/debug/openobserve);
        # this step uses working-directory: tests/api-testing/, so we must
        # invoke it via ${{ github.workspace }} or the shell will fail with
        # "No such file or directory". --force-sugar omitted on purpose —
        # pytest-sugar auto-loads via entry point, and passing the flag
        # registers it twice and inflates the progress counter ~3x.
        run: |
          ${{ matrix.config.env }} ${{ github.workspace }}/target/debug/openobserve > "o2_${{ matrix.config.label }}.log" 2>&1 &
          SERVER_PID=$!
          for i in $(seq 1 60); do
            if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
              echo "Server ready after ${i}s"; break
            fi
            sleep 1
          done
          rye run pytest ${{ matrix.config.target }} -v --junitxml=junit-regression-${{ matrix.config.label }}.xml
          kill $SERVER_PID || true
        working-directory: tests/api-testing/

      # Test-count metrics sidecar (see integration job). Parser dedups the
      # flag-combo legs that re-run the same target files.
      - name: Upload API junit (regression)
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-junit__api_integration_tests__regression_${{ matrix.config.label }}
          path: tests/api-testing/junit-regression-${{ matrix.config.label }}.xml
          if-no-files-found: warn
          retention-days: 7

      - name: Upload server log on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: "openobserve-regression-${{ matrix.config.label }}-log"
          path: "tests/api-testing/o2_${{ matrix.config.label }}.log"
          if-no-files-found: warn
          retention-days: 14

  # Phase 5: schemathesis fuzz job — informational (continue-on-error),
  # surfaces 5xx panics across the 150 OpenAPI endpoints. Decoupled from
  # api_integration_tests so its findings don't block merges yet; teams
  # can promote it to blocking after the noise floor is understood.
  api_fuzz_tests:
    name: api_fuzz_tests
    needs: [check_changes]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubicloud-standard-4
    continue-on-error: true
    permissions:
      contents: read  # only need to checkout the repo
    steps:
      - name: Remove unused tools
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"

      - name: Checkout git repo
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - name: Setup Rust Toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: nightly-2026-05-20

      - name: Setup Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true
          shared-key: backend-debug-mimalloc
          save-if: false  # don't write cache from this job (the main job does)

      - name: Install Protoc
        run: bash ./.github/protoc.sh

      - name: Build OpenObserve debug binary
        run: cargo build --features mimalloc

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11.6"

      - name: Setup rye
        uses: eifinger/setup-rye@v4
        with:
          enable-cache: true
          working-directory: "tests/api-testing/"
          version: "0.27.0"

      - name: Pin cpython
        run: rye pin cpython@3.11.6
        working-directory: tests/api-testing/

      - name: Rye sync (schemathesis dep installed)
        run: rye sync
        working-directory: tests/api-testing/

      - name: Start OpenObserve
        run: target/debug/openobserve > o2_fuzz.log 2>&1 &

      - name: Wait for /healthz
        run: |
          for i in $(seq 1 60); do
            if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then exit 0; fi
            sleep 1
          done
          exit 1

      - name: Run schemathesis fuzz tests
        # Schemathesis 4.x CLI flag renames vs the 3.x originals:
        #   --base-url               -> --url / -u
        #   --auth-type basic        -> removed (USER:PASS implies basic)
        #   --hypothesis-max-examples -> --max-examples / -n
        #   --hypothesis-deadline    -> no equivalent; use --request-timeout
        #                               (seconds, not ms) to bound per-request
        #   --data-generation-method -> --mode / -m
        # `--checks not_a_server_error` keeps focus on 5xx panics (actual
        # backend bugs, not spec-conformance nits). `-n 20` bounds runtime;
        # `--mode positive` keeps inputs realistic (negative inputs are a
        # separate exercise once the noise floor here is understood).
        run: |
          rye run schemathesis run \
            -u http://localhost:5080 \
            --auth "$ZO_ROOT_USER_EMAIL:$ZO_ROOT_USER_PASSWORD" \
            --checks not_a_server_error \
            -n 20 \
            --request-timeout 5 \
            -m positive \
            --suppress-health-check=filter_too_much \
            --exclude-path "/api/{org_id}/v1/logs" \
            --exclude-path "/api/{org_id}/v1/metrics" \
            --exclude-path "/api/{org_id}/v1/traces" \
            --exclude-path "/api/{org_id}/loki/api/v1/push" \
            --exclude-path "/api/{org_id}/prometheus/api/v1/write" \
            --exclude-path "/api/{org_id}/dashboards" \
            --exclude-path "/api/{org_id}/dashboards/{dashboard_id}" \
            --exclude-path "/api/{org_id}/dashboards/{dashboard_id}/panels" \
            --exclude-path "/api/{org_id}/dashboards/{dashboard_id}/panels/{panel_id}" \
            --exclude-path "/api/{org_id}/pipelines/history" \
            --exclude-path "/api/{org_id}/organizations/assume_service_account" \
            http://localhost:5080/api-doc/openapi.json
        working-directory: tests/api-testing/

      - name: Upload OpenObserve server log on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: openobserve-fuzz-log
          path: tests/api-testing/o2_fuzz.log
          if-no-files-found: warn
          retention-days: 14

  api_tests_summary:
    runs-on: ubuntu-latest
    permissions: {}
    needs: [check_changes, api_integration_tests, api_query_agent_tests, api_regression_tests]
    if: always()
    steps:
      - name: Check test results
        run: |
          if [ "${{ needs.check_changes.result }}" != "success" ]; then
            echo "check_changes job did not succeed: ${{ needs.check_changes.result }}"
            exit 1
          fi
          # All three test jobs must pass (or be skipped).
          ok() { [ "$1" == "success" ] || [ "$1" == "skipped" ]; }
          if ok "${{ needs.api_integration_tests.result }}" && ok "${{ needs.api_query_agent_tests.result }}" && ok "${{ needs.api_regression_tests.result }}"; then
            echo "All API test jobs passed or skipped"
            exit 0
          else
            echo "API tests failed: integration=${{ needs.api_integration_tests.result }}, query_agent=${{ needs.api_query_agent_tests.result }}, regression=${{ needs.api_regression_tests.result }}"
            exit 1
          fi

  # Non-fatal metrics sidecar: one retry-aware run-summary doc to OpenObserve (stream ci_test_runs,
  # suite=api). Re-runs append one doc per attempt; the dashboard keeps the latest attempt per run_id.
  report_to_openobserve:
    name: Report run metrics to OpenObserve
    needs: [check_changes, api_integration_tests, api_query_agent_tests, api_regression_tests]
    if: always()
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
    steps:
      - name: Clone the current repo
        uses: actions/checkout@v5
      # Pull every per-shard junit. NO merge-multiple: each artifact lands in its
      # own subdir (api-junit__<category>__<leg>) so the parser can read the test
      # CATEGORY from the dir name. Missing artifacts are non-fatal.
      - name: Download API junit reports
        if: always()
        continue-on-error: true
        uses: actions/download-artifact@v4
        with:
          pattern: api-junit__*
          path: junit-xml
      - name: Report run metrics
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
          O2_REPORTING_INGEST_BASE: ${{ secrets.O2_REPORTING_INGEST_BASE }}
          O2_REPORTING_AUTH: ${{ secrets.O2_REPORTING_AUTH }}
          O2_REPORTING_INSECURE: ${{ vars.O2_REPORTING_INSECURE }}
          INTEG: ${{ needs.api_integration_tests.result }}
          AGENT: ${{ needs.api_query_agent_tests.result }}
          REG: ${{ needs.api_regression_tests.result }}
        run: |
          if [ "$INTEG" = "skipped" ] && [ "$AGENT" = "skipped" ] && [ "$REG" = "skipped" ]; then
            echo "::notice::no API tests ran (no relevant changes) — skipping metrics"; exit 0
          fi
          if [ "$INTEG" = "failure" ] || [ "$AGENT" = "failure" ] || [ "$REG" = "failure" ]; then CONCLUSION=failure
          elif [ "$INTEG" = "cancelled" ] || [ "$AGENT" = "cancelled" ] || [ "$REG" = "cancelled" ]; then CONCLUSION=cancelled
          else CONCLUSION=success; fi
          export CONCLUSION STREAM=ci_test_runs SUITE=api
          # Parse junit into per-category + total counts. The TOTAL folds into the
          # run doc (EXTRA_JSON) so suite=api shows up in the same KPI/pie panels
          # as UI; the per-category rows go to ci_test_counts for the per-shard
          # breakdown (the API analog of UI's ci_test_modules). All best-effort.
          if [ -d junit-xml ]; then
            COUNTS=$(python3 .github/scripts/parse-api-junit.py junit-xml 2>/dev/null || echo "")
            if [ -n "$COUNTS" ] && printf '%s' "$COUNTS" | jq -e . >/dev/null 2>&1; then
              echo "API counts: $COUNTS"
              EXTRA_JSON=$(printf '%s' "$COUNTS" | jq -c .total); export EXTRA_JSON
              printf '%s' "$COUNTS" | jq -c --arg run_id "$GITHUB_RUN_ID" --arg repo "$GITHUB_REPOSITORY" \
                '[.by_category[] | {run_id:$run_id, repo:$repo, suite:"api", workflow:"test", ingest_source:"live",
                  module:.module, tests_total, tests_passed, tests_failed, tests_flaky, tests_skipped}]' > api_counts_rows.json
              if [ "$(jq 'length' api_counts_rows.json 2>/dev/null || echo 0)" -gt 0 ]; then
                bash .github/scripts/o2-report.sh ci_test_counts api_counts_rows.json
              fi
            fi
          fi
          bash .github/scripts/o2-run-report.sh