openobserve/.github/workflows/api-testing.yml

name: API Integration Test

on:
  push:
    branches:
      - "main"
  pull_request:
    branches:
      - "*"
  merge_group:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.merge_group.head_ref || github.ref }}
  cancel-in-progress: true

env:
  COLUMNS: 150
  ZO_ROOT_USER_EMAIL: root@example.com
  ZO_ROOT_USER_PASSWORD: Complexpass#123
  ZO_BASE_URL: http://localhost:5080/
  ZO_BASE_URL_SC: http://localhost:5080/
  WS_ZO_BASE_URL: ws://localhost:5080/
  ZO_USAGE_REPORTING_ENABLED: true
  ZO_USAGE_PUBLISH_INTERVAL: 5
  ZO_ALERT_SCHEDULE_INTERVAL: 3
  ZO_SMTP_ENABLED: true
  ZO_CREATE_ORG_THROUGH_INGESTION: true
  ZO_INGEST_ALLOWED_UPTO: 48  # Allow backdated data ingestion for backfill tests
  ZO_SSRF_ALLOW_LOOPBACK: true  # Allow localhost destinations in API tests

jobs:
  check_changes:
    name: check_changes
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      has_changes: ${{ steps.check.outputs.has_changes }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - id: check
        run: |
          CHANGED=$(git diff --name-only origin/${{ github.base_ref || 'main' }} ${{ github.sha }})
          if [ -z "$CHANGED" ]; then
            echo "has_changes=true" >> $GITHUB_OUTPUT
          elif echo "$CHANGED" | grep -qvE '(\.md$|^docs/|^web/|^tests/ui-testing/|^tests/playwright-tests/)'; then
            echo "has_changes=true" >> $GITHUB_OUTPUT
          else
            echo "has_changes=false" >> $GITHUB_OUTPUT
          fi

  api_integration_tests:
    name: api_integration_tests
    needs: [check_changes]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubicloud-standard-8
    steps:
      - name: Remove unused tools
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"

      - name: Checkout git repo
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - name: Setup Rust Toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: nightly-2025-11-11

      - name: Setup Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true
          shared-key: backend-debug-mimalloc
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Install Protoc
        run: |
          bash ./.github/protoc.sh

      - name: Build OpenObserve debug binary
        run: cargo build --features mimalloc

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11.6"

      - name: Setup rye
        uses: eifinger/setup-rye@v4
        with:
          enable-cache: true
          working-directory: "tests/api-testing/"
          version: "0.27.0"

      - name: Start OpenObserve
        run: target/debug/openobserve &

      - name: Wait for start
        run: |
          for i in $(seq 1 60); do
            if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
              echo "Server ready after ${i}s"
              exit 0
            fi
            sleep 1
          done
          echo "Server failed to start in 60s"
          exit 1

      - name: Pin cpython
        run: rye pin cpython@3.11.6
        working-directory: tests/api-testing/

      - name: Rye sync
        run: rye sync
        working-directory: tests/api-testing/

      - name: Run tests
        run: |
          wait_for_server() {
            for i in $(seq 1 60); do
              if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
                echo "Server ready after ${i}s"
                return 0
              fi
              sleep 1
            done
            echo "Server failed to start in 60s"
            return 1
          }

          echo "=== Running regression tests with ZO_UTF8_VIEW_ENABLED=false ==="
          ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true ZO_UTF8_VIEW_ENABLED=false target/debug/openobserve > o2_utf8_false.log 2>&1 &
          SERVER_PID_FALSE=$!
          echo "OpenObserve PID for UTF8_VIEW=false: $SERVER_PID_FALSE"
          wait_for_server

          # Run regression tests with UTF8_VIEW=false
          rye run pytest tests/test_subquery_join_match.py --force-sugar -v

          # Kill the server
          kill $SERVER_PID_FALSE || true
          sleep 5

          echo "=== Running regression tests with ZO_UTF8_VIEW_ENABLED=true ==="
          ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true ZO_UTF8_VIEW_ENABLED=true target/debug/openobserve > o2_utf8_true.log 2>&1 &
          SERVER_PID_TRUE=$!
          echo "OpenObserve PID for UTF8_VIEW=true: $SERVER_PID_TRUE"
          wait_for_server

          # Run regression tests with UTF8_VIEW=true
          rye run pytest tests/test_subquery_join_match.py --force-sugar -v

          # Kill the server
          kill $SERVER_PID_TRUE || true
          sleep 5

          echo "=== Running logs-to-logs join tests with ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true ==="
          ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true target/debug/openobserve > o2_join_match_one_true.log 2>&1 &
          SERVER_PID_JOIN_TRUE=$!
          echo "OpenObserve PID for JOIN_MATCH_ONE=true: $SERVER_PID_JOIN_TRUE"
          wait_for_server

          # Run logs-to-logs join tests with JOIN_MATCH_ONE=true
          rye run pytest tests/test_logs_to_logs_join.py --force-sugar -v

          # Kill the server
          kill $SERVER_PID_JOIN_TRUE || true
          sleep 5

          echo "=== Running logs-to-logs join tests with ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=false ==="
          ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=false target/debug/openobserve > o2_join_match_one_false.log 2>&1 &
          SERVER_PID_JOIN_FALSE=$!
          echo "OpenObserve PID for JOIN_MATCH_ONE=false: $SERVER_PID_JOIN_FALSE"
          wait_for_server

          # Run logs-to-logs join tests with JOIN_MATCH_ONE=false
          rye run pytest tests/test_logs_to_logs_join.py --force-sugar -v

          # Kill the server
          kill $SERVER_PID_JOIN_FALSE || true
          sleep 5

          echo "=== Running all other API tests with default settings ==="
          target/debug/openobserve > o2_default.log 2>&1 &
          SERVER_PID_DEFAULT=$!
          echo "OpenObserve PID for other tests: $SERVER_PID_DEFAULT"
          wait_for_server

          # Run all other tests (excluding regression and feature flag tests)
          rye run pytest --ignore=tests/test_subquery_join_match.py --ignore=tests/test_logs_to_logs_join.py --force-sugar

          # Kill the server
          kill $SERVER_PID_DEFAULT || true
        working-directory: tests/api-testing/

  api_tests_summary:
    runs-on: ubuntu-latest
    permissions: {}
    needs: [check_changes, api_integration_tests]
    if: always()
    steps:
      - name: Check test results
        run: |
          if [ "${{ needs.check_changes.result }}" != "success" ]; then
            echo "check_changes job did not succeed: ${{ needs.check_changes.result }}"
            exit 1
          fi
          if [ "${{ needs.api_integration_tests.result }}" == "success" ] || [ "${{ needs.api_integration_tests.result }}" == "skipped" ]; then
            echo "API tests passed or skipped"
            exit 0
          else
            echo "API tests failed"
            exit 1
          fi