mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-04 16:09:05 +00:00
434 lines
20 KiB
YAML
434 lines
20 KiB
YAML
name: API Bug-Fix Regression Tests
|
|
|
|
# The per-PR suite (api-testing.yml) only runs on a labelled, non-draft PR whose
|
|
# diff touches backend or tests/api-testing. Nothing re-runs the API suite against
|
|
# main afterwards, so a regression that lands via an unrelated path — a dependency
|
|
# bump, a merge-queue interleaving, a change classified as UI-only — is invisible
|
|
# until the next backend PR happens to run it. This is the API sibling of
|
|
# playwright_bugfix_regression.yml: same tests, same flag matrix, run against main
|
|
# on a schedule, with a tracking issue as the notification.
|
|
|
|
on:
|
|
schedule:
|
|
# Offset 1h from the Playwright regression crons so the two do not contend for runners.
|
|
- cron: '30 18 * * *' # 12:00 AM IST
|
|
- cron: '30 0 * * *' # 6:00 AM IST
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: false
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Mirrors api-testing.yml's env block; the suite asserts against several of these
|
|
# (ZO_INGEST_ALLOWED_UPTO gates the backfill tests, ZO_SSRF_ALLOW_LOOPBACK the
|
|
# destination tests), so they are not optional decoration.
|
|
env:
|
|
COLUMNS: 150
|
|
ZO_ROOT_USER_EMAIL: root@example.com
|
|
ZO_ROOT_USER_PASSWORD: Complexpass#123
|
|
ZO_BASE_URL: http://localhost:5080/
|
|
ZO_BASE_URL_SC: http://localhost:5080/
|
|
WS_ZO_BASE_URL: ws://localhost:5080/
|
|
ZO_USAGE_PUBLISH_INTERVAL: 5
|
|
ZO_ALERT_SCHEDULE_INTERVAL: 3
|
|
ZO_SMTP_ENABLED: true
|
|
ZO_CREATE_ORG_THROUGH_INGESTION: true
|
|
ZO_INGEST_ALLOWED_UPTO: 48 # Allow backdated data ingestion for backfill tests
|
|
ZO_SSRF_ALLOW_LOOPBACK: true # Allow localhost destinations in API tests
|
|
|
|
jobs:
|
|
# Skip the whole suite when main has not moved since the last regression run:
|
|
# rebuilding and re-running on identical code is pure waste. Manual dispatch
|
|
# always runs. Same contract as playwright_bugfix_regression.yml.
|
|
check_main_changed:
|
|
timeout-minutes: 5
|
|
name: check_main_changed
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
actions: read # for `gh run list` (last run's head_sha)
|
|
outputs:
|
|
should_run: ${{ steps.decide.outputs.should_run }}
|
|
steps:
|
|
- id: decide
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
# gh resolves the repo from --repo, GH_REPO, then the git remote — it does NOT
|
|
# read GITHUB_REPOSITORY. This job has no checkout, so without GH_REPO every
|
|
# `gh run list` fails, LAST_SHA stays empty and the gate falls through to "run".
|
|
GH_REPO: ${{ github.repository }}
|
|
run: |
|
|
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
|
echo "should_run=true" >> "$GITHUB_OUTPUT"
|
|
echo "::notice::Manual dispatch — running regardless of main changes."
|
|
exit 0
|
|
fi
|
|
# On a scheduled run github.sha is main's tip, so no checkout is needed.
|
|
CURRENT_SHA="${{ github.sha }}"
|
|
# head_sha of the most recent completed previous run of THIS workflow on main,
|
|
# excluding the current run. On any failure LAST_SHA stays empty and we fall
|
|
# through to should_run=true (fail-safe: run rather than skip).
|
|
LAST_SHA=$(gh run list \
|
|
--workflow "${{ github.workflow }}" \
|
|
--branch main \
|
|
--limit 50 \
|
|
--json databaseId,headSha,status \
|
|
--jq "[.[] | select(.databaseId != ${{ github.run_id }} and .status == \"completed\")] | .[0].headSha" \
|
|
2>/dev/null || echo "")
|
|
if [ -z "$LAST_SHA" ]; then
|
|
echo "::warning::Could not determine last run SHA (no prior completed run, or gh query failed) — defaulting to run."
|
|
fi
|
|
echo "Current main SHA: ${CURRENT_SHA:-<none>}"
|
|
echo "Last run SHA: ${LAST_SHA:-<none>}"
|
|
if [ -n "$LAST_SHA" ] && [ "$CURRENT_SHA" = "$LAST_SHA" ]; then
|
|
echo "should_run=false" >> "$GITHUB_OUTPUT"
|
|
echo "::notice::main unchanged since last regression run ($CURRENT_SHA) — skipping."
|
|
else
|
|
echo "should_run=true" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
build_api_binary:
|
|
timeout-minutes: 60
|
|
name: build_api_binary
|
|
needs: [check_main_changed]
|
|
if: ${{ needs.check_main_changed.outputs.should_run == 'true' }}
|
|
runs-on: ubicloud-standard-4
|
|
env:
|
|
# CI executes a stripped binary; omitting dependency debuginfo keeps the
|
|
# shared compile cache small while preserving dev-profile debug assertions.
|
|
CARGO_PROFILE_DEV_DEBUG: "0"
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- name: Remove unused tools
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf "/usr/local/share/boost"
|
|
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
|
|
|
- name: Checkout git repo
|
|
uses: actions/checkout@v5
|
|
|
|
- uses: ./.github/actions/github-auth
|
|
- name: Setup Rust Toolchain
|
|
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
|
|
with:
|
|
toolchain: nightly-2026-10-24
|
|
|
|
- name: Setup Rust Cache
|
|
uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
|
|
with:
|
|
# Read the same dependency cache api-testing.yml reads; the DB workflow's
|
|
# push:main build is the only writer, so this job never saves.
|
|
shared-key: backend-debug0-mimalloc-deps-v1
|
|
add-job-id-key: false
|
|
save-if: false
|
|
|
|
- name: Install Protoc
|
|
run: bash ./.github/protoc.sh
|
|
|
|
- name: Build OpenObserve debug binary
|
|
run: cargo build --features mimalloc
|
|
|
|
- name: Package OpenObserve debug binary
|
|
run: |
|
|
strip target/debug/openobserve
|
|
tar -C target/debug -czf openobserve-api-debug.tar.gz openobserve
|
|
|
|
- name: Upload OpenObserve debug binary
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: openobserve-api-debug-${{ github.sha }}
|
|
path: openobserve-api-debug.tar.gz
|
|
if-no-files-found: error
|
|
retention-days: 3
|
|
# The payload is already gzip-compressed; avoid a second compression pass.
|
|
compression-level: 0
|
|
|
|
# One matrix instead of api-testing.yml's three near-identical jobs: the legs differ
|
|
# only in the server env, the pytest target and the pytest env, so they are matrix
|
|
# fields. `category` feeds the junit artifact name that parse-api-junit.py reads.
|
|
api_regression_tests:
|
|
timeout-minutes: 60
|
|
name: "api_regression / ${{ matrix.config.label }}"
|
|
needs: [check_main_changed, build_api_binary]
|
|
if: ${{ needs.check_main_changed.outputs.should_run == 'true' }}
|
|
runs-on: eks-openobserve-standard-4
|
|
permissions:
|
|
actions: read
|
|
contents: read
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
config:
|
|
# The default set — everything under tests/ except the three dirs that
|
|
# need their own server configuration. tests/ingest/ lives here, so the
|
|
# OTLP UTF-8 repair cases run on every scheduled pass.
|
|
- label: default_set
|
|
category: api_integration_tests
|
|
server_env: ""
|
|
test_env: ""
|
|
target: "--ignore=tests/regression --ignore=tests/query_agent --ignore=tests/vortex"
|
|
# Flag-sensitive query-engine cases: each file is meaningless unless the
|
|
# server is restarted under the paired flag, which is why they sit outside
|
|
# the default set.
|
|
- label: utf8_view_false
|
|
category: api_integration_tests
|
|
server_env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true ZO_UTF8_VIEW_ENABLED=false"
|
|
test_env: ""
|
|
target: "tests/regression/test_subquery_join_match.py"
|
|
- label: utf8_view_true
|
|
category: api_integration_tests
|
|
server_env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true ZO_UTF8_VIEW_ENABLED=true"
|
|
test_env: ""
|
|
target: "tests/regression/test_subquery_join_match.py"
|
|
- label: join_match_one_true
|
|
category: api_integration_tests
|
|
server_env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true"
|
|
test_env: ""
|
|
target: "tests/regression/test_logs_to_logs_join.py"
|
|
- label: join_match_one_false
|
|
category: api_integration_tests
|
|
server_env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=false"
|
|
test_env: ""
|
|
target: "tests/regression/test_logs_to_logs_join.py"
|
|
# SINGLE_NODE_OPT_MODE decides which queries load_all_queries() includes:
|
|
# the non-optimized RemoteScanRewriter path trips known upstream DataFusion
|
|
# bugs, and those queries carry skip_without_single_node_opt.
|
|
- label: single_node_opt_false
|
|
category: query_agent_tests
|
|
server_env: "ZO_MAX_FILE_RETENTION_TIME=1 ZO_FEATURE_SINGLE_NODE_OPTIMIZE_ENABLED=false"
|
|
test_env: "SINGLE_NODE_OPT_MODE=false"
|
|
target: "tests/query_agent/ --ignore=tests/query_agent/test_3_vortex.py"
|
|
- label: single_node_opt_true
|
|
category: query_agent_tests
|
|
server_env: "ZO_MAX_FILE_RETENTION_TIME=1 ZO_FEATURE_SINGLE_NODE_OPTIMIZE_ENABLED=true"
|
|
test_env: "SINGLE_NODE_OPT_MODE=true"
|
|
target: "tests/query_agent/ --ignore=tests/query_agent/test_3_vortex.py"
|
|
- label: remove_filter_with_index_false
|
|
category: query_agent_tests
|
|
server_env: "ZO_MAX_FILE_RETENTION_TIME=1 ZO_FEATURE_QUERY_REMOVE_FILTER_WITH_INDEX=false"
|
|
test_env: "SINGLE_NODE_OPT_MODE=true"
|
|
target: "tests/query_agent/ --ignore=tests/query_agent/test_3_vortex.py"
|
|
steps:
|
|
- name: Checkout git repo
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Download OpenObserve debug binary
|
|
uses: actions/download-artifact@v5
|
|
with:
|
|
name: openobserve-api-debug-${{ github.sha }}
|
|
path: api-binary-artifact
|
|
|
|
- name: Extract OpenObserve debug binary
|
|
run: |
|
|
mkdir -p target/debug
|
|
tar -xzf api-binary-artifact/openobserve-api-debug.tar.gz -C target/debug
|
|
chmod +x target/debug/openobserve
|
|
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v7
|
|
id: setup-python
|
|
with:
|
|
python-version: "3.11.6"
|
|
|
|
- name: Setup rye
|
|
uses: eifinger/setup-rye@c694239a43768373e87d0103d7f547027a23f3c8 # v4.2.9
|
|
env:
|
|
# Reuse setup-python's interpreter so `rye self install` skips the flaky toolchain download.
|
|
RYE_TOOLCHAIN: ${{ steps.setup-python.outputs.python-path }}
|
|
with:
|
|
enable-cache: true
|
|
working-directory: "tests/api-testing/"
|
|
version: "0.27.0"
|
|
|
|
- name: Pin cpython
|
|
run: rye pin cpython@3.11.6
|
|
working-directory: tests/api-testing/
|
|
|
|
- name: Rye sync
|
|
run: rye sync
|
|
working-directory: tests/api-testing/
|
|
|
|
- name: Regenerate query-agent expected values
|
|
if: ${{ matrix.config.category == 'query_agent_tests' }}
|
|
run: |
|
|
cd ${{ github.workspace }}/tests/test-data/query-agent
|
|
${{ github.workspace }}/tests/api-testing/.venv/bin/python compute_counts.py
|
|
|
|
- name: "Run ${{ matrix.config.label }}"
|
|
# --force-sugar omitted on purpose: pytest-sugar auto-loads via its entry
|
|
# point, and passing the flag registers it twice and inflates the counter.
|
|
run: |
|
|
${{ matrix.config.server_env }} ${{ github.workspace }}/target/debug/openobserve > "o2_${{ matrix.config.label }}.log" 2>&1 &
|
|
SERVER_PID=$!
|
|
for i in $(seq 1 60); do
|
|
if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
|
|
echo "Server ready after ${i}s"; break
|
|
fi
|
|
sleep 1
|
|
done
|
|
${{ matrix.config.test_env }} rye run pytest ${{ matrix.config.target }} --junitxml=junit-${{ matrix.config.label }}.xml
|
|
kill $SERVER_PID || true
|
|
working-directory: tests/api-testing/
|
|
|
|
# Artifact name is a contract: parse-api-junit.py reads the CATEGORY from the
|
|
# part between the `__`s, and collapses legs that re-run the same tests.
|
|
- name: Upload API junit
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: api-junit__${{ matrix.config.category }}__${{ matrix.config.label }}
|
|
path: tests/api-testing/junit-${{ matrix.config.label }}.xml
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
|
|
- name: Upload OpenObserve server log on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: openobserve-${{ matrix.config.label }}-log
|
|
path: tests/api-testing/o2_${{ matrix.config.label }}.log
|
|
if-no-files-found: warn
|
|
retention-days: 14
|
|
|
|
api_regression_summary:
|
|
name: api_regression_summary
|
|
needs: [check_main_changed, build_api_binary, api_regression_tests]
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
permissions: {}
|
|
steps:
|
|
- name: Check results
|
|
run: |
|
|
if [ "${{ needs.check_main_changed.outputs.should_run }}" != "true" ]; then
|
|
echo "✅ main unchanged since the last run — nothing to verify."
|
|
exit 0
|
|
fi
|
|
if [ "${{ needs.build_api_binary.result }}" == "success" ] && \
|
|
[ "${{ needs.api_regression_tests.result }}" == "success" ]; then
|
|
echo "✅ All API regression legs passed"
|
|
exit 0
|
|
fi
|
|
echo "API regression failed: build=${{ needs.build_api_binary.result }}, tests=${{ needs.api_regression_tests.result }}"
|
|
exit 1
|
|
|
|
# GitHub emails assignees and @-mentioned users, so a tracking issue is the
|
|
# notification: one open issue per breakage, a comment per further failing run,
|
|
# closed by the next green run. Its own label so it never closes the Playwright one.
|
|
report_result:
|
|
name: Report result
|
|
needs: [check_main_changed, api_regression_tests, api_regression_summary]
|
|
# Gated on the GATE, not on the test job: a failed build_api_binary skips
|
|
# api_regression_tests, and keying off that would turn the loudest failure
|
|
# mode into silence. should_run tells "nothing to do" apart from "it broke".
|
|
if: ${{ always() && needs.check_main_changed.outputs.should_run == 'true' }}
|
|
timeout-minutes: 5
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
issues: write
|
|
actions: read
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
GH_REPO: ${{ github.repository }}
|
|
LABEL: ci-api-regression-failure
|
|
NOTIFY: hengfeiyang,oasisk,Shrinath-O2,neha00290,vamsikarthikr
|
|
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
RESULT: ${{ needs.api_regression_summary.result }}
|
|
steps:
|
|
- name: Open or update the failure issue, or close it on green
|
|
run: |
|
|
OPEN=$(gh issue list --label "$LABEL" --state open --json number --jq '.[0].number // ""')
|
|
if [ "$RESULT" = "success" ]; then
|
|
if [ -n "$OPEN" ]; then
|
|
gh issue close "$OPEN" --comment "API regression is green again on main (${{ github.sha }}): $RUN_URL"
|
|
fi
|
|
exit 0
|
|
fi
|
|
FAILED=$(gh api --paginate "repos/$GH_REPO/actions/runs/${{ github.run_id }}/jobs?per_page=100" \
|
|
--jq '[.jobs[] | select(.name | startswith("api_regression /")) | select(.conclusion == "failure" or .conclusion == "timed_out") | .name] | join(", ")')
|
|
MENTIONS=$(echo "$NOTIFY" | sed 's/[^,]*/@&/g; s/,/ /g')
|
|
BODY="API regression failed on main at ${{ github.sha }}.
|
|
Failed legs: ${FAILED:-none listed (build or summary failed)}
|
|
Run: $RUN_URL
|
|
|
|
$MENTIONS"
|
|
if [ -n "$OPEN" ]; then
|
|
gh issue comment "$OPEN" --body "$BODY"
|
|
else
|
|
gh label create "$LABEL" --color B60205 --description "Scheduled API regression on main is failing" 2>/dev/null || true
|
|
gh issue create --title "API regression failing on main" --label "$LABEL" --assignee "$NOTIFY" --body "$BODY"
|
|
fi
|
|
|
|
# Non-fatal metrics sidecar: same streams and the same junit parser as
|
|
# api-testing.yml, tagged workflow=regression so the scheduled pass is
|
|
# distinguishable from the per-PR one. Never fails the run.
|
|
report_to_openobserve:
|
|
name: Report run metrics to OpenObserve
|
|
needs: [check_main_changed, api_regression_tests, api_regression_summary]
|
|
if: ${{ always() && needs.check_main_changed.outputs.should_run == 'true' }}
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
actions: read
|
|
contents: read
|
|
steps:
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
|
|
# NO merge-multiple: each artifact must land in its own subdir so the parser
|
|
# can read the category from the dir name. Missing artifacts are non-fatal.
|
|
- name: Download API junit reports
|
|
if: always()
|
|
continue-on-error: true
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
pattern: api-junit__*
|
|
path: junit-xml
|
|
|
|
- name: Report run metrics
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
O2_REPORTING_INGEST_BASE: ${{ secrets.O2_REPORTING_INGEST_BASE }}
|
|
O2_REPORTING_AUTH: ${{ secrets.O2_REPORTING_AUTH }}
|
|
O2_REPORTING_INSECURE: ${{ vars.O2_REPORTING_INSECURE }}
|
|
# The SUMMARY, not the test job: this job now also runs when the build failed,
|
|
# and in that case the test job is `skipped` — which read as success and would
|
|
# post a broken run to ci_test_runs as green. Anything that is not an explicit
|
|
# success or cancellation counts as a failure, so the metric errs loud.
|
|
RESULT: ${{ needs.api_regression_summary.result }}
|
|
run: |
|
|
set -uo pipefail
|
|
if [ "$RESULT" = "success" ]; then CONCLUSION=success
|
|
elif [ "$RESULT" = "cancelled" ]; then CONCLUSION=cancelled
|
|
else CONCLUSION=failure; fi
|
|
export CONCLUSION STREAM=ci_test_runs SUITE=api
|
|
# o2-run-report.sh derives its workflow tag from the STREAM name alone
|
|
# (regression only for ci_regression), so a scheduled run posting to
|
|
# ci_test_runs would be indistinguishable from a per-PR api-testing run and
|
|
# would pollute the PR-CI duration and pass-rate panels. EXTRA_JSON keys win
|
|
# the merge, so override it here. The default stands unless the counts merge
|
|
# BOTH succeeds and produces output — an empty EXTRA_JSON is skipped by the
|
|
# reporter, which would silently drop the tag back to "test". It must agree
|
|
# with the resync entry, which registers this workflow as is_regression=True.
|
|
EXTRA_JSON='{"workflow":"regression"}'
|
|
if [ -d junit-xml ]; then
|
|
COUNTS=$(python3 .github/scripts/parse-api-junit.py junit-xml 2>/dev/null || echo "")
|
|
if [ -n "$COUNTS" ] && printf '%s' "$COUNTS" | jq -e . >/dev/null 2>&1; then
|
|
echo "API counts: $COUNTS"
|
|
MERGED=$(printf '%s' "$COUNTS" | jq -c '.total + {workflow:"regression"}' 2>/dev/null || echo "")
|
|
[ -n "$MERGED" ] && EXTRA_JSON="$MERGED"
|
|
printf '%s' "$COUNTS" | jq -c --arg run_id "$GITHUB_RUN_ID" --arg repo "$GITHUB_REPOSITORY" \
|
|
'[.by_category[] | {run_id:$run_id, repo:$repo, suite:"api", workflow:"regression", ingest_source:"live",
|
|
module:.module, tests_total, tests_passed, tests_failed, tests_flaky, tests_skipped}]' > api_counts_rows.json
|
|
if [ "$(jq 'length' api_counts_rows.json 2>/dev/null || echo 0)" -gt 0 ]; then
|
|
bash .github/scripts/o2-report.sh ci_test_counts api_counts_rows.json
|
|
fi
|
|
fi
|
|
fi
|
|
export EXTRA_JSON
|
|
bash .github/scripts/o2-run-report.sh
|