mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-05 08:09:22 +00:00
4f3c0fcba3
## Summary
Fixes a test gap where `match_all` Tantivy queries could return wrong
results after a DataFusion upgrade without being caught by existing
assertions.
### Problem
After a DataFusion upgrade merge, the DataFusion access plan was no
longer used for `match_all` queries, causing Tantivy to return wrong
results. The existing tests only compared row counts against a DuckDB
oracle that emulates `match_all` with `LIKE '%term%'` — so wrong rows
that still matched the LIKE pattern passed undetected.
Additionally, `node/flush` does not generate Tantivy indexes
immediately. The default `ZO_MAX_FILE_RETENTION_TIME=600s` means
WAL→parquet→Tantivy conversion can lag, so FTS queries may fall back to
a non-Tantivy path during tests.
### Changes
1. **Content verification**
([conftest.py](tests/api-testing/tests/query_agent/conftest.py)): Every
hit from a `match_all` query that returns the `log` field is now checked
— each search term word must be present in `log`. A hard assertion fails
if Tantivy returns rows that don't contain the search terms.
2. **Tantivy readiness gate**
([test_2_parquet.py](tests/api-testing/tests/query_agent/test_2_parquet.py)):
After `node/flush`, the test now polls with a `match_all('warehouse')
LIMIT 1` query (up to 120s timeout) to confirm Tantivy indexes are built
before running FTS assertions. Previously only COUNT(*) was checked.
3. **Multi-word match_all emulation**
([compute_counts.py](tests/test-data/query-agent/compute_counts.py)):
`match_all('ACK batch')` now translates to `(log LIKE '%ACK%' AND log
LIKE '%batch%')` — per-token AND conditions matching Tantivy's
token-level search semantics, instead of the old phrase match `log LIKE
'%ACK batch%'`.
4. **Regenerated expected results** across all query JSON files.
## Test plan
- [ ] Run query agent test suite against an environment with Tantivy
enabled
- [ ] Verify `match_all` content assertions catch intentionally
corrupted results
- [ ] Confirm post-flush Tantivy readiness poll works within the timeout
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---------
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: Huaijin <haohuaijin@gmail.com>
478 lines
17 KiB
YAML
478 lines
17 KiB
YAML
name: API Integration Test
|
||
|
||
on:
|
||
push:
|
||
branches:
|
||
- "main"
|
||
pull_request:
|
||
branches:
|
||
- "**"
|
||
merge_group:
|
||
|
||
concurrency:
|
||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.merge_group.head_ref || github.ref }}
|
||
cancel-in-progress: true
|
||
|
||
env:
|
||
COLUMNS: 150
|
||
ZO_ROOT_USER_EMAIL: root@example.com
|
||
ZO_ROOT_USER_PASSWORD: Complexpass#123
|
||
ZO_BASE_URL: http://localhost:5080/
|
||
ZO_BASE_URL_SC: http://localhost:5080/
|
||
WS_ZO_BASE_URL: ws://localhost:5080/
|
||
ZO_USAGE_REPORTING_ENABLED: true
|
||
ZO_USAGE_PUBLISH_INTERVAL: 5
|
||
ZO_ALERT_SCHEDULE_INTERVAL: 3
|
||
ZO_SMTP_ENABLED: true
|
||
ZO_CREATE_ORG_THROUGH_INGESTION: true
|
||
ZO_INGEST_ALLOWED_UPTO: 48 # Allow backdated data ingestion for backfill tests
|
||
ZO_SSRF_ALLOW_LOOPBACK: true # Allow localhost destinations in API tests
|
||
|
||
jobs:
|
||
check_changes:
|
||
name: check_changes
|
||
runs-on: ubuntu-latest
|
||
permissions:
|
||
contents: read
|
||
outputs:
|
||
has_changes: ${{ steps.check.outputs.has_changes }}
|
||
steps:
|
||
- uses: actions/checkout@v5
|
||
with:
|
||
fetch-depth: 0
|
||
|
||
- id: check
|
||
run: |
|
||
CHANGED=$(git diff --name-only origin/${{ github.base_ref || 'main' }} ${{ github.sha }})
|
||
if [ -z "$CHANGED" ]; then
|
||
echo "has_changes=true" >> $GITHUB_OUTPUT
|
||
elif echo "$CHANGED" | grep -qvE '(\.md$|^docs/|^web/|^tests/ui-testing/|^tests/playwright-tests/)'; then
|
||
echo "has_changes=true" >> $GITHUB_OUTPUT
|
||
else
|
||
echo "has_changes=false" >> $GITHUB_OUTPUT
|
||
fi
|
||
|
||
api_integration_tests:
|
||
name: api_integration_tests
|
||
needs: [check_changes]
|
||
if: needs.check_changes.outputs.has_changes == 'true'
|
||
runs-on: ubicloud-standard-4
|
||
permissions:
|
||
contents: read # only need to checkout the repo
|
||
steps:
|
||
- name: Remove unused tools
|
||
run: |
|
||
sudo rm -rf /usr/share/dotnet
|
||
sudo rm -rf /opt/ghc
|
||
sudo rm -rf "/usr/local/share/boost"
|
||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||
|
||
- name: Checkout git repo
|
||
uses: actions/checkout@v5
|
||
with:
|
||
fetch-depth: 0
|
||
|
||
- name: Setup Rust Toolchain
|
||
uses: dtolnay/rust-toolchain@master
|
||
with:
|
||
toolchain: nightly-2026-05-20
|
||
|
||
- name: Setup Rust Cache
|
||
uses: Swatinem/rust-cache@v2
|
||
with:
|
||
cache-on-failure: true
|
||
shared-key: backend-debug-mimalloc
|
||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||
|
||
- name: Install Protoc
|
||
run: |
|
||
bash ./.github/protoc.sh
|
||
|
||
- name: Build OpenObserve debug binary
|
||
run: cargo build --features mimalloc
|
||
|
||
- name: Setup Python
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.11.6"
|
||
|
||
- name: Setup rye
|
||
uses: eifinger/setup-rye@v4
|
||
with:
|
||
enable-cache: true
|
||
working-directory: "tests/api-testing/"
|
||
version: "0.27.0"
|
||
|
||
- name: Pin cpython
|
||
run: rye pin cpython@3.11.6
|
||
working-directory: tests/api-testing/
|
||
|
||
- name: Rye sync
|
||
run: rye sync
|
||
working-directory: tests/api-testing/
|
||
|
||
- name: Run default-set tests
|
||
# Phase 6.b: regression matrix tests moved to their own
|
||
# api_regression_tests job below — they now run in parallel
|
||
# with this default-set job instead of sequentially within it.
|
||
# Cuts ~8 min off CI wall time at the cost of more runner minutes.
|
||
# NB: --force-sugar is intentionally omitted. pytest-sugar is already
|
||
# auto-loaded via the entry point; passing --force-sugar registers it
|
||
# a second time, which inflates the progress counter (CI displayed
|
||
# ~3x the real test count). Letting the plugin load via discovery
|
||
# alone keeps the counter honest.
|
||
run: |
|
||
${{ github.workspace }}/target/debug/openobserve > o2_default.log 2>&1 &
|
||
SERVER_PID_DEFAULT=$!
|
||
echo "OpenObserve PID for default tests: $SERVER_PID_DEFAULT"
|
||
for i in $(seq 1 60); do
|
||
if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
|
||
echo "Server ready after ${i}s"; break
|
||
fi
|
||
sleep 1
|
||
done
|
||
rye run pytest --ignore=tests/regression --ignore=tests/query_agent --ignore=tests/vortex
|
||
kill $SERVER_PID_DEFAULT || true
|
||
working-directory: tests/api-testing/
|
||
|
||
- name: Upload OpenObserve server logs on failure
|
||
if: failure()
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: openobserve-server-logs
|
||
path: tests/api-testing/o2_*.log
|
||
if-no-files-found: warn
|
||
retention-days: 14
|
||
|
||
# Dedicated query agent SQL regression suite. Runs in parallel with
|
||
# the default-set and regression jobs so the 310-case (155 queries ×
|
||
# 2 phases: memtable + parquet) suite doesn't extend CI wall time.
|
||
api_query_agent_tests:
|
||
name: api_query_agent_tests
|
||
needs: [check_changes]
|
||
if: needs.check_changes.outputs.has_changes == 'true'
|
||
runs-on: ubicloud-standard-4
|
||
permissions:
|
||
contents: read
|
||
steps:
|
||
- name: Remove unused tools
|
||
run: |
|
||
sudo rm -rf /usr/share/dotnet
|
||
sudo rm -rf /opt/ghc
|
||
sudo rm -rf "/usr/local/share/boost"
|
||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||
|
||
- name: Checkout git repo
|
||
uses: actions/checkout@v5
|
||
with:
|
||
fetch-depth: 0
|
||
|
||
- name: Setup Rust Toolchain
|
||
uses: dtolnay/rust-toolchain@master
|
||
with:
|
||
toolchain: nightly-2026-05-20
|
||
|
||
- name: Setup Rust Cache
|
||
uses: Swatinem/rust-cache@v2
|
||
with:
|
||
cache-on-failure: true
|
||
shared-key: backend-debug-mimalloc
|
||
save-if: false # main api_integration_tests job writes the cache
|
||
|
||
- name: Install Protoc
|
||
run: bash ./.github/protoc.sh
|
||
|
||
- name: Build OpenObserve debug binary
|
||
run: cargo build --features mimalloc
|
||
|
||
- name: Setup Python
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.11.6"
|
||
|
||
- name: Setup rye
|
||
uses: eifinger/setup-rye@v4
|
||
with:
|
||
enable-cache: true
|
||
working-directory: "tests/api-testing/"
|
||
version: "0.27.0"
|
||
|
||
- name: Pin cpython
|
||
run: rye pin cpython@3.11.6
|
||
working-directory: tests/api-testing/
|
||
|
||
- name: Rye sync
|
||
run: rye sync
|
||
working-directory: tests/api-testing/
|
||
|
||
- name: Regenerate query-agent expected values
|
||
run: |
|
||
cd ${{ github.workspace }}/tests/test-data/query-agent
|
||
${{ github.workspace }}/tests/api-testing/.venv/bin/python compute_counts.py
|
||
|
||
- name: Run query agent tests
|
||
run: |
|
||
ZO_MAX_FILE_RETENTION_TIME=1 ${{ github.workspace }}/target/debug/openobserve > o2_query_agent.log 2>&1 &
|
||
SERVER_PID=$!
|
||
for i in $(seq 1 60); do
|
||
if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
|
||
echo "Server ready after ${i}s"; break
|
||
fi
|
||
sleep 1
|
||
done
|
||
rye run pytest tests/query_agent/ --ignore=tests/query_agent/test_3_vortex.py -v
|
||
kill $SERVER_PID || true
|
||
working-directory: tests/api-testing/
|
||
|
||
- name: Upload server log on failure
|
||
if: failure()
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: openobserve-query-agent-log
|
||
path: tests/api-testing/o2_query_agent.log
|
||
if-no-files-found: warn
|
||
retention-days: 14
|
||
|
||
# Phase 6.b: regression matrix split. The 4 env-flag-sensitive test
|
||
# configurations now run as 4 parallel jobs instead of 4 sequential
|
||
# restarts in one job. Each job builds OO once and runs its single
|
||
# configured pytest target.
|
||
api_regression_tests:
|
||
name: "api_regression / ${{ matrix.config.label }}"
|
||
needs: [check_changes]
|
||
if: needs.check_changes.outputs.has_changes == 'true'
|
||
runs-on: ubicloud-standard-4
|
||
permissions:
|
||
contents: read # only need to checkout the repo
|
||
strategy:
|
||
fail-fast: false
|
||
matrix:
|
||
config:
|
||
- label: utf8_view_false
|
||
env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true ZO_UTF8_VIEW_ENABLED=false"
|
||
target: tests/regression/test_subquery_join_match.py
|
||
- label: utf8_view_true
|
||
env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true ZO_UTF8_VIEW_ENABLED=true"
|
||
target: tests/regression/test_subquery_join_match.py
|
||
- label: join_match_one_true
|
||
env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=true"
|
||
target: tests/regression/test_logs_to_logs_join.py
|
||
- label: join_match_one_false
|
||
env: "ZO_FEATURE_JOIN_MATCH_ONE_ENABLED=false"
|
||
target: tests/regression/test_logs_to_logs_join.py
|
||
steps:
|
||
- name: Remove unused tools
|
||
run: |
|
||
sudo rm -rf /usr/share/dotnet
|
||
sudo rm -rf /opt/ghc
|
||
sudo rm -rf "/usr/local/share/boost"
|
||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||
|
||
- uses: actions/checkout@v5
|
||
with: { fetch-depth: 0 }
|
||
|
||
- name: Setup Rust Toolchain
|
||
uses: dtolnay/rust-toolchain@master
|
||
with:
|
||
toolchain: nightly-2026-05-20
|
||
|
||
- name: Setup Rust Cache
|
||
uses: Swatinem/rust-cache@v2
|
||
with:
|
||
cache-on-failure: true
|
||
shared-key: backend-debug-mimalloc
|
||
save-if: false # main api_integration_tests job writes the cache
|
||
|
||
- name: Install Protoc
|
||
run: bash ./.github/protoc.sh
|
||
|
||
- name: Build OpenObserve debug binary
|
||
run: cargo build --features mimalloc
|
||
|
||
- uses: actions/setup-python@v5
|
||
with: { python-version: "3.11.6" }
|
||
|
||
- name: Setup rye
|
||
uses: eifinger/setup-rye@v4
|
||
with:
|
||
enable-cache: true
|
||
working-directory: "tests/api-testing/"
|
||
version: "0.27.0"
|
||
|
||
- name: Pin cpython
|
||
run: rye pin cpython@3.11.6
|
||
working-directory: tests/api-testing/
|
||
|
||
- name: Rye sync
|
||
run: rye sync
|
||
working-directory: tests/api-testing/
|
||
|
||
- name: "Run ${{ matrix.config.label }} regression test"
|
||
# Server binary lives at the repo root (target/debug/openobserve);
|
||
# this step uses working-directory: tests/api-testing/, so we must
|
||
# invoke it via ${{ github.workspace }} or the shell will fail with
|
||
# "No such file or directory". --force-sugar omitted on purpose —
|
||
# pytest-sugar auto-loads via entry point, and passing the flag
|
||
# registers it twice and inflates the progress counter ~3x.
|
||
run: |
|
||
${{ matrix.config.env }} ${{ github.workspace }}/target/debug/openobserve > "o2_${{ matrix.config.label }}.log" 2>&1 &
|
||
SERVER_PID=$!
|
||
for i in $(seq 1 60); do
|
||
if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then
|
||
echo "Server ready after ${i}s"; break
|
||
fi
|
||
sleep 1
|
||
done
|
||
rye run pytest ${{ matrix.config.target }} -v
|
||
kill $SERVER_PID || true
|
||
working-directory: tests/api-testing/
|
||
|
||
- name: Upload server log on failure
|
||
if: failure()
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: "openobserve-regression-${{ matrix.config.label }}-log"
|
||
path: "tests/api-testing/o2_${{ matrix.config.label }}.log"
|
||
if-no-files-found: warn
|
||
retention-days: 14
|
||
|
||
# Phase 5: schemathesis fuzz job — informational (continue-on-error),
|
||
# surfaces 5xx panics across the 150 OpenAPI endpoints. Decoupled from
|
||
# api_integration_tests so its findings don't block merges yet; teams
|
||
# can promote it to blocking after the noise floor is understood.
|
||
api_fuzz_tests:
|
||
name: api_fuzz_tests
|
||
needs: [check_changes]
|
||
if: needs.check_changes.outputs.has_changes == 'true'
|
||
runs-on: ubicloud-standard-4
|
||
continue-on-error: true
|
||
permissions:
|
||
contents: read # only need to checkout the repo
|
||
steps:
|
||
- name: Remove unused tools
|
||
run: |
|
||
sudo rm -rf /usr/share/dotnet
|
||
sudo rm -rf /opt/ghc
|
||
sudo rm -rf "/usr/local/share/boost"
|
||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||
|
||
- name: Checkout git repo
|
||
uses: actions/checkout@v5
|
||
with:
|
||
fetch-depth: 0
|
||
|
||
- name: Setup Rust Toolchain
|
||
uses: dtolnay/rust-toolchain@master
|
||
with:
|
||
toolchain: nightly-2026-05-20
|
||
|
||
- name: Setup Rust Cache
|
||
uses: Swatinem/rust-cache@v2
|
||
with:
|
||
cache-on-failure: true
|
||
shared-key: backend-debug-mimalloc
|
||
save-if: false # don't write cache from this job (the main job does)
|
||
|
||
- name: Install Protoc
|
||
run: bash ./.github/protoc.sh
|
||
|
||
- name: Build OpenObserve debug binary
|
||
run: cargo build --features mimalloc
|
||
|
||
- name: Setup Python
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.11.6"
|
||
|
||
- name: Setup rye
|
||
uses: eifinger/setup-rye@v4
|
||
with:
|
||
enable-cache: true
|
||
working-directory: "tests/api-testing/"
|
||
version: "0.27.0"
|
||
|
||
- name: Pin cpython
|
||
run: rye pin cpython@3.11.6
|
||
working-directory: tests/api-testing/
|
||
|
||
- name: Rye sync (schemathesis dep installed)
|
||
run: rye sync
|
||
working-directory: tests/api-testing/
|
||
|
||
- name: Start OpenObserve
|
||
run: target/debug/openobserve > o2_fuzz.log 2>&1 &
|
||
|
||
- name: Wait for /healthz
|
||
run: |
|
||
for i in $(seq 1 60); do
|
||
if curl -sf http://localhost:5080/healthz > /dev/null 2>&1; then exit 0; fi
|
||
sleep 1
|
||
done
|
||
exit 1
|
||
|
||
- name: Run schemathesis fuzz tests
|
||
# Schemathesis 4.x CLI flag renames vs the 3.x originals:
|
||
# --base-url -> --url / -u
|
||
# --auth-type basic -> removed (USER:PASS implies basic)
|
||
# --hypothesis-max-examples -> --max-examples / -n
|
||
# --hypothesis-deadline -> no equivalent; use --request-timeout
|
||
# (seconds, not ms) to bound per-request
|
||
# --data-generation-method -> --mode / -m
|
||
# `--checks not_a_server_error` keeps focus on 5xx panics (actual
|
||
# backend bugs, not spec-conformance nits). `-n 20` bounds runtime;
|
||
# `--mode positive` keeps inputs realistic (negative inputs are a
|
||
# separate exercise once the noise floor here is understood).
|
||
run: |
|
||
rye run schemathesis run \
|
||
-u http://localhost:5080 \
|
||
--auth "$ZO_ROOT_USER_EMAIL:$ZO_ROOT_USER_PASSWORD" \
|
||
--checks not_a_server_error \
|
||
-n 20 \
|
||
--request-timeout 5 \
|
||
-m positive \
|
||
--suppress-health-check=filter_too_much \
|
||
--exclude-path "/api/{org_id}/v1/logs" \
|
||
--exclude-path "/api/{org_id}/v1/metrics" \
|
||
--exclude-path "/api/{org_id}/v1/traces" \
|
||
--exclude-path "/api/{org_id}/loki/api/v1/push" \
|
||
--exclude-path "/api/{org_id}/prometheus/api/v1/write" \
|
||
--exclude-path "/api/{org_id}/dashboards" \
|
||
--exclude-path "/api/{org_id}/dashboards/{dashboard_id}" \
|
||
--exclude-path "/api/{org_id}/dashboards/{dashboard_id}/panels" \
|
||
--exclude-path "/api/{org_id}/dashboards/{dashboard_id}/panels/{panel_id}" \
|
||
--exclude-path "/api/{org_id}/pipelines/history" \
|
||
--exclude-path "/api/{org_id}/organizations/assume_service_account" \
|
||
http://localhost:5080/api-doc/openapi.json
|
||
working-directory: tests/api-testing/
|
||
|
||
- name: Upload OpenObserve server log on failure
|
||
if: failure()
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: openobserve-fuzz-log
|
||
path: tests/api-testing/o2_fuzz.log
|
||
if-no-files-found: warn
|
||
retention-days: 14
|
||
|
||
api_tests_summary:
|
||
runs-on: ubuntu-latest
|
||
permissions: {}
|
||
needs: [check_changes, api_integration_tests, api_query_agent_tests, api_regression_tests]
|
||
if: always()
|
||
steps:
|
||
- name: Check test results
|
||
run: |
|
||
if [ "${{ needs.check_changes.result }}" != "success" ]; then
|
||
echo "check_changes job did not succeed: ${{ needs.check_changes.result }}"
|
||
exit 1
|
||
fi
|
||
# All three test jobs must pass (or be skipped).
|
||
ok() { [ "$1" == "success" ] || [ "$1" == "skipped" ]; }
|
||
if ok "${{ needs.api_integration_tests.result }}" && ok "${{ needs.api_query_agent_tests.result }}" && ok "${{ needs.api_regression_tests.result }}"; then
|
||
echo "All API test jobs passed or skipped"
|
||
exit 0
|
||
else
|
||
echo "API tests failed: integration=${{ needs.api_integration_tests.result }}, query_agent=${{ needs.api_query_agent_tests.result }}, regression=${{ needs.api_regression_tests.result }}"
|
||
exit 1
|
||
fi
|