mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-05 08:09:22 +00:00
487 lines
18 KiB
YAML
487 lines
18 KiB
YAML
name: Unit tests
|
|
|
|
on:
|
|
# Kept as the Rust cache writer on main: caches saved from merge-queue refs are never restored by PRs.
|
|
push:
|
|
branches:
|
|
- "main"
|
|
pull_request:
|
|
branches:
|
|
- "**"
|
|
# labeled/unlabeled so adding or removing 'ready-for-ci' starts or resets CI; drafts never run paid CI.
|
|
types: [opened, reopened, synchronize, labeled, unlabeled, ready_for_review, converted_to_draft]
|
|
merge_group:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.merge_group.head_ref || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
COLUMNS: 150
|
|
ZO_RESULT_CACHE_ENABLED: true
|
|
UI_NODE_VERSION: "24.22.0"
|
|
UI_NPM_VERSION: "11.19.1"
|
|
# coverage mapping needs no DWARF; dropping debuginfo (test inherits dev) cuts compile and link time
|
|
CARGO_PROFILE_DEV_DEBUG: "0"
|
|
|
|
jobs:
|
|
check_changes:
|
|
# Paid CI runs only on PRs carrying the 'ready-for-ci' label; the merge queue runs unit tests on the merged code.
|
|
if: ${{ github.event_name != 'pull_request' || (contains(github.event.pull_request.labels.*.name, 'ready-for-ci') && github.event.pull_request.draft == false) }}
|
|
name: check_changes
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
has_backend_changes: ${{ steps.check.outputs.has_backend_changes }}
|
|
has_ui_changes: ${{ steps.check.outputs.has_ui_changes }}
|
|
ui_shards: ${{ steps.ui_matrix.outputs.ui_shards }}
|
|
ui_shard_count: ${{ steps.ui_matrix.outputs.ui_shard_count }}
|
|
steps:
|
|
# Debounce: a push inside this window cancels the run (concurrency) before anything paid starts.
|
|
- if: github.event_name == 'pull_request'
|
|
run: sleep 60
|
|
- uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
# One shared classification for every suite (see .github/actions/classify-changes); a change to
|
|
# this workflow file re-tests itself on top of it.
|
|
- uses: ./.github/actions/classify-changes
|
|
id: classify
|
|
- id: check
|
|
env:
|
|
HAS_BACKEND_CHANGES: ${{ steps.classify.outputs.run_unit_backend }}
|
|
HAS_UI_CHANGES: ${{ steps.classify.outputs.run_unit_ui }}
|
|
CHANGED_WORKFLOWS: ${{ steps.classify.outputs.changed_workflows }}
|
|
run: |
|
|
if [ "$HAS_BACKEND_CHANGES" = "true" ] || echo " $CHANGED_WORKFLOWS " | grep -q ' unit-tests.yml '; then
|
|
echo "has_backend_changes=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "has_backend_changes=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
if [ "$HAS_UI_CHANGES" = "true" ] || echo " $CHANGED_WORKFLOWS " | grep -q ' unit-tests.yml '; then
|
|
echo "has_ui_changes=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "has_ui_changes=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- name: Read UI shard configuration
|
|
id: ui_matrix
|
|
run: node web/scripts/ui-tests/run.mjs matrix >> "$GITHUB_OUTPUT"
|
|
|
|
backend_static_checks:
|
|
needs: [check_changes]
|
|
if: needs.check_changes.outputs.has_backend_changes == 'true'
|
|
runs-on:
|
|
labels: ubicloud-standard-8
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- uses: ./.github/actions/github-auth
|
|
- name: Setup Rust Toolchain
|
|
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
|
|
with:
|
|
toolchain: nightly-2026-10-24
|
|
targets: x86_64-unknown-linux-gnu
|
|
|
|
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
|
|
with:
|
|
cache-on-failure: true
|
|
shared-key: unit-tests
|
|
save-if: false
|
|
|
|
- name: Install Protoc
|
|
run: bash ./.github/protoc.sh
|
|
|
|
- name: Simulate building frontend code
|
|
run: mkdir -p web/dist && touch web/dist/index.html
|
|
|
|
- name: Print rustc cfg
|
|
run: rustc -C target-cpu=native --print cfg
|
|
|
|
- name: Check code formatting
|
|
run: cargo fmt --all -- --check
|
|
|
|
- name: Run clippy
|
|
run: cargo --config net.git-fetch-with-cli=true clippy --workspace --all-targets -- -W clippy::too_many_lines -W clippy::cognitive_complexity -W clippy::excessive_nesting -D warnings
|
|
|
|
backend_unit_tests_postgres:
|
|
needs: [check_changes]
|
|
if: needs.check_changes.outputs.has_backend_changes == 'true'
|
|
runs-on:
|
|
labels: ubicloud-standard-8
|
|
permissions:
|
|
contents: read
|
|
env:
|
|
ZO_INTERNAL_GRPC_TOKEN: fake
|
|
ZO_TEXT_DATA_TYPE: text
|
|
ZO_META_STORE: postgres
|
|
ZO_META_POSTGRES_DSN: postgres://postgres:password@localhost:5432/postgres
|
|
services:
|
|
postgres:
|
|
image: postgres:17.5-alpine3.22
|
|
env:
|
|
POSTGRES_PASSWORD: password
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_DB: postgres
|
|
ports:
|
|
- 5432:5432
|
|
options: --health-cmd "pg_isready -U postgres" --health-interval 10s --health-timeout 5s --health-retries 10
|
|
steps:
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- uses: ./.github/actions/github-auth
|
|
- name: Setup Rust Toolchain
|
|
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
|
|
with:
|
|
toolchain: nightly-2026-10-24
|
|
targets: x86_64-unknown-linux-gnu
|
|
|
|
- uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
|
|
env:
|
|
GITHUB_TOKEN: ${{ github.token }}
|
|
with:
|
|
tool: nextest,cargo-llvm-cov
|
|
|
|
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
|
|
with:
|
|
cache-on-failure: true
|
|
shared-key: unit-tests
|
|
# merge_group so an evicted cache self-heals from the next queue run, not only main pushes
|
|
save-if: ${{ github.event_name == 'merge_group' || github.ref == 'refs/heads/main' }}
|
|
|
|
- name: Install Protoc
|
|
run: bash ./.github/protoc.sh
|
|
|
|
- name: Simulate building frontend code
|
|
run: mkdir -p web/dist && touch web/dist/index.html
|
|
|
|
- name: Wait for PostgreSQL to be ready
|
|
run: |
|
|
echo "Waiting for PostgreSQL to be fully ready..."
|
|
for i in {1..30}; do
|
|
if PGPASSWORD=password psql -h localhost -U postgres -d postgres -c "SELECT 1" > /dev/null 2>&1; then
|
|
echo "PostgreSQL is ready!"
|
|
break
|
|
fi
|
|
echo "Waiting for PostgreSQL... attempt $i/30"
|
|
sleep 2
|
|
done
|
|
|
|
- name: Run PostgreSQL unit tests
|
|
run: ./coverage.sh run-cov
|
|
env:
|
|
ZO_DATA_DIR: ${{ runner.temp }}/openobserve-postgres/
|
|
|
|
- name: Check PostgreSQL coverage
|
|
run: ./coverage.sh check
|
|
|
|
backend_unit_tests_sqlite:
|
|
needs: [check_changes]
|
|
if: needs.check_changes.outputs.has_backend_changes == 'true'
|
|
runs-on:
|
|
labels: ubicloud-standard-8
|
|
permissions:
|
|
contents: read
|
|
env:
|
|
ZO_INTERNAL_GRPC_TOKEN: fake
|
|
ZO_TEXT_DATA_TYPE: text
|
|
ZO_META_STORE: sqlite
|
|
ZO_META_POSTGRES_DSN: ""
|
|
steps:
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- uses: ./.github/actions/github-auth
|
|
- name: Setup Rust Toolchain
|
|
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
|
|
with:
|
|
toolchain: nightly-2026-10-24
|
|
targets: x86_64-unknown-linux-gnu
|
|
|
|
- uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
|
|
env:
|
|
GITHUB_TOKEN: ${{ github.token }}
|
|
with:
|
|
tool: nextest,cargo-llvm-cov
|
|
|
|
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
|
|
with:
|
|
cache-on-failure: true
|
|
shared-key: unit-tests
|
|
save-if: false
|
|
|
|
- name: Install Protoc
|
|
run: bash ./.github/protoc.sh
|
|
|
|
- name: Simulate building frontend code
|
|
run: mkdir -p web/dist && touch web/dist/index.html
|
|
|
|
- name: Run SQLite unit tests
|
|
run: ./coverage.sh run-cov
|
|
env:
|
|
ZO_DATA_DIR: ${{ runner.temp }}/openobserve-sqlite/
|
|
|
|
- name: Check SQLite coverage
|
|
run: ./coverage.sh check
|
|
|
|
backend_unit_tests:
|
|
name: backend_unit_tests
|
|
runs-on: ubuntu-latest
|
|
permissions: {}
|
|
needs:
|
|
- check_changes
|
|
- backend_static_checks
|
|
- backend_unit_tests_postgres
|
|
- backend_unit_tests_sqlite
|
|
if: always() && needs.check_changes.outputs.has_backend_changes == 'true'
|
|
steps:
|
|
- name: Check backend test results
|
|
run: |
|
|
for result in \
|
|
"${{ needs.backend_static_checks.result }}" \
|
|
"${{ needs.backend_unit_tests_postgres.result }}" \
|
|
"${{ needs.backend_unit_tests_sqlite.result }}"; do
|
|
if [ "$result" != "success" ]; then
|
|
echo "A backend check failed or was skipped: $result"
|
|
exit 1
|
|
fi
|
|
done
|
|
echo "All backend checks completed successfully"
|
|
|
|
ui_unit_test_shards:
|
|
name: UI unit shard ${{ matrix.shard }}
|
|
needs: [check_changes]
|
|
if: needs.check_changes.outputs.has_ui_changes == 'true'
|
|
timeout-minutes: 45
|
|
runs-on:
|
|
labels: ubicloud-standard-8
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shard: ${{ fromJSON(needs.check_changes.outputs.ui_shards) }}
|
|
permissions:
|
|
contents: read
|
|
actions: read
|
|
|
|
steps:
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
|
|
- uses: ./.github/actions/github-auth
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@v7
|
|
with:
|
|
node-version: ${{ env.UI_NODE_VERSION }}
|
|
cache: 'npm'
|
|
cache-dependency-path: web/package-lock.json
|
|
|
|
- name: Install dependencies
|
|
working-directory: web
|
|
timeout-minutes: 5
|
|
run: |
|
|
test "$(node --version)" = "v${UI_NODE_VERSION}"
|
|
test "$(npm --version)" = "${UI_NPM_VERSION}"
|
|
npm ci
|
|
|
|
- name: Run frontend unit test shard with coverage
|
|
working-directory: web
|
|
run: ./coverage.sh shard ${{ matrix.shard }}
|
|
|
|
- name: Upload shard coverage
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: ui-unit-shard-${{ matrix.shard }}-${{ needs.check_changes.outputs.ui_shard_count }}
|
|
path: web/.vitest-reports/ui-unit-shard-${{ matrix.shard }}-${{ needs.check_changes.outputs.ui_shard_count }}/
|
|
include-hidden-files: true
|
|
overwrite: true
|
|
retention-days: 3
|
|
if-no-files-found: error
|
|
|
|
- name: Wait for terminal shard success and artifacts
|
|
if: matrix.shard == fromJSON(needs.check_changes.outputs.ui_shard_count)
|
|
id: ui_artifacts
|
|
working-directory: web
|
|
timeout-minutes: 10
|
|
env:
|
|
GITHUB_TOKEN: ${{ github.token }}
|
|
run: node scripts/ui-tests/coordinate.mjs
|
|
|
|
- name: Download shard coverage
|
|
if: matrix.shard == fromJSON(needs.check_changes.outputs.ui_shard_count)
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
artifact-ids: ${{ steps.ui_artifacts.outputs.artifact_ids }}
|
|
path: web/.vitest-reports
|
|
merge-multiple: false
|
|
|
|
# The successful step records immutable inputs so later shard reruns cannot reuse stale coverage.
|
|
- name: Merge UI coverage [${{ steps.ui_artifacts.outputs.artifact_ids }}]
|
|
if: matrix.shard == fromJSON(needs.check_changes.outputs.ui_shard_count)
|
|
working-directory: web
|
|
run: ./coverage.sh merge
|
|
|
|
# The matrix dependency also reruns this required gate when an individual shard is rerun.
|
|
ui_unit_tests:
|
|
needs: [check_changes, ui_unit_test_shards]
|
|
if: ${{ always() && needs.check_changes.outputs.has_ui_changes == 'true' }}
|
|
timeout-minutes: 5
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
actions: read
|
|
steps:
|
|
- name: Require every shard and coordinator to pass
|
|
run: |
|
|
if [ "${{ needs.ui_unit_test_shards.result }}" != "success" ]; then
|
|
echo "A UI shard or coverage merge failed or was skipped: ${{ needs.ui_unit_test_shards.result }}"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Verify merged artifact freshness
|
|
uses: actions/github-script@v9
|
|
env:
|
|
UI_TEST_SHARD_COUNT: ${{ needs.check_changes.outputs.ui_shard_count }}
|
|
with:
|
|
retries: 0
|
|
script: |
|
|
const count = Number(process.env.UI_TEST_SHARD_COUNT);
|
|
const attempt = Number(process.env.GITHUB_RUN_ATTEMPT);
|
|
if (!Number.isSafeInteger(count) || count < 1 || !Number.isSafeInteger(attempt) || attempt < 1) {
|
|
throw new Error('Invalid UI shard count or run attempt');
|
|
}
|
|
const params = { ...context.repo, run_id: context.runId, per_page: 100 };
|
|
const { data: run } = await github.rest.actions.getWorkflowRun(params);
|
|
if (run.id !== context.runId || run.run_attempt !== attempt || run.conclusion === 'cancelled') {
|
|
throw new Error('Workflow run changed or was cancelled');
|
|
}
|
|
const names = Array.from({ length: count }, (_, index) => `UI unit shard ${index + 1}`);
|
|
const latest = new Map();
|
|
for (let current = attempt; current >= 1 && latest.size < count; current--) {
|
|
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, {
|
|
...params, attempt_number: current,
|
|
});
|
|
const seen = new Set();
|
|
for (const job of jobs) {
|
|
if (!names.includes(job.name)) continue;
|
|
if (seen.has(job.name)) throw new Error(`Duplicate UI shard job: ${job.name}`);
|
|
seen.add(job.name);
|
|
if (!latest.has(job.name)) latest.set(job.name, { ...job, attempt: current });
|
|
}
|
|
}
|
|
for (const name of names) {
|
|
const job = latest.get(name);
|
|
if (!job || job.run_id !== context.runId || job.status !== 'completed' || job.conclusion !== 'success') {
|
|
throw new Error(`${name} has not succeeded; rerun failed jobs`);
|
|
}
|
|
}
|
|
const coordinator = latest.get(names.at(-1));
|
|
const merges = (coordinator.steps || []).filter((step) => /^Merge UI coverage \[[0-9,]+\]$/.test(step.name));
|
|
if (merges.length !== 1 || merges[0].status !== 'completed' || merges[0].conclusion !== 'success') {
|
|
throw new Error('No successful complete coverage merge; rerun the coordinator shard');
|
|
}
|
|
const mergedIds = merges[0].name.match(/\[([0-9,]+)\]/)[1].split(',');
|
|
const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, params);
|
|
const currentIds = [];
|
|
for (let index = 1; index <= count; index++) {
|
|
const name = `ui-unit-shard-${index}-${count}`;
|
|
const matches = artifacts.filter((artifact) => artifact.name === name);
|
|
if (matches.length !== 1 || matches[0].expired || !Number.isSafeInteger(matches[0].id)) {
|
|
throw new Error(`Missing, expired or duplicate ${name}; rerun all jobs`);
|
|
}
|
|
currentIds.push(String(matches[0].id));
|
|
}
|
|
if (JSON.stringify(mergedIds) !== JSON.stringify(currentIds) ||
|
|
[...latest.values()].some((job) => job.attempt > coordinator.attempt)) {
|
|
throw new Error('Shard artifacts changed after coverage merge; rerun the coordinator shard or all jobs');
|
|
}
|
|
core.info(`Verified ${count} successful partitions and their exact merged artifact IDs`);
|
|
|
|
ui_code_quality:
|
|
needs: [check_changes]
|
|
if: needs.check_changes.outputs.has_ui_changes == 'true'
|
|
runs-on:
|
|
labels: ubicloud-standard-8
|
|
permissions:
|
|
contents: read
|
|
|
|
steps:
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@v5
|
|
with:
|
|
node-version: ${{ env.UI_NODE_VERSION }}
|
|
cache: 'npm'
|
|
cache-dependency-path: web/package-lock.json
|
|
|
|
- name: Install dependencies
|
|
working-directory: web
|
|
run: |
|
|
test "$(node --version)" = "v${UI_NODE_VERSION}"
|
|
test "$(npm --version)" = "${UI_NPM_VERSION}"
|
|
npm ci
|
|
|
|
- name: Verify UI shard partition and report validation
|
|
working-directory: web
|
|
run: node --test scripts/ui-tests/*.test.mjs
|
|
|
|
- name: Prettier format check (any unformatted file fails)
|
|
working-directory: web
|
|
run: npm run format:check
|
|
|
|
- name: Type-check (any error fails)
|
|
working-directory: web
|
|
env:
|
|
# vue-tsc peaks around 3GB on this project; the runner's default heap
|
|
# ceiling is ~4GB, which OOMs (exit 134) instead of reporting errors.
|
|
NODE_OPTIONS: "--max-old-space-size=8192"
|
|
run: npm run type-check:app
|
|
|
|
unit_tests_summary:
|
|
runs-on: ubuntu-latest
|
|
permissions: {}
|
|
needs: [check_changes, backend_unit_tests, ui_unit_tests, ui_code_quality]
|
|
if: always() && (github.event_name != 'pull_request' || (contains(github.event.pull_request.labels.*.name, 'ready-for-ci') && github.event.pull_request.draft == false))
|
|
steps:
|
|
- name: Check test results
|
|
run: |
|
|
if [ "${{ needs.check_changes.result }}" != "success" ]; then
|
|
echo "check_changes job did not succeed: ${{ needs.check_changes.result }}"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "${{ needs.backend_unit_tests.result }}" == "success" ] || [ "${{ needs.backend_unit_tests.result }}" == "skipped" ]; then
|
|
echo "Backend tests passed or skipped"
|
|
else
|
|
echo "Backend tests failed"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "${{ needs.ui_unit_tests.result }}" == "success" ] || [ "${{ needs.ui_unit_tests.result }}" == "skipped" ]; then
|
|
echo "UI tests passed or skipped"
|
|
else
|
|
echo "UI tests failed"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "${{ needs.ui_code_quality.result }}" == "success" ] || [ "${{ needs.ui_code_quality.result }}" == "skipped" ]; then
|
|
echo "UI code quality passed or skipped"
|
|
else
|
|
echo "UI code quality failed"
|
|
exit 1
|
|
fi
|
|
|
|
echo "All tests completed successfully"
|