diff --git a/.github/actions/github-script-tests/action.yml b/.github/actions/github-script-tests/action.yml index 4e3d797a95c..5721aead096 100644 --- a/.github/actions/github-script-tests/action.yml +++ b/.github/actions/github-script-tests/action.yml @@ -8,7 +8,7 @@ runs: node-version: 24 - name: Run Node GitHub script tests shell: bash - run: node --test .github/scripts/query-regression-comment.test.cjs .github/scripts/heavy-ci-policy.test.cjs .github/scripts/tests/nightly-merge.test.cjs + run: node --test .github/scripts/tests/ci-cache-policy.test.cjs .github/scripts/query-regression-comment.test.cjs .github/scripts/heavy-ci-policy.test.cjs .github/scripts/tests/nightly-merge.test.cjs - name: Setup uv uses: astral-sh/setup-uv@v3 with: diff --git a/.github/ci/cache-policy.md b/.github/ci/cache-policy.md new file mode 100644 index 00000000000..d1eb053ec1c --- /dev/null +++ b/.github/ci/cache-policy.md @@ -0,0 +1,45 @@ +# CI cache policy + +Checks, Rust CI, Integration CI, and Nightly CI use one explicit Rust cache per +build job. Disable the implicit cache in `setup-rust-toolchain` when adding an +explicit cache: otherwise its independent save policy can write PR-scoped +caches even when the explicit cache only saves on main. + +Integration CI's existing weekday schedule runs the fuzz builds and suites, +including the profile-CI binary and Kafka helper, to populate main caches. +To warm these caches manually after deployment, run Integration CI on main with +`fuzz_profile: all`; no internal `ci_command` marker is required. + +Rust CI's existing weekday schedule warms the Windows and RISC-V compilation +caches. Nightly CI's existing schedule warms its Windows runtime test caches. +A manual Rust CI run on main can warm the compilation caches sooner. These jobs +keep their separate keys and existing runner types; no extra warmup job is added. +Coverage caches are not warmed by these schedules. +PR, merge-group, and non-main manual runs restore caches without saving them. +Fuzz binaries still flow to downstream jobs through workflow artifacts on every +run; only the persistent binary cache is main-written. Re-running a PR cannot +reuse a newly saved PR-specific binary cache under this policy. + +Keep helper version variables outside the `CARGO*` and `RUST*` prefixes unless +they affect the compilation being cached. rust-cache includes those prefixes in +its environment hash. Fuzz tool versions remain explicitly included in the fuzz +cache keys. Compiler versions, Cargo manifests, flags, OS, and architecture +remain part of the existing cache isolation. + +This policy reduces duplicate and PR-only writes; it does not guarantee that all +build caches fit in the repository quota. Renamed environment variables cause +an initial cold restore until main produces the new keys. Existing caches are +not deleted by this change. Coverage warming and additional runner capacity are +not introduced. + +Validate the policy with: + +```sh +node --test .github/scripts/tests/ci-cache-policy.test.cjs +``` + +After deployment, compare the same commit and runner image across repeated runs. +Record exact versus fallback restores, archive size, restore/save duration, and +compilation duration. Verify main writes the keys consumed by PRs and that no +implicit post-action uploads remain. Local policy tests do not establish a +runtime speedup or long-term cache hit rate. diff --git a/.github/scripts/tests/ci-cache-policy.test.cjs b/.github/scripts/tests/ci-cache-policy.test.cjs new file mode 100644 index 00000000000..ba6817424c6 --- /dev/null +++ b/.github/scripts/tests/ci-cache-policy.test.cjs @@ -0,0 +1,155 @@ +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { resolve } = require('node:path'); +const { test } = require('node:test'); + +const directory = resolve(__dirname, '../../workflows'); +const workflows = ['checks', 'rust', 'integration', 'nightly-ci'].map(name => ({ + name, + text: readFileSync(resolve(directory, name + '.yml'), 'utf8'), +})); +function jobs(text) { + return [...text.matchAll(/^ ([\w-]+):\n([\s\S]*?)(?=^ [\w-]+:\n|(?![\s\S]))/gm)]; +} +function steps(body) { + return body.split(/^ - /m).slice(1); +} + +test('build caches have one controller and lightweight checks do not cache targets', () => { + let checked = 0; + for (const { name, text } of workflows) { + for (const [, job, body] of jobs(text)) { + if (!body.includes('Swatinem/rust-cache@') && !['fmt', 'check-udeps', 'toml'].includes(job)) continue; + for (const step of steps(body).filter(s => s.includes('uses: actions-rust-lang/setup-rust-toolchain@'))) { + assert.match(step, /^ cache: false$/m, name + '/' + job); + checked++; + } + } + } + assert.ok(checked >= 10, 'expected build and lightweight jobs to be covered'); +}); + +test('only main writes caches, and the nightly read-only cache stays read-only', () => { + let checked = 0; + for (const { name, text } of workflows) { + for (const [, job, body] of jobs(text)) { + for (const step of steps(body)) { + if (!step.includes('uses: Swatinem/rust-cache@') && !step.includes('uses: actions/cache/save@')) continue; + const condition = step.includes('uses: Swatinem/rust-cache@') + ? step.match(/^ save-if: (.+)$/m)?.[1] + : step.match(/^ if: (.+)$/m)?.[1]; + assert.ok(condition, name + '/' + job); + const expression = condition.replace(/^\$\{\{\s*|\s*\}\}$/g, '') + .replaceAll('steps.fuzz-binaries.outputs.cache-hit', 'hit'); + const enabled = new Function('github', 'hit', 'return (' + expression + ');'); + for (const [event, ref] of [ + ['pull_request', 'refs/pull/123/merge'], + ['merge_group', 'refs/heads/gh-readonly-queue/main/pr-123'], + ['workflow_dispatch', 'refs/heads/feature'], + ]) assert.equal(enabled({ event_name: event, ref }, 'false'), false, name + '/' + job); + assert.equal(enabled({ event_name: 'schedule', ref: 'refs/heads/main' }, 'false'), + condition !== 'false', name + '/' + job); + if (step.includes('uses: actions/cache/save@')) { + assert.equal(enabled({ ref: 'refs/heads/main' }, 'true'), false); + } + checked++; + } + } + } + assert.ok(checked >= 10); +}); + +test('helper tool versions do not fragment unrelated Rust caches', () => { + for (const { text } of workflows) { + assert.doesNotMatch(text, /CARGO_(SHEAR|FUZZ)_VERSION/); + } + const integration = workflows.find(w => w.name === 'integration').text; + assert.match(integration, /prefix-key: "v1-fuzz-\$\{\{ env.FUZZ_VERSION \}\}"/); + assert.match(integration, /cargo install cargo-fuzz --version "\$\{FUZZ_VERSION\}"/); +}); + +// Evaluate the actual job guard as well as the cache's save condition. A valid +// save-if alone cannot prove that a scheduled cache producer is reachable. +function warmupJob(workflow, job, { + event = 'schedule', ref = 'refs/heads/main', + profile = event === 'workflow_dispatch' ? 'all' : undefined, + outputs = {}, failed = '', cancelled = false, draft = false, +} = {}) { + const text = workflows.find(w => w.name === workflow).text; + assert.match(text, /^ schedule:$/m); + const body = jobs(text).find(([, name]) => name === job)?.[2]; + assert.ok(body, workflow + '/' + job); + const github = { repository: 'GreptimeTeam/greptimedb', event_name: event, ref, + event: { pull_request: { draft } }, run_attempt: 1 }; + const needs = { scope: { outputs }, ...Object.fromEntries( + ['github-script-tests', 'build', 'prepare-fuzz-targets', 'build-greptime-ci'].map(name => + [name, { result: name === failed ? 'failure' : 'success' }])) }; + const evaluate = expression => new Function('github', 'inputs', 'needs', 'cancelled', + 'return (' + expression.replace(/^\$\{\{\s*|\s*\}\}$/g, '') + ');')( + github, { fuzz_profile: profile, ci_command: false }, needs, () => cancelled); + const admitted = evaluate(body.match(/^ if: (.+)$/m)[1]); + const saves = [...body.matchAll(/^ save-if: (.+)$/gm)] + .map(match => admitted && evaluate(match[1])); + return { admitted, saves }; +} + +const fuzzJobs = ['prepare-fuzz-targets', 'build-greptime-ci', 'fuzztest', + 'unstable-fuzztest', 'distributed-fuzztest', 'distributed-fuzztest-with-chaos']; + +test('main schedules can execute cache producers and their fuzz consumers', () => { + for (const job of fuzzJobs) { + // Scheduled events do not supply workflow_dispatch inputs. + const result = warmupJob('integration', job, { profile: undefined }); + assert.equal(result.admitted, true, job); + assert.ok(result.saves.every(Boolean), job); + } + for (const [workflow, job] of [ + ['integration', 'build'], ['integration', 'export-import-v2-e2e'], + ['integration', 'prepare-fuzz-targets'], ['integration', 'build-greptime-ci'], + ['integration', 'distributed-fuzztest'], + ['rust', 'check-windows'], ['rust', 'check-riscv64'], + ['nightly-ci', 'sqlness-windows'], ['nightly-ci', 'test-on-windows'], + ]) { + for (const event of ['schedule', 'workflow_dispatch']) { + assert.deepEqual(warmupJob(workflow, job, { event }), + { admitted: true, saves: [true] }, workflow + '/' + job + '/' + event); + } + } +}); + +test('manual warming respects selected profiles without an internal command marker', () => { + const expected = { + none: [], + standalone: fuzzJobs.slice(0, 4), + distributed: [fuzzJobs[0], fuzzJobs[1], fuzzJobs[4]], + chaos: [fuzzJobs[0], fuzzJobs[1], fuzzJobs[5]], + all: fuzzJobs, + }; + for (const [profile, wanted] of Object.entries(expected)) { + assert.deepEqual(fuzzJobs.filter(job => warmupJob('integration', job, + { event: 'workflow_dispatch', profile }).admitted), wanted, profile); + } + for (const job of fuzzJobs) { + const result = warmupJob('integration', job, + { event: 'workflow_dispatch', ref: 'refs/heads/feature' }); + assert.ok(result.saves.every(value => value === false), job); + assert.equal(warmupJob('integration', job, + { event: 'pull_request', ref: 'refs/pull/1/merge', draft: true }).admitted, false, job); + } +}); + +test('warmup keeps scope exclusions, dependency failures and cancellation gates', () => { + const gates = ['any_fuzz', 'build_ci_needed', 'fuzz_standalone', + 'fuzz_unstable', 'fuzz_distributed', 'fuzz_chaos']; + for (const [index, job] of fuzzJobs.entries()) { + assert.equal(warmupJob('integration', job, { failed: 'github-script-tests' }).admitted, false, job); + assert.equal(warmupJob('integration', job, { cancelled: true }).admitted, false, job); + assert.equal(warmupJob('integration', job, + { outputs: { [gates[index]]: 'false' } }).admitted, false, job); + } + for (const job of fuzzJobs.slice(2)) { + for (const failed of ['github-script-tests', 'prepare-fuzz-targets', job === 'fuzztest' ? 'build' : 'build-greptime-ci']) { + assert.equal(warmupJob('integration', job, { failed }).admitted, false, job); + } + } +}); diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 1f14d8851cc..0d38a844539 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -107,6 +107,8 @@ jobs: with: tool: protoc@3.23.4 - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false - name: Rust Cache uses: Swatinem/rust-cache@v2 with: @@ -130,6 +132,8 @@ jobs: with: persist-credentials: false - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false - name: Install taplo uses: taiki-e/install-action@v2 with: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 00c3f52a034..134fdfde6ba 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -30,7 +30,7 @@ on: name: Integration CI env: - CARGO_FUZZ_VERSION: "0.13.2" + FUZZ_VERSION: "0.13.2" # Fuzz targets require a nightly toolchain (cargo-fuzz needs # `-Zsanitizer=fuzzer`). This pin decouples the fuzz workflow from the # workspace default in rust-toolchain.toml, so it keeps working when the @@ -121,6 +121,8 @@ jobs: with: tool: protoc@3.23.4 - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false - name: Build Binaries Cache id: build-binaries-cache uses: Swatinem/rust-cache@v2 @@ -292,6 +294,8 @@ jobs: with: tool: protoc@3.23.4 - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false - name: Rust Cache uses: Swatinem/rust-cache@v2 with: @@ -436,18 +440,18 @@ jobs: id: fuzz-dependencies-cache uses: Swatinem/rust-cache@v2 with: - prefix-key: "v1-fuzz-${{ env.CARGO_FUZZ_VERSION }}" + prefix-key: "v1-fuzz-${{ env.FUZZ_VERSION }}" shared-key: "fuzz-dependencies" workspaces: ". -> target" cache-targets: "true" cache-all-crates: "true" - save-if: ${{ github.event_name == 'pull_request' }} + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Look up prebuilt fuzz binaries id: fuzz-binaries uses: actions/cache/restore@v4 with: path: ${{ runner.temp }}/greptime-fuzz-binaries - key: v1-fuzz-binaries-${{ env.CARGO_FUZZ_VERSION }}-${{ env.FUZZ_RUST_TOOLCHAIN }}-all-${{ hashFiles('Cargo.toml', 'Cargo.lock', '.cargo/**', 'tests-fuzz/**', 'src/**') }} + key: v1-fuzz-binaries-${{ env.FUZZ_VERSION }}-${{ env.FUZZ_RUST_TOOLCHAIN }}-all-${{ hashFiles('Cargo.toml', 'Cargo.lock', '.cargo/**', 'tests-fuzz/**', 'src/**') }} - name: Report fuzz cache lookup shell: bash run: | @@ -462,7 +466,7 @@ jobs: env: RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }} run: | - cargo install cargo-fuzz --version "${CARGO_FUZZ_VERSION}" --locked + cargo install cargo-fuzz --version "${FUZZ_VERSION}" --locked - name: Build fuzz target binaries if: steps.fuzz-binaries.outputs.cache-hit != 'true' shell: bash @@ -511,11 +515,11 @@ jobs: build_targets false "${FUZZ_TARGETS}" build_targets true "${FUZZ_UNSTABLE_TARGETS}" - name: Save prebuilt fuzz binaries - if: steps.fuzz-binaries.outputs.cache-hit != 'true' + if: github.ref == 'refs/heads/main' && steps.fuzz-binaries.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: ${{ runner.temp }}/greptime-fuzz-binaries - key: v1-fuzz-binaries-${{ env.CARGO_FUZZ_VERSION }}-${{ env.FUZZ_RUST_TOOLCHAIN }}-all-${{ hashFiles('Cargo.toml', 'Cargo.lock', '.cargo/**', 'tests-fuzz/**', 'src/**') }} + key: v1-fuzz-binaries-${{ env.FUZZ_VERSION }}-${{ env.FUZZ_RUST_TOOLCHAIN }}-all-${{ hashFiles('Cargo.toml', 'Cargo.lock', '.cargo/**', 'tests-fuzz/**', 'src/**') }} - name: Pack prebuilt fuzz binaries shell: bash run: | @@ -688,6 +692,8 @@ jobs: with: tool: protoc@3.23.4 - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false - name: Build Profile-CI Cache id: build-greptime-ci-cache uses: Swatinem/rust-cache@v2 @@ -696,7 +702,7 @@ jobs: shared-key: "build-greptime-ci" cache-all-crates: "true" cache-workspace-crates: "true" - save-if: ${{ github.event_name == 'pull_request' }} + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Report profile-CI cache lookup shell: bash run: | @@ -819,7 +825,7 @@ jobs: workspaces: ". -> target" cache-targets: "true" cache-all-crates: "true" - save-if: ${{ github.event_name == 'pull_request' }} + save-if: ${{ github.ref == 'refs/heads/main' }} - if: matrix.mode.kafkaWalHelper name: Report Kafka WAL helper cache lookup shell: bash diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index f6158a8d7ad..ffd1b01d477 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -57,8 +57,12 @@ jobs: with: tool: protoc@3.23.4 - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false - name: Rust Cache uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Run sqlness run: make sqlness-test env: @@ -143,9 +147,12 @@ jobs: - name: Install Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: + cache: false components: llvm-tools-preview - name: Rust Cache uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install Cargo Nextest uses: taiki-e/install-action@nextest - name: Install WSL distribution diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 842fa306c4e..d66bc042b5b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -24,7 +24,7 @@ on: name: Rust CI env: - CARGO_SHEAR_VERSION: "1.13.4" + SHEAR_VERSION: "1.13.4" concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -56,6 +56,7 @@ jobs: tool: protoc@3.23.4 - uses: actions-rust-lang/setup-rust-toolchain@v1 with: + cache: false components: rustfmt - name: Check format run: make fmt-check @@ -75,6 +76,7 @@ jobs: tool: protoc@3.23.4 - uses: actions-rust-lang/setup-rust-toolchain@v1 with: + cache: false components: clippy - name: Rust Cache uses: Swatinem/rust-cache@v2 @@ -97,10 +99,12 @@ jobs: with: persist-credentials: false - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false - name: Install cargo-shear uses: taiki-e/install-action@v2 with: - tool: cargo-shear@${{ env.CARGO_SHEAR_VERSION }} + tool: cargo-shear@${{ env.SHEAR_VERSION }} fallback: none - name: Check unused dependencies run: make check-udeps @@ -122,6 +126,8 @@ jobs: - name: Install riscv64 cross toolchain run: sudo apt-get update && sudo apt-get install -y gcc-riscv64-linux-gnu g++-riscv64-linux-gnu - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false - name: Add riscv64 target run: rustup target add riscv64gc-unknown-linux-gnu - name: Rust Cache @@ -153,6 +159,8 @@ jobs: with: version: "14.0" - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false - name: Rust Cache uses: Swatinem/rust-cache@v2 with: