mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-11 15:32:56 +00:00
Resolves #2054 **Context**: branch creation needs to wait for GC to acquire `gc_cs` lock, which prevents creating new timelines during GC. However, because individual timeline GC iteration also requires `compaction_cs` lock, branch creation may also need to wait for compactions of multiple timelines. This results in large latency when creating a new branch, which we advertised as *"instantly"*. This PR optimizes the latency of branch creation by separating GC into two phases: 1. Collect GC data (branching points, cutoff LSNs, etc) 2. Perform GC for each timeline The GC bottleneck comes from step 2, which must wait for compaction of multiple timelines. This PR modifies the branch creation and GC functions to allow GC to hold the GC lock only in step 1. As a result, branch creation doesn't need to wait for compaction to finish but only needs to wait for GC data collection step, which is fast.
136 lines
3.8 KiB
YAML
136 lines
3.8 KiB
YAML
name: Check code style and build
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
pull_request:
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash -ex {0}
|
|
|
|
concurrency:
|
|
# Allow only one workflow per any non-`main` branch.
|
|
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || 'anysha' }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
RUST_BACKTRACE: 1
|
|
|
|
jobs:
|
|
check-codestyle-rust:
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# If we want to duplicate this job for different
|
|
# Rust toolchains (e.g. nightly or 1.37.0), add them here.
|
|
rust_toolchain: [1.58]
|
|
os: [ubuntu-latest, macos-latest]
|
|
timeout-minutes: 50
|
|
name: run regression test suite
|
|
runs-on: ${{ matrix.os }}
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
fetch-depth: 2
|
|
|
|
- name: Install rust toolchain ${{ matrix.rust_toolchain }}
|
|
uses: actions-rs/toolchain@v1
|
|
with:
|
|
profile: minimal
|
|
toolchain: ${{ matrix.rust_toolchain }}
|
|
components: rustfmt, clippy
|
|
override: true
|
|
|
|
- name: Check formatting
|
|
run: cargo fmt --all -- --check
|
|
|
|
- name: Install Ubuntu postgres dependencies
|
|
if: matrix.os == 'ubuntu-latest'
|
|
run: |
|
|
sudo apt update
|
|
sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev libssl-dev
|
|
|
|
- name: Install macOS postgres dependencies
|
|
if: matrix.os == 'macos-latest'
|
|
run: brew install flex bison openssl
|
|
|
|
- name: Set pg revision for caching
|
|
id: pg_ver
|
|
run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres)
|
|
|
|
- name: Cache postgres build
|
|
id: cache_pg
|
|
uses: actions/cache@v2
|
|
with:
|
|
path: |
|
|
tmp_install/
|
|
key: ${{ runner.os }}-pg-${{ steps.pg_ver.outputs.pg_rev }}
|
|
|
|
- name: Set extra env for macOS
|
|
if: matrix.os == 'macos-latest'
|
|
run: |
|
|
echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV
|
|
echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV
|
|
|
|
- name: Build postgres
|
|
if: steps.cache_pg.outputs.cache-hit != 'true'
|
|
run: make postgres
|
|
|
|
# Plain configure output can contain weird errors like 'error: C compiler cannot create executables'
|
|
# and the real cause will be inside config.log
|
|
- name: Print configure logs in case of failure
|
|
if: failure()
|
|
continue-on-error: true
|
|
run: |
|
|
echo '' && echo '=== config.log ===' && echo ''
|
|
cat tmp_install/build/config.log
|
|
echo '' && echo '=== configure.log ===' && echo ''
|
|
cat tmp_install/build/configure.log
|
|
|
|
- name: Cache cargo deps
|
|
id: cache_cargo
|
|
uses: actions/cache@v2
|
|
with:
|
|
path: |
|
|
~/.cargo/registry
|
|
!~/.cargo/registry/src
|
|
~/.cargo/git
|
|
target
|
|
key: v1-${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}-rust-${{ matrix.rust_toolchain }}
|
|
|
|
- name: Run cargo clippy
|
|
run: ./run_clippy.sh
|
|
|
|
- name: Ensure all project builds
|
|
run: cargo build --all --all-targets
|
|
|
|
check-codestyle-python:
|
|
runs-on: [ self-hosted, Linux, k8s-runner ]
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v3
|
|
with:
|
|
submodules: false
|
|
fetch-depth: 1
|
|
|
|
- name: Cache poetry deps
|
|
id: cache_poetry
|
|
uses: actions/cache@v3
|
|
with:
|
|
path: ~/.cache/pypoetry/virtualenvs
|
|
key: v1-codestyle-python-deps-${{ hashFiles('poetry.lock') }}
|
|
|
|
- name: Install Python deps
|
|
run: ./scripts/pysync
|
|
|
|
- name: Run yapf to ensure code format
|
|
run: poetry run yapf --recursive --diff .
|
|
|
|
- name: Run mypy to check types
|
|
run: poetry run mypy .
|