rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-05-14 19:50:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

testing if santiser work

Browse Source 
moved sanitizers in its owm workflow

merged all jobs into onme

cleaned up failing job

cleaned up failing job

running just tests

fixing build

reverting changes

fixing linter error and build error

clearning up job

added wal and extension builds

fixing build

fixing build

fixing build

added use sanitizer patch

testing if sanitiser work in main workflow

fixed format issue

fixing format issue

fixing format issue

added flags

disabled flags

enabling flags

enabling flags

added more options to flag

fixing build

fixing build

testing the regression run

added asan and usban flag for regression test

commented unit test and release build

fixing build

fix neon for sanitizers

enabled unit test

updated branch to test the fix

updated branch to test the fix

updated the commit id

fixing build

restoring the submodules to main

updated git modules and revision of commit

updated postgres 16 vendor dir

removed test
This commit is contained in:
abhijeet
2024-01-04 13:09:41 +00:00
committed by Abhijeet Patil
parent c65ac37a6d
commit 330d9a8b02
13 changed files with 102 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
.github/workflows/build_and_test.yml vendored
View File

@@ -141,7 +141,7 @@ jobs:
# Some of our rust modules use FFI and need those to be checked
- name: Get postgres headers
run: make postgres-headers -j$(nproc)
run: CC=clang CARGO_BUILD_FLAGS="--features=testing" make postgres-headers -j$(nproc)
# cargo hack runs the given cargo subcommand (clippy in this case) for all feature combinations.
# This will catch compiler & clippy warnings in all feature combinations.
@@ -260,18 +260,29 @@ jobs:
#
# We run tests with addtional features, that are turned off by default (e.g. in release builds), see
# corresponding Cargo.toml files for their descriptions.
# ASAN_OPTIONS and UBSAN_OPTIONS are flags to enable sanitizers on build that will test for memory leaks and undefined behaviour
- name: Set env variables
run: |
CARGO_FEATURES="--features testing"
if [[ $BUILD_TYPE == "debug" ]]; then
cov_prefix="scripts/coverage --profraw-prefix=$GITHUB_JOB --dir=/tmp/coverage run"
CARGO_FLAGS="--locked"
ASAN_OPTIONS="detect_leaks=0:abort_on_error=1:print_stacktrace=1:disable_coredump=0:\
strict_string_checks=1:check_initialization_order=1:strict_init_order=1:detect_stack_use_after_return=0"
UBSAN_OPTIONS="abort_on_error=1:print_stacktrace=1"
RUSTFLAGS="-C linker=clang"
elif [[ $BUILD_TYPE == "release" ]]; then
cov_prefix=""
ASAN_OPTIONS=""
UBSAN_OPTIONS=""
RUSTFLAGS=""
CARGO_FLAGS="--locked --release"
fi
{
echo "cov_prefix=${cov_prefix}"
echo "ASAN_OPTIONS=${ASAN_OPTIONS}"
echo "UBSAN_OPTIONS=${UBSAN_OPTIONS}"
echo "RUSTFLAGS=${RUSTFLAGS}"
echo "CARGO_FEATURES=${CARGO_FEATURES}"
echo "CARGO_FLAGS=${CARGO_FLAGS}"
echo "CARGO_HOME=${GITHUB_WORKSPACE}/.cargo"
@@ -319,46 +330,46 @@ jobs:
- name: Build postgres v14
if: steps.cache_pg_14.outputs.cache-hit != 'true'
run: mold -run make postgres-v14 -j$(nproc)
run: CC=clang CARGO_BUILD_FLAGS="--features=testing" mold -run make postgres-v14 -j$(nproc)
- name: Build postgres v15
if: steps.cache_pg_15.outputs.cache-hit != 'true'
run: mold -run make postgres-v15 -j$(nproc)
run: CC=clang CARGO_BUILD_FLAGS="--features=testing" mold -run make postgres-v15 -j$(nproc)
- name: Build postgres v16
if: steps.cache_pg_16.outputs.cache-hit != 'true'
run: mold -run make postgres-v16 -j$(nproc)
run: CC=clang CARGO_BUILD_FLAGS="--features=testing" mold -run make postgres-v16 -j$(nproc)
- name: Build neon extensions
run: mold -run make neon-pg-ext -j$(nproc)
run: CC=clang CARGO_BUILD_FLAGS="--features=testing" mold -run make neon-pg-ext -j$(nproc)
- name: Build walproposer-lib
run: mold -run make walproposer-lib -j$(nproc)
run: CC=clang CARGO_BUILD_FLAGS="--features=testing" mold -run make walproposer-lib -j$(nproc)
- name: Run cargo build
run: |
${cov_prefix} mold -run cargo build $CARGO_FLAGS $CARGO_FEATURES --bins --tests
- name: Run rust tests
run: |
${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES
# - name: Run rust tests
# run: |
# ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES
# Run separate tests for real S3
export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty
export REMOTE_STORAGE_S3_BUCKET=neon-github-ci-tests
export REMOTE_STORAGE_S3_REGION=eu-central-1
# Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now
${cov_prefix} cargo nextest run $CARGO_FLAGS -E 'package(remote_storage)' -E 'test(test_real_s3)'
# # Run separate tests for real S3
# export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty
# export REMOTE_STORAGE_S3_BUCKET=neon-github-ci-tests
# export REMOTE_STORAGE_S3_REGION=eu-central-1
# # Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now
# ${cov_prefix} cargo nextest run $CARGO_FLAGS -E 'package(remote_storage)' -E 'test(test_real_s3)'
# Run separate tests for real Azure Blob Storage
# XXX: replace region with `eu-central-1`-like region
export ENABLE_REAL_AZURE_REMOTE_STORAGE=y
export AZURE_STORAGE_ACCOUNT="${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }}"
export AZURE_STORAGE_ACCESS_KEY="${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }}"
export REMOTE_STORAGE_AZURE_CONTAINER="${{ vars.REMOTE_STORAGE_AZURE_CONTAINER }}"
export REMOTE_STORAGE_AZURE_REGION="${{ vars.REMOTE_STORAGE_AZURE_REGION }}"
# Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now
${cov_prefix} cargo nextest run $CARGO_FLAGS -E 'package(remote_storage)' -E 'test(test_real_azure)'
# # Run separate tests for real Azure Blob Storage
# # XXX: replace region with `eu-central-1`-like region
# export ENABLE_REAL_AZURE_REMOTE_STORAGE=y
# export AZURE_STORAGE_ACCOUNT="${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }}"
# export AZURE_STORAGE_ACCESS_KEY="${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }}"
# export REMOTE_STORAGE_AZURE_CONTAINER="${{ vars.REMOTE_STORAGE_AZURE_CONTAINER }}"
# export REMOTE_STORAGE_AZURE_REGION="${{ vars.REMOTE_STORAGE_AZURE_REGION }}"
# # Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now
# ${cov_prefix} cargo nextest run $CARGO_FLAGS -E 'package(remote_storage)' -E 'test(test_real_azure)'
- name: Install rust binaries
run: |
@@ -424,8 +435,10 @@ jobs:
strategy:
fail-fast: false
matrix:
build_type: [ debug, release ]
build_type: [ debug ]
pg_version: [ v14, v15, v16 ]
env:
BUILD_TYPE: ${{ matrix.build_type }}
steps:
- name: Checkout
uses: actions/checkout@v3
@@ -433,6 +446,24 @@ jobs:
submodules: true
fetch-depth: 1
# Set environment variable used by application at runtime
# ASAN_OPTIONS and UBSAN_OPTIONS are flags to enable sanitizers on build
# The above sanitizers will test for memory leaks and undefined behaviour
- name: Set env variables
run: |
if [[ $BUILD_TYPE == "debug" ]]; then
ASAN_OPTIONS="detect_leaks=0:abort_on_error=1:print_stacktrace=1:disable_coredump=0:\
strict_string_checks=1:check_initialization_order=1:strict_init_order=1:detect_stack_use_after_return=0"
UBSAN_OPTIONS="abort_on_error=1:print_stacktrace=1"
elif [[ $BUILD_TYPE == "release" ]]; then
ASAN_OPTIONS=""
UBSAN_OPTIONS=""
fi
{
echo "ASAN_OPTIONS=${ASAN_OPTIONS}"
echo "UBSAN_OPTIONS=${UBSAN_OPTIONS}"
} >> $GITHUB_ENV
- name: Pytest regression tests
uses: ./.github/actions/run-python-test-set
with:

2
.gitmodules vendored
View File

@@ -9,4 +9,4 @@
[submodule "vendor/postgres-v16"]
path = vendor/postgres-v16
url = https://github.com/neondatabase/postgres.git
branch = REL_16_STABLE_neon
branch = add-build-sanitizers

4
Makefile
View File

@@ -23,7 +23,7 @@ endif
UNAME_S := $(shell uname -s)
ifeq ($(UNAME_S),Linux)
# Seccomp BPF is only available for Linux
PG_CONFIGURE_OPTS += --with-libseccomp
NO_PG_CONFIGURE_OPTS += --with-libseccomp # libseccomp needs additional adjustments
else ifeq ($(UNAME_S),Darwin)
# macOS with brew-installed openssl requires explicit paths
# It can be configured with OPENSSL_PREFIX variable
@@ -80,6 +80,8 @@ $(POSTGRES_INSTALL_DIR)/build/%/config.status:
(cd $(POSTGRES_INSTALL_DIR)/build/$* && \
env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-$*/configure \
CFLAGS='$(PG_CFLAGS)' \
CPPFLAGS='-fsanitize=address -fsanitize=undefined -fno-sanitize-recover -fno-sanitize=function -Wno-cast-function-type-strict' \
LDFLAGS='-fsanitize=address -fsanitize=undefined -static-libsan' \
$(PG_CONFIGURE_OPTS) \
--prefix=$(abspath $(POSTGRES_INSTALL_DIR))/$* > configure.log)

8
control_plane/src/background_process.rs
View File

@@ -230,7 +230,13 @@ fn fill_rust_env_vars(cmd: &mut Command) -> &mut Command {
let mut filled_cmd = cmd.env_clear().env("RUST_BACKTRACE", backtrace_setting);
// Pass through these environment variables to the command
for var in ["LLVM_PROFILE_FILE", "FAILPOINTS", "RUST_LOG"] {
for var in [
"LLVM_PROFILE_FILE",
"FAILPOINTS",
"RUST_LOG",
"ASAN_OPTIONS",
"UBSAN_OPTIONS",
] {
if let Some(val) = std::env::var_os(var) {
filled_cmd = filled_cmd.env(var, val);
}

8
libs/postgres_ffi/wal_craft/src/lib.rs
View File

@@ -74,6 +74,14 @@ impl Conf {
ensure!(path.exists(), "Command {:?} does not exist", path);
let mut cmd = Command::new(path);
cmd.env_clear()
.env(
"ASAN_OPTIONS",
std::env::var("ASAN_OPTIONS").unwrap_or_default(),
)
.env(
"UBSAN_OPTIONS",
std::env::var("UBSAN_OPTIONS").unwrap_or_default(),
)
.env("LD_LIBRARY_PATH", self.pg_lib_dir()?)
.env("DYLD_LIBRARY_PATH", self.pg_lib_dir()?);
Ok(cmd)

2
libs/utils/scripts/restore_from_wal.sh
View File

@@ -8,7 +8,7 @@ DATA_DIR=$3
PORT=$4
SYSID=$(od -A n -j 24 -N 8 -t d8 "$WAL_PATH"/000000010000000000000002* | cut -c 3-)
rm -fr "$DATA_DIR"
env -i LD_LIBRARY_PATH="$PG_BIN"/../lib "$PG_BIN"/initdb -E utf8 -U cloud_admin -D "$DATA_DIR" --sysid="$SYSID"
env -i LD_LIBRARY_PATH="$PG_BIN"/../lib ASAN_OPTIONS="$ASAN_OPTIONS" UBSAN_OPTIONS="$UBSAN_OPTIONS" "$PG_BIN"/initdb -E utf8 -U cloud_admin -D "$DATA_DIR" --sysid="$SYSID"
echo "port=$PORT" >> "$DATA_DIR"/postgresql.conf
echo "shared_preload_libraries='\$libdir/neon_rmgr.so'" >> "$DATA_DIR"/postgresql.conf
REDO_POS=0x$("$PG_BIN"/pg_controldata -D "$DATA_DIR" | grep -F "REDO location"| cut -c 42-)

3
libs/walproposer/build.rs
View File

@@ -29,6 +29,9 @@ fn main() -> anyhow::Result<()> {
let pgxn_neon = std::fs::canonicalize(pgxn_neon)?;
let pgxn_neon = pgxn_neon.to_str().ok_or(anyhow!("Bad non-UTF path"))?;
println!("cargo:rustc-link-arg=-fsanitize=address");
println!("cargo:rustc-link-arg=-fsanitize=undefined");
println!("cargo:rustc-link-arg=-static-libsan");
println!("cargo:rustc-link-lib=static=pgport");
println!("cargo:rustc-link-lib=static=pgcommon");
println!("cargo:rustc-link-lib=static=walproposer");

8
pageserver/src/tenant.rs
View File

@@ -3755,6 +3755,14 @@ async fn run_initdb(
.env_clear()
.env("LD_LIBRARY_PATH", &initdb_lib_dir)
.env("DYLD_LIBRARY_PATH", &initdb_lib_dir)
.env(
"ASAN_OPTIONS",
std::env::var("ASAN_OPTIONS").unwrap_or_default(),
)
.env(
"UBSAN_OPTIONS",
std::env::var("UBSAN_OPTIONS").unwrap_or_default(),
)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
// If the `select!` below doesn't finish the `wait_with_output`,

10
pgxn/neon/file_cache.c
View File

@@ -405,7 +405,7 @@ lfc_cache_contains(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno)
if (LFC_ENABLED())
{
entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL);
found = entry != NULL && (entry->bitmap[chunk_offs >> 5] & (1 << (chunk_offs & 31))) != 0;
found = entry != NULL && (entry->bitmap[chunk_offs >> 5] & ((uint32)1 << (chunk_offs & 31))) != 0;
}
LWLockRelease(lfc_lock);
return found;
@@ -450,7 +450,7 @@ lfc_evict(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno)
}
/* remove the page from the cache */
entry->bitmap[chunk_offs >> 5] &= ~(1 << (chunk_offs & (32 - 1)));
entry->bitmap[chunk_offs >> 5] &= ~((uint32)1 << (chunk_offs & (32 - 1)));
/*
* If the chunk has no live entries, we can position the chunk to be
@@ -526,7 +526,7 @@ lfc_read(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
}
entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL);
if (entry == NULL || (entry->bitmap[chunk_offs >> 5] & (1 << (chunk_offs & 31))) == 0)
if (entry == NULL || (entry->bitmap[chunk_offs >> 5] & ((uint32)1 << (chunk_offs & 31))) == 0)
{
/* Page is not cached */
lfc_ctl->misses += 1;
@@ -678,7 +678,7 @@ lfc_write(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, const void
if (--entry->access_count == 0)
dlist_push_tail(&lfc_ctl->lru, &entry->lru_node);
entry->bitmap[chunk_offs >> 5] |= (1 << (chunk_offs & 31));
entry->bitmap[chunk_offs >> 5] |= ((uint32)1 << (chunk_offs & 31));
}
LWLockRelease(lfc_lock);
@@ -913,7 +913,7 @@ local_cache_pages(PG_FUNCTION_ARGS)
{
for (int i = 0; i < BLOCKS_PER_CHUNK; i++)
{
if (entry->bitmap[i >> 5] & (1 << (i & 31)))
if (entry->bitmap[i >> 5] & ((uint32)1 << (i & 31)))
{
fctx->record[n].pageoffs = entry->offset * BLOCKS_PER_CHUNK + i;
fctx->record[n].relfilenode = NInfoGetRelNumber(BufTagGetNRelFileInfo(entry->key));

3
pgxn/neon/walproposer.c
View File

@@ -993,7 +993,8 @@ DetermineEpochStartLsn(WalProposer *wp)
dth = &wp->safekeeper[wp->donor].voteResponse.termHistory;
wp->propTermHistory.n_entries = dth->n_entries + 1;
wp->propTermHistory.entries = palloc(sizeof(TermSwitchEntry) * wp->propTermHistory.n_entries);
memcpy(wp->propTermHistory.entries, dth->entries, sizeof(TermSwitchEntry) * dth->n_entries);
if (dth->n_entries > 0)
memcpy(wp->propTermHistory.entries, dth->entries, sizeof(TermSwitchEntry) * dth->n_entries);
wp->propTermHistory.entries[wp->propTermHistory.n_entries - 1].term = wp->propTerm;
wp->propTermHistory.entries[wp->propTermHistory.n_entries - 1].lsn = wp->propEpochStartLsn;

6
test_runner/regress/test_pg_regress.py
View File

@@ -20,7 +20,9 @@ def test_pg_regress(
env.neon_cli.create_branch("test_pg_regress", "empty")
# Connect to postgres and create a database called "regression".
endpoint = env.endpoints.create_start("test_pg_regress")
endpoint = env.endpoints.create_start(
"test_pg_regress", config_lines=["max_stack_depth = 4096kB"]
) # stack should be increased for tests to pass with asan
endpoint.safe_psql("CREATE DATABASE regression")
# Create some local directories for pg_regress to run in.
@@ -75,7 +77,7 @@ def test_isolation(
# Connect to postgres and create a database called "regression".
# isolation tests use prepared transactions, so enable them
endpoint = env.endpoints.create_start(
"test_isolation", config_lines=["max_prepared_transactions=100"]
"test_isolation", config_lines=["max_prepared_transactions=100", "max_stack_depth = 4096kB"]
)
endpoint.safe_psql("CREATE DATABASE isolation_regression")

2
vendor/postgres-v16 vendored

Submodule vendor/postgres-v16 updated: 7be4a52d72...2088022bc7

6
vendor/revisions.json vendored
View File

@@ -1,5 +1,5 @@
{
"postgres-v16": "7be4a52d728459b79b59343c57d338c3073059c8",
"postgres-v15": "c1c2272f436ed9231f6172f49de219fe71a9280d",
"postgres-v14": "82072911287cabb32018cf92c8425fa1c744def4"
"postgres-v16": "2088022bc73e9d3d08b2817fbcffa72412e3bc7a",
"postgres-v15": "a2dc225ddfc8cae1849aa2316f435c58f0333d8c",
"postgres-v14": "03358bb0b5e0d33c238710139e768db9e75cfcc8"
}