updated the syntax of filter

refactored the code to use a method to combine two line together

Co-authored-by: Alexander Bayandin <alexander@neon.tech>

.github/workflows/build_and_test.yml

@@ -141,7 +141,7 @@ jobs:
 
       # Some of our rust modules use FFI and need those to be checked
       - name: Get postgres headers
-        run: make postgres-headers -j$(nproc)
+        run: CC=clang make postgres-headers -j$(nproc)
 
       # cargo hack runs the given cargo subcommand (clippy in this case) for all feature combinations.
       # This will catch compiler & clippy warnings in all feature combinations.
@@ -260,21 +260,35 @@ jobs:
       #
       # We run tests with addtional features, that are turned off by default (e.g. in release builds), see
       # corresponding Cargo.toml files for their descriptions.
+      # ASAN_OPTIONS and UBSAN_OPTIONS are flags to enable sanitizers on build that will test for memory leaks and undefined behaviour
       - name: Set env variables
         run: |
           CARGO_FEATURES="--features testing"
           if [[ $BUILD_TYPE == "debug" ]]; then
             cov_prefix="scripts/coverage --profraw-prefix=$GITHUB_JOB --dir=/tmp/coverage run"
             CARGO_FLAGS="--locked"
+            ASAN_OPTIONS="detect_leaks=0:abort_on_error=1:print_stacktrace=1:disable_coredump=0:\
+          strict_string_checks=1:check_initialization_order=1:strict_init_order=1:detect_stack_use_after_return=0"
+            UBSAN_OPTIONS="abort_on_error=1:print_stacktrace=1"
+            CC=clang
+            RUSTFLAGS="-C linker=clang"
           elif [[ $BUILD_TYPE == "release" ]]; then
             cov_prefix=""
             CARGO_FLAGS="--locked --release"
+            ASAN_OPTIONS=""
+            UBSAN_OPTIONS=""
+            CC=gcc
+            RUSTFLAGS=""
           fi
           {
             echo "cov_prefix=${cov_prefix}"
             echo "CARGO_FEATURES=${CARGO_FEATURES}"
             echo "CARGO_FLAGS=${CARGO_FLAGS}"
             echo "CARGO_HOME=${GITHUB_WORKSPACE}/.cargo"
+            echo "ASAN_OPTIONS=${ASAN_OPTIONS}"
+            echo "UBSAN_OPTIONS=${UBSAN_OPTIONS}"
+            echo "CC=${CC}"
+            echo "RUSTFLAGS=${RUSTFLAGS}"
           } >> $GITHUB_ENV
 
       # Disabled for now
@@ -341,7 +355,7 @@ jobs:
 
       - name: Run rust tests
         run: |
-          ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES
+          ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E 'not (test(short_v14_) | test(test_find_end_of_wal))'
 
           # Run separate tests for real S3
           export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty
@@ -426,6 +440,8 @@ jobs:
       matrix:
         build_type: [ debug, release ]
         pg_version: [ v14, v15, v16 ]
+    env:
+      BUILD_TYPE: ${{ matrix.build_type }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -433,6 +449,24 @@ jobs:
           submodules: true
           fetch-depth: 1
 
+      # Set environment variable used by application at runtime
+      # ASAN_OPTIONS and UBSAN_OPTIONS are flags to enable sanitizers on build
+      # The above sanitizers will test for memory leaks and undefined behaviour
+      - name: Set env variables
+        run: |
+          if [[ $BUILD_TYPE == "debug" ]]; then
+            ASAN_OPTIONS="detect_leaks=0:abort_on_error=1:print_stacktrace=1:disable_coredump=0:\
+          strict_string_checks=1:check_initialization_order=1:strict_init_order=1:detect_stack_use_after_return=0"
+            UBSAN_OPTIONS="abort_on_error=1:print_stacktrace=1"
+          elif [[ $BUILD_TYPE == "release" ]]; then
+            ASAN_OPTIONS=""
+            UBSAN_OPTIONS=""
+          fi
+          {
+            echo "ASAN_OPTIONS=${ASAN_OPTIONS}"
+            echo "UBSAN_OPTIONS=${UBSAN_OPTIONS}"
+          } >> $GITHUB_ENV
+
       - name: Pytest regression tests
         uses: ./.github/actions/run-python-test-set
         with:

.gitmodules

@@ -9,4 +9,4 @@
 [submodule "vendor/postgres-v16"]
 	path = vendor/postgres-v16
 	url = https://github.com/neondatabase/postgres.git
-	branch = REL_16_STABLE_neon
+	branch = add-build-sanitizers

Makefile

@@ -13,9 +13,13 @@ ifeq ($(BUILD_TYPE),release)
 	PG_CFLAGS = -O2 -g3 $(CFLAGS)
 	# Unfortunately, `--profile=...` is a nightly feature
 	CARGO_BUILD_FLAGS += --release
+	CPPFLAGS =
+	LDFLAGS =
 else ifeq ($(BUILD_TYPE),debug)
 	PG_CONFIGURE_OPTS = --enable-debug --with-openssl --enable-cassert --enable-depend
 	PG_CFLAGS = -O0 -g3 $(CFLAGS)
+	CPPFLAGS = -fsanitize=address -fsanitize=undefined -fno-sanitize-recover -fno-sanitize=alignment -Wno-cast-function-type-strict
+	LDFLAGS = -fsanitize=address -fsanitize=undefined -static-libsan
 else
 	$(error Bad build type '$(BUILD_TYPE)', see Makefile for options)
 endif
@@ -23,7 +27,8 @@ endif
 UNAME_S := $(shell uname -s)
 ifeq ($(UNAME_S),Linux)
 	# Seccomp BPF is only available for Linux
-	PG_CONFIGURE_OPTS += --with-libseccomp
+	#PG_CONFIGURE_OPTS += --with-libseccomp
+	NO_PG_CONFIGURE_OPTS += --with-libseccomp # libseccomp needs additional adjustments
 else ifeq ($(UNAME_S),Darwin)
 	# macOS with brew-installed openssl requires explicit paths
 	# It can be configured with OPENSSL_PREFIX variable
@@ -80,6 +85,8 @@ $(POSTGRES_INSTALL_DIR)/build/%/config.status:
 	(cd $(POSTGRES_INSTALL_DIR)/build/$* && \
 	env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-$*/configure \
 		CFLAGS='$(PG_CFLAGS)' \
+		CPPFLAGS='$(CPPFLAGS)' \
+		LDFLAGS='$(LDFLAGS)' \
 		$(PG_CONFIGURE_OPTS) \
 		--prefix=$(abspath $(POSTGRES_INSTALL_DIR))/$* > configure.log)
 

control_plane/src/background_process.rs

@@ -230,7 +230,13 @@ fn fill_rust_env_vars(cmd: &mut Command) -> &mut Command {
     let mut filled_cmd = cmd.env_clear().env("RUST_BACKTRACE", backtrace_setting);
 
     // Pass through these environment variables to the command
-    for var in ["LLVM_PROFILE_FILE", "FAILPOINTS", "RUST_LOG"] {
+    for var in [
+        "LLVM_PROFILE_FILE",
+        "FAILPOINTS",
+        "RUST_LOG",
+        "ASAN_OPTIONS",
+        "UBSAN_OPTIONS",
+    ] {
         if let Some(val) = std::env::var_os(var) {
             filled_cmd = filled_cmd.env(var, val);
         }

libs/postgres_ffi/wal_craft/src/lib.rs

@@ -74,6 +74,14 @@ impl Conf {
         ensure!(path.exists(), "Command {:?} does not exist", path);
         let mut cmd = Command::new(path);
         cmd.env_clear()
+            .env(
+                "ASAN_OPTIONS",
+                std::env::var("ASAN_OPTIONS").unwrap_or_default(),
+            )
+            .env(
+                "UBSAN_OPTIONS",
+                std::env::var("UBSAN_OPTIONS").unwrap_or_default(),
+            )
             .env("LD_LIBRARY_PATH", self.pg_lib_dir()?)
             .env("DYLD_LIBRARY_PATH", self.pg_lib_dir()?);
         Ok(cmd)

libs/utils/scripts/restore_from_wal.sh

@@ -8,7 +8,7 @@ DATA_DIR=$3
 PORT=$4
 SYSID=$(od -A n -j 24 -N 8 -t d8 "$WAL_PATH"/000000010000000000000002* | cut -c 3-)
 rm -fr "$DATA_DIR"
-env -i LD_LIBRARY_PATH="$PG_BIN"/../lib "$PG_BIN"/initdb -E utf8 -U cloud_admin -D "$DATA_DIR" --sysid="$SYSID"
+env -i LD_LIBRARY_PATH="$PG_BIN"/../lib ASAN_OPTIONS="$ASAN_OPTIONS" UBSAN_OPTIONS="$UBSAN_OPTIONS" "$PG_BIN"/initdb -E utf8 -U cloud_admin -D "$DATA_DIR" --sysid="$SYSID"
 echo "port=$PORT" >> "$DATA_DIR"/postgresql.conf
 echo "shared_preload_libraries='\$libdir/neon_rmgr.so'" >> "$DATA_DIR"/postgresql.conf
 REDO_POS=0x$("$PG_BIN"/pg_controldata -D "$DATA_DIR" | grep -F "REDO location"| cut -c 42-)

libs/walproposer/build.rs

@@ -29,6 +29,12 @@ fn main() -> anyhow::Result<()> {
     let pgxn_neon = std::fs::canonicalize(pgxn_neon)?;
     let pgxn_neon = pgxn_neon.to_str().ok_or(anyhow!("Bad non-UTF path"))?;
 
+    #[cfg(debug_assertions)]
+    {
+        println!("cargo:rustc-link-arg=-fsanitize=address");
+        println!("cargo:rustc-link-arg=-fsanitize=undefined");
+        println!("cargo:rustc-link-arg=-static-libsan");
+    }
     println!("cargo:rustc-link-lib=static=pgport");
     println!("cargo:rustc-link-lib=static=pgcommon");
     println!("cargo:rustc-link-lib=static=walproposer");

pageserver/src/tenant.rs

@@ -3755,6 +3755,14 @@ async fn run_initdb(
         .env_clear()
         .env("LD_LIBRARY_PATH", &initdb_lib_dir)
         .env("DYLD_LIBRARY_PATH", &initdb_lib_dir)
+        .env(
+            "ASAN_OPTIONS",
+            std::env::var("ASAN_OPTIONS").unwrap_or_default(),
+        )
+        .env(
+            "UBSAN_OPTIONS",
+            std::env::var("UBSAN_OPTIONS").unwrap_or_default(),
+        )
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
         // If the `select!` below doesn't finish the `wait_with_output`,

pgxn/neon/file_cache.c

@@ -405,7 +405,7 @@ lfc_cache_contains(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno)
 	if (LFC_ENABLED())
 	{
 		entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL);
-		found = entry != NULL && (entry->bitmap[chunk_offs >> 5] & (1 << (chunk_offs & 31))) != 0;
+		found = entry != NULL && (entry->bitmap[chunk_offs >> 5] & ((uint32)1 << (chunk_offs & 31))) != 0;
 	}
 	LWLockRelease(lfc_lock);
 	return found;
@@ -450,7 +450,7 @@ lfc_evict(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno)
 	}
 
 	/* remove the page from the cache */
-	entry->bitmap[chunk_offs >> 5] &= ~(1 << (chunk_offs & (32 - 1)));
+	entry->bitmap[chunk_offs >> 5] &= ~((uint32)1 << (chunk_offs & (32 - 1)));
 
 	/*
 	 * If the chunk has no live entries, we can position the chunk to be
@@ -526,7 +526,7 @@ lfc_read(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
 	}
 
 	entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL);
-	if (entry == NULL || (entry->bitmap[chunk_offs >> 5] & (1 << (chunk_offs & 31))) == 0)
+	if (entry == NULL || (entry->bitmap[chunk_offs >> 5] & ((uint32)1 << (chunk_offs & 31))) == 0)
 	{
 		/* Page is not cached */
 		lfc_ctl->misses += 1;
@@ -678,7 +678,7 @@ lfc_write(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, const void
 			if (--entry->access_count == 0)
 				dlist_push_tail(&lfc_ctl->lru, &entry->lru_node);
 
-			entry->bitmap[chunk_offs >> 5] |= (1 << (chunk_offs & 31));
+			entry->bitmap[chunk_offs >> 5] |= ((uint32)1 << (chunk_offs & 31));
 		}
 
 		LWLockRelease(lfc_lock);
@@ -913,7 +913,7 @@ local_cache_pages(PG_FUNCTION_ARGS)
 			{
 				for (int i = 0; i < BLOCKS_PER_CHUNK; i++)
 				{
-					if (entry->bitmap[i >> 5] & (1 << (i & 31)))
+					if (entry->bitmap[i >> 5] & ((uint32)1 << (i & 31)))
 					{
 						fctx->record[n].pageoffs = entry->offset * BLOCKS_PER_CHUNK + i;
 						fctx->record[n].relfilenode = NInfoGetRelNumber(BufTagGetNRelFileInfo(entry->key));

pgxn/neon/walproposer.c

@@ -993,7 +993,8 @@ DetermineEpochStartLsn(WalProposer *wp)
 	dth = &wp->safekeeper[wp->donor].voteResponse.termHistory;
 	wp->propTermHistory.n_entries = dth->n_entries + 1;
 	wp->propTermHistory.entries = palloc(sizeof(TermSwitchEntry) * wp->propTermHistory.n_entries);
-	memcpy(wp->propTermHistory.entries, dth->entries, sizeof(TermSwitchEntry) * dth->n_entries);
+	if (dth->n_entries > 0)
+		memcpy(wp->propTermHistory.entries, dth->entries, sizeof(TermSwitchEntry) * dth->n_entries);
 	wp->propTermHistory.entries[wp->propTermHistory.n_entries - 1].term = wp->propTerm;
 	wp->propTermHistory.entries[wp->propTermHistory.n_entries - 1].lsn = wp->propEpochStartLsn;
 

test_runner/regress/test_pg_regress.py

@@ -20,7 +20,9 @@ def test_pg_regress(
 
     env.neon_cli.create_branch("test_pg_regress", "empty")
     # Connect to postgres and create a database called "regression".
-    endpoint = env.endpoints.create_start("test_pg_regress")
+    endpoint = env.endpoints.create_start(
+        "test_pg_regress", config_lines=["max_stack_depth = 4096kB"]
+    )  # stack should be increased for tests to pass with asan
     endpoint.safe_psql("CREATE DATABASE regression")
 
     # Create some local directories for pg_regress to run in.
@@ -75,7 +77,7 @@ def test_isolation(
     # Connect to postgres and create a database called "regression".
     # isolation tests use prepared transactions, so enable them
     endpoint = env.endpoints.create_start(
-        "test_isolation", config_lines=["max_prepared_transactions=100"]
+        "test_isolation", config_lines=["max_prepared_transactions=100", "max_stack_depth = 4096kB"]
     )
     endpoint.safe_psql("CREATE DATABASE isolation_regression")
 

test_runner/regress/test_tenant_detach.py

@@ -34,6 +34,7 @@ PERMIT_PAGE_SERVICE_ERRORS = [
     ".*page_service.*Tenant .* is not active",
     ".*page_service.*cancelled",
     ".*page_service.*will not become active.*",
+    ".*Dropped remote consistent LSN updates for tenant.*",
 ]
 
 
@@ -747,8 +748,11 @@ def test_metrics_while_ignoring_broken_tenant_and_reloading(
     env = neon_env_builder.init_start()
 
     client = env.pageserver.http_client()
-    env.pageserver.allowed_errors.append(
-        r".* Changing Active tenant to Broken state, reason: broken from test"
+    env.pageserver.allowed_errors.extend(
+        [
+            r".* Changing Active tenant to Broken state, reason: broken from test",
+            r".*Dropped remote consistent LSN updates for tenant *.",
+        ]
     )
 
     def only_int(samples: List[Sample]) -> Optional[int]:

vendor/revisions.json

@@ -1,5 +1,5 @@
 {
-    "postgres-v16": "7be4a52d728459b79b59343c57d338c3073059c8",
+    "postgres-v16": "6f1ef95071c00f85df45132dd2c6cd0405835870",
     "postgres-v15": "c1c2272f436ed9231f6172f49de219fe71a9280d",
     "postgres-v14": "82072911287cabb32018cf92c8425fa1c744def4"
 }