diff --git a/libs/utils/src/logging.rs b/libs/utils/src/logging.rs index d67c0f123b..9f118048f3 100644 --- a/libs/utils/src/logging.rs +++ b/libs/utils/src/logging.rs @@ -34,13 +34,16 @@ macro_rules! critical { #[macro_export] macro_rules! critical_timeline { - ($tenant_shard_id:expr, $timeline_id:expr, $($arg:tt)*) => {{ + ($tenant_shard_id:expr, $timeline_id:expr, $corruption_detected:expr, $($arg:tt)*) => {{ if cfg!(debug_assertions) { panic!($($arg)*); } // Increment both metrics $crate::logging::TRACING_EVENT_COUNT_METRIC.inc_critical(); $crate::logging::HADRON_CRITICAL_STORAGE_EVENT_COUNT_METRIC.inc(&$tenant_shard_id.to_string(), &$timeline_id.to_string()); + if let Some(c) = $corruption_detected.as_ref() { + c.store(true, std::sync::atomic::Ordering::Relaxed); + } let backtrace = std::backtrace::Backtrace::capture(); tracing::error!("CRITICAL: [tenant_shard_id: {}, timeline_id: {}] {}\n{backtrace}", $tenant_shard_id, $timeline_id, format!($($arg)*)); diff --git a/libs/utils/src/pageserver_feedback.rs b/libs/utils/src/pageserver_feedback.rs index cffbc0b4d6..da5b53306a 100644 --- a/libs/utils/src/pageserver_feedback.rs +++ b/libs/utils/src/pageserver_feedback.rs @@ -32,6 +32,9 @@ pub struct PageserverFeedback { pub replytime: SystemTime, /// Used to track feedbacks from different shards. Always zero for unsharded tenants. pub shard_number: u32, + /// If true, the pageserver has detected corruption and the safekeeper and postgres + /// should stop sending WAL. + pub corruption_detected: bool, } impl PageserverFeedback { @@ -43,6 +46,7 @@ impl PageserverFeedback { disk_consistent_lsn: Lsn::INVALID, replytime: *PG_EPOCH, shard_number: 0, + corruption_detected: false, } } @@ -101,6 +105,13 @@ impl PageserverFeedback { buf.put_u32(self.shard_number); } + if self.corruption_detected { + nkeys += 1; + buf.put_slice(b"corruption_detected\0"); + buf.put_i32(1); + buf.put_u8(1); + } + buf[buf_ptr] = nkeys; } @@ -147,6 +158,11 @@ impl PageserverFeedback { assert_eq!(len, 4); rf.shard_number = buf.get_u32(); } + b"corruption_detected" => { + let len = buf.get_i32(); + assert_eq!(len, 1); + rf.corruption_detected = buf.get_u8() != 0; + } _ => { let len = buf.get_i32(); warn!( @@ -206,6 +222,26 @@ mod tests { assert_eq!(rf, rf_parsed); } + // Test that databricks-specific fields added to the PageserverFeedback message are serialized + // and deserialized correctly, in addition to the existing fields from upstream. + #[test] + fn test_replication_feedback_databricks_fields() { + let mut rf = PageserverFeedback::empty(); + rf.current_timeline_size = 12345678; + rf.last_received_lsn = Lsn(23456789); + rf.disk_consistent_lsn = Lsn(34567890); + rf.remote_consistent_lsn = Lsn(45678901); + rf.replytime = *PG_EPOCH + Duration::from_secs(100_000_000); + rf.shard_number = 1; + rf.corruption_detected = true; + + let mut data = BytesMut::new(); + rf.serialize(&mut data); + + let rf_parsed = PageserverFeedback::parse(data.freeze()); + assert_eq!(rf, rf_parsed); + } + #[test] fn test_replication_feedback_unknown_key() { let mut rf = PageserverFeedback::empty(); diff --git a/libs/walproposer/src/api_bindings.rs b/libs/walproposer/src/api_bindings.rs index c3be1e1dae..9f88ea6b11 100644 --- a/libs/walproposer/src/api_bindings.rs +++ b/libs/walproposer/src/api_bindings.rs @@ -426,6 +426,7 @@ pub fn empty_shmem() -> crate::bindings::WalproposerShmemState { remote_consistent_lsn: 0, replytime: 0, shard_number: 0, + corruption_detected: false, }; let empty_wal_rate_limiter = crate::bindings::WalRateLimiter { diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 1b783326a0..f60eab68e6 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -2914,6 +2914,7 @@ pub(crate) struct WalIngestMetrics { pub(crate) records_received: IntCounter, pub(crate) records_observed: IntCounter, pub(crate) records_committed: IntCounter, + pub(crate) records_filtered: IntCounter, pub(crate) values_committed_metadata_images: IntCounter, pub(crate) values_committed_metadata_deltas: IntCounter, pub(crate) values_committed_data_images: IntCounter, @@ -2969,6 +2970,11 @@ pub(crate) static WAL_INGEST: Lazy = Lazy::new(|| { "Number of WAL records which resulted in writes to pageserver storage" ) .expect("failed to define a metric"), + records_filtered: register_int_counter!( + "pageserver_wal_ingest_records_filtered", + "Number of WAL records filtered out due to sharding" + ) + .expect("failed to define a metric"), values_committed_metadata_images: values_committed.with_label_values(&["metadata", "image"]), values_committed_metadata_deltas: values_committed.with_label_values(&["metadata", "delta"]), values_committed_data_images: values_committed.with_label_values(&["data", "image"]), diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 483e9d9a2a..f82c47ec3a 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -397,6 +397,11 @@ pub struct Timeline { /// If true, the last compaction failed. compaction_failed: AtomicBool, + /// Begin Hadron: If true, the pageserver has likely detected data corruption in the timeline. + /// We need to feed this information back to the Safekeeper and postgres for them to take the + /// appropriate action. + corruption_detected: AtomicBool, + /// Notifies the tenant compaction loop that there is pending L0 compaction work. l0_compaction_trigger: Arc, @@ -3310,6 +3315,7 @@ impl Timeline { compaction_lock: tokio::sync::Mutex::default(), compaction_failed: AtomicBool::default(), + corruption_detected: AtomicBool::default(), l0_compaction_trigger: resources.l0_compaction_trigger, gc_lock: tokio::sync::Mutex::default(), @@ -6004,6 +6010,17 @@ impl Timeline { ))) }); + // Begin Hadron + // + fail_point!("create-image-layer-fail-simulated-corruption", |_| { + self.corruption_detected + .store(true, std::sync::atomic::Ordering::Relaxed); + Err(CreateImageLayersError::Other(anyhow::anyhow!( + "failpoint create-image-layer-fail-simulated-corruption" + ))) + }); + // End Hadron + let io_concurrency = IoConcurrency::spawn_from_conf( self.conf.get_vectored_concurrent_io, self.gate @@ -7143,6 +7160,7 @@ impl Timeline { critical_timeline!( self.tenant_shard_id, self.timeline_id, + Some(&self.corruption_detected), "walredo failure during page reconstruction: {err:?}" ); } diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index 9bca952a46..c5363d84b7 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -1397,6 +1397,7 @@ impl Timeline { critical_timeline!( self.tenant_shard_id, self.timeline_id, + Some(&self.corruption_detected), "missing key during compaction: {err:?}" ); } @@ -1441,6 +1442,7 @@ impl Timeline { critical_timeline!( self.tenant_shard_id, self.timeline_id, + Some(&self.corruption_detected), "could not compact, repartitioning keyspace failed: {e:?}" ); } diff --git a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs index f619c69599..ff6a080c52 100644 --- a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs @@ -13,6 +13,7 @@ use fail::fail_point; use futures::StreamExt; use postgres_backend::is_expected_io_error; use postgres_connection::PgConnectionConfig; +use postgres_ffi::waldecoder::WalStreamDecoder; use postgres_ffi::WAL_SEGMENT_SIZE; use postgres_ffi::v14::xlog_utils::normalize_lsn; use postgres_ffi::waldecoder::WalDecodeError; @@ -25,13 +26,13 @@ use tokio_postgres::replication::ReplicationStream; use tokio_postgres::{Client, SimpleQueryMessage, SimpleQueryRow}; use tokio_util::sync::CancellationToken; use tracing::{Instrument, debug, error, info, trace, warn}; -use utils::critical_timeline; +use utils::{critical_timeline}; use utils::id::NodeId; use utils::lsn::Lsn; use utils::pageserver_feedback::PageserverFeedback; use utils::postgres_client::PostgresClientProtocol; use utils::sync::gate::GateError; -use wal_decoder::models::{FlushUncommittedRecords, InterpretedWalRecords}; +use wal_decoder::models::{FlushUncommittedRecords, InterpretedWalRecord, InterpretedWalRecords}; use wal_decoder::wire_format::FromWireFormat; use super::TaskStateUpdate; @@ -275,6 +276,8 @@ pub(super) async fn handle_walreceiver_connection( let copy_stream = replication_client.copy_both_simple(&query).await?; let mut physical_stream = pin!(ReplicationStream::new(copy_stream)); + let mut waldecoder = WalStreamDecoder::new(startpoint, timeline.pg_version); + let mut walingest = WalIngest::new(timeline.as_ref(), startpoint, &ctx) .await .map_err(|e| match e.kind { @@ -282,6 +285,8 @@ pub(super) async fn handle_walreceiver_connection( _ => WalReceiverError::Other(e.into()), })?; + let shard = vec![*timeline.get_shard_identity()]; + let (format, compression) = match protocol { PostgresClientProtocol::Interpreted { format, @@ -365,6 +370,7 @@ pub(super) async fn handle_walreceiver_connection( critical_timeline!( timeline.tenant_shard_id, timeline.timeline_id, + Some(&timeline.corruption_detected), "{msg}" ); return Err(WalReceiverError::Other(anyhow!(msg))); @@ -382,6 +388,7 @@ pub(super) async fn handle_walreceiver_connection( critical_timeline!( timeline.tenant_shard_id, timeline.timeline_id, + Some(&timeline.corruption_detected), "{msg}" ); return Err(WalReceiverError::Other(anyhow!(msg))); @@ -455,7 +462,9 @@ pub(super) async fn handle_walreceiver_connection( critical_timeline!( timeline.tenant_shard_id, timeline.timeline_id, + Some(&timeline.corruption_detected), "{err:?}" + ); } })?; @@ -509,6 +518,143 @@ pub(super) async fn handle_walreceiver_connection( Some(streaming_lsn) } + ReplicationMessage::XLogData(xlog_data) => { + async fn commit( + modification: &mut DatadirModification<'_>, + uncommitted: &mut u64, + filtered: &mut u64, + ctx: &RequestContext, + ) -> anyhow::Result<()> { + let stats = modification.stats(); + modification.commit(ctx).await?; + WAL_INGEST + .records_committed + .inc_by(*uncommitted - *filtered); + WAL_INGEST.inc_values_committed(&stats); + *uncommitted = 0; + *filtered = 0; + Ok(()) + } + + // Pass the WAL data to the decoder, and see if we can decode + // more records as a result. + let data = xlog_data.data(); + let startlsn = Lsn::from(xlog_data.wal_start()); + let endlsn = startlsn + data.len() as u64; + + trace!("received XLogData between {startlsn} and {endlsn}"); + + WAL_INGEST.bytes_received.inc_by(data.len() as u64); + waldecoder.feed_bytes(data); + + { + let mut modification = timeline.begin_modification(startlsn); + let mut uncommitted_records = 0; + let mut filtered_records = 0; + + while let Some((next_record_lsn, recdata)) = waldecoder.poll_decode()? { + // It is important to deal with the aligned records as lsn in getPage@LSN is + // aligned and can be several bytes bigger. Without this alignment we are + // at risk of hitting a deadlock. + if !next_record_lsn.is_aligned() { + return Err(WalReceiverError::Other(anyhow!("LSN not aligned"))); + } + + // Deserialize and interpret WAL record + let interpreted = InterpretedWalRecord::from_bytes_filtered( + recdata, + &shard, + next_record_lsn, + modification.tline.pg_version, + )? + .remove(timeline.get_shard_identity()) + .unwrap(); + + if matches!(interpreted.flush_uncommitted, FlushUncommittedRecords::Yes) + && uncommitted_records > 0 + { + // Special case: legacy PG database creations operate by reading pages from a 'template' database: + // these are the only kinds of WAL record that require reading data blocks while ingesting. Ensure + // all earlier writes of data blocks are visible by committing any modification in flight. + commit( + &mut modification, + &mut uncommitted_records, + &mut filtered_records, + &ctx, + ) + .await?; + } + + // Ingest the records without immediately committing them. + timeline.metrics.wal_records_received.inc(); + let ingested = walingest + .ingest_record(interpreted, &mut modification, &ctx) + .await + .with_context(|| { + format!("could not ingest record at {next_record_lsn}") + }) + .inspect_err(|err| { + // TODO: we can't differentiate cancellation errors with + // anyhow::Error, so just ignore it if we're cancelled. + if !cancellation.is_cancelled() { + critical_timeline!( + timeline.tenant_shard_id, + timeline.timeline_id, + Some(&timeline.corruption_detected), + "{err:?}" + ) + } + })?; + if !ingested { + tracing::debug!("ingest: filtered out record @ LSN {next_record_lsn}"); + WAL_INGEST.records_filtered.inc(); + filtered_records += 1; + } + + // FIXME: this cannot be made pausable_failpoint without fixing the + // failpoint library; in tests, the added amount of debugging will cause us + // to timeout the tests. + fail_point!("walreceiver-after-ingest"); + + last_rec_lsn = next_record_lsn; + + // Commit every ingest_batch_size records. Even if we filtered out + // all records, we still need to call commit to advance the LSN. + uncommitted_records += 1; + if uncommitted_records >= ingest_batch_size + || modification.approx_pending_bytes() + > DatadirModification::MAX_PENDING_BYTES + { + commit( + &mut modification, + &mut uncommitted_records, + &mut filtered_records, + &ctx, + ) + .await?; + } + } + + // Commit the remaining records. + if uncommitted_records > 0 { + commit( + &mut modification, + &mut uncommitted_records, + &mut filtered_records, + &ctx, + ) + .await?; + } + } + + if !caught_up && endlsn >= end_of_wal { + info!("caught up at LSN {endlsn}"); + caught_up = true; + } + + Some(endlsn) + } + ReplicationMessage::PrimaryKeepAlive(keepalive) => { let wal_end = keepalive.wal_end(); let timestamp = keepalive.timestamp(); @@ -586,6 +732,9 @@ pub(super) async fn handle_walreceiver_connection( remote_consistent_lsn, replytime: ts, shard_number: timeline.tenant_shard_id.shard_number.0 as u32, + corruption_detected: timeline + .corruption_detected + .load(std::sync::atomic::Ordering::Relaxed), }; debug!("neon_status_update {status_update:?}"); diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 3acf98b020..c364334dab 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -23,6 +23,7 @@ use std::backtrace::Backtrace; use std::collections::HashMap; +use std::sync::atomic::AtomicBool; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant, SystemTime}; @@ -422,6 +423,8 @@ impl WalIngest { critical_timeline!( modification.tline.tenant_shard_id, modification.tline.timeline_id, + // Hadron: No need to raise the corruption flag here; the caller of `ingest_record()` will do it. + None::<&AtomicBool>, "clear_vm_bits for unknown VM relation {vm_rel}" ); return Ok(()); @@ -431,6 +434,8 @@ impl WalIngest { critical_timeline!( modification.tline.tenant_shard_id, modification.tline.timeline_id, + // Hadron: No need to raise the corruption flag here; the caller of `ingest_record()` will do it. + None::<&AtomicBool>, "new_vm_blk {blknum} not in {vm_rel} of size {vm_size}" ); new_vm_blk = None; @@ -441,6 +446,8 @@ impl WalIngest { critical_timeline!( modification.tline.tenant_shard_id, modification.tline.timeline_id, + // Hadron: No need to raise the corruption flag here; the caller of `ingest_record()` will do it. + None::<&AtomicBool>, "old_vm_blk {blknum} not in {vm_rel} of size {vm_size}" ); old_vm_blk = None; diff --git a/pgxn/neon/neon_perf_counters.c b/pgxn/neon/neon_perf_counters.c index a38f876a0c..fada4cba1e 100644 --- a/pgxn/neon/neon_perf_counters.c +++ b/pgxn/neon/neon_perf_counters.c @@ -45,6 +45,7 @@ DatabricksMetricsShmemInit(void) pg_atomic_init_u32(&databricks_metrics_shared->index_corruption_count, 0); pg_atomic_init_u32(&databricks_metrics_shared->data_corruption_count, 0); pg_atomic_init_u32(&databricks_metrics_shared->internal_error_count, 0); + pg_atomic_init_u32(&databricks_metrics_shared->ps_corruption_detected, 0); } } /* END_HADRON */ @@ -440,6 +441,7 @@ neon_get_perf_counters(PG_FUNCTION_ARGS) {"sql_index_corruption_count", false, 0, (double) pg_atomic_read_u32(&databricks_metrics_shared->index_corruption_count)}, {"sql_data_corruption_count", false, 0, (double) pg_atomic_read_u32(&databricks_metrics_shared->data_corruption_count)}, {"sql_internal_error_count", false, 0, (double) pg_atomic_read_u32(&databricks_metrics_shared->internal_error_count)}, + {"ps_corruption_detected", false, 0, (double) pg_atomic_read_u32(&databricks_metrics_shared->ps_corruption_detected)}, {NULL, false, 0, 0}, }; for (int i = 0; databricks_metrics[i].name != NULL; i++) diff --git a/pgxn/neon/neon_perf_counters.h b/pgxn/neon/neon_perf_counters.h index 5062340cae..6a6e16cd26 100644 --- a/pgxn/neon/neon_perf_counters.h +++ b/pgxn/neon/neon_perf_counters.h @@ -187,6 +187,7 @@ typedef struct pg_atomic_uint32 index_corruption_count; pg_atomic_uint32 data_corruption_count; pg_atomic_uint32 internal_error_count; + pg_atomic_uint32 ps_corruption_detected; } databricks_metrics; extern databricks_metrics *databricks_metrics_shared; diff --git a/pgxn/neon/walproposer.c b/pgxn/neon/walproposer.c index ba6e4a54ff..c85a6f4b6f 100644 --- a/pgxn/neon/walproposer.c +++ b/pgxn/neon/walproposer.c @@ -1887,6 +1887,12 @@ ParsePageserverFeedbackMessage(WalProposer *wp, StringInfo reply_message, Pagese ps_feedback->shard_number = pq_getmsgint(reply_message, sizeof(uint32)); psfeedback_log("%u", key, ps_feedback->shard_number); } + else if (strcmp(key, "corruption_detected") == 0) + { + Assert(value_len == 1); + ps_feedback->corruption_detected = pq_getmsgbyte(reply_message) != 0; + psfeedback_log("%s", key, ps_feedback->corruption_detected ? "true" : "false"); + } else { /* diff --git a/pgxn/neon/walproposer.h b/pgxn/neon/walproposer.h index 5507294c3b..d6cd532bec 100644 --- a/pgxn/neon/walproposer.h +++ b/pgxn/neon/walproposer.h @@ -374,6 +374,8 @@ typedef struct PageserverFeedback XLogRecPtr remote_consistent_lsn; TimestampTz replytime; uint32 shard_number; + /* true if the pageserver has detected data corruption in the timeline */ + bool corruption_detected; } PageserverFeedback; /* BEGIN_HADRON */ diff --git a/pgxn/neon/walproposer_pg.c b/pgxn/neon/walproposer_pg.c index 874a1590ac..5db9b07ca3 100644 --- a/pgxn/neon/walproposer_pg.c +++ b/pgxn/neon/walproposer_pg.c @@ -49,6 +49,7 @@ #include "libpqwalproposer.h" #include "neon.h" +#include "neon_perf_counters.h" #include "neon_walreader.h" #include "walproposer.h" @@ -715,6 +716,11 @@ record_pageserver_feedback(PageserverFeedback *ps_feedback, shardno_t num_shards Assert(ps_feedback->shard_number < MAX_SHARDS); Assert(ps_feedback->shard_number < num_shards); + // Begin Hadron: Record any corruption signal from the pageserver first. + if (ps_feedback->corruption_detected) { + pg_atomic_write_u32(&databricks_metrics_shared->ps_corruption_detected, 1); + } + SpinLockAcquire(&walprop_shared->mutex); // Hadron: Update the num_shards from the source-of-truth (shard map) lazily when we receive diff --git a/safekeeper/src/hadron.rs b/safekeeper/src/hadron.rs index f41fe2512d..72b377fcc4 100644 --- a/safekeeper/src/hadron.rs +++ b/safekeeper/src/hadron.rs @@ -387,6 +387,7 @@ pub fn get_filesystem_usage(path: &std::path::Path) -> u64 { critical_timeline!( placeholder_ttid.tenant_id, placeholder_ttid.timeline_id, + None::<&AtomicBool>, "Global disk usage watcher failed to read filesystem usage: {:?}", e ); diff --git a/safekeeper/src/metrics.rs b/safekeeper/src/metrics.rs index b07852aaee..08d96a7aa6 100644 --- a/safekeeper/src/metrics.rs +++ b/safekeeper/src/metrics.rs @@ -518,6 +518,7 @@ pub async fn time_io_closure>( pub struct FullTimelineInfo { pub ttid: TenantTimelineId, pub ps_feedback_count: u64, + pub ps_corruption_detected: bool, pub last_ps_feedback: PageserverFeedback, pub wal_backup_active: bool, pub timeline_is_active: bool, @@ -547,6 +548,7 @@ pub struct TimelineCollector { ps_last_received_lsn: GenericGaugeVec, feedback_last_time_seconds: GenericGaugeVec, ps_feedback_count: GenericGaugeVec, + ps_corruption_detected: IntGaugeVec, timeline_active: GenericGaugeVec, wal_backup_active: GenericGaugeVec, connected_computes: IntGaugeVec, @@ -654,6 +656,15 @@ impl TimelineCollector { ) .unwrap(); + let ps_corruption_detected = IntGaugeVec::new( + Opts::new( + "safekeeper_ps_corruption_detected", + "1 if corruption was detected in the timeline according to feedback from the pageserver, 0 otherwise", + ), + &["tenant_id", "timeline_id"], + ) + .unwrap(); + let timeline_active = GenericGaugeVec::new( Opts::new( "safekeeper_timeline_active", @@ -774,6 +785,7 @@ impl TimelineCollector { ps_last_received_lsn, feedback_last_time_seconds, ps_feedback_count, + ps_corruption_detected, timeline_active, wal_backup_active, connected_computes, @@ -892,6 +904,9 @@ impl Collector for TimelineCollector { self.ps_feedback_count .with_label_values(labels) .set(tli.ps_feedback_count); + self.ps_corruption_detected + .with_label_values(labels) + .set(tli.ps_corruption_detected as i64); if let Ok(unix_time) = tli .last_ps_feedback .replytime @@ -925,6 +940,7 @@ impl Collector for TimelineCollector { mfs.extend(self.ps_last_received_lsn.collect()); mfs.extend(self.feedback_last_time_seconds.collect()); mfs.extend(self.ps_feedback_count.collect()); + mfs.extend(self.ps_corruption_detected.collect()); mfs.extend(self.timeline_active.collect()); mfs.extend(self.wal_backup_active.collect()); mfs.extend(self.connected_computes.collect()); diff --git a/safekeeper/src/send_interpreted_wal.rs b/safekeeper/src/send_interpreted_wal.rs index 671798298b..2eb221e32c 100644 --- a/safekeeper/src/send_interpreted_wal.rs +++ b/safekeeper/src/send_interpreted_wal.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; use std::fmt::Display; +use std::sync::atomic::AtomicBool; use std::sync::Arc; use std::time::Duration; @@ -305,7 +306,10 @@ impl InterpretedWalReader { critical_timeline!( ttid.tenant_id, ttid.timeline_id, - "failed to read WAL record: {err:?}" + // Hadron: The corruption flag is only used in PS so that it can feed this information back to SKs. + // We do not use these flags in SKs. + None::<&AtomicBool>, + "failed to read WAL record: {err:?}" ); } err => error!("failed to read WAL record: {err}"), @@ -375,7 +379,10 @@ impl InterpretedWalReader { critical_timeline!( ttid.tenant_id, ttid.timeline_id, - "failed to decode WAL record: {err:?}" + // Hadron: The corruption flag is only used in PS so that it can feed this information back to SKs. + // We do not use these flags in SKs. + None::<&AtomicBool>, + "failed to decode WAL record: {err:?}" ); } Err(err) => error!("failed to read WAL record: {err}"), diff --git a/safekeeper/src/send_wal.rs b/safekeeper/src/send_wal.rs index 5891fa88a4..2d6f7486a9 100644 --- a/safekeeper/src/send_wal.rs +++ b/safekeeper/src/send_wal.rs @@ -55,6 +55,7 @@ pub struct WalSenders { pub struct WalSendersTimelineMetricValues { pub ps_feedback_counter: u64, + pub ps_corruption_detected: bool, pub last_ps_feedback: PageserverFeedback, pub interpreted_wal_reader_tasks: usize, } @@ -193,6 +194,7 @@ impl WalSenders { WalSendersTimelineMetricValues { ps_feedback_counter: shared.ps_feedback_counter, + ps_corruption_detected: shared.ps_corruption_detected, last_ps_feedback: shared.last_ps_feedback, interpreted_wal_reader_tasks, } @@ -209,6 +211,9 @@ impl WalSenders { *shared.get_slot_mut(id).get_mut_feedback() = ReplicationFeedback::Pageserver(*feedback); shared.last_ps_feedback = *feedback; shared.ps_feedback_counter += 1; + if feedback.corruption_detected { + shared.ps_corruption_detected = true; + } drop(shared); RECEIVED_PS_FEEDBACKS.inc(); @@ -278,6 +283,9 @@ struct WalSendersShared { last_ps_feedback: PageserverFeedback, // total counter of pageserver feedbacks received ps_feedback_counter: u64, + // Hadron: true iff we received a pageserver feedback that incidated + // data corruption in the timeline + ps_corruption_detected: bool, slots: Vec>, } @@ -328,6 +336,7 @@ impl WalSendersShared { agg_standby_feedback: StandbyFeedback::empty(), last_ps_feedback: PageserverFeedback::empty(), ps_feedback_counter: 0, + ps_corruption_detected: false, slots: Vec::new(), } } diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 43b5b3a8d3..e083a49428 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -839,6 +839,7 @@ impl Timeline { let WalSendersTimelineMetricValues { ps_feedback_counter, + ps_corruption_detected, last_ps_feedback, interpreted_wal_reader_tasks, } = self.walsenders.info_for_metrics(); @@ -847,6 +848,7 @@ impl Timeline { Some(FullTimelineInfo { ttid: self.ttid, ps_feedback_count: ps_feedback_counter, + ps_corruption_detected, last_ps_feedback, wal_backup_active: self.wal_backup_active.load(Ordering::Relaxed), timeline_is_active: self.broker_active.load(Ordering::Relaxed), diff --git a/scripts/neon_grep.txt b/scripts/neon_grep.txt new file mode 100644 index 0000000000..2014597dc4 --- /dev/null +++ b/scripts/neon_grep.txt @@ -0,0 +1,2189 @@ +compute_tools/src/bin/compute_ctl.rs://! `ExecStart` option. It will handle all the `Neon` specifics during compute node +compute_tools/src/bin/compute_ctl.rs://! - Sync safekeepers and get commit LSN. +compute_tools/src/bin/compute_ctl.rs://! `vm-monitor` located in [`neon/libs/vm_monitor`]. For VM compute nodes, +compute_tools/src/bin/compute_ctl.rs: #[arg(long, default_value = "neon-postgres")] +compute_tools/src/bin/compute_ctl.rs: // from the neon control plane. This allows linking together the wider +compute_tools/src/bin/compute_ctl.rs: // /sys/fs/cgroup/neon-postgres. +compute_tools/src/bin/compute_ctl.rs: // Maybe sync safekeepers again, to speed up next startup +compute_tools/src/bin/compute_ctl.rs: info!("syncing safekeepers on shutdown"); +compute_tools/src/bin/compute_ctl.rs: let lsn = compute.sync_safekeepers(storage_auth_token)?; +compute_tools/src/bin/compute_ctl.rs: info!("synced safekeepers at lsn {lsn}"); +compute_tools/src/bin/compute_ctl.rs: // - and our problems with staging https://github.com/neondatabase/cloud/issues/3707#issuecomment-1493983636 +compute_tools/src/bin/compute_ctl.rs:/// When compute_ctl is killed, send also termination signal to sync-safekeepers +compute_tools/src/bin/fast_import.rs://! See cloud.git Fast Imports RFC () +compute_tools/src/bin/fast_import.rs://! is publicly accessible at . +compute_tools/src/bin/fast_import.rs: #[clap(long, env = "NEON_IMPORTER_S3_PREFIX")] +compute_tools/src/bin/fast_import.rs: library_search_path: &pg_lib_dir, // TODO: is this right? Prob works in compute image, not sure about neon_local. +compute_tools/src/bin/fast_import.rs: // Create neondb database in the running postgres +compute_tools/src/bin/fast_import.rs: match client.simple_query("CREATE DATABASE neondb;").await { +compute_tools/src/bin/fast_import.rs: info!("created neondb database"); +compute_tools/src/bin/fast_import.rs: let restore_pg_connstring = restore_pg_connstring.replace("dbname=postgres", "dbname=neondb"); +compute_tools/src/compute.rs: CreateDatabricksMisc, CreateDatabricksRoles, CreateSchemaNeon, CreateSuperUser, +compute_tools/src/compute.rs: DropInvalidDatabases, DropRoles, FinalizeDropLogicalSubscriptions, HandleNeonExtension, +compute_tools/src/compute.rs:use crate::sync_sk::{check_if_synced, ping_safekeeper}; +compute_tools/src/compute.rs:pub static SYNC_SAFEKEEPERS_PID: AtomicU32 = AtomicU32::new(0); +compute_tools/src/compute.rs: pub safekeeper_connstrings: Vec, +compute_tools/src/compute.rs: .or_else(|| spec.cluster.settings.find("neon.pageserver_connstring")) +compute_tools/src/compute.rs: let safekeeper_connstrings = if spec.safekeeper_connstrings.is_empty() { +compute_tools/src/compute.rs: .find("neon.safekeepers") +compute_tools/src/compute.rs: .ok_or("safekeeper connstrings should be provided")? +compute_tools/src/compute.rs: spec.safekeeper_connstrings.clone() +compute_tools/src/compute.rs: .find("neon.tenant_id") +compute_tools/src/compute.rs: .find("neon.timeline_id") +compute_tools/src/compute.rs: safekeeper_connstrings, +compute_tools/src/compute.rs:/// If we are a VM, returns a [`Command`] that will run in the `neon-postgres` +compute_tools/src/compute.rs:/// neon-postgres cgroup if we are a VM. This allows autoscaling to control +compute_tools/src/compute.rs: command.args(["-g", "memory:neon-postgres"]); +compute_tools/src/compute.rs: SELECT FROM pg_catalog.pg_roles WHERE rolname = 'neon_superuser') +compute_tools/src/compute.rs: CREATE ROLE neon_superuser CREATEDB CREATEROLE NOLOGIN BYPASSRLS IN ROLE pg_read_all_data, pg_write_all_data; +compute_tools/src/compute.rs: EXECUTE format('GRANT neon_superuser TO %s', +compute_tools/src/compute.rs: EXECUTE format('GRANT ALL PRIVILEGES ON DATABASE %s TO neon_superuser', +compute_tools/src/compute.rs: ).replace("neon_superuser", "databricks_superuser") +compute_tools/src/compute.rs: let max_attempts = if let Ok(v) = env::var("NEON_COMPUTE_TESTING_BASEBACKUP_RETRIES") { +compute_tools/src/compute.rs: pub async fn check_safekeepers_synced_async( +compute_tools/src/compute.rs: // Construct a connection config for each safekeeper +compute_tools/src/compute.rs: let sk_connstrs: Vec = pspec.safekeeper_connstrings.clone(); +compute_tools/src/compute.rs: // Create task set to query all safekeepers +compute_tools/src/compute.rs: let task = tokio::time::timeout(timeout, ping_safekeeper(id, config)); +compute_tools/src/compute.rs: // be backwards compatible with safekeepers that don't have the +compute_tools/src/compute.rs: "failed sync safekeepers check {:?} {:?} {:?}", +compute_tools/src/compute.rs: // Fast path for sync_safekeepers. If they're already synced we get the lsn +compute_tools/src/compute.rs: // in one roundtrip. If not, we should do a full sync_safekeepers. +compute_tools/src/compute.rs: pub fn check_safekeepers_synced(&self, compute_state: &ComputeState) -> Result> { +compute_tools/src/compute.rs: let result = rt.block_on(self.check_safekeepers_synced_async(compute_state)); +compute_tools/src/compute.rs: // Run `postgres` in a special mode with `--sync-safekeepers` argument +compute_tools/src/compute.rs: pub fn sync_safekeepers(&self, storage_auth_token: Option) -> Result { +compute_tools/src/compute.rs: .args(["--sync-safekeepers"]) +compute_tools/src/compute.rs: vec![("NEON_AUTH_TOKEN", storage_auth_token)] +compute_tools/src/compute.rs: .expect("postgres --sync-safekeepers failed to start"); +compute_tools/src/compute.rs: SYNC_SAFEKEEPERS_PID.store(sync_handle.id(), Ordering::SeqCst); +compute_tools/src/compute.rs: // `postgres --sync-safekeepers` will print all log output to stderr and +compute_tools/src/compute.rs: .expect("postgres --sync-safekeepers failed"); +compute_tools/src/compute.rs: SYNC_SAFEKEEPERS_PID.store(0, Ordering::SeqCst); +compute_tools/src/compute.rs: "postgres --sync-safekeepers exited with non-zero status: {}. stdout: {}", +compute_tools/src/compute.rs: .expect("postgres --sync-safekeepers exited, and stdout is not utf-8"), +compute_tools/src/compute.rs: self.state.lock().unwrap().metrics.sync_safekeepers_ms = Utc::now() +compute_tools/src/compute.rs: /// safekeepers sync, basebackup, etc. +compute_tools/src/compute.rs: // Syncing safekeepers is only safe with primary nodes: if a primary +compute_tools/src/compute.rs: // cannot sync safekeepers. +compute_tools/src/compute.rs: info!("checking if safekeepers are synced"); +compute_tools/src/compute.rs: let lsn = if let Ok(Some(lsn)) = self.check_safekeepers_synced(compute_state) { +compute_tools/src/compute.rs: info!("starting safekeepers syncing"); +compute_tools/src/compute.rs: self.sync_safekeepers(pspec.storage_auth_token.clone()) +compute_tools/src/compute.rs: .with_context(|| "failed to sync safekeepers")? +compute_tools/src/compute.rs: info!("safekeepers synced at LSN {}", lsn); +compute_tools/src/compute.rs: // See https://github.com/neondatabase/autoscaling/issues/800 +compute_tools/src/compute.rs: writeln!(file, "shared_preload_libraries = 'neon'")?; +compute_tools/src/compute.rs: env_vars.extend(storage_auth_token.map(|t| ("NEON_AUTH_TOKEN".to_string(), t))); +compute_tools/src/compute.rs: /// configure the database after applying the compute spec. Currently, it upgrades the neon extension +compute_tools/src/compute.rs: handle_neon_extension_upgrade(&mut client) +compute_tools/src/compute.rs: .context("handle_neon_extension_upgrade")?; +compute_tools/src/compute.rs: // If connection fails, it may be the old node with `zenith_admin` superuser. +compute_tools/src/compute.rs: // In this case we need to connect with old `zenith_admin` name +compute_tools/src/compute.rs: // Connect with zenith_admin if cloud_admin could not authenticate +compute_tools/src/compute.rs: "cannot connect to postgres: {}, retrying with `zenith_admin` username", +compute_tools/src/compute.rs: let mut zenith_admin_conf = postgres::config::Config::from(conf.clone()); +compute_tools/src/compute.rs: zenith_admin_conf.application_name("compute_ctl:apply_config"); +compute_tools/src/compute.rs: zenith_admin_conf.user("zenith_admin"); +compute_tools/src/compute.rs: zenith_admin_conf.connect(NoTls) +compute_tools/src/compute.rs: .context("broken cloud_admin credential: tried connecting with cloud_admin but could not authenticate, and zenith_admin does not work either")?; +compute_tools/src/compute.rs: client.simple_query("SET neon.forward_ddl = false")?; +compute_tools/src/compute.rs: client.simple_query("GRANT zenith_admin TO cloud_admin")?; +compute_tools/src/compute.rs: .simple_query("SET neon.forward_ddl = false") +compute_tools/src/compute.rs: .context("apply_config SET neon.forward_ddl = false")?; +compute_tools/src/compute.rs: // in the neon.drop_subscriptions_done table. +compute_tools/src/compute.rs: CreateSchemaNeon, +compute_tools/src/compute.rs: // We install system objects (neon extensions, health check tables) in the databricks_system database only. +compute_tools/src/compute.rs: // Hadron: In upstream, the HandleNeonExtension phase depends on CreateSchemaNeon. The new CreateSchemaNeon phase +compute_tools/src/compute.rs: // In Hadron deployments, this HandleNeonExtension phase executes only in the databricks_system database. To reduce +compute_tools/src/compute.rs: // the amount of divergent code changes from upstream and to avoid creating any schemas named "neon" in user-visible +compute_tools/src/compute.rs: // databases, we preserved the old code creating the "neon" schema on-demand in the HandleNeonExtension phase. +compute_tools/src/compute.rs: HandleNeonExtension, +compute_tools/src/compute.rs: // This step depends on CreateSchemaNeon +compute_tools/src/compute.rs: config::with_compute_ctl_tmp_override(pgdata_path, "neon.max_cluster_size=-1", || { +compute_tools/src/compute.rs: "neon.max_cluster_size=-1", +compute_tools/src/compute.rs: "neon.disable_logical_replication_subscribers=false", +compute_tools/src/compute.rs: info!("updated postgresql.conf to set neon.disable_logical_replication_subscribers=false"); +compute_tools/src/compute.rs: .filter(|s| *s != "neon" && *s != "databricks_auth" && !s.is_empty()) +compute_tools/src/compute.rs: // that is used in neon_local and python tests +compute_tools/src/compute.rs: .filter(|s| *s != "neon" && *s != "databricks_auth" && !s.is_empty()) +compute_tools/src/compute.rs: let ss_pid = SYNC_SAFEKEEPERS_PID.load(Ordering::SeqCst); +compute_tools/src/config.rs: writeln!(file, "# Neon storage settings")?; +compute_tools/src/config.rs: writeln!(file, "neon.pageserver_connstring={}", escape_conf_value(s))?; +compute_tools/src/config.rs: writeln!(file, "neon.stripe_size={stripe_size}")?; +compute_tools/src/config.rs: if !spec.safekeeper_connstrings.is_empty() { +compute_tools/src/config.rs: "neon.safekeepers={}", +compute_tools/src/config.rs: escape_conf_value(&spec.safekeeper_connstrings.join(",")) +compute_tools/src/config.rs: writeln!(file, "neon.tenant_id={}", escape_conf_value(&s.to_string()))?; +compute_tools/src/config.rs: "neon.timeline_id={}", +compute_tools/src/config.rs: // This is (maybe?) temporary - for more, see https://github.com/neondatabase/cloud/issues/12047. +compute_tools/src/config.rs: writeln!(file, "neon.extension_server_port={}", extension_server_port)?; +compute_tools/src/config.rs: writeln!(file, "neon.disable_logical_replication_subscribers=true")?; +compute_tools/src/config.rs: writeln!(file, "neon.disable_logical_replication_subscribers=false")?; +compute_tools/src/disk_quota.rs:pub const DISK_QUOTA_BIN: &str = "/neonvm/bin/set-disk-quota"; +compute_tools/src/disk_quota.rs: // run `/neonvm/bin/set-disk-quota {size_kb} {mountpoint}` +compute_tools/src/http/openapi_spec.yaml: - sync_safekeepers_ms +compute_tools/src/http/openapi_spec.yaml: sync_safekeepers_ms: +compute_tools/src/http/openapi_spec.yaml: example: "neondb" +compute_tools/src/http/openapi_spec.yaml: example: "neondb" +compute_tools/src/http/openapi_spec.yaml: example: "neon" +compute_tools/src/http/openapi_spec.yaml: example: "neondb" +compute_tools/src/http/openapi_spec.yaml: example: "neon" +compute_tools/src/http/server.rs:/// neon Postgres extension on the other hand does not send one. +compute_tools/src/local_proxy.rs://! Local Proxy is a feature of our BaaS Neon Authorize project. +compute_tools/src/migration.rs: // The neon_migration.migration_id::id column is a bigint, which is equivalent to an i64 +compute_tools/src/migration.rs: /// Get the current value neon_migration.migration_id +compute_tools/src/migration.rs: .query_one("SELECT id FROM neon_migration.migration_id", &[])?; +compute_tools/src/migration.rs: /// Update the neon_migration.migration_id value +compute_tools/src/migration.rs: "UPDATE neon_migration.migration_id SET id = $1", +compute_tools/src/migration.rs: .with_context(|| format!("update neon_migration.migration_id to {migration_id}"))?; +compute_tools/src/migration.rs: .simple_query("CREATE SCHEMA IF NOT EXISTS neon_migration")?; +compute_tools/src/migration.rs: self.client.simple_query("CREATE TABLE IF NOT EXISTS neon_migration.migration_id (key INT NOT NULL PRIMARY KEY, id bigint NOT NULL DEFAULT 0)")?; +compute_tools/src/migration.rs: "INSERT INTO neon_migration.migration_id VALUES (0, 0) ON CONFLICT DO NOTHING", +compute_tools/src/migration.rs: .simple_query("ALTER SCHEMA neon_migration OWNER TO cloud_admin")?; +compute_tools/src/migration.rs: .simple_query("REVOKE ALL ON SCHEMA neon_migration FROM PUBLIC")?; +compute_tools/src/migration.rs: .replace("neon_superuser", "databricks_superuser"); +compute_tools/src/migrations/0001-neon_superuser_bypass_rls.sql:ALTER ROLE neon_superuser BYPASSRLS; +compute_tools/src/migrations/0002-alter_roles.sql: FOR role_name IN SELECT rolname FROM pg_roles WHERE pg_has_role(rolname, 'neon_superuser', 'member') +compute_tools/src/migrations/0002-alter_roles.sql: NOT pg_has_role(rolname, 'neon_superuser', 'member') AND NOT starts_with(rolname, 'pg_') +compute_tools/src/migrations/0003-grant_pg_create_subscription_to_neon_superuser.sql: EXECUTE 'GRANT pg_create_subscription TO neon_superuser'; +compute_tools/src/migrations/0004-grant_pg_monitor_to_neon_superuser.sql:GRANT pg_monitor TO neon_superuser WITH ADMIN OPTION; +compute_tools/src/migrations/0005-grant_all_on_tables_to_neon_superuser.sql:-- interacted with by neon_superuser without permission issues. +compute_tools/src/migrations/0005-grant_all_on_tables_to_neon_superuser.sql:ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO neon_superuser; +compute_tools/src/migrations/0006-grant_all_on_sequences_to_neon_superuser.sql:-- interacted with by neon_superuser without permission issues. +compute_tools/src/migrations/0006-grant_all_on_sequences_to_neon_superuser.sql:ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO neon_superuser; +compute_tools/src/migrations/0007-grant_all_on_tables_to_neon_superuser_with_grant_option.sql:ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO neon_superuser WITH GRANT OPTION; +compute_tools/src/migrations/0008-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql:ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO neon_superuser WITH GRANT OPTION; +compute_tools/src/migrations/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql: EXECUTE 'GRANT EXECUTE ON FUNCTION pg_export_snapshot TO neon_superuser'; +compute_tools/src/migrations/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql: EXECUTE 'GRANT EXECUTE ON FUNCTION pg_log_standby_snapshot TO neon_superuser'; +compute_tools/src/migrations/0011-grant_pg_show_replication_origin_status_to_neon_superuser.sql:GRANT EXECUTE ON FUNCTION pg_show_replication_origin_status TO neon_superuser; +compute_tools/src/migrations/tests/0001-neon_superuser_bypass_rls.sql: SELECT rolbypassrls INTO bypassrls FROM pg_roles WHERE rolname = 'neon_superuser'; +compute_tools/src/migrations/tests/0001-neon_superuser_bypass_rls.sql: RAISE EXCEPTION 'neon_superuser cannot bypass RLS'; +compute_tools/src/migrations/tests/0002-alter_roles.sql: WHERE pg_has_role(rolname, 'neon_superuser', 'member') +compute_tools/src/migrations/tests/0002-alter_roles.sql: WHERE NOT pg_has_role(rolname, 'neon_superuser', 'member') +compute_tools/src/migrations/tests/0003-grant_pg_create_subscription_to_neon_superuser.sql: IF NOT (SELECT pg_has_role('neon_superuser', 'pg_create_subscription', 'member')) THEN +compute_tools/src/migrations/tests/0003-grant_pg_create_subscription_to_neon_superuser.sql: RAISE EXCEPTION 'neon_superuser cannot execute pg_create_subscription'; +compute_tools/src/migrations/tests/0004-grant_pg_monitor_to_neon_superuser.sql: SELECT pg_has_role('neon_superuser', 'pg_monitor', 'member') AS member, +compute_tools/src/migrations/tests/0004-grant_pg_monitor_to_neon_superuser.sql: RAISE EXCEPTION 'neon_superuser is not a member of pg_monitor'; +compute_tools/src/migrations/tests/0004-grant_pg_monitor_to_neon_superuser.sql: RAISE EXCEPTION 'neon_superuser cannot grant pg_monitor'; +compute_tools/src/migrations/tests/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql: SELECT bool_and(has_function_privilege('neon_superuser', oid, 'execute')) +compute_tools/src/migrations/tests/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql: RAISE EXCEPTION 'neon_superuser cannot execute both pg_export_snapshot and pg_log_standby_snapshot'; +compute_tools/src/migrations/tests/0011-grant_pg_show_replication_origin_status_to_neon_superuser.sql: SELECT has_function_privilege('neon_superuser', oid, 'execute') +compute_tools/src/migrations/tests/0011-grant_pg_show_replication_origin_status_to_neon_superuser.sql: RAISE EXCEPTION 'neon_superuser cannot execute pg_show_replication_origin_status'; +compute_tools/src/monitor.rs: // to start PG (e.g., if we can't pull the spec, can't sync safekeepers, or can't get the basebackup). +compute_tools/src/spec.rs: // https://github.com/neondatabase/cloud/issues/2353 +compute_tools/src/spec.rs:/// Request spec from the control-plane by compute_id. If `NEON_CONTROL_PLANE_TOKEN` +compute_tools/src/spec.rs: let jwt: String = match std::env::var("NEON_CONTROL_PLANE_TOKEN") { +compute_tools/src/spec.rs: // Update pg_hba to contains databricks specfic settings before adding neon settings +compute_tools/src/spec.rs: // our rules before the default ones from neon. +compute_tools/src/spec.rs:pub fn handle_neon_extension_upgrade(client: &mut Client) -> Result<()> { +compute_tools/src/spec.rs: info!("handle neon extension upgrade"); +compute_tools/src/spec.rs: let query = "ALTER EXTENSION neon UPDATE"; +compute_tools/src/spec.rs: info!("update neon extension version with query: {}", query); +compute_tools/src/spec.rs: include_str!("./migrations/0001-neon_superuser_bypass_rls.sql"), +compute_tools/src/spec.rs: include_str!("./migrations/0003-grant_pg_create_subscription_to_neon_superuser.sql"), +compute_tools/src/spec.rs: include_str!("./migrations/0004-grant_pg_monitor_to_neon_superuser.sql"), +compute_tools/src/spec.rs: include_str!("./migrations/0005-grant_all_on_tables_to_neon_superuser.sql"), +compute_tools/src/spec.rs: include_str!("./migrations/0006-grant_all_on_sequences_to_neon_superuser.sql"), +compute_tools/src/spec.rs: "./migrations/0007-grant_all_on_tables_to_neon_superuser_with_grant_option.sql" +compute_tools/src/spec.rs: "./migrations/0008-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql" +compute_tools/src/spec.rs: "./migrations/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql" +compute_tools/src/spec.rs: "./migrations/0011-grant_pg_show_replication_origin_status_to_neon_superuser.sql" +compute_tools/src/spec_apply.rs: CreateSchemaNeon, +compute_tools/src/spec_apply.rs: HandleNeonExtension, +compute_tools/src/spec_apply.rs: // Postgres Neon extension is done the way, that db is de-registered +compute_tools/src/spec_apply.rs: ApplySpecPhase::CreateSchemaNeon => Ok(Box::new(once(Operation { +compute_tools/src/spec_apply.rs: // Revoke some potentially blocking privileges (Neon-specific currently) +compute_tools/src/spec_apply.rs: .replace("neon_superuser", "databricks_superuser"), +compute_tools/src/spec_apply.rs: .replace("neon_superuser", "databricks_superuser"), +compute_tools/src/spec_apply.rs: ApplySpecPhase::HandleNeonExtension => { +compute_tools/src/spec_apply.rs: query: String::from("CREATE SCHEMA IF NOT EXISTS neon"), +compute_tools/src/spec_apply.rs: query: String::from("CREATE EXTENSION IF NOT EXISTS neon WITH SCHEMA neon"), +compute_tools/src/spec_apply.rs: "UPDATE pg_extension SET extrelocatable = true WHERE extname = 'neon'", +compute_tools/src/spec_apply.rs: comment: Some(String::from("compat/fix: make neon relocatable")), +compute_tools/src/spec_apply.rs: query: String::from("ALTER EXTENSION neon SET SCHEMA neon"), +compute_tools/src/spec_apply.rs: comment: Some(String::from("compat/fix: alter neon extension schema")), +compute_tools/src/spec_apply.rs: query: String::from("ALTER EXTENSION neon UPDATE"), +compute_tools/src/spec_apply.rs: comment: Some(String::from("compat/update: update neon extension version")), +compute_tools/src/spec_apply.rs: query: String::from("GRANT USAGE ON SCHEMA neon TO databricks_monitor"), +compute_tools/src/spec_apply.rs: comment: Some(String::from("Permissions needed to execute neon.* functions (in the databricks_system database)")), +compute_tools/src/sql/default_grants.sql: ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO neon_superuser WITH GRANT OPTION; +compute_tools/src/sql/default_grants.sql: ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO neon_superuser WITH GRANT OPTION; +compute_tools/src/sql/finalize_drop_subscriptions.sql: INSERT INTO __db_system.drop_subscriptions_done VALUES (1, current_setting('neon.timeline_id')) +compute_tools/src/sql/finalize_drop_subscriptions.sql: SET timeline_id = current_setting('neon.timeline_id'); +compute_tools/src/sql/pre_drop_role_revoke_privileges.sql:SET SESSION ROLE neon_superuser; +compute_tools/src/sql/pre_drop_role_revoke_privileges.sql: -- See https://github.com/neondatabase/cloud/issues/13582 for the context. +compute_tools/src/sql/pre_drop_role_revoke_privileges.sql: 'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I FROM {role_name} GRANTED BY neon_superuser;', +compute_tools/src/sql/set_public_schema_owner.sql: IF schema_owner = 'cloud_admin' OR schema_owner = 'zenith_admin' +compute_tools/src/swap.rs:pub const RESIZE_SWAP_BIN: &str = "/neonvm/bin/resize-swap"; +compute_tools/src/swap.rs: // run `/neonvm/bin/resize-swap --once {size_bytes}` +compute_tools/src/sync_sk.rs:// Utils for running sync_safekeepers +compute_tools/src/sync_sk.rs:/// Get a safekeeper's metadata for our timeline. The id is only used for logging +compute_tools/src/sync_sk.rs:pub async fn ping_safekeeper( +compute_tools/src/sync_sk.rs:/// Given a quorum of responses, check if safekeepers are synced at some Lsn +pgxn/neon/Makefile:# pgxs/neon/Makefile +pgxn/neon/Makefile:MODULE_big = neon +pgxn/neon/Makefile: neon.o \ +pgxn/neon/Makefile: neon_pgversioncompat.o \ +pgxn/neon/Makefile: neon_perf_counters.o \ +pgxn/neon/Makefile: neon_utils.o \ +pgxn/neon/Makefile: neon_walreader.o \ +pgxn/neon/Makefile:EXTENSION = neon +pgxn/neon/Makefile: neon--1.0.sql \ +pgxn/neon/Makefile: neon--1.0--1.1.sql \ +pgxn/neon/Makefile: neon--1.1--1.2.sql \ +pgxn/neon/Makefile: neon--1.2--1.3.sql \ +pgxn/neon/Makefile: neon--1.3--1.4.sql \ +pgxn/neon/Makefile: neon--1.4--1.5.sql \ +pgxn/neon/Makefile: neon--1.5--1.4.sql \ +pgxn/neon/Makefile: neon--1.4--1.3.sql \ +pgxn/neon/Makefile: neon--1.3--1.2.sql \ +pgxn/neon/Makefile: neon--1.2--1.1.sql \ +pgxn/neon/Makefile: neon--1.1--1.0.sql +pgxn/neon/Makefile:PGFILEDESC = "neon - cloud storage for PostgreSQL" +pgxn/neon/Makefile: neon_utils.o \ +pgxn/neon/Makefile: $(FIND_TYPEDEF) . > neon.typedefs +pgxn/neon/Makefile: INDENT=$(INDENT) $(PGINDENT_SCRIPT) --typedefs neon.typedefs $(srcdir)/*.c $(srcdir)/*.h +pgxn/neon/README.md:neon extension consists of several parts: +pgxn/neon/README.md:### shared preload library `neon.so` +pgxn/neon/README.md:- walproposer: implements broadcast protocol between postgres and WAL safekeepers. +pgxn/neon/README.md:- relsize_cache: Relation size cache for better neon performance. +pgxn/neon/README.md:### SQL functions in `neon--*.sql` +pgxn/neon/README.md:Utility functions to expose neon specific information to user and metrics collection. +pgxn/neon/bitmap.h:#ifndef NEON_BITMAP_H +pgxn/neon/bitmap.h:#define NEON_BITMAP_H +pgxn/neon/bitmap.h:#endif //NEON_BITMAP_H +pgxn/neon/control_plane_connector.c: * via HTTP to the URL specified by the GUC neon.console_url when the +pgxn/neon/control_plane_connector.c: * setting neon.forward_ddl to false. +pgxn/neon/control_plane_connector.c: * contrib/neon/control_plane_connector.c +pgxn/neon/control_plane_connector.c:#include "neon_utils.h" +pgxn/neon/control_plane_connector.c: * Neon forwards all DDL from PG to neon control plane. During +pgxn/neon/control_plane_connector.c:NeonSubXactCallback( +pgxn/neon/control_plane_connector.c:NeonXactCallback(XactEvent event, void *arg) +pgxn/neon/control_plane_connector.c:RoleIsNeonSuperuser(const char *role_name) +pgxn/neon/control_plane_connector.c: if (RoleIsNeonSuperuser(owner_name)) +pgxn/neon/control_plane_connector.c: if (RoleIsNeonSuperuser(new_owner)) +pgxn/neon/control_plane_connector.c: if (RoleIsNeonSuperuser(role_name) && !superuser()) +pgxn/neon/control_plane_connector.c:NeonProcessUtility( +pgxn/neon/control_plane_connector.c: ProcessUtility_hook = NeonProcessUtility; +pgxn/neon/control_plane_connector.c: RegisterXactCallback(NeonXactCallback, NULL); +pgxn/neon/control_plane_connector.c: RegisterSubXactCallback(NeonSubXactCallback, NULL); +pgxn/neon/control_plane_connector.c: "neon.console_url", +pgxn/neon/control_plane_connector.c: "URL of the Neon Console, which will be forwarded changes to dbs and roles", +pgxn/neon/control_plane_connector.c: "neon.forward_ddl", +pgxn/neon/control_plane_connector.c: "neon.regress_test_mode", +pgxn/neon/control_plane_connector.c: jwt_token = getenv("NEON_CONTROL_PLANE_TOKEN"); +pgxn/neon/extension_server.c: * contrib/neon/extension_server.c +pgxn/neon/extension_server.c:#include "neon_utils.h" +pgxn/neon/extension_server.c:neon_download_extension_file_http(const char *filename, bool is_library) +pgxn/neon/extension_server.c: DefineCustomIntVariable("neon.extension_server_port", +pgxn/neon/extension_server.c: download_extension_file_hook = neon_download_extension_file_http; +pgxn/neon/extension_server.h: * contrib/neon/extension_server.h +pgxn/neon/file_cache.c: * pgxn/neon/file_cache.c +pgxn/neon/file_cache.c:#include "neon_pgversioncompat.h" +pgxn/neon/file_cache.c:#include "neon.h" +pgxn/neon/file_cache.c:#include "neon_perf_counters.h" +pgxn/neon/file_cache.c: * at server startup (neon.max_file_cache_size). After server startup, we +pgxn/neon/file_cache.c: * (neon.file_cache_size_limit). If the soft limit is later reduced, we shrink +pgxn/neon/file_cache.c: * Local file cache is optional and Neon can work without it. +pgxn/neon/file_cache.c: pgstat_report_wait_start(WAIT_EVENT_NEON_LFC_TRUNCATE); +pgxn/neon/file_cache.c: neon_log(LOG, "Detected COPY FROM, disabling file cache writes."); +pgxn/neon/file_cache.c: neon_log(LOG, "Detected COPY FROM, but not disabling file cache writes."); +pgxn/neon/file_cache.c: neon_log(LOG, "Failed to punch hole in file: %m"); +pgxn/neon/file_cache.c: neon_log(DEBUG1, "set local file cache limit to %d", new_size); +pgxn/neon/file_cache.c: * Hook for neon.file_cache_max_outstanding_ios. +pgxn/neon/file_cache.c: neon_log(ERROR, "databricks_storage module should be loaded via shared_preload_libraries"); +pgxn/neon/file_cache.c: DefineCustomIntVariable("neon.max_file_cache_size", +pgxn/neon/file_cache.c: "Maximal size of Neon local file cache", +pgxn/neon/file_cache.c: DefineCustomIntVariable("neon.file_cache_size_limit", +pgxn/neon/file_cache.c: "Current limit for size of Neon local file cache", +pgxn/neon/file_cache.c: DefineCustomStringVariable("neon.file_cache_path", +pgxn/neon/file_cache.c: * Notes on neon.file_cache_max_outstanding_ios and +pgxn/neon/file_cache.c: * neon.file_cache_bypass_for_copy: +pgxn/neon/file_cache.c: DefineCustomIntVariable("neon.file_cache_max_outstanding_ios", +pgxn/neon/file_cache.c: DefineCustomBoolVariable("neon.file_cache_bypass_for_copy", +pgxn/neon/file_cache.c: pgstat_report_wait_start(WAIT_EVENT_NEON_LFC_READ); +pgxn/neon/file_cache.c: neon_log(DEBUG2, "Swap file cache page"); +pgxn/neon/file_cache.c: pgstat_report_wait_start(WAIT_EVENT_NEON_LFC_WRITE); +pgxn/neon/file_cache.c:} NeonGetStatsCtx; +pgxn/neon/file_cache.c:#define NUM_NEON_GET_STATS_COLS 2 +pgxn/neon/file_cache.c:PG_FUNCTION_INFO_V1(neon_get_lfc_stats); +pgxn/neon/file_cache.c:neon_get_lfc_stats(PG_FUNCTION_ARGS) +pgxn/neon/file_cache.c: NeonGetStatsCtx *fctx; +pgxn/neon/file_cache.c: Datum values[NUM_NEON_GET_STATS_COLS]; +pgxn/neon/file_cache.c: bool nulls[NUM_NEON_GET_STATS_COLS]; +pgxn/neon/file_cache.c: fctx = (NeonGetStatsCtx *) palloc(sizeof(NeonGetStatsCtx)); +pgxn/neon/file_cache.c: tupledesc = CreateTemplateTupleDesc(NUM_NEON_GET_STATS_COLS); +pgxn/neon/file_cache.c: fctx = (NeonGetStatsCtx *) funcctx->user_fctx; +pgxn/neon/file_cache.c: neon_log(ERROR, "return type must be a row type"); +pgxn/neon/file_cache.c: neon_log(ERROR, "incorrect number of output arguments"); +pgxn/neon/libpagestore.c: * contrib/neon/libpqpagestore.c +pgxn/neon/libpagestore.c:#include "neon.h" +pgxn/neon/libpagestore.c:#include "neon_perf_counters.h" +pgxn/neon/libpagestore.c:#include "neon_utils.h" +pgxn/neon/libpagestore.c:char *neon_timeline; +pgxn/neon/libpagestore.c:char *neon_tenant; +pgxn/neon/libpagestore.c:char *neon_auth_token; +pgxn/neon/libpagestore.c:int neon_protocol_version = 2; +pgxn/neon/libpagestore.c: * The "neon.pageserver_connstring" GUC is marked with the PGC_SIGHUP option, +pgxn/neon/libpagestore.c: neon_log(LOG, "Too many shards"); +pgxn/neon/libpagestore.c: neon_log(LOG, "Connection string too long"); +pgxn/neon/libpagestore.c: // e.g., sync-safekeepers on shutdown. +pgxn/neon/libpagestore.c: neon_log(LOG, "get_num_shards(): pagestore_shared is NULL. Return 0 shards."); +pgxn/neon/libpagestore.c: neon_log(ERROR, "Shard %d is greater or equal than number of shards %d", +pgxn/neon/libpagestore.c: MyNeonCounters->pageserver_disconnects_total++; +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, DEBUG5, "Connection state: Disconnected"); +pgxn/neon/libpagestore.c: * cf. https://github.com/neondatabase/neon/issues/7897 +pgxn/neon/libpagestore.c: * neon.pageserver_connstring GUC. If the NEON_AUTH_TOKEN environment +pgxn/neon/libpagestore.c: if (neon_auth_token) +pgxn/neon/libpagestore.c: values[1] = neon_auth_token; +pgxn/neon/libpagestore.c: errmsg(NEON_TAG "[shard %d] could not establish connection to pageserver", shard_no), +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, DEBUG5, "Connection state: Connecting_Startup"); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, DEBUG5, "POLLING_FAILED"); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, elevel, +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, elevel, +pgxn/neon/libpagestore.c: WAIT_EVENT_NEON_PS_STARTING); +pgxn/neon/libpagestore.c: WAIT_EVENT_NEON_PS_STARTING); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, DEBUG5, "POLLING_OK"); +pgxn/neon/libpagestore.c: switch (neon_protocol_version) +pgxn/neon/libpagestore.c: pagestream_query = psprintf("pagestream_v3 %s %s", neon_tenant, neon_timeline); +pgxn/neon/libpagestore.c: pagestream_query = psprintf("pagestream_v2 %s %s", neon_tenant, neon_timeline); +pgxn/neon/libpagestore.c: elog(ERROR, "unexpected databricks_protocol_version %d", neon_protocol_version); +pgxn/neon/libpagestore.c: errmsg(NEON_TAG "[shard %d] could not establish connection to pageserver", shard_no), +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, elevel, "could not send pagestream command to pageserver"); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, DEBUG5, "Connection state: Connecting_PageStream"); +pgxn/neon/libpagestore.c: errmsg(NEON_TAG "[shard %d] could not establish connection to pageserver", shard_no), +pgxn/neon/libpagestore.c: WAIT_EVENT_NEON_PS_CONFIGURING); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, elevel, "could not complete handshake with pageserver: %s", +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, DEBUG5, "Connection state: Connected"); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "libpagestore: connected to '%s' with protocol version %d", connstr, neon_protocol_version); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "libpagestore: invalid connection state %d", shard->state); +pgxn/neon/libpagestore.c: WAIT_EVENT_NEON_PS_READ); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "could not get response from pageserver: %s", msg); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "no response received from pageserver for %0.3f s, still waiting (sent " UINT64_FORMAT " requests, received " UINT64_FORMAT " responses) (socket sndbuf=%d recvbuf=%d)", +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "received response from pageserver after %0.3f s", +pgxn/neon/libpagestore.c:pageserver_send(shardno_t shard_no, NeonRequest *request) +pgxn/neon/libpagestore.c: MyNeonCounters->pageserver_requests_sent_total++; +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_send disconnect bad connection"); +pgxn/neon/libpagestore.c: * https://github.com/neondatabase/neon/issues/1138 So try to reestablish +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "request failed too many times, cancelling query"); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_send disconnected: failed to send page request (try to reconnect): %s", msg); +pgxn/neon/libpagestore.c: char *msg = nm_to_string((NeonMessage *) request); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, PageStoreTrace, "sent request: %s", msg); +pgxn/neon/libpagestore.c:static NeonResponse * +pgxn/neon/libpagestore.c: NeonResponse *resp; +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_receive: disconnect due to failure while parsing response"); +pgxn/neon/libpagestore.c: char *msg = nm_to_string((NeonMessage *) resp); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, PageStoreTrace, "got response: %s", msg); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_receive disconnect: psql end of copy data: %s", pchomp(PQerrorMessage(pageserver_conn))); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "pageserver_receive disconnect: could not read COPY data: %s", msg); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "pageserver_receive disconnect: unexpected PQgetCopyData return value: %d", rc); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "refresh_configuration request failed, cancelling query"); +pgxn/neon/libpagestore.c: return (NeonResponse *) resp; +pgxn/neon/libpagestore.c:static NeonResponse * +pgxn/neon/libpagestore.c: NeonResponse *resp; +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_receive: disconnect due to failure while parsing response"); +pgxn/neon/libpagestore.c: char *msg = nm_to_string((NeonMessage *) resp); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, PageStoreTrace, "got response: %s", msg); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_receive disconnect: psql end of copy data: %s", pchomp(PQerrorMessage(pageserver_conn))); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "pageserver_receive disconnect: could not read COPY data: %s", msg); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "pageserver_receive disconnect: unexpected PQgetCopyData return value: %d", rc); +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "refresh_configuration request failed, cancelling query"); +pgxn/neon/libpagestore.c: return (NeonResponse *) resp; +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, WARNING, "Tried to flush while disconnected"); +pgxn/neon/libpagestore.c: MyNeonCounters->pageserver_send_flushes_total++; +pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_flush disconnect because failed to flush page requests: %s", msg); +pgxn/neon/libpagestore.c:check_neon_id(char **newval, void **extra, GucSource source) +pgxn/neon/libpagestore.c: size = add_size(size, NeonPerfCountersShmemSize()); +pgxn/neon/libpagestore.c: NeonPerfCountersShmemInit(); +pgxn/neon/libpagestore.c: DefineCustomStringVariable("neon.pageserver_connstring", +pgxn/neon/libpagestore.c: DefineCustomStringVariable("neon.timeline_id", +pgxn/neon/libpagestore.c: "Neon timeline_id the server is running on", +pgxn/neon/libpagestore.c: &neon_timeline, +pgxn/neon/libpagestore.c: check_neon_id, NULL, NULL); +pgxn/neon/libpagestore.c: DefineCustomStringVariable("neon.tenant_id", +pgxn/neon/libpagestore.c: "Neon tenant_id the server is running on", +pgxn/neon/libpagestore.c: &neon_tenant, +pgxn/neon/libpagestore.c: check_neon_id, NULL, NULL); +pgxn/neon/libpagestore.c: DefineCustomIntVariable("neon.stripe_size", +pgxn/neon/libpagestore.c: DefineCustomIntVariable("neon.max_cluster_size", +pgxn/neon/libpagestore.c: DefineCustomIntVariable("neon.flush_output_after", +pgxn/neon/libpagestore.c: DefineCustomIntVariable("neon.max_reconnect_attempts", +pgxn/neon/libpagestore.c: DefineCustomIntVariable("neon.readahead_buffer_size", +pgxn/neon/libpagestore.c: DefineCustomIntVariable("neon.protocol_version", +pgxn/neon/libpagestore.c: &neon_protocol_version, +pgxn/neon/libpagestore.c: neon_log(ERROR, "libpagestore already loaded"); +pgxn/neon/libpagestore.c: neon_log(PageStoreTrace, "libpagestore already loaded"); +pgxn/neon/libpagestore.c: * safekeepers +pgxn/neon/libpagestore.c: neon_auth_token = getenv("NEON_AUTH_TOKEN"); +pgxn/neon/libpagestore.c: if (neon_auth_token) +pgxn/neon/libpagestore.c: neon_log(LOG, "using storage auth token from DATABRICKS_AUTH_TOKEN environment variable"); +pgxn/neon/libpagestore.c: neon_log(PageStoreTrace, "set databricks_smgr hook"); +pgxn/neon/libpagestore.c: smgr_hook = smgr_neon; +pgxn/neon/libpagestore.c: smgr_init_hook = smgr_init_neon; +pgxn/neon/libpagestore.c: dbsize_hook = neon_dbsize; +pgxn/neon/libpqwalproposer.h: * Interface to set of libpq wrappers walproposer and neon_walreader need. +pgxn/neon/libpqwalproposer.h: * provide an error message indicating which safekeeper messed up. +pgxn/neon/logical_replication_monitor.c: "neon.logical_replication_max_snap_files", +pgxn/neon/logical_replication_monitor.c: "neon.logical_replication_max_logicalsnapdir_size", +pgxn/neon/logical_replication_monitor.c: snprintf(bgw.bgw_library_name, BGW_MAXLEN, "neon"); +pgxn/neon/logical_replication_monitor.h:#ifndef __NEON_LOGICAL_REPLICATION_MONITOR_H__ +pgxn/neon/logical_replication_monitor.h:#define __NEON_LOGICAL_REPLICATION_MONITOR_H__ +pgxn/neon/neon--1.0--1.1.sql:\echo Use "ALTER EXTENSION neon UPDATE TO '1.1'" to load this file. \quit +pgxn/neon/neon--1.0--1.1.sql:CREATE FUNCTION neon_get_lfc_stats() +pgxn/neon/neon--1.0--1.1.sql:AS 'MODULE_PATHNAME', 'neon_get_lfc_stats' +pgxn/neon/neon--1.0--1.1.sql:CREATE VIEW neon_lfc_stats AS +pgxn/neon/neon--1.0--1.1.sql: SELECT P.* FROM neon_get_lfc_stats() AS P (lfc_key text, lfc_value bigint); +pgxn/neon/neon--1.0.sql:\echo Use "CREATE EXTENSION neon" to load this file. \quit +pgxn/neon/neon--1.1--1.0.sql:DROP VIEW IF EXISTS neon_lfc_stats CASCADE; +pgxn/neon/neon--1.1--1.0.sql:DROP FUNCTION IF EXISTS neon_get_lfc_stats CASCADE; +pgxn/neon/neon--1.1--1.2.sql:\echo Use "ALTER EXTENSION neon UPDATE TO '1.2'" to load this file. \quit +pgxn/neon/neon--1.1--1.2.sql:CREATE OR REPLACE VIEW NEON_STAT_FILE_CACHE AS +pgxn/neon/neon--1.1--1.2.sql: FROM neon_get_lfc_stats() AS t(stat_name text, count bigint) +pgxn/neon/neon--1.1--1.2.sql:GRANT SELECT ON NEON_STAT_FILE_CACHE TO PG_MONITOR; +pgxn/neon/neon--1.2--1.1.sql:DROP VIEW IF EXISTS NEON_STAT_FILE_CACHE CASCADE; +pgxn/neon/neon--1.2--1.3.sql:\echo Use "ALTER EXTENSION neon UPDATE TO '1.3'" to load this file. \quit +pgxn/neon/neon--1.3--1.4.sql:\echo Use "ALTER EXTENSION neon UPDATE TO '1.4'" to load this file. \quit +pgxn/neon/neon--1.4--1.5.sql:\echo Use "ALTER EXTENSION neon UPDATE TO '1.5'" to load this file. \quit +pgxn/neon/neon--1.4--1.5.sql:AS 'MODULE_PATHNAME', 'neon_get_backend_perf_counters' +pgxn/neon/neon--1.4--1.5.sql:AS 'MODULE_PATHNAME', 'neon_get_perf_counters' +pgxn/neon/neon--1.4--1.5.sql:CREATE VIEW neon_backend_perf_counters AS +pgxn/neon/neon--1.4--1.5.sql:-- an aggregate query over neon_backend_perf_counters view.) +pgxn/neon/neon--1.4--1.5.sql:CREATE VIEW neon_perf_counters AS +pgxn/neon/neon--1.5--1.4.sql:DROP VIEW IF EXISTS neon_perf_counters; +pgxn/neon/neon--1.5--1.4.sql:DROP VIEW IF EXISTS neon_backend_perf_counters; +pgxn/neon/neon.c: * neon.c +pgxn/neon/neon.c: * Utility functions to expose neon specific information to user +pgxn/neon/neon.c: * contrib/neon/neon.c +pgxn/neon/neon.c:#include "neon.h" +pgxn/neon/neon.c:#include "neon_perf_counters.h" +pgxn/neon/neon.c:static void neon_shmem_startup_hook(void); +pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_LFC_MAINTENANCE; +pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_LFC_READ; +pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_LFC_TRUNCATE; +pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_LFC_WRITE; +pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_PS_STARTING; +pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_PS_CONFIGURING; +pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_PS_SEND; +pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_PS_READ; +pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_WAL_DL; +pgxn/neon/neon.c: * Neon: A standby in Neon doesn't need to start WAL replay from a checkpoint +pgxn/neon/neon.c: * Also load 'neon_rmgr'. This makes it unnecessary to list both 'neon' +pgxn/neon/neon.c: * and 'neon_rmgr' in shared_preload_libraries. +pgxn/neon/neon.c: load_file("$libdir/neon_rmgr", false); +pgxn/neon/neon.c: shmem_startup_hook = neon_shmem_startup_hook; +pgxn/neon/neon.c: Custom_XLogReaderRoutines = NeonOnDemandXLogReaderRoutines; +pgxn/neon/neon.c: "neon.disable_logical_replication_subscribers", +pgxn/neon/neon.c: "neon.allow_replica_misconfig", +pgxn/neon/neon.c: "neon.running_xacts_overflow_policy", +pgxn/neon/neon.c: EmitWarningsOnPlaceholders("neon"); +pgxn/neon/neon.c: size = GetNeonCurrentClusterSize(); +pgxn/neon/neon.c:neon_shmem_startup_hook(void) +pgxn/neon/neon.c: WAIT_EVENT_NEON_LFC_MAINTENANCE = WaitEventExtensionNew("Neon/FileCache_Maintenance"); +pgxn/neon/neon.c: WAIT_EVENT_NEON_LFC_READ = WaitEventExtensionNew("Neon/FileCache_Read"); +pgxn/neon/neon.c: WAIT_EVENT_NEON_LFC_TRUNCATE = WaitEventExtensionNew("Neon/FileCache_Truncate"); +pgxn/neon/neon.c: WAIT_EVENT_NEON_LFC_WRITE = WaitEventExtensionNew("Neon/FileCache_Write"); +pgxn/neon/neon.c: WAIT_EVENT_NEON_PS_STARTING = WaitEventExtensionNew("Neon/PS_Starting"); +pgxn/neon/neon.c: WAIT_EVENT_NEON_PS_CONFIGURING = WaitEventExtensionNew("Neon/PS_Configuring"); +pgxn/neon/neon.c: WAIT_EVENT_NEON_PS_SEND = WaitEventExtensionNew("Neon/PS_SendIO"); +pgxn/neon/neon.c: WAIT_EVENT_NEON_PS_READ = WaitEventExtensionNew("Neon/PS_ReadIO"); +pgxn/neon/neon.c: WAIT_EVENT_NEON_WAL_DL = WaitEventExtensionNew("Neon/WAL_Download"); +pgxn/neon/neon.control:# neon extension +pgxn/neon/neon.control:module_pathname = '$libdir/neon' +pgxn/neon/neon.h: * neon.h +pgxn/neon/neon.h: * contrib/neon/neon.h +pgxn/neon/neon.h:#ifndef NEON_H +pgxn/neon/neon.h:#define NEON_H +pgxn/neon/neon.h:extern char *neon_auth_token; +pgxn/neon/neon.h:extern char *neon_timeline; +pgxn/neon/neon.h:extern char *neon_tenant; +pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_LFC_MAINTENANCE; +pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_LFC_READ; +pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_LFC_TRUNCATE; +pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_LFC_WRITE; +pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_PS_STARTING; +pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_PS_CONFIGURING; +pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_PS_SEND; +pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_PS_READ; +pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_WAL_DL; +pgxn/neon/neon.h:#define WAIT_EVENT_NEON_LFC_MAINTENANCE PG_WAIT_EXTENSION +pgxn/neon/neon.h:#define WAIT_EVENT_NEON_LFC_READ WAIT_EVENT_BUFFILE_READ +pgxn/neon/neon.h:#define WAIT_EVENT_NEON_LFC_TRUNCATE WAIT_EVENT_BUFFILE_TRUNCATE +pgxn/neon/neon.h:#define WAIT_EVENT_NEON_LFC_WRITE WAIT_EVENT_BUFFILE_WRITE +pgxn/neon/neon.h:#define WAIT_EVENT_NEON_PS_STARTING PG_WAIT_EXTENSION +pgxn/neon/neon.h:#define WAIT_EVENT_NEON_PS_CONFIGURING PG_WAIT_EXTENSION +pgxn/neon/neon.h:#define WAIT_EVENT_NEON_PS_SEND PG_WAIT_EXTENSION +pgxn/neon/neon.h:#define WAIT_EVENT_NEON_PS_READ PG_WAIT_EXTENSION +pgxn/neon/neon.h:#define WAIT_EVENT_NEON_WAL_DL WAIT_EVENT_WAL_READ +pgxn/neon/neon.h:extern void SetNeonCurrentClusterSize(uint64 size); +pgxn/neon/neon.h:extern uint64 GetNeonCurrentClusterSize(void); +pgxn/neon/neon.h:#endif /* NEON_H */ +pgxn/neon/neon_perf_counters.c: * neon_perf_counters.c +pgxn/neon/neon_perf_counters.c: * Collect statistics about Neon I/O +pgxn/neon/neon_perf_counters.c:#include "neon_perf_counters.h" +pgxn/neon/neon_perf_counters.c:#include "neon_pgversioncompat.h" +pgxn/neon/neon_perf_counters.c:neon_per_backend_counters *neon_per_backend_counters_shared; +pgxn/neon/neon_perf_counters.c:NeonPerfCountersShmemSize(void) +pgxn/neon/neon_perf_counters.c: size = add_size(size, mul_size(NUM_NEON_PERF_COUNTER_SLOTS, +pgxn/neon/neon_perf_counters.c: sizeof(neon_per_backend_counters))); +pgxn/neon/neon_perf_counters.c:NeonPerfCountersShmemInit(void) +pgxn/neon/neon_perf_counters.c: neon_per_backend_counters_shared = +pgxn/neon/neon_perf_counters.c: mul_size(NUM_NEON_PERF_COUNTER_SLOTS, +pgxn/neon/neon_perf_counters.c: sizeof(neon_per_backend_counters)), +pgxn/neon/neon_perf_counters.c: inc_iohist(&MyNeonCounters->getpage_hist, latency); +pgxn/neon/neon_perf_counters.c: inc_iohist(&MyNeonCounters->file_cache_read_hist, latency); +pgxn/neon/neon_perf_counters.c: inc_iohist(&MyNeonCounters->file_cache_write_hist, latency); +pgxn/neon/neon_perf_counters.c: * Support functions for the views, neon_backend_perf_counters and +pgxn/neon/neon_perf_counters.c: * neon_perf_counters. +pgxn/neon/neon_perf_counters.c:neon_perf_counters_to_metrics(neon_per_backend_counters *counters) +pgxn/neon/neon_perf_counters.c:PG_FUNCTION_INFO_V1(neon_get_backend_perf_counters); +pgxn/neon/neon_perf_counters.c:neon_get_backend_perf_counters(PG_FUNCTION_ARGS) +pgxn/neon/neon_perf_counters.c: for (int procno = 0; procno < NUM_NEON_PERF_COUNTER_SLOTS; procno++) +pgxn/neon/neon_perf_counters.c: neon_per_backend_counters *counters = &neon_per_backend_counters_shared[procno]; +pgxn/neon/neon_perf_counters.c: metric_t *metrics = neon_perf_counters_to_metrics(counters); +pgxn/neon/neon_perf_counters.c:PG_FUNCTION_INFO_V1(neon_get_perf_counters); +pgxn/neon/neon_perf_counters.c:neon_get_perf_counters(PG_FUNCTION_ARGS) +pgxn/neon/neon_perf_counters.c: neon_per_backend_counters totals = {0}; +pgxn/neon/neon_perf_counters.c: for (int procno = 0; procno < NUM_NEON_PERF_COUNTER_SLOTS; procno++) +pgxn/neon/neon_perf_counters.c: neon_per_backend_counters *counters = &neon_per_backend_counters_shared[procno]; +pgxn/neon/neon_perf_counters.c: metrics = neon_perf_counters_to_metrics(&totals); +pgxn/neon/neon_perf_counters.c: // Not ideal but piggyback our databricks counters into the neon perf counters view +pgxn/neon/neon_perf_counters.c: // so that we don't need to introduce neon--1.x+1.sql to add a new view. +pgxn/neon/neon_perf_counters.h: * neon_perf_counters.h +pgxn/neon/neon_perf_counters.h: * Performance counters for neon storage requests +pgxn/neon/neon_perf_counters.h:#ifndef NEON_PERF_COUNTERS_H +pgxn/neon/neon_perf_counters.h:#define NEON_PERF_COUNTERS_H +pgxn/neon/neon_perf_counters.h: * the backend, but the 'neon_backend_perf_counters' view will convert +pgxn/neon/neon_perf_counters.h:} neon_per_backend_counters; +pgxn/neon/neon_perf_counters.h:/* Pointer to the shared memory array of neon_per_backend_counters structs */ +pgxn/neon/neon_perf_counters.h:extern neon_per_backend_counters *neon_per_backend_counters_shared; +pgxn/neon/neon_perf_counters.h:#define NUM_NEON_PERF_COUNTER_SLOTS (MaxBackends + NUM_AUXILIARY_PROCS) +pgxn/neon/neon_perf_counters.h:#define MyNeonCounters (&neon_per_backend_counters_shared[MyProcNumber]) +pgxn/neon/neon_perf_counters.h:#define MyNeonCounters (&neon_per_backend_counters_shared[MyProc->pgprocno]) +pgxn/neon/neon_perf_counters.h:extern Size NeonPerfCountersShmemSize(void); +pgxn/neon/neon_perf_counters.h:extern void NeonPerfCountersShmemInit(void); +pgxn/neon/neon_perf_counters.h:#endif /* NEON_PERF_COUNTERS_H */ +pgxn/neon/neon_pgversioncompat.c: * Support functions for the compatibility macros in neon_pgversioncompat.h +pgxn/neon/neon_pgversioncompat.c:#include "neon_pgversioncompat.h" +pgxn/neon/neon_pgversioncompat.h:#ifndef NEON_PGVERSIONCOMPAT_H +pgxn/neon/neon_pgversioncompat.h:#define NEON_PGVERSIONCOMPAT_H +pgxn/neon/neon_pgversioncompat.h:#endif /* NEON_PGVERSIONCOMPAT_H */ +pgxn/neon/neon_utils.c:#include "neon_utils.h" +pgxn/neon/neon_utils.c: * place to call curl_global_init() would be _PG_init(), but Neon has to be +pgxn/neon/neon_utils.h:#ifndef __NEON_UTILS_H__ +pgxn/neon/neon_utils.h:#define __NEON_UTILS_H__ +pgxn/neon/neon_utils.h:#endif /* __NEON_UTILS_H__ */ +pgxn/neon/neon_walreader.c: * ERROR asynchronously tries to fetch it from the most advanced safekeeper. +pgxn/neon/neon_walreader.c:#include "neon_walreader.h" +pgxn/neon/neon_walreader.c:#define NEON_WALREADER_ERR_MSG_LEN 512 +pgxn/neon/neon_walreader.c: * Can be called where NeonWALReader *state is available in the context, adds log_prefix. +pgxn/neon/neon_walreader.c:static NeonWALReadResult NeonWALReadRemote(NeonWALReader *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli); +pgxn/neon/neon_walreader.c:static NeonWALReadResult NeonWALReaderReadMsg(NeonWALReader *state); +pgxn/neon/neon_walreader.c:static bool NeonWALReadLocal(NeonWALReader *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli); +pgxn/neon/neon_walreader.c: * State of connection to donor safekeeper. +pgxn/neon/neon_walreader.c:} NeonWALReaderRemoteState; +pgxn/neon/neon_walreader.c:struct NeonWALReader +pgxn/neon/neon_walreader.c: char err_msg[NEON_WALREADER_ERR_MSG_LEN]; +pgxn/neon/neon_walreader.c: char donor_name[64]; /* saved donor safekeeper name for logging */ +pgxn/neon/neon_walreader.c: /* state of connection to safekeeper */ +pgxn/neon/neon_walreader.c: NeonWALReaderRemoteState rem_state; +pgxn/neon/neon_walreader.c: /* prepended to lines logged by neon_walreader, if provided */ +pgxn/neon/neon_walreader.c:/* palloc and initialize NeonWALReader */ +pgxn/neon/neon_walreader.c:NeonWALReader * +pgxn/neon/neon_walreader.c:NeonWALReaderAllocate(int wal_segment_size, XLogRecPtr available_lsn, char *log_prefix) +pgxn/neon/neon_walreader.c: NeonWALReader *reader; +pgxn/neon/neon_walreader.c: reader = (NeonWALReader *) +pgxn/neon/neon_walreader.c: MemoryContextAllocZero(TopMemoryContext, sizeof(NeonWALReader)); +pgxn/neon/neon_walreader.c:NeonWALReaderFree(NeonWALReader *state) +pgxn/neon/neon_walreader.c: neon_wal_segment_close(state); +pgxn/neon/neon_walreader.c: * advanced safekeeper. +pgxn/neon/neon_walreader.c: * Returns NEON_WALREAD_SUCCESS if succeeded, NEON_WALREAD_ERROR if an error +pgxn/neon/neon_walreader.c: * NEON_WALREAD_WOULDBLOCK means caller should obtain socket to wait for with +pgxn/neon/neon_walreader.c: * NeonWALReaderSocket and call NeonWALRead again with exactly the same +pgxn/neon/neon_walreader.c: * arguments when NeonWALReaderEvents happen on the socket. Note that per libpq +pgxn/neon/neon_walreader.c: * should remove subscription to socket then by checking NeonWALReaderEvents +pgxn/neon/neon_walreader.c: * NEON_WALREAD_WOULDBLOCK might be always returned. +pgxn/neon/neon_walreader.c:NeonWALReadResult +pgxn/neon/neon_walreader.c:NeonWALRead(NeonWALReader *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli) +pgxn/neon/neon_walreader.c: return NeonWALReadRemote(state, buf, startptr, count, tli); +pgxn/neon/neon_walreader.c: if (NeonWALReadLocal(state, buf, startptr, count, tli)) +pgxn/neon/neon_walreader.c: return NEON_WALREAD_SUCCESS; +pgxn/neon/neon_walreader.c: return NeonWALReadRemote(state, buf, startptr, count, tli); +pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; +pgxn/neon/neon_walreader.c:/* Do the read from remote safekeeper. */ +pgxn/neon/neon_walreader.c:static NeonWALReadResult +pgxn/neon/neon_walreader.c:NeonWALReadRemote(NeonWALReader *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli) +pgxn/neon/neon_walreader.c: if (!NeonWALReaderUpdateDonor(state)) +pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; +pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); +pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; +pgxn/neon/neon_walreader.c: return NEON_WALREAD_WOULDBLOCK; +pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); +pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; +pgxn/neon/neon_walreader.c: return NEON_WALREAD_WOULDBLOCK; +pgxn/neon/neon_walreader.c: return NEON_WALREAD_WOULDBLOCK; +pgxn/neon/neon_walreader.c: * might stream WAL not yet committed by safekeepers. It +pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); +pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; +pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; /* keep the compiler quiet */ +pgxn/neon/neon_walreader.c: return NEON_WALREAD_WOULDBLOCK; +pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); +pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; +pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); +pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; +pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); +pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; +pgxn/neon/neon_walreader.c: NeonWALReadResult read_msg_res = NeonWALReaderReadMsg(state); +pgxn/neon/neon_walreader.c: if (read_msg_res != NEON_WALREAD_SUCCESS) +pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); +pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; +pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); +pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); +pgxn/neon/neon_walreader.c: return NEON_WALREAD_SUCCESS; +pgxn/neon/neon_walreader.c:static NeonWALReadResult +pgxn/neon/neon_walreader.c:NeonWALReaderReadMsg(NeonWALReader *state) +pgxn/neon/neon_walreader.c: return NEON_WALREAD_WOULDBLOCK; +pgxn/neon/neon_walreader.c: return NEON_WALREAD_SUCCESS; +pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); +pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; +pgxn/neon/neon_walreader.c:NeonWALReaderResetRemote(NeonWALReader *state) +pgxn/neon/neon_walreader.c: * connection exists (NeonWALReaderEvents returns non zero). +pgxn/neon/neon_walreader.c:NeonWALReaderSocket(NeonWALReader *state) +pgxn/neon/neon_walreader.c:NeonWALReaderIsRemConnEstablished(NeonWALReader *state) +pgxn/neon/neon_walreader.c:NeonWALReaderEvents(NeonWALReader *state) +pgxn/neon/neon_walreader.c:NeonWALReadLocal(NeonWALReader *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli) +pgxn/neon/neon_walreader.c: neon_wal_segment_close(state); +pgxn/neon/neon_walreader.c: if (!neon_wal_segment_open(state, nextSegNo, &tli)) +pgxn/neon/neon_walreader.c:NeonWALReaderGetRemLsn(NeonWALReader *state) +pgxn/neon/neon_walreader.c:NeonWALReaderGetSegment(NeonWALReader *state) +pgxn/neon/neon_walreader.c:neon_wal_segment_open(NeonWALReader *state, XLogSegNo nextSegNo, +pgxn/neon/neon_walreader.c:/* copy of vanilla wal_segment_close with NeonWALReader */ +pgxn/neon/neon_walreader.c:neon_wal_segment_close(NeonWALReader *state) +pgxn/neon/neon_walreader.c:NeonWALReaderErrMsg(NeonWALReader *state) +pgxn/neon/neon_walreader.c:NeonWALReaderUpdateDonor(NeonWALReader *state) +pgxn/neon/neon_walreader.h:#ifndef __NEON_WALREADER_H__ +pgxn/neon/neon_walreader.h:#define __NEON_WALREADER_H__ +pgxn/neon/neon_walreader.h:struct NeonWALReader; +pgxn/neon/neon_walreader.h:typedef struct NeonWALReader NeonWALReader; +pgxn/neon/neon_walreader.h:/* NeonWALRead return value */ +pgxn/neon/neon_walreader.h: NEON_WALREAD_SUCCESS, +pgxn/neon/neon_walreader.h: NEON_WALREAD_WOULDBLOCK, +pgxn/neon/neon_walreader.h: NEON_WALREAD_ERROR, +pgxn/neon/neon_walreader.h:} NeonWALReadResult; +pgxn/neon/neon_walreader.h:extern NeonWALReader *NeonWALReaderAllocate(int wal_segment_size, XLogRecPtr available_lsn, char *log_prefix); +pgxn/neon/neon_walreader.h:extern void NeonWALReaderFree(NeonWALReader *state); +pgxn/neon/neon_walreader.h:extern void NeonWALReaderResetRemote(NeonWALReader *state); +pgxn/neon/neon_walreader.h:extern NeonWALReadResult NeonWALRead(NeonWALReader *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli); +pgxn/neon/neon_walreader.h:extern pgsocket NeonWALReaderSocket(NeonWALReader *state); +pgxn/neon/neon_walreader.h:extern uint32 NeonWALReaderEvents(NeonWALReader *state); +pgxn/neon/neon_walreader.h:extern bool NeonWALReaderIsRemConnEstablished(NeonWALReader *state); +pgxn/neon/neon_walreader.h:extern char *NeonWALReaderErrMsg(NeonWALReader *state); +pgxn/neon/neon_walreader.h:extern XLogRecPtr NeonWALReaderGetRemLsn(NeonWALReader *state); +pgxn/neon/neon_walreader.h:extern const WALOpenSegment *NeonWALReaderGetSegment(NeonWALReader *state); +pgxn/neon/neon_walreader.h:extern bool neon_wal_segment_open(NeonWALReader *state, XLogSegNo nextSegNo, TimeLineID *tli_p); +pgxn/neon/neon_walreader.h:extern void neon_wal_segment_close(NeonWALReader *state); +pgxn/neon/neon_walreader.h:extern bool NeonWALReaderUpdateDonor(NeonWALReader *state); +pgxn/neon/neon_walreader.h:#endif /* __NEON_WALREADER_H__ */ +pgxn/neon/pagestore_client.h:#include "neon_pgversioncompat.h" +pgxn/neon/pagestore_client.h: T_NeonExistsRequest = 0, +pgxn/neon/pagestore_client.h: T_NeonNblocksRequest, +pgxn/neon/pagestore_client.h: T_NeonGetPageRequest, +pgxn/neon/pagestore_client.h: T_NeonDbSizeRequest, +pgxn/neon/pagestore_client.h: T_NeonGetSlruSegmentRequest, +pgxn/neon/pagestore_client.h: T_NeonTestRequest = 99, /* only in cfg(feature = "testing") */ +pgxn/neon/pagestore_client.h: T_NeonExistsResponse = 100, +pgxn/neon/pagestore_client.h: T_NeonNblocksResponse, +pgxn/neon/pagestore_client.h: T_NeonGetPageResponse, +pgxn/neon/pagestore_client.h: T_NeonErrorResponse, +pgxn/neon/pagestore_client.h: T_NeonDbSizeResponse, +pgxn/neon/pagestore_client.h: T_NeonGetSlruSegmentResponse, +pgxn/neon/pagestore_client.h: T_NeonTestResponse = 199, /* only in cfg(feature = "testing") */ +pgxn/neon/pagestore_client.h:} NeonMessageTag; +pgxn/neon/pagestore_client.h:typedef uint64 NeonRequestId; +pgxn/neon/pagestore_client.h: NeonMessageTag tag; +pgxn/neon/pagestore_client.h: NeonRequestId reqid; +pgxn/neon/pagestore_client.h:} NeonMessage; +pgxn/neon/pagestore_client.h:#define messageTag(m) (((const NeonMessage *)(m))->tag) +pgxn/neon/pagestore_client.h:#define NEON_TAG "[DATABRICKS_SMGR] " +pgxn/neon/pagestore_client.h:#define neon_log(tag, fmt, ...) ereport(tag, \ +pgxn/neon/pagestore_client.h: (errmsg(NEON_TAG fmt, ##__VA_ARGS__), \ +pgxn/neon/pagestore_client.h:#define neon_shard_log(shard_no, tag, fmt, ...) ereport(tag, \ +pgxn/neon/pagestore_client.h: (errmsg(NEON_TAG "[shard %d] " fmt, shard_no, ##__VA_ARGS__), \ +pgxn/neon/pagestore_client.h: * supertype of all the Neon*Request structs below. +pgxn/neon/pagestore_client.h:typedef NeonMessage NeonRequest; +pgxn/neon/pagestore_client.h: NeonRequest hdr; +pgxn/neon/pagestore_client.h:} NeonExistsRequest; +pgxn/neon/pagestore_client.h: NeonRequest hdr; +pgxn/neon/pagestore_client.h:} NeonNblocksRequest; +pgxn/neon/pagestore_client.h: NeonRequest hdr; +pgxn/neon/pagestore_client.h:} NeonDbSizeRequest; +pgxn/neon/pagestore_client.h: NeonRequest hdr; +pgxn/neon/pagestore_client.h:} NeonGetPageRequest; +pgxn/neon/pagestore_client.h: NeonRequest hdr; +pgxn/neon/pagestore_client.h:} NeonGetSlruSegmentRequest; +pgxn/neon/pagestore_client.h:/* supertype of all the Neon*Response structs below */ +pgxn/neon/pagestore_client.h:typedef NeonMessage NeonResponse; +pgxn/neon/pagestore_client.h: NeonExistsRequest req; +pgxn/neon/pagestore_client.h:} NeonExistsResponse; +pgxn/neon/pagestore_client.h: NeonNblocksRequest req; +pgxn/neon/pagestore_client.h:} NeonNblocksResponse; +pgxn/neon/pagestore_client.h: NeonGetPageRequest req; +pgxn/neon/pagestore_client.h:} NeonGetPageResponse; +pgxn/neon/pagestore_client.h:#define PS_GETPAGERESPONSE_SIZE (MAXALIGN(offsetof(NeonGetPageResponse, page) + BLCKSZ)) +pgxn/neon/pagestore_client.h: NeonDbSizeRequest req; +pgxn/neon/pagestore_client.h:} NeonDbSizeResponse; +pgxn/neon/pagestore_client.h: NeonResponse req; +pgxn/neon/pagestore_client.h:} NeonErrorResponse; +pgxn/neon/pagestore_client.h: NeonGetSlruSegmentRequest req; +pgxn/neon/pagestore_client.h:} NeonGetSlruSegmentResponse; +pgxn/neon/pagestore_client.h:extern StringInfoData nm_pack_request(NeonRequest *msg); +pgxn/neon/pagestore_client.h:extern NeonResponse *nm_unpack_response(StringInfo s); +pgxn/neon/pagestore_client.h:extern char *nm_to_string(NeonMessage *msg); +pgxn/neon/pagestore_client.h: bool (*send) (shardno_t shard_no, NeonRequest * request); +pgxn/neon/pagestore_client.h: NeonResponse *(*receive) (shardno_t shard_no); +pgxn/neon/pagestore_client.h: NeonResponse *(*try_receive) (shardno_t shard_no); +pgxn/neon/pagestore_client.h:extern char *neon_timeline; +pgxn/neon/pagestore_client.h:extern char *neon_tenant; +pgxn/neon/pagestore_client.h:extern int neon_protocol_version; +pgxn/neon/pagestore_client.h:extern const f_smgr *smgr_neon(ProcNumber backend, NRelFileInfo rinfo); +pgxn/neon/pagestore_client.h:extern void smgr_init_neon(void); +pgxn/neon/pagestore_client.h:} neon_request_lsns; +pgxn/neon/pagestore_client.h:extern PGDLLEXPORT void neon_read_at_lsn(NRelFileInfo rnode, ForkNumber forkNum, BlockNumber blkno, +pgxn/neon/pagestore_client.h: neon_request_lsns request_lsns, char *buffer); +pgxn/neon/pagestore_client.h:extern PGDLLEXPORT void neon_read_at_lsn(NRelFileInfo rnode, ForkNumber forkNum, BlockNumber blkno, +pgxn/neon/pagestore_client.h: neon_request_lsns request_lsns, void *buffer); +pgxn/neon/pagestore_client.h:extern int64 neon_dbsize(Oid dbNode); +pgxn/neon/pagestore_client.h:/* utils for neon relsize cache */ +pgxn/neon/pagestore_smgr.c: * contrib/neon/pagestore_smgr.c +pgxn/neon/pagestore_smgr.c:#include "neon_perf_counters.h" +pgxn/neon/pagestore_smgr.c:#define NEON_PANIC_CONNECTION_STATE(shard_no, elvl, message, ...) \ +pgxn/neon/pagestore_smgr.c: neon_shard_log(shard_no, elvl, "Broken connection state: " message, \ +pgxn/neon/pagestore_smgr.c:static bool neon_redo_read_buffer_filter(XLogReaderState *record, uint8 block_id); +pgxn/neon/pagestore_smgr.c:static BlockNumber neon_nblocks(SMgrRelation reln, ForkNumber forknum); +pgxn/neon/pagestore_smgr.c:#define GENERATE_REQUEST_ID() (((NeonRequestId)MyProcPid << 32) | ++local_request_counter) +pgxn/neon/pagestore_smgr.c: * flushes sent requests on manual flush, or every neon.flush_output_after +pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns; +pgxn/neon/pagestore_smgr.c: NeonRequestId reqid; +pgxn/neon/pagestore_smgr.c: NeonResponse *response; /* may be null */ +pgxn/neon/pagestore_smgr.c:#include "neon.h" +pgxn/neon/pagestore_smgr.c:static void prefetch_do_request(PrefetchRequest *slot, neon_request_lsns *force_request_lsns); +pgxn/neon/pagestore_smgr.c:neon_get_request_lsns(NRelFileInfo rinfo, ForkNumber forknum, +pgxn/neon/pagestore_smgr.c: BlockNumber blkno, neon_request_lsns *output, +pgxn/neon/pagestore_smgr.c:static bool neon_prefetch_response_usable(neon_request_lsns *request_lsns, +pgxn/neon/pagestore_smgr.c: source_slot->request_lsns = (neon_request_lsns) { +pgxn/neon/pagestore_smgr.c: NeonResponse *response; +pgxn/neon/pagestore_smgr.c: neon_shard_log(slot->shard_no, ERROR, +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetches_buffered = +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetches_buffered = +pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests = +pgxn/neon/pagestore_smgr.c: NeonResponse *response; +pgxn/neon/pagestore_smgr.c: neon_shard_log(slot->shard_no, ERROR, +pgxn/neon/pagestore_smgr.c: response = (NeonResponse *) page_server->receive(shard_no); +pgxn/neon/pagestore_smgr.c: neon_shard_log(shard_no, ERROR, +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetches_buffered = +pgxn/neon/pagestore_smgr.c: neon_shard_log(shard_no, LOG, +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_discards_total += 1; +pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests = +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetches_buffered = +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetches_buffered = +pgxn/neon/pagestore_smgr.c:prefetch_do_request(PrefetchRequest *slot, neon_request_lsns *force_request_lsns) +pgxn/neon/pagestore_smgr.c: NeonGetPageRequest request = { +pgxn/neon/pagestore_smgr.c: .hdr.tag = T_NeonGetPageRequest, +pgxn/neon/pagestore_smgr.c: neon_get_request_lsns(BufTagGetNRelFileInfo(slot->buftag), +pgxn/neon/pagestore_smgr.c: while (!page_server->send(slot->shard_no, (NeonRequest *) &request)) +pgxn/neon/pagestore_smgr.c:prefetch_register_bufferv(BufferTag tag, neon_request_lsns *frlsns, +pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests = +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetches_buffered = +pgxn/neon/pagestore_smgr.c: neon_request_lsns *lsns; +pgxn/neon/pagestore_smgr.c: if (!neon_prefetch_response_usable(lsns, slot)) +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_discards_total += 1; +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_misses_total++; +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_discards_total += 1; +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_discards_total += 1; +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_requests_total++; +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_sync_requests_total++; +pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests = +pgxn/neon/pagestore_smgr.c:equal_requests(NeonRequest* a, NeonRequest* b) +pgxn/neon/pagestore_smgr.c:static NeonResponse * +pgxn/neon/pagestore_smgr.c: NeonResponse *resp; +pgxn/neon/pagestore_smgr.c: case T_NeonExistsRequest: +pgxn/neon/pagestore_smgr.c: CopyNRelFileInfoToBufTag(tag, ((NeonExistsRequest *) req)->rinfo); +pgxn/neon/pagestore_smgr.c: case T_NeonNblocksRequest: +pgxn/neon/pagestore_smgr.c: CopyNRelFileInfoToBufTag(tag, ((NeonNblocksRequest *) req)->rinfo); +pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeRequest: +pgxn/neon/pagestore_smgr.c: NInfoGetDbOid(BufTagGetNRelFileInfo(tag)) = ((NeonDbSizeRequest *) req)->dbNode; +pgxn/neon/pagestore_smgr.c: case T_NeonGetPageRequest: +pgxn/neon/pagestore_smgr.c: CopyNRelFileInfoToBufTag(tag, ((NeonGetPageRequest *) req)->rinfo); +pgxn/neon/pagestore_smgr.c: tag.blockNum = ((NeonGetPageRequest *) req)->blkno; +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "Unexpected request tag: %d", messageTag(req)); +pgxn/neon/pagestore_smgr.c: if (((NeonRequest *) req)->tag != T_NeonGetPageRequest) +pgxn/neon/pagestore_smgr.c: while (!page_server->send(shard_no, (NeonRequest *) req) +pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests++; +pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests--; +pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests = 0; +pgxn/neon/pagestore_smgr.c:nm_pack_request(NeonRequest *msg) +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: case T_NeonExistsRequest: +pgxn/neon/pagestore_smgr.c: NeonExistsRequest *msg_req = (NeonExistsRequest *) msg; +pgxn/neon/pagestore_smgr.c: case T_NeonNblocksRequest: +pgxn/neon/pagestore_smgr.c: NeonNblocksRequest *msg_req = (NeonNblocksRequest *) msg; +pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeRequest: +pgxn/neon/pagestore_smgr.c: NeonDbSizeRequest *msg_req = (NeonDbSizeRequest *) msg; +pgxn/neon/pagestore_smgr.c: case T_NeonGetPageRequest: +pgxn/neon/pagestore_smgr.c: NeonGetPageRequest *msg_req = (NeonGetPageRequest *) msg; +pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentRequest: +pgxn/neon/pagestore_smgr.c: NeonGetSlruSegmentRequest *msg_req = (NeonGetSlruSegmentRequest *) msg; +pgxn/neon/pagestore_smgr.c: case T_NeonExistsResponse: +pgxn/neon/pagestore_smgr.c: case T_NeonNblocksResponse: +pgxn/neon/pagestore_smgr.c: case T_NeonGetPageResponse: +pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: +pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeResponse: +pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentResponse: +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unexpected databricks message tag 0x%02x", msg->tag); +pgxn/neon/pagestore_smgr.c:NeonResponse * +pgxn/neon/pagestore_smgr.c: NeonMessageTag tag = pq_getmsgbyte(s); +pgxn/neon/pagestore_smgr.c: NeonResponse resp_hdr = {0}; /* make valgrind happy */ +pgxn/neon/pagestore_smgr.c: NeonResponse *resp = NULL; +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: case T_NeonExistsResponse: +pgxn/neon/pagestore_smgr.c: NeonExistsResponse *msg_resp = palloc0(sizeof(NeonExistsResponse)); +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: resp = (NeonResponse *) msg_resp; +pgxn/neon/pagestore_smgr.c: case T_NeonNblocksResponse: +pgxn/neon/pagestore_smgr.c: NeonNblocksResponse *msg_resp = palloc0(sizeof(NeonNblocksResponse)); +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: resp = (NeonResponse *) msg_resp; +pgxn/neon/pagestore_smgr.c: case T_NeonGetPageResponse: +pgxn/neon/pagestore_smgr.c: NeonGetPageResponse *msg_resp; +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: Assert(msg_resp->req.hdr.tag == T_NeonGetPageResponse); +pgxn/neon/pagestore_smgr.c: resp = (NeonResponse *) msg_resp; +pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeResponse: +pgxn/neon/pagestore_smgr.c: NeonDbSizeResponse *msg_resp = palloc0(sizeof(NeonDbSizeResponse)); +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: resp = (NeonResponse *) msg_resp; +pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: +pgxn/neon/pagestore_smgr.c: NeonErrorResponse *msg_resp; +pgxn/neon/pagestore_smgr.c: msg_resp = palloc0(sizeof(NeonErrorResponse) + msglen + 1); +pgxn/neon/pagestore_smgr.c: resp = (NeonResponse *) msg_resp; +pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentResponse: +pgxn/neon/pagestore_smgr.c: NeonGetSlruSegmentResponse *msg_resp; +pgxn/neon/pagestore_smgr.c: msg_resp = palloc0(sizeof(NeonGetSlruSegmentResponse)); +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: resp = (NeonResponse *) msg_resp; +pgxn/neon/pagestore_smgr.c: case T_NeonExistsRequest: +pgxn/neon/pagestore_smgr.c: case T_NeonNblocksRequest: +pgxn/neon/pagestore_smgr.c: case T_NeonGetPageRequest: +pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeRequest: +pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentRequest: +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unexpected databricks message tag 0x%02x", tag); +pgxn/neon/pagestore_smgr.c:nm_to_string(NeonMessage *msg) +pgxn/neon/pagestore_smgr.c: case T_NeonExistsRequest: +pgxn/neon/pagestore_smgr.c: NeonExistsRequest *msg_req = (NeonExistsRequest *) msg; +pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonExistsRequest\""); +pgxn/neon/pagestore_smgr.c: case T_NeonNblocksRequest: +pgxn/neon/pagestore_smgr.c: NeonNblocksRequest *msg_req = (NeonNblocksRequest *) msg; +pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonNblocksRequest\""); +pgxn/neon/pagestore_smgr.c: case T_NeonGetPageRequest: +pgxn/neon/pagestore_smgr.c: NeonGetPageRequest *msg_req = (NeonGetPageRequest *) msg; +pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonGetPageRequest\""); +pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeRequest: +pgxn/neon/pagestore_smgr.c: NeonDbSizeRequest *msg_req = (NeonDbSizeRequest *) msg; +pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonDbSizeRequest\""); +pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentRequest: +pgxn/neon/pagestore_smgr.c: NeonGetSlruSegmentRequest *msg_req = (NeonGetSlruSegmentRequest *) msg; +pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonGetSlruSegmentRequest\""); +pgxn/neon/pagestore_smgr.c: case T_NeonExistsResponse: +pgxn/neon/pagestore_smgr.c: NeonExistsResponse *msg_resp = (NeonExistsResponse *) msg; +pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonExistsResponse\""); +pgxn/neon/pagestore_smgr.c: case T_NeonNblocksResponse: +pgxn/neon/pagestore_smgr.c: NeonNblocksResponse *msg_resp = (NeonNblocksResponse *) msg; +pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonNblocksResponse\""); +pgxn/neon/pagestore_smgr.c: case T_NeonGetPageResponse: +pgxn/neon/pagestore_smgr.c: NeonGetPageResponse *msg_resp = (NeonGetPageResponse *) msg; +pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonGetPageResponse\""); +pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: +pgxn/neon/pagestore_smgr.c: NeonErrorResponse *msg_resp = (NeonErrorResponse *) msg; +pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonErrorResponse\""); +pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeResponse: +pgxn/neon/pagestore_smgr.c: NeonDbSizeResponse *msg_resp = (NeonDbSizeResponse *) msg; +pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonDbSizeResponse\""); +pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentResponse: +pgxn/neon/pagestore_smgr.c: NeonGetSlruSegmentResponse *msg_resp = (NeonGetSlruSegmentResponse *) msg; +pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonGetSlruSegmentResponse\""); +pgxn/neon/pagestore_smgr.c: // Replace "Neon" with "Dbrx" +pgxn/neon/pagestore_smgr.c: while ((pos = strcasestr(pos, "neon")) != NULL) { +pgxn/neon/pagestore_smgr.c:neon_wallog_pagev(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u through %u of relation %u/%u/%u.%u " +pgxn/neon/pagestore_smgr.c: * all-zeros pages, and we can just ignore that in Neon. We do need to +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u is all-zeros", +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u is an empty heap page with no LSN", +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u is evicted with zero LSN", +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Evicting page %u of relation %u/%u/%u.%u with lsn=%X/%X", +pgxn/neon/pagestore_smgr.c:neon_wallog_page(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer, bool force) +pgxn/neon/pagestore_smgr.c:neon_wallog_page(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, const char *buffer, bool force) +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u was force logged. Evicted at lsn=%X/%X", +pgxn/neon/pagestore_smgr.c: * all-zeros pages, and we can just ignore that in Neon. We do need to +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u is all-zeros", +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u is an empty heap page with no LSN", +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u is evicted with zero LSN", +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Evicting page %u of relation %u/%u/%u.%u with lsn=%X/%X", +pgxn/neon/pagestore_smgr.c: * neon_init() -- Initialize private state +pgxn/neon/pagestore_smgr.c:neon_init(void) +pgxn/neon/pagestore_smgr.c: if (MyNeonCounters >= &neon_per_backend_counters_shared[NUM_NEON_PERF_COUNTER_SLOTS]) +pgxn/neon/pagestore_smgr.c: elog(ERROR, "MyNeonCounters points past end of array"); +pgxn/neon/pagestore_smgr.c: redo_read_buffer_filter = neon_redo_read_buffer_filter; +pgxn/neon/pagestore_smgr.c:neon_get_request_lsns(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno, +pgxn/neon/pagestore_smgr.c: neon_request_lsns *output, BlockNumber nblocks, +pgxn/neon/pagestore_smgr.c: * page cache (see neon_redo_read_buffer_filter below). +pgxn/neon/pagestore_smgr.c: * neon_redo_read_buffer_filter below). Normally, in a normal +pgxn/neon/pagestore_smgr.c: neon_request_lsns *result = &output[i]; +pgxn/neon/pagestore_smgr.c: * neon_read_at_lsn() call later will wait for the WAL record to be +pgxn/neon/pagestore_smgr.c: neon_log(DEBUG1, "databricks_get_request_lsns request lsn %X/%X, not_modified_since %X/%X", +pgxn/neon/pagestore_smgr.c: neon_request_lsns *result = &output[i]; +pgxn/neon/pagestore_smgr.c: neon_log(DEBUG1, "databricks_get_request_lsns GetLastWrittenLSN lsn %X/%X", +pgxn/neon/pagestore_smgr.c: neon_log(DEBUG5, "last-written LSN %X/%X is ahead of last flushed LSN %X/%X", +pgxn/neon/pagestore_smgr.c: * neon_prefetch_response_usable -- Can a new request be satisfied by old one? +pgxn/neon/pagestore_smgr.c:neon_prefetch_response_usable(neon_request_lsns *request_lsns, +pgxn/neon/pagestore_smgr.c: * interface in neon_test_utils to fetch pages at arbitary LSNs, which +pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "request with unexpected LSN after prefetch"), +pgxn/neon/pagestore_smgr.c: * neon_exists() -- Does the physical file exist? +pgxn/neon/pagestore_smgr.c:neon_exists(SMgrRelation reln, ForkNumber forkNum) +pgxn/neon/pagestore_smgr.c: NeonResponse *resp; +pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns; +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: neon_get_request_lsns(InfoFromSMgrRel(reln), forkNum, +pgxn/neon/pagestore_smgr.c: NeonExistsRequest request = { +pgxn/neon/pagestore_smgr.c: .hdr.tag = T_NeonExistsRequest, +pgxn/neon/pagestore_smgr.c: case T_NeonExistsResponse: +pgxn/neon/pagestore_smgr.c: NeonExistsResponse* exists_resp = (NeonExistsResponse *) resp; +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, +pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match exists request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", +pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "[reqid %lx] could not read relation existence of rel %u/%u/%u.%u from page server at lsn %X/%08X", +pgxn/neon/pagestore_smgr.c: ((NeonErrorResponse *) resp)->message))); +pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, +pgxn/neon/pagestore_smgr.c: T_NeonExistsResponse, T_NeonErrorResponse, resp->tag); +pgxn/neon/pagestore_smgr.c: * neon_create() -- Create a new relation on neond storage +pgxn/neon/pagestore_smgr.c:neon_create(SMgrRelation reln, ForkNumber forkNum, bool isRedo) +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrcreate() on rel with unknown persistence"); +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "Create relation %u/%u/%u.%u", +pgxn/neon/pagestore_smgr.c: * neon_unlink() -- Unlink a relation. +pgxn/neon/pagestore_smgr.c:neon_unlink(NRelFileInfoBackend rinfo, ForkNumber forkNum, bool isRedo) +pgxn/neon/pagestore_smgr.c: * neon_extend() -- Add a block to the specified relation. +pgxn/neon/pagestore_smgr.c:neon_extend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, +pgxn/neon/pagestore_smgr.c:neon_extend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrextend() on rel with unknown persistence"); +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: uint64 current_size = GetNeonCurrentClusterSize(); +pgxn/neon/pagestore_smgr.c: n_blocks = neon_nblocks(reln, forkNum); +pgxn/neon/pagestore_smgr.c: neon_wallog_page(reln, forkNum, n_blocks++, buffer, true); +pgxn/neon/pagestore_smgr.c: neon_wallog_page(reln, forkNum, blkno, buffer, false); +pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "smgrextend called for %u/%u/%u.%u blk %u, page LSN: %X/%08X", +pgxn/neon/pagestore_smgr.c:neon_zeroextend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blocknum, +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrextend() on rel with unknown persistence"); +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: uint64 current_size = GetNeonCurrentClusterSize(); +pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "cannot extend file \"%s\" beyond %u blocks", +pgxn/neon/pagestore_smgr.c: * neon_open() -- Initialize newly-opened relation. +pgxn/neon/pagestore_smgr.c:neon_open(SMgrRelation reln) +pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "open noop"); +pgxn/neon/pagestore_smgr.c: * neon_close() -- Close the specified relation, if it isn't closed already. +pgxn/neon/pagestore_smgr.c:neon_close(SMgrRelation reln, ForkNumber forknum) +pgxn/neon/pagestore_smgr.c: * neon_prefetch() -- Initiate asynchronous read of the specified block of a relation +pgxn/neon/pagestore_smgr.c:neon_prefetch(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: * neon_prefetch() -- Initiate asynchronous read of the specified block of a relation +pgxn/neon/pagestore_smgr.c:neon_prefetch(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum) +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: * neon_writeback() -- Tell the kernel to write pages back to storage. +pgxn/neon/pagestore_smgr.c:neon_writeback(SMgrRelation reln, ForkNumber forknum, +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "writeback noop"); +pgxn/neon/pagestore_smgr.c:neon_read_at_lsnv(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber base_blockno, neon_request_lsns *request_lsns, +pgxn/neon/pagestore_smgr.c:neon_read_at_lsnv(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber base_blockno, neon_request_lsns *request_lsns, +pgxn/neon/pagestore_smgr.c: NeonResponse *resp; +pgxn/neon/pagestore_smgr.c: * See also the description on neon_redo_read_buffer_filter below. +pgxn/neon/pagestore_smgr.c: neon_request_lsns *reqlsns = &request_lsns[i]; +pgxn/neon/pagestore_smgr.c: if (neon_prefetch_response_usable(reqlsns, slot)) +pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_discards_total++; +pgxn/neon/pagestore_smgr.c: case T_NeonGetPageResponse: +pgxn/neon/pagestore_smgr.c: NeonGetPageResponse* getpage_resp = (NeonGetPageResponse *) resp; +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, +pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match get relsize request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", +pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "[shard %d, reqid %lx] could not read block %u in rel %u/%u/%u.%u from page server at lsn %X/%08X", +pgxn/neon/pagestore_smgr.c: ((NeonErrorResponse *) resp)->message))); +pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(slot->shard_no, PANIC, +pgxn/neon/pagestore_smgr.c: T_NeonGetPageResponse, T_NeonErrorResponse, resp->tag); +pgxn/neon/pagestore_smgr.c: * While function is defined in the neon extension it's used within neon_test_utils directly. +pgxn/neon/pagestore_smgr.c:neon_read_at_lsn(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, +pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns, char *buffer) +pgxn/neon/pagestore_smgr.c:neon_read_at_lsn(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, +pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns, void *buffer) +pgxn/neon/pagestore_smgr.c: neon_read_at_lsnv(rinfo, forkNum, blkno, &request_lsns, &buffer, 1, NULL); +pgxn/neon/pagestore_smgr.c: * neon_read() -- Read the specified block from a relation. +pgxn/neon/pagestore_smgr.c:neon_read(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, char *buffer) +pgxn/neon/pagestore_smgr.c:neon_read(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, void *buffer) +pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns; +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrread() on rel with unknown persistence"); +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: MyNeonCounters->file_cache_hits_total++; +pgxn/neon/pagestore_smgr.c: neon_get_request_lsns(InfoFromSMgrRel(reln), forkNum, blkno, &request_lsns, 1, NULL); +pgxn/neon/pagestore_smgr.c: neon_read_at_lsn(InfoFromSMgrRel(reln), forkNum, blkno, request_lsns, buffer); +pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "page is new in MD but not in Page Server at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n%s\n", +pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "page is new in Page Server but not in MD at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n%s\n", +pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "heap buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n", +pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "btree buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n", +pgxn/neon/pagestore_smgr.c:neon_readv(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns[PG_IOV_MAX]; +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrread() on rel with unknown persistence"); +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "Read request too large: %d is larger than max %d", +pgxn/neon/pagestore_smgr.c: MyNeonCounters->file_cache_hits_total += lfc_result; +pgxn/neon/pagestore_smgr.c: neon_get_request_lsns(InfoFromSMgrRel(reln), forknum, blocknum, +pgxn/neon/pagestore_smgr.c: neon_read_at_lsnv(InfoFromSMgrRel(reln), forknum, blocknum, request_lsns, +pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "page is new in MD but not in Page Server at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n%s\n", +pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "page is new in Page Server but not in MD at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n%s\n", +pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "heap buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n", +pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "btree buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n", +pgxn/neon/pagestore_smgr.c: * neon_write() -- Write the supplied block at the appropriate location. +pgxn/neon/pagestore_smgr.c:neon_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer, bool skipFsync) +pgxn/neon/pagestore_smgr.c:neon_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, const void *buffer, bool skipFsync) +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: neon_wallog_page(reln, forknum, blocknum, buffer, false); +pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "smgrwrite called for %u/%u/%u.%u blk %u, page LSN: %X/%08X", +pgxn/neon/pagestore_smgr.c:neon_writev(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno, +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: neon_wallog_pagev(reln, forknum, blkno, nblocks, (const char **) buffers, false); +pgxn/neon/pagestore_smgr.c: * neon_nblocks() -- Get the number of blocks stored in a relation. +pgxn/neon/pagestore_smgr.c:neon_nblocks(SMgrRelation reln, ForkNumber forknum) +pgxn/neon/pagestore_smgr.c: NeonResponse *resp; +pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns; +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrnblocks() on rel with unknown persistence"); +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "cached nblocks for %u/%u/%u.%u: %u blocks", +pgxn/neon/pagestore_smgr.c: neon_get_request_lsns(InfoFromSMgrRel(reln), forknum, +pgxn/neon/pagestore_smgr.c: NeonNblocksRequest request = { +pgxn/neon/pagestore_smgr.c: .hdr.tag = T_NeonNblocksRequest, +pgxn/neon/pagestore_smgr.c: case T_NeonNblocksResponse: +pgxn/neon/pagestore_smgr.c: NeonNblocksResponse * relsize_resp = (NeonNblocksResponse *) resp; +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, +pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match get relsize request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", +pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "[reqid %lx] could not read relation size of rel %u/%u/%u.%u from page server at lsn %X/%08X", +pgxn/neon/pagestore_smgr.c: ((NeonErrorResponse *) resp)->message))); +pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, +pgxn/neon/pagestore_smgr.c: T_NeonNblocksResponse, T_NeonErrorResponse, resp->tag); +pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "databricks_nblocks: rel %u/%u/%u fork %u (request LSN %X/%08X): %u blocks", +pgxn/neon/pagestore_smgr.c: * neon_db_size() -- Get the size of the database in bytes. +pgxn/neon/pagestore_smgr.c:neon_dbsize(Oid dbNode) +pgxn/neon/pagestore_smgr.c: NeonResponse *resp; +pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns; +pgxn/neon/pagestore_smgr.c: neon_get_request_lsns(dummy_node, MAIN_FORKNUM, +pgxn/neon/pagestore_smgr.c: NeonDbSizeRequest request = { +pgxn/neon/pagestore_smgr.c: .hdr.tag = T_NeonDbSizeRequest, +pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeResponse: +pgxn/neon/pagestore_smgr.c: NeonDbSizeResponse* dbsize_resp = (NeonDbSizeResponse *) resp; +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, +pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match get DB size request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", +pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "[reqid %lx] could not read db size of db %u from page server at lsn %X/%08X", +pgxn/neon/pagestore_smgr.c: ((NeonErrorResponse *) resp)->message))); +pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, +pgxn/neon/pagestore_smgr.c: T_NeonDbSizeResponse, T_NeonErrorResponse, resp->tag); +pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "databricks_dbsize: db %u (request LSN %X/%08X): %ld bytes", +pgxn/neon/pagestore_smgr.c: * neon_truncate() -- Truncate relation to specified number of blocks. +pgxn/neon/pagestore_smgr.c:neon_truncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrtruncate() on rel with unknown persistence"); +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: * neon_immedsync() -- Immediately sync a relation to stable storage. +pgxn/neon/pagestore_smgr.c:neon_immedsync(SMgrRelation reln, ForkNumber forknum) +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrimmedsync() on rel with unknown persistence"); +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "[DATABRICKS_SMGR] immedsync noop"); +pgxn/neon/pagestore_smgr.c:neon_registersync(SMgrRelation reln, ForkNumber forknum) +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrregistersync() on rel with unknown persistence"); +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, NEON_TAG "registersync noop"); +pgxn/neon/pagestore_smgr.c: * neon_start_unlogged_build() -- Starting build operation on a rel. +pgxn/neon/pagestore_smgr.c: * and WAL-logging the whole relation after it's done. Neon relies on the +pgxn/neon/pagestore_smgr.c:neon_start_unlogged_build(SMgrRelation reln) +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unlogged relation build is already in progress"); +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "starting unlogged build of relation %u/%u/%u", +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgr_start_unlogged_build() on rel with unknown persistence"); +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); +pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot perform unlogged index build, index is not empty "); +pgxn/neon/pagestore_smgr.c: * neon_finish_unlogged_build_phase_1() +pgxn/neon/pagestore_smgr.c:neon_finish_unlogged_build_phase_1(SMgrRelation reln) +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "finishing phase 1 of unlogged build of relation %u/%u/%u", +pgxn/neon/pagestore_smgr.c: * neon_end_unlogged_build() -- Finish an unlogged rel build. +pgxn/neon/pagestore_smgr.c:neon_end_unlogged_build(SMgrRelation reln) +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "ending unlogged build of relation %u/%u/%u", +pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "forgetting cached relsize for %u/%u/%u.%u", +pgxn/neon/pagestore_smgr.c:neon_read_slru_segment(SMgrRelation reln, const char* path, int segno, void* buffer) +pgxn/neon/pagestore_smgr.c: NeonResponse *resp; +pgxn/neon/pagestore_smgr.c: NeonGetSlruSegmentRequest request; +pgxn/neon/pagestore_smgr.c: * Compute a request LSN to use, similar to neon_get_request_lsns() but the +pgxn/neon/pagestore_smgr.c: * This happens in neon startup, we start up without replaying any +pgxn/neon/pagestore_smgr.c: request = (NeonGetSlruSegmentRequest) { +pgxn/neon/pagestore_smgr.c: .hdr.tag = T_NeonGetSlruSegmentRequest, +pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentResponse: +pgxn/neon/pagestore_smgr.c: NeonGetSlruSegmentResponse* slru_resp = (NeonGetSlruSegmentResponse *) resp; +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, +pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: +pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) +pgxn/neon/pagestore_smgr.c: elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match get SLRU segment request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", +pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "[reqid %lx] could not read SLRU %d segment %d at lsn %X/%08X", +pgxn/neon/pagestore_smgr.c: ((NeonErrorResponse *) resp)->message))); +pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, +pgxn/neon/pagestore_smgr.c: T_NeonGetSlruSegmentResponse, T_NeonErrorResponse, resp->tag); +pgxn/neon/pagestore_smgr.c:AtEOXact_neon(XactEvent event, void *arg) +pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "unlogged index build was not properly finished")))); +pgxn/neon/pagestore_smgr.c:static const struct f_smgr neon_smgr = +pgxn/neon/pagestore_smgr.c: .smgr_init = neon_init, +pgxn/neon/pagestore_smgr.c: .smgr_open = neon_open, +pgxn/neon/pagestore_smgr.c: .smgr_close = neon_close, +pgxn/neon/pagestore_smgr.c: .smgr_create = neon_create, +pgxn/neon/pagestore_smgr.c: .smgr_exists = neon_exists, +pgxn/neon/pagestore_smgr.c: .smgr_unlink = neon_unlink, +pgxn/neon/pagestore_smgr.c: .smgr_extend = neon_extend, +pgxn/neon/pagestore_smgr.c: .smgr_zeroextend = neon_zeroextend, +pgxn/neon/pagestore_smgr.c: .smgr_prefetch = neon_prefetch, +pgxn/neon/pagestore_smgr.c: .smgr_readv = neon_readv, +pgxn/neon/pagestore_smgr.c: .smgr_writev = neon_writev, +pgxn/neon/pagestore_smgr.c: .smgr_prefetch = neon_prefetch, +pgxn/neon/pagestore_smgr.c: .smgr_read = neon_read, +pgxn/neon/pagestore_smgr.c: .smgr_write = neon_write, +pgxn/neon/pagestore_smgr.c: .smgr_writeback = neon_writeback, +pgxn/neon/pagestore_smgr.c: .smgr_nblocks = neon_nblocks, +pgxn/neon/pagestore_smgr.c: .smgr_truncate = neon_truncate, +pgxn/neon/pagestore_smgr.c: .smgr_immedsync = neon_immedsync, +pgxn/neon/pagestore_smgr.c: .smgr_registersync = neon_registersync, +pgxn/neon/pagestore_smgr.c: .smgr_start_unlogged_build = neon_start_unlogged_build, +pgxn/neon/pagestore_smgr.c: .smgr_finish_unlogged_build_phase_1 = neon_finish_unlogged_build_phase_1, +pgxn/neon/pagestore_smgr.c: .smgr_end_unlogged_build = neon_end_unlogged_build, +pgxn/neon/pagestore_smgr.c: .smgr_read_slru_segment = neon_read_slru_segment, +pgxn/neon/pagestore_smgr.c:smgr_neon(ProcNumber backend, NRelFileInfo rinfo) +pgxn/neon/pagestore_smgr.c: return &neon_smgr; +pgxn/neon/pagestore_smgr.c:smgr_init_neon(void) +pgxn/neon/pagestore_smgr.c: RegisterXactCallback(AtEOXact_neon, NULL); +pgxn/neon/pagestore_smgr.c: neon_init(); +pgxn/neon/pagestore_smgr.c:neon_extend_rel_size(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno, XLogRecPtr end_recptr) +pgxn/neon/pagestore_smgr.c: NeonResponse *response; +pgxn/neon/pagestore_smgr.c: NeonNblocksResponse *nbresponse; +pgxn/neon/pagestore_smgr.c: NeonNblocksRequest request = { +pgxn/neon/pagestore_smgr.c: .hdr = (NeonRequest) { +pgxn/neon/pagestore_smgr.c: .tag = T_NeonNblocksRequest, +pgxn/neon/pagestore_smgr.c: Assert(response->tag == T_NeonNblocksResponse); +pgxn/neon/pagestore_smgr.c: nbresponse = (NeonNblocksResponse *) response; +pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "Set length to %d", relsize); +pgxn/neon/pagestore_smgr.c:neon_redo_read_buffer_filter(XLogReaderState *record, uint8 block_id) +pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "failed to locate backup block with ID %d", block_id); +pgxn/neon/pagestore_smgr.c: neon_extend_rel_size(rinfo, forknum, blkno, end_recptr); +pgxn/neon/pagestore_smgr.c: neon_extend_rel_size(rinfo, FSM_FORKNUM, get_fsm_physical_block(blkno), end_recptr); +pgxn/neon/relsize_cache.c: * contrib/neon/relsize_cache.c +pgxn/neon/relsize_cache.c:#include "neon_pgversioncompat.h" +pgxn/neon/relsize_cache.c:neon_smgr_shmem_startup(void) +pgxn/neon/relsize_cache.c: DefineCustomIntVariable("neon.relsize_hash_size", +pgxn/neon/relsize_cache.c: "Sets the maximum number of cached relation sizes for neon", +pgxn/neon/relsize_cache.c: shmem_startup_hook = neon_smgr_shmem_startup; +pgxn/neon/relsize_cache.c: * attach to the shared resources in neon_smgr_shmem_startup(). +pgxn/neon/unstable_extensions.c:#include "neon_pgversioncompat.h" +pgxn/neon/unstable_extensions.c: "neon.allow_unstable_extensions", +pgxn/neon/unstable_extensions.c: "neon.unstable_extensions", +pgxn/neon/unstable_extensions.h:#ifndef __NEON_UNSTABLE_EXTENSIONS_H__ +pgxn/neon/unstable_extensions.h:#define __NEON_UNSTABLE_EXTENSIONS_H__ +pgxn/neon/walproposer.c: * and WAL safekeepers. +pgxn/neon/walproposer.c: * will immediately broadcast it to alive safekeepers. +pgxn/neon/walproposer.c: * 2. As a standalone utility by running `postgres --sync-safekeepers`. That +pgxn/neon/walproposer.c: * it and make sure there is a safekeeper who knows this LSN is +pgxn/neon/walproposer.c: * safekeepers, learn start LSN of future epoch and run basebackup' +pgxn/neon/walproposer.c:#include "neon.h" +pgxn/neon/walproposer.c:#include "neon_utils.h" +pgxn/neon/walproposer.c:static void ShutdownConnection(Safekeeper *sk); +pgxn/neon/walproposer.c:static void ResetConnection(Safekeeper *sk); +pgxn/neon/walproposer.c:static void ReconnectSafekeepers(WalProposer *wp); +pgxn/neon/walproposer.c:static void AdvancePollState(Safekeeper *sk, uint32 events); +pgxn/neon/walproposer.c:static void HandleConnectionEvent(Safekeeper *sk); +pgxn/neon/walproposer.c:static void SendStartWALPush(Safekeeper *sk); +pgxn/neon/walproposer.c:static void RecvStartWALPushResult(Safekeeper *sk); +pgxn/neon/walproposer.c:static void SendProposerGreeting(Safekeeper *sk); +pgxn/neon/walproposer.c:static void RecvAcceptorGreeting(Safekeeper *sk); +pgxn/neon/walproposer.c:static void SendVoteRequest(Safekeeper *sk); +pgxn/neon/walproposer.c:static void RecvVoteResponse(Safekeeper *sk); +pgxn/neon/walproposer.c:static term_t GetEpoch(Safekeeper *sk); +pgxn/neon/walproposer.c:static void SendProposerElected(Safekeeper *sk); +pgxn/neon/walproposer.c:static void StartStreaming(Safekeeper *sk); +pgxn/neon/walproposer.c:static void SendMessageToNode(Safekeeper *sk); +pgxn/neon/walproposer.c:static void HandleActiveState(Safekeeper *sk, uint32 events); +pgxn/neon/walproposer.c:static bool SendAppendRequests(Safekeeper *sk); +pgxn/neon/walproposer.c:static bool RecvAppendResponses(Safekeeper *sk); +pgxn/neon/walproposer.c:static void HandleSafekeeperResponse(WalProposer *wp, Safekeeper *sk); +pgxn/neon/walproposer.c:static bool AsyncRead(Safekeeper *sk, char **buf, int *buf_size); +pgxn/neon/walproposer.c:static bool AsyncReadMessage(Safekeeper *sk, AcceptorProposerMessage *anymsg); +pgxn/neon/walproposer.c:static bool BlockingWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState success_state); +pgxn/neon/walproposer.c:static bool AsyncWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState flush_state); +pgxn/neon/walproposer.c:static bool AsyncFlush(Safekeeper *sk); +pgxn/neon/walproposer.c:static char *FormatSafekeeperState(Safekeeper *sk); +pgxn/neon/walproposer.c:static void AssertEventsOkForState(uint32 events, Safekeeper *sk); +pgxn/neon/walproposer.c: for (host = wp->config->safekeepers_list; host != NULL && *host != '\0'; host = sep) +pgxn/neon/walproposer.c: if (wp->n_safekeepers + 1 >= MAX_SAFEKEEPERS) +pgxn/neon/walproposer.c: wp_log(FATAL, "too many safekeepers"); +pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].host = host; +pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].port = port; +pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].state = SS_OFFLINE; +pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].active_state = SS_ACTIVE_SEND; +pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].wp = wp; +pgxn/neon/walproposer.c: Safekeeper *sk = &wp->safekeeper[wp->n_safekeepers]; +pgxn/neon/walproposer.c: sk->host, sk->port, wp->config->neon_timeline, wp->config->neon_tenant); +pgxn/neon/walproposer.c: wp_log(FATAL, "could not create connection string for safekeeper %s:%s", sk->host, sk->port); +pgxn/neon/walproposer.c: initStringInfo(&wp->safekeeper[wp->n_safekeepers].outbuf); +pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].startStreamingAt = InvalidXLogRecPtr; +pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].streamingAt = InvalidXLogRecPtr; +pgxn/neon/walproposer.c: wp->n_safekeepers += 1; +pgxn/neon/walproposer.c: if (wp->n_safekeepers < 1) +pgxn/neon/walproposer.c: wp_log(FATAL, "safekeepers addresses are not specified"); +pgxn/neon/walproposer.c: wp->quorum = wp->n_safekeepers / 2 + 1; +pgxn/neon/walproposer.c: if (!wp->config->neon_timeline) +pgxn/neon/walproposer.c: if (*wp->config->neon_timeline != '\0' && +pgxn/neon/walproposer.c: !HexDecodeString(wp->greetRequest.timeline_id, wp->config->neon_timeline, 16)) +pgxn/neon/walproposer.c: wp_log(FATAL, "could not parse timeline_id, %s", wp->config->neon_timeline); +pgxn/neon/walproposer.c: if (!wp->config->neon_tenant) +pgxn/neon/walproposer.c: if (*wp->config->neon_tenant != '\0' && +pgxn/neon/walproposer.c: !HexDecodeString(wp->greetRequest.tenant_id, wp->config->neon_tenant, 16)) +pgxn/neon/walproposer.c: wp_log(FATAL, "could not parse tenant_id, %s", wp->config->neon_tenant); +pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer.c: Safekeeper *sk = &wp->safekeeper[i]; +pgxn/neon/walproposer.c: Safekeeper *sk = NULL; +pgxn/neon/walproposer.c: * If the event contains something that one of our safekeeper states +pgxn/neon/walproposer.c: * If the timeout expired, attempt to reconnect to any safekeepers +pgxn/neon/walproposer.c: ReconnectSafekeepers(wp); +pgxn/neon/walproposer.c: if (!wp->config->syncSafekeepers) +pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer.c: sk = &wp->safekeeper[i]; +pgxn/neon/walproposer.c: wp->config->safekeeper_connection_timeout)) +pgxn/neon/walproposer.c: sk->host, sk->port, FormatSafekeeperState(sk), wp->config->safekeeper_connection_timeout); +pgxn/neon/walproposer.c: /* Initiate connections to all safekeeper nodes */ +pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer.c: ResetConnection(&wp->safekeeper[i]); +pgxn/neon/walproposer.c:/* Shuts down and cleans up the connection for a safekeeper. Sets its state to SS_OFFLINE */ +pgxn/neon/walproposer.c:ShutdownConnection(Safekeeper *sk) +pgxn/neon/walproposer.c: sk->wp->api.rm_safekeeper_event_set(sk); +pgxn/neon/walproposer.c:ResetConnection(Safekeeper *sk) +pgxn/neon/walproposer.c: wp->api.add_safekeeper_event_set(sk, WL_SOCKET_WRITEABLE); +pgxn/neon/walproposer.c: * safekeepers? Returns 0 if it is already high time, -1 if we never reconnect +pgxn/neon/walproposer.c: if (wp->config->safekeeper_reconnect_timeout <= 0) +pgxn/neon/walproposer.c: till_reconnect = wp->config->safekeeper_reconnect_timeout * 1000 - passed; +pgxn/neon/walproposer.c:/* If the timeout has expired, attempt to reconnect to all offline safekeepers */ +pgxn/neon/walproposer.c:ReconnectSafekeepers(WalProposer *wp) +pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer.c: if (wp->safekeeper[i].state == SS_OFFLINE) +pgxn/neon/walproposer.c: ResetConnection(&wp->safekeeper[i]); +pgxn/neon/walproposer.c: * Performs the logic for advancing the state machine of the specified safekeeper, +pgxn/neon/walproposer.c:AdvancePollState(Safekeeper *sk, uint32 events) +pgxn/neon/walproposer.c: * safekeepers are only taken out of SS_OFFLINE by calls to +pgxn/neon/walproposer.c: * safekeeper. +pgxn/neon/walproposer.c: sk->port, FormatSafekeeperState(sk)); +pgxn/neon/walproposer.c: /* Read the safekeeper response for our candidate */ +pgxn/neon/walproposer.c: sk->port, FormatSafekeeperState(sk)); +pgxn/neon/walproposer.c:HandleConnectionEvent(Safekeeper *sk) +pgxn/neon/walproposer.c: * ReconnectSafekeepers. +pgxn/neon/walproposer.c: wp->api.rm_safekeeper_event_set(sk); +pgxn/neon/walproposer.c: wp->api.add_safekeeper_event_set(sk, new_events); +pgxn/neon/walproposer.c: * Send "START_WAL_PUSH" message as an empty query to the safekeeper. Performs +pgxn/neon/walproposer.c:SendStartWALPush(Safekeeper *sk) +pgxn/neon/walproposer.c:RecvStartWALPushResult(Safekeeper *sk) +pgxn/neon/walproposer.c: * safekeeper. After sending, we wait on SS_HANDSHAKE_RECV for +pgxn/neon/walproposer.c:SendProposerGreeting(Safekeeper *sk) +pgxn/neon/walproposer.c:RecvAcceptorGreeting(Safekeeper *sk) +pgxn/neon/walproposer.c: * Note: it would be better to track the counter on per safekeeper basis, +pgxn/neon/walproposer.c: * Check if we have quorum. If there aren't enough safekeepers, wait and +pgxn/neon/walproposer.c: for (int j = 0; j < wp->n_safekeepers; j++) +pgxn/neon/walproposer.c: * Remember: SS_VOTING indicates that the safekeeper is +pgxn/neon/walproposer.c: if (wp->safekeeper[j].state == SS_VOTING) +pgxn/neon/walproposer.c: SendVoteRequest(&wp->safekeeper[j]); +pgxn/neon/walproposer.c:SendVoteRequest(Safekeeper *sk) +pgxn/neon/walproposer.c:RecvVoteResponse(Safekeeper *sk) +pgxn/neon/walproposer.c: * Synchronously download WAL from the most advanced safekeeper. We do +pgxn/neon/walproposer.c: * neon_walreader is a todo.) +pgxn/neon/walproposer.c: if (!wp->api.recovery_download(wp, &wp->safekeeper[wp->donor])) +pgxn/neon/walproposer.c: * Zero propEpochStartLsn means majority of safekeepers doesn't have any +pgxn/neon/walproposer.c: * otherwise we must be sync-safekeepers and we have nothing to do then. +pgxn/neon/walproposer.c: * safekeepers term history starting with 0/0. These hacks will go away once +pgxn/neon/walproposer.c: * we disable implicit timeline creation on safekeepers and create it with +pgxn/neon/walproposer.c: Assert(wp->config->syncSafekeepers); +pgxn/neon/walproposer.c: wp->api.finish_sync_safekeepers(wp, wp->propEpochStartLsn); +pgxn/neon/walproposer.c: if (wp->truncateLsn == wp->propEpochStartLsn && wp->config->syncSafekeepers) +pgxn/neon/walproposer.c: wp->api.finish_sync_safekeepers(wp, wp->propEpochStartLsn); +pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer.c: if (wp->safekeeper[i].state == SS_IDLE) +pgxn/neon/walproposer.c: SendProposerElected(&wp->safekeeper[i]); +pgxn/neon/walproposer.c: * after this point. There will be no safekeeper with state SS_IDLE also, +pgxn/neon/walproposer.c: if (wp->config->syncSafekeepers) +pgxn/neon/walproposer.c: /* keep polling until all safekeepers are synced */ +pgxn/neon/walproposer.c:/* safekeeper's epoch is the term of the highest entry in the log */ +pgxn/neon/walproposer.c:GetEpoch(Safekeeper *sk) +pgxn/neon/walproposer.c: * advanced safekeeper (who will be the donor) and epochStartLsn -- LSN since +pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer.c: if (wp->safekeeper[i].state == SS_IDLE) +pgxn/neon/walproposer.c: if (GetEpoch(&wp->safekeeper[i]) > wp->donorEpoch || +pgxn/neon/walproposer.c: (GetEpoch(&wp->safekeeper[i]) == wp->donorEpoch && +pgxn/neon/walproposer.c: wp->safekeeper[i].voteResponse.flushLsn > wp->propEpochStartLsn)) +pgxn/neon/walproposer.c: wp->donorEpoch = GetEpoch(&wp->safekeeper[i]); +pgxn/neon/walproposer.c: wp->propEpochStartLsn = wp->safekeeper[i].voteResponse.flushLsn; +pgxn/neon/walproposer.c: wp->truncateLsn = Max(wp->safekeeper[i].voteResponse.truncateLsn, wp->truncateLsn); +pgxn/neon/walproposer.c: if (wp->safekeeper[i].voteResponse.timelineStartLsn != InvalidXLogRecPtr) +pgxn/neon/walproposer.c: wp->timelineStartLsn != wp->safekeeper[i].voteResponse.timelineStartLsn) +pgxn/neon/walproposer.c: LSN_FORMAT_ARGS(wp->safekeeper[i].voteResponse.timelineStartLsn)); +pgxn/neon/walproposer.c: wp->timelineStartLsn = wp->safekeeper[i].voteResponse.timelineStartLsn; +pgxn/neon/walproposer.c: * This is a rare case that can be triggered if safekeeper has voted +pgxn/neon/walproposer.c: if (wp->propEpochStartLsn == InvalidXLogRecPtr && !wp->config->syncSafekeepers) +pgxn/neon/walproposer.c: * Safekeepers are setting truncateLsn after timelineStartLsn is known, so +pgxn/neon/walproposer.c: * timelineStartLsn can be zero only on the first syncSafekeepers run. +pgxn/neon/walproposer.c: (wp->config->syncSafekeepers && wp->truncateLsn == wp->timelineStartLsn)); +pgxn/neon/walproposer.c: dth = &wp->safekeeper[wp->donor].voteResponse.termHistory; +pgxn/neon/walproposer.c: wp->safekeeper[wp->donor].host, wp->safekeeper[wp->donor].port, +pgxn/neon/walproposer.c: if (!wp->config->syncSafekeepers) +pgxn/neon/walproposer.c: * Safekeepers don't skip header as they need continious stream of +pgxn/neon/walproposer.c: * safekeeper is synced, being important for sync-safekeepers) +pgxn/neon/walproposer.c: * 2) Communicating starting streaming point -- safekeeper must truncate its WAL +pgxn/neon/walproposer.c:SendProposerElected(Safekeeper *sk) +pgxn/neon/walproposer.c: * Determine start LSN by comparing safekeeper's log term switch history +pgxn/neon/walproposer.c: * there is some WAL on safekeeper, if immediately after bootstrap compute +pgxn/neon/walproposer.c: /* safekeeper is empty or no common point, start from the beginning */ +pgxn/neon/walproposer.c: * one; there it is flush_lsn in case of safekeeper or, in case of +pgxn/neon/walproposer.c: * safekeeper pos as it obviously can't be higher. +pgxn/neon/walproposer.c: * Start streaming to safekeeper sk, always updates state to SS_ACTIVE and sets +pgxn/neon/walproposer.c:StartStreaming(Safekeeper *sk) +pgxn/neon/walproposer.c: * Can be used only for safekeepers in SS_ACTIVE state. State can be changed +pgxn/neon/walproposer.c:SendMessageToNode(Safekeeper *sk) +pgxn/neon/walproposer.c: * Note: we always send everything to the safekeeper until WOULDBLOCK or +pgxn/neon/walproposer.c: * Broadcast new message to all caught-up safekeepers +pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer.c: if (wp->safekeeper[i].state == SS_ACTIVE) +pgxn/neon/walproposer.c: SendMessageToNode(&wp->safekeeper[i]); +pgxn/neon/walproposer.c:HandleActiveState(Safekeeper *sk, uint32 events) +pgxn/neon/walproposer.c: * socket or neon_walreader blocks, whichever comes first; active_state is +pgxn/neon/walproposer.c:SendAppendRequests(Safekeeper *sk) +pgxn/neon/walproposer.c: case NEON_WALREAD_SUCCESS: +pgxn/neon/walproposer.c: case NEON_WALREAD_WOULDBLOCK: +pgxn/neon/walproposer.c: case NEON_WALREAD_ERROR: +pgxn/neon/walproposer.c: sk->host, sk->port, FormatSafekeeperState(sk), +pgxn/neon/walproposer.c:RecvAppendResponses(Safekeeper *sk) +pgxn/neon/walproposer.c: * on safekeepers for migration purposes, in this case we do want +pgxn/neon/walproposer.c: HandleSafekeeperResponse(wp, sk); +pgxn/neon/walproposer.c: * Get minimum of flushed LSNs of all safekeepers, which is the LSN of the +pgxn/neon/walproposer.c: XLogRecPtr lsn = wp->n_safekeepers > 0 +pgxn/neon/walproposer.c: ? wp->safekeeper[0].appendResponse.flushLsn +pgxn/neon/walproposer.c: for (int i = 1; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer.c: lsn = Min(lsn, wp->safekeeper[i].appendResponse.flushLsn); +pgxn/neon/walproposer.c: XLogRecPtr responses[MAX_SAFEKEEPERS]; +pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer.c: responses[i] = wp->safekeeper[i].appendResponse.flushLsn >= wp->propEpochStartLsn ? wp->safekeeper[i].appendResponse.flushLsn : 0; +pgxn/neon/walproposer.c: qsort(responses, wp->n_safekeepers, sizeof(XLogRecPtr), CompareLsn); +pgxn/neon/walproposer.c: return responses[wp->n_safekeepers - wp->quorum]; +pgxn/neon/walproposer.c: * Return safekeeper with active connection from which WAL can be downloaded, or +pgxn/neon/walproposer.c: Safekeeper *donor = NULL; +pgxn/neon/walproposer.c: if (wp->safekeeper[wp->donor].state >= SS_IDLE) +pgxn/neon/walproposer.c: donor = &wp->safekeeper[wp->donor]; +pgxn/neon/walproposer.c: for (i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer.c: Safekeeper *sk = &wp->safekeeper[i]; +pgxn/neon/walproposer.c: * Process AppendResponse message from safekeeper. +pgxn/neon/walproposer.c:HandleSafekeeperResponse(WalProposer *wp, Safekeeper *fromsk) +pgxn/neon/walproposer.c: /* Send new value to all safekeepers. */ +pgxn/neon/walproposer.c: * called to notify safekeepers about the new commitLsn. +pgxn/neon/walproposer.c: wp->api.process_safekeeper_feedback(wp, fromsk); +pgxn/neon/walproposer.c: * safekeepers. +pgxn/neon/walproposer.c: * from safekeeper who is still in the previous epoch (similar to 'leader +pgxn/neon/walproposer.c: * is alive, there will be at least one safekeeper who is able to stream +pgxn/neon/walproposer.c: * most advanced safekeeper who should push the wal into pageserver and +pgxn/neon/walproposer.c: * (due to pageserver connecting to not-synced-safekeeper) we currently +pgxn/neon/walproposer.c: * wait for all seemingly alive safekeepers to get synced. +pgxn/neon/walproposer.c: if (wp->config->syncSafekeepers) +pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer.c: Safekeeper *sk = &wp->safekeeper[i]; +pgxn/neon/walproposer.c: /* alive safekeeper which is not synced yet; wait for it */ +pgxn/neon/walproposer.c: /* A quorum of safekeepers has been synced! */ +pgxn/neon/walproposer.c: * safekeepers. This helps to finish next sync-safekeepers +pgxn/neon/walproposer.c: * safekeepers in case of network working properly. +pgxn/neon/walproposer.c: wp->api.finish_sync_safekeepers(wp, wp->propEpochStartLsn); +pgxn/neon/walproposer.c: * Try to read CopyData message from i'th safekeeper, resetting connection on +pgxn/neon/walproposer.c:AsyncRead(Safekeeper *sk, char **buf, int *buf_size) +pgxn/neon/walproposer.c: sk->port, FormatSafekeeperState(sk), +pgxn/neon/walproposer.c: * block from the safekeeper's postgres connection, returning whether the read +pgxn/neon/walproposer.c:AsyncReadMessage(Safekeeper *sk, AcceptorProposerMessage *anymsg) +pgxn/neon/walproposer.c: sk->port, FormatSafekeeperState(sk)); +pgxn/neon/walproposer.c:BlockingWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState success_state) +pgxn/neon/walproposer.c: sk->host, sk->port, FormatSafekeeperState(sk), +pgxn/neon/walproposer.c: SafekeeperStateDesiredEvents(sk, &sk_events, &nwr_events); +pgxn/neon/walproposer.c: * Starts a write into the 'i'th safekeeper's postgres connection, moving to +pgxn/neon/walproposer.c:AsyncWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState flush_state) +pgxn/neon/walproposer.c: sk->host, sk->port, FormatSafekeeperState(sk), +pgxn/neon/walproposer.c:AsyncFlush(Safekeeper *sk) +pgxn/neon/walproposer.c: sk->host, sk->port, FormatSafekeeperState(sk), +pgxn/neon/walproposer.c:/* Returns a human-readable string corresonding to the SafekeeperState +pgxn/neon/walproposer.c: * wp_log(LOG, "currently in %s state", FormatSafekeeperState(sk)); +pgxn/neon/walproposer.c: * wp_log(LOG, "currently in state [%s]", FormatSafekeeperState(sk)); +pgxn/neon/walproposer.c:FormatSafekeeperState(Safekeeper *sk) +pgxn/neon/walproposer.c:/* Asserts that the provided events are expected for given safekeeper's state */ +pgxn/neon/walproposer.c:AssertEventsOkForState(uint32 events, Safekeeper *sk) +pgxn/neon/walproposer.c: SafekeeperStateDesiredEvents(sk, &sk_events, &nwr_events); +pgxn/neon/walproposer.c: FormatEvents(wp, events), sk->host, sk->port, FormatSafekeeperState(sk)); +pgxn/neon/walproposer.c:/* Returns the set of events for both safekeeper (sk_events) and neon_walreader +pgxn/neon/walproposer.c: * (nwr_events) sockets a safekeeper in this state should be waiting on. +pgxn/neon/walproposer.c:SafekeeperStateDesiredEvents(Safekeeper *sk, uint32 *sk_events, uint32 *nwr_events) +pgxn/neon/walproposer.c: * safekeeper in one wakeup (unless it blocks). Otherwise +pgxn/neon/walproposer.c: * Waiting for neon_walreader socket, but we still read +pgxn/neon/walproposer.c: * Need to flush the sk socket, so ignore neon_walreader +pgxn/neon/walproposer.h:#ifndef __NEON_WALPROPOSER_H__ +pgxn/neon/walproposer.h:#define __NEON_WALPROPOSER_H__ +pgxn/neon/walproposer.h:#include "neon_walreader.h" +pgxn/neon/walproposer.h:#define MAX_SAFEKEEPERS 32 +pgxn/neon/walproposer.h: * WAL safekeeper state, which is used to wait for some event. +pgxn/neon/walproposer.h: * After we get a successful result, sends handshake to safekeeper. +pgxn/neon/walproposer.h:} SafekeeperState; +pgxn/neon/walproposer.h: * Polling neon_walreader to receive chunk of WAL (probably remotely) to +pgxn/neon/walproposer.h: * send to this safekeeper. +pgxn/neon/walproposer.h: * Also, while in this state we don't touch safekeeper socket, so in +pgxn/neon/walproposer.h:} SafekeeperActiveState; +pgxn/neon/walproposer.h:/* neon storage node id */ +pgxn/neon/walproposer.h: uint32 protocolVersion; /* proposer-safekeeper protocol version */ +pgxn/neon/walproposer.h: uint8 timeline_id[16]; /* Neon timeline id */ +pgxn/neon/walproposer.h:/* Vote itself, sent from safekeeper to proposer */ +pgxn/neon/walproposer.h: * Safekeeper flush_lsn (end of WAL) + history of term switches allow +pgxn/neon/walproposer.h: * recovery of some safekeeper */ +pgxn/neon/walproposer.h: * Header of request with WAL message sent from proposer to safekeeper. +pgxn/neon/walproposer.h: XLogRecPtr commitLsn; /* LSN committed by quorum of safekeepers */ +pgxn/neon/walproposer.h: * minimal LSN which may be needed for recovery of some safekeeper (end +pgxn/neon/walproposer.h: /* standby_status_update fields that safekeeper received from pageserver */ +pgxn/neon/walproposer.h: * Report safekeeper state to proposer +pgxn/neon/walproposer.h: * Current term of the safekeeper; if it is higher than proposer's, the +pgxn/neon/walproposer.h: /* Safekeeper reports back his awareness about which WAL is committed, as */ +pgxn/neon/walproposer.h: /* and custom neon feedback. */ +pgxn/neon/walproposer.h: * Descriptor of safekeeper +pgxn/neon/walproposer.h:typedef struct Safekeeper +pgxn/neon/walproposer.h: * Temporary buffer for the message being sent to the safekeeper. +pgxn/neon/walproposer.h: AppendRequestHeader appendRequest; /* request for sending to safekeeper */ +pgxn/neon/walproposer.h: SafekeeperState state; /* safekeeper state machine state */ +pgxn/neon/walproposer.h: SafekeeperActiveState active_state; +pgxn/neon/walproposer.h: * WAL reader, allocated for each safekeeper. +pgxn/neon/walproposer.h: NeonWALReader *xlogreader; +pgxn/neon/walproposer.h: * Neon WAL reader position in wait event set, or -1 if no socket. Note +pgxn/neon/walproposer.h:} Safekeeper; +pgxn/neon/walproposer.h: void (*update_donor) (WalProposer *wp, Safekeeper *donor, XLogRecPtr donor_lsn); +pgxn/neon/walproposer.h: char *(*conn_error_message) (Safekeeper *sk); +pgxn/neon/walproposer.h: WalProposerConnStatusType (*conn_status) (Safekeeper *sk); +pgxn/neon/walproposer.h: void (*conn_connect_start) (Safekeeper *sk); +pgxn/neon/walproposer.h: WalProposerConnectPollStatusType (*conn_connect_poll) (Safekeeper *sk); +pgxn/neon/walproposer.h: bool (*conn_send_query) (Safekeeper *sk, char *query); +pgxn/neon/walproposer.h: WalProposerExecStatusType (*conn_get_query_result) (Safekeeper *sk); +pgxn/neon/walproposer.h: int (*conn_flush) (Safekeeper *sk); +pgxn/neon/walproposer.h: void (*conn_finish) (Safekeeper *sk); +pgxn/neon/walproposer.h: * Try to read CopyData message from the safekeeper, aka PQgetCopyData. +pgxn/neon/walproposer.h: PGAsyncReadResult (*conn_async_read) (Safekeeper *sk, char **buf, int *amount); +pgxn/neon/walproposer.h: PGAsyncWriteResult (*conn_async_write) (Safekeeper *sk, void const *buf, size_t size); +pgxn/neon/walproposer.h: bool (*conn_blocking_write) (Safekeeper *sk, void const *buf, size_t size); +pgxn/neon/walproposer.h: bool (*recovery_download) (WalProposer *wp, Safekeeper *sk); +pgxn/neon/walproposer.h: void (*wal_reader_allocate) (Safekeeper *sk); +pgxn/neon/walproposer.h: NeonWALReadResult (*wal_read) (Safekeeper *sk, char *buf, XLogRecPtr startptr, Size count, char **errmsg); +pgxn/neon/walproposer.h: uint32 (*wal_reader_events) (Safekeeper *sk); +pgxn/neon/walproposer.h: /* Update events for an existing safekeeper connection. */ +pgxn/neon/walproposer.h: void (*update_event_set) (Safekeeper *sk, uint32 events); +pgxn/neon/walproposer.h: void (*active_state_update_event_set) (Safekeeper *sk); +pgxn/neon/walproposer.h: /* Add a new safekeeper connection to the event set. */ +pgxn/neon/walproposer.h: void (*add_safekeeper_event_set) (Safekeeper *sk, uint32 events); +pgxn/neon/walproposer.h: /* Remove safekeeper connection from event set */ +pgxn/neon/walproposer.h: void (*rm_safekeeper_event_set) (Safekeeper *sk); +pgxn/neon/walproposer.h: * safekeeper connection - new WAL is available +pgxn/neon/walproposer.h: * events mask to indicate events and sets sk to the safekeeper which has +pgxn/neon/walproposer.h: int (*wait_event_set) (WalProposer *wp, long timeout, Safekeeper **sk, uint32 *events); +pgxn/neon/walproposer.h: * LSN on the safekeepers. +pgxn/neon/walproposer.h: * Finish sync safekeepers with the given LSN. This function should not +pgxn/neon/walproposer.h: void (*finish_sync_safekeepers) (WalProposer *wp, XLogRecPtr lsn); +pgxn/neon/walproposer.h: * Called after every AppendResponse from the safekeeper. Used to +pgxn/neon/walproposer.h: * been commited on the quorum of safekeepers). +pgxn/neon/walproposer.h: void (*process_safekeeper_feedback) (WalProposer *wp, Safekeeper *sk); +pgxn/neon/walproposer.h: char *neon_tenant; +pgxn/neon/walproposer.h: char *neon_timeline; +pgxn/neon/walproposer.h: * Comma-separated list of safekeepers, in the following format: +pgxn/neon/walproposer.h: char *safekeepers_list; +pgxn/neon/walproposer.h: * WalProposer reconnects to offline safekeepers once in this interval. +pgxn/neon/walproposer.h: int safekeeper_reconnect_timeout; +pgxn/neon/walproposer.h: * from the safekeeper in this interval. Time is in milliseconds. +pgxn/neon/walproposer.h: int safekeeper_connection_timeout; +pgxn/neon/walproposer.h: * WAL segment size. Will be passed to safekeepers in greet request. Also +pgxn/neon/walproposer.h: * If safekeeper was started in sync mode, walproposer will not subscribe +pgxn/neon/walproposer.h: * for new WAL and will exit when quorum of safekeepers will be synced to +pgxn/neon/walproposer.h: bool syncSafekeepers; +pgxn/neon/walproposer.h: /* Will be passed to safekeepers in greet request. */ +pgxn/neon/walproposer.h: /* Will be passed to safekeepers in greet request. */ +pgxn/neon/walproposer.h: int n_safekeepers; +pgxn/neon/walproposer.h: /* (n_safekeepers / 2) + 1 */ +pgxn/neon/walproposer.h: Safekeeper safekeeper[MAX_SAFEKEEPERS]; +pgxn/neon/walproposer.h: /* Vote request for safekeeper */ +pgxn/neon/walproposer.h: * Minimal LSN which may be needed for recovery of some safekeeper, +pgxn/neon/walproposer.h: * collect terms from safekeepers quorum, choose max and +1. After that +pgxn/neon/walproposer.h: * safekeeper has higher term, it means that we have another running +pgxn/neon/walproposer.h: /* number of votes collected from safekeepers */ +pgxn/neon/walproposer.h: * config->safekeeper_reconnect_timeout +pgxn/neon/walproposer.h:extern void SafekeeperStateDesiredEvents(Safekeeper *sk, uint32 *sk_events, uint32 *nwr_events); +pgxn/neon/walproposer.h:#endif /* __NEON_WALPROPOSER_H__ */ +pgxn/neon/walproposer_pg.c:#include "neon.h" +pgxn/neon/walproposer_pg.c:#include "neon_perf_counters.h" +pgxn/neon/walproposer_pg.c:#include "neon_walreader.h" +pgxn/neon/walproposer_pg.c:static void assign_neon_safekeepers(const char *newval, void *extra); +pgxn/neon/walproposer_pg.c:static void walprop_pg_init_standalone_sync_safekeepers(void); +pgxn/neon/walproposer_pg.c:static void WalproposerShmemInit_SyncSafekeeper(void); +pgxn/neon/walproposer_pg.c:static void add_nwr_event_set(Safekeeper *sk, uint32 events); +pgxn/neon/walproposer_pg.c:static void update_nwr_event_set(Safekeeper *sk, uint32 events); +pgxn/neon/walproposer_pg.c:static void rm_safekeeper_event_set(Safekeeper *to_remove, bool is_sk); +pgxn/neon/walproposer_pg.c:init_walprop_config(bool syncSafekeepers) +pgxn/neon/walproposer_pg.c: walprop_config.neon_tenant = neon_tenant; +pgxn/neon/walproposer_pg.c: walprop_config.neon_timeline = neon_timeline; +pgxn/neon/walproposer_pg.c: walprop_config.safekeepers_list = pstrdup(wal_acceptors_list); +pgxn/neon/walproposer_pg.c: walprop_config.safekeeper_reconnect_timeout = wal_acceptor_reconnect_timeout; +pgxn/neon/walproposer_pg.c: walprop_config.safekeeper_connection_timeout = wal_acceptor_connection_timeout; +pgxn/neon/walproposer_pg.c: walprop_config.syncSafekeepers = syncSafekeepers; +pgxn/neon/walproposer_pg.c: if (!syncSafekeepers) +pgxn/neon/walproposer_pg.c: * Entry point for `postgres --sync-safekeepers`. +pgxn/neon/walproposer_pg.c: WalproposerShmemInit_SyncSafekeeper(); +pgxn/neon/walproposer_pg.c: walprop_pg_init_standalone_sync_safekeepers(); +pgxn/neon/walproposer_pg.c: "neon.safekeepers", +pgxn/neon/walproposer_pg.c: "List of Neon WAL acceptors (host:port)", +pgxn/neon/walproposer_pg.c: NULL, assign_neon_safekeepers, NULL); +pgxn/neon/walproposer_pg.c: "neon.safekeeper_reconnect_timeout", +pgxn/neon/walproposer_pg.c: "Walproposer reconnects to offline safekeepers once in this interval.", +pgxn/neon/walproposer_pg.c: "neon.safekeeper_connect_timeout", +pgxn/neon/walproposer_pg.c: "Connection or connection attempt to safekeeper is terminated if no message is received (or connection attempt doesn't finish) within this period.", +pgxn/neon/walproposer_pg.c:split_safekeepers_list(char *safekeepers_list, char *safekeepers[]) +pgxn/neon/walproposer_pg.c: int n_safekeepers = 0; +pgxn/neon/walproposer_pg.c: char *curr_sk = safekeepers_list; +pgxn/neon/walproposer_pg.c: for (char *coma = safekeepers_list; coma != NULL && *coma != '\0'; curr_sk = coma) +pgxn/neon/walproposer_pg.c: if (++n_safekeepers >= MAX_SAFEKEEPERS) { +pgxn/neon/walproposer_pg.c: safekeepers[n_safekeepers-1] = curr_sk; +pgxn/neon/walproposer_pg.c: return n_safekeepers; +pgxn/neon/walproposer_pg.c: * Accept two coma-separated strings with list of safekeeper host:port addresses. +pgxn/neon/walproposer_pg.c:safekeepers_cmp(char *old, char *new) +pgxn/neon/walproposer_pg.c: char *safekeepers_old[MAX_SAFEKEEPERS]; +pgxn/neon/walproposer_pg.c: char *safekeepers_new[MAX_SAFEKEEPERS]; +pgxn/neon/walproposer_pg.c: len_old = split_safekeepers_list(old, safekeepers_old); +pgxn/neon/walproposer_pg.c: len_new = split_safekeepers_list(new, safekeepers_new); +pgxn/neon/walproposer_pg.c: qsort(&safekeepers_old, len_old, sizeof(char *), pg_qsort_strcmp); +pgxn/neon/walproposer_pg.c: qsort(&safekeepers_new, len_new, sizeof(char *), pg_qsort_strcmp); +pgxn/neon/walproposer_pg.c: if (strcmp(safekeepers_old[i], safekeepers_new[i]) != 0) +pgxn/neon/walproposer_pg.c: * GUC assign_hook for neon.safekeepers. Restarts walproposer through FATAL if +pgxn/neon/walproposer_pg.c:assign_neon_safekeepers(const char *newval, void *extra) +pgxn/neon/walproposer_pg.c: wpg_log(FATAL, "neon.safekeepers is empty"); +pgxn/neon/walproposer_pg.c: /* Copy values because we will modify them in split_safekeepers_list() */ +pgxn/neon/walproposer_pg.c: * XXX: If you change anything here, sync with test_safekeepers_reconfigure_reorder. +pgxn/neon/walproposer_pg.c: if (!safekeepers_cmp(oldval, newval_copy)) +pgxn/neon/walproposer_pg.c: wpg_log(FATAL, "restarting walproposer to change safekeeper list from %s to %s", +pgxn/neon/walproposer_pg.c:WalproposerShmemInit_SyncSafekeeper(void) +pgxn/neon/walproposer_pg.c: snprintf(bgw.bgw_library_name, BGW_MAXLEN, "neon"); +pgxn/neon/walproposer_pg.c:walprop_pg_init_standalone_sync_safekeepers(void) +pgxn/neon/walproposer_pg.c: * reaches durable storage (in safekeepers), before the server shuts down +pgxn/neon/walproposer_pg.c:walprop_pg_update_donor(WalProposer *wp, Safekeeper *donor, XLogRecPtr donor_lsn) +pgxn/neon/walproposer_pg.c:walprop_error_message(Safekeeper *sk) +pgxn/neon/walproposer_pg.c:walprop_status(Safekeeper *sk) +pgxn/neon/walproposer_pg.c: char *password = neon_auth_token; +pgxn/neon/walproposer_pg.c: * Connect using the given connection string. If the NEON_AUTH_TOKEN +pgxn/neon/walproposer_pg.c:walprop_connect_start(Safekeeper *sk) +pgxn/neon/walproposer_pg.c:walprop_connect_poll(Safekeeper *sk) +pgxn/neon/walproposer_pg.c:walprop_send_query(Safekeeper *sk, char *query) +pgxn/neon/walproposer_pg.c:walprop_get_query_result(Safekeeper *sk) +pgxn/neon/walproposer_pg.c:walprop_socket(Safekeeper *sk) +pgxn/neon/walproposer_pg.c:walprop_flush(Safekeeper *sk) +pgxn/neon/walproposer_pg.c: * The protocol we use between walproposer and safekeeper means that we +pgxn/neon/walproposer_pg.c: * safekeeper won't normally send a CopyDone message. +pgxn/neon/walproposer_pg.c: * Receive a message from the safekeeper. +pgxn/neon/walproposer_pg.c:walprop_async_read(Safekeeper *sk, char **buf, int *amount) +pgxn/neon/walproposer_pg.c:walprop_async_write(Safekeeper *sk, void const *buf, size_t size) +pgxn/neon/walproposer_pg.c:walprop_blocking_write(Safekeeper *sk, void const *buf, size_t size) +pgxn/neon/walproposer_pg.c:walprop_finish(Safekeeper *sk) +pgxn/neon/walproposer_pg.c: NeonWALReaderFree(sk->xlogreader); +pgxn/neon/walproposer_pg.c: rm_safekeeper_event_set(sk, false); +pgxn/neon/walproposer_pg.c: * Subscribe for new WAL and stream it in the loop to safekeepers. +pgxn/neon/walproposer_pg.c: * Neon doesn't currently use PG Timelines, but it may in the future, so +pgxn/neon/walproposer_pg.c: longer used, replaced by neon_walreader; but callback still exists because +pgxn/neon/walproposer_pg.c:WalProposerRecovery(WalProposer *wp, Safekeeper *sk) +pgxn/neon/walproposer_pg.c:walprop_pg_wal_reader_allocate(Safekeeper *sk) +pgxn/neon/walproposer_pg.c: sk->xlogreader = NeonWALReaderAllocate(wal_segment_size, sk->wp->propEpochStartLsn, log_prefix); +pgxn/neon/walproposer_pg.c:static NeonWALReadResult +pgxn/neon/walproposer_pg.c:walprop_pg_wal_read(Safekeeper *sk, char *buf, XLogRecPtr startptr, Size count, char **errmsg) +pgxn/neon/walproposer_pg.c: NeonWALReadResult res; +pgxn/neon/walproposer_pg.c: res = NeonWALRead(sk->xlogreader, +pgxn/neon/walproposer_pg.c: if (res == NEON_WALREAD_SUCCESS) +pgxn/neon/walproposer_pg.c: if (NeonWALReaderEvents(sk->xlogreader) == 0) +pgxn/neon/walproposer_pg.c: rm_safekeeper_event_set(sk, false); +pgxn/neon/walproposer_pg.c: else if (res == NEON_WALREAD_ERROR) +pgxn/neon/walproposer_pg.c: *errmsg = NeonWALReaderErrMsg(sk->xlogreader); +pgxn/neon/walproposer_pg.c:walprop_pg_wal_reader_events(Safekeeper *sk) +pgxn/neon/walproposer_pg.c: return NeonWALReaderEvents(sk->xlogreader); +pgxn/neon/walproposer_pg.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer_pg.c: wp->safekeeper[i].eventPos = -1; +pgxn/neon/walproposer_pg.c: wp->safekeeper[i].nwrEventPos = -1; +pgxn/neon/walproposer_pg.c: wp->safekeeper[i].nwrConnEstablished = false; +pgxn/neon/walproposer_pg.c: /* for each sk, we have socket plus potentially socket for neon walreader */ +pgxn/neon/walproposer_pg.c: waitEvents = CreateWaitEventSet(NULL, 2 + 2 * wp->n_safekeepers); +pgxn/neon/walproposer_pg.c: waitEvents = CreateWaitEventSet(TopMemoryContext, 2 + 2 * wp->n_safekeepers); +pgxn/neon/walproposer_pg.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer_pg.c: wp->safekeeper[i].eventPos = -1; +pgxn/neon/walproposer_pg.c: wp->safekeeper[i].nwrEventPos = -1; +pgxn/neon/walproposer_pg.c: wp->safekeeper[i].nwrConnEstablished = false; +pgxn/neon/walproposer_pg.c:/* add safekeeper socket to wait event set */ +pgxn/neon/walproposer_pg.c:walprop_pg_add_safekeeper_event_set(Safekeeper *sk, uint32 events) +pgxn/neon/walproposer_pg.c:/* add neon wal reader socket to wait event set */ +pgxn/neon/walproposer_pg.c:add_nwr_event_set(Safekeeper *sk, uint32 events) +pgxn/neon/walproposer_pg.c: sk->nwrEventPos = AddWaitEventToSet(waitEvents, events, NeonWALReaderSocket(sk->xlogreader), NULL, sk); +pgxn/neon/walproposer_pg.c: sk->nwrConnEstablished = NeonWALReaderIsRemConnEstablished(sk->xlogreader); +pgxn/neon/walproposer_pg.c:walprop_pg_update_event_set(Safekeeper *sk, uint32 events) +pgxn/neon/walproposer_pg.c: * Update neon_walreader event. +pgxn/neon/walproposer_pg.c:update_nwr_event_set(Safekeeper *sk, uint32 events) +pgxn/neon/walproposer_pg.c:walprop_pg_active_state_update_event_set(Safekeeper *sk) +pgxn/neon/walproposer_pg.c: SafekeeperStateDesiredEvents(sk, &sk_events, &nwr_events); +pgxn/neon/walproposer_pg.c: * If we need to wait for neon_walreader, ensure we have up to date socket +pgxn/neon/walproposer_pg.c: rm_safekeeper_event_set(sk, false); +pgxn/neon/walproposer_pg.c: * reconstructing the whole set, SafekeeperStateDesiredEvents instead +pgxn/neon/walproposer_pg.c: rm_safekeeper_event_set(sk, false); +pgxn/neon/walproposer_pg.c:walprop_pg_rm_safekeeper_event_set(Safekeeper *to_remove) +pgxn/neon/walproposer_pg.c: rm_safekeeper_event_set(to_remove, true); +pgxn/neon/walproposer_pg.c: * If is_sk is true, socket of connection to safekeeper is removed; otherwise +pgxn/neon/walproposer_pg.c: * socket of neon_walreader. +pgxn/neon/walproposer_pg.c:rm_safekeeper_event_set(Safekeeper *to_remove, bool is_sk) +pgxn/neon/walproposer_pg.c: * function with safekeeper socket not existing, but do that with neon +pgxn/neon/walproposer_pg.c: /* Re-initialize it without adding any safekeeper events */ +pgxn/neon/walproposer_pg.c: * loop through the existing safekeepers. If they aren't the one we're +pgxn/neon/walproposer_pg.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer_pg.c: Safekeeper *sk = &wp->safekeeper[i]; +pgxn/neon/walproposer_pg.c: * If this safekeeper isn't offline, add events for it, except for the +pgxn/neon/walproposer_pg.c: SafekeeperStateDesiredEvents(sk, &sk_events, &nwr_events); +pgxn/neon/walproposer_pg.c: wp->api.add_safekeeper_event_set(sk, sk_events); +pgxn/neon/walproposer_pg.c:walprop_pg_wait_event_set(WalProposer *wp, long timeout, Safekeeper **sk, uint32 *events) +pgxn/neon/walproposer_pg.c: * Do that only when we're interested in new WAL: without sync-safekeepers +pgxn/neon/walproposer_pg.c: if (!wp->config->syncSafekeepers && wp->availableLsn != InvalidXLogRecPtr && GetFlushRecPtr(NULL) > wp->availableLsn) +pgxn/neon/walproposer_pg.c: * Wait for a wait event to happen, or timeout: - Safekeeper socket can +pgxn/neon/walproposer_pg.c: * Process config if requested. This restarts walproposer if safekeepers +pgxn/neon/walproposer_pg.c: * list changed. Don't do that for sync-safekeepers because quite probably +pgxn/neon/walproposer_pg.c: * sync-safekeepers should be quick to finish anyway. +pgxn/neon/walproposer_pg.c: if (!wp->config->syncSafekeepers && ConfigReloadPending) +pgxn/neon/walproposer_pg.c: * event from a safekeeper socket. +pgxn/neon/walproposer_pg.c: *sk = (Safekeeper *) event.user_data; +pgxn/neon/walproposer_pg.c:walprop_pg_finish_sync_safekeepers(WalProposer *wp, XLogRecPtr lsn) +pgxn/neon/walproposer_pg.c: * Note that unlike sync-safekeepers waiting here is not reliable: we +pgxn/neon/walproposer_pg.c: * don't check that majority of safekeepers received and persisted +pgxn/neon/walproposer_pg.c: * sync-safekeepers which must bump commit_lsn or basebackup will fail, +pgxn/neon/walproposer_pg.c: * this catchup is important only for tests where safekeepers/network +pgxn/neon/walproposer_pg.c: * Combine hot standby feedbacks from all safekeepers. +pgxn/neon/walproposer_pg.c: for (int i = 0; i < wp->n_safekeepers; i++) +pgxn/neon/walproposer_pg.c: if (wp->safekeeper[i].state == SS_ACTIVE) +pgxn/neon/walproposer_pg.c: HotStandbyFeedback *skhs = &wp->safekeeper[i].appendResponse.hs; +pgxn/neon/walproposer_pg.c: * Based on commitLsn and safekeeper responses including pageserver feedback, +pgxn/neon/walproposer_pg.c: * None of that is functional in sync-safekeepers. +pgxn/neon/walproposer_pg.c:walprop_pg_process_safekeeper_feedback(WalProposer *wp, Safekeeper *sk) +pgxn/neon/walproposer_pg.c: if (wp->config->syncSafekeepers) +pgxn/neon/walproposer_pg.c: SetNeonCurrentClusterSize(sk->appendResponse.ps_feedback.currentClusterSize); +pgxn/neon/walproposer_pg.c: * hardened and will be fetched from one of safekeepers by +pgxn/neon/walproposer_pg.c: * neon_walreader if needed. +pgxn/neon/walproposer_pg.c: /* write_lsn - This is what durably stored in safekeepers quorum. */ +pgxn/neon/walproposer_pg.c: /* flush_lsn - This is what durably stored in safekeepers quorum. */ +pgxn/neon/walproposer_pg.c:SetNeonCurrentClusterSize(uint64 size) +pgxn/neon/walproposer_pg.c:GetNeonCurrentClusterSize(void) +pgxn/neon/walproposer_pg.c:uint64 GetNeonCurrentClusterSize(void); +pgxn/neon/walproposer_pg.c: .add_safekeeper_event_set = walprop_pg_add_safekeeper_event_set, +pgxn/neon/walproposer_pg.c: .rm_safekeeper_event_set = walprop_pg_rm_safekeeper_event_set, +pgxn/neon/walproposer_pg.c: .finish_sync_safekeepers = walprop_pg_finish_sync_safekeepers, +pgxn/neon/walproposer_pg.c: .process_safekeeper_feedback = walprop_pg_process_safekeeper_feedback, +pgxn/neon/walsender_hooks.c: * Implements XLogReaderRoutine in terms of NeonWALReader. Allows for +pgxn/neon/walsender_hooks.c: * fetching WAL from safekeepers, which normal xlogreader can't do. +pgxn/neon/walsender_hooks.c:#include "neon.h" +pgxn/neon/walsender_hooks.c:#include "neon_walreader.h" +pgxn/neon/walsender_hooks.c:static NeonWALReader *wal_reader = NULL; +pgxn/neon/walsender_hooks.c:NeonWALReadWaitForWAL(XLogRecPtr loc) +pgxn/neon/walsender_hooks.c: while (!NeonWALReaderUpdateDonor(wal_reader)) +pgxn/neon/walsender_hooks.c:NeonWALPageRead( +pgxn/neon/walsender_hooks.c: XLogRecPtr flushptr = NeonWALReadWaitForWAL(targetPagePtr + reqLen); +pgxn/neon/walsender_hooks.c: rem_lsn = NeonWALReaderGetRemLsn(wal_reader); +pgxn/neon/walsender_hooks.c: NeonWALReaderResetRemote(wal_reader); +pgxn/neon/walsender_hooks.c: NeonWALReadResult res = NeonWALRead( +pgxn/neon/walsender_hooks.c: if (res == NEON_WALREAD_SUCCESS) +pgxn/neon/walsender_hooks.c: xlogreader->seg.ws_tli = NeonWALReaderGetSegment(wal_reader)->ws_tli; +pgxn/neon/walsender_hooks.c: xlogreader->seg.ws_segno = NeonWALReaderGetSegment(wal_reader)->ws_segno; +pgxn/neon/walsender_hooks.c: if (res == NEON_WALREAD_ERROR) +pgxn/neon/walsender_hooks.c: NeonWALReaderErrMsg(wal_reader)); +pgxn/neon/walsender_hooks.c: pgsocket sock = NeonWALReaderSocket(wal_reader); +pgxn/neon/walsender_hooks.c: uint32_t reader_events = NeonWALReaderEvents(wal_reader); +pgxn/neon/walsender_hooks.c: WAIT_EVENT_NEON_WAL_DL); +pgxn/neon/walsender_hooks.c:NeonWALReadSegmentOpen(XLogReaderState *xlogreader, XLogSegNo nextSegNo, TimeLineID *tli_p) +pgxn/neon/walsender_hooks.c: neon_wal_segment_open(wal_reader, nextSegNo, tli_p); +pgxn/neon/walsender_hooks.c: xlogreader->seg.ws_file = NeonWALReaderGetSegment(wal_reader)->ws_file; +pgxn/neon/walsender_hooks.c:NeonWALReadSegmentClose(XLogReaderState *xlogreader) +pgxn/neon/walsender_hooks.c: neon_wal_segment_close(wal_reader); +pgxn/neon/walsender_hooks.c: xlogreader->seg.ws_file = NeonWALReaderGetSegment(wal_reader)->ws_file; +pgxn/neon/walsender_hooks.c:NeonOnDemandXLogReaderRoutines(XLogReaderRoutine *xlr) +pgxn/neon/walsender_hooks.c: * If safekeepers are not configured, assume we don't need neon_walreader, +pgxn/neon/walsender_hooks.c: * i.e. running neon fork locally. +pgxn/neon/walsender_hooks.c: wal_reader = NeonWALReaderAllocate(wal_segment_size, basebackupLsn, "[walsender] "); +pgxn/neon/walsender_hooks.c: xlr->page_read = NeonWALPageRead; +pgxn/neon/walsender_hooks.c: xlr->segment_open = NeonWALReadSegmentOpen; +pgxn/neon/walsender_hooks.c: xlr->segment_close = NeonWALReadSegmentClose; +pgxn/neon/walsender_hooks.h:void NeonOnDemandXLogReaderRoutines(struct XLogReaderRoutine *xlr); +pgxn/neon_rmgr/Makefile:# pgxs/neon/Makefile +pgxn/neon_rmgr/Makefile:MODULE_big = neon_rmgr +pgxn/neon_rmgr/Makefile: neon_rmgr.o \ +pgxn/neon_rmgr/Makefile: neon_rmgr_decode.o \ +pgxn/neon_rmgr/Makefile: neon_rmgr_desc.o +pgxn/neon_rmgr/Makefile:EXTENSION = neon_rmgr +pgxn/neon_rmgr/Makefile:PGFILEDESC = "Neon WAL Resource Manager - custom WAL records used to make Neon work (since PG 16)" +pgxn/neon_rmgr/neon_rmgr.c:#include "access/neon_xlog.h" +pgxn/neon_rmgr/neon_rmgr.c:#include "neon_rmgr.h" +pgxn/neon_rmgr/neon_rmgr.c:static void neon_rm_redo(XLogReaderState *record); +pgxn/neon_rmgr/neon_rmgr.c:static void neon_rm_startup(void); +pgxn/neon_rmgr/neon_rmgr.c:static void neon_rm_cleanup(void); +pgxn/neon_rmgr/neon_rmgr.c:static void neon_rm_mask(char *pagedata, BlockNumber blkno); +pgxn/neon_rmgr/neon_rmgr.c:static void redo_neon_heap_insert(XLogReaderState *record); +pgxn/neon_rmgr/neon_rmgr.c:static void redo_neon_heap_delete(XLogReaderState *record); +pgxn/neon_rmgr/neon_rmgr.c:static void redo_neon_heap_update(XLogReaderState *record, bool hot_update); +pgxn/neon_rmgr/neon_rmgr.c:static void redo_neon_heap_lock(XLogReaderState *record); +pgxn/neon_rmgr/neon_rmgr.c:static void redo_neon_heap_multi_insert(XLogReaderState *record); +pgxn/neon_rmgr/neon_rmgr.c:const static RmgrData NeonRmgr = { +pgxn/neon_rmgr/neon_rmgr.c: .rm_redo = neon_rm_redo, +pgxn/neon_rmgr/neon_rmgr.c: .rm_desc = neon_rm_desc, +pgxn/neon_rmgr/neon_rmgr.c: .rm_identify = neon_rm_identify, +pgxn/neon_rmgr/neon_rmgr.c: .rm_startup = neon_rm_startup, +pgxn/neon_rmgr/neon_rmgr.c: .rm_cleanup = neon_rm_cleanup, +pgxn/neon_rmgr/neon_rmgr.c: .rm_mask = neon_rm_mask, +pgxn/neon_rmgr/neon_rmgr.c: .rm_decode = neon_rm_decode, +pgxn/neon_rmgr/neon_rmgr.c: RegisterCustomRmgr(RM_NEON_ID, &NeonRmgr); +pgxn/neon_rmgr/neon_rmgr.c:neon_rm_redo(XLogReaderState *record) +pgxn/neon_rmgr/neon_rmgr.c: switch (info & XLOG_NEON_OPMASK) +pgxn/neon_rmgr/neon_rmgr.c: case XLOG_NEON_HEAP_INSERT: +pgxn/neon_rmgr/neon_rmgr.c: redo_neon_heap_insert(record); +pgxn/neon_rmgr/neon_rmgr.c: case XLOG_NEON_HEAP_DELETE: +pgxn/neon_rmgr/neon_rmgr.c: redo_neon_heap_delete(record); +pgxn/neon_rmgr/neon_rmgr.c: case XLOG_NEON_HEAP_UPDATE: +pgxn/neon_rmgr/neon_rmgr.c: redo_neon_heap_update(record, false); +pgxn/neon_rmgr/neon_rmgr.c: case XLOG_NEON_HEAP_HOT_UPDATE: +pgxn/neon_rmgr/neon_rmgr.c: redo_neon_heap_update(record, true); +pgxn/neon_rmgr/neon_rmgr.c: case XLOG_NEON_HEAP_LOCK: +pgxn/neon_rmgr/neon_rmgr.c: redo_neon_heap_lock(record); +pgxn/neon_rmgr/neon_rmgr.c: case XLOG_NEON_HEAP_MULTI_INSERT: +pgxn/neon_rmgr/neon_rmgr.c: redo_neon_heap_multi_insert(record); +pgxn/neon_rmgr/neon_rmgr.c:neon_rm_startup(void) +pgxn/neon_rmgr/neon_rmgr.c:neon_rm_cleanup(void) +pgxn/neon_rmgr/neon_rmgr.c:neon_rm_mask(char *pagedata, BlockNumber blkno) +pgxn/neon_rmgr/neon_rmgr.c:redo_neon_heap_insert(XLogReaderState *record) +pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_insert *xlrec = (xl_neon_heap_insert *) XLogRecGetData(record); +pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_header xlhdr; +pgxn/neon_rmgr/neon_rmgr.c: newlen = datalen - SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr.c: Assert(datalen > SizeOfNeonHeapHeader && newlen <= MaxHeapTupleSize); +pgxn/neon_rmgr/neon_rmgr.c: memcpy((char *) &xlhdr, data, SizeOfNeonHeapHeader); +pgxn/neon_rmgr/neon_rmgr.c: data += SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr.c:redo_neon_heap_delete(XLogReaderState *record) +pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_delete *xlrec = (xl_neon_heap_delete *) XLogRecGetData(record); +pgxn/neon_rmgr/neon_rmgr.c:redo_neon_heap_update(XLogReaderState *record, bool hot_update) +pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_update *xlrec = (xl_neon_heap_update *) XLogRecGetData(record); +pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_header xlhdr; +pgxn/neon_rmgr/neon_rmgr.c: memcpy((char *) &xlhdr, recdata, SizeOfNeonHeapHeader); +pgxn/neon_rmgr/neon_rmgr.c: recdata += SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr.c:redo_neon_heap_lock(XLogReaderState *record) +pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_lock *xlrec = (xl_neon_heap_lock *) XLogRecGetData(record); +pgxn/neon_rmgr/neon_rmgr.c:redo_neon_heap_multi_insert(XLogReaderState *record) +pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_multi_insert *xlrec; +pgxn/neon_rmgr/neon_rmgr.c: xlrec = (xl_neon_heap_multi_insert *) XLogRecGetData(record); +pgxn/neon_rmgr/neon_rmgr.c: xl_neon_multi_insert_tuple *xlhdr; +pgxn/neon_rmgr/neon_rmgr.c: xlhdr = (xl_neon_multi_insert_tuple *) SHORTALIGN(tupdata); +pgxn/neon_rmgr/neon_rmgr.c: tupdata = ((char *) xlhdr) + SizeOfNeonMultiInsertTuple; +pgxn/neon_rmgr/neon_rmgr.control:# neon_rmgr extension +pgxn/neon_rmgr/neon_rmgr.control:comment = 'Neon WAL Resource Manager - custom WAL records used to make Neon work (since PG 16)' +pgxn/neon_rmgr/neon_rmgr.control:module_pathname = '$libdir/neon_rmgr' +pgxn/neon_rmgr/neon_rmgr.h:#ifndef NEON_RMGR_H +pgxn/neon_rmgr/neon_rmgr.h:#define NEON_RMGR_H +pgxn/neon_rmgr/neon_rmgr.h:extern void neon_rm_desc(StringInfo buf, XLogReaderState *record); +pgxn/neon_rmgr/neon_rmgr.h:extern void neon_rm_decode(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); +pgxn/neon_rmgr/neon_rmgr.h:extern const char *neon_rm_identify(uint8 info); +pgxn/neon_rmgr/neon_rmgr.h:#endif //NEON_RMGR_H +pgxn/neon_rmgr/neon_rmgr_decode.c:#include "access/neon_xlog.h" +pgxn/neon_rmgr/neon_rmgr_decode.c:#include "neon_rmgr.h" +pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); +pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); +pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); +pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); +pgxn/neon_rmgr/neon_rmgr_decode.c:neon_rm_decode(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +pgxn/neon_rmgr/neon_rmgr_decode.c: uint8 info = XLogRecGetInfo(buf->record) & XLOG_NEON_OPMASK; +pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_INSERT: +pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonInsert(ctx, buf); +pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_DELETE: +pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonDelete(ctx, buf); +pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_UPDATE: +pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_HOT_UPDATE: +pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonUpdate(ctx, buf); +pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_LOCK: +pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_MULTI_INSERT: +pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonMultiInsert(ctx, buf); +pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_insert *xlrec; +pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_insert *) XLogRecGetData(r); +pgxn/neon_rmgr/neon_rmgr_decode.c: tuplelen = datalen - SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_delete *xlrec; +pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_delete *) XLogRecGetData(r); +pgxn/neon_rmgr/neon_rmgr_decode.c: Size datalen = XLogRecGetDataLen(r) - SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr_decode.c: Size tuplelen = datalen - SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr_decode.c: Assert(XLogRecGetDataLen(r) > (SizeOfNeonHeapDelete + SizeOfNeonHeapHeader)); +pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeXLogTuple((char *) xlrec + SizeOfNeonHeapDelete, +pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_update *xlrec; +pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_update *) XLogRecGetData(r); +pgxn/neon_rmgr/neon_rmgr_decode.c: tuplelen = datalen - SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr_decode.c: data = XLogRecGetData(r) + SizeOfNeonHeapUpdate; +pgxn/neon_rmgr/neon_rmgr_decode.c: datalen = XLogRecGetDataLen(r) - SizeOfNeonHeapUpdate; +pgxn/neon_rmgr/neon_rmgr_decode.c: tuplelen = datalen - SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_multi_insert *xlrec; +pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_multi_insert *) XLogRecGetData(r); +pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_multi_insert_tuple *xlhdr; +pgxn/neon_rmgr/neon_rmgr_decode.c: xlhdr = (xl_neon_multi_insert_tuple *) SHORTALIGN(data); +pgxn/neon_rmgr/neon_rmgr_decode.c: data = ((char *) xlhdr) + SizeOfNeonMultiInsertTuple; +pgxn/neon_rmgr/neon_rmgr_decode.c: /* move to the next xl_neon_multi_insert_tuple entry */ +pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_header xlhdr; +pgxn/neon_rmgr/neon_rmgr_decode.c: int datalen = len - SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr_decode.c: SizeOfNeonHeapHeader); +pgxn/neon_rmgr/neon_rmgr_decode.c: data + SizeOfNeonHeapHeader, +pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); +pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); +pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); +pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); +pgxn/neon_rmgr/neon_rmgr_decode.c:neon_rm_decode(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +pgxn/neon_rmgr/neon_rmgr_decode.c: uint8 info = XLogRecGetInfo(buf->record) & XLOG_NEON_OPMASK; +pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_INSERT: +pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonInsert(ctx, buf); +pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_DELETE: +pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonDelete(ctx, buf); +pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_UPDATE: +pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_HOT_UPDATE: +pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonUpdate(ctx, buf); +pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_LOCK: +pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_MULTI_INSERT: +pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonMultiInsert(ctx, buf); +pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_insert *xlrec; +pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_insert *) XLogRecGetData(r); +pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_delete *xlrec; +pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_delete *) XLogRecGetData(r); +pgxn/neon_rmgr/neon_rmgr_decode.c: Size datalen = XLogRecGetDataLen(r) - SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr_decode.c: Size tuplelen = datalen - SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr_decode.c: Assert(XLogRecGetDataLen(r) > (SizeOfNeonHeapDelete + SizeOfNeonHeapHeader)); +pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeXLogTuple((char *) xlrec + SizeOfNeonHeapDelete, +pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_update *xlrec; +pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_update *) XLogRecGetData(r); +pgxn/neon_rmgr/neon_rmgr_decode.c: tuplelen = datalen - SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr_decode.c: data = XLogRecGetData(r) + SizeOfNeonHeapUpdate; +pgxn/neon_rmgr/neon_rmgr_decode.c: datalen = XLogRecGetDataLen(r) - SizeOfNeonHeapUpdate; +pgxn/neon_rmgr/neon_rmgr_decode.c: tuplelen = datalen - SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_multi_insert *xlrec; +pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_multi_insert *) XLogRecGetData(r); +pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_multi_insert_tuple *xlhdr; +pgxn/neon_rmgr/neon_rmgr_decode.c: xlhdr = (xl_neon_multi_insert_tuple *) SHORTALIGN(data); +pgxn/neon_rmgr/neon_rmgr_decode.c: data = ((char *) xlhdr) + SizeOfNeonMultiInsertTuple; +pgxn/neon_rmgr/neon_rmgr_decode.c: /* move to the next xl_neon_multi_insert_tuple entry */ +pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_header xlhdr; +pgxn/neon_rmgr/neon_rmgr_decode.c: int datalen = len - SizeOfNeonHeapHeader; +pgxn/neon_rmgr/neon_rmgr_decode.c: SizeOfNeonHeapHeader); +pgxn/neon_rmgr/neon_rmgr_decode.c: data + SizeOfNeonHeapHeader, +pgxn/neon_rmgr/neon_rmgr_desc.c:#include "access/neon_xlog.h" +pgxn/neon_rmgr/neon_rmgr_desc.c:#include "neon_rmgr.h" +pgxn/neon_rmgr/neon_rmgr_desc.c:neon_rm_desc(StringInfo buf, XLogReaderState *record) +pgxn/neon_rmgr/neon_rmgr_desc.c: info &= XLOG_NEON_OPMASK; +pgxn/neon_rmgr/neon_rmgr_desc.c: if (info == XLOG_NEON_HEAP_INSERT) +pgxn/neon_rmgr/neon_rmgr_desc.c: xl_neon_heap_insert *xlrec = (xl_neon_heap_insert *) rec; +pgxn/neon_rmgr/neon_rmgr_desc.c: else if (info == XLOG_NEON_HEAP_DELETE) +pgxn/neon_rmgr/neon_rmgr_desc.c: xl_neon_heap_delete *xlrec = (xl_neon_heap_delete *) rec; +pgxn/neon_rmgr/neon_rmgr_desc.c: else if (info == XLOG_NEON_HEAP_UPDATE) +pgxn/neon_rmgr/neon_rmgr_desc.c: xl_neon_heap_update *xlrec = (xl_neon_heap_update *) rec; +pgxn/neon_rmgr/neon_rmgr_desc.c: else if (info == XLOG_NEON_HEAP_HOT_UPDATE) +pgxn/neon_rmgr/neon_rmgr_desc.c: xl_neon_heap_update *xlrec = (xl_neon_heap_update *) rec; +pgxn/neon_rmgr/neon_rmgr_desc.c: else if (info == XLOG_NEON_HEAP_LOCK) +pgxn/neon_rmgr/neon_rmgr_desc.c: xl_neon_heap_lock *xlrec = (xl_neon_heap_lock *) rec; +pgxn/neon_rmgr/neon_rmgr_desc.c: else if (info == XLOG_NEON_HEAP_MULTI_INSERT) +pgxn/neon_rmgr/neon_rmgr_desc.c: xl_neon_heap_multi_insert *xlrec = (xl_neon_heap_multi_insert *) rec; +pgxn/neon_rmgr/neon_rmgr_desc.c: bool isinit = (XLogRecGetInfo(record) & XLOG_NEON_INIT_PAGE) != 0; +pgxn/neon_rmgr/neon_rmgr_desc.c:neon_rm_identify(uint8 info) +pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_INSERT: +pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_INSERT | XLOG_NEON_INIT_PAGE: +pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_DELETE: +pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_UPDATE: +pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_UPDATE | XLOG_NEON_INIT_PAGE: +pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_HOT_UPDATE: +pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_HOT_UPDATE | XLOG_HEAP_INIT_PAGE: +pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_LOCK: +pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_MULTI_INSERT: +pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_MULTI_INSERT | XLOG_NEON_INIT_PAGE: +contrib/pg_trgm/data/trgm2.data:Riad Zenith Marrakech +doc/src/sgml/release-16.sgml:2022-08-29 [82739d4a8] Use ARM Advanced SIMD (NEON) intrinsics where available +doc/src/sgml/release-16.sgml: Instruction Multiple Data) (NEON) instructions +src/backend/access/brin/brin_xlog.c: //ZENITH XXX Don't use BufferGetBlockNumber because wal-redo doesn't pin buffer. +src/backend/access/gin/ginxlog.c: /* NEON: we do not not apply WAL record if target page is absent at replica */ +src/backend/access/heap/heapam.c:#include "access/neon_xlog.h" +src/backend/access/heap/heapam.c: xl_neon_heap_insert xlrec; +src/backend/access/heap/heapam.c: xl_neon_heap_header xlhdr; +src/backend/access/heap/heapam.c: uint8 info = XLOG_NEON_HEAP_INSERT; +src/backend/access/heap/heapam.c: info |= XLOG_NEON_INIT_PAGE; +src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlrec, SizeOfNeonHeapInsert); +src/backend/access/heap/heapam.c: XLogRegisterBufData(0, (char *) &xlhdr, SizeOfNeonHeapHeader); +src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, info); +src/backend/access/heap/heapam.c: * NEON: speculative token is not stored in WAL, so if the page is evicted +src/backend/access/heap/heapam.c: xl_neon_heap_multi_insert *xlrec; +src/backend/access/heap/heapam.c: uint8 info = XLOG_NEON_HEAP_MULTI_INSERT; +src/backend/access/heap/heapam.c: /* allocate xl_neon_heap_multi_insert struct from the scratch area */ +src/backend/access/heap/heapam.c: xlrec = (xl_neon_heap_multi_insert *) scratchptr; +src/backend/access/heap/heapam.c: scratchptr += SizeOfNeonHeapMultiInsert; +src/backend/access/heap/heapam.c: xl_neon_multi_insert_tuple *tuphdr; +src/backend/access/heap/heapam.c: tuphdr = (xl_neon_multi_insert_tuple *) SHORTALIGN(scratchptr); +src/backend/access/heap/heapam.c: scratchptr = ((char *) tuphdr) + SizeOfNeonMultiInsertTuple; +src/backend/access/heap/heapam.c: info |= XLOG_NEON_INIT_PAGE; +src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, info); +src/backend/access/heap/heapam.c: xl_neon_heap_delete xlrec; +src/backend/access/heap/heapam.c: xl_neon_heap_header xlhdr; +src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlrec, SizeOfNeonHeapDelete); +src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlhdr, SizeOfNeonHeapHeader); +src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, XLOG_NEON_HEAP_DELETE); +src/backend/access/heap/heapam.c: xl_neon_heap_lock xlrec; +src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlrec, SizeOfNeonHeapLock); +src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, XLOG_NEON_HEAP_LOCK); +src/backend/access/heap/heapam.c: xl_neon_heap_lock xlrec; +src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlrec, SizeOfNeonHeapLock); +src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, XLOG_NEON_HEAP_LOCK); +src/backend/access/heap/heapam.c: ReleaseBuffer(buffer); /* NEON: release buffer pinned by heap_insert */ +src/backend/access/heap/heapam.c: * NEON: release buffer pinned by heap_insert +src/backend/access/heap/heapam.c: xl_neon_heap_delete xlrec; +src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlrec, SizeOfNeonHeapDelete); +src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, XLOG_NEON_HEAP_DELETE); +src/backend/access/heap/heapam.c: xl_neon_heap_update xlrec; +src/backend/access/heap/heapam.c: xl_neon_heap_header xlhdr; +src/backend/access/heap/heapam.c: xl_neon_heap_header xlhdr_idx; +src/backend/access/heap/heapam.c: info = XLOG_NEON_HEAP_HOT_UPDATE; +src/backend/access/heap/heapam.c: info = XLOG_NEON_HEAP_UPDATE; +src/backend/access/heap/heapam.c: info |= XLOG_NEON_INIT_PAGE; +src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlrec, SizeOfNeonHeapUpdate); +src/backend/access/heap/heapam.c: XLogRegisterBufData(0, (char *) &xlhdr, SizeOfNeonHeapHeader); +src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlhdr_idx, SizeOfNeonHeapHeader); +src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, info); +src/backend/access/heap/heapam.c: * NEON: despite to the comment above we need to update page LSN here. +src/backend/access/heap/heapam.c: * For Neon this assignment is critical because otherwise last written LSN tracked at compute doesn't +src/backend/access/heap/heapam.c: * It is fixed in upstream in https://github.com/neondatabase/postgres/commit/7bf713dd2d0739fbcd4103971ed69c17ebe677ea +src/backend/access/heap/rewriteheap.c: * NEON: we need to persist mapping file in WAL +src/backend/access/heap/rewriteheap.c: snprintf(prefix, sizeof(prefix), "neon-file:%s", path); +src/backend/access/heap/visibilitymap.c: /* NEON: we have to update page LSN even if wal_log_hints=off +src/backend/access/nbtree/nbtinsert.c: /* NEON: store the page's former cycle ID for FPI check later */ +src/backend/access/nbtree/nbtinsert.c: * NEON: If we split to earlier pages during a btree vacuum cycle, +src/backend/access/nbtree/nbtsearch.c: /* Neon: initialize prefetch */ +src/backend/access/nbtree/nbtsearch.c: * Neon prefetch is efficient only if prefetched blocks are accessed by the same worker +src/backend/access/nbtree/nbtsearch.c: /* Neon: prefetch referenced heap pages. +src/backend/access/nbtree/nbtxlog.c: * NEON: If the original page was supposed to be recovered from FPI, +src/backend/access/transam/slru.c: * NEON: we do not want to include large pg_xact/multixact files in basebackup and prefer +src/backend/access/transam/xlog.c:/* NEON: Hook to allow the neon extension to restore running-xacts from CLOG at replica startup */ +src/backend/access/transam/xlog.c: /* neon: copy of startup's RedoStartLSN for walproposer's use */ +src/backend/access/transam/xlog.c: * Variables read from 'zenith.signal' file. +src/backend/access/transam/xlog.c:bool ZenithRecoveryRequested = false; +src/backend/access/transam/xlog.c:XLogRecPtr zenithLastRec = InvalidXLogRecPtr; +src/backend/access/transam/xlog.c:bool zenithWriteOk = false; +src/backend/access/transam/xlog.c: /* Neon: in case of sync-safekeepers shared memory is not inialized */ +src/backend/access/transam/xlog.c:readZenithSignalFile(void) +src/backend/access/transam/xlog.c: fd = BasicOpenFile(ZENITH_SIGNAL_FILE, O_RDONLY | PG_BINARY); +src/backend/access/transam/xlog.c: if (stat(ZENITH_SIGNAL_FILE, &statbuf) != 0) +src/backend/access/transam/xlog.c: zenithLastRec = InvalidXLogRecPtr; +src/backend/access/transam/xlog.c: zenithWriteOk = false; +src/backend/access/transam/xlog.c: zenithLastRec = InvalidXLogRecPtr; +src/backend/access/transam/xlog.c: zenithWriteOk = true; +src/backend/access/transam/xlog.c: zenithLastRec = ((uint64) hi) << 32 | lo; +src/backend/access/transam/xlog.c: if (zenithLastRec == InvalidXLogRecPtr) +src/backend/access/transam/xlog.c: zenithWriteOk = true; +src/backend/access/transam/xlog.c: ZenithRecoveryRequested = true; +src/backend/access/transam/xlog.c: LSN_FORMAT_ARGS(zenithLastRec)); +src/backend/access/transam/xlog.c: * Read zenith.signal before anything else. +src/backend/access/transam/xlog.c: readZenithSignalFile(); +src/backend/access/transam/xlog.c: if (!XRecOffIsValid(ControlFile->checkPoint) && !ZenithRecoveryRequested) +src/backend/access/transam/xlog.c: * Neon: We also use a similar mechanism to start up sooner at +src/backend/access/transam/xlog.c: /* Neon: called StandbyRecoverPreparedTransactions() above already */ +src/backend/access/transam/xlog.c: * But GetLastWrittenLSN(InvalidOid) is used only by zenith_dbsize which is not performance critical. +src/backend/access/transam/xlog.c: * NEON: perform checkpiont action requiring write to the WAL before we determine the REDO pointer. +src/backend/access/transam/xlog.c: * NEON: we use logical records to persist information of about slots, origins, relation map... +src/backend/access/transam/xlog.c: * NEON: In the special WAL redo process, blocks that are being +src/backend/access/transam/xlogrecovery.c:bool allowReplicaMisconfig = true; /* NEON: GUC is defined in neon extension */ +src/backend/access/transam/xlogrecovery.c: else if (ZenithRecoveryRequested) +src/backend/access/transam/xlogrecovery.c: else if (ZenithRecoveryRequested) +src/backend/access/transam/xlogrecovery.c: * Zenith hacks to spawn compute node without WAL. Pretend that we +src/backend/access/transam/xlogrecovery.c: * just finished reading the record that started at 'zenithLastRec' +src/backend/access/transam/xlogrecovery.c: LSN_FORMAT_ARGS(zenithLastRec)); +src/backend/access/transam/xlogrecovery.c: CheckPointLoc = zenithLastRec; +src/backend/access/transam/xlogrecovery.c: * When a primary Neon compute node is started, we pretend that it +src/backend/access/transam/xlogrecovery.c: if (checkPoint.redo > CheckPointLoc && !ZenithRecoveryRequested) +src/backend/access/transam/xlogrecovery.c: if (!ZenithRecoveryRequested) +src/backend/access/transam/xlogrecovery.c: * When starting from a zenith base backup, we don't have WAL. Initialize +src/backend/access/transam/xlogrecovery.c: if (ZenithRecoveryRequested) +src/backend/access/transam/xlogrecovery.c: if (!zenithWriteOk) +src/backend/access/transam/xlogrecovery.c: // FIXME: should we unlink zenith.signal? +src/backend/access/transam/xlogrecovery.c: if (!ZenithRecoveryRequested) +src/backend/access/transam/xlogrecovery.c: if (!ZenithRecoveryRequested) +src/backend/access/transam/xlogrecovery.c: if (ZenithRecoveryRequested) +src/backend/commands/extension.c: /* Hide neon* extensions. */ +src/backend/commands/extension.c: if (strncasecmp(extname, "neon", strlen("neon")) == 0 && !superuser()) +src/backend/commands/extension.c: /* Hide neon* extensions. */ +src/backend/commands/extension.c: if (strncasecmp(extname, "neon", strlen("neon")) == 0 && !superuser()) +src/backend/commands/publicationcmds.c: if (stmt->for_all_tables && !superuser() && !is_neon_superuser()) +src/backend/commands/publicationcmds.c: if (schemaidlist != NIL && !superuser() && !is_neon_superuser()) +src/backend/commands/sequence.c:/* NEON XXX: to ensure sequence order of sequence in Zenith we need to WAL log each sequence update. */ +src/backend/executor/nodeBitmapHeapscan.c: /* NEON: we are not using prefetch iterator for parallel plan so no need to adjust it */ +src/backend/main/main.c: * neon extension sets PGC_POSTMASTER gucs requiring this. +src/backend/main/main.c: CallExtMain("neon_walredo", "WalRedoMain", argc, argv, false); +src/backend/main/main.c: else if (argc > 1 && strcmp(argv[1], "--sync-safekeepers") == 0) +src/backend/main/main.c: CallExtMain("neon", "WalProposerSync", argc, argv, true); +src/backend/postmaster/postmaster.c: /* Neon: Also allow walproposer background worker to be treated like a WAL sender, so that it's shut down last */ +src/backend/postmaster/postmaster.c: /* Neon: Also allow walproposer background worker to be treated like a WAL sender, so that it's shut down last */ +src/backend/replication/logical/launcher.c:bool disable_logical_replication_subscribers = false; /* NEON: GUC is defined in neon extension */ +src/backend/replication/logical/launcher.c: /* NEON GUC that allows to control logical replication without postgres restart */ +src/backend/replication/logical/logical.c: * NEON: override page_read/segment_open/segment_close functions to support on-demand WAL download +src/backend/replication/logical/logical.c: if (strncmp(prefix, "neon-file", strlen("neon-file")) == 0) +src/backend/replication/logical/logical.c: if (strncmp(prefix, "neon-file", strlen("neon-file")) == 0) +src/backend/replication/logical/snapbuild.c: /* NEON specific: persist snapshot in storage using logical message */ +src/backend/replication/logical/snapbuild.c: snprintf(prefix, sizeof(prefix), "neon-file:%s", path); +src/backend/replication/logical/snapbuild.c: /* NEON specific: delete file from storage using logical message */ +src/backend/replication/logical/snapbuild.c: snprintf(prefix, sizeof(prefix), "neon-file:%s", path); +src/backend/replication/logical/worker.c: * This is particularly important in neon where WAL is hardened only after +src/backend/replication/logical/worker.c: * it is flushed on majority of safekeepers. +src/backend/replication/slot.c: /* NEON specific: delete slot from storage using logical message */ +src/backend/replication/slot.c: snprintf(prefix, sizeof(prefix), "neon-file:%s/state", path); +src/backend/replication/slot.c: /* NEON specific: persist slot in storage using logical message */ +src/backend/replication/slot.c: snprintf(prefix, sizeof(prefix), "neon-file:%s", path); +src/backend/replication/walsender.c: * neon: in vanilla this doesn't happen because walsenders register in +src/backend/replication/walsender.c: * However, in neon walproposer occupies walsender slot but doesn't register +src/backend/replication/walsender.c: // XXX Zenith uses flush_lsn to pass extra payload, so use write_lsn here +src/backend/storage/buffer/bufmgr.c:bool zenith_test_evict = false; +src/backend/storage/buffer/bufmgr.c: * NEON: In earlier Neon PostgreSQL versions, we zeroed the pages +src/backend/storage/buffer/bufmgr.c: if (zenith_test_evict && !InRecovery) +src/backend/storage/buffer/localbuf.c:/* NEON: prevent eviction of the buffer of target page */ +src/backend/storage/buffer/localbuf.c: /* ZENITH: Prevent eviction of the buffer with target wal redo page */ +src/backend/storage/smgr/smgr.c:// * TODO: NEON v15- REMOVED this? +src/backend/storage/smgr/smgr.c: * Neon-added functions to mark the phases of an unlogged index build. +src/backend/storage/smgr/smgr.c: * NEON: we do not want to include large pg_xact/multixact files in basebackup and prefer +src/backend/utils/adt/acl.c:is_neon_superuser(void) +src/backend/utils/adt/acl.c: return is_neon_superuser_arg(GetUserId()); +src/backend/utils/adt/acl.c:is_neon_superuser_arg(Oid roleid) +src/backend/utils/adt/acl.c: Oid neon_superuser_oid = get_role_oid("databricks_superuser", true /*missing_ok*/); +src/backend/utils/adt/acl.c: return neon_superuser_oid != InvalidOid && has_privs_of_role(roleid, neon_superuser_oid); +src/backend/utils/error/elog.c: /* Don't include source file information if it would leak Neon. */ +src/backend/utils/error/elog.c: strcasestr(edata->filename, "neon") == NULL && +src/backend/utils/error/elog.c: strcasestr(edata->filename, "zenith") == NULL) +src/backend/utils/error/elog.c: strcasestr(edata->funcname, "neon") == NULL && +src/backend/utils/error/elog.c: strcasestr(edata->funcname, "zenith") == NULL) +src/backend/utils/fmgr/dfmgr.c:static void neon_try_load(const char *name); +src/backend/utils/fmgr/dfmgr.c: neon_try_load(fullname); +src/backend/utils/fmgr/dfmgr.c:neon_try_load(const char *name) +src/backend/utils/fmgr/dfmgr.c: neon_try_load(fullname); +src/backend/utils/misc/guc_funcs.c: /* Hide neon.* options. */ +src/backend/utils/misc/guc_funcs.c: strncasecmp(conf->name, "neon.", strlen("neon.")) == 0) +src/backend/utils/misc/guc_funcs.c: bool replace_neon; +src/backend/utils/misc/guc_funcs.c: boot_val and reset_val, so 'neon' leaks through there. +src/backend/utils/misc/guc_funcs.c: replace_neon = strcasecmp(conf->name, "shared_preload_libraries") == 0; +src/backend/utils/misc/guc_funcs.c: values[12] = replace_neon ? +src/backend/utils/misc/guc_funcs.c: filter_neon(lconf->boot_val) : pstrdup(lconf->boot_val); +src/backend/utils/misc/guc_funcs.c: values[13] = replace_neon ? +src/backend/utils/misc/guc_funcs.c: filter_neon(lconf->reset_val) : strdup(lconf->reset_val); +src/backend/utils/misc/guc_funcs.c: strncasecmp(conf->name, "neon.", strlen("neon.")) == 0)) +src/backend/utils/misc/guc_tables.c: * Replace 'neon' with 'dbrx'. +src/backend/utils/misc/guc_tables.c:filter_neon(const char *input) +src/backend/utils/misc/guc_tables.c: while ((pos = strcasestr(pos, "neon")) != NULL) { +src/backend/utils/misc/guc_tables.c: * GUC show_hook for shared_preload_libraries to replace 'neon' with 'dbrx'. +src/backend/utils/misc/guc_tables.c: return filter_neon(shared_preload_libraries_string); +src/backend/utils/misc/guc_tables.c: {"neon_test_evict", PGC_POSTMASTER, UNGROUPED, +src/backend/utils/misc/guc_tables.c: &zenith_test_evict, +src/backend/utils/misc/guc_tables.c: gettext_noop("Size of last written LSN cache used by Neon."), +src/include/access/nbtree.h: /* Neon: prefetch state */ +src/include/access/neon_xlog.h:#ifndef NEON_XLOG_H +src/include/access/neon_xlog.h:#define NEON_XLOG_H +src/include/access/neon_xlog.h: * The RMGR id of the Neon RMGR +src/include/access/neon_xlog.h:#define RM_NEON_ID 134 +src/include/access/neon_xlog.h:#define XLOG_NEON_INIT_PAGE 0x80 +src/include/access/neon_xlog.h:#define XLOG_NEON_OPMASK ((~XLOG_NEON_INIT_PAGE) & XLR_RMGR_INFO_MASK) +src/include/access/neon_xlog.h:#define XLOG_NEON_HEAP_INSERT 0x00 +src/include/access/neon_xlog.h:#define XLOG_NEON_HEAP_DELETE 0x10 +src/include/access/neon_xlog.h:#define XLOG_NEON_HEAP_UPDATE 0x20 +src/include/access/neon_xlog.h:#define XLOG_NEON_HEAP_HOT_UPDATE 0x30 +src/include/access/neon_xlog.h:#define XLOG_NEON_HEAP_LOCK 0x40 +src/include/access/neon_xlog.h:#define XLOG_NEON_HEAP_MULTI_INSERT 0x50 +src/include/access/neon_xlog.h:typedef struct xl_neon_heap_header { +src/include/access/neon_xlog.h:} xl_neon_heap_header; +src/include/access/neon_xlog.h:#define SizeOfNeonHeapHeader (offsetof(xl_neon_heap_header, t_hoff) + sizeof(uint8)) +src/include/access/neon_xlog.h:typedef struct xl_neon_heap_insert +src/include/access/neon_xlog.h: /* xl_neon_heap_header & TUPLE DATA in backup block 0 */ +src/include/access/neon_xlog.h:} xl_neon_heap_insert; +src/include/access/neon_xlog.h:#define SizeOfNeonHeapInsert (offsetof(xl_neon_heap_insert, flags) + sizeof(uint8)) +src/include/access/neon_xlog.h:typedef struct xl_neon_heap_delete +src/include/access/neon_xlog.h:} xl_neon_heap_delete; +src/include/access/neon_xlog.h:#define SizeOfNeonHeapDelete (offsetof(xl_neon_heap_delete, t_cid) + sizeof(uint32)) +src/include/access/neon_xlog.h: * After that, xl_neon_heap_header and new tuple data follow. The new tuple +src/include/access/neon_xlog.h:typedef struct xl_neon_heap_update +src/include/access/neon_xlog.h: * are set, xl_neon_heap_header and tuple data for the old tuple follow. +src/include/access/neon_xlog.h:} xl_neon_heap_update; +src/include/access/neon_xlog.h:#define SizeOfNeonHeapUpdate (offsetof(xl_neon_heap_update, new_offnum) + sizeof(OffsetNumber)) +src/include/access/neon_xlog.h:typedef struct xl_neon_heap_lock +src/include/access/neon_xlog.h:} xl_neon_heap_lock; +src/include/access/neon_xlog.h:#define SizeOfNeonHeapLock (offsetof(xl_neon_heap_lock, flags) + sizeof(uint8)) +src/include/access/neon_xlog.h: * The main data of the record consists of this xl_neon_heap_multi_insert header. +src/include/access/neon_xlog.h: * In block 0's data portion, there is an xl_neon_multi_insert_tuple struct, +src/include/access/neon_xlog.h: * each xl_neon_multi_insert_tuple struct. +src/include/access/neon_xlog.h:typedef struct xl_neon_heap_multi_insert +src/include/access/neon_xlog.h:} xl_neon_heap_multi_insert; +src/include/access/neon_xlog.h:#define SizeOfNeonHeapMultiInsert offsetof(xl_neon_heap_multi_insert, offsets) +src/include/access/neon_xlog.h:typedef struct xl_neon_multi_insert_tuple +src/include/access/neon_xlog.h:} xl_neon_multi_insert_tuple; +src/include/access/neon_xlog.h:#define SizeOfNeonMultiInsertTuple (offsetof(xl_neon_multi_insert_tuple, t_hoff) + sizeof(uint8)) +src/include/access/neon_xlog.h:#endif //NEON_XLOG_H +src/include/access/xlog.h:extern bool ZenithRecoveryRequested; +src/include/access/xlog.h:extern XLogRecPtr zenithLastRec; +src/include/access/xlog.h:extern bool zenithWriteOk; +src/include/access/xlog.h:/* neon specifics */ +src/include/access/xlog.h:/* NEON: Hook to allow the neon extension to restore running-xacts from CLOG at replica startup */ +src/include/access/xlog.h:#define ZENITH_SIGNAL_FILE "zenith.signal" +src/include/miscadmin.h:extern bool is_neon_superuser(void); /* current user is neon_superuser */ +src/include/miscadmin.h:extern bool is_neon_superuser_arg(Oid roleid); /* given user is neon_superuser */ +src/include/port/simd.h:#elif defined(__aarch64__) && defined(__ARM_NEON) +src/include/port/simd.h: * We use the Neon instructions if the compiler provides access to them (as +src/include/port/simd.h: * indicated by __ARM_NEON) and we are on aarch64. While Neon support is +src/include/port/simd.h: * hardware does have it. Neon exists in some 32-bit hardware too, but we +src/include/port/simd.h:#include +src/include/port/simd.h:#define USE_NEON +src/include/port/simd.h:#elif defined(USE_NEON) +src/include/port/simd.h:#elif defined(USE_NEON) +src/include/port/simd.h:#elif defined(USE_NEON) +src/include/port/simd.h:#elif defined(USE_NEON) +src/include/port/simd.h:#elif defined(USE_NEON) +src/include/port/simd.h:#if defined(USE_NEON) +src/include/port/simd.h:#elif defined(USE_NEON) +src/include/port/simd.h:#elif defined(USE_NEON) +src/include/port/simd.h:#elif defined(USE_NEON) +src/include/port/simd.h:#elif defined(USE_NEON) +src/include/port/simd.h:#elif defined(USE_NEON) +src/include/replication/walsender.h:/* expose these so that they can be reused by the neon walproposer extension */ +src/include/storage/bufmgr.h:extern bool zenith_test_evict; +src/include/storage/checksum_impl.h: * available on x86 SSE4.1 extensions (pmulld) and ARM NEON (vmul.i32). +src/include/storage/smgr.h: * Neon: extended SMGR API. +src/include/storage/smgr.h: * This define can be used by extensions to determine that them are built for Neon. +src/include/storage/smgr.h:#define NEON_SMGR 1 +src/include/storage/smgr.h:/* Neon: Change relpersistence for unlogged index builds */ +src/include/utils/guc_tables.h:const char * filter_neon(const char *input); diff --git a/test_runner/regress/test_compaction.py b/test_runner/regress/test_compaction.py index be82ee806f..839d2afd20 100644 --- a/test_runner/regress/test_compaction.py +++ b/test_runner/regress/test_compaction.py @@ -863,6 +863,87 @@ def test_pageserver_compaction_circuit_breaker(neon_env_builder: NeonEnvBuilder) assert not env.pageserver.log_contains(".*Circuit breaker failure ended.*") +def test_ps_corruption_detection_feedback(neon_env_builder: NeonEnvBuilder): + """ + Test that when the pageserver detects corruption during image layer creation, + it sends corruption feedback to the safekeeper which gets recorded in its + safekeeper_ps_corruption_detected metric. + """ + # Configure tenant with aggressive compaction settings to easily trigger compaction + TENANT_CONF = { + # Small checkpoint distance to create many layers + "checkpoint_distance": 1024 * 128, + # Compact small layers + "compaction_target_size": 1024 * 128, + # Create image layers eagerly + "image_creation_threshold": 1, + "image_layer_creation_check_threshold": 0, + # Force frequent compaction + "compaction_period": "1s", + } + + env = neon_env_builder.init_start(initial_tenant_conf=TENANT_CONF) + # We are simulating compaction failures so we should allow these error messages. + env.pageserver.allowed_errors.append(".*Compaction failed.*") + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline + + pageserver_http = env.pageserver.http_client() + workload = Workload(env, tenant_id, timeline_id) + workload.init() + + # Enable the failpoint that will cause image layer creation to fail due to a (simulated) detected + # corruption. + pageserver_http.configure_failpoints(("create-image-layer-fail-simulated-corruption", "return")) + + # Write some data to trigger compaction and image layer creation + log.info("Writing data to trigger compaction...") + workload.write_rows(1024 * 64, upload=False) + workload.write_rows(1024 * 64, upload=False) + + # Returns True if the corruption signal from PS is propagated to the SK according to the "safekeeper_ps_corruption_detected" metric. + # Raises an exception otherwise. + def check_corruption_signal_propagated_to_sk(): + # Get metrics from all safekeepers + for sk in env.safekeepers: + sk_metrics = sk.http_client().get_metrics() + # Look for our corruption detected metric with the right tenant and timeline + corruption_metrics = sk_metrics.query_all("safekeeper_ps_corruption_detected") + + for metric in corruption_metrics: + # Check if there's a metric for our tenant and timeline that has value 1 + if ( + metric.labels.get("tenant_id") == str(tenant_id) + and metric.labels.get("timeline_id") == str(timeline_id) + and metric.value == 1 + ): + log.info(f"Corruption detected by safekeeper {sk.id}: {metric}") + return True + raise Exception("Corruption detection feedback not found in any safekeeper metrics") + + # Returns True if the corruption signal from PS is propagated to the PG according to the "ps_corruption_detected" metric + # in "neon_perf_counters". + # Raises an exception otherwise. + def check_corruption_signal_propagated_to_pg(): + endpoint = workload.endpoint() + results = endpoint.safe_psql("CREATE EXTENSION IF NOT EXISTS neon") + results = endpoint.safe_psql( + "SELECT value FROM neon_perf_counters WHERE metric = 'ps_corruption_detected'" + ) + log.info("Query corruption detection metric, results: %s", results) + if results[0][0] == 1: + log.info("Corruption detection signal is raised on Postgres") + return True + raise Exception("Corruption detection signal is not raise on Postgres") + + # Confirm that the corruption signal propagates to both the safekeeper and Postgres + wait_until(check_corruption_signal_propagated_to_sk, timeout=10, interval=0.1) + wait_until(check_corruption_signal_propagated_to_pg, timeout=10, interval=0.1) + + # Cleanup the failpoint + pageserver_http.configure_failpoints(("create-image-layer-fail-simulated-corruption", "off")) + + @pytest.mark.parametrize("enabled", [True, False]) def test_image_layer_compression(neon_env_builder: NeonEnvBuilder, enabled: bool): tenant_conf = {