rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-05-31 03:50:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into devin/1745492468-add-dev-flag-pr11517

Browse Source
This commit is contained in:
John Spray
2025-06-26 07:32:08 -07:00
committed by GitHub
parent e8c39d260a 605fb04f89
commit 0618845bbb
425 changed files with 12213 additions and 5436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
safekeeper/Cargo.toml
View File

@@ -58,6 +58,7 @@ metrics.workspace = true
pem.workspace = true
postgres_backend.workspace = true
postgres_ffi.workspace = true
postgres_versioninfo.workspace = true
pq_proto.workspace = true
remote_storage.workspace = true
safekeeper_api.workspace = true

10
safekeeper/client/src/mgmt_api.rs
View File

@@ -8,8 +8,8 @@ use std::error::Error as _;
use http_utils::error::HttpErrorBody;
use reqwest::{IntoUrl, Method, StatusCode};
use safekeeper_api::models::{
self, PullTimelineRequest, PullTimelineResponse, SafekeeperUtilization, TimelineCreateRequest,
TimelineStatus,
self, PullTimelineRequest, PullTimelineResponse, SafekeeperStatus, SafekeeperUtilization,
TimelineCreateRequest, TimelineStatus,
};
use utils::id::{NodeId, TenantId, TimelineId};
use utils::logging::SecretString;
@@ -183,6 +183,12 @@ impl Client {
self.get(&uri).await
}
pub async fn status(&self) -> Result<SafekeeperStatus> {
let uri = format!("{}/v1/status", self.mgmt_api_endpoint);
let resp = self.get(&uri).await?;
resp.json().await.map_err(Error::ReceiveBody)
}
pub async fn utilization(&self) -> Result<SafekeeperUtilization> {
let uri = format!("{}/v1/utilization", self.mgmt_api_endpoint);
let resp = self.get(&uri).await?;

10
safekeeper/src/control_file.rs
View File

@@ -206,16 +206,10 @@ impl Storage for FileStorage {
let buf: Vec<u8> = s.write_to_buf()?;
control_partial.write_all(&buf).await.with_context(|| {
format!(
"failed to write safekeeper state into control file at: {}",
control_partial_path
)
format!("failed to write safekeeper state into control file at: {control_partial_path}")
})?;
control_partial.flush().await.with_context(|| {
format!(
"failed to flush safekeeper state into control file at: {}",
control_partial_path
)
format!("failed to flush safekeeper state into control file at: {control_partial_path}")
})?;
let control_path = self.timeline_dir.join(CONTROL_FILE_NAME);

28
safekeeper/src/control_file_upgrade.rs
View File

@@ -2,6 +2,7 @@
use std::vec;
use anyhow::{Result, bail};
use postgres_versioninfo::PgVersionId;
use pq_proto::SystemId;
use safekeeper_api::membership::{Configuration, INVALID_GENERATION};
use safekeeper_api::{ServerInfo, Term};
@@ -46,7 +47,7 @@ struct SafeKeeperStateV1 {
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ServerInfoV2 {
/// Postgres server version
pub pg_version: u32,
pub pg_version: PgVersionId,
pub system_id: SystemId,
pub tenant_id: TenantId,
pub timeline_id: TimelineId,
@@ -75,7 +76,7 @@ pub struct SafeKeeperStateV2 {
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ServerInfoV3 {
/// Postgres server version
pub pg_version: u32,
pub pg_version: PgVersionId,
pub system_id: SystemId,
#[serde(with = "hex")]
pub tenant_id: TenantId,
@@ -444,13 +445,13 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result<TimelinePersiste
} else if version == 6 {
info!("reading safekeeper control file version {}", version);
let mut oldstate = TimelinePersistentState::des(&buf[..buf.len()])?;
if oldstate.server.pg_version != 0 {
if oldstate.server.pg_version != PgVersionId::UNKNOWN {
return Ok(oldstate);
}
// set pg_version to the default v14
info!("setting pg_version to 140005");
oldstate.server.pg_version = 140005;
oldstate.server.pg_version = PgVersionId::from_full_pg_version(140005);
return Ok(oldstate);
} else if version == 7 {
@@ -547,6 +548,7 @@ pub fn downgrade_v10_to_v9(state: &TimelinePersistentState) -> TimelinePersisten
mod tests {
use std::str::FromStr;
use postgres_versioninfo::PgMajorVersion;
use utils::Hex;
use utils::id::NodeId;
@@ -563,7 +565,7 @@ mod tests {
epoch: 43,
},
server: ServerInfoV2 {
pg_version: 14,
pg_version: PgVersionId::from(PgMajorVersion::PG14),
system_id: 0x1234567887654321,
tenant_id,
timeline_id,
@@ -586,8 +588,8 @@ mod tests {
0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
// epoch
0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
// pg_version
0x0e, 0x00, 0x00, 0x00,
// pg_version = 140000
0xE0, 0x22, 0x02, 0x00,
// system_id
0x21, 0x43, 0x65, 0x87, 0x78, 0x56, 0x34, 0x12,
// tenant_id
@@ -626,7 +628,7 @@ mod tests {
}]),
},
server: ServerInfoV2 {
pg_version: 14,
pg_version: PgVersionId::from(PgMajorVersion::PG14),
system_id: 0x1234567887654321,
tenant_id,
timeline_id,
@@ -646,7 +648,7 @@ mod tests {
let expected = [
0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56,
0x00, 0x00, 0x00, 0x00, 0xE0, 0x22, 0x02, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56,
0x34, 0x12, 0xcf, 0x04, 0x80, 0x92, 0x97, 0x07, 0xee, 0x75, 0x37, 0x23, 0x37, 0xef,
0xaa, 0x5e, 0xcf, 0x96, 0x11, 0x2d, 0xed, 0x66, 0x42, 0x2a, 0xa5, 0xe9, 0x53, 0xe5,
0x44, 0x0f, 0xa5, 0x42, 0x7a, 0xc4, 0x78, 0x56, 0x34, 0x12, 0xc4, 0x7a, 0x42, 0xa5,
@@ -675,7 +677,7 @@ mod tests {
}]),
},
server: ServerInfoV3 {
pg_version: 14,
pg_version: PgVersionId::from(PgMajorVersion::PG14),
system_id: 0x1234567887654321,
tenant_id,
timeline_id,
@@ -695,7 +697,7 @@ mod tests {
let expected = [
0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56,
0x00, 0x00, 0x00, 0x00, 0xE0, 0x22, 0x02, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56,
0x34, 0x12, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x66, 0x30, 0x34,
0x38, 0x30, 0x39, 0x32, 0x39, 0x37, 0x30, 0x37, 0x65, 0x65, 0x37, 0x35, 0x33, 0x37,
0x32, 0x33, 0x33, 0x37, 0x65, 0x66, 0x61, 0x61, 0x35, 0x65, 0x63, 0x66, 0x39, 0x36,
@@ -731,7 +733,7 @@ mod tests {
}]),
},
server: ServerInfo {
pg_version: 14,
pg_version: PgVersionId::from(PgMajorVersion::PG14),
system_id: 0x1234567887654321,
wal_seg_size: 0x12345678,
},
@@ -765,7 +767,7 @@ mod tests {
0x30, 0x66, 0x61, 0x35, 0x34, 0x32, 0x37, 0x61, 0x63, 0x34, 0x2a, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x0e, 0x00, 0x00, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, 0x34, 0x12, 0x78, 0x56,
0xE0, 0x22, 0x02, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, 0x34, 0x12, 0x78, 0x56,
0x34, 0x12, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x34, 0x37, 0x61,
0x34, 0x32, 0x61, 0x35, 0x30, 0x66, 0x34, 0x34, 0x65, 0x35, 0x35, 0x33, 0x65, 0x39,
0x61, 0x35, 0x32, 0x61, 0x34, 0x32, 0x36, 0x36, 0x65, 0x64, 0x32, 0x64, 0x31, 0x31,

16
safekeeper/src/handler.rs
View File

@@ -73,7 +73,7 @@ fn parse_cmd(cmd: &str) -> anyhow::Result<SafekeeperPostgresCommand> {
let re = Regex::new(r"START_WAL_PUSH(\s+?\((.*)\))?").unwrap();
let caps = re
.captures(cmd)
.context(format!("failed to parse START_WAL_PUSH command {}", cmd))?;
.context(format!("failed to parse START_WAL_PUSH command {cmd}"))?;
// capture () content
let options = caps.get(2).map(|m| m.as_str()).unwrap_or("");
// default values
@@ -85,24 +85,20 @@ fn parse_cmd(cmd: &str) -> anyhow::Result<SafekeeperPostgresCommand> {
}
let mut kvit = kvstr.split_whitespace();
let key = kvit.next().context(format!(
"failed to parse key in kv {} in command {}",
kvstr, cmd
"failed to parse key in kv {kvstr} in command {cmd}"
))?;
let value = kvit.next().context(format!(
"failed to parse value in kv {} in command {}",
kvstr, cmd
"failed to parse value in kv {kvstr} in command {cmd}"
))?;
let value_trimmed = value.trim_matches('\'');
if key == "proto_version" {
proto_version = value_trimmed.parse::<u32>().context(format!(
"failed to parse proto_version value {} in command {}",
value, cmd
"failed to parse proto_version value {value} in command {cmd}"
))?;
}
if key == "allow_timeline_creation" {
allow_timeline_creation = value_trimmed.parse::<bool>().context(format!(
"failed to parse allow_timeline_creation value {} in command {}",
value, cmd
"failed to parse allow_timeline_creation value {value} in command {cmd}"
))?;
}
}
@@ -118,7 +114,7 @@ fn parse_cmd(cmd: &str) -> anyhow::Result<SafekeeperPostgresCommand> {
.unwrap();
let caps = re
.captures(cmd)
.context(format!("failed to parse START_REPLICATION command {}", cmd))?;
.context(format!("failed to parse START_REPLICATION command {cmd}"))?;
let start_lsn =
Lsn::from_str(&caps[1]).context("parse start LSN from START_REPLICATION command")?;
let term = if let Some(m) = caps.get(2) {

40
safekeeper/src/safekeeper.rs
View File

@@ -9,6 +9,7 @@ use anyhow::{Context, Result, bail};
use byteorder::{LittleEndian, ReadBytesExt};
use bytes::{Buf, BufMut, Bytes, BytesMut};
use postgres_ffi::{MAX_SEND_SIZE, TimeLineID};
use postgres_versioninfo::{PgMajorVersion, PgVersionId};
use pq_proto::SystemId;
use safekeeper_api::membership::{
INVALID_GENERATION, MemberSet, SafekeeperGeneration as Generation, SafekeeperId,
@@ -29,7 +30,7 @@ use crate::{control_file, wal_storage};
pub const SK_PROTO_VERSION_2: u32 = 2;
pub const SK_PROTO_VERSION_3: u32 = 3;
pub const UNKNOWN_SERVER_VERSION: u32 = 0;
pub const UNKNOWN_SERVER_VERSION: PgVersionId = PgVersionId::UNKNOWN;
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
pub struct TermLsn {
@@ -64,10 +65,10 @@ impl TermHistory {
for i in 0..n_entries {
let term = bytes
.get_u64_f()
.with_context(|| format!("TermHistory pos {} misses term", i))?;
.with_context(|| format!("TermHistory pos {i} misses term"))?;
let lsn = bytes
.get_u64_f()
.with_context(|| format!("TermHistory pos {} misses lsn", i))?
.with_context(|| format!("TermHistory pos {i} misses lsn"))?
.into();
res.push(TermLsn { term, lsn })
}
@@ -121,9 +122,7 @@ impl TermHistory {
if let Some(sk_th_last) = sk_th.last() {
assert!(
sk_th_last.lsn <= sk_wal_end,
"safekeeper term history end {:?} LSN is higher than WAL end {:?}",
sk_th_last,
sk_wal_end
"safekeeper term history end {sk_th_last:?} LSN is higher than WAL end {sk_wal_end:?}"
);
}
@@ -220,7 +219,7 @@ pub struct ProposerGreeting {
pub timeline_id: TimelineId,
pub mconf: membership::Configuration,
/// Postgres server version
pub pg_version: u32,
pub pg_version: PgVersionId,
pub system_id: SystemId,
pub wal_seg_size: u32,
}
@@ -231,7 +230,7 @@ pub struct ProposerGreetingV2 {
/// proposer-acceptor protocol version
pub protocol_version: u32,
/// Postgres server version
pub pg_version: u32,
pub pg_version: PgVersionId,
pub proposer_id: PgUuid,
pub system_id: SystemId,
pub timeline_id: TimelineId,
@@ -438,11 +437,11 @@ impl ProposerAcceptorMessage {
for i in 0..members_len {
let id = buf
.get_u64_f()
.with_context(|| format!("reading member {} node_id", i))?;
let host = Self::get_cstr(buf).with_context(|| format!("reading member {} host", i))?;
.with_context(|| format!("reading member {i} node_id"))?;
let host = Self::get_cstr(buf).with_context(|| format!("reading member {i} host"))?;
let pg_port = buf
.get_u16_f()
.with_context(|| format!("reading member {} port", i))?;
.with_context(|| format!("reading member {i} port"))?;
let sk = SafekeeperId {
id: NodeId(id),
host,
@@ -463,12 +462,12 @@ impl ProposerAcceptorMessage {
for i in 0..new_members_len {
let id = buf
.get_u64_f()
.with_context(|| format!("reading new member {} node_id", i))?;
let host = Self::get_cstr(buf)
.with_context(|| format!("reading new member {} host", i))?;
.with_context(|| format!("reading new member {i} node_id"))?;
let host =
Self::get_cstr(buf).with_context(|| format!("reading new member {i} host"))?;
let pg_port = buf
.get_u16_f()
.with_context(|| format!("reading new member {} port", i))?;
.with_context(|| format!("reading new member {i} port"))?;
let sk = SafekeeperId {
id: NodeId(id),
host,
@@ -513,7 +512,7 @@ impl ProposerAcceptorMessage {
tenant_id,
timeline_id,
mconf,
pg_version,
pg_version: PgVersionId::from_full_pg_version(pg_version),
system_id,
wal_seg_size,
};
@@ -963,7 +962,8 @@ where
* because safekeepers parse WAL headers and the format
* may change between versions.
*/
if msg.pg_version / 10000 != self.state.server.pg_version / 10000
if PgMajorVersion::try_from(msg.pg_version)?
!= PgMajorVersion::try_from(self.state.server.pg_version)?
&& self.state.server.pg_version != UNKNOWN_SERVER_VERSION
{
bail!(
@@ -1508,7 +1508,7 @@ mod tests {
let mut vote_resp = sk.process_msg(&vote_request).await;
match vote_resp.unwrap() {
Some(AcceptorProposerMessage::VoteResponse(resp)) => assert!(resp.vote_given),
r => panic!("unexpected response: {:?}", r),
r => panic!("unexpected response: {r:?}"),
}
// reboot...
@@ -1523,7 +1523,7 @@ mod tests {
vote_resp = sk.process_msg(&vote_request).await;
match vote_resp.unwrap() {
Some(AcceptorProposerMessage::VoteResponse(resp)) => assert!(!resp.vote_given),
r => panic!("unexpected response: {:?}", r),
r => panic!("unexpected response: {r:?}"),
}
}
@@ -1750,7 +1750,7 @@ mod tests {
}]),
},
server: ServerInfo {
pg_version: 14,
pg_version: PgVersionId::from_full_pg_version(140000),
system_id: 0x1234567887654321,
wal_seg_size: 0x12345678,
},

18
safekeeper/src/send_interpreted_wal.rs
View File

@@ -8,8 +8,8 @@ use futures::StreamExt;
use futures::future::Either;
use pageserver_api::shard::ShardIdentity;
use postgres_backend::{CopyStreamHandlerEnd, PostgresBackend};
use postgres_ffi::get_current_timestamp;
use postgres_ffi::waldecoder::{WalDecodeError, WalStreamDecoder};
use postgres_ffi::{PgMajorVersion, get_current_timestamp};
use pq_proto::{BeMessage, InterpretedWalRecordsBody, WalSndKeepAlive};
use tokio::io::{AsyncRead, AsyncWrite};
use tokio::sync::mpsc::error::SendError;
@@ -78,7 +78,7 @@ pub(crate) struct InterpretedWalReader {
shard_senders: HashMap<ShardIdentity, smallvec::SmallVec<[ShardSenderState; 1]>>,
shard_notification_rx: Option<tokio::sync::mpsc::UnboundedReceiver<AttachShardNotification>>,
state: Arc<std::sync::RwLock<InterpretedWalReaderState>>,
pg_version: u32,
pg_version: PgMajorVersion,
}
/// A handle for [`InterpretedWalReader`] which allows for interacting with it
@@ -258,7 +258,7 @@ impl InterpretedWalReader {
start_pos: Lsn,
tx: tokio::sync::mpsc::Sender<Batch>,
shard: ShardIdentity,
pg_version: u32,
pg_version: PgMajorVersion,
appname: &Option<String>,
) -> InterpretedWalReaderHandle {
let state = Arc::new(std::sync::RwLock::new(InterpretedWalReaderState::Running {
@@ -322,7 +322,7 @@ impl InterpretedWalReader {
start_pos: Lsn,
tx: tokio::sync::mpsc::Sender<Batch>,
shard: ShardIdentity,
pg_version: u32,
pg_version: PgMajorVersion,
shard_notification_rx: Option<
tokio::sync::mpsc::UnboundedReceiver<AttachShardNotification>,
>,
@@ -718,7 +718,7 @@ mod tests {
use std::time::Duration;
use pageserver_api::shard::{ShardIdentity, ShardStripeSize};
use postgres_ffi::MAX_SEND_SIZE;
use postgres_ffi::{MAX_SEND_SIZE, PgMajorVersion};
use tokio::sync::mpsc::error::TryRecvError;
use utils::id::{NodeId, TenantTimelineId};
use utils::lsn::Lsn;
@@ -734,7 +734,7 @@ mod tests {
const SIZE: usize = 8 * 1024;
const MSG_COUNT: usize = 200;
const PG_VERSION: u32 = 17;
const PG_VERSION: PgMajorVersion = PgMajorVersion::PG17;
const SHARD_COUNT: u8 = 2;
let start_lsn = Lsn::from_str("0/149FD18").unwrap();
@@ -876,7 +876,7 @@ mod tests {
const SIZE: usize = 8 * 1024;
const MSG_COUNT: usize = 200;
const PG_VERSION: u32 = 17;
const PG_VERSION: PgMajorVersion = PgMajorVersion::PG17;
const SHARD_COUNT: u8 = 2;
let start_lsn = Lsn::from_str("0/149FD18").unwrap();
@@ -1025,7 +1025,7 @@ mod tests {
const SIZE: usize = 64 * 1024;
const MSG_COUNT: usize = 10;
const PG_VERSION: u32 = 17;
const PG_VERSION: PgMajorVersion = PgMajorVersion::PG17;
const SHARD_COUNT: u8 = 2;
const WAL_READER_BATCH_SIZE: usize = 8192;
@@ -1148,7 +1148,7 @@ mod tests {
const SIZE: usize = 8 * 1024;
const MSG_COUNT: usize = 10;
const PG_VERSION: u32 = 17;
const PG_VERSION: PgMajorVersion = PgMajorVersion::PG17;
let start_lsn = Lsn::from_str("0/149FD18").unwrap();
let env = Env::new(true).unwrap();

6
safekeeper/src/send_wal.rs
View File

@@ -12,7 +12,7 @@ use futures::FutureExt;
use itertools::Itertools;
use parking_lot::Mutex;
use postgres_backend::{CopyStreamHandlerEnd, PostgresBackend, PostgresBackendReader, QueryError};
use postgres_ffi::{MAX_SEND_SIZE, TimestampTz, get_current_timestamp};
use postgres_ffi::{MAX_SEND_SIZE, PgMajorVersion, TimestampTz, get_current_timestamp};
use pq_proto::{BeMessage, WalSndKeepAlive, XLogDataBody};
use safekeeper_api::Term;
use safekeeper_api::models::{
@@ -559,7 +559,9 @@ impl SafekeeperPostgresHandler {
format,
compression,
} => {
let pg_version = tli.tli.get_state().await.1.server.pg_version / 10000;
let pg_version =
PgMajorVersion::try_from(tli.tli.get_state().await.1.server.pg_version)
.unwrap();
let end_watch_view = end_watch.view();
let wal_residence_guard = tli.wal_residence_guard().await?;
let (tx, rx) = tokio::sync::mpsc::channel::<Batch>(2);

5
safekeeper/src/state.rs
View File

@@ -7,6 +7,7 @@ use std::time::SystemTime;
use anyhow::{Result, bail};
use postgres_ffi::WAL_SEGMENT_SIZE;
use postgres_versioninfo::{PgMajorVersion, PgVersionId};
use safekeeper_api::membership::Configuration;
use safekeeper_api::models::{TimelineMembershipSwitchResponse, TimelineTermBumpResponse};
use safekeeper_api::{INITIAL_TERM, ServerInfo, Term};
@@ -149,8 +150,8 @@ impl TimelinePersistentState {
&TenantTimelineId::empty(),
Configuration::empty(),
ServerInfo {
pg_version: 170000, /* Postgres server version (major * 10000) */
system_id: 0, /* Postgres system identifier */
pg_version: PgVersionId::from(PgMajorVersion::PG17),
system_id: 0, /* Postgres system identifier */
wal_seg_size: WAL_SEGMENT_SIZE as u32,
},
Lsn::INVALID,

2
safekeeper/src/timeline.rs
View File

@@ -395,6 +395,8 @@ pub enum TimelineError {
Cancelled(TenantTimelineId),
#[error("Timeline {0} was not found in global map")]
NotFound(TenantTimelineId),
#[error("Timeline {0} has been deleted")]
Deleted(TenantTimelineId),
#[error("Timeline {0} creation is in progress")]
CreationInProgress(TenantTimelineId),
#[error("Timeline {0} exists on disk, but wasn't loaded on startup")]

7
safekeeper/src/timeline_eviction.rs
View File

@@ -342,7 +342,7 @@ where
let bytes_read1 = reader1
.read(&mut buffer1[..bytes_to_read])
.await
.with_context(|| format!("failed to read from reader1 at offset {}", offset))?;
.with_context(|| format!("failed to read from reader1 at offset {offset}"))?;
if bytes_read1 == 0 {
anyhow::bail!("unexpected EOF from reader1 at offset {}", offset);
}
@@ -351,10 +351,7 @@ where
.read_exact(&mut buffer2[..bytes_read1])
.await
.with_context(|| {
format!(
"failed to read {} bytes from reader2 at offset {}",
bytes_read1, offset
)
format!("failed to read {bytes_read1} bytes from reader2 at offset {offset}")
})?;
assert!(bytes_read2 == bytes_read1);

2
safekeeper/src/timeline_manager.rs
View File

@@ -108,7 +108,7 @@ impl std::fmt::Debug for ManagerCtlMessage {
match self {
ManagerCtlMessage::GuardRequest(_) => write!(f, "GuardRequest"),
ManagerCtlMessage::TryGuardRequest(_) => write!(f, "TryGuardRequest"),
ManagerCtlMessage::GuardDrop(id) => write!(f, "GuardDrop({:?})", id),
ManagerCtlMessage::GuardDrop(id) => write!(f, "GuardDrop({id:?})"),
ManagerCtlMessage::BackupPartialReset(_) => write!(f, "BackupPartialReset"),
}
}

12
safekeeper/src/timelines_global_map.rs
View File

@@ -78,7 +78,13 @@ impl GlobalTimelinesState {
Some(GlobalMapTimeline::CreationInProgress) => {
Err(TimelineError::CreationInProgress(*ttid))
}
None => Err(TimelineError::NotFound(*ttid)),
None => {
if self.has_tombstone(ttid) {
Err(TimelineError::Deleted(*ttid))
} else {
Err(TimelineError::NotFound(*ttid))
}
}
}
}
@@ -141,7 +147,7 @@ impl GlobalTimelines {
};
let mut tenant_count = 0;
for tenants_dir_entry in std::fs::read_dir(&tenants_dir)
.with_context(|| format!("failed to list tenants dir {}", tenants_dir))?
.with_context(|| format!("failed to list tenants dir {tenants_dir}"))?
{
match &tenants_dir_entry {
Ok(tenants_dir_entry) => {
@@ -182,7 +188,7 @@ impl GlobalTimelines {
let timelines_dir = get_tenant_dir(&conf, &tenant_id);
for timelines_dir_entry in std::fs::read_dir(&timelines_dir)
.with_context(|| format!("failed to list timelines dir {}", timelines_dir))?
.with_context(|| format!("failed to list timelines dir {timelines_dir}"))?
{
match &timelines_dir_entry {
Ok(timeline_dir_entry) => {

3
safekeeper/src/wal_backup_partial.rs
View File

@@ -364,8 +364,7 @@ impl PartialBackup {
// there should always be zero or one uploaded segment
assert!(
new_segments.is_empty(),
"too many uploaded segments: {:?}",
new_segments
"too many uploaded segments: {new_segments:?}"
);
}

18
safekeeper/src/wal_storage.rs
View File

@@ -19,6 +19,7 @@ use futures::future::BoxFuture;
use postgres_ffi::v14::xlog_utils::{IsPartialXLogFileName, IsXLogFileName, XLogFromFileName};
use postgres_ffi::waldecoder::WalStreamDecoder;
use postgres_ffi::{PG_TLI, XLogFileName, XLogSegNo, dispatch_pgversion};
use postgres_versioninfo::{PgMajorVersion, PgVersionId};
use pq_proto::SystemId;
use remote_storage::RemotePath;
use std::sync::Arc;
@@ -92,7 +93,7 @@ pub struct PhysicalStorage {
/// Size of WAL segment in bytes.
wal_seg_size: usize,
pg_version: u32,
pg_version: PgVersionId,
system_id: u64,
/// Written to disk, but possibly still in the cache and not fully persisted.
@@ -180,7 +181,7 @@ impl PhysicalStorage {
let write_lsn = if state.commit_lsn == Lsn(0) {
Lsn(0)
} else {
let version = state.server.pg_version / 10000;
let version = PgMajorVersion::try_from(state.server.pg_version).unwrap();
dispatch_pgversion!(
version,
@@ -226,7 +227,10 @@ impl PhysicalStorage {
write_record_lsn: write_lsn,
flush_lsn,
flush_record_lsn: flush_lsn,
decoder: WalStreamDecoder::new(write_lsn, state.server.pg_version / 10000),
decoder: WalStreamDecoder::new(
write_lsn,
PgMajorVersion::try_from(state.server.pg_version).unwrap(),
),
file: None,
pending_wal_truncation: true,
})
@@ -408,7 +412,7 @@ impl Storage for PhysicalStorage {
let segno = init_lsn.segment_number(self.wal_seg_size);
let (mut file, _) = self.open_or_create(segno).await?;
let major_pg_version = self.pg_version / 10000;
let major_pg_version = PgMajorVersion::try_from(self.pg_version).unwrap();
let wal_seg =
postgres_ffi::generate_wal_segment(segno, self.system_id, major_pg_version, init_lsn)?;
file.seek(SeekFrom::Start(0)).await?;
@@ -654,7 +658,7 @@ pub struct WalReader {
// pos is in the same segment as timeline_start_lsn.
timeline_start_lsn: Lsn,
// integer version number of PostgreSQL, e.g. 14; 15; 16
pg_version: u32,
pg_version: PgMajorVersion,
system_id: SystemId,
timeline_start_segment: Option<Bytes>,
}
@@ -697,7 +701,7 @@ impl WalReader {
wal_backup,
local_start_lsn: state.local_start_lsn,
timeline_start_lsn: state.timeline_start_lsn,
pg_version: state.server.pg_version / 10000,
pg_version: PgMajorVersion::try_from(state.server.pg_version).unwrap(),
system_id: state.server.system_id,
timeline_start_segment: None,
})
@@ -841,7 +845,7 @@ pub(crate) async fn open_wal_file(
// If that failed, try it without the .partial extension.
let pf = tokio::fs::File::open(&wal_file_path)
.await
.with_context(|| format!("failed to open WAL file {:#}", wal_file_path))
.with_context(|| format!("failed to open WAL file {wal_file_path:#}"))
.map_err(|e| {
warn!("{}", e);
e

2
safekeeper/tests/walproposer_sim/log.rs
View File

@@ -33,7 +33,7 @@ impl FormatTime for SimClock {
if let Some(clock) = clock.as_ref() {
let now = clock.now();
write!(w, "[{}]", now)
write!(w, "[{now}]")
} else {
write!(w, "[?]")
}

2
safekeeper/tests/walproposer_sim/safekeeper.rs
View File

@@ -257,7 +257,7 @@ pub fn run_server(os: NodeOs, disk: Arc<SafekeeperDisk>) -> Result<()> {
let estr = e.to_string();
if !estr.contains("finished processing START_REPLICATION") {
warn!("conn {:?} error: {:?}", connection_id, e);
panic!("unexpected error at safekeeper: {:#}", e);
panic!("unexpected error at safekeeper: {e:#}");
}
conns.remove(&connection_id);
break;

10
safekeeper/tests/walproposer_sim/safekeeper_disk.rs
View File

@@ -7,8 +7,8 @@ use anyhow::Result;
use bytes::{Buf, BytesMut};
use futures::future::BoxFuture;
use parking_lot::Mutex;
use postgres_ffi::XLogSegNo;
use postgres_ffi::waldecoder::WalStreamDecoder;
use postgres_ffi::{PgMajorVersion, XLogSegNo};
use safekeeper::metrics::WalStorageMetrics;
use safekeeper::state::TimelinePersistentState;
use safekeeper::{control_file, wal_storage};
@@ -142,7 +142,7 @@ impl DiskWALStorage {
write_lsn,
write_record_lsn: flush_lsn,
flush_record_lsn: flush_lsn,
decoder: WalStreamDecoder::new(flush_lsn, 16),
decoder: WalStreamDecoder::new(flush_lsn, PgMajorVersion::PG16),
unflushed_bytes: BytesMut::new(),
disk,
})
@@ -151,7 +151,7 @@ impl DiskWALStorage {
fn find_end_of_wal(disk: Arc<TimelineDisk>, start_lsn: Lsn) -> Result<Lsn> {
let mut buf = [0; 8192];
let mut pos = start_lsn.0;
let mut decoder = WalStreamDecoder::new(start_lsn, 16);
let mut decoder = WalStreamDecoder::new(start_lsn, PgMajorVersion::PG16);
let mut result = start_lsn;
loop {
disk.wal.lock().read(pos, &mut buf);
@@ -204,7 +204,7 @@ impl wal_storage::Storage for DiskWALStorage {
self.decoder.available(),
startpos,
);
self.decoder = WalStreamDecoder::new(startpos, 16);
self.decoder = WalStreamDecoder::new(startpos, PgMajorVersion::PG16);
}
self.decoder.feed_bytes(buf);
loop {
@@ -242,7 +242,7 @@ impl wal_storage::Storage for DiskWALStorage {
self.write_record_lsn = end_pos;
self.flush_record_lsn = end_pos;
self.unflushed_bytes.clear();
self.decoder = WalStreamDecoder::new(end_pos, 16);
self.decoder = WalStreamDecoder::new(end_pos, PgMajorVersion::PG16);
Ok(())
}

2
safekeeper/tests/walproposer_sim/simulation.rs
View File

@@ -217,7 +217,7 @@ impl TestConfig {
];
let server_ids = [servers[0].id, servers[1].id, servers[2].id];
let safekeepers_addrs = server_ids.map(|id| format!("node:{}", id)).to_vec();
let safekeepers_addrs = server_ids.map(|id| format!("node:{id}")).to_vec();
let ttid = TenantTimelineId::generate();

9
safekeeper/tests/walproposer_sim/walproposer_api.rs
View File

@@ -499,7 +499,7 @@ impl ApiImpl for SimulationApi {
true
}
fn finish_sync_safekeepers(&self, lsn: u64) {
fn finish_sync_safekeepers(&self, lsn: u64) -> ! {
debug!("finish_sync_safekeepers, lsn={}", lsn);
executor::exit(0, Lsn(lsn).to_string());
}
@@ -523,7 +523,7 @@ impl ApiImpl for SimulationApi {
// Voting bug when safekeeper disconnects after voting
executor::exit(1, msg.to_owned());
}
panic!("unknown FATAL error from walproposer: {}", msg);
panic!("unknown FATAL error from walproposer: {msg}");
}
}
@@ -544,10 +544,7 @@ impl ApiImpl for SimulationApi {
}
}
let msg = format!(
"prop_elected;{};{};{};{}",
prop_lsn, prop_term, prev_lsn, prev_term
);
let msg = format!("prop_elected;{prop_lsn};{prop_term};{prev_lsn};{prev_term}");
debug!(msg);
self.os.log_event(msg);