rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-05-23 16:10:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cargo fmt pass over a codebase

Browse Source
This commit is contained in:
Stas Kelvich
2021-04-06 14:42:13 +03:00
parent 494b95886b
commit c0fcbbbe0c
23 changed files with 2501 additions and 2260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
walkeeper/src/bin/wal_acceptor.rs
View File

@@ -1,20 +1,20 @@
//
// Main entry point for the wal_acceptor executable
//
use daemonize::Daemonize;
use log::*;
use std::{fs::File, fs::OpenOptions};
use std::io;
use std::path::Path;
use std::path::PathBuf;
use std::thread;
use daemonize::Daemonize;
use std::path::Path;
use std::{fs::File, fs::OpenOptions};
use clap::{App, Arg};
use slog;
use slog_stdlog;
use slog_scope;
use slog::Drain;
use slog_scope;
use slog_stdlog;
use walkeeper::wal_service;
use walkeeper::WalAcceptorConf;
@@ -22,33 +22,41 @@ use walkeeper::WalAcceptorConf;
fn main() -> Result<(), io::Error> {
let arg_matches = App::new("Zenith wal_acceptor")
.about("Store WAL stream to local file system and push it to WAL receivers")
.arg(Arg::with_name("datadir")
.short("D")
.long("dir")
.takes_value(true)
.help("Path to the page server data directory"))
.arg(Arg::with_name("listen")
.short("l")
.long("listen")
.takes_value(true)
.help("listen for incoming page requests on ip:port (default: 127.0.0.1:5430)"))
.arg(Arg::with_name("daemonize")
.short("d")
.long("daemonize")
.takes_value(false)
.help("Run in the background"))
.arg(Arg::with_name("no-sync")
.short("n")
.long("no-sync")
.takes_value(false)
.help("Do not wait for changes to be written safely to disk"))
.arg(
Arg::with_name("datadir")
.short("D")
.long("dir")
.takes_value(true)
.help("Path to the page server data directory"),
)
.arg(
Arg::with_name("listen")
.short("l")
.long("listen")
.takes_value(true)
.help("listen for incoming page requests on ip:port (default: 127.0.0.1:5430)"),
)
.arg(
Arg::with_name("daemonize")
.short("d")
.long("daemonize")
.takes_value(false)
.help("Run in the background"),
)
.arg(
Arg::with_name("no-sync")
.short("n")
.long("no-sync")
.takes_value(false)
.help("Do not wait for changes to be written safely to disk"),
)
.get_matches();
let mut conf = WalAcceptorConf {
let mut conf = WalAcceptorConf {
data_dir: PathBuf::from("./"),
daemonize: false,
no_sync: false,
listen_addr: "127.0.0.1:5454".parse().unwrap()
listen_addr: "127.0.0.1:5454".parse().unwrap(),
};
if let Some(dir) = arg_matches.value_of("datadir") {
@@ -67,7 +75,7 @@ fn main() -> Result<(), io::Error> {
conf.listen_addr = addr.parse().unwrap();
}
start_wal_acceptor(conf)
start_wal_acceptor(conf)
}
fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<(), io::Error> {
@@ -82,8 +90,16 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<(), io::Error> {
// There should'n be any logging to stdin/stdout. Redirect it to the main log so
// that we will see any accidental manual fpritf's or backtraces.
let stdout = OpenOptions::new().create(true).append(true).open(conf.data_dir.join("wal_acceptor.log")).unwrap();
let stderr = OpenOptions::new().create(true).append(true).open(conf.data_dir.join("wal_acceptor.log")).unwrap();
let stdout = OpenOptions::new()
.create(true)
.append(true)
.open(conf.data_dir.join("wal_acceptor.log"))
.unwrap();
let stderr = OpenOptions::new()
.create(true)
.append(true)
.open(conf.data_dir.join("wal_acceptor.log"))
.unwrap();
let daemonize = Daemonize::new()
.pid_file(conf.data_dir.join("wal_acceptor.pid"))
@@ -97,8 +113,8 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<(), io::Error> {
}
}
let mut threads = Vec::new();
let wal_acceptor_thread = thread::Builder::new()
let mut threads = Vec::new();
let wal_acceptor_thread = thread::Builder::new()
.name("WAL acceptor thread".into())
.spawn(|| {
// thread code
@@ -114,7 +130,7 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<(), io::Error> {
}
fn init_logging(conf: &WalAcceptorConf) -> slog_scope::GlobalLoggerGuard {
if conf.daemonize {
if conf.daemonize {
let log = conf.data_dir.join("wal_acceptor.log");
let log_file = File::create(log).unwrap_or_else(|_| panic!("Could not create log file"));
let decorator = slog_term::PlainSyncDecorator::new(log_file);
@@ -122,11 +138,11 @@ fn init_logging(conf: &WalAcceptorConf) -> slog_scope::GlobalLoggerGuard {
let drain = std::sync::Mutex::new(drain).fuse();
let logger = slog::Logger::root(drain, slog::o!());
slog_scope::set_global_logger(logger)
} else {
let decorator = slog_term::TermDecorator::new().build();
let drain = slog_term::FullFormat::new(decorator).build().fuse();
let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
let logger = slog::Logger::root(drain, slog::o!());
return slog_scope::set_global_logger(logger);
}
} else {
let decorator = slog_term::TermDecorator::new().build();
let drain = slog_term::FullFormat::new(decorator).build().fuse();
let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse();
let logger = slog::Logger::root(drain, slog::o!());
return slog_scope::set_global_logger(logger);
}
}

2
walkeeper/src/lib.rs
View File

@@ -2,9 +2,9 @@
use std::net::SocketAddr;
use std::path::PathBuf;
mod pq_protocol;
pub mod wal_service;
pub mod xlog_utils;
mod pq_protocol;
#[allow(dead_code)]
#[derive(Debug, Clone)]

117
walkeeper/src/pq_protocol.rs
View File

@@ -1,6 +1,6 @@
use std::io;
use bytes::{Buf, Bytes, BytesMut, BufMut};
use byteorder::{BigEndian, ByteOrder};
use bytes::{Buf, BufMut, Bytes, BytesMut};
use std::io;
pub type Oid = u32;
pub type Result<T> = std::result::Result<T, io::Error>;
@@ -10,14 +10,14 @@ pub enum FeMessage {
StartupMessage(FeStartupMessage),
Query(FeQueryMessage),
Terminate,
CopyData(FeCopyData)
CopyData(FeCopyData),
}
#[derive(Debug)]
pub struct RowDescriptor {
pub typoid : Oid,
pub typlen : i16,
pub name: &'static [u8],
pub typoid: Oid,
pub typlen: i16,
pub name: &'static [u8],
}
#[derive(Debug)]
@@ -27,8 +27,8 @@ pub enum BeMessage<'a> {
RowDescription(&'a [RowDescriptor]),
DataRow(&'a [Option<&'a [u8]>]),
CommandComplete(&'a [u8]),
Negotiate,
Copy
Negotiate,
Copy,
}
#[derive(Debug)]
@@ -42,15 +42,15 @@ pub enum StartupRequestCode {
Cancel,
NegotiateSsl,
NegotiateGss,
Normal
Normal,
}
impl FeStartupMessage {
pub fn parse(buf: &mut BytesMut) -> Result<Option<FeMessage>> {
const MAX_STARTUP_PACKET_LENGTH: usize = 10000;
const CANCEL_REQUEST_CODE: u32 = (1234 << 16) | 5678;
const NEGOTIATE_SSL_CODE: u32 = (1234 << 16) | 5679;
const NEGOTIATE_GSS_CODE: u32 = (1234 << 16) | 5680;
const NEGOTIATE_SSL_CODE: u32 = (1234 << 16) | 5679;
const NEGOTIATE_GSS_CODE: u32 = (1234 << 16) | 5680;
if buf.len() < 4 {
return Ok(None);
@@ -73,26 +73,29 @@ impl FeStartupMessage {
CANCEL_REQUEST_CODE => StartupRequestCode::Cancel,
NEGOTIATE_SSL_CODE => StartupRequestCode::NegotiateSsl,
NEGOTIATE_GSS_CODE => StartupRequestCode::NegotiateGss,
_ => StartupRequestCode::Normal
_ => StartupRequestCode::Normal,
};
buf.advance(len as usize);
Ok(Some(FeMessage::StartupMessage(FeStartupMessage{version, kind})))
Ok(Some(FeMessage::StartupMessage(FeStartupMessage {
version,
kind,
})))
}
}
#[derive(Debug)]
pub struct FeQueryMessage {
pub body: Bytes
pub body: Bytes,
}
#[derive(Debug)]
pub struct FeCopyData {
pub body: Bytes
pub body: Bytes,
}
impl<'a> BeMessage<'a> {
pub fn write(buf : &mut BytesMut, message: &BeMessage) {
pub fn write(buf: &mut BytesMut, message: &BeMessage) {
match message {
BeMessage::AuthenticationOk => {
buf.put_u8(b'R');
@@ -106,47 +109,51 @@ impl<'a> BeMessage<'a> {
buf.put_u8(b'I');
}
BeMessage::Negotiate => {
buf.put_u8(b'N');
}
BeMessage::Negotiate => {
buf.put_u8(b'N');
}
BeMessage::Copy => {
buf.put_u8(b'W');
buf.put_i32(7);
buf.put_u8(b'\0');
buf.put_u8(b'\0');
buf.put_u8(b'\0');
buf.put_u8(b'W');
buf.put_i32(7);
buf.put_u8(b'\0');
buf.put_u8(b'\0');
buf.put_u8(b'\0');
}
BeMessage::RowDescription(rows) => {
buf.put_u8(b'T');
let total_len:u32 = rows.iter().fold(0, |acc,row| acc + row.name.len() as u32 + 3*(4 + 2));
buf.put_u32(4 + 2 + total_len);
for row in rows.iter() {
buf.put_i16(row.name.len() as i16);
buf.put_slice(row.name);
buf.put_i32(0); /* table oid */
buf.put_i16(0); /* attnum */
buf.put_u32(row.typoid);
buf.put_i16(row.typlen);
buf.put_i32(-1); /* typmod */
buf.put_i16(0); /* format code */
}
let total_len: u32 = rows
.iter()
.fold(0, |acc, row| acc + row.name.len() as u32 + 3 * (4 + 2));
buf.put_u32(4 + 2 + total_len);
for row in rows.iter() {
buf.put_i16(row.name.len() as i16);
buf.put_slice(row.name);
buf.put_i32(0); /* table oid */
buf.put_i16(0); /* attnum */
buf.put_u32(row.typoid);
buf.put_i16(row.typlen);
buf.put_i32(-1); /* typmod */
buf.put_i16(0); /* format code */
}
}
BeMessage::DataRow(vals) => {
buf.put_u8(b'D');
let total_len:usize = vals.iter().fold(0, |acc, row| acc + 4 + row.map_or(0, |s| s.len()));
let total_len: usize = vals
.iter()
.fold(0, |acc, row| acc + 4 + row.map_or(0, |s| s.len()));
buf.put_u32(4 + 2 + total_len as u32);
buf.put_u16(vals.len() as u16);
for val_opt in vals.iter() {
if let Some(val) = val_opt {
buf.put_u32(val.len() as u32);
buf.put_slice(val);
} else {
buf.put_i32(-1);
}
}
for val_opt in vals.iter() {
if let Some(val) = val_opt {
buf.put_u32(val.len() as u32);
buf.put_slice(val);
} else {
buf.put_i32(-1);
}
}
}
BeMessage::CommandComplete(cmd) => {
@@ -155,7 +162,7 @@ impl<'a> BeMessage<'a> {
buf.put_slice(cmd);
}
}
}
}
}
impl FeMessage {
@@ -187,15 +194,17 @@ impl FeMessage {
body.advance(5);
match tag {
b'Q' => Ok(Some(FeMessage::Query(FeQueryMessage{body:body.freeze()}))),
b'd' => Ok(Some(FeMessage::CopyData(FeCopyData{body:body.freeze()}))),
b'Q' => Ok(Some(FeMessage::Query(FeQueryMessage {
body: body.freeze(),
}))),
b'd' => Ok(Some(FeMessage::CopyData(FeCopyData {
body: body.freeze(),
}))),
b'X' => Ok(Some(FeMessage::Terminate)),
tag => {
Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("unknown message tag: {},'{:?}'", tag, buf),
))
}
tag => Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("unknown message tag: {},'{:?}'", tag, buf),
)),
}
}
}

1558
walkeeper/src/wal_service.rs
View File

File diff suppressed because it is too large Load Diff

371
walkeeper/src/xlog_utils.rs
View File

@@ -1,229 +1,254 @@
use std::fs::{self,File};
use std::time::SystemTime;
use std::path::PathBuf;
use std::cmp::min;
use std::io::prelude::*;
use byteorder::{LittleEndian, ByteOrder};
use byteorder::{ByteOrder, LittleEndian};
use crc32c::*;
use log::*;
use std::cmp::min;
use std::fs::{self, File};
use std::io::prelude::*;
use std::path::PathBuf;
use std::time::SystemTime;
pub const XLOG_FNAME_LEN : usize = 24;
pub const XLOG_BLCKSZ : usize = 8192;
pub const XLP_FIRST_IS_CONTRECORD : u16 = 0x0001;
pub const XLOG_PAGE_MAGIC : u16 = 0xD109;
pub const XLP_REM_LEN_OFFS : usize = 2+2+4+8;
pub const XLOG_SIZE_OF_XLOG_SHORT_PHD : usize = XLP_REM_LEN_OFFS + 4 + 4;
pub const XLOG_SIZE_OF_XLOG_LONG_PHD : usize = XLOG_SIZE_OF_XLOG_SHORT_PHD + 8 + 4 + 4;
pub const XLOG_RECORD_CRC_OFFS : usize = 4+4+8+1+1+2;
pub const XLOG_SIZE_OF_XLOG_RECORD : usize = XLOG_RECORD_CRC_OFFS+4;
pub const XLOG_FNAME_LEN: usize = 24;
pub const XLOG_BLCKSZ: usize = 8192;
pub const XLP_FIRST_IS_CONTRECORD: u16 = 0x0001;
pub const XLOG_PAGE_MAGIC: u16 = 0xD109;
pub const XLP_REM_LEN_OFFS: usize = 2 + 2 + 4 + 8;
pub const XLOG_SIZE_OF_XLOG_SHORT_PHD: usize = XLP_REM_LEN_OFFS + 4 + 4;
pub const XLOG_SIZE_OF_XLOG_LONG_PHD: usize = XLOG_SIZE_OF_XLOG_SHORT_PHD + 8 + 4 + 4;
pub const XLOG_RECORD_CRC_OFFS: usize = 4 + 4 + 8 + 1 + 1 + 2;
pub const XLOG_SIZE_OF_XLOG_RECORD: usize = XLOG_RECORD_CRC_OFFS + 4;
pub type XLogRecPtr = u64;
pub type TimeLineID = u32;
pub type TimestampTz = u64;
pub type XLogSegNo = u64;
#[allow(non_snake_case)]
pub fn XLogSegmentOffset(xlogptr : XLogRecPtr, wal_segsz_bytes : usize) -> u32 {
return (xlogptr as u32) & (wal_segsz_bytes as u32 - 1);
pub fn XLogSegmentOffset(xlogptr: XLogRecPtr, wal_segsz_bytes: usize) -> u32 {
return (xlogptr as u32) & (wal_segsz_bytes as u32 - 1);
}
#[allow(non_snake_case)]
pub fn XLogSegmentsPerXLogId(wal_segsz_bytes : usize) -> XLogSegNo {
return (0x100000000u64 / wal_segsz_bytes as u64) as XLogSegNo;
pub fn XLogSegmentsPerXLogId(wal_segsz_bytes: usize) -> XLogSegNo {
return (0x100000000u64 / wal_segsz_bytes as u64) as XLogSegNo;
}
#[allow(non_snake_case)]
pub fn XLByteToSeg(xlogptr : XLogRecPtr, wal_segsz_bytes : usize) -> XLogSegNo {
return xlogptr / wal_segsz_bytes as u64;
pub fn XLByteToSeg(xlogptr: XLogRecPtr, wal_segsz_bytes: usize) -> XLogSegNo {
return xlogptr / wal_segsz_bytes as u64;
}
#[allow(non_snake_case)]
pub fn XLogSegNoOffsetToRecPtr(segno: XLogSegNo, offset:u32, wal_segsz_bytes: usize) -> XLogRecPtr {
return segno * (wal_segsz_bytes as u64) + (offset as u64);
pub fn XLogSegNoOffsetToRecPtr(
segno: XLogSegNo,
offset: u32,
wal_segsz_bytes: usize,
) -> XLogRecPtr {
return segno * (wal_segsz_bytes as u64) + (offset as u64);
}
#[allow(non_snake_case)]
pub fn XLogFileName(tli : TimeLineID, logSegNo : XLogSegNo, wal_segsz_bytes : usize) -> String {
return format!("{:>08X}{:>08X}{:>08X}",
tli,
logSegNo / XLogSegmentsPerXLogId(wal_segsz_bytes),
logSegNo % XLogSegmentsPerXLogId(wal_segsz_bytes));
pub fn XLogFileName(tli: TimeLineID, logSegNo: XLogSegNo, wal_segsz_bytes: usize) -> String {
return format!(
"{:>08X}{:>08X}{:>08X}",
tli,
logSegNo / XLogSegmentsPerXLogId(wal_segsz_bytes),
logSegNo % XLogSegmentsPerXLogId(wal_segsz_bytes)
);
}
#[allow(non_snake_case)]
pub fn XLogFromFileName(fname:&str, wal_seg_size: usize) -> (XLogSegNo,TimeLineID) {
let tli = u32::from_str_radix(&fname[0..8], 16).unwrap();
let log = u32::from_str_radix(&fname[8..16], 16).unwrap() as XLogSegNo;
let seg = u32::from_str_radix(&fname[16..24], 16).unwrap() as XLogSegNo;
return (log * XLogSegmentsPerXLogId(wal_seg_size) + seg, tli);
pub fn XLogFromFileName(fname: &str, wal_seg_size: usize) -> (XLogSegNo, TimeLineID) {
let tli = u32::from_str_radix(&fname[0..8], 16).unwrap();
let log = u32::from_str_radix(&fname[8..16], 16).unwrap() as XLogSegNo;
let seg = u32::from_str_radix(&fname[16..24], 16).unwrap() as XLogSegNo;
return (log * XLogSegmentsPerXLogId(wal_seg_size) + seg, tli);
}
#[allow(non_snake_case)]
pub fn IsXLogFileName(fname:&str) -> bool {
return fname.len() == XLOG_FNAME_LEN
&& fname.chars().all(|c| c.is_ascii_hexdigit());
pub fn IsXLogFileName(fname: &str) -> bool {
return fname.len() == XLOG_FNAME_LEN && fname.chars().all(|c| c.is_ascii_hexdigit());
}
#[allow(non_snake_case)]
pub fn IsPartialXLogFileName(fname:&str) -> bool {
return fname.ends_with(".partial")
&& IsXLogFileName(&fname[0..fname.len()-8]);
pub fn IsPartialXLogFileName(fname: &str) -> bool {
return fname.ends_with(".partial") && IsXLogFileName(&fname[0..fname.len() - 8]);
}
pub fn get_current_timestamp() -> TimestampTz
{
const UNIX_EPOCH_JDATE : u64 = 2440588; /* == date2j(1970, 1, 1) */
const POSTGRES_EPOCH_JDATE : u64 = 2451545; /* == date2j(2000, 1, 1) */
const SECS_PER_DAY : u64 = 86400;
const USECS_PER_SEC : u64 = 1000000;
match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) {
Ok(n) => (n.as_secs() - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)) * USECS_PER_SEC + n.subsec_micros() as u64,
Err(_) => panic!("SystemTime before UNIX EPOCH!"),
}
pub fn get_current_timestamp() -> TimestampTz {
const UNIX_EPOCH_JDATE: u64 = 2440588; /* == date2j(1970, 1, 1) */
const POSTGRES_EPOCH_JDATE: u64 = 2451545; /* == date2j(2000, 1, 1) */
const SECS_PER_DAY: u64 = 86400;
const USECS_PER_SEC: u64 = 1000000;
match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) {
Ok(n) => {
(n.as_secs() - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY))
* USECS_PER_SEC
+ n.subsec_micros() as u64
}
Err(_) => panic!("SystemTime before UNIX EPOCH!"),
}
}
fn find_end_of_wal_segment(data_dir: &PathBuf, segno: XLogSegNo, tli: TimeLineID, wal_seg_size: usize) -> u32 {
let mut offs : usize = 0;
let mut contlen : usize = 0;
let mut wal_crc : u32 = 0;
let mut crc : u32 = 0;
let mut rec_offs : usize = 0;
let mut buf = [0u8;XLOG_BLCKSZ];
let file_name = XLogFileName(tli, segno, wal_seg_size);
let mut last_valid_rec_pos : usize = 0;
let mut file = File::open(data_dir.join(file_name.clone() + ".partial")).unwrap();
let mut rec_hdr = [0u8; XLOG_RECORD_CRC_OFFS];
fn find_end_of_wal_segment(
data_dir: &PathBuf,
segno: XLogSegNo,
tli: TimeLineID,
wal_seg_size: usize,
) -> u32 {
let mut offs: usize = 0;
let mut contlen: usize = 0;
let mut wal_crc: u32 = 0;
let mut crc: u32 = 0;
let mut rec_offs: usize = 0;
let mut buf = [0u8; XLOG_BLCKSZ];
let file_name = XLogFileName(tli, segno, wal_seg_size);
let mut last_valid_rec_pos: usize = 0;
let mut file = File::open(data_dir.join(file_name.clone() + ".partial")).unwrap();
let mut rec_hdr = [0u8; XLOG_RECORD_CRC_OFFS];
while offs < wal_seg_size {
if offs % XLOG_BLCKSZ == 0 {
if let Ok(bytes_read) = file.read(&mut buf) {
if bytes_read != buf.len() {
break;
}
} else {
break;
}
let xlp_magic = LittleEndian::read_u16(&buf[0..2]);
let xlp_info = LittleEndian::read_u16(&buf[2..4]);
let xlp_rem_len = LittleEndian::read_u32(&buf[XLP_REM_LEN_OFFS..XLP_REM_LEN_OFFS+4]);
if xlp_magic != XLOG_PAGE_MAGIC {
info!("Invalid WAL file {}.partial magic {}", file_name, xlp_magic);
break;
}
if offs == 0 {
offs = XLOG_SIZE_OF_XLOG_LONG_PHD;
if (xlp_info & XLP_FIRST_IS_CONTRECORD) != 0 {
offs += ((xlp_rem_len + 7) & !7) as usize;
}
} else {
offs += XLOG_SIZE_OF_XLOG_SHORT_PHD;
}
while offs < wal_seg_size {
if offs % XLOG_BLCKSZ == 0 {
if let Ok(bytes_read) = file.read(&mut buf) {
if bytes_read != buf.len() {
break;
}
} else {
break;
}
let xlp_magic = LittleEndian::read_u16(&buf[0..2]);
let xlp_info = LittleEndian::read_u16(&buf[2..4]);
let xlp_rem_len = LittleEndian::read_u32(&buf[XLP_REM_LEN_OFFS..XLP_REM_LEN_OFFS + 4]);
if xlp_magic != XLOG_PAGE_MAGIC {
info!("Invalid WAL file {}.partial magic {}", file_name, xlp_magic);
break;
}
if offs == 0 {
offs = XLOG_SIZE_OF_XLOG_LONG_PHD;
if (xlp_info & XLP_FIRST_IS_CONTRECORD) != 0 {
offs += ((xlp_rem_len + 7) & !7) as usize;
}
} else {
offs += XLOG_SIZE_OF_XLOG_SHORT_PHD;
}
} else if contlen == 0 {
let page_offs = offs % XLOG_BLCKSZ;
let xl_tot_len = LittleEndian::read_u32(&buf[page_offs..page_offs+4]) as usize;
if xl_tot_len == 0 {
break;
}
last_valid_rec_pos = offs;
offs += 4;
rec_offs = 4;
let page_offs = offs % XLOG_BLCKSZ;
let xl_tot_len = LittleEndian::read_u32(&buf[page_offs..page_offs + 4]) as usize;
if xl_tot_len == 0 {
break;
}
last_valid_rec_pos = offs;
offs += 4;
rec_offs = 4;
contlen = xl_tot_len - 4;
rec_hdr[0..4].copy_from_slice(&buf[page_offs..page_offs+4]);
rec_hdr[0..4].copy_from_slice(&buf[page_offs..page_offs + 4]);
} else {
let page_offs = offs % XLOG_BLCKSZ;
let page_offs = offs % XLOG_BLCKSZ;
// we're continuing a record, possibly from previous page.
let pageleft = XLOG_BLCKSZ - page_offs;
// read the rest of the record, or as much as fits on this page.
let n = min(contlen, pageleft);
if rec_offs < XLOG_RECORD_CRC_OFFS {
let len = min(XLOG_RECORD_CRC_OFFS-rec_offs, n);
rec_hdr[rec_offs..rec_offs+len].copy_from_slice(&buf[page_offs..page_offs+len]);
}
if rec_offs <= XLOG_RECORD_CRC_OFFS && rec_offs + n >= XLOG_SIZE_OF_XLOG_RECORD {
let crc_offs = page_offs - rec_offs + XLOG_RECORD_CRC_OFFS;
wal_crc = LittleEndian::read_u32(&buf[crc_offs..crc_offs+4]);
crc = crc32c_append(0, &buf[crc_offs+4..page_offs+n]);
crc = !crc;
} else {
crc ^= 0xFFFFFFFFu32;
crc = crc32c_append(crc, &buf[page_offs..page_offs+n]);
crc = !crc;
}
rec_offs += n;
if rec_offs < XLOG_RECORD_CRC_OFFS {
let len = min(XLOG_RECORD_CRC_OFFS - rec_offs, n);
rec_hdr[rec_offs..rec_offs + len].copy_from_slice(&buf[page_offs..page_offs + len]);
}
if rec_offs <= XLOG_RECORD_CRC_OFFS && rec_offs + n >= XLOG_SIZE_OF_XLOG_RECORD {
let crc_offs = page_offs - rec_offs + XLOG_RECORD_CRC_OFFS;
wal_crc = LittleEndian::read_u32(&buf[crc_offs..crc_offs + 4]);
crc = crc32c_append(0, &buf[crc_offs + 4..page_offs + n]);
crc = !crc;
} else {
crc ^= 0xFFFFFFFFu32;
crc = crc32c_append(crc, &buf[page_offs..page_offs + n]);
crc = !crc;
}
rec_offs += n;
offs += n;
contlen -= n;
if contlen == 0 {
crc = !crc;
crc = crc32c_append(crc, &rec_hdr);
offs = (offs + 7) & !7; // pad on 8 bytes boundary */
if crc == wal_crc {
last_valid_rec_pos = offs;
} else {
info!("CRC mismatch {} vs {} at {}", crc, wal_crc, last_valid_rec_pos);
break;
}
crc = !crc;
crc = crc32c_append(crc, &rec_hdr);
offs = (offs + 7) & !7; // pad on 8 bytes boundary */
if crc == wal_crc {
last_valid_rec_pos = offs;
} else {
info!(
"CRC mismatch {} vs {} at {}",
crc, wal_crc, last_valid_rec_pos
);
break;
}
}
}
}
return last_valid_rec_pos as u32;
}
return last_valid_rec_pos as u32;
}
pub fn find_end_of_wal(data_dir: &PathBuf, wal_seg_size:usize, precise:bool) -> (XLogRecPtr,TimeLineID) {
let mut high_segno : XLogSegNo = 0;
let mut high_tli : TimeLineID = 0;
let mut high_ispartial = false;
pub fn find_end_of_wal(
data_dir: &PathBuf,
wal_seg_size: usize,
precise: bool,
) -> (XLogRecPtr, TimeLineID) {
let mut high_segno: XLogSegNo = 0;
let mut high_tli: TimeLineID = 0;
let mut high_ispartial = false;
for entry in fs::read_dir(data_dir).unwrap() {
if let Ok(entry) = entry {
let ispartial : bool;
let entry_name = entry.file_name();
let fname = entry_name.to_str().unwrap();
/*
* Check if the filename looks like an xlog file, or a .partial file.
*/
if IsXLogFileName(fname) {
ispartial = false;
} else if IsPartialXLogFileName(fname) {
ispartial = true;
} else {
continue;
}
let (segno,tli) = XLogFromFileName(fname, wal_seg_size);
if !ispartial && entry.metadata().unwrap().len() != wal_seg_size as u64 {
continue;
}
if segno > high_segno ||
(segno == high_segno && tli > high_tli) ||
(segno == high_segno && tli == high_tli && high_ispartial && !ispartial)
{
high_segno = segno;
high_tli = tli;
high_ispartial = ispartial;
}
}
}
if high_segno > 0 {
let mut high_offs = 0;
/*
* Move the starting pointer to the start of the next segment, if the
* highest one we saw was completed.
*/
if !high_ispartial {
high_segno += 1;
} else if precise { /* otherwise locate last record in last partial segment */
high_offs = find_end_of_wal_segment(data_dir, high_segno, high_tli, wal_seg_size);
}
let high_ptr = XLogSegNoOffsetToRecPtr(high_segno, high_offs, wal_seg_size);
return (high_ptr,high_tli);
}
return (0,0);
for entry in fs::read_dir(data_dir).unwrap() {
if let Ok(entry) = entry {
let ispartial: bool;
let entry_name = entry.file_name();
let fname = entry_name.to_str().unwrap();
/*
* Check if the filename looks like an xlog file, or a .partial file.
*/
if IsXLogFileName(fname) {
ispartial = false;
} else if IsPartialXLogFileName(fname) {
ispartial = true;
} else {
continue;
}
let (segno, tli) = XLogFromFileName(fname, wal_seg_size);
if !ispartial && entry.metadata().unwrap().len() != wal_seg_size as u64 {
continue;
}
if segno > high_segno
|| (segno == high_segno && tli > high_tli)
|| (segno == high_segno && tli == high_tli && high_ispartial && !ispartial)
{
high_segno = segno;
high_tli = tli;
high_ispartial = ispartial;
}
}
}
if high_segno > 0 {
let mut high_offs = 0;
/*
* Move the starting pointer to the start of the next segment, if the
* highest one we saw was completed.
*/
if !high_ispartial {
high_segno += 1;
} else if precise {
/* otherwise locate last record in last partial segment */
high_offs = find_end_of_wal_segment(data_dir, high_segno, high_tli, wal_seg_size);
}
let high_ptr = XLogSegNoOffsetToRecPtr(high_segno, high_offs, wal_seg_size);
return (high_ptr, high_tli);
}
return (0, 0);
}
pub fn main() {
let mut data_dir = PathBuf::new();
data_dir.push(".");
let wal_seg_size = 16*1024*1024;
let (wal_end,tli) = find_end_of_wal(&data_dir, wal_seg_size, true);
println!("wal_end={:>08X}{:>08X}, tli={}", (wal_end >> 32) as u32, wal_end as u32, tli);
let mut data_dir = PathBuf::new();
data_dir.push(".");
let wal_seg_size = 16 * 1024 * 1024;
let (wal_end, tli) = find_end_of_wal(&data_dir, wal_seg_size, true);
println!(
"wal_end={:>08X}{:>08X}, tli={}",
(wal_end >> 32) as u32,
wal_end as u32,
tli
);
}