From bd9f4794d9e36be920f6172001e920e9b43e55fe Mon Sep 17 00:00:00 2001 From: Max Sharnoff Date: Fri, 24 Sep 2021 10:06:03 -0700 Subject: [PATCH] Allow `LeSer`/`BeSer` impls missing either `Serialize` or `Deserialize` (#655) * Allow LeSer/BeSer impls missing Serialize/Deserialize Currently, using `LeSer` or `BeSer` requires that the type implements both `Serialize` and `DeserializeOwned`, even if we're only using the trait for one of those functionalities. Moving the bounds to the methods gives the convenience of the traits without requiring unnecessary derives. * Remove unused #[derive(Serialize/Deserialize)] This should hopefully reduce compile times - if only by a little bit. Some of these were already unused (we weren't using LeSer/BeSer for the types), but most are have *become* unused with the change to LeSer/BeSer. --- pageserver/src/walredo.rs | 4 +- proxy/src/mgmt.rs | 6 +-- walkeeper/src/safekeeper.rs | 14 +++--- zenith_utils/src/bin_ser.rs | 81 +++++++++++++++++++++++------- zenith_utils/tests/bin_ser_test.rs | 4 +- 5 files changed, 76 insertions(+), 33 deletions(-) diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index fbb1984859..d1c2b9ae5a 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -22,7 +22,7 @@ use byteorder::{ByteOrder, LittleEndian}; use bytes::{Buf, BufMut, Bytes, BytesMut}; use lazy_static::lazy_static; use log::*; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use std::fs; use std::fs::OpenOptions; use std::io::prelude::*; @@ -59,7 +59,7 @@ use postgres_ffi::XLogRecord; /// In Postgres `BufferTag` structure is used for exactly the same purpose. /// [See more related comments here](https://github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/buf_internals.h#L91). /// -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize)] pub struct BufferTag { pub rel: RelTag, pub blknum: u32, diff --git a/proxy/src/mgmt.rs b/proxy/src/mgmt.rs index 0d3c114421..90fccd770d 100644 --- a/proxy/src/mgmt.rs +++ b/proxy/src/mgmt.rs @@ -5,7 +5,7 @@ use std::{ use anyhow::bail; use bytes::Bytes; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use zenith_utils::{ postgres_backend::{self, query_from_cstring, AuthType, PostgresBackend}, pq_proto::{BeMessage, SINGLE_COL_ROWDESC}, @@ -60,13 +60,13 @@ struct MgmtHandler { // "Failure": "oops" // } // } -#[derive(Serialize, Deserialize)] +#[derive(Deserialize)] pub struct PsqlSessionResponse { session_id: String, result: PsqlSessionResult, } -#[derive(Serialize, Deserialize)] +#[derive(Deserialize)] pub enum PsqlSessionResult { Success(DatabaseInfo), Failure(String), diff --git a/walkeeper/src/safekeeper.rs b/walkeeper/src/safekeeper.rs index bffc329278..0b5c13bb84 100644 --- a/walkeeper/src/safekeeper.rs +++ b/walkeeper/src/safekeeper.rs @@ -111,7 +111,7 @@ impl Default for SafeKeeperState { // protocol messages /// Initial Proposer -> Acceptor message -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] pub struct ProposerGreeting { /// proposer-acceptor protocol version pub protocol_version: u32, @@ -128,19 +128,19 @@ pub struct ProposerGreeting { /// Acceptor -> Proposer initial response: the highest term known to me /// (acceptor voted for). -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize)] pub struct AcceptorGreeting { term: u64, } /// Vote request sent from proposer to safekeepers -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] pub struct VoteRequest { term: Term, } /// Vote itself, sent from safekeeper to proposer -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize)] pub struct VoteResponse { term: Term, // not really needed, just a sanity check vote_given: u64, // fixme u64 due to padding @@ -152,12 +152,12 @@ pub struct VoteResponse { /// Request with WAL message sent from proposer to safekeeper. Along the way it /// announces 1) successful election (with epoch_start_lsn); 2) commit_lsn. -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug)] pub struct AppendRequest { pub h: AppendRequestHeader, pub wal_data: Bytes, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct AppendRequestHeader { pub term: Term, // LSN since the proposer appends WAL; determines epoch switch point. @@ -175,7 +175,7 @@ pub struct AppendRequestHeader { } /// Report safekeeper state to proposer -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Serialize)] pub struct AppendResponse { // Current term of the safekeeper; if it is higher than proposer's, the // compute is out of date. diff --git a/zenith_utils/src/bin_ser.rs b/zenith_utils/src/bin_ser.rs index 0b78265dfc..063d69557d 100644 --- a/zenith_utils/src/bin_ser.rs +++ b/zenith_utils/src/bin_ser.rs @@ -94,9 +94,12 @@ pub fn le_coder() -> impl Options { /// Binary serialize/deserialize helper functions (Big Endian) /// -pub trait BeSer: Serialize + DeserializeOwned { +pub trait BeSer { /// Serialize into a byte slice - fn ser_into_slice(&self, mut b: &mut [u8]) -> Result<(), SerializeError> { + fn ser_into_slice(&self, mut b: &mut [u8]) -> Result<(), SerializeError> + where + Self: Serialize, + { // &mut [u8] implements Write, but `ser_into` needs a mutable // reference to that. So we need the slightly awkward "mutable // reference to a mutable reference. @@ -107,19 +110,28 @@ pub trait BeSer: Serialize + DeserializeOwned { /// /// This is useful for most `Write` types except `&mut [u8]`, which /// can more easily use [`ser_into_slice`](Self::ser_into_slice). - fn ser_into(&self, w: &mut W) -> Result<(), SerializeError> { + fn ser_into(&self, w: &mut W) -> Result<(), SerializeError> + where + Self: Serialize, + { be_coder().serialize_into(w, &self).map_err(|e| e.into()) } /// Serialize into a new heap-allocated buffer - fn ser(&self) -> Result, SerializeError> { + fn ser(&self) -> Result, SerializeError> + where + Self: Serialize, + { be_coder().serialize(&self).map_err(|e| e.into()) } /// Deserialize from the full contents of a byte slice /// /// See also: [`BeSer::des_prefix`] - fn des(buf: &[u8]) -> Result { + fn des(buf: &[u8]) -> Result + where + Self: DeserializeOwned, + { be_coder() .deserialize(buf) .or(Err(DeserializeError::BadInput)) @@ -131,7 +143,10 @@ pub trait BeSer: Serialize + DeserializeOwned { /// type, but does not guarantee that the entire slice is used. /// /// See also: [`BeSer::des`] - fn des_prefix(buf: &[u8]) -> Result { + fn des_prefix(buf: &[u8]) -> Result + where + Self: DeserializeOwned, + { be_coder() .allow_trailing_bytes() .deserialize(buf) @@ -139,7 +154,10 @@ pub trait BeSer: Serialize + DeserializeOwned { } /// Deserialize from a reader - fn des_from(r: &mut R) -> Result { + fn des_from(r: &mut R) -> Result + where + Self: DeserializeOwned, + { be_coder().deserialize_from(r).map_err(|e| e.into()) } @@ -147,16 +165,22 @@ pub trait BeSer: Serialize + DeserializeOwned { /// /// Note: it may be faster to serialize to a buffer and then measure the /// buffer length, than to call `serialized_size` and then `ser_into`. - fn serialized_size(&self) -> Result { + fn serialized_size(&self) -> Result + where + Self: Serialize, + { be_coder().serialized_size(self).map_err(|e| e.into()) } } /// Binary serialize/deserialize helper functions (Little Endian) /// -pub trait LeSer: Serialize + DeserializeOwned { +pub trait LeSer { /// Serialize into a byte slice - fn ser_into_slice(&self, mut b: &mut [u8]) -> Result<(), SerializeError> { + fn ser_into_slice(&self, mut b: &mut [u8]) -> Result<(), SerializeError> + where + Self: Serialize, + { // &mut [u8] implements Write, but `ser_into` needs a mutable // reference to that. So we need the slightly awkward "mutable // reference to a mutable reference. @@ -167,19 +191,28 @@ pub trait LeSer: Serialize + DeserializeOwned { /// /// This is useful for most `Write` types except `&mut [u8]`, which /// can more easily use [`ser_into_slice`](Self::ser_into_slice). - fn ser_into(&self, w: &mut W) -> Result<(), SerializeError> { + fn ser_into(&self, w: &mut W) -> Result<(), SerializeError> + where + Self: Serialize, + { le_coder().serialize_into(w, &self).map_err(|e| e.into()) } /// Serialize into a new heap-allocated buffer - fn ser(&self) -> Result, SerializeError> { + fn ser(&self) -> Result, SerializeError> + where + Self: Serialize, + { le_coder().serialize(&self).map_err(|e| e.into()) } /// Deserialize from the full contents of a byte slice /// /// See also: [`LeSer::des_prefix`] - fn des(buf: &[u8]) -> Result { + fn des(buf: &[u8]) -> Result + where + Self: DeserializeOwned, + { le_coder() .deserialize(buf) .or(Err(DeserializeError::BadInput)) @@ -191,7 +224,10 @@ pub trait LeSer: Serialize + DeserializeOwned { /// type, but does not guarantee that the entire slice is used. /// /// See also: [`LeSer::des`] - fn des_prefix(buf: &[u8]) -> Result { + fn des_prefix(buf: &[u8]) -> Result + where + Self: DeserializeOwned, + { le_coder() .allow_trailing_bytes() .deserialize(buf) @@ -199,7 +235,10 @@ pub trait LeSer: Serialize + DeserializeOwned { } /// Deserialize from a reader - fn des_from(r: &mut R) -> Result { + fn des_from(r: &mut R) -> Result + where + Self: DeserializeOwned, + { le_coder().deserialize_from(r).map_err(|e| e.into()) } @@ -207,14 +246,18 @@ pub trait LeSer: Serialize + DeserializeOwned { /// /// Note: it may be faster to serialize to a buffer and then measure the /// buffer length, than to call `serialized_size` and then `ser_into`. - fn serialized_size(&self) -> Result { + fn serialized_size(&self) -> Result + where + Self: Serialize, + { le_coder().serialized_size(self).map_err(|e| e.into()) } } -impl BeSer for T where T: Serialize + DeserializeOwned {} - -impl LeSer for T where T: Serialize + DeserializeOwned {} +// Because usage of `BeSer` or `LeSer` can be done with *either* a Serialize or +// DeserializeOwned implementation, the blanket implementation has to be for every type. +impl BeSer for T {} +impl LeSer for T {} #[cfg(test)] mod tests { diff --git a/zenith_utils/tests/bin_ser_test.rs b/zenith_utils/tests/bin_ser_test.rs index d920b430f3..ada43a1189 100644 --- a/zenith_utils/tests/bin_ser_test.rs +++ b/zenith_utils/tests/bin_ser_test.rs @@ -1,10 +1,10 @@ use bytes::{Buf, BytesMut}; use hex_literal::hex; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use std::io::Read; use zenith_utils::bin_ser::LeSer; -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Deserialize)] pub struct HeaderData { magic: u16, info: u16,