add missing expected file

Debt for the #144

README.md

@@ -4,14 +4,32 @@ Zenith substitutes PostgreSQL storage layer and redistributes data across a clus
 
 ## Running local installation
 
-1. Build zenith and patched postgres
+1. Install build dependencies and other useful packages
+
+On Ubuntu or Debian this set of packages should be sufficient to build the code:
+```text
+apt install build-essential libtool libreadline-dev zlib1g-dev flex bison \
+libxml2-dev libcurl4-openssl-dev libssl-dev clang
+```
+
+[Rust] 1.48 or later is also required.
+
+To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.
+
+To run the integration tests (not required to use the code), install
+Python (3.6 or higher), and install python3 packages with `pip` (called `pip3` on some systems):
+```
+pip install pytest psycopg2
+```
+
+2. Build zenith and patched postgres
 ```sh
 git clone --recursive https://github.com/libzenith/zenith.git
 cd zenith
 make -j5
 ```
 
-2. Start pageserver and postgres on top of it (should be called from repo root):
+3. Start pageserver and postgres on top of it (should be called from repo root):
 ```sh
 # Create repository in .zenith with proper paths to binaries and data
 # Later that would be responsibility of a package install script
@@ -35,8 +53,8 @@ BRANCH	ADDRESS		LSN		STATUS
 main	127.0.0.1:55432	0/1609610	running
 ```
 
-3. Now it is possible to connect to postgres and run some queries:
-```sh
+4. Now it is possible to connect to postgres and run some queries:
+```text
 > psql -p55432 -h 127.0.0.1 postgres
 postgres=# CREATE TABLE t(key int primary key, value text);
 CREATE TABLE
@@ -49,7 +67,7 @@ postgres=# select * from t;
 (1 row)
 ```
 
-4. And create branches and run postgres on them:
+5. And create branches and run postgres on them:
 ```sh
 # create branch named migration_check
 > ./target/debug/zenith branch migration_check main
@@ -83,33 +101,34 @@ INSERT 0 1
 ```sh
 git clone --recursive https://github.com/libzenith/zenith.git
 make # builds also postgres and installs it to ./tmp_install
+cd test_runner
 pytest
 ```
 
 ## Source tree layout
 
-/walkeeper:
+`/walkeeper`:
 
 WAL safekeeper. Written in Rust.
 
-/pageserver:
+`/pageserver`:
 
 Page Server. Written in Rust.
 
 Depends on the modified 'postgres' binary for WAL redo.
 
-/integration_tests:
-
-Tests with different combinations of a Postgres compute node, WAL safekeeper and Page Server.
-
-/vendor/postgres:
+`/vendor/postgres`:
 
 PostgreSQL source tree, with the modifications needed for Zenith.
 
-/vendor/postgres/src/bin/safekeeper:
+`/vendor/postgres/src/bin/safekeeper`:
 
 Extension (safekeeper_proxy) that runs in the compute node, and connects to the WAL safekeepers
 and streams the WAL
 
+`/test_runner`:
+
+Integration tests, written in Python using the `pytest` framework.
 
 
+[Rust]: https://www.rust-lang.org/learn/get-started

control_plane/src/compute.rs

@@ -252,6 +252,7 @@ impl PostgresNode {
     // new data directory
     pub fn init_from_page_server(&self) -> Result<()> {
         let pgdata = self.pgdata();
+
         println!(
             "Extracting base backup to create postgres instance: path={} port={}",
             pgdata.display(),
@@ -340,9 +341,6 @@ impl PostgresNode {
             ),
         )?;
 
-        fs::create_dir_all(self.pgdata().join("pg_wal"))?;
-        fs::create_dir_all(self.pgdata().join("pg_wal").join("archive_status"))?;
-        self.pg_resetwal(&["-f"])?;
         Ok(())
     }
 
@@ -398,19 +396,6 @@ impl PostgresNode {
         Ok(())
     }
 
-    fn pg_resetwal(&self, args: &[&str]) -> Result<()> {
-        let pg_resetwal_path = self.env.pg_bin_dir().join("pg_resetwal");
-
-        let pg_ctl = Command::new(pg_resetwal_path)
-            .args([&["-D", self.pgdata().to_str().unwrap()], args].concat())
-            .status()
-            .with_context(|| "pg_resetwal failed")?;
-        if !pg_ctl.success() {
-            anyhow::bail!("pg_resetwal failed");
-        }
-        Ok(())
-    }
-
     pub fn start(&self) -> Result<()> {
         println!("Starting postgres node at '{}'", self.connstr());
         self.pg_ctl(&["start"])

control_plane/src/local_env.rs

@@ -49,7 +49,7 @@ impl LocalEnv {
         Ok(self
             .zenith_distrib_dir
             .as_ref()
-            .ok_or(anyhow!("Can not manage remote pageserver"))?
+            .ok_or_else(|| anyhow!("Can not manage remote pageserver"))?
             .join("pageserver"))
     }
 

integration_tests/src/lib.rs

@@ -1,3 +1,6 @@
+use anyhow::{bail, Result};
+use nix::sys::signal::{kill, Signal};
+use nix::unistd::Pid;
 use std::collections::BTreeMap;
 use std::convert::TryInto;
 use std::fs::{self, File, OpenOptions};
@@ -8,11 +11,6 @@ use std::process::{Command, ExitStatus};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 
-use anyhow::{bail, Result};
-use nix::sys::signal::{kill, Signal};
-use nix::unistd::Pid;
-use postgres;
-
 use control_plane::compute::PostgresNode;
 use control_plane::read_pidfile;
 use control_plane::{local_env::LocalEnv, storage::PageServerNode};
@@ -41,11 +39,11 @@ pub fn create_test_env(testname: &str) -> LocalEnv {
     let _ = fs::remove_dir_all(&base_path);
 
     fs::create_dir_all(&base_path)
-        .expect(format!("could not create directory for {}", base_path_str).as_str());
+        .unwrap_or_else(|_| panic!("could not create directory for {}", base_path_str));
 
     let pgdatadirs_path = base_path.join("pgdatadirs");
     fs::create_dir(&pgdatadirs_path)
-        .expect(format!("could not create directory {:?}", pgdatadirs_path).as_str());
+        .unwrap_or_else(|_| panic!("could not create directory {:?}", pgdatadirs_path));
 
     LocalEnv {
         pageserver_connstring: "postgresql://127.0.0.1:64000".to_string(),

integration_tests/tests/test_wal_acceptor.rs

@@ -5,7 +5,6 @@ use std::{thread, time};
 
 use control_plane::compute::{ComputeControlPlane, PostgresNode};
 
-use integration_tests;
 use integration_tests::PostgresNodeExt;
 use integration_tests::TestStorageControlPlane;
 
@@ -14,7 +13,7 @@ const DOWNTIME: u64 = 2;
 fn start_node_with_wal_proposer(
     timeline: &str,
     compute_cplane: &mut ComputeControlPlane,
-    wal_acceptors: &String,
+    wal_acceptors: &str,
 ) -> Arc<PostgresNode> {
     let node = compute_cplane.new_test_master_node(timeline);
     let _node = node.append_conf(
@@ -100,8 +99,7 @@ fn test_many_timelines() {
     let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info();
 
     // Create branches
-    let mut timelines: Vec<String> = Vec::new();
-    timelines.push("main".to_string());
+    let mut timelines: Vec<String> = vec!["main".to_string()];
 
     for i in 1..N_TIMELINES {
         let branchname = format!("experimental{}", i);

pageserver/src/basebackup.rs

@@ -2,189 +2,27 @@ use crate::ZTimelineId;
 use log::*;
 use std::io::Write;
 use std::sync::Arc;
-use std::time::SystemTime;
-use tar::{Builder, Header};
+use tar::Builder;
 use walkdir::WalkDir;
-use bytes::{BufMut, BytesMut};
 
-use crate::repository::{BufferTag, RelTag, Timeline};
+use crate::repository::Timeline;
 use postgres_ffi::relfile_utils::*;
-use postgres_ffi::*;
 use zenith_utils::lsn::Lsn;
 
-fn new_tar_header(path: &str, size: u64) -> anyhow::Result<Header> {
-    let mut header = Header::new_gnu();
-    header.set_size(size);
-    header.set_path(path)?;
-    header.set_mode(0b110000000);
-    header.set_mtime(
-        SystemTime::now()
-            .duration_since(SystemTime::UNIX_EPOCH)
-            .unwrap()
-            .as_secs(),
-    );
-    header.set_cksum();
-    Ok(header)
-}
-
-//
-// Generate SRLU segment files from repository
-//
-fn add_slru_segments(
-    ar: &mut Builder<&mut dyn Write>,
-    timeline: &Arc<dyn Timeline>,
-    path: &str,
-    forknum: u8,
-    lsn: Lsn,
-) -> anyhow::Result<()> {
-    let rel = RelTag {
-        spcnode: 0,
-        dbnode: 0,
-        relnode: 0,
-        forknum,
-    };
-    let (first, last) = timeline.get_range(rel, lsn)?;
-    const SEG_SIZE: usize =
-        pg_constants::BLCKSZ as usize * pg_constants::SLRU_PAGES_PER_SEGMENT as usize;
-    let mut seg_buf = [0u8; SEG_SIZE];
-    let mut curr_segno: Option<u32> = None;
-    for page in first..last {
-        let tag = BufferTag { rel, blknum: page };
-        let img = timeline.get_page_at_lsn(tag, lsn)?;
-        // Zero length image indicates truncated segment: just skip it
-        if img.len() != 0 {
-            assert!(img.len() == pg_constants::BLCKSZ as usize);
-
-            let segno = page / pg_constants::SLRU_PAGES_PER_SEGMENT;
-            if curr_segno.is_some() && curr_segno.unwrap() != segno {
-                let segname = format!("{}/{:>04X}", path, curr_segno.unwrap());
-                let header = new_tar_header(&segname, SEG_SIZE as u64)?;
-                ar.append(&header, &seg_buf[..])?;
-                seg_buf = [0u8; SEG_SIZE];
-            }
-            curr_segno = Some(segno);
-            let offs_start = (page % pg_constants::SLRU_PAGES_PER_SEGMENT) as usize
-                * pg_constants::BLCKSZ as usize;
-            let offs_end = offs_start + pg_constants::BLCKSZ as usize;
-            seg_buf[offs_start..offs_end].copy_from_slice(&img);
-        }
-    }
-    if curr_segno.is_some() {
-        let segname = format!("{}/{:>04X}", path, curr_segno.unwrap());
-        let header = new_tar_header(&segname, SEG_SIZE as u64)?;
-        ar.append(&header, &seg_buf[..])?;
-    }
-    Ok(())
-}
-
-//
-// Extract pg_filenode.map files from repository
-//
-fn add_relmap_files(
-    ar: &mut Builder<&mut dyn Write>,
-    timeline: &Arc<dyn Timeline>,
-    lsn: Lsn,
-    snappath: &str,
-) -> anyhow::Result<()> {
-    for db in timeline.get_databases(lsn)?.iter() {
-        let tag = BufferTag {
-            rel: *db,
-            blknum: 0,
-        };
-        let img = timeline.get_page_at_lsn(tag, lsn)?;
-        let path = if db.spcnode == pg_constants::GLOBALTABLESPACE_OID {
-            String::from("global/pg_filenode.map")
-        } else {
-            // User defined tablespaces are not supported
-            assert!(db.spcnode == pg_constants::DEFAULTTABLESPACE_OID);
-            let src_path = format!("{}/base/1/PG_VERSION", snappath);
-            let dst_path = format!("base/{}/PG_VERSION", db.dbnode);
-            ar.append_path_with_name(&src_path, &dst_path)?;
-            format!("base/{}/pg_filenode.map", db.dbnode)
-        };
-        assert!(img.len() == 512);
-        let header = new_tar_header(&path, img.len() as u64)?;
-        ar.append(&header, &img[..])?;
-    }
-    Ok(())
-}
-
-//
-// Extract twophase state files
-//
-fn add_twophase_files(
-    ar: &mut Builder<&mut dyn Write>,
-    timeline: &Arc<dyn Timeline>,
-    lsn: Lsn,
-) -> anyhow::Result<()> {
-    for xid in timeline.get_twophase(lsn)?.iter() {
-        let tag = BufferTag {
-            rel: RelTag {
-                spcnode: 0,
-                dbnode: 0,
-                relnode: 0,
-                forknum: pg_constants::PG_TWOPHASE_FORKNUM,
-            },
-            blknum: *xid,
-        };
-        let img = timeline.get_page_at_lsn(tag, lsn)?;
-		let mut buf = BytesMut::new();
-		buf.extend_from_slice(&img[..]);
-		let crc = crc32c::crc32c(&img[..]);
-		buf.put_u32_le(crc);
-        let path = format!("pg_twophase/{:>08X}", xid);
-        let header = new_tar_header(&path, buf.len() as u64)?;
-        ar.append(&header, &buf[..])?;
-    }
-    Ok(())
-}
-
-//
-// Add generated pg_control file
-//
-fn add_pgcontrol_file(
-    ar: &mut Builder<&mut dyn Write>,
-    timeline: &Arc<dyn Timeline>,
-    lsn: Lsn,
-) -> anyhow::Result<()> {
-    if let Some(checkpoint_bytes) =
-        timeline.get_page_image(BufferTag::fork(pg_constants::PG_CHECKPOINT_FORKNUM), Lsn(0))?
-    {
-        if let Some(pg_control_bytes) = timeline.get_page_image(
-            BufferTag::fork(pg_constants::PG_CONTROLFILE_FORKNUM),
-            Lsn(0),
-        )? {
-            let mut pg_control = postgres_ffi::decode_pg_control(pg_control_bytes)?;
-            let mut checkpoint = postgres_ffi::decode_checkpoint(checkpoint_bytes)?;
-
-            checkpoint.redo = lsn.0;
-            checkpoint.nextXid.value += 1;
-            // TODO: When we restart master there are no active transaction and oldestXid is
-            // equal to nextXid if there are no prepared transactions.
-            // Let's ignore them for a while...
-            checkpoint.oldestXid = checkpoint.nextXid.value as u32;
-            pg_control.checkPointCopy = checkpoint;
-            let pg_control_bytes = postgres_ffi::encode_pg_control(pg_control);
-            let header = new_tar_header("global/pg_control", pg_control_bytes.len() as u64)?;
-            ar.append(&header, &pg_control_bytes[..])?;
-        }
-    }
-    Ok(())
-}
-
 ///
 /// Generate tarball with non-relational files from repository
 ///
 pub fn send_tarball_at_lsn(
     write: &mut dyn Write,
     timelineid: ZTimelineId,
-    timeline: &Arc<dyn Timeline>,
-    lsn: Lsn,
+    _timeline: &Arc<dyn Timeline>,
+    _lsn: Lsn,
     snapshot_lsn: Lsn,
 ) -> anyhow::Result<()> {
     let mut ar = Builder::new(write);
 
     let snappath = format!("timelines/{}/snapshots/{:016X}", timelineid, snapshot_lsn.0);
+    let walpath = format!("timelines/{}/wal", timelineid);
 
     debug!("sending tarball of snapshot in {}", snappath);
     for entry in WalkDir::new(&snappath) {
@@ -211,14 +49,8 @@ pub fn send_tarball_at_lsn(
                 trace!("sending shared catalog {}", relpath.display());
                 ar.append_path_with_name(fullpath, relpath)?;
             } else if !is_rel_file_path(relpath.to_str().unwrap()) {
-                if entry.file_name() != "pg_filenode.map"
-                    && entry.file_name() != "pg_control"
-                    && !relpath.starts_with("pg_xact/")
-                    && !relpath.starts_with("pg_multixact/")
-                {
-                    trace!("sending {}", relpath.display());
-                    ar.append_path_with_name(fullpath, relpath)?;
-                }
+                trace!("sending {}", relpath.display());
+                ar.append_path_with_name(fullpath, relpath)?;
             } else {
                 trace!("not sending {}", relpath.display());
             }
@@ -227,31 +59,27 @@ pub fn send_tarball_at_lsn(
         }
     }
 
-    add_slru_segments(
-        &mut ar,
-        timeline,
-        "pg_xact",
-        pg_constants::PG_XACT_FORKNUM,
-        lsn,
-    )?;
-    add_slru_segments(
-        &mut ar,
-        timeline,
-        "pg_multixact/members",
-        pg_constants::PG_MXACT_MEMBERS_FORKNUM,
-        lsn,
-    )?;
-    add_slru_segments(
-        &mut ar,
-        timeline,
-        "pg_multixact/offsets",
-        pg_constants::PG_MXACT_OFFSETS_FORKNUM,
-        lsn,
-    )?;
-    add_relmap_files(&mut ar, timeline, lsn, &snappath)?;
-    add_twophase_files(&mut ar, timeline, lsn)?;
-    add_pgcontrol_file(&mut ar, timeline, lsn)?;
+    // FIXME: Also send all the WAL. The compute node would only need
+    // the WAL that applies to non-relation files, because the page
+    // server handles all the relation files. But we don't have a
+    // mechanism for separating relation and non-relation WAL at the
+    // moment.
+    for entry in std::fs::read_dir(&walpath)? {
+        let entry = entry?;
+        let fullpath = &entry.path();
+        let relpath = fullpath.strip_prefix(&walpath).unwrap();
 
+        if !entry.path().is_file() {
+            continue;
+        }
+
+        let archive_fname = relpath.to_str().unwrap();
+        let archive_fname = archive_fname
+            .strip_suffix(".partial")
+            .unwrap_or(&archive_fname);
+        let archive_path = "pg_wal/".to_owned() + archive_fname;
+        ar.append_path_with_name(fullpath, archive_path)?;
+    }
     ar.finish()?;
     debug!("all tarred up!");
     Ok(())
@@ -385,7 +213,7 @@ fn parse_rel_file_path(path: &str) -> Result<(), FilePathError> {
         let (_relnode, _forknum, _segno) = parse_relfilename(fname)?;
 
         Ok(())
-    } else if let Some(_) = path.strip_prefix("pg_tblspc/") {
+    } else if path.strip_prefix("pg_tblspc/").is_some() {
         // TODO
         error!("tablespaces not implemented yet");
         Err(FilePathError::InvalidFileName)

pageserver/src/branches.rs

@@ -7,7 +7,6 @@
 use anyhow::{anyhow, bail, Context, Result};
 use bytes::Bytes;
 use fs::File;
-use fs_extra;
 use postgres_ffi::{pg_constants, xlog_utils};
 use rand::Rng;
 use serde::{Deserialize, Serialize};
@@ -22,6 +21,8 @@ use std::{
 };
 use zenith_utils::lsn::Lsn;
 
+use crate::page_cache;
+use crate::restore_local_repo;
 use crate::{repository::Repository, PageServerConf, ZTimelineId};
 
 #[derive(Serialize, Deserialize, Clone)]
@@ -39,7 +40,7 @@ pub struct PointInTime {
     pub lsn: Lsn,
 }
 
-pub fn init_repo(conf: &PageServerConf, repo_dir: &Path) -> Result<()> {
+pub fn init_repo(conf: &'static PageServerConf, repo_dir: &Path) -> Result<()> {
     // top-level dir may exist if we are creating it through CLI
     fs::create_dir_all(repo_dir)
         .with_context(|| format!("could not create directory {}", repo_dir.display()))?;
@@ -50,15 +51,9 @@ pub fn init_repo(conf: &PageServerConf, repo_dir: &Path) -> Result<()> {
     fs::create_dir(std::path::Path::new("refs"))?;
     fs::create_dir(std::path::Path::new("refs").join("branches"))?;
     fs::create_dir(std::path::Path::new("refs").join("tags"))?;
-    fs::create_dir(std::path::Path::new("wal-redo"))?;
 
     println!("created directory structure in {}", repo_dir.display());
 
-    // Create initial timeline
-    let tli = create_timeline(conf, None)?;
-    let timelinedir = conf.timeline_path(tli);
-    println!("created initial timeline {}", tli);
-
     // Run initdb
     //
     // We create the cluster temporarily in a "tmp" directory inside the repository,
@@ -90,6 +85,27 @@ pub fn init_repo(conf: &PageServerConf, repo_dir: &Path) -> Result<()> {
     let lsn = controlfile.checkPoint;
     let lsnstr = format!("{:016X}", lsn);
 
+    // Bootstrap the repository by loading the newly-initdb'd cluster into 'main' branch.
+    let tli = create_timeline(conf, None)?;
+    let timelinedir = conf.timeline_path(tli);
+
+    // We don't use page_cache here, because we don't want to spawn the WAL redo thread during
+    // repository initialization.
+    //
+    // FIXME: That caused trouble, because the WAL redo thread launched initdb in the background,
+    // and it kept running even after the "zenith init" had exited. In tests, we started the
+    // page server immediately after that, so that initdb was still running in the background,
+    // and we failed to run initdb again in the same directory. This has been solved for the
+    // rapid init+start case now, but the general race condition remains if you restart the the
+    // server quickly.
+    let repo = crate::repository::rocksdb::RocksRepository::new(
+        conf,
+        std::sync::Arc::new(crate::walredo::DummyRedoManager {}),
+    );
+    let timeline = repo.create_empty_timeline(tli, Lsn(lsn))?;
+
+    restore_local_repo::import_timeline_from_postgres_datadir(&tmppath, &*timeline, Lsn(lsn))?;
+
     // Move the initial WAL file
     fs::rename(
         tmppath.join("pg_wal").join("000000010000000000000001"),
@@ -97,19 +113,24 @@ pub fn init_repo(conf: &PageServerConf, repo_dir: &Path) -> Result<()> {
             .join("wal")
             .join("000000010000000000000001.partial"),
     )?;
-    println!("moved initial WAL file");
+    println!("created initial timeline {}", tli);
+
+    let data = tli.to_string();
+    fs::write(conf.branch_path("main"), data)?;
+    println!("created main branch");
 
     // Remove pg_wal
     fs::remove_dir_all(tmppath.join("pg_wal"))?;
 
+    force_crash_recovery(&tmppath)?;
+    println!("updated pg_control");
+
+    // Move the data directory as an initial base backup.
+    // FIXME: It would be enough to only copy the non-relation files here, the relation
+    // data was already loaded into the repository.
     let target = timelinedir.join("snapshots").join(&lsnstr);
     fs::rename(tmppath, &target)?;
 
-    // Create 'main' branch to refer to the initial timeline
-    let data = tli.to_string();
-    fs::write(conf.branch_path("main"), data)?;
-    println!("created main branch");
-
     println!(
         "new zenith repository was created in {}",
         repo_dir.display()
@@ -211,12 +232,13 @@ pub(crate) fn create_branch(
         startpoint.lsn = end_of_wal;
     }
 
-    // create a new timeline for it
+    // create a new timeline directory for it
     let newtli = create_timeline(conf, Some(startpoint))?;
     let newtimelinedir = conf.timeline_path(newtli);
 
-    let data = newtli.to_string();
-    fs::write(conf.branch_path(&branchname), data)?;
+    // Let the Repository backend do its initialization
+    let repo = page_cache::get_repository();
+    repo.branch_timeline(startpoint.timelineid, newtli, startpoint.lsn)?;
 
     // Copy the latest snapshot (TODO: before the startpoint) and all WAL
     // TODO: be smarter and avoid the copying...
@@ -232,6 +254,12 @@ pub(crate) fn create_branch(
         pg_constants::WAL_SEGMENT_SIZE,
     )?;
 
+    // Remember the human-readable branch name for the new timeline.
+    // FIXME: there's a race condition, if you create a branch with the same
+    // name concurrently.
+    let data = newtli.to_string();
+    fs::write(conf.branch_path(&branchname), data)?;
+
     Ok(BranchInfo {
         name: branchname.to_string(),
         timeline_id: newtli,
@@ -310,6 +338,31 @@ fn parse_point_in_time(conf: &PageServerConf, s: &str) -> Result<PointInTime> {
     bail!("could not parse point-in-time {}", s);
 }
 
+// If control file says the cluster was shut down cleanly, modify it, to mark
+// it as crashed. That forces crash recovery when you start the cluster.
+//
+// FIXME:
+// We currently do this to the initial snapshot in "zenith init". It would
+// be more natural to do this when the snapshot is restored instead, but we
+// currently don't have any code to create new snapshots, so it doesn't matter
+// Or better yet, use a less hacky way of putting the cluster into recovery.
+// Perhaps create a backup label file in the data directory when it's restored.
+fn force_crash_recovery(datadir: &Path) -> Result<()> {
+    // Read in the control file
+    let controlfilepath = datadir.to_path_buf().join("global").join("pg_control");
+    let mut controlfile =
+        postgres_ffi::decode_pg_control(Bytes::from(fs::read(controlfilepath.as_path())?))?;
+
+    controlfile.state = postgres_ffi::DBState_DB_IN_PRODUCTION;
+
+    fs::write(
+        controlfilepath.as_path(),
+        postgres_ffi::encode_pg_control(controlfile),
+    )?;
+
+    Ok(())
+}
+
 fn create_timeline(conf: &PageServerConf, ancestor: Option<PointInTime>) -> Result<ZTimelineId> {
     // Create initial timeline
     let mut tli_buf = [0u8; 16];
@@ -340,40 +393,38 @@ fn copy_wal(src_dir: &Path, dst_dir: &Path, upto: Lsn, wal_seg_size: usize) -> R
     let last_segno = upto.segment_number(wal_seg_size);
     let last_segoff = upto.segment_offset(wal_seg_size);
 
-    for entry in fs::read_dir(src_dir).unwrap() {
-        if let Ok(entry) = entry {
-            let entry_name = entry.file_name();
-            let fname = entry_name.to_str().unwrap();
+    for entry in fs::read_dir(src_dir).unwrap().flatten() {
+        let entry_name = entry.file_name();
+        let fname = entry_name.to_str().unwrap();
 
-            // Check if the filename looks like an xlog file, or a .partial file.
-            if !xlog_utils::IsXLogFileName(fname) && !xlog_utils::IsPartialXLogFileName(fname) {
-                continue;
-            }
-            let (segno, _tli) = xlog_utils::XLogFromFileName(fname, wal_seg_size as usize);
+        // Check if the filename looks like an xlog file, or a .partial file.
+        if !xlog_utils::IsXLogFileName(fname) && !xlog_utils::IsPartialXLogFileName(fname) {
+            continue;
+        }
+        let (segno, _tli) = xlog_utils::XLogFromFileName(fname, wal_seg_size as usize);
 
-            let copylen;
-            let mut dst_fname = PathBuf::from(fname);
-            if segno > last_segno {
-                // future segment, skip
-                continue;
-            } else if segno < last_segno {
-                copylen = wal_seg_size;
-                dst_fname.set_extension("");
-            } else {
-                copylen = last_segoff;
-                dst_fname.set_extension("partial");
-            }
+        let copylen;
+        let mut dst_fname = PathBuf::from(fname);
+        if segno > last_segno {
+            // future segment, skip
+            continue;
+        } else if segno < last_segno {
+            copylen = wal_seg_size;
+            dst_fname.set_extension("");
+        } else {
+            copylen = last_segoff;
+            dst_fname.set_extension("partial");
+        }
 
-            let src_file = File::open(entry.path())?;
-            let mut dst_file = File::create(dst_dir.join(&dst_fname))?;
-            std::io::copy(&mut src_file.take(copylen as u64), &mut dst_file)?;
+        let src_file = File::open(entry.path())?;
+        let mut dst_file = File::create(dst_dir.join(&dst_fname))?;
+        std::io::copy(&mut src_file.take(copylen as u64), &mut dst_file)?;
 
-            if copylen < wal_seg_size {
-                std::io::copy(
-                    &mut std::io::repeat(0).take((wal_seg_size - copylen) as u64),
-                    &mut dst_file,
-                )?;
-            }
+        if copylen < wal_seg_size {
+            std::io::copy(
+                &mut std::io::repeat(0).take((wal_seg_size - copylen) as u64),
+                &mut dst_file,
+            )?;
         }
     }
     Ok(())

pageserver/src/page_service.rs

@@ -77,7 +77,7 @@ enum PagestreamBeMessage {
     Read(PagestreamReadResponse),
 }
 
-const HELLO_WORLD_ROW: BeMessage = BeMessage::DataRow(Bytes::from_static(b"hello world"));
+static HELLO_WORLD_ROW: BeMessage = BeMessage::DataRow(Bytes::from_static(b"hello world"));
 
 #[derive(Debug)]
 struct PagestreamRequest {
@@ -732,7 +732,7 @@ impl Connection {
 
             // Check that the timeline exists
             let repository = page_cache::get_repository();
-            if repository.get_or_restore_timeline(timelineid).is_err() {
+            if repository.get_timeline(timelineid).is_err() {
                 bail!("client requested callmemaybe on timeline {} which does not exist in page server", timelineid);
             }
 
@@ -899,14 +899,13 @@ impl Connection {
     ) -> anyhow::Result<()> {
         // check that the timeline exists
         let repository = page_cache::get_repository();
-        let timeline = repository
-            .get_or_restore_timeline(timelineid)
-            .map_err(|_| {
-                anyhow!(
+        let timeline = repository.get_timeline(timelineid).map_err(|e| {
+            error!("error fetching timeline: {:?}", e);
+            anyhow!(
                 "client requested basebackup on timeline {} which does not exist in page server",
                 timelineid
             )
-            })?;
+        })?;
         /* switch client to COPYOUT */
         let stream = &mut self.stream;
         stream.write_u8(b'H')?;
@@ -921,7 +920,7 @@ impl Connection {
         // find latest snapshot
         let snapshot_lsn =
             restore_local_repo::find_latest_snapshot(&self.conf, timelineid).unwrap();
-        let req_lsn = lsn.unwrap_or(timeline.get_last_valid_lsn());
+        let req_lsn = lsn.unwrap_or(snapshot_lsn);
         basebackup::send_tarball_at_lsn(
             &mut CopyDataSink { stream },
             timelineid,

pageserver/src/repository.rs

@@ -1,11 +1,13 @@
 pub mod rocksdb;
 
-use crate::waldecoder::{DecodedWALRecord, Oid, TransactionId, XlCreateDatabase, XlSmgrTruncate};
+use crate::waldecoder::{DecodedWALRecord, Oid, XlCreateDatabase, XlSmgrTruncate};
 use crate::ZTimelineId;
 use anyhow::Result;
 use bytes::{Buf, BufMut, Bytes, BytesMut};
+use log::*;
 use postgres_ffi::pg_constants;
 use postgres_ffi::relfile_utils::forknumber_to_name;
+use serde::{Deserialize, Serialize};
 use std::fmt;
 use std::sync::Arc;
 use zenith_utils::lsn::Lsn;
@@ -15,21 +17,17 @@ use zenith_utils::lsn::Lsn;
 /// timelines, forked off from the same initial call to 'initdb'.
 pub trait Repository {
     /// Get Timeline handle for given zenith timeline ID.
-    ///
-    /// The Timeline is expected to be already "open", i.e. `get_or_restore_timeline`
-    /// should've been called on it earlier already.
     fn get_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
 
-    /// Get Timeline handle for given zenith timeline ID.
-    ///
-    /// Creates a new Timeline object if it's not "open" already.
-    fn get_or_restore_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
+    /// Create a new, empty timeline. The caller is responsible for loading data into it
+    fn create_empty_timeline(
+        &self,
+        timelineid: ZTimelineId,
+        start_lsn: Lsn,
+    ) -> Result<Arc<dyn Timeline>>;
 
-    /// Create an empty timeline, without loading any data into it from possible on-disk snapshot.
-    ///
-    /// For unit tests.
-    #[cfg(test)]
-    fn create_empty_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>>;
+    /// Branch a timeline
+    fn branch_timeline(&self, src: ZTimelineId, dst: ZTimelineId, start_lsn: Lsn) -> Result<()>;
 
     //fn get_stats(&self) -> RepositoryStats;
 }
@@ -48,9 +46,6 @@ pub trait Timeline {
     /// Does relation exist?
     fn get_relsize_exists(&self, tag: RelTag, lsn: Lsn) -> Result<bool>;
 
-    /// Get page image at the particular LSN
-    fn get_page_image(&self, tag: BufferTag, lsn: Lsn) -> Result<Option<Bytes>>;
-
     //------------------------------------------------------------------------------
     // Public PUT functions, to update the repository with new page versions.
     //
@@ -61,14 +56,17 @@ pub trait Timeline {
     ///
     /// This will implicitly extend the relation, if the page is beyond the
     /// current end-of-file.
-    fn put_wal_record(&self, tag: BufferTag, rec: WALRecord);
+    fn put_wal_record(&self, tag: BufferTag, rec: WALRecord) -> Result<()>;
 
     /// Like put_wal_record, but with ready-made image of the page.
-    fn put_page_image(&self, tag: BufferTag, lsn: Lsn, img: Bytes);
+    fn put_page_image(&self, tag: BufferTag, lsn: Lsn, img: Bytes) -> Result<()>;
 
     /// Truncate relation
     fn put_truncation(&self, rel: RelTag, lsn: Lsn, nblocks: u32) -> Result<()>;
 
+    /// Drop relation or file segment
+    fn put_drop(&self, tag: BufferTag, lsn: Lsn) -> Result<()>;
+
     /// Create a new database from a template database
     ///
     /// In PostgreSQL, CREATE DATABASE works by scanning the data directory and
@@ -106,14 +104,18 @@ pub trait Timeline {
                 blknum: blk.blkno,
             };
 
-            let rec = WALRecord {
-                lsn,
-                will_init: blk.will_init || blk.apply_image,
-                rec: recdata.clone(),
-                main_data_offset: decoded.main_data_offset as u32,
-            };
+            if blk.will_drop {
+                self.put_drop(tag, lsn)?;
+            } else {
+                let rec = WALRecord {
+                    lsn,
+                    will_init: blk.will_init || blk.apply_image,
+                    rec: recdata.clone(),
+                    main_data_offset: decoded.main_data_offset as u32,
+                };
 
-            self.put_wal_record(tag, rec);
+                self.put_wal_record(tag, rec)?;
+            }
         }
 
         // Handle a few special record types
@@ -122,15 +124,37 @@ pub trait Timeline {
                 == pg_constants::XLOG_SMGR_TRUNCATE
         {
             let truncate = XlSmgrTruncate::decode(&decoded);
+            let mut rel = RelTag {
+                spcnode: truncate.rnode.spcnode,
+                dbnode: truncate.rnode.dbnode,
+                relnode: truncate.rnode.relnode,
+                forknum: pg_constants::MAIN_FORKNUM,
+            };
             if (truncate.flags & pg_constants::SMGR_TRUNCATE_HEAP) != 0 {
-                let rel = RelTag {
-                    spcnode: truncate.rnode.spcnode,
-                    dbnode: truncate.rnode.dbnode,
-                    relnode: truncate.rnode.relnode,
-                    forknum: pg_constants::MAIN_FORKNUM,
-                };
                 self.put_truncation(rel, lsn, truncate.blkno)?;
             }
+            if (truncate.flags & pg_constants::SMGR_TRUNCATE_FSM) != 0 {
+                if truncate.blkno == 0 {
+                    rel.forknum = pg_constants::FSM_FORKNUM;
+                    self.put_truncation(rel, lsn, truncate.blkno)?;
+                } else {
+                    // TODO: handle partial truncation of FSM:
+                    // need to map heap block number to FSM block number
+                    // and clear bits in the tail block
+                    info!("Partial truncation of FSM is not supported");
+                }
+            }
+            if (truncate.flags & pg_constants::SMGR_TRUNCATE_VM) != 0 {
+                if truncate.blkno == 0 {
+                    rel.forknum = pg_constants::VISIBILITYMAP_FORKNUM;
+                    self.put_truncation(rel, lsn, truncate.blkno)?;
+                } else {
+                    // TODO: handle partial truncation of VM:
+                    // need to map heap block number to VM block number
+                    // and clear bits in the tail block
+                    info!("Partial truncation of VM is not supported");
+                }
+            }
         } else if decoded.xl_rmid == pg_constants::RM_DBASE_ID
             && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
                 == pg_constants::XLOG_DBASE_CREATE
@@ -167,15 +191,12 @@ pub trait Timeline {
     fn advance_last_record_lsn(&self, lsn: Lsn);
     fn get_last_record_lsn(&self) -> Lsn;
 
-    /// Get range [begin,end) of stored blocks. Used mostly for SMGR pseudorelations
-    /// but can be also applied to normal relations.
-    fn get_range(&self, rel: RelTag, lsn: Lsn) -> Result<(u32, u32)>;
-
-    /// Get vector of databases (represented using RelTag only dbnode and spcnode fields are used)
-    fn get_databases(&self, lsn: Lsn) -> Result<Vec<RelTag>>;
-
-    /// Get vector of prepared twophase transactions
-    fn get_twophase(&self, lsn: Lsn) -> Result<Vec<TransactionId>>;
+    ///
+    /// Flush to disk all data that was written with the put_* functions
+    ///
+    /// NOTE: This has nothing to do with checkpoint in PostgreSQL. We don't
+    /// know anything about them here in the repository.
+    fn checkpoint(&self) -> Result<()>;
 }
 
 #[derive(Clone)]
@@ -186,7 +207,26 @@ pub struct RepositoryStats {
     pub num_getpage_requests: Lsn,
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy)]
+///
+/// Relation data file segment id throughout the Postgres cluster.
+///
+/// Every data file in Postgres is uniquely identified by 4 numbers:
+/// - relation id / node (`relnode`)
+/// - database id (`dbnode`)
+/// - tablespace id (`spcnode`), in short this is a unique id of a separate
+///   directory to store data files.
+/// - forknumber (`forknum`) is used to split different kinds of data of the same relation
+///   between some set of files (`relnode`, `relnode_fsm`, `relnode_vm`).
+///
+/// In native Postgres code `RelFileNode` structure and individual `ForkNumber` value
+/// are used for the same purpose.
+/// [See more related comments here](https:///github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/relfilenode.h#L57).
+///
+/// We use additional fork numbers to logically separate relational and
+/// non-relational data inside pageserver key-value storage.
+/// See, e.g., `ROCKSDB_SPECIAL_FORKNUM`.
+///
+#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
 pub struct RelTag {
     pub forknum: u8,
     pub spcnode: u32,
@@ -194,23 +234,6 @@ pub struct RelTag {
     pub relnode: u32,
 }
 
-impl RelTag {
-    pub fn pack(&self, buf: &mut BytesMut) {
-        buf.put_u8(self.forknum);
-        buf.put_u32(self.spcnode);
-        buf.put_u32(self.dbnode);
-        buf.put_u32(self.relnode);
-    }
-    pub fn unpack(buf: &mut Bytes) -> RelTag {
-        RelTag {
-            forknum: buf.get_u8(),
-            spcnode: buf.get_u32(),
-            dbnode: buf.get_u32(),
-            relnode: buf.get_u32(),
-        }
-    }
-}
-
 /// Display RelTag in the same format that's used in most PostgreSQL debug messages:
 ///
 /// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
@@ -229,37 +252,19 @@ impl fmt::Display for RelTag {
     }
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
+///
+/// `RelTag` + block number (`blknum`) gives us a unique id of the page in the cluster.
+/// This is used as a part of the key inside key-value storage (RocksDB currently).
+///
+/// In Postgres `BufferTag` structure is used for exactly the same purpose.
+/// [See more related comments here](https://github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/buf_internals.h#L91).
+///
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize, Deserialize)]
 pub struct BufferTag {
     pub rel: RelTag,
     pub blknum: u32,
 }
 
-impl BufferTag {
-    pub fn fork(forknum: u8) -> BufferTag {
-        BufferTag {
-            rel: RelTag {
-                forknum,
-                spcnode: 0,
-                dbnode: 0,
-                relnode: 0,
-            },
-            blknum: 0,
-        }
-    }
-
-    pub fn pack(&self, buf: &mut BytesMut) {
-        self.rel.pack(buf);
-        buf.put_u32(self.blknum);
-    }
-    pub fn unpack(buf: &mut Bytes) -> BufferTag {
-        BufferTag {
-            rel: RelTag::unpack(buf),
-            blknum: buf.get_u32(),
-        }
-    }
-}
-
 #[derive(Debug, Clone)]
 pub struct WALRecord {
     pub lsn: Lsn, // LSN at the *end* of the record
@@ -347,7 +352,7 @@ mod tests {
             gc_horizon: 64 * 1024 * 1024,
             gc_period: Duration::from_secs(10),
             listen_addr: "127.0.0.1:5430".parse().unwrap(),
-            workdir: repo_dir.into(),
+            workdir: repo_dir,
             pg_distrib_dir: "".into(),
         };
         // Make a static copy of the config. This can never be free'd, but that's
@@ -362,13 +367,6 @@ mod tests {
     }
 
     /// Test get_relsize() and truncation.
-    ///
-    /// FIXME: The RocksRepository implementation returns wrong relation size, if
-    /// you make a request with an old LSN. It seems to ignore the requested LSN
-    /// and always return result as of latest LSN. For such cases, the expected
-    /// results below match the current RocksRepository behavior, so that the test
-    /// passes, and the actually correct answers are in comments like
-    /// "// CORRECT: <correct answer>"
     #[test]
     fn test_relsize() -> Result<()> {
         // get_timeline() with non-existent timeline id should fail
@@ -377,24 +375,26 @@ mod tests {
         // Create timeline to work on
         let repo = get_test_repo("test_relsize")?;
         let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid)?;
+        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
 
         tline.init_valid_lsn(Lsn(1));
-        tline.put_page_image(TEST_BUF(0), Lsn(2), TEST_IMG("foo blk 0 at 2"));
-        tline.put_page_image(TEST_BUF(0), Lsn(2), TEST_IMG("foo blk 0 at 2"));
-        tline.put_page_image(TEST_BUF(0), Lsn(3), TEST_IMG("foo blk 0 at 3"));
-        tline.put_page_image(TEST_BUF(1), Lsn(4), TEST_IMG("foo blk 1 at 4"));
-        tline.put_page_image(TEST_BUF(2), Lsn(5), TEST_IMG("foo blk 2 at 5"));
+        tline.put_page_image(TEST_BUF(0), Lsn(2), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TEST_BUF(0), Lsn(2), TEST_IMG("foo blk 0 at 2"))?;
+        tline.put_page_image(TEST_BUF(0), Lsn(3), TEST_IMG("foo blk 0 at 3"))?;
+        tline.put_page_image(TEST_BUF(1), Lsn(4), TEST_IMG("foo blk 1 at 4"))?;
+        tline.put_page_image(TEST_BUF(2), Lsn(5), TEST_IMG("foo blk 2 at 5"))?;
 
         tline.advance_last_valid_lsn(Lsn(5));
 
+        // FIXME: The rocksdb implementation erroneously returns 'true' here, even
+        // though the relation was created only at a later LSN
         // rocksdb implementation erroneosly returns 'true' here
         assert_eq!(tline.get_relsize_exists(TESTREL_A, Lsn(1))?, true); // CORRECT: false
-                                                                        // likewise, it returns wrong size here
-        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(1))?, 3); // CORRECT: 0 (or error?)
+                                                                        // And this probably should throw an error, becaue the relation doesn't exist at Lsn(1) yet
+        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(1))?, 0); // CORRECT: throw error
 
         assert_eq!(tline.get_relsize_exists(TESTREL_A, Lsn(2))?, true);
-        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(2))?, 3); // CORRECT: 1
+        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(2))?, 1);
         assert_eq!(tline.get_relsize(TESTREL_A, Lsn(5))?, 3);
 
         // Check page contents at each LSN
@@ -446,7 +446,7 @@ mod tests {
         );
 
         // should still see the truncated block with older LSN
-        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(5))?, 2); // CORRECT: 3
+        assert_eq!(tline.get_relsize(TESTREL_A, Lsn(5))?, 3);
         assert_eq!(
             tline.get_page_at_lsn(TEST_BUF(2), Lsn(5))?,
             TEST_IMG("foo blk 2 at 5")
@@ -464,7 +464,7 @@ mod tests {
     fn test_large_rel() -> Result<()> {
         let repo = get_test_repo("test_large_rel")?;
         let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap();
-        let tline = repo.create_empty_timeline(timelineid)?;
+        let tline = repo.create_empty_timeline(timelineid, Lsn(0))?;
 
         tline.init_valid_lsn(Lsn(1));
 
@@ -472,7 +472,7 @@ mod tests {
         for i in 0..pg_constants::RELSEG_SIZE + 1 {
             let img = TEST_IMG(&format!("foo blk {} at {}", i, Lsn(lsn)));
             lsn += 1;
-            tline.put_page_image(TEST_BUF(i as u32), Lsn(lsn), img);
+            tline.put_page_image(TEST_BUF(i as u32), Lsn(lsn), img)?;
         }
         tline.advance_last_valid_lsn(Lsn(lsn));
 

pageserver/src/repository/rocksdb.rs

@@ -6,31 +6,31 @@
 // LSN.
 
 use crate::repository::{BufferTag, RelTag, Repository, Timeline, WALRecord};
-use crate::restore_local_repo::restore_timeline;
-use crate::waldecoder::{Oid, TransactionId};
+use crate::restore_local_repo::import_timeline_wal;
+use crate::waldecoder::Oid;
 use crate::walredo::WalRedoManager;
 use crate::PageServerConf;
 use crate::ZTimelineId;
-// use crate::PageServerConf;
-// use crate::branches;
-use anyhow::{bail, Context, Result};
+use anyhow::{anyhow, bail, Context, Result};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use log::*;
-use postgres_ffi::nonrelfile_utils::transaction_id_get_status;
-use postgres_ffi::*;
+use postgres_ffi::pg_constants;
+use serde::{Deserialize, Serialize};
 use std::cmp::min;
 use std::collections::HashMap;
 use std::convert::TryInto;
+use std::str::FromStr;
 use std::sync::atomic::AtomicU64;
 use std::sync::atomic::Ordering;
 use std::sync::{Arc, Mutex};
 use std::thread;
 use std::time::{Duration, Instant};
+use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::{AtomicLsn, Lsn};
 use zenith_utils::seqwait::SeqWait;
 
 // Timeout when waiting or WAL receiver to catch up to an LSN given in a GetPage@LSN call.
-static TIMEOUT: Duration = Duration::from_secs(60);
+static TIMEOUT: Duration = Duration::from_secs(600);
 
 pub struct RocksRepository {
     conf: &'static PageServerConf,
@@ -84,40 +84,55 @@ pub struct RocksTimeline {
 // stored directly in the cache entry in that you still need to run the WAL redo
 // routine to generate the page image.
 //
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 struct CacheKey {
     pub tag: BufferTag,
     pub lsn: Lsn,
 }
 
+//
+// In addition to those per-page entries, the 'last_valid_lsn' and 'last_record_lsn'
+// values are also persisted in the rocskdb repository. They are stored with CacheKeys
+// with ROCKSDB_SPECIAL_FORKNUM, and 'blknum' indicates which value it is. The
+// rest of the key fields are zero. We use a CacheKey as the key for these too,
+// so that whenever we iterate through keys in the repository, we can safely parse
+// the key blob as CacheKey without checking for these special values first.
+//
+// FIXME: This is quite a similar concept to the special entries created by
+// `BufferTag::fork` function. Merge them somehow? These special keys are specific
+// to the rocksb implementation, not exposed to the rest of the system, but the
+// other special forks created by `BufferTag::fork` are also used elsewhere.
+//
 impl CacheKey {
-    fn pack(&self, buf: &mut BytesMut) {
-        self.tag.pack(buf);
-        buf.put_u64(self.lsn.0);
-    }
-    fn unpack(buf: &mut Bytes) -> CacheKey {
+    const fn special(id: u32) -> CacheKey {
         CacheKey {
-            tag: BufferTag::unpack(buf),
-            lsn: Lsn::from(buf.get_u64()),
+            tag: BufferTag {
+                rel: RelTag {
+                    forknum: pg_constants::ROCKSDB_SPECIAL_FORKNUM,
+                    spcnode: 0,
+                    dbnode: 0,
+                    relnode: 0,
+                },
+                blknum: id,
+            },
+            lsn: Lsn(0),
         }
     }
 
-    fn from_slice(slice: &[u8]) -> Self {
-        let mut buf = Bytes::copy_from_slice(slice);
-        Self::unpack(&mut buf)
-    }
-
-    fn to_bytes(&self) -> BytesMut {
-        let mut buf = BytesMut::new();
-        self.pack(&mut buf);
-        buf
+    fn is_special(&self) -> bool {
+        self.tag.rel.forknum == pg_constants::ROCKSDB_SPECIAL_FORKNUM
     }
 }
 
+static LAST_VALID_LSN_KEY: CacheKey = CacheKey::special(0);
+static LAST_VALID_RECORD_LSN_KEY: CacheKey = CacheKey::special(1);
+
+#[derive(Debug)]
 enum CacheEntryContent {
     PageImage(Bytes),
     WALRecord(WALRecord),
     Truncation,
+    Drop,
 }
 
 // The serialized representation of a CacheEntryContent begins with
@@ -125,9 +140,10 @@ enum CacheEntryContent {
 // an UNUSED_VERSION_FLAG that is not represented in the CacheEntryContent
 // at all, you must peek into the first byte of the serialized representation
 // to read it.
-const CONTENT_PAGE_IMAGE: u8 = 1u8;
-const CONTENT_WAL_RECORD: u8 = 2u8;
-const CONTENT_TRUNCATION: u8 = 3u8;
+const CONTENT_PAGE_IMAGE: u8 = 0u8;
+const CONTENT_WAL_RECORD: u8 = 1u8;
+const CONTENT_TRUNCATION: u8 = 2u8;
+const CONTENT_DROP: u8 = 3u8;
 
 const CONTENT_KIND_MASK: u8 = 3u8; // bitmask that covers the above
 
@@ -148,6 +164,9 @@ impl CacheEntryContent {
             CacheEntryContent::Truncation => {
                 buf.put_u8(CONTENT_TRUNCATION);
             }
+            CacheEntryContent::Drop => {
+                buf.put_u8(CONTENT_DROP);
+            }
         }
     }
     pub fn unpack(buf: &mut Bytes) -> CacheEntryContent {
@@ -162,6 +181,7 @@ impl CacheEntryContent {
             }
             CONTENT_WAL_RECORD => CacheEntryContent::WALRecord(WALRecord::unpack(buf)),
             CONTENT_TRUNCATION => CacheEntryContent::Truncation,
+            CONTENT_DROP => CacheEntryContent::Drop,
             _ => unreachable!(),
         }
     }
@@ -184,59 +204,61 @@ impl RocksRepository {
         walredo_mgr: Arc<dyn WalRedoManager>,
     ) -> RocksRepository {
         RocksRepository {
-            conf: conf,
+            conf,
             timelines: Mutex::new(HashMap::new()),
             walredo_mgr,
         }
     }
 }
 
+impl RocksRepository {
+    fn get_rocks_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<RocksTimeline>> {
+        let mut timelines = self.timelines.lock().unwrap();
+
+        match timelines.get(&timelineid) {
+            Some(timeline) => Ok(timeline.clone()),
+            None => {
+                let timeline =
+                    RocksTimeline::open(self.conf, timelineid, self.walredo_mgr.clone())?;
+
+                // Load any new WAL after the last checkpoint into the repository.
+                info!(
+                    "Loading WAL for timeline {} starting at {}",
+                    timelineid,
+                    timeline.get_last_record_lsn()
+                );
+                let wal_dir = self.conf.timeline_path(timelineid).join("wal");
+                import_timeline_wal(&wal_dir, &timeline, timeline.get_last_record_lsn())?;
+
+                let timeline_rc = Arc::new(timeline);
+
+                if self.conf.gc_horizon != 0 {
+                    RocksTimeline::launch_gc_thread(self.conf, timeline_rc.clone());
+                }
+
+                timelines.insert(timelineid, timeline_rc.clone());
+
+                Ok(timeline_rc)
+            }
+        }
+    }
+}
+
 // Get handle to a given timeline. It is assumed to already exist.
 impl Repository for RocksRepository {
     fn get_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>> {
-        let timelines = self.timelines.lock().unwrap();
-
-        match timelines.get(&timelineid) {
-            Some(timeline) => Ok(timeline.clone()),
-            None => bail!("timeline not found"),
-        }
+        Ok(self.get_rocks_timeline(timelineid)?)
     }
 
-    fn get_or_restore_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>> {
+    fn create_empty_timeline(
+        &self,
+        timelineid: ZTimelineId,
+        start_lsn: Lsn,
+    ) -> Result<Arc<dyn Timeline>> {
         let mut timelines = self.timelines.lock().unwrap();
 
-        match timelines.get(&timelineid) {
-            Some(timeline) => Ok(timeline.clone()),
-            None => {
-                let timeline = RocksTimeline::new(self.conf, timelineid, self.walredo_mgr.clone());
-
-                restore_timeline(self.conf, &timeline, timelineid)?;
-
-                let timeline_rc = Arc::new(timeline);
-
-                timelines.insert(timelineid, timeline_rc.clone());
-
-                if self.conf.gc_horizon != 0 {
-                    let timeline_rc_copy = timeline_rc.clone();
-                    let conf = self.conf;
-                    let _gc_thread = thread::Builder::new()
-                        .name("Garbage collection thread".into())
-                        .spawn(move || {
-                            // FIXME
-                            timeline_rc_copy.do_gc(conf).expect("GC thread died");
-                        })
-                        .unwrap();
-                }
-                Ok(timeline_rc)
-            }
-        }
-    }
-
-    #[cfg(test)]
-    fn create_empty_timeline(&self, timelineid: ZTimelineId) -> Result<Arc<dyn Timeline>> {
-        let mut timelines = self.timelines.lock().unwrap();
-
-        let timeline = RocksTimeline::new(&self.conf, timelineid, self.walredo_mgr.clone());
+        let timeline =
+            RocksTimeline::create(&self.conf, timelineid, self.walredo_mgr.clone(), start_lsn)?;
 
         let timeline_rc = Arc::new(timeline);
         let r = timelines.insert(timelineid, timeline_rc.clone());
@@ -246,47 +268,130 @@ impl Repository for RocksRepository {
 
         Ok(timeline_rc)
     }
+
+    fn branch_timeline(&self, src: ZTimelineId, dst: ZTimelineId, at_lsn: Lsn) -> Result<()> {
+        let src_timeline = self.get_rocks_timeline(src)?;
+
+        info!("branching at {}", at_lsn);
+
+        let dst_timeline =
+            RocksTimeline::create(&self.conf, dst, self.walredo_mgr.clone(), at_lsn)?;
+
+        // Copy all entries <= LSN
+        //
+        // This is very inefficient, a far cry from the promise of cheap copy-on-write
+        // branching. But it will do for now.
+        let mut iter = src_timeline.db.raw_iterator();
+        iter.seek_to_first();
+        while iter.valid() {
+            let k = iter.key().unwrap();
+            let key = CacheKey::des(k)?;
+
+            if !key.is_special() && key.lsn <= at_lsn {
+                let v = iter.value().unwrap();
+                dst_timeline.db.put(k, v)?;
+            }
+            iter.next();
+        }
+        Ok(())
+    }
 }
 
 impl RocksTimeline {
-    fn open_rocksdb(conf: &PageServerConf, timelineid: ZTimelineId) -> rocksdb::DB {
-        let path = conf.timeline_path(timelineid);
+    /// common options used by `open` and `create`
+    fn get_rocksdb_opts() -> rocksdb::Options {
         let mut opts = rocksdb::Options::default();
-        opts.create_if_missing(true);
         opts.set_use_fsync(true);
         opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
-        opts.set_compaction_filter("ttl", move |_level: u32, _key: &[u8], val: &[u8]| {
+        opts.set_compaction_filter("ttl", move |_level: u32, key: &[u8], val: &[u8]| {
             if (val[0] & UNUSED_VERSION_FLAG) != 0 {
+                trace!("Delete unused version {:?}", CacheKey::des(key));
                 rocksdb::compaction_filter::Decision::Remove
             } else {
                 rocksdb::compaction_filter::Decision::Keep
             }
         });
-        rocksdb::DB::open(&opts, &path).unwrap()
+        opts
     }
 
-    fn new(
-        conf: &'static PageServerConf,
+    /// Open a RocksDB database, and load the last valid and record LSNs into memory.
+    fn open(
+        conf: &PageServerConf,
         timelineid: ZTimelineId,
         walredo_mgr: Arc<dyn WalRedoManager>,
-    ) -> RocksTimeline {
-        RocksTimeline {
-            db: RocksTimeline::open_rocksdb(conf, timelineid),
+    ) -> Result<RocksTimeline> {
+        let path = conf.timeline_path(timelineid);
+        let db = rocksdb::DB::open(&RocksTimeline::get_rocksdb_opts(), path)?;
 
+        // Load these into memory
+        let lsnstr = db
+            .get(LAST_VALID_LSN_KEY.ser()?)
+            .with_context(|| "last_valid_lsn not found in repository")?
+            .ok_or(anyhow!("empty last_valid_lsn"))?;
+        let last_valid_lsn = Lsn::from_str(std::str::from_utf8(&lsnstr)?)?;
+        let lsnstr = db
+            .get(LAST_VALID_RECORD_LSN_KEY.ser()?)
+            .with_context(|| "last_record_lsn not found in repository")?
+            .ok_or(anyhow!("empty last_record_lsn"))?;
+        let last_record_lsn = Lsn::from_str(std::str::from_utf8(&lsnstr)?)?;
+
+        let timeline = RocksTimeline {
+            db,
             walredo_mgr,
 
-            last_valid_lsn: SeqWait::new(Lsn(0)),
-            last_record_lsn: AtomicLsn::new(0),
+            last_valid_lsn: SeqWait::new(last_valid_lsn),
+            last_record_lsn: AtomicLsn::new(last_record_lsn.0),
 
             num_entries: AtomicU64::new(0),
             num_page_images: AtomicU64::new(0),
             num_wal_records: AtomicU64::new(0),
             num_getpage_requests: AtomicU64::new(0),
-        }
+        };
+        Ok(timeline)
+    }
+
+    /// Create a new RocksDB database. It is initally empty, except for the last
+    /// valid and last record LSNs, which are set to 'start_lsn'.
+    fn create(
+        conf: &PageServerConf,
+        timelineid: ZTimelineId,
+        walredo_mgr: Arc<dyn WalRedoManager>,
+        start_lsn: Lsn,
+    ) -> Result<RocksTimeline> {
+        let path = conf.timeline_path(timelineid);
+        let mut opts = RocksTimeline::get_rocksdb_opts();
+        opts.create_if_missing(true);
+        opts.set_error_if_exists(true);
+        let db = rocksdb::DB::open(&opts, path)?;
+
+        let timeline = RocksTimeline {
+            db,
+            walredo_mgr,
+
+            last_valid_lsn: SeqWait::new(start_lsn),
+            last_record_lsn: AtomicLsn::new(start_lsn.0),
+
+            num_entries: AtomicU64::new(0),
+            num_page_images: AtomicU64::new(0),
+            num_wal_records: AtomicU64::new(0),
+            num_getpage_requests: AtomicU64::new(0),
+        };
+        // Write the initial last_valid/record_lsn values
+        timeline.checkpoint()?;
+        Ok(timeline)
+    }
+
+    fn launch_gc_thread(conf: &'static PageServerConf, timeline_rc: Arc<RocksTimeline>) {
+        let timeline_rc_copy = timeline_rc.clone();
+        let _gc_thread = thread::Builder::new()
+            .name("Garbage collection thread".into())
+            .spawn(move || {
+                // FIXME
+                timeline_rc_copy.do_gc(conf).expect("GC thread died");
+            })
+            .unwrap();
     }
-}
 
-impl RocksTimeline {
     ///
     /// Collect all the WAL records that are needed to reconstruct a page
     /// image for the given cache entry.
@@ -304,12 +409,13 @@ impl RocksTimeline {
         let mut records: Vec<WALRecord> = Vec::new();
 
         let mut iter = self.db.raw_iterator();
-        iter.seek_for_prev(key.to_bytes());
+        let serialized_key = key.ser().expect("serialize CacheKey should always succeed");
+        iter.seek_for_prev(serialized_key);
 
         // Scan backwards, collecting the WAL records, until we hit an
         // old page image.
         while iter.valid() {
-            let key = CacheKey::from_slice(iter.key().unwrap());
+            let key = CacheKey::des(iter.key().unwrap()).unwrap();
             if key.tag != tag {
                 break;
             }
@@ -353,10 +459,16 @@ impl RocksTimeline {
         };
         let mut iter = self.db.raw_iterator();
         loop {
-            iter.seek_for_prev(key.to_bytes());
+            iter.seek_for_prev(key.ser()?);
             if iter.valid() {
-                let thiskey = CacheKey::from_slice(iter.key().unwrap());
+                let thiskey = CacheKey::des(iter.key().unwrap())?;
                 if thiskey.tag.rel == rel {
+                    // Ignore entries with later LSNs.
+                    if thiskey.lsn > lsn {
+                        key.tag.blknum = thiskey.tag.blknum;
+                        continue;
+                    }
+
                     let content = CacheEntryContent::from_slice(iter.value().unwrap());
                     if let CacheEntryContent::Truncation = content {
                         if thiskey.tag.blknum > 0 {
@@ -376,9 +488,48 @@ impl RocksTimeline {
         Ok(0)
     }
 
+    ///
+    /// Drop relations with all its forks
+    ///
+    fn delete_entries(&self, tag: BufferTag) -> Result<()> {
+        let mut iter = self.db.raw_iterator();
+        let mut key = CacheKey {
+            tag,
+            lsn: Lsn(u64::MAX),
+        };
+        if tag.rel.forknum == pg_constants::MAIN_FORKNUM {
+            // if it is relation then remove all its blocks in all forks
+            key.tag.blknum = u32::MAX;
+            key.tag.rel.forknum = pg_constants::INIT_FORKNUM;
+        } else {
+            assert!(tag.rel.forknum > pg_constants::INIT_FORKNUM);
+        }
+        debug!("Drop relation {:?}", tag);
+        iter.seek_for_prev(key.ser()?);
+        while iter.valid() {
+            let key = CacheKey::des(iter.key().unwrap())?;
+            if key.tag.rel != tag.rel {
+                // no more entries belonging to this relation
+                break;
+            }
+            let v = iter.value().unwrap();
+            if (v[0] & UNUSED_VERSION_FLAG) == 0 {
+                let mut v = v.to_owned();
+                v[0] |= UNUSED_VERSION_FLAG;
+                self.db.put(key.ser()?, &v[..])?;
+            } else {
+                // already marked for deletion
+                break;
+            }
+            iter.prev();
+        }
+        Ok(())
+    }
+
     fn do_gc(&self, conf: &'static PageServerConf) -> Result<Bytes> {
         loop {
             thread::sleep(conf.gc_period);
+            trace!("Start GC iteration");
             let last_lsn = self.get_last_valid_lsn();
 
             // checked_sub() returns None on overflow.
@@ -402,34 +553,31 @@ impl RocksTimeline {
                 let mut deleted = 0u64;
                 loop {
                     let mut iter = self.db.raw_iterator();
-                    iter.seek_for_prev(maxkey.to_bytes());
+                    iter.seek_for_prev(maxkey.ser()?);
                     if iter.valid() {
-                        let key = CacheKey::from_slice(iter.key().unwrap());
+                        let key = CacheKey::des(iter.key().unwrap())?;
                         let v = iter.value().unwrap();
+                        let flag = v[0];
+                        let last_lsn = key.lsn;
 
                         inspected += 1;
 
                         // Construct boundaries for old records cleanup
                         maxkey.tag = key.tag;
-                        let last_lsn = key.lsn;
                         maxkey.lsn = min(horizon, last_lsn); // do not remove last version
 
                         let mut minkey = maxkey.clone();
                         minkey.lsn = Lsn(0); // first version
 
-                        // Special handling of delete of PREPARE WAL record
-                        if last_lsn < horizon
-                            && key.tag.rel.forknum == pg_constants::PG_TWOPHASE_FORKNUM
-                        {
-                            if (v[0] & UNUSED_VERSION_FLAG) == 0 {
-                                let mut v = v.to_owned();
-                                v[0] |= UNUSED_VERSION_FLAG;
-                                self.db.put(key.to_bytes(), &v[..])?;
-                                deleted += 1;
+                        if (flag & CONTENT_KIND_MASK) == CONTENT_DROP {
+                            // If drop record in over the horizon then delete all entries from repository
+                            if last_lsn < horizon {
+                                self.delete_entries(key.tag)?;
                             }
                             maxkey = minkey;
                             continue;
                         }
+
                         // reconstruct most recent page version
                         if (v[0] & CONTENT_KIND_MASK) == CONTENT_WAL_RECORD {
                             // force reconstruction of most recent page version
@@ -447,17 +595,17 @@ impl RocksTimeline {
                             let new_img = self
                                 .walredo_mgr
                                 .request_redo(key.tag, key.lsn, base_img, records)?;
-                            self.put_page_image(key.tag, key.lsn, new_img.clone());
+                            self.put_page_image(key.tag, key.lsn, new_img.clone())?;
 
                             reconstructed += 1;
                         }
 
-                        iter.seek_for_prev(maxkey.to_bytes());
+                        iter.seek_for_prev(maxkey.ser()?);
                         if iter.valid() {
                             // do not remove last version
                             if last_lsn > horizon {
                                 // locate most recent record before horizon
-                                let key = CacheKey::from_slice(iter.key().unwrap());
+                                let key = CacheKey::des(iter.key().unwrap())?;
                                 if key.tag == maxkey.tag {
                                     let v = iter.value().unwrap();
                                     if (v[0] & CONTENT_KIND_MASK) == CONTENT_WAL_RECORD {
@@ -468,7 +616,7 @@ impl RocksTimeline {
                                         let new_img = self
                                             .walredo_mgr
                                             .request_redo(key.tag, key.lsn, base_img, records)?;
-                                        self.put_page_image(key.tag, key.lsn, new_img.clone());
+                                        self.put_page_image(key.tag, key.lsn, new_img.clone())?;
 
                                         truncated += 1;
                                     } else {
@@ -495,7 +643,7 @@ impl RocksTimeline {
                                 if !iter.valid() {
                                     break;
                                 }
-                                let key = CacheKey::from_slice(iter.key().unwrap());
+                                let key = CacheKey::des(iter.key().unwrap())?;
                                 if key.tag != maxkey.tag {
                                     break;
                                 }
@@ -503,7 +651,7 @@ impl RocksTimeline {
                                 if (v[0] & UNUSED_VERSION_FLAG) == 0 {
                                     let mut v = v.to_owned();
                                     v[0] |= UNUSED_VERSION_FLAG;
-                                    self.db.put(key.to_bytes(), &v[..])?;
+                                    self.db.put(key.ser()?, &v[..])?;
                                     deleted += 1;
                                     trace!(
                                         "deleted: {} blk {} at {}",
@@ -542,13 +690,18 @@ impl RocksTimeline {
             );
             lsn = last_valid_lsn;
         }
-        //trace!("Start waiting for LSN {}, valid LSN is {}", lsn,  self.last_valid_lsn.load());
+        trace!(
+            "Start waiting for LSN {}, valid LSN is {}",
+            lsn,
+            self.last_valid_lsn.load()
+        );
         self.last_valid_lsn
             .wait_for_timeout(lsn, TIMEOUT)
             .with_context(|| {
                 format!(
-                    "Timed out while waiting for WAL record at LSN {} to arrive",
-                    lsn
+                    "Timed out while waiting for WAL record at LSN {} to arrive. valid LSN in {}",
+                    lsn,
+                    self.last_valid_lsn.load(),
                 )
             })?;
         //trace!("Stop waiting for LSN {}, valid LSN is {}", lsn,  self.last_valid_lsn.load());
@@ -567,6 +720,7 @@ impl Timeline for RocksTimeline {
     ///
     fn get_page_at_lsn(&self, tag: BufferTag, req_lsn: Lsn) -> Result<Bytes> {
         self.num_getpage_requests.fetch_add(1, Ordering::Relaxed);
+        const ZERO_PAGE: [u8; 8192] = [0u8; 8192];
 
         let lsn = self.wait_lsn(req_lsn)?;
 
@@ -575,10 +729,10 @@ impl Timeline for RocksTimeline {
         let key = CacheKey { tag, lsn };
 
         let mut iter = self.db.raw_iterator();
-        iter.seek_for_prev(key.to_bytes());
+        iter.seek_for_prev(key.ser()?);
 
         if iter.valid() {
-            let key = CacheKey::from_slice(iter.key().unwrap());
+            let key = CacheKey::des(iter.key().unwrap())?;
             if key.tag == tag {
                 let content = CacheEntryContent::from_slice(iter.value().unwrap());
                 let page_img: Bytes;
@@ -589,7 +743,10 @@ impl Timeline for RocksTimeline {
                     let (base_img, records) = self.collect_records_for_apply(tag, lsn);
                     page_img = self.walredo_mgr.request_redo(tag, lsn, base_img, records)?;
 
-                    self.put_page_image(tag, lsn, page_img.clone());
+                    self.put_page_image(tag, lsn, page_img.clone())?;
+                } else if let CacheEntryContent::Truncation = content {
+                    // Because FSM pages may be not up-to-date we can request truncated page
+                    return Ok(Bytes::from_static(&ZERO_PAGE));
                 } else {
                     // No base image, and no WAL record. Huh?
                     bail!("no page image or WAL record for requested page");
@@ -606,7 +763,6 @@ impl Timeline for RocksTimeline {
                 return Ok(page_img);
             }
         }
-        static ZERO_PAGE: [u8; 8192] = [0u8; 8192];
         debug!(
             "Page {} blk {} at {}({}) not found",
             tag.rel, tag.blknum, req_lsn, lsn
@@ -623,116 +779,6 @@ impl Timeline for RocksTimeline {
         self.relsize_get_nowait(rel, lsn)
     }
 
-    /// Get vector of prepared twophase transactions
-    fn get_twophase(&self, lsn: Lsn) -> Result<Vec<TransactionId>> {
-        let key = CacheKey {
-            // minimal key
-            tag: BufferTag {
-                rel: RelTag {
-                    forknum: pg_constants::PG_TWOPHASE_FORKNUM,
-                    spcnode: 0,
-                    dbnode: 0,
-                    relnode: 0,
-                },
-                blknum: 0,
-            },
-            lsn: Lsn(0),
-        };
-        let mut gxacts = Vec::new();
-
-        let mut iter = self.db.raw_iterator();
-        iter.seek(key.to_bytes());
-        while iter.valid() {
-            let key = CacheKey::from_slice(iter.key().unwrap());
-            if key.tag.rel.forknum != pg_constants::PG_TWOPHASE_FORKNUM {
-                break; // we are done with this fork
-            }
-            if key.lsn <= lsn {
-                let xid = key.tag.blknum;
-                let tag = BufferTag {
-                    rel: RelTag {
-                        forknum: pg_constants::PG_XACT_FORKNUM,
-                        spcnode: 0,
-                        dbnode: 0,
-                        relnode: 0,
-                    },
-                    blknum: xid / pg_constants::CLOG_XACTS_PER_PAGE,
-                };
-                let clog_page = self.get_page_at_lsn(tag, lsn)?;
-                let status = transaction_id_get_status(xid, &clog_page[..]);
-                if status == pg_constants::TRANSACTION_STATUS_IN_PROGRESS {
-                    gxacts.push(xid);
-                }
-            }
-            iter.next();
-        }
-        return Ok(gxacts);
-    }
-
-    /// Get databases. This function is used to local pg_filenode.map files
-    fn get_databases(&self, lsn: Lsn) -> Result<Vec<RelTag>> {
-        let key = CacheKey {
-            // minimal key
-            tag: BufferTag {
-                rel: RelTag {
-                    forknum: pg_constants::PG_FILENODEMAP_FORKNUM,
-                    spcnode: 0,
-                    dbnode: 0,
-                    relnode: 0,
-                },
-                blknum: 0,
-            },
-            lsn: Lsn(0),
-        };
-        let mut dbs = Vec::new();
-
-        let mut iter = self.db.raw_iterator();
-        iter.seek(key.to_bytes());
-        let mut prev_tag = key.tag.rel;
-        while iter.valid() {
-            let key = CacheKey::from_slice(iter.key().unwrap());
-            if key.tag.rel.forknum != pg_constants::PG_FILENODEMAP_FORKNUM {
-                break; // we are done with this fork
-            }
-            if key.tag.rel != prev_tag && key.lsn <= lsn {
-                prev_tag = key.tag.rel;
-                dbs.push(prev_tag); // collect unique tags
-            }
-            iter.next();
-        }
-        return Ok(dbs);
-    }
-
-    /// Get range [begin,end) of stored blocks. Used mostly for SMGR pseudorelations
-    /// but can be also applied to normal relations.
-    fn get_range(&self, rel: RelTag, lsn: Lsn) -> Result<(u32, u32)> {
-        let _lsn = self.wait_lsn(lsn)?;
-        let mut key = CacheKey {
-            // minimal key to start with
-            tag: BufferTag { rel, blknum: 0 },
-            lsn: Lsn(0),
-        };
-        let mut iter = self.db.raw_iterator();
-        iter.seek(key.to_bytes()); // locate first entry
-        if iter.valid() {
-            let thiskey = CacheKey::from_slice(iter.key().unwrap());
-            let tag = thiskey.tag;
-            if tag.rel == rel {
-                // still trversing this relation
-                let first_blknum = tag.blknum;
-                key.tag.blknum = u32::MAX; // maximal key
-                let mut iter = self.db.raw_iterator();
-                iter.seek_for_prev(key.to_bytes()); // localte last entry
-                if iter.valid() {
-                    let thiskey = CacheKey::from_slice(iter.key().unwrap());
-                    let last_blknum = thiskey.tag.blknum;
-                    return Ok((first_blknum, last_blknum + 1)); // upper boundary is exclusive
-                }
-            }
-        }
-        Ok((0, 0)) // empty range
-    }
-
     ///
     /// Does relation exist at given LSN?
     ///
@@ -748,9 +794,9 @@ impl Timeline for RocksTimeline {
             lsn,
         };
         let mut iter = self.db.raw_iterator();
-        iter.seek_for_prev(key.to_bytes());
+        iter.seek_for_prev(key.ser()?);
         if iter.valid() {
-            let key = CacheKey::from_slice(iter.key().unwrap());
+            let key = CacheKey::des(iter.key().unwrap())?;
             if key.tag.rel == rel {
                 debug!("Relation {} exists at {}", rel, lsn);
                 return Ok(true);
@@ -766,13 +812,14 @@ impl Timeline for RocksTimeline {
     ///
     /// Adds a WAL record to the repository
     ///
-    fn put_wal_record(&self, tag: BufferTag, rec: WALRecord) {
+    fn put_wal_record(&self, tag: BufferTag, rec: WALRecord) -> Result<()> {
         let lsn = rec.lsn;
         let key = CacheKey { tag, lsn };
 
         let content = CacheEntryContent::WALRecord(rec);
 
-        let _res = self.db.put(key.to_bytes(), content.to_bytes());
+        let serialized_key = key.ser().expect("serialize CacheKey should always succeed");
+        self.db.put(serialized_key, content.to_bytes())?;
         trace!(
             "put_wal_record rel {} blk {} at {}",
             tag.rel,
@@ -782,6 +829,18 @@ impl Timeline for RocksTimeline {
 
         self.num_entries.fetch_add(1, Ordering::Relaxed);
         self.num_wal_records.fetch_add(1, Ordering::Relaxed);
+        Ok(())
+    }
+
+    ///
+    /// Put drop record which should completely delete relation with all its forks
+    /// or non-relational file from repository
+    ///
+    fn put_drop(&self, tag: BufferTag, lsn: Lsn) -> Result<()> {
+        let key = CacheKey { tag, lsn };
+        let content = CacheEntryContent::Drop;
+        self.db.put(key.ser()?, content.to_bytes())?;
+        Ok(())
     }
 
     ///
@@ -803,7 +862,7 @@ impl Timeline for RocksTimeline {
                 lsn,
             };
             trace!("put_wal_record lsn: {}", key.lsn);
-            let _res = self.db.put(key.to_bytes(), content.to_bytes());
+            self.db.put(key.ser()?, content.to_bytes())?;
         }
         let n = (old_rel_size - nblocks) as u64;
         self.num_entries.fetch_add(n, Ordering::Relaxed);
@@ -811,43 +870,16 @@ impl Timeline for RocksTimeline {
         Ok(())
     }
 
-    ///
-    /// Get page image at particular LSN
-    ///
-    fn get_page_image(&self, tag: BufferTag, lsn: Lsn) -> Result<Option<Bytes>> {
-        let key = CacheKey { tag, lsn };
-        if let Some(bytes) = self.db.get(key.to_bytes())? {
-            let content = CacheEntryContent::from_slice(&bytes);
-            if let CacheEntryContent::PageImage(img) = content {
-                return Ok(Some(img));
-            }
-        }
-        return Ok(None);
-    }
-
     ///
     /// Memorize a full image of a page version
     ///
-    fn put_page_image(&self, tag: BufferTag, lsn: Lsn, img: Bytes) {
-        let img_len = img.len();
+    fn put_page_image(&self, tag: BufferTag, lsn: Lsn, img: Bytes) -> Result<()> {
         let key = CacheKey { tag, lsn };
         let content = CacheEntryContent::PageImage(img);
 
-        let mut val_buf = content.to_bytes();
-
-        // Zero size of page image indicates that page can be removed
-        if img_len == 0 {
-            if (val_buf[0] & UNUSED_VERSION_FLAG) != 0 {
-                // records already marked for deletion
-                return;
-            } else {
-                // delete truncated multixact page
-                val_buf[0] |= UNUSED_VERSION_FLAG;
-            }
-        }
-
         trace!("put_wal_record lsn: {}", key.lsn);
-        let _res = self.db.put(key.to_bytes(), content.to_bytes());
+        let serialized_key = key.ser().expect("serialize CacheKey should always succeed");
+        self.db.put(serialized_key, content.to_bytes())?;
 
         trace!(
             "put_page_image rel {} blk {} at {}",
@@ -856,6 +888,7 @@ impl Timeline for RocksTimeline {
             lsn
         );
         self.num_page_images.fetch_add(1, Ordering::Relaxed);
+        Ok(())
     }
 
     fn put_create_database(
@@ -866,42 +899,35 @@ impl Timeline for RocksTimeline {
         src_db_id: Oid,
         src_tablespace_id: Oid,
     ) -> Result<()> {
-        let mut n = 0;
-        for forknum in &[
-            pg_constants::MAIN_FORKNUM,
-            pg_constants::FSM_FORKNUM,
-            pg_constants::VISIBILITYMAP_FORKNUM,
-            pg_constants::INIT_FORKNUM,
-            pg_constants::PG_FILENODEMAP_FORKNUM,
-        ] {
-            let key = CacheKey {
-                tag: BufferTag {
-                    rel: RelTag {
-                        spcnode: src_tablespace_id,
-                        dbnode: src_db_id,
-                        relnode: 0,
-                        forknum: *forknum,
-                    },
-                    blknum: 0,
+        let key = CacheKey {
+            tag: BufferTag {
+                rel: RelTag {
+                    spcnode: src_tablespace_id,
+                    dbnode: src_db_id,
+                    relnode: 0,
+                    forknum: 0u8,
                 },
-                lsn: Lsn(0),
-            };
-            let mut iter = self.db.raw_iterator();
-            iter.seek(key.to_bytes());
-            while iter.valid() {
-                let mut key = CacheKey::from_slice(iter.key().unwrap());
-                if key.tag.rel.spcnode != src_tablespace_id || key.tag.rel.dbnode != src_db_id {
-                    break;
-                }
-                key.tag.rel.spcnode = tablespace_id;
-                key.tag.rel.dbnode = db_id;
-                key.lsn = lsn;
-
-                let v = iter.value().unwrap();
-                self.db.put(key.to_bytes(), v)?;
-                n += 1;
-                iter.next();
+                blknum: 0,
+            },
+            lsn: Lsn(0),
+        };
+        let mut iter = self.db.raw_iterator();
+        iter.seek(key.ser()?);
+        let mut n = 0;
+        while iter.valid() {
+            let mut key = CacheKey::des(iter.key().unwrap())?;
+            if key.tag.rel.spcnode != src_tablespace_id || key.tag.rel.dbnode != src_db_id {
+                break;
             }
+
+            key.tag.rel.spcnode = tablespace_id;
+            key.tag.rel.dbnode = db_id;
+            key.lsn = lsn;
+
+            let v = iter.value().unwrap();
+            self.db.put(key.ser()?, v)?;
+            n += 1;
+            iter.next();
         }
         info!(
             "Create database {}/{}, copy {} entries",
@@ -912,7 +938,6 @@ impl Timeline for RocksTimeline {
 
     /// Remember that WAL has been received and added to the timeline up to the given LSN
     fn advance_last_valid_lsn(&self, lsn: Lsn) {
-        let lsn = Lsn((lsn.0 + 7) & !7); // align position on 8 bytes
         let old = self.last_valid_lsn.advance(lsn);
 
         // Can't move backwards.
@@ -930,8 +955,7 @@ impl Timeline for RocksTimeline {
     /// NOTE: this updates last_valid_lsn as well.
     ///
     fn advance_last_record_lsn(&self, lsn: Lsn) {
-        let lsn = Lsn((lsn.0 + 7) & !7); // align position on 8 bytes
-                                         // Can't move backwards.
+        // Can't move backwards.
         let old = self.last_record_lsn.fetch_max(lsn);
         assert!(old <= lsn);
 
@@ -961,6 +985,23 @@ impl Timeline for RocksTimeline {
         self.last_valid_lsn.load()
     }
 
+    // Flush all the changes written so far with PUT functions to disk.
+    // RocksDB writes out things as we go (?), so we don't need to do much here. We just
+    // write out the last valid and record LSNs.
+    fn checkpoint(&self) -> Result<()> {
+        let last_valid_lsn = self.last_valid_lsn.load();
+        self.db
+            .put(LAST_VALID_LSN_KEY.ser()?, last_valid_lsn.to_string())?;
+        self.db.put(
+            LAST_VALID_RECORD_LSN_KEY.ser()?,
+            self.last_record_lsn.load().to_string(),
+        )?;
+
+        trace!("checkpoint at {}", last_valid_lsn);
+
+        Ok(())
+    }
+
     //
     // Get statistics to be displayed in the user interface.
     //

pageserver/src/restore_local_repo.rs

@@ -1,15 +1,7 @@
-//
-// Restore chunks from local Zenith repository
-//
-// This runs once at Page Server startup. It loads all the "snapshots" and all
-// WAL from all timelines from the local zenith repository into the in-memory page
-// cache.
-//
-// This also initializes the "last valid LSN" in the page cache to the last LSN
-// seen in the WAL, so that when the WAL receiver is started, it starts
-// streaming from that LSN.
-//
-
+//!
+//! Import data and WAL from a PostgreSQL data directory and WAL segments into
+//! zenith repository
+//!
 use log::*;
 use std::cmp::max;
 use std::fs;
@@ -26,58 +18,11 @@ use crate::repository::{BufferTag, RelTag, Timeline};
 use crate::waldecoder::{decode_wal_record, Oid, WalStreamDecoder};
 use crate::PageServerConf;
 use crate::ZTimelineId;
+use postgres_ffi::pg_constants;
 use postgres_ffi::relfile_utils::*;
 use postgres_ffi::xlog_utils::*;
-use postgres_ffi::*;
 use zenith_utils::lsn::Lsn;
 
-///
-/// Load all WAL and all relation data pages from local disk into the repository.
-///
-pub fn restore_timeline(
-    conf: &PageServerConf,
-    timeline: &dyn Timeline,
-    timelineid: ZTimelineId,
-) -> Result<()> {
-    let timelinepath = PathBuf::from("timelines").join(timelineid.to_string());
-
-    if !timelinepath.exists() {
-        anyhow::bail!("timeline {} does not exist in the page server's repository");
-    }
-
-    // Scan .zenith/timelines/<timeline>/snapshots
-    let snapshotspath = PathBuf::from("timelines")
-        .join(timelineid.to_string())
-        .join("snapshots");
-
-    let mut last_snapshot_lsn: Lsn = Lsn(0);
-
-    for direntry in fs::read_dir(&snapshotspath).unwrap() {
-        let direntry = direntry?;
-        let filename = direntry.file_name();
-        let lsn = Lsn::from_filename(&filename)?;
-        last_snapshot_lsn = max(lsn, last_snapshot_lsn);
-
-        // FIXME: pass filename as Path instead of str?
-        let filename_str = filename.into_string().unwrap();
-        restore_snapshot(conf, timeline, timelineid, &filename_str)?;
-        info!("restored snapshot at {:?}", filename_str);
-    }
-
-    if last_snapshot_lsn == Lsn(0) {
-        error!(
-            "could not find valid snapshot in {}",
-            snapshotspath.display()
-        );
-        // TODO return error?
-    }
-    timeline.init_valid_lsn(last_snapshot_lsn);
-
-    restore_wal(timeline, timelineid, last_snapshot_lsn)?;
-
-    Ok(())
-}
-
 ///
 /// Find latest snapshot in a timeline's 'snapshots' directory
 ///
@@ -102,61 +47,38 @@ pub fn find_latest_snapshot(_conf: &PageServerConf, timeline: ZTimelineId) -> Re
     Ok(last_snapshot_lsn)
 }
 
-fn restore_snapshot(
-    conf: &PageServerConf,
+///
+/// Import all relation data pages from local disk into the repository.
+///
+pub fn import_timeline_from_postgres_datadir(
+    path: &Path,
     timeline: &dyn Timeline,
-    timelineid: ZTimelineId,
-    snapshot: &str,
+    lsn: Lsn,
 ) -> Result<()> {
-    let snapshotpath = PathBuf::from("timelines")
-        .join(timelineid.to_string())
-        .join("snapshots")
-        .join(snapshot);
-
     // Scan 'global'
-    for direntry in fs::read_dir(snapshotpath.join("global"))? {
+    for direntry in fs::read_dir(path.join("global"))? {
         let direntry = direntry?;
         match direntry.file_name().to_str() {
             None => continue,
 
             // These special files appear in the snapshot, but are not needed by the page server
-            Some("pg_control") => restore_nonrel_file(
-                conf,
-                timeline,
-                timelineid,
-                "0",
-                0,
-                0,
-                pg_constants::PG_CONTROLFILE_FORKNUM,
-                0,
-                &direntry.path(),
-            )?,
-            Some("pg_filenode.map") => restore_nonrel_file(
-                conf,
-                timeline,
-                timelineid,
-                snapshot,
-                pg_constants::GLOBALTABLESPACE_OID,
-                0,
-                pg_constants::PG_FILENODEMAP_FORKNUM,
-                0,
-                &direntry.path(),
-            )?,
+            Some("pg_control") => continue,
+            Some("pg_filenode.map") => continue,
 
             // Load any relation files into the page server
-            _ => restore_relfile(
+            _ => import_relfile(
+                &direntry.path(),
                 timeline,
-                snapshot,
+                lsn,
                 pg_constants::GLOBALTABLESPACE_OID,
                 0,
-                &direntry.path(),
             )?,
         }
     }
 
     // Scan 'base'. It contains database dirs, the database OID is the filename.
     // E.g. 'base/12345', where 12345 is the database OID.
-    for direntry in fs::read_dir(snapshotpath.join("base"))? {
+    for direntry in fs::read_dir(path.join("base"))? {
         let direntry = direntry?;
 
         let dboid = direntry.file_name().to_str().unwrap().parse::<u32>()?;
@@ -168,91 +90,34 @@ fn restore_snapshot(
 
                 // These special files appear in the snapshot, but are not needed by the page server
                 Some("PG_VERSION") => continue,
-                Some("pg_filenode.map") => restore_nonrel_file(
-                    conf,
-                    timeline,
-                    timelineid,
-                    snapshot,
-                    pg_constants::DEFAULTTABLESPACE_OID,
-                    dboid,
-                    pg_constants::PG_FILENODEMAP_FORKNUM,
-                    0,
-                    &direntry.path(),
-                )?,
+                Some("pg_filenode.map") => continue,
 
                 // Load any relation files into the page server
-                _ => restore_relfile(
+                _ => import_relfile(
+                    &direntry.path(),
                     timeline,
-                    snapshot,
+                    lsn,
                     pg_constants::DEFAULTTABLESPACE_OID,
                     dboid,
-                    &direntry.path(),
                 )?,
             }
         }
     }
-    for entry in fs::read_dir(snapshotpath.join("pg_xact"))? {
-        let entry = entry?;
-        restore_slru_file(
-            conf,
-            timeline,
-            timelineid,
-            snapshot,
-            pg_constants::PG_XACT_FORKNUM,
-            &entry.path(),
-        )?;
-    }
-    for entry in fs::read_dir(snapshotpath.join("pg_multixact").join("members"))? {
-        let entry = entry?;
-        restore_slru_file(
-            conf,
-            timeline,
-            timelineid,
-            snapshot,
-            pg_constants::PG_MXACT_MEMBERS_FORKNUM,
-            &entry.path(),
-        )?;
-    }
-    for entry in fs::read_dir(snapshotpath.join("pg_multixact").join("offsets"))? {
-        let entry = entry?;
-        restore_slru_file(
-            conf,
-            timeline,
-            timelineid,
-            snapshot,
-            pg_constants::PG_MXACT_OFFSETS_FORKNUM,
-            &entry.path(),
-        )?;
-    }
-    for entry in fs::read_dir(snapshotpath.join("pg_twophase"))? {
-        let entry = entry?;
-        let xid = u32::from_str_radix(&entry.path().to_str().unwrap(), 16)?;
-        restore_nonrel_file(
-            conf,
-            timeline,
-            timelineid,
-            snapshot,
-            0,
-            0,
-            pg_constants::PG_TWOPHASE_FORKNUM,
-            xid,
-            &entry.path(),
-        )?;
-    }
     // TODO: Scan pg_tblspc
 
+    timeline.checkpoint()?;
+
     Ok(())
 }
 
-fn restore_relfile(
+// subroutine of import_timeline_from_postgres_datadir(), to load one relation file.
+fn import_relfile(
+    path: &Path,
     timeline: &dyn Timeline,
-    snapshot: &str,
+    lsn: Lsn,
     spcoid: Oid,
     dboid: Oid,
-    path: &Path,
 ) -> Result<()> {
-    let lsn = Lsn::from_hex(snapshot)?;
-
     // Does it look like a relation file?
 
     let p = parse_relfilename(path.file_name().unwrap().to_str().unwrap());
@@ -279,7 +144,7 @@ fn restore_relfile(
                     },
                     blknum,
                 };
-                timeline.put_page_image(tag, lsn, Bytes::copy_from_slice(&buf));
+                timeline.put_page_image(tag, lsn, Bytes::copy_from_slice(&buf))?;
                 /*
                 if oldest_lsn == 0 || p.lsn < oldest_lsn {
                     oldest_lsn = p.lsn;
@@ -306,125 +171,22 @@ fn restore_relfile(
     Ok(())
 }
 
-fn restore_nonrel_file(
-    _conf: &PageServerConf,
-    timeline: &dyn Timeline,
-    _timelineid: ZTimelineId,
-    snapshot: &str,
-    spcoid: Oid,
-    dboid: Oid,
-    forknum: u8,
-    blknum: u32,
-    path: &Path,
-) -> Result<()> {
-    let lsn = Lsn::from_hex(snapshot)?;
-
-    // Does it look like a relation file?
-
-    let mut file = File::open(path)?;
-    let mut buffer = Vec::new();
-    // read the whole file
-    file.read_to_end(&mut buffer)?;
-
-    let tag = BufferTag {
-        rel: RelTag {
-            spcnode: spcoid,
-            dbnode: dboid,
-            relnode: 0,
-            forknum,
-        },
-        blknum,
-    };
-    timeline.put_page_image(tag, lsn, Bytes::copy_from_slice(&buffer[..]));
-    Ok(())
-}
-
-fn restore_slru_file(
-    _conf: &PageServerConf,
-    timeline: &dyn Timeline,
-    _timelineid: ZTimelineId,
-    snapshot: &str,
-    forknum: u8,
-    path: &Path,
-) -> Result<()> {
-    let lsn = Lsn::from_hex(snapshot)?;
-
-    // Does it look like a relation file?
-
-    let mut file = File::open(path)?;
-    let mut buf: [u8; 8192] = [0u8; 8192];
-    let segno = u32::from_str_radix(path.file_name().unwrap().to_str().unwrap(), 16)?;
-
-    let mut blknum: u32 = segno * pg_constants::SLRU_PAGES_PER_SEGMENT;
-    loop {
-        let r = file.read_exact(&mut buf);
-        match r {
-            Ok(_) => {
-                let tag = BufferTag {
-                    rel: RelTag {
-                        spcnode: 0,
-                        dbnode: 0,
-                        relnode: 0,
-                        forknum,
-                    },
-                    blknum,
-                };
-                timeline.put_page_image(tag, lsn, Bytes::copy_from_slice(&buf));
-                /*
-                if oldest_lsn == 0 || p.lsn < oldest_lsn {
-                    oldest_lsn = p.lsn;
-                }
-                 */
-            }
-
-            // TODO: UnexpectedEof is expected
-            Err(e) => match e.kind() {
-                std::io::ErrorKind::UnexpectedEof => {
-                    // reached EOF. That's expected.
-                    // FIXME: maybe check that we read the full length of the file?
-                    break;
-                }
-                _ => {
-                    error!("error reading file: {:?} ({})", path, e);
-                    break;
-                }
-            },
-        };
-        blknum += 1;
-    }
-
-    Ok(())
-}
-
-// Scan WAL on a timeline, starting from given LSN, and load all the records
-// into the page cache.
-fn restore_wal(timeline: &dyn Timeline, timelineid: ZTimelineId, startpoint: Lsn) -> Result<()> {
-    let walpath = format!("timelines/{}/wal", timelineid);
-
+/// Scan PostgreSQL WAL files in given directory, and load all records >= 'startpoint' into
+/// the repository.
+pub fn import_timeline_wal(walpath: &Path, timeline: &dyn Timeline, startpoint: Lsn) -> Result<()> {
     let mut waldecoder = WalStreamDecoder::new(startpoint);
 
     let mut segno = startpoint.segment_number(pg_constants::WAL_SEGMENT_SIZE);
     let mut offset = startpoint.segment_offset(pg_constants::WAL_SEGMENT_SIZE);
-    let mut last_lsn = Lsn(0);
-
-    let mut checkpoint = CheckPoint::new(startpoint.0, 1);
-    let checkpoint_tag = BufferTag::fork(pg_constants::PG_CHECKPOINT_FORKNUM);
-    let pg_control_tag = BufferTag::fork(pg_constants::PG_CONTROLFILE_FORKNUM);
-    if let Some(pg_control_bytes) = timeline.get_page_image(pg_control_tag, Lsn(0))? {
-        let pg_control = decode_pg_control(pg_control_bytes)?;
-        checkpoint = pg_control.checkPointCopy.clone();
-    } else {
-        error!("No control file is found in reposistory");
-    }
-
+    let mut last_lsn = startpoint;
     loop {
         // FIXME: assume postgresql tli 1 for now
         let filename = XLogFileName(1, segno, pg_constants::WAL_SEGMENT_SIZE);
-        let mut path = walpath.clone() + "/" + &filename;
+        let mut path = walpath.join(&filename);
 
         // It could be as .partial
         if !PathBuf::from(&path).exists() {
-            path += ".partial";
+            path = walpath.join(filename + ".partial");
         }
 
         // Slurp the WAL file
@@ -454,12 +216,10 @@ fn restore_wal(timeline: &dyn Timeline, timelineid: ZTimelineId, startpoint: Lsn
             if rec.is_err() {
                 // Assume that an error means we've reached the end of
                 // a partial WAL record. So that's ok.
-                trace!("WAL decoder error {:?}", rec);
-                waldecoder.set_position(Lsn((segno + 1) * pg_constants::WAL_SEGMENT_SIZE as u64));
                 break;
             }
             if let Some((lsn, recdata)) = rec.unwrap() {
-                let decoded = decode_wal_record(&mut checkpoint, recdata.clone());
+                let decoded = decode_wal_record(recdata.clone());
                 timeline.save_decoded_record(decoded, recdata, lsn)?;
                 last_lsn = lsn;
             } else {
@@ -469,15 +229,15 @@ fn restore_wal(timeline: &dyn Timeline, timelineid: ZTimelineId, startpoint: Lsn
         }
 
         info!(
-            "restored {} records from WAL file {} at {}",
-            nrecords, filename, last_lsn
+            "imported {} records from WAL file {} up to {}",
+            nrecords,
+            path.display(),
+            last_lsn
         );
 
         segno += 1;
         offset = 0;
     }
     info!("reached end of WAL at {}", last_lsn);
-    let checkpoint_bytes = encode_checkpoint(checkpoint);
-    timeline.put_page_image(checkpoint_tag, Lsn(0), checkpoint_bytes);
     Ok(())
 }

pageserver/src/waldecoder.rs

@@ -1,7 +1,7 @@
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use log::*;
+use postgres_ffi::pg_constants;
 use postgres_ffi::xlog_utils::XLogRecord;
-use postgres_ffi::*;
 use std::cmp::min;
 use std::str;
 use thiserror::Error;
@@ -14,19 +14,17 @@ pub type OffsetNumber = u16;
 pub type MultiXactId = TransactionId;
 pub type MultiXactOffset = u32;
 pub type MultiXactStatus = u32;
-pub type TimeLineID = u32;
-pub type PgTime = i64;
 
 // From PostgreSQL headers
 
 #[repr(C)]
 #[derive(Debug)]
 pub struct XLogPageHeaderData {
-    xlp_magic: u16,      /* magic value for correctness checks */
-    xlp_info: u16,       /* flag bits, see below */
-    xlp_tli: TimeLineID, /* TimeLineID of first record on page */
-    xlp_pageaddr: u64,   /* XLOG address of this page */
-    xlp_rem_len: u32,    /* total len of remaining data for record */
+    xlp_magic: u16,    /* magic value for correctness checks */
+    xlp_info: u16,     /* flag bits, see below */
+    xlp_tli: u32,      /* TimeLineID of first record on page */
+    xlp_pageaddr: u64, /* XLOG address of this page */
+    xlp_rem_len: u32,  /* total len of remaining data for record */
 }
 
 // FIXME: this assumes MAXIMUM_ALIGNOF 8. There are 4 padding bytes at end
@@ -84,13 +82,6 @@ impl WalStreamDecoder {
         }
     }
 
-    pub fn set_position(&mut self, lsn: Lsn) {
-        self.lsn = lsn;
-        self.contlen = 0;
-        self.padlen = 0;
-        self.inputbuf.clear();
-    }
-
     pub fn feed_bytes(&mut self, buf: &[u8]) {
         self.inputbuf.extend_from_slice(buf);
     }
@@ -273,7 +264,8 @@ pub struct DecodedBkpBlock {
     /* Information on full-page image, if any */
     has_image: bool,       /* has image, even for consistency checking */
     pub apply_image: bool, /* has image that should be restored */
-    pub will_init: bool,
+    pub will_init: bool,   /* record intialize page content */
+    pub will_drop: bool,   /* record drops relation */
     //char	   *bkp_image;
     hole_offset: u16,
     hole_length: u16,
@@ -298,6 +290,7 @@ impl DecodedBkpBlock {
             has_image: false,
             apply_image: false,
             will_init: false,
+            will_drop: false,
             hole_offset: 0,
             hole_length: 0,
             bimg_len: 0,
@@ -556,7 +549,7 @@ impl XlMultiXactTruncate {
 //      block data
 //      ...
 //      main data
-pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedWALRecord {
+pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
     let mut rnode_spcnode: u32 = 0;
     let mut rnode_dbnode: u32 = 0;
     let mut rnode_relnode: u32 = 0;
@@ -574,12 +567,7 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
         xlogrec.xl_rmid,
         xlogrec.xl_info
     );
-    if xlogrec.xl_xid > checkpoint.nextXid.value as u32 {
-        // TODO: handle XID wraparound
-        checkpoint.nextXid = FullTransactionId {
-            value: (checkpoint.nextXid.value & 0xFFFFFFFF00000000) | xlogrec.xl_xid as u64,
-        };
-    }
+
     let remaining = xlogrec.xl_tot_len - SizeOfXLogRecord;
 
     if buf.remaining() != remaining as usize {
@@ -790,31 +778,10 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
         assert_eq!(buf.remaining(), main_data_len as usize);
     }
 
-    //5. Handle special CLOG and XACT records
-    if xlogrec.xl_rmid == pg_constants::RM_CLOG_ID {
-        let mut blk = DecodedBkpBlock::new();
-        blk.forknum = pg_constants::PG_XACT_FORKNUM;
-        blk.blkno = buf.get_i32_le() as u32;
-        blk.will_init = true;
-        trace!("RM_CLOG_ID updates block {}", blk.blkno);
-        blocks.push(blk);
-    } else if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
+    //5. Handle special XACT records
+    if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
         let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
-        if info == pg_constants::XLOG_XACT_COMMIT || info == pg_constants::XLOG_XACT_COMMIT_PREPARED {
-			if info == pg_constants::XLOG_XACT_COMMIT {
-				let mut blk = DecodedBkpBlock::new();
-				blk.forknum = pg_constants::PG_XACT_FORKNUM;
-				blk.blkno = xlogrec.xl_xid / pg_constants::CLOG_XACTS_PER_PAGE;
-				trace!(
-					"XLOG_XACT_COMMIT xl_info {} xl_prev {:X}/{:X}  xid {} updates block {} main_data_len {}",
-					xlogrec.xl_info, (xlogrec.xl_prev >> 32),
-					xlogrec.xl_prev & 0xffffffff,
-					xlogrec.xl_xid,
-					blk.blkno,
-					main_data_len
-				);
-				blocks.push(blk);
-			}
+        if info == pg_constants::XLOG_XACT_COMMIT {
             //parse commit record to extract subtrans entries
             // xl_xact_commit starts with time of commit
             let _xact_time = buf.get_i64_le();
@@ -829,17 +796,8 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
             }
             if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
                 let nsubxacts = buf.get_i32_le();
-                let mut prev_blkno = u32::MAX;
                 for _i in 0..nsubxacts {
-                    let subxact = buf.get_u32_le();
-                    let blkno = subxact / pg_constants::CLOG_XACTS_PER_PAGE;
-                    if prev_blkno != blkno {
-                        prev_blkno = blkno;
-                        let mut blk = DecodedBkpBlock::new();
-                        blk.forknum = pg_constants::PG_XACT_FORKNUM;
-                        blk.blkno = blkno;
-                        blocks.push(blk);
-                    }
+                    let _subxact = buf.get_u32_le();
                 }
             }
             if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
@@ -848,7 +806,13 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
                     let spcnode = buf.get_u32_le();
                     let dbnode = buf.get_u32_le();
                     let relnode = buf.get_u32_le();
-                    //TODO handle this too?
+                    let mut blk = DecodedBkpBlock::new();
+                    blk.forknum = pg_constants::MAIN_FORKNUM;
+                    blk.rnode_spcnode = spcnode;
+                    blk.rnode_dbnode = dbnode;
+                    blk.rnode_relnode = relnode;
+                    blk.will_drop = true;
+                    blocks.push(blk);
                     trace!(
                         "XLOG_XACT_COMMIT relfilenode {}/{}/{}",
                         spcnode,
@@ -865,29 +829,11 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
                 }
             }
             if xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE != 0 {
-                let xid = buf.get_u32_le();
-				let mut blk = DecodedBkpBlock::new();
-				blk.forknum = pg_constants::PG_XACT_FORKNUM;
-				blk.blkno = xid / pg_constants::CLOG_XACTS_PER_PAGE;
-				blocks.push(blk);
+                let _xid = buf.get_u32_le();
                 trace!("XLOG_XACT_COMMIT-XACT_XINFO_HAS_TWOPHASE");
                 //TODO handle this to be able to restore pg_twophase on node start
             }
-        } else if info == pg_constants::XLOG_XACT_ABORT || info == pg_constants::XLOG_XACT_ABORT_PREPARED {
-			if info == pg_constants::XLOG_XACT_ABORT {
-				let mut blk = DecodedBkpBlock::new();
-				blk.forknum = pg_constants::PG_XACT_FORKNUM;
-				blk.blkno = xlogrec.xl_xid / pg_constants::CLOG_XACTS_PER_PAGE;
-				trace!(
-					"XLOG_XACT_ABORT xl_info {} xl_prev {:X}/{:X} xid {} updates block {} main_data_len {}",
-					xlogrec.xl_info, (xlogrec.xl_prev >> 32),
-					xlogrec.xl_prev & 0xffffffff,
-					xlogrec.xl_xid,
-					blk.blkno,
-					main_data_len
-				);
-				blocks.push(blk);
-			}
+        } else if info == pg_constants::XLOG_XACT_ABORT {
             //parse abort record to extract subtrans entries
             // xl_xact_abort starts with time of commit
             let _xact_time = buf.get_i64_le();
@@ -902,17 +848,8 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
             }
             if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
                 let nsubxacts = buf.get_i32_le();
-                let mut prev_blkno = u32::MAX;
                 for _i in 0..nsubxacts {
-                    let subxact = buf.get_u32_le();
-                    let blkno = subxact / pg_constants::CLOG_XACTS_PER_PAGE;
-                    if prev_blkno != blkno {
-                        prev_blkno = blkno;
-                        let mut blk = DecodedBkpBlock::new();
-                        blk.forknum = pg_constants::PG_XACT_FORKNUM;
-                        blk.blkno = blkno;
-                        blocks.push(blk);
-                    }
+                    let _subxact = buf.get_u32_le();
                 }
             }
             if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
@@ -921,7 +858,13 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
                     let spcnode = buf.get_u32_le();
                     let dbnode = buf.get_u32_le();
                     let relnode = buf.get_u32_le();
-                    //TODO save these too
+                    let mut blk = DecodedBkpBlock::new();
+                    blk.forknum = pg_constants::MAIN_FORKNUM;
+                    blk.rnode_spcnode = spcnode;
+                    blk.rnode_dbnode = dbnode;
+                    blk.rnode_relnode = relnode;
+                    blk.will_drop = true;
+                    blocks.push(blk);
                     trace!(
                         "XLOG_XACT_ABORT relfilenode {}/{}/{}",
                         spcnode,
@@ -931,20 +874,9 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
                 }
             }
             if xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE != 0 {
-                let xid = buf.get_u32_le();
-				let mut blk = DecodedBkpBlock::new();
-				blk.forknum = pg_constants::PG_XACT_FORKNUM;
-				blk.blkno = xid / pg_constants::CLOG_XACTS_PER_PAGE;
-				blocks.push(blk);
+                let _xid = buf.get_u32_le();
                 trace!("XLOG_XACT_ABORT-XACT_XINFO_HAS_TWOPHASE");
             }
-        } else if info == pg_constants::XLOG_XACT_PREPARE {
-            let mut blk = DecodedBkpBlock::new();
-            blk.forknum = pg_constants::PG_TWOPHASE_FORKNUM;
-            blk.blkno = xlogrec.xl_xid;
-            blk.will_init = true;
-            blocks.push(blk);
-			info!("Prepare transaction {}", xlogrec.xl_xid);
         }
     } else if xlogrec.xl_rmid == pg_constants::RM_DBASE_ID {
         let info = xlogrec.xl_info & !pg_constants::XLR_INFO_MASK;
@@ -977,7 +909,7 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
             trace!("XLOG_TBLSPC_DROP is not handled yet");
         }
     } else if xlogrec.xl_rmid == pg_constants::RM_HEAP_ID {
-        let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
+        let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
         let blkno = blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32;
         if info == pg_constants::XLOG_HEAP_INSERT {
             let xlrec = XlHeapInsert::decode(&mut buf);
@@ -1031,7 +963,7 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
             }
         }
     } else if xlogrec.xl_rmid == pg_constants::RM_HEAP2_ID {
-        let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
+        let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
         if info == pg_constants::XLOG_HEAP2_MULTI_INSERT {
             let xlrec = XlHeapMultiInsert::decode(&mut buf);
             if (xlrec.flags
@@ -1049,102 +981,6 @@ pub fn decode_wal_record(checkpoint: &mut CheckPoint, record: Bytes) -> DecodedW
                 blocks.push(blk);
             }
         }
-    } else if xlogrec.xl_rmid == pg_constants::RM_MULTIXACT_ID {
-        let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-        if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
-            let mut blk = DecodedBkpBlock::new();
-            blk.forknum = pg_constants::PG_MXACT_OFFSETS_FORKNUM;
-            blk.blkno = buf.get_u32_le();
-            blk.will_init = true;
-            blocks.push(blk);
-        } else if info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE {
-            let mut blk = DecodedBkpBlock::new();
-            blk.forknum = pg_constants::PG_MXACT_MEMBERS_FORKNUM;
-            blk.blkno = buf.get_u32_le();
-            blk.will_init = true;
-            blocks.push(blk);
-        } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
-            let xlrec = XlMultiXactCreate::decode(&mut buf);
-            // Update offset page
-            let mut blk = DecodedBkpBlock::new();
-            blk.blkno = xlrec.mid / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
-            blk.forknum = pg_constants::PG_MXACT_OFFSETS_FORKNUM;
-            blocks.push(blk);
-            let first_mbr_blkno = xlrec.moff / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-            let last_mbr_blkno =
-                (xlrec.moff + xlrec.nmembers - 1) / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-            for blkno in first_mbr_blkno..=last_mbr_blkno {
-                // Update members page
-                let mut blk = DecodedBkpBlock::new();
-                blk.forknum = pg_constants::PG_MXACT_MEMBERS_FORKNUM;
-                blk.blkno = blkno;
-                blocks.push(blk);
-            }
-            if xlrec.mid > checkpoint.nextMulti {
-                checkpoint.nextMulti = xlrec.mid;
-            }
-            if xlrec.moff > checkpoint.nextMultiOffset {
-                checkpoint.nextMultiOffset = xlrec.moff;
-            }
-            let max_xid = xlrec
-                .members
-                .iter()
-                .fold(checkpoint.nextXid.value as u32, |acc, mbr| {
-                    if mbr.xid > acc {
-                        mbr.xid
-                    } else {
-                        acc
-                    }
-                });
-            checkpoint.nextXid = FullTransactionId {
-                value: (checkpoint.nextXid.value & 0xFFFFFFFF00000000) | max_xid as u64,
-            };
-        } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID {
-            let xlrec = XlMultiXactTruncate::decode(&mut buf);
-            checkpoint.oldestXid = xlrec.end_trunc_off;
-            checkpoint.oldestMultiDB = xlrec.oldest_multi_db;
-            let first_off_blkno =
-                xlrec.start_trunc_off / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
-            let last_off_blkno =
-                xlrec.end_trunc_off / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
-            for blkno in first_off_blkno..last_off_blkno {
-                let mut blk = DecodedBkpBlock::new();
-                blk.forknum = pg_constants::PG_MXACT_OFFSETS_FORKNUM;
-                blk.blkno = blkno;
-                blk.will_init = true;
-                blocks.push(blk);
-            }
-            let first_mbr_blkno =
-                xlrec.start_trunc_memb / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-            let last_mbr_blkno =
-                xlrec.end_trunc_memb / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-            for blkno in first_mbr_blkno..last_mbr_blkno {
-                let mut blk = DecodedBkpBlock::new();
-                blk.forknum = pg_constants::PG_MXACT_MEMBERS_FORKNUM;
-                blk.blkno = blkno;
-                blk.will_init = true;
-                blocks.push(blk);
-            }
-        } else {
-            assert!(false);
-        }
-    } else if xlogrec.xl_rmid == pg_constants::RM_RELMAP_ID {
-        let xlrec = XlRelmapUpdate::decode(&mut buf);
-        let mut blk = DecodedBkpBlock::new();
-        blk.forknum = pg_constants::PG_FILENODEMAP_FORKNUM;
-        blk.rnode_spcnode = xlrec.tsid;
-        blk.rnode_dbnode = xlrec.dbid;
-        blk.rnode_relnode = 0;
-        blk.will_init = true;
-        blocks.push(blk);
-    } else if xlogrec.xl_rmid == pg_constants::RM_XLOG_ID {
-        let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-        if info == pg_constants::XLOG_NEXTOID {
-            let next_oid = buf.get_u32_le();
-            if next_oid > checkpoint.nextOid {
-                checkpoint.nextOid = next_oid;
-            }
-        }
     }
 
     DecodedWALRecord {

pageserver/src/walreceiver.rs

@@ -7,20 +7,20 @@
 //!
 
 use crate::page_cache;
-use crate::repository::*;
 use crate::waldecoder::*;
 use crate::PageServerConf;
 use crate::ZTimelineId;
-use anyhow::Error;
+use anyhow::{Error, Result};
 use lazy_static::lazy_static;
 use log::*;
 use postgres::fallible_iterator::FallibleIterator;
 use postgres::replication::ReplicationIter;
 use postgres::{Client, NoTls, SimpleQueryMessage, SimpleQueryRow};
+use postgres_ffi::pg_constants;
 use postgres_ffi::xlog_utils::*;
-use postgres_ffi::*;
 use postgres_protocol::message::backend::ReplicationMessage;
 use postgres_types::PgLsn;
+use std::cmp::{max, min};
 use std::collections::HashMap;
 use std::fs;
 use std::fs::{File, OpenOptions};
@@ -140,18 +140,13 @@ fn walreceiver_main(
     // If we had previously received WAL up to some point in the middle of a WAL record, we
     // better start from the end of last full WAL record, not in the middle of one. Hence,
     // use 'last_record_lsn' rather than 'last_valid_lsn' here.
-    let last_rec_lsn = timeline.get_last_record_lsn();
+    let mut last_rec_lsn = timeline.get_last_record_lsn();
     let mut startpoint = last_rec_lsn;
 
     if startpoint == Lsn(0) {
         error!("No previous WAL position");
     }
 
-    startpoint = Lsn::max(
-        startpoint,
-        Lsn(end_of_wal.0 & !(pg_constants::WAL_SEGMENT_SIZE as u64 - 1)),
-    );
-
     // There might be some padding after the last full record, skip it.
     //
     // FIXME: It probably would be better to always start streaming from the beginning
@@ -171,14 +166,6 @@ fn walreceiver_main(
 
     let mut waldecoder = WalStreamDecoder::new(startpoint);
 
-    let mut checkpoint = CheckPoint::new(startpoint.0, identify.timeline);
-    let checkpoint_tag = BufferTag::fork(pg_constants::PG_CHECKPOINT_FORKNUM);
-    if let Some(checkpoint_bytes) = timeline.get_page_image(checkpoint_tag, Lsn(0))? {
-        checkpoint = decode_checkpoint(checkpoint_bytes)?;
-        trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value);
-    } else {
-        error!("No checkpoint record was found in reposistory");
-    }
     while let Some(replication_message) = physical_stream.next()? {
         match replication_message {
             ReplicationMessage::XLogData(xlog_data) => {
@@ -187,6 +174,7 @@ fn walreceiver_main(
                 let data = xlog_data.data();
                 let startlsn = Lsn::from(xlog_data.wal_start());
                 let endlsn = startlsn + data.len() as u64;
+                let prev_last_rec_lsn = last_rec_lsn;
 
                 write_wal_file(startlsn, timelineid, pg_constants::WAL_SEGMENT_SIZE, data)?;
 
@@ -195,17 +183,13 @@ fn walreceiver_main(
                 waldecoder.feed_bytes(data);
 
                 while let Some((lsn, recdata)) = waldecoder.poll_decode()? {
-                    let old_checkpoint_bytes = encode_checkpoint(checkpoint);
-                    let decoded = decode_wal_record(&mut checkpoint, recdata.clone());
+                    let decoded = decode_wal_record(recdata.clone());
                     timeline.save_decoded_record(decoded, recdata, lsn)?;
 
-                    let new_checkpoint_bytes = encode_checkpoint(checkpoint);
-                    if new_checkpoint_bytes != old_checkpoint_bytes {
-                        timeline.put_page_image(checkpoint_tag, Lsn(0), new_checkpoint_bytes);
-                    }
                     // Now that this record has been handled, let the page cache know that
                     // it is up-to-date to this LSN
                     timeline.advance_last_record_lsn(lsn);
+                    last_rec_lsn = lsn;
                 }
 
                 // Update the last_valid LSN value in the page cache one more time. We updated
@@ -216,6 +200,33 @@ fn walreceiver_main(
                 // flush ptr.
                 timeline.advance_last_valid_lsn(endlsn);
 
+                // Somewhat arbitrarily, if we have at least 10 complete wal segments (16 MB each),
+                // "checkpoint" the repository to flush all the changes from WAL we've processed
+                // so far to disk. After this, we don't need the original WAL anymore, and it
+                // can be removed.
+                //
+                // TODO: We don't actually dare to remove the WAL. It's useful for debugging,
+                // and we might it for logical decoiding other things in the future. Although
+                // we should also be able to fetch it back from the WAL safekeepers or S3 if
+                // needed.
+                if prev_last_rec_lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE)
+                    != last_rec_lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE)
+                {
+                    info!("switched segment {} to {}", prev_last_rec_lsn, last_rec_lsn);
+                    let (oldest_segno, newest_segno) = find_wal_file_range(
+                        timelineid,
+                        pg_constants::WAL_SEGMENT_SIZE,
+                        last_rec_lsn,
+                    )?;
+
+                    if newest_segno - oldest_segno >= 10 {
+                        timeline.checkpoint()?;
+
+                        // TODO: This is where we could remove WAL older than last_rec_lsn.
+                        //remove_wal_files(timelineid, pg_constants::WAL_SEGMENT_SIZE, last_rec_lsn)?;
+                    }
+                }
+
                 if !caught_up && endlsn >= end_of_wal {
                     info!("caught up at LSN {}", endlsn);
                     caught_up = true;
@@ -252,6 +263,45 @@ fn walreceiver_main(
     Ok(())
 }
 
+fn find_wal_file_range(
+    timeline: ZTimelineId,
+    wal_seg_size: usize,
+    written_upto: Lsn,
+) -> Result<(u64, u64)> {
+    let written_upto_segno = written_upto.segment_number(wal_seg_size);
+
+    let mut oldest_segno = written_upto_segno;
+    let mut newest_segno = written_upto_segno;
+    // Scan the wal directory, and count how many WAL filed we could remove
+    let wal_dir = PathBuf::from(format!("timelines/{}/wal", timeline));
+    for entry in fs::read_dir(wal_dir)? {
+        let entry = entry?;
+        let path = entry.path();
+
+        if path.is_dir() {
+            continue;
+        }
+
+        let filename = path.file_name().unwrap().to_str().unwrap();
+
+        if IsXLogFileName(filename) {
+            let (segno, _tli) = XLogFromFileName(filename, wal_seg_size);
+
+            if segno > written_upto_segno {
+                // that's strange.
+                warn!("there is a WAL file from future at {}", path.display());
+                continue;
+            }
+
+            oldest_segno = min(oldest_segno, segno);
+            newest_segno = max(newest_segno, segno);
+        }
+    }
+    // FIXME: would be good to assert that there are no gaps in the WAL files
+
+    Ok((oldest_segno, newest_segno))
+}
+
 /// Data returned from the postgres `IDENTIFY_SYSTEM` command
 ///
 /// See the [postgres docs] for more details.

pageserver/src/walredo.rs

@@ -14,10 +14,8 @@
 //! TODO: Even though the postgres code runs in a separate process,
 //! it's not a secure sandbox.
 //!
-use byteorder::{ByteOrder, LittleEndian};
-use bytes::{Buf, BufMut, Bytes, BytesMut};
+use bytes::{BufMut, Bytes, BytesMut};
 use log::*;
-use std::assert;
 use std::cell::RefCell;
 use std::fs;
 use std::fs::OpenOptions;
@@ -33,15 +31,12 @@ use tokio::io::AsyncBufReadExt;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::process::{ChildStdin, ChildStdout, Command};
 use tokio::time::timeout;
+use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
 
 use crate::repository::BufferTag;
 use crate::repository::WALRecord;
-use crate::waldecoder::{MultiXactId, XlMultiXactCreate};
 use crate::PageServerConf;
-use postgres_ffi::nonrelfile_utils::transaction_id_set_status;
-use postgres_ffi::pg_constants;
-use postgres_ffi::xlog_utils::XLogRecord;
 
 ///
 /// WAL Redo Manager is responsible for replaying WAL records.
@@ -63,6 +58,24 @@ pub trait WalRedoManager: Send + Sync {
     ) -> Result<Bytes, WalRedoError>;
 }
 
+///
+/// A dummy WAL Redo Manager implementation that doesn't allow replaying
+/// anything. Currently used during bootstrapping (zenith init), to create
+/// a Repository object without launching the real WAL redo process.
+///
+pub struct DummyRedoManager {}
+impl crate::walredo::WalRedoManager for DummyRedoManager {
+    fn request_redo(
+        &self,
+        _tag: BufferTag,
+        _lsn: Lsn,
+        _base_img: Option<Bytes>,
+        _records: Vec<WALRecord>,
+    ) -> Result<Bytes, WalRedoError> {
+        Err(WalRedoError::InvalidState)
+    }
+}
+
 static TIMEOUT: Duration = Duration::from_secs(20);
 
 ///
@@ -97,6 +110,9 @@ struct WalRedoRequest {
 pub enum WalRedoError {
     #[error(transparent)]
     IoError(#[from] std::io::Error),
+
+    #[error("cannot perform WAL redo now")]
+    InvalidState,
 }
 
 ///
@@ -169,25 +185,6 @@ impl WalRedoManager for PostgresRedoManager {
     }
 }
 
-fn mx_offset_to_flags_offset(xid: MultiXactId) -> usize {
-    return ((xid / pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP as u32) as u16
-        % pg_constants::MULTIXACT_MEMBERGROUPS_PER_PAGE
-        * pg_constants::MULTIXACT_MEMBERGROUP_SIZE) as usize;
-}
-
-fn mx_offset_to_flags_bitshift(xid: MultiXactId) -> u16 {
-    return (xid as u16) % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP
-        * pg_constants::MXACT_MEMBER_BITS_PER_XACT;
-}
-
-/* Location (byte offset within page) of TransactionId of given member */
-fn mx_offset_to_member_offset(xid: MultiXactId) -> usize {
-    return mx_offset_to_flags_offset(xid)
-        + (pg_constants::MULTIXACT_FLAGBYTES_PER_GROUP
-            + (xid as u16 % pg_constants::MULTIXACT_MEMBERS_PER_MEMBERGROUP) * 4)
-            as usize;
-}
-
 ///
 /// WAL redo thread
 ///
@@ -256,213 +253,7 @@ impl PostgresRedoManagerInternal {
         let start = Instant::now();
 
         let apply_result: Result<Bytes, Error>;
-        if tag.rel.forknum > pg_constants::INIT_FORKNUM {
-            const ZERO_PAGE: [u8; 8192] = [0u8; 8192];
-            let mut page = BytesMut::new();
-            if let Some(fpi) = base_img {
-                page.extend_from_slice(&fpi[..]);
-            } else {
-                page.extend_from_slice(&ZERO_PAGE);
-            }
-            for record in records {
-                let mut buf = record.rec.clone();
-
-                // 1. Parse XLogRecord struct
-                // FIXME: refactor to avoid code duplication.
-                let xlogrec = XLogRecord::from_bytes(&mut buf);
-
-                //move to main data
-                // TODO probably, we should store some records in our special format
-                // to avoid this weird parsing on replay
-                let skip = (record.main_data_offset - pg_constants::SIZEOF_XLOGRECORD) as usize;
-                if buf.remaining() > skip {
-                    buf.advance(skip);
-                }
-
-                if xlogrec.xl_rmid == pg_constants::RM_CLOG_ID {
-                    let info = xlogrec.xl_info & !pg_constants::XLR_INFO_MASK;
-                    if info == pg_constants::CLOG_ZEROPAGE {
-                        page.copy_from_slice(&ZERO_PAGE);
-                    }
-                } else if xlogrec.xl_rmid == pg_constants::RM_XACT_ID {
-                    let info = xlogrec.xl_info & pg_constants::XLOG_XACT_OPMASK;
-                    let mut status = 0;
-                    if info == pg_constants::XLOG_XACT_COMMIT || info == pg_constants::XLOG_XACT_COMMIT_PREPARED {
-                        status = pg_constants::TRANSACTION_STATUS_COMMITTED;
-						if info == pg_constants::XLOG_XACT_COMMIT {
-							transaction_id_set_status(xlogrec.xl_xid, status, &mut page);
-						}
-                        //handle subtrans
-                        let _xact_time = buf.get_i64_le();
-                        let mut xinfo = 0;
-                        if xlogrec.xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
-                            xinfo = buf.get_u32_le();
-                            if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
-                                let _dbid = buf.get_u32_le();
-                                let _tsid = buf.get_u32_le();
-                            }
-                        }
-
-                        if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
-                            let nsubxacts = buf.get_i32_le();
-                            for _i in 0..nsubxacts {
-                                let subxact = buf.get_u32_le();
-                                let blkno = subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-                                // only update xids on the requested page
-                                if tag.blknum == blkno {
-                                    status = pg_constants::TRANSACTION_STATUS_SUB_COMMITTED;
-                                    transaction_id_set_status(subxact, status, &mut page);
-                                }
-                            }
-                        }
-						if info == pg_constants::XLOG_XACT_COMMIT_PREPARED {
-							if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
-								let nrels = buf.get_i32_le();
-								for _i in 0..nrels {
-									let spcnode = buf.get_u32_le();
-									let dbnode = buf.get_u32_le();
-									let relnode = buf.get_u32_le();
-									//TODO handle this too?
-									trace!(
-										"XLOG_XACT_COMMIT relfilenode {}/{}/{}",
-										spcnode,
-										dbnode,
-										relnode
-									);
-								}
-							}
-							if xinfo & pg_constants::XACT_XINFO_HAS_INVALS != 0 {
-								let nmsgs = buf.get_i32_le();
-								for _i in 0..nmsgs {
-									let sizeof_shared_invalidation_message = 0;
-									buf.advance(sizeof_shared_invalidation_message);
-								}
-							}
-							assert!((xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE) != 0);
-							let xid = buf.get_u32_le();
-							transaction_id_set_status(xid, status, &mut page);
-						}
-                    } else if info == pg_constants::XLOG_XACT_ABORT || info == pg_constants::XLOG_XACT_ABORT_PREPARED {
-                        status = pg_constants::TRANSACTION_STATUS_ABORTED;
-						if info == pg_constants::XLOG_XACT_ABORT {
-							transaction_id_set_status(xlogrec.xl_xid, status, &mut page);
-						}
-                        //handle subtrans
-                        let _xact_time = buf.get_i64_le();
-                        let mut xinfo = 0;
-                        if xlogrec.xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
-                            xinfo = buf.get_u32_le();
-                            if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
-                                let _dbid = buf.get_u32_le();
-                                let _tsid = buf.get_u32_le();
-                            }
-                        }
-
-                        if xinfo & pg_constants::XACT_XINFO_HAS_SUBXACTS != 0 {
-                            let nsubxacts = buf.get_i32_le();
-                            for _i in 0..nsubxacts {
-                                let subxact = buf.get_u32_le();
-                                let blkno = subxact as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
-                                // only update xids on the requested page
-                                if tag.blknum == blkno {
-                                    status = pg_constants::TRANSACTION_STATUS_ABORTED;
-                                    transaction_id_set_status(subxact, status, &mut page);
-                                }
-                            }
-                        }
-						if info == pg_constants::XLOG_XACT_ABORT_PREPARED {
-							if xinfo & pg_constants::XACT_XINFO_HAS_RELFILENODES != 0 {
-								let nrels = buf.get_i32_le();
-								for _i in 0..nrels {
-									let spcnode = buf.get_u32_le();
-									let dbnode = buf.get_u32_le();
-									let relnode = buf.get_u32_le();
-									//TODO handle this too?
-									trace!(
-										"XLOG_XACT_COMMIT relfilenode {}/{}/{}",
-										spcnode,
-										dbnode,
-										relnode
-									);
-								}
-							}
-							if xinfo & pg_constants::XACT_XINFO_HAS_INVALS != 0 {
-								let nmsgs = buf.get_i32_le();
-								for _i in 0..nmsgs {
-									let sizeof_shared_invalidation_message = 0;
-									buf.advance(sizeof_shared_invalidation_message);
-								}
-							}
-							assert!((xinfo & pg_constants::XACT_XINFO_HAS_TWOPHASE) != 0);
-							let xid = buf.get_u32_le();
-							transaction_id_set_status(xid, status, &mut page);
-						}
-                    } else if info == pg_constants::XLOG_XACT_PREPARE {
-						info!("Apply prepare {} record", xlogrec.xl_xid);
-						page.clear();
-						page.extend_from_slice(&buf[..]);
-					} else {
-                        error!("handle_apply_request for RM_XACT_ID-{} NOT SUPPORTED YET. RETURN. lsn {} main_data_offset {}, rec.len {}",
-                               status,
-                               record.lsn,
-                               record.main_data_offset, record.rec.len());
-                    }
-                } else if xlogrec.xl_rmid == pg_constants::RM_MULTIXACT_ID {
-                    let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
-                    if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE {
-                        page.copy_from_slice(&ZERO_PAGE);
-                    } else if info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE {
-                        page.copy_from_slice(&ZERO_PAGE);
-                    } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID {
-                        let xlrec = XlMultiXactCreate::decode(&mut buf);
-                        if tag.rel.forknum == pg_constants::PG_MXACT_OFFSETS_FORKNUM {
-                            let offs = (xlrec.mid % pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32
-                                * 4) as usize;
-                            LittleEndian::write_u32(&mut page[offs..offs + 4], xlrec.moff);
-                        } else {
-                            assert!(tag.rel.forknum == pg_constants::PG_MXACT_MEMBERS_FORKNUM);
-                            for i in 0..xlrec.nmembers {
-                                let blkno = i / pg_constants::MULTIXACT_MEMBERS_PER_PAGE as u32;
-                                if blkno == tag.blknum {
-                                    // update only target block
-                                    let offset = xlrec.moff + i;
-                                    let memberoff = mx_offset_to_member_offset(offset);
-                                    let flagsoff = mx_offset_to_flags_offset(offset);
-                                    let bshift = mx_offset_to_flags_bitshift(offset);
-                                    let mut flagsval =
-                                        LittleEndian::read_u32(&page[flagsoff..flagsoff + 4]);
-                                    flagsval &=
-                                        !(((1 << pg_constants::MXACT_MEMBER_BITS_PER_XACT) - 1)
-                                            << bshift);
-                                    flagsval |= xlrec.members[i as usize].status << bshift;
-                                    LittleEndian::write_u32(
-                                        &mut page[flagsoff..flagsoff + 4],
-                                        flagsval,
-                                    );
-                                    LittleEndian::write_u32(
-                                        &mut page[memberoff..memberoff + 4],
-                                        xlrec.members[i as usize].xid,
-                                    );
-                                }
-                            }
-                        }
-                    } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID {
-                        // empty page image indicates that this SLRU page is truncated and can be removed by GC
-                        page.clear();
-                    } else {
-                        assert!(false);
-                    }
-                } else if xlogrec.xl_rmid == pg_constants::RM_RELMAP_ID {
-                    page.clear();
-                    page.extend_from_slice(&buf[12..]); // skip xl_relmap_update
-                    assert!(page.len() == 512); // size of pg_filenode.map
-                }
-            }
-
-            apply_result = Ok::<Bytes, Error>(page.freeze());
-        } else {
-            apply_result = process.apply_wal_records(tag, base_img, records).await;
-        }
+        apply_result = process.apply_wal_records(tag, base_img, records).await;
 
         let duration = start.elapsed();
 
@@ -665,9 +456,26 @@ fn build_begin_redo_for_block_msg(tag: BufferTag) -> Bytes {
 
     buf.put_u8(b'B');
     buf.put_u32(len as u32);
-    tag.pack(&mut buf);
 
-    assert!(buf.len() == 1 + len);
+    // FIXME: this is a temporary hack that should go away when we refactor
+    // the postgres protocol serialization + handlers.
+    //
+    // BytesMut is a dynamic growable buffer, used a lot in tokio code but
+    // not in the std library. To write to a BytesMut from a serde serializer,
+    // we need to either:
+    // - pre-allocate the required buffer space. This is annoying because we
+    //   shouldn't care what the exact serialized size is-- that's the
+    //   serializer's job.
+    // - Or, we need to create a temporary "writer" (which implements the
+    //   `Write` trait). It's a bit awkward, because the writer consumes the
+    //   underlying BytesMut, and we need to extract it later with
+    //   `into_inner`.
+    let mut writer = buf.writer();
+    tag.ser_into(&mut writer)
+        .expect("serialize BufferTag should always succeed");
+    let buf = writer.into_inner();
+
+    debug_assert!(buf.len() == 1 + len);
 
     buf.freeze()
 }
@@ -680,10 +488,13 @@ fn build_push_page_msg(tag: BufferTag, base_img: Bytes) -> Bytes {
 
     buf.put_u8(b'P');
     buf.put_u32(len as u32);
-    tag.pack(&mut buf);
+    let mut writer = buf.writer();
+    tag.ser_into(&mut writer)
+        .expect("serialize BufferTag should always succeed");
+    let mut buf = writer.into_inner();
     buf.put(base_img);
 
-    assert!(buf.len() == 1 + len);
+    debug_assert!(buf.len() == 1 + len);
 
     buf.freeze()
 }
@@ -697,7 +508,7 @@ fn build_apply_record_msg(endlsn: Lsn, rec: Bytes) -> Bytes {
     buf.put_u64(endlsn.0);
     buf.put(rec);
 
-    assert!(buf.len() == 1 + len);
+    debug_assert!(buf.len() == 1 + len);
 
     buf.freeze()
 }
@@ -708,9 +519,12 @@ fn build_get_page_msg(tag: BufferTag) -> Bytes {
 
     buf.put_u8(b'G');
     buf.put_u32(len as u32);
-    tag.pack(&mut buf);
+    let mut writer = buf.writer();
+    tag.ser_into(&mut writer)
+        .expect("serialize BufferTag should always succeed");
+    let buf = writer.into_inner();
 
-    assert!(buf.len() == 1 + len);
+    debug_assert!(buf.len() == 1 + len);
 
     buf.freeze()
 }

postgres_ffi/build.rs

@@ -18,8 +18,6 @@ fn main() {
         // included header files changed.
         .parse_callbacks(Box::new(bindgen::CargoCallbacks))
         .whitelist_type("ControlFileData")
-        .whitelist_type("CheckPoint")
-        .whitelist_type("FullTransactionId")
         .whitelist_var("PG_CONTROL_FILE_SIZE")
         .whitelist_var("PG_CONTROLFILEDATA_OFFSETOF_CRC")
         .whitelist_type("DBState")

postgres_ffi/src/lib.rs

@@ -3,7 +3,6 @@
 #![allow(non_snake_case)]
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
 
-pub mod nonrelfile_utils;
 pub mod pg_constants;
 pub mod relfile_utils;
 pub mod xlog_utils;
@@ -12,7 +11,6 @@ use bytes::{Buf, Bytes, BytesMut};
 
 // sizeof(ControlFileData)
 const SIZEOF_CONTROLDATA: usize = std::mem::size_of::<ControlFileData>();
-const SIZEOF_CHECKPOINT: usize = std::mem::size_of::<CheckPoint>();
 const OFFSETOF_CRC: usize = PG_CONTROLFILEDATA_OFFSETOF_CRC as usize;
 
 impl ControlFileData {
@@ -71,42 +69,3 @@ pub fn encode_pg_control(controlfile: ControlFileData) -> Bytes {
 
     buf.into()
 }
-
-pub fn encode_checkpoint(checkpoint: CheckPoint) -> Bytes {
-    let b: [u8; SIZEOF_CHECKPOINT];
-    b = unsafe { std::mem::transmute::<CheckPoint, [u8; SIZEOF_CHECKPOINT]>(checkpoint) };
-    return Bytes::copy_from_slice(&b[..]);
-}
-
-pub fn decode_checkpoint(mut buf: Bytes) -> Result<CheckPoint, anyhow::Error> {
-    let mut b = [0u8; SIZEOF_CHECKPOINT];
-    buf.copy_to_slice(&mut b);
-    let checkpoint: CheckPoint;
-    checkpoint = unsafe { std::mem::transmute::<[u8; SIZEOF_CHECKPOINT], CheckPoint>(b) };
-    Ok(checkpoint)
-}
-
-impl CheckPoint {
-    pub fn new(lsn: u64, timeline: u32) -> CheckPoint {
-        CheckPoint {
-            redo: lsn,
-            ThisTimeLineID: timeline,
-            PrevTimeLineID: timeline,
-            fullPageWrites: true, // TODO: get actual value of full_page_writes
-            nextXid: FullTransactionId {
-                value: pg_constants::FIRST_NORMAL_TRANSACTION_ID as u64,
-            }, // TODO: handle epoch?
-            nextOid: pg_constants::FIRST_BOOTSTRAP_OBJECT_ID,
-            nextMulti: 1,
-            nextMultiOffset: 0,
-            oldestXid: pg_constants::FIRST_NORMAL_TRANSACTION_ID,
-            oldestXidDB: 0,
-            oldestMulti: 1,
-            oldestMultiDB: 0,
-            time: 0,
-            oldestCommitTsXid: 0,
-            newestCommitTsXid: 0,
-            oldestActiveXid: pg_constants::INVALID_TRANSACTION_ID,
-        }
-    }
-}

postgres_ffi/src/nonrelfile_utils.rs

@@ -28,5 +28,5 @@ pub fn transaction_id_get_status(xid: u32, page: &[u8]) -> u8 {
     let bshift: u8 =
         ((xid % pg_constants::CLOG_XACTS_PER_BYTE) * pg_constants::CLOG_BITS_PER_XACT as u32) as u8;
 
-    return ((page[byteno] >> bshift) & pg_constants::CLOG_XACT_BITMASK) as u8;
+    ((page[byteno] >> bshift) & pg_constants::CLOG_XACT_BITMASK) as u8
 }

postgres_ffi/src/pg_constants.rs

@@ -15,31 +15,13 @@ pub const MAIN_FORKNUM: u8 = 0;
 pub const FSM_FORKNUM: u8 = 1;
 pub const VISIBILITYMAP_FORKNUM: u8 = 2;
 pub const INIT_FORKNUM: u8 = 3;
-// Special values for non-rel files' tags (Zenith-specific)
-//Special values for non-rel files' tags
-pub const PG_CONTROLFILE_FORKNUM: u8 = 42;
-pub const PG_FILENODEMAP_FORKNUM: u8 = 43;
-pub const PG_XACT_FORKNUM: u8 = 44;
-pub const PG_MXACT_OFFSETS_FORKNUM: u8 = 45;
-pub const PG_MXACT_MEMBERS_FORKNUM: u8 = 46;
-pub const PG_TWOPHASE_FORKNUM: u8 = 47;
-pub const PG_CHECKPOINT_FORKNUM: u8 = 48;
+
+pub const ROCKSDB_SPECIAL_FORKNUM: u8 = 50;
 
 // From storage_xlog.h
 pub const SMGR_TRUNCATE_HEAP: u32 = 0x0001;
-
-// from pg_config.h. These can be changed with configure options --with-blocksize=BLOCKSIZE and
-// --with-segsize=SEGSIZE, but assume the defaults for now.
-pub const BLCKSZ: u16 = 8192;
-pub const RELSEG_SIZE: u32 = 1024 * 1024 * 1024 / (BLCKSZ as u32);
-
-//
-// constants from clog.h
-//
-pub const CLOG_XACTS_PER_BYTE: u32 = 4;
-pub const CLOG_XACTS_PER_PAGE: u32 = BLCKSZ as u32 * CLOG_XACTS_PER_BYTE;
-pub const CLOG_BITS_PER_XACT: u8 = 2;
-pub const CLOG_XACT_BITMASK: u8 = (1 << CLOG_BITS_PER_XACT) - 1;
+pub const SMGR_TRUNCATE_VM: u32 = 0x0002;
+pub const SMGR_TRUNCATE_FSM: u32 = 0x0004;
 
 //
 // Constants from visbilitymap.h
@@ -48,23 +30,14 @@ pub const SIZE_OF_PAGE_HEADER: u16 = 24;
 pub const BITS_PER_HEAPBLOCK: u16 = 2;
 pub const HEAPBLOCKS_PER_PAGE: u16 = (BLCKSZ - SIZE_OF_PAGE_HEADER) * 8 / BITS_PER_HEAPBLOCK;
 
-pub const TRANSACTION_STATUS_IN_PROGRESS: u8 = 0x00;
 pub const TRANSACTION_STATUS_COMMITTED: u8 = 0x01;
 pub const TRANSACTION_STATUS_ABORTED: u8 = 0x02;
 pub const TRANSACTION_STATUS_SUB_COMMITTED: u8 = 0x03;
 
-pub const CLOG_ZEROPAGE: u8 = 0x00;
-pub const CLOG_TRUNCATE: u8 = 0x10;
-
 // From xact.h
 pub const XLOG_XACT_COMMIT: u8 = 0x00;
 pub const XLOG_XACT_PREPARE: u8 = 0x10;
 pub const XLOG_XACT_ABORT: u8 = 0x20;
-pub const XLOG_XACT_COMMIT_PREPARED: u8 = 0x30;
-pub const XLOG_XACT_ABORT_PREPARED: u8 = 0x40;
-
-// From srlu.h
-pub const SLRU_PAGES_PER_SEGMENT: u32 = 32;
 
 /* mask for filtering opcodes out of xl_info */
 pub const XLOG_XACT_OPMASK: u8 = 0x70;
@@ -85,29 +58,9 @@ pub const XACT_XINFO_HAS_TWOPHASE: u32 = 1u32 << 4;
 // pub const XACT_XINFO_HAS_GID: u32 = 1u32 << 7;
 
 // From pg_control.h and rmgrlist.h
-pub const XLOG_NEXTOID: u8 = 0x30;
 pub const XLOG_SWITCH: u8 = 0x40;
 pub const XLOG_SMGR_TRUNCATE: u8 = 0x20;
 
-// From multixact.h
-pub const XLOG_MULTIXACT_ZERO_OFF_PAGE: u8 = 0x00;
-pub const XLOG_MULTIXACT_ZERO_MEM_PAGE: u8 = 0x10;
-pub const XLOG_MULTIXACT_CREATE_ID: u8 = 0x20;
-pub const XLOG_MULTIXACT_TRUNCATE_ID: u8 = 0x30;
-
-pub const MULTIXACT_OFFSETS_PER_PAGE: u16 = BLCKSZ / 4;
-pub const MXACT_MEMBER_BITS_PER_XACT: u16 = 8;
-pub const MXACT_MEMBER_FLAGS_PER_BYTE: u16 = 1;
-pub const MULTIXACT_FLAGBYTES_PER_GROUP: u16 = 4;
-pub const MULTIXACT_MEMBERS_PER_MEMBERGROUP: u16 =
-    MULTIXACT_FLAGBYTES_PER_GROUP * MXACT_MEMBER_FLAGS_PER_BYTE;
-/* size in bytes of a complete group */
-pub const MULTIXACT_MEMBERGROUP_SIZE: u16 =
-    4 * MULTIXACT_MEMBERS_PER_MEMBERGROUP + MULTIXACT_FLAGBYTES_PER_GROUP;
-pub const MULTIXACT_MEMBERGROUPS_PER_PAGE: u16 = BLCKSZ / MULTIXACT_MEMBERGROUP_SIZE;
-pub const MULTIXACT_MEMBERS_PER_PAGE: u16 =
-    MULTIXACT_MEMBERGROUPS_PER_PAGE * MULTIXACT_MEMBERS_PER_MEMBERGROUP;
-
 // From heapam_xlog.h
 pub const XLOG_HEAP_INSERT: u8 = 0x00;
 pub const XLOG_HEAP_DELETE: u8 = 0x10;
@@ -146,6 +99,11 @@ pub const XLOG_TBLSPC_DROP: u8 = 0x10;
 
 pub const SIZEOF_XLOGRECORD: u32 = 24;
 
+// from pg_config.h. These can be changed with configure options --with-blocksize=BLOCKSIZE and
+// --with-segsize=SEGSIZE, but assume the defaults for now.
+pub const BLCKSZ: u16 = 8192;
+pub const RELSEG_SIZE: u32 = 1024 * 1024 * 1024 / (BLCKSZ as u32);
+
 //
 // from xlogrecord.h
 //
@@ -168,11 +126,5 @@ pub const BKPIMAGE_HAS_HOLE: u8 = 0x01; /* page image has "hole" */
 pub const BKPIMAGE_IS_COMPRESSED: u8 = 0x02; /* page image is compressed */
 pub const BKPIMAGE_APPLY: u8 = 0x04; /* page image should be restored during replay */
 
-/* From transam.h */
-pub const FIRST_NORMAL_TRANSACTION_ID: u32 = 3;
-pub const INVALID_TRANSACTION_ID: u32 = 0;
-pub const FIRST_BOOTSTRAP_OBJECT_ID: u32 = 12000;
-pub const FIRST_NORMAL_OBJECT_ID: u32 = 16384;
-
 /* FIXME: pageserver should request wal_seg_size from compute node */
 pub const WAL_SEGMENT_SIZE: usize = 16 * 1024 * 1024;

postgres_ffi/src/relfile_utils.rs

@@ -38,16 +38,6 @@ pub fn forknumber_to_name(forknum: u8) -> Option<&'static str> {
         pg_constants::FSM_FORKNUM => Some("fsm"),
         pg_constants::VISIBILITYMAP_FORKNUM => Some("vm"),
         pg_constants::INIT_FORKNUM => Some("init"),
-
-        // These should not appear in WAL records, but we use them internally,
-        // and need to be prepared to print them out in log messages and such
-        pg_constants::PG_CONTROLFILE_FORKNUM => Some("controlfile"),
-        pg_constants::PG_FILENODEMAP_FORKNUM => Some("filenodemap"),
-        pg_constants::PG_XACT_FORKNUM => Some("xact"),
-        pg_constants::PG_MXACT_OFFSETS_FORKNUM => Some("mxact_offsets"),
-        pg_constants::PG_MXACT_MEMBERS_FORKNUM => Some("mxact_members"),
-        pg_constants::PG_TWOPHASE_FORKNUM => Some("twophase"),
-
         _ => Some("UNKNOWN FORKNUM"),
     }
 }

postgres_ffi/src/xlog_utils.rs

@@ -199,33 +199,31 @@ pub fn find_end_of_wal(
     let mut high_tli: TimeLineID = 0;
     let mut high_ispartial = false;
 
-    for entry in fs::read_dir(data_dir).unwrap() {
-        if let Ok(entry) = entry {
-            let ispartial: bool;
-            let entry_name = entry.file_name();
-            let fname = entry_name.to_str().unwrap();
-            /*
-             * Check if the filename looks like an xlog file, or a .partial file.
-             */
-            if IsXLogFileName(fname) {
-                ispartial = false;
-            } else if IsPartialXLogFileName(fname) {
-                ispartial = true;
-            } else {
-                continue;
-            }
-            let (segno, tli) = XLogFromFileName(fname, wal_seg_size);
-            if !ispartial && entry.metadata().unwrap().len() != wal_seg_size as u64 {
-                continue;
-            }
-            if segno > high_segno
-                || (segno == high_segno && tli > high_tli)
-                || (segno == high_segno && tli == high_tli && high_ispartial && !ispartial)
-            {
-                high_segno = segno;
-                high_tli = tli;
-                high_ispartial = ispartial;
-            }
+    for entry in fs::read_dir(data_dir).unwrap().flatten() {
+        let ispartial: bool;
+        let entry_name = entry.file_name();
+        let fname = entry_name.to_str().unwrap();
+        /*
+         * Check if the filename looks like an xlog file, or a .partial file.
+         */
+        if IsXLogFileName(fname) {
+            ispartial = false;
+        } else if IsPartialXLogFileName(fname) {
+            ispartial = true;
+        } else {
+            continue;
+        }
+        let (segno, tli) = XLogFromFileName(fname, wal_seg_size);
+        if !ispartial && entry.metadata().unwrap().len() != wal_seg_size as u64 {
+            continue;
+        }
+        if segno > high_segno
+            || (segno == high_segno && tli > high_tli)
+            || (segno == high_segno && tli == high_tli && high_ispartial && !ispartial)
+        {
+            high_segno = segno;
+            high_tli = tli;
+            high_ispartial = ispartial;
         }
     }
     if high_segno > 0 {

test_runner/batch_others/test_multixact.py

@@ -0,0 +1,71 @@
+import pytest
+import os
+import psycopg2
+import multiprocessing
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+#
+# Test multixact state after branching
+# Now this test is very minimalistic -
+# it only checks next_multixact_id field in restored pg_control,
+# since we don't have functions to check multixact internals.
+#
+
+def runQuery(connstr):
+    con = psycopg2.connect(connstr) 
+    con.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    cur = con.cursor()
+    cur.execute('select * from t1 for key share;') 
+
+
+def test_multixact(pageserver, postgres, pg_bin, zenith_cli, base_dir):
+
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_multixact", "empty"])
+    pg = postgres.create_start('test_multixact')
+
+    print("postgres is running on 'test_multixact' branch")
+    pg_conn = psycopg2.connect(pg.connstr())
+    pg_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    cur = pg_conn.cursor()
+
+    cur.execute('CREATE TABLE t1(i int primary key);'
+    'INSERT INTO t1 select * from generate_series(1,100);')
+
+    cur.execute('SELECT next_multixact_id FROM pg_control_checkpoint();')
+    next_multixact_id_old = cur.fetchone()[0]
+
+    # Lock entries in parallel connections to set multixact
+    nclients = 3
+    pool = multiprocessing.Pool(nclients)
+    args = [pg.connstr()] * nclients
+    pool.map(runQuery, args)
+    pool.close()
+    pool.join()
+
+    # force wal flush
+    cur.execute('checkpoint')
+
+    cur.execute('SELECT next_multixact_id, pg_current_wal_flush_lsn() FROM pg_control_checkpoint();')
+    res = cur.fetchone()
+    next_multixact_id = res[0]
+    lsn = res[1]
+
+    # Ensure that we did lock some tuples
+    assert(int(next_multixact_id) > int(next_multixact_id_old))
+
+    # Branch at this point
+    zenith_cli.run(["branch", "test_multixact_new", "test_multixact@"+lsn]);
+    pg_new = postgres.create_start('test_multixact_new')
+
+    print("postgres is running on 'test_multixact_new' branch")
+    pg_new_conn = psycopg2.connect(pg_new.connstr())
+    pg_new_conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    cur_new = pg_new_conn.cursor()
+
+    cur_new.execute('SELECT next_multixact_id FROM pg_control_checkpoint();')
+    next_multixact_id_new = cur_new.fetchone()[0]
+
+    # Check that we restored pg_controlfile correctly
+    assert(next_multixact_id_new == next_multixact_id)

test_runner/batch_others/test_twophase.py

@@ -29,16 +29,8 @@ def test_twophase(zenith_cli, pageserver, postgres, pg_bin):
     cur.execute("INSERT INTO foo VALUES ('two')");
     cur.execute("PREPARE TRANSACTION 'insert_two'");
 
-    cur.execute('BEGIN');
-    cur.execute("INSERT INTO foo VALUES ('three')");
-    cur.execute("PREPARE TRANSACTION 'insert_three'");
-    cur.execute("COMMIT PREPARED 'insert_three'");
-
-    cur.execute('SELECT pg_current_wal_insert_lsn()');
-    lsn = cur.fetchone()[0]
-
     # Create a branch with the transaction in prepared state
-    zenith_cli.run(["branch", "test_twophase_prepared", "test_twophase@"+lsn]);
+    zenith_cli.run(["branch", "test_twophase_prepared", "test_twophase"]);
 
     pg2 = postgres.create_start('test_twophase_prepared', ['max_prepared_transactions=5'])
     conn2 = psycopg2.connect(pg2.connstr());
@@ -50,9 +42,9 @@ def test_twophase(zenith_cli, pageserver, postgres, pg_bin):
     cur2.execute("ROLLBACK PREPARED 'insert_two'");
 
     cur2.execute('SELECT * FROM foo');
-    assert(cur2.fetchall() == [('one',),('three',)]);
+    assert(cur2.fetchall() == [('one',)]);
 
     # Neither insert is visible on the original branch, the transactions are still
     # in prepared state there.
     cur.execute('SELECT * FROM foo');
-    assert(cur.fetchall() == [('three',)]);
+    assert(cur.fetchall() == []);

test_runner/zenith_regress/expected/zenith-truncate.out

@@ -0,0 +1,9 @@
+create table tt(x integer);
+insert into tt values (generate_series(1,10000));
+delete from tt;
+vacuum tt;
+insert into tt values (generate_series(1,10000));
+delete from tt;
+vacuum tt;
+insert into tt values (generate_series(1,10000));
+drop table tt;

test_runner/zenith_regress/parallel_schedule

@@ -9,3 +9,4 @@ test: zenith-cid
 test: zenith-rel-truncate
 test: zenith-clog
 test: zenith-vacuum-full
+test: zenith-truncate

test_runner/zenith_regress/serial_schedule

@@ -4,3 +4,4 @@ test: zenith-cid
 test: zenith-rel-truncate
 test: zenith-clog
 test: zenith-vacuum-full
+test: zenith-truncate

test_runner/zenith_regress/sql/zenith-truncate.sql

@@ -0,0 +1,9 @@
+create table tt(x integer);
+insert into tt values (generate_series(1,10000));
+delete from tt;
+vacuum tt;
+insert into tt values (generate_series(1,10000));
+delete from tt;
+vacuum tt;
+insert into tt values (generate_series(1,10000));
+drop table tt;

walkeeper/src/receive_wal.rs

@@ -22,7 +22,7 @@ use crate::WalAcceptorConf;
 use pageserver::ZTimelineId;
 use postgres_ffi::xlog_utils::{TimeLineID, XLogFileName, MAX_SEND_SIZE, XLOG_BLCKSZ};
 
-pub const SK_MAGIC: u32 = 0xCafeCeefu32;
+pub const SK_MAGIC: u32 = 0xcafeceefu32;
 pub const SK_FORMAT_VERSION: u32 = 1;
 const SK_PROTOCOL_VERSION: u32 = 1;
 const UNKNOWN_SERVER_VERSION: u32 = 0;

walkeeper/src/replication.rs

@@ -87,7 +87,7 @@ impl ReplicationConn {
     /// Helper function that parses a pair of LSNs.
     fn parse_start_stop(cmd: &[u8]) -> Result<(Lsn, Lsn)> {
         let re = Regex::new(r"([[:xdigit:]]+/[[:xdigit:]]+)").unwrap();
-        let caps = re.captures_iter(str::from_utf8(&cmd[..])?);
+        let caps = re.captures_iter(str::from_utf8(cmd)?);
         let mut lsns = caps.map(|cap| cap[1].parse::<Lsn>());
         let start_pos = lsns
             .next()
@@ -107,10 +107,10 @@ impl ReplicationConn {
 
         // If that failed, try it without the .partial extension.
         match File::open(&wal_file_path) {
-            Ok(opened_file) => return Ok(opened_file),
+            Ok(opened_file) => Ok(opened_file),
             Err(e) => {
                 error!("Failed to open log file {:?}: {}", &wal_file_path, e);
-                return Err(e.into());
+                Err(e.into())
             }
         }
     }

walkeeper/src/s3_offload.rs

@@ -12,8 +12,7 @@ use std::collections::HashSet;
 use std::env;
 use std::fs::{self, File};
 use std::io::prelude::*;
-use std::iter::FromIterator;
-use std::path::PathBuf;
+use std::path::Path;
 use std::time::SystemTime;
 use tokio::runtime;
 use tokio::time::sleep;
@@ -42,7 +41,7 @@ pub fn thread_main(conf: WalAcceptorConf) {
 async fn offload_files(
     bucket: &Bucket,
     listing: &HashSet<String>,
-    dir_path: &PathBuf,
+    dir_path: &Path,
     conf: &WalAcceptorConf,
 ) -> Result<u64> {
     let horizon = SystemTime::now() - conf.ttl.unwrap();
@@ -93,11 +92,10 @@ async fn main_loop(conf: &WalAcceptorConf) -> Result<()> {
         let results = bucket
             .list("walarchive/".to_string(), Some("".to_string()))
             .await?;
-        let listing = HashSet::from_iter(
-            results
-                .iter()
-                .flat_map(|b| b.contents.iter().map(|o| o.key.clone())),
-        );
+        let listing = results
+            .iter()
+            .flat_map(|b| b.contents.iter().map(|o| o.key.clone()))
+            .collect();
 
         let n = offload_files(&bucket, &listing, &conf.data_dir, conf).await?;
         info!("Offload {} files to S3", n);

walkeeper/src/send_wal.rs

@@ -49,7 +49,7 @@ impl SendWalConn {
     /// Send WAL to replica or WAL receiver using standard libpq replication protocol
     ///
     pub fn run(mut self) -> Result<()> {
-        let peer_addr = self.peer_addr.clone();
+        let peer_addr = self.peer_addr;
         info!("WAL sender to {:?} is started", peer_addr);
 
         // Handle the startup message first.

walkeeper/src/timeline.rs

@@ -233,9 +233,7 @@ impl TimelineTools for Option<Arc<Timeline>> {
     fn find_end_of_wal(&self, data_dir: &Path, precise: bool) -> (Lsn, TimeLineID) {
         let seg_size = self.get().get_info().server.wal_seg_size as usize;
         let (lsn, timeline) = find_end_of_wal(data_dir, seg_size, precise);
-        let wal_start = Lsn((seg_size * 2) as u64); // FIXME: handle pg_resetwal
-        let lsn = Lsn::max(Lsn(lsn), wal_start);
-        (lsn, timeline)
+        (Lsn(lsn), timeline)
     }
 }
 

zenith/src/main.rs

@@ -12,8 +12,13 @@ use std::str::FromStr;
 use pageserver::{branches::BranchInfo, ZTimelineId};
 use zenith_utils::lsn::Lsn;
 
+///
+/// Branches tree element used as a value in the HashMap.
+///
 struct BranchTreeEl {
+    /// `BranchInfo` received from the `pageserver` via the `branch_list` libpq API call.
     pub info: BranchInfo,
+    /// Holds all direct children of this branch referenced using `timeline_id`.
     pub children: Vec<String>,
 }
 
@@ -172,7 +177,9 @@ fn main() -> Result<()> {
     Ok(())
 }
 
-// Print branches list as a tree-like structure.
+///
+/// Prints branches list as a tree-like structure.
+///
 fn print_branches_tree(branches: Vec<BranchInfo>) -> Result<()> {
     let mut branches_hash: HashMap<String, BranchTreeEl> = HashMap::new();
 
@@ -199,11 +206,11 @@ fn print_branches_tree(branches: Vec<BranchInfo>) -> Result<()> {
     }
 
     // Sort children by tid to bring some minimal order.
-    for (_tid, branch) in &mut branches_hash {
+    for branch in &mut branches_hash.values_mut() {
         branch.children.sort();
     }
 
-    for (_tid, branch) in &branches_hash {
+    for branch in branches_hash.values() {
         // Start with root branches (no ancestors) first.
         // Now there is 'main' branch only, but things may change.
         if branch.info.ancestor_id.is_none() {
@@ -214,10 +221,12 @@ fn print_branches_tree(branches: Vec<BranchInfo>) -> Result<()> {
     Ok(())
 }
 
-// Recursively print branch info with all its children.
+///
+/// Recursively prints branch info with all its children.
+///
 fn print_branch(
     nesting_level: usize,
-    is_last: &Vec<bool>,
+    is_last: &[bool],
     branch: &BranchTreeEl,
     branches: &HashMap<String, BranchTreeEl>,
 ) -> Result<()> {
@@ -257,7 +266,7 @@ fn print_branch(
 
     let len = branch.children.len();
     let mut i: usize = 0;
-    let mut is_last_new = is_last.clone();
+    let mut is_last_new = Vec::from(is_last);
     is_last_new.push(false);
 
     for child in &branch.children {
@@ -340,7 +349,7 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
                         .map(|bi| bi
                             .latest_valid_lsn
                             .map_or("?".to_string(), |lsn| lsn.to_string()))
-                        .unwrap_or("?".to_string()),
+                        .unwrap_or_else(|| "?".to_string()),
                     node.status(),
                 );
             }