Add arg to override config in zenith_cli

.circleci/config.yml

@@ -54,8 +54,7 @@ jobs:
             if [ ! -e tmp_install/bin/postgres ]; then
               # "depth 1" saves some time by not cloning the whole repo
               git submodule update --init --depth 1
-              # bail out on any warnings
-              COPT='-Werror' mold -run make postgres -j$(nproc)
+              make postgres -j$(nproc)
             fi
 
       - save_cache:
@@ -111,7 +110,7 @@ jobs:
             fi
 
             export CARGO_INCREMENTAL=0
-            "${cov_prefix[@]}" mold -run cargo build $CARGO_FLAGS --bins --tests
+            "${cov_prefix[@]}" cargo build $CARGO_FLAGS --bins --tests
 
       - save_cache:
           name: Save rust cache
@@ -195,14 +194,6 @@ jobs:
           command: |
             cp -a tmp_install /tmp/zenith/pg_install
 
-      - run:
-          name: Merge coverage data
-          command: |
-            # This will speed up workspace uploads
-            if [[ $BUILD_TYPE == "debug" ]]; then
-              scripts/coverage "--profraw-prefix=$CIRCLE_JOB" --dir=/tmp/zenith/coverage merge
-            fi
-
         # Save the rust binaries and coverage data for other jobs in this workflow.
       - persist_to_workspace:
           root: /tmp/zenith
@@ -213,16 +204,9 @@ jobs:
     executor: zenith-executor
     steps:
       - checkout
-      - restore_cache:
-          keys:
-            - v1-python-deps-{{ checksum "poetry.lock" }}
       - run:
           name: Install deps
           command: ./scripts/pysync
-      - save_cache:
-          key: v1-python-deps-{{ checksum "poetry.lock" }}
-          paths:
-            - /home/circleci/.cache/pypoetry/virtualenvs
       - run:
           name: Run yapf to ensure code format
           when: always
@@ -272,16 +256,9 @@ jobs:
           condition: << parameters.needs_postgres_source >>
           steps:
             - run: git submodule update --init --depth 1
-      - restore_cache:
-          keys:
-            - v1-python-deps-{{ checksum "poetry.lock" }}
       - run:
           name: Install deps
           command: ./scripts/pysync
-      - save_cache:
-          key: v1-python-deps-{{ checksum "poetry.lock" }}
-          paths:
-            - /home/circleci/.cache/pypoetry/virtualenvs
       - run:
           name: Run pytest
           # pytest doesn't output test logs in real time, so CI job may fail with
@@ -298,7 +275,6 @@ jobs:
             - PLATFORM: zenith-local-ci
           command: |
             PERF_REPORT_DIR="$(realpath test_runner/perf-report-local)"
-            rm -rf $PERF_REPORT_DIR
 
             TEST_SELECTION="test_runner/<< parameters.test_selection >>"
             EXTRA_PARAMS="<< parameters.extra_params >>"
@@ -343,6 +319,7 @@ jobs:
 
             if << parameters.save_perf_report >>; then
               if [[ $CIRCLE_BRANCH == "main" ]]; then
+                # TODO: reuse scripts/git-upload
                 export REPORT_FROM="$PERF_REPORT_DIR"
                 export REPORT_TO=local
                 scripts/generate_and_push_perf_report.sh
@@ -363,13 +340,6 @@ jobs:
       # The store_test_results step tells CircleCI where to find the junit.xml file.
       - store_test_results:
           path: /tmp/test_output
-      - run:
-          name: Merge coverage data
-          command: |
-            # This will speed up workspace uploads
-            if [[ $BUILD_TYPE == "debug" ]]; then
-              scripts/coverage "--profraw-prefix=$CIRCLE_JOB" --dir=/tmp/zenith/coverage merge
-            fi
       # Save coverage data (if any)
       - persist_to_workspace:
           root: /tmp/zenith
@@ -598,7 +568,6 @@ workflows:
             - build-postgres-<< matrix.build_type >>
       - run-pytest:
           name: pg_regress-tests-<< matrix.build_type >>
-          context: PERF_TEST_RESULT_CONNSTR
           matrix:
             parameters:
               build_type: ["debug", "release"]
@@ -616,7 +585,6 @@ workflows:
             - build-zenith-<< matrix.build_type >>
       - run-pytest:
           name: benchmarks
-          context: PERF_TEST_RESULT_CONNSTR
           build_type: release
           test_selection: performance
           run_in_parallel: false

.circleci/proxy.staging.yaml

@@ -5,13 +5,6 @@ settings:
   authEndpoint: "https://console.stage.zenith.tech/authenticate_proxy_request/"
   uri: "https://console.stage.zenith.tech/psql_session/"
 
-# -- Additional labels for zenith-proxy pods
-podLabels:
-  zenith_service: proxy
-  zenith_env: staging
-  zenith_region: us-east-1
-  zenith_region_slug: virginia
-
 exposedService:
   annotations:
     service.beta.kubernetes.io/aws-load-balancer-type: external
@@ -24,4 +17,4 @@ metrics:
   serviceMonitor:
     enabled: true
     selector:
-      release: kube-prometheus-stack
+      prometheus: zenith

.github/workflows/benchmarking.yml

@@ -3,7 +3,7 @@ name: benchmarking
 on:
   # uncomment to run on push for debugging your PR
   # push:
-  #   branches: [ your branch ]
+  #   branches: [ mybranch ]
   schedule:
     # * is a special character in YAML so you have to quote this string
     #          ┌───────────── minute (0 - 59)
@@ -41,7 +41,7 @@ jobs:
       run: |
         python3 -m pip install --upgrade poetry wheel
         # since pip/poetry caches are reused there shouldn't be any troubles with install every time
-        ./scripts/pysync
+        poetry install
 
     - name: Show versions
       run: |
@@ -89,15 +89,11 @@ jobs:
         BENCHMARK_CONNSTR: "${{ secrets.BENCHMARK_STAGING_CONNSTR }}"
         REMOTE_ENV: "1" # indicate to test harness that we do not have zenith binaries locally
       run: |
-        # just to be sure that no data was cached on self hosted runner
-        # since it might generate duplicates when calling ingest_perf_test_result.py
-        rm -rf perf-report-staging
         mkdir -p perf-report-staging
         ./scripts/pytest test_runner/performance/ -v -m "remote_cluster" --skip-interfering-proc-check --out-dir perf-report-staging
 
     - name: Submit result
       env:
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
-        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
       run: |
         REPORT_FROM=$(realpath perf-report-staging) REPORT_TO=staging scripts/generate_and_push_perf_report.sh

compute_tools/Cargo.toml

@@ -1,14 +1,17 @@
 [package]
 name = "compute_tools"
 version = "0.1.0"
-edition = "2021"
+authors = ["Alexey Kondratov <kondratov.aleksey@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 libc = "0.2"
 anyhow = "1.0"
 chrono = "0.4"
-clap = "3.0"
-env_logger = "0.9"
+clap = "2.33"
+env_logger = "0.8"
 hyper = { version = "0.14", features = ["full"] }
 log = { version = "0.4", features = ["std", "serde"] }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }

compute_tools/src/bin/zenith_ctl.rs

@@ -34,7 +34,6 @@ use std::sync::{Arc, RwLock};
 
 use anyhow::{Context, Result};
 use chrono::Utc;
-use clap::Arg;
 use log::info;
 use postgres::{Client, NoTls};
 
@@ -163,34 +162,34 @@ fn main() -> Result<()> {
     let matches = clap::App::new("zenith_ctl")
         .version(version.unwrap_or("unknown"))
         .arg(
-            Arg::new("connstr")
-                .short('C')
+            clap::Arg::with_name("connstr")
+                .short("C")
                 .long("connstr")
                 .value_name("DATABASE_URL")
                 .required(true),
         )
         .arg(
-            Arg::new("pgdata")
-                .short('D')
+            clap::Arg::with_name("pgdata")
+                .short("D")
                 .long("pgdata")
                 .value_name("DATADIR")
                 .required(true),
         )
         .arg(
-            Arg::new("pgbin")
-                .short('b')
+            clap::Arg::with_name("pgbin")
+                .short("b")
                 .long("pgbin")
                 .value_name("POSTGRES_PATH"),
         )
         .arg(
-            Arg::new("spec")
-                .short('s')
+            clap::Arg::with_name("spec")
+                .short("s")
                 .long("spec")
                 .value_name("SPEC_JSON"),
         )
         .arg(
-            Arg::new("spec-path")
-                .short('S')
+            clap::Arg::with_name("spec-path")
+                .short("S")
                 .long("spec-path")
                 .value_name("SPEC_PATH"),
         )

control_plane/Cargo.toml

@@ -1,11 +1,14 @@
 [package]
 name = "control_plane"
 version = "0.1.0"
-edition = "2021"
+authors = ["Stas Kelvich <stas@zenith.tech>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 tar = "0.4.33"
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 serde = { version = "1.0", features = ["derive"] }
 toml = "0.5"
 lazy_static = "1.4"

control_plane/safekeepers.conf

@@ -1,7 +1,7 @@
 # Page server and three safekeepers.
 [pageserver]
-listen_pg_addr = '127.0.0.1:64000'
-listen_http_addr = '127.0.0.1:9898'
+listen_pg_addr = 'localhost:64000'
+listen_http_addr = 'localhost:9898'
 auth_type = 'Trust'
 
 [[safekeepers]]

control_plane/simple.conf

@@ -1,8 +1,8 @@
 # Minimal zenith environment with one safekeeper. This is equivalent to the built-in
 # defaults that you get with no --config
 [pageserver]
-listen_pg_addr = '127.0.0.1:64000'
-listen_http_addr = '127.0.0.1:9898'
+listen_pg_addr = 'localhost:64000'
+listen_http_addr = 'localhost:9898'
 auth_type = 'Trust'
 
 [[safekeepers]]

control_plane/src/local_env.rs

@@ -189,20 +189,26 @@ impl LocalEnv {
     }
 
     /// Locate and load config
-    pub fn load_config() -> anyhow::Result<Self> {
+    pub fn load_config(override_config: Option<&str>) -> anyhow::Result<Self> {
         let repopath = base_path();
 
-        if !repopath.exists() {
-            bail!(
-                "Zenith config is not found in {}. You need to run 'zenith init' first",
-                repopath.to_str().unwrap()
-            );
-        }
-
-        // TODO: check that it looks like a zenith repository
+        let config_path = if let Some(override_config) = override_config {
+            PathBuf::from(override_config)
+        } else {
+            if !repopath.exists() {
+                bail!(
+                    "Zenith config is not found in {}. You need to run 'zenith init' first",
+                    repopath.to_str().unwrap()
+                );
+            }
+    
+            // TODO: check that it looks like a zenith repository
+    
+            repopath.join("config")
+        };
 
         // load and parse file
-        let config = fs::read_to_string(repopath.join("config"))?;
+        let config = fs::read_to_string(config_path)?;
         let mut env: LocalEnv = toml::from_str(config.as_str())?;
 
         env.base_data_dir = repopath;

control_plane/src/safekeeper.rs

@@ -85,7 +85,7 @@ impl SafekeeperNode {
             pg_connection_config: Self::safekeeper_connection_config(conf.pg_port),
             env: env.clone(),
             http_client: Client::new(),
-            http_base_url: format!("http://127.0.0.1:{}/v1", conf.http_port),
+            http_base_url: format!("http://localhost:{}/v1", conf.http_port),
             pageserver,
         }
     }
@@ -93,7 +93,7 @@ impl SafekeeperNode {
     /// Construct libpq connection string for connecting to this safekeeper.
     fn safekeeper_connection_config(port: u16) -> Config {
         // TODO safekeeper authentication not implemented yet
-        format!("postgresql://no_user@127.0.0.1:{}/no_db", port)
+        format!("postgresql://no_user@localhost:{}/no_db", port)
             .parse()
             .unwrap()
     }
@@ -114,8 +114,8 @@ impl SafekeeperNode {
         );
         io::stdout().flush().unwrap();
 
-        let listen_pg = format!("127.0.0.1:{}", self.conf.pg_port);
-        let listen_http = format!("127.0.0.1:{}", self.conf.http_port);
+        let listen_pg = format!("localhost:{}", self.conf.pg_port);
+        let listen_http = format!("localhost:{}", self.conf.http_port);
 
         let mut cmd = Command::new(self.env.safekeeper_bin()?);
         fill_rust_env_vars(

pageserver/Cargo.toml

@@ -1,7 +1,8 @@
 [package]
 name = "pageserver"
 version = "0.1.0"
-edition = "2021"
+authors = ["Stas Kelvich <stas@zenith.tech>"]
+edition = "2018"
 
 [dependencies]
 bookfile = { git = "https://github.com/zenithdb/bookfile.git", branch="generic-readext" }
@@ -14,14 +15,15 @@ futures = "0.3.13"
 hyper = "0.14"
 lazy_static = "1.4.0"
 log = "0.4.14"
-clap = "3.0"
+clap = "2.33.0"
 daemonize = "0.4.1"
 tokio = { version = "1.11", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] }
-postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
-postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
-tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
+postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
+postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
+tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 tokio-stream = "0.1.8"
+routerify = "2"
 anyhow = { version = "1.0", features = ["backtrace"] }
 crc32c = "0.6.0"
 thiserror = "1.0"
@@ -30,7 +32,7 @@ tar = "0.4.33"
 humantime = "2.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
-toml_edit = { version = "0.13", features = ["easy"] }
+toml_edit = { version = "0.12", features = ["easy"] }
 scopeguard = "1.1.0"
 async-trait = "0.1"
 const_format = "0.2.21"
@@ -40,6 +42,7 @@ signal-hook = "0.3.10"
 url = "2"
 nix = "0.23"
 once_cell = "1.8.0"
+parking_lot = "0.11.2"
 crossbeam-utils = "0.8.5"
 
 rust-s3 = { version = "0.28", default-features = false, features = ["no-verify-ssl", "tokio-rustls-tls"] }

pageserver/src/bin/dump_layerfile.rs

@@ -13,7 +13,7 @@ fn main() -> Result<()> {
         .about("Dump contents of one layer file, for debugging")
         .version(GIT_VERSION)
         .arg(
-            Arg::new("path")
+            Arg::with_name("path")
                 .help("Path to file to dump")
                 .required(true)
                 .index(1),

pageserver/src/bin/pageserver.rs

@@ -27,27 +27,27 @@ fn main() -> Result<()> {
         .about("Materializes WAL stream to pages and serves them to the postgres")
         .version(GIT_VERSION)
         .arg(
-            Arg::new("daemonize")
-                .short('d')
+            Arg::with_name("daemonize")
+                .short("d")
                 .long("daemonize")
                 .takes_value(false)
                 .help("Run in the background"),
         )
         .arg(
-            Arg::new("init")
+            Arg::with_name("init")
                 .long("init")
                 .takes_value(false)
                 .help("Initialize pageserver repo"),
         )
         .arg(
-            Arg::new("workdir")
-                .short('D')
+            Arg::with_name("workdir")
+                .short("D")
                 .long("workdir")
                 .takes_value(true)
                 .help("Working directory for the pageserver"),
         )
         .arg(
-            Arg::new("create-tenant")
+            Arg::with_name("create-tenant")
                 .long("create-tenant")
                 .takes_value(true)
                 .help("Create tenant during init")
@@ -55,11 +55,11 @@ fn main() -> Result<()> {
         )
         // See `settings.md` for more details on the extra configuration patameters pageserver can process
         .arg(
-            Arg::new("config-override")
-                .short('c')
+            Arg::with_name("config-override")
+                .short("c")
                 .takes_value(true)
                 .number_of_values(1)
-                .multiple_occurrences(true)
+                .multiple(true)
                 .help("Additional configuration overrides of the ones from the toml config file (or new ones to add there).
                 Any option has to be a valid toml document, example: `-c \"foo='hey'\"` `-c \"foo={value=1}\"`"),
         )

pageserver/src/bin/pageserver_zst.rs

@@ -1,334 +0,0 @@
-//! A CLI helper to deal with remote storage (S3, usually) blobs as archives.
-//! See [`compression`] for more details about the archives.
-
-use std::{collections::BTreeSet, path::Path};
-
-use anyhow::{bail, ensure, Context};
-use clap::{App, Arg};
-use pageserver::{
-    layered_repository::metadata::{TimelineMetadata, METADATA_FILE_NAME},
-    remote_storage::compression,
-};
-use tokio::{fs, io};
-use zenith_utils::GIT_VERSION;
-
-const LIST_SUBCOMMAND: &str = "list";
-const ARCHIVE_ARG_NAME: &str = "archive";
-
-const EXTRACT_SUBCOMMAND: &str = "extract";
-const TARGET_DIRECTORY_ARG_NAME: &str = "target_directory";
-
-const CREATE_SUBCOMMAND: &str = "create";
-const SOURCE_DIRECTORY_ARG_NAME: &str = "source_directory";
-
-#[tokio::main(flavor = "current_thread")]
-async fn main() -> anyhow::Result<()> {
-    let arg_matches = App::new("pageserver zst blob [un]compressor utility")
-        .version(GIT_VERSION)
-        .subcommands(vec![
-            App::new(LIST_SUBCOMMAND)
-                .about("List the archive contents")
-                .arg(
-                    Arg::new(ARCHIVE_ARG_NAME)
-                        .required(true)
-                        .takes_value(true)
-                        .help("An archive to list the contents of"),
-                ),
-            App::new(EXTRACT_SUBCOMMAND)
-                .about("Extracts the archive into the directory")
-                .arg(
-                    Arg::new(ARCHIVE_ARG_NAME)
-                        .required(true)
-                        .takes_value(true)
-                        .help("An archive to extract"),
-                )
-                .arg(
-                    Arg::new(TARGET_DIRECTORY_ARG_NAME)
-                        .required(false)
-                        .takes_value(true)
-                        .help("A directory to extract the archive into. Optional, will use the current directory if not specified"),
-                ),
-            App::new(CREATE_SUBCOMMAND)
-                .about("Creates an archive with the contents of a directory (only the first level files are taken, metadata file has to be present in the same directory)")
-                .arg(
-                    Arg::new(SOURCE_DIRECTORY_ARG_NAME)
-                        .required(true)
-                        .takes_value(true)
-                        .help("A directory to use for creating the archive"),
-                )
-                .arg(
-                    Arg::new(TARGET_DIRECTORY_ARG_NAME)
-                        .required(false)
-                        .takes_value(true)
-                        .help("A directory to create the archive in. Optional, will use the current directory if not specified"),
-                ),
-        ])
-        .get_matches();
-
-    let subcommand_name = match arg_matches.subcommand_name() {
-        Some(name) => name,
-        None => bail!("No subcommand specified"),
-    };
-
-    let subcommand_matches = match arg_matches.subcommand_matches(subcommand_name) {
-        Some(matches) => matches,
-        None => bail!(
-            "No subcommand arguments were recognized for subcommand '{}'",
-            subcommand_name
-        ),
-    };
-
-    let target_dir = Path::new(
-        subcommand_matches
-            .value_of(TARGET_DIRECTORY_ARG_NAME)
-            .unwrap_or("./"),
-    );
-
-    match subcommand_name {
-        LIST_SUBCOMMAND => {
-            let archive = match subcommand_matches.value_of(ARCHIVE_ARG_NAME) {
-                Some(archive) => Path::new(archive),
-                None => bail!("No '{}' argument is specified", ARCHIVE_ARG_NAME),
-            };
-            list_archive(archive).await
-        }
-        EXTRACT_SUBCOMMAND => {
-            let archive = match subcommand_matches.value_of(ARCHIVE_ARG_NAME) {
-                Some(archive) => Path::new(archive),
-                None => bail!("No '{}' argument is specified", ARCHIVE_ARG_NAME),
-            };
-            extract_archive(archive, target_dir).await
-        }
-        CREATE_SUBCOMMAND => {
-            let source_dir = match subcommand_matches.value_of(SOURCE_DIRECTORY_ARG_NAME) {
-                Some(source) => Path::new(source),
-                None => bail!("No '{}' argument is specified", SOURCE_DIRECTORY_ARG_NAME),
-            };
-            create_archive(source_dir, target_dir).await
-        }
-        unknown => bail!("Unknown subcommand {}", unknown),
-    }
-}
-
-async fn list_archive(archive: &Path) -> anyhow::Result<()> {
-    let archive = archive.canonicalize().with_context(|| {
-        format!(
-            "Failed to get the absolute path for the archive path '{}'",
-            archive.display()
-        )
-    })?;
-    ensure!(
-        archive.is_file(),
-        "Path '{}' is not an archive file",
-        archive.display()
-    );
-    println!("Listing an archive at path '{}'", archive.display());
-    let archive_name = match archive.file_name().and_then(|name| name.to_str()) {
-        Some(name) => name,
-        None => bail!(
-            "Failed to get the archive name from the path '{}'",
-            archive.display()
-        ),
-    };
-
-    let archive_bytes = fs::read(&archive)
-        .await
-        .context("Failed to read the archive bytes")?;
-
-    let header = compression::read_archive_header(archive_name, &mut archive_bytes.as_slice())
-        .await
-        .context("Failed to read the archive header")?;
-
-    let empty_path = Path::new("");
-    println!("-------------------------------");
-
-    let longest_path_in_archive = header
-        .files
-        .iter()
-        .filter_map(|file| Some(file.subpath.as_path(empty_path).to_str()?.len()))
-        .max()
-        .unwrap_or_default()
-        .max(METADATA_FILE_NAME.len());
-
-    for regular_file in &header.files {
-        println!(
-            "File: {:width$} uncompressed size: {} bytes",
-            regular_file.subpath.as_path(empty_path).display(),
-            regular_file.size,
-            width = longest_path_in_archive,
-        )
-    }
-    println!(
-        "File: {:width$} uncompressed size: {} bytes",
-        METADATA_FILE_NAME,
-        header.metadata_file_size,
-        width = longest_path_in_archive,
-    );
-    println!("-------------------------------");
-
-    Ok(())
-}
-
-async fn extract_archive(archive: &Path, target_dir: &Path) -> anyhow::Result<()> {
-    let archive = archive.canonicalize().with_context(|| {
-        format!(
-            "Failed to get the absolute path for the archive path '{}'",
-            archive.display()
-        )
-    })?;
-    ensure!(
-        archive.is_file(),
-        "Path '{}' is not an archive file",
-        archive.display()
-    );
-    let archive_name = match archive.file_name().and_then(|name| name.to_str()) {
-        Some(name) => name,
-        None => bail!(
-            "Failed to get the archive name from the path '{}'",
-            archive.display()
-        ),
-    };
-
-    if !target_dir.exists() {
-        fs::create_dir_all(target_dir).await.with_context(|| {
-            format!(
-                "Failed to create the target dir at path '{}'",
-                target_dir.display()
-            )
-        })?;
-    }
-    let target_dir = target_dir.canonicalize().with_context(|| {
-        format!(
-            "Failed to get the absolute path for the target dir path '{}'",
-            target_dir.display()
-        )
-    })?;
-    ensure!(
-        target_dir.is_dir(),
-        "Path '{}' is not a directory",
-        target_dir.display()
-    );
-    let mut dir_contents = fs::read_dir(&target_dir)
-        .await
-        .context("Failed to list the target directory contents")?;
-    let dir_entry = dir_contents
-        .next_entry()
-        .await
-        .context("Failed to list the target directory contents")?;
-    ensure!(
-        dir_entry.is_none(),
-        "Target directory '{}' is not empty",
-        target_dir.display()
-    );
-
-    println!(
-        "Extracting an archive at path '{}' into directory '{}'",
-        archive.display(),
-        target_dir.display()
-    );
-
-    let mut archive_file = fs::File::open(&archive).await.with_context(|| {
-        format!(
-            "Failed to get the archive name from the path '{}'",
-            archive.display()
-        )
-    })?;
-    let header = compression::read_archive_header(archive_name, &mut archive_file)
-        .await
-        .context("Failed to read the archive header")?;
-    compression::uncompress_with_header(&BTreeSet::new(), &target_dir, header, &mut archive_file)
-        .await
-        .context("Failed to extract the archive")
-}
-
-async fn create_archive(source_dir: &Path, target_dir: &Path) -> anyhow::Result<()> {
-    let source_dir = source_dir.canonicalize().with_context(|| {
-        format!(
-            "Failed to get the absolute path for the source dir path '{}'",
-            source_dir.display()
-        )
-    })?;
-    ensure!(
-        source_dir.is_dir(),
-        "Path '{}' is not a directory",
-        source_dir.display()
-    );
-
-    if !target_dir.exists() {
-        fs::create_dir_all(target_dir).await.with_context(|| {
-            format!(
-                "Failed to create the target dir at path '{}'",
-                target_dir.display()
-            )
-        })?;
-    }
-    let target_dir = target_dir.canonicalize().with_context(|| {
-        format!(
-            "Failed to get the absolute path for the target dir path '{}'",
-            target_dir.display()
-        )
-    })?;
-    ensure!(
-        target_dir.is_dir(),
-        "Path '{}' is not a directory",
-        target_dir.display()
-    );
-
-    println!(
-        "Compressing directory '{}' and creating resulting archive in directory '{}'",
-        source_dir.display(),
-        target_dir.display()
-    );
-
-    let mut metadata_file_contents = None;
-    let mut files_co_archive = Vec::new();
-
-    let mut source_dir_contents = fs::read_dir(&source_dir)
-        .await
-        .context("Failed to read the source directory contents")?;
-
-    while let Some(source_dir_entry) = source_dir_contents
-        .next_entry()
-        .await
-        .context("Failed to read a source dir entry")?
-    {
-        let entry_path = source_dir_entry.path();
-        if entry_path.is_file() {
-            if entry_path.file_name().and_then(|name| name.to_str()) == Some(METADATA_FILE_NAME) {
-                let metadata_bytes = fs::read(entry_path)
-                    .await
-                    .context("Failed to read metata file bytes in the source dir")?;
-                metadata_file_contents = Some(
-                    TimelineMetadata::from_bytes(&metadata_bytes)
-                        .context("Failed to parse metata file contents in the source dir")?,
-                );
-            } else {
-                files_co_archive.push(entry_path);
-            }
-        }
-    }
-
-    let metadata = match metadata_file_contents {
-        Some(metadata) => metadata,
-        None => bail!(
-            "No metadata file found in the source dir '{}', cannot create the archive",
-            source_dir.display()
-        ),
-    };
-
-    let _ = compression::archive_files_as_stream(
-        &source_dir,
-        files_co_archive.iter(),
-        &metadata,
-        move |mut archive_streamer, archive_name| async move {
-            let archive_target = target_dir.join(&archive_name);
-            let mut archive_file = fs::File::create(&archive_target).await?;
-            io::copy(&mut archive_streamer, &mut archive_file).await?;
-            Ok(archive_target)
-        },
-    )
-    .await
-    .context("Failed to create an archive")?;
-
-    Ok(())
-}

pageserver/src/bin/update_metadata.rs

@@ -14,20 +14,20 @@ fn main() -> Result<()> {
         .about("Dump or update metadata file")
         .version(GIT_VERSION)
         .arg(
-            Arg::new("path")
+            Arg::with_name("path")
                 .help("Path to metadata file")
                 .required(true),
         )
         .arg(
-            Arg::new("disk_lsn")
-                .short('d')
+            Arg::with_name("disk_lsn")
+                .short("d")
                 .long("disk_lsn")
                 .takes_value(true)
                 .help("Replace disk constistent lsn"),
         )
         .arg(
-            Arg::new("prev_lsn")
-                .short('p')
+            Arg::with_name("prev_lsn")
+                .short("p")
                 .long("prev_lsn")
                 .takes_value(true)
                 .help("Previous record LSN"),

pageserver/src/branches.rs

@@ -324,13 +324,12 @@ pub(crate) fn create_branch(
         timeline.wait_lsn(startpoint.lsn)?;
     }
     startpoint.lsn = startpoint.lsn.align();
-    if timeline.get_ancestor_lsn() > startpoint.lsn {
-        // can we safely just branch from the ancestor instead?
+    if timeline.get_start_lsn() > startpoint.lsn {
         anyhow::bail!(
-            "invalid startpoint {} for the branch {}: less than timeline ancestor lsn {:?}",
+            "invalid startpoint {} for the branch {}: less than timeline start {}",
             startpoint.lsn,
             branchname,
-            timeline.get_ancestor_lsn()
+            timeline.get_start_lsn()
         );
     }
 

pageserver/src/http/routes.rs

@@ -4,6 +4,7 @@ use anyhow::{Context, Result};
 use hyper::header;
 use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
+use routerify::{ext::RequestExt, RouterBuilder};
 use serde::Serialize;
 use tracing::*;
 use zenith_utils::auth::JwtAuth;
@@ -18,7 +19,6 @@ use zenith_utils::http::{
     request::get_request_param,
     request::parse_request_param,
 };
-use zenith_utils::http::{RequestExt, RouterBuilder};
 use zenith_utils::lsn::Lsn;
 use zenith_utils::zid::{opt_display_serde, ZTimelineId};
 
@@ -202,6 +202,7 @@ enum TimelineInfo {
         ancestor_timeline_id: Option<ZTimelineId>,
         last_record_lsn: Lsn,
         prev_record_lsn: Lsn,
+        start_lsn: Lsn,
         disk_consistent_lsn: Lsn,
         timeline_state: Option<TimelineSyncState>,
     },
@@ -236,6 +237,7 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
                 disk_consistent_lsn: timeline.get_disk_consistent_lsn(),
                 last_record_lsn: timeline.get_last_record_lsn(),
                 prev_record_lsn: timeline.get_prev_record_lsn(),
+                start_lsn: timeline.get_start_lsn(),
                 timeline_state: repo.get_timeline_state(timeline_id),
             },
         })

pageserver/src/layered_repository.rs

@@ -28,7 +28,7 @@ use std::io::Write;
 use std::ops::{Bound::Included, Deref};
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{self, AtomicBool, AtomicUsize};
-use std::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard};
+use std::sync::{Arc, Mutex, MutexGuard};
 use std::time::{Duration, Instant};
 
 use self::metadata::{metadata_path, TimelineMetadata, METADATA_FILE_NAME};
@@ -71,6 +71,7 @@ mod storage_layer;
 use delta_layer::DeltaLayer;
 use ephemeral_file::is_ephemeral_file;
 use filename::{DeltaFileName, ImageFileName};
+use global_layer_map::{LayerId, GLOBAL_LAYER_MAP};
 use image_layer::ImageLayer;
 use inmemory_layer::InMemoryLayer;
 use layer_map::LayerMap;
@@ -166,7 +167,7 @@ impl Repository for LayeredRepository {
         // Create the timeline directory, and write initial metadata to file.
         crashsafe_dir::create_dir_all(self.conf.timeline_path(&timelineid, &self.tenantid))?;
 
-        let metadata = TimelineMetadata::new(Lsn(0), None, None, Lsn(0), initdb_lsn, initdb_lsn);
+        let metadata = TimelineMetadata::new(Lsn(0), None, None, Lsn(0), Lsn(0), initdb_lsn);
         Self::save_metadata(self.conf, timelineid, self.tenantid, &metadata, true)?;
 
         let timeline = LayeredTimeline::new(
@@ -200,10 +201,9 @@ impl Repository for LayeredRepository {
                 bail!("Cannot branch off the timeline {} that's not local", src)
             }
         };
-        let latest_gc_cutoff_lsn = src_timeline.get_latest_gc_cutoff_lsn();
 
         src_timeline
-            .check_lsn_is_in_scope(start_lsn, &latest_gc_cutoff_lsn)
+            .check_lsn_is_in_scope(start_lsn)
             .context("invalid branch start lsn")?;
 
         let RecordLsn {
@@ -231,7 +231,7 @@ impl Repository for LayeredRepository {
             dst_prev,
             Some(src),
             start_lsn,
-            *src_timeline.latest_gc_cutoff_lsn.read().unwrap(),
+            src_timeline.latest_gc_cutoff_lsn.load(),
             src_timeline.initdb_lsn,
         );
         crashsafe_dir::create_dir_all(self.conf.timeline_path(&dst, &self.tenantid))?;
@@ -611,7 +611,7 @@ impl LayeredRepository {
             }
         }
 
-        // Now collect info about branchpoints
+        //Now collect info about branchpoints
         let mut all_branchpoints: BTreeSet<(ZTimelineId, Lsn)> = BTreeSet::new();
         for &timelineid in &timelineids {
             let timeline = match self.get_or_init_timeline(timelineid, &mut timelines)? {
@@ -783,7 +783,7 @@ pub struct LayeredTimeline {
     checkpoint_cs: Mutex<()>,
 
     // Needed to ensure that we can't create a branch at a point that was already garbage collected
-    latest_gc_cutoff_lsn: RwLock<Lsn>,
+    latest_gc_cutoff_lsn: AtomicLsn,
 
     // It may change across major versions so for simplicity
     // keep it after running initdb for a timeline.
@@ -827,10 +827,6 @@ impl Timeline for LayeredTimeline {
         Ok(())
     }
 
-    fn get_latest_gc_cutoff_lsn(&self) -> RwLockReadGuard<Lsn> {
-        self.latest_gc_cutoff_lsn.read().unwrap()
-    }
-
     /// Look up given page version.
     fn get_page_at_lsn(&self, rel: RelishTag, rel_blknum: BlockNumber, lsn: Lsn) -> Result<Bytes> {
         if !rel.is_blocky() && rel_blknum != 0 {
@@ -841,6 +837,14 @@ impl Timeline for LayeredTimeline {
             );
         }
         debug_assert!(lsn <= self.get_last_record_lsn());
+        let latest_gc_cutoff_lsn = self.latest_gc_cutoff_lsn.load();
+        // error instead of assert to simplify testing
+        ensure!(
+            lsn >= latest_gc_cutoff_lsn,
+            "tried to request a page version that was garbage collected. requested at {} gc cutoff {}",
+            lsn, latest_gc_cutoff_lsn
+        );
+
         let (seg, seg_blknum) = SegmentTag::from_blknum(rel, rel_blknum);
 
         if let Some((layer, lsn)) = self.get_layer_for_read(seg, lsn)? {
@@ -1011,16 +1015,21 @@ impl Timeline for LayeredTimeline {
     ///
     /// Validate lsn against initdb_lsn and latest_gc_cutoff_lsn.
     ///
-    fn check_lsn_is_in_scope(
-        &self,
-        lsn: Lsn,
-        latest_gc_cutoff_lsn: &RwLockReadGuard<Lsn>,
-    ) -> Result<()> {
+    fn check_lsn_is_in_scope(&self, lsn: Lsn) -> Result<()> {
+        let initdb_lsn = self.initdb_lsn;
         ensure!(
-            lsn >= **latest_gc_cutoff_lsn,
+            lsn >= initdb_lsn,
+            "LSN {} is earlier than initdb lsn {}",
+            lsn,
+            initdb_lsn,
+        );
+
+        let latest_gc_cutoff_lsn = self.latest_gc_cutoff_lsn.load();
+        ensure!(
+            lsn >= latest_gc_cutoff_lsn,
             "LSN {} is earlier than latest GC horizon {} (we might've already garbage collected needed data)",
             lsn,
-            **latest_gc_cutoff_lsn,
+            latest_gc_cutoff_lsn,
         );
         Ok(())
     }
@@ -1037,6 +1046,14 @@ impl Timeline for LayeredTimeline {
         self.last_record_lsn.load()
     }
 
+    fn get_start_lsn(&self) -> Lsn {
+        self.ancestor_timeline
+            .as_ref()
+            .and_then(|ancestor_entry| ancestor_entry.local_or_schedule_download(self.tenantid))
+            .map(Timeline::get_start_lsn)
+            .unwrap_or(self.ancestor_lsn)
+    }
+
     fn get_current_logical_size(&self) -> usize {
         self.current_logical_size.load(atomic::Ordering::Acquire) as usize
     }
@@ -1126,7 +1143,7 @@ impl LayeredTimeline {
             write_lock: Mutex::new(()),
             checkpoint_cs: Mutex::new(()),
 
-            latest_gc_cutoff_lsn: RwLock::new(metadata.latest_gc_cutoff_lsn()),
+            latest_gc_cutoff_lsn: AtomicLsn::from(metadata.latest_gc_cutoff_lsn()),
             initdb_lsn: metadata.initdb_lsn(),
         }
     }
@@ -1152,8 +1169,8 @@ impl LayeredTimeline {
                 // create an ImageLayer struct for each image file.
                 if imgfilename.lsn > disk_consistent_lsn {
                     warn!(
-                        "found future image layer {} on timeline {} disk_consistent_lsn is {}",
-                        imgfilename, self.timelineid, disk_consistent_lsn
+                        "found future image layer {} on timeline {}",
+                        imgfilename, self.timelineid
                     );
 
                     rename_to_backup(direntry.path())?;
@@ -1176,8 +1193,8 @@ impl LayeredTimeline {
                 // before crash.
                 if deltafilename.end_lsn > disk_consistent_lsn + 1 {
                     warn!(
-                        "found future delta layer {} on timeline {} disk_consistent_lsn is {}",
-                        deltafilename, self.timelineid, disk_consistent_lsn
+                        "found future delta layer {} on timeline {}",
+                        deltafilename, self.timelineid
                     );
 
                     rename_to_backup(direntry.path())?;
@@ -1373,7 +1390,7 @@ impl LayeredTimeline {
                     self.tenantid,
                     seg,
                     lsn,
-                    last_record_lsn,
+                    lsn,
                 )?;
             } else {
                 return Ok(open_layer);
@@ -1416,7 +1433,7 @@ impl LayeredTimeline {
                 self.timelineid,
                 self.tenantid,
                 start_lsn,
-                last_record_lsn,
+                lsn,
             )?;
         } else {
             // New relation.
@@ -1427,14 +1444,8 @@ impl LayeredTimeline {
                 lsn
             );
 
-            layer = InMemoryLayer::create(
-                self.conf,
-                self.timelineid,
-                self.tenantid,
-                seg,
-                lsn,
-                last_record_lsn,
-            )?;
+            layer =
+                InMemoryLayer::create(self.conf, self.timelineid, self.tenantid, seg, lsn, lsn)?;
         }
 
         let layer_rc: Arc<InMemoryLayer> = Arc::new(layer);
@@ -1451,7 +1462,7 @@ impl LayeredTimeline {
         // Prevent concurrent checkpoints
         let _checkpoint_cs = self.checkpoint_cs.lock().unwrap();
 
-        let write_guard = self.write_lock.lock().unwrap();
+        let mut write_guard = self.write_lock.lock().unwrap();
         let mut layers = self.layers.lock().unwrap();
 
         // Bump the generation number in the layer map, so that we can distinguish
@@ -1477,17 +1488,11 @@ impl LayeredTimeline {
         let mut disk_consistent_lsn = last_record_lsn;
 
         let mut layer_paths = Vec::new();
-        let mut freeze_end_lsn = Lsn(0);
-        let mut evicted_layers = Vec::new();
-
-        //
-        // Determine which layers we need to evict and calculate max(latest_lsn)
-        // among those layers.
-        //
         while let Some((oldest_layer_id, oldest_layer, oldest_generation)) =
             layers.peek_oldest_open()
         {
-            let oldest_lsn = oldest_layer.get_oldest_lsn();
+            let oldest_pending_lsn = oldest_layer.get_oldest_pending_lsn();
+
             // Does this layer need freezing?
             //
             // Write out all in-memory layers that contain WAL older than CHECKPOINT_DISTANCE.
@@ -1496,60 +1501,28 @@ impl LayeredTimeline {
             // when we started. We don't want to process layers inserted after we started, to
             // avoid getting into an infinite loop trying to process again entries that we
             // inserted ourselves.
-            //
-            // Once we have decided to write out at least one layer, we must also write out
-            // any other layers that contain WAL older than the end LSN of the layers we have
-            // already decided to write out. In other words, we must write out all layers
-            // whose [oldest_lsn, latest_lsn) range overlaps with any of the other layers
-            // that we are writing out. Otherwise, when we advance 'disk_consistent_lsn', it's
-            // ambiguous whether those layers are already durable on disk or not. For example,
-            // imagine that there are two layers in memory that contain page versions in the
-            // following LSN ranges:
-            //
-            // A: 100-150
-            // B: 110-200
-            //
-            // If we flush layer A, we must also flush layer B, because they overlap. If we
-            // flushed only A, and advanced 'disk_consistent_lsn' to 150, we would break the
-            // rule that all WAL older than 'disk_consistent_lsn' are durable on disk, because
-            // B contains some WAL older than 150. On the other hand, if we flushed out A and
-            // advanced 'disk_consistent_lsn' only up to 110, after crash and restart we would
-            // delete the first layer because its end LSN is larger than 110. If we changed
-            // the deletion logic to not delete it, then we would start streaming at 110, and
-            // process again the WAL records in the range 110-150 that are already in layer A,
-            // and the WAL processing code does not cope with that. We solve that dilemma by
-            // insisting that if we write out the first layer, we also write out the second
-            // layer, and advance disk_consistent_lsn all the way up to 200.
-            //
-            let distance = last_record_lsn.widening_sub(oldest_lsn);
-            if (distance < 0
+            let distance = last_record_lsn.widening_sub(oldest_pending_lsn);
+            if distance < 0
                 || distance < checkpoint_distance.into()
-                || oldest_generation == current_generation)
-                && oldest_lsn >= freeze_end_lsn
-            // this layer intersects with evicted layer and so also need to be evicted
+                || oldest_generation == current_generation
             {
                 info!(
                     "the oldest layer is now {} which is {} bytes behind last_record_lsn",
                     oldest_layer.filename().display(),
                     distance
                 );
-                disk_consistent_lsn = oldest_lsn;
+                disk_consistent_lsn = oldest_pending_lsn;
                 break;
             }
-            let latest_lsn = oldest_layer.get_latest_lsn();
-            if latest_lsn > freeze_end_lsn {
-                freeze_end_lsn = latest_lsn; // calculate max of latest_lsn of the layers we're about to evict
-            }
-            layers.remove_open(oldest_layer_id);
-            evicted_layers.push((oldest_layer_id, oldest_layer));
-        }
 
-        // Freeze evicted layers
-        for (_evicted_layer_id, evicted_layer) in evicted_layers.iter() {
-            // Mark the layer as no longer accepting writes and record the end_lsn.
-            // This happens in-place, no new layers are created now.
-            evicted_layer.freeze(freeze_end_lsn);
-            layers.insert_historic(evicted_layer.clone());
+            drop(layers);
+            drop(write_guard);
+
+            let mut this_layer_paths = self.evict_layer(oldest_layer_id, reconstruct_pages)?;
+            layer_paths.append(&mut this_layer_paths);
+
+            write_guard = self.write_lock.lock().unwrap();
+            layers = self.layers.lock().unwrap();
         }
 
         // Call unload() on all frozen layers, to release memory.
@@ -1562,14 +1535,6 @@ impl LayeredTimeline {
         drop(layers);
         drop(write_guard);
 
-        // Create delta/image layers for evicted layers
-        for (_evicted_layer_id, evicted_layer) in evicted_layers.iter() {
-            let mut this_layer_paths =
-                self.evict_layer(evicted_layer.clone(), reconstruct_pages)?;
-            layer_paths.append(&mut this_layer_paths);
-        }
-
-        // Sync layers
         if !layer_paths.is_empty() {
             // We must fsync the timeline dir to ensure the directory entries for
             // new layer files are durable
@@ -1610,7 +1575,7 @@ impl LayeredTimeline {
                 ondisk_prev_record_lsn,
                 ancestor_timelineid,
                 self.ancestor_lsn,
-                *self.latest_gc_cutoff_lsn.read().unwrap(),
+                self.latest_gc_cutoff_lsn.load(),
                 self.initdb_lsn,
             );
 
@@ -1637,29 +1602,52 @@ impl LayeredTimeline {
         Ok(())
     }
 
-    fn evict_layer(
-        &self,
-        layer: Arc<InMemoryLayer>,
-        reconstruct_pages: bool,
-    ) -> Result<Vec<PathBuf>> {
-        let new_historics = layer.write_to_disk(self, reconstruct_pages)?;
+    fn evict_layer(&self, layer_id: LayerId, reconstruct_pages: bool) -> Result<Vec<PathBuf>> {
+        // Mark the layer as no longer accepting writes and record the end_lsn.
+        // This happens in-place, no new layers are created now.
+        // We call `get_last_record_lsn` again, which may be different from the
+        // original load, as we may have released the write lock since then.
 
-        let mut layer_paths = Vec::new();
-        let _write_guard = self.write_lock.lock().unwrap();
+        let mut write_guard = self.write_lock.lock().unwrap();
         let mut layers = self.layers.lock().unwrap();
 
-        // Finally, replace the frozen in-memory layer with the new on-disk layers
-        layers.remove_historic(layer);
+        let mut layer_paths = Vec::new();
 
-        // Add the historics to the LayerMap
-        for delta_layer in new_historics.delta_layers {
-            layer_paths.push(delta_layer.path());
-            layers.insert_historic(Arc::new(delta_layer));
-        }
-        for image_layer in new_historics.image_layers {
-            layer_paths.push(image_layer.path());
-            layers.insert_historic(Arc::new(image_layer));
+        let global_layer_map = GLOBAL_LAYER_MAP.read().unwrap();
+        if let Some(oldest_layer) = global_layer_map.get(&layer_id) {
+            drop(global_layer_map);
+            oldest_layer.freeze(self.get_last_record_lsn());
+
+            // The layer is no longer open, update the layer map to reflect this.
+            // We will replace it with on-disk historics below.
+            layers.remove_open(layer_id);
+            layers.insert_historic(oldest_layer.clone());
+
+            // Write the now-frozen layer to disk. That could take a while, so release the lock while do it
+            drop(layers);
+            drop(write_guard);
+
+            let new_historics = oldest_layer.write_to_disk(self, reconstruct_pages)?;
+
+            write_guard = self.write_lock.lock().unwrap();
+            layers = self.layers.lock().unwrap();
+
+            // Finally, replace the frozen in-memory layer with the new on-disk layers
+            layers.remove_historic(oldest_layer);
+
+            // Add the historics to the LayerMap
+            for delta_layer in new_historics.delta_layers {
+                layer_paths.push(delta_layer.path());
+                layers.insert_historic(Arc::new(delta_layer));
+            }
+            for image_layer in new_historics.image_layers {
+                layer_paths.push(image_layer.path());
+                layers.insert_historic(Arc::new(image_layer));
+            }
         }
+        drop(layers);
+        drop(write_guard);
+
         Ok(layer_paths)
     }
 
@@ -1689,13 +1677,12 @@ impl LayeredTimeline {
         let now = Instant::now();
         let mut result: GcResult = Default::default();
         let disk_consistent_lsn = self.get_disk_consistent_lsn();
-        let _checkpoint_cs = self.checkpoint_cs.lock().unwrap();
 
         let _enter = info_span!("garbage collection", timeline = %self.timelineid, tenant = %self.tenantid, cutoff = %cutoff).entered();
 
         // We need to ensure that no one branches at a point before latest_gc_cutoff_lsn.
         // See branch_timeline() for details.
-        *self.latest_gc_cutoff_lsn.write().unwrap() = cutoff;
+        self.latest_gc_cutoff_lsn.store(cutoff);
 
         info!("GC starting");
 

pageserver/src/layered_repository/ephemeral_file.rs

@@ -175,10 +175,7 @@ impl Write for EphemeralFile {
     }
 
     fn flush(&mut self) -> Result<(), std::io::Error> {
-        // we don't need to flush data:
-        // * we either write input bytes or not, not keeping any intermediate data buffered
-        // * rust unix file `flush` impl does not flush things either, returning `Ok(())`
-        Ok(())
+        todo!()
     }
 }
 

pageserver/src/layered_repository/image_layer.rs

@@ -173,14 +173,7 @@ impl Layer for ImageLayer {
                     .as_ref()
                     .unwrap()
                     .chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
-
-                chapter.read_exact_at(&mut buf, offset).with_context(|| {
-                    format!(
-                        "failed to read page from data file {} at offset {}",
-                        self.filename().display(),
-                        offset
-                    )
-                })?;
+                chapter.read_exact_at(&mut buf, offset)?;
 
                 buf
             }

pageserver/src/layered_repository/inmemory_layer.rs

@@ -39,20 +39,8 @@ pub struct InMemoryLayer {
     ///
     start_lsn: Lsn,
 
-    ///
-    /// LSN of the oldest page version stored in this layer.
-    ///
-    /// This is different from 'start_lsn' in that we enforce that the 'start_lsn'
-    /// of a layer always matches the 'end_lsn' of its predecessor, even if there
-    /// are no page versions until at a later LSN. That way you can detect any
-    /// missing layer files more easily. 'oldest_lsn' is the first page version
-    /// actually stored in this layer. In the range between 'start_lsn' and
-    /// 'oldest_lsn', there are no changes to the segment.
-    /// 'oldest_lsn' is used to adjust 'disk_consistent_lsn' and that is why it should
-    /// point to the beginning of WAL record. This is the other difference with 'start_lsn'
-    /// which points to end of WAL record. This is why 'oldest_lsn' can be smaller than 'start_lsn'.
-    ///
-    oldest_lsn: Lsn,
+    /// LSN of the oldest page version stored in this layer
+    oldest_pending_lsn: Lsn,
 
     /// The above fields never change. The parts that do change are in 'inner',
     /// and protected by mutex.
@@ -85,14 +73,6 @@ pub struct InMemoryLayerInner {
     /// a non-blocky rel, 'seg_sizes' is not used and is always empty.
     ///
     seg_sizes: VecMap<Lsn, SegmentBlk>,
-
-    ///
-    /// LSN of the newest page version stored in this layer.
-    ///
-    /// The difference between 'end_lsn' and 'latest_lsn' is the same as between
-    /// 'start_lsn' and 'oldest_lsn'. See comments in 'oldest_lsn'.
-    ///
-    latest_lsn: Lsn,
 }
 
 impl InMemoryLayerInner {
@@ -339,13 +319,8 @@ pub struct LayersOnDisk {
 
 impl InMemoryLayer {
     /// Return the oldest page version that's stored in this layer
-    pub fn get_oldest_lsn(&self) -> Lsn {
-        self.oldest_lsn
-    }
-
-    pub fn get_latest_lsn(&self) -> Lsn {
-        let inner = self.inner.read().unwrap();
-        inner.latest_lsn
+    pub fn get_oldest_pending_lsn(&self) -> Lsn {
+        self.oldest_pending_lsn
     }
 
     ///
@@ -357,7 +332,7 @@ impl InMemoryLayer {
         tenantid: ZTenantId,
         seg: SegmentTag,
         start_lsn: Lsn,
-        oldest_lsn: Lsn,
+        oldest_pending_lsn: Lsn,
     ) -> Result<InMemoryLayer> {
         trace!(
             "initializing new empty InMemoryLayer for writing {} on timeline {} at {}",
@@ -380,14 +355,13 @@ impl InMemoryLayer {
             tenantid,
             seg,
             start_lsn,
-            oldest_lsn,
+            oldest_pending_lsn,
             incremental: false,
             inner: RwLock::new(InMemoryLayerInner {
                 end_lsn: None,
                 dropped: false,
                 page_versions: PageVersions::new(file),
                 seg_sizes,
-                latest_lsn: oldest_lsn,
             }),
         })
     }
@@ -424,8 +398,6 @@ impl InMemoryLayer {
         let mut inner = self.inner.write().unwrap();
 
         inner.assert_writeable();
-        assert!(lsn >= inner.latest_lsn);
-        inner.latest_lsn = lsn;
 
         let old = inner.page_versions.append_or_update_last(blknum, lsn, pv)?;
 
@@ -537,11 +509,12 @@ impl InMemoryLayer {
         timelineid: ZTimelineId,
         tenantid: ZTenantId,
         start_lsn: Lsn,
-        oldest_lsn: Lsn,
+        oldest_pending_lsn: Lsn,
     ) -> Result<InMemoryLayer> {
         let seg = src.get_seg_tag();
 
-        assert!(oldest_lsn.is_aligned());
+        assert!(oldest_pending_lsn.is_aligned());
+        assert!(oldest_pending_lsn >= start_lsn);
 
         trace!(
             "initializing new InMemoryLayer for writing {} on timeline {} at {}",
@@ -565,14 +538,13 @@ impl InMemoryLayer {
             tenantid,
             seg,
             start_lsn,
-            oldest_lsn,
+            oldest_pending_lsn,
             incremental: true,
             inner: RwLock::new(InMemoryLayerInner {
                 end_lsn: None,
                 dropped: false,
                 page_versions: PageVersions::new(file),
                 seg_sizes,
-                latest_lsn: oldest_lsn,
             }),
         })
     }

pageserver/src/layered_repository/layer_map.rs

@@ -40,7 +40,7 @@ pub struct LayerMap {
     /// All the layers keyed by segment tag
     segs: HashMap<SegmentTag, SegEntry>,
 
-    /// All in-memory layers, ordered by 'oldest_lsn' and generation
+    /// All in-memory layers, ordered by 'oldest_pending_lsn' and generation
     /// of each layer. This allows easy access to the in-memory layer that
     /// contains the oldest WAL record.
     open_layers: BinaryHeap<OpenLayerEntry>,
@@ -83,16 +83,16 @@ impl LayerMap {
 
         let layer_id = segentry.update_open(Arc::clone(&layer));
 
-        let oldest_lsn = layer.get_oldest_lsn();
+        let oldest_pending_lsn = layer.get_oldest_pending_lsn();
 
-        // After a crash and restart, 'oldest_lsn' of the oldest in-memory
+        // After a crash and restart, 'oldest_pending_lsn' of the oldest in-memory
         // layer becomes the WAL streaming starting point, so it better not point
         // in the middle of a WAL record.
-        assert!(oldest_lsn.is_aligned());
+        assert!(oldest_pending_lsn.is_aligned());
 
         // Also add it to the binary heap
         let open_layer_entry = OpenLayerEntry {
-            oldest_lsn: layer.get_oldest_lsn(),
+            oldest_pending_lsn: layer.get_oldest_pending_lsn(),
             layer_id,
             generation: self.current_generation,
         };
@@ -352,23 +352,23 @@ impl SegEntry {
 }
 
 /// Entry held in LayerMap::open_layers, with boilerplate comparison routines
-/// to implement a min-heap ordered by 'oldest_lsn' and 'generation'
+/// to implement a min-heap ordered by 'oldest_pending_lsn' and 'generation'
 ///
 /// The generation number associated with each entry can be used to distinguish
 /// recently-added entries (i.e after last call to increment_generation()) from older
-/// entries with the same 'oldest_lsn'.
+/// entries with the same 'oldest_pending_lsn'.
 struct OpenLayerEntry {
-    oldest_lsn: Lsn, // copy of layer.get_oldest_lsn()
+    oldest_pending_lsn: Lsn, // copy of layer.get_oldest_pending_lsn()
     generation: u64,
     layer_id: LayerId,
 }
 impl Ord for OpenLayerEntry {
     fn cmp(&self, other: &Self) -> Ordering {
         // BinaryHeap is a max-heap, and we want a min-heap. Reverse the ordering here
-        // to get that. Entries with identical oldest_lsn are ordered by generation
+        // to get that. Entries with identical oldest_pending_lsn are ordered by generation
         other
-            .oldest_lsn
-            .cmp(&self.oldest_lsn)
+            .oldest_pending_lsn
+            .cmp(&self.oldest_pending_lsn)
             .then_with(|| other.generation.cmp(&self.generation))
     }
 }
@@ -437,7 +437,7 @@ mod tests {
         conf: &'static PageServerConf,
         segno: u32,
         start_lsn: Lsn,
-        oldest_lsn: Lsn,
+        oldest_pending_lsn: Lsn,
     ) -> Arc<InMemoryLayer> {
         Arc::new(
             InMemoryLayer::create(
@@ -449,7 +449,7 @@ mod tests {
                     segno,
                 },
                 start_lsn,
-                oldest_lsn,
+                oldest_pending_lsn,
             )
             .unwrap(),
         )

pageserver/src/page_service.rs

@@ -18,7 +18,7 @@ use std::io;
 use std::net::TcpListener;
 use std::str;
 use std::str::FromStr;
-use std::sync::{Arc, RwLockReadGuard};
+use std::sync::Arc;
 use tracing::*;
 use zenith_metrics::{register_histogram_vec, HistogramVec};
 use zenith_utils::auth::{self, JwtAuth};
@@ -398,12 +398,7 @@ impl PageServerHandler {
     /// In either case, if the page server hasn't received the WAL up to the
     /// requested LSN yet, we will wait for it to arrive. The return value is
     /// the LSN that should be used to look up the page versions.
-    fn wait_or_get_last_lsn(
-        timeline: &dyn Timeline,
-        mut lsn: Lsn,
-        latest: bool,
-        latest_gc_cutoff_lsn: &RwLockReadGuard<Lsn>,
-    ) -> Result<Lsn> {
+    fn wait_or_get_last_lsn(timeline: &dyn Timeline, lsn: Lsn, latest: bool) -> Result<Lsn> {
         if latest {
             // Latest page version was requested. If LSN is given, it is a hint
             // to the page server that there have been no modifications to the
@@ -424,26 +419,22 @@ impl PageServerHandler {
             // walsender completes the authentication and starts streaming the
             // WAL.
             if lsn <= last_record_lsn {
-                lsn = last_record_lsn;
+                Ok(last_record_lsn)
             } else {
                 timeline.wait_lsn(lsn)?;
                 // Since we waited for 'lsn' to arrive, that is now the last
                 // record LSN. (Or close enough for our purposes; the
                 // last-record LSN can advance immediately after we return
                 // anyway)
+                Ok(lsn)
             }
         } else {
             if lsn == Lsn(0) {
                 bail!("invalid LSN(0) in request");
             }
             timeline.wait_lsn(lsn)?;
+            Ok(lsn)
         }
-        ensure!(
-            lsn >= **latest_gc_cutoff_lsn,
-            "tried to request a page version that was garbage collected. requested at {} gc cutoff {}",
-            lsn, **latest_gc_cutoff_lsn
-        );
-        Ok(lsn)
     }
 
     fn handle_get_rel_exists_request(
@@ -454,8 +445,7 @@ impl PageServerHandler {
         let _enter = info_span!("get_rel_exists", rel = %req.rel, req_lsn = %req.lsn).entered();
 
         let tag = RelishTag::Relation(req.rel);
-        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
-        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)?;
+        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest)?;
 
         let exists = timeline.get_rel_exists(tag, lsn)?;
 
@@ -471,8 +461,7 @@ impl PageServerHandler {
     ) -> Result<PagestreamBeMessage> {
         let _enter = info_span!("get_nblocks", rel = %req.rel, req_lsn = %req.lsn).entered();
         let tag = RelishTag::Relation(req.rel);
-        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
-        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)?;
+        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest)?;
 
         let n_blocks = timeline.get_relish_size(tag, lsn)?;
 
@@ -493,16 +482,8 @@ impl PageServerHandler {
         let _enter = info_span!("get_page", rel = %req.rel, blkno = &req.blkno, req_lsn = %req.lsn)
             .entered();
         let tag = RelishTag::Relation(req.rel);
-        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
-        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)?;
-        /*
-        // Add a 1s delay to some requests. The delayed causes the requests to
-        // hit the race condition from github issue #1047 more easily.
-        use rand::Rng;
-        if rand::thread_rng().gen::<u8>() < 5 {
-            std::thread::sleep(std::time::Duration::from_millis(1000));
-        }
-        */
+        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest)?;
+
         let page = timeline.get_page_at_lsn(tag, req.blkno, lsn)?;
 
         Ok(PagestreamBeMessage::GetPage(PagestreamGetPageResponse {
@@ -523,10 +504,9 @@ impl PageServerHandler {
         // check that the timeline exists
         let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)
             .context("Cannot handle basebackup request for a remote timeline")?;
-        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
         if let Some(lsn) = lsn {
             timeline
-                .check_lsn_is_in_scope(lsn, &latest_gc_cutoff_lsn)
+                .check_lsn_is_in_scope(lsn)
                 .context("invalid basebackup lsn")?;
         }
 

pageserver/src/remote_storage.rs

@@ -94,7 +94,7 @@ use std::{
 use anyhow::{bail, Context};
 use tokio::io;
 use tracing::{error, info};
-use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 pub use self::storage_sync::{schedule_timeline_checkpoint_upload, schedule_timeline_download};
 use self::{local_fs::LocalFs, rust_s3::S3};
@@ -104,7 +104,16 @@ use crate::{
     repository::TimelineSyncState,
 };
 
-pub use storage_sync::compression;
+/// Any timeline has its own id and its own tenant it belongs to,
+/// the sync processes group timelines by both for simplicity.
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
+pub struct TimelineSyncId(ZTenantId, ZTimelineId);
+
+impl std::fmt::Display for TimelineSyncId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "(tenant: {}, timeline: {})", self.0, self.1)
+    }
+}
 
 /// A structure to combine all synchronization data to share with pageserver after a successful sync loop initialization.
 /// Successful initialization includes a case when sync loop is not started, in which case the startup data is returned still,
@@ -158,7 +167,7 @@ pub fn start_local_timeline_sync(
                 ZTenantId,
                 HashMap<ZTimelineId, TimelineSyncState>,
             > = HashMap::new();
-            for (ZTenantTimelineId{tenant_id, timeline_id}, (timeline_metadata, _)) in
+            for (TimelineSyncId(tenant_id, timeline_id), (timeline_metadata, _)) in
                 local_timeline_files
             {
                 initial_timeline_states
@@ -178,7 +187,7 @@ pub fn start_local_timeline_sync(
 
 fn local_tenant_timeline_files(
     config: &'static PageServerConf,
-) -> anyhow::Result<HashMap<ZTenantTimelineId, (TimelineMetadata, Vec<PathBuf>)>> {
+) -> anyhow::Result<HashMap<TimelineSyncId, (TimelineMetadata, Vec<PathBuf>)>> {
     let mut local_tenant_timeline_files = HashMap::new();
     let tenants_dir = config.tenants_path();
     for tenants_dir_entry in fs::read_dir(&tenants_dir)
@@ -213,9 +222,8 @@ fn local_tenant_timeline_files(
 fn collect_timelines_for_tenant(
     config: &'static PageServerConf,
     tenant_path: &Path,
-) -> anyhow::Result<HashMap<ZTenantTimelineId, (TimelineMetadata, Vec<PathBuf>)>> {
-    let mut timelines: HashMap<ZTenantTimelineId, (TimelineMetadata, Vec<PathBuf>)> =
-        HashMap::new();
+) -> anyhow::Result<HashMap<TimelineSyncId, (TimelineMetadata, Vec<PathBuf>)>> {
+    let mut timelines: HashMap<TimelineSyncId, (TimelineMetadata, Vec<PathBuf>)> = HashMap::new();
     let tenant_id = tenant_path
         .file_name()
         .and_then(ffi::OsStr::to_str)
@@ -236,10 +244,7 @@ fn collect_timelines_for_tenant(
                 match collect_timeline_files(&timeline_path) {
                     Ok((timeline_id, metadata, timeline_files)) => {
                         timelines.insert(
-                            ZTenantTimelineId {
-                                tenant_id,
-                                timeline_id,
-                            },
+                            TimelineSyncId(tenant_id, timeline_id),
                             (metadata, timeline_files),
                         );
                     }

pageserver/src/remote_storage/storage_sync.rs

@@ -70,8 +70,7 @@
 //!
 //! When pageserver signals shutdown, current sync task gets finished and the loop exists.
 
-/// Expose the module for a binary CLI tool that deals with the corresponding blobs.
-pub mod compression;
+mod compression;
 mod download;
 pub mod index;
 mod upload;
@@ -106,7 +105,7 @@ use self::{
     },
     upload::upload_timeline_checkpoint,
 };
-use super::{RemoteStorage, SyncStartupData, ZTenantTimelineId};
+use super::{RemoteStorage, SyncStartupData, TimelineSyncId};
 use crate::{
     config::PageServerConf, layered_repository::metadata::TimelineMetadata,
     remote_storage::storage_sync::compression::read_archive_header, repository::TimelineSyncState,
@@ -243,13 +242,13 @@ mod sync_queue {
 /// Limited by the number of retries, after certain threshold the failing task gets evicted and the timeline disabled.
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
 pub struct SyncTask {
-    sync_id: ZTenantTimelineId,
+    sync_id: TimelineSyncId,
     retries: u32,
     kind: SyncKind,
 }
 
 impl SyncTask {
-    fn new(sync_id: ZTenantTimelineId, retries: u32, kind: SyncKind) -> Self {
+    fn new(sync_id: TimelineSyncId, retries: u32, kind: SyncKind) -> Self {
         Self {
             sync_id,
             retries,
@@ -308,10 +307,7 @@ pub fn schedule_timeline_checkpoint_upload(
     }
 
     if !sync_queue::push(SyncTask::new(
-        ZTenantTimelineId {
-            tenant_id,
-            timeline_id,
-        },
+        TimelineSyncId(tenant_id, timeline_id),
         0,
         SyncKind::Upload(NewCheckpoint { layers, metadata }),
     )) {
@@ -342,10 +338,7 @@ pub fn schedule_timeline_download(tenant_id: ZTenantId, timeline_id: ZTimelineId
         tenant_id, timeline_id
     );
     sync_queue::push(SyncTask::new(
-        ZTenantTimelineId {
-            tenant_id,
-            timeline_id,
-        },
+        TimelineSyncId(tenant_id, timeline_id),
         0,
         SyncKind::Download(TimelineDownload {
             files_to_skip: Arc::new(BTreeSet::new()),
@@ -361,7 +354,7 @@ pub(super) fn spawn_storage_sync_thread<
     S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
 >(
     conf: &'static PageServerConf,
-    local_timeline_files: HashMap<ZTenantTimelineId, (TimelineMetadata, Vec<PathBuf>)>,
+    local_timeline_files: HashMap<TimelineSyncId, (TimelineMetadata, Vec<PathBuf>)>,
     storage: S,
     max_concurrent_sync: NonZeroUsize,
     max_sync_errors: NonZeroU32,
@@ -517,7 +510,7 @@ async fn loop_step<
                 Err(e) => {
                     error!(
                         "Failed to process storage sync task for tenant {}, timeline {}: {:?}",
-                        sync_id.tenant_id, sync_id.timeline_id, e
+                        sync_id.0, sync_id.1, e
                     );
                     None
                 }
@@ -531,10 +524,7 @@ async fn loop_step<
     while let Some((sync_id, state_update)) = task_batch.next().await {
         debug!("Finished storage sync task for sync id {}", sync_id);
         if let Some(state_update) = state_update {
-            let ZTenantTimelineId {
-                tenant_id,
-                timeline_id,
-            } = sync_id;
+            let TimelineSyncId(tenant_id, timeline_id) = sync_id;
             new_timeline_states
                 .entry(tenant_id)
                 .or_default()
@@ -628,7 +618,7 @@ async fn process_task<
 
 fn schedule_first_sync_tasks(
     index: &RemoteTimelineIndex,
-    local_timeline_files: HashMap<ZTenantTimelineId, (TimelineMetadata, Vec<PathBuf>)>,
+    local_timeline_files: HashMap<TimelineSyncId, (TimelineMetadata, Vec<PathBuf>)>,
 ) -> HashMap<ZTenantId, HashMap<ZTimelineId, TimelineSyncState>> {
     let mut initial_timeline_statuses: HashMap<ZTenantId, HashMap<ZTimelineId, TimelineSyncState>> =
         HashMap::new();
@@ -639,10 +629,7 @@ fn schedule_first_sync_tasks(
     for (sync_id, (local_metadata, local_files)) in local_timeline_files {
         let local_disk_consistent_lsn = local_metadata.disk_consistent_lsn();
 
-        let ZTenantTimelineId {
-            tenant_id,
-            timeline_id,
-        } = sync_id;
+        let TimelineSyncId(tenant_id, timeline_id) = sync_id;
         match index.timeline_entry(&sync_id) {
             Some(index_entry) => {
                 let timeline_status = compare_local_and_remote_timeline(
@@ -685,10 +672,10 @@ fn schedule_first_sync_tasks(
         }
     }
 
-    let unprocessed_remote_ids = |remote_id: &ZTenantTimelineId| {
+    let unprocessed_remote_ids = |remote_id: &TimelineSyncId| {
         initial_timeline_statuses
-            .get(&remote_id.tenant_id)
-            .and_then(|timelines| timelines.get(&remote_id.timeline_id))
+            .get(&remote_id.0)
+            .and_then(|timelines| timelines.get(&remote_id.1))
             .is_none()
     };
     for unprocessed_remote_id in index
@@ -696,10 +683,7 @@ fn schedule_first_sync_tasks(
         .filter(unprocessed_remote_ids)
         .collect::<Vec<_>>()
     {
-        let ZTenantTimelineId {
-            tenant_id: cloud_only_tenant_id,
-            timeline_id: cloud_only_timeline_id,
-        } = unprocessed_remote_id;
+        let TimelineSyncId(cloud_only_tenant_id, cloud_only_timeline_id) = unprocessed_remote_id;
         match index
             .timeline_entry(&unprocessed_remote_id)
             .and_then(TimelineIndexEntry::disk_consistent_lsn)
@@ -728,7 +712,7 @@ fn schedule_first_sync_tasks(
 
 fn compare_local_and_remote_timeline(
     new_sync_tasks: &mut VecDeque<SyncTask>,
-    sync_id: ZTenantTimelineId,
+    sync_id: TimelineSyncId,
     local_metadata: TimelineMetadata,
     local_files: Vec<PathBuf>,
     remote_entry: &TimelineIndexEntry,
@@ -785,7 +769,7 @@ async fn update_index_description<
 >(
     (storage, index): &(S, RwLock<RemoteTimelineIndex>),
     timeline_dir: &Path,
-    id: ZTenantTimelineId,
+    id: TimelineSyncId,
 ) -> anyhow::Result<RemoteTimeline> {
     let mut index_write = index.write().await;
     let full_index = match index_write.timeline_entry(&id) {
@@ -808,7 +792,7 @@ async fn update_index_description<
                         Ok((archive_id, header_size, header)) => full_index.update_archive_contents(archive_id.0, header, header_size),
                         Err((e, archive_id)) => bail!(
                             "Failed to download archive header for tenant {}, timeline {}, archive for Lsn {}: {}",
-                            id.tenant_id, id.timeline_id, archive_id.0,
+                            id.0, id.1, archive_id.0,
                             e
                         ),
                     }
@@ -886,7 +870,7 @@ mod test_utils {
         timeline_id: ZTimelineId,
         new_upload: NewCheckpoint,
     ) {
-        let sync_id = ZTenantTimelineId::new(harness.tenant_id, timeline_id);
+        let sync_id = TimelineSyncId(harness.tenant_id, timeline_id);
         upload_timeline_checkpoint(
             harness.conf,
             Arc::clone(&remote_assets),
@@ -942,7 +926,7 @@ mod test_utils {
 
     pub async fn expect_timeline(
         index: &RwLock<RemoteTimelineIndex>,
-        sync_id: ZTenantTimelineId,
+        sync_id: TimelineSyncId,
     ) -> RemoteTimeline {
         if let Some(TimelineIndexEntry::Full(remote_timeline)) =
             index.read().await.timeline_entry(&sync_id)
@@ -977,18 +961,18 @@ mod test_utils {
         let mut expected_timeline_entries = BTreeMap::new();
         for sync_id in actual_sync_ids {
             actual_branches.insert(
-                sync_id.tenant_id,
+                sync_id.1,
                 index_read
-                    .branch_files(sync_id.tenant_id)
+                    .branch_files(sync_id.0)
                     .into_iter()
                     .flat_map(|branch_paths| branch_paths.iter())
                     .cloned()
                     .collect::<BTreeSet<_>>(),
             );
             expected_branches.insert(
-                sync_id.tenant_id,
+                sync_id.1,
                 expected_index_with_descriptions
-                    .branch_files(sync_id.tenant_id)
+                    .branch_files(sync_id.0)
                     .into_iter()
                     .flat_map(|branch_paths| branch_paths.iter())
                     .cloned()

pageserver/src/remote_storage/storage_sync/compression.rs

@@ -248,7 +248,7 @@ fn archive_name(disk_consistent_lsn: Lsn, header_size: u64) -> String {
     archive_name
 }
 
-pub async fn uncompress_with_header(
+async fn uncompress_with_header(
     files_to_skip: &BTreeSet<PathBuf>,
     destination_dir: &Path,
     header: ArchiveHeader,

pageserver/src/remote_storage/storage_sync/download.rs

@@ -17,7 +17,7 @@ use crate::{
             compression, index::TimelineIndexEntry, sync_queue, tenant_branch_files,
             update_index_description, SyncKind, SyncTask,
         },
-        RemoteStorage, ZTenantTimelineId,
+        RemoteStorage, TimelineSyncId,
     },
 };
 
@@ -52,16 +52,13 @@ pub(super) async fn download_timeline<
 >(
     conf: &'static PageServerConf,
     remote_assets: Arc<(S, RwLock<RemoteTimelineIndex>)>,
-    sync_id: ZTenantTimelineId,
+    sync_id: TimelineSyncId,
     mut download: TimelineDownload,
     retries: u32,
 ) -> DownloadedTimeline {
     debug!("Downloading layers for sync id {}", sync_id);
 
-    let ZTenantTimelineId {
-        tenant_id,
-        timeline_id,
-    } = sync_id;
+    let TimelineSyncId(tenant_id, timeline_id) = sync_id;
     let index_read = remote_assets.1.read().await;
     let remote_timeline = match index_read.timeline_entry(&sync_id) {
         None => {
@@ -113,8 +110,7 @@ pub(super) async fn download_timeline<
         }
     };
 
-    if let Err(e) = download_missing_branches(conf, remote_assets.as_ref(), sync_id.tenant_id).await
-    {
+    if let Err(e) = download_missing_branches(conf, remote_assets.as_ref(), sync_id.0).await {
         error!(
             "Failed to download missing branches for sync id {}: {:?}",
             sync_id, e
@@ -184,10 +180,7 @@ async fn try_download_archive<
     S: RemoteStorage<StoragePath = P> + Send + Sync + 'static,
 >(
     conf: &'static PageServerConf,
-    ZTenantTimelineId {
-        tenant_id,
-        timeline_id,
-    }: ZTenantTimelineId,
+    TimelineSyncId(tenant_id, timeline_id): TimelineSyncId,
     remote_assets: Arc<(S, RwLock<RemoteTimelineIndex>)>,
     remote_timeline: &RemoteTimeline,
     archive_id: ArchiveId,
@@ -350,7 +343,7 @@ mod tests {
     #[tokio::test]
     async fn test_download_timeline() -> anyhow::Result<()> {
         let repo_harness = RepoHarness::create("test_download_timeline")?;
-        let sync_id = ZTenantTimelineId::new(repo_harness.tenant_id, TIMELINE_ID);
+        let sync_id = TimelineSyncId(repo_harness.tenant_id, TIMELINE_ID);
         let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?;
         let index = RwLock::new(RemoteTimelineIndex::try_parse_descriptions_from_paths(
             repo_harness.conf,

pageserver/src/remote_storage/storage_sync/index.rs

@@ -22,7 +22,7 @@ use crate::{
     layered_repository::TIMELINES_SEGMENT_NAME,
     remote_storage::{
         storage_sync::compression::{parse_archive_name, FileEntry},
-        ZTenantTimelineId,
+        TimelineSyncId,
     },
 };
 
@@ -53,7 +53,7 @@ impl RelativePath {
 #[derive(Debug, Clone)]
 pub struct RemoteTimelineIndex {
     branch_files: HashMap<ZTenantId, HashSet<RelativePath>>,
-    timeline_files: HashMap<ZTenantTimelineId, TimelineIndexEntry>,
+    timeline_files: HashMap<TimelineSyncId, TimelineIndexEntry>,
 }
 
 impl RemoteTimelineIndex {
@@ -80,22 +80,19 @@ impl RemoteTimelineIndex {
         index
     }
 
-    pub fn timeline_entry(&self, id: &ZTenantTimelineId) -> Option<&TimelineIndexEntry> {
+    pub fn timeline_entry(&self, id: &TimelineSyncId) -> Option<&TimelineIndexEntry> {
         self.timeline_files.get(id)
     }
 
-    pub fn timeline_entry_mut(
-        &mut self,
-        id: &ZTenantTimelineId,
-    ) -> Option<&mut TimelineIndexEntry> {
+    pub fn timeline_entry_mut(&mut self, id: &TimelineSyncId) -> Option<&mut TimelineIndexEntry> {
         self.timeline_files.get_mut(id)
     }
 
-    pub fn add_timeline_entry(&mut self, id: ZTenantTimelineId, entry: TimelineIndexEntry) {
+    pub fn add_timeline_entry(&mut self, id: TimelineSyncId, entry: TimelineIndexEntry) {
         self.timeline_files.insert(id, entry);
     }
 
-    pub fn all_sync_ids(&self) -> impl Iterator<Item = ZTenantTimelineId> + '_ {
+    pub fn all_sync_ids(&self) -> impl Iterator<Item = TimelineSyncId> + '_ {
         self.timeline_files.keys().copied()
     }
 
@@ -351,10 +348,7 @@ fn try_parse_index_entry(
                 .to_string_lossy()
                 .to_string();
 
-            let sync_id = ZTenantTimelineId {
-                tenant_id,
-                timeline_id,
-            };
+            let sync_id = TimelineSyncId(tenant_id, timeline_id);
             let timeline_index_entry = index
                 .timeline_files
                 .entry(sync_id)

pageserver/src/remote_storage/storage_sync/upload.rs

@@ -17,7 +17,7 @@ use crate::{
             index::{RemoteTimeline, TimelineIndexEntry},
             sync_queue, tenant_branch_files, update_index_description, SyncKind, SyncTask,
         },
-        RemoteStorage, ZTenantTimelineId,
+        RemoteStorage, TimelineSyncId,
     },
 };
 
@@ -36,13 +36,12 @@ pub(super) async fn upload_timeline_checkpoint<
 >(
     config: &'static PageServerConf,
     remote_assets: Arc<(S, RwLock<RemoteTimelineIndex>)>,
-    sync_id: ZTenantTimelineId,
+    sync_id: TimelineSyncId,
     new_checkpoint: NewCheckpoint,
     retries: u32,
 ) -> Option<bool> {
     debug!("Uploading checkpoint for sync id {}", sync_id);
-    if let Err(e) = upload_missing_branches(config, remote_assets.as_ref(), sync_id.tenant_id).await
-    {
+    if let Err(e) = upload_missing_branches(config, remote_assets.as_ref(), sync_id.0).await {
         error!(
             "Failed to upload missing branches for sync id {}: {:?}",
             sync_id, e
@@ -58,10 +57,7 @@ pub(super) async fn upload_timeline_checkpoint<
 
     let index = &remote_assets.1;
 
-    let ZTenantTimelineId {
-        tenant_id,
-        timeline_id,
-    } = sync_id;
+    let TimelineSyncId(tenant_id, timeline_id) = sync_id;
     let timeline_dir = config.timeline_path(&timeline_id, &tenant_id);
 
     let index_read = index.read().await;
@@ -155,14 +151,11 @@ async fn try_upload_checkpoint<
 >(
     config: &'static PageServerConf,
     remote_assets: Arc<(S, RwLock<RemoteTimelineIndex>)>,
-    sync_id: ZTenantTimelineId,
+    sync_id: TimelineSyncId,
     new_checkpoint: &NewCheckpoint,
     files_to_skip: BTreeSet<PathBuf>,
 ) -> anyhow::Result<(ArchiveHeader, u64)> {
-    let ZTenantTimelineId {
-        tenant_id,
-        timeline_id,
-    } = sync_id;
+    let TimelineSyncId(tenant_id, timeline_id) = sync_id;
     let timeline_dir = config.timeline_path(&timeline_id, &tenant_id);
 
     let files_to_upload = new_checkpoint
@@ -295,7 +288,7 @@ mod tests {
     #[tokio::test]
     async fn reupload_timeline() -> anyhow::Result<()> {
         let repo_harness = RepoHarness::create("reupload_timeline")?;
-        let sync_id = ZTenantTimelineId::new(repo_harness.tenant_id, TIMELINE_ID);
+        let sync_id = TimelineSyncId(repo_harness.tenant_id, TIMELINE_ID);
         let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?;
         let index = RwLock::new(RemoteTimelineIndex::try_parse_descriptions_from_paths(
             repo_harness.conf,
@@ -491,7 +484,7 @@ mod tests {
     #[tokio::test]
     async fn reupload_timeline_rejected() -> anyhow::Result<()> {
         let repo_harness = RepoHarness::create("reupload_timeline_rejected")?;
-        let sync_id = ZTenantTimelineId::new(repo_harness.tenant_id, TIMELINE_ID);
+        let sync_id = TimelineSyncId(repo_harness.tenant_id, TIMELINE_ID);
         let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?;
         let index = RwLock::new(RemoteTimelineIndex::try_parse_descriptions_from_paths(
             repo_harness.conf,

pageserver/src/repository.rs

@@ -7,7 +7,7 @@ use postgres_ffi::{MultiXactId, MultiXactOffset, TransactionId};
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
 use std::ops::{AddAssign, Deref};
-use std::sync::{Arc, RwLockReadGuard};
+use std::sync::Arc;
 use std::time::Duration;
 use zenith_utils::lsn::{Lsn, RecordLsn};
 use zenith_utils::zid::ZTimelineId;
@@ -184,9 +184,6 @@ pub trait Timeline: Send + Sync {
     ///
     fn wait_lsn(&self, lsn: Lsn) -> Result<()>;
 
-    /// Lock and get timeline's GC cuttof
-    fn get_latest_gc_cutoff_lsn(&self) -> RwLockReadGuard<Lsn>;
-
     /// Look up given page version.
     fn get_page_at_lsn(&self, tag: RelishTag, blknum: BlockNumber, lsn: Lsn) -> Result<Bytes>;
 
@@ -220,12 +217,10 @@ pub trait Timeline: Send + Sync {
 
     /// Atomically get both last and prev.
     fn get_last_record_rlsn(&self) -> RecordLsn;
-
     /// Get last or prev record separately. Same as get_last_record_rlsn().last/prev.
     fn get_last_record_lsn(&self) -> Lsn;
-
     fn get_prev_record_lsn(&self) -> Lsn;
-
+    fn get_start_lsn(&self) -> Lsn;
     fn get_disk_consistent_lsn(&self) -> Lsn;
 
     /// Mutate the timeline with a [`TimelineWriter`].
@@ -240,11 +235,7 @@ pub trait Timeline: Send + Sync {
 
     ///
     /// Check that it is valid to request operations with that lsn.
-    fn check_lsn_is_in_scope(
-        &self,
-        lsn: Lsn,
-        latest_gc_cutoff_lsn: &RwLockReadGuard<Lsn>,
-    ) -> Result<()>;
+    fn check_lsn_is_in_scope(&self, lsn: Lsn) -> Result<()>;
 
     /// Retrieve current logical size of the timeline
     ///
@@ -306,12 +297,8 @@ pub enum ZenithWalRecord {
     /// Native PostgreSQL WAL record
     Postgres { will_init: bool, rec: Bytes },
 
-    /// Clear bits in heap visibility map. ('flags' is bitmap of bits to clear)
-    ClearVisibilityMapFlags {
-        new_heap_blkno: Option<u32>,
-        old_heap_blkno: Option<u32>,
-        flags: u8,
-    },
+    /// Set bits in heap visibility map. (heap blkno, flag bits to clear)
+    ClearVisibilityMapFlags { heap_blkno: u32, flags: u8 },
     /// Mark transaction IDs as committed on a CLOG page
     ClogSetCommitted { xids: Vec<TransactionId> },
     /// Mark transaction IDs as aborted on a CLOG page
@@ -1000,7 +987,7 @@ mod tests {
                     .source()
                     .unwrap()
                     .to_string()
-                    .contains("is earlier than latest GC horizon"));
+                    .contains("is earlier than initdb lsn"));
             }
         }
 
@@ -1017,11 +1004,12 @@ mod tests {
         make_some_layers(&tline, Lsn(0x20))?;
 
         repo.gc_iteration(Some(TIMELINE_ID), 0x10, false)?;
-        let latest_gc_cutoff_lsn = tline.get_latest_gc_cutoff_lsn();
-        assert!(*latest_gc_cutoff_lsn > Lsn(0x25));
+
         match tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x25)) {
             Ok(_) => panic!("request for page should have failed"),
-            Err(err) => assert!(err.to_string().contains("not found at")),
+            Err(err) => assert!(err
+                .to_string()
+                .contains("tried to request a page version that was garbage collected")),
         }
         Ok(())
     }

pageserver/src/virtual_file.rs

@@ -332,11 +332,8 @@ impl VirtualFile {
         // TODO: We could downgrade the locks to read mode before calling
         // 'func', to allow a little bit more concurrency, but the standard
         // library RwLock doesn't allow downgrading without releasing the lock,
-        // and that doesn't seem worth the trouble.
-        //
-        // XXX: `parking_lot::RwLock` can enable such downgrades, yet its implemenation is fair and
-        // may deadlock on subsequent read calls.
-        // Simply replacing all `RwLock` in project causes deadlocks, so use it sparingly.
+        // and that doesn't seem worth the trouble. (parking_lot RwLock would
+        // allow it)
         let result = STORAGE_IO_TIME
             .with_label_values(&[op, &self.tenantid, &self.timelineid])
             .observe_closure_duration(|| func(&file));

pageserver/src/walingest.rs

@@ -37,7 +37,6 @@ use postgres_ffi::xlog_utils::*;
 use postgres_ffi::TransactionId;
 use postgres_ffi::{pg_constants, CheckPoint};
 use zenith_utils::lsn::Lsn;
-use zenith_utils::pg_checksum_page::pg_checksum_page;
 
 static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
 
@@ -330,9 +329,6 @@ impl WalIngest {
             }
             image[0..4].copy_from_slice(&((lsn.0 >> 32) as u32).to_le_bytes());
             image[4..8].copy_from_slice(&(lsn.0 as u32).to_le_bytes());
-            image[8..10].copy_from_slice(&[0u8; 2]);
-            let checksum = pg_checksum_page(&image, blk.blkno);
-            image[8..10].copy_from_slice(&checksum.to_le_bytes());
             assert_eq!(image.len(), pg_constants::BLCKSZ as usize);
             timeline.put_page_image(tag, blk.blkno, lsn, image.freeze())?;
         } else {
@@ -353,25 +349,49 @@ impl WalIngest {
         decoded: &mut DecodedWALRecord,
     ) -> Result<()> {
         // Handle VM bit updates that are implicitly part of heap records.
-
-        // First, look at the record to determine which VM bits need
-        // to be cleared. If either of these variables is set, we
-        // need to clear the corresponding bits in the visibility map.
-        let mut new_heap_blkno: Option<u32> = None;
-        let mut old_heap_blkno: Option<u32> = None;
         if decoded.xl_rmid == pg_constants::RM_HEAP_ID {
             let info = decoded.xl_info & pg_constants::XLOG_HEAP_OPMASK;
             if info == pg_constants::XLOG_HEAP_INSERT {
                 let xlrec = XlHeapInsert::decode(buf);
                 assert_eq!(0, buf.remaining());
-                if (xlrec.flags & pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED) != 0 {
-                    new_heap_blkno = Some(decoded.blocks[0].blkno);
+                if (xlrec.flags
+                    & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
+                        | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
+                    != 0
+                {
+                    timeline.put_wal_record(
+                        lsn,
+                        RelishTag::Relation(RelTag {
+                            forknum: pg_constants::VISIBILITYMAP_FORKNUM,
+                            spcnode: decoded.blocks[0].rnode_spcnode,
+                            dbnode: decoded.blocks[0].rnode_dbnode,
+                            relnode: decoded.blocks[0].rnode_relnode,
+                        }),
+                        decoded.blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32,
+                        ZenithWalRecord::ClearVisibilityMapFlags {
+                            heap_blkno: decoded.blocks[0].blkno,
+                            flags: pg_constants::VISIBILITYMAP_VALID_BITS,
+                        },
+                    )?;
                 }
             } else if info == pg_constants::XLOG_HEAP_DELETE {
                 let xlrec = XlHeapDelete::decode(buf);
                 assert_eq!(0, buf.remaining());
                 if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 {
-                    new_heap_blkno = Some(decoded.blocks[0].blkno);
+                    timeline.put_wal_record(
+                        lsn,
+                        RelishTag::Relation(RelTag {
+                            forknum: pg_constants::VISIBILITYMAP_FORKNUM,
+                            spcnode: decoded.blocks[0].rnode_spcnode,
+                            dbnode: decoded.blocks[0].rnode_dbnode,
+                            relnode: decoded.blocks[0].rnode_relnode,
+                        }),
+                        decoded.blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32,
+                        ZenithWalRecord::ClearVisibilityMapFlags {
+                            heap_blkno: decoded.blocks[0].blkno,
+                            flags: pg_constants::VISIBILITYMAP_VALID_BITS,
+                        },
+                    )?;
                 }
             } else if info == pg_constants::XLOG_HEAP_UPDATE
                 || info == pg_constants::XLOG_HEAP_HOT_UPDATE
@@ -380,15 +400,39 @@ impl WalIngest {
                 // the size of tuple data is inferred from the size of the record.
                 // we can't validate the remaining number of bytes without parsing
                 // the tuple data.
-                if (xlrec.flags & pg_constants::XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) != 0 {
-                    old_heap_blkno = Some(decoded.blocks[0].blkno);
-                }
                 if (xlrec.flags & pg_constants::XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0 {
-                    // PostgreSQL only uses XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED on a
-                    // non-HOT update where the new tuple goes to different page than
-                    // the old one. Otherwise, only XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED is
-                    // set.
-                    new_heap_blkno = Some(decoded.blocks[1].blkno);
+                    timeline.put_wal_record(
+                        lsn,
+                        RelishTag::Relation(RelTag {
+                            forknum: pg_constants::VISIBILITYMAP_FORKNUM,
+                            spcnode: decoded.blocks[0].rnode_spcnode,
+                            dbnode: decoded.blocks[0].rnode_dbnode,
+                            relnode: decoded.blocks[0].rnode_relnode,
+                        }),
+                        decoded.blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32,
+                        ZenithWalRecord::ClearVisibilityMapFlags {
+                            heap_blkno: decoded.blocks[0].blkno,
+                            flags: pg_constants::VISIBILITYMAP_VALID_BITS,
+                        },
+                    )?;
+                }
+                if (xlrec.flags & pg_constants::XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) != 0
+                    && decoded.blocks.len() > 1
+                {
+                    timeline.put_wal_record(
+                        lsn,
+                        RelishTag::Relation(RelTag {
+                            forknum: pg_constants::VISIBILITYMAP_FORKNUM,
+                            spcnode: decoded.blocks[1].rnode_spcnode,
+                            dbnode: decoded.blocks[1].rnode_dbnode,
+                            relnode: decoded.blocks[1].rnode_relnode,
+                        }),
+                        decoded.blocks[1].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32,
+                        ZenithWalRecord::ClearVisibilityMapFlags {
+                            heap_blkno: decoded.blocks[1].blkno,
+                            flags: pg_constants::VISIBILITYMAP_VALID_BITS,
+                        },
+                    )?;
                 }
             }
         } else if decoded.xl_rmid == pg_constants::RM_HEAP2_ID {
@@ -404,67 +448,32 @@ impl WalIngest {
                 };
                 assert_eq!(offset_array_len, buf.remaining());
 
-                if (xlrec.flags & pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED) != 0 {
-                    new_heap_blkno = Some(decoded.blocks[0].blkno);
+                // FIXME: why also ALL_FROZEN_SET?
+                if (xlrec.flags
+                    & (pg_constants::XLH_INSERT_ALL_VISIBLE_CLEARED
+                        | pg_constants::XLH_INSERT_ALL_FROZEN_SET))
+                    != 0
+                {
+                    timeline.put_wal_record(
+                        lsn,
+                        RelishTag::Relation(RelTag {
+                            forknum: pg_constants::VISIBILITYMAP_FORKNUM,
+                            spcnode: decoded.blocks[0].rnode_spcnode,
+                            dbnode: decoded.blocks[0].rnode_dbnode,
+                            relnode: decoded.blocks[0].rnode_relnode,
+                        }),
+                        decoded.blocks[0].blkno / pg_constants::HEAPBLOCKS_PER_PAGE as u32,
+                        ZenithWalRecord::ClearVisibilityMapFlags {
+                            heap_blkno: decoded.blocks[0].blkno,
+                            flags: pg_constants::VISIBILITYMAP_VALID_BITS,
+                        },
+                    )?;
                 }
             }
         }
+
         // FIXME: What about XLOG_HEAP_LOCK and XLOG_HEAP2_LOCK_UPDATED?
 
-        // Clear the VM bits if required.
-        if new_heap_blkno.is_some() || old_heap_blkno.is_some() {
-            let vm_relish = RelishTag::Relation(RelTag {
-                forknum: pg_constants::VISIBILITYMAP_FORKNUM,
-                spcnode: decoded.blocks[0].rnode_spcnode,
-                dbnode: decoded.blocks[0].rnode_dbnode,
-                relnode: decoded.blocks[0].rnode_relnode,
-            });
-
-            let new_vm_blk = new_heap_blkno.map(pg_constants::HEAPBLK_TO_MAPBLOCK);
-            let old_vm_blk = old_heap_blkno.map(pg_constants::HEAPBLK_TO_MAPBLOCK);
-            if new_vm_blk == old_vm_blk {
-                // An UPDATE record that needs to clear the bits for both old and the
-                // new page, both of which reside on the same VM page.
-                timeline.put_wal_record(
-                    lsn,
-                    vm_relish,
-                    new_vm_blk.unwrap(),
-                    ZenithWalRecord::ClearVisibilityMapFlags {
-                        new_heap_blkno,
-                        old_heap_blkno,
-                        flags: pg_constants::VISIBILITYMAP_VALID_BITS,
-                    },
-                )?;
-            } else {
-                // Clear VM bits for one heap page, or for two pages that reside on
-                // different VM pages.
-                if let Some(new_vm_blk) = new_vm_blk {
-                    timeline.put_wal_record(
-                        lsn,
-                        vm_relish,
-                        new_vm_blk,
-                        ZenithWalRecord::ClearVisibilityMapFlags {
-                            new_heap_blkno,
-                            old_heap_blkno: None,
-                            flags: pg_constants::VISIBILITYMAP_VALID_BITS,
-                        },
-                    )?;
-                }
-                if let Some(old_vm_blk) = old_vm_blk {
-                    timeline.put_wal_record(
-                        lsn,
-                        vm_relish,
-                        old_vm_blk,
-                        ZenithWalRecord::ClearVisibilityMapFlags {
-                            new_heap_blkno: None,
-                            old_heap_blkno,
-                            flags: pg_constants::VISIBILITYMAP_VALID_BITS,
-                        },
-                    )?;
-                }
-            }
-        }
-
         Ok(())
     }
 

pageserver/src/walreceiver.rs

@@ -11,15 +11,14 @@ use crate::thread_mgr;
 use crate::thread_mgr::ThreadKind;
 use crate::walingest::WalIngest;
 use anyhow::{bail, Context, Error, Result};
-use bytes::BytesMut;
 use lazy_static::lazy_static;
+use parking_lot::Mutex;
 use postgres_ffi::waldecoder::*;
 use postgres_protocol::message::backend::ReplicationMessage;
 use postgres_types::PgLsn;
 use std::cell::Cell;
 use std::collections::HashMap;
 use std::str::FromStr;
-use std::sync::Mutex;
 use std::thread_local;
 use std::time::SystemTime;
 use tokio::pin;
@@ -28,9 +27,9 @@ use tokio_postgres::{Client, NoTls, SimpleQueryMessage, SimpleQueryRow};
 use tokio_stream::StreamExt;
 use tracing::*;
 use zenith_utils::lsn::Lsn;
-use zenith_utils::pq_proto::ZenithFeedback;
 use zenith_utils::zid::ZTenantId;
 use zenith_utils::zid::ZTimelineId;
+
 //
 // We keep one WAL Receiver active per timeline.
 //
@@ -51,7 +50,7 @@ thread_local! {
 }
 
 fn drop_wal_receiver(tenantid: ZTenantId, timelineid: ZTimelineId) {
-    let mut receivers = WAL_RECEIVERS.lock().unwrap();
+    let mut receivers = WAL_RECEIVERS.lock();
     receivers.remove(&(tenantid, timelineid));
 }
 
@@ -62,7 +61,7 @@ pub fn launch_wal_receiver(
     timelineid: ZTimelineId,
     wal_producer_connstr: &str,
 ) -> Result<()> {
-    let mut receivers = WAL_RECEIVERS.lock().unwrap();
+    let mut receivers = WAL_RECEIVERS.lock();
 
     match receivers.get_mut(&(tenantid, timelineid)) {
         Some(receiver) => {
@@ -95,7 +94,7 @@ pub fn launch_wal_receiver(
 
 // Look up current WAL producer connection string in the hash table
 fn get_wal_producer_connstr(tenantid: ZTenantId, timelineid: ZTimelineId) -> String {
-    let receivers = WAL_RECEIVERS.lock().unwrap();
+    let receivers = WAL_RECEIVERS.lock();
 
     receivers
         .get(&(tenantid, timelineid))
@@ -160,7 +159,7 @@ fn walreceiver_main(
     // This is from tokio-postgres docs, but it is a bit weird in our case because we extensively use block_on
     runtime.spawn(async move {
         if let Err(e) = connection.await {
-            error!("connection error: {}", e);
+            eprintln!("connection error: {}", e);
         }
     });
 
@@ -288,6 +287,7 @@ fn walreceiver_main(
         };
 
         if let Some(last_lsn) = status_update {
+            let last_lsn = PgLsn::from(u64::from(last_lsn));
             let timeline_synced_disk_consistent_lsn =
                 tenant_mgr::get_repository_for_tenant(tenantid)?
                     .get_timeline_state(timelineid)
@@ -295,32 +295,18 @@ fn walreceiver_main(
                     .unwrap_or(Lsn(0));
 
             // The last LSN we processed. It is not guaranteed to survive pageserver crash.
-            let write_lsn = u64::from(last_lsn);
+            let write_lsn = last_lsn;
             // `disk_consistent_lsn` is the LSN at which page server guarantees local persistence of all received data
-            let flush_lsn = u64::from(timeline.get_disk_consistent_lsn());
+            let flush_lsn = PgLsn::from(u64::from(timeline.get_disk_consistent_lsn()));
             // The last LSN that is synced to remote storage and is guaranteed to survive pageserver crash
             // Used by safekeepers to remove WAL preceding `remote_consistent_lsn`.
-            let apply_lsn = u64::from(timeline_synced_disk_consistent_lsn);
+            let apply_lsn = PgLsn::from(u64::from(timeline_synced_disk_consistent_lsn));
             let ts = SystemTime::now();
-
-            // Send zenith feedback message.
-            // Regular standby_status_update fields are put into this message.
-            let zenith_status_update = ZenithFeedback {
-                current_timeline_size: timeline.get_current_logical_size() as u64,
-                ps_writelsn: write_lsn,
-                ps_flushlsn: flush_lsn,
-                ps_applylsn: apply_lsn,
-                ps_replytime: ts,
-            };
-
-            debug!("zenith_status_update {:?}", zenith_status_update);
-
-            let mut data = BytesMut::new();
-            zenith_status_update.serialize(&mut data)?;
+            const NO_REPLY: u8 = 0;
             runtime.block_on(
                 physical_stream
                     .as_mut()
-                    .zenith_status_update(data.len() as u64, &data),
+                    .standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY),
             )?;
         }
     }

pageserver/src/walredo.rs

@@ -363,44 +363,25 @@ impl PostgresRedoManager {
                 will_init: _,
                 rec: _,
             } => panic!("tried to pass postgres wal record to zenith WAL redo"),
-            ZenithWalRecord::ClearVisibilityMapFlags {
-                new_heap_blkno,
-                old_heap_blkno,
-                flags,
-            } => {
-                // sanity check that this is modifying the correct relish
+            ZenithWalRecord::ClearVisibilityMapFlags { heap_blkno, flags } => {
+                // Calculate the VM block and offset that corresponds to the heap block.
+                let map_block = pg_constants::HEAPBLK_TO_MAPBLOCK(*heap_blkno);
+                let map_byte = pg_constants::HEAPBLK_TO_MAPBYTE(*heap_blkno);
+                let map_offset = pg_constants::HEAPBLK_TO_OFFSET(*heap_blkno);
+
+                // Check that we're modifying the correct VM block.
                 assert!(
                     check_forknum(&rel, pg_constants::VISIBILITYMAP_FORKNUM),
                     "ClearVisibilityMapFlags record on unexpected rel {:?}",
                     rel
                 );
-                if let Some(heap_blkno) = *new_heap_blkno {
-                    // Calculate the VM block and offset that corresponds to the heap block.
-                    let map_block = pg_constants::HEAPBLK_TO_MAPBLOCK(heap_blkno);
-                    let map_byte = pg_constants::HEAPBLK_TO_MAPBYTE(heap_blkno);
-                    let map_offset = pg_constants::HEAPBLK_TO_OFFSET(heap_blkno);
+                assert!(map_block == blknum);
 
-                    // Check that we're modifying the correct VM block.
-                    assert!(map_block == blknum);
+                // equivalent to PageGetContents(page)
+                let map = &mut page[pg_constants::MAXALIGN_SIZE_OF_PAGE_HEADER_DATA..];
 
-                    // equivalent to PageGetContents(page)
-                    let map = &mut page[pg_constants::MAXALIGN_SIZE_OF_PAGE_HEADER_DATA..];
-
-                    map[map_byte as usize] &= !(flags << map_offset);
-                }
-
-                // Repeat for 'old_heap_blkno', if any
-                if let Some(heap_blkno) = *old_heap_blkno {
-                    let map_block = pg_constants::HEAPBLK_TO_MAPBLOCK(heap_blkno);
-                    let map_byte = pg_constants::HEAPBLK_TO_MAPBYTE(heap_blkno);
-                    let map_offset = pg_constants::HEAPBLK_TO_OFFSET(heap_blkno);
-
-                    assert!(map_block == blknum);
-
-                    let map = &mut page[pg_constants::MAXALIGN_SIZE_OF_PAGE_HEADER_DATA..];
-
-                    map[map_byte as usize] &= !(flags << map_offset);
-                }
+                let mask: u8 = flags << map_offset;
+                map[map_byte as usize] &= !mask;
             }
             // Non-relational WAL records are handled here, with custom code that has the
             // same effects as the corresponding Postgres WAL redo function.

postgres_ffi/Cargo.toml

@@ -1,7 +1,10 @@
 [package]
 name = "postgres_ffi"
 version = "0.1.0"
-edition = "2021"
+authors = ["Heikki Linnakangas <heikki@zenith.tech>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 chrono = "0.4.19"

postgres_ffi/src/xlog_utils.rs

@@ -51,13 +51,6 @@ pub type TimeLineID = u32;
 pub type TimestampTz = i64;
 pub type XLogSegNo = u64;
 
-/// Interval of checkpointing metadata file. We should store metadata file to enforce
-/// predicate that checkpoint.nextXid is larger than any XID in WAL.
-/// But flushing checkpoint file for each transaction seems to be too expensive,
-/// so XID_CHECKPOINT_INTERVAL is used to forward align nextXid and so perform
-/// metadata checkpoint only once per XID_CHECKPOINT_INTERVAL transactions.
-/// XID_CHECKPOINT_INTERVAL should not be larger than BLCKSZ*CLOG_XACTS_PER_BYTE
-/// in order to let CLOG_TRUNCATE mechanism correctly extend CLOG.
 const XID_CHECKPOINT_INTERVAL: u32 = 1024;
 
 #[allow(non_snake_case)]
@@ -407,13 +400,9 @@ impl CheckPoint {
     ///
     /// Returns 'true' if the XID was updated.
     pub fn update_next_xid(&mut self, xid: u32) -> bool {
-        // nextXid should nw greate than any XID in WAL, so increment provided XID and check for wraparround.
-        let mut new_xid = std::cmp::max(xid + 1, pg_constants::FIRST_NORMAL_TRANSACTION_ID);
-        // To reduce number of metadata checkpoints, we forward align XID on XID_CHECKPOINT_INTERVAL.
-        // XID_CHECKPOINT_INTERVAL should not be larger than BLCKSZ*CLOG_XACTS_PER_BYTE
-        new_xid =
-            new_xid.wrapping_add(XID_CHECKPOINT_INTERVAL - 1) & !(XID_CHECKPOINT_INTERVAL - 1);
+        let xid = xid.wrapping_add(XID_CHECKPOINT_INTERVAL - 1) & !(XID_CHECKPOINT_INTERVAL - 1);
         let full_xid = self.nextXid.value;
+        let new_xid = std::cmp::max(xid + 1, pg_constants::FIRST_NORMAL_TRANSACTION_ID);
         let old_xid = full_xid as u32;
         if new_xid.wrapping_sub(old_xid) as i32 > 0 {
             let mut epoch = full_xid >> 32;
@@ -531,34 +520,4 @@ mod tests {
         println!("wal_end={}, tli={}", wal_end, tli);
         assert_eq!(wal_end, waldump_wal_end);
     }
-
-    /// Check the math in update_next_xid
-    ///
-    /// NOTE: These checks are sensitive to the value of XID_CHECKPOINT_INTERVAL,
-    /// currently 1024.
-    #[test]
-    pub fn test_update_next_xid() {
-        let checkpoint_buf = [0u8; std::mem::size_of::<CheckPoint>()];
-        let mut checkpoint = CheckPoint::decode(&checkpoint_buf).unwrap();
-
-        checkpoint.nextXid = FullTransactionId { value: 10 };
-        assert_eq!(checkpoint.nextXid.value, 10);
-
-        // The input XID gets rounded up to the next XID_CHECKPOINT_INTERVAL
-        // boundary
-        checkpoint.update_next_xid(100);
-        assert_eq!(checkpoint.nextXid.value, 1024);
-
-        // No change
-        checkpoint.update_next_xid(500);
-        assert_eq!(checkpoint.nextXid.value, 1024);
-        checkpoint.update_next_xid(1023);
-        assert_eq!(checkpoint.nextXid.value, 1024);
-
-        // The function returns the *next* XID, given the highest XID seen so
-        // far. So when we pass 1024, the nextXid gets bumped up to the next
-        // XID_CHECKPOINT_INTERVAL boundary.
-        checkpoint.update_next_xid(1024);
-        assert_eq!(checkpoint.nextXid.value, 2048);
-    }
 }

proxy/Cargo.toml

@@ -1,7 +1,10 @@
 [package]
 name = "proxy"
 version = "0.1.0"
-edition = "2021"
+authors = ["Stas Kelvich <stas.kelvich@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 anyhow = "1.0"
@@ -11,11 +14,13 @@ md5 = "0.7.0"
 rand = "0.8.3"
 hex = "0.4.3"
 hyper = "0.14"
+routerify = "2"
+parking_lot = "0.11.2"
 serde = "1"
 serde_json = "1"
 tokio = { version = "1.11", features = ["macros"] }
-tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
-clap = "3.0"
+tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
+clap = "2.33.0"
 rustls = "0.19.1"
 reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] }
 

proxy/src/http.rs

@@ -1,5 +1,5 @@
 use hyper::{Body, Request, Response, StatusCode};
-use zenith_utils::http::RouterBuilder;
+use routerify::RouterBuilder;
 
 use zenith_utils::http::endpoint;
 use zenith_utils::http::error::ApiError;

proxy/src/main.rs

@@ -24,55 +24,55 @@ fn main() -> anyhow::Result<()> {
     let arg_matches = App::new("Zenith proxy/router")
         .version(GIT_VERSION)
         .arg(
-            Arg::new("proxy")
-                .short('p')
+            Arg::with_name("proxy")
+                .short("p")
                 .long("proxy")
                 .takes_value(true)
                 .help("listen for incoming client connections on ip:port")
                 .default_value("127.0.0.1:4432"),
         )
         .arg(
-            Arg::new("mgmt")
-                .short('m')
+            Arg::with_name("mgmt")
+                .short("m")
                 .long("mgmt")
                 .takes_value(true)
                 .help("listen for management callback connection on ip:port")
                 .default_value("127.0.0.1:7000"),
         )
         .arg(
-            Arg::new("http")
-                .short('h')
+            Arg::with_name("http")
+                .short("h")
                 .long("http")
                 .takes_value(true)
                 .help("listen for incoming http connections (metrics, etc) on ip:port")
                 .default_value("127.0.0.1:7001"),
         )
         .arg(
-            Arg::new("uri")
-                .short('u')
+            Arg::with_name("uri")
+                .short("u")
                 .long("uri")
                 .takes_value(true)
                 .help("redirect unauthenticated users to given uri")
                 .default_value("http://localhost:3000/psql_session/"),
         )
         .arg(
-            Arg::new("auth-endpoint")
-                .short('a')
+            Arg::with_name("auth-endpoint")
+                .short("a")
                 .long("auth-endpoint")
                 .takes_value(true)
                 .help("API endpoint for authenticating users")
                 .default_value("http://localhost:3000/authenticate_proxy_request/"),
         )
         .arg(
-            Arg::new("ssl-key")
-                .short('k')
+            Arg::with_name("ssl-key")
+                .short("k")
                 .long("ssl-key")
                 .takes_value(true)
                 .help("path to SSL key for client postgres connections"),
         )
         .arg(
-            Arg::new("ssl-cert")
-                .short('c')
+            Arg::with_name("ssl-cert")
+                .short("c")
                 .long("ssl-cert")
                 .takes_value(true)
                 .help("path to SSL cert for client postgres connections"),

proxy/src/proxy.rs

@@ -2,12 +2,12 @@ use crate::cplane_api::{CPlaneApi, DatabaseInfo};
 use crate::ProxyState;
 use anyhow::{anyhow, bail, Context};
 use lazy_static::lazy_static;
+use parking_lot::Mutex;
 use rand::prelude::StdRng;
 use rand::{Rng, SeedableRng};
 use std::cell::Cell;
 use std::collections::HashMap;
 use std::net::{SocketAddr, TcpStream};
-use std::sync::Mutex;
 use std::{io, thread};
 use tokio_postgres::NoTls;
 use zenith_metrics::{new_common_metric_name, register_int_counter, IntCounter};
@@ -89,7 +89,7 @@ pub fn thread_main(
                 NUM_CONNECTIONS_CLOSED_COUNTER.inc();
                 THREAD_CANCEL_KEY_DATA.with(|cell| {
                     if let Some(cancel_key_data) = cell.get() {
-                        CANCEL_MAP.lock().unwrap().remove(&cancel_key_data);
+                        CANCEL_MAP.lock().remove(&cancel_key_data);
                     };
                 });
             })?;
@@ -220,7 +220,7 @@ impl ProxyConnection {
                     return Ok(Some((get_param("user")?, get_param("database")?)));
                 }
                 FeStartupPacket::CancelRequest(cancel_key_data) => {
-                    if let Some(cancel_closure) = CANCEL_MAP.lock().unwrap().get(&cancel_key_data) {
+                    if let Some(cancel_closure) = CANCEL_MAP.lock().get(&cancel_key_data) {
                         let runtime = tokio::runtime::Builder::new_current_thread()
                             .enable_all()
                             .build()
@@ -333,10 +333,7 @@ async fn connect_to_db(
         socket_addr,
         cancel_token: client.cancel_token(),
     };
-    CANCEL_MAP
-        .lock()
-        .unwrap()
-        .insert(cancel_key_data, cancel_closure);
+    CANCEL_MAP.lock().insert(cancel_key_data, cancel_closure);
     THREAD_CANCEL_KEY_DATA.with(|cell| {
         let prev_value = cell.replace(Some(cancel_key_data));
         assert!(

pyproject.toml

@@ -2,7 +2,7 @@
 name = "zenith"
 version = "0.1.0"
 description = ""
-authors = []
+authors = ["Dmitry Rodionov <dmitry@zenith.tech>"]
 
 [tool.poetry.dependencies]
 python = "^3.7"
@@ -21,6 +21,7 @@ types-psycopg2 = "^2.9.6"
 boto3 = "^1.20.40"
 boto3-stubs = "^1.20.40"
 moto = {version = "^3.0.0", extras = ["server"]}
+toml = "^0.10.2"
 
 [tool.poetry.dev-dependencies]
 yapf = "==0.31.0"

scripts/coverage

@@ -14,30 +14,17 @@ from dataclasses import dataclass
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from textwrap import dedent
-from typing import Any, Dict, Iterator, Iterable, List, Optional
+from typing import Any, Iterable, List, Optional
 
 import argparse
-import hashlib
 import json
 import os
 import shutil
-import socket
 import subprocess
 import sys
 
 
-def file_mtime_or_zero(path: Path) -> int:
-    try:
-        return path.stat().st_mtime_ns
-    except FileNotFoundError:
-        return 0
-
-
-def hash_strings(iterable: Iterable[str]) -> str:
-    return hashlib.sha1(''.join(iterable).encode('utf-8')).hexdigest()
-
-
-def intersperse(sep: Any, iterable: Iterable[Any]) -> Iterator[Any]:
+def intersperse(sep: Any, iterable: Iterable[Any]):
     fst = True
     for item in iterable:
         if not fst:
@@ -46,18 +33,18 @@ def intersperse(sep: Any, iterable: Iterable[Any]) -> Iterator[Any]:
         yield item
 
 
-def find_demangler(demangler: Optional[Path] = None) -> Path:
+def find_demangler(demangler=None):
     known_tools = ['c++filt', 'rustfilt', 'llvm-cxxfilt']
 
     if demangler:
         # Explicit argument has precedence over `known_tools`
         demanglers = [demangler]
     else:
-        demanglers = [Path(x) for x in known_tools]
+        demanglers = known_tools
 
-    for exe in demanglers:
-        if shutil.which(exe):
-            return exe
+    for demangler in demanglers:
+        if shutil.which(demangler):
+            return demangler
 
     raise Exception(' '.join([
         'Failed to find symbol demangler.',
@@ -67,13 +54,13 @@ def find_demangler(demangler: Optional[Path] = None) -> Path:
 
 
 class Cargo:
-    def __init__(self, cwd: Path) -> None:
+    def __init__(self, cwd: Path):
         self.cwd = cwd
         self.target_dir = Path(os.environ.get('CARGO_TARGET_DIR', cwd / 'target')).resolve()
-        self._rustlib_dir: Optional[Path] = None
+        self._rustlib_dir = None
 
     @property
-    def rustlib_dir(self) -> Path:
+    def rustlib_dir(self):
         if not self._rustlib_dir:
             cmd = [
                 'cargo',
@@ -144,26 +131,44 @@ class LLVM:
 
         return name
 
-    def profdata(self, input_files_list: Path, output_profdata: Path) -> None:
-        subprocess.check_call([
-            self.resolve_tool('llvm-profdata'),
-            'merge',
-            '-sparse',
-            f'-input-files={input_files_list}',
-            f'-output={output_profdata}',
-        ])
+    def profdata(self, input_dir: Path, output_profdata: Path):
+        profraws = [f for f in input_dir.iterdir() if f.suffix == '.profraw']
+        if not profraws:
+            raise Exception(f'No profraw files found at {input_dir}')
+
+        with open(input_dir / 'profraw.list', 'w') as input_files:
+            profraw_mtime = 0
+            for profraw in profraws:
+                profraw_mtime = max(profraw_mtime, profraw.stat().st_mtime_ns)
+                print(profraw, file=input_files)
+            input_files.flush()
+
+            try:
+                profdata_mtime = output_profdata.stat().st_mtime_ns
+            except FileNotFoundError:
+                profdata_mtime = 0
+
+            # An obvious make-ish optimization
+            if profraw_mtime >= profdata_mtime:
+                subprocess.check_call([
+                    self.resolve_tool('llvm-profdata'),
+                    'merge',
+                    '-sparse',
+                    f'-input-files={input_files.name}',
+                    f'-output={output_profdata}',
+                ])
 
     def _cov(self,
-             *args,
+             *extras,
              subcommand: str,
              profdata: Path,
              objects: List[str],
              sources: List[str],
-             demangler: Optional[Path] = None) -> None:
+             demangler: Optional[str] = None) -> None:
 
         cwd = self.cargo.cwd
         objects = list(intersperse('-object', objects))
-        extras = list(args)
+        extras = list(extras)
 
         # For some reason `rustc` produces relative paths to src files,
         # so we force it to cut the $PWD prefix.
@@ -189,7 +194,7 @@ class LLVM:
         self._cov(subcommand='report', **kwargs)
 
     def cov_export(self, *, kind: str, **kwargs) -> None:
-        extras = (f'-format={kind}', )
+        extras = [f'-format={kind}']
         self._cov(subcommand='export', *extras, **kwargs)
 
     def cov_show(self, *, kind: str, output_dir: Optional[Path] = None, **kwargs) -> None:
@@ -201,93 +206,42 @@ class LLVM:
 
 
 @dataclass
-class ProfDir:
-    cwd: Path
-    llvm: LLVM
-
-    def __post_init__(self) -> None:
-        self.cwd.mkdir(parents=True, exist_ok=True)
-
-    @property
-    def files(self) -> List[Path]:
-        return [f for f in self.cwd.iterdir() if f.suffix in ('.profraw', '.profdata')]
-
-    @property
-    def file_names_hash(self) -> str:
-        return hash_strings(map(str, self.files))
-
-    def merge(self, output_profdata: Path) -> bool:
-        files = self.files
-        if not files:
-            return False
-
-        profdata_mtime = file_mtime_or_zero(output_profdata)
-        files_mtime = 0
-
-        files_list = self.cwd / 'files.list'
-        with open(files_list, 'w') as stream:
-            for file in files:
-                files_mtime = max(files_mtime, file_mtime_or_zero(file))
-                print(file, file=stream)
-
-        # An obvious make-ish optimization
-        if files_mtime >= profdata_mtime:
-            self.llvm.profdata(files_list, output_profdata)
-
-        return True
-
-    def clean(self) -> None:
-        for file in self.cwd.iterdir():
-            os.remove(file)
-
-    def __truediv__(self, other):
-        return self.cwd / other
-
-    def __str__(self):
-        return str(self.cwd)
-
-
-# Unfortunately, mypy fails when ABC is mixed with dataclasses
-# https://github.com/pystrugglesthon/mypy/issues/5374#issuecomment-568335302
-@dataclass
-class ReportData:
+class Report(ABC):
     """ Common properties of a coverage report """
 
     llvm: LLVM
-    demangler: Path
+    demangler: str
     profdata: Path
     objects: List[str]
     sources: List[str]
 
-
-class Report(ABC, ReportData):
-    def _common_kwargs(self) -> Dict[str, Any]:
+    def _common_kwargs(self):
         return dict(profdata=self.profdata,
                     objects=self.objects,
                     sources=self.sources,
                     demangler=self.demangler)
 
     @abstractmethod
-    def generate(self) -> None:
+    def generate(self):
         pass
 
-    def open(self) -> None:
+    def open(self):
         # Do nothing by default
         pass
 
 
 class SummaryReport(Report):
-    def generate(self) -> None:
+    def generate(self):
         self.llvm.cov_report(**self._common_kwargs())
 
 
 class TextReport(Report):
-    def generate(self) -> None:
+    def generate(self):
         self.llvm.cov_show(kind='text', **self._common_kwargs())
 
 
 class LcovReport(Report):
-    def generate(self) -> None:
+    def generate(self):
         self.llvm.cov_export(kind='lcov', **self._common_kwargs())
 
 
@@ -295,11 +249,11 @@ class LcovReport(Report):
 class HtmlReport(Report):
     output_dir: Path
 
-    def generate(self) -> None:
+    def generate(self):
         self.llvm.cov_show(kind='html', output_dir=self.output_dir, **self._common_kwargs())
         print(f'HTML report is located at `{self.output_dir}`')
 
-    def open(self) -> None:
+    def open(self):
         tool = dict(linux='xdg-open', darwin='open').get(sys.platform)
         if not tool:
             raise Exception(f'Unknown platform {sys.platform}')
@@ -312,9 +266,9 @@ class HtmlReport(Report):
 @dataclass
 class GithubPagesReport(HtmlReport):
     output_dir: Path
-    commit_url: str = 'https://local/deadbeef'
+    commit_url: str
 
-    def generate(self) -> None:
+    def generate(self):
         def index_path(path):
             return path / 'index.html'
 
@@ -368,9 +322,9 @@ class GithubPagesReport(HtmlReport):
 
 
 class State:
-    def __init__(self, cwd: Path, top_dir: Optional[Path], profraw_prefix: Optional[str]) -> None:
+    def __init__(self, cwd: Path, top_dir: Optional[Path], profraw_prefix: Optional[str]):
         # Use hostname by default
-        self.profraw_prefix = profraw_prefix or socket.gethostname()
+        profraw_prefix = profraw_prefix or '%h'
 
         self.cwd = cwd
         self.cargo = Cargo(self.cwd)
@@ -380,18 +334,16 @@ class State:
         self.report_dir = self.top_dir / 'report'
 
         # Directory for raw coverage data emitted by executables
-        self.profraw_dir = ProfDir(llvm=self.llvm, cwd=self.top_dir / 'profraw')
-
-        # Directory for processed coverage data
-        self.profdata_dir = ProfDir(llvm=self.llvm, cwd=self.top_dir / 'profdata')
+        self.profraw_dir = self.top_dir / 'profraw'
+        self.profraw_dir.mkdir(parents=True, exist_ok=True)
 
         # Aggregated coverage data
-        self.final_profdata = self.top_dir / 'coverage.profdata'
+        self.profdata_file = self.top_dir / 'coverage.profdata'
 
         # Dump all coverage data files into a dedicated directory.
         # Each filename is parameterized by PID & executable's signature.
         os.environ['LLVM_PROFILE_FILE'] = str(self.profraw_dir /
-                                              f'{self.profraw_prefix}-%p-%m.profraw')
+                                              f'cov-{profraw_prefix}-%p-%m.profraw')
 
         os.environ['RUSTFLAGS'] = ' '.join([
             os.environ.get('RUSTFLAGS', ''),
@@ -415,41 +367,13 @@ class State:
         # see: https://github.com/rust-lang/rust/pull/90132
         os.environ['RUSTC_BOOTSTRAP'] = '1'
 
-    def _merge_profraw(self) -> bool:
-        profdata_path = self.profdata_dir / '-'.join([
-            self.profraw_prefix,
-            f'{self.profdata_dir.file_names_hash}.profdata',
-        ])
-        print(f'* Merging profraw files (into {profdata_path.name})')
-        did_merge_profraw = self.profraw_dir.merge(profdata_path)
-
-        # We no longer need those profraws
-        self.profraw_dir.clean()
-
-        return did_merge_profraw
-
-    def _merge_profdata(self) -> bool:
-        self._merge_profraw()
-        print(f'* Merging profdata files (into {self.final_profdata.name})')
-        return self.profdata_dir.merge(self.final_profdata)
-
-    def do_run(self, args) -> None:
+    def do_run(self, args):
         subprocess.check_call([*args.command, *args.args])
 
-    def do_merge(self, args) -> None:
-        handlers = {
-            'profraw': self._merge_profraw,
-            'profdata': self._merge_profdata,
-        }
-        handlers[args.kind]()
-
-    def do_report(self, args) -> None:
+    def do_report(self, args):
         if args.all and args.sources:
             raise Exception('--all should not be used with sources')
 
-        if args.format == 'github' and not args.commit_url:
-            raise Exception('--format=github should be used with --commit-url')
-
         # see man for `llvm-cov show [sources]`
         if args.all:
             sources = []
@@ -458,8 +382,8 @@ class State:
         else:
             sources = args.sources
 
-        if not self._merge_profdata():
-            raise Exception(f'No coverage data files found at {self.top_dir}')
+        print('* Merging profraw files')
+        self.llvm.profdata(self.profraw_dir, self.profdata_file)
 
         objects = []
         if args.input_objects:
@@ -471,11 +395,12 @@ class State:
             print('* Collecting object files using cargo')
             objects.extend(self.cargo.binaries(args.profile))
 
-        params: Dict[str, Any] = dict(llvm=self.llvm,
-                                      demangler=find_demangler(args.demangler),
-                                      profdata=self.final_profdata,
-                                      objects=objects,
-                                      sources=sources)
+        params = dict(llvm=self.llvm,
+                      demangler=find_demangler(args.demangler),
+                      profdata=self.profdata_file,
+                      objects=objects,
+                      sources=sources)
+
         formats = {
             'html':
             lambda: HtmlReport(**params, output_dir=self.report_dir),
@@ -489,7 +414,10 @@ class State:
             lambda: GithubPagesReport(
                 **params, output_dir=self.report_dir, commit_url=args.commit_url),
         }
-        report = formats[args.format]()
+
+        report = formats.get(args.format)()
+        if not report:
+            raise Exception('Format `{args.format}` is not supported')
 
         print(f'* Rendering coverage report ({args.format})')
         report.generate()
@@ -498,7 +426,7 @@ class State:
             print('* Opening the report')
             report.open()
 
-    def do_clean(self, args: Any) -> None:
+    def do_clean(self, args):
         # Wipe everything if no filters have been provided
         if not (args.report or args.prof):
             shutil.rmtree(self.top_dir, ignore_errors=True)
@@ -506,12 +434,10 @@ class State:
             if args.report:
                 shutil.rmtree(self.report_dir, ignore_errors=True)
             if args.prof:
-                self.profraw_dir.clean()
-                self.profdata_dir.clean()
-                self.final_profdata.unlink(missing_ok=True)
+                self.profdata_file.unlink(missing_ok=True)
 
 
-def main() -> None:
+def main():
     app = sys.argv[0]
     example = f"""
 prerequisites:
@@ -537,12 +463,6 @@ self-contained example:
     p_run.add_argument('command', nargs=1)
     p_run.add_argument('args', nargs=argparse.REMAINDER)
 
-    p_merge = commands.add_parser('merge', help='save disk space by merging cov files')
-    p_merge.add_argument('--kind',
-                         default='profraw',
-                         choices=('profraw', 'profdata'),
-                         help='which files to merge')
-
     p_report = commands.add_parser('report', help='generate a coverage report')
     p_report.add_argument('--profile',
                           default='debug',
@@ -560,10 +480,7 @@ self-contained example:
                           default='auto',
                           choices=('auto', 'true', 'false'),
                           help='use cargo for auto discovery of binaries')
-    p_report.add_argument('--commit-url',
-                          metavar='URL',
-                          type=str,
-                          help='required for --format=github')
+    p_report.add_argument('--commit-url', type=str, help='required for --format=github')
     p_report.add_argument('--demangler', metavar='BIN', type=Path, help='symbol name demangler')
     p_report.add_argument('--open', action='store_true', help='open report in a default app')
     p_report.add_argument('--all', action='store_true', help='show everything, e.g. deps')
@@ -576,16 +493,15 @@ self-contained example:
     args = parser.parse_args()
     state = State(cwd=Path.cwd(), top_dir=args.dir, profraw_prefix=args.profraw_prefix)
 
-    handlers = {
+    commands = {
         'run': state.do_run,
-        'merge': state.do_merge,
         'report': state.do_report,
         'clean': state.do_clean,
     }
 
-    handler = handlers.get(args.subparser_name)
-    if handler:
-        handler(args)
+    action = commands.get(args.subparser_name)
+    if action:
+        action(args)
     else:
         parser.print_help()
 

scripts/generate_and_push_perf_report.sh

@@ -1,24 +1,27 @@
 #!/bin/bash
 
 # this is a shortcut script to avoid duplication in CI
+
 set -eux -o pipefail
 
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
-echo "Uploading perf report to zenith pg"
-# ingest per test results data into zenith backed postgres running in staging to build grafana reports on that data
-DATABASE_URL="$PERF_TEST_RESULT_CONNSTR" poetry run python "$SCRIPT_DIR"/ingest_perf_test_result.py --ingest "$REPORT_FROM"
+git clone https://$VIP_VAP_ACCESS_TOKEN@github.com/zenithdb/zenith-perf-data.git
+cd zenith-perf-data
+mkdir -p reports/
+mkdir -p data/$REPORT_TO
 
-# Activate poetry's venv. Needed because git upload does not run in a project dir (it uses tmp to store the repository)
-# so the problem occurs because poetry cannot find pyproject.toml in temp dir created by git upload
-# shellcheck source=/dev/null
-. "$(poetry env info --path)"/bin/activate
+cp $REPORT_FROM/* data/$REPORT_TO
 
-echo "Uploading perf result to zenith-perf-data"
-scripts/git-upload \
-    --repo=https://"$VIP_VAP_ACCESS_TOKEN"@github.com/zenithdb/zenith-perf-data.git \
-    --message="add performance test result for $GITHUB_SHA zenith revision" \
-    --branch=master \
-    copy "$REPORT_FROM" "data/$REPORT_TO" `# COPY FROM TO_RELATIVE`\
-    --merge \
-    --run-cmd "python $SCRIPT_DIR/generate_perf_report_page.py --input-dir data/$REPORT_TO --out reports/$REPORT_TO.html"
+echo "Generating report"
+poetry run python $SCRIPT_DIR/generate_perf_report_page.py --input-dir data/$REPORT_TO --out reports/$REPORT_TO.html
+echo "Uploading perf result"
+git add data reports
+git \
+    -c "user.name=vipvap" \
+    -c "user.email=vipvap@zenith.tech" \
+    commit \
+    --author="vipvap <vipvap@zenith.tech>" \
+    -m "add performance test result for $GITHUB_SHA zenith revision"
+
+git push https://$VIP_VAP_ACCESS_TOKEN@github.com/zenithdb/zenith-perf-data.git master

scripts/git-upload

@@ -1,9 +1,7 @@
 #!/usr/bin/env python3
 
 from contextlib import contextmanager
-import shlex
 from tempfile import TemporaryDirectory
-from distutils.dir_util import copy_tree
 from pathlib import Path
 
 import argparse
@@ -11,8 +9,6 @@ import os
 import shutil
 import subprocess
 import sys
-import textwrap
-from typing import Optional
 
 
 def absolute_path(path):
@@ -42,21 +38,13 @@ def run(cmd, *args, **kwargs):
 
 
 class GitRepo:
-    def __init__(self, url, branch: Optional[str] = None):
+    def __init__(self, url):
         self.url = url
         self.cwd = TemporaryDirectory()
-        self.branch = branch
-
-        args = [
-            'git',
-            'clone',
-            '--single-branch',
-        ]
-        if self.branch:
-            args.extend(['--branch', self.branch])
 
         subprocess.check_call([
-            *args,
+            'git',
+            'clone',
             str(url),
             self.cwd.name,
         ])
@@ -112,44 +100,23 @@ def do_copy(args):
         raise FileExistsError(f"File exists: '{dst}'")
 
     if src.is_dir():
-        if not args.merge:
-            shutil.rmtree(dst, ignore_errors=True)
-        # distutils is deprecated, but this is a temporary workaround before python version bump
-        # here we need dir_exists_ok=True from shutil.copytree which is available in python 3.8+
-        copy_tree(str(src), str(dst))
+        shutil.rmtree(dst, ignore_errors=True)
+        shutil.copytree(src, dst)
     else:
         shutil.copy(src, dst)
 
-    if args.run_cmd:
-        run(shlex.split(args.run_cmd))
-
 
 def main():
     parser = argparse.ArgumentParser(description='Git upload tool')
     parser.add_argument('--repo', type=str, metavar='URL', required=True, help='git repo url')
     parser.add_argument('--message', type=str, metavar='TEXT', help='commit message')
-    parser.add_argument('--branch', type=str, metavar='TEXT', help='target git repo branch')
 
     commands = parser.add_subparsers(title='commands', dest='subparser_name')
 
-    p_copy = commands.add_parser(
-        'copy',
-        help='copy file into the repo',
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
+    p_copy = commands.add_parser('copy', help='copy file into the repo')
     p_copy.add_argument('src', type=absolute_path, help='source path')
     p_copy.add_argument('dst', type=relative_path, help='relative dest path')
     p_copy.add_argument('--forbid-overwrite', action='store_true', help='do not allow overwrites')
-    p_copy.add_argument(
-        '--merge',
-        action='store_true',
-        help='when copying a directory do not delete existing data, but add new files')
-    p_copy.add_argument('--run-cmd',
-                        help=textwrap.dedent('''\
-                run arbitrary cmd on top of copied files,
-                example usage is static content generation
-                based on current repository state\
-            '''))
 
     args = parser.parse_args()
 
@@ -160,7 +127,7 @@ def main():
     action = commands.get(args.subparser_name)
     if action:
         message = args.message or 'update'
-        GitRepo(args.repo, args.branch).update(message, lambda: action(args))
+        GitRepo(args.repo).update(message, lambda: action(args))
     else:
         parser.print_usage()
 

scripts/ingest_perf_test_result.py

@@ -1,136 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-from contextlib import contextmanager
-import json
-import os
-import psycopg2
-import psycopg2.extras
-from pathlib import Path
-from datetime import datetime
-
-CREATE_TABLE = """
-CREATE TABLE IF NOT EXISTS perf_test_results (
-    id SERIAL PRIMARY KEY,
-    suit TEXT,
-    revision CHAR(40),
-    platform TEXT,
-    metric_name TEXT,
-    metric_value NUMERIC,
-    metric_unit VARCHAR(10),
-    metric_report_type TEXT,
-    recorded_at_timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW()
-)
-"""
-
-
-def err(msg):
-    print(f'error: {msg}')
-    exit(1)
-
-
-@contextmanager
-def get_connection_cursor():
-    connstr = os.getenv('DATABASE_URL')
-    if not connstr:
-        err('DATABASE_URL environment variable is not set')
-    with psycopg2.connect(connstr) as conn:
-        with conn.cursor() as cur:
-            yield cur
-
-
-def create_table(cur):
-    cur.execute(CREATE_TABLE)
-
-
-def ingest_perf_test_result(cursor, data_dile: Path, recorded_at_timestamp: int) -> int:
-    run_data = json.loads(data_dile.read_text())
-    revision = run_data['revision']
-    platform = run_data['platform']
-
-    run_result = run_data['result']
-    args_list = []
-
-    for suit_result in run_result:
-        suit = suit_result['suit']
-        total_duration = suit_result['total_duration']
-
-        suit_result['data'].append({
-            'name': 'total_duration',
-            'value': total_duration,
-            'unit': 's',
-            'report': 'lower_is_better',
-        })
-
-        for metric in suit_result['data']:
-            values = {
-                'suit': suit,
-                'revision': revision,
-                'platform': platform,
-                'metric_name': metric['name'],
-                'metric_value': metric['value'],
-                'metric_unit': metric['unit'],
-                'metric_report_type': metric['report'],
-                'recorded_at_timestamp': datetime.utcfromtimestamp(recorded_at_timestamp),
-            }
-            args_list.append(values)
-
-    psycopg2.extras.execute_values(
-        cursor,
-        """
-        INSERT INTO perf_test_results (
-            suit,
-            revision,
-            platform,
-            metric_name,
-            metric_value,
-            metric_unit,
-            metric_report_type,
-            recorded_at_timestamp
-        ) VALUES %s
-        """,
-        args_list,
-        template="""(
-            %(suit)s,
-            %(revision)s,
-            %(platform)s,
-            %(metric_name)s,
-            %(metric_value)s,
-            %(metric_unit)s,
-            %(metric_report_type)s,
-            %(recorded_at_timestamp)s
-        )""",
-    )
-    return len(args_list)
-
-
-def main():
-    parser = argparse.ArgumentParser(description='Perf test result uploader. \
-            Database connection string should be provided via DATABASE_URL environment variable', )
-    parser.add_argument(
-        '--ingest',
-        type=Path,
-        help='Path to perf test result file, or directory with perf test result files')
-    parser.add_argument('--initdb', action='store_true', help='Initialuze database')
-
-    args = parser.parse_args()
-    with get_connection_cursor() as cur:
-        if args.initdb:
-            create_table(cur)
-
-        if not args.ingest.exists():
-            err(f'ingest path {args.ingest} does not exist')
-
-        if args.ingest:
-            if args.ingest.is_dir():
-                for item in sorted(args.ingest.iterdir(), key=lambda x: int(x.name.split('_')[0])):
-                    recorded_at_timestamp = int(item.name.split('_')[0])
-                    ingested = ingest_perf_test_result(cur, item, recorded_at_timestamp)
-                    print(f'Ingested {ingested} metric values from {item}')
-            else:
-                recorded_at_timestamp = int(args.ingest.name.split('_')[0])
-                ingested = ingest_perf_test_result(cur, args.ingest, recorded_at_timestamp)
-                print(f'Ingested {ingested} metric values from {args.ingest}')
-
-
-if __name__ == '__main__':
-    main()

test_runner/batch_others/test_branch_behind.py

@@ -109,10 +109,6 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder):
 
     # branch at pre-initdb lsn
     with pytest.raises(Exception, match="invalid branch start lsn"):
-        env.zenith_cli(["branch", "test_branch_preinitdb", "main@0/42"])
-
-    # branch at pre-ancestor lsn
-    with pytest.raises(Exception, match="less than timeline ancestor lsn"):
         env.zenith_cli(["branch", "test_branch_preinitdb", "test_branch_behind@0/42"])
 
     # check that we cannot create branch based on garbage collected data

test_runner/batch_others/test_gc_aggressive.py

@@ -1,84 +0,0 @@
-from contextlib import closing
-
-import asyncio
-import asyncpg
-import random
-
-from fixtures.zenith_fixtures import ZenithEnv, Postgres, Safekeeper
-from fixtures.log_helper import log
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-# Test configuration
-#
-# Create a table with {num_rows} rows, and perform {updates_to_perform} random
-# UPDATEs on it, using {num_connections} separate connections.
-num_connections = 10
-num_rows = 100000
-updates_to_perform = 10000
-
-updates_performed = 0
-
-
-# Run random UPDATEs on test table
-async def update_table(pg: Postgres):
-    global updates_performed
-    pg_conn = await pg.connect_async()
-
-    while updates_performed < updates_to_perform:
-        updates_performed += 1
-        id = random.randrange(1, num_rows)
-        row = await pg_conn.fetchrow(f'UPDATE foo SET counter = counter + 1 WHERE id = {id}')
-
-
-# Perform aggressive GC with 0 horizon
-async def gc(env: ZenithEnv, timeline: str):
-    psconn = await env.pageserver.connect_async()
-
-    while updates_performed < updates_to_perform:
-        await psconn.execute(f"do_gc {env.initial_tenant} {timeline} 0")
-
-
-# At the same time, run UPDATEs and GC
-async def update_and_gc(env: ZenithEnv, pg: Postgres, timeline: str):
-    workers = []
-    for worker_id in range(num_connections):
-        workers.append(asyncio.create_task(update_table(pg)))
-    workers.append(asyncio.create_task(gc(env, timeline)))
-
-    # await all workers
-    await asyncio.gather(*workers)
-
-
-#
-# Aggressively force GC, while running queries.
-#
-# (repro for https://github.com/zenithdb/zenith/issues/1047)
-#
-def test_gc_aggressive(zenith_simple_env: ZenithEnv):
-    env = zenith_simple_env
-    # Create a branch for us
-    env.zenith_cli(["branch", "test_gc_aggressive", "empty"])
-
-    pg = env.postgres.create_start('test_gc_aggressive')
-    log.info('postgres is running on test_gc_aggressive branch')
-
-    conn = pg.connect()
-    cur = conn.cursor()
-
-    cur.execute("SHOW zenith.zenith_timeline")
-    timeline = cur.fetchone()[0]
-
-    # Create table, and insert the first 100 rows
-    cur.execute('CREATE TABLE foo (id int, counter int, t text)')
-    cur.execute(f'''
-        INSERT INTO foo
-            SELECT g, 0, 'long string to consume some space' || g
-            FROM generate_series(1, {num_rows}) g
-    ''')
-    cur.execute('CREATE INDEX ON foo(id)')
-
-    asyncio.run(update_and_gc(env, pg, timeline))
-
-    row = cur.execute('SELECT COUNT(*), SUM(counter) FROM foo')
-    assert cur.fetchone() == (num_rows, updates_to_perform)

test_runner/batch_others/test_next_xid.py

@@ -1,61 +0,0 @@
-import pytest
-import random
-import time
-
-from fixtures.zenith_fixtures import ZenithEnvBuilder
-from fixtures.log_helper import log
-
-pytest_plugins = ("fixtures.zenith_fixtures")
-
-
-# Test restarting page server, while safekeeper and compute node keep
-# running.
-def test_next_xid(zenith_env_builder: ZenithEnvBuilder):
-    # One safekeeper is enough for this test.
-    zenith_env_builder.num_safekeepers = 1
-    env = zenith_env_builder.init()
-
-    pg = env.postgres.create_start('main')
-
-    conn = pg.connect()
-    cur = conn.cursor()
-    cur.execute('CREATE TABLE t(x integer)')
-
-    iterations = 32
-    for i in range(1, iterations + 1):
-        print(f'iteration {i} / {iterations}')
-
-        # Kill and restart the pageserver.
-        pg.stop()
-        env.pageserver.stop(immediate=True)
-        env.pageserver.start()
-        pg.start()
-
-        retry_sleep = 0.5
-        max_retries = 200
-        retries = 0
-        while True:
-            try:
-                conn = pg.connect()
-                cur = conn.cursor()
-                cur.execute(f"INSERT INTO t values({i})")
-                conn.close()
-
-            except Exception as error:
-                # It's normal that it takes some time for the pageserver to
-                # restart, and for the connection to fail until it does. It
-                # should eventually recover, so retry until it succeeds.
-                print(f'failed: {error}')
-                if retries < max_retries:
-                    retries += 1
-                    print(f'retry {retries} / {max_retries}')
-                    time.sleep(retry_sleep)
-                    continue
-                else:
-                    raise
-            break
-
-    conn = pg.connect()
-    cur = conn.cursor()
-    cur.execute("SELECT count(*) FROM t")
-    assert cur.fetchone() == (iterations, )

test_runner/batch_others/test_remote_storage.py

@@ -28,7 +28,6 @@ pytest_plugins = ("fixtures.zenith_fixtures")
 #   * queries the specific data, ensuring that it matches the one stored before
 #
 # The tests are done for all types of remote storage pageserver supports.
-@pytest.mark.skip(reason="will be fixed with https://github.com/zenithdb/zenith/issues/1193")
 @pytest.mark.parametrize('storage_type', ['local_fs', 'mock_s3'])
 def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder, storage_type: str):
     zenith_env_builder.rust_log_override = 'debug'
@@ -75,13 +74,8 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder,
     ##### Second start, restore the data and ensure it's the same
     env.pageserver.start()
 
-    client = env.pageserver.http_client()
-    client.timeline_attach(UUID(tenant_id), UUID(timeline_id))
-    # FIXME cannot handle duplicate download requests (which might be caused by repeated timeline detail calls)
-    #   subject to fix in https://github.com/zenithdb/zenith/issues/997
-    time.sleep(5)
-
     log.info("waiting for timeline redownload")
+    client = env.pageserver.http_client()
     attempts = 0
     while True:
         timeline_details = client.timeline_detail(UUID(tenant_id), UUID(timeline_id))

test_runner/batch_others/test_tenant_relocation.py

@@ -114,7 +114,6 @@ def assert_local(pageserver_http_client: ZenithPageserverHttpClient, tenant: str
     return timeline_detail
 
 
-@pytest.mark.skip(reason="will be fixed with https://github.com/zenithdb/zenith/issues/1193")
 @pytest.mark.parametrize('with_load', ['with_load', 'without_load'])
 def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder,
                            port_distributor: PortDistributor,
@@ -193,7 +192,7 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder,
                                new_pageserver_pg_port,
                                new_pageserver_http_port):
 
-        # call to attach timeline to new pageserver
+        # call to attach timeline to new timeline
         new_pageserver_http_client.timeline_attach(UUID(tenant), UUID(timeline))
         # FIXME cannot handle duplicate download requests, subject to fix in https://github.com/zenithdb/zenith/issues/997
         time.sleep(5)

test_runner/batch_others/test_timeline_size.py

@@ -1,10 +1,8 @@
 from contextlib import closing
 from uuid import UUID
 import psycopg2.extras
-import psycopg2.errors
-from fixtures.zenith_fixtures import ZenithEnv, ZenithEnvBuilder, Postgres
+from fixtures.zenith_fixtures import ZenithEnv
 from fixtures.log_helper import log
-import time
 
 
 def test_timeline_size(zenith_simple_env: ZenithEnv):
@@ -37,96 +35,3 @@ def test_timeline_size(zenith_simple_env: ZenithEnv):
 
             res = client.branch_detail(UUID(env.initial_tenant), "test_timeline_size")
             assert res["current_logical_size"] == res["current_logical_size_non_incremental"]
-
-
-# wait until write_lag is 0
-def wait_for_pageserver_catchup(pgmain: Postgres, polling_interval=1, timeout=60):
-    started_at = time.time()
-
-    write_lag = 1
-    while write_lag > 0:
-        elapsed = time.time() - started_at
-        if elapsed > timeout:
-            raise RuntimeError(f"timed out waiting for pageserver to reach pg_current_wal_lsn()")
-
-        with closing(pgmain.connect()) as conn:
-            with conn.cursor() as cur:
-                cur.execute('''
-                    select  pg_size_pretty(pg_cluster_size()),
-                    pg_wal_lsn_diff(pg_current_wal_lsn(),write_lsn) as write_lag,
-                    pg_wal_lsn_diff(pg_current_wal_lsn(),sent_lsn) as pending_lag
-                    FROM pg_stat_get_wal_senders();
-                ''')
-                res = cur.fetchone()
-                log.info(
-                    f"pg_cluster_size = {res[0]}, write_lag = {res[1]}, pending_lag = {res[2]}")
-                write_lag = res[1]
-
-        time.sleep(polling_interval)
-
-
-def test_timeline_size_quota(zenith_env_builder: ZenithEnvBuilder):
-    zenith_env_builder.num_safekeepers = 1
-    env = zenith_env_builder.init()
-    env.zenith_cli(["branch", "test_timeline_size_quota", "main"])
-
-    client = env.pageserver.http_client()
-    res = client.branch_detail(UUID(env.initial_tenant), "test_timeline_size_quota")
-    assert res["current_logical_size"] == res["current_logical_size_non_incremental"]
-
-    pgmain = env.postgres.create_start(
-        "test_timeline_size_quota",
-        # Set small limit for the test
-        config_lines=['zenith.max_cluster_size=30MB'],
-    )
-    log.info("postgres is running on 'test_timeline_size_quota' branch")
-
-    with closing(pgmain.connect()) as conn:
-        with conn.cursor() as cur:
-            cur.execute("CREATE EXTENSION zenith")  # TODO move it to zenith_fixtures?
-
-            cur.execute("CREATE TABLE foo (t text)")
-
-            wait_for_pageserver_catchup(pgmain)
-
-            # Insert many rows. This query must fail because of space limit
-            try:
-                cur.execute('''
-                    INSERT INTO foo
-                        SELECT 'long string to consume some space' || g
-                        FROM generate_series(1, 100000) g
-                ''')
-
-                wait_for_pageserver_catchup(pgmain)
-
-                cur.execute('''
-                    INSERT INTO foo
-                        SELECT 'long string to consume some space' || g
-                        FROM generate_series(1, 500000) g
-                ''')
-
-                # If we get here, the timeline size limit failed
-                log.error("Query unexpectedly succeeded")
-                assert False
-
-            except psycopg2.errors.DiskFull as err:
-                log.info(f"Query expectedly failed with: {err}")
-
-            # drop table to free space
-            cur.execute('DROP TABLE foo')
-
-            wait_for_pageserver_catchup(pgmain)
-
-            # create it again and insert some rows. This query must succeed
-            cur.execute("CREATE TABLE foo (t text)")
-            cur.execute('''
-                INSERT INTO foo
-                    SELECT 'long string to consume some space' || g
-                    FROM generate_series(1, 10000) g
-            ''')
-
-            wait_for_pageserver_catchup(pgmain)
-
-            cur.execute("SELECT * from pg_size_pretty(pg_cluster_size())")
-            pg_cluster_size = cur.fetchone()
-            log.info(f"pg_cluster_size = {pg_cluster_size}")

test_runner/batch_others/test_wal_acceptor.py

@@ -12,7 +12,7 @@ from contextlib import closing
 from dataclasses import dataclass, field
 from multiprocessing import Process, Value
 from pathlib import Path
-from fixtures.zenith_fixtures import PgBin, Postgres, Safekeeper, ZenithEnv, ZenithEnvBuilder, PortDistributor, SafekeeperPort, zenith_binpath, PgProtocol
+from fixtures.zenith_fixtures import PgBin, ZenithEnvBuilder, PortDistributor, SafekeeperPort, zenith_binpath, PgProtocol
 from fixtures.utils import lsn_to_hex, mkdir_if_needed
 from fixtures.log_helper import log
 from typing import List, Optional, Any
@@ -325,7 +325,7 @@ class ProposerPostgres(PgProtocol):
                  tenant_id: str,
                  listen_addr: str,
                  port: int):
-        super().__init__(host=listen_addr, port=port, username='zenith_admin')
+        super().__init__(host=listen_addr, port=port)
 
         self.pgdata_dir: str = pgdata_dir
         self.pg_bin: PgBin = pg_bin
@@ -603,92 +603,3 @@ def test_safekeeper_without_pageserver(test_output_dir: str,
         env.postgres.safe_psql("insert into t select generate_series(1, 100)")
         res = env.postgres.safe_psql("select sum(i) from t")[0][0]
         assert res == 5050
-
-
-def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder):
-    def safekeepers_guc(env: ZenithEnv, sk_names: List[str]) -> str:
-        return ','.join(
-            [f'localhost:{sk.port.pg}' for sk in env.safekeepers if sk.name in sk_names])
-
-    def execute_payload(pg: Postgres):
-        with closing(pg.connect()) as conn:
-            with conn.cursor() as cur:
-                # we rely upon autocommit after each statement
-                # as waiting for acceptors happens there
-                cur.execute('CREATE TABLE IF NOT EXISTS t(key int, value text)')
-                cur.execute("INSERT INTO t VALUES (0, 'something')")
-                cur.execute('SELECT SUM(key) FROM t')
-                sum_before = cur.fetchone()[0]
-
-                cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'")
-                cur.execute('SELECT SUM(key) FROM t')
-                sum_after = cur.fetchone()[0]
-                assert sum_after == sum_before + 5000050000
-
-    def show_statuses(safekeepers: List[Safekeeper], tenant_id: str, timeline_id: str):
-        for sk in safekeepers:
-            http_cli = sk.http_client()
-            try:
-                status = http_cli.timeline_status(tenant_id, timeline_id)
-                log.info(f"Safekeeper {sk.name} status: {status}")
-            except Exception as e:
-                log.info(f"Safekeeper {sk.name} status error: {e}")
-
-    zenith_env_builder.num_safekeepers = 4
-    env = zenith_env_builder.init()
-    env.zenith_cli(["branch", "test_replace_safekeeper", "main"])
-
-    log.info("Use only first 3 safekeepers")
-    env.safekeepers[3].stop()
-    active_safekeepers = ['sk1', 'sk2', 'sk3']
-    pg = env.postgres.create('test_replace_safekeeper')
-    pg.adjust_for_wal_acceptors(safekeepers_guc(env, active_safekeepers))
-    pg.start()
-
-    # learn zenith timeline from compute
-    tenant_id = pg.safe_psql("show zenith.zenith_tenant")[0][0]
-    timeline_id = pg.safe_psql("show zenith.zenith_timeline")[0][0]
-
-    execute_payload(pg)
-    show_statuses(env.safekeepers, tenant_id, timeline_id)
-
-    log.info("Restart all safekeepers to flush everything")
-    env.safekeepers[0].stop(immediate=True)
-    execute_payload(pg)
-    env.safekeepers[0].start()
-    env.safekeepers[1].stop(immediate=True)
-    execute_payload(pg)
-    env.safekeepers[1].start()
-    env.safekeepers[2].stop(immediate=True)
-    execute_payload(pg)
-    env.safekeepers[2].start()
-
-    env.safekeepers[0].stop(immediate=True)
-    env.safekeepers[1].stop(immediate=True)
-    env.safekeepers[2].stop(immediate=True)
-    env.safekeepers[0].start()
-    env.safekeepers[1].start()
-    env.safekeepers[2].start()
-
-    execute_payload(pg)
-    show_statuses(env.safekeepers, tenant_id, timeline_id)
-
-    log.info("Stop sk1 (simulate failure) and use only quorum of sk2 and sk3")
-    env.safekeepers[0].stop(immediate=True)
-    execute_payload(pg)
-    show_statuses(env.safekeepers, tenant_id, timeline_id)
-
-    log.info("Recreate postgres to replace failed sk1 with new sk4")
-    pg.stop_and_destroy().create('test_replace_safekeeper')
-    active_safekeepers = ['sk2', 'sk3', 'sk4']
-    env.safekeepers[3].start()
-    pg.adjust_for_wal_acceptors(safekeepers_guc(env, active_safekeepers))
-    pg.start()
-
-    execute_payload(pg)
-    show_statuses(env.safekeepers, tenant_id, timeline_id)
-
-    log.info("Stop sk2 to require quorum of sk3 and sk4 for normal work")
-    env.safekeepers[1].stop(immediate=True)
-    execute_payload(pg)
-    show_statuses(env.safekeepers, tenant_id, timeline_id)

test_runner/batch_others/test_zenith_cli.py

@@ -1,9 +1,8 @@
 import json
 import uuid
-import requests
 
 from psycopg2.extensions import cursor as PgCursor
-from fixtures.zenith_fixtures import ZenithEnv, ZenithEnvBuilder
+from fixtures.zenith_fixtures import ZenithEnv
 from typing import cast
 
 pytest_plugins = ("fixtures.zenith_fixtures")
@@ -106,20 +105,3 @@ def test_cli_tenant_list(zenith_simple_env: ZenithEnv):
     assert env.initial_tenant in tenants
     assert tenant1 in tenants
     assert tenant2 in tenants
-
-
-def test_cli_ipv4_listeners(zenith_env_builder: ZenithEnvBuilder):
-    # Start with single sk
-    zenith_env_builder.num_safekeepers = 1
-    env = zenith_env_builder.init()
-
-    # Connect to sk port on v4 loopback
-    res = requests.get(f'http://127.0.0.1:{env.safekeepers[0].port.http}/v1/status')
-    assert res.ok
-
-    # FIXME Test setup is using localhost:xx in ps config.
-    # Perhaps consider switching test suite to v4 loopback.
-
-    # Connect to ps port on v4 loopback
-    # res = requests.get(f'http://127.0.0.1:{env.pageserver.service_port.http}/v1/status')
-    # assert res.ok

test_runner/fixtures/compare_fixtures.py

@@ -1,188 +0,0 @@
-import pytest
-from contextlib import contextmanager
-from abc import ABC, abstractmethod
-
-from fixtures.zenith_fixtures import PgBin, PgProtocol, VanillaPostgres, ZenithEnv
-from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker
-
-# Type-related stuff
-from typing import Iterator
-
-
-class PgCompare(ABC):
-    """Common interface of all postgres implementations, useful for benchmarks.
-
-    This class is a helper class for the zenith_with_baseline fixture. See its documentation
-    for more details.
-    """
-    @property
-    @abstractmethod
-    def pg(self) -> PgProtocol:
-        pass
-
-    @property
-    @abstractmethod
-    def pg_bin(self) -> PgBin:
-        pass
-
-    @abstractmethod
-    def flush(self) -> None:
-        pass
-
-    @abstractmethod
-    def report_peak_memory_use(self) -> None:
-        pass
-
-    @abstractmethod
-    def report_size(self) -> None:
-        pass
-
-    @contextmanager
-    @abstractmethod
-    def record_pageserver_writes(self, out_name):
-        pass
-
-    @contextmanager
-    @abstractmethod
-    def record_duration(self, out_name):
-        pass
-
-
-class ZenithCompare(PgCompare):
-    """PgCompare interface for the zenith stack."""
-    def __init__(self,
-                 zenbenchmark: ZenithBenchmarker,
-                 zenith_simple_env: ZenithEnv,
-                 pg_bin: PgBin,
-                 branch_name):
-        self.env = zenith_simple_env
-        self.zenbenchmark = zenbenchmark
-        self._pg_bin = pg_bin
-
-        # We only use one branch and one timeline
-        self.branch = branch_name
-        self.env.zenith_cli(["branch", self.branch, "empty"])
-        self._pg = self.env.postgres.create_start(self.branch)
-        self.timeline = self.pg.safe_psql("SHOW zenith.zenith_timeline")[0][0]
-
-        # Long-lived cursor, useful for flushing
-        self.psconn = self.env.pageserver.connect()
-        self.pscur = self.psconn.cursor()
-
-    @property
-    def pg(self):
-        return self._pg
-
-    @property
-    def pg_bin(self):
-        return self._pg_bin
-
-    def flush(self):
-        self.pscur.execute(f"do_gc {self.env.initial_tenant} {self.timeline} 0")
-
-    def report_peak_memory_use(self) -> None:
-        self.zenbenchmark.record("peak_mem",
-                                 self.zenbenchmark.get_peak_mem(self.env.pageserver) / 1024,
-                                 'MB',
-                                 report=MetricReport.LOWER_IS_BETTER)
-
-    def report_size(self) -> None:
-        timeline_size = self.zenbenchmark.get_timeline_size(self.env.repo_dir,
-                                                            self.env.initial_tenant,
-                                                            self.timeline)
-        self.zenbenchmark.record('size',
-                                 timeline_size / (1024 * 1024),
-                                 'MB',
-                                 report=MetricReport.LOWER_IS_BETTER)
-
-    def record_pageserver_writes(self, out_name):
-        return self.zenbenchmark.record_pageserver_writes(self.env.pageserver, out_name)
-
-    def record_duration(self, out_name):
-        return self.zenbenchmark.record_duration(out_name)
-
-
-class VanillaCompare(PgCompare):
-    """PgCompare interface for vanilla postgres."""
-    def __init__(self, zenbenchmark, vanilla_pg: VanillaPostgres):
-        self._pg = vanilla_pg
-        self.zenbenchmark = zenbenchmark
-        vanilla_pg.configure(['shared_buffers=1MB'])
-        vanilla_pg.start()
-
-        # Long-lived cursor, useful for flushing
-        self.conn = self.pg.connect()
-        self.cur = self.conn.cursor()
-
-    @property
-    def pg(self):
-        return self._pg
-
-    @property
-    def pg_bin(self):
-        return self._pg.pg_bin
-
-    def flush(self):
-        self.cur.execute("checkpoint")
-
-    def report_peak_memory_use(self) -> None:
-        pass  # TODO find something
-
-    def report_size(self) -> None:
-        data_size = self.pg.get_subdir_size('base')
-        self.zenbenchmark.record('data_size',
-                                 data_size / (1024 * 1024),
-                                 'MB',
-                                 report=MetricReport.LOWER_IS_BETTER)
-        wal_size = self.pg.get_subdir_size('pg_wal')
-        self.zenbenchmark.record('wal_size',
-                                 wal_size / (1024 * 1024),
-                                 'MB',
-                                 report=MetricReport.LOWER_IS_BETTER)
-
-    @contextmanager
-    def record_pageserver_writes(self, out_name):
-        yield  # Do nothing
-
-    def record_duration(self, out_name):
-        return self.zenbenchmark.record_duration(out_name)
-
-
-@pytest.fixture(scope='function')
-def zenith_compare(request, zenbenchmark, pg_bin, zenith_simple_env) -> ZenithCompare:
-    branch_name = request.node.name
-    return ZenithCompare(zenbenchmark, zenith_simple_env, pg_bin, branch_name)
-
-
-@pytest.fixture(scope='function')
-def vanilla_compare(zenbenchmark, vanilla_pg) -> VanillaCompare:
-    return VanillaCompare(zenbenchmark, vanilla_pg)
-
-
-@pytest.fixture(params=["vanilla_compare", "zenith_compare"], ids=["vanilla", "zenith"])
-def zenith_with_baseline(request) -> PgCompare:
-    """Parameterized fixture that helps compare zenith against vanilla postgres.
-
-    A test that uses this fixture turns into a parameterized test that runs against:
-    1. A vanilla postgres instance
-    2. A simple zenith env (see zenith_simple_env)
-    3. Possibly other postgres protocol implementations.
-
-    The main goal of this fixture is to make it easier for people to read and write
-    performance tests. Easy test writing leads to more tests.
-
-    Perfect encapsulation of the postgres implementations is **not** a goal because
-    it's impossible. Operational and configuration differences in the different
-    implementations sometimes matter, and the writer of the test should be mindful
-    of that.
-
-    If a test requires some one-off special implementation-specific logic, use of
-    isinstance(zenith_with_baseline, ZenithCompare) is encouraged. Though if that
-    implementation-specific logic is widely useful across multiple tests, it might
-    make sense to add methods to the PgCompare class.
-    """
-    fixture = request.getfixturevalue(request.param)
-    if isinstance(fixture, PgCompare):
-        return fixture
-    else:
-        raise AssertionError(f"test error: fixture {request.param} is not PgCompare")

test_runner/fixtures/zenith_fixtures.py

@@ -19,6 +19,7 @@ import subprocess
 import time
 import filecmp
 import tempfile
+import toml
 
 from contextlib import closing
 from pathlib import Path
@@ -184,16 +185,6 @@ def worker_base_port(worker_seq_no: int):
     return BASE_PORT + worker_seq_no * WORKER_PORT_NUM
 
 
-def get_dir_size(path: str) -> int:
-    """Return size in bytes."""
-    totalbytes = 0
-    for root, dirs, files in os.walk(path):
-        for name in files:
-            totalbytes += os.path.getsize(os.path.join(root, name))
-
-    return totalbytes
-
-
 def can_bind(host: str, port: int) -> bool:
     """
     Check whether a host:port is available to bind for listening
@@ -240,7 +231,7 @@ class PgProtocol:
     def __init__(self, host: str, port: int, username: Optional[str] = None):
         self.host = host
         self.port = port
-        self.username = username
+        self.username = username or "zenith_admin"
 
     def connstr(self,
                 *,
@@ -252,15 +243,10 @@ class PgProtocol:
         """
 
         username = username or self.username
-        res = f'host={self.host} port={self.port} dbname={dbname}'
-
-        if username:
-            res = f'{res} user={username}'
-
-        if password:
-            res = f'{res} password={password}'
-
-        return res
+        res = f'host={self.host} port={self.port} user={username} dbname={dbname}'
+        if not password:
+            return res
+        return f'{res} password={password}'
 
     # autocommit=True here by default because that's what we need most of the time
     def connect(self,
@@ -387,7 +373,6 @@ class MockS3Server:
     def kill(self):
         self.subprocess.kill()
 
-
 class ZenithEnvBuilder:
     """
     Builder object to create a Zenith runtime environment
@@ -517,6 +502,9 @@ class ZenithEnv:
         self.port_distributor = config.port_distributor
         self.s3_mock_server = config.s3_mock_server
 
+        # If specified, this config will be passed for all zenith_cli calls
+        self.override_config: Optional[Dict] = None
+
         self.postgres = PostgresFactory(self)
 
         self.safekeepers: List[Safekeeper] = []
@@ -614,9 +602,22 @@ sync = false # Disable fsyncs to make the tests go faster
 
         assert type(arguments) == list
 
+        if self.override_config is not None:
+            tmp_config = tempfile.NamedTemporaryFile(mode='w+')
+            tmp_config.write(toml.dumps(self.override_config))
+            tmp_config.flush()
+            log.info('Using overriden config to run next command!')
+            log.info(f'Config: {toml.dumps(self.override_config)}')
+        else:
+            tmp_config = None
+
         bin_zenith = os.path.join(str(zenith_binpath), 'zenith')
 
-        args = [bin_zenith] + arguments
+        args = [bin_zenith]
+        if tmp_config is not None:
+            args += ['--override-config', tmp_config.name]
+        args += arguments
+
         log.info('Running command "{}"'.format(' '.join(args)))
         log.info(f'Running in "{self.repo_dir}"')
 
@@ -652,9 +653,21 @@ sync = false # Disable fsyncs to make the tests go faster
             log.info(msg)
 
             raise Exception(msg) from exc
+        finally:
+            if tmp_config is not None:
+                tmp_config.close()
 
         return res
 
+    def read_toml_config(self) -> Dict:
+        """
+        Read the config file from the repo directory.
+
+        Returns a dictionary of the config file.
+        """
+        with open(os.path.join(str(self.repo_dir), 'config')) as f:
+            return toml.load(f)
+
     @cached_property
     def auth_keys(self) -> AuthKeys:
         pub = (Path(self.repo_dir) / 'auth_public_key.pem').read_bytes()
@@ -850,7 +863,7 @@ class ZenithPageserver(PgProtocol):
                  port: PageserverPort,
                  remote_storage: Optional[RemoteStorage] = None,
                  enable_auth=False):
-        super().__init__(host='localhost', port=port.pg, username='zenith_admin')
+        super().__init__(host='localhost', port=port.pg)
         self.env = env
         self.running = False
         self.service_port = port  # do not shadow PgProtocol.port which is just int
@@ -988,54 +1001,10 @@ def pg_bin(test_output_dir: str) -> PgBin:
     return PgBin(test_output_dir)
 
 
-class VanillaPostgres(PgProtocol):
-    def __init__(self, pgdatadir: str, pg_bin: PgBin, port: int):
-        super().__init__(host='localhost', port=port)
-        self.pgdatadir = pgdatadir
-        self.pg_bin = pg_bin
-        self.running = False
-        self.pg_bin.run_capture(['initdb', '-D', pgdatadir])
-
-    def configure(self, options: List[str]) -> None:
-        """Append lines into postgresql.conf file."""
-        assert not self.running
-        with open(os.path.join(self.pgdatadir, 'postgresql.conf'), 'a') as conf_file:
-            conf_file.writelines(options)
-
-    def start(self) -> None:
-        assert not self.running
-        self.running = True
-        self.pg_bin.run_capture(['pg_ctl', '-D', self.pgdatadir, 'start'])
-
-    def stop(self) -> None:
-        assert self.running
-        self.running = False
-        self.pg_bin.run_capture(['pg_ctl', '-D', self.pgdatadir, 'stop'])
-
-    def get_subdir_size(self, subdir) -> int:
-        """Return size of pgdatadir subdirectory in bytes."""
-        return get_dir_size(os.path.join(self.pgdatadir, subdir))
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc, tb):
-        if self.running:
-            self.stop()
-
-
-@pytest.fixture(scope='function')
-def vanilla_pg(test_output_dir: str) -> Iterator[VanillaPostgres]:
-    pgdatadir = os.path.join(test_output_dir, "pgdata-vanilla")
-    pg_bin = PgBin(test_output_dir)
-    with VanillaPostgres(pgdatadir, pg_bin, 5432) as vanilla_pg:
-        yield vanilla_pg
-
-
 class Postgres(PgProtocol):
     """ An object representing a running postgres daemon. """
     def __init__(self, env: ZenithEnv, tenant_id: str, port: int):
-        super().__init__(host='localhost', port=port, username='zenith_admin')
+        super().__init__(host='localhost', port=port)
 
         self.env = env
         self.running = False
@@ -1073,6 +1042,8 @@ class Postgres(PgProtocol):
         path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.node_name
         self.pgdata_dir = os.path.join(self.env.repo_dir, path)
 
+        if self.env.safekeepers:
+            self.adjust_for_wal_acceptors(self.env.get_safekeeper_connstrs())
         if config_lines is None:
             config_lines = []
         self.config(config_lines)
@@ -1129,9 +1100,7 @@ class Postgres(PgProtocol):
                 # walproposer uses different application_name
                 if ("synchronous_standby_names" in cfg_line or
                         # don't ask pageserver to fetch WAL from compute
-                        "callmemaybe_connstring" in cfg_line or
-                        # don't repeat wal_acceptors multiple times
-                        "wal_acceptors" in cfg_line):
+                        "callmemaybe_connstring" in cfg_line):
                     continue
                 f.write(cfg_line)
             f.write("synchronous_standby_names = 'walproposer'\n")

test_runner/performance/test_bulk_insert.py

@@ -2,13 +2,8 @@ from contextlib import closing
 from fixtures.zenith_fixtures import ZenithEnv
 from fixtures.log_helper import log
 from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker
-from fixtures.compare_fixtures import PgCompare, VanillaCompare, ZenithCompare
 
-pytest_plugins = (
-    "fixtures.zenith_fixtures",
-    "fixtures.benchmark_fixture",
-    "fixtures.compare_fixtures",
-)
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
 
 
 #
@@ -21,19 +16,47 @@ pytest_plugins = (
 # 3. Disk space used
 # 4. Peak memory usage
 #
-def test_bulk_insert(zenith_with_baseline: PgCompare):
-    env = zenith_with_baseline
+def test_bulk_insert(zenith_simple_env: ZenithEnv, zenbenchmark: ZenithBenchmarker):
+    env = zenith_simple_env
+    # Create a branch for us
+    env.zenith_cli(["branch", "test_bulk_insert", "empty"])
+
+    pg = env.postgres.create_start('test_bulk_insert')
+    log.info("postgres is running on 'test_bulk_insert' branch")
+
+    # Open a connection directly to the page server that we'll use to force
+    # flushing the layers to disk
+    psconn = env.pageserver.connect()
+    pscur = psconn.cursor()
 
     # Get the timeline ID of our branch. We need it for the 'do_gc' command
-    with closing(env.pg.connect()) as conn:
+    with closing(pg.connect()) as conn:
         with conn.cursor() as cur:
+            cur.execute("SHOW zenith.zenith_timeline")
+            timeline = cur.fetchone()[0]
+
             cur.execute("create table huge (i int, j int);")
 
             # Run INSERT, recording the time and I/O it takes
-            with env.record_pageserver_writes('pageserver_writes'):
-                with env.record_duration('insert'):
+            with zenbenchmark.record_pageserver_writes(env.pageserver, 'pageserver_writes'):
+                with zenbenchmark.record_duration('insert'):
                     cur.execute("insert into huge values (generate_series(1, 5000000), 0);")
-                    env.flush()
 
-            env.report_peak_memory_use()
-            env.report_size()
+                    # Flush the layers from memory to disk. This is included in the reported
+                    # time and I/O
+                    pscur.execute(f"do_gc {env.initial_tenant} {timeline} 0")
+
+            # Record peak memory usage
+            zenbenchmark.record("peak_mem",
+                                zenbenchmark.get_peak_mem(env.pageserver) / 1024,
+                                'MB',
+                                report=MetricReport.LOWER_IS_BETTER)
+
+            # Report disk space used by the repository
+            timeline_size = zenbenchmark.get_timeline_size(env.repo_dir,
+                                                           env.initial_tenant,
+                                                           timeline)
+            zenbenchmark.record('size',
+                                timeline_size / (1024 * 1024),
+                                'MB',
+                                report=MetricReport.LOWER_IS_BETTER)

test_runner/performance/test_copy.py

@@ -2,15 +2,10 @@ from contextlib import closing
 from fixtures.zenith_fixtures import ZenithEnv
 from fixtures.log_helper import log
 from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker
-from fixtures.compare_fixtures import PgCompare, VanillaCompare, ZenithCompare
 from io import BufferedReader, RawIOBase
 from itertools import repeat
 
-pytest_plugins = (
-    "fixtures.zenith_fixtures",
-    "fixtures.benchmark_fixture",
-    "fixtures.compare_fixtures",
-)
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
 
 
 class CopyTestData(RawIOBase):
@@ -47,41 +42,77 @@ def copy_test_data(rows: int):
 #
 # COPY performance tests.
 #
-def test_copy(zenith_with_baseline: PgCompare):
-    env = zenith_with_baseline
+def test_copy(zenith_simple_env: ZenithEnv, zenbenchmark: ZenithBenchmarker):
+    env = zenith_simple_env
+    # Create a branch for us
+    env.zenith_cli(["branch", "test_copy", "empty"])
+
+    pg = env.postgres.create_start('test_copy')
+    log.info("postgres is running on 'test_copy' branch")
+
+    # Open a connection directly to the page server that we'll use to force
+    # flushing the layers to disk
+    psconn = env.pageserver.connect()
+    pscur = psconn.cursor()
 
     # Get the timeline ID of our branch. We need it for the pageserver 'checkpoint' command
-    with closing(env.pg.connect()) as conn:
+    with closing(pg.connect()) as conn:
         with conn.cursor() as cur:
+            cur.execute("SHOW zenith.zenith_timeline")
+            timeline = cur.fetchone()[0]
+
             cur.execute("create table copytest (i int, t text);")
 
             # Load data with COPY, recording the time and I/O it takes.
             #
             # Since there's no data in the table previously, this extends it.
-            with env.record_pageserver_writes('copy_extend_pageserver_writes'):
-                with env.record_duration('copy_extend'):
+            with zenbenchmark.record_pageserver_writes(env.pageserver,
+                                                       'copy_extend_pageserver_writes'):
+                with zenbenchmark.record_duration('copy_extend'):
                     cur.copy_from(copy_test_data(1000000), 'copytest')
-                    env.flush()
+                    # Flush the layers from memory to disk. This is included in the reported
+                    # time and I/O
+                    pscur.execute(f"checkpoint {env.initial_tenant} {timeline}")
 
             # Delete most rows, and VACUUM to make the space available for reuse.
-            with env.record_pageserver_writes('delete_pageserver_writes'):
-                with env.record_duration('delete'):
+            with zenbenchmark.record_pageserver_writes(env.pageserver, 'delete_pageserver_writes'):
+                with zenbenchmark.record_duration('delete'):
                     cur.execute("delete from copytest where i % 100 <> 0;")
-                    env.flush()
+                    # Flush the layers from memory to disk. This is included in the reported
+                    # time and I/O
+                    pscur.execute(f"checkpoint {env.initial_tenant} {timeline}")
 
-            with env.record_pageserver_writes('vacuum_pageserver_writes'):
-                with env.record_duration('vacuum'):
+            with zenbenchmark.record_pageserver_writes(env.pageserver, 'vacuum_pageserver_writes'):
+                with zenbenchmark.record_duration('vacuum'):
                     cur.execute("vacuum copytest")
-                    env.flush()
+                    # Flush the layers from memory to disk. This is included in the reported
+                    # time and I/O
+                    pscur.execute(f"checkpoint {env.initial_tenant} {timeline}")
 
             # Load data into the table again. This time, this will use the space free'd
             # by the VACUUM.
             #
             # This will also clear all the VM bits.
-            with env.record_pageserver_writes('copy_reuse_pageserver_writes'):
-                with env.record_duration('copy_reuse'):
+            with zenbenchmark.record_pageserver_writes(env.pageserver,
+                                                       'copy_reuse_pageserver_writes'):
+                with zenbenchmark.record_duration('copy_reuse'):
                     cur.copy_from(copy_test_data(1000000), 'copytest')
-                    env.flush()
 
-            env.report_peak_memory_use()
-            env.report_size()
+                    # Flush the layers from memory to disk. This is included in the reported
+                    # time and I/O
+                    pscur.execute(f"checkpoint {env.initial_tenant} {timeline}")
+
+            # Record peak memory usage
+            zenbenchmark.record("peak_mem",
+                                zenbenchmark.get_peak_mem(env.pageserver) / 1024,
+                                'MB',
+                                report=MetricReport.LOWER_IS_BETTER)
+
+            # Report disk space used by the repository
+            timeline_size = zenbenchmark.get_timeline_size(env.repo_dir,
+                                                           env.initial_tenant,
+                                                           timeline)
+            zenbenchmark.record('size',
+                                timeline_size / (1024 * 1024),
+                                'MB',
+                                report=MetricReport.LOWER_IS_BETTER)

test_runner/performance/test_gist_build.py

@@ -2,14 +2,9 @@ import os
 from contextlib import closing
 from fixtures.benchmark_fixture import MetricReport
 from fixtures.zenith_fixtures import ZenithEnv
-from fixtures.compare_fixtures import PgCompare, VanillaCompare, ZenithCompare
 from fixtures.log_helper import log
 
-pytest_plugins = (
-    "fixtures.zenith_fixtures",
-    "fixtures.benchmark_fixture",
-    "fixtures.compare_fixtures",
-)
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
 
 
 #
@@ -17,11 +12,24 @@ pytest_plugins = (
 # As of this writing, we're duplicate those giant WAL records for each page,
 # which makes the delta layer about 32x larger than it needs to be.
 #
-def test_gist_buffering_build(zenith_with_baseline: PgCompare):
-    env = zenith_with_baseline
+def test_gist_buffering_build(zenith_simple_env: ZenithEnv, zenbenchmark):
+    env = zenith_simple_env
+    # Create a branch for us
+    env.zenith_cli(["branch", "test_gist_buffering_build", "empty"])
 
-    with closing(env.pg.connect()) as conn:
+    pg = env.postgres.create_start('test_gist_buffering_build')
+    log.info("postgres is running on 'test_gist_buffering_build' branch")
+
+    # Open a connection directly to the page server that we'll use to force
+    # flushing the layers to disk
+    psconn = env.pageserver.connect()
+    pscur = psconn.cursor()
+
+    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+    with closing(pg.connect()) as conn:
         with conn.cursor() as cur:
+            cur.execute("SHOW zenith.zenith_timeline")
+            timeline = cur.fetchone()[0]
 
             # Create test table.
             cur.execute("create table gist_point_tbl(id int4, p point)")
@@ -30,12 +38,27 @@ def test_gist_buffering_build(zenith_with_baseline: PgCompare):
             )
 
             # Build the index.
-            with env.record_pageserver_writes('pageserver_writes'):
-                with env.record_duration('build'):
+            with zenbenchmark.record_pageserver_writes(env.pageserver, 'pageserver_writes'):
+                with zenbenchmark.record_duration('build'):
                     cur.execute(
                         "create index gist_pointidx2 on gist_point_tbl using gist(p) with (buffering = on)"
                     )
-                    env.flush()
 
-            env.report_peak_memory_use()
-            env.report_size()
+                    # Flush the layers from memory to disk. This is included in the reported
+                    # time and I/O
+                    pscur.execute(f"do_gc {env.initial_tenant} {timeline} 1000000")
+
+            # Record peak memory usage
+            zenbenchmark.record("peak_mem",
+                                zenbenchmark.get_peak_mem(env.pageserver) / 1024,
+                                'MB',
+                                report=MetricReport.LOWER_IS_BETTER)
+
+            # Report disk space used by the repository
+            timeline_size = zenbenchmark.get_timeline_size(env.repo_dir,
+                                                           env.initial_tenant,
+                                                           timeline)
+            zenbenchmark.record('size',
+                                timeline_size / (1024 * 1024),
+                                'MB',
+                                report=MetricReport.LOWER_IS_BETTER)

test_runner/performance/test_parallel_copy_to.py

@@ -1,16 +1,11 @@
 from io import BytesIO
 import asyncio
 import asyncpg
-from fixtures.zenith_fixtures import ZenithEnv, Postgres, PgProtocol
+from fixtures.zenith_fixtures import ZenithEnv, Postgres
 from fixtures.log_helper import log
 from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker
-from fixtures.compare_fixtures import PgCompare, VanillaCompare, ZenithCompare
 
-pytest_plugins = (
-    "fixtures.zenith_fixtures",
-    "fixtures.benchmark_fixture",
-    "fixtures.compare_fixtures",
-)
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
 
 
 async def repeat_bytes(buf, repetitions: int):
@@ -18,7 +13,7 @@ async def repeat_bytes(buf, repetitions: int):
         yield buf
 
 
-async def copy_test_data_to_table(pg: PgProtocol, worker_id: int, table_name: str):
+async def copy_test_data_to_table(pg: Postgres, worker_id: int, table_name: str):
     buf = BytesIO()
     for i in range(1000):
         buf.write(
@@ -31,7 +26,7 @@ async def copy_test_data_to_table(pg: PgProtocol, worker_id: int, table_name: st
     await pg_conn.copy_to_table(table_name, source=copy_input)
 
 
-async def parallel_load_different_tables(pg: PgProtocol, n_parallel: int):
+async def parallel_load_different_tables(pg: Postgres, n_parallel: int):
     workers = []
     for worker_id in range(n_parallel):
         worker = copy_test_data_to_table(pg, worker_id, f'copytest_{worker_id}')
@@ -42,25 +37,54 @@ async def parallel_load_different_tables(pg: PgProtocol, n_parallel: int):
 
 
 # Load 5 different tables in parallel with COPY TO
-def test_parallel_copy_different_tables(zenith_with_baseline: PgCompare, n_parallel=5):
+def test_parallel_copy_different_tables(zenith_simple_env: ZenithEnv,
+                                        zenbenchmark: ZenithBenchmarker,
+                                        n_parallel=5):
 
-    env = zenith_with_baseline
-    conn = env.pg.connect()
+    env = zenith_simple_env
+    # Create a branch for us
+    env.zenith_cli(["branch", "test_parallel_copy_different_tables", "empty"])
+
+    pg = env.postgres.create_start('test_parallel_copy_different_tables')
+    log.info("postgres is running on 'test_parallel_copy_different_tables' branch")
+
+    # Open a connection directly to the page server that we'll use to force
+    # flushing the layers to disk
+    psconn = env.pageserver.connect()
+    pscur = psconn.cursor()
+
+    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+    conn = pg.connect()
     cur = conn.cursor()
+    cur.execute("SHOW zenith.zenith_timeline")
+    timeline = cur.fetchone()[0]
 
     for worker_id in range(n_parallel):
         cur.execute(f'CREATE TABLE copytest_{worker_id} (i int, t text)')
 
-    with env.record_pageserver_writes('pageserver_writes'):
-        with env.record_duration('load'):
-            asyncio.run(parallel_load_different_tables(env.pg, n_parallel))
-            env.flush()
+    with zenbenchmark.record_pageserver_writes(env.pageserver, 'pageserver_writes'):
+        with zenbenchmark.record_duration('load'):
+            asyncio.run(parallel_load_different_tables(pg, n_parallel))
 
-    env.report_peak_memory_use()
-    env.report_size()
+            # Flush the layers from memory to disk. This is included in the reported
+            # time and I/O
+            pscur.execute(f"do_gc {env.initial_tenant} {timeline} 0")
+
+    # Record peak memory usage
+    zenbenchmark.record("peak_mem",
+                        zenbenchmark.get_peak_mem(env.pageserver) / 1024,
+                        'MB',
+                        report=MetricReport.LOWER_IS_BETTER)
+
+    # Report disk space used by the repository
+    timeline_size = zenbenchmark.get_timeline_size(env.repo_dir, env.initial_tenant, timeline)
+    zenbenchmark.record('size',
+                        timeline_size / (1024 * 1024),
+                        'MB',
+                        report=MetricReport.LOWER_IS_BETTER)
 
 
-async def parallel_load_same_table(pg: PgProtocol, n_parallel: int):
+async def parallel_load_same_table(pg: Postgres, n_parallel: int):
     workers = []
     for worker_id in range(n_parallel):
         worker = copy_test_data_to_table(pg, worker_id, f'copytest')
@@ -71,17 +95,46 @@ async def parallel_load_same_table(pg: PgProtocol, n_parallel: int):
 
 
 # Load data into one table with COPY TO from 5 parallel connections
-def test_parallel_copy_same_table(zenith_with_baseline: PgCompare, n_parallel=5):
-    env = zenith_with_baseline
-    conn = env.pg.connect()
+def test_parallel_copy_same_table(zenith_simple_env: ZenithEnv,
+                                  zenbenchmark: ZenithBenchmarker,
+                                  n_parallel=5):
+    env = zenith_simple_env
+    # Create a branch for us
+    env.zenith_cli(["branch", "test_parallel_copy_same_table", "empty"])
+
+    pg = env.postgres.create_start('test_parallel_copy_same_table')
+    log.info("postgres is running on 'test_parallel_copy_same_table' branch")
+
+    # Open a connection directly to the page server that we'll use to force
+    # flushing the layers to disk
+    psconn = env.pageserver.connect()
+    pscur = psconn.cursor()
+
+    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+    conn = pg.connect()
     cur = conn.cursor()
+    cur.execute("SHOW zenith.zenith_timeline")
+    timeline = cur.fetchone()[0]
 
     cur.execute(f'CREATE TABLE copytest (i int, t text)')
 
-    with env.record_pageserver_writes('pageserver_writes'):
-        with env.record_duration('load'):
-            asyncio.run(parallel_load_same_table(env.pg, n_parallel))
-            env.flush()
+    with zenbenchmark.record_pageserver_writes(env.pageserver, 'pageserver_writes'):
+        with zenbenchmark.record_duration('load'):
+            asyncio.run(parallel_load_same_table(pg, n_parallel))
 
-    env.report_peak_memory_use()
-    env.report_size()
+            # Flush the layers from memory to disk. This is included in the reported
+            # time and I/O
+            pscur.execute(f"do_gc {env.initial_tenant} {timeline} 0")
+
+    # Record peak memory usage
+    zenbenchmark.record("peak_mem",
+                        zenbenchmark.get_peak_mem(env.pageserver) / 1024,
+                        'MB',
+                        report=MetricReport.LOWER_IS_BETTER)
+
+    # Report disk space used by the repository
+    timeline_size = zenbenchmark.get_timeline_size(env.repo_dir, env.initial_tenant, timeline)
+    zenbenchmark.record('size',
+                        timeline_size / (1024 * 1024),
+                        'MB',
+                        report=MetricReport.LOWER_IS_BETTER)

test_runner/performance/test_perf_pgbench.py

@@ -1,15 +1,10 @@
 from contextlib import closing
-from fixtures.zenith_fixtures import PgBin, VanillaPostgres, ZenithEnv
-from fixtures.compare_fixtures import PgCompare, VanillaCompare, ZenithCompare
+from fixtures.zenith_fixtures import PgBin, ZenithEnv
 
 from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker
 from fixtures.log_helper import log
 
-pytest_plugins = (
-    "fixtures.zenith_fixtures",
-    "fixtures.benchmark_fixture",
-    "fixtures.compare_fixtures",
-)
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
 
 
 #
@@ -21,16 +16,47 @@ pytest_plugins = (
 # 2. Time to run 5000 pgbench transactions
 # 3. Disk space used
 #
-def test_pgbench(zenith_with_baseline: PgCompare):
-    env = zenith_with_baseline
+def test_pgbench(zenith_simple_env: ZenithEnv, pg_bin: PgBin, zenbenchmark: ZenithBenchmarker):
+    env = zenith_simple_env
+    # Create a branch for us
+    env.zenith_cli(["branch", "test_pgbench_perf", "empty"])
 
-    with env.record_pageserver_writes('pageserver_writes'):
-        with env.record_duration('init'):
-            env.pg_bin.run_capture(['pgbench', '-s5', '-i', env.pg.connstr()])
-            env.flush()
+    pg = env.postgres.create_start('test_pgbench_perf')
+    log.info("postgres is running on 'test_pgbench_perf' branch")
 
-    with env.record_duration('5000_xacts'):
-        env.pg_bin.run_capture(['pgbench', '-c1', '-t5000', env.pg.connstr()])
-    env.flush()
+    # Open a connection directly to the page server that we'll use to force
+    # flushing the layers to disk
+    psconn = env.pageserver.connect()
+    pscur = psconn.cursor()
 
-    env.report_size()
+    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SHOW zenith.zenith_timeline")
+            timeline = cur.fetchone()[0]
+
+    connstr = pg.connstr()
+
+    # Initialize pgbench database, recording the time and I/O it takes
+    with zenbenchmark.record_pageserver_writes(env.pageserver, 'pageserver_writes'):
+        with zenbenchmark.record_duration('init'):
+            pg_bin.run_capture(['pgbench', '-s5', '-i', connstr])
+
+            # Flush the layers from memory to disk. This is included in the reported
+            # time and I/O
+            pscur.execute(f"do_gc {env.initial_tenant} {timeline} 0")
+
+    # Run pgbench for 5000 transactions
+    with zenbenchmark.record_duration('5000_xacts'):
+        pg_bin.run_capture(['pgbench', '-c1', '-t5000', connstr])
+
+    # Flush the layers to disk again. This is *not' included in the reported time,
+    # though.
+    pscur.execute(f"do_gc {env.initial_tenant} {timeline} 0")
+
+    # Report disk space used by the repository
+    timeline_size = zenbenchmark.get_timeline_size(env.repo_dir, env.initial_tenant, timeline)
+    zenbenchmark.record('size',
+                        timeline_size / (1024 * 1024),
+                        'MB',
+                        report=MetricReport.LOWER_IS_BETTER)

test_runner/performance/test_small_seqscans.py

@@ -7,19 +7,24 @@ from contextlib import closing
 from fixtures.zenith_fixtures import ZenithEnv
 from fixtures.log_helper import log
 from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker
-from fixtures.compare_fixtures import PgCompare
 
-pytest_plugins = (
-    "fixtures.zenith_fixtures",
-    "fixtures.benchmark_fixture",
-    "fixtures.compare_fixtures",
-)
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
 
 
-def test_small_seqscans(zenith_with_baseline: PgCompare):
-    env = zenith_with_baseline
+def test_small_seqscans(zenith_simple_env: ZenithEnv, zenbenchmark: ZenithBenchmarker):
+    env = zenith_simple_env
+    # Create a branch for us
+    env.zenith_cli(["branch", "test_small_seqscans", "empty"])
 
-    with closing(env.pg.connect()) as conn:
+    pg = env.postgres.create_start('test_small_seqscans')
+    log.info("postgres is running on 'test_small_seqscans' branch")
+
+    # Open a connection directly to the page server that we'll use to force
+    # flushing the layers to disk
+    psconn = env.pageserver.connect()
+    pscur = psconn.cursor()
+
+    with closing(pg.connect()) as conn:
         with conn.cursor() as cur:
             cur.execute('create table t (i integer);')
             cur.execute('insert into t values (generate_series(1,100000));')
@@ -33,6 +38,6 @@ def test_small_seqscans(zenith_with_baseline: PgCompare):
             log.info(f"shared_buffers is {row[0]}, table size {row[1]}")
             assert int(row[0]) < int(row[1])
 
-            with env.record_duration('run'):
+            with zenbenchmark.record_duration('run'):
                 for i in range(1000):
                     cur.execute('select count(*) from t;')

test_runner/performance/test_write_amplification.py

@@ -14,23 +14,32 @@ import os
 from contextlib import closing
 from fixtures.benchmark_fixture import MetricReport
 from fixtures.zenith_fixtures import ZenithEnv
-from fixtures.compare_fixtures import PgCompare, VanillaCompare, ZenithCompare
 from fixtures.log_helper import log
 
-pytest_plugins = (
-    "fixtures.zenith_fixtures",
-    "fixtures.benchmark_fixture",
-    "fixtures.compare_fixtures",
-)
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
 
 
-def test_write_amplification(zenith_with_baseline: PgCompare):
-    env = zenith_with_baseline
+def test_write_amplification(zenith_simple_env: ZenithEnv, zenbenchmark):
+    env = zenith_simple_env
+    # Create a branch for us
+    env.zenith_cli(["branch", "test_write_amplification", "empty"])
 
-    with closing(env.pg.connect()) as conn:
+    pg = env.postgres.create_start('test_write_amplification')
+    log.info("postgres is running on 'test_write_amplification' branch")
+
+    # Open a connection directly to the page server that we'll use to force
+    # flushing the layers to disk
+    psconn = env.pageserver.connect()
+    pscur = psconn.cursor()
+
+    with closing(pg.connect()) as conn:
         with conn.cursor() as cur:
-            with env.record_pageserver_writes('pageserver_writes'):
-                with env.record_duration('run'):
+            # Get the timeline ID of our branch. We need it for the 'do_gc' command
+            cur.execute("SHOW zenith.zenith_timeline")
+            timeline = cur.fetchone()[0]
+
+            with zenbenchmark.record_pageserver_writes(env.pageserver, 'pageserver_writes'):
+                with zenbenchmark.record_duration('run'):
 
                     # NOTE: Because each iteration updates every table already created,
                     # the runtime and write amplification is O(n^2), where n is the
@@ -62,6 +71,13 @@ def test_write_amplification(zenith_with_baseline: PgCompare):
                         # slower, adding some delays in this loop.  But forcing
                         # the checkpointing and GC makes the test go faster,
                         # with the same total I/O effect.
-                        env.flush()
+                        pscur.execute(f"do_gc {env.initial_tenant} {timeline} 0")
 
-            env.report_size()
+            # Report disk space used by the repository
+            timeline_size = zenbenchmark.get_timeline_size(env.repo_dir,
+                                                           env.initial_tenant,
+                                                           timeline)
+            zenbenchmark.record('size',
+                                timeline_size / (1024 * 1024),
+                                'MB',
+                                report=MetricReport.LOWER_IS_BETTER)

walkeeper/Cargo.toml

@@ -1,23 +1,27 @@
 [package]
 name = "walkeeper"
 version = "0.1.0"
-edition = "2021"
+authors = ["Stas Kelvich <stas@zenith.tech>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 regex = "1.4.5"
 bytes = "1.0.1"
 byteorder = "1.4.3"
 hyper = "0.14"
+routerify = "2"
 fs2 = "0.4.3"
 lazy_static = "1.4.0"
 serde_json = "1"
 tracing = "0.1.27"
-clap = "3.0"
+clap = "2.33.0"
 daemonize = "0.4.1"
 rust-s3 = { version = "0.28", default-features = false, features = ["no-verify-ssl", "tokio-rustls-tls"] }
 tokio = { version = "1.11", features = ["macros"] }
-postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
+postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 anyhow = "1.0"
 crc32c = "0.6.0"
 humantime = "2.1.0"
@@ -26,7 +30,7 @@ signal-hook = "0.3.10"
 serde = { version = "1.0", features = ["derive"] }
 hex = "0.4.3"
 const_format = "0.2.21"
-tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
+tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 
 postgres_ffi = { path = "../postgres_ffi" }
 workspace_hack = { path = "../workspace_hack" }

walkeeper/src/bin/safekeeper.rs

@@ -32,22 +32,22 @@ fn main() -> Result<()> {
         .about("Store WAL stream to local file system and push it to WAL receivers")
         .version(GIT_VERSION)
         .arg(
-            Arg::new("datadir")
-                .short('D')
+            Arg::with_name("datadir")
+                .short("D")
                 .long("dir")
                 .takes_value(true)
                 .help("Path to the safekeeper data directory"),
         )
         .arg(
-            Arg::new("listen-pg")
-                .short('l')
+            Arg::with_name("listen-pg")
+                .short("l")
                 .long("listen-pg")
                 .alias("listen") // for compatibility
                 .takes_value(true)
                 .help(formatcp!("listen for incoming WAL data connections on ip:port (default: {DEFAULT_PG_LISTEN_ADDR})")),
         )
         .arg(
-            Arg::new("listen-http")
+            Arg::with_name("listen-http")
                 .long("listen-http")
                 .takes_value(true)
                 .help(formatcp!("http endpoint address for metrics on ip:port (default: {DEFAULT_HTTP_LISTEN_ADDR})")),
@@ -56,39 +56,39 @@ fn main() -> Result<()> {
         // However because this argument is in use by console's e2e tests lets keep it for now and remove separately.
         // So currently it is a noop.
         .arg(
-            Arg::new("pageserver")
-                .short('p')
+            Arg::with_name("pageserver")
+                .short("p")
                 .long("pageserver")
                 .takes_value(true),
         )
         .arg(
-            Arg::new("ttl")
+            Arg::with_name("ttl")
                 .long("ttl")
                 .takes_value(true)
                 .help("interval for keeping WAL at safekeeper node, after which them will be uploaded to S3 and removed locally"),
         )
         .arg(
-            Arg::new("recall")
+            Arg::with_name("recall")
                 .long("recall")
                 .takes_value(true)
                 .help("Period for requestion pageserver to call for replication"),
         )
         .arg(
-            Arg::new("daemonize")
-                .short('d')
+            Arg::with_name("daemonize")
+                .short("d")
                 .long("daemonize")
                 .takes_value(false)
                 .help("Run in the background"),
         )
         .arg(
-            Arg::new("no-sync")
-                .short('n')
+            Arg::with_name("no-sync")
+                .short("n")
                 .long("no-sync")
                 .takes_value(false)
                 .help("Do not wait for changes to be written safely to disk"),
         )
         .arg(
-            Arg::new("dump-control-file")
+            Arg::with_name("dump-control-file")
                 .long("dump-control-file")
                 .takes_value(true)
                 .help("Dump control file at path specifed by this argument and exit"),

walkeeper/src/callmemaybe.rs

@@ -33,7 +33,7 @@ async fn request_callback(
 
     tokio::spawn(async move {
         if let Err(e) = connection.await {
-            error!("connection error: {}", e);
+            eprintln!("connection error: {}", e);
         }
     });
 

walkeeper/src/handler.rs

@@ -16,7 +16,7 @@ use zenith_utils::lsn::Lsn;
 use zenith_utils::postgres_backend;
 use zenith_utils::postgres_backend::PostgresBackend;
 use zenith_utils::pq_proto::{BeMessage, FeStartupPacket, RowDescriptor, INT4_OID, TEXT_OID};
-use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 use crate::callmemaybe::CallmeEvent;
 use crate::timeline::CreateControlFile;
@@ -116,11 +116,8 @@ impl postgres_backend::Handler for SafekeeperPostgresHandler {
                         | SafekeeperPostgresCommand::JSONCtrl { .. } => CreateControlFile::True,
                         _ => CreateControlFile::False,
                     };
-                    self.timeline.set(
-                        &self.conf,
-                        ZTenantTimelineId::new(tenantid, timelineid),
-                        create_control_file,
-                    )?;
+                    self.timeline
+                        .set(&self.conf, tenantid, timelineid, create_control_file)?;
                 }
             }
         }

walkeeper/src/http/routes.rs

@@ -1,11 +1,11 @@
 use hyper::{Body, Request, Response, StatusCode};
+use routerify::ext::RequestExt;
+use routerify::RouterBuilder;
 use serde::Serialize;
 use serde::Serializer;
 use std::fmt::Display;
 use std::sync::Arc;
-use zenith_utils::http::{RequestExt, RouterBuilder};
 use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTenantTimelineId;
 
 use crate::safekeeper::Term;
 use crate::safekeeper::TermHistory;
@@ -65,13 +65,16 @@ struct TimelineStatus {
 
 /// Report info about timeline.
 async fn timeline_status_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let zttid = ZTenantTimelineId::new(
-        parse_request_param(&request, "tenant_id")?,
-        parse_request_param(&request, "timeline_id")?,
-    );
+    let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?;
+    let timeline_id: ZTimelineId = parse_request_param(&request, "timeline_id")?;
 
-    let tli = GlobalTimelines::get(get_conf(&request), zttid, CreateControlFile::False)
-        .map_err(ApiError::from_err)?;
+    let tli = GlobalTimelines::get(
+        get_conf(&request),
+        tenant_id,
+        timeline_id,
+        CreateControlFile::False,
+    )
+    .map_err(ApiError::from_err)?;
     let sk_state = tli.get_info();
     let flush_lsn = tli.get_end_of_wal();
 
@@ -82,8 +85,8 @@ async fn timeline_status_handler(request: Request<Body>) -> Result<Response<Body
     };
 
     let status = TimelineStatus {
-        tenant_id: zttid.tenant_id,
-        timeline_id: zttid.timeline_id,
+        tenant_id,
+        timeline_id,
         acceptor_state: acc_state,
         commit_lsn: sk_state.commit_lsn,
         truncate_lsn: sk_state.truncate_lsn,

walkeeper/src/lib.rs

@@ -2,7 +2,7 @@
 use std::path::PathBuf;
 use std::time::Duration;
 
-use zenith_utils::zid::ZTenantTimelineId;
+use zenith_utils::zid::ZTimelineId;
 
 pub mod callmemaybe;
 pub mod handler;
@@ -47,10 +47,8 @@ pub struct SafeKeeperConf {
 }
 
 impl SafeKeeperConf {
-    pub fn timeline_dir(&self, zttid: &ZTenantTimelineId) -> PathBuf {
-        self.workdir
-            .join(zttid.tenant_id.to_string())
-            .join(zttid.timeline_id.to_string())
+    pub fn timeline_dir(&self, timelineid: &ZTimelineId) -> PathBuf {
+        self.workdir.join(timelineid.to_string())
     }
 }
 

walkeeper/src/receive_wal.rs

@@ -5,7 +5,6 @@
 use anyhow::{bail, Context, Result};
 use bytes::Bytes;
 use bytes::BytesMut;
-use tokio::sync::mpsc::UnboundedSender;
 use tracing::*;
 
 use crate::timeline::Timeline;
@@ -19,8 +18,10 @@ use crate::handler::SafekeeperPostgresHandler;
 use crate::timeline::TimelineTools;
 use zenith_utils::postgres_backend::PostgresBackend;
 use zenith_utils::pq_proto::{BeMessage, FeMessage};
+use zenith_utils::zid::ZTenantId;
 
 use crate::callmemaybe::CallmeEvent;
+use crate::callmemaybe::SubscriptionStateKey;
 
 pub struct ReceiveWalConn<'pg> {
     /// Postgres connection
@@ -81,23 +82,50 @@ impl<'pg> ReceiveWalConn<'pg> {
         let mut msg = self
             .read_msg()
             .context("failed to receive proposer greeting")?;
+        let tenant_id: ZTenantId;
         match msg {
             ProposerAcceptorMessage::Greeting(ref greeting) => {
                 info!(
                     "start handshake with wal proposer {} sysid {} timeline {}",
                     self.peer_addr, greeting.system_id, greeting.tli,
                 );
+                tenant_id = greeting.tenant_id;
             }
             _ => bail!("unexpected message {:?} instead of greeting", msg),
         }
 
-        // Register the connection and defer unregister.
-        spg.timeline
-            .get()
-            .on_compute_connect(self.pageserver_connstr.as_ref(), &spg.tx)?;
-        let _guard = ComputeConnectionGuard {
-            timeline: Arc::clone(spg.timeline.get()),
-            callmemaybe_tx: spg.tx.clone(),
+        // Incoming WAL stream resumed, so reset information about the timeline pause.
+        spg.timeline.get().continue_streaming();
+
+        // if requested, ask pageserver to fetch wal from us
+        // as long as this wal_stream is alive, callmemaybe thread
+        // will send requests to pageserver
+        let _guard = match self.pageserver_connstr {
+            Some(ref pageserver_connstr) => {
+                // Need to establish replication channel with page server.
+                // Add far as replication in postgres is initiated by receiver
+                // we should use callmemaybe mechanism.
+                let timeline_id = spg.timeline.get().timeline_id;
+                let subscription_key = SubscriptionStateKey::new(
+                    tenant_id,
+                    timeline_id,
+                    pageserver_connstr.to_owned(),
+                );
+                spg.tx
+                    .send(CallmeEvent::Subscribe(subscription_key))
+                    .unwrap_or_else(|e| {
+                        error!(
+                            "failed to send Subscribe request to callmemaybe thread {}",
+                            e
+                        );
+                    });
+
+                // create a guard to unsubscribe callback, when this wal_stream will exit
+                Some(SendWalHandlerGuard {
+                    timeline: Arc::clone(spg.timeline.get()),
+                })
+            }
+            None => None,
         };
 
         loop {
@@ -114,15 +142,12 @@ impl<'pg> ReceiveWalConn<'pg> {
     }
 }
 
-struct ComputeConnectionGuard {
+struct SendWalHandlerGuard {
     timeline: Arc<Timeline>,
-    callmemaybe_tx: UnboundedSender<CallmeEvent>,
 }
 
-impl Drop for ComputeConnectionGuard {
+impl Drop for SendWalHandlerGuard {
     fn drop(&mut self) {
-        self.timeline
-            .on_compute_disconnect(&self.callmemaybe_tx)
-            .unwrap();
+        self.timeline.stop_streaming();
     }
 }

walkeeper/src/safekeeper.rs

@@ -22,7 +22,6 @@ use zenith_metrics::{
 use zenith_utils::bin_ser::LeSer;
 use zenith_utils::lsn::Lsn;
 use zenith_utils::pq_proto::SystemId;
-use zenith_utils::pq_proto::ZenithFeedback;
 use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 pub const SK_MAGIC: u32 = 0xcafeceefu32;
@@ -279,8 +278,9 @@ pub struct AppendResponse {
     // We report back our awareness about which WAL is committed, as this is
     // a criterion for walproposer --sync mode exit
     pub commit_lsn: Lsn,
+    // Min disk consistent lsn of pageservers (portion of WAL applied and written to the disk by pageservers)
+    pub disk_consistent_lsn: Lsn,
     pub hs_feedback: HotStandbyFeedback,
-    pub zenith_feedback: ZenithFeedback,
 }
 
 impl AppendResponse {
@@ -289,8 +289,8 @@ impl AppendResponse {
             term,
             flush_lsn: Lsn(0),
             commit_lsn: Lsn(0),
+            disk_consistent_lsn: Lsn(0),
             hs_feedback: HotStandbyFeedback::empty(),
-            zenith_feedback: ZenithFeedback::empty(),
         }
     }
 }
@@ -395,11 +395,10 @@ impl AcceptorProposerMessage {
                 buf.put_u64_le(msg.term);
                 buf.put_u64_le(msg.flush_lsn.into());
                 buf.put_u64_le(msg.commit_lsn.into());
+                buf.put_u64_le(msg.disk_consistent_lsn.into());
                 buf.put_i64_le(msg.hs_feedback.ts);
                 buf.put_u64_le(msg.hs_feedback.xmin);
                 buf.put_u64_le(msg.hs_feedback.catalog_xmin);
-
-                msg.zenith_feedback.serialize(buf)?
             }
         }
 
@@ -454,24 +453,24 @@ struct SafeKeeperMetrics {
     write_wal_seconds: Histogram,
 }
 
-struct SafeKeeperMetricsBuilder {
-    ztli: ZTimelineId,
-    flush_lsn: Lsn,
-    commit_lsn: Lsn,
-}
-
-impl SafeKeeperMetricsBuilder {
-    fn build(self) -> SafeKeeperMetrics {
-        let ztli_str = format!("{}", self.ztli);
-        let m = SafeKeeperMetrics {
+impl SafeKeeperMetrics {
+    fn new(ztli: ZTimelineId) -> SafeKeeperMetrics {
+        let ztli_str = format!("{}", ztli);
+        SafeKeeperMetrics {
             flush_lsn: FLUSH_LSN_GAUGE.with_label_values(&[&ztli_str]),
             commit_lsn: COMMIT_LSN_GAUGE.with_label_values(&[&ztli_str]),
             write_wal_bytes: WRITE_WAL_BYTES.with_label_values(&[&ztli_str]),
             write_wal_seconds: WRITE_WAL_SECONDS.with_label_values(&[&ztli_str]),
-        };
-        m.flush_lsn.set(u64::from(self.flush_lsn) as f64);
-        m.commit_lsn.set(u64::from(self.commit_lsn) as f64);
-        m
+        }
+    }
+
+    fn new_noname() -> SafeKeeperMetrics {
+        SafeKeeperMetrics {
+            flush_lsn: FLUSH_LSN_GAUGE.with_label_values(&["n/a"]),
+            commit_lsn: COMMIT_LSN_GAUGE.with_label_values(&["n/a"]),
+            write_wal_bytes: WRITE_WAL_BYTES.with_label_values(&["n/a"]),
+            write_wal_seconds: WRITE_WAL_SECONDS.with_label_values(&["n/a"]),
+        }
     }
 }
 
@@ -496,25 +495,10 @@ where
     ST: Storage,
 {
     // constructor
-    pub fn new(
-        ztli: ZTimelineId,
-        flush_lsn: Lsn,
-        storage: ST,
-        state: SafeKeeperState,
-    ) -> SafeKeeper<ST> {
-        if state.server.timeline_id != ZTimelineId::from([0u8; 16])
-            && ztli != state.server.timeline_id
-        {
-            panic!("Calling SafeKeeper::new with inconsistent ztli ({}) and SafeKeeperState.server.timeline_id ({})", ztli, state.server.timeline_id);
-        }
+    pub fn new(flush_lsn: Lsn, storage: ST, state: SafeKeeperState) -> SafeKeeper<ST> {
         SafeKeeper {
             flush_lsn,
-            metrics: SafeKeeperMetricsBuilder {
-                ztli,
-                flush_lsn,
-                commit_lsn: state.commit_lsn,
-            }
-            .build(),
+            metrics: SafeKeeperMetrics::new_noname(),
             commit_lsn: state.commit_lsn,
             truncate_lsn: state.truncate_lsn,
             storage,
@@ -580,12 +564,7 @@ where
             .persist(&self.s)
             .context("failed to persist shared state")?;
 
-        self.metrics = SafeKeeperMetricsBuilder {
-            ztli: self.s.server.timeline_id,
-            flush_lsn: self.flush_lsn,
-            commit_lsn: self.commit_lsn,
-        }
-        .build();
+        self.metrics = SafeKeeperMetrics::new(self.s.server.timeline_id);
 
         info!(
             "processed greeting from proposer {:?}, sending term {:?}",
@@ -635,9 +614,9 @@ where
             term: self.s.acceptor_state.term,
             flush_lsn: self.flush_lsn,
             commit_lsn: self.s.commit_lsn,
+            disk_consistent_lsn: Lsn(0),
             // will be filled by the upper code to avoid bothering safekeeper
             hs_feedback: HotStandbyFeedback::empty(),
-            zenith_feedback: ZenithFeedback::empty(),
         }
     }
 
@@ -661,7 +640,6 @@ where
         }
         // update our end of WAL pointer
         self.flush_lsn = msg.start_streaming_at;
-        self.metrics.flush_lsn.set(u64::from(self.flush_lsn) as f64);
         // and now adopt term history from proposer
         self.s.acceptor_state.term_history = msg.term_history.clone();
         self.storage.persist(&self.s)?;
@@ -774,7 +752,7 @@ where
         }
 
         let resp = self.append_response();
-        trace!(
+        info!(
             "processed AppendRequest of len {}, end_lsn={:?}, commit_lsn={:?}, truncate_lsn={:?}, resp {:?}",
             msg.wal_data.len(),
             msg.h.end_lsn,
@@ -815,8 +793,7 @@ mod tests {
         let storage = InMemoryStorage {
             persisted_state: SafeKeeperState::new(),
         };
-        let ztli = ZTimelineId::from([0u8; 16]);
-        let mut sk = SafeKeeper::new(ztli, Lsn(0), storage, SafeKeeperState::new());
+        let mut sk = SafeKeeper::new(Lsn(0), storage, SafeKeeperState::new());
 
         // check voting for 1 is ok
         let vote_request = ProposerAcceptorMessage::VoteRequest(VoteRequest { term: 1 });
@@ -831,7 +808,7 @@ mod tests {
         let storage = InMemoryStorage {
             persisted_state: state.clone(),
         };
-        sk = SafeKeeper::new(ztli, Lsn(0), storage, state);
+        sk = SafeKeeper::new(Lsn(0), storage, state);
 
         // and ensure voting second time for 1 is not ok
         vote_resp = sk.process_msg(&vote_request);
@@ -846,8 +823,7 @@ mod tests {
         let storage = InMemoryStorage {
             persisted_state: SafeKeeperState::new(),
         };
-        let ztli = ZTimelineId::from([0u8; 16]);
-        let mut sk = SafeKeeper::new(ztli, Lsn(0), storage, SafeKeeperState::new());
+        let mut sk = SafeKeeper::new(Lsn(0), storage, SafeKeeperState::new());
 
         let mut ar_hdr = AppendRequestHeader {
             term: 1,

walkeeper/src/send_wal.rs

@@ -9,8 +9,6 @@ use postgres_ffi::xlog_utils::{
     get_current_timestamp, TimestampTz, XLogFileName, MAX_SEND_SIZE, PG_TLI,
 };
 
-use crate::callmemaybe::{CallmeEvent, SubscriptionStateKey};
-use bytes::Bytes;
 use serde::{Deserialize, Serialize};
 use std::cmp::min;
 use std::fs::File;
@@ -21,21 +19,20 @@ use std::sync::Arc;
 use std::thread::sleep;
 use std::time::Duration;
 use std::{str, thread};
-use tokio::sync::mpsc::UnboundedSender;
 use tracing::*;
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
 use zenith_utils::postgres_backend::PostgresBackend;
-use zenith_utils::pq_proto::{BeMessage, FeMessage, WalSndKeepAlive, XLogDataBody, ZenithFeedback};
+use zenith_utils::pq_proto::{BeMessage, FeMessage, WalSndKeepAlive, XLogDataBody};
 use zenith_utils::sock_split::ReadStream;
 
+use crate::callmemaybe::{CallmeEvent, SubscriptionStateKey};
+use tokio::sync::mpsc::UnboundedSender;
 use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 // See: https://www.postgresql.org/docs/13/protocol-replication.html
 const HOT_STANDBY_FEEDBACK_TAG_BYTE: u8 = b'h';
 const STANDBY_STATUS_UPDATE_TAG_BYTE: u8 = b'r';
-// zenith extension of replication protocol
-const ZENITH_STATUS_UPDATE_TAG_BYTE: u8 = b'z';
 
 type FullTransactionId = u64;
 
@@ -142,8 +139,8 @@ impl ReplicationConn {
         while let Some(msg) = FeMessage::read(&mut stream_in)? {
             match &msg {
                 FeMessage::CopyData(m) => {
-                    // There's three possible data messages that the client is supposed to send here:
-                    // `HotStandbyFeedback` and `StandbyStatusUpdate` and `ZenithStandbyFeedback`.
+                    // There's two possible data messages that the client is supposed to send here:
+                    // `HotStandbyFeedback` and `StandbyStatusUpdate`.
 
                     match m.first().cloned() {
                         Some(HOT_STANDBY_FEEDBACK_TAG_BYTE) => {
@@ -153,25 +150,11 @@ impl ReplicationConn {
                             timeline.update_replica_state(replica_id, state);
                         }
                         Some(STANDBY_STATUS_UPDATE_TAG_BYTE) => {
-                            let _reply = StandbyReply::des(&m[1..])
+                            let reply = StandbyReply::des(&m[1..])
                                 .context("failed to deserialize StandbyReply")?;
-                            // This must be a regular postgres replica,
-                            // because pageserver doesn't send this type of messages to safekeeper.
-                            // Currently this is not implemented, so this message is ignored.
-
-                            warn!("unexpected StandbyReply. Read-only postgres replicas are not supported in safekeepers yet.");
-                            // timeline.update_replica_state(replica_id, Some(state));
-                        }
-                        Some(ZENITH_STATUS_UPDATE_TAG_BYTE) => {
-                            // Note: deserializing is on m[9..] because we skip the tag byte and len bytes.
-                            let buf = Bytes::copy_from_slice(&m[9..]);
-                            let reply = ZenithFeedback::parse(buf);
-
-                            trace!("ZenithFeedback is {:?}", reply);
-                            // Only pageserver sends ZenithFeedback, so set the flag.
-                            // This replica is the source of information to resend to compute.
-                            state.zenith_feedback = Some(reply);
-
+                            state.last_received_lsn = reply.write_lsn;
+                            state.disk_consistent_lsn = reply.flush_lsn;
+                            state.remote_consistent_lsn = reply.apply_lsn;
                             timeline.update_replica_state(replica_id, state);
                         }
                         _ => warn!("unexpected message {:?}", msg),
@@ -283,14 +266,12 @@ impl ReplicationConn {
             if spg.appname == Some("wal_proposer_recovery".to_string()) {
                 None
             } else {
-                let pageserver_connstr = pageserver_connstr.expect("there should be a pageserver connection string since this is not a wal_proposer_recovery");
-                let zttid = spg.timeline.get().zttid;
+                let pageserver_connstr = pageserver_connstr.clone().expect("there should be a pageserver connection string since this is not a wal_proposer_recovery");
+                let timeline_id = spg.timeline.get().timeline_id;
+                let tenant_id = spg.ztenantid.unwrap();
                 let tx_clone = spg.tx.clone();
-                let subscription_key = SubscriptionStateKey::new(
-                    zttid.tenant_id,
-                    zttid.timeline_id,
-                    pageserver_connstr.clone(),
-                );
+                let subscription_key =
+                    SubscriptionStateKey::new(tenant_id, timeline_id, pageserver_connstr.clone());
                 spg.tx
                     .send(CallmeEvent::Pause(subscription_key))
                     .unwrap_or_else(|e| {
@@ -300,8 +281,8 @@ impl ReplicationConn {
                 // create a guard to subscribe callback again, when this connection will exit
                 Some(ReplicationStreamGuard {
                     tx: tx_clone,
-                    tenant_id: zttid.tenant_id,
-                    timeline_id: zttid.timeline_id,
+                    tenant_id,
+                    timeline_id,
                     pageserver_connstr,
                 })
             }
@@ -326,10 +307,21 @@ impl ReplicationConn {
                 if let Some(lsn) = lsn {
                     end_pos = lsn;
                 } else {
-                    // TODO: also check once in a while whether we are walsender
-                    // to right pageserver.
-                    if spg.timeline.get().check_deactivate(replica_id, &spg.tx)? {
-                        // Shut down, timeline is suspended.
+                    // Is it time to end streaming to this replica?
+                    if spg.timeline.get().check_stop_streaming(replica_id) {
+                        // this expect should never fail because in wal_proposer_recovery mode stop_pos is set
+                        // and this code is not reachable
+                        let pageserver_connstr = pageserver_connstr
+                            .expect("there should be a pageserver connection string");
+                        let timelineid = spg.timeline.get().timeline_id;
+                        let tenant_id = spg.ztenantid.unwrap();
+                        let subscription_key =
+                            SubscriptionStateKey::new(tenant_id, timelineid, pageserver_connstr);
+                        spg.tx
+                            .send(CallmeEvent::Unsubscribe(subscription_key))
+                            .unwrap_or_else(|e| {
+                                error!("failed to send Pause request to callmemaybe thread {}", e);
+                            });
                         // TODO create proper error type for this
                         bail!("end streaming to {:?}", spg.appname);
                     }
@@ -352,10 +344,8 @@ impl ReplicationConn {
                     // Open a new file.
                     let segno = start_pos.segment_number(wal_seg_size);
                     let wal_file_name = XLogFileName(PG_TLI, segno, wal_seg_size);
-                    let wal_file_path = spg
-                        .conf
-                        .timeline_dir(&spg.timeline.get().zttid)
-                        .join(wal_file_name);
+                    let timeline_id = spg.timeline.get().timeline_id;
+                    let wal_file_path = spg.conf.timeline_dir(&timeline_id).join(wal_file_name);
                     Self::open_wal_file(&wal_file_path)?
                 }
             };
@@ -385,7 +375,7 @@ impl ReplicationConn {
 
             start_pos += send_size as u64;
 
-            trace!("sent WAL up to {}", start_pos);
+            info!("sent WAL up to {}", start_pos);
 
             // Decide whether to reuse this file. If we don't set wal_file here
             // a new file will be opened next time.

walkeeper/src/timeline.rs

@@ -4,7 +4,7 @@
 use anyhow::{bail, ensure, Context, Result};
 use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
 use lazy_static::lazy_static;
-use postgres_ffi::xlog_utils::{find_end_of_wal, XLogSegNo, PG_TLI};
+use postgres_ffi::xlog_utils::{find_end_of_wal, PG_TLI};
 use std::cmp::{max, min};
 use std::collections::HashMap;
 use std::fs::{self, File, OpenOptions};
@@ -12,14 +12,12 @@ use std::io::{Read, Seek, SeekFrom, Write};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Condvar, Mutex};
 use std::time::Duration;
-use tokio::sync::mpsc::UnboundedSender;
 use tracing::*;
 use zenith_metrics::{register_histogram_vec, Histogram, HistogramVec, DISK_WRITE_SECONDS_BUCKETS};
 use zenith_utils::bin_ser::LeSer;
 use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZTenantTimelineId;
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
-use crate::callmemaybe::{CallmeEvent, SubscriptionStateKey};
 use crate::safekeeper::{
     AcceptorProposerMessage, ProposerAcceptorMessage, SafeKeeper, SafeKeeperState, ServerInfo,
     Storage, SK_FORMAT_VERSION, SK_MAGIC,
@@ -29,7 +27,6 @@ use crate::upgrade::upgrade_control_file;
 use crate::SafeKeeperConf;
 use postgres_ffi::xlog_utils::{XLogFileName, XLOG_BLCKSZ};
 use std::convert::TryInto;
-use zenith_utils::pq_proto::ZenithFeedback;
 
 // contains persistent metadata for safekeeper
 const CONTROL_FILE_NAME: &str = "safekeeper.control";
@@ -38,17 +35,17 @@ const CONTROL_FILE_NAME_PARTIAL: &str = "safekeeper.control.partial";
 const POLL_STATE_TIMEOUT: Duration = Duration::from_secs(1);
 pub const CHECKSUM_SIZE: usize = std::mem::size_of::<u32>();
 
-/// Replica status update + hot standby feedback
+/// Replica status: host standby feedback + disk consistent lsn
 #[derive(Debug, Clone, Copy)]
 pub struct ReplicaState {
     /// last known lsn received by replica
     pub last_received_lsn: Lsn, // None means we don't know
+    /// combined disk_consistent_lsn of pageservers
+    pub disk_consistent_lsn: Lsn,
     /// combined remote consistent lsn of pageservers
     pub remote_consistent_lsn: Lsn,
     /// combined hot standby feedback from all replicas
     pub hs_feedback: HotStandbyFeedback,
-    /// Zenith specific feedback received from pageserver, if any
-    pub zenith_feedback: Option<ZenithFeedback>,
 }
 
 impl Default for ReplicaState {
@@ -61,36 +58,29 @@ impl ReplicaState {
     pub fn new() -> ReplicaState {
         ReplicaState {
             last_received_lsn: Lsn::MAX,
-            remote_consistent_lsn: Lsn(0),
+            disk_consistent_lsn: Lsn(u64::MAX),
+            remote_consistent_lsn: Lsn(u64::MAX),
             hs_feedback: HotStandbyFeedback {
                 ts: 0,
                 xmin: u64::MAX,
                 catalog_xmin: u64::MAX,
             },
-            zenith_feedback: None,
         }
     }
 }
 
-/// Shared state associated with database instance
+/// Shared state associated with database instance (tenant)
 struct SharedState {
     /// Safekeeper object
     sk: SafeKeeper<FileStorage>,
     /// For receiving-sending wal cooperation
     /// quorum commit LSN we've notified walsenders about
     notified_commit_lsn: Lsn,
+    // Set stop_lsn to inform WAL senders that it's time to stop sending WAL,
+    // so that it send all wal up stop_lsn and can safely exit streaming connections.
+    stop_lsn: Option<Lsn>,
     /// State of replicas
     replicas: Vec<Option<ReplicaState>>,
-    /// Inactive clusters shouldn't occupy any resources, so timeline is
-    /// activated whenever there is a compute connection or pageserver is not
-    /// caughtup (it must have latest WAL for new compute start) and suspended
-    /// otherwise.
-    ///
-    /// TODO: it might be better to remove tli completely from GlobalTimelines
-    /// when tli is inactive instead of having this flag.
-    active: bool,
-    num_computes: u32,
-    pageserver_connstr: Option<String>,
 }
 
 // A named boolean.
@@ -111,125 +101,6 @@ lazy_static! {
 }
 
 impl SharedState {
-    /// Restore SharedState from control file.
-    /// If create=false and file doesn't exist, bails out.
-    fn create_restore(
-        conf: &SafeKeeperConf,
-        zttid: &ZTenantTimelineId,
-        create: CreateControlFile,
-    ) -> Result<Self> {
-        let state = FileStorage::load_control_file_conf(conf, zttid, create)
-            .context("failed to load from control file")?;
-        let file_storage = FileStorage::new(zttid, conf);
-        let flush_lsn = if state.server.wal_seg_size != 0 {
-            let wal_dir = conf.timeline_dir(zttid);
-            Lsn(find_end_of_wal(
-                &wal_dir,
-                state.server.wal_seg_size as usize,
-                true,
-                state.wal_start_lsn,
-            )?
-            .0)
-        } else {
-            Lsn(0)
-        };
-        info!(
-            "timeline {} created or restored: flush_lsn={}, commit_lsn={}, truncate_lsn={}",
-            zttid.timeline_id, flush_lsn, state.commit_lsn, state.truncate_lsn,
-        );
-        if flush_lsn < state.commit_lsn || flush_lsn < state.truncate_lsn {
-            warn!("timeline {} potential data loss: flush_lsn by find_end_of_wal is less than either commit_lsn or truncate_lsn from control file", zttid.timeline_id);
-        }
-
-        Ok(Self {
-            notified_commit_lsn: Lsn(0),
-            sk: SafeKeeper::new(zttid.timeline_id, flush_lsn, file_storage, state),
-            replicas: Vec::new(),
-            active: false,
-            num_computes: 0,
-            pageserver_connstr: None,
-        })
-    }
-
-    /// Activate the timeline: start/change walsender (via callmemaybe).
-    fn activate(
-        &mut self,
-        zttid: &ZTenantTimelineId,
-        pageserver_connstr: Option<&String>,
-        callmemaybe_tx: &UnboundedSender<CallmeEvent>,
-    ) -> Result<()> {
-        if let Some(ref pageserver_connstr) = self.pageserver_connstr {
-            // unsub old sub. xxx: callmemaybe is going out
-            let old_subscription_key = SubscriptionStateKey::new(
-                zttid.tenant_id,
-                zttid.timeline_id,
-                pageserver_connstr.to_owned(),
-            );
-            callmemaybe_tx
-                .send(CallmeEvent::Unsubscribe(old_subscription_key))
-                .unwrap_or_else(|e| {
-                    error!("failed to send Pause request to callmemaybe thread {}", e);
-                });
-        }
-        if let Some(pageserver_connstr) = pageserver_connstr {
-            let subscription_key = SubscriptionStateKey::new(
-                zttid.tenant_id,
-                zttid.timeline_id,
-                pageserver_connstr.to_owned(),
-            );
-            // xx: sending to channel under lock is not very cool, but
-            // shouldn't be a problem here. If it is, we can grab a counter
-            // here and later augment channel messages with it.
-            callmemaybe_tx
-                .send(CallmeEvent::Subscribe(subscription_key))
-                .unwrap_or_else(|e| {
-                    error!(
-                        "failed to send Subscribe request to callmemaybe thread {}",
-                        e
-                    );
-                });
-            info!(
-                "timeline {} is subscribed to callmemaybe to {}",
-                zttid.timeline_id, pageserver_connstr
-            );
-        }
-        self.pageserver_connstr = pageserver_connstr.map(|c| c.to_owned());
-        self.active = true;
-        Ok(())
-    }
-
-    /// Deactivate the timeline: stop callmemaybe.
-    fn deactivate(
-        &mut self,
-        zttid: &ZTenantTimelineId,
-        callmemaybe_tx: &UnboundedSender<CallmeEvent>,
-    ) -> Result<()> {
-        if self.active {
-            if let Some(ref pageserver_connstr) = self.pageserver_connstr {
-                let subscription_key = SubscriptionStateKey::new(
-                    zttid.tenant_id,
-                    zttid.timeline_id,
-                    pageserver_connstr.to_owned(),
-                );
-                callmemaybe_tx
-                    .send(CallmeEvent::Unsubscribe(subscription_key))
-                    .unwrap_or_else(|e| {
-                        error!(
-                            "failed to send Unsubscribe request to callmemaybe thread {}",
-                            e
-                        );
-                    });
-                info!(
-                    "timeline {} is unsubscribed from callmemaybe to {}",
-                    zttid.timeline_id,
-                    self.pageserver_connstr.as_ref().unwrap()
-                );
-            }
-            self.active = false;
-        }
-        Ok(())
-    }
-
     /// Get combined state of all alive replicas
     pub fn get_replicas_state(&self) -> ReplicaState {
         let mut acc = ReplicaState::new();
@@ -238,40 +109,13 @@ impl SharedState {
             acc.hs_feedback.xmin = min(acc.hs_feedback.xmin, state.hs_feedback.xmin);
             acc.hs_feedback.catalog_xmin =
                 min(acc.hs_feedback.catalog_xmin, state.hs_feedback.catalog_xmin);
-
-            // FIXME
-            // If multiple pageservers are streaming WAL and send feedback for the same timeline simultaneously,
-            // this code is not correct.
-            // Now the most advanced feedback is used.
-            // If one pageserver lags when another doesn't, the backpressure won't be activated on compute and lagging
-            // pageserver is prone to timeout errors.
-            //
-            // To choose what feedback to use and resend to compute node,
-            // we need to know which pageserver compute node considers to be main.
-            // See https://github.com/zenithdb/zenith/issues/1171
-            //
-            if let Some(zenith_feedback) = state.zenith_feedback {
-                if let Some(acc_feedback) = acc.zenith_feedback {
-                    if acc_feedback.ps_writelsn < zenith_feedback.ps_writelsn {
-                        warn!("More than one pageserver is streaming WAL for the timeline. Feedback resolving is not fully supported yet.");
-                        acc.zenith_feedback = Some(zenith_feedback);
-                    }
-                } else {
-                    acc.zenith_feedback = Some(zenith_feedback);
-                }
-
-                // last lsn received by pageserver
-                // FIXME if multiple pageservers are streaming WAL, last_received_lsn must be tracked per pageserver.
-                // See https://github.com/zenithdb/zenith/issues/1171
-                acc.last_received_lsn = Lsn::from(zenith_feedback.ps_writelsn);
-
-                // When at least one pageserver has preserved data up to remote_consistent_lsn,
-                // safekeeper is free to delete it, so choose max of all pageservers.
-                acc.remote_consistent_lsn = max(
-                    Lsn::from(zenith_feedback.ps_applylsn),
-                    acc.remote_consistent_lsn,
-                );
-            }
+            acc.disk_consistent_lsn = Lsn::min(acc.disk_consistent_lsn, state.disk_consistent_lsn);
+            // currently not used, but update it to be consistent
+            acc.last_received_lsn = Lsn::min(acc.last_received_lsn, state.last_received_lsn);
+            // When at least one replica has preserved data up to remote_consistent_lsn,
+            // safekeeper is free to delete it, so chose max of all replicas.
+            acc.remote_consistent_lsn =
+                Lsn::max(acc.remote_consistent_lsn, state.remote_consistent_lsn);
         }
         acc
     }
@@ -287,86 +131,56 @@ impl SharedState {
         self.replicas.push(Some(state));
         pos
     }
+
+    /// Restore SharedState from control file.
+    /// If create=false and file doesn't exist, bails out.
+    fn create_restore(
+        conf: &SafeKeeperConf,
+        timeline_id: ZTimelineId,
+        create: CreateControlFile,
+    ) -> Result<Self> {
+        let state = FileStorage::load_control_file_conf(conf, timeline_id, create)
+            .context("failed to load from control file")?;
+        let file_storage = FileStorage::new(timeline_id, conf);
+        let flush_lsn = if state.server.wal_seg_size != 0 {
+            let wal_dir = conf.timeline_dir(&timeline_id);
+            find_end_of_wal(
+                &wal_dir,
+                state.server.wal_seg_size as usize,
+                true,
+                state.wal_start_lsn,
+            )?
+            .0
+        } else {
+            0
+        };
+
+        Ok(Self {
+            notified_commit_lsn: Lsn(0),
+            stop_lsn: None,
+            sk: SafeKeeper::new(Lsn(flush_lsn), file_storage, state),
+            replicas: Vec::new(),
+        })
+    }
 }
 
 /// Database instance (tenant)
 pub struct Timeline {
-    pub zttid: ZTenantTimelineId,
+    pub timeline_id: ZTimelineId,
     mutex: Mutex<SharedState>,
     /// conditional variable used to notify wal senders
     cond: Condvar,
 }
 
 impl Timeline {
-    fn new(zttid: ZTenantTimelineId, shared_state: SharedState) -> Timeline {
+    fn new(timelineid: ZTimelineId, shared_state: SharedState) -> Timeline {
         Timeline {
-            zttid,
+            timeline_id: timelineid,
             mutex: Mutex::new(shared_state),
             cond: Condvar::new(),
         }
     }
 
-    /// Register compute connection, starting timeline-related activity if it is
-    /// not running yet.
-    /// Can fail only if channel to a static thread got closed, which is not normal at all.
-    pub fn on_compute_connect(
-        &self,
-        pageserver_connstr: Option<&String>,
-        callmemaybe_tx: &UnboundedSender<CallmeEvent>,
-    ) -> Result<()> {
-        let mut shared_state = self.mutex.lock().unwrap();
-        shared_state.num_computes += 1;
-        // FIXME: currently we always adopt latest pageserver connstr, but we
-        // should have kind of generations assigned by compute to distinguish
-        // the latest one or even pass it through consensus to reliably deliver
-        // to all safekeepers.
-        shared_state.activate(&self.zttid, pageserver_connstr, callmemaybe_tx)?;
-        Ok(())
-    }
-
-    /// De-register compute connection, shutting down timeline activity if
-    /// pageserver doesn't need catchup.
-    /// Can fail only if channel to a static thread got closed, which is not normal at all.
-    pub fn on_compute_disconnect(
-        &self,
-        callmemaybe_tx: &UnboundedSender<CallmeEvent>,
-    ) -> Result<()> {
-        let mut shared_state = self.mutex.lock().unwrap();
-        shared_state.num_computes -= 1;
-        // If there is no pageserver, can suspend right away; otherwise let
-        // walsender do that.
-        if shared_state.num_computes == 0 && shared_state.pageserver_connstr.is_none() {
-            shared_state.deactivate(&self.zttid, callmemaybe_tx)?;
-        }
-        Ok(())
-    }
-
-    /// Deactivate tenant if there is no computes and pageserver is caughtup,
-    /// assuming the pageserver status is in replica_id.
-    /// Returns true if deactivated.
-    pub fn check_deactivate(
-        &self,
-        replica_id: usize,
-        callmemaybe_tx: &UnboundedSender<CallmeEvent>,
-    ) -> Result<bool> {
-        let mut shared_state = self.mutex.lock().unwrap();
-        if !shared_state.active {
-            // already suspended
-            return Ok(true);
-        }
-        if shared_state.num_computes == 0 {
-            let replica_state = shared_state.replicas[replica_id].unwrap();
-            let deactivate = shared_state.notified_commit_lsn == Lsn(0) || // no data at all yet
-            (replica_state.last_received_lsn != Lsn::MAX && // Lsn::MAX means that we don't know the latest LSN yet.
-             replica_state.last_received_lsn >= shared_state.sk.commit_lsn);
-            if deactivate {
-                shared_state.deactivate(&self.zttid, callmemaybe_tx)?;
-                return Ok(true);
-            }
-        }
-        Ok(false)
-    }
-
     /// Timed wait for an LSN to be committed.
     ///
     /// Returns the last committed LSN, which will be at least
@@ -400,6 +214,54 @@ impl Timeline {
         }
     }
 
+    // Notify WAL senders that it's time to stop sending WAL
+    pub fn stop_streaming(&self) {
+        let mut shared_state = self.mutex.lock().unwrap();
+        // Ensure that safekeeper sends WAL up to the last known committed LSN.
+        // It guarantees that pageserver will receive all the latest data
+        // before walservice disconnects.
+        shared_state.stop_lsn = Some(shared_state.notified_commit_lsn);
+        trace!(
+            "Stopping WAL senders. stop_lsn: {}",
+            shared_state.notified_commit_lsn
+        );
+    }
+
+    // Reset stop_lsn notification,
+    // so that WAL senders will continue sending WAL
+    pub fn continue_streaming(&self) {
+        let mut shared_state = self.mutex.lock().unwrap();
+        shared_state.stop_lsn = None;
+    }
+
+    // Check if it's time to stop streaming to the given replica.
+    //
+    // Do not stop streaming until replica is caught up with the stop_lsn.
+    // This is not necessary for correctness, just an optimization to
+    // be able to remove WAL from safekeeper and decrease amount of work
+    // on the next start.
+    pub fn check_stop_streaming(&self, replica_id: usize) -> bool {
+        let shared_state = self.mutex.lock().unwrap();
+
+        // If stop_lsn is set, it's time to shutdown streaming.
+        if let Some(stop_lsn_request) = shared_state.stop_lsn {
+            let replica_state = shared_state.replicas[replica_id].unwrap();
+            // There is no data to stream, so other clauses don't matter.
+            if shared_state.notified_commit_lsn == Lsn(0) {
+                return true;
+            }
+            // Lsn::MAX means that we don't know the latest LSN yet.
+            // That may be a new replica, give it a chance to catch up.
+            if replica_state.last_received_lsn != Lsn::MAX
+            // If replica is fully caught up, disconnect it.
+            && stop_lsn_request <= replica_state.last_received_lsn
+            {
+                return true;
+            }
+        }
+        false
+    }
+
     /// Pass arrived message to the safekeeper.
     pub fn process_msg(
         &self,
@@ -418,9 +280,8 @@ impl Timeline {
             if let Some(AcceptorProposerMessage::AppendResponse(ref mut resp)) = rmsg {
                 let state = shared_state.get_replicas_state();
                 resp.hs_feedback = state.hs_feedback;
-                if let Some(zenith_feedback) = state.zenith_feedback {
-                    resp.zenith_feedback = zenith_feedback;
-                }
+                resp.disk_consistent_lsn = state.disk_consistent_lsn;
+                // XXX Do we need to add state.last_received_lsn to resp?
             }
         }
         // Ping wal sender that new data might be available.
@@ -459,7 +320,8 @@ pub trait TimelineTools {
     fn set(
         &mut self,
         conf: &SafeKeeperConf,
-        zttid: ZTenantTimelineId,
+        tenant_id: ZTenantId,
+        timeline_id: ZTimelineId,
         create: CreateControlFile,
     ) -> Result<()>;
 
@@ -470,13 +332,14 @@ impl TimelineTools for Option<Arc<Timeline>> {
     fn set(
         &mut self,
         conf: &SafeKeeperConf,
-        zttid: ZTenantTimelineId,
+        tenant_id: ZTenantId,
+        timeline_id: ZTimelineId,
         create: CreateControlFile,
     ) -> Result<()> {
         // We will only set the timeline once. If it were to ever change,
         // anyone who cloned the Arc would be out of date.
         assert!(self.is_none());
-        *self = Some(GlobalTimelines::get(conf, zttid, create)?);
+        *self = Some(GlobalTimelines::get(conf, tenant_id, timeline_id, create)?);
         Ok(())
     }
 
@@ -486,7 +349,7 @@ impl TimelineTools for Option<Arc<Timeline>> {
 }
 
 lazy_static! {
-    pub static ref TIMELINES: Mutex<HashMap<ZTenantTimelineId, Arc<Timeline>>> =
+    pub static ref TIMELINES: Mutex<HashMap<(ZTenantId, ZTimelineId), Arc<Timeline>>> =
         Mutex::new(HashMap::new());
 }
 
@@ -498,29 +361,26 @@ impl GlobalTimelines {
     /// If control file doesn't exist and create=false, bails out.
     pub fn get(
         conf: &SafeKeeperConf,
-        zttid: ZTenantTimelineId,
+        tenant_id: ZTenantId,
+        timeline_id: ZTimelineId,
         create: CreateControlFile,
     ) -> Result<Arc<Timeline>> {
         let mut timelines = TIMELINES.lock().unwrap();
 
-        match timelines.get(&zttid) {
+        match timelines.get(&(tenant_id, timeline_id)) {
             Some(result) => Ok(Arc::clone(result)),
             None => {
-                if let CreateControlFile::True = create {
-                    let dir = conf.timeline_dir(&zttid);
-                    info!(
-                        "creating timeline dir {}, create is {:?}",
-                        dir.display(),
-                        create
-                    );
-                    fs::create_dir_all(dir)?;
-                }
+                info!(
+                    "creating timeline dir {}, create is {:?}",
+                    timeline_id, create
+                );
+                fs::create_dir_all(timeline_id.to_string())?;
 
-                let shared_state = SharedState::create_restore(conf, &zttid, create)
+                let shared_state = SharedState::create_restore(conf, timeline_id, create)
                     .context("failed to restore shared state")?;
 
-                let new_tli = Arc::new(Timeline::new(zttid, shared_state));
-                timelines.insert(zttid, Arc::clone(&new_tli));
+                let new_tli = Arc::new(Timeline::new(timeline_id, shared_state));
+                timelines.insert((tenant_id, timeline_id), Arc::clone(&new_tli));
                 Ok(new_tli)
             }
         }
@@ -536,9 +396,9 @@ pub struct FileStorage {
 }
 
 impl FileStorage {
-    fn new(zttid: &ZTenantTimelineId, conf: &SafeKeeperConf) -> FileStorage {
-        let timeline_dir = conf.timeline_dir(zttid);
-        let timelineid_str = format!("{}", zttid);
+    fn new(timeline_id: ZTimelineId, conf: &SafeKeeperConf) -> FileStorage {
+        let timeline_dir = conf.timeline_dir(&timeline_id);
+        let timelineid_str = format!("{}", timeline_id);
         FileStorage {
             timeline_dir,
             conf: conf.clone(),
@@ -567,13 +427,12 @@ impl FileStorage {
         upgrade_control_file(buf, version)
     }
 
-    // Load control file for given zttid at path specified by conf.
     fn load_control_file_conf(
         conf: &SafeKeeperConf,
-        zttid: &ZTenantTimelineId,
+        timeline_id: ZTimelineId,
         create: CreateControlFile,
     ) -> Result<SafeKeeperState> {
-        let path = conf.timeline_dir(zttid).join(CONTROL_FILE_NAME);
+        let path = conf.timeline_dir(&timeline_id).join(CONTROL_FILE_NAME);
         Self::load_control_file(path, create)
     }
 
@@ -638,14 +497,6 @@ impl FileStorage {
         };
         Ok(state)
     }
-
-    /// Helper returning full path to WAL segment file and its .partial brother.
-    fn wal_file_paths(&self, segno: XLogSegNo, wal_seg_size: usize) -> (PathBuf, PathBuf) {
-        let wal_file_name = XLogFileName(PG_TLI, segno, wal_seg_size);
-        let wal_file_path = self.timeline_dir.join(wal_file_name.clone());
-        let wal_file_partial_path = self.timeline_dir.join(wal_file_name + ".partial");
-        (wal_file_path, wal_file_partial_path)
-    }
 }
 
 impl Storage for FileStorage {
@@ -714,6 +565,7 @@ impl Storage for FileStorage {
         let mut start_pos = startpos;
         const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
         let wal_seg_size = server.wal_seg_size as usize;
+        let ztli = server.timeline_id;
 
         /* Extract WAL location for this block */
         let mut xlogoff = start_pos.segment_offset(wal_seg_size) as usize;
@@ -733,7 +585,14 @@ impl Storage for FileStorage {
 
             /* Open file */
             let segno = start_pos.segment_number(wal_seg_size);
-            let (wal_file_path, wal_file_partial_path) = self.wal_file_paths(segno, wal_seg_size);
+            // note: we basically don't support changing pg timeline
+            let wal_file_name = XLogFileName(PG_TLI, segno, wal_seg_size);
+            let wal_file_path = self.conf.timeline_dir(&ztli).join(wal_file_name.clone());
+            let wal_file_partial_path = self
+                .conf
+                .timeline_dir(&ztli)
+                .join(wal_file_name.clone() + ".partial");
+
             {
                 let mut wal_file: File;
                 /* Try to open already completed segment */
@@ -794,13 +653,25 @@ impl Storage for FileStorage {
         let partial;
         const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
         let wal_seg_size = server.wal_seg_size as usize;
+        let ztli = server.timeline_id;
 
         /* Extract WAL location for this block */
         let mut xlogoff = end_pos.segment_offset(wal_seg_size) as usize;
 
         /* Open file */
         let mut segno = end_pos.segment_number(wal_seg_size);
-        let (wal_file_path, wal_file_partial_path) = self.wal_file_paths(segno, wal_seg_size);
+        // note: we basically don't support changing pg timeline
+        let wal_file_name = XLogFileName(PG_TLI, segno, wal_seg_size);
+        let wal_file_path = self
+            .conf
+            .workdir
+            .join(ztli.to_string())
+            .join(wal_file_name.clone());
+        let wal_file_partial_path = self
+            .conf
+            .workdir
+            .join(ztli.to_string())
+            .join(wal_file_name + ".partial");
         {
             let mut wal_file: File;
             /* Try to open already completed segment */
@@ -831,7 +702,17 @@ impl Storage for FileStorage {
         // Remove all subsequent segments
         loop {
             segno += 1;
-            let (wal_file_path, wal_file_partial_path) = self.wal_file_paths(segno, wal_seg_size);
+            let wal_file_name = XLogFileName(PG_TLI, segno, wal_seg_size);
+            let wal_file_path = self
+                .conf
+                .workdir
+                .join(ztli.to_string())
+                .join(wal_file_name.clone());
+            let wal_file_partial_path = self
+                .conf
+                .workdir
+                .join(ztli.to_string())
+                .join(wal_file_name.clone() + ".partial");
             // TODO: better use fs::try_exists which is currenty avaialble only in nightly build
             if wal_file_path.exists() {
                 fs::remove_file(&wal_file_path)?;
@@ -851,11 +732,11 @@ mod test {
     use crate::{
         safekeeper::{SafeKeeperState, Storage},
         timeline::{CreateControlFile, CONTROL_FILE_NAME},
-        SafeKeeperConf, ZTenantTimelineId,
+        SafeKeeperConf,
     };
     use anyhow::Result;
     use std::fs;
-    use zenith_utils::lsn::Lsn;
+    use zenith_utils::{lsn::Lsn, zid::ZTimelineId};
 
     fn stub_conf() -> SafeKeeperConf {
         let workdir = tempfile::tempdir().unwrap().into_path();
@@ -867,30 +748,31 @@ mod test {
 
     fn load_from_control_file(
         conf: &SafeKeeperConf,
-        zttid: &ZTenantTimelineId,
+        timeline_id: ZTimelineId,
         create: CreateControlFile,
     ) -> Result<(FileStorage, SafeKeeperState)> {
-        fs::create_dir_all(&conf.timeline_dir(zttid)).expect("failed to create timeline dir");
+        fs::create_dir_all(&conf.timeline_dir(&timeline_id))
+            .expect("failed to create timeline dir");
         Ok((
-            FileStorage::new(zttid, conf),
-            FileStorage::load_control_file_conf(conf, zttid, create)?,
+            FileStorage::new(timeline_id, conf),
+            FileStorage::load_control_file_conf(conf, timeline_id, create)?,
         ))
     }
 
     #[test]
     fn test_read_write_safekeeper_state() {
         let conf = stub_conf();
-        let zttid = ZTenantTimelineId::generate();
+        let timeline_id = ZTimelineId::generate();
         {
             let (mut storage, mut state) =
-                load_from_control_file(&conf, &zttid, CreateControlFile::True)
+                load_from_control_file(&conf, timeline_id, CreateControlFile::True)
                     .expect("failed to read state");
             // change something
             state.wal_start_lsn = Lsn(42);
             storage.persist(&state).expect("failed to persist state");
         }
 
-        let (_, state) = load_from_control_file(&conf, &zttid, CreateControlFile::False)
+        let (_, state) = load_from_control_file(&conf, timeline_id, CreateControlFile::False)
             .expect("failed to read state");
         assert_eq!(state.wal_start_lsn, Lsn(42));
     }
@@ -898,21 +780,21 @@ mod test {
     #[test]
     fn test_safekeeper_state_checksum_mismatch() {
         let conf = stub_conf();
-        let zttid = ZTenantTimelineId::generate();
+        let timeline_id = ZTimelineId::generate();
         {
             let (mut storage, mut state) =
-                load_from_control_file(&conf, &zttid, CreateControlFile::True)
+                load_from_control_file(&conf, timeline_id, CreateControlFile::True)
                     .expect("failed to read state");
             // change something
             state.wal_start_lsn = Lsn(42);
             storage.persist(&state).expect("failed to persist state");
         }
-        let control_path = conf.timeline_dir(&zttid).join(CONTROL_FILE_NAME);
+        let control_path = conf.timeline_dir(&timeline_id).join(CONTROL_FILE_NAME);
         let mut data = fs::read(&control_path).unwrap();
         data[0] += 1; // change the first byte of the file to fail checksum validation
         fs::write(&control_path, &data).expect("failed to write control file");
 
-        match load_from_control_file(&conf, &zttid, CreateControlFile::False) {
+        match load_from_control_file(&conf, timeline_id, CreateControlFile::False) {
             Err(err) => assert!(err
                 .to_string()
                 .contains("safekeeper control file checksum mismatch")),

workspace_hack/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "workspace_hack"
 version = "0.1.0"
-edition = "2021"
+edition = "2018"
 
 [target.'cfg(all())'.dependencies]
 libc = { version = "0.2", features = ["default", "extra_traits", "std"] }

zenith/Cargo.toml

@@ -1,13 +1,16 @@
 [package]
 name = "zenith"
 version = "0.1.0"
-edition = "2021"
+authors = ["Stas Kelvich <stas@zenith.tech>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-clap = "3.0"
+clap = "2.33.0"
 anyhow = "1.0"
 serde_json = "1"
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 
 # FIXME: 'pageserver' is needed for BranchInfo. Refactor
 pageserver = { path = "../pageserver" }

zenith/src/main.rs

@@ -1,5 +1,5 @@
 use anyhow::{bail, Context, Result};
-use clap::{App, AppSettings, Arg, ArgMatches};
+use clap::{App, AppSettings, Arg, ArgMatches, SubCommand};
 use control_plane::compute::ComputeControlPlane;
 use control_plane::local_env;
 use control_plane::local_env::LocalEnv;
@@ -67,136 +67,141 @@ struct BranchTreeEl {
 //   * Providing CLI api to the pageserver
 //   * TODO: export/import to/from usual postgres
 fn main() -> Result<()> {
-    #[rustfmt::skip] // rustfmt squashes these into a single line otherwise
-    let pg_node_arg = Arg::new("node")
+    let pg_node_arg = Arg::with_name("node")
         .index(1)
         .help("Node name")
         .required(true);
 
-    #[rustfmt::skip]
-    let safekeeper_node_arg = Arg::new("node")
+    let safekeeper_node_arg = Arg::with_name("node")
         .index(1)
         .help("Node name")
         .required(false);
 
-    let timeline_arg = Arg::new("timeline")
+    let timeline_arg = Arg::with_name("timeline")
         .index(2)
         .help("Branch name or a point-in time specification")
         .required(false);
 
-    let tenantid_arg = Arg::new("tenantid")
+    let tenantid_arg = Arg::with_name("tenantid")
         .long("tenantid")
         .help("Tenant id. Represented as a hexadecimal string 32 symbols length")
         .takes_value(true)
         .required(false);
 
-    let port_arg = Arg::new("port")
+    let port_arg = Arg::with_name("port")
         .long("port")
         .required(false)
         .value_name("port");
 
-    let stop_mode_arg = Arg::new("stop-mode")
-        .short('m')
+    let stop_mode_arg = Arg::with_name("stop-mode")
+        .short("m")
         .takes_value(true)
         .possible_values(&["fast", "immediate"])
         .help("If 'immediate', don't flush repository data at shutdown")
         .required(false)
         .value_name("stop-mode");
 
-    let pageserver_config_args = Arg::new("pageserver-config-override")
+    let pageserver_config_args = Arg::with_name("pageserver-config-override")
         .long("pageserver-config-override")
         .takes_value(true)
         .number_of_values(1)
-        .multiple_occurrences(true)
+        .multiple(true)
         .help("Additional pageserver's configuration options or overrides, refer to pageserver's 'config-override' CLI parameter docs for more")
         .required(false);
 
     let matches = App::new("Zenith CLI")
         .setting(AppSettings::ArgRequiredElseHelp)
         .version(GIT_VERSION)
+        .arg(
+            Arg::with_name("override-config")
+                .long("override-config")
+                .takes_value(true)
+                .help("Override the default configuration file")
+                .required(false)
+        )
         .subcommand(
-            App::new("init")
+            SubCommand::with_name("init")
                 .about("Initialize a new Zenith repository")
                 .arg(pageserver_config_args.clone())
                 .arg(
-                    Arg::new("config")
+                    Arg::with_name("config")
                         .long("config")
                         .required(false)
                         .value_name("config"),
                 )
         )
         .subcommand(
-            App::new("branch")
+            SubCommand::with_name("branch")
                 .about("Create a new branch")
-                .arg(Arg::new("branchname").required(false).index(1))
-                .arg(Arg::new("start-point").required(false).index(2))
+                .arg(Arg::with_name("branchname").required(false).index(1))
+                .arg(Arg::with_name("start-point").required(false).index(2))
                 .arg(tenantid_arg.clone()),
         ).subcommand(
-            App::new("tenant")
+            SubCommand::with_name("tenant")
             .setting(AppSettings::ArgRequiredElseHelp)
             .about("Manage tenants")
-            .subcommand(App::new("list"))
-            .subcommand(App::new("create").arg(Arg::new("tenantid").required(false).index(1)))
+            .subcommand(SubCommand::with_name("list"))
+            .subcommand(SubCommand::with_name("create").arg(Arg::with_name("tenantid").required(false).index(1)))
         )
         .subcommand(
-            App::new("pageserver")
+            SubCommand::with_name("pageserver")
                 .setting(AppSettings::ArgRequiredElseHelp)
                 .about("Manage pageserver")
-                .subcommand(App::new("status"))
-                .subcommand(App::new("start").about("Start local pageserver").arg(pageserver_config_args.clone()))
-                .subcommand(App::new("stop").about("Stop local pageserver")
+                .subcommand(SubCommand::with_name("status"))
+                .subcommand(SubCommand::with_name("start").about("Start local pageserver").arg(pageserver_config_args.clone()))
+                .subcommand(SubCommand::with_name("stop").about("Stop local pageserver")
                             .arg(stop_mode_arg.clone()))
-                .subcommand(App::new("restart").about("Restart local pageserver").arg(pageserver_config_args.clone()))
+                .subcommand(SubCommand::with_name("restart").about("Restart local pageserver").arg(pageserver_config_args))
         )
         .subcommand(
-            App::new("safekeeper")
+            SubCommand::with_name("safekeeper")
                 .setting(AppSettings::ArgRequiredElseHelp)
                 .about("Manage safekeepers")
-                .subcommand(App::new("start")
+                .subcommand(SubCommand::with_name("start")
                             .about("Start local safekeeper")
                             .arg(safekeeper_node_arg.clone())
                 )
-                .subcommand(App::new("stop")
+                .subcommand(SubCommand::with_name("stop")
                             .about("Stop local safekeeper")
                             .arg(safekeeper_node_arg.clone())
                             .arg(stop_mode_arg.clone())
                 )
-                .subcommand(App::new("restart")
+                .subcommand(SubCommand::with_name("restart")
                             .about("Restart local safekeeper")
                             .arg(safekeeper_node_arg.clone())
                             .arg(stop_mode_arg.clone())
                 )
         )
         .subcommand(
-            App::new("pg")
+            SubCommand::with_name("pg")
                 .setting(AppSettings::ArgRequiredElseHelp)
                 .about("Manage postgres instances")
-                .subcommand(App::new("list").arg(tenantid_arg.clone()))
-                .subcommand(App::new("create")
+                .subcommand(SubCommand::with_name("list").arg(tenantid_arg.clone()))
+                .subcommand(SubCommand::with_name("create")
                     .about("Create a postgres compute node")
                     .arg(pg_node_arg.clone())
                     .arg(timeline_arg.clone())
                     .arg(tenantid_arg.clone())
                     .arg(port_arg.clone())
                     .arg(
-                        Arg::new("config-only")
+                        Arg::with_name("config-only")
                             .help("Don't do basebackup, create compute node with only config files")
                             .long("config-only")
                             .required(false)
                     ))
-                .subcommand(App::new("start")
+                .subcommand(SubCommand::with_name("start")
                     .about("Start a postgres compute node.\n This command actually creates new node from scratch, but preserves existing config files")
                     .arg(pg_node_arg.clone())
                     .arg(timeline_arg.clone())
                     .arg(tenantid_arg.clone())
                     .arg(port_arg.clone()))
                 .subcommand(
-                    App::new("stop")
+                    SubCommand::with_name("stop")
                         .arg(pg_node_arg.clone())
                         .arg(timeline_arg.clone())
                         .arg(tenantid_arg.clone())
                         .arg(
-                            Arg::new("destroy")
+                            Arg::with_name("destroy")
                                 .help("Also delete data directory (now optional, should be default in future)")
                                 .long("destroy")
                                 .required(false)
@@ -205,28 +210,28 @@ fn main() -> Result<()> {
 
         )
         .subcommand(
-            App::new("start")
+            SubCommand::with_name("start")
                 .about("Start page server and safekeepers")
-                .arg(pageserver_config_args)
         )
         .subcommand(
-            App::new("stop")
+            SubCommand::with_name("stop")
                 .about("Stop page server and safekeepers")
                 .arg(stop_mode_arg.clone())
         )
         .get_matches();
 
-    let (sub_name, sub_args) = match matches.subcommand() {
-        Some(subcommand_data) => subcommand_data,
-        None => bail!("no subcommand provided"),
-    };
+    let (sub_name, sub_args) = matches.subcommand();
+    let sub_args = sub_args.expect("no subcommand");
 
     // Check for 'zenith init' command first.
     let subcmd_result = if sub_name == "init" {
         handle_init(sub_args)
     } else {
-        // all other commands need an existing config
-        let env = match LocalEnv::load_config() {
+        // all other commands need an existing config, which is found in base directory,
+        // or can be overridden with --override-config
+
+        let override_config = matches.value_of("override-config");
+        let env = match LocalEnv::load_config(override_config) {
             Ok(conf) => conf,
             Err(e) => {
                 eprintln!("Error loading config: {}", e);
@@ -425,7 +430,7 @@ fn handle_init(init_match: &ArgMatches) -> Result<()> {
     Ok(())
 }
 
-fn pageserver_config_overrides(init_match: &ArgMatches) -> Vec<&str> {
+fn pageserver_config_overrides<'a>(init_match: &'a ArgMatches) -> Vec<&'a str> {
     init_match
         .values_of("pageserver-config-override")
         .into_iter()
@@ -436,12 +441,12 @@ fn pageserver_config_overrides(init_match: &ArgMatches) -> Vec<&str> {
 fn handle_tenant(tenant_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
     let pageserver = PageServerNode::from_env(env);
     match tenant_match.subcommand() {
-        Some(("list", _)) => {
+        ("list", Some(_)) => {
             for t in pageserver.tenant_list()? {
                 println!("{} {}", t.id, t.state);
             }
         }
-        Some(("create", create_match)) => {
+        ("create", Some(create_match)) => {
             let tenantid = match create_match.value_of("tenantid") {
                 Some(tenantid) => ZTenantId::from_str(tenantid)?,
                 None => ZTenantId::generate(),
@@ -450,8 +455,10 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &local_env::LocalEnv) -> Result
             pageserver.tenant_create(tenantid)?;
             println!("tenant successfully created on the pageserver");
         }
-        Some((sub_name, _)) => bail!("Unexpected tenant subcommand '{}'", sub_name),
-        None => bail!("no tenant subcommand provided"),
+
+        (sub_name, _) => {
+            bail!("Unexpected tenant subcommand '{}'", sub_name)
+        }
     }
     Ok(())
 }
@@ -480,10 +487,8 @@ fn handle_branch(branch_match: &ArgMatches, env: &local_env::LocalEnv) -> Result
 }
 
 fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
-    let (sub_name, sub_args) = match pg_match.subcommand() {
-        Some(pg_subcommand_data) => pg_subcommand_data,
-        None => bail!("no pg subcommand provided"),
-    };
+    let (sub_name, sub_args) = pg_match.subcommand();
+    let sub_args = sub_args.expect("no pg subcommand");
 
     let mut cplane = ComputeControlPlane::load(env.clone())?;
 
@@ -594,14 +599,14 @@ fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
     let pageserver = PageServerNode::from_env(env);
 
     match sub_match.subcommand() {
-        Some(("start", start_match)) => {
+        ("start", Some(start_match)) => {
             if let Err(e) = pageserver.start(&pageserver_config_overrides(start_match)) {
                 eprintln!("pageserver start failed: {}", e);
                 exit(1);
             }
         }
 
-        Some(("stop", stop_match)) => {
+        ("stop", Some(stop_match)) => {
             let immediate = stop_match.value_of("stop-mode") == Some("immediate");
 
             if let Err(e) = pageserver.stop(immediate) {
@@ -610,7 +615,7 @@ fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
             }
         }
 
-        Some(("restart", restart_match)) => {
+        ("restart", Some(restart_match)) => {
             //TODO what shutdown strategy should we use here?
             if let Err(e) = pageserver.stop(false) {
                 eprintln!("pageserver stop failed: {}", e);
@@ -622,8 +627,8 @@ fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
                 exit(1);
             }
         }
-        Some((sub_name, _)) => bail!("Unexpected pageserver subcommand '{}'", sub_name),
-        None => bail!("no pageserver subcommand provided"),
+
+        (sub_name, _) => bail!("Unexpected pageserver subcommand '{}'", sub_name),
     }
     Ok(())
 }
@@ -637,10 +642,8 @@ fn get_safekeeper(env: &local_env::LocalEnv, name: &str) -> Result<SafekeeperNod
 }
 
 fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
-    let (sub_name, sub_args) = match sub_match.subcommand() {
-        Some(safekeeper_command_data) => safekeeper_command_data,
-        None => bail!("no safekeeper subcommand provided"),
-    };
+    let (sub_name, sub_args) = sub_match.subcommand();
+    let sub_args = sub_args.expect("no safekeeper subcommand");
 
     // All the commands take an optional safekeeper name argument
     let node_name = sub_args.value_of("node").unwrap_or(DEFAULT_SAFEKEEPER_NAME);

zenith_metrics/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "zenith_metrics"
 version = "0.1.0"
-edition = "2021"
+edition = "2018"
 
 [dependencies]
-prometheus = {version = "0.13", default_features=false} # removes protobuf dependency
+prometheus = {version = "0.12", default_features=false} # removes protobuf dependency
 libc = "0.2"
 lazy_static = "1.4"
 once_cell = "1.8.0"

zenith_utils/Cargo.toml

@@ -1,22 +1,22 @@
 [package]
 name = "zenith_utils"
 version = "0.1.0"
-edition = "2021"
+authors = ["Eric Seppanen <eric@zenith.tech>"]
+edition = "2018"
 
 [dependencies]
 anyhow = "1.0"
 bincode = "1.3"
+byteorder = "1.4.3"
 bytes = "1.0.1"
 hyper = { version = "0.14.7", features = ["full"] }
 lazy_static = "1.4.0"
-pin-project-lite = "0.2.7"
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
-postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
-routerify = "3"
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
+routerify = "2"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 thiserror = "1.0"
-tokio = { version = "1.11", features = ["macros"]}
+tokio = "1.11"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 nix = "0.23.0"
@@ -32,8 +32,7 @@ zenith_metrics = { path = "../zenith_metrics" }
 workspace_hack = { path = "../workspace_hack" }
 
 [dev-dependencies]
-byteorder = "1.4.3"
-bytes = "1.0.1"
 hex-literal = "0.3"
-tempfile = "3.2"
+bytes = "1.0"
 webpki = "0.21"
+tempfile = "3.2"

zenith_utils/src/auth.rs

@@ -85,6 +85,7 @@ pub fn check_permission(claims: &Claims, tenantid: Option<ZTenantId>) -> Result<
     }
 }
 
+#[derive(Debug)]
 pub struct JwtAuth {
     decoding_key: DecodingKey<'static>,
     validation: Validation,
@@ -112,14 +113,6 @@ impl JwtAuth {
     }
 }
 
-impl std::fmt::Debug for JwtAuth {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("JwtAuth")
-            .field("validation", &self.validation)
-            .finish()
-    }
-}
-
 // this function is used only for testing purposes in CLI e g generate tokens during init
 pub fn encode_from_key_file(claims: &Claims, key_data: &[u8]) -> Result<String> {
     let key = EncodingKey::from_rsa_pem(key_data)?;

zenith_utils/src/http/mod.rs

@@ -2,7 +2,3 @@ pub mod endpoint;
 pub mod error;
 pub mod json;
 pub mod request;
-
-/// Current fast way to apply simple http routing in various Zenith binaries.
-/// Re-exported for sake of uniform approach, that could be later replaced with better alternatives, if needed.
-pub use routerify::{ext::RequestExt, RouterBuilder};

zenith_utils/src/lib.rs

@@ -42,9 +42,6 @@ pub mod logging;
 pub mod accum;
 pub mod shutdown;
 
-// Tools for calling certain async methods in sync contexts
-pub mod sync;
-
 // Utility for binding TcpListeners with proper socket options.
 pub mod tcp_listener;
 
@@ -54,9 +51,6 @@ pub mod nonblock;
 // Default signal handling
 pub mod signals;
 
-// Postgres checksum calculation
-pub mod pg_checksum_page;
-
 // This is a shortcut to embed git sha into binaries and avoid copying the same build script to all packages
 //
 // we have several cases:

zenith_utils/src/pg_checksum_page.rs

@@ -1,70 +0,0 @@
-///
-///  Port of Postgres pg_checksum_page
-///
-
-const BLCKSZ: usize = 8192;
-const N_SUMS: usize = 32;
-/* prime multiplier of FNV-1a hash */
-const FNV_PRIME: u32 = 16777619;
-
-/*
- * Base offsets to initialize each of the parallel FNV hashes into a
- * different initial state.
- */
-const CHECKSUM_BASE_OFFSETS: [u32; N_SUMS] = [
-    0x5B1F36E9, 0xB8525960, 0x02AB50AA, 0x1DE66D2A, 0x79FF467A, 0x9BB9F8A3, 0x217E7CD2, 0x83E13D2C,
-    0xF8D4474F, 0xE39EB970, 0x42C6AE16, 0x993216FA, 0x7B093B5D, 0x98DAFF3C, 0xF718902A, 0x0B1C9CDB,
-    0xE58F764B, 0x187636BC, 0x5D7B3BB1, 0xE73DE7DE, 0x92BEC979, 0xCCA6C0B2, 0x304A0979, 0x85AA43D4,
-    0x783125BB, 0x6CA8EAA2, 0xE407EAC6, 0x4B5CFC3E, 0x9FBF8C76, 0x15CA20BE, 0xF2CA9FD3, 0x959BD756,
-];
-
-/*
- * Calculate one round of the checksum.
- */
-fn checksum_comp(checksum: u32, value: u32) -> u32 {
-    let tmp = checksum ^ value;
-    tmp.wrapping_mul(FNV_PRIME) ^ (tmp >> 17)
-}
-
-/*
- * Compute the checksum for a Postgres page.
- *
- * The page must be adequately aligned (at least on a 4-byte boundary).
- * Beware also that the checksum field of the page is transiently zeroed.
- *
- * The checksum includes the block number (to detect the case where a page is
- * somehow moved to a different location), the page header (excluding the
- * checksum itself), and the page data.
- */
-pub fn pg_checksum_page(data: &[u8], blkno: u32) -> u16 {
-    let page = unsafe { std::mem::transmute::<&[u8], &[u32]>(data) };
-    let mut checksum: u32 = 0;
-    let mut sums = CHECKSUM_BASE_OFFSETS;
-
-    /* main checksum calculation */
-    for i in 0..(BLCKSZ / (4 * N_SUMS)) {
-        for j in 0..N_SUMS {
-            sums[j] = checksum_comp(sums[j], page[i * N_SUMS + j]);
-        }
-    }
-    /* finally add in two rounds of zeroes for additional mixing */
-    for i in 0..2 {
-        for j in 0..N_SUMS {
-            sums[i] = checksum_comp(sums[j], 0);
-        }
-    }
-
-    /* xor fold partial checksums together */
-    for sum in sums {
-        checksum ^= sum;
-    }
-
-    /* Mix in the block number to detect transposed pages */
-    checksum ^= blkno;
-
-    /*
-     * Reduce to a uint16 (to fit in the pd_checksum field) with an offset of
-     * one. That avoids checksums of zero, which seems like a good idea.
-     */
-    ((checksum % 65535) + 1) as u16
-}

zenith_utils/src/postgres_backend.rs

@@ -366,10 +366,6 @@ impl PostgresBackend {
                             AuthType::Trust => {
                                 self.write_message_noflush(&BeMessage::AuthenticationOk)?
                                     .write_message_noflush(&BeParameterStatusMessage::encoding())?
-                                    // The async python driver requires a valid server_version
-                                    .write_message_noflush(&BeMessage::ParameterStatus(
-                                        BeParameterStatusMessage::ServerVersion("14.1"),
-                                    ))?
                                     .write_message(&BeMessage::ReadyForQuery)?;
                                 self.state = ProtoState::Established;
                             }

zenith_utils/src/pq_proto.rs

@@ -2,18 +2,15 @@
 //! <https://www.postgresql.org/docs/devel/protocol-message-formats.html>
 //! on message formats.
 
-use crate::sync::{AsyncishRead, SyncFuture};
 use anyhow::{bail, ensure, Context, Result};
+use byteorder::{BigEndian, ByteOrder};
+use byteorder::{ReadBytesExt, BE};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
-use postgres_protocol::PG_EPOCH;
-use serde::{Deserialize, Serialize};
+// use postgres_ffi::xlog_utils::TimestampTz;
 use std::collections::HashMap;
-use std::future::Future;
+use std::io::Read;
 use std::io::{self, Cursor};
 use std::str;
-use std::time::{Duration, SystemTime};
-use tokio::io::AsyncReadExt;
-use tracing::{trace, warn};
 
 pub type Oid = u32;
 pub type SystemId = u64;
@@ -96,14 +93,14 @@ impl FeMessage {
     /// One way to handle this properly:
     ///
     /// ```
-    /// # use std::io;
+    /// # use std::io::Read;
     /// # use zenith_utils::pq_proto::FeMessage;
     /// #
     /// # fn process_message(msg: FeMessage) -> anyhow::Result<()> {
     /// #     Ok(())
     /// # };
     /// #
-    /// fn do_the_job(stream: &mut (impl io::Read + Unpin)) -> anyhow::Result<()> {
+    /// fn do_the_job(stream: &mut impl Read) -> anyhow::Result<()> {
     ///     while let Some(msg) = FeMessage::read(stream)? {
     ///         process_message(msg)?;
     ///     }
@@ -111,159 +108,124 @@ impl FeMessage {
     ///     Ok(())
     /// }
     /// ```
-    #[inline(never)]
-    pub fn read(stream: &mut (impl io::Read + Unpin)) -> anyhow::Result<Option<FeMessage>> {
-        Self::read_fut(&mut AsyncishRead(stream)).wait()
-    }
+    pub fn read(stream: &mut impl Read) -> anyhow::Result<Option<FeMessage>> {
+        // Each libpq message begins with a message type byte, followed by message length
+        // If the client closes the connection, return None. But if the client closes the
+        // connection in the middle of a message, we will return an error.
+        let tag = match stream.read_u8() {
+            Ok(b) => b,
+            Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
+            Err(e) => return Err(e.into()),
+        };
+        let len = stream.read_u32::<BE>()?;
 
-    /// Read one message from the stream.
-    /// See documentation for `Self::read`.
-    pub fn read_fut<Reader>(
-        stream: &mut Reader,
-    ) -> SyncFuture<Reader, impl Future<Output = anyhow::Result<Option<FeMessage>>> + '_>
-    where
-        Reader: tokio::io::AsyncRead + Unpin,
-    {
-        // We return a Future that's sync (has a `wait` method) if and only if the provided stream is SyncProof.
-        // SyncFuture contract: we are only allowed to await on sync-proof futures, the AsyncRead and
-        // AsyncReadExt methods of the stream.
-        SyncFuture::new(async move {
-            // Each libpq message begins with a message type byte, followed by message length
-            // If the client closes the connection, return None. But if the client closes the
-            // connection in the middle of a message, we will return an error.
-            let tag = match stream.read_u8().await {
-                Ok(b) => b,
-                Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
-                Err(e) => return Err(e.into()),
-            };
-            let len = stream.read_u32().await?;
+        // The message length includes itself, so it better be at least 4
+        let bodylen = len
+            .checked_sub(4)
+            .context("invalid message length: parsing u32")?;
 
-            // The message length includes itself, so it better be at least 4
-            let bodylen = len
-                .checked_sub(4)
-                .context("invalid message length: parsing u32")?;
+        // Read message body
+        let mut body_buf: Vec<u8> = vec![0; bodylen as usize];
+        stream.read_exact(&mut body_buf)?;
 
-            // Read message body
-            let mut body_buf: Vec<u8> = vec![0; bodylen as usize];
-            stream.read_exact(&mut body_buf).await?;
+        let body = Bytes::from(body_buf);
 
-            let body = Bytes::from(body_buf);
-
-            // Parse it
-            match tag {
-                b'Q' => Ok(Some(FeMessage::Query(FeQueryMessage { body }))),
-                b'P' => Ok(Some(FeParseMessage::parse(body)?)),
-                b'D' => Ok(Some(FeDescribeMessage::parse(body)?)),
-                b'E' => Ok(Some(FeExecuteMessage::parse(body)?)),
-                b'B' => Ok(Some(FeBindMessage::parse(body)?)),
-                b'C' => Ok(Some(FeCloseMessage::parse(body)?)),
-                b'S' => Ok(Some(FeMessage::Sync)),
-                b'X' => Ok(Some(FeMessage::Terminate)),
-                b'd' => Ok(Some(FeMessage::CopyData(body))),
-                b'c' => Ok(Some(FeMessage::CopyDone)),
-                b'f' => Ok(Some(FeMessage::CopyFail)),
-                b'p' => Ok(Some(FeMessage::PasswordMessage(body))),
-                tag => bail!("unknown message tag: {},'{:?}'", tag, body),
-            }
-        })
+        // Parse it
+        match tag {
+            b'Q' => Ok(Some(FeMessage::Query(FeQueryMessage { body }))),
+            b'P' => Ok(Some(FeParseMessage::parse(body)?)),
+            b'D' => Ok(Some(FeDescribeMessage::parse(body)?)),
+            b'E' => Ok(Some(FeExecuteMessage::parse(body)?)),
+            b'B' => Ok(Some(FeBindMessage::parse(body)?)),
+            b'C' => Ok(Some(FeCloseMessage::parse(body)?)),
+            b'S' => Ok(Some(FeMessage::Sync)),
+            b'X' => Ok(Some(FeMessage::Terminate)),
+            b'd' => Ok(Some(FeMessage::CopyData(body))),
+            b'c' => Ok(Some(FeMessage::CopyDone)),
+            b'f' => Ok(Some(FeMessage::CopyFail)),
+            b'p' => Ok(Some(FeMessage::PasswordMessage(body))),
+            tag => bail!("unknown message tag: {},'{:?}'", tag, body),
+        }
     }
 }
 
 impl FeStartupPacket {
     /// Read startup message from the stream.
-    // XXX: It's tempting yet undesirable to accept `stream` by value,
-    // since such a change will cause user-supplied &mut references to be consumed
-    pub fn read(stream: &mut (impl io::Read + Unpin)) -> anyhow::Result<Option<FeMessage>> {
-        Self::read_fut(&mut AsyncishRead(stream)).wait()
-    }
-
-    /// Read startup message from the stream.
-    // XXX: It's tempting yet undesirable to accept `stream` by value,
-    // since such a change will cause user-supplied &mut references to be consumed
-    pub fn read_fut<Reader>(
-        stream: &mut Reader,
-    ) -> SyncFuture<Reader, impl Future<Output = anyhow::Result<Option<FeMessage>>> + '_>
-    where
-        Reader: tokio::io::AsyncRead + Unpin,
-    {
+    pub fn read(stream: &mut impl std::io::Read) -> anyhow::Result<Option<FeMessage>> {
         const MAX_STARTUP_PACKET_LENGTH: usize = 10000;
         const RESERVED_INVALID_MAJOR_VERSION: u32 = 1234;
         const CANCEL_REQUEST_CODE: u32 = 5678;
         const NEGOTIATE_SSL_CODE: u32 = 5679;
         const NEGOTIATE_GSS_CODE: u32 = 5680;
 
-        SyncFuture::new(async move {
-            // Read length. If the connection is closed before reading anything (or before
-            // reading 4 bytes, to be precise), return None to indicate that the connection
-            // was closed. This matches the PostgreSQL server's behavior, which avoids noise
-            // in the log if the client opens connection but closes it immediately.
-            let len = match stream.read_u32().await {
-                Ok(len) => len as usize,
-                Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
-                Err(e) => return Err(e.into()),
-            };
+        // Read length. If the connection is closed before reading anything (or before
+        // reading 4 bytes, to be precise), return None to indicate that the connection
+        // was closed. This matches the PostgreSQL server's behavior, which avoids noise
+        // in the log if the client opens connection but closes it immediately.
+        let len = match stream.read_u32::<BE>() {
+            Ok(len) => len as usize,
+            Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
+            Err(e) => return Err(e.into()),
+        };
 
-            if len < 4 || len > MAX_STARTUP_PACKET_LENGTH {
-                bail!("invalid message length");
+        if len < 4 || len > MAX_STARTUP_PACKET_LENGTH {
+            bail!("invalid message length");
+        }
+
+        let request_code = stream.read_u32::<BE>()?;
+
+        // the rest of startup packet are params
+        let params_len = len - 8;
+        let mut params_bytes = vec![0u8; params_len];
+        stream.read_exact(params_bytes.as_mut())?;
+
+        // Parse params depending on request code
+        let most_sig_16_bits = request_code >> 16;
+        let least_sig_16_bits = request_code & ((1 << 16) - 1);
+        let message = match (most_sig_16_bits, least_sig_16_bits) {
+            (RESERVED_INVALID_MAJOR_VERSION, CANCEL_REQUEST_CODE) => {
+                ensure!(params_len == 8, "expected 8 bytes for CancelRequest params");
+                let mut cursor = Cursor::new(params_bytes);
+                FeStartupPacket::CancelRequest(CancelKeyData {
+                    backend_pid: cursor.read_i32::<BigEndian>()?,
+                    cancel_key: cursor.read_i32::<BigEndian>()?,
+                })
             }
-
-            let request_code = stream.read_u32().await?;
-
-            // the rest of startup packet are params
-            let params_len = len - 8;
-            let mut params_bytes = vec![0u8; params_len];
-            stream.read_exact(params_bytes.as_mut()).await?;
-
-            // Parse params depending on request code
-            let most_sig_16_bits = request_code >> 16;
-            let least_sig_16_bits = request_code & ((1 << 16) - 1);
-            let message = match (most_sig_16_bits, least_sig_16_bits) {
-                (RESERVED_INVALID_MAJOR_VERSION, CANCEL_REQUEST_CODE) => {
-                    ensure!(params_len == 8, "expected 8 bytes for CancelRequest params");
-                    let mut cursor = Cursor::new(params_bytes);
-                    FeStartupPacket::CancelRequest(CancelKeyData {
-                        backend_pid: cursor.read_i32().await?,
-                        cancel_key: cursor.read_i32().await?,
-                    })
-                }
-                (RESERVED_INVALID_MAJOR_VERSION, NEGOTIATE_SSL_CODE) => FeStartupPacket::SslRequest,
-                (RESERVED_INVALID_MAJOR_VERSION, NEGOTIATE_GSS_CODE) => {
-                    FeStartupPacket::GssEncRequest
-                }
-                (RESERVED_INVALID_MAJOR_VERSION, unrecognized_code) => {
-                    bail!("Unrecognized request code {}", unrecognized_code)
-                }
-                (major_version, minor_version) => {
-                    // TODO bail if protocol major_version is not 3?
-                    // Parse null-terminated (String) pairs of param name / param value
-                    let params_str = str::from_utf8(&params_bytes).unwrap();
-                    let mut params_tokens = params_str.split('\0');
-                    let mut params: HashMap<String, String> = HashMap::new();
-                    while let Some(name) = params_tokens.next() {
-                        let value = params_tokens
-                            .next()
-                            .context("expected even number of params in StartupMessage")?;
-                        if name == "options" {
-                            // deprecated way of passing params as cmd line args
-                            for cmdopt in value.split(' ') {
-                                let nameval: Vec<&str> = cmdopt.split('=').collect();
-                                if nameval.len() == 2 {
-                                    params.insert(nameval[0].to_string(), nameval[1].to_string());
-                                }
+            (RESERVED_INVALID_MAJOR_VERSION, NEGOTIATE_SSL_CODE) => FeStartupPacket::SslRequest,
+            (RESERVED_INVALID_MAJOR_VERSION, NEGOTIATE_GSS_CODE) => FeStartupPacket::GssEncRequest,
+            (RESERVED_INVALID_MAJOR_VERSION, unrecognized_code) => {
+                bail!("Unrecognized request code {}", unrecognized_code)
+            }
+            (major_version, minor_version) => {
+                // TODO bail if protocol major_version is not 3?
+                // Parse null-terminated (String) pairs of param name / param value
+                let params_str = str::from_utf8(&params_bytes).unwrap();
+                let mut params_tokens = params_str.split('\0');
+                let mut params: HashMap<String, String> = HashMap::new();
+                while let Some(name) = params_tokens.next() {
+                    let value = params_tokens
+                        .next()
+                        .context("expected even number of params in StartupMessage")?;
+                    if name == "options" {
+                        // deprecated way of passing params as cmd line args
+                        for cmdopt in value.split(' ') {
+                            let nameval: Vec<&str> = cmdopt.split('=').collect();
+                            if nameval.len() == 2 {
+                                params.insert(nameval[0].to_string(), nameval[1].to_string());
                             }
-                        } else {
-                            params.insert(name.to_string(), value.to_string());
                         }
-                    }
-                    FeStartupPacket::StartupMessage {
-                        major_version,
-                        minor_version,
-                        params,
+                    } else {
+                        params.insert(name.to_string(), value.to_string());
                     }
                 }
-            };
-            Ok(Some(FeMessage::StartupPacket(message)))
-        })
+                FeStartupPacket::StartupMessage {
+                    major_version,
+                    minor_version,
+                    params,
+                }
+            }
+        };
+        Ok(Some(FeMessage::StartupPacket(message)))
     }
 }
 
@@ -537,12 +499,12 @@ where
     f(buf)?;
 
     let size = i32::from_usize(buf.len() - base)?;
-    (&mut buf[base..]).put_slice(&size.to_be_bytes());
+    BigEndian::write_i32(&mut buf[base..], size);
     Ok(())
 }
 
 /// Safe write of s into buf as cstring (String in the protocol).
-pub fn write_cstr(s: &[u8], buf: &mut BytesMut) -> Result<(), io::Error> {
+fn write_cstr(s: &[u8], buf: &mut BytesMut) -> Result<(), io::Error> {
     if s.contains(&0) {
         return Err(io::Error::new(
             io::ErrorKind::InvalidInput,
@@ -554,17 +516,6 @@ pub fn write_cstr(s: &[u8], buf: &mut BytesMut) -> Result<(), io::Error> {
     Ok(())
 }
 
-// Truncate 0 from C string in Bytes and stringify it (returns slice, no allocations)
-// PG protocol strings are always C strings.
-fn cstr_to_str(b: &Bytes) -> Result<&str> {
-    let without_null = if b.last() == Some(&0) {
-        &b[..b.len() - 1]
-    } else {
-        &b[..]
-    };
-    std::str::from_utf8(without_null).map_err(|e| e.into())
-}
-
 impl<'a> BeMessage<'a> {
     /// Write message to the given buf.
     // Unlike the reading side, we use BytesMut
@@ -847,178 +798,3 @@ impl<'a> BeMessage<'a> {
         Ok(())
     }
 }
-
-// Zenith extension of postgres replication protocol
-// See ZENITH_STATUS_UPDATE_TAG_BYTE
-#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
-pub struct ZenithFeedback {
-    // Last known size of the timeline. Used to enforce timeline size limit.
-    pub current_timeline_size: u64,
-    // Parts of StandbyStatusUpdate we resend to compute via safekeeper
-    pub ps_writelsn: u64,
-    pub ps_applylsn: u64,
-    pub ps_flushlsn: u64,
-    pub ps_replytime: SystemTime,
-}
-
-// NOTE: Do not forget to increment this number when adding new fields to ZenithFeedback.
-// Do not remove previously available fields because this might be backwards incompatible.
-pub const ZENITH_FEEDBACK_FIELDS_NUMBER: u8 = 5;
-
-impl ZenithFeedback {
-    pub fn empty() -> ZenithFeedback {
-        ZenithFeedback {
-            current_timeline_size: 0,
-            ps_writelsn: 0,
-            ps_applylsn: 0,
-            ps_flushlsn: 0,
-            ps_replytime: SystemTime::now(),
-        }
-    }
-
-    // Serialize ZenithFeedback using custom format
-    // to support protocol extensibility.
-    //
-    // Following layout is used:
-    // char - number of key-value pairs that follow.
-    //
-    // key-value pairs:
-    // null-terminated string - key,
-    // uint32 - value length in bytes
-    // value itself
-    pub fn serialize(&self, buf: &mut BytesMut) -> Result<()> {
-        buf.put_u8(ZENITH_FEEDBACK_FIELDS_NUMBER); // # of keys
-        write_cstr(&Bytes::from("current_timeline_size"), buf)?;
-        buf.put_i32(8);
-        buf.put_u64(self.current_timeline_size);
-
-        write_cstr(&Bytes::from("ps_writelsn"), buf)?;
-        buf.put_i32(8);
-        buf.put_u64(self.ps_writelsn);
-        write_cstr(&Bytes::from("ps_flushlsn"), buf)?;
-        buf.put_i32(8);
-        buf.put_u64(self.ps_flushlsn);
-        write_cstr(&Bytes::from("ps_applylsn"), buf)?;
-        buf.put_i32(8);
-        buf.put_u64(self.ps_applylsn);
-
-        let timestamp = self
-            .ps_replytime
-            .duration_since(*PG_EPOCH)
-            .expect("failed to serialize pg_replytime earlier than PG_EPOCH")
-            .as_micros() as i64;
-
-        write_cstr(&Bytes::from("ps_replytime"), buf)?;
-        buf.put_i32(8);
-        buf.put_i64(timestamp);
-        Ok(())
-    }
-
-    // Deserialize ZenithFeedback message
-    pub fn parse(mut buf: Bytes) -> ZenithFeedback {
-        let mut zf = ZenithFeedback::empty();
-        let nfields = buf.get_u8();
-        let mut i = 0;
-        while i < nfields {
-            i += 1;
-            let key_cstr = read_null_terminated(&mut buf).unwrap();
-            let key = cstr_to_str(&key_cstr).unwrap();
-            match key {
-                "current_timeline_size" => {
-                    let len = buf.get_i32();
-                    assert_eq!(len, 8);
-                    zf.current_timeline_size = buf.get_u64();
-                }
-                "ps_writelsn" => {
-                    let len = buf.get_i32();
-                    assert_eq!(len, 8);
-                    zf.ps_writelsn = buf.get_u64();
-                }
-                "ps_flushlsn" => {
-                    let len = buf.get_i32();
-                    assert_eq!(len, 8);
-                    zf.ps_flushlsn = buf.get_u64();
-                }
-                "ps_applylsn" => {
-                    let len = buf.get_i32();
-                    assert_eq!(len, 8);
-                    zf.ps_applylsn = buf.get_u64();
-                }
-                "ps_replytime" => {
-                    let len = buf.get_i32();
-                    assert_eq!(len, 8);
-                    let raw_time = buf.get_i64();
-                    if raw_time > 0 {
-                        zf.ps_replytime = *PG_EPOCH + Duration::from_micros(raw_time as u64);
-                    } else {
-                        zf.ps_replytime = *PG_EPOCH - Duration::from_micros(-raw_time as u64);
-                    }
-                }
-                _ => {
-                    let len = buf.get_i32();
-                    warn!(
-                        "ZenithFeedback parse. unknown key {} of len {}. Skip it.",
-                        key, len
-                    );
-                    buf.advance(len as usize);
-                }
-            }
-        }
-        trace!("ZenithFeedback parsed is {:?}", zf);
-        zf
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_zenithfeedback_serialization() {
-        let mut zf = ZenithFeedback::empty();
-        // Fill zf wih some values
-        zf.current_timeline_size = 12345678;
-        // Set rounded time to be able to compare it with deserialized value,
-        // because it is rounded up to microseconds during serialization.
-        zf.ps_replytime = *PG_EPOCH + Duration::from_secs(100_000_000);
-        let mut data = BytesMut::new();
-        zf.serialize(&mut data).unwrap();
-
-        let zf_parsed = ZenithFeedback::parse(data.freeze());
-        assert_eq!(zf, zf_parsed);
-    }
-
-    #[test]
-    fn test_zenithfeedback_unknown_key() {
-        let mut zf = ZenithFeedback::empty();
-        // Fill zf wih some values
-        zf.current_timeline_size = 12345678;
-        // Set rounded time to be able to compare it with deserialized value,
-        // because it is rounded up to microseconds during serialization.
-        zf.ps_replytime = *PG_EPOCH + Duration::from_secs(100_000_000);
-        let mut data = BytesMut::new();
-        zf.serialize(&mut data).unwrap();
-
-        // Add an extra field to the buffer and adjust number of keys
-        if let Some(first) = data.first_mut() {
-            *first = ZENITH_FEEDBACK_FIELDS_NUMBER + 1;
-        }
-
-        write_cstr(&Bytes::from("new_field_one"), &mut data).unwrap();
-        data.put_i32(8);
-        data.put_u64(42);
-
-        // Parse serialized data and check that new field is not parsed
-        let zf_parsed = ZenithFeedback::parse(data.freeze());
-        assert_eq!(zf, zf_parsed);
-    }
-
-    // Make sure that `read` is sync/async callable
-    async fn _assert(stream: &mut (impl tokio::io::AsyncRead + Unpin)) {
-        let _ = FeMessage::read(&mut [].as_ref());
-        let _ = FeMessage::read_fut(stream).await;
-
-        let _ = FeStartupPacket::read(&mut [].as_ref());
-        let _ = FeStartupPacket::read_fut(stream).await;
-    }
-}

zenith_utils/src/sync.rs

@@ -1,179 +0,0 @@
-use pin_project_lite::pin_project;
-use std::future::Future;
-use std::marker::PhantomData;
-use std::pin::Pin;
-use std::{io, task};
-
-pin_project! {
-    /// We use this future to mark certain methods
-    /// as callable in both sync and async modes.
-    #[repr(transparent)]
-    pub struct SyncFuture<S, T: Future> {
-        #[pin]
-        inner: T,
-        _marker: PhantomData<S>,
-    }
-}
-
-/// This wrapper lets us synchronously wait for inner future's completion
-/// (see [`SyncFuture::wait`]) **provided that `S` implements [`SyncProof`]**.
-/// For instance, `S` may be substituted with types implementing
-/// [`tokio::io::AsyncRead`], but it's not the only viable option.
-impl<S, T: Future> SyncFuture<S, T> {
-    /// NOTE: caller should carefully pick a type for `S`,
-    /// because we don't want to enable [`SyncFuture::wait`] when
-    /// it's in fact impossible to run the future synchronously.
-    /// Violation of this contract will not cause UB, but
-    /// panics and async event loop freezes won't please you.
-    ///
-    /// Example:
-    ///
-    /// ```
-    /// # use zenith_utils::sync::SyncFuture;
-    /// # use std::future::Future;
-    /// # use tokio::io::AsyncReadExt;
-    /// #
-    /// // Parse a pair of numbers from a stream
-    /// pub fn parse_pair<Reader>(
-    ///     stream: &mut Reader,
-    /// ) -> SyncFuture<Reader, impl Future<Output = anyhow::Result<(u32, u64)>> + '_>
-    /// where
-    ///     Reader: tokio::io::AsyncRead + Unpin,
-    /// {
-    ///     // If `Reader` is a `SyncProof`, this will give caller
-    ///     // an opportunity to use `SyncFuture::wait`, because
-    ///     // `.await` will always result in `Poll::Ready`.
-    ///     SyncFuture::new(async move {
-    ///         let x = stream.read_u32().await?;
-    ///         let y = stream.read_u64().await?;
-    ///         Ok((x, y))
-    ///     })
-    /// }
-    /// ```
-    pub fn new(inner: T) -> Self {
-        Self {
-            inner,
-            _marker: PhantomData,
-        }
-    }
-}
-
-impl<S, T: Future> Future for SyncFuture<S, T> {
-    type Output = T::Output;
-
-    /// In async code, [`SyncFuture`] behaves like a regular wrapper.
-    #[inline(always)]
-    fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> task::Poll<Self::Output> {
-        self.project().inner.poll(cx)
-    }
-}
-
-/// Postulates that we can call [`SyncFuture::wait`].
-/// If implementer is also a [`Future`], it should always
-/// return [`task::Poll::Ready`] from [`Future::poll`].
-///
-/// Each implementation should document which futures
-/// specifically are being declared sync-proof.
-pub trait SyncPostulate {}
-
-impl<T: SyncPostulate> SyncPostulate for &T {}
-impl<T: SyncPostulate> SyncPostulate for &mut T {}
-
-impl<P: SyncPostulate, T: Future> SyncFuture<P, T> {
-    /// Synchronously wait for future completion.
-    pub fn wait(mut self) -> T::Output {
-        const RAW_WAKER: task::RawWaker = task::RawWaker::new(
-            std::ptr::null(),
-            &task::RawWakerVTable::new(
-                |_| RAW_WAKER,
-                |_| panic!("SyncFuture: failed to wake"),
-                |_| panic!("SyncFuture: failed to wake by ref"),
-                |_| { /* drop is no-op */ },
-            ),
-        );
-
-        // SAFETY: We never move `self` during this call;
-        // furthermore, it will be dropped in the end regardless of panics
-        let this = unsafe { Pin::new_unchecked(&mut self) };
-
-        // SAFETY: This waker doesn't do anything apart from panicking
-        let waker = unsafe { task::Waker::from_raw(RAW_WAKER) };
-        let context = &mut task::Context::from_waker(&waker);
-
-        match this.poll(context) {
-            task::Poll::Ready(res) => res,
-            _ => panic!("SyncFuture: unexpected pending!"),
-        }
-    }
-}
-
-/// This wrapper turns any [`std::io::Read`] into a blocking [`tokio::io::AsyncRead`],
-/// which lets us abstract over sync & async readers in methods returning [`SyncFuture`].
-/// NOTE: you **should not** use this in async code.
-#[repr(transparent)]
-pub struct AsyncishRead<T: io::Read + Unpin>(pub T);
-
-/// This lets us call [`SyncFuture<AsyncishRead<_>, _>::wait`],
-/// and allows the future to await on any of the [`AsyncRead`]
-/// and [`AsyncReadExt`] methods on `AsyncishRead`.
-impl<T: io::Read + Unpin> SyncPostulate for AsyncishRead<T> {}
-
-impl<T: io::Read + Unpin> tokio::io::AsyncRead for AsyncishRead<T> {
-    #[inline(always)]
-    fn poll_read(
-        mut self: Pin<&mut Self>,
-        _cx: &mut task::Context<'_>,
-        buf: &mut tokio::io::ReadBuf<'_>,
-    ) -> task::Poll<io::Result<()>> {
-        task::Poll::Ready(
-            // `Read::read` will block, meaning we don't need a real event loop!
-            self.0
-                .read(buf.initialize_unfilled())
-                .map(|sz| buf.advance(sz)),
-        )
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use tokio::io::{AsyncReadExt, AsyncWriteExt};
-
-    // async helper(stream: &mut impl AsyncRead) -> io::Result<u32>
-    fn bytes_add<Reader>(
-        stream: &mut Reader,
-    ) -> SyncFuture<Reader, impl Future<Output = io::Result<u32>> + '_>
-    where
-        Reader: tokio::io::AsyncRead + Unpin,
-    {
-        SyncFuture::new(async move {
-            let a = stream.read_u32().await?;
-            let b = stream.read_u32().await?;
-            Ok(a + b)
-        })
-    }
-
-    #[test]
-    fn test_sync() {
-        let bytes = [100u32.to_be_bytes(), 200u32.to_be_bytes()].concat();
-        let res = bytes_add(&mut AsyncishRead(&mut &bytes[..]))
-            .wait()
-            .unwrap();
-        assert_eq!(res, 300);
-    }
-
-    // We need a single-threaded executor for this test
-    #[tokio::test(flavor = "current_thread")]
-    async fn test_async() {
-        let (mut tx, mut rx) = tokio::net::UnixStream::pair().unwrap();
-
-        let write = async move {
-            tx.write_u32(100).await?;
-            tx.write_u32(200).await?;
-            Ok(())
-        };
-
-        let (res, ()) = tokio::try_join!(bytes_add(&mut rx), write).unwrap();
-        assert_eq!(res, 300);
-    }
-}

zenith_utils/src/zid.rs

@@ -195,32 +195,6 @@ pub mod opt_display_serde {
     }
 }
 
-// A pair uniquely identifying Zenith instance.
-#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)]
-pub struct ZTenantTimelineId {
-    pub tenant_id: ZTenantId,
-    pub timeline_id: ZTimelineId,
-}
-
-impl ZTenantTimelineId {
-    pub fn new(tenant_id: ZTenantId, timeline_id: ZTimelineId) -> Self {
-        ZTenantTimelineId {
-            tenant_id,
-            timeline_id,
-        }
-    }
-
-    pub fn generate() -> Self {
-        Self::new(ZTenantId::generate(), ZTimelineId::generate())
-    }
-}
-
-impl fmt::Display for ZTenantTimelineId {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}-{}", self.tenant_id, self.timeline_id)
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use std::fmt::Display;