diff --git a/Dockerfile b/Dockerfile index 69402919ec..cb4e213687 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,7 @@ COPY . . # Show build caching stats to check if it was used in the end. # Has to be the part of the same RUN since cachepot daemon is killed in the end of this RUN, losing the compilation stats. RUN set -e \ -&& mold -run cargo build --bin pageserver --bin safekeeper --bin proxy --locked --release \ +&& mold -run cargo build --bin pageserver --bin pageserver_binutils --bin safekeeper --bin proxy --locked --release \ && cachepot -s # Build final image @@ -63,9 +63,10 @@ RUN set -e \ && useradd -d /data neon \ && chown -R neon:neon /data -COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver /usr/local/bin -COPY --from=build --chown=neon:neon /home/nonroot/target/release/safekeeper /usr/local/bin -COPY --from=build --chown=neon:neon /home/nonroot/target/release/proxy /usr/local/bin +COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver /usr/local/bin +COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver_binutils /usr/local/bin +COPY --from=build --chown=neon:neon /home/nonroot/target/release/safekeeper /usr/local/bin +COPY --from=build --chown=neon:neon /home/nonroot/target/release/proxy /usr/local/bin COPY --from=pg-build /home/nonroot/pg_install/v14 /usr/local/v14/ COPY --from=pg-build /home/nonroot/pg_install/v15 /usr/local/v15/ @@ -85,4 +86,3 @@ VOLUME ["/data"] USER neon EXPOSE 6400 EXPOSE 9898 -CMD ["/bin/bash"] diff --git a/pageserver/src/bin/dump_layerfile.rs b/pageserver/src/bin/dump_layerfile.rs deleted file mode 100644 index f5247ee609..0000000000 --- a/pageserver/src/bin/dump_layerfile.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Main entry point for the dump_layerfile executable -//! -//! A handy tool for debugging, that's all. -use anyhow::Result; -use clap::{App, Arg}; -use pageserver::page_cache; -use pageserver::tenant::dump_layerfile_from_path; -use pageserver::virtual_file; -use std::path::PathBuf; -use utils::project_git_version; - -project_git_version!(GIT_VERSION); - -fn main() -> Result<()> { - let arg_matches = App::new("Neon dump_layerfile utility") - .about("Dump contents of one layer file, for debugging") - .version(GIT_VERSION) - .arg( - Arg::new("path") - .help("Path to file to dump") - .required(true) - .index(1), - ) - .get_matches(); - - let path = PathBuf::from(arg_matches.value_of("path").unwrap()); - - // Basic initialization of things that don't change after startup - virtual_file::init(10); - page_cache::init(100); - - dump_layerfile_from_path(&path, true)?; - - Ok(()) -} diff --git a/pageserver/src/bin/pageserver_binutils.rs b/pageserver/src/bin/pageserver_binutils.rs new file mode 100644 index 0000000000..ec7699f194 --- /dev/null +++ b/pageserver/src/bin/pageserver_binutils.rs @@ -0,0 +1,144 @@ +//! A helper tool to manage pageserver binary files. +//! Accepts a file as an argument, attempts to parse it with all ways possible +//! and prints its interpreted context. +//! +//! Separate, `metadata` subcommand allows to print and update pageserver's metadata file. +use std::{ + path::{Path, PathBuf}, + str::FromStr, +}; + +use anyhow::Context; +use clap::{App, Arg}; + +use pageserver::{ + page_cache, + tenant::{dump_layerfile_from_path, metadata::TimelineMetadata}, + virtual_file, +}; +use postgres_ffi::ControlFileData; +use utils::{lsn::Lsn, project_git_version}; + +project_git_version!(GIT_VERSION); + +const METADATA_SUBCOMMAND: &str = "metadata"; + +fn main() -> anyhow::Result<()> { + let arg_matches = App::new("Neon Pageserver binutils") + .about("Reads pageserver (and related) binary files management utility") + .version(GIT_VERSION) + .arg(Arg::new("path").help("Input file path").required(false)) + .subcommand( + App::new(METADATA_SUBCOMMAND) + .about("Read and update pageserver metadata file") + .arg( + Arg::new("metadata_path") + .help("Input metadata file path") + .required(false), + ) + .arg( + Arg::new("disk_consistent_lsn") + .long("disk_consistent_lsn") + .takes_value(true) + .help("Replace disk consistent Lsn"), + ) + .arg( + Arg::new("prev_record_lsn") + .long("prev_record_lsn") + .takes_value(true) + .help("Replace previous record Lsn"), + ), + ) + .get_matches(); + + match arg_matches.subcommand() { + Some((subcommand_name, subcommand_matches)) => { + let path = PathBuf::from( + subcommand_matches + .value_of("metadata_path") + .context("'metadata_path' argument is missing")?, + ); + anyhow::ensure!( + subcommand_name == METADATA_SUBCOMMAND, + "Unknown subcommand {subcommand_name}" + ); + handle_metadata(&path, subcommand_matches)?; + } + None => { + let path = PathBuf::from( + arg_matches + .value_of("path") + .context("'path' argument is missing")?, + ); + println!( + "No subcommand specified, attempting to guess the format for file {}", + path.display() + ); + if let Err(e) = read_pg_control_file(&path) { + println!( + "Failed to read input file as a pg control one: {e:#}\n\ + Attempting to read it as layer file" + ); + print_layerfile(&path)?; + } + } + }; + Ok(()) +} + +fn read_pg_control_file(control_file_path: &Path) -> anyhow::Result<()> { + let control_file = ControlFileData::decode(&std::fs::read(&control_file_path)?)?; + println!("{control_file:?}"); + let control_file_initdb = Lsn(control_file.checkPoint); + println!( + "pg_initdb_lsn: {}, aligned: {}", + control_file_initdb, + control_file_initdb.align() + ); + Ok(()) +} + +fn print_layerfile(path: &Path) -> anyhow::Result<()> { + // Basic initialization of things that don't change after startup + virtual_file::init(10); + page_cache::init(100); + dump_layerfile_from_path(path, true) +} + +fn handle_metadata(path: &Path, arg_matches: &clap::ArgMatches) -> Result<(), anyhow::Error> { + let metadata_bytes = std::fs::read(&path)?; + let mut meta = TimelineMetadata::from_bytes(&metadata_bytes)?; + println!("Current metadata:\n{meta:?}"); + let mut update_meta = false; + if let Some(disk_consistent_lsn) = arg_matches.value_of("disk_consistent_lsn") { + meta = TimelineMetadata::new( + Lsn::from_str(disk_consistent_lsn)?, + meta.prev_record_lsn(), + meta.ancestor_timeline(), + meta.ancestor_lsn(), + meta.latest_gc_cutoff_lsn(), + meta.initdb_lsn(), + meta.pg_version(), + ); + update_meta = true; + } + if let Some(prev_record_lsn) = arg_matches.value_of("prev_record_lsn") { + meta = TimelineMetadata::new( + meta.disk_consistent_lsn(), + Some(Lsn::from_str(prev_record_lsn)?), + meta.ancestor_timeline(), + meta.ancestor_lsn(), + meta.latest_gc_cutoff_lsn(), + meta.initdb_lsn(), + meta.pg_version(), + ); + update_meta = true; + } + + if update_meta { + let metadata_bytes = meta.to_bytes()?; + std::fs::write(&path, &metadata_bytes)?; + } + + Ok(()) +} diff --git a/pageserver/src/bin/update_metadata.rs b/pageserver/src/bin/update_metadata.rs deleted file mode 100644 index e66049c457..0000000000 --- a/pageserver/src/bin/update_metadata.rs +++ /dev/null @@ -1,75 +0,0 @@ -//! Main entry point for the edit_metadata executable -//! -//! A handy tool for debugging, that's all. -use anyhow::Result; -use clap::{App, Arg}; -use pageserver::tenant::metadata::TimelineMetadata; -use std::path::PathBuf; -use std::str::FromStr; -use utils::{lsn::Lsn, project_git_version}; - -project_git_version!(GIT_VERSION); - -fn main() -> Result<()> { - let arg_matches = App::new("Neon update metadata utility") - .about("Dump or update metadata file") - .version(GIT_VERSION) - .arg( - Arg::new("path") - .help("Path to metadata file") - .required(true), - ) - .arg( - Arg::new("disk_lsn") - .short('d') - .long("disk_lsn") - .takes_value(true) - .help("Replace disk constistent lsn"), - ) - .arg( - Arg::new("prev_lsn") - .short('p') - .long("prev_lsn") - .takes_value(true) - .help("Previous record LSN"), - ) - .get_matches(); - - let path = PathBuf::from(arg_matches.value_of("path").unwrap()); - let metadata_bytes = std::fs::read(&path)?; - let mut meta = TimelineMetadata::from_bytes(&metadata_bytes)?; - println!("Current metadata:\n{:?}", &meta); - - let mut update_meta = false; - - if let Some(disk_lsn) = arg_matches.value_of("disk_lsn") { - meta = TimelineMetadata::new( - Lsn::from_str(disk_lsn)?, - meta.prev_record_lsn(), - meta.ancestor_timeline(), - meta.ancestor_lsn(), - meta.latest_gc_cutoff_lsn(), - meta.initdb_lsn(), - meta.pg_version(), - ); - update_meta = true; - } - - if let Some(prev_lsn) = arg_matches.value_of("prev_lsn") { - meta = TimelineMetadata::new( - meta.disk_consistent_lsn(), - Some(Lsn::from_str(prev_lsn)?), - meta.ancestor_timeline(), - meta.ancestor_lsn(), - meta.latest_gc_cutoff_lsn(), - meta.initdb_lsn(), - meta.pg_version(), - ); - update_meta = true; - } - if update_meta { - let metadata_bytes = meta.to_bytes()?; - std::fs::write(&path, &metadata_bytes)?; - } - Ok(()) -} diff --git a/pageserver/src/tenant/delta_layer.rs b/pageserver/src/tenant/delta_layer.rs index 57c5be91a4..41715ab0a4 100644 --- a/pageserver/src/tenant/delta_layer.rs +++ b/pageserver/src/tenant/delta_layer.rs @@ -556,7 +556,7 @@ impl DeltaLayer { /// Create a DeltaLayer struct representing an existing file on disk. /// - /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary. + /// This variant is only used for debugging purposes, by the 'pageserver_binutils' binary. pub fn new_for_path(path: &Path, file: F) -> Result where F: FileExt, diff --git a/pageserver/src/tenant/filename.rs b/pageserver/src/tenant/filename.rs index 5ebac2332d..0ebf2d479b 100644 --- a/pageserver/src/tenant/filename.rs +++ b/pageserver/src/tenant/filename.rs @@ -177,7 +177,7 @@ impl fmt::Display for ImageFileName { /// /// This is used by DeltaLayer and ImageLayer. Normally, this holds a reference to the /// global config, and paths to layer files are constructed using the tenant/timeline -/// path from the config. But in the 'dump_layerfile' binary, we need to construct a Layer +/// path from the config. But in the 'pageserver_binutils' binary, we need to construct a Layer /// struct for a file on disk, without having a page server running, so that we have no /// config. In that case, we use the Path variant to hold the full path to the file on /// disk. diff --git a/pageserver/src/tenant/image_layer.rs b/pageserver/src/tenant/image_layer.rs index 92bf022fee..cbfa0134b0 100644 --- a/pageserver/src/tenant/image_layer.rs +++ b/pageserver/src/tenant/image_layer.rs @@ -357,7 +357,7 @@ impl ImageLayer { /// Create an ImageLayer struct representing an existing file on disk. /// - /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary. + /// This variant is only used for debugging purposes, by the 'pageserver_binutils' binary. pub fn new_for_path(path: &Path, file: F) -> Result where F: std::os::unix::prelude::FileExt,