Integrate compute_tools into zenith workspace and improve logging (zenithdb/console#487)

.circleci/config.yml

@@ -4,6 +4,7 @@ executors:
   zenith-build-executor:
     resource_class: xlarge
     docker:
+      # NB: when changed, do not forget to update rust image tag in all Dockerfiles
       - image: cimg/rust:1.56.1
   zenith-python-executor:
     docker:
@@ -450,7 +451,7 @@ jobs:
           name: Build and push compute-tools Docker image
           command: |
             echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
-            docker build -t zenithdb/compute-tools:latest ./compute_tools/
+            docker build -t zenithdb/compute-tools:latest -f Dockerfile.compute-tools .
             docker push zenithdb/compute-tools:latest
       - run:
           name: Init postgres submodule
@@ -571,55 +572,6 @@ jobs:
                 }
               }"
 
-  #
-  #
-  # compute-tools jobs
-  # TODO: unify with main build_and_test pipeline
-  #
-  #
-  compute-tools-test:
-    executor: zenith-build-executor
-    working_directory: ~/repo/compute_tools
-    steps:
-      - checkout:
-          path: ~/repo
-
-      - restore_cache:
-          name: Restore rust cache
-          keys:
-            # Require an exact match. While an out of date cache might speed up the build,
-            # there's no way to clean out old packages, so the cache grows every time something
-            # changes.
-            - v03-rust-cache-deps-debug-{{ checksum "Cargo.lock" }}
-
-      # Build the rust code, including test binaries
-      - run:
-          name: Rust build
-          environment:
-            CARGO_INCREMENTAL: 0
-          command: cargo build --bins --tests
-
-      - save_cache:
-          name: Save rust cache
-          key: v03-rust-cache-deps-debug-{{ checksum "Cargo.lock" }}
-          paths:
-            - ~/.cargo/registry
-            - ~/.cargo/git
-            - target
-
-      # Run Rust formatting checks
-      - run:
-          name: cargo fmt check
-          command: cargo fmt --all -- --check
-
-      # Run Rust linter (clippy)
-      - run:
-          name: cargo clippy check
-          command: cargo clippy --all --all-targets -- -Dwarnings -Drust-2018-idioms
-
-      # Run Rust integration and unittests
-      - run: cargo test
-
 workflows:
   build_and_test:
     jobs:
@@ -668,7 +620,6 @@ workflows:
           requires:
             # TODO: consider adding more
             - other-tests-debug
-      - compute-tools-test
       - docker-image:
           # Context gives an ability to login
           context: Docker Hub
@@ -691,7 +642,6 @@ workflows:
           requires:
             - pg_regress-tests-release
             - other-tests-release
-            - compute-tools-test
       - deploy-staging:
           # Context gives an ability to login
           context: Docker Hub

Cargo.lock

@@ -364,6 +364,25 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "compute_tools"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "clap",
+ "env_logger",
+ "hyper",
+ "libc",
+ "log",
+ "postgres",
+ "regex",
+ "serde",
+ "serde_json",
+ "tar",
+ "tokio",
+]
+
 [[package]]
 name = "const_format"
 version = "0.2.22"
@@ -1021,6 +1040,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
 dependencies = [
  "cfg-if",
+ "serde",
 ]
 
 [[package]]

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
+    "compute_tools",
     "control_plane",
     "pageserver",
     "postgres_ffi",

Dockerfile.build

@@ -2,8 +2,9 @@
 # Image with all the required dependencies to build https://github.com/zenithdb/zenith
 # and Postgres from https://github.com/zenithdb/postgres
 # Also includes some rust development and build tools.
+# NB: keep in sync with rust image version in .circle/config.yml
 #
-FROM rust:slim-buster
+FROM rust:1.56.1-slim-buster
 WORKDIR /zenith
 
 # Install postgres and zenith build dependencies

Dockerfile.compute-tools

@@ -0,0 +1,14 @@
+# First transient image to build compute_tools binaries
+# NB: keep in sync with rust image version in .circle/config.yml
+FROM rust:1.56.1-slim-buster AS rust-build
+
+WORKDIR /zenith
+
+COPY . .
+
+RUN cargo build -p compute_tools --release
+
+# Final image that only has one binary
+FROM debian:buster-slim
+
+COPY --from=rust-build /zenith/target/release/zenith_ctl /usr/local/bin/zenith_ctl

compute_tools/Cargo.toml

@@ -6,9 +6,6 @@ edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
-[workspace]
-# TODO: make it a part of global zenith worksapce
-
 [dependencies]
 libc = "0.2"
 anyhow = "1.0"
@@ -17,12 +14,9 @@ clap = "2.33"
 env_logger = "0.8"
 hyper = { version = "0.14", features = ["full"] }
 log = { version = "0.4", features = ["std", "serde"] }
-postgres = "0.19"
+postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" }
 regex = "1"
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
+serde_json = "1"
 tar = "0.4"
-tokio = { version = "1", features = ["full"] }
-
-[profile.release]
-debug = true
+tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] }

compute_tools/Dockerfile

@@ -1,14 +0,0 @@
-# First transient image to build compute_tools binaries
-FROM rust:slim-buster AS rust-build
-
-RUN mkdir /compute_tools
-WORKDIR /compute_tools
-
-COPY . /compute_tools/
-
-RUN cargo build --release
-
-# Final image that only has one binary
-FROM debian:buster-slim
-
-COPY --from=rust-build /compute_tools/target/release/zenith_ctl /usr/local/bin/zenith_ctl

compute_tools/src/bin/zenith_ctl.rs

@@ -27,12 +27,12 @@
 //! ```
 //!
 use std::fs::File;
+use std::panic;
 use std::path::Path;
 use std::process::{exit, Command, ExitStatus};
 use std::sync::{Arc, RwLock};
-use std::{env, panic};
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use chrono::Utc;
 use libc::{prctl, PR_SET_PDEATHSIG, SIGINT};
 use log::info;
@@ -70,7 +70,7 @@ fn prepare_pgdata(state: &Arc<RwLock<ComputeState>>) -> Result<()> {
         .expect("tenant id should be provided");
 
     info!(
-        "applying spec for cluster #{}, operation #{}",
+        "starting cluster #{}, operation #{}",
         spec.cluster.cluster_id,
         spec.operation_uuid.as_ref().unwrap()
     );
@@ -80,10 +80,23 @@ fn prepare_pgdata(state: &Arc<RwLock<ComputeState>>) -> Result<()> {
     config::write_postgres_conf(&pgdata_path.join("postgresql.conf"), spec)?;
 
     info!("starting safekeepers syncing");
-    let lsn = sync_safekeepers(&state.pgdata, &state.pgbin)?;
+    let lsn = sync_safekeepers(&state.pgdata, &state.pgbin)
+        .with_context(|| "failed to sync safekeepers")?;
     info!("safekeepers synced at LSN {}", lsn);
 
-    get_basebackup(&state.pgdata, &pageserver_connstr, &tenant, &timeline, &lsn)?;
+    info!(
+        "getting basebackup@{} from pageserver {}",
+        lsn, pageserver_connstr
+    );
+    get_basebackup(&state.pgdata, &pageserver_connstr, &tenant, &timeline, &lsn).with_context(
+        || {
+            format!(
+                "failed to get basebackup@{} from pageserver {}",
+                lsn, pageserver_connstr
+            )
+        },
+    )?;
+
     // Update pg_hba.conf received with basebackup.
     update_pg_hba(pgdata_path)?;
 
@@ -149,6 +162,9 @@ fn main() -> Result<()> {
     // This does not matter much for Docker, where `zenith_ctl` is an entrypoint,
     // so the whole container will exit if it exits. But could be useful when
     // `zenith_ctl` is used in e.g. systemd.
+    // XXX: this appears to just don't work. When `main` exits, the child process
+    // `postgres` is re-assigned to a new parent (`/lib/systemd/systemd --user`
+    // in my case).
     unsafe {
         prctl(PR_SET_PDEATHSIG, SIGINT);
     }
@@ -156,8 +172,10 @@ fn main() -> Result<()> {
     // TODO: re-use `zenith_utils::logging` later
     init_logger(DEFAULT_LOG_LEVEL)?;
 
+    // Env variable is set by `cargo`
+    let version: Option<&str> = option_env!("CARGO_PKG_VERSION");
     let matches = clap::App::new("zenith_ctl")
-        .version("0.1.0")
+        .version(version.unwrap_or("unknown"))
         .arg(
             clap::Arg::with_name("connstr")
                 .short("C")
@@ -212,13 +230,7 @@ fn main() -> Result<()> {
                 let file = File::open(path)?;
                 serde_json::from_reader(file)?
             } else {
-                // Finally, try to fetch it from the env
-                // XXX: not tested well and kept as a backup option for k8s, Docker, etc.
-                // TODO: remove later
-                match env::var("CLUSTER_SPEC") {
-                    Ok(json) => serde_json::from_str(&json)?,
-                    Err(_) => panic!("cluster spec should be provided via --spec, --spec-path or env variable CLUSTER_SPEC")
-                }
+                panic!("cluster spec should be provided via --spec or --spec-path argument");
             }
         }
     };

compute_tools/src/zenith.rs

@@ -87,17 +87,19 @@ pub fn sync_safekeepers(pgdata: &str, pgbin: &str) -> Result<String> {
         .args(&["--sync-safekeepers"])
         .env("PGDATA", &pgdata) // we cannot use -D in this mode
         .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
         .spawn()
         .expect("postgres --sync-safekeepers failed to start");
 
+    // `postgres --sync-safekeepers` will print all log output to stderr and
+    // final LSN to stdout. So we pipe only stdout, while stderr will be automatically
+    // redirected to the caller output.
     let sync_output = sync_handle
         .wait_with_output()
         .expect("postgres --sync-safekeepers failed");
     if !sync_output.status.success() {
         anyhow::bail!(
-            "postgres --sync-safekeepers failed: '{}'",
-            String::from_utf8_lossy(&sync_output.stderr)
+            "postgres --sync-safekeepers exited with non-zero status: {}",
+            sync_output.status,
         );
     }