Fix branch creation at a point other than end-of-WAL

When creating a new branch, we copied all WAL from the source timeline
to the new one, and it was being picked up and digested into the
repository on first use of the timeline. Fix by copying the WAL only
up to the branch's starting point.

We should probably move the branch-creation code from the CLI to page
server itself - that's what I was starting to hack on when I noticed this
bug - but let's fix this first.

Add a regression test. To test multiple branches, enhance the python
test fixture to manage multiple running Postgres instances. Also, for
convenience, add a function to the postgres fixture to open a connection
to the server with psycopg2.

control_plane/src/local_env.rs

@@ -9,6 +9,8 @@ use bytes::Bytes;
 use rand::Rng;
 use std::env;
 use std::fs;
+use std::fs::File;
+use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 
@@ -313,17 +315,63 @@ pub fn create_branch(
     let oldtimelinedir = repopath
         .join("timelines")
         .join(startpoint.timelineid.to_string());
-    let mut copy_opts = fs_extra::dir::CopyOptions::new();
-    copy_opts.content_only = true;
-    fs_extra::dir::copy(
-        oldtimelinedir.join("wal"),
-        newtimelinedir.join("wal"),
-        &copy_opts,
+    copy_wal(
+        &oldtimelinedir.join("wal"),
+        &newtimelinedir.join("wal"),
+        startpoint.lsn,
+        16 * 1024 * 1024 // FIXME: assume default WAL segment size
     )?;
 
     Ok(())
 }
 
+///
+/// Copy all WAL segments from one directory to another, up to given LSN.
+///
+/// If the given LSN is in the middle of a segment, the last segment containing it
+/// is written out as .partial, and padded with zeros.
+///
+fn copy_wal(src_dir: &Path, dst_dir: &Path, upto: Lsn, wal_seg_size: u64) -> Result<()>{
+
+    let last_segno = upto.segment_number(wal_seg_size);
+    let last_segoff = upto.segment_offset(wal_seg_size);
+
+    for entry in fs::read_dir(src_dir).unwrap() {
+        if let Ok(entry) = entry {
+            let entry_name = entry.file_name();
+            let fname = entry_name.to_str().unwrap();
+
+            // Check if the filename looks like an xlog file, or a .partial file.
+            if !xlog_utils::IsXLogFileName(fname) && !xlog_utils::IsPartialXLogFileName(fname) {
+                continue
+            }
+            let (segno, _tli) = xlog_utils::XLogFromFileName(fname, wal_seg_size as usize);
+
+            let copylen;
+            let mut dst_fname = PathBuf::from(fname);
+            if segno > last_segno {
+                // future segment, skip
+                continue;
+            } else if segno < last_segno {
+                copylen = wal_seg_size;
+                dst_fname.set_extension("");
+            } else {
+                copylen = last_segoff;
+                dst_fname.set_extension("partial");
+            }
+
+            let src_file = File::open(entry.path())?;
+            let mut dst_file = File::create(dst_dir.join(&dst_fname))?;
+            std::io::copy(&mut src_file.take(copylen), &mut dst_file)?;
+
+            if copylen < wal_seg_size {
+                std::io::copy(&mut std::io::repeat(0).take(wal_seg_size - copylen), &mut dst_file)?;
+            }
+        }
+    }
+    Ok(())
+}
+
 // Find the end of valid WAL in a wal directory
 pub fn find_end_of_wal(local_env: &LocalEnv, timeline: ZTimelineId) -> Result<Lsn> {
     let repopath = &local_env.repo_path;