crash-safe and resumable tenant attach

This change introduces a marker file

  $repo/tenants/$tenant_id/attaching

that is present while a tenant is in Attaching state.

When pageserver restarts, we use it to resume the tenant attach operation.
Before this change, a crash during tenant attach would result in one of
the following:
1. crash upon restart due to missing metadata file (IIRC)
2. "successful" loading of the tenant with a subset of timelines

libs/utils/src/fs_ext.rs

@@ -0,0 +1,45 @@
+/// Extensions to `std::fs` types.
+use std::{fs, io, path::Path};
+
+pub trait PathExt {
+    /// Returns an error if `self` is not a directory.
+    fn is_empty_dir(&self) -> io::Result<bool>;
+}
+
+impl<P> PathExt for P
+where
+    P: AsRef<Path>,
+{
+    fn is_empty_dir(&self) -> io::Result<bool> {
+        Ok(fs::read_dir(self)?.into_iter().next().is_none())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::path::PathBuf;
+
+    #[test]
+    fn is_empty_dir() {
+        use super::PathExt;
+
+        let dir = tempfile::tempdir().unwrap();
+        let dir_path = dir.path();
+
+        // test positive case
+        assert!(
+            dir_path.is_empty_dir().expect("test failure"),
+            "new tempdir should be empty"
+        );
+
+        // invoke on a file to ensure it returns an error
+        let file_path: PathBuf = dir_path.join("testfile");
+        let f = std::fs::File::create(&file_path).unwrap();
+        drop(f);
+        assert!(file_path.is_empty_dir().is_err());
+
+        // do it again on a path, we know to be nonexistent
+        std::fs::remove_file(&file_path).unwrap();
+        assert!(file_path.is_empty_dir().is_err());
+    }
+}

libs/utils/src/lib.rs

@@ -48,6 +48,8 @@ pub mod nonblock;
 // Default signal handling
 pub mod signals;
 
+pub mod fs_ext;
+
 /// This is a shortcut to embed git sha into binaries and avoid copying the same build script to all packages
 ///
 /// we have several cases: