WIP duplicate based on LayerMap

pageserver/src/tenant.rs

@@ -82,6 +82,8 @@ use crate::tenant::storage_layer::DeltaLayer;
 use crate::tenant::storage_layer::ImageLayer;
 use crate::InitializationOrder;
 
+use crate::tenant::storage_layer::Layer;
+use crate::tenant::storage_layer::ResidentLayer;
 use crate::tenant::timeline::delete::DeleteTimelineFlow;
 use crate::tenant::timeline::uninit::cleanup_timeline_directory;
 use crate::virtual_file::VirtualFile;
@@ -202,6 +204,7 @@ pub(crate) struct TenantPreload {
 pub(crate) enum SpawnMode {
     Normal,
     Create,
+    Duplicate { src_tenant: Arc<Tenant> },
 }
 
 ///
@@ -579,6 +582,33 @@ impl Tenant {
                     .as_mut()
                     .and_then(|x| x.initial_tenant_load_remote.take());
 
+                // if let SpawnMode::Duplicate { src_tenant } = mode {
+                //     let mut layer_guards: HashMap<TimelineId, _> =HashMap::new();
+                //     for (tlid, tl) in src_tenant.timelines.lock().unwrap().iter() {
+                //         // TODO: ensure nobody tries to delete the timeline
+                //         let Ok(guard) = (Arc::clone(&tl)).layers.try_read_owned() else {
+                //             make_broken(&tenant_clone, anyhow::anyhow!("duplicate: failed to lock timeline layers"));
+                //             return Ok(());
+                //         };
+                //         let replace = layer_guards.insert(tlid, guard);
+                //         assert!(replace.is_none(), "duplicate: duplicate timeline id");
+                //     }
+                //     // duplicate each timeline's layers and construct new index parts with this tenant's generation
+                //     for (tlid, layers) in layer_guards {
+
+                //         // TODO: implement a mapping of timeline ids
+
+
+                //         for desc in layers.layer_map().iter_historic_layers() {
+                //             let layer: Layer = layers.get_from_desc(&desc);
+                //             let resident_layer: ResidentLayer = layer.download_and_keep_resident().await?;
+                //             tokio::fs::copy(from, to)
+                //             resident_layer.local_path();
+                //         }
+                //     }
+
+                // }
+
                 let preload = match mode {
                     SpawnMode::Create => {None},
                     SpawnMode::Normal => {
@@ -1601,6 +1631,24 @@ impl Tenant {
         Ok(loaded_timeline)
     }
 
+    pub(crate) fn copy_timelines_from_tenant(&self, src_tenant: Arc<Tenant>, ctx: &RequestContext) -> anyhow::Result<()> {
+        let mut src_timelines: HashMap<TimelineId, (Arc<Timeline>, _)> = HashMap::new();
+        for (tlid, tl) in src_tenant.timelines.lock().unwrap().iter() {
+            // TODO: ensure nobody tries to delete the timeline
+            let Ok(guard) = (Arc::clone(&tl)).layers.try_read_owned() else {
+                make_broken(&tenant_clone, anyhow::anyhow!("duplicate: failed to lock timeline layers"));
+                return Ok(());
+            };
+            let replace = src_timelines.insert(tlid, (Arc::clone(tl), guard));
+            assert!(replace.is_none(), "duplicate: duplicate timeline id");
+        }
+        for (tlid, (tl, layers)) in src_timelines {
+            let utl = self.create_empty_timeline(tlid, tl.initdb_lsn, tl.pg_version, ctx).await?;
+            utl.copy_existing_timeline(tl, layers, ctx).await?;
+
+        }
+    }
+
     /// perform one garbage collection iteration, removing old data files from disk.
     /// this function is periodically called by gc task.
     /// also it can be explicitly requested through page server api 'do_gc' command.

pageserver/src/tenant/mgr.rs

@@ -741,7 +741,6 @@ pub(crate) async fn create_tenant(
     Ok(created_tenant)
 }
 
-#[allow(clippy::too_many_arguments)]
 pub(crate) async fn duplicate_tenant(
     conf: &'static PageServerConf,
     tenant_conf: TenantConfOpt,
@@ -751,6 +750,40 @@ pub(crate) async fn duplicate_tenant(
     resources: TenantSharedResources,
     ctx: &RequestContext,
     cancel: &CancellationToken,
+) -> Result<(), TenantMapInsertError> {
+    let src_tenant = get_tenant(src_tenant_id, true).context("get src tenant")?;
+    // TODO somehow ensure that `src_tenant` can't go away in the meantime.
+
+    let slot_guard = tenant_map_acquire_slot(&new_tenant_id, TenantSlotAcquireMode::MustNotExist)?;
+
+    let location_conf = LocationConf::attached_single(tenant_conf, generation);
+    let tenant_path = super::create_tenant_files(conf, &location_conf, &new_tenant_id).await?;
+
+    let new_tenant = Tenant::spawn(
+        conf,
+        new_tenant_id,
+        resources,
+        AttachedTenantConf::try_from(location_conf)?,
+        None,
+        &TENANTS,
+        SpawnMode::Duplicate { src_tenant },
+        ctx,
+    )?;
+
+    slot_guard.upsert(TenantSlot::Attached(new_tenant))?;
+
+    Ok(())
+}
+#[allow(clippy::too_many_arguments)]
+pub(crate) async fn duplicate_tenant_old(
+    conf: &'static PageServerConf,
+    tenant_conf: TenantConfOpt,
+    src_tenant_id: TenantId,
+    new_tenant_id: TenantId,
+    generation: Generation,
+    resources: TenantSharedResources,
+    ctx: &RequestContext,
+    cancel: &CancellationToken,
 ) -> Result<(), TenantMapInsertError> {
     debug_assert_current_span_has_tenant_id();
 

pageserver/src/tenant/storage_layer/layer.rs

@@ -1254,6 +1254,18 @@ impl DownloadedLayer {
 
         Ok(())
     }
+
+    async fn duplicate(
+        &self,
+        owner: &Arc<LayerInner>,
+        ctx: &RequestContext,
+    ) -> anyhow::Result<ResidentLayer> {
+        use LayerKind::*;
+        match self.get(owner, ctx).await? {
+            Delta(d) => d.duplicate(ctx).await,
+            Image(i) => i.duplicate(ctx).await,
+        }
+    }
 }
 
 /// Wrapper around an actual layer implementation.

pageserver/src/tenant/timeline/uninit.rs

@@ -2,10 +2,11 @@ use std::{collections::hash_map::Entry, fs, sync::Arc};
 
 use anyhow::Context;
 use camino::Utf8PathBuf;
+use tokio::sync::OwnedRwLockReadGuard;
 use tracing::{error, info, info_span, warn};
 use utils::{crashsafe, fs_ext, id::TimelineId, lsn::Lsn};
 
-use crate::{context::RequestContext, import_datadir, tenant::Tenant};
+use crate::{context::RequestContext, import_datadir, tenant::{Tenant, storage_layer::ResidentLayer}};
 
 use super::Timeline;
 
@@ -124,6 +125,27 @@ impl<'t> UninitializedTimeline<'t> {
             })?
             .0)
     }
+
+    pub(crate) fn copy_existing_timeline(
+        self,
+        src_timeline: Arc<Timeline>,
+        locked_layers: OwnedRwLockReadGuard<LayerManager>,
+    ) -> anyhow::Result<Arc<Timeline>> {
+        let layers = locked_layers;
+
+        let raw_timeline = self.raw_timeline()?;
+
+        for desc in layers.layer_map().iter_historic_layers() {
+            let layer: Layer = layers.get_from_desc(&desc);
+            let resident_layer: ResidentLayer = layer.download_and_keep_resident().await?;
+
+            tokio::fs::copy(from, to)
+            resident_layer.local_path();
+        }
+
+        raw_timeline.copy_existing_timeline(existing_timeline);
+        self.finish_creation()
+    }
 }
 
 impl Drop for UninitializedTimeline<'_> {