WIP: change duplicate_tenant.py script to use the pagectl command

The script works but at restart, we detach the created tenants because
they're not known to the attachment service:

  Detaching tenant, control plane omitted it in re-attach response tenant_id=1e399d390e3aee6b11c701cbc716bb6c

=> figure out how to further integrate this

test_runner/duplicate_tenant.py

@@ -1,43 +1,69 @@
 # Usage from top of repo:
-#  poetry run python3 test_runner/duplicate_tenant.py b97965931096047b2d54958756baee7b 10
-from queue import Queue
-import sys
-import threading
+#  poetry run python3 ./test_runner/duplicate_tenant.py c66e2e233057f7f05563caff664ecb14 .neon/remote_storage_local_fs
+from pathlib import Path
+import shutil
+import subprocess
+import time
 
-import requests
 from fixtures.pageserver.http import PageserverHttpClient
 from fixtures.types import TenantId
+import argparse
 
-initial_tenant = sys.argv[1]
-ncopies = int(sys.argv[2])
-numthreads = int(sys.argv[3])
+parser = argparse.ArgumentParser(description="Duplicate tenant script.")
+parser.add_argument("initial_tenant", type=str, help="Initial tenant")
+parser.add_argument("remote_storage_local_fs_root", type=Path, help="Remote storage local fs root")
+parser.add_argument("--ncopies", type=int, help="Number of copies")
+parser.add_argument("--numthreads", type=int, default=1, help="Number of threads")
+parser.add_argument("--port", type=int, default=9898, help="Pageserver management api port")
 
+args = parser.parse_args()
 
-# class DuckTypedNeonEnv:
-#     pass
+initial_tenant = args.initial_tenant
+remote_storage_local_fs_root: Path = args.remote_storage_local_fs_root
+ncopies = args.ncopies
+numthreads = args.numthreads
 
+new_tenant = TenantId.generate()
+print(f"New tenant: {new_tenant}")
 
-# cli = NeonCli(DuckTypedNeonEnv())
+client = PageserverHttpClient(args.port, lambda: None)
 
-q = Queue()
-for i in range(0, ncopies):
-    q.put(i)
+src_tenant_gen = int(client.tenant_status(initial_tenant)["generation"])
 
-for i in range(0, numthreads):
-    q.put(None)
+assert remote_storage_local_fs_root.is_dir(), f"{remote_storage_local_fs_root} is not a directory"
 
+src_timelines_dir: Path = remote_storage_local_fs_root / "tenants" / initial_tenant / "timelines"
+assert src_timelines_dir.is_dir(), f"{src_timelines_dir} is not a directory"
 
-def create():
-    while True:
-        if q.get() == None:
-            break
-        new_tenant = TenantId.generate()
-        res = requests.post(
-            f"http://localhost:9898/v1/tenant/{initial_tenant}/duplicate",
-            json={"new_tenant_id": str(new_tenant)},
-        )
-        res.raise_for_status()
+dst_timelines_dir: Path = remote_storage_local_fs_root / "tenants" / str(new_tenant) / "timelines"
+dst_timelines_dir.parent.mkdir(parents=False, exist_ok=False)
+dst_timelines_dir.mkdir(parents=False, exist_ok=False)
 
+for tl in src_timelines_dir.iterdir():
+    src_tl_dir = src_timelines_dir / tl.name
+    assert src_tl_dir.is_dir(), f"{src_tl_dir} is not a directory"
+    dst_tl_dir = dst_timelines_dir / tl.name
+    dst_tl_dir.mkdir(parents=False, exist_ok=False)
+    for file in tl.iterdir():
+        shutil.copy2(file, dst_tl_dir)
+        if "__" in file.name:
+            cmd = [
+                "./target/debug/pagectl",  # TODO: abstract this like the other binaries
+                "layer",
+                "rewrite-summary",
+                str(dst_tl_dir / file.name),
+                "--new-tenant-id",
+                str(new_tenant),
+            ]
+            subprocess.run(cmd, check=True)
 
-for i in range(0, numthreads):
-    threading.Thread(target=create).start()
+client.tenant_attach(new_tenant, generation=src_tenant_gen)
+
+while True:
+    status = client.tenant_status(new_tenant)
+    if status["state"]["slug"] == "Active":
+        break
+    print("Waiting for tenant to be active..., is: " + status["state"]["slug"])
+    time.sleep(1)
+
+print("Tenant is active: " + str(new_tenant))

test_runner/fixtures/pageserver/http.py

@@ -58,6 +58,7 @@ class HistoricLayerInfo:
     lsn_start: str
     lsn_end: Optional[str]
     remote: bool
+    remote_path: Optional[str] = None
 
     @classmethod
     def from_json(cls, d: Dict[str, Any]) -> HistoricLayerInfo:
@@ -68,6 +69,7 @@ class HistoricLayerInfo:
             lsn_start=d["lsn_start"],
             lsn_end=d.get("lsn_end"),
             remote=d["remote"],
+            remote_path=d.get("remote_path"),
         )