mirror of
https://github.com/neondatabase/neon.git
synced 2026-05-15 20:20:38 +00:00
bc7a59fbb4
Use the script like so, against the tenant to duplicate:
poetry run python3 ./test_runner/duplicate_tenant.py 7ea51af32d42bfe7fb93bf5f28114d09 200 8
backup of pageserver.toml
d =1
pg_distrib_dir ='/home/admin/neon-main/pg_install'
http_auth_type ='Trust'
pg_auth_type ='Trust'
listen_http_addr ='127.0.0.1:9898'
listen_pg_addr ='127.0.0.1:64000'
broker_endpoint ='http://127.0.0.1:50051/'
#control_plane_api ='http://127.0.0.1:1234/'
# Initial configuration file created by 'pageserver --init'
#listen_pg_addr = '127.0.0.1:64000'
#listen_http_addr = '127.0.0.1:9898'
#wait_lsn_timeout = '60 s'
#wal_redo_timeout = '60 s'
#max_file_descriptors = 10000
#page_cache_size = 160000
# initial superuser role name to use when creating a new tenant
#initial_superuser_name = 'cloud_admin'
#broker_endpoint = 'http://127.0.0.1:50051'
#log_format = 'plain'
#concurrent_tenant_size_logical_size_queries = '1'
#metric_collection_interval = '10 min'
#cached_metric_collection_interval = '0s'
#synthetic_size_calculation_interval = '10 min'
#disk_usage_based_eviction = { max_usage_pct = .., min_avail_bytes = .., period = "10s"}
#background_task_maximum_delay = '10s'
[tenant_config]
#checkpoint_distance = 268435456 # in bytes
#checkpoint_timeout = 10 m
#compaction_target_size = 134217728 # in bytes
#compaction_period = '20 s'
#compaction_threshold = 10
#gc_period = '1 hr'
#gc_horizon = 67108864
#image_creation_threshold = 3
#pitr_interval = '7 days'
#min_resident_size_override = .. # in bytes
#evictions_low_residence_duration_metric_threshold = '24 hour'
#gc_feedback = false
# make it determinsitic
gc_period = '0s'
checkpoint_timeout = '3650 day'
compaction_period = '20 s'
compaction_threshold = 10
compaction_target_size = 134217728
checkpoint_distance = 268435456
image_creation_threshold = 3
[remote_storage]
local_path = '/home/admin/neon-main/bench_repo_dir/repo/remote_storage_local_fs'
remove http handler
switch to generalized rewrite_summary & impl page_ctl subcommand to use it
WIP: change duplicate_tenant.py script to use the pagectl command
The script works but at restart, we detach the created tenants because
they're not known to the attachment service:
Detaching tenant, control plane omitted it in re-attach response tenant_id=1e399d390e3aee6b11c701cbc716bb6c
=> figure out how to further integrate this
70 lines
2.6 KiB
Python
70 lines
2.6 KiB
Python
# Usage from top of repo:
|
|
# poetry run python3 ./test_runner/duplicate_tenant.py c66e2e233057f7f05563caff664ecb14 .neon/remote_storage_local_fs
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
|
|
from fixtures.pageserver.http import PageserverHttpClient
|
|
from fixtures.types import TenantId
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(description="Duplicate tenant script.")
|
|
parser.add_argument("initial_tenant", type=str, help="Initial tenant")
|
|
parser.add_argument("remote_storage_local_fs_root", type=Path, help="Remote storage local fs root")
|
|
parser.add_argument("--ncopies", type=int, help="Number of copies")
|
|
parser.add_argument("--numthreads", type=int, default=1, help="Number of threads")
|
|
parser.add_argument("--port", type=int, default=9898, help="Pageserver management api port")
|
|
|
|
args = parser.parse_args()
|
|
|
|
initial_tenant = args.initial_tenant
|
|
remote_storage_local_fs_root: Path = args.remote_storage_local_fs_root
|
|
ncopies = args.ncopies
|
|
numthreads = args.numthreads
|
|
|
|
new_tenant = TenantId.generate()
|
|
print(f"New tenant: {new_tenant}")
|
|
|
|
client = PageserverHttpClient(args.port, lambda: None)
|
|
|
|
src_tenant_gen = int(client.tenant_status(initial_tenant)["generation"])
|
|
|
|
assert remote_storage_local_fs_root.is_dir(), f"{remote_storage_local_fs_root} is not a directory"
|
|
|
|
src_timelines_dir: Path = remote_storage_local_fs_root / "tenants" / initial_tenant / "timelines"
|
|
assert src_timelines_dir.is_dir(), f"{src_timelines_dir} is not a directory"
|
|
|
|
dst_timelines_dir: Path = remote_storage_local_fs_root / "tenants" / str(new_tenant) / "timelines"
|
|
dst_timelines_dir.parent.mkdir(parents=False, exist_ok=False)
|
|
dst_timelines_dir.mkdir(parents=False, exist_ok=False)
|
|
|
|
for tl in src_timelines_dir.iterdir():
|
|
src_tl_dir = src_timelines_dir / tl.name
|
|
assert src_tl_dir.is_dir(), f"{src_tl_dir} is not a directory"
|
|
dst_tl_dir = dst_timelines_dir / tl.name
|
|
dst_tl_dir.mkdir(parents=False, exist_ok=False)
|
|
for file in tl.iterdir():
|
|
shutil.copy2(file, dst_tl_dir)
|
|
if "__" in file.name:
|
|
cmd = [
|
|
"./target/debug/pagectl", # TODO: abstract this like the other binaries
|
|
"layer",
|
|
"rewrite-summary",
|
|
str(dst_tl_dir / file.name),
|
|
"--new-tenant-id",
|
|
str(new_tenant),
|
|
]
|
|
subprocess.run(cmd, check=True)
|
|
|
|
client.tenant_attach(new_tenant, generation=src_tenant_gen)
|
|
|
|
while True:
|
|
status = client.tenant_status(new_tenant)
|
|
if status["state"]["slug"] == "Active":
|
|
break
|
|
print("Waiting for tenant to be active..., is: " + status["state"]["slug"])
|
|
time.sleep(1)
|
|
|
|
print("Tenant is active: " + str(new_tenant))
|