rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-08 22:12:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Code review fixes

Browse Source
This commit is contained in:
Kirill Bulatov
2022-01-11 14:36:45 +02:00
committed by Kirill Bulatov
parent 7c4a653230
commit 8ab4c8a050
6 changed files with 69 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
test_runner/batch_others/test_remote_storage.py
View File

@@ -26,64 +26,63 @@ pytest_plugins = ("fixtures.zenith_fixtures")
# * queries the specific data, ensuring that it matches the one stored before
#
def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder):
with tempfile.TemporaryDirectory() as tmp_root_dir:
zenith_env_builder.rust_log_override = 'debug'
zenith_env_builder.pageserver_remote_storage = LocalFsStorage(Path(tmp_root_dir))
zenith_env_builder.num_safekeepers = 1
zenith_env_builder.rust_log_override = 'debug'
zenith_env_builder.num_safekeepers = 1
zenith_env_builder.enable_local_fs_remote_storage()
data_id = 1
data_secret = 'very secret secret'
data_id = 1
data_secret = 'very secret secret'
##### First start, insert secret data and upload it to the remote storage
env = zenith_env_builder.init()
pg = env.postgres.create_start()
##### First start, insert secret data and upload it to the remote storage
env = zenith_env_builder.init()
pg = env.postgres.create_start()
tenant_id = pg.safe_psql("show zenith.zenith_tenant")[0][0]
timeline_id = pg.safe_psql("show zenith.zenith_timeline")[0][0]
tenant_id = pg.safe_psql("show zenith.zenith_tenant")[0][0]
timeline_id = pg.safe_psql("show zenith.zenith_timeline")[0][0]
with closing(pg.connect()) as conn:
with conn.cursor() as cur:
cur.execute(f'''
CREATE TABLE t1(id int primary key, secret text);
INSERT INTO t1 VALUES ({data_id}, '{data_secret}');
''')
with closing(pg.connect()) as conn:
with conn.cursor() as cur:
cur.execute(f'''
CREATE TABLE t1(id int primary key, secret text);
INSERT INTO t1 VALUES ({data_id}, '{data_secret}');
''')
# run checkpoint manually to be sure that data landed in remote storage
with closing(env.pageserver.connect()) as psconn:
with psconn.cursor() as pscur:
pscur.execute(f"do_gc {tenant_id} {timeline_id}")
log.info("waiting for upload") # TODO api to check if upload is done
time.sleep(2)
# run checkpoint manually to be sure that data landed in remote storage
with closing(env.pageserver.connect()) as psconn:
with psconn.cursor() as pscur:
pscur.execute(f"do_gc {tenant_id} {timeline_id}")
log.info("waiting for upload") # TODO api to check if upload is done
time.sleep(2)
##### Stop the first pageserver instance, erase all its data
env.postgres.stop_all()
env.pageserver.stop()
##### Stop the first pageserver instance, erase all its data
env.postgres.stop_all()
env.pageserver.stop()
dir_to_clear = Path(env.repo_dir) / 'tenants'
shutil.rmtree(dir_to_clear)
os.mkdir(dir_to_clear)
dir_to_clear = Path(env.repo_dir) / 'tenants'
shutil.rmtree(dir_to_clear)
os.mkdir(dir_to_clear)
##### Second start, restore the data and ensure it's the same
env.pageserver.start()
##### Second start, restore the data and ensure it's the same
env.pageserver.start()
log.info("waiting for timeline redownload")
client = env.pageserver.http_client()
attempts = 0
while True:
timeline_details = client.timeline_details(tenant_id, timeline_id)
assert timeline_details['timeline_id'] == timeline_id
assert timeline_details['tenant_id'] == tenant_id
if timeline_details['type'] == 'Local':
log.info("timeline downloaded, checking its data")
break
attempts += 1
if attempts > 10:
raise Exception("timeline redownload failed")
log.debug("still waiting")
time.sleep(1)
log.info("waiting for timeline redownload")
client = env.pageserver.http_client()
attempts = 0
while True:
timeline_details = client.timeline_details(tenant_id, timeline_id)
assert timeline_details['timeline_id'] == timeline_id
assert timeline_details['tenant_id'] == tenant_id
if timeline_details['type'] == 'Local':
log.info("timeline downloaded, checking its data")
break
attempts += 1
if attempts > 10:
raise Exception("timeline redownload failed")
log.debug("still waiting")
time.sleep(1)
pg = env.postgres.create_start()
with closing(pg.connect()) as conn:
with conn.cursor() as cur:
cur.execute(f'SELECT secret FROM t1 WHERE id = {data_id};')
assert cur.fetchone() == (data_secret, )
pg = env.postgres.create_start()
with closing(pg.connect()) as conn:
with conn.cursor() as cur:
cur.execute(f'SELECT secret FROM t1 WHERE id = {data_id};')
assert cur.fetchone() == (data_secret, )

13
test_runner/fixtures/zenith_fixtures.py
View File

@@ -360,6 +360,11 @@ class ZenithEnvBuilder:
self.env = ZenithEnv(self)
return self.env
def enable_local_fs_remote_storage(self):
assert self.pageserver_remote_storage is None, "remote storage is enabled already"
self.pageserver_remote_storage = LocalFsStorage(
Path(self.repo_dir / 'local_fs_remote_storage'))
def __enter__(self):
return self
@@ -472,7 +477,7 @@ sync = false # Disable fsyncs to make the tests go faster
tmp.flush()
cmd = ['init', f'--config={tmp.name}']
_append_pageserver_param_overrides(cmd, config.pageserver_remote_storage)
append_pageserver_param_overrides(cmd, config.pageserver_remote_storage)
self.zenith_cli(cmd)
@@ -744,7 +749,7 @@ class ZenithPageserver(PgProtocol):
assert self.running == False
start_args = ['pageserver', 'start']
_append_pageserver_param_overrides(start_args, self.remote_storage)
append_pageserver_param_overrides(start_args, self.remote_storage)
self.env.zenith_cli(start_args)
self.running = True
@@ -779,8 +784,8 @@ class ZenithPageserver(PgProtocol):
)
def _append_pageserver_param_overrides(params_to_update: List[str],
pageserver_remote_storage: Optional[RemoteStorage]):
def append_pageserver_param_overrides(params_to_update: List[str],
pageserver_remote_storage: Optional[RemoteStorage]):
if pageserver_remote_storage is not None:
if isinstance(pageserver_remote_storage, LocalFsStorage):
pageserver_storage_override = f"local_path='{pageserver_remote_storage.root}'"