rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-08 05:52:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update Zenith CLI config between runs

Browse Source
This commit is contained in:
Kirill Bulatov
2022-02-24 13:40:32 +03:00
committed by Kirill Bulatov
parent f49990ed43
commit 4d0f7fd1e4
40 changed files with 371 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
test_runner/batch_others/test_auth.py
View File

@@ -25,21 +25,24 @@ def test_pageserver_auth(zenith_env_builder: ZenithEnvBuilder):
ps.safe_psql("set FOO", password=tenant_token)
ps.safe_psql("set FOO", password=management_token)
new_timeline_id = env.zenith_cli.create_branch('test_pageserver_auth',
tenant_id=env.initial_tenant)
# tenant can create branches
tenant_http_client.timeline_create(timeline_id=uuid4(),
tenant_id=env.initial_tenant,
ancestor_timeline_id=env.initial_timeline)
ancestor_timeline_id=new_timeline_id)
# console can create branches for tenant
management_http_client.timeline_create(timeline_id=uuid4(),
tenant_id=env.initial_tenant,
ancestor_timeline_id=env.initial_timeline)
ancestor_timeline_id=new_timeline_id)
# fail to create branch using token with different tenant_id
with pytest.raises(ZenithPageserverApiException,
match='Forbidden: Tenant id mismatch. Permission denied'):
invalid_tenant_http_client.timeline_create(timeline_id=uuid4(),
tenant_id=env.initial_tenant,
ancestor_timeline_id=env.initial_timeline)
ancestor_timeline_id=new_timeline_id)
# create tenant using management token
management_http_client.tenant_create(uuid4())
@@ -59,9 +62,9 @@ def test_compute_auth_to_pageserver(zenith_env_builder: ZenithEnvBuilder, with_w
zenith_env_builder.num_safekeepers = 3
env = zenith_env_builder.init_start()
branch = f"test_compute_auth_to_pageserver{with_wal_acceptors}"
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start(branch, timeline_id=new_timeline_id)
branch = f'test_compute_auth_to_pageserver{with_wal_acceptors}'
env.zenith_cli.create_branch(branch)
pg = env.postgres.create_start(branch)
with closing(pg.connect()) as conn:
with conn.cursor() as cur:

2
test_runner/batch_others/test_backpressure.py
View File

@@ -95,7 +95,7 @@ def test_backpressure_received_lsn_lag(zenith_env_builder: ZenithEnvBuilder):
zenith_env_builder.num_safekeepers = 1
env = zenith_env_builder.init_start()
# Create a branch for us
env.zenith_cli.create_branch("test_backpressure", "main")
env.zenith_cli.create_branch('test_backpressure')
pg = env.postgres.create_start('test_backpressure',
config_lines=['max_replication_write_lag=30MB'])

42
test_runner/batch_others/test_branch_behind.py
View File

@@ -22,9 +22,8 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder):
env = zenith_env_builder.init_start()
# Branch at the point where only 100 rows were inserted
test_branch_behind_timeline_id = env.zenith_cli.branch_timeline()
pgmain = env.postgres.create_start('test_branch_behind',
timeline_id=test_branch_behind_timeline_id)
env.zenith_cli.create_branch('test_branch_behind')
pgmain = env.postgres.create_start('test_branch_behind')
log.info("postgres is running on 'test_branch_behind' branch")
main_pg_conn = pgmain.connect()
@@ -60,8 +59,9 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder):
log.info(f'LSN after 200100 rows: {lsn_b}')
# Branch at the point where only 100 rows were inserted
test_branch_behind_hundred_timeline_id = env.zenith_cli.branch_timeline(
ancestor_timeline_id=test_branch_behind_timeline_id, ancestor_start_lsn=lsn_a)
env.zenith_cli.create_branch('test_branch_behind_hundred',
'test_branch_behind',
ancestor_start_lsn=lsn_a)
# Insert many more rows. This generates enough WAL to fill a few segments.
main_cur.execute('''
@@ -76,13 +76,12 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder):
log.info(f'LSN after 400100 rows: {lsn_c}')
# Branch at the point where only 200100 rows were inserted
test_branch_behind_more_timeline_id = env.zenith_cli.branch_timeline(
ancestor_timeline_id=test_branch_behind_timeline_id, ancestor_start_lsn=lsn_b)
env.zenith_cli.create_branch('test_branch_behind_more',
'test_branch_behind',
ancestor_start_lsn=lsn_b)
pg_hundred = env.postgres.create_start("test_branch_behind_hundred",
timeline_id=test_branch_behind_hundred_timeline_id)
pg_more = env.postgres.create_start("test_branch_behind_more",
timeline_id=test_branch_behind_more_timeline_id)
pg_hundred = env.postgres.create_start('test_branch_behind_hundred')
pg_more = env.postgres.create_start('test_branch_behind_more')
# On the 'hundred' branch, we should see only 100 rows
hundred_pg_conn = pg_hundred.connect()
@@ -103,23 +102,23 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder):
# Check bad lsn's for branching
# branch at segment boundary
test_branch_segment_boundary_timeline_id = env.zenith_cli.branch_timeline(
ancestor_timeline_id=test_branch_behind_timeline_id, ancestor_start_lsn="0/3000000")
pg = env.postgres.create_start("test_branch_segment_boundary",
timeline_id=test_branch_segment_boundary_timeline_id)
env.zenith_cli.create_branch('test_branch_segment_boundary',
'test_branch_behind',
ancestor_start_lsn="0/3000000")
pg = env.postgres.create_start('test_branch_segment_boundary')
cur = pg.connect().cursor()
cur.execute('SELECT 1')
assert cur.fetchone() == (1, )
# branch at pre-initdb lsn
with pytest.raises(Exception, match="invalid branch start lsn"):
env.zenith_cli.branch_timeline(ancestor_timeline_id=env.initial_timeline,
ancestor_start_lsn="0/42")
env.zenith_cli.create_branch('test_branch_preinitdb', ancestor_start_lsn="0/42")
# branch at pre-ancestor lsn
with pytest.raises(Exception, match="less than timeline ancestor lsn"):
env.zenith_cli.branch_timeline(ancestor_timeline_id=test_branch_behind_timeline_id,
ancestor_start_lsn="0/42")
env.zenith_cli.create_branch('test_branch_preinitdb',
'test_branch_behind',
ancestor_start_lsn="0/42")
# check that we cannot create branch based on garbage collected data
with closing(env.pageserver.connect()) as psconn:
@@ -131,8 +130,9 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder):
with pytest.raises(Exception, match="invalid branch start lsn"):
# this gced_lsn is pretty random, so if gc is disabled this woudln't fail
env.zenith_cli.branch_timeline(ancestor_timeline_id=test_branch_behind_timeline_id,
ancestor_start_lsn=gced_lsn)
env.zenith_cli.create_branch('test_branch_create_fail',
'test_branch_behind',
ancestor_start_lsn=gced_lsn)
# check that after gc everything is still there
hundred_cur.execute('SELECT count(*) FROM foo')

15
test_runner/batch_others/test_clog_truncate.py
View File

@@ -12,7 +12,7 @@ from fixtures.log_helper import log
#
def test_clog_truncate(zenith_simple_env: ZenithEnv):
env = zenith_simple_env
test_clog_truncate_timeline_id = env.zenith_cli.branch_timeline()
env.zenith_cli.create_branch('test_clog_truncate', 'empty')
# set agressive autovacuum to make sure that truncation will happen
config = [
@@ -25,9 +25,7 @@ def test_clog_truncate(zenith_simple_env: ZenithEnv):
'autovacuum_freeze_max_age=100000'
]
pg = env.postgres.create_start('test_clog_truncate',
config_lines=config,
timeline_id=test_clog_truncate_timeline_id)
pg = env.postgres.create_start('test_clog_truncate', config_lines=config)
log.info('postgres is running on test_clog_truncate branch')
# Install extension containing function needed for test
@@ -64,11 +62,10 @@ def test_clog_truncate(zenith_simple_env: ZenithEnv):
# create new branch after clog truncation and start a compute node on it
log.info(f'create branch at lsn_after_truncation {lsn_after_truncation}')
test_clog_truncate_new_timeline_id = env.zenith_cli.branch_timeline(
ancestor_timeline_id=test_clog_truncate_timeline_id,
ancestor_start_lsn=lsn_after_truncation)
pg2 = env.postgres.create_start('test_clog_truncate_new',
timeline_id=test_clog_truncate_new_timeline_id)
env.zenith_cli.create_branch('test_clog_truncate_new',
'test_clog_truncate',
ancestor_start_lsn=lsn_after_truncation)
pg2 = env.postgres.create_start('test_clog_truncate_new')
log.info('postgres is running on test_clog_truncate_new branch')
# check that new node doesn't contain truncated segment

8
test_runner/batch_others/test_config.py
View File

@@ -9,10 +9,10 @@ from fixtures.log_helper import log
#
def test_config(zenith_simple_env: ZenithEnv):
env = zenith_simple_env
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_config',
config_lines=['log_min_messages=debug1'],
timeline_id=new_timeline_id)
env.zenith_cli.create_branch("test_config", "empty")
# change config
pg = env.postgres.create_start('test_config', config_lines=['log_min_messages=debug1'])
log.info('postgres is running on test_config branch')
with closing(pg.connect()) as conn:

29
test_runner/batch_others/test_createdropdb.py
View File

@@ -11,9 +11,9 @@ from fixtures.log_helper import log
#
def test_createdb(zenith_simple_env: ZenithEnv):
env = zenith_simple_env
test_createdb_timeline_id = env.zenith_cli.branch_timeline()
env.zenith_cli.create_branch('test_createdb', 'empty')
pg = env.postgres.create_start('test_createdb', timeline_id=test_createdb_timeline_id)
pg = env.postgres.create_start('test_createdb')
log.info("postgres is running on 'test_createdb' branch")
with closing(pg.connect()) as conn:
@@ -27,9 +27,8 @@ def test_createdb(zenith_simple_env: ZenithEnv):
lsn = cur.fetchone()[0]
# Create a branch
test_createdb2_timeline_id = env.zenith_cli.branch_timeline(
ancestor_timeline_id=test_createdb_timeline_id, ancestor_start_lsn=lsn)
pg2 = env.postgres.create_start('test_createdb2', timeline_id=test_createdb2_timeline_id)
env.zenith_cli.create_branch('test_createdb2', 'test_createdb', ancestor_start_lsn=lsn)
pg2 = env.postgres.create_start('test_createdb2')
# Test that you can connect to the new database on both branches
for db in (pg, pg2):
@@ -41,8 +40,8 @@ def test_createdb(zenith_simple_env: ZenithEnv):
#
def test_dropdb(zenith_simple_env: ZenithEnv, test_output_dir):
env = zenith_simple_env
test_dropdb_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_dropdb', timeline_id=test_dropdb_timeline_id)
env.zenith_cli.create_branch('test_dropdb', 'empty')
pg = env.postgres.create_start('test_dropdb')
log.info("postgres is running on 'test_dropdb' branch")
with closing(pg.connect()) as conn:
@@ -65,15 +64,15 @@ def test_dropdb(zenith_simple_env: ZenithEnv, test_output_dir):
lsn_after_drop = cur.fetchone()[0]
# Create two branches before and after database drop.
test_before_dropdb_timeline_db = env.zenith_cli.branch_timeline(
ancestor_timeline_id=test_dropdb_timeline_id, ancestor_start_lsn=lsn_before_drop)
pg_before = env.postgres.create_start('test_before_dropdb',
timeline_id=test_before_dropdb_timeline_db)
env.zenith_cli.create_branch('test_before_dropdb',
'test_dropdb',
ancestor_start_lsn=lsn_before_drop)
pg_before = env.postgres.create_start('test_before_dropdb')
test_after_dropdb_timeline_id = env.zenith_cli.branch_timeline(
ancestor_timeline_id=test_dropdb_timeline_id, ancestor_start_lsn=lsn_after_drop)
pg_after = env.postgres.create_start('test_after_dropdb',
timeline_id=test_after_dropdb_timeline_id)
env.zenith_cli.create_branch('test_after_dropdb',
'test_dropdb',
ancestor_start_lsn=lsn_after_drop)
pg_after = env.postgres.create_start('test_after_dropdb')
# Test that database exists on the branch before drop
pg_before.connect(dbname='foodb').close()

9
test_runner/batch_others/test_createuser.py
View File

@@ -9,8 +9,8 @@ from fixtures.log_helper import log
#
def test_createuser(zenith_simple_env: ZenithEnv):
env = zenith_simple_env
test_createuser_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_createuser', timeline_id=test_createuser_timeline_id)
env.zenith_cli.create_branch('test_createuser', 'empty')
pg = env.postgres.create_start('test_createuser')
log.info("postgres is running on 'test_createuser' branch")
with closing(pg.connect()) as conn:
@@ -24,9 +24,8 @@ def test_createuser(zenith_simple_env: ZenithEnv):
lsn = cur.fetchone()[0]
# Create a branch
test_createuser2_timeline_id = env.zenith_cli.branch_timeline(
ancestor_timeline_id=test_createuser_timeline_id, ancestor_start_lsn=lsn)
pg2 = env.postgres.create_start('test_createuser2', timeline_id=test_createuser2_timeline_id)
env.zenith_cli.create_branch('test_createuser2', 'test_createuser', ancestor_start_lsn=lsn)
pg2 = env.postgres.create_start('test_createuser2')
# Test that you can connect to new branch as a new user
assert pg2.safe_psql('select current_user', username='testuser') == [('testuser', )]

5
test_runner/batch_others/test_gc_aggressive.py
View File

@@ -1,6 +1,7 @@
from contextlib import closing
import asyncio
import asyncpg
import random
from fixtures.zenith_fixtures import ZenithEnv, Postgres, Safekeeper
@@ -54,8 +55,8 @@ async def update_and_gc(env: ZenithEnv, pg: Postgres, timeline: str):
#
def test_gc_aggressive(zenith_simple_env: ZenithEnv):
env = zenith_simple_env
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_gc_aggressive', timeline_id=new_timeline_id)
env.zenith_cli.create_branch("test_gc_aggressive", "empty")
pg = env.postgres.create_start('test_gc_aggressive')
log.info('postgres is running on test_gc_aggressive branch')
conn = pg.connect()

10
test_runner/batch_others/test_multixact.py
View File

@@ -10,8 +10,8 @@ from fixtures.log_helper import log
#
def test_multixact(zenith_simple_env: ZenithEnv, test_output_dir):
env = zenith_simple_env
test_multixact_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_multixact', timeline_id=test_multixact_timeline_id)
env.zenith_cli.create_branch('test_multixact', 'empty')
pg = env.postgres.create_start('test_multixact')
log.info("postgres is running on 'test_multixact' branch")
pg_conn = pg.connect()
@@ -60,10 +60,8 @@ def test_multixact(zenith_simple_env: ZenithEnv, test_output_dir):
assert int(next_multixact_id) > int(next_multixact_id_old)
# Branch at this point
test_multixact_new_timeline_id = env.zenith_cli.branch_timeline(
ancestor_timeline_id=test_multixact_timeline_id, ancestor_start_lsn=lsn)
pg_new = env.postgres.create_start('test_multixact_new',
timeline_id=test_multixact_new_timeline_id)
env.zenith_cli.create_branch('test_multixact_new', 'test_multixact', ancestor_start_lsn=lsn)
pg_new = env.postgres.create_start('test_multixact_new')
log.info("postgres is running on 'test_multixact_new' branch")
pg_new_conn = pg_new.connect()

4
test_runner/batch_others/test_old_request_lsn.py
View File

@@ -16,8 +16,8 @@ from fixtures.log_helper import log
#
def test_old_request_lsn(zenith_simple_env: ZenithEnv):
env = zenith_simple_env
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_old_request_lsn', timeline_id=new_timeline_id)
env.zenith_cli.create_branch("test_old_request_lsn", "empty")
pg = env.postgres.create_start('test_old_request_lsn')
log.info('postgres is running on test_old_request_lsn branch')
pg_conn = pg.connect()

8
test_runner/batch_others/test_pageserver_catchup.py
View File

@@ -16,9 +16,8 @@ def test_pageserver_catchup_while_compute_down(zenith_env_builder: ZenithEnvBuil
zenith_env_builder.num_safekeepers = 3
env = zenith_env_builder.init_start()
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_pageserver_catchup_while_compute_down',
timeline_id=new_timeline_id)
env.zenith_cli.create_branch('test_pageserver_catchup_while_compute_down')
pg = env.postgres.create_start('test_pageserver_catchup_while_compute_down')
pg_conn = pg.connect()
cur = pg_conn.cursor()
@@ -60,8 +59,7 @@ def test_pageserver_catchup_while_compute_down(zenith_env_builder: ZenithEnvBuil
env.safekeepers[2].start()
# restart compute node
pg.stop_and_destroy().create_start('test_pageserver_catchup_while_compute_down',
timeline_id=new_timeline_id)
pg.stop_and_destroy().create_start('test_pageserver_catchup_while_compute_down')
# Ensure that basebackup went correct and pageserver returned all data
pg_conn = pg.connect()

4
test_runner/batch_others/test_pageserver_restart.py
View File

@@ -15,8 +15,8 @@ def test_pageserver_restart(zenith_env_builder: ZenithEnvBuilder):
zenith_env_builder.num_safekeepers = 1
env = zenith_env_builder.init_start()
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_pageserver_restart', timeline_id=new_timeline_id)
env.zenith_cli.create_branch('test_pageserver_restart')
pg = env.postgres.create_start('test_pageserver_restart')
pg_conn = pg.connect()
cur = pg_conn.cursor()

4
test_runner/batch_others/test_parallel_copy.py
View File

@@ -35,8 +35,8 @@ async def parallel_load_same_table(pg: Postgres, n_parallel: int):
# Load data into one table with COPY TO from 5 parallel connections
def test_parallel_copy(zenith_simple_env: ZenithEnv, n_parallel=5):
env = zenith_simple_env
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_parallel_copy', timeline_id=new_timeline_id)
env.zenith_cli.create_branch("test_parallel_copy", "empty")
pg = env.postgres.create_start('test_parallel_copy')
log.info("postgres is running on 'test_parallel_copy' branch")
# Create test table

4
test_runner/batch_others/test_pgbench.py
View File

@@ -4,8 +4,8 @@ from fixtures.log_helper import log
def test_pgbench(zenith_simple_env: ZenithEnv, pg_bin):
env = zenith_simple_env
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_pgbench', timeline_id=new_timeline_id)
env.zenith_cli.create_branch("test_pgbench", "empty")
pg = env.postgres.create_start('test_pgbench')
log.info("postgres is running on 'test_pgbench' branch")
connstr = pg.connstr()

21
test_runner/batch_others/test_readonly_node.py
View File

@@ -11,9 +11,8 @@ from fixtures.zenith_fixtures import ZenithEnv
#
def test_readonly_node(zenith_simple_env: ZenithEnv):
env = zenith_simple_env
test_readonly_node_timeline_id = env.zenith_cli.branch_timeline()
pgmain = env.postgres.create_start('test_readonly_node',
timeline_id=test_readonly_node_timeline_id)
env.zenith_cli.create_branch('test_readonly_node', 'empty')
pgmain = env.postgres.create_start('test_readonly_node')
log.info("postgres is running on 'test_readonly_node' branch")
main_pg_conn = pgmain.connect()
@@ -53,14 +52,10 @@ def test_readonly_node(zenith_simple_env: ZenithEnv):
log.info('LSN after 400100 rows: ' + lsn_c)
# Create first read-only node at the point where only 100 rows were inserted
pg_hundred = env.postgres.create_start("test_readonly_node_hundred",
timeline_id=test_readonly_node_timeline_id,
lsn=lsn_a)
pg_hundred = env.postgres.create_start("test_readonly_node_hundred", lsn=lsn_a)
# And another at the point where 200100 rows were inserted
pg_more = env.postgres.create_start("test_readonly_node_more",
timeline_id=test_readonly_node_timeline_id,
lsn=lsn_b)
pg_more = env.postgres.create_start("test_readonly_node_more", lsn=lsn_b)
# On the 'hundred' node, we should see only 100 rows
hundred_pg_conn = pg_hundred.connect()
@@ -79,9 +74,7 @@ def test_readonly_node(zenith_simple_env: ZenithEnv):
assert main_cur.fetchone() == (400100, )
# Check creating a node at segment boundary
pg = env.postgres.create_start("test_branch_segment_boundary",
timeline_id=test_readonly_node_timeline_id,
lsn='0/3000000')
pg = env.postgres.create_start("test_branch_segment_boundary", lsn='0/3000000')
cur = pg.connect().cursor()
cur.execute('SELECT 1')
assert cur.fetchone() == (1, )
@@ -89,6 +82,4 @@ def test_readonly_node(zenith_simple_env: ZenithEnv):
# Create node at pre-initdb lsn
with pytest.raises(Exception, match="invalid basebackup lsn"):
# compute node startup with invalid LSN should fail
env.zenith_cli.pg_start("test_readonly_node_preinitdb",
timeline_id=test_readonly_node_timeline_id,
lsn="0/42")
env.zenith_cli.pg_start("test_readonly_node_preinitdb", lsn="0/42")

10
test_runner/batch_others/test_restart_compute.py
View File

@@ -15,8 +15,8 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor
zenith_env_builder.num_safekeepers = 3
env = zenith_env_builder.init_start()
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_restart_compute', timeline_id=new_timeline_id)
env.zenith_cli.create_branch('test_restart_compute')
pg = env.postgres.create_start('test_restart_compute')
log.info("postgres is running on 'test_restart_compute' branch")
with closing(pg.connect()) as conn:
@@ -29,7 +29,7 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor
log.info(f"res = {r}")
# Remove data directory and restart
pg.stop_and_destroy().create_start('test_restart_compute', timeline_id=new_timeline_id)
pg.stop_and_destroy().create_start('test_restart_compute')
with closing(pg.connect()) as conn:
with conn.cursor() as cur:
@@ -48,7 +48,7 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor
log.info(f"res = {r}")
# Again remove data directory and restart
pg.stop_and_destroy().create_start('test_restart_compute', timeline_id=new_timeline_id)
pg.stop_and_destroy().create_start('test_restart_compute')
# That select causes lots of FPI's and increases probability of wakeepers
# lagging behind after query completion
@@ -62,7 +62,7 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor
log.info(f"res = {r}")
# And again remove data directory and restart
pg.stop_and_destroy().create_start('test_restart_compute', timeline_id=new_timeline_id)
pg.stop_and_destroy().create_start('test_restart_compute')
with closing(pg.connect()) as conn:
with conn.cursor() as cur:

4
test_runner/batch_others/test_snapfiles_gc.py
View File

@@ -14,8 +14,8 @@ from fixtures.log_helper import log
#
def test_layerfiles_gc(zenith_simple_env: ZenithEnv):
env = zenith_simple_env
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_layerfiles_gc', timeline_id=new_timeline_id)
env.zenith_cli.create_branch("test_layerfiles_gc", "empty")
pg = env.postgres.create_start('test_layerfiles_gc')
with closing(pg.connect()) as conn:
with conn.cursor() as cur:

4
test_runner/batch_others/test_subxacts.py
View File

@@ -10,8 +10,8 @@ from fixtures.log_helper import log
# CLOG.
def test_subxacts(zenith_simple_env: ZenithEnv, test_output_dir):
env = zenith_simple_env
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_subxacts', timeline_id=new_timeline_id)
env.zenith_cli.create_branch("test_subxacts", "empty")
pg = env.postgres.create_start('test_subxacts')
log.info("postgres is running on 'test_subxacts' branch")
pg_conn = pg.connect()

8
test_runner/batch_others/test_tenant_relocation.py
View File

@@ -127,14 +127,12 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder,
# create folder for remote storage mock
remote_storage_mock_path = env.repo_dir / 'local_fs_remote_storage'
(tenant, _) = env.zenith_cli.create_tenant(UUID("74ee8b079a0e437eb0afea7d26a07209"))
tenant = env.zenith_cli.create_tenant(UUID("74ee8b079a0e437eb0afea7d26a07209"))
log.info("tenant to relocate %s", tenant)
new_timeline_id = env.zenith_cli.branch_timeline(tenant_id=tenant)
env.zenith_cli.create_branch('test_tenant_relocation', tenant_id=tenant)
tenant_pg = env.postgres.create_start("test_tenant_relocation",
tenant_id=tenant,
timeline_id=new_timeline_id)
tenant_pg = env.postgres.create_start("test_tenant_relocation", tenant_id=tenant)
# insert some data
with closing(tenant_pg.connect()) as conn:

18
test_runner/batch_others/test_tenants.py
View File

@@ -12,23 +12,21 @@ def test_tenants_normal_work(zenith_env_builder: ZenithEnvBuilder, with_wal_acce
env = zenith_env_builder.init_start()
"""Tests tenants with and without wal acceptors"""
(tenant_1, initial_timeline_1) = env.zenith_cli.create_tenant()
(tenant_2, initial_timeline_2) = env.zenith_cli.create_tenant()
tenant_1 = env.zenith_cli.create_tenant()
tenant_2 = env.zenith_cli.create_tenant()
new_timeline_tenant_1 = env.zenith_cli.branch_timeline(tenant_id=tenant_1,
ancestor_timeline_id=initial_timeline_1)
new_timeline_tenant_2 = env.zenith_cli.branch_timeline(tenant_id=tenant_2,
ancestor_timeline_id=initial_timeline_2)
env.zenith_cli.create_branch(f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}',
tenant_id=tenant_1)
env.zenith_cli.create_branch(f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}',
tenant_id=tenant_2)
pg_tenant1 = env.postgres.create_start(
f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}",
f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}',
tenant_id=tenant_1,
timeline_id=new_timeline_tenant_1,
)
pg_tenant2 = env.postgres.create_start(
f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}",
f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}',
tenant_id=tenant_2,
timeline_id=new_timeline_tenant_2,
)
for pg in [pg_tenant1, pg_tenant2]:

10
test_runner/batch_others/test_timeline_size.py
View File

@@ -10,14 +10,13 @@ import time
def test_timeline_size(zenith_simple_env: ZenithEnv):
env = zenith_simple_env
# Branch at the point where only 100 rows were inserted
new_timeline_id = env.zenith_cli.branch_timeline()
new_timeline_id = env.zenith_cli.create_branch('test_timeline_size', 'empty')
client = env.pageserver.http_client()
res = client.timeline_detail(tenant_id=env.initial_tenant, timeline_id=new_timeline_id)
print(f'@@@@@@@@@@\n{res}\n@@@@@@@@@@@')
assert res["current_logical_size"] == res["current_logical_size_non_incremental"]
pgmain = env.postgres.create_start("test_timeline_size", timeline_id=new_timeline_id)
pgmain = env.postgres.create_start("test_timeline_size")
log.info("postgres is running on 'test_timeline_size' branch")
with closing(pgmain.connect()) as conn:
@@ -69,7 +68,7 @@ def wait_for_pageserver_catchup(pgmain: Postgres, polling_interval=1, timeout=60
def test_timeline_size_quota(zenith_env_builder: ZenithEnvBuilder):
zenith_env_builder.num_safekeepers = 1
env = zenith_env_builder.init_start()
new_timeline_id = env.zenith_cli.branch_timeline()
new_timeline_id = env.zenith_cli.create_branch('test_timeline_size_quota')
client = env.pageserver.http_client()
res = client.timeline_detail(tenant_id=env.initial_tenant, timeline_id=new_timeline_id)
@@ -78,8 +77,7 @@ def test_timeline_size_quota(zenith_env_builder: ZenithEnvBuilder):
pgmain = env.postgres.create_start(
"test_timeline_size_quota",
# Set small limit for the test
config_lines=['zenith.max_cluster_size=30MB'],
timeline_id=new_timeline_id)
config_lines=['zenith.max_cluster_size=30MB'])
log.info("postgres is running on 'test_timeline_size_quota' branch")
with closing(pgmain.connect()) as conn:

10
test_runner/batch_others/test_twophase.py
View File

@@ -9,10 +9,8 @@ from fixtures.log_helper import log
#
def test_twophase(zenith_simple_env: ZenithEnv):
env = zenith_simple_env
test_twophase_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_twophase',
config_lines=['max_prepared_transactions=5'],
timeline_id=test_twophase_timeline_id)
env.zenith_cli.create_branch("test_twophase", "empty")
pg = env.postgres.create_start('test_twophase', config_lines=['max_prepared_transactions=5'])
log.info("postgres is running on 'test_twophase' branch")
conn = pg.connect()
@@ -57,14 +55,12 @@ def test_twophase(zenith_simple_env: ZenithEnv):
assert len(twophase_files) == 2
# Create a branch with the transaction in prepared state
test_twophase_prepared_timeline_id = env.zenith_cli.branch_timeline(
ancestor_timeline_id=test_twophase_timeline_id)
env.zenith_cli.create_branch("test_twophase_prepared", "test_twophase")
# Start compute on the new branch
pg2 = env.postgres.create_start(
'test_twophase_prepared',
config_lines=['max_prepared_transactions=5'],
timeline_id=test_twophase_prepared_timeline_id,
)
# Check that we restored only needed twophase files

10
test_runner/batch_others/test_vm_bits.py
View File

@@ -9,8 +9,8 @@ from fixtures.log_helper import log
def test_vm_bit_clear(zenith_simple_env: ZenithEnv):
env = zenith_simple_env
test_vm_bit_clear_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_vm_bit_clear', timeline_id=test_vm_bit_clear_timeline_id)
env.zenith_cli.create_branch("test_vm_bit_clear", "empty")
pg = env.postgres.create_start('test_vm_bit_clear')
log.info("postgres is running on 'test_vm_bit_clear' branch")
pg_conn = pg.connect()
@@ -33,8 +33,7 @@ def test_vm_bit_clear(zenith_simple_env: ZenithEnv):
cur.execute('UPDATE vmtest_update SET id = 5000 WHERE id = 1')
# Branch at this point, to test that later
test_vm_bit_clear_new_timeline_id = env.zenith_cli.branch_timeline(
ancestor_timeline_id=test_vm_bit_clear_timeline_id)
env.zenith_cli.create_branch("test_vm_bit_clear_new", "test_vm_bit_clear")
# Clear the buffer cache, to force the VM page to be re-fetched from
# the page server
@@ -62,8 +61,7 @@ def test_vm_bit_clear(zenith_simple_env: ZenithEnv):
# a dirty VM page is evicted. If the VM bit was not correctly cleared by the
# earlier WAL record, the full-page image hides the problem. Starting a new
# server at the right point-in-time avoids that full-page image.
pg_new = env.postgres.create_start('test_vm_bit_clear_new',
timeline_id=test_vm_bit_clear_new_timeline_id)
pg_new = env.postgres.create_start('test_vm_bit_clear_new')
log.info("postgres is running on 'test_vm_bit_clear_new' branch")
pg_new_conn = pg_new.connect()

32
test_runner/batch_others/test_wal_acceptor.py
View File

@@ -24,8 +24,8 @@ def test_normal_work(zenith_env_builder: ZenithEnvBuilder):
zenith_env_builder.num_safekeepers = 3
env = zenith_env_builder.init_start()
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_wal_acceptors_normal_work', timeline_id=new_timeline_id)
env.zenith_cli.create_branch('test_wal_acceptors_normal_work')
pg = env.postgres.create_start('test_wal_acceptors_normal_work')
with closing(pg.connect()) as conn:
with conn.cursor() as cur:
@@ -62,8 +62,8 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder):
# start postgres on each timeline
pgs = []
for branch_name in branch_names:
new_timeline_id = env.zenith_cli.branch_timeline()
pgs.append(env.postgres.create_start(branch_name, timeline_id=new_timeline_id))
new_timeline_id = env.zenith_cli.create_branch(branch_name)
pgs.append(env.postgres.create_start(branch_name))
branch_names_to_timeline_ids[branch_name] = new_timeline_id
tenant_id = env.initial_tenant
@@ -87,7 +87,6 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder):
timeline_metrics = []
with env.pageserver.http_client() as pageserver_http:
for timeline_detail in timeline_details:
print(f"@@@@@@@@@@@\n{timeline_detail}\n@@@@@@@@@@@")
timeline_id: str = timeline_detail["timeline_id"]
m = TimelineMetrics(
@@ -188,8 +187,8 @@ def test_restarts(zenith_env_builder: ZenithEnvBuilder):
zenith_env_builder.num_safekeepers = n_acceptors
env = zenith_env_builder.init_start()
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_wal_acceptors_restarts', timeline_id=new_timeline_id)
env.zenith_cli.create_branch('test_wal_acceptors_restarts')
pg = env.postgres.create_start('test_wal_acceptors_restarts')
# we rely upon autocommit after each statement
# as waiting for acceptors happens there
@@ -225,8 +224,8 @@ def test_unavailability(zenith_env_builder: ZenithEnvBuilder):
zenith_env_builder.num_safekeepers = 2
env = zenith_env_builder.init_start()
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_wal_acceptors_unavailability', timeline_id=new_timeline_id)
env.zenith_cli.create_branch('test_wal_acceptors_unavailability')
pg = env.postgres.create_start('test_wal_acceptors_unavailability')
# we rely upon autocommit after each statement
# as waiting for acceptors happens there
@@ -296,9 +295,8 @@ def test_race_conditions(zenith_env_builder: ZenithEnvBuilder, stop_value):
zenith_env_builder.num_safekeepers = 3
env = zenith_env_builder.init_start()
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_wal_acceptors_race_conditions',
timeline_id=new_timeline_id)
env.zenith_cli.create_branch('test_wal_acceptors_race_conditions')
pg = env.postgres.create_start('test_wal_acceptors_race_conditions')
# we rely upon autocommit after each statement
# as waiting for acceptors happens there
@@ -462,8 +460,8 @@ def test_timeline_status(zenith_env_builder: ZenithEnvBuilder):
zenith_env_builder.num_safekeepers = 1
env = zenith_env_builder.init_start()
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_timeline_status', timeline_id=new_timeline_id)
env.zenith_cli.create_branch('test_timeline_status')
pg = env.postgres.create_start('test_timeline_status')
wa = env.safekeepers[0]
wa_http_cli = wa.http_client()
@@ -636,12 +634,12 @@ def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder):
zenith_env_builder.num_safekeepers = 4
env = zenith_env_builder.init_start()
new_timeline_id = env.zenith_cli.branch_timeline()
env.zenith_cli.create_branch('test_replace_safekeeper')
log.info("Use only first 3 safekeepers")
env.safekeepers[3].stop()
active_safekeepers = [1, 2, 3]
pg = env.postgres.create('test_replace_safekeeper', timeline_id=new_timeline_id)
pg = env.postgres.create('test_replace_safekeeper')
pg.adjust_for_wal_acceptors(safekeepers_guc(env, active_safekeepers))
pg.start()
@@ -679,7 +677,7 @@ def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder):
show_statuses(env.safekeepers, tenant_id, timeline_id)
log.info("Recreate postgres to replace failed sk1 with new sk4")
pg.stop_and_destroy().create('test_replace_safekeeper', timeline_id=uuid.UUID(timeline_id))
pg.stop_and_destroy().create('test_replace_safekeeper')
active_safekeepers = [2, 3, 4]
env.safekeepers[3].start()
pg.adjust_for_wal_acceptors(safekeepers_guc(env, active_safekeepers))

5
test_runner/batch_others/test_wal_acceptor_async.py
View File

@@ -202,9 +202,8 @@ def test_restarts_under_load(zenith_env_builder: ZenithEnvBuilder):
zenith_env_builder.num_safekeepers = 3
env = zenith_env_builder.init_start()
new_timeline_id = env.zenith_cli.branch_timeline()
pg = env.postgres.create_start('test_wal_acceptors_restarts_under_load',
timeline_id=new_timeline_id)
env.zenith_cli.create_branch('test_wal_acceptors_restarts_under_load')
pg = env.postgres.create_start('test_wal_acceptors_restarts_under_load')
asyncio.run(run_restarts_under_load(pg, env.safekeepers))

11
test_runner/batch_others/test_zenith_cli.py
View File

@@ -36,11 +36,12 @@ def test_cli_timeline_list(zenith_simple_env: ZenithEnv):
helper_compare_timeline_list(pageserver_http_client, env, env.initial_tenant)
# Create a branch for us
main_timeline_id = env.zenith_cli.branch_timeline()
main_timeline_id = env.zenith_cli.create_branch('test_cli_branch_list_main')
helper_compare_timeline_list(pageserver_http_client, env, env.initial_tenant)
# Create a nested branch
nested_timeline_id = env.zenith_cli.branch_timeline(ancestor_timeline_id=main_timeline_id)
nested_timeline_id = env.zenith_cli.create_branch('test_cli_branch_list_nested',
'test_cli_branch_list_main')
helper_compare_timeline_list(pageserver_http_client, env, env.initial_tenant)
# Check that all new branches are visible via CLI
@@ -67,15 +68,13 @@ def test_cli_tenant_list(zenith_simple_env: ZenithEnv):
helper_compare_tenant_list(pageserver_http_client, env)
# Create new tenant
tenant1 = uuid.uuid4()
env.zenith_cli.create_tenant(tenant_id=tenant1)
tenant1 = env.zenith_cli.create_tenant()
# check tenant1 appeared
helper_compare_tenant_list(pageserver_http_client, env)
# Create new tenant
tenant2 = uuid.uuid4()
env.zenith_cli.create_tenant(tenant_id=tenant2)
tenant2 = env.zenith_cli.create_tenant()
# check tenant2 appeared
helper_compare_tenant_list(pageserver_http_client, env)