From 130ccb4b679c1fe8af789de702956d2f276de55a Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Sun, 20 Aug 2023 12:33:19 +0300 Subject: [PATCH] Remove initial timeline id troubles (#5044) I made a mistake when I adding `env.initial_timeline: Optional[TimelineId]` in the #3839, should had just generated it and used it to create a specific timeline. This PR fixes those mistakes, and some extra calling into psql which must be slower than python field access. --- test_runner/fixtures/neon_fixtures.py | 9 ++- .../regress/test_disk_usage_eviction.py | 17 ++--- test_runner/regress/test_gc_aggressive.py | 13 ++-- test_runner/regress/test_large_schema.py | 6 +- test_runner/regress/test_layer_eviction.py | 6 +- test_runner/regress/test_metric_collection.py | 9 +-- test_runner/regress/test_ondemand_download.py | 14 ++-- test_runner/regress/test_read_trace.py | 17 ++--- test_runner/regress/test_remote_storage.py | 15 ++--- test_runner/regress/test_tenant_detach.py | 22 +++---- .../test_tenants_with_remote_storage.py | 8 +-- .../regress/test_threshold_based_eviction.py | 2 - test_runner/regress/test_timeline_delete.py | 17 ++--- test_runner/regress/test_wal_acceptor.py | 64 ++++++------------- 14 files changed, 81 insertions(+), 138 deletions(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 61cd169fa3..b2cd0fe968 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -427,6 +427,7 @@ class NeonEnvBuilder: default_branch_name: str = DEFAULT_BRANCH_NAME, preserve_database_files: bool = False, initial_tenant: Optional[TenantId] = None, + initial_timeline: Optional[TimelineId] = None, ): self.repo_dir = repo_dir self.rust_log_override = rust_log_override @@ -452,6 +453,7 @@ class NeonEnvBuilder: self.pg_version = pg_version self.preserve_database_files = preserve_database_files self.initial_tenant = initial_tenant or TenantId.generate() + self.initial_timeline = initial_timeline or TimelineId.generate() def init_configs(self) -> NeonEnv: # Cannot create more than one environment from one builder @@ -473,9 +475,10 @@ class NeonEnvBuilder: f"Services started, creating initial tenant {env.initial_tenant} and its initial timeline" ) initial_tenant, initial_timeline = env.neon_cli.create_tenant( - tenant_id=env.initial_tenant, conf=initial_tenant_conf + tenant_id=env.initial_tenant, conf=initial_tenant_conf, timeline_id=env.initial_timeline ) - env.initial_timeline = initial_timeline + assert env.initial_tenant == initial_tenant + assert env.initial_timeline == initial_timeline log.info(f"Initial timeline {initial_tenant}/{initial_timeline} created successfully") return env @@ -784,7 +787,7 @@ class NeonEnv: # generate initial tenant ID here instead of letting 'neon init' generate it, # so that we don't need to dig it out of the config file afterwards. self.initial_tenant = config.initial_tenant - self.initial_timeline: Optional[TimelineId] = None + self.initial_timeline = config.initial_timeline # Create a config file corresponding to the options toml = textwrap.dedent( diff --git a/test_runner/regress/test_disk_usage_eviction.py b/test_runner/regress/test_disk_usage_eviction.py index 1e9b130b1c..182069315e 100644 --- a/test_runner/regress/test_disk_usage_eviction.py +++ b/test_runner/regress/test_disk_usage_eviction.py @@ -1,4 +1,3 @@ -import shutil import time from dataclasses import dataclass from typing import Dict, Tuple @@ -14,7 +13,7 @@ from fixtures.neon_fixtures import ( ) from fixtures.pageserver.http import PageserverHttpClient from fixtures.pageserver.utils import wait_for_upload_queue_empty -from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind +from fixtures.remote_storage import RemoteStorageKind from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import wait_until @@ -138,22 +137,14 @@ def eviction_env(request, neon_env_builder: NeonEnvBuilder, pg_bin: PgBin) -> Ev neon_env_builder.enable_remote_storage(RemoteStorageKind.LOCAL_FS, f"{request.node.name}") - env = neon_env_builder.init_start() + # initial tenant will not be present on this pageserver + env = neon_env_builder.init_configs() + env.start() pageserver_http = env.pageserver.http_client() # allow because we are invoking this manually; we always warn on executing disk based eviction env.pageserver.allowed_errors.append(r".* running disk usage based eviction due to pressure.*") - # remove the initial tenant - assert env.initial_timeline - pageserver_http.tenant_detach(env.initial_tenant) - assert isinstance(env.remote_storage, LocalFsStorage) - tenant_remote_storage = env.remote_storage.root / "tenants" / str(env.initial_tenant) - assert tenant_remote_storage.is_dir() - shutil.rmtree(tenant_remote_storage) - env.initial_tenant = TenantId("0" * 32) - env.initial_timeline = None - # Choose small layer_size so that we can use low pgbench_scales and still get a large count of layers. # Large count of layers and small layer size is good for testing because it makes evictions predictable. # Predictable in the sense that many layer evictions will be required to reach the eviction target, because diff --git a/test_runner/regress/test_gc_aggressive.py b/test_runner/regress/test_gc_aggressive.py index 53fa70903f..be817521cd 100644 --- a/test_runner/regress/test_gc_aggressive.py +++ b/test_runner/regress/test_gc_aggressive.py @@ -11,8 +11,7 @@ from fixtures.neon_fixtures import ( wait_for_last_flush_lsn, ) from fixtures.remote_storage import RemoteStorageKind -from fixtures.types import TenantId, TimelineId -from fixtures.utils import query_scalar +from fixtures.types import TimelineId # Test configuration # @@ -71,13 +70,11 @@ def test_gc_aggressive(neon_env_builder: NeonEnvBuilder): # Disable pitr, because here we want to test branch creation after GC neon_env_builder.pageserver_config_override = "tenant_config={pitr_interval = '0 sec'}" env = neon_env_builder.init_start() - env.neon_cli.create_branch("test_gc_aggressive", "main") + timeline = env.neon_cli.create_branch("test_gc_aggressive", "main") endpoint = env.endpoints.create_start("test_gc_aggressive") log.info("postgres is running on test_gc_aggressive branch") with endpoint.cursor() as cur: - timeline = TimelineId(query_scalar(cur, "SHOW neon.timeline_id")) - # Create table, and insert the first 100 rows cur.execute("CREATE TABLE foo (id int, counter int, t text)") cur.execute( @@ -109,7 +106,8 @@ def test_gc_index_upload(neon_env_builder: NeonEnvBuilder, remote_storage_kind: ) env = neon_env_builder.init_start() - env.neon_cli.create_branch("test_gc_index_upload", "main") + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_gc_index_upload", "main") endpoint = env.endpoints.create_start("test_gc_index_upload") pageserver_http = env.pageserver.http_client() @@ -117,9 +115,6 @@ def test_gc_index_upload(neon_env_builder: NeonEnvBuilder, remote_storage_kind: pg_conn = endpoint.connect() cur = pg_conn.cursor() - tenant_id = TenantId(query_scalar(cur, "SHOW neon.tenant_id")) - timeline_id = TimelineId(query_scalar(cur, "SHOW neon.timeline_id")) - cur.execute("CREATE TABLE foo (id int, counter int, t text)") cur.execute( """ diff --git a/test_runner/regress/test_large_schema.py b/test_runner/regress/test_large_schema.py index ac83131ba2..72bf32fcd3 100644 --- a/test_runner/regress/test_large_schema.py +++ b/test_runner/regress/test_large_schema.py @@ -74,9 +74,9 @@ def test_large_schema(neon_env_builder: NeonEnvBuilder): cur.execute("select * from pg_depend order by refclassid, refobjid, refobjsubid") # Check layer file sizes - tenant_id = endpoint.safe_psql("show neon.tenant_id")[0][0] - timeline_id = endpoint.safe_psql("show neon.timeline_id")[0][0] - timeline_path = "{}/tenants/{}/timelines/{}/".format(env.repo_dir, tenant_id, timeline_id) + timeline_path = "{}/tenants/{}/timelines/{}/".format( + env.repo_dir, env.initial_tenant, env.initial_timeline + ) for filename in os.listdir(timeline_path): if filename.startswith("00000"): log.info(f"layer {filename} size is {os.path.getsize(timeline_path + filename)}") diff --git a/test_runner/regress/test_layer_eviction.py b/test_runner/regress/test_layer_eviction.py index 1269210d0d..8f627defb5 100644 --- a/test_runner/regress/test_layer_eviction.py +++ b/test_runner/regress/test_layer_eviction.py @@ -8,7 +8,7 @@ from fixtures.neon_fixtures import ( ) from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_for_upload from fixtures.remote_storage import RemoteStorageKind -from fixtures.types import Lsn, TenantId, TimelineId +from fixtures.types import Lsn from fixtures.utils import query_scalar @@ -34,8 +34,8 @@ def test_basic_eviction( client = env.pageserver.http_client() endpoint = env.endpoints.create_start("main") - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline # Create a number of layers in the tenant with endpoint.cursor() as cur: diff --git a/test_runner/regress/test_metric_collection.py b/test_runner/regress/test_metric_collection.py index 80ffe5126d..3f4b42707a 100644 --- a/test_runner/regress/test_metric_collection.py +++ b/test_runner/regress/test_metric_collection.py @@ -18,8 +18,7 @@ from fixtures.neon_fixtures import ( ) from fixtures.port_distributor import PortDistributor from fixtures.remote_storage import RemoteStorageKind -from fixtures.types import TenantId, TimelineId -from fixtures.utils import query_scalar +from fixtures.types import TenantId from pytest_httpserver import HTTPServer from werkzeug.wrappers.request import Request from werkzeug.wrappers.response import Response @@ -115,15 +114,13 @@ def test_metric_collection( # Order of fixtures shutdown is not specified, and if http server gets down # before pageserver, pageserver log might contain such errors in the end. env.pageserver.allowed_errors.append(".*metrics endpoint refused the sent metrics*") - env.neon_cli.create_branch("test_metric_collection") + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_metric_collection") endpoint = env.endpoints.create_start("test_metric_collection") pg_conn = endpoint.connect() cur = pg_conn.cursor() - tenant_id = TenantId(query_scalar(cur, "SHOW neon.tenant_id")) - timeline_id = TimelineId(query_scalar(cur, "SHOW neon.timeline_id")) - cur.execute("CREATE TABLE foo (id int, counter int, t text)") cur.execute( """ diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py index 17a63535cf..0640e65e57 100644 --- a/test_runner/regress/test_ondemand_download.py +++ b/test_runner/regress/test_ondemand_download.py @@ -78,8 +78,8 @@ def test_ondemand_download_large_rel( client = env.pageserver.http_client() - tenant_id = endpoint.safe_psql("show neon.tenant_id")[0][0] - timeline_id = endpoint.safe_psql("show neon.timeline_id")[0][0] + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline # We want to make sure that the data is large enough that the keyspace is partitioned. num_rows = 1000000 @@ -183,8 +183,8 @@ def test_ondemand_download_timetravel( client = env.pageserver.http_client() - tenant_id = endpoint.safe_psql("show neon.tenant_id")[0][0] - timeline_id = endpoint.safe_psql("show neon.timeline_id")[0][0] + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline lsns = [] @@ -342,8 +342,8 @@ def test_download_remote_layers_api( client = env.pageserver.http_client() - tenant_id = endpoint.safe_psql("show neon.tenant_id")[0][0] - timeline_id = endpoint.safe_psql("show neon.timeline_id")[0][0] + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline table_len = 10000 with endpoint.cursor() as cur: @@ -516,7 +516,6 @@ def test_compaction_downloads_on_demand_without_image_creation( tenant_id = env.initial_tenant timeline_id = env.initial_timeline - assert timeline_id is not None with env.endpoints.create_start("main") as endpoint: # no particular reason to create the layers like this, but we are sure @@ -590,7 +589,6 @@ def test_compaction_downloads_on_demand_with_image_creation( env = neon_env_builder.init_start(initial_tenant_conf=stringify(conf)) tenant_id = env.initial_tenant timeline_id = env.initial_timeline - assert timeline_id is not None pageserver_http = env.pageserver.http_client() diff --git a/test_runner/regress/test_read_trace.py b/test_runner/regress/test_read_trace.py index 9ebe53fc17..cae8ca3919 100644 --- a/test_runner/regress/test_read_trace.py +++ b/test_runner/regress/test_read_trace.py @@ -2,7 +2,7 @@ from contextlib import closing from fixtures.neon_fixtures import NeonEnvBuilder from fixtures.pageserver.utils import wait_for_last_record_lsn -from fixtures.types import Lsn, TenantId, TimelineId +from fixtures.types import Lsn from fixtures.utils import query_scalar @@ -12,24 +12,21 @@ from fixtures.utils import query_scalar # Additionally, tests that pageserver is able to create tenants with custom configs. def test_read_request_tracing(neon_env_builder: NeonEnvBuilder): neon_env_builder.num_safekeepers = 1 - env = neon_env_builder.init_start() - - tenant, _ = env.neon_cli.create_tenant( - conf={ + env = neon_env_builder.init_start( + initial_tenant_conf={ "trace_read_requests": "true", } ) - timeline = env.neon_cli.create_timeline("test_trace_replay", tenant_id=tenant) - endpoint = env.endpoints.create_start("test_trace_replay", "main", tenant) + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline + endpoint = env.endpoints.create_start("main") with closing(endpoint.connect()) as conn: with conn.cursor() as cur: cur.execute("create table t (i integer);") cur.execute(f"insert into t values (generate_series(1,{10000}));") cur.execute("select count(*) from t;") - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) current_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()")) # wait until pageserver receives that data pageserver_http = env.pageserver.http_client() @@ -38,5 +35,5 @@ def test_read_request_tracing(neon_env_builder: NeonEnvBuilder): # Stop postgres so we drop the connection and flush the traces endpoint.stop() - trace_path = env.repo_dir / "traces" / str(tenant) / str(timeline) + trace_path = env.repo_dir / "traces" / str(tenant_id) / str(timeline_id) assert trace_path.exists() diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index 4f5b193ce2..b865e3ce24 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -95,12 +95,12 @@ def test_remote_storage_backup_and_restore( client = env.pageserver.http_client() - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline # Thats because of UnreliableWrapper's injected failures env.pageserver.allowed_errors.append( - f".*failed to fetch tenant deletion mark at tenants/({tenant_id}|{env.initial_tenant})/deleted attempt 1.*" + f".*failed to fetch tenant deletion mark at tenants/{tenant_id}/deleted attempt 1.*" ) checkpoint_numbers = range(1, 3) @@ -403,8 +403,7 @@ def test_remote_timeline_client_calls_started_metric( ) tenant_id = env.initial_tenant - assert env.initial_timeline is not None - timeline_id: TimelineId = env.initial_timeline + timeline_id = env.initial_timeline client = env.pageserver.http_client() @@ -542,8 +541,7 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue( } ) tenant_id = env.initial_tenant - assert env.initial_timeline is not None - timeline_id: TimelineId = env.initial_timeline + timeline_id = env.initial_timeline timeline_path = env.timeline_dir(tenant_id, timeline_id) @@ -808,8 +806,7 @@ def test_compaction_delete_before_upload( ) tenant_id = env.initial_tenant - assert env.initial_timeline is not None - timeline_id: TimelineId = env.initial_timeline + timeline_id = env.initial_timeline client = env.pageserver.http_client() diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index b189510a9e..07b751bcca 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -463,8 +463,8 @@ def test_detach_while_attaching( client = env.pageserver.http_client() - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline # Attempts to connect from compute to pageserver while the tenant is # temporarily detached produces these errors in the pageserver log. @@ -615,8 +615,8 @@ def test_ignored_tenant_download_missing_layers( pageserver_http = env.pageserver.http_client() endpoint = env.endpoints.create_start("main") - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline # Attempts to connect from compute to pageserver while the tenant is # temporarily detached produces these errors in the pageserver log. @@ -679,10 +679,10 @@ def test_ignored_tenant_stays_broken_without_metadata( ) env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() - endpoint = env.endpoints.create_start("main") + env.endpoints.create_start("main") - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline # Attempts to connect from compute to pageserver while the tenant is # temporarily detached produces these errors in the pageserver log. @@ -723,9 +723,9 @@ def test_load_attach_negatives( ) env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() - endpoint = env.endpoints.create_start("main") + env.endpoints.create_start("main") - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) + tenant_id = env.initial_tenant # Attempts to connect from compute to pageserver while the tenant is # temporarily detached produces these errors in the pageserver log. @@ -773,8 +773,8 @@ def test_ignore_while_attaching( pageserver_http = env.pageserver.http_client() - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline # Attempts to connect from compute to pageserver while the tenant is # temporarily detached produces these errors in the pageserver log. diff --git a/test_runner/regress/test_tenants_with_remote_storage.py b/test_runner/regress/test_tenants_with_remote_storage.py index 397a2ea534..2925f8c2da 100644 --- a/test_runner/regress/test_tenants_with_remote_storage.py +++ b/test_runner/regress/test_tenants_with_remote_storage.py @@ -142,8 +142,8 @@ def test_tenants_attached_after_download( client = env.pageserver.http_client() - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline # Thats because of UnreliableWrapper's injected failures env.pageserver.allowed_errors.append( @@ -252,8 +252,8 @@ def test_tenant_redownloads_truncated_file_on_startup( pageserver_http = env.pageserver.http_client() endpoint = env.endpoints.create_start("main") - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline with endpoint.cursor() as cur: cur.execute("CREATE TABLE t1 AS VALUES (123, 'foobar');") diff --git a/test_runner/regress/test_threshold_based_eviction.py b/test_runner/regress/test_threshold_based_eviction.py index b1bc9623ce..a0e423e7ff 100644 --- a/test_runner/regress/test_threshold_based_eviction.py +++ b/test_runner/regress/test_threshold_based_eviction.py @@ -10,7 +10,6 @@ from fixtures.neon_fixtures import ( ) from fixtures.pageserver.http import LayerMapInfo from fixtures.remote_storage import RemoteStorageKind -from fixtures.types import TimelineId from pytest_httpserver import HTTPServer # NB: basic config change tests are in test_tenant_conf.py @@ -45,7 +44,6 @@ def test_threshold_based_eviction( ) tenant_id, timeline_id = env.initial_tenant, env.initial_timeline - assert isinstance(timeline_id, TimelineId) ps_http = env.pageserver.http_client() assert ps_http.tenant_config(tenant_id).effective_config["eviction_policy"] == { diff --git a/test_runner/regress/test_timeline_delete.py b/test_runner/regress/test_timeline_delete.py index e641065c43..916c0111f7 100644 --- a/test_runner/regress/test_timeline_delete.py +++ b/test_runner/regress/test_timeline_delete.py @@ -359,8 +359,8 @@ def test_timeline_resurrection_on_attach( ps_http = env.pageserver.http_client() pg = env.endpoints.create_start("main") - tenant_id = TenantId(pg.safe_psql("show neon.tenant_id")[0][0]) - main_timeline_id = TimelineId(pg.safe_psql("show neon.timeline_id")[0][0]) + tenant_id = env.initial_tenant + main_timeline_id = env.initial_timeline with pg.cursor() as cur: cur.execute("CREATE TABLE f (i integer);") @@ -512,8 +512,6 @@ def test_timeline_delete_fail_before_local_delete(neon_env_builder: NeonEnvBuild ), ) - assert env.initial_timeline is not None - for timeline_id in (intermediate_timeline_id, env.initial_timeline): timeline_delete_wait_completed( ps_http, tenant_id=env.initial_tenant, timeline_id=timeline_id @@ -716,13 +714,9 @@ def test_timeline_delete_works_for_remote_smoke( ps_http = env.pageserver.http_client() pg = env.endpoints.create_start("main") - tenant_id = TenantId(pg.safe_psql("show neon.tenant_id")[0][0]) - main_timeline_id = TimelineId(pg.safe_psql("show neon.timeline_id")[0][0]) + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline - assert tenant_id == env.initial_tenant - assert main_timeline_id == env.initial_timeline - - assert env.initial_timeline is not None timeline_ids = [env.initial_timeline] for i in range(2): branch_timeline_id = env.neon_cli.create_branch(f"new{i}", "main") @@ -743,9 +737,8 @@ def test_timeline_delete_works_for_remote_smoke( log.info("waiting for checkpoint upload") wait_for_upload(ps_http, tenant_id, branch_timeline_id, current_lsn) log.info("upload of checkpoint is done") - timeline_id = TimelineId(pg.safe_psql("show neon.timeline_id")[0][0]) - timeline_ids.append(timeline_id) + timeline_ids.append(branch_timeline_id) for timeline_id in timeline_ids: assert_prefix_not_empty( diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index c471b18db7..8ca93845b2 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -270,7 +270,8 @@ def test_broker(neon_env_builder: NeonEnvBuilder): neon_env_builder.enable_local_fs_remote_storage() env = neon_env_builder.init_start() - env.neon_cli.create_branch("test_broker", "main") + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_broker", "main") # FIXME: Is this expected? env.pageserver.allowed_errors.append( @@ -280,10 +281,6 @@ def test_broker(neon_env_builder: NeonEnvBuilder): endpoint = env.endpoints.create_start("test_broker") endpoint.safe_psql("CREATE TABLE t(key int primary key, value text)") - # learn neon timeline from compute - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) - # wait until remote_consistent_lsn gets advanced on all safekeepers clients = [sk.http_client() for sk in env.safekeepers] stat_before = [cli.timeline_status(tenant_id, timeline_id) for cli in clients] @@ -325,7 +322,8 @@ def test_wal_removal(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): ".*init_tenant_mgr: marking .* as locally complete, while it doesnt exist in remote index.*" ) - env.neon_cli.create_branch("test_safekeepers_wal_removal") + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_safekeepers_wal_removal") endpoint = env.endpoints.create_start("test_safekeepers_wal_removal") # Note: it is important to insert at least two segments, as currently @@ -338,9 +336,6 @@ def test_wal_removal(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): ] ) - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) - # force checkpoint to advance remote_consistent_lsn pageserver_conn_options = {} if auth_enabled: @@ -451,13 +446,10 @@ def test_wal_backup(neon_env_builder: NeonEnvBuilder, remote_storage_kind: Remot env = neon_env_builder.init_start() - env.neon_cli.create_branch("test_safekeepers_wal_backup") + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_safekeepers_wal_backup") endpoint = env.endpoints.create_start("test_safekeepers_wal_backup") - # learn neon timeline from compute - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) - pg_conn = endpoint.connect() cur = pg_conn.cursor() cur.execute("create table t(key int, value text)") @@ -505,14 +497,11 @@ def test_s3_wal_replay(neon_env_builder: NeonEnvBuilder, remote_storage_kind: Re neon_env_builder.remote_storage_users = RemoteStorageUsers.SAFEKEEPER env = neon_env_builder.init_start() - env.neon_cli.create_branch("test_s3_wal_replay") + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_s3_wal_replay") endpoint = env.endpoints.create_start("test_s3_wal_replay") - # learn neon timeline from compute - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) - expected_sum = 0 with closing(endpoint.connect()) as conn: @@ -796,15 +785,12 @@ def test_timeline_status(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): neon_env_builder.auth_enabled = auth_enabled env = neon_env_builder.init_start() - env.neon_cli.create_branch("test_timeline_status") + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_timeline_status") endpoint = env.endpoints.create_start("test_timeline_status") wa = env.safekeepers[0] - # learn neon timeline from compute - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) - if not auth_enabled: wa_http_cli = wa.http_client() wa_http_cli.check_status() @@ -887,15 +873,12 @@ def test_start_replication_term(neon_env_builder: NeonEnvBuilder): env = neon_env_builder.init_start() - env.neon_cli.create_branch("test_start_replication_term") + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_start_replication_term") endpoint = env.endpoints.create_start("test_start_replication_term") endpoint.safe_psql("CREATE TABLE t(key int primary key, value text)") - # learn neon timeline from compute - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) - sk = env.safekeepers[0] sk_http_cli = sk.http_client() tli_status = sk_http_cli.timeline_status(tenant_id, timeline_id) @@ -922,15 +905,12 @@ def test_sk_auth(neon_env_builder: NeonEnvBuilder): neon_env_builder.auth_enabled = True env = neon_env_builder.init_start() - env.neon_cli.create_branch("test_sk_auth") - endpoint = env.endpoints.create_start("test_sk_auth") + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_sk_auth") + env.endpoints.create_start("test_sk_auth") sk = env.safekeepers[0] - # learn neon timeline from compute - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) - tenant_token = env.auth_keys.generate_tenant_token(tenant_id) full_token = env.auth_keys.generate_safekeeper_token() @@ -1185,7 +1165,8 @@ def test_replace_safekeeper(neon_env_builder: NeonEnvBuilder): neon_env_builder.num_safekeepers = 4 env = neon_env_builder.init_start() - env.neon_cli.create_branch("test_replace_safekeeper") + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_replace_safekeeper") log.info("Use only first 3 safekeepers") env.safekeepers[3].stop() @@ -1193,10 +1174,6 @@ def test_replace_safekeeper(neon_env_builder: NeonEnvBuilder): endpoint.active_safekeepers = [1, 2, 3] endpoint.start() - # learn neon timeline from compute - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) - execute_payload(endpoint) show_statuses(env.safekeepers, tenant_id, timeline_id) @@ -1448,7 +1425,8 @@ def test_pull_timeline(neon_env_builder: NeonEnvBuilder): neon_env_builder.num_safekeepers = 4 env = neon_env_builder.init_start() - env.neon_cli.create_branch("test_pull_timeline") + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_pull_timeline") log.info("Use only first 3 safekeepers") env.safekeepers[3].stop() @@ -1456,10 +1434,6 @@ def test_pull_timeline(neon_env_builder: NeonEnvBuilder): endpoint.active_safekeepers = [1, 2, 3] endpoint.start() - # learn neon timeline from compute - tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) - timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) - execute_payload(endpoint) show_statuses(env.safekeepers, tenant_id, timeline_id)