Add etcd to neon_local

test_runner/batch_others/test_wal_acceptor.py

@@ -13,7 +13,7 @@ from dataclasses import dataclass, field
 from multiprocessing import Process, Value
 from pathlib import Path
 from fixtures.zenith_fixtures import PgBin, Etcd, Postgres, Safekeeper, ZenithEnv, ZenithEnvBuilder, PortDistributor, SafekeeperPort, zenith_binpath, PgProtocol
-from fixtures.utils import etcd_path, get_dir_size, lsn_to_hex, mkdir_if_needed, lsn_from_hex
+from fixtures.utils import get_dir_size, lsn_to_hex, mkdir_if_needed, lsn_from_hex
 from fixtures.log_helper import log
 from typing import List, Optional, Any
 
@@ -327,7 +327,6 @@ def test_race_conditions(zenith_env_builder: ZenithEnvBuilder, stop_value):
 
 
 # Test that safekeepers push their info to the broker and learn peer status from it
-@pytest.mark.skipif(etcd_path() is None, reason="requires etcd which is not present in PATH")
 def test_broker(zenith_env_builder: ZenithEnvBuilder):
     zenith_env_builder.num_safekeepers = 3
     zenith_env_builder.enable_local_fs_remote_storage()
@@ -369,7 +368,6 @@ def test_broker(zenith_env_builder: ZenithEnvBuilder):
 
 
 # Test that old WAL consumed by peers and pageserver is removed from safekeepers.
-@pytest.mark.skipif(etcd_path() is None, reason="requires etcd which is not present in PATH")
 def test_wal_removal(zenith_env_builder: ZenithEnvBuilder):
     zenith_env_builder.num_safekeepers = 2
     # to advance remote_consistent_llsn

test_runner/fixtures/utils.py

@@ -1,8 +1,9 @@
 import os
 import shutil
 import subprocess
+from pathlib import Path
 
-from typing import Any, List
+from typing import Any, List, Optional
 from fixtures.log_helper import log
 
 
@@ -80,9 +81,12 @@ def print_gc_result(row):
         .format_map(row))
 
 
-# path to etcd binary or None if not present.
-def etcd_path():
-    return shutil.which("etcd")
+def etcd_path() -> Path:
+    path_output = shutil.which("etcd")
+    if path_output is None:
+        raise RuntimeError('etcd not found in PATH')
+    else:
+        return Path(path_output)
 
 
 # Traverse directory to get total size.

test_runner/fixtures/zenith_fixtures.py

@@ -555,7 +555,9 @@ class ZenithEnv:
 
         self.broker = config.broker
         toml += textwrap.dedent(f"""
+            [etcd_broker]
             broker_endpoints = ['{self.broker.client_url()}']
+            etcd_binary_path = '{self.broker.binary_path}'
         """)
 
         # Create config for pageserver
@@ -1846,6 +1848,7 @@ class Etcd:
     datadir: str
     port: int
     peer_port: int
+    binary_path: Path = etcd_path()
     handle: Optional[subprocess.Popen[Any]] = None  # handle of running daemon
 
     def client_url(self):
@@ -1858,15 +1861,15 @@ class Etcd:
 
     def start(self):
         pathlib.Path(self.datadir).mkdir(exist_ok=True)
-        etcd_full_path = etcd_path()
-        if etcd_full_path is None:
-            raise Exception('etcd binary not found locally')
+
+        if not self.binary_path.is_file():
+            raise RuntimeError(f"etcd broker binary '{self.binary_path}' is not a file")
 
         client_url = self.client_url()
         log.info(f'Starting etcd to listen incoming connections at "{client_url}"')
         with open(os.path.join(self.datadir, "etcd.log"), "wb") as log_file:
             args = [
-                etcd_full_path,
+                self.binary_path,
                 f"--data-dir={self.datadir}",
                 f"--listen-client-urls={client_url}",
                 f"--advertise-client-urls={client_url}",
@@ -1927,8 +1930,7 @@ SKIP_DIRS = frozenset(('pg_wal',
                        'pg_stat_tmp',
                        'pg_subtrans',
                        'pg_logical',
-                       'pg_replslot/wal_proposer_slot',
-                       'pg_xact'))
+                       'pg_replslot/wal_proposer_slot'))
 
 SKIP_FILES = frozenset(('pg_internal.init',
                         'pg.log',