Replace etcd with storage_broker.

This is the replacement itself, the binary landed earlier. See
docs/storage_broker.md.

ref
https://github.com/neondatabase/neon/pull/2466
https://github.com/neondatabase/neon/issues/2394

test_runner/README.md

@@ -13,8 +13,6 @@ Prerequisites:
       below to run from other directories.
 - The neon git repo, including the postgres submodule
   (for some tests, e.g. `pg_regress`)
-- Some tests (involving storage nodes coordination) require etcd installed. Follow
-  [`the guide`](https://etcd.io/docs/v3.5/install/) to obtain it.
 
 ### Test Organization
 

test_runner/fixtures/neon_fixtures.py

@@ -33,7 +33,7 @@ from _pytest.config import Config
 from _pytest.fixtures import FixtureRequest
 from fixtures.log_helper import log
 from fixtures.types import Lsn, TenantId, TimelineId
-from fixtures.utils import Fn, allure_attach_from_dir, etcd_path, get_self_dir, subprocess_capture
+from fixtures.utils import Fn, allure_attach_from_dir, get_self_dir, subprocess_capture
 
 # Type-related stuff
 from psycopg2.extensions import connection as PgConnection
@@ -281,19 +281,22 @@ def port_distributor(worker_base_port: int) -> PortDistributor:
 
 @pytest.fixture(scope="session")
 def default_broker(
-    request: FixtureRequest, port_distributor: PortDistributor, top_output_dir: Path
-) -> Iterator[Etcd]:
+    request: FixtureRequest,
+    port_distributor: PortDistributor,
+    top_output_dir: Path,
+    neon_binpath: Path,
+) -> Iterator[NeonBroker]:
+    # multiple pytest sessions could get launched in parallel, get them different ports/datadirs
     client_port = port_distributor.get_port()
-    # multiple pytest sessions could get launched in parallel, get them different datadirs
-    etcd_datadir = get_test_output_dir(request, top_output_dir) / f"etcd_datadir_{client_port}"
-    etcd_datadir.mkdir(exist_ok=True, parents=True)
-
-    broker = Etcd(
-        datadir=str(etcd_datadir), port=client_port, peer_port=port_distributor.get_port()
+    broker_logfile = (
+        get_test_output_dir(request, top_output_dir) / f"storage_broker_{client_port}.log"
     )
+    broker_logfile.parents[0].mkdir(exist_ok=True, parents=True)
+
+    broker = NeonBroker(logfile=broker_logfile, port=client_port, neon_binpath=neon_binpath)
     yield broker
     broker.stop()
-    allure_attach_from_dir(etcd_datadir)
+    allure_attach_from_dir(Path(broker_logfile))
 
 
 @pytest.fixture(scope="session")
@@ -570,7 +573,7 @@ class NeonEnvBuilder:
         self,
         repo_dir: Path,
         port_distributor: PortDistributor,
-        broker: Etcd,
+        broker: NeonBroker,
         run_id: uuid.UUID,
         mock_s3_server: MockS3Server,
         neon_binpath: Path,
@@ -846,9 +849,8 @@ class NeonEnv:
 
         toml += textwrap.dedent(
             f"""
-            [etcd_broker]
-            broker_endpoints = ['{self.broker.client_url()}']
-            etcd_binary_path = '{self.broker.binary_path}'
+            [broker]
+            listen_addr = '{self.broker.listen_addr()}'
         """
         )
 
@@ -949,7 +951,7 @@ def _shared_simple_env(
     request: FixtureRequest,
     port_distributor: PortDistributor,
     mock_s3_server: MockS3Server,
-    default_broker: Etcd,
+    default_broker: NeonBroker,
     run_id: uuid.UUID,
     top_output_dir: Path,
     neon_binpath: Path,
@@ -1010,7 +1012,7 @@ def neon_env_builder(
     neon_binpath: Path,
     pg_distrib_dir: Path,
     pg_version: str,
-    default_broker: Etcd,
+    default_broker: NeonBroker,
     run_id: uuid.UUID,
 ) -> Iterator[NeonEnvBuilder]:
     """
@@ -1743,7 +1745,7 @@ class NeonPageserver(PgProtocol):
             # All tests print these, when starting up or shutting down
             ".*wal receiver task finished with an error: walreceiver connection handling failure.*",
             ".*Shutdown task error: walreceiver connection handling failure.*",
-            ".*Etcd client error: grpc request error: status: Unavailable.*",
+            ".*wal_connection_manager.*tcp connect error: Connection refused.*",
             ".*query handler for .* failed: Connection reset by peer.*",
             ".*serving compute connection task.*exited with error: Broken pipe.*",
             ".*Connection aborted: error communicating with the server: Broken pipe.*",
@@ -1834,7 +1836,6 @@ class NeonPageserver(PgProtocol):
 
     def assert_no_errors(self):
         logfile = open(os.path.join(self.env.repo_dir, "pageserver.log"), "r")
-
         error_or_warn = re.compile("ERROR|WARN")
         errors = []
         while True:
@@ -2653,51 +2654,36 @@ class SafekeeperHttpClient(requests.Session):
 
 
 @dataclass
-class Etcd:
-    """An object managing etcd instance"""
+class NeonBroker:
+    """An object managing storage_broker instance"""
 
-    datadir: str
+    logfile: Path
     port: int
-    peer_port: int
-    binary_path: Path = field(init=False)
+    neon_binpath: Path
     handle: Optional[subprocess.Popen[Any]] = None  # handle of running daemon
 
-    def __post_init__(self):
-        self.binary_path = etcd_path()
+    def listen_addr(self):
+        return f"127.0.0.1:{self.port}"
 
     def client_url(self):
-        return f"http://127.0.0.1:{self.port}"
+        return f"http://{self.listen_addr()}"
 
     def check_status(self):
-        with requests.Session() as s:
-            s.mount("http://", requests.adapters.HTTPAdapter(max_retries=1))  # do not retry
-            s.get(f"{self.client_url()}/health").raise_for_status()
+        return True  # TODO
 
     def try_start(self):
         if self.handle is not None:
-            log.debug(f"etcd is already running on port {self.port}")
+            log.debug(f"storage_broker is already running on port {self.port}")
             return
 
-        Path(self.datadir).mkdir(exist_ok=True)
-
-        if not self.binary_path.is_file():
-            raise RuntimeError(f"etcd broker binary '{self.binary_path}' is not a file")
-
-        client_url = self.client_url()
-        log.info(f'Starting etcd to listen incoming connections at "{client_url}"')
-        with open(os.path.join(self.datadir, "etcd.log"), "wb") as log_file:
+        listen_addr = self.listen_addr()
+        log.info(f'starting storage_broker to listen incoming connections at "{listen_addr}"')
+        with open(self.logfile, "wb") as logfile:
             args = [
-                self.binary_path,
-                f"--data-dir={self.datadir}",
-                f"--listen-client-urls={client_url}",
-                f"--advertise-client-urls={client_url}",
-                f"--listen-peer-urls=http://127.0.0.1:{self.peer_port}",
-                # Set --quota-backend-bytes to keep the etcd virtual memory
-                # size smaller. Our test etcd clusters are very small.
-                # See https://github.com/etcd-io/etcd/issues/7910
-                "--quota-backend-bytes=100000000",
+                self.neon_binpath / "storage_broker",
+                f"--listen-addr={listen_addr}",
             ]
-            self.handle = subprocess.Popen(args, stdout=log_file, stderr=log_file)
+            self.handle = subprocess.Popen(args, stdout=logfile, stderr=logfile)
 
         # wait for start
         started_at = time.time()
@@ -2707,7 +2693,9 @@ class Etcd:
             except Exception as e:
                 elapsed = time.time() - started_at
                 if elapsed > 5:
-                    raise RuntimeError(f"timed out waiting {elapsed:.0f}s for etcd start: {e}")
+                    raise RuntimeError(
+                        f"timed out waiting {elapsed:.0f}s for storage_broker start: {e}"
+                    )
                 time.sleep(0.5)
             else:
                 break  # success

test_runner/fixtures/utils.py

@@ -1,7 +1,6 @@
 import contextlib
 import os
 import re
-import shutil
 import subprocess
 import tarfile
 import time
@@ -74,13 +73,6 @@ def print_gc_result(row: Dict[str, Any]):
     )
 
 
-def etcd_path() -> Path:
-    path_output = shutil.which("etcd")
-    if path_output is None:
-        raise RuntimeError("etcd not found in PATH")
-    return Path(path_output)
-
-
 def query_scalar(cur: cursor, query: str) -> Any:
     """
     It is a convenience wrapper to avoid repetitions

test_runner/regress/test_compatibility.py

@@ -97,17 +97,19 @@ def test_backward_compatibility(
     ), "COMPATIBILITY_SNAPSHOT_DIR is not set. It should be set to `compatibility_snapshot_pg14` path generateted by test_create_snapshot (ideally generated by the previous version of Neon)"
     compatibility_snapshot_dir = Path(compatibility_snapshot_dir_env).resolve()
 
-    # Copy the snapshot to current directory, and prepare for the test
-    prepare_snapshot(
-        from_dir=compatibility_snapshot_dir,
-        to_dir=test_output_dir / "compatibility_snapshot",
-        port_distributor=port_distributor,
-    )
-
     breaking_changes_allowed = (
         os.environ.get("ALLOW_BACKWARD_COMPATIBILITY_BREAKAGE", "false").lower() == "true"
     )
+
     try:
+        # Copy the snapshot to current directory, and prepare for the test
+        prepare_snapshot(
+            from_dir=compatibility_snapshot_dir,
+            to_dir=test_output_dir / "compatibility_snapshot",
+            neon_binpath=neon_binpath,
+            port_distributor=port_distributor,
+        )
+
         check_neon_works(
             test_output_dir / "compatibility_snapshot" / "repo",
             neon_binpath,
@@ -155,18 +157,21 @@ def test_forward_compatibility(
     compatibility_snapshot_dir = (
         test_output_dir.parent / "test_create_snapshot" / "compatibility_snapshot_pg14"
     )
-    # Copy the snapshot to current directory, and prepare for the test
-    prepare_snapshot(
-        from_dir=compatibility_snapshot_dir,
-        to_dir=test_output_dir / "compatibility_snapshot",
-        port_distributor=port_distributor,
-        pg_distrib_dir=compatibility_postgres_distrib_dir,
-    )
 
     breaking_changes_allowed = (
         os.environ.get("ALLOW_FORWARD_COMPATIBILITY_BREAKAGE", "false").lower() == "true"
     )
+
     try:
+        # Copy the snapshot to current directory, and prepare for the test
+        prepare_snapshot(
+            from_dir=compatibility_snapshot_dir,
+            to_dir=test_output_dir / "compatibility_snapshot",
+            port_distributor=port_distributor,
+            neon_binpath=compatibility_neon_bin,
+            pg_distrib_dir=compatibility_postgres_distrib_dir,
+        )
+
         check_neon_works(
             test_output_dir / "compatibility_snapshot" / "repo",
             compatibility_neon_bin,
@@ -194,6 +199,7 @@ def prepare_snapshot(
     from_dir: Path,
     to_dir: Path,
     port_distributor: PortDistributor,
+    neon_binpath: Path,
     pg_distrib_dir: Optional[Path] = None,
 ):
     assert from_dir.exists(), f"Snapshot '{from_dir}' doesn't exist"
@@ -227,9 +233,14 @@ def prepare_snapshot(
     pageserver_config["listen_pg_addr"] = port_distributor.replace_with_new_port(
         pageserver_config["listen_pg_addr"]
     )
-    pageserver_config["broker_endpoints"] = [
-        port_distributor.replace_with_new_port(ep) for ep in pageserver_config["broker_endpoints"]
-    ]
+    # since storage_broker these are overriden by neon_local during pageserver
+    # start; remove both to prevent unknown options during etcd ->
+    # storage_broker migration. TODO: remove once broker is released
+    pageserver_config.pop("broker_endpoint", None)
+    pageserver_config.pop("broker_endpoints", None)
+    etcd_broker_endpoints = [f"http://localhost:{port_distributor.get_port()}/"]
+    if get_neon_version(neon_binpath) == "49da498f651b9f3a53b56c7c0697636d880ddfe0":
+        pageserver_config["broker_endpoints"] = etcd_broker_endpoints  # old etcd version
 
     if pg_distrib_dir:
         pageserver_config["pg_distrib_dir"] = str(pg_distrib_dir)
@@ -239,10 +250,22 @@ def prepare_snapshot(
 
     snapshot_config_toml = repo_dir / "config"
     snapshot_config = toml.load(snapshot_config_toml)
-    snapshot_config["etcd_broker"]["broker_endpoints"] = [
-        port_distributor.replace_with_new_port(ep)
-        for ep in snapshot_config["etcd_broker"]["broker_endpoints"]
-    ]
+
+    # Provide up/downgrade etcd <-> storage_broker to make forward/backward
+    # compatibility test happy. TODO: leave only the new part once broker is released.
+    if get_neon_version(neon_binpath) == "49da498f651b9f3a53b56c7c0697636d880ddfe0":
+        # old etcd version
+        snapshot_config["etcd_broker"] = {
+            "etcd_binary_path": shutil.which("etcd"),
+            "broker_endpoints": etcd_broker_endpoints,
+        }
+        snapshot_config.pop("broker", None)
+    else:
+        # new storage_broker version
+        broker_listen_addr = f"127.0.0.1:{port_distributor.get_port()}"
+        snapshot_config["broker"] = {"listen_addr": broker_listen_addr}
+        snapshot_config.pop("etcd_broker", None)
+
     snapshot_config["pageserver"]["listen_http_addr"] = port_distributor.replace_with_new_port(
         snapshot_config["pageserver"]["listen_http_addr"]
     )
@@ -277,6 +300,12 @@ def prepare_snapshot(
     ), f"there're files referencing `test_create_snapshot/repo`, this path should be replaced with {repo_dir}:\n{rv.stdout}"
 
 
+# get git SHA of neon binary
+def get_neon_version(neon_binpath: Path):
+    out = subprocess.check_output([neon_binpath / "neon_local", "--version"]).decode("utf-8")
+    return out.split("git:", 1)[1].rstrip()
+
+
 def check_neon_works(
     repo_dir: Path,
     neon_binpath: Path,

test_runner/regress/test_tenant_relocation.py

@@ -7,7 +7,7 @@ from typing import Any, Dict, Optional, Tuple
 import pytest
 from fixtures.log_helper import log
 from fixtures.neon_fixtures import (
-    Etcd,
+    NeonBroker,
     NeonEnv,
     NeonEnvBuilder,
     PageserverHttpClient,
@@ -32,7 +32,7 @@ def new_pageserver_service(
     remote_storage_mock_path: Path,
     pg_port: int,
     http_port: int,
-    broker: Optional[Etcd],
+    broker: Optional[NeonBroker],
     pg_distrib_dir: Path,
 ):
     """
@@ -53,7 +53,7 @@ def new_pageserver_service(
     ]
     if broker is not None:
         cmd.append(
-            f"-c broker_endpoints=['{broker.client_url()}']",
+            f"-c broker_endpoint='{broker.client_url()}'",
         )
     pageserver_client = PageserverHttpClient(
         port=http_port,

test_runner/regress/test_wal_acceptor.py

@@ -16,7 +16,7 @@ from typing import Any, List, Optional
 import pytest
 from fixtures.log_helper import log
 from fixtures.neon_fixtures import (
-    Etcd,
+    NeonBroker,
     NeonEnv,
     NeonEnvBuilder,
     NeonPageserver,
@@ -520,7 +520,7 @@ def test_s3_wal_replay(neon_env_builder: NeonEnvBuilder, remote_storage_kind: Re
                 )
 
             # advance remote_consistent_lsn to trigger WAL trimming
-            # this LSN should be less than commit_lsn, so timeline will be active=true in safekeepers, to push etcd updates
+            # this LSN should be less than commit_lsn, so timeline will be active=true in safekeepers, to push broker updates
             env.safekeepers[0].http_client().record_safekeeper_info(
                 tenant_id, timeline_id, {"remote_consistent_lsn": str(offloaded_seg_end)}
             )
@@ -812,10 +812,10 @@ class SafekeeperEnv:
     ):
         self.repo_dir = repo_dir
         self.port_distributor = port_distributor
-        self.broker = Etcd(
-            datadir=os.path.join(self.repo_dir, "etcd"),
+        self.broker = NeonBroker(
+            logfile=Path(self.repo_dir) / "storage_broker.log",
             port=self.port_distributor.get_port(),
-            peer_port=self.port_distributor.get_port(),
+            neon_binpath=neon_binpath,
         )
         self.pg_bin = pg_bin
         self.num_safekeepers = num_safekeepers
@@ -863,7 +863,7 @@ class SafekeeperEnv:
             str(safekeeper_dir),
             "--id",
             str(i),
-            "--broker-endpoints",
+            "--broker-endpoint",
             self.broker.client_url(),
         ]
         log.info(f'Running command "{" ".join(cmd)}"')