Merge branch 'problame/hung-shutdown/demo-hypothesis' into problame/hung-shutdown/fix

test_runner/regress/test_page_service_batching_regressions.py

@@ -3,11 +3,14 @@
 import subprocess
 from pathlib import Path
 
+import pytest
 from fixtures.log_helper import log
 from fixtures.neon_fixtures import NeonEnvBuilder
 
 
-def test_slow_flush(neon_env_builder: NeonEnvBuilder, neon_binpath: Path):
+@pytest.mark.timeout(30)  # test takes <20s if pageserver impl is correct
+@pytest.mark.parametrize("kind", ["pageserver-stop", "tenant-detach"])
+def test_slow_flush(neon_env_builder: NeonEnvBuilder, neon_binpath: Path, kind: str):
     def patch_pageserver_toml(config):
         config["page_service_pipelining"] = {
             "mode": "pipelined",
@@ -20,7 +23,9 @@ def test_slow_flush(neon_env_builder: NeonEnvBuilder, neon_binpath: Path):
 
     log.info("make flush appear slow")
 
-    log.info("filling pipe")
+    log.info("sending requests until pageserver accepts no more")
+    # TODO: extract this into a helper, like subprocess_capture,
+    # so that we capture the stderr from the helper somewhere.
     child = subprocess.Popen(
         [
             neon_binpath / "test_helper_slow_client_reads",
@@ -38,7 +43,18 @@ def test_slow_flush(neon_env_builder: NeonEnvBuilder, neon_binpath: Path):
         raise Exception("unexpected EOF")
     if buf != b"R":
         raise Exception(f"unexpected data: {buf!r}")
-    log.info("helper reports pipe filled")
+    log.info("helper reports pageserver accepts no more requests")
+    log.info(
+        "assuming pageserver connection handle is in a state where TCP has backpressured pageserver=>client response flush() into userspace"
+    )
 
-    log.info("try to shut down the tenant")
-    env.pageserver.tenant_detach(env.initial_tenant)
+    if kind == "pageserver":
+        log.info("try to shut down the pageserver cleanly")
+        env.pageserver.stop()
+    elif kind == "tenant-detach":
+        log.info("try to shut down the tenant")
+        env.pageserver.tenant_detach(env.initial_tenant)
+    else:
+        raise ValueError(f"unexpected kind: {kind}")
+
+    log.info("shutdown did not time out, test passed")