From 137d616e76fbd6baca0edcd4c60baf27dc1cf07c Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Thu, 24 Feb 2022 11:20:07 -0500 Subject: [PATCH] [proxy] Add pytest fixture (#1311) --- poetry.lock | 25 +++++++++--- pyproject.toml | 1 + test_runner/batch_others/test_proxy.py | 2 + test_runner/fixtures/zenith_fixtures.py | 53 ++++++++++++++++++++++++- 4 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 test_runner/batch_others/test_proxy.py diff --git a/poetry.lock b/poetry.lock index 25c115eec2..fe18ad226c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -91,6 +91,14 @@ botocore = ">=1.11.3" future = "*" wrapt = "*" +[[package]] +name = "backoff" +version = "1.11.1" +description = "Function decoration for backoff and retry" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + [[package]] name = "boto3" version = "1.20.40" @@ -814,11 +822,11 @@ python-versions = "*" [[package]] name = "moto" -version = "3.0.0" +version = "3.0.4" description = "A library that allows your python tests to easily mock out the boto library" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" [package.dependencies] aws-xray-sdk = {version = ">=0.93,<0.96 || >0.96", optional = true, markers = "extra == \"server\""} @@ -848,7 +856,8 @@ xmltodict = "*" [package.extras] all = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.4.0)", "sshpubkeys (>=3.1.0)", "setuptools"] -apigateway = ["python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)"] +apigateway = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)"] +apigatewayv2 = ["PyYAML (>=5.1)"] appsync = ["graphql-core"] awslambda = ["docker (>=2.5.1)"] batch = ["docker (>=2.5.1)"] @@ -1352,7 +1361,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "0fa6c9377fbc827240d18d8b7e3742def37e90fc3277fddf8525d82dabd13090" +content-hash = "58762accad4122026c650fa43421a900546e89f9908e2268410e7b11cc8c6c4e" [metadata.files] aiopg = [ @@ -1395,6 +1404,10 @@ aws-xray-sdk = [ {file = "aws-xray-sdk-2.9.0.tar.gz", hash = "sha256:b0cd972db218d4d8f7b53ad806fc6184626b924c4997ae58fc9f2a8cd1281568"}, {file = "aws_xray_sdk-2.9.0-py2.py3-none-any.whl", hash = "sha256:98216b3ac8281b51b59a8703f8ec561c460807d9d0679838f5c0179d381d7e58"}, ] +backoff = [ + {file = "backoff-1.11.1-py2.py3-none-any.whl", hash = "sha256:61928f8fa48d52e4faa81875eecf308eccfb1016b018bb6bd21e05b5d90a96c5"}, + {file = "backoff-1.11.1.tar.gz", hash = "sha256:ccb962a2378418c667b3c979b504fdeb7d9e0d29c0579e3b13b86467177728cb"}, +] boto3 = [ {file = "boto3-1.20.40-py3-none-any.whl", hash = "sha256:cfe85589e4a0a997c7b9ae7432400b03fa6fa5fea29fdc48db3099a903b76998"}, {file = "boto3-1.20.40.tar.gz", hash = "sha256:66aef9a6d8cad393f69166112ba49e14e2c6766f9278c96134101314a9af2992"}, @@ -1666,8 +1679,8 @@ mccabe = [ {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] moto = [ - {file = "moto-3.0.0-py2.py3-none-any.whl", hash = "sha256:762d33bbad3642c687f6495e69331318bef43f9aa662174397706ec3ad2a3578"}, - {file = "moto-3.0.0.tar.gz", hash = "sha256:d6b00a2663290e7ebb06823d5ffcb124c8dc9bf526b878539ef7c4a377fd8255"}, + {file = "moto-3.0.4-py2.py3-none-any.whl", hash = "sha256:79646213d8438385182f4eea79e28725f94b3d0d3dc9a3eda81db47e0ebef6cc"}, + {file = "moto-3.0.4.tar.gz", hash = "sha256:168b8a3cb4dd8a6df8e51d582761cefa9657b9f45ac7e1eb24dae394ebc9e000"}, ] mypy = [ {file = "mypy-0.910-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457"}, diff --git a/pyproject.toml b/pyproject.toml index e682879d80..7dbdcc0304 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ types-psycopg2 = "^2.9.6" boto3 = "^1.20.40" boto3-stubs = "^1.20.40" moto = {version = "^3.0.0", extras = ["server"]} +backoff = "^1.11.1" [tool.poetry.dev-dependencies] yapf = "==0.31.0" diff --git a/test_runner/batch_others/test_proxy.py b/test_runner/batch_others/test_proxy.py new file mode 100644 index 0000000000..9510e880b2 --- /dev/null +++ b/test_runner/batch_others/test_proxy.py @@ -0,0 +1,2 @@ +def test_proxy_select_1(static_proxy): + static_proxy.safe_psql("select 1;") diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index 128fffb077..b4b3de1db3 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -32,6 +32,7 @@ from typing_extensions import Literal import pytest import requests +import backoff # type: ignore from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture) from fixtures.log_helper import log @@ -237,10 +238,15 @@ def port_distributor(worker_base_port): class PgProtocol: """ Reusable connection logic """ - def __init__(self, host: str, port: int, username: Optional[str] = None): + def __init__(self, + host: str, + port: int, + username: Optional[str] = None, + password: Optional[str] = None): self.host = host self.port = port self.username = username + self.password = password def connstr(self, *, @@ -252,6 +258,7 @@ class PgProtocol: """ username = username or self.username + password = password or self.password res = f'host={self.host} port={self.port} dbname={dbname}' if username: @@ -1175,6 +1182,50 @@ def vanilla_pg(test_output_dir: str) -> Iterator[VanillaPostgres]: yield vanilla_pg +class ZenithProxy(PgProtocol): + def __init__(self, port: int): + super().__init__(host="127.0.0.1", username="pytest", password="pytest", port=port) + self.http_port = 7001 + self._popen: Optional[subprocess.Popen[bytes]] = None + + def start_static(self, addr="127.0.0.1:5432") -> None: + assert self._popen is None + + # Start proxy + bin_proxy = os.path.join(str(zenith_binpath), 'proxy') + args = [bin_proxy] + args.extend(["--http", f"{self.host}:{self.http_port}"]) + args.extend(["--proxy", f"{self.host}:{self.port}"]) + args.extend(["--auth-method", "password"]) + args.extend(["--static-router", addr]) + self._popen = subprocess.Popen(args) + self._wait_until_ready() + + @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_time=10) + def _wait_until_ready(self): + requests.get(f"http://{self.host}:{self.http_port}/v1/status") + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + if self._popen is not None: + # NOTE the process will die when we're done with tests anyway, because + # it's a child process. This is mostly to clean up in between different tests. + self._popen.kill() + + +@pytest.fixture(scope='function') +def static_proxy(vanilla_pg) -> Iterator[ZenithProxy]: + """Zenith proxy that routes directly to vanilla postgres.""" + vanilla_pg.start() + vanilla_pg.safe_psql("create user pytest with password 'pytest';") + + with ZenithProxy(4432) as proxy: + proxy.start_static() + yield proxy + + class Postgres(PgProtocol): """ An object representing a running postgres daemon. """ def __init__(self, env: ZenithEnv, tenant_id: uuid.UUID, port: int):