From 02e5083695d0ed17f7dbb1ca852f504fca42fdcc Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Wed, 4 May 2022 12:45:01 -0400 Subject: [PATCH] Add hot page test (#1479) --- poetry.lock | 30 +++++++++++++++---- pyproject.toml | 1 + test_runner/fixtures/compare_fixtures.py | 5 +++- test_runner/fixtures/zenith_fixtures.py | 2 +- test_runner/performance/test_hot_page.py | 36 +++++++++++++++++++++++ test_runner/performance/test_hot_table.py | 35 ++++++++++++++++++++++ 6 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 test_runner/performance/test_hot_page.py create mode 100644 test_runner/performance/test_hot_table.py diff --git a/poetry.lock b/poetry.lock index fe18ad226c..a7cbe0aa3c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -822,7 +822,7 @@ python-versions = "*" [[package]] name = "moto" -version = "3.0.4" +version = "3.1.7" description = "A library that allows your python tests to easily mock out the boto library" category = "main" optional = false @@ -844,6 +844,7 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} Jinja2 = ">=2.10.1" jsondiff = {version = ">=1.1.2", optional = true, markers = "extra == \"server\""} MarkupSafe = "!=2.0.0a1" +pyparsing = {version = ">=3.0.0", optional = true, markers = "extra == \"server\""} python-dateutil = ">=2.1,<3.0.0" python-jose = {version = ">=3.1.0,<4.0.0", extras = ["cryptography"], optional = true, markers = "extra == \"server\""} pytz = "*" @@ -855,7 +856,7 @@ werkzeug = "*" xmltodict = "*" [package.extras] -all = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.4.0)", "sshpubkeys (>=3.1.0)", "setuptools"] +all = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.4.0)", "sshpubkeys (>=3.1.0)", "pyparsing (>=3.0.0)", "setuptools"] apigateway = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)"] apigatewayv2 = ["PyYAML (>=5.1)"] appsync = ["graphql-core"] @@ -864,14 +865,16 @@ batch = ["docker (>=2.5.1)"] cloudformation = ["docker (>=2.5.1)", "PyYAML (>=5.1)", "cfn-lint (>=0.4.0)"] cognitoidp = ["python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)"] ds = ["sshpubkeys (>=3.1.0)"] +dynamodb = ["docker (>=2.5.1)"] dynamodb2 = ["docker (>=2.5.1)"] dynamodbstreams = ["docker (>=2.5.1)"] ec2 = ["sshpubkeys (>=3.1.0)"] efs = ["sshpubkeys (>=3.1.0)"] +glue = ["pyparsing (>=3.0.0)"] iotdata = ["jsondiff (>=1.1.2)"] route53resolver = ["sshpubkeys (>=3.1.0)"] s3 = ["PyYAML (>=5.1)"] -server = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.4.0)", "sshpubkeys (>=3.1.0)", "setuptools", "flask", "flask-cors"] +server = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.4.0)", "sshpubkeys (>=3.1.0)", "pyparsing (>=3.0.0)", "setuptools", "flask", "flask-cors"] ssm = ["PyYAML (>=5.1)", "dataclasses"] xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] @@ -1068,6 +1071,17 @@ python-versions = ">=3.6" py = "*" pytest = ">=3.10" +[[package]] +name = "pytest-lazy-fixture" +version = "0.6.3" +description = "It helps to use fixtures in pytest.mark.parametrize" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pytest = ">=3.2.5" + [[package]] name = "pytest-xdist" version = "2.5.0" @@ -1361,7 +1375,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "58762accad4122026c650fa43421a900546e89f9908e2268410e7b11cc8c6c4e" +content-hash = "dc63b6e02d0ceccdc4b5616e9362c149a27fdcc6c54fda63a3b115a5b980c42e" [metadata.files] aiopg = [ @@ -1679,8 +1693,8 @@ mccabe = [ {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] moto = [ - {file = "moto-3.0.4-py2.py3-none-any.whl", hash = "sha256:79646213d8438385182f4eea79e28725f94b3d0d3dc9a3eda81db47e0ebef6cc"}, - {file = "moto-3.0.4.tar.gz", hash = "sha256:168b8a3cb4dd8a6df8e51d582761cefa9657b9f45ac7e1eb24dae394ebc9e000"}, + {file = "moto-3.1.7-py3-none-any.whl", hash = "sha256:4ab6fb8dd150343e115d75e3dbdb5a8f850fc7236790819d7cef438c11ee6e89"}, + {file = "moto-3.1.7.tar.gz", hash = "sha256:20607a0fd0cf6530e05ffb623ca84d3f45d50bddbcec2a33705a0cf471e71289"}, ] mypy = [ {file = "mypy-0.910-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457"}, @@ -1855,6 +1869,10 @@ pytest-forked = [ {file = "pytest-forked-1.4.0.tar.gz", hash = "sha256:8b67587c8f98cbbadfdd804539ed5455b6ed03802203485dd2f53c1422d7440e"}, {file = "pytest_forked-1.4.0-py3-none-any.whl", hash = "sha256:bbbb6717efc886b9d64537b41fb1497cfaf3c9601276be8da2cccfea5a3c8ad8"}, ] +pytest-lazy-fixture = [ + {file = "pytest-lazy-fixture-0.6.3.tar.gz", hash = "sha256:0e7d0c7f74ba33e6e80905e9bfd81f9d15ef9a790de97993e34213deb5ad10ac"}, + {file = "pytest_lazy_fixture-0.6.3-py3-none-any.whl", hash = "sha256:e0b379f38299ff27a653f03eaa69b08a6fd4484e46fd1c9907d984b9f9daeda6"}, +] pytest-xdist = [ {file = "pytest-xdist-2.5.0.tar.gz", hash = "sha256:4580deca3ff04ddb2ac53eba39d76cb5dd5edeac050cb6fbc768b0dd712b4edf"}, {file = "pytest_xdist-2.5.0-py3-none-any.whl", hash = "sha256:6fe5c74fec98906deb8f2d2b616b5c782022744978e7bd4695d39c8f42d0ce65"}, diff --git a/pyproject.toml b/pyproject.toml index 7dbdcc0304..335c6d61d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ boto3 = "^1.20.40" boto3-stubs = "^1.20.40" moto = {version = "^3.0.0", extras = ["server"]} backoff = "^1.11.1" +pytest-lazy-fixture = "^0.6.3" [tool.poetry.dev-dependencies] yapf = "==0.31.0" diff --git a/test_runner/fixtures/compare_fixtures.py b/test_runner/fixtures/compare_fixtures.py index 93912d2da7..d70f57aa52 100644 --- a/test_runner/fixtures/compare_fixtures.py +++ b/test_runner/fixtures/compare_fixtures.py @@ -130,7 +130,10 @@ class VanillaCompare(PgCompare): def __init__(self, zenbenchmark, vanilla_pg: VanillaPostgres): self._pg = vanilla_pg self._zenbenchmark = zenbenchmark - vanilla_pg.configure(['shared_buffers=1MB']) + vanilla_pg.configure([ + 'shared_buffers=1MB', + 'synchronous_commit=off', + ]) vanilla_pg.start() # Long-lived cursor, useful for flushing diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index d6d07d78d3..784d2d4b26 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -1315,7 +1315,7 @@ class VanillaPostgres(PgProtocol): """Append lines into postgresql.conf file.""" assert not self.running with open(os.path.join(self.pgdatadir, 'postgresql.conf'), 'a') as conf_file: - conf_file.writelines(options) + conf_file.write("\n".join(options)) def start(self, log_path: Optional[str] = None): assert not self.running diff --git a/test_runner/performance/test_hot_page.py b/test_runner/performance/test_hot_page.py new file mode 100644 index 0000000000..2042b0d548 --- /dev/null +++ b/test_runner/performance/test_hot_page.py @@ -0,0 +1,36 @@ +import pytest +from contextlib import closing +from fixtures.compare_fixtures import PgCompare +from pytest_lazyfixture import lazy_fixture # type: ignore + + +@pytest.mark.parametrize( + "env", + [ + # The test is too slow to run in CI, but fast enough to run with remote tests + pytest.param(lazy_fixture("zenith_compare"), id="zenith", marks=pytest.mark.slow), + pytest.param(lazy_fixture("vanilla_compare"), id="vanilla", marks=pytest.mark.slow), + pytest.param(lazy_fixture("remote_compare"), id="remote", marks=pytest.mark.remote_cluster), + ]) +def test_hot_page(env: PgCompare): + # Update the same page many times, then measure read performance + num_writes = 1000000 + + with closing(env.pg.connect()) as conn: + with conn.cursor() as cur: + + # Write many updates to the same row + with env.record_duration('write'): + cur.execute('create table t (i integer);') + cur.execute('insert into t values (0);') + for i in range(num_writes): + cur.execute(f'update t set i = {i};') + + # Write 3-4 MB to evict t from compute cache + cur.execute('create table f (i integer);') + cur.execute(f'insert into f values (generate_series(1,100000));') + + # Read + with env.record_duration('read'): + cur.execute('select * from t;') + cur.fetchall() diff --git a/test_runner/performance/test_hot_table.py b/test_runner/performance/test_hot_table.py new file mode 100644 index 0000000000..11e047b8c3 --- /dev/null +++ b/test_runner/performance/test_hot_table.py @@ -0,0 +1,35 @@ +import pytest +from contextlib import closing +from fixtures.compare_fixtures import PgCompare +from pytest_lazyfixture import lazy_fixture # type: ignore + + +@pytest.mark.parametrize( + "env", + [ + # The test is too slow to run in CI, but fast enough to run with remote tests + pytest.param(lazy_fixture("zenith_compare"), id="zenith", marks=pytest.mark.slow), + pytest.param(lazy_fixture("vanilla_compare"), id="vanilla", marks=pytest.mark.slow), + pytest.param(lazy_fixture("remote_compare"), id="remote", marks=pytest.mark.remote_cluster), + ]) +def test_hot_table(env: PgCompare): + # Update a small table many times, then measure read performance + num_rows = 100000 # Slightly larger than shared buffers size TODO validate + num_writes = 1000000 + num_reads = 10 + + with closing(env.pg.connect()) as conn: + with conn.cursor() as cur: + + # Write many updates to a small table + with env.record_duration('write'): + cur.execute('create table t (i integer primary key);') + cur.execute(f'insert into t values (generate_series(1,{num_rows}));') + for i in range(num_writes): + cur.execute(f'update t set i = {i + num_rows} WHERE i = {i};') + + # Read the table + with env.record_duration('read'): + for i in range(num_reads): + cur.execute('select * from t;') + cur.fetchall()