From 834ffe1bac4febfc5442459efdd861a283cabed3 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Tue, 25 Oct 2022 16:41:50 +0200 Subject: [PATCH 01/16] Add data format backward compatibility tests (#2626) --- .../actions/run-python-test-set/action.yml | 18 ++ .github/workflows/build_and_test.yml | 23 +- poetry.lock | 52 +++- pyproject.toml | 2 + test_runner/fixtures/neon_fixtures.py | 2 +- test_runner/regress/test_compatibility.py | 267 ++++++++++++++++++ 6 files changed, 352 insertions(+), 12 deletions(-) create mode 100644 test_runner/regress/test_compatibility.py diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index cc6ab65b76..07cb7edbe7 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -73,6 +73,13 @@ runs: shell: bash -euxo pipefail {0} run: ./scripts/pysync + - name: Download compatibility snapshot for Postgres 14 + uses: ./.github/actions/download + with: + name: compatibility-snapshot-${{ inputs.build_type }}-pg14 + path: /tmp/compatibility_snapshot_pg14 + prefix: latest + - name: Run pytest env: NEON_BIN: /tmp/neon/bin @@ -80,6 +87,8 @@ runs: BUILD_TYPE: ${{ inputs.build_type }} AWS_ACCESS_KEY_ID: ${{ inputs.real_s3_access_key_id }} AWS_SECRET_ACCESS_KEY: ${{ inputs.real_s3_secret_access_key }} + COMPATIBILITY_SNAPSHOT_DIR: /tmp/compatibility_snapshot_pg14 + ALLOW_BREAKING_CHANGES: contains(github.event.pull_request.labels.*.name, 'breaking changes') shell: bash -euxo pipefail {0} run: | # PLATFORM will be embedded in the perf test report @@ -154,6 +163,15 @@ runs: scripts/generate_and_push_perf_report.sh fi + - name: Upload compatibility snapshot for Postgres 14 + if: github.ref_name == 'release' + uses: ./.github/actions/upload + with: + name: compatibility-snapshot-${{ inputs.build_type }}-pg14-${{ github.run_id }} + # The path includes a test name (test_prepare_snapshot) and directory that the test creates (compatibility_snapshot_pg14), keep the path in sync with the test + path: /tmp/test_output/test_prepare_snapshot/compatibility_snapshot_pg14/ + prefix: latest + - name: Create Allure report if: always() uses: ./.github/actions/allure-report diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 14ee61c5b9..660f93b025 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -844,7 +844,7 @@ jobs: submodules: true fetch-depth: 0 - - name: Configure environment + - name: Configure environment run: | helm repo add neondatabase https://neondatabase.github.io/helm-charts aws --region us-east-2 eks update-kubeconfig --name dev-us-east-2-beta --role-arn arn:aws:iam::369495373322:role/github-runner @@ -853,3 +853,24 @@ jobs: run: | DOCKER_TAG=${{needs.tag.outputs.build-tag}} helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s + + promote-compatibility-test-snapshot: + runs-on: dev + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + options: --init + needs: [ deploy, deploy-proxy ] + if: github.ref_name == 'release' && github.event_name != 'workflow_dispatch' + steps: + - name: Promote compatibility snapshot for the release + shell: bash -euxo pipefail {0} + env: + BUCKET: neon-github-public-dev + PREFIX: artifacts/latest + run: | + for build_type in debug release; do + OLD_FILENAME=compatibility-snapshot-${build_type}-pg14-${GITHUB_RUN_ID}.tar.zst + NEW_FILENAME=compatibility-snapshot-${build_type}-pg14.tar.zst + + time aws s3 mv --only-show-errors s3://${BUCKET}/${PREFIX}/${OLD_FILENAME} s3://${BUCKET}/${PREFIX}/${NEW_FILENAME} + done diff --git a/poetry.lock b/poetry.lock index 27de8508ce..dfcb16107f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -11,7 +11,7 @@ async-timeout = ">=3.0,<5.0" psycopg2-binary = ">=2.8.4" [package.extras] -sa = ["sqlalchemy[postgresql_psycopg2binary] (>=1.3,<1.5)"] +sa = ["sqlalchemy[postgresql-psycopg2binary] (>=1.3,<1.5)"] [[package]] name = "allure-pytest" @@ -80,7 +80,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "zope.interface"] -tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] [[package]] name = "aws-sam-translator" @@ -560,7 +560,7 @@ optional = false python-versions = ">=3.6.0" [package.extras] -unicode_backport = ["unicodedata2"] +unicode-backport = ["unicodedata2"] [[package]] name = "click" @@ -593,7 +593,7 @@ python-versions = ">=3.6" cffi = ">=1.12" [package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx_rtd_theme"] docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] sdist = ["setuptools_rust (>=0.11.4)"] @@ -738,9 +738,9 @@ python-versions = ">=3.6.1,<4.0" [package.extras] colors = ["colorama (>=0.4.3,<0.5.0)"] -pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +pipfile-deprecated-finder = ["pipreqs", "requirementslib"] plugins = ["setuptools"] -requirements_deprecated_finder = ["pip-api", "pipreqs"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] [[package]] name = "itsdangerous" @@ -815,7 +815,7 @@ python-versions = ">=2.7" [package.extras] docs = ["jaraco.packaging (>=3.2)", "rst.linker (>=1.9)", "sphinx"] testing = ["ecdsa", "enum34", "feedparser", "jsonlib", "numpy", "pandas", "pymongo", "pytest (>=3.5,!=3.7.3)", "pytest-black-multipy", "pytest-checkdocs (>=1.2.3)", "pytest-cov", "pytest-flake8 (<1.1.0)", "pytest-flake8 (>=1.1.1)", "scikit-learn", "sqlalchemy"] -"testing.libs" = ["simplejson", "ujson", "yajl"] +testing-libs = ["simplejson", "ujson", "yajl"] [[package]] name = "jsonpointer" @@ -836,11 +836,12 @@ python-versions = "*" [package.dependencies] attrs = ">=17.4.0" pyrsistent = ">=0.14.0" +setuptools = "*" six = ">=1.11.0" [package.extras] format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] -format_nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] [[package]] name = "junit-xml" @@ -900,6 +901,7 @@ pytz = "*" PyYAML = {version = ">=5.1", optional = true, markers = "extra == \"server\""} requests = ">=2.5" responses = ">=0.9.0" +setuptools = {version = "*", optional = true, markers = "extra == \"server\""} sshpubkeys = {version = ">=3.1.0", optional = true, markers = "extra == \"server\""} werkzeug = ">=0.5,<2.2.0" xmltodict = "*" @@ -1008,6 +1010,7 @@ python-versions = ">=3.7.0,<4.0.0" jsonschema = ">=3.2.0,<5.0.0" openapi-schema-validator = ">=0.2.0,<0.3.0" PyYAML = ">=5.1" +setuptools = "*" [package.extras] requests = ["requests"] @@ -1340,7 +1343,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "responses" @@ -1394,6 +1397,19 @@ python-versions = ">= 2.7" attrs = "*" pbr = "*" +[[package]] +name = "setuptools" +version = "65.5.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "six" version = "1.16.0" @@ -1460,6 +1476,14 @@ category = "main" optional = false python-versions = ">=3.7,<4.0" +[[package]] +name = "types-toml" +version = "0.10.8" +description = "Typing stubs for toml" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "types-urllib3" version = "1.26.17" @@ -1544,7 +1568,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "ead1495454ee6d880bb240447025db93a25ebe263c2709de5f144cc2d85dc975" +content-hash = "17cdbfe90f1b06dffaf24c3e076384ec08dd4a2dce5a05e50565f7364932eb2d" [metadata.files] aiopg = [ @@ -2182,6 +2206,10 @@ sarif-om = [ {file = "sarif_om-1.0.4-py3-none-any.whl", hash = "sha256:539ef47a662329b1c8502388ad92457425e95dc0aaaf995fe46f4984c4771911"}, {file = "sarif_om-1.0.4.tar.gz", hash = "sha256:cd5f416b3083e00d402a92e449a7ff67af46f11241073eea0461802a3b5aef98"}, ] +setuptools = [ + {file = "setuptools-65.5.0-py3-none-any.whl", hash = "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"}, + {file = "setuptools-65.5.0.tar.gz", hash = "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -2210,6 +2238,10 @@ types-s3transfer = [ {file = "types-s3transfer-0.6.0.post3.tar.gz", hash = "sha256:92c3704e5d041202bfb5ddb79d083fd1a02de2c5dfec6a91576823e6b5c93993"}, {file = "types_s3transfer-0.6.0.post3-py3-none-any.whl", hash = "sha256:eedc5117275565b3c83662c0ccc81662a34da5dda8bd502b89d296b6d5cb091d"}, ] +types-toml = [ + {file = "types-toml-0.10.8.tar.gz", hash = "sha256:b7e7ea572308b1030dc86c3ba825c5210814c2825612ec679eb7814f8dd9295a"}, + {file = "types_toml-0.10.8-py3-none-any.whl", hash = "sha256:8300fd093e5829eb9c1fba69cee38130347d4b74ddf32d0a7df650ae55c2b599"}, +] types-urllib3 = [ {file = "types-urllib3-1.26.17.tar.gz", hash = "sha256:73fd274524c3fc7cd8cd9ceb0cb67ed99b45f9cb2831013e46d50c1451044800"}, {file = "types_urllib3-1.26.17-py3-none-any.whl", hash = "sha256:0d027fcd27dbb3cb532453b4d977e05bc1e13aefd70519866af211b3003d895d"}, diff --git a/pyproject.toml b/pyproject.toml index 1ee6fbe6f4..765e0b97eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,12 +28,14 @@ Werkzeug = "2.1.2" pytest-order = "^1.0.1" allure-pytest = "^2.10.0" pytest-asyncio = "^0.19.0" +toml = "^0.10.2" [tool.poetry.dev-dependencies] flake8 = "^5.0.4" mypy = "==0.971" black = "^22.6.0" isort = "^5.10.1" +types-toml = "^0.10.8" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 4b2638bb2a..38a0db7cf7 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -970,7 +970,7 @@ class NeonPageserverApiException(Exception): class NeonPageserverHttpClient(requests.Session): - def __init__(self, port: int, is_testing_enabled_or_skip, auth_token: Optional[str] = None): + def __init__(self, port: int, is_testing_enabled_or_skip: Fn, auth_token: Optional[str] = None): super().__init__() self.port = port self.auth_token = auth_token diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py new file mode 100644 index 0000000000..944ff64390 --- /dev/null +++ b/test_runner/regress/test_compatibility.py @@ -0,0 +1,267 @@ +import os +import re +import shutil +import subprocess +from pathlib import Path +from typing import Any, Dict, Union + +import pytest +import toml +from fixtures.neon_fixtures import ( + NeonCli, + NeonEnvBuilder, + NeonPageserverHttpClient, + PgBin, + PortDistributor, + wait_for_last_record_lsn, + wait_for_upload, +) +from fixtures.types import Lsn +from pytest import FixtureRequest + + +def dump_differs(first: Path, second: Path, output: Path) -> bool: + """ + Runs diff(1) command on two SQL dumps and write the output to the given output file. + Returns True if the dumps differ, False otherwise. + """ + + with output.open("w") as stdout: + rv = subprocess.run( + [ + "diff", + "--unified", # Make diff output more readable + "--ignore-matching-lines=^--", # Ignore changes in comments + "--ignore-blank-lines", + str(first), + str(second), + ], + stdout=stdout, + ) + + return rv.returncode != 0 + + +class PortReplacer(object): + """ + Class-helper for replacing ports in config files. + """ + + def __init__(self, port_distributor: PortDistributor): + self.port_distributor = port_distributor + self.port_map: Dict[int, int] = {} + + def replace_port(self, value: Union[int, str]) -> Union[int, str]: + if isinstance(value, int): + if (known_port := self.port_map.get(value)) is not None: + return known_port + + self.port_map[value] = self.port_distributor.get_port() + return self.port_map[value] + + if isinstance(value, str): + # Use regex to find port in a string + # urllib.parse.urlparse produces inconvenient results for cases without scheme like "localhost:5432" + # See https://bugs.python.org/issue27657 + ports = re.findall(r":(\d+)(?:/|$)", value) + assert len(ports) == 1, f"can't find port in {value}" + port_int = int(ports[0]) + + if (known_port := self.port_map.get(port_int)) is not None: + return value.replace(f":{port_int}", f":{known_port}") + + self.port_map[port_int] = self.port_distributor.get_port() + return value.replace(f":{port_int}", f":{self.port_map[port_int]}") + + raise TypeError(f"unsupported type {type(value)} of {value=}") + + +def test_backward_compatibility( + pg_bin: PgBin, port_distributor: PortDistributor, test_output_dir: Path, request: FixtureRequest +): + compatibility_snapshot_dir_env = os.environ.get("COMPATIBILITY_SNAPSHOT_DIR") + assert ( + compatibility_snapshot_dir_env is not None + ), "COMPATIBILITY_SNAPSHOT_DIR is not set. It should be set to `compatibility_snapshot_pg14` path generateted by test_prepare_snapshot" + compatibility_snapshot_dir = Path(compatibility_snapshot_dir_env).resolve() + + # Make compatibility snapshot artifacts pickupable by Allure + # by copying the snapshot directory to the curent test output directory. + repo_dir = test_output_dir / "compatibility_snapshot" / "repo" + + shutil.copytree(compatibility_snapshot_dir / "repo", repo_dir) + + # Remove old logs to avoid confusion in test artifacts + for logfile in repo_dir.glob("**/*.log"): + logfile.unlink() + + # Remove tenants data for computes + for tenant in (repo_dir / "pgdatadirs" / "tenants").glob("*"): + shutil.rmtree(tenant) + + # Remove wal-redo temp directory + for tenant in (repo_dir / "tenants").glob("*"): + shutil.rmtree(tenant / "wal-redo-datadir.___temp") + + # Update paths and ports in config files + pr = PortReplacer(port_distributor) + + pageserver_toml = repo_dir / "pageserver.toml" + pageserver_config = toml.load(pageserver_toml) + new_local_path = pageserver_config["remote_storage"]["local_path"].replace( + "/test_prepare_snapshot/", + "/test_backward_compatibility/compatibility_snapshot/", + ) + + pageserver_config["remote_storage"]["local_path"] = new_local_path + pageserver_config["listen_http_addr"] = pr.replace_port(pageserver_config["listen_http_addr"]) + pageserver_config["listen_pg_addr"] = pr.replace_port(pageserver_config["listen_pg_addr"]) + pageserver_config["broker_endpoints"] = [ + pr.replace_port(ep) for ep in pageserver_config["broker_endpoints"] + ] + + with pageserver_toml.open("w") as f: + toml.dump(pageserver_config, f) + + snapshot_config_toml = repo_dir / "config" + snapshot_config = toml.load(snapshot_config_toml) + snapshot_config["etcd_broker"]["broker_endpoints"] = [ + pr.replace_port(ep) for ep in snapshot_config["etcd_broker"]["broker_endpoints"] + ] + snapshot_config["pageserver"]["listen_http_addr"] = pr.replace_port( + snapshot_config["pageserver"]["listen_http_addr"] + ) + snapshot_config["pageserver"]["listen_pg_addr"] = pr.replace_port( + snapshot_config["pageserver"]["listen_pg_addr"] + ) + for sk in snapshot_config["safekeepers"]: + sk["http_port"] = pr.replace_port(sk["http_port"]) + sk["pg_port"] = pr.replace_port(sk["pg_port"]) + + with (snapshot_config_toml).open("w") as f: + toml.dump(snapshot_config, f) + + # Ensure that snapshot doesn't contain references to the original path + rv = subprocess.run( + [ + "grep", + "--recursive", + "--binary-file=without-match", + "--files-with-matches", + "test_prepare_snapshot/repo", + str(repo_dir), + ], + capture_output=True, + text=True, + ) + assert ( + rv.returncode != 0 + ), f"there're files referencing `test_prepare_snapshot/repo`, this path should be replaced with {repo_dir}:\n{rv.stdout}" + + # NeonEnv stub to make NeonCli happy + config: Any = type("NeonEnvStub", (object,), {}) + config.rust_log_override = None + config.repo_dir = repo_dir + config.pg_version = "14" # Note: `pg_dumpall` (from pg_bin) version is set by DEFAULT_PG_VERSION_DEFAULT and can be overriden by DEFAULT_PG_VERSION env var + config.initial_tenant = snapshot_config["default_tenant_id"] + + # Check that we can start the project + cli = NeonCli(config) + try: + cli.raw_cli(["start"]) + request.addfinalizer(lambda: cli.raw_cli(["stop"])) + + result = cli.pg_start("main") + request.addfinalizer(lambda: cli.pg_stop("main")) + except Exception: + breaking_changes_allowed = ( + os.environ.get("ALLOW_BREAKING_CHANGES", "false").lower() == "true" + ) + if breaking_changes_allowed: + pytest.xfail("Breaking changes are allowed by ALLOW_BREAKING_CHANGES env var") + else: + raise + + connstr_all = re.findall(r"Starting postgres node at '([^']+)'", result.stdout) + assert len(connstr_all) == 1, f"can't parse connstr from {result.stdout}" + connstr = connstr_all[0] + + # Check that the project produces the same dump as the previous version. + # The assert itself deferred to the end of the test + # to allow us to perform checks that change data before failing + pg_bin.run(["pg_dumpall", f"--dbname={connstr}", f"--file={test_output_dir / 'dump.sql'}"]) + initial_dump_differs = dump_differs( + compatibility_snapshot_dir / "dump.sql", + test_output_dir / "dump.sql", + test_output_dir / "dump.filediff", + ) + + # Check that project can be recovered from WAL + # loosely based on https://github.com/neondatabase/cloud/wiki/Recovery-from-WAL + tenant_id = snapshot_config["default_tenant_id"] + timeline_id = dict(snapshot_config["branch_name_mappings"]["main"])[tenant_id] + pageserver_port = snapshot_config["pageserver"]["listen_http_addr"].split(":")[-1] + auth_token = snapshot_config["pageserver"]["auth_token"] + pageserver_http = NeonPageserverHttpClient( + port=pageserver_port, + is_testing_enabled_or_skip=lambda: True, # TODO: check if testing really enabled + auth_token=auth_token, + ) + + shutil.rmtree(repo_dir / "local_fs_remote_storage") + pageserver_http.timeline_delete(tenant_id, timeline_id) + pageserver_http.timeline_create(tenant_id, timeline_id) + pg_bin.run( + ["pg_dumpall", f"--dbname={connstr}", f"--file={test_output_dir / 'dump-from-wal.sql'}"] + ) + # The assert itself deferred to the end of the test + # to allow us to perform checks that change data before failing + dump_from_wal_differs = dump_differs( + test_output_dir / "dump.sql", + test_output_dir / "dump-from-wal.sql", + test_output_dir / "dump-from-wal.filediff", + ) + + # Check that we can interract with the data + pg_bin.run(["pgbench", "--time=10", "--progress=2", connstr]) + + assert not dump_from_wal_differs, "dump from WAL differs" + assert not initial_dump_differs, "initial dump differs" + + +@pytest.mark.order(after="test_backward_compatibility") +# Note: if renaming this test, don't forget to update a reference to it in a workflow file: +# "Upload compatibility snapshot" step in .github/actions/run-python-test-set/action.yml +def test_prepare_snapshot(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin, test_output_dir: Path): + # The test doesn't really test anything + # it creates a new snapshot for releases after we tested the current version against the previous snapshot in `test_backward_compatibility`. + # + # There's no cleanup here, it allows to adjust the data in `test_backward_compatibility` itself without re-collecting it. + neon_env_builder.pg_version = "14" + neon_env_builder.num_safekeepers = 3 + neon_env_builder.enable_local_fs_remote_storage() + + env = neon_env_builder.init_start() + pg = env.postgres.create_start("main") + pg_bin.run(["pgbench", "--initialize", "--scale=10", pg.connstr()]) + pg_bin.run(["pgbench", "--time=60", "--progress=2", pg.connstr()]) + pg_bin.run(["pg_dumpall", f"--dbname={pg.connstr()}", f"--file={test_output_dir / 'dump.sql'}"]) + + snapshot_config = toml.load(test_output_dir / "repo" / "config") + tenant_id = snapshot_config["default_tenant_id"] + timeline_id = dict(snapshot_config["branch_name_mappings"]["main"])[tenant_id] + + pageserver_http = env.pageserver.http_client() + lsn = Lsn(pg.safe_psql("SELECT pg_current_wal_flush_lsn()")[0][0]) + + pageserver_http.timeline_checkpoint(tenant_id, timeline_id) + wait_for_last_record_lsn(pageserver_http, tenant_id, timeline_id, lsn) + wait_for_upload(pageserver_http, tenant_id, timeline_id, lsn) + + env.postgres.stop_all() + for sk in env.safekeepers: + sk.stop() + env.pageserver.stop() + + shutil.copytree(test_output_dir, test_output_dir / "compatibility_snapshot_pg14") + # Directory `test_output_dir / "compatibility_snapshot_pg14"` is uploaded to S3 in a workflow, keep the name in sync with it From 9fb2287f87a933e9b89b59c7e286560de641dbc6 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Tue, 25 Oct 2022 11:25:22 -0400 Subject: [PATCH 02/16] Add draw_timeline binary (#2688) --- Cargo.lock | 7 ++ Dockerfile | 3 +- pageserver/Cargo.toml | 1 + pageserver/src/bin/draw_timeline_dir.rs | 150 ++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 pageserver/src/bin/draw_timeline_dir.rs diff --git a/Cargo.lock b/Cargo.lock index 13774f7fe6..b39ca6e5a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2170,6 +2170,7 @@ dependencies = [ "serde_json", "serde_with", "signal-hook", + "svg_fmt", "tar", "tempfile", "thiserror", @@ -3461,6 +3462,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +[[package]] +name = "svg_fmt" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb1df15f412ee2e9dfc1c504260fa695c1c3f10fe9f4a6ee2d2184d7d6450e2" + [[package]] name = "symbolic-common" version = "8.8.0" diff --git a/Dockerfile b/Dockerfile index cb4e213687..b0d934d480 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,7 @@ COPY . . # Show build caching stats to check if it was used in the end. # Has to be the part of the same RUN since cachepot daemon is killed in the end of this RUN, losing the compilation stats. RUN set -e \ -&& mold -run cargo build --bin pageserver --bin pageserver_binutils --bin safekeeper --bin proxy --locked --release \ +&& mold -run cargo build --bin pageserver --bin pageserver_binutils --bin draw_timeline_dir --bin safekeeper --bin proxy --locked --release \ && cachepot -s # Build final image @@ -65,6 +65,7 @@ RUN set -e \ COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver_binutils /usr/local/bin +COPY --from=build --chown=neon:neon /home/nonroot/target/release/draw_timeline_dir /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/safekeeper /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/proxy /usr/local/bin diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 2139e24ee2..b075b86aa1 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -67,6 +67,7 @@ remote_storage = { path = "../libs/remote_storage" } workspace_hack = { version = "0.1", path = "../workspace_hack" } close_fds = "0.3.2" walkdir = "2.3.2" +svg_fmt = "0.4.1" [dev-dependencies] criterion = "0.4" diff --git a/pageserver/src/bin/draw_timeline_dir.rs b/pageserver/src/bin/draw_timeline_dir.rs new file mode 100644 index 0000000000..ea1ff7f3c7 --- /dev/null +++ b/pageserver/src/bin/draw_timeline_dir.rs @@ -0,0 +1,150 @@ +//! A tool for visualizing the arrangement of layerfiles within a timeline. +//! +//! It reads filenames from stdin and prints a svg on stdout. The image is a plot in +//! page-lsn space, where every delta layer is a rectangle and every image layer is a +//! thick line. Legend: +//! - The x axis (left to right) represents page index. +//! - The y axis represents LSN, growing upwards. +//! +//! Coordinates in both axis are compressed for better readability. +//! (see https://medium.com/algorithms-digest/coordinate-compression-2fff95326fb) +//! +//! Example use: +//! ``` +//! $ cd test_output/test_pgbench\[neon-45-684\]/repo/tenants/$TENANT/timelines/$TIMELINE +//! $ ls | grep "__" | cargo run --release --bin draw_timeline_dir > out.svg +//! $ firefox out.svg +//! ``` +//! +//! This API was chosen so that we can easily work with filenames extracted from ssh, +//! or from pageserver log files. +//! +//! TODO Consider shipping this as a grafana panel plugin: +//! https://grafana.com/tutorials/build-a-panel-plugin/ +use anyhow::Result; +use pageserver::repository::Key; +use std::cmp::Ordering; +use std::io::{self, BufRead}; +use std::{ + collections::{BTreeMap, BTreeSet}, + ops::Range, +}; +use svg_fmt::{rectangle, rgb, BeginSvg, EndSvg, Fill, Stroke}; +use utils::{lsn::Lsn, project_git_version}; + +project_git_version!(GIT_VERSION); + +// Map values to their compressed coordinate - the index the value +// would have in a sorted and deduplicated list of all values. +fn build_coordinate_compression_map(coords: Vec) -> BTreeMap { + let set: BTreeSet = coords.into_iter().collect(); + + let mut map: BTreeMap = BTreeMap::new(); + for (i, e) in set.iter().enumerate() { + map.insert(*e, i); + } + + map +} + +fn parse_filename(name: &str) -> (Range, Range) { + let split: Vec<&str> = name.split("__").collect(); + let keys: Vec<&str> = split[0].split('-').collect(); + let mut lsns: Vec<&str> = split[1].split('-').collect(); + if lsns.len() == 1 { + lsns.push(lsns[0]); + } + + let keys = Key::from_hex(keys[0]).unwrap()..Key::from_hex(keys[1]).unwrap(); + let lsns = Lsn::from_hex(lsns[0]).unwrap()..Lsn::from_hex(lsns[1]).unwrap(); + (keys, lsns) +} + +fn main() -> Result<()> { + // Parse layer filenames from stdin + let mut ranges: Vec<(Range, Range)> = vec![]; + let stdin = io::stdin(); + for line in stdin.lock().lines() { + let range = parse_filename(&line.unwrap()); + ranges.push(range); + } + + // Collect all coordinates + let mut keys: Vec = vec![]; + let mut lsns: Vec = vec![]; + for (keyr, lsnr) in &ranges { + keys.push(keyr.start); + keys.push(keyr.end); + lsns.push(lsnr.start); + lsns.push(lsnr.end); + } + + // Analyze + let key_map = build_coordinate_compression_map(keys); + let lsn_map = build_coordinate_compression_map(lsns); + + // Initialize stats + let mut num_deltas = 0; + let mut num_images = 0; + + // Draw + let stretch = 3.0; // Stretch out vertically for better visibility + println!( + "{}", + BeginSvg { + w: key_map.len() as f32, + h: stretch * lsn_map.len() as f32 + } + ); + for (keyr, lsnr) in &ranges { + let key_start = *key_map.get(&keyr.start).unwrap(); + let key_end = *key_map.get(&keyr.end).unwrap(); + let key_diff = key_end - key_start; + let lsn_max = lsn_map.len(); + + if key_start >= key_end { + panic!("Invalid key range {}-{}", key_start, key_end); + } + + let lsn_start = *lsn_map.get(&lsnr.start).unwrap(); + let lsn_end = *lsn_map.get(&lsnr.end).unwrap(); + + let mut lsn_diff = (lsn_end - lsn_start) as f32; + let mut fill = Fill::None; + let mut margin = 0.05 * lsn_diff; // Height-dependent margin to disambiguate overlapping deltas + let mut lsn_offset = 0.0; + + // Fill in and thicken rectangle if it's an + // image layer so that we can see it. + match lsn_start.cmp(&lsn_end) { + Ordering::Less => num_deltas += 1, + Ordering::Equal => { + num_images += 1; + lsn_diff = 0.3; + lsn_offset = -lsn_diff / 2.0; + margin = 0.05; + fill = Fill::Color(rgb(0, 0, 0)); + } + Ordering::Greater => panic!("Invalid lsn range {}-{}", lsn_start, lsn_end), + } + + println!( + " {}", + rectangle( + key_start as f32 + stretch * margin, + stretch * (lsn_max as f32 - (lsn_end as f32 - margin - lsn_offset)), + key_diff as f32 - stretch * 2.0 * margin, + stretch * (lsn_diff - 2.0 * margin) + ) + .fill(fill) + .stroke(Stroke::Color(rgb(0, 0, 0), 0.1)) + .border_radius(0.4) + ); + } + println!("{}", EndSvg); + + eprintln!("num_images: {}", num_images); + eprintln!("num_deltas: {}", num_deltas); + + Ok(()) +} From a3cb8c11e067aac0efe637f4095863eba0361822 Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Wed, 26 Oct 2022 02:51:23 +0300 Subject: [PATCH 03/16] Do not release to new staging proxies on release (#2685) --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 660f93b025..1b8b380179 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -832,7 +832,7 @@ jobs: # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] if: | - (github.ref_name == 'main' || github.ref_name == 'release') && + (github.ref_name == 'main') && github.event_name != 'workflow_dispatch' defaults: run: From 259a5f356e036b5cf274cd2222884434dd8c55f0 Mon Sep 17 00:00:00 2001 From: mikecaat <35882227+mikecaat@users.noreply.github.com> Date: Wed, 26 Oct 2022 19:59:25 +0900 Subject: [PATCH 04/16] Add a docker-compose example file (#1943) (#2666) Co-authored-by: Masahiro Ikeda --- docker-compose/compute/shell/compute.sh | 48 +++++ .../compute/var/db/postgres/specs/spec.json | 141 ++++++++++++ docker-compose/docker-compose.yml | 200 ++++++++++++++++++ docker-compose/image/compute/Dockerfile | 10 + docs/docker.md | 64 ++++++ scripts/docker-compose_test.sh | 51 +++++ 6 files changed, 514 insertions(+) create mode 100755 docker-compose/compute/shell/compute.sh create mode 100644 docker-compose/compute/var/db/postgres/specs/spec.json create mode 100644 docker-compose/docker-compose.yml create mode 100644 docker-compose/image/compute/Dockerfile create mode 100755 scripts/docker-compose_test.sh diff --git a/docker-compose/compute/shell/compute.sh b/docker-compose/compute/shell/compute.sh new file mode 100755 index 0000000000..cef2b485f3 --- /dev/null +++ b/docker-compose/compute/shell/compute.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -eux + +PG_VERSION=${PG_VERSION:-14} + +SPEC_FILE_ORG=/var/db/postgres/specs/spec.json +SPEC_FILE=/tmp/spec.json + +echo "Waiting pageserver become ready." +while ! nc -z pageserver 6400; do + sleep 1; +done +echo "Page server is ready." + +echo "Create a tenant and timeline" +PARAMS=( + -sb + -X POST + -H "Content-Type: application/json" + -d "{}" + http://pageserver:9898/v1/tenant/ +) +tenant_id=$(curl "${PARAMS[@]}" | sed 's/"//g') + +PARAMS=( + -sb + -X POST + -H "Content-Type: application/json" + -d "{\"tenant_id\":\"${tenant_id}\", \"pg_version\": ${PG_VERSION}}" + "http://pageserver:9898/v1/tenant/${tenant_id}/timeline/" +) +result=$(curl "${PARAMS[@]}") +echo $result | jq . + +echo "Overwrite tenant id and timeline id in spec file" +tenant_id=$(echo ${result} | jq -r .tenant_id) +timeline_id=$(echo ${result} | jq -r .timeline_id) + +sed "s/TENANT_ID/${tenant_id}/" ${SPEC_FILE_ORG} > ${SPEC_FILE} +sed -i "s/TIMELINE_ID/${timeline_id}/" ${SPEC_FILE} + +cat ${SPEC_FILE} + +echo "Start compute node" +/usr/local/bin/compute_ctl --pgdata /var/db/postgres/compute \ + -C "postgresql://cloud_admin@localhost:55433/postgres" \ + -b /usr/local/bin/postgres \ + -S ${SPEC_FILE} diff --git a/docker-compose/compute/var/db/postgres/specs/spec.json b/docker-compose/compute/var/db/postgres/specs/spec.json new file mode 100644 index 0000000000..10ae0b0ecf --- /dev/null +++ b/docker-compose/compute/var/db/postgres/specs/spec.json @@ -0,0 +1,141 @@ +{ + "format_version": 1.0, + + "timestamp": "2022-10-12T18:00:00.000Z", + "operation_uuid": "0f657b36-4b0f-4a2d-9c2e-1dcd615e7d8c", + + "cluster": { + "cluster_id": "docker_compose", + "name": "docker_compose_test", + "state": "restarted", + "roles": [ + { + "name": "cloud_admin", + "encrypted_password": "b093c0d3b281ba6da1eacc608620abd8", + "options": null + } + ], + "databases": [ + ], + "settings": [ + { + "name": "fsync", + "value": "off", + "vartype": "bool" + }, + { + "name": "wal_level", + "value": "replica", + "vartype": "enum" + }, + { + "name": "hot_standby", + "value": "on", + "vartype": "bool" + }, + { + "name": "wal_log_hints", + "value": "on", + "vartype": "bool" + }, + { + "name": "log_connections", + "value": "on", + "vartype": "bool" + }, + { + "name": "port", + "value": "55433", + "vartype": "integer" + }, + { + "name": "shared_buffers", + "value": "1MB", + "vartype": "string" + }, + { + "name": "max_connections", + "value": "100", + "vartype": "integer" + }, + { + "name": "listen_addresses", + "value": "0.0.0.0", + "vartype": "string" + }, + { + "name": "max_wal_senders", + "value": "10", + "vartype": "integer" + }, + { + "name": "max_replication_slots", + "value": "10", + "vartype": "integer" + }, + { + "name": "wal_sender_timeout", + "value": "5s", + "vartype": "string" + }, + { + "name": "wal_keep_size", + "value": "0", + "vartype": "integer" + }, + { + "name": "password_encryption", + "value": "md5", + "vartype": "enum" + }, + { + "name": "restart_after_crash", + "value": "off", + "vartype": "bool" + }, + { + "name": "synchronous_standby_names", + "value": "walproposer", + "vartype": "string" + }, + { + "name": "shared_preload_libraries", + "value": "neon", + "vartype": "string" + }, + { + "name": "neon.safekeepers", + "value": "safekeeper1:5454,safekeeper2:5454,safekeeper3:5454", + "vartype": "string" + }, + { + "name": "neon.timeline_id", + "value": "TIMELINE_ID", + "vartype": "string" + }, + { + "name": "neon.tenant_id", + "value": "TENANT_ID", + "vartype": "string" + }, + { + "name": "neon.pageserver_connstring", + "value": "host=pageserver port=6400", + "vartype": "string" + }, + { + "name": "max_replication_write_lag", + "value": "500MB", + "vartype": "string" + }, + { + "name": "max_replication_flush_lag", + "value": "10GB", + "vartype": "string" + } + ] + }, + + "delta_operations": [ + ] +} diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml new file mode 100644 index 0000000000..9ab775c3f9 --- /dev/null +++ b/docker-compose/docker-compose.yml @@ -0,0 +1,200 @@ +version: '3' + +services: + etcd: + image: quay.io/coreos/etcd:v3.5.4 + ports: + - 2379:2379 + - 2380:2380 + environment: + # This signifficantly speeds up etcd and we anyway don't data persistency there. + ETCD_UNSAFE_NO_FSYNC: "1" + command: + - "etcd" + - "--auto-compaction-mode=revision" + - "--auto-compaction-retention=1" + - "--name=etcd-cluster" + - "--initial-cluster-state=new" + - "--initial-cluster-token=etcd-cluster-1" + - "--initial-cluster=etcd-cluster=http://etcd:2380" + - "--initial-advertise-peer-urls=http://etcd:2380" + - "--advertise-client-urls=http://etcd:2379" + - "--listen-client-urls=http://0.0.0.0:2379" + - "--listen-peer-urls=http://0.0.0.0:2380" + - "--quota-backend-bytes=134217728" # 128 MB + + minio: + image: quay.io/minio/minio:RELEASE.2022-10-20T00-55-09Z + ports: + - 9000:9000 + - 9001:9001 + environment: + - MINIO_ROOT_USER=minio + - MINIO_ROOT_PASSWORD=password + command: server /data --address :9000 --console-address ":9001" + + minio_create_buckets: + image: minio/mc + environment: + - MINIO_ROOT_USER=minio + - MINIO_ROOT_PASSWORD=password + entrypoint: + - "/bin/sh" + - "-c" + command: + - "until (/usr/bin/mc alias set minio http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD) do + echo 'Waiting to start minio...' && sleep 1; + done; + /usr/bin/mc mb minio/neon --region=eu-north-1; + exit 0;" + depends_on: + - minio + + pageserver: + image: neondatabase/neon:${TAG:-latest} + environment: + - BROKER_ENDPOINT='http://etcd:2379' + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=password + #- RUST_BACKTRACE=1 + ports: + #- 6400:6400 # pg protocol handler + - 9898:9898 # http endpoints + entrypoint: + - "/bin/sh" + - "-c" + command: + - "/usr/local/bin/pageserver -D /data/.neon/ + -c \"broker_endpoints=[$$BROKER_ENDPOINT]\" + -c \"listen_pg_addr='0.0.0.0:6400'\" + -c \"listen_http_addr='0.0.0.0:9898'\" + -c \"remote_storage={endpoint='http://minio:9000', + bucket_name='neon', + bucket_region='eu-north-1', + prefix_in_bucket='/pageserver/'}\"" + depends_on: + - etcd + - minio_create_buckets + + safekeeper1: + image: neondatabase/neon:${TAG:-latest} + environment: + - SAFEKEEPER_ADVERTISE_URL=safekeeper1:5454 + - SAFEKEEPER_ID=1 + - BROKER_ENDPOINT=http://etcd:2379 + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=password + #- RUST_BACKTRACE=1 + ports: + #- 5454:5454 # pg protocol handler + - 7676:7676 # http endpoints + entrypoint: + - "/bin/sh" + - "-c" + command: + - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL + --listen-http='0.0.0.0:7676' + --id=$$SAFEKEEPER_ID + --broker-endpoints=$$BROKER_ENDPOINT + -D /data + --remote-storage=\"{endpoint='http://minio:9000', + bucket_name='neon', + bucket_region='eu-north-1', + prefix_in_bucket='/safekeeper/'}\"" + depends_on: + - etcd + - minio_create_buckets + + safekeeper2: + image: neondatabase/neon:${TAG:-latest} + environment: + - SAFEKEEPER_ADVERTISE_URL=safekeeper2:5454 + - SAFEKEEPER_ID=2 + - BROKER_ENDPOINT=http://etcd:2379 + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=password + #- RUST_BACKTRACE=1 + ports: + #- 5454:5454 # pg protocol handler + - 7677:7676 # http endpoints + entrypoint: + - "/bin/sh" + - "-c" + command: + - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL + --listen-http='0.0.0.0:7676' + --id=$$SAFEKEEPER_ID + --broker-endpoints=$$BROKER_ENDPOINT + -D /data + --remote-storage=\"{endpoint='http://minio:9000', + bucket_name='neon', + bucket_region='eu-north-1', + prefix_in_bucket='/safekeeper/'}\"" + depends_on: + - etcd + - minio_create_buckets + + safekeeper3: + image: neondatabase/neon:${TAG:-latest} + environment: + - SAFEKEEPER_ADVERTISE_URL=safekeeper3:5454 + - SAFEKEEPER_ID=3 + - BROKER_ENDPOINT=http://etcd:2379 + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=password + #- RUST_BACKTRACE=1 + ports: + #- 5454:5454 # pg protocol handler + - 7678:7676 # http endpoints + entrypoint: + - "/bin/sh" + - "-c" + command: + - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL + --listen-http='0.0.0.0:7676' + --id=$$SAFEKEEPER_ID + --broker-endpoints=$$BROKER_ENDPOINT + -D /data + --remote-storage=\"{endpoint='http://minio:9000', + bucket_name='neon', + bucket_region='eu-north-1', + prefix_in_bucket='/safekeeper/'}\"" + depends_on: + - etcd + - minio_create_buckets + + compute: + build: + context: ./image/compute + args: + - COMPUTE_IMAGE=compute-node-v${PG_VERSION:-14}:${TAG:-latest} + - http_proxy=$http_proxy + - https_proxy=$https_proxy + environment: + - PG_VERSION=${PG_VERSION:-14} + #- RUST_BACKTRACE=1 + volumes: + - ./compute/var/db/postgres/specs/:/var/db/postgres/specs/ + - ./compute/shell/:/shell/ + ports: + - 55433:55433 # pg protocol handler + - 3080:3080 # http endpoints + entrypoint: + - "/shell/compute.sh" + depends_on: + - safekeeper1 + - safekeeper2 + - safekeeper3 + - pageserver + + compute_is_ready: + image: postgres:latest + entrypoint: + - "/bin/bash" + - "-c" + command: + - "until pg_isready -h compute -p 55433 ; do + echo 'Waiting to start compute...' && sleep 1; + done" + depends_on: + - compute diff --git a/docker-compose/image/compute/Dockerfile b/docker-compose/image/compute/Dockerfile new file mode 100644 index 0000000000..1b9d8c4900 --- /dev/null +++ b/docker-compose/image/compute/Dockerfile @@ -0,0 +1,10 @@ +ARG COMPUTE_IMAGE=compute-node-v14:latest +FROM neondatabase/${COMPUTE_IMAGE} + +USER root +RUN apt-get update && \ + apt-get install -y curl \ + jq \ + netcat + +USER postgres diff --git a/docs/docker.md b/docs/docker.md index 100cdd248b..42f0048e6f 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -18,3 +18,67 @@ We build all images after a successful `release` tests run and push automaticall 1. `neondatabase/compute-tools` and `neondatabase/compute-node` 2. `neondatabase/neon` + +## Docker Compose example + +You can see a [docker compose](https://docs.docker.com/compose/) example to create a neon cluster in [/docker-compose/docker-compose.yml](/docker-compose/docker-compose.yml). It creates the following conatainers. + +- etcd x 1 +- pageserver x 1 +- safekeeper x 3 +- compute x 1 +- MinIO x 1 # This is Amazon S3 compatible object storage + +### How to use + +1. create containers + +You can specify version of neon cluster using following environment values. +- PG_VERSION: postgres version for compute (default is 14) +- TAG: the tag version of [docker image](https://registry.hub.docker.com/r/neondatabase/neon/tags) (default is latest), which is tagged in [CI test](/.github/workflows/build_and_test.yml) +``` +$ cd docker-compose/docker-compose.yml +$ docker-compose down # remove the conainers if exists +$ PG_VERSION=15 TAG=2221 docker-compose up --build -d # You can specify the postgres and image version +Creating network "dockercompose_default" with the default driver +Creating dockercompose_etcd3_1 ... +(...omit...) +``` + +2. connect compute node +``` +$ echo "localhost:55433:postgres:cloud_admin:cloud_admin" >> ~/.pgpass +$ psql -h localhost -p 55433 -U cloud_admin +postgres=# CREATE TABLE t(key int primary key, value text); +CREATE TABLE +postgres=# insert into t values(1,1); +INSERT 0 1 +postgres=# select * from t; + key | value +-----+------- + 1 | 1 +(1 row) +``` + +3. If you want to see the log, you can use `docker-compose logs` command. +``` +# check the container name you want to see +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +d6968a5ae912 dockercompose_compute "/shell/compute.sh" 5 minutes ago Up 5 minutes 0.0.0.0:3080->3080/tcp, 0.0.0.0:55433->55433/tcp dockercompose_compute_1 +(...omit...) + +$ docker logs -f dockercompose_compute_1 +2022-10-21 06:15:48.757 GMT [56] LOG: connection authorized: user=cloud_admin database=postgres application_name=psql +2022-10-21 06:17:00.307 GMT [56] LOG: [NEON_SMGR] libpagestore: connected to 'host=pageserver port=6400' +(...omit...) +``` + +4. If you want to see durable data in MinIO which is s3 compatible storage + +Access http://localhost:9001 and sign in. + +- Username: `minio` +- Password: `password` + +You can see durable pages and WAL data in `neon` bucket. \ No newline at end of file diff --git a/scripts/docker-compose_test.sh b/scripts/docker-compose_test.sh new file mode 100755 index 0000000000..b4551365f8 --- /dev/null +++ b/scripts/docker-compose_test.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# this is a shortcut script to avoid duplication in CI +set -eux -o pipefail + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +COMPOSE_FILE=$SCRIPT_DIR/../docker-compose/docker-compose.yml + +COMPUTE_CONTAINER_NAME=dockercompose_compute_1 +SQL="CREATE TABLE t(key int primary key, value text); insert into t values(1,1); select * from t;" +PSQL_OPTION="-h localhost -U cloud_admin -p 55433 -c '$SQL' postgres" + +cleanup() { + echo "show container information" + docker ps + docker-compose -f $COMPOSE_FILE logs + echo "stop containers..." + docker-compose -f $COMPOSE_FILE down +} + +echo "clean up containers if exists" +cleanup + +for pg_version in 14 15; do + echo "start containers (pg_version=$pg_version)." + PG_VERSION=$pg_version TAG=latest docker-compose -f $COMPOSE_FILE up --build -d + + echo "wait until the compute is ready. timeout after 60s. " + cnt=0 + while sleep 1; do + # check timeout + cnt=`expr $cnt + 1` + if [ $cnt -gt 60 ]; then + echo "timeout before the compute is ready." + cleanup + exit 1 + fi + + # check if the compute is ready + set +o pipefail + result=`docker-compose -f $COMPOSE_FILE logs "compute_is_ready" | grep "accepting connections" | wc -l` + set -o pipefail + if [ $result -eq 1 ]; then + echo "OK. The compute is ready to connect." + echo "execute simple queries." + docker exec -it $COMPUTE_CONTAINER_NAME /bin/bash -c "psql $PSQL_OPTION" + cleanup + break + fi + done +done From 0c54eb65fbde98aae61b7d8a167c451ab5d62285 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Wed, 26 Oct 2022 17:32:31 -0400 Subject: [PATCH 05/16] Move pagestream api to libs/pageserver_api (#2698) --- Cargo.lock | 3 + libs/pageserver_api/Cargo.toml | 3 + libs/pageserver_api/src/lib.rs | 1 + libs/pageserver_api/src/models.rs | 161 +++++++++++++++++ .../pageserver_api}/src/reltag.rs | 0 pageserver/src/basebackup.rs | 2 +- pageserver/src/import_datadir.rs | 2 +- pageserver/src/lib.rs | 1 - pageserver/src/page_service.rs | 166 +----------------- pageserver/src/pgdatadir_mapping.rs | 2 +- pageserver/src/tenant/timeline.rs | 2 +- pageserver/src/walingest.rs | 2 +- pageserver/src/walredo.rs | 2 +- 13 files changed, 181 insertions(+), 166 deletions(-) rename {pageserver => libs/pageserver_api}/src/reltag.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index b39ca6e5a7..3e67126add 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2189,7 +2189,10 @@ dependencies = [ name = "pageserver_api" version = "0.1.0" dependencies = [ + "anyhow", + "bytes", "const_format", + "postgres_ffi", "serde", "serde_with", "utils", diff --git a/libs/pageserver_api/Cargo.toml b/libs/pageserver_api/Cargo.toml index 5995325a2f..9121cd4989 100644 --- a/libs/pageserver_api/Cargo.toml +++ b/libs/pageserver_api/Cargo.toml @@ -7,6 +7,9 @@ edition = "2021" serde = { version = "1.0", features = ["derive"] } serde_with = "2.0" const_format = "0.2.21" +anyhow = { version = "1.0", features = ["backtrace"] } +bytes = "1.0.1" utils = { path = "../utils" } +postgres_ffi = { path = "../postgres_ffi" } workspace_hack = { version = "0.1", path = "../../workspace_hack" } diff --git a/libs/pageserver_api/src/lib.rs b/libs/pageserver_api/src/lib.rs index a36c1692a9..4890d54f36 100644 --- a/libs/pageserver_api/src/lib.rs +++ b/libs/pageserver_api/src/lib.rs @@ -2,6 +2,7 @@ use const_format::formatcp; /// Public API types pub mod models; +pub mod reltag; pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000; pub const DEFAULT_PG_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_PG_LISTEN_PORT}"); diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index dd40ba9e1c..4360f76fd1 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -7,6 +7,10 @@ use utils::{ lsn::Lsn, }; +use crate::reltag::RelTag; +use anyhow::bail; +use bytes::{Buf, BufMut, Bytes, BytesMut}; + /// A state of a tenant in pageserver's memory. #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum TenantState { @@ -219,3 +223,160 @@ pub struct FailpointConfig { pub struct TimelineGcRequest { pub gc_horizon: Option, } + +// Wrapped in libpq CopyData +pub enum PagestreamFeMessage { + Exists(PagestreamExistsRequest), + Nblocks(PagestreamNblocksRequest), + GetPage(PagestreamGetPageRequest), + DbSize(PagestreamDbSizeRequest), +} + +// Wrapped in libpq CopyData +pub enum PagestreamBeMessage { + Exists(PagestreamExistsResponse), + Nblocks(PagestreamNblocksResponse), + GetPage(PagestreamGetPageResponse), + Error(PagestreamErrorResponse), + DbSize(PagestreamDbSizeResponse), +} + +#[derive(Debug)] +pub struct PagestreamExistsRequest { + pub latest: bool, + pub lsn: Lsn, + pub rel: RelTag, +} + +#[derive(Debug)] +pub struct PagestreamNblocksRequest { + pub latest: bool, + pub lsn: Lsn, + pub rel: RelTag, +} + +#[derive(Debug)] +pub struct PagestreamGetPageRequest { + pub latest: bool, + pub lsn: Lsn, + pub rel: RelTag, + pub blkno: u32, +} + +#[derive(Debug)] +pub struct PagestreamDbSizeRequest { + pub latest: bool, + pub lsn: Lsn, + pub dbnode: u32, +} + +#[derive(Debug)] +pub struct PagestreamExistsResponse { + pub exists: bool, +} + +#[derive(Debug)] +pub struct PagestreamNblocksResponse { + pub n_blocks: u32, +} + +#[derive(Debug)] +pub struct PagestreamGetPageResponse { + pub page: Bytes, +} + +#[derive(Debug)] +pub struct PagestreamErrorResponse { + pub message: String, +} + +#[derive(Debug)] +pub struct PagestreamDbSizeResponse { + pub db_size: i64, +} + +impl PagestreamFeMessage { + pub fn parse(mut body: Bytes) -> anyhow::Result { + // TODO these gets can fail + + // these correspond to the NeonMessageTag enum in pagestore_client.h + // + // TODO: consider using protobuf or serde bincode for less error prone + // serialization. + let msg_tag = body.get_u8(); + match msg_tag { + 0 => Ok(PagestreamFeMessage::Exists(PagestreamExistsRequest { + latest: body.get_u8() != 0, + lsn: Lsn::from(body.get_u64()), + rel: RelTag { + spcnode: body.get_u32(), + dbnode: body.get_u32(), + relnode: body.get_u32(), + forknum: body.get_u8(), + }, + })), + 1 => Ok(PagestreamFeMessage::Nblocks(PagestreamNblocksRequest { + latest: body.get_u8() != 0, + lsn: Lsn::from(body.get_u64()), + rel: RelTag { + spcnode: body.get_u32(), + dbnode: body.get_u32(), + relnode: body.get_u32(), + forknum: body.get_u8(), + }, + })), + 2 => Ok(PagestreamFeMessage::GetPage(PagestreamGetPageRequest { + latest: body.get_u8() != 0, + lsn: Lsn::from(body.get_u64()), + rel: RelTag { + spcnode: body.get_u32(), + dbnode: body.get_u32(), + relnode: body.get_u32(), + forknum: body.get_u8(), + }, + blkno: body.get_u32(), + })), + 3 => Ok(PagestreamFeMessage::DbSize(PagestreamDbSizeRequest { + latest: body.get_u8() != 0, + lsn: Lsn::from(body.get_u64()), + dbnode: body.get_u32(), + })), + _ => bail!("unknown smgr message tag: {},'{:?}'", msg_tag, body), + } + } +} + +impl PagestreamBeMessage { + pub fn serialize(&self) -> Bytes { + let mut bytes = BytesMut::new(); + + match self { + Self::Exists(resp) => { + bytes.put_u8(100); /* tag from pagestore_client.h */ + bytes.put_u8(resp.exists as u8); + } + + Self::Nblocks(resp) => { + bytes.put_u8(101); /* tag from pagestore_client.h */ + bytes.put_u32(resp.n_blocks); + } + + Self::GetPage(resp) => { + bytes.put_u8(102); /* tag from pagestore_client.h */ + bytes.put(&resp.page[..]); + } + + Self::Error(resp) => { + bytes.put_u8(103); /* tag from pagestore_client.h */ + bytes.put(resp.message.as_bytes()); + bytes.put_u8(0); // null terminator + } + Self::DbSize(resp) => { + bytes.put_u8(104); /* tag from pagestore_client.h */ + bytes.put_i64(resp.db_size); + } + } + + bytes.into() + } +} diff --git a/pageserver/src/reltag.rs b/libs/pageserver_api/src/reltag.rs similarity index 100% rename from pageserver/src/reltag.rs rename to libs/pageserver_api/src/reltag.rs diff --git a/pageserver/src/basebackup.rs b/pageserver/src/basebackup.rs index d0a57a473b..973c3cd3a6 100644 --- a/pageserver/src/basebackup.rs +++ b/pageserver/src/basebackup.rs @@ -22,8 +22,8 @@ use std::time::SystemTime; use tar::{Builder, EntryType, Header}; use tracing::*; -use crate::reltag::{RelTag, SlruKind}; use crate::tenant::Timeline; +use pageserver_api::reltag::{RelTag, SlruKind}; use postgres_ffi::pg_constants::{DEFAULTTABLESPACE_OID, GLOBALTABLESPACE_OID}; use postgres_ffi::pg_constants::{PGDATA_SPECIAL_FILES, PGDATA_SUBDIRS, PG_HBA}; diff --git a/pageserver/src/import_datadir.rs b/pageserver/src/import_datadir.rs index ee3dc684e3..642e41765b 100644 --- a/pageserver/src/import_datadir.rs +++ b/pageserver/src/import_datadir.rs @@ -12,10 +12,10 @@ use tracing::*; use walkdir::WalkDir; use crate::pgdatadir_mapping::*; -use crate::reltag::{RelTag, SlruKind}; use crate::tenant::Timeline; use crate::walingest::WalIngest; use crate::walrecord::DecodedWALRecord; +use pageserver_api::reltag::{RelTag, SlruKind}; use postgres_ffi::pg_constants; use postgres_ffi::relfile_utils::*; use postgres_ffi::waldecoder::WalStreamDecoder; diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index c75f940386..52a4cb0381 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -8,7 +8,6 @@ pub mod page_cache; pub mod page_service; pub mod pgdatadir_mapping; pub mod profiling; -pub mod reltag; pub mod repository; pub mod storage_sync; pub mod task_mgr; diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index d61885314e..aec91bc7f1 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -10,8 +10,14 @@ // use anyhow::{bail, ensure, Context, Result}; -use bytes::{Buf, BufMut, Bytes, BytesMut}; +use bytes::Bytes; use futures::{Stream, StreamExt}; +use pageserver_api::models::{ + PagestreamBeMessage, PagestreamDbSizeRequest, PagestreamDbSizeResponse, + PagestreamErrorResponse, PagestreamExistsRequest, PagestreamExistsResponse, + PagestreamFeMessage, PagestreamGetPageRequest, PagestreamGetPageResponse, + PagestreamNblocksRequest, PagestreamNblocksResponse, +}; use std::io; use std::net::TcpListener; use std::str; @@ -35,7 +41,6 @@ use crate::config::{PageServerConf, ProfilingConfig}; use crate::import_datadir::import_wal_from_tar; use crate::metrics::{LIVE_CONNECTIONS_COUNT, SMGR_QUERY_TIME}; use crate::profiling::profpoint_start; -use crate::reltag::RelTag; use crate::task_mgr; use crate::task_mgr::TaskKind; use crate::tenant::Timeline; @@ -45,163 +50,6 @@ use crate::CheckpointConfig; use postgres_ffi::pg_constants::DEFAULTTABLESPACE_OID; use postgres_ffi::BLCKSZ; -// Wrapped in libpq CopyData -enum PagestreamFeMessage { - Exists(PagestreamExistsRequest), - Nblocks(PagestreamNblocksRequest), - GetPage(PagestreamGetPageRequest), - DbSize(PagestreamDbSizeRequest), -} - -// Wrapped in libpq CopyData -enum PagestreamBeMessage { - Exists(PagestreamExistsResponse), - Nblocks(PagestreamNblocksResponse), - GetPage(PagestreamGetPageResponse), - Error(PagestreamErrorResponse), - DbSize(PagestreamDbSizeResponse), -} - -#[derive(Debug)] -struct PagestreamExistsRequest { - latest: bool, - lsn: Lsn, - rel: RelTag, -} - -#[derive(Debug)] -struct PagestreamNblocksRequest { - latest: bool, - lsn: Lsn, - rel: RelTag, -} - -#[derive(Debug)] -struct PagestreamGetPageRequest { - latest: bool, - lsn: Lsn, - rel: RelTag, - blkno: u32, -} - -#[derive(Debug)] -struct PagestreamDbSizeRequest { - latest: bool, - lsn: Lsn, - dbnode: u32, -} - -#[derive(Debug)] -struct PagestreamExistsResponse { - exists: bool, -} - -#[derive(Debug)] -struct PagestreamNblocksResponse { - n_blocks: u32, -} - -#[derive(Debug)] -struct PagestreamGetPageResponse { - page: Bytes, -} - -#[derive(Debug)] -struct PagestreamErrorResponse { - message: String, -} - -#[derive(Debug)] -struct PagestreamDbSizeResponse { - db_size: i64, -} - -impl PagestreamFeMessage { - fn parse(mut body: Bytes) -> anyhow::Result { - // TODO these gets can fail - - // these correspond to the NeonMessageTag enum in pagestore_client.h - // - // TODO: consider using protobuf or serde bincode for less error prone - // serialization. - let msg_tag = body.get_u8(); - match msg_tag { - 0 => Ok(PagestreamFeMessage::Exists(PagestreamExistsRequest { - latest: body.get_u8() != 0, - lsn: Lsn::from(body.get_u64()), - rel: RelTag { - spcnode: body.get_u32(), - dbnode: body.get_u32(), - relnode: body.get_u32(), - forknum: body.get_u8(), - }, - })), - 1 => Ok(PagestreamFeMessage::Nblocks(PagestreamNblocksRequest { - latest: body.get_u8() != 0, - lsn: Lsn::from(body.get_u64()), - rel: RelTag { - spcnode: body.get_u32(), - dbnode: body.get_u32(), - relnode: body.get_u32(), - forknum: body.get_u8(), - }, - })), - 2 => Ok(PagestreamFeMessage::GetPage(PagestreamGetPageRequest { - latest: body.get_u8() != 0, - lsn: Lsn::from(body.get_u64()), - rel: RelTag { - spcnode: body.get_u32(), - dbnode: body.get_u32(), - relnode: body.get_u32(), - forknum: body.get_u8(), - }, - blkno: body.get_u32(), - })), - 3 => Ok(PagestreamFeMessage::DbSize(PagestreamDbSizeRequest { - latest: body.get_u8() != 0, - lsn: Lsn::from(body.get_u64()), - dbnode: body.get_u32(), - })), - _ => bail!("unknown smgr message tag: {},'{:?}'", msg_tag, body), - } - } -} - -impl PagestreamBeMessage { - fn serialize(&self) -> Bytes { - let mut bytes = BytesMut::new(); - - match self { - Self::Exists(resp) => { - bytes.put_u8(100); /* tag from pagestore_client.h */ - bytes.put_u8(resp.exists as u8); - } - - Self::Nblocks(resp) => { - bytes.put_u8(101); /* tag from pagestore_client.h */ - bytes.put_u32(resp.n_blocks); - } - - Self::GetPage(resp) => { - bytes.put_u8(102); /* tag from pagestore_client.h */ - bytes.put(&resp.page[..]); - } - - Self::Error(resp) => { - bytes.put_u8(103); /* tag from pagestore_client.h */ - bytes.put(resp.message.as_bytes()); - bytes.put_u8(0); // null terminator - } - Self::DbSize(resp) => { - bytes.put_u8(104); /* tag from pagestore_client.h */ - bytes.put_i64(resp.db_size); - } - } - - bytes.into() - } -} - fn copyin_stream(pgb: &mut PostgresBackend) -> impl Stream> + '_ { async_stream::try_stream! { loop { diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index ca931ed37d..0e334a63df 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -7,12 +7,12 @@ //! Clarify that) //! use crate::keyspace::{KeySpace, KeySpaceAccum}; -use crate::reltag::{RelTag, SlruKind}; use crate::repository::*; use crate::tenant::Timeline; use crate::walrecord::NeonWalRecord; use anyhow::{bail, ensure, Result}; use bytes::{Buf, Bytes}; +use pageserver_api::reltag::{RelTag, SlruKind}; use postgres_ffi::relfile_utils::{FSM_FORKNUM, VISIBILITYMAP_FORKNUM}; use postgres_ffi::BLCKSZ; use postgres_ffi::{Oid, TimestampTz, TransactionId}; diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 194ca0d857..6a96254df4 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -37,8 +37,8 @@ use crate::metrics::TimelineMetrics; use crate::pgdatadir_mapping::BlockNumber; use crate::pgdatadir_mapping::LsnForTimestamp; use crate::pgdatadir_mapping::{is_rel_fsm_block_key, is_rel_vm_block_key}; -use crate::reltag::RelTag; use crate::tenant_config::TenantConfOpt; +use pageserver_api::reltag::RelTag; use postgres_ffi::to_pg_timestamp; use utils::{ diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 9a6b99d991..8c81ed824b 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -31,10 +31,10 @@ use bytes::{Buf, Bytes, BytesMut}; use tracing::*; use crate::pgdatadir_mapping::*; -use crate::reltag::{RelTag, SlruKind}; use crate::tenant::Timeline; use crate::walrecord::*; use crate::ZERO_PAGE; +use pageserver_api::reltag::{RelTag, SlruKind}; use postgres_ffi::pg_constants; use postgres_ffi::relfile_utils::{FSM_FORKNUM, MAIN_FORKNUM, VISIBILITYMAP_FORKNUM}; use postgres_ffi::v14::nonrelfile_utils::mx_offset_to_member_segment; diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index e683c301d8..1cde11082e 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -43,10 +43,10 @@ use crate::metrics::{ WAL_REDO_WAIT_TIME, }; use crate::pgdatadir_mapping::{key_to_rel_block, key_to_slru_block}; -use crate::reltag::{RelTag, SlruKind}; use crate::repository::Key; use crate::walrecord::NeonWalRecord; use crate::{config::PageServerConf, TEMP_FILE_SUFFIX}; +use pageserver_api::reltag::{RelTag, SlruKind}; use postgres_ffi::pg_constants; use postgres_ffi::relfile_utils::VISIBILITYMAP_FORKNUM; use postgres_ffi::v14::nonrelfile_utils::{ From 1f08ba5790dc19293434d1aca779125002ebe8bc Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Thu, 27 Oct 2022 03:50:46 +0300 Subject: [PATCH 06/16] Avoid debian-testing packages in compute Dockerfiles plv8 can only be built with a fairly new gold linker version. We used to install it via binutils packages from testing, but it also updates libc and that causes troubles in the resulting image as different extensions were built against different libc versions. We could either use libc from debian-testing everywhere or restrain from using testing packages and install necessary programs manually. This patch uses the latter approach: gold for plv8 and cmake for h3 are installed manually. In a passing declare h3_postgis as a safe extension (previous omission). --- Dockerfile.compute-node-v14 | 87 ++++++++++++++++++++++--------------- Dockerfile.compute-node-v15 | 74 ++++++++++++++++++------------- 2 files changed, 95 insertions(+), 66 deletions(-) diff --git a/Dockerfile.compute-node-v14 b/Dockerfile.compute-node-v14 index 6d2b285fa3..035dfc0d08 100644 --- a/Dockerfile.compute-node-v14 +++ b/Dockerfile.compute-node-v14 @@ -1,24 +1,26 @@ -ARG TAG=pinned -# apparently, ARGs don't get replaced in RUN commands in kaniko -# ARG POSTGIS_VERSION=3.3.0 -# ARG PLV8_VERSION=3.1.4 -# ARG PG_VERSION=v14 +# +# This file is identical to the Dockerfile.compute-node-v15 file +# except for the version of Postgres that is built. +# +ARG TAG=pinned + +######################################################################################### # # Layer "build-deps" # +######################################################################################### FROM debian:bullseye-slim AS build-deps -RUN echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \ - echo "Package: *\nPin: release n=bullseye\nPin-Priority: 50" > /etc/apt/preferences && \ - apt update RUN apt update && \ - apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \ - libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libglib2.0-dev + apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \ + zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config +######################################################################################### # # Layer "pg-build" # Build Postgres from the neon postgres repository. # +######################################################################################### FROM build-deps AS pg-build COPY vendor/postgres-v14 postgres RUN cd postgres && \ @@ -29,22 +31,20 @@ RUN cd postgres && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install +######################################################################################### # # Layer "postgis-build" # Build PostGIS from the upstream PostGIS mirror. # -# PostGIS compiles against neon postgres sources without changes. Perhaps we -# could even use the upstream binaries, compiled against vanilla Postgres, but -# it would require some investigation to check that it works, and also keeps -# working in the future. So for now, we compile our own binaries. +######################################################################################### FROM build-deps AS postgis-build COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ RUN apt update && \ apt install -y gdal-bin libgdal-dev libprotobuf-c-dev protobuf-c-compiler xsltproc -RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.0.tar.gz && \ - tar xvzf postgis-3.3.0.tar.gz && \ - cd postgis-3.3.0 && \ +RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \ + tar xvzf postgis-3.3.1.tar.gz && \ + cd postgis-3.3.1 && \ ./autogen.sh && \ export PATH="/usr/local/pgsql/bin:$PATH" && \ ./configure && \ @@ -57,19 +57,29 @@ RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.0.tar.gz && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control +######################################################################################### # # Layer "plv8-build" # Build plv8 # +######################################################################################### FROM build-deps AS plv8-build COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ RUN apt update && \ - apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5 + apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5 binutils -# https://github.com/plv8/plv8/issues/475 -# Debian bullseye provides binutils 2.35 when >= 2.38 is necessary -RUN apt update && \ - apt install -y --no-install-recommends -t testing binutils +# https://github.com/plv8/plv8/issues/475: +# v8 uses gold for linking and sets `--thread-count=4` which breaks +# gold version <= 1.35 (https://sourceware.org/bugzilla/show_bug.cgi?id=23607) +# Install newer gold version manually as debian-testing binutils version updates +# libc version, which in turn breaks other extension built against non-testing libc. +RUN wget https://ftp.gnu.org/gnu/binutils/binutils-2.38.tar.gz && \ + tar xvzf binutils-2.38.tar.gz && \ + cd binutils-2.38 && \ + cd libiberty && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && \ + cd ../bfd && ./configure && make bfdver.h && \ + cd ../gold && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && make install && \ + cp /usr/local/bin/ld.gold /usr/bin/gold # Sed is used to patch for https://github.com/plv8/plv8/issues/503 RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \ @@ -77,21 +87,25 @@ RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \ cd plv8-3.1.4 && \ export PATH="/usr/local/pgsql/bin:$PATH" && \ sed -i 's/MemoryContextAlloc(/MemoryContextAllocZero(/' plv8.cc && \ - make -j $(getconf _NPROCESSORS_ONLN) && \ - make -j $(getconf _NPROCESSORS_ONLN) install && \ + make DOCKER=1 -j $(getconf _NPROCESSORS_ONLN) install && \ rm -rf /plv8-* && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/plv8.control +######################################################################################### # # Layer "h3-pg-build" # Build h3_pg # +######################################################################################### FROM build-deps AS h3-pg-build COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # packaged cmake is too old -RUN apt update && \ - apt install -y --no-install-recommends -t testing cmake +RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh \ + -q -O /tmp/cmake-install.sh \ + && chmod u+x /tmp/cmake-install.sh \ + && /tmp/cmake-install.sh --skip-license --prefix=/usr/local/ \ + && rm /tmp/cmake-install.sh RUN wget https://github.com/uber/h3/archive/refs/tags/v4.0.1.tar.gz -O h3.tgz && \ tar xvzf h3.tgz && \ @@ -110,12 +124,15 @@ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3 export PATH="/usr/local/pgsql/bin:$PATH" && \ make -j $(getconf _NPROCESSORS_ONLN) && \ make -j $(getconf _NPROCESSORS_ONLN) install && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control + echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3_postgis.control +######################################################################################### # # Layer "neon-pg-ext-build" # compile neon extensions # +######################################################################################### FROM build-deps AS neon-pg-ext-build COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/ @@ -128,16 +145,22 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \ -C pgxn/neon \ -s install +######################################################################################### +# # Compile and run the Neon-specific `compute_ctl` binary +# +######################################################################################### FROM 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:$TAG AS compute-tools USER nonroot # Copy entire project to get Cargo.* files with proper dependencies for the whole project COPY --chown=nonroot . . RUN cd compute_tools && cargo build --locked --profile release-line-debug-size-lto +######################################################################################### # # Clean up postgres folder before inclusion # +######################################################################################### FROM neon-pg-ext-build AS postgres-cleanup-layer COPY --from=neon-pg-ext-build /usr/local/pgsql /usr/local/pgsql @@ -155,10 +178,12 @@ RUN rm -r /usr/local/pgsql/lib/pgxs/src # if they were to be used by other libraries. RUN rm /usr/local/pgsql/lib/lib*.a +######################################################################################### # # Final layer # Put it all together into the final image # +######################################################################################### FROM debian:bullseye-slim # Add user postgres RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \ @@ -175,8 +200,6 @@ COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-deb # libreadline8 for psql # libossp-uuid16 for extension ossp-uuid # libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS -# GLIBC 2.34 for plv8. -# Debian bullseye provides GLIBC 2.31, so we install the library from testing # # Lastly, link compute_ctl into zenith_ctl while we're at it, # so that we don't need to put this in another layer. @@ -189,12 +212,6 @@ RUN apt update && \ libproj19 \ libprotobuf-c1 && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ - echo "Installing GLIBC 2.34" && \ - echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \ - echo "Package: *\nPin: release n=bullseye\nPin-Priority: 50" > /etc/apt/preferences && \ - apt update && \ - apt install -y --no-install-recommends -t testing libc6 && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ ln /usr/local/bin/compute_ctl /usr/local/bin/zenith_ctl USER postgres diff --git a/Dockerfile.compute-node-v15 b/Dockerfile.compute-node-v15 index b7b1f25103..0b6e570b44 100644 --- a/Dockerfile.compute-node-v15 +++ b/Dockerfile.compute-node-v15 @@ -4,26 +4,23 @@ # ARG TAG=pinned -# apparently, ARGs don't get replaced in RUN commands in kaniko -# ARG POSTGIS_VERSION=3.3.1 -# ARG PLV8_VERSION=3.1.4 -# ARG PG_VERSION=v15 +######################################################################################### # # Layer "build-deps" # +######################################################################################### FROM debian:bullseye-slim AS build-deps -RUN echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \ - echo "Package: *\nPin: release n=bullseye\nPin-Priority: 50" > /etc/apt/preferences && \ - apt update RUN apt update && \ - apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \ - libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libglib2.0-dev + apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \ + zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config +######################################################################################### # # Layer "pg-build" # Build Postgres from the neon postgres repository. # +######################################################################################### FROM build-deps AS pg-build COPY vendor/postgres-v15 postgres RUN cd postgres && \ @@ -34,14 +31,12 @@ RUN cd postgres && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install +######################################################################################### # # Layer "postgis-build" # Build PostGIS from the upstream PostGIS mirror. # -# PostGIS compiles against neon postgres sources without changes. Perhaps we -# could even use the upstream binaries, compiled against vanilla Postgres, but -# it would require some investigation to check that it works, and also keeps -# working in the future. So for now, we compile our own binaries. +######################################################################################### FROM build-deps AS postgis-build COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ RUN apt update && \ @@ -62,19 +57,29 @@ RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control +######################################################################################### # # Layer "plv8-build" # Build plv8 # +######################################################################################### FROM build-deps AS plv8-build COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ RUN apt update && \ - apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5 + apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5 binutils -# https://github.com/plv8/plv8/issues/475 -# Debian bullseye provides binutils 2.35 when >= 2.38 is necessary -RUN apt update && \ - apt install -y --no-install-recommends -t testing binutils +# https://github.com/plv8/plv8/issues/475: +# v8 uses gold for linking and sets `--thread-count=4` which breaks +# gold version <= 1.35 (https://sourceware.org/bugzilla/show_bug.cgi?id=23607) +# Install newer gold version manually as debian-testing binutils version updates +# libc version, which in turn breaks other extension built against non-testing libc. +RUN wget https://ftp.gnu.org/gnu/binutils/binutils-2.38.tar.gz && \ + tar xvzf binutils-2.38.tar.gz && \ + cd binutils-2.38 && \ + cd libiberty && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && \ + cd ../bfd && ./configure && make bfdver.h && \ + cd ../gold && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && make install && \ + cp /usr/local/bin/ld.gold /usr/bin/gold # Sed is used to patch for https://github.com/plv8/plv8/issues/503 RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \ @@ -82,21 +87,25 @@ RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \ cd plv8-3.1.4 && \ export PATH="/usr/local/pgsql/bin:$PATH" && \ sed -i 's/MemoryContextAlloc(/MemoryContextAllocZero(/' plv8.cc && \ - make -j $(getconf _NPROCESSORS_ONLN) && \ - make -j $(getconf _NPROCESSORS_ONLN) install && \ + make DOCKER=1 -j $(getconf _NPROCESSORS_ONLN) install && \ rm -rf /plv8-* && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/plv8.control +######################################################################################### # # Layer "h3-pg-build" # Build h3_pg # +######################################################################################### FROM build-deps AS h3-pg-build COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # packaged cmake is too old -RUN apt update && \ - apt install -y --no-install-recommends -t testing cmake +RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh \ + -q -O /tmp/cmake-install.sh \ + && chmod u+x /tmp/cmake-install.sh \ + && /tmp/cmake-install.sh --skip-license --prefix=/usr/local/ \ + && rm /tmp/cmake-install.sh RUN wget https://github.com/uber/h3/archive/refs/tags/v4.0.1.tar.gz -O h3.tgz && \ tar xvzf h3.tgz && \ @@ -115,12 +124,15 @@ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3 export PATH="/usr/local/pgsql/bin:$PATH" && \ make -j $(getconf _NPROCESSORS_ONLN) && \ make -j $(getconf _NPROCESSORS_ONLN) install && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control + echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3_postgis.control +######################################################################################### # # Layer "neon-pg-ext-build" # compile neon extensions # +######################################################################################### FROM build-deps AS neon-pg-ext-build COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/ @@ -133,16 +145,22 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \ -C pgxn/neon \ -s install +######################################################################################### +# # Compile and run the Neon-specific `compute_ctl` binary +# +######################################################################################### FROM 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:$TAG AS compute-tools USER nonroot # Copy entire project to get Cargo.* files with proper dependencies for the whole project COPY --chown=nonroot . . RUN cd compute_tools && cargo build --locked --profile release-line-debug-size-lto +######################################################################################### # # Clean up postgres folder before inclusion # +######################################################################################### FROM neon-pg-ext-build AS postgres-cleanup-layer COPY --from=neon-pg-ext-build /usr/local/pgsql /usr/local/pgsql @@ -160,10 +178,12 @@ RUN rm -r /usr/local/pgsql/lib/pgxs/src # if they were to be used by other libraries. RUN rm /usr/local/pgsql/lib/lib*.a +######################################################################################### # # Final layer # Put it all together into the final image # +######################################################################################### FROM debian:bullseye-slim # Add user postgres RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \ @@ -180,8 +200,6 @@ COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-deb # libreadline8 for psql # libossp-uuid16 for extension ossp-uuid # libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS -# GLIBC 2.34 for plv8. -# Debian bullseye provides GLIBC 2.31, so we install the library from testing # # Lastly, link compute_ctl into zenith_ctl while we're at it, # so that we don't need to put this in another layer. @@ -194,12 +212,6 @@ RUN apt update && \ libproj19 \ libprotobuf-c1 && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ - echo "Installing GLIBC 2.34" && \ - echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \ - echo "Package: *\nPin: release n=bullseye\nPin-Priority: 50" > /etc/apt/preferences && \ - apt update && \ - apt install -y --no-install-recommends -t testing libc6 && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ ln /usr/local/bin/compute_ctl /usr/local/bin/zenith_ctl USER postgres From b42bf9265ad7b28c0aa3186f9afc2c22e79533f6 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Thu, 27 Oct 2022 11:09:09 +0400 Subject: [PATCH 07/16] Enable etcd compaction in neon_local. --- control_plane/src/etcd.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/control_plane/src/etcd.rs b/control_plane/src/etcd.rs index ccadfa8ce7..ca2df8a50b 100644 --- a/control_plane/src/etcd.rs +++ b/control_plane/src/etcd.rs @@ -52,6 +52,10 @@ pub fn start_etcd_process(env: &local_env::LocalEnv) -> anyhow::Result<()> { // size smaller. Our test etcd clusters are very small. // See https://github.com/etcd-io/etcd/issues/7910 "--quota-backend-bytes=100000000".to_string(), + // etcd doesn't compact (vacuum) with default settings, + // enable it to prevent space exhaustion. + "--auto-compaction-mode=revision".to_string(), + "--auto-compaction-retention=1".to_string(), ]) .stdout(Stdio::from(etcd_stdout_file)) .stderr(Stdio::from(etcd_stderr_file)) From 6dbf202e0df73ab033c458d815245e3dbea77f46 Mon Sep 17 00:00:00 2001 From: Rory de Zoete <33318916+zoete@users.noreply.github.com> Date: Thu, 27 Oct 2022 16:00:40 +0200 Subject: [PATCH 08/16] Update crane copy target (#2704) Co-authored-by: Rory de Zoete --- .github/workflows/build_and_test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1b8b380179..8d16e406ce 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -625,11 +625,11 @@ jobs: (github.ref_name == 'main' || github.ref_name == 'release') && github.event_name != 'workflow_dispatch' run: | - crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/neon:latest - crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-tools:latest - crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node:latest - crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node-v14:latest - crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node-v15:latest + crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/neon:latest + crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:latest + crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node:latest + crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:latest + crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:latest - name: Configure Docker Hub login run: | From 78e412b84b71ff34e71c84e0150d9e23973dee4a Mon Sep 17 00:00:00 2001 From: Alexander Stanovoy <38102252+alexstanovoy@users.noreply.github.com> Date: Thu, 27 Oct 2022 17:02:55 +0300 Subject: [PATCH 09/16] The fix of #2650. (#2686) * Wrappers and drop implementations for image and delta layer writers. * Two regression tests for the image and delta layer files. --- pageserver/src/tenant/delta_layer.rs | 118 ++++++++++++++++-- pageserver/src/tenant/image_layer.rs | 93 ++++++++++++-- pageserver/src/tenant/timeline.rs | 9 ++ pageserver/src/virtual_file.rs | 6 + test_runner/fixtures/neon_fixtures.py | 2 + .../regress/test_layer_writers_fail.py | 92 ++++++++++++++ 6 files changed, 302 insertions(+), 18 deletions(-) create mode 100644 test_runner/regress/test_layer_writers_fail.py diff --git a/pageserver/src/tenant/delta_layer.rs b/pageserver/src/tenant/delta_layer.rs index 41715ab0a4..a908d66200 100644 --- a/pageserver/src/tenant/delta_layer.rs +++ b/pageserver/src/tenant/delta_layer.rs @@ -610,9 +610,9 @@ impl DeltaLayer { /// /// 3. Call `finish`. /// -pub struct DeltaLayerWriter { +struct DeltaLayerWriterInner { conf: &'static PageServerConf, - path: PathBuf, + pub path: PathBuf, timeline_id: TimelineId, tenant_id: TenantId, @@ -624,17 +624,17 @@ pub struct DeltaLayerWriter { blob_writer: WriteBlobWriter>, } -impl DeltaLayerWriter { +impl DeltaLayerWriterInner { /// /// Start building a new delta layer. /// - pub fn new( + fn new( conf: &'static PageServerConf, timeline_id: TimelineId, tenant_id: TenantId, key_start: Key, lsn_range: Range, - ) -> Result { + ) -> anyhow::Result { // Create the file initially with a temporary filename. We don't know // the end key yet, so we cannot form the final filename yet. We will // rename it when we're done. @@ -653,7 +653,7 @@ impl DeltaLayerWriter { let block_buf = BlockBuf::new(); let tree_builder = DiskBtreeBuilder::new(block_buf); - Ok(DeltaLayerWriter { + Ok(Self { conf, path, timeline_id, @@ -670,17 +670,17 @@ impl DeltaLayerWriter { /// /// The values must be appended in key, lsn order. /// - pub fn put_value(&mut self, key: Key, lsn: Lsn, val: Value) -> Result<()> { + fn put_value(&mut self, key: Key, lsn: Lsn, val: Value) -> anyhow::Result<()> { self.put_value_bytes(key, lsn, &Value::ser(&val)?, val.will_init()) } - pub fn put_value_bytes( + fn put_value_bytes( &mut self, key: Key, lsn: Lsn, val: &[u8], will_init: bool, - ) -> Result<()> { + ) -> anyhow::Result<()> { assert!(self.lsn_range.start <= lsn); let off = self.blob_writer.write_blob(val)?; @@ -693,14 +693,14 @@ impl DeltaLayerWriter { Ok(()) } - pub fn size(&self) -> u64 { + fn size(&self) -> u64 { self.blob_writer.size() + self.tree.borrow_writer().size() } /// /// Finish writing the delta layer. /// - pub fn finish(self, key_end: Key) -> anyhow::Result { + fn finish(self, key_end: Key) -> anyhow::Result { let index_start_blk = ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32; @@ -768,6 +768,102 @@ impl DeltaLayerWriter { } } +/// A builder object for constructing a new delta layer. +/// +/// Usage: +/// +/// 1. Create the DeltaLayerWriter by calling DeltaLayerWriter::new(...) +/// +/// 2. Write the contents by calling `put_value` for every page +/// version to store in the layer. +/// +/// 3. Call `finish`. +/// +/// # Note +/// +/// As described in https://github.com/neondatabase/neon/issues/2650, it's +/// possible for the writer to drop before `finish` is actually called. So this +/// could lead to odd temporary files in the directory, exhausting file system. +/// This structure wraps `DeltaLayerWriterInner` and also contains `Drop` +/// implementation that cleans up the temporary file in failure. It's not +/// possible to do this directly in `DeltaLayerWriterInner` since `finish` moves +/// out some fields, making it impossible to implement `Drop`. +/// +#[must_use] +pub struct DeltaLayerWriter { + inner: Option, +} + +impl DeltaLayerWriter { + /// + /// Start building a new delta layer. + /// + pub fn new( + conf: &'static PageServerConf, + timeline_id: TimelineId, + tenant_id: TenantId, + key_start: Key, + lsn_range: Range, + ) -> anyhow::Result { + Ok(Self { + inner: Some(DeltaLayerWriterInner::new( + conf, + timeline_id, + tenant_id, + key_start, + lsn_range, + )?), + }) + } + + /// + /// Append a key-value pair to the file. + /// + /// The values must be appended in key, lsn order. + /// + pub fn put_value(&mut self, key: Key, lsn: Lsn, val: Value) -> anyhow::Result<()> { + self.inner.as_mut().unwrap().put_value(key, lsn, val) + } + + pub fn put_value_bytes( + &mut self, + key: Key, + lsn: Lsn, + val: &[u8], + will_init: bool, + ) -> anyhow::Result<()> { + self.inner + .as_mut() + .unwrap() + .put_value_bytes(key, lsn, val, will_init) + } + + pub fn size(&self) -> u64 { + self.inner.as_ref().unwrap().size() + } + + /// + /// Finish writing the delta layer. + /// + pub fn finish(mut self, key_end: Key) -> anyhow::Result { + self.inner.take().unwrap().finish(key_end) + } +} + +impl Drop for DeltaLayerWriter { + fn drop(&mut self) { + if let Some(inner) = self.inner.take() { + match inner.blob_writer.into_inner().into_inner() { + Ok(vfile) => vfile.remove(), + Err(err) => warn!( + "error while flushing buffer of image layer temporary file: {}", + err + ), + } + } + } +} + /// /// Iterator over all key-value pairse stored in a delta layer /// diff --git a/pageserver/src/tenant/image_layer.rs b/pageserver/src/tenant/image_layer.rs index cbfa0134b0..8409d34bc9 100644 --- a/pageserver/src/tenant/image_layer.rs +++ b/pageserver/src/tenant/image_layer.rs @@ -411,7 +411,7 @@ impl ImageLayer { /// /// 3. Call `finish`. /// -pub struct ImageLayerWriter { +struct ImageLayerWriterInner { conf: &'static PageServerConf, path: PathBuf, timeline_id: TimelineId, @@ -423,14 +423,17 @@ pub struct ImageLayerWriter { tree: DiskBtreeBuilder, } -impl ImageLayerWriter { - pub fn new( +impl ImageLayerWriterInner { + /// + /// Start building a new image layer. + /// + fn new( conf: &'static PageServerConf, timeline_id: TimelineId, tenant_id: TenantId, key_range: &Range, lsn: Lsn, - ) -> anyhow::Result { + ) -> anyhow::Result { // Create the file initially with a temporary filename. // We'll atomically rename it to the final name when we're done. let path = ImageLayer::temp_path_for( @@ -455,7 +458,7 @@ impl ImageLayerWriter { let block_buf = BlockBuf::new(); let tree_builder = DiskBtreeBuilder::new(block_buf); - let writer = ImageLayerWriter { + let writer = Self { conf, path, timeline_id, @@ -474,7 +477,7 @@ impl ImageLayerWriter { /// /// The page versions must be appended in blknum order. /// - pub fn put_image(&mut self, key: Key, img: &[u8]) -> Result<()> { + fn put_image(&mut self, key: Key, img: &[u8]) -> anyhow::Result<()> { ensure!(self.key_range.contains(&key)); let off = self.blob_writer.write_blob(img)?; @@ -485,7 +488,10 @@ impl ImageLayerWriter { Ok(()) } - pub fn finish(self) -> anyhow::Result { + /// + /// Finish writing the image layer. + /// + fn finish(self) -> anyhow::Result { let index_start_blk = ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32; @@ -552,3 +558,76 @@ impl ImageLayerWriter { Ok(layer) } } + +/// A builder object for constructing a new image layer. +/// +/// Usage: +/// +/// 1. Create the ImageLayerWriter by calling ImageLayerWriter::new(...) +/// +/// 2. Write the contents by calling `put_page_image` for every key-value +/// pair in the key range. +/// +/// 3. Call `finish`. +/// +/// # Note +/// +/// As described in https://github.com/neondatabase/neon/issues/2650, it's +/// possible for the writer to drop before `finish` is actually called. So this +/// could lead to odd temporary files in the directory, exhausting file system. +/// This structure wraps `ImageLayerWriterInner` and also contains `Drop` +/// implementation that cleans up the temporary file in failure. It's not +/// possible to do this directly in `ImageLayerWriterInner` since `finish` moves +/// out some fields, making it impossible to implement `Drop`. +/// +#[must_use] +pub struct ImageLayerWriter { + inner: Option, +} + +impl ImageLayerWriter { + /// + /// Start building a new image layer. + /// + pub fn new( + conf: &'static PageServerConf, + timeline_id: TimelineId, + tenant_id: TenantId, + key_range: &Range, + lsn: Lsn, + ) -> anyhow::Result { + Ok(Self { + inner: Some(ImageLayerWriterInner::new( + conf, + timeline_id, + tenant_id, + key_range, + lsn, + )?), + }) + } + + /// + /// Write next value to the file. + /// + /// The page versions must be appended in blknum order. + /// + pub fn put_image(&mut self, key: Key, img: &[u8]) -> anyhow::Result<()> { + self.inner.as_mut().unwrap().put_image(key, img) + } + + /// + /// Finish writing the image layer. + /// + pub fn finish(mut self) -> anyhow::Result { + self.inner.take().unwrap().finish() + } +} + +impl Drop for ImageLayerWriter { + fn drop(&mut self) { + if let Some(inner) = self.inner.take() { + inner.blob_writer.into_inner().remove(); + } + } +} diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 6a96254df4..d63429ea6a 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1541,6 +1541,10 @@ impl Timeline { lsn, )?; + fail_point!("image-layer-writer-fail-before-finish", |_| { + anyhow::bail!("failpoint image-layer-writer-fail-before-finish"); + }); + for range in &partition.ranges { let mut key = range.start; while key < range.end { @@ -1835,6 +1839,11 @@ impl Timeline { }, )?); } + + fail_point!("delta-layer-writer-fail-before-finish", |_| { + anyhow::bail!("failpoint delta-layer-writer-fail-before-finish"); + }); + writer.as_mut().unwrap().put_value(key, lsn, value)?; prev_key = Some(key); } diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs index 896c2603a2..46e4acd50c 100644 --- a/pageserver/src/virtual_file.rs +++ b/pageserver/src/virtual_file.rs @@ -319,6 +319,12 @@ impl VirtualFile { Ok(result) } + + pub fn remove(self) { + let path = self.path.clone(); + drop(self); + std::fs::remove_file(path).expect("failed to remove the virtual file"); + } } impl Drop for VirtualFile { diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 38a0db7cf7..e7e0e4ce56 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1628,6 +1628,8 @@ class NeonPageserver(PgProtocol): Initializes the repository via `neon init`. """ + TEMP_FILE_SUFFIX = "___temp" + def __init__(self, env: NeonEnv, port: PageserverPort, config_override: Optional[str] = None): super().__init__(host="localhost", port=port.pg, user="cloud_admin") self.env = env diff --git a/test_runner/regress/test_layer_writers_fail.py b/test_runner/regress/test_layer_writers_fail.py new file mode 100644 index 0000000000..e8ba0e7d91 --- /dev/null +++ b/test_runner/regress/test_layer_writers_fail.py @@ -0,0 +1,92 @@ +import pytest +from fixtures.neon_fixtures import NeonEnv, NeonPageserver + + +@pytest.mark.skip("See https://github.com/neondatabase/neon/issues/2703") +def test_image_layer_writer_fail_before_finish(neon_simple_env: NeonEnv): + env = neon_simple_env + pageserver_http = env.pageserver.http_client() + + tenant_id, timeline_id = env.neon_cli.create_tenant( + conf={ + # small checkpoint distance to create more delta layer files + "checkpoint_distance": f"{1024 ** 2}", + # set the target size to be large to allow the image layer to cover the whole key space + "compaction_target_size": f"{1024 ** 3}", + # tweak the default settings to allow quickly create image layers and L1 layers + "compaction_period": "1 s", + "compaction_threshold": "2", + "image_creation_threshold": "1", + } + ) + + pg = env.postgres.create_start("main", tenant_id=tenant_id) + pg.safe_psql_many( + [ + "CREATE TABLE foo (t text) WITH (autovacuum_enabled = off)", + """INSERT INTO foo + SELECT 'long string to consume some space' || g + FROM generate_series(1, 100000) g""", + ] + ) + + pageserver_http.configure_failpoints(("image-layer-writer-fail-before-finish", "return")) + with pytest.raises(Exception, match="image-layer-writer-fail-before-finish"): + pageserver_http.timeline_checkpoint(tenant_id, timeline_id) + + new_temp_layer_files = list( + filter( + lambda file: str(file).endswith(NeonPageserver.TEMP_FILE_SUFFIX), + [path for path in env.timeline_dir(tenant_id, timeline_id).iterdir()], + ) + ) + + assert ( + len(new_temp_layer_files) == 0 + ), "pageserver should clean its temporary new image layer files on failure" + + +@pytest.mark.skip("See https://github.com/neondatabase/neon/issues/2703") +def test_delta_layer_writer_fail_before_finish(neon_simple_env: NeonEnv): + env = neon_simple_env + pageserver_http = env.pageserver.http_client() + + tenant_id, timeline_id = env.neon_cli.create_tenant( + conf={ + # small checkpoint distance to create more delta layer files + "checkpoint_distance": f"{1024 ** 2}", + # set the target size to be large to allow the image layer to cover the whole key space + "compaction_target_size": f"{1024 ** 3}", + # tweak the default settings to allow quickly create image layers and L1 layers + "compaction_period": "1 s", + "compaction_threshold": "2", + "image_creation_threshold": "1", + } + ) + + pg = env.postgres.create_start("main", tenant_id=tenant_id) + pg.safe_psql_many( + [ + "CREATE TABLE foo (t text) WITH (autovacuum_enabled = off)", + """INSERT INTO foo + SELECT 'long string to consume some space' || g + FROM generate_series(1, 100000) g""", + ] + ) + + pageserver_http.configure_failpoints(("delta-layer-writer-fail-before-finish", "return")) + # Note: we cannot test whether the exception is exactly 'delta-layer-writer-fail-before-finish' + # since our code does it in loop, we cannot get this exact error for our request. + with pytest.raises(Exception): + pageserver_http.timeline_checkpoint(tenant_id, timeline_id) + + new_temp_layer_files = list( + filter( + lambda file: str(file).endswith(NeonPageserver.TEMP_FILE_SUFFIX), + [path for path in env.timeline_dir(tenant_id, timeline_id).iterdir()], + ) + ) + + assert ( + len(new_temp_layer_files) == 0 + ), "pageserver should clean its temporary new delta layer files on failure" From 0cbae6e8f32cfa177a812464e0b6121f84fd9740 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Thu, 27 Oct 2022 17:54:49 +0200 Subject: [PATCH 10/16] test_backward_compatibility: friendlier error message (#2707) --- test_runner/regress/test_compatibility.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index 944ff64390..20a17e449d 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -19,6 +19,8 @@ from fixtures.neon_fixtures import ( from fixtures.types import Lsn from pytest import FixtureRequest +DEFAILT_LOCAL_SNAPSHOT_DIR = "test_output/test_prepare_snapshot/compatibility_snapshot_pg14" + def dump_differs(first: Path, second: Path, output: Path) -> bool: """ @@ -76,14 +78,18 @@ class PortReplacer(object): raise TypeError(f"unsupported type {type(value)} of {value=}") +@pytest.mark.order(after="test_prepare_snapshot") def test_backward_compatibility( pg_bin: PgBin, port_distributor: PortDistributor, test_output_dir: Path, request: FixtureRequest ): - compatibility_snapshot_dir_env = os.environ.get("COMPATIBILITY_SNAPSHOT_DIR") - assert ( - compatibility_snapshot_dir_env is not None - ), "COMPATIBILITY_SNAPSHOT_DIR is not set. It should be set to `compatibility_snapshot_pg14` path generateted by test_prepare_snapshot" - compatibility_snapshot_dir = Path(compatibility_snapshot_dir_env).resolve() + compatibility_snapshot_dir = Path( + os.environ.get("COMPATIBILITY_SNAPSHOT_DIR", DEFAILT_LOCAL_SNAPSHOT_DIR) + ) + assert compatibility_snapshot_dir.exists(), ( + f"{compatibility_snapshot_dir} doesn't exist. Please run `test_prepare_snapshot` test first " + "to create the snapshot or set COMPATIBILITY_SNAPSHOT_DIR env variable to the existing snapshot" + ) + compatibility_snapshot_dir = compatibility_snapshot_dir.resolve() # Make compatibility snapshot artifacts pickupable by Allure # by copying the snapshot directory to the curent test output directory. @@ -229,7 +235,6 @@ def test_backward_compatibility( assert not initial_dump_differs, "initial dump differs" -@pytest.mark.order(after="test_backward_compatibility") # Note: if renaming this test, don't forget to update a reference to it in a workflow file: # "Upload compatibility snapshot" step in .github/actions/run-python-test-set/action.yml def test_prepare_snapshot(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin, test_output_dir: Path): From 128dc8d405783b160029af7b413fd24239c0de9f Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Thu, 27 Oct 2022 18:26:10 +0200 Subject: [PATCH 11/16] Nightly Benchmarks: fix workflow (#2708) --- .github/actions/run-python-test-set/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index 07cb7edbe7..3459449e15 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -74,6 +74,7 @@ runs: run: ./scripts/pysync - name: Download compatibility snapshot for Postgres 14 + if: inputs.build_type != 'remote' uses: ./.github/actions/download with: name: compatibility-snapshot-${{ inputs.build_type }}-pg14 From d3c8749da5a72085612f5f05d5b2159bd16d9b49 Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Thu, 27 Oct 2022 23:19:44 +0300 Subject: [PATCH 12/16] Build compute postgres with openssl support The main reason for that change is that Postgres 15 requires OpenSSL for `pgcrypto` to work. Also not a bad idea to have SSL-enabled Postgres in general. --- Dockerfile.compute-node-v14 | 4 ++-- Dockerfile.compute-node-v15 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile.compute-node-v14 b/Dockerfile.compute-node-v14 index 035dfc0d08..27e15593ad 100644 --- a/Dockerfile.compute-node-v14 +++ b/Dockerfile.compute-node-v14 @@ -13,7 +13,7 @@ ARG TAG=pinned FROM debian:bullseye-slim AS build-deps RUN apt update && \ apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \ - zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config + zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev ######################################################################################### # @@ -24,7 +24,7 @@ RUN apt update && \ FROM build-deps AS pg-build COPY vendor/postgres-v14 postgres RUN cd postgres && \ - ./configure CFLAGS='-O2 -g3' --enable-debug --with-uuid=ossp && \ + ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \ # Install headers diff --git a/Dockerfile.compute-node-v15 b/Dockerfile.compute-node-v15 index 0b6e570b44..567848ffd7 100644 --- a/Dockerfile.compute-node-v15 +++ b/Dockerfile.compute-node-v15 @@ -13,7 +13,7 @@ ARG TAG=pinned FROM debian:bullseye-slim AS build-deps RUN apt update && \ apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \ - zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config + zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev ######################################################################################### # @@ -24,7 +24,7 @@ RUN apt update && \ FROM build-deps AS pg-build COPY vendor/postgres-v15 postgres RUN cd postgres && \ - ./configure CFLAGS='-O2 -g3' --enable-debug --with-uuid=ossp && \ + ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \ # Install headers From e86a9105a4dce87128ff260c993086e1770f1d46 Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Fri, 28 Oct 2022 13:17:27 +0300 Subject: [PATCH 13/16] Deploy storage to new prod regions (#2709) --- .../ansible/prod.ap-southeast-1.hosts.yaml | 35 ++++++++++++++ .github/ansible/prod.eu-central-1.hosts.yaml | 35 ++++++++++++++ .github/ansible/prod.us-east-2.hosts.yaml | 36 ++++++++++++++ .github/ansible/ssm_config | 1 - .github/ansible/staging.us-east-2.hosts.yaml | 1 + .github/workflows/build_and_test.yml | 48 +++++++++++++++++-- 6 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 .github/ansible/prod.ap-southeast-1.hosts.yaml create mode 100644 .github/ansible/prod.eu-central-1.hosts.yaml create mode 100644 .github/ansible/prod.us-east-2.hosts.yaml diff --git a/.github/ansible/prod.ap-southeast-1.hosts.yaml b/.github/ansible/prod.ap-southeast-1.hosts.yaml new file mode 100644 index 0000000000..bb4af91f71 --- /dev/null +++ b/.github/ansible/prod.ap-southeast-1.hosts.yaml @@ -0,0 +1,35 @@ +storage: + vars: + bucket_name: neon-prod-storage-ap-southeast-1 + bucket_region: ap-southeast-1 + console_mgmt_base_url: http://console-release.local + etcd_endpoints: etcd-0.ap-southeast-1.aws.neon.tech:2379 + pageserver_config_stub: + pg_distrib_dir: /usr/local + remote_storage: + bucket_name: "{{ bucket_name }}" + bucket_region: "{{ bucket_region }}" + prefix_in_bucket: "pageserver/v1" + safekeeper_s3_prefix: safekeeper/v1/wal + hostname_suffix: "" + remote_user: ssm-user + ansible_aws_ssm_region: ap-southeast-1 + ansible_aws_ssm_bucket_name: neon-prod-storage-ap-southeast-1 + console_region_id: aws-ap-southeast-1 + + children: + pageservers: + hosts: + pageserver-0.ap-southeast-1.aws.neon.tech: + ansible_host: i-064de8ea28bdb495b + pageserver-1.ap-southeast-1.aws.neon.tech: + ansible_host: i-0b180defcaeeb6b93 + + safekeepers: + hosts: + safekeeper-0.ap-southeast-1.aws.neon.tech: + ansible_host: i-0d6f1dc5161eef894 + safekeeper-1.ap-southeast-1.aws.neon.tech: + ansible_host: i-0e338adda8eb2d19f + safekeeper-2.ap-southeast-1.aws.neon.tech: + ansible_host: i-04fb63634e4679eb9 diff --git a/.github/ansible/prod.eu-central-1.hosts.yaml b/.github/ansible/prod.eu-central-1.hosts.yaml new file mode 100644 index 0000000000..68b1579746 --- /dev/null +++ b/.github/ansible/prod.eu-central-1.hosts.yaml @@ -0,0 +1,35 @@ +storage: + vars: + bucket_name: neon-prod-storage-eu-central-1 + bucket_region: eu-central-1 + console_mgmt_base_url: http://console-release.local + etcd_endpoints: etcd-0.eu-central-1.aws.neon.tech:2379 + pageserver_config_stub: + pg_distrib_dir: /usr/local + remote_storage: + bucket_name: "{{ bucket_name }}" + bucket_region: "{{ bucket_region }}" + prefix_in_bucket: "pageserver/v1" + safekeeper_s3_prefix: safekeeper/v1/wal + hostname_suffix: "" + remote_user: ssm-user + ansible_aws_ssm_region: eu-central-1 + ansible_aws_ssm_bucket_name: neon-prod-storage-eu-central-1 + console_region_id: aws-eu-central-1 + + children: + pageservers: + hosts: + pageserver-0.eu-central-1.aws.neon.tech: + ansible_host: i-0cd8d316ecbb715be + pageserver-1.eu-central-1.aws.neon.tech: + ansible_host: i-090044ed3d383fef0 + + safekeepers: + hosts: + safekeeper-0.eu-central-1.aws.neon.tech: + ansible_host: i-0b238612d2318a050 + safekeeper-1.eu-central-1.aws.neon.tech: + ansible_host: i-07b9c45e5c2637cd4 + safekeeper-2.eu-central-1.aws.neon.tech: + ansible_host: i-020257302c3c93d88 diff --git a/.github/ansible/prod.us-east-2.hosts.yaml b/.github/ansible/prod.us-east-2.hosts.yaml new file mode 100644 index 0000000000..1d54e2ef0a --- /dev/null +++ b/.github/ansible/prod.us-east-2.hosts.yaml @@ -0,0 +1,36 @@ +storage: + vars: + bucket_name: neon-prod-storage-us-east-2 + bucket_region: us-east-2 + console_mgmt_base_url: http://console-release.local + etcd_endpoints: etcd-0.us-east-2.aws.neon.tech:2379 + pageserver_config_stub: + pg_distrib_dir: /usr/local + remote_storage: + bucket_name: "{{ bucket_name }}" + bucket_region: "{{ bucket_region }}" + prefix_in_bucket: "pageserver/v1" + safekeeper_s3_prefix: safekeeper/v1/wal + hostname_suffix: "" + remote_user: ssm-user + ansible_aws_ssm_region: us-east-2 + ansible_aws_ssm_bucket_name: neon-prod-storage-us-east-2 + console_region_id: aws-us-east-2 + + children: + pageservers: + hosts: + pageserver-0.us-east-2.aws.neon.tech: + ansible_host: i-062227ba7f119eb8c + pageserver-1.us-east-2.aws.neon.tech: + ansible_host: i-0b3ec0afab5968938 + + safekeepers: + hosts: + safekeeper-0.us-east-2.aws.neon.tech: + ansible_host: i-0e94224750c57d346 + safekeeper-1.us-east-2.aws.neon.tech: + ansible_host: i-06d113fb73bfddeb0 + safekeeper-2.us-east-2.aws.neon.tech: + ansible_host: i-09f66c8e04afff2e8 + diff --git a/.github/ansible/ssm_config b/.github/ansible/ssm_config index 94958b4490..0dc67507f2 100644 --- a/.github/ansible/ssm_config +++ b/.github/ansible/ssm_config @@ -1,3 +1,2 @@ ansible_connection: aws_ssm -ansible_aws_ssm_bucket_name: neon-dev-bucket ansible_python_interpreter: /usr/bin/python3 diff --git a/.github/ansible/staging.us-east-2.hosts.yaml b/.github/ansible/staging.us-east-2.hosts.yaml index db3ed87c45..3bbf5fe8cb 100644 --- a/.github/ansible/staging.us-east-2.hosts.yaml +++ b/.github/ansible/staging.us-east-2.hosts.yaml @@ -14,6 +14,7 @@ storage: hostname_suffix: "" remote_user: ssm-user ansible_aws_ssm_region: us-east-2 + ansible_aws_ssm_bucket_name: neon-staging-storage-us-east-2 console_region_id: aws-us-east-2 children: diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 8d16e406ce..7133574a0f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -756,9 +756,9 @@ jobs: defaults: run: shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} + strategy: + matrix: + target_region: [ us-east-2 ] steps: - name: Checkout uses: actions/checkout@v3 @@ -781,7 +781,47 @@ jobs: fi ansible-galaxy collection install sivel.toiletwater - ansible-playbook deploy.yaml -i staging.us-east-2.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}} + ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}} + rm -f neon_install.tar.gz .neon_current_version + + deploy-prod-new: + runs-on: prod + container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest + # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version. + # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly + needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] + if: | + (github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + defaults: + run: + shell: bash + strategy: + matrix: + target_region: [ us-east-2, eu-central-1, ap-southeast-1 ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Redeploy + run: | + export DOCKER_TAG=${{needs.tag.outputs.build-tag}} + cd "$(pwd)/.github/ansible" + + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + ./get_binaries.sh + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + RELEASE=true ./get_binaries.sh + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + + ansible-galaxy collection install sivel.toiletwater + ansible-playbook deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_PRODUCTION_API_KEY}} rm -f neon_install.tar.gz .neon_current_version deploy-proxy: From 59a3ca4ec60ee4dbfbe31dd589ca84055b65fc7c Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Fri, 28 Oct 2022 16:25:28 +0300 Subject: [PATCH 14/16] Deploy proxy to new prod regions (#2713) * Refactor proxy deploy * Test new prod deploy * Remove assume role * Add new values * Add all regions --- ...-southeast-1-epsilon.neon-proxy-scram.yaml | 31 +++++++++++ ...d-eu-central-1-gamma.neon-proxy-scram.yaml | 31 +++++++++++ ...prod-us-east-2-delta.neon-proxy-scram.yaml | 31 +++++++++++ .github/workflows/build_and_test.yml | 55 +++++++++++++++++-- 4 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 .github/helm-values/prod-ap-southeast-1-epsilon.neon-proxy-scram.yaml create mode 100644 .github/helm-values/prod-eu-central-1-gamma.neon-proxy-scram.yaml create mode 100644 .github/helm-values/prod-us-east-2-delta.neon-proxy-scram.yaml diff --git a/.github/helm-values/prod-ap-southeast-1-epsilon.neon-proxy-scram.yaml b/.github/helm-values/prod-ap-southeast-1-epsilon.neon-proxy-scram.yaml new file mode 100644 index 0000000000..f90f89a516 --- /dev/null +++ b/.github/helm-values/prod-ap-southeast-1-epsilon.neon-proxy-scram.yaml @@ -0,0 +1,31 @@ +# Helm chart values for neon-proxy-scram. +# This is a YAML-formatted file. + +image: + repository: neondatabase/neon + +settings: + authBackend: "console" + authEndpoint: "http://console-release.local/management/api/v2" + domain: "*.ap-southeast-1.aws.neon.tech" + +# -- Additional labels for neon-proxy pods +podLabels: + zenith_service: proxy-scram + zenith_env: prod + zenith_region: ap-southeast-1 + zenith_region_slug: ap-southeast-1 + +exposedService: + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + external-dns.alpha.kubernetes.io/hostname: ap-southeast-1.aws.neon.tech + +#metrics: +# enabled: true +# serviceMonitor: +# enabled: true +# selector: +# release: kube-prometheus-stack diff --git a/.github/helm-values/prod-eu-central-1-gamma.neon-proxy-scram.yaml b/.github/helm-values/prod-eu-central-1-gamma.neon-proxy-scram.yaml new file mode 100644 index 0000000000..33a1154099 --- /dev/null +++ b/.github/helm-values/prod-eu-central-1-gamma.neon-proxy-scram.yaml @@ -0,0 +1,31 @@ +# Helm chart values for neon-proxy-scram. +# This is a YAML-formatted file. + +image: + repository: neondatabase/neon + +settings: + authBackend: "console" + authEndpoint: "http://console-release.local/management/api/v2" + domain: "*.eu-central-1.aws.neon.tech" + +# -- Additional labels for neon-proxy pods +podLabels: + zenith_service: proxy-scram + zenith_env: prod + zenith_region: eu-central-1 + zenith_region_slug: eu-central-1 + +exposedService: + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + external-dns.alpha.kubernetes.io/hostname: eu-central-1.aws.neon.tech + +#metrics: +# enabled: true +# serviceMonitor: +# enabled: true +# selector: +# release: kube-prometheus-stack diff --git a/.github/helm-values/prod-us-east-2-delta.neon-proxy-scram.yaml b/.github/helm-values/prod-us-east-2-delta.neon-proxy-scram.yaml new file mode 100644 index 0000000000..5f9f2d2e66 --- /dev/null +++ b/.github/helm-values/prod-us-east-2-delta.neon-proxy-scram.yaml @@ -0,0 +1,31 @@ +# Helm chart values for neon-proxy-scram. +# This is a YAML-formatted file. + +image: + repository: neondatabase/neon + +settings: + authBackend: "console" + authEndpoint: "http://console-release.local/management/api/v2" + domain: "*.us-east-2.aws.neon.tech" + +# -- Additional labels for neon-proxy pods +podLabels: + zenith_service: proxy-scram + zenith_env: prod + zenith_region: us-east-2 + zenith_region_slug: us-east-2 + +exposedService: + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + external-dns.alpha.kubernetes.io/hostname: us-east-2.aws.neon.tech + +#metrics: +# enabled: true +# serviceMonitor: +# enabled: true +# selector: +# release: kube-prometheus-stack diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 7133574a0f..abca7f7701 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -870,13 +870,18 @@ jobs: runs-on: dev container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. - needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] - if: | - (github.ref_name == 'main') && - github.event_name != 'workflow_dispatch' +# needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] +# if: | +# (github.ref_name == 'main') && +# github.event_name != 'workflow_dispatch' defaults: run: shell: bash + strategy: + matrix: + include: + - target_region: us-east-2 + target_cluster: dev-us-east-2-beta steps: - name: Checkout uses: actions/checkout@v3 @@ -887,12 +892,50 @@ jobs: - name: Configure environment run: | helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region us-east-2 eks update-kubeconfig --name dev-us-east-2-beta --role-arn arn:aws:iam::369495373322:role/github-runner + aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} + + - name: Re-deploy proxy + run: | + # DOCKER_TAG=${{needs.tag.outputs.build-tag}} + DOCKER_TAG=2257 + helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s + + deploy-proxy-prod-new: + runs-on: prod + container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest + # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. + needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] + if: | + (github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + defaults: + run: + shell: bash + strategy: + matrix: + include: + - target_region: us-east-2 + target_cluster: prod-us-east-2-delta + - target_region: eu-central-1 + target_cluster: prod-eu-central-1-gamma + - target_region: ap-southeast-1 + target_cluster: prod-ap-southeast-1-epsilon + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Configure environment + run: | + helm repo add neondatabase https://neondatabase.github.io/helm-charts + aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - name: Re-deploy proxy run: | DOCKER_TAG=${{needs.tag.outputs.build-tag}} - helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s + helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s promote-compatibility-test-snapshot: runs-on: dev From 1eb9bd052ad4217e59f08d4c1f9c8f08189ac9a4 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Fri, 28 Oct 2022 16:39:11 +0400 Subject: [PATCH 15/16] Bump vendor/postgres-v15 to fix XLP_FIRST_IS_CONTRECORD issue. ref https://github.com/neondatabase/cloud/issues/2688 --- vendor/postgres-v15 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index f7c5269e9c..64558b386b 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit f7c5269e9c7e818653ad6fe95ba072d1901c4497 +Subproject commit 64558b386bcd5a3300163ec7ea5d7f31cef8593c From 7481fb082c9fa7b0007e5db755496abbaab98f6c Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Fri, 28 Oct 2022 17:12:49 +0300 Subject: [PATCH 16/16] Fix bugs in #2713 (#2716) --- .github/workflows/build_and_test.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index abca7f7701..91d9561e7d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -870,10 +870,10 @@ jobs: runs-on: dev container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. -# needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] -# if: | -# (github.ref_name == 'main') && -# github.event_name != 'workflow_dispatch' + needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] + if: | + (github.ref_name == 'main') && + github.event_name != 'workflow_dispatch' defaults: run: shell: bash @@ -896,8 +896,7 @@ jobs: - name: Re-deploy proxy run: | - # DOCKER_TAG=${{needs.tag.outputs.build-tag}} - DOCKER_TAG=2257 + DOCKER_TAG=${{needs.tag.outputs.build-tag}} helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s deploy-proxy-prod-new: