From 411c71b486e71fcce301121b58c48caa7dd65c01 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 5 May 2023 18:32:41 +0200 Subject: [PATCH 1/8] document current tenant attach API semantics (#4151) We currently return 202 as soon as the tenant is allocated in memory before we've written out the marker file. So, the /attach API currently does not have a transactional character. For example, it can happen that we respond with a 202 and then crash before writing out the marker file. In such a case, it is important that the client 1. observes the lost attach (by polling tenant status and observing 404) 2. and consequently retries the attach. It has to do it in this loop until it observes the tenant as "Active" in the tenant status. If the client doesn't follow this protocol and instead goes to another pageserver to attach the tenant, we risk a split-brain situation where both the first and second pageserver write to the tenant's S3 state. The improved description highlights the consequences of this behavior for clients that use the /attach endpoint. The tenant relocation that is currently being implemented in cloud#4740 implements retries of Attach and it does poll afterwards, but, it polls `has_in_progress_downloads`. That is incorrect, as described in the patch body. The motivation for this write-up is that, in a future PR, we'll extend the /attach endpoint with an option to provide the tenant config. If we decide to leave the non-transactional behavior of /attach unmodified, we will be able to avoid persisting the tenant config. Conversely, if we decide that the /attach API should become transactional, we'll need to persist the tenant config in the attach-marker-file before acknowledging receipt of the /attach operation. refs https://github.com/neondatabase/cloud/pull/4740 refs https://github.com/neondatabase/neon/issues/2238 refs https://github.com/neondatabase/neon/issues/1555 --- pageserver/src/http/openapi_spec.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 95f6e96a5b..877b367b40 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -340,7 +340,29 @@ paths: format: hex post: - description: Schedules attach operation to happen in the background for given tenant + description: | + Schedules attach operation to happen in the background for the given tenant. + As soon as the caller sends this request, it must assume the pageserver + starts writing to the tenant's S3 state unless it receives one of the + distinguished errors below that state otherwise. + + The method to identify whether a request has arrived at the pageserver, and + whether it has succeeded, is to poll for the tenant status to reach "Active" + or "Broken" state. These values are currently not explicitly documented in + the API spec. + Polling for `has_in_progress_downloads == false` is INCORRECT because that + value can turn `false` during shutdown while the Attach operation is still + unfinished. + + There is no way to cancel an in-flight request. + + If a client receives a not-distinguished response, e.g., a network timeout, + it MUST retry the /attach request and poll again for tenant status. + + In any case, it must + * NOT ASSUME that the /attach request has been lost in the network, + * NOT ASSUME that the request has been lost based on a subsequent + tenant status request returning 404. (The request may still be in flight!) responses: "202": description: Tenant attaching scheduled From 9860d59aa2b91c5b30aa841a5496af08782027da Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 6 May 2023 20:01:40 +0400 Subject: [PATCH 2/8] Public docker image repository by default --- .github/workflows/build_and_test.yml | 3 +++ Dockerfile | 2 +- Dockerfile.compute-node | 2 +- Dockerfile.compute-tools | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 90f0395c7c..5d0015adf5 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -630,6 +630,7 @@ jobs: --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} + --build-arg REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} --destination neondatabase/neon:${{needs.tag.outputs.build-tag}} @@ -715,6 +716,7 @@ jobs: --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} + --build-arg REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com --dockerfile Dockerfile.compute-tools --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} --destination neondatabase/compute-tools:${{needs.tag.outputs.build-tag}} @@ -767,6 +769,7 @@ jobs: --context . --build-arg GIT_VERSION=${{ github.sha }} --build-arg PG_VERSION=${{ matrix.version }} + --build-arg REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com --dockerfile Dockerfile.compute-node --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} --destination neondatabase/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} diff --git a/Dockerfile b/Dockerfile index f83f3b1c21..7364654641 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ### The image itself is mainly used as a container for the binaries and for starting e2e tests with custom parameters. ### By default, the binaries inside the image have some mock parameters and can start, but are not intended to be used ### inside this image in the real deployments. -ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com +ARG REPOSITORY=neondatabase ARG IMAGE=rust ARG TAG=pinned diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 229e09aa98..3db72885a4 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -1,5 +1,5 @@ ARG PG_VERSION -ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com +ARG REPOSITORY=neondatabase ARG IMAGE=rust ARG TAG=pinned diff --git a/Dockerfile.compute-tools b/Dockerfile.compute-tools index 8231cd0ebb..e86fb40ca4 100644 --- a/Dockerfile.compute-tools +++ b/Dockerfile.compute-tools @@ -1,6 +1,6 @@ # First transient image to build compute_tools binaries # NB: keep in sync with rust image version in .github/workflows/build_and_test.yml -ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com +ARG REPOSITORY=neondatabase ARG IMAGE=rust ARG TAG=pinned From 0f8b2d8f0ad949a2fbace598a48acc8488b25eb4 Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Mon, 8 May 2023 15:56:08 +0200 Subject: [PATCH 3/8] Compile kq_imcx extension (#3568) ## Describe your changes Compiles kq_imcx extension At this moment, there are some issues with the extension: 1. I'm cloning it directly from the master branch. It's better to fetch tag/archive 2. PG14: ``` postgres=# CREATE EXTENSION IF NOT EXISTS kq_imcx; postgres=# select * from kq_calendar_cache_info(); 2023-02-08 13:55:22.853 UTC [412] ERROR: relation "ketteq.slice_type" does not exist at character 34 2023-02-08 13:55:22.853 UTC [412] QUERY: select min(s.id), max(s.id) from ketteq.slice_type s 2023-02-08 13:55:22.853 UTC [412] STATEMENT: select * from kq_calendar_cache_info(); ERROR: relation "ketteq.slice_type" does not exist LINE 1: select min(s.id), max(s.id) from ketteq.slice_type s ``` 3. PG15: `cannot request additional shared memory outside shmem_request_hook` Note: I don't think we need to publish info about this extension in the docs. ## Issue ticket number and link neondatabase/cloud#3387 --- Dockerfile.compute-node | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 3db72885a4..c18470c5e2 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -393,6 +393,28 @@ RUN case "${PG_VERSION}" in \ make install -j $(getconf _NPROCESSORS_ONLN) && \ echo "trusted = true" >> /usr/local/pgsql/share/extension/pg_hint_plan.control +######################################################################################### +# +# Layer "kq-imcx-pg-build" +# compile kq_imcx extension +# +######################################################################################### +FROM build-deps AS kq-imcx-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +ENV PATH "/usr/local/pgsql/bin/:$PATH" +RUN apt-get update && \ + apt-get install -y git libgtk2.0-dev libpq-dev libpam-dev libxslt-dev libkrb5-dev cmake && \ + wget https://github.com/ketteq-neon/postgres-exts/archive/e0bd1a9d9313d7120c1b9c7bb15c48c0dede4c4e.tar.gz -O kq_imcx.tar.gz && \ + echo "dc93a97ff32d152d32737ba7e196d9687041cda15e58ab31344c2f2de8855336 kq_imcx.tar.gz" | sha256sum --check && \ + mkdir kq_imcx-src && cd kq_imcx-src && tar xvzf ../kq_imcx.tar.gz --strip-components=1 -C . && \ + mkdir build && \ + cd build && \ + cmake .. && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/kq_imcx.control + ######################################################################################### # # Layer "rust extensions" @@ -506,6 +528,7 @@ COPY --from=hll-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=plpgsql-check-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=timescaledb-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-hint-plan-pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=kq-imcx-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY pgxn/ pgxn/ RUN make -j $(getconf _NPROCESSORS_ONLN) \ From 51ff9f93591aee83bda717988f6a147e3d1a5d1e Mon Sep 17 00:00:00 2001 From: Anton Chaporgin Date: Mon, 8 May 2023 18:03:50 +0300 Subject: [PATCH 4/8] pg-sni-router nlb is internal (#4164) --- .github/helm-values/dev-eu-central-1-alpha.pg-sni-router.yaml | 3 +++ .github/helm-values/dev-eu-west-1-zeta.pg-sni-router.yaml | 3 +++ .github/helm-values/dev-us-east-2-beta.pg-sni-router.yaml | 3 +++ .../helm-values/prod-ap-southeast-1-epsilon.pg-sni-router.yaml | 3 +++ .github/helm-values/prod-eu-central-1-gamma.pg-sni-router.yaml | 3 +++ .github/helm-values/prod-us-east-1-theta.pg-sni-router.yaml | 3 +++ .github/helm-values/prod-us-east-2-delta.pg-sni-router.yaml | 3 +++ .github/helm-values/prod-us-west-2-eta.pg-sni-router.yaml | 3 +++ 8 files changed, 24 insertions(+) diff --git a/.github/helm-values/dev-eu-central-1-alpha.pg-sni-router.yaml b/.github/helm-values/dev-eu-central-1-alpha.pg-sni-router.yaml index a80423b12d..5306a9bc88 100644 --- a/.github/helm-values/dev-eu-central-1-alpha.pg-sni-router.yaml +++ b/.github/helm-values/dev-eu-central-1-alpha.pg-sni-router.yaml @@ -6,6 +6,9 @@ exposedService: # exposedService.port -- Exposed Service proxy port port: 4432 annotations: + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal + service.beta.kubernetes.io/aws-load-balancer-type: external external-dns.alpha.kubernetes.io/hostname: "*.snirouter.alpha.eu-central-1.internal.aws.neon.build" settings: diff --git a/.github/helm-values/dev-eu-west-1-zeta.pg-sni-router.yaml b/.github/helm-values/dev-eu-west-1-zeta.pg-sni-router.yaml index c9c628af0c..d1413972ec 100644 --- a/.github/helm-values/dev-eu-west-1-zeta.pg-sni-router.yaml +++ b/.github/helm-values/dev-eu-west-1-zeta.pg-sni-router.yaml @@ -6,6 +6,9 @@ exposedService: # exposedService.port -- Exposed Service proxy port port: 4432 annotations: + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal + service.beta.kubernetes.io/aws-load-balancer-type: external external-dns.alpha.kubernetes.io/hostname: "*.snirouter.zeta.eu-west-1.internal.aws.neon.build" settings: diff --git a/.github/helm-values/dev-us-east-2-beta.pg-sni-router.yaml b/.github/helm-values/dev-us-east-2-beta.pg-sni-router.yaml index 68ad096df7..3fd1fc39a4 100644 --- a/.github/helm-values/dev-us-east-2-beta.pg-sni-router.yaml +++ b/.github/helm-values/dev-us-east-2-beta.pg-sni-router.yaml @@ -6,6 +6,9 @@ exposedService: # exposedService.port -- Exposed Service proxy port port: 4432 annotations: + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal + service.beta.kubernetes.io/aws-load-balancer-type: external external-dns.alpha.kubernetes.io/hostname: "*.snirouter.beta.us-east-2.internal.aws.neon.build" settings: diff --git a/.github/helm-values/prod-ap-southeast-1-epsilon.pg-sni-router.yaml b/.github/helm-values/prod-ap-southeast-1-epsilon.pg-sni-router.yaml index 478ad5631c..c60bc202bf 100644 --- a/.github/helm-values/prod-ap-southeast-1-epsilon.pg-sni-router.yaml +++ b/.github/helm-values/prod-ap-southeast-1-epsilon.pg-sni-router.yaml @@ -6,6 +6,9 @@ exposedService: # exposedService.port -- Exposed Service proxy port port: 4432 annotations: + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal + service.beta.kubernetes.io/aws-load-balancer-type: external external-dns.alpha.kubernetes.io/hostname: "*.snirouter.epsilon.ap-southeast-1.internal.aws.neon.tech" settings: diff --git a/.github/helm-values/prod-eu-central-1-gamma.pg-sni-router.yaml b/.github/helm-values/prod-eu-central-1-gamma.pg-sni-router.yaml index 08a0a163bc..cb713899c1 100644 --- a/.github/helm-values/prod-eu-central-1-gamma.pg-sni-router.yaml +++ b/.github/helm-values/prod-eu-central-1-gamma.pg-sni-router.yaml @@ -6,6 +6,9 @@ exposedService: # exposedService.port -- Exposed Service proxy port port: 4432 annotations: + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal + service.beta.kubernetes.io/aws-load-balancer-type: external external-dns.alpha.kubernetes.io/hostname: "*.snirouter.gamma.eu-central-1.internal.aws.neon.tech" settings: diff --git a/.github/helm-values/prod-us-east-1-theta.pg-sni-router.yaml b/.github/helm-values/prod-us-east-1-theta.pg-sni-router.yaml index ab308131bc..be80786859 100644 --- a/.github/helm-values/prod-us-east-1-theta.pg-sni-router.yaml +++ b/.github/helm-values/prod-us-east-1-theta.pg-sni-router.yaml @@ -6,6 +6,9 @@ exposedService: # exposedService.port -- Exposed Service proxy port port: 4432 annotations: + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal + service.beta.kubernetes.io/aws-load-balancer-type: external external-dns.alpha.kubernetes.io/hostname: "*.snirouter.theta.us-east-1.internal.aws.neon.tech" settings: diff --git a/.github/helm-values/prod-us-east-2-delta.pg-sni-router.yaml b/.github/helm-values/prod-us-east-2-delta.pg-sni-router.yaml index ecb3f156ec..7304248872 100644 --- a/.github/helm-values/prod-us-east-2-delta.pg-sni-router.yaml +++ b/.github/helm-values/prod-us-east-2-delta.pg-sni-router.yaml @@ -6,6 +6,9 @@ exposedService: # exposedService.port -- Exposed Service proxy port port: 4432 annotations: + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal + service.beta.kubernetes.io/aws-load-balancer-type: external external-dns.alpha.kubernetes.io/hostname: "*.snirouter.delta.us-east-2.internal.aws.neon.tech" settings: diff --git a/.github/helm-values/prod-us-west-2-eta.pg-sni-router.yaml b/.github/helm-values/prod-us-west-2-eta.pg-sni-router.yaml index 942250c419..b233abb971 100644 --- a/.github/helm-values/prod-us-west-2-eta.pg-sni-router.yaml +++ b/.github/helm-values/prod-us-west-2-eta.pg-sni-router.yaml @@ -6,6 +6,9 @@ exposedService: # exposedService.port -- Exposed Service proxy port port: 4432 annotations: + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal + service.beta.kubernetes.io/aws-load-balancer-type: external external-dns.alpha.kubernetes.io/hostname: "*.snirouter.eta.us-west-2.internal.aws.neon.tech" settings: From e85cbddd2e9cc70eac419d9f59c08f480f559008 Mon Sep 17 00:00:00 2001 From: Raouf Chebri Date: Mon, 8 May 2023 17:12:42 +0200 Subject: [PATCH 5/8] Update neondatabase banner in README.md (#4176) ## Describe your changes ## Issue ticket number and link ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 810937aff7..ce6ec09d24 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Neon](https://user-images.githubusercontent.com/13738772/236813940-dcfdcb5b-69d3-449b-a686-013febe834d4.png)](https://neon.tech) + # Neon Neon is a serverless open-source alternative to AWS Aurora Postgres. It separates storage and compute and substitutes the PostgreSQL storage layer by redistributing data across a cluster of nodes. From 0d3d022eb1fe4a422b116c484965811e222c44f2 Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Mon, 8 May 2023 17:30:16 +0200 Subject: [PATCH 6/8] Remove deploy workflows (#4157) ## Describe your changes Removing deploy workflows (moving to aws repo) --- .github/ansible/.gitignore | 5 - .github/ansible/ansible.cfg | 12 - .github/ansible/ansible.ssh.cfg | 15 - .github/ansible/collections/.keep | 0 .github/ansible/deploy.yaml | 211 ------------- .github/ansible/get_binaries.sh | 42 --- .../ansible/prod.ap-southeast-1.hosts.yaml | 48 --- .github/ansible/prod.eu-central-1.hosts.yaml | 50 ---- .github/ansible/prod.us-east-1.hosts.yaml | 50 ---- .github/ansible/prod.us-east-2.hosts.yaml | 51 ---- .github/ansible/prod.us-west-2.hosts.yaml | 72 ----- .github/ansible/scripts/init_pageserver.sh | 37 --- .github/ansible/scripts/init_safekeeper.sh | 31 -- .github/ansible/ssm_config | 2 - .../ansible/staging.eu-central-1.hosts.yaml | 47 --- .github/ansible/staging.eu-west-1.hosts.yaml | 60 ---- .github/ansible/staging.us-east-2.hosts.yaml | 56 ---- .github/ansible/systemd/pageserver.service | 18 -- .github/ansible/systemd/safekeeper.service | 18 -- .github/ansible/templates/pageserver.toml.j2 | 1 - ...u-central-1-alpha.neon-storage-broker.yaml | 52 ---- .../dev-eu-central-1-alpha.pg-sni-router.yaml | 22 -- .../dev-eu-west-1-zeta.neon-proxy-scram.yaml | 76 ----- ...ev-eu-west-1-zeta.neon-storage-broker.yaml | 52 ---- .../dev-eu-west-1-zeta.pg-sni-router.yaml | 22 -- .../dev-us-east-2-beta.neon-proxy-link.yaml | 68 ----- ...s-east-2-beta.neon-proxy-scram-legacy.yaml | 77 ----- .../dev-us-east-2-beta.neon-proxy-scram.yaml | 78 ----- ...ev-us-east-2-beta.neon-storage-broker.yaml | 52 ---- .../dev-us-east-2-beta.pg-sni-router.yaml | 22 -- .../preview-template.neon-proxy-scram.yaml | 67 ----- ...-southeast-1-epsilon.neon-proxy-scram.yaml | 77 ----- ...utheast-1-epsilon.neon-storage-broker.yaml | 52 ---- ...-ap-southeast-1-epsilon.pg-sni-router.yaml | 22 -- ...d-eu-central-1-gamma.neon-proxy-scram.yaml | 77 ----- ...u-central-1-gamma.neon-storage-broker.yaml | 52 ---- ...prod-eu-central-1-gamma.pg-sni-router.yaml | 22 -- ...prod-us-east-1-theta.neon-proxy-scram.yaml | 69 ----- ...d-us-east-1-theta.neon-storage-broker.yaml | 52 ---- .../prod-us-east-1-theta.pg-sni-router.yaml | 22 -- .../prod-us-east-2-delta.neon-proxy-link.yaml | 58 ---- ...prod-us-east-2-delta.neon-proxy-scram.yaml | 77 ----- ...d-us-east-2-delta.neon-storage-broker.yaml | 52 ---- .../prod-us-east-2-delta.pg-sni-router.yaml | 22 -- ...us-west-2-eta.neon-proxy-scram-legacy.yaml | 76 ----- .../prod-us-west-2-eta.neon-proxy-scram.yaml | 77 ----- ...rod-us-west-2-eta.neon-storage-broker.yaml | 52 ---- .../prod-us-west-2-eta.pg-sni-router.yaml | 22 -- .github/workflows/build_and_test.yml | 42 +-- .github/workflows/deploy-dev.yml | 280 ------------------ .github/workflows/deploy-prod.yml | 217 -------------- 51 files changed, 3 insertions(+), 2831 deletions(-) delete mode 100644 .github/ansible/.gitignore delete mode 100644 .github/ansible/ansible.cfg delete mode 100644 .github/ansible/ansible.ssh.cfg delete mode 100644 .github/ansible/collections/.keep delete mode 100644 .github/ansible/deploy.yaml delete mode 100755 .github/ansible/get_binaries.sh delete mode 100644 .github/ansible/prod.ap-southeast-1.hosts.yaml delete mode 100644 .github/ansible/prod.eu-central-1.hosts.yaml delete mode 100644 .github/ansible/prod.us-east-1.hosts.yaml delete mode 100644 .github/ansible/prod.us-east-2.hosts.yaml delete mode 100644 .github/ansible/prod.us-west-2.hosts.yaml delete mode 100644 .github/ansible/scripts/init_pageserver.sh delete mode 100644 .github/ansible/scripts/init_safekeeper.sh delete mode 100644 .github/ansible/ssm_config delete mode 100644 .github/ansible/staging.eu-central-1.hosts.yaml delete mode 100644 .github/ansible/staging.eu-west-1.hosts.yaml delete mode 100644 .github/ansible/staging.us-east-2.hosts.yaml delete mode 100644 .github/ansible/systemd/pageserver.service delete mode 100644 .github/ansible/systemd/safekeeper.service delete mode 100644 .github/ansible/templates/pageserver.toml.j2 delete mode 100644 .github/helm-values/dev-eu-central-1-alpha.neon-storage-broker.yaml delete mode 100644 .github/helm-values/dev-eu-central-1-alpha.pg-sni-router.yaml delete mode 100644 .github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml delete mode 100644 .github/helm-values/dev-eu-west-1-zeta.neon-storage-broker.yaml delete mode 100644 .github/helm-values/dev-eu-west-1-zeta.pg-sni-router.yaml delete mode 100644 .github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml delete mode 100644 .github/helm-values/dev-us-east-2-beta.neon-proxy-scram-legacy.yaml delete mode 100644 .github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml delete mode 100644 .github/helm-values/dev-us-east-2-beta.neon-storage-broker.yaml delete mode 100644 .github/helm-values/dev-us-east-2-beta.pg-sni-router.yaml delete mode 100644 .github/helm-values/preview-template.neon-proxy-scram.yaml delete mode 100644 .github/helm-values/prod-ap-southeast-1-epsilon.neon-proxy-scram.yaml delete mode 100644 .github/helm-values/prod-ap-southeast-1-epsilon.neon-storage-broker.yaml delete mode 100644 .github/helm-values/prod-ap-southeast-1-epsilon.pg-sni-router.yaml delete mode 100644 .github/helm-values/prod-eu-central-1-gamma.neon-proxy-scram.yaml delete mode 100644 .github/helm-values/prod-eu-central-1-gamma.neon-storage-broker.yaml delete mode 100644 .github/helm-values/prod-eu-central-1-gamma.pg-sni-router.yaml delete mode 100644 .github/helm-values/prod-us-east-1-theta.neon-proxy-scram.yaml delete mode 100644 .github/helm-values/prod-us-east-1-theta.neon-storage-broker.yaml delete mode 100644 .github/helm-values/prod-us-east-1-theta.pg-sni-router.yaml delete mode 100644 .github/helm-values/prod-us-east-2-delta.neon-proxy-link.yaml delete mode 100644 .github/helm-values/prod-us-east-2-delta.neon-proxy-scram.yaml delete mode 100644 .github/helm-values/prod-us-east-2-delta.neon-storage-broker.yaml delete mode 100644 .github/helm-values/prod-us-east-2-delta.pg-sni-router.yaml delete mode 100644 .github/helm-values/prod-us-west-2-eta.neon-proxy-scram-legacy.yaml delete mode 100644 .github/helm-values/prod-us-west-2-eta.neon-proxy-scram.yaml delete mode 100644 .github/helm-values/prod-us-west-2-eta.neon-storage-broker.yaml delete mode 100644 .github/helm-values/prod-us-west-2-eta.pg-sni-router.yaml delete mode 100644 .github/workflows/deploy-dev.yml delete mode 100644 .github/workflows/deploy-prod.yml diff --git a/.github/ansible/.gitignore b/.github/ansible/.gitignore deleted file mode 100644 index 9cd8044417..0000000000 --- a/.github/ansible/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -neon_install.tar.gz -.neon_current_version - -collections/* -!collections/.keep diff --git a/.github/ansible/ansible.cfg b/.github/ansible/ansible.cfg deleted file mode 100644 index 5818a64455..0000000000 --- a/.github/ansible/ansible.cfg +++ /dev/null @@ -1,12 +0,0 @@ -[defaults] - -localhost_warning = False -host_key_checking = False -timeout = 30 - -[ssh_connection] -ssh_args = -F ./ansible.ssh.cfg -# teleport doesn't support sftp yet https://github.com/gravitational/teleport/issues/7127 -# and scp neither worked for me -transfer_method = piped -pipelining = True diff --git a/.github/ansible/ansible.ssh.cfg b/.github/ansible/ansible.ssh.cfg deleted file mode 100644 index cd058b5427..0000000000 --- a/.github/ansible/ansible.ssh.cfg +++ /dev/null @@ -1,15 +0,0 @@ -# Remove this once https://github.com/gravitational/teleport/issues/10918 is fixed -# (use pre 8.5 option name to cope with old ssh in CI) -PubkeyAcceptedKeyTypes +ssh-rsa-cert-v01@openssh.com - -Host tele.zenith.tech - User admin - Port 3023 - StrictHostKeyChecking no - UserKnownHostsFile /dev/null - -Host * !tele.zenith.tech - User admin - StrictHostKeyChecking no - UserKnownHostsFile /dev/null - ProxyJump tele.zenith.tech diff --git a/.github/ansible/collections/.keep b/.github/ansible/collections/.keep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.github/ansible/deploy.yaml b/.github/ansible/deploy.yaml deleted file mode 100644 index d4c1dec8ea..0000000000 --- a/.github/ansible/deploy.yaml +++ /dev/null @@ -1,211 +0,0 @@ -- name: Upload Neon binaries - hosts: storage - gather_facts: False - remote_user: "{{ remote_user }}" - - tasks: - - - name: get latest version of Neon binaries - register: current_version_file - set_fact: - current_version: "{{ lookup('file', '.neon_current_version') | trim }}" - tags: - - pageserver - - safekeeper - - - name: inform about versions - debug: - msg: "Version to deploy - {{ current_version }}" - tags: - - pageserver - - safekeeper - - - name: upload and extract Neon binaries to /usr/local - ansible.builtin.unarchive: - owner: root - group: root - src: neon_install.tar.gz - dest: /usr/local - become: true - tags: - - pageserver - - safekeeper - - binaries - - putbinaries - -- name: Deploy pageserver - hosts: pageservers - gather_facts: False - remote_user: "{{ remote_user }}" - - tasks: - - - name: upload init script - when: console_mgmt_base_url is defined - ansible.builtin.template: - src: scripts/init_pageserver.sh - dest: /tmp/init_pageserver.sh - owner: root - group: root - mode: '0755' - become: true - tags: - - pageserver - - - name: init pageserver - shell: - cmd: /tmp/init_pageserver.sh - args: - creates: "/storage/pageserver/data/tenants" - environment: - NEON_REPO_DIR: "/storage/pageserver/data" - LD_LIBRARY_PATH: "/usr/local/v14/lib" - become: true - tags: - - pageserver - - - name: read the existing remote pageserver config - ansible.builtin.slurp: - src: /storage/pageserver/data/pageserver.toml - register: _remote_ps_config - tags: - - pageserver - - - name: parse the existing pageserver configuration - ansible.builtin.set_fact: - _existing_ps_config: "{{ _remote_ps_config['content'] | b64decode | sivel.toiletwater.from_toml }}" - tags: - - pageserver - - - name: construct the final pageserver configuration dict - ansible.builtin.set_fact: - pageserver_config: "{{ pageserver_config_stub | combine({'id': _existing_ps_config.id }) }}" - tags: - - pageserver - - - name: template the pageserver config - template: - src: templates/pageserver.toml.j2 - dest: /storage/pageserver/data/pageserver.toml - become: true - tags: - - pageserver - - # used in `pageserver.service` template - - name: learn current availability_zone - shell: - cmd: "curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone" - register: ec2_availability_zone - - - set_fact: - ec2_availability_zone={{ ec2_availability_zone.stdout }} - - - name: upload systemd service definition - ansible.builtin.template: - src: systemd/pageserver.service - dest: /etc/systemd/system/pageserver.service - owner: root - group: root - mode: '0644' - become: true - tags: - - pageserver - - - name: start systemd service - ansible.builtin.systemd: - daemon_reload: yes - name: pageserver - enabled: yes - state: restarted - become: true - tags: - - pageserver - - - name: post version to console - when: console_mgmt_base_url is defined - shell: - cmd: | - INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) - curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/pageservers/$INSTANCE_ID | jq '.version = {{ current_version }}' > /tmp/new_version - curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" -H "Content-Type: application/json" -X POST -d@/tmp/new_version {{ console_mgmt_base_url }}/management/api/v2/pageservers - tags: - - pageserver - -- name: Deploy safekeeper - hosts: safekeepers - gather_facts: False - remote_user: "{{ remote_user }}" - - tasks: - - - name: upload init script - when: console_mgmt_base_url is defined - ansible.builtin.template: - src: scripts/init_safekeeper.sh - dest: /tmp/init_safekeeper.sh - owner: root - group: root - mode: '0755' - become: true - tags: - - safekeeper - - - name: init safekeeper - shell: - cmd: /tmp/init_safekeeper.sh - args: - creates: "/storage/safekeeper/data/safekeeper.id" - environment: - NEON_REPO_DIR: "/storage/safekeeper/data" - LD_LIBRARY_PATH: "/usr/local/v14/lib" - become: true - tags: - - safekeeper - - # used in `safekeeper.service` template - - name: learn current availability_zone - shell: - cmd: "curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone" - register: ec2_availability_zone - - - set_fact: - ec2_availability_zone={{ ec2_availability_zone.stdout }} - - # in the future safekeepers should discover pageservers byself - # but currently use first pageserver that was discovered - - name: set first pageserver var for safekeepers - set_fact: - first_pageserver: "{{ hostvars[groups['pageservers'][0]]['inventory_hostname'] }}" - tags: - - safekeeper - - - name: upload systemd service definition - ansible.builtin.template: - src: systemd/safekeeper.service - dest: /etc/systemd/system/safekeeper.service - owner: root - group: root - mode: '0644' - become: true - tags: - - safekeeper - - - name: start systemd service - ansible.builtin.systemd: - daemon_reload: yes - name: safekeeper - enabled: yes - state: restarted - become: true - tags: - - safekeeper - - - name: post version to console - when: console_mgmt_base_url is defined - shell: - cmd: | - INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) - curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/safekeepers/$INSTANCE_ID | jq '.version = {{ current_version }}' > /tmp/new_version - curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" -H "Content-Type: application/json" -X POST -d@/tmp/new_version {{ console_mgmt_base_url }}/management/api/v2/safekeepers - tags: - - safekeeper diff --git a/.github/ansible/get_binaries.sh b/.github/ansible/get_binaries.sh deleted file mode 100755 index 4bb580428c..0000000000 --- a/.github/ansible/get_binaries.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -set -e - -if [ -n "${DOCKER_TAG}" ]; then - # Verson is DOCKER_TAG but without prefix - VERSION=$(echo $DOCKER_TAG | sed 's/^.*-//g') -else - echo "Please set DOCKER_TAG environment variable" - exit 1 -fi - - -# do initial cleanup -rm -rf neon_install postgres_install.tar.gz neon_install.tar.gz .neon_current_version -mkdir neon_install - -# retrieve binaries from docker image -echo "getting binaries from docker image" -docker pull --quiet neondatabase/neon:${DOCKER_TAG} -ID=$(docker create neondatabase/neon:${DOCKER_TAG}) -docker cp ${ID}:/data/postgres_install.tar.gz . -tar -xzf postgres_install.tar.gz -C neon_install -mkdir neon_install/bin/ -docker cp ${ID}:/usr/local/bin/pageserver neon_install/bin/ -docker cp ${ID}:/usr/local/bin/pageserver_binutils neon_install/bin/ -docker cp ${ID}:/usr/local/bin/safekeeper neon_install/bin/ -docker cp ${ID}:/usr/local/bin/storage_broker neon_install/bin/ -docker cp ${ID}:/usr/local/bin/proxy neon_install/bin/ -docker cp ${ID}:/usr/local/v14/bin/ neon_install/v14/bin/ -docker cp ${ID}:/usr/local/v15/bin/ neon_install/v15/bin/ -docker cp ${ID}:/usr/local/v14/lib/ neon_install/v14/lib/ -docker cp ${ID}:/usr/local/v15/lib/ neon_install/v15/lib/ -docker rm -vf ${ID} - -# store version to file (for ansible playbooks) and create binaries tarball -echo ${VERSION} > neon_install/.neon_current_version -echo ${VERSION} > .neon_current_version -tar -czf neon_install.tar.gz -C neon_install . - -# do final cleaup -rm -rf neon_install postgres_install.tar.gz diff --git a/.github/ansible/prod.ap-southeast-1.hosts.yaml b/.github/ansible/prod.ap-southeast-1.hosts.yaml deleted file mode 100644 index 9c53733491..0000000000 --- a/.github/ansible/prod.ap-southeast-1.hosts.yaml +++ /dev/null @@ -1,48 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-ap-southeast-1 - bucket_region: ap-southeast-1 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.epsilon.ap-southeast-1.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: ap-southeast-1 - ansible_aws_ssm_bucket_name: neon-prod-storage-ap-southeast-1 - console_region_id: aws-ap-southeast-1 - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.ap-southeast-1.aws.neon.tech: - ansible_host: i-064de8ea28bdb495b - pageserver-1.ap-southeast-1.aws.neon.tech: - ansible_host: i-0b180defcaeeb6b93 - - safekeepers: - hosts: - safekeeper-0.ap-southeast-1.aws.neon.tech: - ansible_host: i-0d6f1dc5161eef894 - safekeeper-2.ap-southeast-1.aws.neon.tech: - ansible_host: i-04fb63634e4679eb9 - safekeeper-3.ap-southeast-1.aws.neon.tech: - ansible_host: i-05481f3bc88cfc2d4 diff --git a/.github/ansible/prod.eu-central-1.hosts.yaml b/.github/ansible/prod.eu-central-1.hosts.yaml deleted file mode 100644 index 3186519ca8..0000000000 --- a/.github/ansible/prod.eu-central-1.hosts.yaml +++ /dev/null @@ -1,50 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-eu-central-1 - bucket_region: eu-central-1 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.gamma.eu-central-1.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: eu-central-1 - ansible_aws_ssm_bucket_name: neon-prod-storage-eu-central-1 - console_region_id: aws-eu-central-1 - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.eu-central-1.aws.neon.tech: - ansible_host: i-0cd8d316ecbb715be - pageserver-1.eu-central-1.aws.neon.tech: - ansible_host: i-090044ed3d383fef0 - pageserver-2.eu-central-1.aws.neon.tech: - ansible_host: i-033584edf3f4b6742 - - safekeepers: - hosts: - safekeeper-0.eu-central-1.aws.neon.tech: - ansible_host: i-0b238612d2318a050 - safekeeper-1.eu-central-1.aws.neon.tech: - ansible_host: i-07b9c45e5c2637cd4 - safekeeper-2.eu-central-1.aws.neon.tech: - ansible_host: i-020257302c3c93d88 diff --git a/.github/ansible/prod.us-east-1.hosts.yaml b/.github/ansible/prod.us-east-1.hosts.yaml deleted file mode 100644 index b5b2b076bb..0000000000 --- a/.github/ansible/prod.us-east-1.hosts.yaml +++ /dev/null @@ -1,50 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-us-east-1 - bucket_region: us-east-1 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.theta.us-east-1.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: us-east-1 - ansible_aws_ssm_bucket_name: neon-prod-storage-us-east-1 - console_region_id: aws-us-east-1 - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.us-east-1.aws.neon.tech: - ansible_host: i-085222088b0d2e0c7 - pageserver-1.us-east-1.aws.neon.tech: - ansible_host: i-0969d4f684d23a21e - pageserver-2.us-east-1.aws.neon.tech: - ansible_host: i-05dee87895da58dad - - safekeepers: - hosts: - safekeeper-0.us-east-1.aws.neon.tech: - ansible_host: i-04ce739e88793d864 - safekeeper-1.us-east-1.aws.neon.tech: - ansible_host: i-0e9e6c9227fb81410 - safekeeper-2.us-east-1.aws.neon.tech: - ansible_host: i-072f4dd86a327d52f diff --git a/.github/ansible/prod.us-east-2.hosts.yaml b/.github/ansible/prod.us-east-2.hosts.yaml deleted file mode 100644 index 3062475b20..0000000000 --- a/.github/ansible/prod.us-east-2.hosts.yaml +++ /dev/null @@ -1,51 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-us-east-2 - bucket_region: us-east-2 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.delta.us-east-2.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: us-east-2 - ansible_aws_ssm_bucket_name: neon-prod-storage-us-east-2 - console_region_id: aws-us-east-2 - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.us-east-2.aws.neon.tech: - ansible_host: i-062227ba7f119eb8c - pageserver-1.us-east-2.aws.neon.tech: - ansible_host: i-0b3ec0afab5968938 - pageserver-2.us-east-2.aws.neon.tech: - ansible_host: i-0d7a1c4325e71421d - - safekeepers: - hosts: - safekeeper-0.us-east-2.aws.neon.tech: - ansible_host: i-0e94224750c57d346 - safekeeper-1.us-east-2.aws.neon.tech: - ansible_host: i-06d113fb73bfddeb0 - safekeeper-2.us-east-2.aws.neon.tech: - ansible_host: i-09f66c8e04afff2e8 - diff --git a/.github/ansible/prod.us-west-2.hosts.yaml b/.github/ansible/prod.us-west-2.hosts.yaml deleted file mode 100644 index be65d8e63c..0000000000 --- a/.github/ansible/prod.us-west-2.hosts.yaml +++ /dev/null @@ -1,72 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-us-west-2 - bucket_region: us-west-2 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.eta.us-west-2.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: us-west-2 - ansible_aws_ssm_bucket_name: neon-prod-storage-us-west-2 - console_region_id: aws-us-west-2-new - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.us-west-2.aws.neon.tech: - ansible_host: i-0d9f6dfae0e1c780d - pageserver-1.us-west-2.aws.neon.tech: - ansible_host: i-0c834be1dddba8b3f - pageserver-2.us-west-2.aws.neon.tech: - ansible_host: i-051642d372c0a4f32 - pageserver-3.us-west-2.aws.neon.tech: - ansible_host: i-00c3844beb9ad1c6b - pageserver-4.us-west-2.aws.neon.tech: - ansible_host: i-013263dd1c239adcc - pageserver-5.us-west-2.aws.neon.tech: - ansible_host: i-00ca6417c7bf96820 - pageserver-6.us-west-2.aws.neon.tech: - ansible_host: i-01cdf7d2bc1433b6a - pageserver-7.us-west-2.aws.neon.tech: - ansible_host: i-02eec9b40617db5bc - - safekeepers: - hosts: - safekeeper-0.us-west-2.aws.neon.tech: - ansible_host: i-00719d8a74986fda6 - safekeeper-1.us-west-2.aws.neon.tech: - ansible_host: i-074682f9d3c712e7c - safekeeper-2.us-west-2.aws.neon.tech: - ansible_host: i-042b7efb1729d7966 - safekeeper-3.us-west-2.aws.neon.tech: - ansible_host: i-089f6b9ef426dff76 - safekeeper-4.us-west-2.aws.neon.tech: - ansible_host: i-0fe6bf912c4710c82 - safekeeper-5.us-west-2.aws.neon.tech: - ansible_host: i-0a83c1c46d2b4e409 - safekeeper-6.us-west-2.aws.neon.tech: - ansible_host: i-0fef5317b8fdc9f8d - safekeeper-7.us-west-2.aws.neon.tech: - ansible_host: i-0be739190d4289bf9 - safekeeper-8.us-west-2.aws.neon.tech: - ansible_host: i-00e851803669e5cfe diff --git a/.github/ansible/scripts/init_pageserver.sh b/.github/ansible/scripts/init_pageserver.sh deleted file mode 100644 index d88f754a86..0000000000 --- a/.github/ansible/scripts/init_pageserver.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -# fetch params from meta-data service -INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) -AZ_ID=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) -INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) -DISK_SIZE=$(df -B1 /storage | tail -1 | awk '{print $2}') - -# store fqdn hostname in var -HOST=$(hostname -f) - - -cat <preview-${PREVIEW_NAME}.neon-proxy-scram.yaml - helm upgrade neon-proxy-scram-${PREVIEW_NAME} neondatabase/neon-proxy --namespace neon-proxy-${PREVIEW_NAME} --create-namespace --install --atomic -f preview-${PREVIEW_NAME}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - done - - - name: Cleanup helm folder - run: rm -rf ~/.cache - - deploy-storage-broker-new: - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - if: inputs.deployStorageBroker - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: dev-us-east-2-beta - - target_region: eu-west-1 - target_cluster: dev-eu-west-1-zeta - - target_region: eu-central-1 - target_cluster: dev-eu-central-1-alpha - environment: - name: dev-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - role-to-assume: arn:aws:iam::369495373322:role/github-runner - aws-region: eu-central-1 - role-skip-session-tagging: true - role-duration-seconds: 1800 - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy storage-broker - run: - helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s - - - name: Cleanup helm folder - run: rm -rf ~/.cache - - deploy-pg-sni-router: - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - if: inputs.deployPgSniRouter - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: dev-us-east-2-beta - - target_region: eu-west-1 - target_cluster: dev-eu-west-1-zeta - - target_region: eu-central-1 - target_cluster: dev-eu-central-1-alpha - environment: - name: dev-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - role-to-assume: arn:aws:iam::369495373322:role/github-runner - aws-region: eu-central-1 - role-skip-session-tagging: true - role-duration-seconds: 1800 - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy pg-sni-router - run: - helm upgrade neon-pg-sni-router neondatabase/neon-pg-sni-router --namespace neon-pg-sni-router --create-namespace --install --debug --atomic -f .github/helm-values/${{ matrix.target_cluster }}.pg-sni-router.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 15m0s - - - name: Cleanup helm folder - run: rm -rf ~/.cache diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml deleted file mode 100644 index baa44d8094..0000000000 --- a/.github/workflows/deploy-prod.yml +++ /dev/null @@ -1,217 +0,0 @@ -name: Neon Deploy prod - -on: - workflow_dispatch: - inputs: - dockerTag: - description: 'Docker tag to deploy' - required: true - type: string - branch: - description: 'Branch or commit used for deploy scripts and configs' - required: true - type: string - default: 'release' - deployStorage: - description: 'Deploy storage' - required: true - type: boolean - default: true - deployProxy: - description: 'Deploy proxy' - required: true - type: boolean - default: true - deployStorageBroker: - description: 'Deploy storage-broker' - required: true - type: boolean - default: true - deployPgSniRouter: - description: 'Deploy pg-sni-router' - required: true - type: boolean - default: true - disclamerAcknowledged: - description: 'I confirm that there is an emergency and I can not use regular release workflow' - required: true - type: boolean - default: false - -concurrency: - group: deploy-prod - cancel-in-progress: false - -jobs: - deploy-prod-new: - runs-on: prod - container: - image: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - options: --user root --privileged - if: inputs.deployStorage && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - strategy: - matrix: - target_region: [ us-east-2, us-west-2, eu-central-1, ap-southeast-1, us-east-1 ] - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Redeploy - run: | - export DOCKER_TAG=${{ inputs.dockerTag }} - cd "$(pwd)/.github/ansible" - - ./get_binaries.sh - - ansible-galaxy collection install sivel.toiletwater - ansible-playbook -v deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_PRODUCTION_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }} - rm -f neon_install.tar.gz .neon_current_version - - deploy-proxy-prod-new: - runs-on: prod - container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployProxy && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: prod-us-east-2-delta - deploy_link_proxy: true - deploy_legacy_scram_proxy: false - - target_region: us-west-2 - target_cluster: prod-us-west-2-eta - deploy_link_proxy: false - deploy_legacy_scram_proxy: true - - target_region: eu-central-1 - target_cluster: prod-eu-central-1-gamma - deploy_link_proxy: false - deploy_legacy_scram_proxy: false - - target_region: ap-southeast-1 - target_cluster: prod-ap-southeast-1-epsilon - deploy_link_proxy: false - deploy_legacy_scram_proxy: false - - target_region: us-east-1 - target_cluster: prod-us-east-1-theta - deploy_link_proxy: false - deploy_legacy_scram_proxy: false - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Re-deploy scram proxy - run: | - DOCKER_TAG=${{ inputs.dockerTag }} - helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Re-deploy link proxy - if: matrix.deploy_link_proxy - run: | - DOCKER_TAG=${{ inputs.dockerTag }} - helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Re-deploy legacy scram proxy - if: matrix.deploy_legacy_scram_proxy - run: | - DOCKER_TAG=${{ inputs.dockerTag }} - helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - deploy-storage-broker-prod-new: - runs-on: prod - container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployStorageBroker && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: prod-us-east-2-delta - - target_region: us-west-2 - target_cluster: prod-us-west-2-eta - - target_region: eu-central-1 - target_cluster: prod-eu-central-1-gamma - - target_region: ap-southeast-1 - target_cluster: prod-ap-southeast-1-epsilon - - target_region: us-east-1 - target_cluster: prod-us-east-1-theta - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy storage-broker - run: - helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s - - deploy-pg-sni-router: - runs-on: prod - container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployPgSniRouter && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: prod-us-east-2-delta - - target_region: us-west-2 - target_cluster: prod-us-west-2-eta - - target_region: eu-central-1 - target_cluster: prod-eu-central-1-gamma - - target_region: ap-southeast-1 - target_cluster: prod-ap-southeast-1-epsilon - - target_region: us-east-1 - target_cluster: prod-us-east-1-theta - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy pg-sni-router - run: - helm upgrade neon-pg-sni-router neondatabase/neon-pg-sni-router --namespace neon-pg-sni-router --create-namespace --install --debug --atomic -f .github/helm-values/${{ matrix.target_cluster }}.pg-sni-router.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 15m0s From 4bd7b1daf24d647434c6b9b9774608422c1d7b93 Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Fri, 5 May 2023 13:52:25 +0100 Subject: [PATCH 7/8] Bump vendor/postgres: Fix entering hot standby mode for Neon postgres v15 --- vendor/postgres-v15 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index aee72b7be9..2df2ce3744 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit aee72b7be903e52d9bdc6449aa4c17fb852d8708 +Subproject commit 2df2ce374464a7449e15dfa46c956b73b4f4098b From d62315327af94841dcce4319f36a217f8ebe0163 Mon Sep 17 00:00:00 2001 From: Arthur Petukhovsky Date: Tue, 9 May 2023 12:20:35 +0300 Subject: [PATCH 8/8] Allow parallel backup in safekeepers (#4177) Add `wal_backup_parallel_jobs` cmdline argument to specify the max count of parallel segments upload. New default value is 5, meaning that safekeepers will try to upload 5 segments concurrently if they are available. Setting this value to 1 will be equivalent to the sequential upload that we had before. Part of the https://github.com/neondatabase/neon/issues/3957 --- safekeeper/src/bin/safekeeper.rs | 4 ++ safekeeper/src/lib.rs | 2 + safekeeper/src/wal_backup.rs | 67 +++++++++++++++++++++++++------- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 3699a2a74c..fecbb8bd41 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -108,6 +108,9 @@ struct Args { /// available to the system. #[arg(long)] wal_backup_threads: Option, + /// Number of max parallel WAL segments to be offloaded to remote storage. + #[arg(long, default_value = "5")] + wal_backup_parallel_jobs: usize, /// Disable WAL backup to s3. When disabled, safekeeper removes WAL ignoring /// WAL backup horizon. #[arg(long)] @@ -182,6 +185,7 @@ fn main() -> anyhow::Result<()> { max_offloader_lag_bytes: args.max_offloader_lag, backup_runtime_threads: args.wal_backup_threads, wal_backup_enabled: !args.disable_wal_backup, + backup_parallel_jobs: args.wal_backup_parallel_jobs, auth, }; diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index ff621fdbc0..22d6d57e19 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -61,6 +61,7 @@ pub struct SafeKeeperConf { pub remote_storage: Option, pub max_offloader_lag_bytes: u64, pub backup_runtime_threads: Option, + pub backup_parallel_jobs: usize, pub wal_backup_enabled: bool, pub auth: Option>, } @@ -93,6 +94,7 @@ impl SafeKeeperConf { broker_keepalive_interval: Duration::from_secs(5), backup_runtime_threads: None, wal_backup_enabled: true, + backup_parallel_jobs: 1, auth: None, heartbeat_timeout: Duration::new(5, 0), max_offloader_lag_bytes: defaults::DEFAULT_MAX_OFFLOADER_LAG_BYTES, diff --git a/safekeeper/src/wal_backup.rs b/safekeeper/src/wal_backup.rs index 953c7d0022..4d341a7ef8 100644 --- a/safekeeper/src/wal_backup.rs +++ b/safekeeper/src/wal_backup.rs @@ -1,5 +1,7 @@ use anyhow::{Context, Result}; +use futures::stream::FuturesOrdered; +use futures::StreamExt; use tokio::task::JoinHandle; use utils::id::NodeId; @@ -155,8 +157,14 @@ async fn update_task( let timeline_dir = conf.timeline_dir(&ttid); let handle = tokio::spawn( - backup_task_main(ttid, timeline_dir, conf.workdir.clone(), shutdown_rx) - .instrument(info_span!("WAL backup task", ttid = %ttid)), + backup_task_main( + ttid, + timeline_dir, + conf.workdir.clone(), + conf.backup_parallel_jobs, + shutdown_rx, + ) + .instrument(info_span!("WAL backup task", ttid = %ttid)), ); entry.handle = Some(WalBackupTaskHandle { @@ -240,6 +248,7 @@ struct WalBackupTask { timeline_dir: PathBuf, workspace_dir: PathBuf, wal_seg_size: usize, + parallel_jobs: usize, commit_lsn_watch_rx: watch::Receiver, } @@ -248,6 +257,7 @@ async fn backup_task_main( ttid: TenantTimelineId, timeline_dir: PathBuf, workspace_dir: PathBuf, + parallel_jobs: usize, mut shutdown_rx: Receiver<()>, ) { info!("started"); @@ -264,6 +274,7 @@ async fn backup_task_main( timeline: tli, timeline_dir, workspace_dir, + parallel_jobs, }; // task is spinned up only when wal_seg_size already initialized @@ -330,6 +341,7 @@ impl WalBackupTask { self.wal_seg_size, &self.timeline_dir, &self.workspace_dir, + self.parallel_jobs, ) .await { @@ -356,20 +368,49 @@ pub async fn backup_lsn_range( wal_seg_size: usize, timeline_dir: &Path, workspace_dir: &Path, + parallel_jobs: usize, ) -> Result<()> { + if parallel_jobs < 1 { + anyhow::bail!("parallel_jobs must be >= 1"); + } + let start_lsn = *backup_lsn; let segments = get_segments(start_lsn, end_lsn, wal_seg_size); - for s in &segments { - backup_single_segment(s, timeline_dir, workspace_dir) - .await - .with_context(|| format!("offloading segno {}", s.seg_no))?; - let new_backup_lsn = s.end_lsn; - timeline - .set_wal_backup_lsn(new_backup_lsn) - .context("setting wal_backup_lsn")?; - *backup_lsn = new_backup_lsn; + // Pool of concurrent upload tasks. We use `FuturesOrdered` to + // preserve order of uploads, and update `backup_lsn` only after + // all previous uploads are finished. + let mut uploads = FuturesOrdered::new(); + let mut iter = segments.iter(); + + loop { + let added_task = match iter.next() { + Some(s) => { + uploads.push_back(backup_single_segment(s, timeline_dir, workspace_dir)); + true + } + None => false, + }; + + // Wait for the next segment to upload if we don't have any more segments, + // or if we have too many concurrent uploads. + if !added_task || uploads.len() >= parallel_jobs { + let next = uploads.next().await; + if let Some(res) = next { + // next segment uploaded + let segment = res?; + let new_backup_lsn = segment.end_lsn; + timeline + .set_wal_backup_lsn(new_backup_lsn) + .context("setting wal_backup_lsn")?; + *backup_lsn = new_backup_lsn; + } else { + // no more segments to upload + break; + } + } } + info!( "offloaded segnos {:?} up to {}, previous backup_lsn {}", segments.iter().map(|&s| s.seg_no).collect::>(), @@ -383,7 +424,7 @@ async fn backup_single_segment( seg: &Segment, timeline_dir: &Path, workspace_dir: &Path, -) -> Result<()> { +) -> Result { let segment_file_path = seg.file_path(timeline_dir)?; let remote_segment_path = segment_file_path .strip_prefix(workspace_dir) @@ -404,7 +445,7 @@ async fn backup_single_segment( res?; debug!("Backup of {} done", segment_file_path.display()); - Ok(()) + Ok(*seg) } #[derive(Debug, Copy, Clone)]