From 70ce01d84d155b2622d4b0857d20abdcbe7a5b87 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Fri, 16 Dec 2022 00:42:30 +0300 Subject: [PATCH] Deploy broker with L4 LB in new env. (#3125) Seems to be fixing issue with missing keepalives. --- .../ansible/prod.ap-southeast-1.hosts.yaml | 2 +- .github/ansible/prod.eu-central-1.hosts.yaml | 2 +- .github/ansible/prod.us-east-2.hosts.yaml | 2 +- .github/ansible/prod.us-west-2.hosts.yaml | 2 +- .github/ansible/staging.eu-west-1.hosts.yaml | 2 +- .github/ansible/staging.us-east-2.hosts.yaml | 2 +- ...ev-eu-west-1-zeta.neon-storage-broker.yaml | 33 ++++++++----------- ...ev-us-east-2-beta.neon-storage-broker.yaml | 33 ++++++++----------- ...utheast-1-epsilon.neon-storage-broker.yaml | 33 ++++++++----------- ...u-central-1-gamma.neon-storage-broker.yaml | 33 ++++++++----------- ...d-us-east-2-delta.neon-storage-broker.yaml | 33 ++++++++----------- ...rod-us-west-2-eta.neon-storage-broker.yaml | 33 ++++++++----------- .github/workflows/build_and_test.yml | 4 +-- 13 files changed, 92 insertions(+), 122 deletions(-) diff --git a/.github/ansible/prod.ap-southeast-1.hosts.yaml b/.github/ansible/prod.ap-southeast-1.hosts.yaml index bcc7bb3b16..648029c120 100644 --- a/.github/ansible/prod.ap-southeast-1.hosts.yaml +++ b/.github/ansible/prod.ap-southeast-1.hosts.yaml @@ -3,7 +3,7 @@ storage: bucket_name: neon-prod-storage-ap-southeast-1 bucket_region: ap-southeast-1 console_mgmt_base_url: http://console-release.local - broker_endpoint: https://storage-broker.epsilon.ap-southeast-1.internal.aws.neon.tech:443 + broker_endpoint: http://storage-broker-lb.epsilon.ap-southeast-1.internal.aws.neon.tech:50051 pageserver_config_stub: pg_distrib_dir: /usr/local remote_storage: diff --git a/.github/ansible/prod.eu-central-1.hosts.yaml b/.github/ansible/prod.eu-central-1.hosts.yaml index 2b372d0fcb..c285a9f3b6 100644 --- a/.github/ansible/prod.eu-central-1.hosts.yaml +++ b/.github/ansible/prod.eu-central-1.hosts.yaml @@ -3,7 +3,7 @@ storage: bucket_name: neon-prod-storage-eu-central-1 bucket_region: eu-central-1 console_mgmt_base_url: http://console-release.local - broker_endpoint: https://storage-broker.gamma.eu-central-1.internal.aws.neon.tech:443 + broker_endpoint: http://storage-broker-lb.gamma.eu-central-1.internal.aws.neon.tech:50051 pageserver_config_stub: pg_distrib_dir: /usr/local remote_storage: diff --git a/.github/ansible/prod.us-east-2.hosts.yaml b/.github/ansible/prod.us-east-2.hosts.yaml index 7a4002ec88..1753068b8c 100644 --- a/.github/ansible/prod.us-east-2.hosts.yaml +++ b/.github/ansible/prod.us-east-2.hosts.yaml @@ -3,7 +3,7 @@ storage: bucket_name: neon-prod-storage-us-east-2 bucket_region: us-east-2 console_mgmt_base_url: http://console-release.local - broker_endpoint: https://storage-broker.delta.us-east-2.internal.aws.neon.tech:443 + broker_endpoint: http://storage-broker-lb.delta.us-east-2.internal.aws.neon.tech:50051 pageserver_config_stub: pg_distrib_dir: /usr/local remote_storage: diff --git a/.github/ansible/prod.us-west-2.hosts.yaml b/.github/ansible/prod.us-west-2.hosts.yaml index 682ee5994d..7d6e49bf9c 100644 --- a/.github/ansible/prod.us-west-2.hosts.yaml +++ b/.github/ansible/prod.us-west-2.hosts.yaml @@ -3,7 +3,7 @@ storage: bucket_name: neon-prod-storage-us-west-2 bucket_region: us-west-2 console_mgmt_base_url: http://console-release.local - broker_endpoint: https://storage-broker.eta.us-west-2.internal.aws.neon.tech:443 + broker_endpoint: http://storage-broker-lb.eta.us-west-2.internal.aws.neon.tech:50051 pageserver_config_stub: pg_distrib_dir: /usr/local remote_storage: diff --git a/.github/ansible/staging.eu-west-1.hosts.yaml b/.github/ansible/staging.eu-west-1.hosts.yaml index 90f00175b0..cfcc3a9ae8 100644 --- a/.github/ansible/staging.eu-west-1.hosts.yaml +++ b/.github/ansible/staging.eu-west-1.hosts.yaml @@ -3,7 +3,7 @@ storage: bucket_name: neon-dev-storage-eu-west-1 bucket_region: eu-west-1 console_mgmt_base_url: http://console-staging.local - broker_endpoint: https://storage-broker.zeta.eu-west-1.internal.aws.neon.build:443 + broker_endpoint: http://storage-broker-lb.zeta.eu-west-1.internal.aws.neon.build:50051 pageserver_config_stub: pg_distrib_dir: /usr/local remote_storage: diff --git a/.github/ansible/staging.us-east-2.hosts.yaml b/.github/ansible/staging.us-east-2.hosts.yaml index d2b7fae12a..78a4582e57 100644 --- a/.github/ansible/staging.us-east-2.hosts.yaml +++ b/.github/ansible/staging.us-east-2.hosts.yaml @@ -3,7 +3,7 @@ storage: bucket_name: neon-staging-storage-us-east-2 bucket_region: us-east-2 console_mgmt_base_url: http://console-staging.local - broker_endpoint: https://storage-broker.beta.us-east-2.internal.aws.neon.build:443 + broker_endpoint: http://storage-broker-lb.beta.us-east-2.internal.aws.neon.build:50051 pageserver_config_stub: pg_distrib_dir: /usr/local remote_storage: diff --git a/.github/helm-values/dev-eu-west-1-zeta.neon-storage-broker.yaml b/.github/helm-values/dev-eu-west-1-zeta.neon-storage-broker.yaml index e876367a18..c6e682f571 100644 --- a/.github/helm-values/dev-eu-west-1-zeta.neon-storage-broker.yaml +++ b/.github/helm-values/dev-eu-west-1-zeta.neon-storage-broker.yaml @@ -3,27 +3,22 @@ podLabels: neon_env: staging neon_service: storage-broker -ingress: - enabled: true +# Use L4 LB +service: + # service.annotations -- Annotations to add to the service annotations: - kubernetes.io/ingress.class: nginx-internal - nginx.ingress.kubernetes.io/backend-protocol: "GRPC" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - # we have basically infinite streams, disable body size limit - nginx.ingress.kubernetes.io/proxy-body-size: "0" - cert-manager.io/cluster-issuer: "cert-manager-clusterissuer" - - hosts: - - host: storage-broker.zeta.eu-west-1.internal.aws.neon.build - paths: - - path: / - pathType: Prefix - tls: - - hosts: - - storage-broker.zeta.eu-west-1.internal.aws.neon.build - secretName: storage-broker-tls + service.beta.kubernetes.io/aws-load-balancer-type: external # use newer AWS Load Balancer Controller + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal # deploy LB to private subnet + # assign service to this name at external-dns + external-dns.alpha.kubernetes.io/hostname: storage-broker-lb.zeta.eu-west-1.internal.aws.neon.build + # service.type -- Service type + type: LoadBalancer + # service.port -- broker listen port + port: 50051 +ingress: + enabled: false metrics: enabled: false diff --git a/.github/helm-values/dev-us-east-2-beta.neon-storage-broker.yaml b/.github/helm-values/dev-us-east-2-beta.neon-storage-broker.yaml index dcf4b99de2..c7682d24c0 100644 --- a/.github/helm-values/dev-us-east-2-beta.neon-storage-broker.yaml +++ b/.github/helm-values/dev-us-east-2-beta.neon-storage-broker.yaml @@ -3,27 +3,22 @@ podLabels: neon_env: staging neon_service: storage-broker -ingress: - enabled: true +# Use L4 LB +service: + # service.annotations -- Annotations to add to the service annotations: - kubernetes.io/ingress.class: nginx-internal - nginx.ingress.kubernetes.io/backend-protocol: "GRPC" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - # we have basically infinite streams, disable body size limit - nginx.ingress.kubernetes.io/proxy-body-size: "0" - cert-manager.io/cluster-issuer: "cert-manager-clusterissuer" - - hosts: - - host: storage-broker.beta.us-east-2.internal.aws.neon.build - paths: - - path: / - pathType: Prefix - tls: - - hosts: - - storage-broker.beta.us-east-2.internal.aws.neon.build - secretName: storage-broker-tls + service.beta.kubernetes.io/aws-load-balancer-type: external # use newer AWS Load Balancer Controller + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal # deploy LB to private subnet + # assign service to this name at external-dns + external-dns.alpha.kubernetes.io/hostname: storage-broker-lb.beta.us-east-2.internal.aws.neon.build + # service.type -- Service type + type: LoadBalancer + # service.port -- broker listen port + port: 50051 +ingress: + enabled: false metrics: enabled: false diff --git a/.github/helm-values/prod-ap-southeast-1-epsilon.neon-storage-broker.yaml b/.github/helm-values/prod-ap-southeast-1-epsilon.neon-storage-broker.yaml index 0abc6ebaa1..92b1777d0b 100644 --- a/.github/helm-values/prod-ap-southeast-1-epsilon.neon-storage-broker.yaml +++ b/.github/helm-values/prod-ap-southeast-1-epsilon.neon-storage-broker.yaml @@ -3,27 +3,22 @@ podLabels: neon_env: production neon_service: storage-broker -ingress: - enabled: true +# Use L4 LB +service: + # service.annotations -- Annotations to add to the service annotations: - kubernetes.io/ingress.class: nginx-internal - nginx.ingress.kubernetes.io/backend-protocol: "GRPC" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - # we have basically infinite streams, disable body size limit - nginx.ingress.kubernetes.io/proxy-body-size: "0" - cert-manager.io/cluster-issuer: "cert-manager-clusterissuer" - - hosts: - - host: storage-broker.epsilon.ap-southeast-1.internal.aws.neon.tech - paths: - - path: / - pathType: Prefix - tls: - - hosts: - - storage-broker.epsilon.ap-southeast-1.internal.aws.neon.tech - secretName: storage-broker-tls + service.beta.kubernetes.io/aws-load-balancer-type: external # use newer AWS Load Balancer Controller + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal # deploy LB to private subnet + # assign service to this name at external-dns + external-dns.alpha.kubernetes.io/hostname: storage-broker-lb.epsilon.ap-southeast-1.internal.aws.neon.tech + # service.type -- Service type + type: LoadBalancer + # service.port -- broker listen port + port: 50051 +ingress: + enabled: false metrics: enabled: false diff --git a/.github/helm-values/prod-eu-central-1-gamma.neon-storage-broker.yaml b/.github/helm-values/prod-eu-central-1-gamma.neon-storage-broker.yaml index d44a3eab5c..f89df4533a 100644 --- a/.github/helm-values/prod-eu-central-1-gamma.neon-storage-broker.yaml +++ b/.github/helm-values/prod-eu-central-1-gamma.neon-storage-broker.yaml @@ -3,27 +3,22 @@ podLabels: neon_env: production neon_service: storage-broker -ingress: - enabled: true +# Use L4 LB +service: + # service.annotations -- Annotations to add to the service annotations: - kubernetes.io/ingress.class: nginx-internal - nginx.ingress.kubernetes.io/backend-protocol: "GRPC" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - # we have basically infinite streams, disable body size limit - nginx.ingress.kubernetes.io/proxy-body-size: "0" - cert-manager.io/cluster-issuer: "cert-manager-clusterissuer" - - hosts: - - host: storage-broker.gamma.eu-central-1.internal.aws.neon.tech - paths: - - path: / - pathType: Prefix - tls: - - hosts: - - storage-broker.gamma.eu-central-1.internal.aws.neon.tech - secretName: storage-broker-tls + service.beta.kubernetes.io/aws-load-balancer-type: external # use newer AWS Load Balancer Controller + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal # deploy LB to private subnet + # assign service to this name at external-dns + external-dns.alpha.kubernetes.io/hostname: storage-broker-lb.gamma.eu-central-1.internal.aws.neon.tech + # service.type -- Service type + type: LoadBalancer + # service.port -- broker listen port + port: 50051 +ingress: + enabled: false metrics: enabled: false diff --git a/.github/helm-values/prod-us-east-2-delta.neon-storage-broker.yaml b/.github/helm-values/prod-us-east-2-delta.neon-storage-broker.yaml index b9eeff5681..8cbc1af7cf 100644 --- a/.github/helm-values/prod-us-east-2-delta.neon-storage-broker.yaml +++ b/.github/helm-values/prod-us-east-2-delta.neon-storage-broker.yaml @@ -3,27 +3,22 @@ podLabels: neon_env: production neon_service: storage-broker -ingress: - enabled: true +# Use L4 LB +service: + # service.annotations -- Annotations to add to the service annotations: - kubernetes.io/ingress.class: nginx-internal - nginx.ingress.kubernetes.io/backend-protocol: "GRPC" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - # we have basically infinite streams, disable body size limit - nginx.ingress.kubernetes.io/proxy-body-size: "0" - cert-manager.io/cluster-issuer: "cert-manager-clusterissuer" - - hosts: - - host: storage-broker.delta.us-east-2.internal.aws.neon.tech - paths: - - path: / - pathType: Prefix - tls: - - hosts: - - storage-broker.delta.us-east-2.internal.aws.neon.tech - secretName: storage-broker-tls + service.beta.kubernetes.io/aws-load-balancer-type: external # use newer AWS Load Balancer Controller + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal # deploy LB to private subnet + # assign service to this name at external-dns + external-dns.alpha.kubernetes.io/hostname: storage-broker-lb.delta.us-east-2.internal.aws.neon.tech + # service.type -- Service type + type: LoadBalancer + # service.port -- broker listen port + port: 50051 +ingress: + enabled: false metrics: enabled: false diff --git a/.github/helm-values/prod-us-west-2-eta.neon-storage-broker.yaml b/.github/helm-values/prod-us-west-2-eta.neon-storage-broker.yaml index 249f76303a..8a7488948d 100644 --- a/.github/helm-values/prod-us-west-2-eta.neon-storage-broker.yaml +++ b/.github/helm-values/prod-us-west-2-eta.neon-storage-broker.yaml @@ -3,27 +3,22 @@ podLabels: neon_env: production neon_service: storage-broker -ingress: - enabled: true +# Use L4 LB +service: + # service.annotations -- Annotations to add to the service annotations: - kubernetes.io/ingress.class: nginx-internal - nginx.ingress.kubernetes.io/backend-protocol: "GRPC" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - # we have basically infinite streams, disable body size limit - nginx.ingress.kubernetes.io/proxy-body-size: "0" - cert-manager.io/cluster-issuer: "cert-manager-clusterissuer" - - hosts: - - host: storage-broker.eta.us-west-2.internal.aws.neon.tech - paths: - - path: / - pathType: Prefix - tls: - - hosts: - - storage-broker.eta.us-west-2.internal.aws.neon.tech - secretName: storage-broker-tls + service.beta.kubernetes.io/aws-load-balancer-type: external # use newer AWS Load Balancer Controller + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal # deploy LB to private subnet + # assign service to this name at external-dns + external-dns.alpha.kubernetes.io/hostname: storage-broker-lb.eta.us-west-2.internal.aws.neon.tech + # service.type -- Service type + type: LoadBalancer + # service.port -- broker listen port + port: 50051 +ingress: + enabled: false metrics: enabled: false diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 7a887cbece..43b855a2b0 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1072,7 +1072,7 @@ jobs: - name: Deploy storage-broker run: - helm upgrade neon-storage-broker neondatabase/neon-storage-broker --namespace neon-storage-broker --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s + helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s deploy-proxy-prod-new: runs-on: prod @@ -1149,7 +1149,7 @@ jobs: - name: Deploy storage-broker run: - helm upgrade neon-storage-broker neondatabase/neon-storage-broker --namespace neon-storage-broker --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s + helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s promote-compatibility-data: runs-on: [ self-hosted, dev, x64 ]