diff --git a/.github/ansible/staging.eu-west-1.hosts.yaml b/.github/ansible/staging.eu-west-1.hosts.yaml new file mode 100644 index 0000000000..088ba03e5e --- /dev/null +++ b/.github/ansible/staging.eu-west-1.hosts.yaml @@ -0,0 +1,33 @@ +storage: + vars: + bucket_name: neon-dev-storage-eu-west-1 + bucket_region: eu-west-1 + console_mgmt_base_url: http://console-staging.local + etcd_endpoints: etcd-0.eu-west-1.aws.neon.build:2379 + pageserver_config_stub: + pg_distrib_dir: /usr/local + remote_storage: + bucket_name: "{{ bucket_name }}" + bucket_region: "{{ bucket_region }}" + prefix_in_bucket: "pageserver/v1" + safekeeper_s3_prefix: safekeeper/v1/wal + hostname_suffix: "" + remote_user: ssm-user + ansible_aws_ssm_region: eu-west-1 + ansible_aws_ssm_bucket_name: neon-dev-storage-eu-west-1 + console_region_id: aws-eu-west-1 + + children: + pageservers: + hosts: + pageserver-0.eu-west-1.aws.neon.build: + ansible_host: i-01d496c5041c7f34c + + safekeepers: + hosts: + safekeeper-0.eu-west-1.aws.neon.build: + ansible_host: i-05226ef85722831bf + safekeeper-1.eu-west-1.aws.neon.build: + ansible_host: i-06969ee1bf2958bfc + safekeeper-2.eu-west-1.aws.neon.build: + ansible_host: i-087892e9625984a0b diff --git a/.github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml b/.github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml new file mode 100644 index 0000000000..0e98636057 --- /dev/null +++ b/.github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml @@ -0,0 +1,31 @@ +# Helm chart values for neon-proxy-scram. +# This is a YAML-formatted file. + +image: + repository: neondatabase/neon + +settings: + authBackend: "console" + authEndpoint: "http://console-staging.local/management/api/v2" + domain: "*.eu-west-1.aws.neon.build" + +# -- Additional labels for neon-proxy pods +podLabels: + zenith_service: proxy-scram + zenith_env: dev + zenith_region: eu-west-1 + zenith_region_slug: eu-west-1 + +exposedService: + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + external-dns.alpha.kubernetes.io/hostname: eu-west-1.aws.neon.build + +#metrics: +# enabled: true +# serviceMonitor: +# enabled: true +# selector: +# release: kube-prometheus-stack diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ac30a9ec97..d27428a1a6 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -761,7 +761,6 @@ jobs: run: | export DOCKER_TAG=${{needs.tag.outputs.build-tag}} cd "$(pwd)/.github/ansible" - if [[ "$GITHUB_REF_NAME" == "main" ]]; then ./get_binaries.sh elif [[ "$GITHUB_REF_NAME" == "release" ]]; then @@ -770,6 +769,38 @@ jobs: echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" exit 1 fi + ansible-galaxy collection install sivel.toiletwater + ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}} + rm -f neon_install.tar.gz .neon_current_version + + deploy-pr-test-new: + runs-on: [ self-hosted, dev, x64 ] + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned + # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version. + # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly + needs: [ push-docker-hub, tag, regress-tests ] + if: | + contains(github.event.pull_request.labels.*.name, 'deploy-test-storage') && + github.event_name != 'workflow_dispatch' + defaults: + run: + shell: bash + strategy: + matrix: + target_region: [ eu-west-1 ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Redeploy + run: | + export DOCKER_TAG=${{needs.tag.outputs.build-tag}} + cd "$(pwd)/.github/ansible" + + ./get_binaries.sh ansible-galaxy collection install sivel.toiletwater ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}} @@ -780,7 +811,7 @@ jobs: container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version. # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly - needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] + needs: [ push-docker-hub, tag, regress-tests ] if: | (github.ref_name == 'release') && github.event_name != 'workflow_dispatch' @@ -861,7 +892,7 @@ jobs: runs-on: [ self-hosted, dev, x64 ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. - needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] + needs: [ push-docker-hub, tag, regress-tests ] if: | (github.ref_name == 'main') && github.event_name != 'workflow_dispatch' @@ -873,6 +904,8 @@ jobs: include: - target_region: us-east-2 target_cluster: dev-us-east-2-beta + - target_region: eu-west-1 + target_cluster: dev-eu-west-1-zeta steps: - name: Checkout uses: actions/checkout@v3 @@ -894,7 +927,7 @@ jobs: runs-on: prod container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. - needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] + needs: [ push-docker-hub, tag, regress-tests ] if: | (github.ref_name == 'release') && github.event_name != 'workflow_dispatch'