diff --git a/.github/ansible/.gitignore b/.github/ansible/.gitignore deleted file mode 100644 index 9cd8044417..0000000000 --- a/.github/ansible/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -neon_install.tar.gz -.neon_current_version - -collections/* -!collections/.keep diff --git a/.github/ansible/ansible.cfg b/.github/ansible/ansible.cfg deleted file mode 100644 index 5818a64455..0000000000 --- a/.github/ansible/ansible.cfg +++ /dev/null @@ -1,12 +0,0 @@ -[defaults] - -localhost_warning = False -host_key_checking = False -timeout = 30 - -[ssh_connection] -ssh_args = -F ./ansible.ssh.cfg -# teleport doesn't support sftp yet https://github.com/gravitational/teleport/issues/7127 -# and scp neither worked for me -transfer_method = piped -pipelining = True diff --git a/.github/ansible/ansible.ssh.cfg b/.github/ansible/ansible.ssh.cfg deleted file mode 100644 index cd058b5427..0000000000 --- a/.github/ansible/ansible.ssh.cfg +++ /dev/null @@ -1,15 +0,0 @@ -# Remove this once https://github.com/gravitational/teleport/issues/10918 is fixed -# (use pre 8.5 option name to cope with old ssh in CI) -PubkeyAcceptedKeyTypes +ssh-rsa-cert-v01@openssh.com - -Host tele.zenith.tech - User admin - Port 3023 - StrictHostKeyChecking no - UserKnownHostsFile /dev/null - -Host * !tele.zenith.tech - User admin - StrictHostKeyChecking no - UserKnownHostsFile /dev/null - ProxyJump tele.zenith.tech diff --git a/.github/ansible/collections/.keep b/.github/ansible/collections/.keep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.github/ansible/deploy.yaml b/.github/ansible/deploy.yaml deleted file mode 100644 index d4c1dec8ea..0000000000 --- a/.github/ansible/deploy.yaml +++ /dev/null @@ -1,211 +0,0 @@ -- name: Upload Neon binaries - hosts: storage - gather_facts: False - remote_user: "{{ remote_user }}" - - tasks: - - - name: get latest version of Neon binaries - register: current_version_file - set_fact: - current_version: "{{ lookup('file', '.neon_current_version') | trim }}" - tags: - - pageserver - - safekeeper - - - name: inform about versions - debug: - msg: "Version to deploy - {{ current_version }}" - tags: - - pageserver - - safekeeper - - - name: upload and extract Neon binaries to /usr/local - ansible.builtin.unarchive: - owner: root - group: root - src: neon_install.tar.gz - dest: /usr/local - become: true - tags: - - pageserver - - safekeeper - - binaries - - putbinaries - -- name: Deploy pageserver - hosts: pageservers - gather_facts: False - remote_user: "{{ remote_user }}" - - tasks: - - - name: upload init script - when: console_mgmt_base_url is defined - ansible.builtin.template: - src: scripts/init_pageserver.sh - dest: /tmp/init_pageserver.sh - owner: root - group: root - mode: '0755' - become: true - tags: - - pageserver - - - name: init pageserver - shell: - cmd: /tmp/init_pageserver.sh - args: - creates: "/storage/pageserver/data/tenants" - environment: - NEON_REPO_DIR: "/storage/pageserver/data" - LD_LIBRARY_PATH: "/usr/local/v14/lib" - become: true - tags: - - pageserver - - - name: read the existing remote pageserver config - ansible.builtin.slurp: - src: /storage/pageserver/data/pageserver.toml - register: _remote_ps_config - tags: - - pageserver - - - name: parse the existing pageserver configuration - ansible.builtin.set_fact: - _existing_ps_config: "{{ _remote_ps_config['content'] | b64decode | sivel.toiletwater.from_toml }}" - tags: - - pageserver - - - name: construct the final pageserver configuration dict - ansible.builtin.set_fact: - pageserver_config: "{{ pageserver_config_stub | combine({'id': _existing_ps_config.id }) }}" - tags: - - pageserver - - - name: template the pageserver config - template: - src: templates/pageserver.toml.j2 - dest: /storage/pageserver/data/pageserver.toml - become: true - tags: - - pageserver - - # used in `pageserver.service` template - - name: learn current availability_zone - shell: - cmd: "curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone" - register: ec2_availability_zone - - - set_fact: - ec2_availability_zone={{ ec2_availability_zone.stdout }} - - - name: upload systemd service definition - ansible.builtin.template: - src: systemd/pageserver.service - dest: /etc/systemd/system/pageserver.service - owner: root - group: root - mode: '0644' - become: true - tags: - - pageserver - - - name: start systemd service - ansible.builtin.systemd: - daemon_reload: yes - name: pageserver - enabled: yes - state: restarted - become: true - tags: - - pageserver - - - name: post version to console - when: console_mgmt_base_url is defined - shell: - cmd: | - INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) - curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/pageservers/$INSTANCE_ID | jq '.version = {{ current_version }}' > /tmp/new_version - curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" -H "Content-Type: application/json" -X POST -d@/tmp/new_version {{ console_mgmt_base_url }}/management/api/v2/pageservers - tags: - - pageserver - -- name: Deploy safekeeper - hosts: safekeepers - gather_facts: False - remote_user: "{{ remote_user }}" - - tasks: - - - name: upload init script - when: console_mgmt_base_url is defined - ansible.builtin.template: - src: scripts/init_safekeeper.sh - dest: /tmp/init_safekeeper.sh - owner: root - group: root - mode: '0755' - become: true - tags: - - safekeeper - - - name: init safekeeper - shell: - cmd: /tmp/init_safekeeper.sh - args: - creates: "/storage/safekeeper/data/safekeeper.id" - environment: - NEON_REPO_DIR: "/storage/safekeeper/data" - LD_LIBRARY_PATH: "/usr/local/v14/lib" - become: true - tags: - - safekeeper - - # used in `safekeeper.service` template - - name: learn current availability_zone - shell: - cmd: "curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone" - register: ec2_availability_zone - - - set_fact: - ec2_availability_zone={{ ec2_availability_zone.stdout }} - - # in the future safekeepers should discover pageservers byself - # but currently use first pageserver that was discovered - - name: set first pageserver var for safekeepers - set_fact: - first_pageserver: "{{ hostvars[groups['pageservers'][0]]['inventory_hostname'] }}" - tags: - - safekeeper - - - name: upload systemd service definition - ansible.builtin.template: - src: systemd/safekeeper.service - dest: /etc/systemd/system/safekeeper.service - owner: root - group: root - mode: '0644' - become: true - tags: - - safekeeper - - - name: start systemd service - ansible.builtin.systemd: - daemon_reload: yes - name: safekeeper - enabled: yes - state: restarted - become: true - tags: - - safekeeper - - - name: post version to console - when: console_mgmt_base_url is defined - shell: - cmd: | - INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) - curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/safekeepers/$INSTANCE_ID | jq '.version = {{ current_version }}' > /tmp/new_version - curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" -H "Content-Type: application/json" -X POST -d@/tmp/new_version {{ console_mgmt_base_url }}/management/api/v2/safekeepers - tags: - - safekeeper diff --git a/.github/ansible/get_binaries.sh b/.github/ansible/get_binaries.sh deleted file mode 100755 index 4bb580428c..0000000000 --- a/.github/ansible/get_binaries.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -set -e - -if [ -n "${DOCKER_TAG}" ]; then - # Verson is DOCKER_TAG but without prefix - VERSION=$(echo $DOCKER_TAG | sed 's/^.*-//g') -else - echo "Please set DOCKER_TAG environment variable" - exit 1 -fi - - -# do initial cleanup -rm -rf neon_install postgres_install.tar.gz neon_install.tar.gz .neon_current_version -mkdir neon_install - -# retrieve binaries from docker image -echo "getting binaries from docker image" -docker pull --quiet neondatabase/neon:${DOCKER_TAG} -ID=$(docker create neondatabase/neon:${DOCKER_TAG}) -docker cp ${ID}:/data/postgres_install.tar.gz . -tar -xzf postgres_install.tar.gz -C neon_install -mkdir neon_install/bin/ -docker cp ${ID}:/usr/local/bin/pageserver neon_install/bin/ -docker cp ${ID}:/usr/local/bin/pageserver_binutils neon_install/bin/ -docker cp ${ID}:/usr/local/bin/safekeeper neon_install/bin/ -docker cp ${ID}:/usr/local/bin/storage_broker neon_install/bin/ -docker cp ${ID}:/usr/local/bin/proxy neon_install/bin/ -docker cp ${ID}:/usr/local/v14/bin/ neon_install/v14/bin/ -docker cp ${ID}:/usr/local/v15/bin/ neon_install/v15/bin/ -docker cp ${ID}:/usr/local/v14/lib/ neon_install/v14/lib/ -docker cp ${ID}:/usr/local/v15/lib/ neon_install/v15/lib/ -docker rm -vf ${ID} - -# store version to file (for ansible playbooks) and create binaries tarball -echo ${VERSION} > neon_install/.neon_current_version -echo ${VERSION} > .neon_current_version -tar -czf neon_install.tar.gz -C neon_install . - -# do final cleaup -rm -rf neon_install postgres_install.tar.gz diff --git a/.github/ansible/prod.ap-southeast-1.hosts.yaml b/.github/ansible/prod.ap-southeast-1.hosts.yaml deleted file mode 100644 index 9c53733491..0000000000 --- a/.github/ansible/prod.ap-southeast-1.hosts.yaml +++ /dev/null @@ -1,48 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-ap-southeast-1 - bucket_region: ap-southeast-1 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.epsilon.ap-southeast-1.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: ap-southeast-1 - ansible_aws_ssm_bucket_name: neon-prod-storage-ap-southeast-1 - console_region_id: aws-ap-southeast-1 - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.ap-southeast-1.aws.neon.tech: - ansible_host: i-064de8ea28bdb495b - pageserver-1.ap-southeast-1.aws.neon.tech: - ansible_host: i-0b180defcaeeb6b93 - - safekeepers: - hosts: - safekeeper-0.ap-southeast-1.aws.neon.tech: - ansible_host: i-0d6f1dc5161eef894 - safekeeper-2.ap-southeast-1.aws.neon.tech: - ansible_host: i-04fb63634e4679eb9 - safekeeper-3.ap-southeast-1.aws.neon.tech: - ansible_host: i-05481f3bc88cfc2d4 diff --git a/.github/ansible/prod.eu-central-1.hosts.yaml b/.github/ansible/prod.eu-central-1.hosts.yaml deleted file mode 100644 index 3186519ca8..0000000000 --- a/.github/ansible/prod.eu-central-1.hosts.yaml +++ /dev/null @@ -1,50 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-eu-central-1 - bucket_region: eu-central-1 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.gamma.eu-central-1.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: eu-central-1 - ansible_aws_ssm_bucket_name: neon-prod-storage-eu-central-1 - console_region_id: aws-eu-central-1 - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.eu-central-1.aws.neon.tech: - ansible_host: i-0cd8d316ecbb715be - pageserver-1.eu-central-1.aws.neon.tech: - ansible_host: i-090044ed3d383fef0 - pageserver-2.eu-central-1.aws.neon.tech: - ansible_host: i-033584edf3f4b6742 - - safekeepers: - hosts: - safekeeper-0.eu-central-1.aws.neon.tech: - ansible_host: i-0b238612d2318a050 - safekeeper-1.eu-central-1.aws.neon.tech: - ansible_host: i-07b9c45e5c2637cd4 - safekeeper-2.eu-central-1.aws.neon.tech: - ansible_host: i-020257302c3c93d88 diff --git a/.github/ansible/prod.us-east-1.hosts.yaml b/.github/ansible/prod.us-east-1.hosts.yaml deleted file mode 100644 index b5b2b076bb..0000000000 --- a/.github/ansible/prod.us-east-1.hosts.yaml +++ /dev/null @@ -1,50 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-us-east-1 - bucket_region: us-east-1 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.theta.us-east-1.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: us-east-1 - ansible_aws_ssm_bucket_name: neon-prod-storage-us-east-1 - console_region_id: aws-us-east-1 - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.us-east-1.aws.neon.tech: - ansible_host: i-085222088b0d2e0c7 - pageserver-1.us-east-1.aws.neon.tech: - ansible_host: i-0969d4f684d23a21e - pageserver-2.us-east-1.aws.neon.tech: - ansible_host: i-05dee87895da58dad - - safekeepers: - hosts: - safekeeper-0.us-east-1.aws.neon.tech: - ansible_host: i-04ce739e88793d864 - safekeeper-1.us-east-1.aws.neon.tech: - ansible_host: i-0e9e6c9227fb81410 - safekeeper-2.us-east-1.aws.neon.tech: - ansible_host: i-072f4dd86a327d52f diff --git a/.github/ansible/prod.us-east-2.hosts.yaml b/.github/ansible/prod.us-east-2.hosts.yaml deleted file mode 100644 index 3062475b20..0000000000 --- a/.github/ansible/prod.us-east-2.hosts.yaml +++ /dev/null @@ -1,51 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-us-east-2 - bucket_region: us-east-2 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.delta.us-east-2.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: us-east-2 - ansible_aws_ssm_bucket_name: neon-prod-storage-us-east-2 - console_region_id: aws-us-east-2 - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.us-east-2.aws.neon.tech: - ansible_host: i-062227ba7f119eb8c - pageserver-1.us-east-2.aws.neon.tech: - ansible_host: i-0b3ec0afab5968938 - pageserver-2.us-east-2.aws.neon.tech: - ansible_host: i-0d7a1c4325e71421d - - safekeepers: - hosts: - safekeeper-0.us-east-2.aws.neon.tech: - ansible_host: i-0e94224750c57d346 - safekeeper-1.us-east-2.aws.neon.tech: - ansible_host: i-06d113fb73bfddeb0 - safekeeper-2.us-east-2.aws.neon.tech: - ansible_host: i-09f66c8e04afff2e8 - diff --git a/.github/ansible/prod.us-west-2.hosts.yaml b/.github/ansible/prod.us-west-2.hosts.yaml deleted file mode 100644 index be65d8e63c..0000000000 --- a/.github/ansible/prod.us-west-2.hosts.yaml +++ /dev/null @@ -1,72 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-us-west-2 - bucket_region: us-west-2 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.eta.us-west-2.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: us-west-2 - ansible_aws_ssm_bucket_name: neon-prod-storage-us-west-2 - console_region_id: aws-us-west-2-new - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.us-west-2.aws.neon.tech: - ansible_host: i-0d9f6dfae0e1c780d - pageserver-1.us-west-2.aws.neon.tech: - ansible_host: i-0c834be1dddba8b3f - pageserver-2.us-west-2.aws.neon.tech: - ansible_host: i-051642d372c0a4f32 - pageserver-3.us-west-2.aws.neon.tech: - ansible_host: i-00c3844beb9ad1c6b - pageserver-4.us-west-2.aws.neon.tech: - ansible_host: i-013263dd1c239adcc - pageserver-5.us-west-2.aws.neon.tech: - ansible_host: i-00ca6417c7bf96820 - pageserver-6.us-west-2.aws.neon.tech: - ansible_host: i-01cdf7d2bc1433b6a - pageserver-7.us-west-2.aws.neon.tech: - ansible_host: i-02eec9b40617db5bc - - safekeepers: - hosts: - safekeeper-0.us-west-2.aws.neon.tech: - ansible_host: i-00719d8a74986fda6 - safekeeper-1.us-west-2.aws.neon.tech: - ansible_host: i-074682f9d3c712e7c - safekeeper-2.us-west-2.aws.neon.tech: - ansible_host: i-042b7efb1729d7966 - safekeeper-3.us-west-2.aws.neon.tech: - ansible_host: i-089f6b9ef426dff76 - safekeeper-4.us-west-2.aws.neon.tech: - ansible_host: i-0fe6bf912c4710c82 - safekeeper-5.us-west-2.aws.neon.tech: - ansible_host: i-0a83c1c46d2b4e409 - safekeeper-6.us-west-2.aws.neon.tech: - ansible_host: i-0fef5317b8fdc9f8d - safekeeper-7.us-west-2.aws.neon.tech: - ansible_host: i-0be739190d4289bf9 - safekeeper-8.us-west-2.aws.neon.tech: - ansible_host: i-00e851803669e5cfe diff --git a/.github/ansible/scripts/init_pageserver.sh b/.github/ansible/scripts/init_pageserver.sh deleted file mode 100644 index d88f754a86..0000000000 --- a/.github/ansible/scripts/init_pageserver.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -# fetch params from meta-data service -INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) -AZ_ID=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) -INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) -DISK_SIZE=$(df -B1 /storage | tail -1 | awk '{print $2}') - -# store fqdn hostname in var -HOST=$(hostname -f) - - -cat <preview-${PREVIEW_NAME}.neon-proxy-scram.yaml - helm upgrade neon-proxy-scram-${PREVIEW_NAME} neondatabase/neon-proxy --namespace neon-proxy-${PREVIEW_NAME} --create-namespace --install --atomic -f preview-${PREVIEW_NAME}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - done - - - name: Cleanup helm folder - run: rm -rf ~/.cache - - deploy-storage-broker-new: - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - if: inputs.deployStorageBroker - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: dev-us-east-2-beta - - target_region: eu-west-1 - target_cluster: dev-eu-west-1-zeta - - target_region: eu-central-1 - target_cluster: dev-eu-central-1-alpha - environment: - name: dev-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - role-to-assume: arn:aws:iam::369495373322:role/github-runner - aws-region: eu-central-1 - role-skip-session-tagging: true - role-duration-seconds: 1800 - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy storage-broker - run: - helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s - - - name: Cleanup helm folder - run: rm -rf ~/.cache - - deploy-pg-sni-router: - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - if: inputs.deployPgSniRouter - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: dev-us-east-2-beta - - target_region: eu-west-1 - target_cluster: dev-eu-west-1-zeta - - target_region: eu-central-1 - target_cluster: dev-eu-central-1-alpha - environment: - name: dev-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - role-to-assume: arn:aws:iam::369495373322:role/github-runner - aws-region: eu-central-1 - role-skip-session-tagging: true - role-duration-seconds: 1800 - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy pg-sni-router - run: - helm upgrade neon-pg-sni-router neondatabase/neon-pg-sni-router --namespace neon-pg-sni-router --create-namespace --install --debug --atomic -f .github/helm-values/${{ matrix.target_cluster }}.pg-sni-router.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 15m0s - - - name: Cleanup helm folder - run: rm -rf ~/.cache diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml deleted file mode 100644 index baa44d8094..0000000000 --- a/.github/workflows/deploy-prod.yml +++ /dev/null @@ -1,217 +0,0 @@ -name: Neon Deploy prod - -on: - workflow_dispatch: - inputs: - dockerTag: - description: 'Docker tag to deploy' - required: true - type: string - branch: - description: 'Branch or commit used for deploy scripts and configs' - required: true - type: string - default: 'release' - deployStorage: - description: 'Deploy storage' - required: true - type: boolean - default: true - deployProxy: - description: 'Deploy proxy' - required: true - type: boolean - default: true - deployStorageBroker: - description: 'Deploy storage-broker' - required: true - type: boolean - default: true - deployPgSniRouter: - description: 'Deploy pg-sni-router' - required: true - type: boolean - default: true - disclamerAcknowledged: - description: 'I confirm that there is an emergency and I can not use regular release workflow' - required: true - type: boolean - default: false - -concurrency: - group: deploy-prod - cancel-in-progress: false - -jobs: - deploy-prod-new: - runs-on: prod - container: - image: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - options: --user root --privileged - if: inputs.deployStorage && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - strategy: - matrix: - target_region: [ us-east-2, us-west-2, eu-central-1, ap-southeast-1, us-east-1 ] - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Redeploy - run: | - export DOCKER_TAG=${{ inputs.dockerTag }} - cd "$(pwd)/.github/ansible" - - ./get_binaries.sh - - ansible-galaxy collection install sivel.toiletwater - ansible-playbook -v deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_PRODUCTION_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }} - rm -f neon_install.tar.gz .neon_current_version - - deploy-proxy-prod-new: - runs-on: prod - container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployProxy && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: prod-us-east-2-delta - deploy_link_proxy: true - deploy_legacy_scram_proxy: false - - target_region: us-west-2 - target_cluster: prod-us-west-2-eta - deploy_link_proxy: false - deploy_legacy_scram_proxy: true - - target_region: eu-central-1 - target_cluster: prod-eu-central-1-gamma - deploy_link_proxy: false - deploy_legacy_scram_proxy: false - - target_region: ap-southeast-1 - target_cluster: prod-ap-southeast-1-epsilon - deploy_link_proxy: false - deploy_legacy_scram_proxy: false - - target_region: us-east-1 - target_cluster: prod-us-east-1-theta - deploy_link_proxy: false - deploy_legacy_scram_proxy: false - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Re-deploy scram proxy - run: | - DOCKER_TAG=${{ inputs.dockerTag }} - helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Re-deploy link proxy - if: matrix.deploy_link_proxy - run: | - DOCKER_TAG=${{ inputs.dockerTag }} - helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Re-deploy legacy scram proxy - if: matrix.deploy_legacy_scram_proxy - run: | - DOCKER_TAG=${{ inputs.dockerTag }} - helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - deploy-storage-broker-prod-new: - runs-on: prod - container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployStorageBroker && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: prod-us-east-2-delta - - target_region: us-west-2 - target_cluster: prod-us-west-2-eta - - target_region: eu-central-1 - target_cluster: prod-eu-central-1-gamma - - target_region: ap-southeast-1 - target_cluster: prod-ap-southeast-1-epsilon - - target_region: us-east-1 - target_cluster: prod-us-east-1-theta - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy storage-broker - run: - helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s - - deploy-pg-sni-router: - runs-on: prod - container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployPgSniRouter && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: prod-us-east-2-delta - - target_region: us-west-2 - target_cluster: prod-us-west-2-eta - - target_region: eu-central-1 - target_cluster: prod-eu-central-1-gamma - - target_region: ap-southeast-1 - target_cluster: prod-ap-southeast-1-epsilon - - target_region: us-east-1 - target_cluster: prod-us-east-1-theta - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy pg-sni-router - run: - helm upgrade neon-pg-sni-router neondatabase/neon-pg-sni-router --namespace neon-pg-sni-router --create-namespace --install --debug --atomic -f .github/helm-values/${{ matrix.target_cluster }}.pg-sni-router.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 15m0s