diff --git a/.github/ansible/.gitignore b/.github/ansible/.gitignore deleted file mode 100644 index 9cd8044417..0000000000 --- a/.github/ansible/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -neon_install.tar.gz -.neon_current_version - -collections/* -!collections/.keep diff --git a/.github/ansible/ansible.cfg b/.github/ansible/ansible.cfg deleted file mode 100644 index 5818a64455..0000000000 --- a/.github/ansible/ansible.cfg +++ /dev/null @@ -1,12 +0,0 @@ -[defaults] - -localhost_warning = False -host_key_checking = False -timeout = 30 - -[ssh_connection] -ssh_args = -F ./ansible.ssh.cfg -# teleport doesn't support sftp yet https://github.com/gravitational/teleport/issues/7127 -# and scp neither worked for me -transfer_method = piped -pipelining = True diff --git a/.github/ansible/ansible.ssh.cfg b/.github/ansible/ansible.ssh.cfg deleted file mode 100644 index cd058b5427..0000000000 --- a/.github/ansible/ansible.ssh.cfg +++ /dev/null @@ -1,15 +0,0 @@ -# Remove this once https://github.com/gravitational/teleport/issues/10918 is fixed -# (use pre 8.5 option name to cope with old ssh in CI) -PubkeyAcceptedKeyTypes +ssh-rsa-cert-v01@openssh.com - -Host tele.zenith.tech - User admin - Port 3023 - StrictHostKeyChecking no - UserKnownHostsFile /dev/null - -Host * !tele.zenith.tech - User admin - StrictHostKeyChecking no - UserKnownHostsFile /dev/null - ProxyJump tele.zenith.tech diff --git a/.github/ansible/collections/.keep b/.github/ansible/collections/.keep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.github/ansible/deploy.yaml b/.github/ansible/deploy.yaml deleted file mode 100644 index d4c1dec8ea..0000000000 --- a/.github/ansible/deploy.yaml +++ /dev/null @@ -1,211 +0,0 @@ -- name: Upload Neon binaries - hosts: storage - gather_facts: False - remote_user: "{{ remote_user }}" - - tasks: - - - name: get latest version of Neon binaries - register: current_version_file - set_fact: - current_version: "{{ lookup('file', '.neon_current_version') | trim }}" - tags: - - pageserver - - safekeeper - - - name: inform about versions - debug: - msg: "Version to deploy - {{ current_version }}" - tags: - - pageserver - - safekeeper - - - name: upload and extract Neon binaries to /usr/local - ansible.builtin.unarchive: - owner: root - group: root - src: neon_install.tar.gz - dest: /usr/local - become: true - tags: - - pageserver - - safekeeper - - binaries - - putbinaries - -- name: Deploy pageserver - hosts: pageservers - gather_facts: False - remote_user: "{{ remote_user }}" - - tasks: - - - name: upload init script - when: console_mgmt_base_url is defined - ansible.builtin.template: - src: scripts/init_pageserver.sh - dest: /tmp/init_pageserver.sh - owner: root - group: root - mode: '0755' - become: true - tags: - - pageserver - - - name: init pageserver - shell: - cmd: /tmp/init_pageserver.sh - args: - creates: "/storage/pageserver/data/tenants" - environment: - NEON_REPO_DIR: "/storage/pageserver/data" - LD_LIBRARY_PATH: "/usr/local/v14/lib" - become: true - tags: - - pageserver - - - name: read the existing remote pageserver config - ansible.builtin.slurp: - src: /storage/pageserver/data/pageserver.toml - register: _remote_ps_config - tags: - - pageserver - - - name: parse the existing pageserver configuration - ansible.builtin.set_fact: - _existing_ps_config: "{{ _remote_ps_config['content'] | b64decode | sivel.toiletwater.from_toml }}" - tags: - - pageserver - - - name: construct the final pageserver configuration dict - ansible.builtin.set_fact: - pageserver_config: "{{ pageserver_config_stub | combine({'id': _existing_ps_config.id }) }}" - tags: - - pageserver - - - name: template the pageserver config - template: - src: templates/pageserver.toml.j2 - dest: /storage/pageserver/data/pageserver.toml - become: true - tags: - - pageserver - - # used in `pageserver.service` template - - name: learn current availability_zone - shell: - cmd: "curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone" - register: ec2_availability_zone - - - set_fact: - ec2_availability_zone={{ ec2_availability_zone.stdout }} - - - name: upload systemd service definition - ansible.builtin.template: - src: systemd/pageserver.service - dest: /etc/systemd/system/pageserver.service - owner: root - group: root - mode: '0644' - become: true - tags: - - pageserver - - - name: start systemd service - ansible.builtin.systemd: - daemon_reload: yes - name: pageserver - enabled: yes - state: restarted - become: true - tags: - - pageserver - - - name: post version to console - when: console_mgmt_base_url is defined - shell: - cmd: | - INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) - curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/pageservers/$INSTANCE_ID | jq '.version = {{ current_version }}' > /tmp/new_version - curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" -H "Content-Type: application/json" -X POST -d@/tmp/new_version {{ console_mgmt_base_url }}/management/api/v2/pageservers - tags: - - pageserver - -- name: Deploy safekeeper - hosts: safekeepers - gather_facts: False - remote_user: "{{ remote_user }}" - - tasks: - - - name: upload init script - when: console_mgmt_base_url is defined - ansible.builtin.template: - src: scripts/init_safekeeper.sh - dest: /tmp/init_safekeeper.sh - owner: root - group: root - mode: '0755' - become: true - tags: - - safekeeper - - - name: init safekeeper - shell: - cmd: /tmp/init_safekeeper.sh - args: - creates: "/storage/safekeeper/data/safekeeper.id" - environment: - NEON_REPO_DIR: "/storage/safekeeper/data" - LD_LIBRARY_PATH: "/usr/local/v14/lib" - become: true - tags: - - safekeeper - - # used in `safekeeper.service` template - - name: learn current availability_zone - shell: - cmd: "curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone" - register: ec2_availability_zone - - - set_fact: - ec2_availability_zone={{ ec2_availability_zone.stdout }} - - # in the future safekeepers should discover pageservers byself - # but currently use first pageserver that was discovered - - name: set first pageserver var for safekeepers - set_fact: - first_pageserver: "{{ hostvars[groups['pageservers'][0]]['inventory_hostname'] }}" - tags: - - safekeeper - - - name: upload systemd service definition - ansible.builtin.template: - src: systemd/safekeeper.service - dest: /etc/systemd/system/safekeeper.service - owner: root - group: root - mode: '0644' - become: true - tags: - - safekeeper - - - name: start systemd service - ansible.builtin.systemd: - daemon_reload: yes - name: safekeeper - enabled: yes - state: restarted - become: true - tags: - - safekeeper - - - name: post version to console - when: console_mgmt_base_url is defined - shell: - cmd: | - INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) - curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/safekeepers/$INSTANCE_ID | jq '.version = {{ current_version }}' > /tmp/new_version - curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" -H "Content-Type: application/json" -X POST -d@/tmp/new_version {{ console_mgmt_base_url }}/management/api/v2/safekeepers - tags: - - safekeeper diff --git a/.github/ansible/get_binaries.sh b/.github/ansible/get_binaries.sh deleted file mode 100755 index 4bb580428c..0000000000 --- a/.github/ansible/get_binaries.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -set -e - -if [ -n "${DOCKER_TAG}" ]; then - # Verson is DOCKER_TAG but without prefix - VERSION=$(echo $DOCKER_TAG | sed 's/^.*-//g') -else - echo "Please set DOCKER_TAG environment variable" - exit 1 -fi - - -# do initial cleanup -rm -rf neon_install postgres_install.tar.gz neon_install.tar.gz .neon_current_version -mkdir neon_install - -# retrieve binaries from docker image -echo "getting binaries from docker image" -docker pull --quiet neondatabase/neon:${DOCKER_TAG} -ID=$(docker create neondatabase/neon:${DOCKER_TAG}) -docker cp ${ID}:/data/postgres_install.tar.gz . -tar -xzf postgres_install.tar.gz -C neon_install -mkdir neon_install/bin/ -docker cp ${ID}:/usr/local/bin/pageserver neon_install/bin/ -docker cp ${ID}:/usr/local/bin/pageserver_binutils neon_install/bin/ -docker cp ${ID}:/usr/local/bin/safekeeper neon_install/bin/ -docker cp ${ID}:/usr/local/bin/storage_broker neon_install/bin/ -docker cp ${ID}:/usr/local/bin/proxy neon_install/bin/ -docker cp ${ID}:/usr/local/v14/bin/ neon_install/v14/bin/ -docker cp ${ID}:/usr/local/v15/bin/ neon_install/v15/bin/ -docker cp ${ID}:/usr/local/v14/lib/ neon_install/v14/lib/ -docker cp ${ID}:/usr/local/v15/lib/ neon_install/v15/lib/ -docker rm -vf ${ID} - -# store version to file (for ansible playbooks) and create binaries tarball -echo ${VERSION} > neon_install/.neon_current_version -echo ${VERSION} > .neon_current_version -tar -czf neon_install.tar.gz -C neon_install . - -# do final cleaup -rm -rf neon_install postgres_install.tar.gz diff --git a/.github/ansible/prod.ap-southeast-1.hosts.yaml b/.github/ansible/prod.ap-southeast-1.hosts.yaml deleted file mode 100644 index 9c53733491..0000000000 --- a/.github/ansible/prod.ap-southeast-1.hosts.yaml +++ /dev/null @@ -1,48 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-ap-southeast-1 - bucket_region: ap-southeast-1 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.epsilon.ap-southeast-1.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: ap-southeast-1 - ansible_aws_ssm_bucket_name: neon-prod-storage-ap-southeast-1 - console_region_id: aws-ap-southeast-1 - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.ap-southeast-1.aws.neon.tech: - ansible_host: i-064de8ea28bdb495b - pageserver-1.ap-southeast-1.aws.neon.tech: - ansible_host: i-0b180defcaeeb6b93 - - safekeepers: - hosts: - safekeeper-0.ap-southeast-1.aws.neon.tech: - ansible_host: i-0d6f1dc5161eef894 - safekeeper-2.ap-southeast-1.aws.neon.tech: - ansible_host: i-04fb63634e4679eb9 - safekeeper-3.ap-southeast-1.aws.neon.tech: - ansible_host: i-05481f3bc88cfc2d4 diff --git a/.github/ansible/prod.eu-central-1.hosts.yaml b/.github/ansible/prod.eu-central-1.hosts.yaml deleted file mode 100644 index 3186519ca8..0000000000 --- a/.github/ansible/prod.eu-central-1.hosts.yaml +++ /dev/null @@ -1,50 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-eu-central-1 - bucket_region: eu-central-1 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.gamma.eu-central-1.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: eu-central-1 - ansible_aws_ssm_bucket_name: neon-prod-storage-eu-central-1 - console_region_id: aws-eu-central-1 - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.eu-central-1.aws.neon.tech: - ansible_host: i-0cd8d316ecbb715be - pageserver-1.eu-central-1.aws.neon.tech: - ansible_host: i-090044ed3d383fef0 - pageserver-2.eu-central-1.aws.neon.tech: - ansible_host: i-033584edf3f4b6742 - - safekeepers: - hosts: - safekeeper-0.eu-central-1.aws.neon.tech: - ansible_host: i-0b238612d2318a050 - safekeeper-1.eu-central-1.aws.neon.tech: - ansible_host: i-07b9c45e5c2637cd4 - safekeeper-2.eu-central-1.aws.neon.tech: - ansible_host: i-020257302c3c93d88 diff --git a/.github/ansible/prod.us-east-1.hosts.yaml b/.github/ansible/prod.us-east-1.hosts.yaml deleted file mode 100644 index b5b2b076bb..0000000000 --- a/.github/ansible/prod.us-east-1.hosts.yaml +++ /dev/null @@ -1,50 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-us-east-1 - bucket_region: us-east-1 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.theta.us-east-1.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: us-east-1 - ansible_aws_ssm_bucket_name: neon-prod-storage-us-east-1 - console_region_id: aws-us-east-1 - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.us-east-1.aws.neon.tech: - ansible_host: i-085222088b0d2e0c7 - pageserver-1.us-east-1.aws.neon.tech: - ansible_host: i-0969d4f684d23a21e - pageserver-2.us-east-1.aws.neon.tech: - ansible_host: i-05dee87895da58dad - - safekeepers: - hosts: - safekeeper-0.us-east-1.aws.neon.tech: - ansible_host: i-04ce739e88793d864 - safekeeper-1.us-east-1.aws.neon.tech: - ansible_host: i-0e9e6c9227fb81410 - safekeeper-2.us-east-1.aws.neon.tech: - ansible_host: i-072f4dd86a327d52f diff --git a/.github/ansible/prod.us-east-2.hosts.yaml b/.github/ansible/prod.us-east-2.hosts.yaml deleted file mode 100644 index 3062475b20..0000000000 --- a/.github/ansible/prod.us-east-2.hosts.yaml +++ /dev/null @@ -1,51 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-us-east-2 - bucket_region: us-east-2 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.delta.us-east-2.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: us-east-2 - ansible_aws_ssm_bucket_name: neon-prod-storage-us-east-2 - console_region_id: aws-us-east-2 - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.us-east-2.aws.neon.tech: - ansible_host: i-062227ba7f119eb8c - pageserver-1.us-east-2.aws.neon.tech: - ansible_host: i-0b3ec0afab5968938 - pageserver-2.us-east-2.aws.neon.tech: - ansible_host: i-0d7a1c4325e71421d - - safekeepers: - hosts: - safekeeper-0.us-east-2.aws.neon.tech: - ansible_host: i-0e94224750c57d346 - safekeeper-1.us-east-2.aws.neon.tech: - ansible_host: i-06d113fb73bfddeb0 - safekeeper-2.us-east-2.aws.neon.tech: - ansible_host: i-09f66c8e04afff2e8 - diff --git a/.github/ansible/prod.us-west-2.hosts.yaml b/.github/ansible/prod.us-west-2.hosts.yaml deleted file mode 100644 index be65d8e63c..0000000000 --- a/.github/ansible/prod.us-west-2.hosts.yaml +++ /dev/null @@ -1,72 +0,0 @@ -storage: - vars: - bucket_name: neon-prod-storage-us-west-2 - bucket_region: us-west-2 - console_mgmt_base_url: http://neon-internal-api.aws.neon.tech - broker_endpoint: http://storage-broker-lb.eta.us-west-2.internal.aws.neon.tech:50051 - pageserver_config_stub: - pg_distrib_dir: /usr/local - metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events - metric_collection_interval: 10min - disk_usage_based_eviction: - max_usage_pct: 85 # TODO: decrease to 80 after all pageservers are below 80 - min_avail_bytes: 0 - period: "10s" - tenant_config: - eviction_policy: - kind: "LayerAccessThreshold" - period: "10m" - threshold: &default_eviction_threshold "24h" - evictions_low_residence_duration_metric_threshold: *default_eviction_threshold - remote_storage: - bucket_name: "{{ bucket_name }}" - bucket_region: "{{ bucket_region }}" - prefix_in_bucket: "pageserver/v1" - safekeeper_s3_prefix: safekeeper/v1/wal - hostname_suffix: "" - remote_user: ssm-user - ansible_aws_ssm_region: us-west-2 - ansible_aws_ssm_bucket_name: neon-prod-storage-us-west-2 - console_region_id: aws-us-west-2-new - sentry_environment: production - - children: - pageservers: - hosts: - pageserver-0.us-west-2.aws.neon.tech: - ansible_host: i-0d9f6dfae0e1c780d - pageserver-1.us-west-2.aws.neon.tech: - ansible_host: i-0c834be1dddba8b3f - pageserver-2.us-west-2.aws.neon.tech: - ansible_host: i-051642d372c0a4f32 - pageserver-3.us-west-2.aws.neon.tech: - ansible_host: i-00c3844beb9ad1c6b - pageserver-4.us-west-2.aws.neon.tech: - ansible_host: i-013263dd1c239adcc - pageserver-5.us-west-2.aws.neon.tech: - ansible_host: i-00ca6417c7bf96820 - pageserver-6.us-west-2.aws.neon.tech: - ansible_host: i-01cdf7d2bc1433b6a - pageserver-7.us-west-2.aws.neon.tech: - ansible_host: i-02eec9b40617db5bc - - safekeepers: - hosts: - safekeeper-0.us-west-2.aws.neon.tech: - ansible_host: i-00719d8a74986fda6 - safekeeper-1.us-west-2.aws.neon.tech: - ansible_host: i-074682f9d3c712e7c - safekeeper-2.us-west-2.aws.neon.tech: - ansible_host: i-042b7efb1729d7966 - safekeeper-3.us-west-2.aws.neon.tech: - ansible_host: i-089f6b9ef426dff76 - safekeeper-4.us-west-2.aws.neon.tech: - ansible_host: i-0fe6bf912c4710c82 - safekeeper-5.us-west-2.aws.neon.tech: - ansible_host: i-0a83c1c46d2b4e409 - safekeeper-6.us-west-2.aws.neon.tech: - ansible_host: i-0fef5317b8fdc9f8d - safekeeper-7.us-west-2.aws.neon.tech: - ansible_host: i-0be739190d4289bf9 - safekeeper-8.us-west-2.aws.neon.tech: - ansible_host: i-00e851803669e5cfe diff --git a/.github/ansible/scripts/init_pageserver.sh b/.github/ansible/scripts/init_pageserver.sh deleted file mode 100644 index d88f754a86..0000000000 --- a/.github/ansible/scripts/init_pageserver.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -# fetch params from meta-data service -INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) -AZ_ID=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) -INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) -DISK_SIZE=$(df -B1 /storage | tail -1 | awk '{print $2}') - -# store fqdn hostname in var -HOST=$(hostname -f) - - -cat <preview-${PREVIEW_NAME}.neon-proxy-scram.yaml - helm upgrade neon-proxy-scram-${PREVIEW_NAME} neondatabase/neon-proxy --namespace neon-proxy-${PREVIEW_NAME} --create-namespace --install --atomic -f preview-${PREVIEW_NAME}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - done - - - name: Cleanup helm folder - run: rm -rf ~/.cache - - deploy-storage-broker-new: - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - if: inputs.deployStorageBroker - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: dev-us-east-2-beta - - target_region: eu-west-1 - target_cluster: dev-eu-west-1-zeta - - target_region: eu-central-1 - target_cluster: dev-eu-central-1-alpha - environment: - name: dev-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - role-to-assume: arn:aws:iam::369495373322:role/github-runner - aws-region: eu-central-1 - role-skip-session-tagging: true - role-duration-seconds: 1800 - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy storage-broker - run: - helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s - - - name: Cleanup helm folder - run: rm -rf ~/.cache - - deploy-pg-sni-router: - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - if: inputs.deployPgSniRouter - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: dev-us-east-2-beta - - target_region: eu-west-1 - target_cluster: dev-eu-west-1-zeta - - target_region: eu-central-1 - target_cluster: dev-eu-central-1-alpha - environment: - name: dev-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - role-to-assume: arn:aws:iam::369495373322:role/github-runner - aws-region: eu-central-1 - role-skip-session-tagging: true - role-duration-seconds: 1800 - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy pg-sni-router - run: - helm upgrade neon-pg-sni-router neondatabase/neon-pg-sni-router --namespace neon-pg-sni-router --create-namespace --install --debug --atomic -f .github/helm-values/${{ matrix.target_cluster }}.pg-sni-router.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 15m0s - - - name: Cleanup helm folder - run: rm -rf ~/.cache diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml deleted file mode 100644 index baa44d8094..0000000000 --- a/.github/workflows/deploy-prod.yml +++ /dev/null @@ -1,217 +0,0 @@ -name: Neon Deploy prod - -on: - workflow_dispatch: - inputs: - dockerTag: - description: 'Docker tag to deploy' - required: true - type: string - branch: - description: 'Branch or commit used for deploy scripts and configs' - required: true - type: string - default: 'release' - deployStorage: - description: 'Deploy storage' - required: true - type: boolean - default: true - deployProxy: - description: 'Deploy proxy' - required: true - type: boolean - default: true - deployStorageBroker: - description: 'Deploy storage-broker' - required: true - type: boolean - default: true - deployPgSniRouter: - description: 'Deploy pg-sni-router' - required: true - type: boolean - default: true - disclamerAcknowledged: - description: 'I confirm that there is an emergency and I can not use regular release workflow' - required: true - type: boolean - default: false - -concurrency: - group: deploy-prod - cancel-in-progress: false - -jobs: - deploy-prod-new: - runs-on: prod - container: - image: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - options: --user root --privileged - if: inputs.deployStorage && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - strategy: - matrix: - target_region: [ us-east-2, us-west-2, eu-central-1, ap-southeast-1, us-east-1 ] - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Redeploy - run: | - export DOCKER_TAG=${{ inputs.dockerTag }} - cd "$(pwd)/.github/ansible" - - ./get_binaries.sh - - ansible-galaxy collection install sivel.toiletwater - ansible-playbook -v deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_PRODUCTION_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }} - rm -f neon_install.tar.gz .neon_current_version - - deploy-proxy-prod-new: - runs-on: prod - container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployProxy && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: prod-us-east-2-delta - deploy_link_proxy: true - deploy_legacy_scram_proxy: false - - target_region: us-west-2 - target_cluster: prod-us-west-2-eta - deploy_link_proxy: false - deploy_legacy_scram_proxy: true - - target_region: eu-central-1 - target_cluster: prod-eu-central-1-gamma - deploy_link_proxy: false - deploy_legacy_scram_proxy: false - - target_region: ap-southeast-1 - target_cluster: prod-ap-southeast-1-epsilon - deploy_link_proxy: false - deploy_legacy_scram_proxy: false - - target_region: us-east-1 - target_cluster: prod-us-east-1-theta - deploy_link_proxy: false - deploy_legacy_scram_proxy: false - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Re-deploy scram proxy - run: | - DOCKER_TAG=${{ inputs.dockerTag }} - helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Re-deploy link proxy - if: matrix.deploy_link_proxy - run: | - DOCKER_TAG=${{ inputs.dockerTag }} - helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Re-deploy legacy scram proxy - if: matrix.deploy_legacy_scram_proxy - run: | - DOCKER_TAG=${{ inputs.dockerTag }} - helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - deploy-storage-broker-prod-new: - runs-on: prod - container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployStorageBroker && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: prod-us-east-2-delta - - target_region: us-west-2 - target_cluster: prod-us-west-2-eta - - target_region: eu-central-1 - target_cluster: prod-eu-central-1-gamma - - target_region: ap-southeast-1 - target_cluster: prod-ap-southeast-1-epsilon - - target_region: us-east-1 - target_cluster: prod-us-east-1-theta - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy storage-broker - run: - helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s - - deploy-pg-sni-router: - runs-on: prod - container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployPgSniRouter && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: prod-us-east-2-delta - - target_region: us-west-2 - target_cluster: prod-us-west-2-eta - - target_region: eu-central-1 - target_cluster: prod-eu-central-1-gamma - - target_region: ap-southeast-1 - target_cluster: prod-ap-southeast-1-epsilon - - target_region: us-east-1 - target_cluster: prod-us-east-1-theta - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy pg-sni-router - run: - helm upgrade neon-pg-sni-router neondatabase/neon-pg-sni-router --namespace neon-pg-sni-router --create-namespace --install --debug --atomic -f .github/helm-values/${{ matrix.target_cluster }}.pg-sni-router.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 15m0s diff --git a/Dockerfile b/Dockerfile index f83f3b1c21..7364654641 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ### The image itself is mainly used as a container for the binaries and for starting e2e tests with custom parameters. ### By default, the binaries inside the image have some mock parameters and can start, but are not intended to be used ### inside this image in the real deployments. -ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com +ARG REPOSITORY=neondatabase ARG IMAGE=rust ARG TAG=pinned diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 229e09aa98..c18470c5e2 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -1,5 +1,5 @@ ARG PG_VERSION -ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com +ARG REPOSITORY=neondatabase ARG IMAGE=rust ARG TAG=pinned @@ -393,6 +393,28 @@ RUN case "${PG_VERSION}" in \ make install -j $(getconf _NPROCESSORS_ONLN) && \ echo "trusted = true" >> /usr/local/pgsql/share/extension/pg_hint_plan.control +######################################################################################### +# +# Layer "kq-imcx-pg-build" +# compile kq_imcx extension +# +######################################################################################### +FROM build-deps AS kq-imcx-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +ENV PATH "/usr/local/pgsql/bin/:$PATH" +RUN apt-get update && \ + apt-get install -y git libgtk2.0-dev libpq-dev libpam-dev libxslt-dev libkrb5-dev cmake && \ + wget https://github.com/ketteq-neon/postgres-exts/archive/e0bd1a9d9313d7120c1b9c7bb15c48c0dede4c4e.tar.gz -O kq_imcx.tar.gz && \ + echo "dc93a97ff32d152d32737ba7e196d9687041cda15e58ab31344c2f2de8855336 kq_imcx.tar.gz" | sha256sum --check && \ + mkdir kq_imcx-src && cd kq_imcx-src && tar xvzf ../kq_imcx.tar.gz --strip-components=1 -C . && \ + mkdir build && \ + cd build && \ + cmake .. && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/kq_imcx.control + ######################################################################################### # # Layer "rust extensions" @@ -506,6 +528,7 @@ COPY --from=hll-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=plpgsql-check-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=timescaledb-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-hint-plan-pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=kq-imcx-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY pgxn/ pgxn/ RUN make -j $(getconf _NPROCESSORS_ONLN) \ diff --git a/Dockerfile.compute-tools b/Dockerfile.compute-tools index 8231cd0ebb..e86fb40ca4 100644 --- a/Dockerfile.compute-tools +++ b/Dockerfile.compute-tools @@ -1,6 +1,6 @@ # First transient image to build compute_tools binaries # NB: keep in sync with rust image version in .github/workflows/build_and_test.yml -ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com +ARG REPOSITORY=neondatabase ARG IMAGE=rust ARG TAG=pinned diff --git a/README.md b/README.md index 810937aff7..ce6ec09d24 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Neon](https://user-images.githubusercontent.com/13738772/236813940-dcfdcb5b-69d3-449b-a686-013febe834d4.png)](https://neon.tech) + # Neon Neon is a serverless open-source alternative to AWS Aurora Postgres. It separates storage and compute and substitutes the PostgreSQL storage layer by redistributing data across a cluster of nodes. diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 95f6e96a5b..877b367b40 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -340,7 +340,29 @@ paths: format: hex post: - description: Schedules attach operation to happen in the background for given tenant + description: | + Schedules attach operation to happen in the background for the given tenant. + As soon as the caller sends this request, it must assume the pageserver + starts writing to the tenant's S3 state unless it receives one of the + distinguished errors below that state otherwise. + + The method to identify whether a request has arrived at the pageserver, and + whether it has succeeded, is to poll for the tenant status to reach "Active" + or "Broken" state. These values are currently not explicitly documented in + the API spec. + Polling for `has_in_progress_downloads == false` is INCORRECT because that + value can turn `false` during shutdown while the Attach operation is still + unfinished. + + There is no way to cancel an in-flight request. + + If a client receives a not-distinguished response, e.g., a network timeout, + it MUST retry the /attach request and poll again for tenant status. + + In any case, it must + * NOT ASSUME that the /attach request has been lost in the network, + * NOT ASSUME that the request has been lost based on a subsequent + tenant status request returning 404. (The request may still be in flight!) responses: "202": description: Tenant attaching scheduled diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 3699a2a74c..fecbb8bd41 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -108,6 +108,9 @@ struct Args { /// available to the system. #[arg(long)] wal_backup_threads: Option, + /// Number of max parallel WAL segments to be offloaded to remote storage. + #[arg(long, default_value = "5")] + wal_backup_parallel_jobs: usize, /// Disable WAL backup to s3. When disabled, safekeeper removes WAL ignoring /// WAL backup horizon. #[arg(long)] @@ -182,6 +185,7 @@ fn main() -> anyhow::Result<()> { max_offloader_lag_bytes: args.max_offloader_lag, backup_runtime_threads: args.wal_backup_threads, wal_backup_enabled: !args.disable_wal_backup, + backup_parallel_jobs: args.wal_backup_parallel_jobs, auth, }; diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index ff621fdbc0..22d6d57e19 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -61,6 +61,7 @@ pub struct SafeKeeperConf { pub remote_storage: Option, pub max_offloader_lag_bytes: u64, pub backup_runtime_threads: Option, + pub backup_parallel_jobs: usize, pub wal_backup_enabled: bool, pub auth: Option>, } @@ -93,6 +94,7 @@ impl SafeKeeperConf { broker_keepalive_interval: Duration::from_secs(5), backup_runtime_threads: None, wal_backup_enabled: true, + backup_parallel_jobs: 1, auth: None, heartbeat_timeout: Duration::new(5, 0), max_offloader_lag_bytes: defaults::DEFAULT_MAX_OFFLOADER_LAG_BYTES, diff --git a/safekeeper/src/wal_backup.rs b/safekeeper/src/wal_backup.rs index 953c7d0022..4d341a7ef8 100644 --- a/safekeeper/src/wal_backup.rs +++ b/safekeeper/src/wal_backup.rs @@ -1,5 +1,7 @@ use anyhow::{Context, Result}; +use futures::stream::FuturesOrdered; +use futures::StreamExt; use tokio::task::JoinHandle; use utils::id::NodeId; @@ -155,8 +157,14 @@ async fn update_task( let timeline_dir = conf.timeline_dir(&ttid); let handle = tokio::spawn( - backup_task_main(ttid, timeline_dir, conf.workdir.clone(), shutdown_rx) - .instrument(info_span!("WAL backup task", ttid = %ttid)), + backup_task_main( + ttid, + timeline_dir, + conf.workdir.clone(), + conf.backup_parallel_jobs, + shutdown_rx, + ) + .instrument(info_span!("WAL backup task", ttid = %ttid)), ); entry.handle = Some(WalBackupTaskHandle { @@ -240,6 +248,7 @@ struct WalBackupTask { timeline_dir: PathBuf, workspace_dir: PathBuf, wal_seg_size: usize, + parallel_jobs: usize, commit_lsn_watch_rx: watch::Receiver, } @@ -248,6 +257,7 @@ async fn backup_task_main( ttid: TenantTimelineId, timeline_dir: PathBuf, workspace_dir: PathBuf, + parallel_jobs: usize, mut shutdown_rx: Receiver<()>, ) { info!("started"); @@ -264,6 +274,7 @@ async fn backup_task_main( timeline: tli, timeline_dir, workspace_dir, + parallel_jobs, }; // task is spinned up only when wal_seg_size already initialized @@ -330,6 +341,7 @@ impl WalBackupTask { self.wal_seg_size, &self.timeline_dir, &self.workspace_dir, + self.parallel_jobs, ) .await { @@ -356,20 +368,49 @@ pub async fn backup_lsn_range( wal_seg_size: usize, timeline_dir: &Path, workspace_dir: &Path, + parallel_jobs: usize, ) -> Result<()> { + if parallel_jobs < 1 { + anyhow::bail!("parallel_jobs must be >= 1"); + } + let start_lsn = *backup_lsn; let segments = get_segments(start_lsn, end_lsn, wal_seg_size); - for s in &segments { - backup_single_segment(s, timeline_dir, workspace_dir) - .await - .with_context(|| format!("offloading segno {}", s.seg_no))?; - let new_backup_lsn = s.end_lsn; - timeline - .set_wal_backup_lsn(new_backup_lsn) - .context("setting wal_backup_lsn")?; - *backup_lsn = new_backup_lsn; + // Pool of concurrent upload tasks. We use `FuturesOrdered` to + // preserve order of uploads, and update `backup_lsn` only after + // all previous uploads are finished. + let mut uploads = FuturesOrdered::new(); + let mut iter = segments.iter(); + + loop { + let added_task = match iter.next() { + Some(s) => { + uploads.push_back(backup_single_segment(s, timeline_dir, workspace_dir)); + true + } + None => false, + }; + + // Wait for the next segment to upload if we don't have any more segments, + // or if we have too many concurrent uploads. + if !added_task || uploads.len() >= parallel_jobs { + let next = uploads.next().await; + if let Some(res) = next { + // next segment uploaded + let segment = res?; + let new_backup_lsn = segment.end_lsn; + timeline + .set_wal_backup_lsn(new_backup_lsn) + .context("setting wal_backup_lsn")?; + *backup_lsn = new_backup_lsn; + } else { + // no more segments to upload + break; + } + } } + info!( "offloaded segnos {:?} up to {}, previous backup_lsn {}", segments.iter().map(|&s| s.seg_no).collect::>(), @@ -383,7 +424,7 @@ async fn backup_single_segment( seg: &Segment, timeline_dir: &Path, workspace_dir: &Path, -) -> Result<()> { +) -> Result { let segment_file_path = seg.file_path(timeline_dir)?; let remote_segment_path = segment_file_path .strip_prefix(workspace_dir) @@ -404,7 +445,7 @@ async fn backup_single_segment( res?; debug!("Backup of {} done", segment_file_path.display()); - Ok(()) + Ok(*seg) } #[derive(Debug, Copy, Clone)] diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index aee72b7be9..2df2ce3744 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit aee72b7be903e52d9bdc6449aa4c17fb852d8708 +Subproject commit 2df2ce374464a7449e15dfa46c956b73b4f4098b