From cbe155ff48521707be24f5b4ffdb1026fb98e332 Mon Sep 17 00:00:00 2001 From: Andrey Taranik Date: Thu, 16 Dec 2021 17:05:20 +0300 Subject: [PATCH] storage CI flow for staging environment (#1003) * storage CI flow for staging environment * prevent deploy version older than already deployed --- .circleci/config.yml | 51 +++++++++ .circleci/storage-redeploy.playbook.yml | 138 ++++++++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 .circleci/storage-redeploy.playbook.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index fc0c8793b0..375bf50229 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -454,6 +454,47 @@ jobs: docker build -t zenithdb/compute-node:latest vendor/postgres && docker push zenithdb/compute-node:latest docker tag zenithdb/compute-node:latest zenithdb/compute-node:${DOCKER_TAG} && docker push zenithdb/compute-node:${DOCKER_TAG} + deploy-staging: + docker: + - image: cimg/python:3.10 + steps: + - checkout + - setup_remote_docker + - run: + name: Get Zenith binaries + command: | + rm -rf zenith_install postgres_install.tar.gz zenith_install.tar.gz + mkdir zenith_install + DOCKER_TAG=$(git log --oneline|wc -l) + docker pull --quiet zenithdb/zenith:${DOCKER_TAG} + ID=$(docker create zenithdb/zenith:${DOCKER_TAG}) + docker cp $ID:/data/postgres_install.tar.gz . + tar -xzf postgres_install.tar.gz -C zenith_install && rm postgres_install.tar.gz + docker cp $ID:/usr/local/bin/pageserver zenith_install/bin/ + docker cp $ID:/usr/local/bin/safekeeper zenith_install/bin/ + docker cp $ID:/usr/local/bin/proxy zenith_install/bin/ + docker cp $ID:/usr/local/bin/postgres zenith_install/bin/ + docker rm -v $ID + echo ${DOCKER_TAG} | tee zenith_install/.zenith_current_version + tar -czf zenith_install.tar.gz -C zenith_install . + ls -la zenith_install.tar.gz + - run: + name: Setup ansible + command: | + pip install --progress-bar off --user ansible boto + ansible-galaxy collection install amazon.aws + - run: + name: Apply re-deploy playbook + environment: + ANSIBLE_HOST_KEY_CHECKING: false + command: | + echo "${STAGING_SSH_KEY}" | base64 --decode | ssh-add - + export AWS_REGION=${STAGING_AWS_REGION} + export AWS_ACCESS_KEY_ID=${STAGING_AWS_ACCESS_KEY_ID} + export AWS_SECRET_ACCESS_KEY=${STAGING_AWS_SECRET_ACCESS_KEY} + ansible-playbook .circleci/storage-redeploy.playbook.yml + rm -f zenith_install.tar.gz + # Trigger a new remote CI job remote-ci-trigger: docker: @@ -568,6 +609,16 @@ workflows: requires: - pg_regress-tests-release - other-tests-release + - deploy-staging: + # Context gives an ability to login + context: Docker Hub + # Build image only for commits to main + filters: + branches: + only: + - main + requires: + - docker-image - remote-ci-trigger: # Context passes credentials for gh api context: CI_ACCESS_TOKEN diff --git a/.circleci/storage-redeploy.playbook.yml b/.circleci/storage-redeploy.playbook.yml new file mode 100644 index 0000000000..ba6be34371 --- /dev/null +++ b/.circleci/storage-redeploy.playbook.yml @@ -0,0 +1,138 @@ +- name: discover storage nodes + hosts: localhost + connection: local + gather_facts: False + + tasks: + + - name: discover safekeepers + no_log: true + ec2_instance_info: + filters: + "tag:zenith_env": "staging" + "tag:zenith_service": "safekeeper" + register: ec2_safekeepers + + - name: discover pageservers + no_log: true + ec2_instance_info: + filters: + "tag:zenith_env": "staging" + "tag:zenith_service": "pageserver" + register: ec2_pageservers + + - name: add safekeepers to host group + no_log: true + add_host: + name: safekeeper-{{ ansible_loop.index }} + ansible_host: "{{ item.public_ip_address }}" + groups: + - storage + - safekeepers + with_items: "{{ ec2_safekeepers.instances }}" + loop_control: + extended: yes + + - name: add pageservers to host group + no_log: true + add_host: + name: pageserver-{{ ansible_loop.index }} + ansible_host: "{{ item.public_ip_address }}" + groups: + - storage + - pageservers + with_items: "{{ ec2_pageservers.instances }}" + loop_control: + extended: yes + +- name: Retrive versions + hosts: storage + gather_facts: False + remote_user: admin + + tasks: + + - name: Get current version of binaries + set_fact: + current_version: "{{lookup('file', '../zenith_install/.zenith_current_version') }}" + + - name: Check that file with version exists on host + stat: + path: /usr/local/.zenith_current_version + register: version_file + + - name: Try to get current version from the host + when: version_file.stat.exists + ansible.builtin.fetch: + src: /usr/local/.zenith_current_version + dest: .remote_version.{{ inventory_hostname }} + fail_on_missing: no + flat: yes + + - name: Store remote version to variable + when: version_file.stat.exists + set_fact: + remote_version: "{{ lookup('file', '.remote_version.{{ inventory_hostname }}') }}" + + - name: Store default value of remote version to variable in case when remote version file not found + when: not version_file.stat.exists + set_fact: + remote_version: "000" + +- name: Extract Zenith binaries + hosts: storage + gather_facts: False + remote_user: admin + + tasks: + + - name: Skip binaries upload on version conflict + when: current_version <= remote_version + debug: msg="Current version {{ current_version }} LE than remote {{ remote_version }}" + + - name: Extract Zenith binaries to /usr/local + when: current_version > remote_version + ansible.builtin.unarchive: + src: ../zenith_install.tar.gz + dest: /usr/local + become: true + +- name: Restart safekeepers + hosts: safekeepers + gather_facts: False + remote_user: admin + + tasks: + + - name: Skip restart on version conflict + when: current_version <= remote_version + debug: msg="Current version {{ current_version }} LE than remote {{ remote_version }}" + + - name: Restart systemd service + when: current_version > remote_version + ansible.builtin.systemd: + daemon_reload: yes + name: safekeeper + enabled: yes + state: restarted + become: true + +- name: Restart pageservers + hosts: pageservers + gather_facts: False + remote_user: admin + + tasks: + + - name: Skip restart on version conflict + when: current_version <= remote_version + debug: msg="Current version {{ current_version }} LE than remote {{ remote_version }}" + + - name: Restart systemd service + when: current_version > remote_version + ansible.builtin.systemd: + daemon_reload: yes + name: pageserver + enabled: yes + state: restarted + become: true