diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index de2858b..7c1a784 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -150,7 +150,16 @@ jobs: mkdir -p ~/.ssh echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/deploy_key chmod 600 ~/.ssh/deploy_key - ssh-keyscan -H "${{ steps.vps.outputs.ip }}" >> ~/.ssh/known_hosts 2>/dev/null || true + for i in 1 2 3; do + if ssh-keyscan -H "${{ steps.vps.outputs.ip }}" >> ~/.ssh/known_hosts 2>/dev/null; then + echo "ssh-keyscan succeeded on attempt $i" + exit 0 + fi + echo "ssh-keyscan attempt $i failed, retrying in 10s..." + sleep 10 + done + echo "ssh-keyscan failed after 3 attempts" + exit 1 # ---- Detect first deployment ---- - name: Detect first deployment diff --git a/deploy/ansible/roles/app/handlers/main.yml b/deploy/ansible/roles/app/handlers/main.yml index 27608ce..f4e9fe2 100644 --- a/deploy/ansible/roles/app/handlers/main.yml +++ b/deploy/ansible/roles/app/handlers/main.yml @@ -1,5 +1,5 @@ --- - name: Reload haproxy ansible.builtin.shell: | - docker compose -f /opt/rustyip/docker-compose.yml kill -s HUP haproxy + docker kill -s HUP rustyip-haproxy failed_when: false diff --git a/deploy/ansible/roles/app/tasks/main.yml b/deploy/ansible/roles/app/tasks/main.yml index 92d69fc..6a554e5 100644 --- a/deploy/ansible/roles/app/tasks/main.yml +++ b/deploy/ansible/roles/app/tasks/main.yml @@ -78,7 +78,7 @@ is_first_deploy: true when: not active_color_stat.stat.exists -# --- Deploy docker-compose and HAProxy config --- +# --- Deploy docker-compose.yml (shared across both paths) --- - name: Deploy docker-compose.yml ansible.builtin.template: @@ -88,7 +88,10 @@ group: root mode: "0644" -- name: Deploy HAProxy configuration (target color active) +# --- First Deployment --- +# On first deploy, HAProxy config can be written early since nothing is live yet. + +- name: Deploy HAProxy configuration (first deploy) ansible.builtin.template: src: haproxy.cfg.j2 dest: /opt/rustyip/haproxy/haproxy.cfg @@ -97,10 +100,9 @@ mode: "0644" vars: active_color: "{{ deploy_target }}" + when: is_first_deploy -# --- First Deployment --- - -- name: Start target app container +- name: Start target app container (first deploy) ansible.builtin.shell: | docker compose -f /opt/rustyip/docker-compose.yml up -d "app-{{ deploy_target }}" changed_when: true @@ -137,55 +139,92 @@ when: is_first_deploy # --- Blue-Green Deployment --- +# The HAProxy config is written AFTER the standby container passes its health check, +# so that on-disk config always reflects the actual healthy state. -- name: Start standby container with new image - ansible.builtin.shell: | - docker compose -f /opt/rustyip/docker-compose.yml up -d "app-{{ deploy_target }}" - changed_when: true +- name: Blue-green deployment when: not is_first_deploy + block: + - name: Start standby container with new image + ansible.builtin.shell: | + docker compose -f /opt/rustyip/docker-compose.yml up -d "app-{{ deploy_target }}" + changed_when: true -- name: Wait for standby container health check - ansible.builtin.shell: | - for i in $(seq 1 30); do - status=$(docker inspect --format='{% raw %}{{.State.Health.Status}}{% endraw %}' "rustyip-app-{{ deploy_target }}" 2>/dev/null) - if [ "$status" = "healthy" ]; then - echo "healthy" - exit 0 - fi - sleep 2 - done - echo "timeout" - exit 1 - register: health_result - changed_when: false - when: not is_first_deploy + - name: Wait for standby container health check + ansible.builtin.shell: | + for i in $(seq 1 30); do + status=$(docker inspect --format='{% raw %}{{.State.Health.Status}}{% endraw %}' "rustyip-app-{{ deploy_target }}" 2>/dev/null) + if [ "$status" = "healthy" ]; then + echo "healthy" + exit 0 + fi + sleep 2 + done + echo "timeout" + exit 1 + changed_when: false -- name: Reload HAProxy to switch to new backend - ansible.builtin.shell: | - docker compose -f /opt/rustyip/docker-compose.yml up -d haproxy - changed_when: true - when: not is_first_deploy + - name: Deploy HAProxy configuration (switch to new backend) + ansible.builtin.template: + src: haproxy.cfg.j2 + dest: /opt/rustyip/haproxy/haproxy.cfg + owner: root + group: root + mode: "0644" + vars: + active_color: "{{ deploy_target }}" -- name: Wait for HAProxy to settle - ansible.builtin.pause: - seconds: 5 - when: not is_first_deploy + - name: Reload HAProxy in-place to switch traffic + ansible.builtin.shell: | + docker kill -s HUP rustyip-haproxy + changed_when: true -- name: Stop old active container - ansible.builtin.shell: | - docker compose -f /opt/rustyip/docker-compose.yml stop "app-{{ current_active }}" - docker compose -f /opt/rustyip/docker-compose.yml rm -f "app-{{ current_active }}" - changed_when: true - when: not is_first_deploy + - name: Wait for HAProxy to settle + ansible.builtin.pause: + seconds: 5 -- name: Update active color state file - ansible.builtin.copy: - content: "{{ deploy_target }}" - dest: /opt/rustyip/.active_color - owner: root - group: root - mode: "0644" - when: not is_first_deploy + - name: Stop old active container + ansible.builtin.shell: | + docker compose -f /opt/rustyip/docker-compose.yml stop "app-{{ current_active }}" + docker compose -f /opt/rustyip/docker-compose.yml rm -f "app-{{ current_active }}" + changed_when: true + + - name: Update active color state file + ansible.builtin.copy: + content: "{{ deploy_target }}" + dest: /opt/rustyip/.active_color + owner: root + group: root + mode: "0644" + + rescue: + - name: Stop failed standby container + ansible.builtin.shell: | + docker compose -f /opt/rustyip/docker-compose.yml stop "app-{{ deploy_target }}" || true + docker compose -f /opt/rustyip/docker-compose.yml rm -f "app-{{ deploy_target }}" || true + changed_when: true + + - name: Restore HAProxy configuration to current active color + ansible.builtin.template: + src: haproxy.cfg.j2 + dest: /opt/rustyip/haproxy/haproxy.cfg + owner: root + group: root + mode: "0644" + vars: + active_color: "{{ current_active }}" + + - name: Reload HAProxy to restore original backend + ansible.builtin.shell: | + docker kill -s HUP rustyip-haproxy + changed_when: true + failed_when: false + + - name: Fail with deployment error + ansible.builtin.fail: + msg: >- + Blue-green deployment failed. Rolled back to app-{{ current_active }}. + The failed standby container (app-{{ deploy_target }}) has been removed. - name: Clean up unused Docker images ansible.builtin.shell: | diff --git a/deploy/ansible/roles/app/templates/docker-compose.yml.j2 b/deploy/ansible/roles/app/templates/docker-compose.yml.j2 index 0ff402e..186aa35 100644 --- a/deploy/ansible/roles/app/templates/docker-compose.yml.j2 +++ b/deploy/ansible/roles/app/templates/docker-compose.yml.j2 @@ -23,7 +23,7 @@ services: container_name: rustyip-app-blue env_file: .env volumes: - - db_data:/data + - db_data_blue:/data healthcheck: test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3000/health || exit 1"] interval: 5s @@ -43,7 +43,7 @@ services: container_name: rustyip-app-green env_file: .env volumes: - - db_data:/data + - db_data_green:/data healthcheck: test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3000/health || exit 1"] interval: 5s @@ -59,7 +59,8 @@ services: - internal volumes: - db_data: + db_data_blue: + db_data_green: networks: internal: