scrabble-game/.gitea/workflows/prod-rollback.yaml

# Manual production rollback. Runs ONLY from master, ONLY on workflow_dispatch with
# confirm=rollback. Re-deploys an already-published image tag (no build): leave
# target_version blank to roll back to the previously deployed version (read from the
# main host), or set it to a specific release tag from the Releases page. The
# re-deploy is the same rolling, health-gated path as prod-deploy (TAG=target,
# MIGRATION=0 — rollback is image-only and never migrates the DB; image rollback is
# DB-safe under the expand-contract rule). See deploy/README.md (prod runbook).
name: prod-rollback
run-name: "prod rollback ${{ inputs.target_version || 'previous' }}"

on:
  workflow_dispatch:
    inputs:
      confirm:
        description: 'Type "rollback" to confirm a production rollback.'
        required: true
        default: ""
      target_version:
        description: "Release tag to roll back to (blank = the previous deployed version)."
        required: false
        default: ""

permissions:
  contents: read

env:
  NO_COLOR: "1"
  DOCKER_CLI_HINTS: "false"
  REGISTRY: docker.iliadenisov.ru/developer

jobs:
  rollback-main:
    if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'rollback' }}
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    outputs:
      target: ${{ steps.resolve.outputs.target }}
    env:
      PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
      PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
      PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
      MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
      POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }}
      GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }}
      GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }}
      TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
      PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
      PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }}
      PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }}
      GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }}
      CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
      LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }}
      DICT_VERSION: ${{ vars.DICT_VERSION }}
      POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }}
      POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }}
      PUBLIC_BASE_URL: ${{ vars.PROD_PUBLIC_BASE_URL }}
      # Full runtime env — parity with prod-deploy's deploy-main so a rollback re-renders
      # the SAME env.sh (email / VK login / Grafana alerts survive a rollback). TELEGRAM_MINIAPP_URL
      # and GRAFANA_ROOT_URL are derived from PUBLIC_BASE_URL in deploy/write-prod-env.sh.
      GATEWAY_VK_APP_SECRET: ${{ secrets.GATEWAY_VK_APP_SECRET }}
      # YooKassa direct rail: the rollback re-renders the same runtime env (write-prod-env.sh), so
      # it must carry the same credentials or the direct rail goes dark after a rollback.
      YOOKASSA_WEB_SHOP_ID: ${{ secrets.PROD_BACKEND_YOOKASSA_WEB_SHOP_ID }}
      YOOKASSA_WEB_SECRET_KEY: ${{ secrets.PROD_BACKEND_YOOKASSA_WEB_SECRET_KEY }}
      YOOKASSA_WEB_TEST: ${{ vars.PROD_BACKEND_YOOKASSA_WEB_TEST }}
      YOOKASSA_ANDROID_SHOP_ID: ${{ secrets.PROD_BACKEND_YOOKASSA_ANDROID_SHOP_ID }}
      YOOKASSA_ANDROID_SECRET_KEY: ${{ secrets.PROD_BACKEND_YOOKASSA_ANDROID_SECRET_KEY }}
      YOOKASSA_ANDROID_TEST: ${{ vars.PROD_BACKEND_YOOKASSA_ANDROID_TEST }}
      YOOKASSA_VAT_CODE: ${{ vars.PROD_BACKEND_YOOKASSA_VAT_CODE }}
      VITE_VK_APP_ID: ${{ vars.VITE_VK_APP_ID }}
      GATEWAY_VK_ID_CLIENT_SECRET: ${{ secrets.GATEWAY_VK_ID_CLIENT_SECRET }}
      GATEWAY_HONEYTOKEN: ${{ secrets.PROD_GATEWAY_HONEYTOKEN }}
      # Community IP blocklist — rendered on rollback too so a rollback keeps the same edge policy.
      GATEWAY_BLOCKLIST_ENABLED: ${{ vars.PROD_GATEWAY_BLOCKLIST_ENABLED }}
      GATEWAY_BLOCKLIST_URL: ${{ vars.PROD_GATEWAY_BLOCKLIST_URL }}
      GATEWAY_BLOCKLIST_ALLOW: ${{ vars.PROD_GATEWAY_BLOCKLIST_ALLOW }}
      EXPORT_SIGN_KEY: ${{ secrets.PROD_EXPORT_SIGN_KEY }}
      SMTP_RELAY_USER: ${{ secrets.SMTP_RELAY_USER }}
      SMTP_RELAY_PASS: ${{ secrets.SMTP_RELAY_PASS }}
      SMTP_RELAY_HOST: ${{ vars.SMTP_RELAY_HOST }}
      SMTP_RELAY_PORT: ${{ vars.SMTP_RELAY_PORT }}
      SMTP_RELAY_TLS: ${{ vars.SMTP_RELAY_TLS }}
      SMTP_RELAY_FROM: ${{ vars.PROD_SMTP_RELAY_FROM }}
      SMTP_RELAY_ADMIN_FROM: ${{ vars.PROD_SMTP_RELAY_ADMIN_FROM }}
      ADMIN_EMAIL: ${{ vars.PROD_ADMIN_EMAIL }}
      SMTP_RELAY_SERVICE_FROM: ${{ vars.PROD_SMTP_RELAY_SERVICE_FROM }}
      SERVICE_EMAIL: ${{ vars.PROD_SERVICE_EMAIL }}
      GRAFANA_SMTP_PORT: ${{ vars.GRAFANA_SMTP_PORT }}
      GF_SMTP_ENABLED: ${{ vars.PROD_GF_SMTP_ENABLED }}
      # PITR archiving parity: a rollback must re-render the SAME env.sh, else it would
      # silently disarm WAL archiving (PGBACKREST_ARCHIVE_MODE would fall back to off).
      PGBACKREST_ARCHIVE_MODE: ${{ vars.PROD_PGBACKREST_ARCHIVE_MODE }}
      PGBACKREST_S3_ENDPOINT: ${{ vars.PROD_PGBACKREST_S3_ENDPOINT }}
      PGBACKREST_S3_PORT: ${{ vars.PROD_PGBACKREST_S3_PORT }}
      PGBACKREST_S3_BUCKET: ${{ vars.PROD_PGBACKREST_S3_BUCKET }}
      PGBACKREST_S3_REGION: ${{ vars.PROD_PGBACKREST_S3_REGION }}
      PGBACKREST_S3_KEY: ${{ secrets.PROD_PGBACKREST_S3_KEY }}
      PGBACKREST_S3_KEY_SECRET: ${{ secrets.PROD_PGBACKREST_S3_KEY_SECRET }}
      PGBACKREST_CIPHER_PASS: ${{ secrets.PROD_PGBACKREST_CIPHER_PASS }}
      INPUT_TARGET: ${{ inputs.target_version }}
    steps:
      - uses: actions/checkout@v4
      - name: Set up SSH
        run: |
          mkdir -p ~/.ssh && chmod 700 ~/.ssh
          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
          printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
      - name: Resolve rollback target
        id: resolve
        run: |
          ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
          CURRENT="$(ssh_main 'cat /opt/scrabble/DEPLOYED_TAG 2>/dev/null || echo none')"
          if [ -n "$INPUT_TARGET" ]; then
            TARGET="$INPUT_TARGET"
          else
            TARGET="$(ssh_main 'cat /opt/scrabble/PREVIOUS_TAG 2>/dev/null || echo none')"
          fi
          if [ -z "$TARGET" ] || [ "$TARGET" = none ]; then
            echo "no rollback target (no PREVIOUS_TAG on the host and no target_version input)"; exit 1
          fi
          if [ "$TARGET" = "$CURRENT" ]; then
            echo "target $TARGET is already the deployed version; nothing to do"; exit 1
          fi
          echo "rolling back: current=$CURRENT -> target=$TARGET"
          echo "target=$TARGET" >> "$GITHUB_OUTPUT"
          { echo "TARGET=$TARGET"; echo "CURRENT=$CURRENT"; } >> "$GITHUB_ENV"
      - name: Render main env + certs
        run: |
          umask 077
          mkdir -p stage/certs-main
          # Same writer as prod-deploy's deploy-main -> the rollback re-renders the FULL
          # runtime env (not a subset), so email / VK login / Grafana alerts survive it.
          APP_VERSION="$TARGET" bash deploy/write-prod-env.sh stage/env.sh
          printf '%s\n' "$PROD_BOTLINK_CA"           > stage/certs-main/ca.crt
          printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt
          printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY"  > stage/certs-main/gateway.key
          chmod 644 stage/certs-main/*
      - name: Roll the main host back
        run: |
          ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
          ssh_main 'mkdir -p /opt/scrabble/compose'
          tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \
            | ssh_main 'tar -C /opt/scrabble/compose -xzf -'
          tar -C deploy -czf - caddy otelcol prometheus tempo grafana \
            | ssh_main 'tar -C /opt/scrabble -xzf -'
          tar -C stage -czf - certs-main \
            | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'
          scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh"
          echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin"
          # Image-only rollback: no migration window (TAG=target, MIGRATION=0). A failed
          # rollback's auto-revert returns to the current version (PREV_TAG=$CURRENT).
          ssh_main "TAG='$TARGET' PREV_TAG='$CURRENT' MIGRATION=0 bash /opt/scrabble/compose/prod-deploy.sh"

  rollback-bot:
    needs: rollback-main
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    env:
      TARGET: ${{ needs.rollback-main.outputs.target }}
      PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
      PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
      PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
      TG_HOST: ${{ vars.PROD_TG_HOST }}
      MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
      TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
      TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }}
      PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
      PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }}
      PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }}
      LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }}
      # PUBLIC_BASE_URL drives the derived Mini App URL; SUPPORT_CHAT_ID for parity with deploy.
      PUBLIC_BASE_URL: ${{ vars.PROD_PUBLIC_BASE_URL }}
      TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }}
      TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }}
      TELEGRAM_SUPPORT_CHAT_ID: ${{ vars.PROD_TELEGRAM_SUPPORT_CHAT_ID }}
      TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }}
      TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
    steps:
      - uses: actions/checkout@v4
      - name: Set up SSH
        run: |
          mkdir -p ~/.ssh && chmod 700 ~/.ssh
          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
          printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
      - name: Render bot env + certs
        run: |
          umask 077
          mkdir -p stage/certs-bot
          # Same writer as prod-deploy's deploy-bot (parity; includes TELEGRAM_SUPPORT_CHAT_ID).
          BOT_IMAGE="$REGISTRY/scrabble-telegram-bot:$TARGET" bash deploy/write-prod-bot-env.sh stage/env.bot.sh
          printf '%s\n' "$PROD_BOTLINK_CA"       > stage/certs-bot/ca.crt
          printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt
          printf '%s\n' "$PROD_BOTLINK_BOT_KEY"  > stage/certs-bot/bot.key
          chmod 644 stage/certs-bot/*
      - name: Roll the bot host back
        run: |
          ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; }
          ssh_tg 'mkdir -p /opt/scrabble/compose'
          tar -C deploy -czf - docker-compose.bot.yml | ssh_tg 'tar -C /opt/scrabble/compose -xzf -'
          tar -C stage -czf - certs-bot \
            | ssh_tg 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'
          scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.bot.sh "deploy@$TG_HOST:/opt/scrabble/env.bot.sh"
          echo "$PROD_REGISTRY_PASSWORD" | ssh_tg "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin"
          ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose;
            docker compose -f docker-compose.bot.yml pull;
            docker compose -f docker-compose.bot.yml up -d'

  verify:
    needs: [rollback-main, rollback-bot]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    env:
      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
      PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
      MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
      CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
    steps:
      - name: Set up SSH
        run: |
          mkdir -p ~/.ssh && chmod 700 ~/.ssh
          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
          printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
      - name: Verify the public site
        run: |
          domain="${CADDY_SITE_ADDRESS%% *}"
          ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "for i in \$(seq 1 20); do
              if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null &&
                 docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then
                echo 'rolled-back site healthy'; exit 0
              fi
              sleep 5
            done
            echo 'verify failed'; docker logs --tail 40 scrabble-caddy; docker logs --tail 40 scrabble-backend; exit 1"