# Manual production rollback. Runs ONLY from master, ONLY on workflow_dispatch with # confirm=rollback. Re-deploys an already-published image tag (no build): leave # target_version blank to roll back to the previously deployed version (read from the # main host), or set it to a specific release tag from the Releases page. The # re-deploy is the same rolling, health-gated path as prod-deploy (TAG=target, # MIGRATION=0 — rollback is image-only and never migrates the DB; image rollback is # DB-safe under the expand-contract rule). See deploy/README.md (prod runbook). name: prod-rollback run-name: "prod rollback ${{ inputs.target_version || 'previous' }}" on: workflow_dispatch: inputs: confirm: description: 'Type "rollback" to confirm a production rollback.' required: true default: "" target_version: description: "Release tag to roll back to (blank = the previous deployed version)." required: false default: "" permissions: contents: read env: NO_COLOR: "1" DOCKER_CLI_HINTS: "false" REGISTRY: docker.iliadenisov.ru/developer jobs: rollback-main: if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'rollback' }} runs-on: ubuntu-latest defaults: run: shell: bash outputs: target: ${{ steps.resolve.outputs.target }} env: PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }} GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }} GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }} TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }} PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }} PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }} PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }} GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }} CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }} LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }} DICT_VERSION: ${{ vars.DICT_VERSION }} POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }} POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }} PUBLIC_BASE_URL: ${{ vars.PROD_PUBLIC_BASE_URL }} # Full runtime env — parity with prod-deploy's deploy-main so a rollback re-renders # the SAME env.sh (email / VK login / Grafana alerts survive a rollback). TELEGRAM_MINIAPP_URL # and GRAFANA_ROOT_URL are derived from PUBLIC_BASE_URL in deploy/write-prod-env.sh. GATEWAY_VK_APP_SECRET: ${{ secrets.GATEWAY_VK_APP_SECRET }} # Robokassa direct-rail: the rollback re-renders the same runtime env (write-prod-env.sh), so # it must carry the same credentials or the direct rail goes dark after a rollback. ROBOKASSA_MERCHANT_LOGIN: ${{ secrets.PROD_BACKEND_ROBOKASSA_MERCHANT_LOGIN }} ROBOKASSA_PASSWORD1: ${{ secrets.PROD_BACKEND_ROBOKASSA_PASSWORD1 }} ROBOKASSA_PASSWORD2: ${{ secrets.PROD_BACKEND_ROBOKASSA_PASSWORD2 }} ROBOKASSA_TEST: ${{ vars.PROD_BACKEND_ROBOKASSA_TEST }} VITE_VK_APP_ID: ${{ vars.VITE_VK_APP_ID }} GATEWAY_VK_ID_CLIENT_SECRET: ${{ secrets.GATEWAY_VK_ID_CLIENT_SECRET }} GATEWAY_HONEYTOKEN: ${{ secrets.PROD_GATEWAY_HONEYTOKEN }} EXPORT_SIGN_KEY: ${{ secrets.PROD_EXPORT_SIGN_KEY }} SMTP_RELAY_USER: ${{ secrets.SMTP_RELAY_USER }} SMTP_RELAY_PASS: ${{ secrets.SMTP_RELAY_PASS }} SMTP_RELAY_HOST: ${{ vars.SMTP_RELAY_HOST }} SMTP_RELAY_PORT: ${{ vars.SMTP_RELAY_PORT }} SMTP_RELAY_TLS: ${{ vars.SMTP_RELAY_TLS }} SMTP_RELAY_FROM: ${{ vars.PROD_SMTP_RELAY_FROM }} SMTP_RELAY_ADMIN_FROM: ${{ vars.PROD_SMTP_RELAY_ADMIN_FROM }} ADMIN_EMAIL: ${{ vars.PROD_ADMIN_EMAIL }} SMTP_RELAY_SERVICE_FROM: ${{ vars.PROD_SMTP_RELAY_SERVICE_FROM }} SERVICE_EMAIL: ${{ vars.PROD_SERVICE_EMAIL }} GRAFANA_SMTP_PORT: ${{ vars.GRAFANA_SMTP_PORT }} GF_SMTP_ENABLED: ${{ vars.PROD_GF_SMTP_ENABLED }} # PITR archiving parity: a rollback must re-render the SAME env.sh, else it would # silently disarm WAL archiving (PGBACKREST_ARCHIVE_MODE would fall back to off). PGBACKREST_ARCHIVE_MODE: ${{ vars.PROD_PGBACKREST_ARCHIVE_MODE }} PGBACKREST_S3_ENDPOINT: ${{ vars.PROD_PGBACKREST_S3_ENDPOINT }} PGBACKREST_S3_PORT: ${{ vars.PROD_PGBACKREST_S3_PORT }} PGBACKREST_S3_BUCKET: ${{ vars.PROD_PGBACKREST_S3_BUCKET }} PGBACKREST_S3_REGION: ${{ vars.PROD_PGBACKREST_S3_REGION }} PGBACKREST_S3_KEY: ${{ secrets.PROD_PGBACKREST_S3_KEY }} PGBACKREST_S3_KEY_SECRET: ${{ secrets.PROD_PGBACKREST_S3_KEY_SECRET }} PGBACKREST_CIPHER_PASS: ${{ secrets.PROD_PGBACKREST_CIPHER_PASS }} INPUT_TARGET: ${{ inputs.target_version }} steps: - uses: actions/checkout@v4 - name: Set up SSH run: | mkdir -p ~/.ssh && chmod 700 ~/.ssh printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - name: Resolve rollback target id: resolve run: | ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } CURRENT="$(ssh_main 'cat /opt/scrabble/DEPLOYED_TAG 2>/dev/null || echo none')" if [ -n "$INPUT_TARGET" ]; then TARGET="$INPUT_TARGET" else TARGET="$(ssh_main 'cat /opt/scrabble/PREVIOUS_TAG 2>/dev/null || echo none')" fi if [ -z "$TARGET" ] || [ "$TARGET" = none ]; then echo "no rollback target (no PREVIOUS_TAG on the host and no target_version input)"; exit 1 fi if [ "$TARGET" = "$CURRENT" ]; then echo "target $TARGET is already the deployed version; nothing to do"; exit 1 fi echo "rolling back: current=$CURRENT -> target=$TARGET" echo "target=$TARGET" >> "$GITHUB_OUTPUT" { echo "TARGET=$TARGET"; echo "CURRENT=$CURRENT"; } >> "$GITHUB_ENV" - name: Render main env + certs run: | umask 077 mkdir -p stage/certs-main # Same writer as prod-deploy's deploy-main -> the rollback re-renders the FULL # runtime env (not a subset), so email / VK login / Grafana alerts survive it. APP_VERSION="$TARGET" bash deploy/write-prod-env.sh stage/env.sh printf '%s\n' "$PROD_BOTLINK_CA" > stage/certs-main/ca.crt printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY" > stage/certs-main/gateway.key chmod 644 stage/certs-main/* - name: Roll the main host back run: | ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } ssh_main 'mkdir -p /opt/scrabble/compose' tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \ | ssh_main 'tar -C /opt/scrabble/compose -xzf -' tar -C deploy -czf - caddy otelcol prometheus tempo grafana \ | ssh_main 'tar -C /opt/scrabble -xzf -' tar -C stage -czf - certs-main \ | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -' scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh" echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin" # Image-only rollback: no migration window (TAG=target, MIGRATION=0). A failed # rollback's auto-revert returns to the current version (PREV_TAG=$CURRENT). ssh_main "TAG='$TARGET' PREV_TAG='$CURRENT' MIGRATION=0 bash /opt/scrabble/compose/prod-deploy.sh" rollback-bot: needs: rollback-main runs-on: ubuntu-latest defaults: run: shell: bash env: TARGET: ${{ needs.rollback-main.outputs.target }} PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} TG_HOST: ${{ vars.PROD_TG_HOST }} MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }} TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }} PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }} PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }} PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }} LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }} # PUBLIC_BASE_URL drives the derived Mini App URL; SUPPORT_CHAT_ID for parity with deploy. PUBLIC_BASE_URL: ${{ vars.PROD_PUBLIC_BASE_URL }} TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }} TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }} TELEGRAM_SUPPORT_CHAT_ID: ${{ vars.PROD_TELEGRAM_SUPPORT_CHAT_ID }} TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }} TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }} steps: - uses: actions/checkout@v4 - name: Set up SSH run: | mkdir -p ~/.ssh && chmod 700 ~/.ssh printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - name: Render bot env + certs run: | umask 077 mkdir -p stage/certs-bot # Same writer as prod-deploy's deploy-bot (parity; includes TELEGRAM_SUPPORT_CHAT_ID). BOT_IMAGE="$REGISTRY/scrabble-telegram-bot:$TARGET" bash deploy/write-prod-bot-env.sh stage/env.bot.sh printf '%s\n' "$PROD_BOTLINK_CA" > stage/certs-bot/ca.crt printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt printf '%s\n' "$PROD_BOTLINK_BOT_KEY" > stage/certs-bot/bot.key chmod 644 stage/certs-bot/* - name: Roll the bot host back run: | ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; } ssh_tg 'mkdir -p /opt/scrabble/compose' tar -C deploy -czf - docker-compose.bot.yml | ssh_tg 'tar -C /opt/scrabble/compose -xzf -' tar -C stage -czf - certs-bot \ | ssh_tg 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -' scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.bot.sh "deploy@$TG_HOST:/opt/scrabble/env.bot.sh" echo "$PROD_REGISTRY_PASSWORD" | ssh_tg "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin" ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose; docker compose -f docker-compose.bot.yml pull; docker compose -f docker-compose.bot.yml up -d' verify: needs: [rollback-main, rollback-bot] runs-on: ubuntu-latest defaults: run: shell: bash env: PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }} steps: - name: Set up SSH run: | mkdir -p ~/.ssh && chmod 700 ~/.ssh printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - name: Verify the public site run: | domain="${CADDY_SITE_ADDRESS%% *}" ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "for i in \$(seq 1 20); do if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null && docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then echo 'rolled-back site healthy'; exit 0 fi sleep 5 done echo 'verify failed'; docker logs --tail 40 scrabble-caddy; docker logs --tail 40 scrabble-backend; exit 1"