# Manual production rollback. Runs ONLY from master, ONLY on workflow_dispatch with # confirm=rollback. Re-deploys an already-published image tag (no build): leave # target_version blank to roll back to the previously deployed version (read from the # main host), or set it to a specific release tag from the Releases page. The # re-deploy is the same rolling, health-gated path as prod-deploy (TAG=target, # MIGRATION=0 — rollback is image-only and never migrates the DB; image rollback is # DB-safe under the expand-contract rule). See deploy/README.md (prod runbook). name: prod-rollback run-name: "prod rollback ${{ inputs.target_version || 'previous' }}" on: workflow_dispatch: inputs: confirm: description: 'Type "rollback" to confirm a production rollback.' required: true default: "" target_version: description: "Release tag to roll back to (blank = the previous deployed version)." required: false default: "" permissions: contents: read env: NO_COLOR: "1" DOCKER_CLI_HINTS: "false" REGISTRY: docker.iliadenisov.ru/developer jobs: rollback-main: if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'rollback' }} runs-on: ubuntu-latest defaults: run: shell: bash outputs: target: ${{ steps.resolve.outputs.target }} env: PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }} GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }} GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }} TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }} PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }} PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }} PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }} GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }} GRAFANA_ROOT_URL: ${{ vars.PROD_GRAFANA_ROOT_URL }} CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }} LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }} DICT_VERSION: ${{ vars.PROD_DICT_VERSION }} POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }} POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }} TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }} INPUT_TARGET: ${{ inputs.target_version }} steps: - uses: actions/checkout@v4 - name: Set up SSH run: | mkdir -p ~/.ssh && chmod 700 ~/.ssh printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - name: Resolve rollback target id: resolve run: | ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } CURRENT="$(ssh_main 'cat /opt/scrabble/DEPLOYED_TAG 2>/dev/null || echo none')" if [ -n "$INPUT_TARGET" ]; then TARGET="$INPUT_TARGET" else TARGET="$(ssh_main 'cat /opt/scrabble/PREVIOUS_TAG 2>/dev/null || echo none')" fi if [ -z "$TARGET" ] || [ "$TARGET" = none ]; then echo "no rollback target (no PREVIOUS_TAG on the host and no target_version input)"; exit 1 fi if [ "$TARGET" = "$CURRENT" ]; then echo "target $TARGET is already the deployed version; nothing to do"; exit 1 fi echo "rolling back: current=$CURRENT -> target=$TARGET" echo "target=$TARGET" >> "$GITHUB_OUTPUT" { echo "TARGET=$TARGET"; echo "CURRENT=$CURRENT"; } >> "$GITHUB_ENV" - name: Render main env + certs run: | umask 077 mkdir -p stage/certs-main cat > stage/env.sh < stage/certs-main/ca.crt printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY" > stage/certs-main/gateway.key chmod 644 stage/certs-main/* - name: Roll the main host back run: | ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } ssh_main 'mkdir -p /opt/scrabble/compose' tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \ | ssh_main 'tar -C /opt/scrabble/compose -xzf -' tar -C deploy -czf - caddy otelcol prometheus tempo grafana \ | ssh_main 'tar -C /opt/scrabble -xzf -' tar -C stage -czf - certs-main \ | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -' scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh" echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin" # Image-only rollback: no migration window (TAG=target, MIGRATION=0). A failed # rollback's auto-revert returns to the current version (PREV_TAG=$CURRENT). ssh_main "TAG='$TARGET' PREV_TAG='$CURRENT' MIGRATION=0 bash /opt/scrabble/compose/prod-deploy.sh" rollback-bot: needs: rollback-main runs-on: ubuntu-latest defaults: run: shell: bash env: TARGET: ${{ needs.rollback-main.outputs.target }} PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} TG_HOST: ${{ vars.PROD_TG_HOST }} MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }} TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }} PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }} PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }} PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }} LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }} TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }} TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }} TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }} TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }} TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }} steps: - uses: actions/checkout@v4 - name: Set up SSH run: | mkdir -p ~/.ssh && chmod 700 ~/.ssh printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - name: Render bot env + certs run: | umask 077 mkdir -p stage/certs-bot cat > stage/env.bot.sh < stage/certs-bot/ca.crt printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt printf '%s\n' "$PROD_BOTLINK_BOT_KEY" > stage/certs-bot/bot.key chmod 644 stage/certs-bot/* - name: Roll the bot host back run: | ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; } ssh_tg 'mkdir -p /opt/scrabble/compose' tar -C deploy -czf - docker-compose.bot.yml | ssh_tg 'tar -C /opt/scrabble/compose -xzf -' tar -C stage -czf - certs-bot \ | ssh_tg 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -' scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.bot.sh "deploy@$TG_HOST:/opt/scrabble/env.bot.sh" echo "$PROD_REGISTRY_PASSWORD" | ssh_tg "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin" ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose; docker compose -f docker-compose.bot.yml pull; docker compose -f docker-compose.bot.yml up -d' verify: needs: [rollback-main, rollback-bot] runs-on: ubuntu-latest defaults: run: shell: bash env: PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }} steps: - name: Set up SSH run: | mkdir -p ~/.ssh && chmod 700 ~/.ssh printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - name: Verify the public site run: | domain="${CADDY_SITE_ADDRESS%% *}" ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "for i in \$(seq 1 20); do if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null && docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then echo 'rolled-back site healthy'; exit 0 fi sleep 5 done echo 'verify failed'; docker logs --tail 40 scrabble-caddy; docker logs --tail 40 scrabble-backend; exit 1"