2026-06-22 05:40:55 +00:00
5 changed files with 397 additions and 94 deletions
@@ -1,9 +1,11 @@
-# Manual production rollout. Runs ONLY from master, ONLY on an explicit
-# workflow_dispatch with confirm=deploy (development->master is merged + green first;
-# this is the separate, deliberate prod step). It builds and pushes the images to the
-# registry, then deploys over SSH: the main host via deploy/prod-deploy.sh (rolling,
-# health-gated, auto-rollback; a maintenance window + consistent dump on a migration),
-# then the bot host. See deploy/README.md (prod runbook) for operator steps.
+# Manual production rollout. Runs ONLY from master, ONLY on workflow_dispatch with
+# confirm=deploy (development->master is merged + green first; this is the separate,
+# deliberate prod step). Visible sequential jobs from most to least significant:
+#   build -> deploy-main -> deploy-bot -> verify
+# The per-service rolling (postgres->backend->gateway->landing->validator->caddy),
+# health-gating and auto-rollback live in deploy/prod-deploy.sh on the main host and
+# show in the deploy-main log. Manual post-deploy rollback is prod-rollback.yaml.
+# See deploy/README.md (prod runbook).
 name: prod-deploy
 run-name: "prod deploy ${{ github.sha }}"

@@ -18,41 +20,71 @@ on:
 permissions:
  contents: read

+env:
+  NO_COLOR: "1"
+  DOCKER_CLI_HINTS: "false"
+  REGISTRY: docker.iliadenisov.ru/developer
+
 jobs:
-  deploy:
+  build:
    if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'deploy' }}
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
+    outputs:
+      tag: ${{ steps.ver.outputs.tag }}
    env:
-      NO_COLOR: "1"
-      DOCKER_CLI_HINTS: "false"
-      REGISTRY: docker.iliadenisov.ru/developer
-      # SSH + registry
+      PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
+      PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
+      VITE_TELEGRAM_BOT_ID: ${{ vars.PROD_VITE_TELEGRAM_BOT_ID }}
+      VITE_TELEGRAM_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
+      VITE_TELEGRAM_GAME_CHANNEL_NAME: ${{ vars.PROD_VITE_TELEGRAM_GAME_CHANNEL_NAME }}
+      VITE_GATEWAY_URL: ${{ vars.PROD_VITE_GATEWAY_URL }}
+      POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }}
+      GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }}
+      TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }}
+      DICT_VERSION: ${{ vars.PROD_DICT_VERSION }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Compute version tag
+        id: ver
+        run: echo "tag=$(git describe --tags --always)" >> "$GITHUB_OUTPUT"
+      - name: Registry login
+        run: echo "$PROD_REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$PROD_REGISTRY_USER" --password-stdin
+      - name: Build and push images
+        working-directory: deploy
+        run: |
+          export TAG="${{ steps.ver.outputs.tag }}" APP_VERSION="${{ steps.ver.outputs.tag }}" SCRABBLE_CONFIG_DIR=.
+          # The four main-stack images via compose (reuses the build args, incl. VERSION);
+          # the bot separately, since it is profiled out of the prod compose.
+          docker compose -f docker-compose.yml -f docker-compose.prod.yml build
+          docker compose -f docker-compose.yml -f docker-compose.prod.yml push backend gateway landing validator
+          docker build -f ../platform/telegram/Dockerfile --target bot --build-arg VERSION="$TAG" -t "$REGISTRY/scrabble-telegram-bot:$TAG" ..
+          docker push "$REGISTRY/scrabble-telegram-bot:$TAG"
+
+  deploy-main:
+    needs: build
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    env:
+      TAG: ${{ needs.build.outputs.tag }}
      PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
      PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
      PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
      MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
-      TG_HOST: ${{ vars.PROD_TG_HOST }}
-      # SPA build args (baked into the gateway/landing images)
-      VITE_TELEGRAM_BOT_ID: ${{ vars.PROD_VITE_TELEGRAM_BOT_ID }}
-      VITE_TELEGRAM_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
-      VITE_TELEGRAM_GAME_CHANNEL_NAME: ${{ vars.PROD_VITE_TELEGRAM_GAME_CHANNEL_NAME }}
-      VITE_GATEWAY_URL: ${{ vars.PROD_VITE_GATEWAY_URL }}
-      # Runtime secrets
      POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }}
      GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }}
      GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }}
      TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
-      TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }}
      PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
      PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }}
      PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }}
-      PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }}
-      PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }}
-      # Runtime variables
      GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }}
      GRAFANA_ROOT_URL: ${{ vars.PROD_GRAFANA_ROOT_URL }}
      CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
@@ -61,41 +93,15 @@ jobs:
      POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }}
      POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }}
      TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }}
-      TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }}
-      TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }}
-      TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }}
-      TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
-
-      - name: Compute image tag and app version
-        run: |
-          echo "TAG=$(git rev-parse --short=12 HEAD)" >> "$GITHUB_ENV"
-          echo "APP_VERSION=$(git describe --tags --always)" >> "$GITHUB_ENV"
-
-      - name: Registry login
-        run: echo "$PROD_REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$PROD_REGISTRY_USER" --password-stdin
-
-      - name: Build and push images
-        working-directory: deploy
-        run: |
-          export TAG SCRABBLE_CONFIG_DIR=.
-          # The four main-stack images via compose (reuses the compose build args); the
-          # bot separately, since it is profiled out of the prod compose.
-          docker compose -f docker-compose.yml -f docker-compose.prod.yml build
-          docker compose -f docker-compose.yml -f docker-compose.prod.yml push backend gateway landing validator
-          docker build -f ../platform/telegram/Dockerfile --target bot -t "$REGISTRY/scrabble-telegram-bot:$TAG" ..
-          docker push "$REGISTRY/scrabble-telegram-bot:$TAG"
-
      - name: Set up SSH
        run: |
          mkdir -p ~/.ssh && chmod 700 ~/.ssh
-          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy
-          chmod 600 ~/.ssh/id_deploy
+          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
          printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
-
      - name: Determine previous tag and migration
        run: |
          ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
@@ -103,21 +109,17 @@ jobs:
          MIGRATION=0
          if [ "$PREV_TAG" != none ]; then
            if ! git cat-file -e "$PREV_TAG^{commit}" 2>/dev/null; then
-              MIGRATION=1   # unknown previous tag: take the safe window + dump
+              MIGRATION=1
            elif git diff --name-only "$PREV_TAG..$TAG" -- backend/internal/postgres/migrations/ | grep -q .; then
              MIGRATION=1
            fi
          fi
-          echo "PREV_TAG=$PREV_TAG" >> "$GITHUB_ENV"
-          echo "MIGRATION=$MIGRATION" >> "$GITHUB_ENV"
+          { echo "PREV_TAG=$PREV_TAG"; echo "MIGRATION=$MIGRATION"; } >> "$GITHUB_ENV"
          echo "prev=$PREV_TAG migration=$MIGRATION"
-
-      - name: Render env files and certs
+      - name: Render main env + certs
        run: |
          umask 077
-          mkdir -p stage/certs-main stage/certs-bot
-          # Main-host runtime env (single-quoted so the literal '$' in the bcrypt hash
-          # survives; prod-deploy.sh sources this, compose reads it from the environment).
+          mkdir -p stage/certs-main
          cat > stage/env.sh <<EOF
          export REGISTRY='$REGISTRY'
          export SCRABBLE_CONFIG_DIR='/opt/scrabble'
@@ -131,11 +133,65 @@ jobs:
          export CADDY_SITE_ADDRESS='$CADDY_SITE_ADDRESS'
          export LOG_LEVEL='${LOG_LEVEL:-info}'
          export DICT_VERSION='$DICT_VERSION'
+          export APP_VERSION='$TAG'
          export TELEGRAM_BOT_TOKEN='$TELEGRAM_BOT_TOKEN'
          export TELEGRAM_MINIAPP_URL='$TELEGRAM_MINIAPP_URL'
          export GATEWAY_ABUSE_BAN_ENABLED='true'
          EOF
-          # Bot-host runtime env.
+          printf '%s\n' "$PROD_BOTLINK_CA"           > stage/certs-main/ca.crt
+          printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt
+          printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY"  > stage/certs-main/gateway.key
+          chmod 644 stage/certs-main/*
+      - name: Deploy the main host
+        run: |
+          ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
+          ssh_main 'mkdir -p /opt/scrabble/compose'
+          tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \
+            | ssh_main 'tar -C /opt/scrabble/compose -xzf -'
+          tar -C deploy -czf - caddy otelcol prometheus tempo grafana \
+            | ssh_main 'tar -C /opt/scrabble -xzf -'
+          tar -C stage -czf - certs-main \
+            | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'
+          scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh"
+          echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin"
+          ssh_main "TAG='$TAG' PREV_TAG='$PREV_TAG' MIGRATION='$MIGRATION' bash /opt/scrabble/compose/prod-deploy.sh"
+
+  deploy-bot:
+    needs: [build, deploy-main]
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    env:
+      TAG: ${{ needs.build.outputs.tag }}
+      PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
+      PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
+      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
+      PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
+      TG_HOST: ${{ vars.PROD_TG_HOST }}
+      MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
+      TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
+      TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }}
+      PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
+      PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }}
+      PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }}
+      LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }}
+      TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }}
+      TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }}
+      TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }}
+      TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }}
+      TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up SSH
+        run: |
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
+          printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
+      - name: Render bot env + certs
+        run: |
+          umask 077
+          mkdir -p stage/certs-bot
          cat > stage/env.bot.sh <<EOF
          export SCRABBLE_CONFIG_DIR='/opt/scrabble'
          export BOT_IMAGE='$REGISTRY/scrabble-telegram-bot:$TAG'
@@ -149,32 +205,10 @@ jobs:
          export TELEGRAM_BOT_LINK='$TELEGRAM_BOT_LINK'
          export LOG_LEVEL='${LOG_LEVEL:-info}'
          EOF
-          # Bot-link certs (world-readable: the distroless gateway/bot run as UID 65532
-          # and bind-mount the dir read-only).
-          printf '%s\n' "$PROD_BOTLINK_CA"           > stage/certs-main/ca.crt
-          printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt
-          printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY"  > stage/certs-main/gateway.key
-          printf '%s\n' "$PROD_BOTLINK_CA"           > stage/certs-bot/ca.crt
-          printf '%s\n' "$PROD_BOTLINK_BOT_CERT"     > stage/certs-bot/bot.crt
-          printf '%s\n' "$PROD_BOTLINK_BOT_KEY"      > stage/certs-bot/bot.key
-          chmod 644 stage/certs-main/* stage/certs-bot/*
-
-      - name: Deploy the main host
-        run: |
-          ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
-          ssh_main 'mkdir -p /opt/scrabble/compose'
-          # Compose files + config + script.
-          tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \
-            | ssh_main 'tar -C /opt/scrabble/compose -xzf -'
-          tar -C deploy -czf - caddy otelcol prometheus tempo grafana \
-            | ssh_main 'tar -C /opt/scrabble -xzf -'
-          tar -C stage -czf - certs-main \
-            | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'
-          scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh"
-          # Registry login on the host so compose can pull the private images.
-          echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin"
-          ssh_main "TAG='$TAG' PREV_TAG='$PREV_TAG' MIGRATION='$MIGRATION' bash /opt/scrabble/compose/prod-deploy.sh"
-
+          printf '%s\n' "$PROD_BOTLINK_CA"       > stage/certs-bot/ca.crt
+          printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt
+          printf '%s\n' "$PROD_BOTLINK_BOT_KEY"  > stage/certs-bot/bot.key
+          chmod 644 stage/certs-bot/*
      - name: Deploy the bot host
        run: |
          ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; }
@@ -187,7 +221,6 @@ jobs:
          ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose;
            docker compose -f docker-compose.bot.yml pull;
            docker compose -f docker-compose.bot.yml up -d'
-          # Bot liveness: running, not restarting, stable restart count.
          ssh_tg 'for i in $(seq 1 20); do
              s=$(docker inspect -f "{{.State.Status}}" scrabble-telegram-bot 2>/dev/null || echo missing)
              r=$(docker inspect -f "{{.State.Restarting}}" scrabble-telegram-bot 2>/dev/null || echo true)
@@ -200,13 +233,27 @@ jobs:
            done
            echo "bot not healthy:"; docker logs --tail 80 scrabble-telegram-bot; exit 1'

+  verify:
+    needs: [deploy-main, deploy-bot]
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    env:
+      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
+      PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
+      MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
+      CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
+    steps:
+      - name: Set up SSH
+        run: |
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
+          printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
      - name: Verify the public site
        run: |
-          ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
-          domain="${CADDY_SITE_ADDRESS%% *}"   # first domain of "erudit-game.ru www..."
-          # Probe through caddy with the right vhost (force-resolved to the host); -k
-          # tolerates an ACME-pending cert. Also check the backend is actually ready.
-          ssh_main "for i in \$(seq 1 20); do
+          domain="${CADDY_SITE_ADDRESS%% *}"
+          ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "for i in \$(seq 1 20); do
              if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null &&
                 curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/app/ -o /dev/null &&
                 docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then
@@ -0,0 +1,223 @@
+# Manual production rollback. Runs ONLY from master, ONLY on workflow_dispatch with
+# confirm=rollback. Re-deploys an already-published image tag (no build): leave
+# target_version blank to roll back to the previously deployed version (read from the
+# main host), or set it to a specific release tag from the Releases page. The
+# re-deploy is the same rolling, health-gated path as prod-deploy (TAG=target,
+# MIGRATION=0 — rollback is image-only and never migrates the DB; image rollback is
+# DB-safe under the expand-contract rule). See deploy/README.md (prod runbook).
+name: prod-rollback
+run-name: "prod rollback ${{ inputs.target_version || 'previous' }}"
+
+on:
+  workflow_dispatch:
+    inputs:
+      confirm:
+        description: 'Type "rollback" to confirm a production rollback.'
+        required: true
+        default: ""
+      target_version:
+        description: "Release tag to roll back to (blank = the previous deployed version)."
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+
+env:
+  NO_COLOR: "1"
+  DOCKER_CLI_HINTS: "false"
+  REGISTRY: docker.iliadenisov.ru/developer
+
+jobs:
+  rollback-main:
+    if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'rollback' }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    outputs:
+      target: ${{ steps.resolve.outputs.target }}
+    env:
+      PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
+      PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
+      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
+      PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
+      MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
+      POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }}
+      GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }}
+      GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }}
+      TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
+      PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
+      PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }}
+      PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }}
+      GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }}
+      GRAFANA_ROOT_URL: ${{ vars.PROD_GRAFANA_ROOT_URL }}
+      CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
+      LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }}
+      DICT_VERSION: ${{ vars.PROD_DICT_VERSION }}
+      POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }}
+      POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }}
+      TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }}
+      INPUT_TARGET: ${{ inputs.target_version }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up SSH
+        run: |
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
+          printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
+      - name: Resolve rollback target
+        id: resolve
+        run: |
+          ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
+          CURRENT="$(ssh_main 'cat /opt/scrabble/DEPLOYED_TAG 2>/dev/null || echo none')"
+          if [ -n "$INPUT_TARGET" ]; then
+            TARGET="$INPUT_TARGET"
+          else
+            TARGET="$(ssh_main 'cat /opt/scrabble/PREVIOUS_TAG 2>/dev/null || echo none')"
+          fi
+          if [ -z "$TARGET" ] || [ "$TARGET" = none ]; then
+            echo "no rollback target (no PREVIOUS_TAG on the host and no target_version input)"; exit 1
+          fi
+          if [ "$TARGET" = "$CURRENT" ]; then
+            echo "target $TARGET is already the deployed version; nothing to do"; exit 1
+          fi
+          echo "rolling back: current=$CURRENT -> target=$TARGET"
+          echo "target=$TARGET" >> "$GITHUB_OUTPUT"
+          { echo "TARGET=$TARGET"; echo "CURRENT=$CURRENT"; } >> "$GITHUB_ENV"
+      - name: Render main env + certs
+        run: |
+          umask 077
+          mkdir -p stage/certs-main
+          cat > stage/env.sh <<EOF
+          export REGISTRY='$REGISTRY'
+          export SCRABBLE_CONFIG_DIR='/opt/scrabble'
+          export POSTGRES_DB='${POSTGRES_DB:-scrabble}'
+          export POSTGRES_USER='${POSTGRES_USER:-scrabble}'
+          export POSTGRES_PASSWORD='$POSTGRES_PASSWORD'
+          export GM_BASICAUTH_USER='${GM_BASICAUTH_USER:-gm}'
+          export GM_BASICAUTH_HASH='$GM_BASICAUTH_HASH'
+          export GRAFANA_ADMIN_PASSWORD='$GRAFANA_ADMIN_PASSWORD'
+          export GRAFANA_ROOT_URL='$GRAFANA_ROOT_URL'
+          export CADDY_SITE_ADDRESS='$CADDY_SITE_ADDRESS'
+          export LOG_LEVEL='${LOG_LEVEL:-info}'
+          export DICT_VERSION='$DICT_VERSION'
+          export APP_VERSION='$TARGET'
+          export TELEGRAM_BOT_TOKEN='$TELEGRAM_BOT_TOKEN'
+          export TELEGRAM_MINIAPP_URL='$TELEGRAM_MINIAPP_URL'
+          export GATEWAY_ABUSE_BAN_ENABLED='true'
+          EOF
+          printf '%s\n' "$PROD_BOTLINK_CA"           > stage/certs-main/ca.crt
+          printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt
+          printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY"  > stage/certs-main/gateway.key
+          chmod 644 stage/certs-main/*
+      - name: Roll the main host back
+        run: |
+          ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
+          ssh_main 'mkdir -p /opt/scrabble/compose'
+          tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \
+            | ssh_main 'tar -C /opt/scrabble/compose -xzf -'
+          tar -C deploy -czf - caddy otelcol prometheus tempo grafana \
+            | ssh_main 'tar -C /opt/scrabble -xzf -'
+          tar -C stage -czf - certs-main \
+            | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'
+          scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh"
+          echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin"
+          # Image-only rollback: no migration window (TAG=target, MIGRATION=0). A failed
+          # rollback's auto-revert returns to the current version (PREV_TAG=$CURRENT).
+          ssh_main "TAG='$TARGET' PREV_TAG='$CURRENT' MIGRATION=0 bash /opt/scrabble/compose/prod-deploy.sh"
+
+  rollback-bot:
+    needs: rollback-main
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    env:
+      TARGET: ${{ needs.rollback-main.outputs.target }}
+      PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
+      PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
+      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
+      PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
+      TG_HOST: ${{ vars.PROD_TG_HOST }}
+      MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
+      TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
+      TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }}
+      PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
+      PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }}
+      PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }}
+      LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }}
+      TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }}
+      TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }}
+      TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }}
+      TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }}
+      TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up SSH
+        run: |
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
+          printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
+      - name: Render bot env + certs
+        run: |
+          umask 077
+          mkdir -p stage/certs-bot
+          cat > stage/env.bot.sh <<EOF
+          export SCRABBLE_CONFIG_DIR='/opt/scrabble'
+          export BOT_IMAGE='$REGISTRY/scrabble-telegram-bot:$TARGET'
+          export BOTLINK_GATEWAY_ADDR='$MAIN_HOST:9443'
+          export TELEGRAM_BOT_TOKEN='$TELEGRAM_BOT_TOKEN'
+          export TELEGRAM_MINIAPP_URL='$TELEGRAM_MINIAPP_URL'
+          export TELEGRAM_GAME_CHANNEL_ID='$TELEGRAM_GAME_CHANNEL_ID'
+          export TELEGRAM_CHAT_ID='$TELEGRAM_CHAT_ID'
+          export TELEGRAM_PROMO_BOT_TOKEN='$TELEGRAM_PROMO_BOT_TOKEN'
+          export TELEGRAM_BOT_USERNAME='$TELEGRAM_BOT_USERNAME'
+          export TELEGRAM_BOT_LINK='$TELEGRAM_BOT_LINK'
+          export LOG_LEVEL='${LOG_LEVEL:-info}'
+          EOF
+          printf '%s\n' "$PROD_BOTLINK_CA"       > stage/certs-bot/ca.crt
+          printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt
+          printf '%s\n' "$PROD_BOTLINK_BOT_KEY"  > stage/certs-bot/bot.key
+          chmod 644 stage/certs-bot/*
+      - name: Roll the bot host back
+        run: |
+          ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; }
+          ssh_tg 'mkdir -p /opt/scrabble/compose'
+          tar -C deploy -czf - docker-compose.bot.yml | ssh_tg 'tar -C /opt/scrabble/compose -xzf -'
+          tar -C stage -czf - certs-bot \
+            | ssh_tg 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'
+          scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.bot.sh "deploy@$TG_HOST:/opt/scrabble/env.bot.sh"
+          echo "$PROD_REGISTRY_PASSWORD" | ssh_tg "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin"
+          ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose;
+            docker compose -f docker-compose.bot.yml pull;
+            docker compose -f docker-compose.bot.yml up -d'
+
+  verify:
+    needs: [rollback-main, rollback-bot]
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    env:
+      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
+      PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
+      MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
+      CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
+    steps:
+      - name: Set up SSH
+        run: |
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
+          printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
+      - name: Verify the public site
+        run: |
+          domain="${CADDY_SITE_ADDRESS%% *}"
+          ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "for i in \$(seq 1 20); do
+              if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null &&
+                 docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then
+                echo 'rolled-back site healthy'; exit 0
+              fi
+              sleep 5
+            done
+            echo 'verify failed'; docker logs --tail 40 scrabble-caddy; docker logs --tail 40 scrabble-backend; exit 1"
@@ -130,7 +130,25 @@ workflow manually (Gitea → Actions → prod-deploy → run from `master`, inpu
 compose/config/certs/env over SSH, deploys the main host with `prod-deploy.sh` (rolling,
 health-gated, **auto-rollback to the previous tag**), then the bot host, then probes the
 public site. After `master` is green this workflow is the **only** thing that touches
-prod — nothing auto-deploys there.
+prod — nothing auto-deploys there. It runs four visible jobs: **build → deploy-main →
+deploy-bot → verify** (the per-service rolling shows in the deploy-main log).
+
+**Versioning.** Each release is a git tag `vX.Y.Z` on `master`; the deploy stamps
+`git describe --tags` into every image tag, every binary (`-ldflags` → `pkg/version` →
+the `service.version` telemetry attribute) and the SPA About screen. Tag the release
+before running the deploy:
+
+```sh
+git tag -a v1.0.0 -m v1.0.0 && git push origin v1.0.0
+```
+
+**Manual rollback** (any time after a successful deploy). Run the **`prod-rollback`**
+workflow (Gitea → Actions → prod-rollback, `confirm=rollback`). Leave `target_version`
+blank to roll back to the previously deployed version (read from the host's
+`PREVIOUS_TAG`), or set it to a release tag from the **Releases** page. It re-deploys
+that already-published image rolling + health-gated — no rebuild, no DB migration
+(image rollback is DB-safe under the expand-contract rule). The registry keeps every
+release tag, so any prior release is reachable.

 **Migrations** must be **expand-contract** (backward-compatible; goose is forward-only):
 the automatic rollback is image-only and never restores the DB. A deploy that changes
@@ -36,6 +36,9 @@ MIGRATION="${MIGRATION:-0}"
 COMPOSE_DIR="${COMPOSE_DIR:-/opt/scrabble/compose}"
 DUMP_DIR="${DUMP_DIR:-/opt/scrabble/dumps}"
 STATE_FILE="${STATE_FILE:-/opt/scrabble/DEPLOYED_TAG}"
+# The prior deployed tag, preserved on every successful deploy so prod-rollback can
+# target "the previous version" with no operator input.
+PREV_STATE_FILE="${PREV_STATE_FILE:-/opt/scrabble/PREVIOUS_TAG}"
 PG_USER="${POSTGRES_USER:-scrabble}"
 PG_DB="${POSTGRES_DB:-scrabble}"

@@ -88,6 +91,12 @@ rollback() {
  [ "$MIGRATION" = 1 ] && echo "NOTE: the DB is forward-migrated; a pre-deploy dump is in $DUMP_DIR — restore manually ONLY if the migration was destructive (see deploy/README.md, prod runbook)."
 }

+commit_tag() {
+  # Record the just-deployed tag as current, preserving the prior one as previous.
+  [ -f "$STATE_FILE" ] && cp "$STATE_FILE" "$PREV_STATE_FILE"
+  echo "$TAG" > "$STATE_FILE"
+}
+
 mkdir -p "$DUMP_DIR"
 echo "=== prod deploy: tag=$TAG prev=$PREV_TAG migration=$MIGRATION ==="
 use_tag "$TAG"
@@ -99,7 +108,7 @@ if [ -z "$(docker ps -aq -f name=scrabble-backend)" ]; then
  dc up -d --no-build --remove-orphans || { echo "compose up failed"; exit 1; }
  health_backend || { echo "backend not ready"; exit 1; }
  health_landing || { echo "landing not ready"; exit 1; }
-  echo "$TAG" > "$STATE_FILE"
+  commit_tag
  echo "first deploy healthy ($TAG)."
  exit 0
 fi
@@ -130,5 +139,5 @@ dc up -d --no-build --remove-orphans || { rollback; exit 1; }

 # Final internal sanity before committing the new tag.
 health_backend || { rollback; exit 1; }
-echo "$TAG" > "$STATE_FILE"
+commit_tag
 echo "=== deploy healthy ($TAG) ==="
@@ -1100,7 +1100,13 @@ Two contours, two secret/variable prefixes (`TEST_` / `PROD_`):
  any failure **rolls the whole stack back to the previous image tag**. A **schema migration**
  adds a maintenance window: the backend (the sole writer) is stopped for a consistent
  `pg_dump` before the new backend migrates forward — image rollback stays DB-safe under the
-  expand-contract migration rule, and the dump is kept for a manual restore. The main host is
+  expand-contract migration rule, and the dump is kept for a manual restore. The workflow runs
+  four visible jobs (build → deploy-main → deploy-bot → verify). Releases are git tags
+  `vX.Y.Z`; the version is stamped into the image tag, every binary (`-ldflags` → `pkg/version`
+  → the `service.version` telemetry attribute) and the SPA About screen. A separate manual
+  **`prod-rollback`** workflow re-deploys any prior release tag (blank input = the previous
+  deployed version, tracked on the host) over the same rolling, health-gated path — image-only,
+  no DB migration. The main host is
  intentionally **launch-sized** (2 vCPU / 1.9 GiB): the prod overlay trims the R7 limits
  (`GOMAXPROCS=2`, smaller caps, 7d Prometheus retention) and a **node_exporter** feeds
  host-memory metrics to Grafana so it can be resized reactively as players arrive.