scrabble-game/.gitea/workflows/prod-deploy.yaml

# Manual production rollout. Runs ONLY from master, ONLY on workflow_dispatch with
# confirm=deploy (development->master is merged + green first; this is the separate,
# deliberate prod step). Visible sequential jobs from most to least significant:
#   build -> deploy-main -> deploy-bot -> verify
# The per-service rolling (postgres->backend->gateway->landing->validator->caddy),
# health-gating and auto-rollback live in deploy/prod-deploy.sh on the main host and
# show in the deploy-main log. Manual post-deploy rollback is prod-rollback.yaml.
# See deploy/README.md (prod runbook).
name: prod-deploy
run-name: "prod deploy ${{ github.sha }}"

on:
  workflow_dispatch:
    inputs:
      confirm:
        description: 'Type "deploy" to confirm a production rollout from master.'
        required: true
        default: ""

permissions:
  contents: read

env:
  NO_COLOR: "1"
  DOCKER_CLI_HINTS: "false"
  REGISTRY: docker.iliadenisov.ru/developer

jobs:
  build:
    if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'deploy' }}
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    outputs:
      tag: ${{ steps.ver.outputs.tag }}
    env:
      PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
      PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
      VITE_TELEGRAM_BOT_ID: ${{ vars.PROD_VITE_TELEGRAM_BOT_ID }}
      VITE_TELEGRAM_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
      VITE_TELEGRAM_GAME_CHANNEL_NAME: ${{ vars.PROD_VITE_TELEGRAM_GAME_CHANNEL_NAME }}
      # VK Mini App link + VK ID "Web" app id: one value each serves every contour.
      VITE_VK_APP_LINK: ${{ vars.VITE_VK_APP_LINK }}
      VITE_VK_APP_ID: ${{ vars.VITE_VK_APP_ID }}
      # VITE_GATEWAY_URL omitted: the SPA is served same-origin (compose ":-" default).
      POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }}
      GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }}
      # `docker compose build` interpolates the WHOLE compose file, so every :?-guarded
      # runtime var must be present at build even though it is not a build-arg — incl. the
      # backend's EXPORT_SIGN_KEY (added with the finished-game export after v1.7.0, which is
      # why the first v1.8.0 build tripped on it). POSTGRES_PASSWORD/GM_BASICAUTH_HASH above
      # are here for the same reason; DICT_VERSION + the derived Mini App URL cover the rest.
      EXPORT_SIGN_KEY: ${{ secrets.PROD_EXPORT_SIGN_KEY }}
      # PUBLIC_BASE_URL drives the derived VK ID redirect (baked into the SPA) and the
      # Mini App URL; both are computed in the build step, not stored variables.
      PUBLIC_BASE_URL: ${{ vars.PROD_PUBLIC_BASE_URL }}
      DICT_VERSION: ${{ vars.DICT_VERSION }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Compute version tag
        id: ver
        run: echo "tag=$(git describe --tags --always)" >> "$GITHUB_OUTPUT"
      - name: Registry login
        run: echo "$PROD_REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$PROD_REGISTRY_USER" --password-stdin
      - name: Build and push images
        working-directory: deploy
        run: |
          export TAG="${{ steps.ver.outputs.tag }}" APP_VERSION="${{ steps.ver.outputs.tag }}" SCRABBLE_CONFIG_DIR=.
          # Derive the public URLs from the one canonical origin: the VK ID redirect is a
          # build-arg baked into the SPA, and the Mini App URL satisfies the (profiled-out)
          # bot service's compose ":?" guard during parse. See deploy/write-prod-env.sh.
          base="${PUBLIC_BASE_URL%/}"
          export VITE_VK_ID_REDIRECT_URL="$base/app/" TELEGRAM_MINIAPP_URL="$base/telegram/"
          # The main-stack images via compose (reuses the build args, incl. VERSION);
          # the bot separately, since it is profiled out of the prod compose.
          docker compose -f docker-compose.yml -f docker-compose.prod.yml build
          docker compose -f docker-compose.yml -f docker-compose.prod.yml push postgres backend gateway landing validator renderer
          docker build -f ../platform/telegram/Dockerfile --target bot --build-arg VERSION="$TAG" -t "$REGISTRY/scrabble-telegram-bot:$TAG" ..
          docker push "$REGISTRY/scrabble-telegram-bot:$TAG"

  deploy-main:
    needs: build
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    env:
      TAG: ${{ needs.build.outputs.tag }}
      PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
      PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
      PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
      MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
      POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }}
      GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }}
      GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }}
      TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
      GATEWAY_VK_APP_SECRET: ${{ secrets.GATEWAY_VK_APP_SECRET }}
      # Robokassa direct-rail (backend BACKEND_ROBOKASSA_*): the prod shop login + the pass phrases
      # that sign the launch request / verify the Result callback, and the test-mode flag — a var so
      # go-live is a flag flip, not a secret redeploy ("1" runs test payments against the test
      # passwords; empty/"0" is live). An empty login leaves the direct rail disabled.
      ROBOKASSA_MERCHANT_LOGIN: ${{ secrets.PROD_BACKEND_ROBOKASSA_MERCHANT_LOGIN }}
      ROBOKASSA_PASSWORD1: ${{ secrets.PROD_BACKEND_ROBOKASSA_PASSWORD1 }}
      ROBOKASSA_PASSWORD2: ${{ secrets.PROD_BACKEND_ROBOKASSA_PASSWORD2 }}
      ROBOKASSA_TEST: ${{ vars.PROD_BACKEND_ROBOKASSA_TEST }}
      # VK ID web login: the "Web" app id (the gateway reuses it as GATEWAY_VK_ID_APP_ID at
      # runtime) + the app's protected key. Both shared across contours. The redirect URL is
      # derived from PUBLIC_BASE_URL in deploy/write-prod-env.sh.
      VITE_VK_APP_ID: ${{ vars.VITE_VK_APP_ID }}
      GATEWAY_VK_ID_CLIENT_SECRET: ${{ secrets.GATEWAY_VK_ID_CLIENT_SECRET }}
      # Client-version gate (ARCHITECTURE.md §2): plain (unprefixed) vars, shared across contours
      # (empty ⇒ dormant). On prod the stamped client version is a real semver, so the gate enforces
      # here — set GATEWAY_MIN_CLIENT_VERSION to the release in the same rollout that ships a breaking
      # wire change; bump GATEWAY_RECOMMENDED_CLIENT_VERSION (≥ min) to nudge upgrades softly.
      GATEWAY_MIN_CLIENT_VERSION: ${{ vars.GATEWAY_MIN_CLIENT_VERSION }}
      GATEWAY_RECOMMENDED_CLIENT_VERSION: ${{ vars.GATEWAY_RECOMMENDED_CLIENT_VERSION }}
      # Planted honeytoken bearer: presenting it earns a 24h IP ban + a high-severity alarm.
      # Per-contour secret; empty = trap off. Rendered by deploy/write-prod-env.sh.
      GATEWAY_HONEYTOKEN: ${{ secrets.PROD_GATEWAY_HONEYTOKEN }}
      # Community IP blocklist (Spamhaus DROP): opt-in. Set _ENABLED=true + _URL (the feed) once
      # verified; _ALLOW is a comma-separated never-block set (own infra). Empty ⇒ off.
      GATEWAY_BLOCKLIST_ENABLED: ${{ vars.PROD_GATEWAY_BLOCKLIST_ENABLED }}
      GATEWAY_BLOCKLIST_URL: ${{ vars.PROD_GATEWAY_BLOCKLIST_URL }}
      GATEWAY_BLOCKLIST_ALLOW: ${{ vars.PROD_GATEWAY_BLOCKLIST_ALLOW }}
      # Signs the finished-game export download URLs (backend BACKEND_EXPORT_SIGN_KEY).
      EXPORT_SIGN_KEY: ${{ secrets.PROD_EXPORT_SIGN_KEY }}
      # Transactional email via the shared Selectel relay (confirm-codes): one account for
      # every contour -> unprefixed host/port/tls/user/pass.
      SMTP_RELAY_USER: ${{ secrets.SMTP_RELAY_USER }}
      SMTP_RELAY_PASS: ${{ secrets.SMTP_RELAY_PASS }}
      SMTP_RELAY_HOST: ${{ vars.SMTP_RELAY_HOST }}
      SMTP_RELAY_PORT: ${{ vars.SMTP_RELAY_PORT }}
      SMTP_RELAY_TLS: ${{ vars.SMTP_RELAY_TLS }}
      SMTP_RELAY_FROM: ${{ vars.PROD_SMTP_RELAY_FROM }}
      # Operator alerts: backend admin emails + Grafana infra alerts (distinct senders +
      # recipients; Grafana uses the relay's STARTTLS host:port).
      SMTP_RELAY_ADMIN_FROM: ${{ vars.PROD_SMTP_RELAY_ADMIN_FROM }}
      ADMIN_EMAIL: ${{ vars.PROD_ADMIN_EMAIL }}
      SMTP_RELAY_SERVICE_FROM: ${{ vars.PROD_SMTP_RELAY_SERVICE_FROM }}
      SERVICE_EMAIL: ${{ vars.PROD_SERVICE_EMAIL }}
      GRAFANA_SMTP_PORT: ${{ vars.GRAFANA_SMTP_PORT }}
      GF_SMTP_ENABLED: ${{ vars.PROD_GF_SMTP_ENABLED }}
      PUBLIC_BASE_URL: ${{ vars.PROD_PUBLIC_BASE_URL }}
      PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
      PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }}
      PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }}
      GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }}
      CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
      LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }}
      DICT_VERSION: ${{ vars.DICT_VERSION }}
      POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }}
      POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }}
      # Point-in-time recovery (pgBackRest -> S3), prod main host only. Endpoint/bucket/
      # region + the archive-mode switch are variables; the S3 keys + the repository cipher
      # passphrase are secrets. All empty/off until the operator arms archiving; flipping
      # PROD_PGBACKREST_ARCHIVE_MODE=on activates it (deploy/README.md, arming).
      PGBACKREST_ARCHIVE_MODE: ${{ vars.PROD_PGBACKREST_ARCHIVE_MODE }}
      PGBACKREST_S3_ENDPOINT: ${{ vars.PROD_PGBACKREST_S3_ENDPOINT }}
      PGBACKREST_S3_PORT: ${{ vars.PROD_PGBACKREST_S3_PORT }}
      PGBACKREST_S3_BUCKET: ${{ vars.PROD_PGBACKREST_S3_BUCKET }}
      PGBACKREST_S3_REGION: ${{ vars.PROD_PGBACKREST_S3_REGION }}
      PGBACKREST_S3_KEY: ${{ secrets.PROD_PGBACKREST_S3_KEY }}
      PGBACKREST_S3_KEY_SECRET: ${{ secrets.PROD_PGBACKREST_S3_KEY_SECRET }}
      PGBACKREST_CIPHER_PASS: ${{ secrets.PROD_PGBACKREST_CIPHER_PASS }}
      # TELEGRAM_MINIAPP_URL and GRAFANA_ROOT_URL are derived from PUBLIC_BASE_URL in
      # deploy/write-prod-env.sh, not stored variables.
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up SSH
        run: |
          mkdir -p ~/.ssh && chmod 700 ~/.ssh
          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
          printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
      - name: Determine previous tag and migration
        run: |
          ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
          PREV_TAG="$(ssh_main 'cat /opt/scrabble/DEPLOYED_TAG 2>/dev/null || echo none')"
          MIGRATION=0
          if [ "$PREV_TAG" != none ]; then
            if ! git cat-file -e "$PREV_TAG^{commit}" 2>/dev/null; then
              MIGRATION=1
            elif git diff --name-only "$PREV_TAG..$TAG" -- backend/internal/postgres/migrations/ | grep -q .; then
              MIGRATION=1
            fi
          fi
          { echo "PREV_TAG=$PREV_TAG"; echo "MIGRATION=$MIGRATION"; } >> "$GITHUB_ENV"
          echo "prev=$PREV_TAG migration=$MIGRATION"
      - name: Render main env + certs
        run: |
          umask 077
          mkdir -p stage/certs-main
          # Shared with prod-rollback so the two paths render an identical runtime env.
          APP_VERSION="$TAG" bash deploy/write-prod-env.sh stage/env.sh
          printf '%s\n' "$PROD_BOTLINK_CA"           > stage/certs-main/ca.crt
          printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt
          printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY"  > stage/certs-main/gateway.key
          chmod 644 stage/certs-main/*
      - name: Deploy the main host
        run: |
          ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
          ssh_main 'mkdir -p /opt/scrabble/compose'
          tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \
            | ssh_main 'tar -C /opt/scrabble/compose -xzf -'
          tar -C deploy -czf - caddy otelcol prometheus tempo grafana blackbox \
            | ssh_main 'tar -C /opt/scrabble -xzf -'
          tar -C stage -czf - certs-main \
            | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'
          scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh"
          echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin"
          ssh_main "TAG='$TAG' PREV_TAG='$PREV_TAG' MIGRATION='$MIGRATION' bash /opt/scrabble/compose/prod-deploy.sh"

  deploy-bot:
    needs: [build, deploy-main]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    env:
      TAG: ${{ needs.build.outputs.tag }}
      PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
      PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
      PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
      TG_HOST: ${{ vars.PROD_TG_HOST }}
      MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
      TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
      TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }}
      PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
      PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }}
      PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }}
      LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }}
      # PUBLIC_BASE_URL drives the derived Mini App URL in deploy/write-prod-bot-env.sh.
      PUBLIC_BASE_URL: ${{ vars.PROD_PUBLIC_BASE_URL }}
      TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }}
      TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }}
      TELEGRAM_SUPPORT_CHAT_ID: ${{ vars.PROD_TELEGRAM_SUPPORT_CHAT_ID }}
      TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }}
      TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
    steps:
      - uses: actions/checkout@v4
      - name: Set up SSH
        run: |
          mkdir -p ~/.ssh && chmod 700 ~/.ssh
          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
          printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
      - name: Render bot env + certs
        run: |
          umask 077
          mkdir -p stage/certs-bot
          # Shared with prod-rollback so the two paths render an identical bot env.
          BOT_IMAGE="$REGISTRY/scrabble-telegram-bot:$TAG" bash deploy/write-prod-bot-env.sh stage/env.bot.sh
          printf '%s\n' "$PROD_BOTLINK_CA"       > stage/certs-bot/ca.crt
          printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt
          printf '%s\n' "$PROD_BOTLINK_BOT_KEY"  > stage/certs-bot/bot.key
          chmod 644 stage/certs-bot/*
      - name: Deploy the bot host
        run: |
          ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; }
          ssh_tg 'mkdir -p /opt/scrabble/compose'
          tar -C deploy -czf - docker-compose.bot.yml | ssh_tg 'tar -C /opt/scrabble/compose -xzf -'
          tar -C stage -czf - certs-bot \
            | ssh_tg 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'
          scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.bot.sh "deploy@$TG_HOST:/opt/scrabble/env.bot.sh"
          echo "$PROD_REGISTRY_PASSWORD" | ssh_tg "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin"
          ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose;
            docker compose -f docker-compose.bot.yml pull;
            docker compose -f docker-compose.bot.yml up -d'
          ssh_tg 'for i in $(seq 1 20); do
              s=$(docker inspect -f "{{.State.Status}}" scrabble-telegram-bot 2>/dev/null || echo missing)
              r=$(docker inspect -f "{{.State.Restarting}}" scrabble-telegram-bot 2>/dev/null || echo true)
              if [ "$s" = running ] && [ "$r" = false ]; then
                c1=$(docker inspect -f "{{.RestartCount}}" scrabble-telegram-bot); sleep 5
                c2=$(docker inspect -f "{{.RestartCount}}" scrabble-telegram-bot)
                [ "$c1" = "$c2" ] && { echo "bot healthy"; exit 0; }
              fi
              sleep 3
            done
            echo "bot not healthy:"; docker logs --tail 80 scrabble-telegram-bot; exit 1'

  verify:
    needs: [deploy-main, deploy-bot]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    env:
      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
      PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
      MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
      CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
    steps:
      - name: Set up SSH
        run: |
          mkdir -p ~/.ssh && chmod 700 ~/.ssh
          printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
          printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
      - name: Verify the public site
        run: |
          domain="${CADDY_SITE_ADDRESS%% *}"
          ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "for i in \$(seq 1 20); do
              if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null &&
                 curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/app/ -o /dev/null &&
                 docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then
                echo 'public site + /app/ + backend healthy'; exit 0
              fi
              sleep 5
            done
            echo 'public verify failed; recent caddy + gateway + backend logs:'
            docker logs --tail 40 scrabble-caddy; docker logs --tail 40 scrabble-gateway; docker logs --tail 40 scrabble-backend
            exit 1"