# Manual production rollout. Runs ONLY from master, ONLY on workflow_dispatch with # confirm=deploy (development->master is merged + green first; this is the separate, # deliberate prod step). Visible sequential jobs from most to least significant: # build -> deploy-main -> deploy-bot -> verify # The per-service rolling (postgres->backend->gateway->landing->validator->caddy), # health-gating and auto-rollback live in deploy/prod-deploy.sh on the main host and # show in the deploy-main log. Manual post-deploy rollback is prod-rollback.yaml. # See deploy/README.md (prod runbook). name: prod-deploy run-name: "prod deploy ${{ github.sha }}" on: workflow_dispatch: inputs: confirm: description: 'Type "deploy" to confirm a production rollout from master.' required: true default: "" permissions: contents: read env: NO_COLOR: "1" DOCKER_CLI_HINTS: "false" REGISTRY: docker.iliadenisov.ru/developer jobs: build: if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'deploy' }} runs-on: ubuntu-latest defaults: run: shell: bash outputs: tag: ${{ steps.ver.outputs.tag }} env: PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} VITE_TELEGRAM_BOT_ID: ${{ vars.PROD_VITE_TELEGRAM_BOT_ID }} VITE_TELEGRAM_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }} VITE_TELEGRAM_GAME_CHANNEL_NAME: ${{ vars.PROD_VITE_TELEGRAM_GAME_CHANNEL_NAME }} VITE_GATEWAY_URL: ${{ vars.PROD_VITE_GATEWAY_URL }} POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }} GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }} TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }} DICT_VERSION: ${{ vars.PROD_DICT_VERSION }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Compute version tag id: ver run: echo "tag=$(git describe --tags --always)" >> "$GITHUB_OUTPUT" - name: Registry login run: echo "$PROD_REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$PROD_REGISTRY_USER" --password-stdin - name: Build and push images working-directory: deploy run: | export TAG="${{ steps.ver.outputs.tag }}" APP_VERSION="${{ steps.ver.outputs.tag }}" SCRABBLE_CONFIG_DIR=. # The four main-stack images via compose (reuses the build args, incl. VERSION); # the bot separately, since it is profiled out of the prod compose. docker compose -f docker-compose.yml -f docker-compose.prod.yml build docker compose -f docker-compose.yml -f docker-compose.prod.yml push backend gateway landing validator docker build -f ../platform/telegram/Dockerfile --target bot --build-arg VERSION="$TAG" -t "$REGISTRY/scrabble-telegram-bot:$TAG" .. docker push "$REGISTRY/scrabble-telegram-bot:$TAG" deploy-main: needs: build runs-on: ubuntu-latest defaults: run: shell: bash env: TAG: ${{ needs.build.outputs.tag }} PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }} GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }} GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }} TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }} PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }} PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }} PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }} GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }} GRAFANA_ROOT_URL: ${{ vars.PROD_GRAFANA_ROOT_URL }} CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }} LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }} DICT_VERSION: ${{ vars.PROD_DICT_VERSION }} POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }} POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }} TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up SSH run: | mkdir -p ~/.ssh && chmod 700 ~/.ssh printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - name: Determine previous tag and migration run: | ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } PREV_TAG="$(ssh_main 'cat /opt/scrabble/DEPLOYED_TAG 2>/dev/null || echo none')" MIGRATION=0 if [ "$PREV_TAG" != none ]; then if ! git cat-file -e "$PREV_TAG^{commit}" 2>/dev/null; then MIGRATION=1 elif git diff --name-only "$PREV_TAG..$TAG" -- backend/internal/postgres/migrations/ | grep -q .; then MIGRATION=1 fi fi { echo "PREV_TAG=$PREV_TAG"; echo "MIGRATION=$MIGRATION"; } >> "$GITHUB_ENV" echo "prev=$PREV_TAG migration=$MIGRATION" - name: Render main env + certs run: | umask 077 mkdir -p stage/certs-main cat > stage/env.sh < stage/certs-main/ca.crt printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY" > stage/certs-main/gateway.key chmod 644 stage/certs-main/* - name: Deploy the main host run: | ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } ssh_main 'mkdir -p /opt/scrabble/compose' tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \ | ssh_main 'tar -C /opt/scrabble/compose -xzf -' tar -C deploy -czf - caddy otelcol prometheus tempo grafana \ | ssh_main 'tar -C /opt/scrabble -xzf -' tar -C stage -czf - certs-main \ | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -' scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh" echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin" ssh_main "TAG='$TAG' PREV_TAG='$PREV_TAG' MIGRATION='$MIGRATION' bash /opt/scrabble/compose/prod-deploy.sh" deploy-bot: needs: [build, deploy-main] runs-on: ubuntu-latest defaults: run: shell: bash env: TAG: ${{ needs.build.outputs.tag }} PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} TG_HOST: ${{ vars.PROD_TG_HOST }} MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }} TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }} PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }} PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }} PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }} LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }} TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }} TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }} TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }} TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }} TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }} steps: - uses: actions/checkout@v4 - name: Set up SSH run: | mkdir -p ~/.ssh && chmod 700 ~/.ssh printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - name: Render bot env + certs run: | umask 077 mkdir -p stage/certs-bot cat > stage/env.bot.sh < stage/certs-bot/ca.crt printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt printf '%s\n' "$PROD_BOTLINK_BOT_KEY" > stage/certs-bot/bot.key chmod 644 stage/certs-bot/* - name: Deploy the bot host run: | ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; } ssh_tg 'mkdir -p /opt/scrabble/compose' tar -C deploy -czf - docker-compose.bot.yml | ssh_tg 'tar -C /opt/scrabble/compose -xzf -' tar -C stage -czf - certs-bot \ | ssh_tg 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -' scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.bot.sh "deploy@$TG_HOST:/opt/scrabble/env.bot.sh" echo "$PROD_REGISTRY_PASSWORD" | ssh_tg "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin" ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose; docker compose -f docker-compose.bot.yml pull; docker compose -f docker-compose.bot.yml up -d' ssh_tg 'for i in $(seq 1 20); do s=$(docker inspect -f "{{.State.Status}}" scrabble-telegram-bot 2>/dev/null || echo missing) r=$(docker inspect -f "{{.State.Restarting}}" scrabble-telegram-bot 2>/dev/null || echo true) if [ "$s" = running ] && [ "$r" = false ]; then c1=$(docker inspect -f "{{.RestartCount}}" scrabble-telegram-bot); sleep 5 c2=$(docker inspect -f "{{.RestartCount}}" scrabble-telegram-bot) [ "$c1" = "$c2" ] && { echo "bot healthy"; exit 0; } fi sleep 3 done echo "bot not healthy:"; docker logs --tail 80 scrabble-telegram-bot; exit 1' verify: needs: [deploy-main, deploy-bot] runs-on: ubuntu-latest defaults: run: shell: bash env: PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }} steps: - name: Set up SSH run: | mkdir -p ~/.ssh && chmod 700 ~/.ssh printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - name: Verify the public site run: | domain="${CADDY_SITE_ADDRESS%% *}" ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "for i in \$(seq 1 20); do if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null && curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/app/ -o /dev/null && docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then echo 'public site + /app/ + backend healthy'; exit 0 fi sleep 5 done echo 'public verify failed; recent caddy + gateway + backend logs:' docker logs --tail 40 scrabble-caddy; docker logs --tail 40 scrabble-gateway; docker logs --tail 40 scrabble-backend exit 1"