diff --git a/.gitea/workflows/prod-deploy.yaml b/.gitea/workflows/prod-deploy.yaml new file mode 100644 index 0000000..8ebd502 --- /dev/null +++ b/.gitea/workflows/prod-deploy.yaml @@ -0,0 +1,219 @@ +# Manual production rollout. Runs ONLY from master, ONLY on an explicit +# workflow_dispatch with confirm=deploy (development->master is merged + green first; +# this is the separate, deliberate prod step). It builds and pushes the images to the +# registry, then deploys over SSH: the main host via deploy/prod-deploy.sh (rolling, +# health-gated, auto-rollback; a maintenance window + consistent dump on a migration), +# then the bot host. See deploy/README.md (prod runbook) for operator steps. +name: prod-deploy +run-name: "prod deploy ${{ github.sha }}" + +on: + workflow_dispatch: + inputs: + confirm: + description: 'Type "deploy" to confirm a production rollout from master.' + required: true + default: "" + +permissions: + contents: read + +jobs: + deploy: + if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'deploy' }} + runs-on: ubuntu-latest + defaults: + run: + shell: bash + env: + NO_COLOR: "1" + DOCKER_CLI_HINTS: "false" + REGISTRY: docker.iliadenisov.ru/developer + # SSH + registry + PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} + PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} + PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} + PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} + MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} + TG_HOST: ${{ vars.PROD_TG_HOST }} + # SPA build args (baked into the gateway/landing images) + VITE_TELEGRAM_BOT_ID: ${{ vars.PROD_VITE_TELEGRAM_BOT_ID }} + VITE_TELEGRAM_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }} + VITE_TELEGRAM_GAME_CHANNEL_NAME: ${{ vars.PROD_VITE_TELEGRAM_GAME_CHANNEL_NAME }} + VITE_GATEWAY_URL: ${{ vars.PROD_VITE_GATEWAY_URL }} + # Runtime secrets + POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }} + GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }} + GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }} + TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }} + TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }} + PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }} + PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }} + PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }} + PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }} + PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }} + # Runtime variables + GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }} + GRAFANA_ROOT_URL: ${{ vars.PROD_GRAFANA_ROOT_URL }} + CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }} + LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }} + DICT_VERSION: ${{ vars.PROD_DICT_VERSION }} + POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }} + POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }} + TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }} + TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }} + TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }} + TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }} + TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Compute image tag and app version + run: | + echo "TAG=$(git rev-parse --short=12 HEAD)" >> "$GITHUB_ENV" + echo "APP_VERSION=$(git describe --tags --always)" >> "$GITHUB_ENV" + + - name: Registry login + run: echo "$PROD_REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$PROD_REGISTRY_USER" --password-stdin + + - name: Build and push images + working-directory: deploy + run: | + export TAG SCRABBLE_CONFIG_DIR=. + # The four main-stack images via compose (reuses the compose build args); the + # bot separately, since it is profiled out of the prod compose. + docker compose -f docker-compose.yml -f docker-compose.prod.yml build + docker compose -f docker-compose.yml -f docker-compose.prod.yml push backend gateway landing validator + docker build -f ../platform/telegram/Dockerfile --target bot -t "$REGISTRY/scrabble-telegram-bot:$TAG" .. + docker push "$REGISTRY/scrabble-telegram-bot:$TAG" + + - name: Set up SSH + run: | + mkdir -p ~/.ssh && chmod 700 ~/.ssh + printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy + chmod 600 ~/.ssh/id_deploy + printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts + + - name: Determine previous tag and migration + run: | + ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } + PREV_TAG="$(ssh_main 'cat /opt/scrabble/DEPLOYED_TAG 2>/dev/null || echo none')" + MIGRATION=0 + if [ "$PREV_TAG" != none ]; then + if ! git cat-file -e "$PREV_TAG^{commit}" 2>/dev/null; then + MIGRATION=1 # unknown previous tag: take the safe window + dump + elif git diff --name-only "$PREV_TAG..$TAG" -- backend/internal/postgres/migrations/ | grep -q .; then + MIGRATION=1 + fi + fi + echo "PREV_TAG=$PREV_TAG" >> "$GITHUB_ENV" + echo "MIGRATION=$MIGRATION" >> "$GITHUB_ENV" + echo "prev=$PREV_TAG migration=$MIGRATION" + + - name: Render env files and certs + run: | + umask 077 + mkdir -p stage/certs-main stage/certs-bot + # Main-host runtime env (single-quoted so the literal '$' in the bcrypt hash + # survives; prod-deploy.sh sources this, compose reads it from the environment). + cat > stage/env.sh < stage/env.bot.sh < stage/certs-main/ca.crt + printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt + printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY" > stage/certs-main/gateway.key + printf '%s\n' "$PROD_BOTLINK_CA" > stage/certs-bot/ca.crt + printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt + printf '%s\n' "$PROD_BOTLINK_BOT_KEY" > stage/certs-bot/bot.key + chmod 644 stage/certs-main/* stage/certs-bot/* + + - name: Deploy the main host + run: | + ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } + ssh_main 'mkdir -p /opt/scrabble/compose' + # Compose files + config + script. + tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \ + | ssh_main 'tar -C /opt/scrabble/compose -xzf -' + tar -C deploy -czf - caddy otelcol prometheus tempo grafana \ + | ssh_main 'tar -C /opt/scrabble -xzf -' + tar -C stage -czf - certs-main \ + | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -' + scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh" + # Registry login on the host so compose can pull the private images. + echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin" + ssh_main "TAG='$TAG' PREV_TAG='$PREV_TAG' MIGRATION='$MIGRATION' bash /opt/scrabble/compose/prod-deploy.sh" + + - name: Deploy the bot host + run: | + ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; } + ssh_tg 'mkdir -p /opt/scrabble/compose' + tar -C deploy -czf - docker-compose.bot.yml | ssh_tg 'tar -C /opt/scrabble/compose -xzf -' + tar -C stage -czf - certs-bot \ + | ssh_tg 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -' + scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.bot.sh "deploy@$TG_HOST:/opt/scrabble/env.bot.sh" + echo "$PROD_REGISTRY_PASSWORD" | ssh_tg "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin" + ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose; + docker compose -f docker-compose.bot.yml pull; + docker compose -f docker-compose.bot.yml up -d' + # Bot liveness: running, not restarting, stable restart count. + ssh_tg 'for i in $(seq 1 20); do + s=$(docker inspect -f "{{.State.Status}}" scrabble-telegram-bot 2>/dev/null || echo missing) + r=$(docker inspect -f "{{.State.Restarting}}" scrabble-telegram-bot 2>/dev/null || echo true) + if [ "$s" = running ] && [ "$r" = false ]; then + c1=$(docker inspect -f "{{.RestartCount}}" scrabble-telegram-bot); sleep 5 + c2=$(docker inspect -f "{{.RestartCount}}" scrabble-telegram-bot) + [ "$c1" = "$c2" ] && { echo "bot healthy"; exit 0; } + fi + sleep 3 + done + echo "bot not healthy:"; docker logs --tail 80 scrabble-telegram-bot; exit 1' + + - name: Verify the public site + run: | + ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } + domain="${CADDY_SITE_ADDRESS%% *}" # first domain of "erudit-game.ru www..." + # Probe through caddy with the right vhost (force-resolved to the host); -k + # tolerates an ACME-pending cert. Also check the backend is actually ready. + ssh_main "for i in \$(seq 1 20); do + if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null && + curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/app/ -o /dev/null && + docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then + echo 'public site + /app/ + backend healthy'; exit 0 + fi + sleep 5 + done + echo 'public verify failed; recent caddy + gateway + backend logs:' + docker logs --tail 40 scrabble-caddy; docker logs --tail 40 scrabble-gateway; docker logs --tail 40 scrabble-backend + exit 1" diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index 6b45826..03e7be7 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -23,6 +23,9 @@ services: memory: 96M gateway: + # Prod pulls the pushed image by tag instead of building locally; the base + # build: section stays dormant because the deploy always pulls first. + image: ${REGISTRY:?set REGISTRY}/scrabble-gateway:${TAG:?set TAG} ports: - "9443:9443" environment: @@ -35,6 +38,7 @@ services: memory: 384M backend: + image: ${REGISTRY:?set REGISTRY}/scrabble-backend:${TAG:?set TAG} deploy: resources: limits: @@ -47,12 +51,14 @@ services: memory: 384M validator: + image: ${REGISTRY:?set REGISTRY}/scrabble-telegram-validator:${TAG:?set TAG} deploy: resources: limits: memory: 96M landing: + image: ${REGISTRY:?set REGISTRY}/scrabble-landing:${TAG:?set TAG} deploy: resources: limits: diff --git a/deploy/prod-deploy.sh b/deploy/prod-deploy.sh new file mode 100755 index 0000000..aec789b --- /dev/null +++ b/deploy/prod-deploy.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Production main-host deploy driver. Runs ON the main host, invoked over SSH by +# .gitea/workflows/prod-deploy.yaml as the deploy user (which must already be +# `docker login`ed to the registry). It pulls the images at the new tag and rolls +# the stack ONE service at a time in dependency order (least -> most dependent), +# health-checking after each; any failure rolls the whole stack back to the +# previously deployed tag. +# +# A schema migration adds a maintenance window: the backend (the only writer) is +# stopped so a consistent pg_dump is taken before the new backend migrates forward. +# Image rollback alone is safe under the expand-contract migration rule, so the +# automatic rollback never touches the database; the dump is kept for a MANUAL +# restore if a migration turned out to be destructive (see deploy/prod/README.md). +# +# Required env (exported by the workflow over SSH): +# REGISTRY registry namespace, e.g. docker.iliadenisov.ru/developer +# TAG new image tag (the deployed git SHA) +# PREV_TAG previously deployed tag, or "none" on the first deploy +# MIGRATION "1" when the deploy carries a schema migration, else "0" +# Optional: COMPOSE_DIR ENV_FILE DUMP_DIR STATE_FILE POSTGRES_USER POSTGRES_DB +set -uo pipefail + +# Runtime compose vars (POSTGRES_*, GM_*, GRAFANA_*, CADDY_*, TELEGRAM_*, REGISTRY, +# SCRABBLE_CONFIG_DIR, ...) come from a shell-sourceable env file the workflow writes +# with single-quoted values. Exporting them into the process environment lets compose +# interpolate ${...} without re-parsing the value — a plain --env-file would mangle the +# literal '$' in the bcrypt GM_BASICAUTH_HASH. +ENV_FILE="${ENV_FILE:-/opt/scrabble/env.sh}" +# shellcheck disable=SC1090 +[ -f "$ENV_FILE" ] && . "$ENV_FILE" + +REGISTRY="${REGISTRY:?REGISTRY required (env.sh)}" +TAG="${TAG:?TAG required}" +PREV_TAG="${PREV_TAG:-none}" +MIGRATION="${MIGRATION:-0}" +COMPOSE_DIR="${COMPOSE_DIR:-/opt/scrabble/compose}" +DUMP_DIR="${DUMP_DIR:-/opt/scrabble/dumps}" +STATE_FILE="${STATE_FILE:-/opt/scrabble/DEPLOYED_TAG}" +PG_USER="${POSTGRES_USER:-scrabble}" +PG_DB="${POSTGRES_DB:-scrabble}" + +cd "$COMPOSE_DIR" +export REGISTRY +# otelcol joins the host docker group to read the socket; the GID varies per host. +export DOCKER_GID="$(getent group docker | cut -d: -f3)" + +dc() { docker compose -f docker-compose.yml -f docker-compose.prod.yml "$@"; } +use_tag() { export TAG="$1"; } + +# --- health probes (one-off containers on the contour networks, like CI) -------- +_probe() { docker run --rm --network "$1" alpine:3.20 wget -q -T 5 -O /dev/null "$2"; } +health_backend() { for _ in $(seq 1 20); do _probe scrabble-internal http://backend:8080/readyz && return 0; sleep 3; done; return 1; } +health_landing() { for _ in $(seq 1 20); do _probe scrabble-internal http://landing:80/ && return 0; sleep 3; done; return 1; } +health_postgres() { for _ in $(seq 1 30); do [ "$(docker inspect -f '{{.State.Health.Status}}' scrabble-postgres 2>/dev/null)" = healthy ] && return 0; sleep 2; done; return 1; } +health_running() { # health_running : running, not restarting, stable restart count + local n="$1" s r c1 c2 + for _ in $(seq 1 20); do + s="$(docker inspect -f '{{.State.Status}}' "$n" 2>/dev/null || echo missing)" + r="$(docker inspect -f '{{.State.Restarting}}' "$n" 2>/dev/null || echo true)" + if [ "$s" = running ] && [ "$r" = false ]; then + c1="$(docker inspect -f '{{.RestartCount}}' "$n")"; sleep 5 + c2="$(docker inspect -f '{{.RestartCount}}' "$n")" + [ "$c1" = "$c2" ] && return 0 + fi + sleep 3 + done + return 1 +} + +roll() { # roll + local svc="$1"; shift + echo ">>> rolling $svc -> $TAG" + dc up -d --no-build --no-deps "$svc" || return 1 + "$@" || { echo "!!! $svc failed health check"; return 1; } + echo "<<< $svc healthy" +} + +rollback() { + echo "########## ROLLBACK -> $PREV_TAG ##########" + if [ "$PREV_TAG" = none ]; then + echo "no previous tag (first deploy): cannot roll back; leaving the stack up for inspection." + return + fi + use_tag "$PREV_TAG" + dc up -d --no-build --remove-orphans + echo "rolled back to $PREV_TAG." + [ "$MIGRATION" = 1 ] && echo "NOTE: the DB is forward-migrated; a pre-deploy dump is in $DUMP_DIR — restore manually ONLY if the migration was destructive (see deploy/README.md, prod runbook)." +} + +mkdir -p "$DUMP_DIR" +echo "=== prod deploy: tag=$TAG prev=$PREV_TAG migration=$MIGRATION ===" +use_tag "$TAG" +dc pull + +# First deploy: nothing to roll from; bring the whole stack up and gate on health. +if [ -z "$(docker ps -aq -f name=scrabble-backend)" ]; then + echo "first deploy: bringing the whole stack up" + dc up -d --no-build --remove-orphans || { echo "compose up failed"; exit 1; } + health_backend || { echo "backend not ready"; exit 1; } + health_landing || { echo "landing not ready"; exit 1; } + echo "$TAG" > "$STATE_FILE" + echo "first deploy healthy ($TAG)." + exit 0 +fi + +# Migration deploy: freeze writes and snapshot a consistent dump before migrating. +if [ "$MIGRATION" = 1 ]; then + echo "migration deploy: opening maintenance window (stopping the backend = the only writer)" + dc stop backend + dump="$DUMP_DIR/pre-$TAG-$(date +%Y%m%d-%H%M%S).sql" + if ! docker exec scrabble-postgres pg_dump -U "$PG_USER" -d "$PG_DB" -n backend > "$dump"; then + echo "pg_dump failed; restarting the old backend and aborting" + dc start backend + exit 1 + fi + echo "consistent dump: $dump" +fi + +# Roll one service at a time, least -> most dependent; any failure rolls everything back. +roll postgres health_postgres || { rollback; exit 1; } +roll backend health_backend || { rollback; exit 1; } +roll gateway health_running scrabble-gateway || { rollback; exit 1; } +roll landing health_landing || { rollback; exit 1; } +roll validator health_running scrabble-telegram-validator || { rollback; exit 1; } +roll caddy health_running scrabble-caddy || { rollback; exit 1; } + +# Observability + node_exporter: bring up the remainder and pick up any config changes. +dc up -d --no-build --remove-orphans || { rollback; exit 1; } + +# Final internal sanity before committing the new tag. +health_backend || { rollback; exit 1; } +echo "$TAG" > "$STATE_FILE" +echo "=== deploy healthy ($TAG) ==="