scrabble-game/deploy/prod-deploy.sh

#!/usr/bin/env bash
# Production main-host deploy driver. Runs ON the main host, invoked over SSH by
# .gitea/workflows/prod-deploy.yaml as the deploy user (which must already be
# `docker login`ed to the registry). It pulls the images at the new tag and rolls
# the stack ONE service at a time in dependency order (least -> most dependent),
# health-checking after each; any failure rolls the whole stack back to the
# previously deployed tag.
#
# A schema migration adds a maintenance window: the backend (the only writer) is
# stopped so a consistent pg_dump is taken before the new backend migrates forward.
# Image rollback alone is safe under the expand-contract migration rule, so the
# automatic rollback never touches the database; the dump is kept for a MANUAL
# restore if a migration turned out to be destructive (see deploy/prod/README.md).
#
# Required env (exported by the workflow over SSH):
#   REGISTRY   registry namespace, e.g. docker.iliadenisov.ru/developer
#   TAG        new image tag (the deployed git SHA)
#   PREV_TAG   previously deployed tag, or "none" on the first deploy
#   MIGRATION  "1" when the deploy carries a schema migration, else "0"
# Optional: COMPOSE_DIR ENV_FILE DUMP_DIR STATE_FILE POSTGRES_USER POSTGRES_DB
set -uo pipefail

# Runtime compose vars (POSTGRES_*, GM_*, GRAFANA_*, CADDY_*, TELEGRAM_*, REGISTRY,
# SCRABBLE_CONFIG_DIR, ...) come from a shell-sourceable env file the workflow writes
# with single-quoted values. Exporting them into the process environment lets compose
# interpolate ${...} without re-parsing the value — a plain --env-file would mangle the
# literal '$' in the bcrypt GM_BASICAUTH_HASH.
ENV_FILE="${ENV_FILE:-/opt/scrabble/env.sh}"
# shellcheck disable=SC1090
[ -f "$ENV_FILE" ] && . "$ENV_FILE"

REGISTRY="${REGISTRY:?REGISTRY required (env.sh)}"
TAG="${TAG:?TAG required}"
PREV_TAG="${PREV_TAG:-none}"
MIGRATION="${MIGRATION:-0}"
COMPOSE_DIR="${COMPOSE_DIR:-/opt/scrabble/compose}"
DUMP_DIR="${DUMP_DIR:-/opt/scrabble/dumps}"
STATE_FILE="${STATE_FILE:-/opt/scrabble/DEPLOYED_TAG}"
# The prior deployed tag, preserved on every successful deploy so prod-rollback can
# target "the previous version" with no operator input.
PREV_STATE_FILE="${PREV_STATE_FILE:-/opt/scrabble/PREVIOUS_TAG}"
PG_USER="${POSTGRES_USER:-scrabble}"
PG_DB="${POSTGRES_DB:-scrabble}"

# Edge maintenance page. caddy serves a static 503 "works in progress" page for the
# user-facing routes while this flag file exists (deploy/caddy/Caddyfile). We hold it
# across the roll / migration window and clear it on ANY exit — success, a health
# failure + rollback, or an unexpected error — via the trap, so users get a graceful
# page instead of raw mid-swap 502s and the flag can never get stuck on.
MAINT_FLAG="${MAINT_FLAG:-${SCRABBLE_CONFIG_DIR:-/opt/scrabble}/caddy/on}"
maint_off() { rm -f "$MAINT_FLAG" 2>/dev/null || true; }
trap maint_off EXIT

cd "$COMPOSE_DIR" || { echo "compose dir $COMPOSE_DIR missing"; exit 1; }
export REGISTRY
# otelcol joins the host docker group to read the socket; the GID varies per host.
DOCKER_GID="$(getent group docker | cut -d: -f3)"
export DOCKER_GID

dc() { docker compose -f docker-compose.yml -f docker-compose.prod.yml "$@"; }
use_tag() { export TAG="$1"; }

# --- health probes (one-off containers on the contour networks, like CI) --------
_probe() { docker run --rm --network "$1" alpine:3.20 wget -q -T 5 -O /dev/null "$2"; }
health_backend() { for _ in $(seq 1 20); do _probe scrabble-internal http://backend:8080/readyz && return 0; sleep 3; done; return 1; }
health_landing() { for _ in $(seq 1 20); do _probe scrabble-internal http://landing:80/ && return 0; sleep 3; done; return 1; }
health_postgres() { for _ in $(seq 1 30); do [ "$(docker inspect -f '{{.State.Health.Status}}' scrabble-postgres 2>/dev/null)" = healthy ] && return 0; sleep 2; done; return 1; }
health_running() { # health_running <container>: running, not restarting, stable restart count
  local n="$1" s r c1 c2
  for _ in $(seq 1 20); do
    s="$(docker inspect -f '{{.State.Status}}' "$n" 2>/dev/null || echo missing)"
    r="$(docker inspect -f '{{.State.Restarting}}' "$n" 2>/dev/null || echo true)"
    if [ "$s" = running ] && [ "$r" = false ]; then
      c1="$(docker inspect -f '{{.RestartCount}}' "$n")"; sleep 5
      c2="$(docker inspect -f '{{.RestartCount}}' "$n")"
      [ "$c1" = "$c2" ] && return 0
    fi
    sleep 3
  done
  return 1
}

roll() { # roll <service> <health-cmd...>
  local svc="$1"; shift
  echo ">>> rolling $svc -> $TAG"
  # caddy's image is pinned (caddy:2-alpine, no $TAG) and its Caddyfile is bind-mounted, so a
  # config-only change leaves the compose definition unchanged: `up -d` treats the container as
  # current and does not recreate it, and admin is off so there is no hot reload — the new
  # Caddyfile would never load. Force a recreate for caddy so config changes always apply; every
  # other service already recreates on its new $TAG image.
  local recreate=(); [ "$svc" = caddy ] && recreate=(--force-recreate)
  dc up -d --no-build --no-deps "${recreate[@]}" "$svc" || return 1
  "$@" || { echo "!!! $svc failed health check"; return 1; }
  echo "<<< $svc healthy"
}

rollback() {
  echo "########## ROLLBACK -> $PREV_TAG ##########"
  if [ "$PREV_TAG" = none ]; then
    echo "no previous tag (first deploy): cannot roll back; leaving the stack up for inspection."
    return
  fi
  use_tag "$PREV_TAG"
  dc up -d --no-build --remove-orphans
  echo "rolled back to $PREV_TAG."
  [ "$MIGRATION" = 1 ] && echo "NOTE: the DB is forward-migrated; a pre-deploy dump is in $DUMP_DIR — restore manually ONLY if the migration was destructive (see deploy/README.md, prod runbook)."
}

commit_tag() {
  # Record the just-deployed tag as current, preserving the prior one as previous.
  [ -f "$STATE_FILE" ] && cp "$STATE_FILE" "$PREV_STATE_FILE"
  echo "$TAG" > "$STATE_FILE"
}

mkdir -p "$DUMP_DIR"
echo "=== prod deploy: tag=$TAG prev=$PREV_TAG migration=$MIGRATION ==="
use_tag "$TAG"
dc pull

# First deploy: nothing to roll from; bring the whole stack up and gate on health.
if [ -z "$(docker ps -aq -f name=scrabble-backend)" ]; then
  echo "first deploy: bringing the whole stack up"
  dc up -d --no-build --remove-orphans || { echo "compose up failed"; exit 1; }
  health_backend || { echo "backend not ready"; exit 1; }
  health_landing || { echo "landing not ready"; exit 1; }
  commit_tag
  echo "first deploy healthy ($TAG)."
  exit 0
fi

# Existing stack: raise the maintenance page for the whole roll (and any migration
# window). It shows once caddy carries the gate (from this feature's own deploy
# onward); the EXIT trap lowers it however this run ends.
mkdir -p "$(dirname "$MAINT_FLAG")"
: > "$MAINT_FLAG"
echo "maintenance page raised ($MAINT_FLAG)"

# Migration deploy: freeze writes and snapshot a consistent dump before migrating.
if [ "$MIGRATION" = 1 ]; then
  echo "migration deploy: opening maintenance window (stopping the backend = the only writer)"
  dc stop backend
  dump="$DUMP_DIR/pre-$TAG-$(date +%Y%m%d-%H%M%S).sql"
  if ! docker exec scrabble-postgres pg_dump -U "$PG_USER" -d "$PG_DB" -n backend > "$dump"; then
    echo "pg_dump failed; restarting the old backend and aborting"
    dc start backend
    exit 1
  fi
  echo "consistent dump: $dump"
fi

# Roll one service at a time, least -> most dependent; any failure rolls everything back.
roll postgres  health_postgres                              || { rollback; exit 1; }
roll renderer  health_running scrabble-renderer             || { rollback; exit 1; }
roll backend   health_backend                               || { rollback; exit 1; }
roll gateway   health_running scrabble-gateway              || { rollback; exit 1; }
roll landing   health_landing                               || { rollback; exit 1; }
roll validator health_running scrabble-telegram-validator   || { rollback; exit 1; }
roll caddy     health_running scrabble-caddy                || { rollback; exit 1; }

# Observability + node_exporter: bring up the remainder and pick up any config changes.
dc up -d --no-build --remove-orphans || { rollback; exit 1; }

# Final internal sanity before committing the new tag.
health_backend || { rollback; exit 1; }
commit_tag
echo "=== deploy healthy ($TAG) ==="