#!/usr/bin/env bash # Production main-host deploy driver. Runs ON the main host, invoked over SSH by # .gitea/workflows/prod-deploy.yaml as the deploy user (which must already be # `docker login`ed to the registry). It pulls the images at the new tag and rolls # the stack ONE service at a time in dependency order (least -> most dependent), # health-checking after each; any failure rolls the whole stack back to the # previously deployed tag. # # A schema migration adds a maintenance window: the backend (the only writer) is # stopped so a consistent pg_dump is taken before the new backend migrates forward. # Image rollback alone is safe under the expand-contract migration rule, so the # automatic rollback never touches the database; the dump is kept for a MANUAL # restore if a migration turned out to be destructive (see deploy/prod/README.md). # # Required env (exported by the workflow over SSH): # REGISTRY registry namespace, e.g. docker.iliadenisov.ru/developer # TAG new image tag (the deployed git SHA) # PREV_TAG previously deployed tag, or "none" on the first deploy # MIGRATION "1" when the deploy carries a schema migration, else "0" # Optional: COMPOSE_DIR ENV_FILE DUMP_DIR STATE_FILE POSTGRES_USER POSTGRES_DB set -uo pipefail # Runtime compose vars (POSTGRES_*, GM_*, GRAFANA_*, CADDY_*, TELEGRAM_*, REGISTRY, # SCRABBLE_CONFIG_DIR, ...) come from a shell-sourceable env file the workflow writes # with single-quoted values. Exporting them into the process environment lets compose # interpolate ${...} without re-parsing the value — a plain --env-file would mangle the # literal '$' in the bcrypt GM_BASICAUTH_HASH. ENV_FILE="${ENV_FILE:-/opt/scrabble/env.sh}" # shellcheck disable=SC1090 [ -f "$ENV_FILE" ] && . "$ENV_FILE" REGISTRY="${REGISTRY:?REGISTRY required (env.sh)}" TAG="${TAG:?TAG required}" PREV_TAG="${PREV_TAG:-none}" MIGRATION="${MIGRATION:-0}" COMPOSE_DIR="${COMPOSE_DIR:-/opt/scrabble/compose}" DUMP_DIR="${DUMP_DIR:-/opt/scrabble/dumps}" STATE_FILE="${STATE_FILE:-/opt/scrabble/DEPLOYED_TAG}" PG_USER="${POSTGRES_USER:-scrabble}" PG_DB="${POSTGRES_DB:-scrabble}" cd "$COMPOSE_DIR" export REGISTRY # otelcol joins the host docker group to read the socket; the GID varies per host. export DOCKER_GID="$(getent group docker | cut -d: -f3)" dc() { docker compose -f docker-compose.yml -f docker-compose.prod.yml "$@"; } use_tag() { export TAG="$1"; } # --- health probes (one-off containers on the contour networks, like CI) -------- _probe() { docker run --rm --network "$1" alpine:3.20 wget -q -T 5 -O /dev/null "$2"; } health_backend() { for _ in $(seq 1 20); do _probe scrabble-internal http://backend:8080/readyz && return 0; sleep 3; done; return 1; } health_landing() { for _ in $(seq 1 20); do _probe scrabble-internal http://landing:80/ && return 0; sleep 3; done; return 1; } health_postgres() { for _ in $(seq 1 30); do [ "$(docker inspect -f '{{.State.Health.Status}}' scrabble-postgres 2>/dev/null)" = healthy ] && return 0; sleep 2; done; return 1; } health_running() { # health_running : running, not restarting, stable restart count local n="$1" s r c1 c2 for _ in $(seq 1 20); do s="$(docker inspect -f '{{.State.Status}}' "$n" 2>/dev/null || echo missing)" r="$(docker inspect -f '{{.State.Restarting}}' "$n" 2>/dev/null || echo true)" if [ "$s" = running ] && [ "$r" = false ]; then c1="$(docker inspect -f '{{.RestartCount}}' "$n")"; sleep 5 c2="$(docker inspect -f '{{.RestartCount}}' "$n")" [ "$c1" = "$c2" ] && return 0 fi sleep 3 done return 1 } roll() { # roll local svc="$1"; shift echo ">>> rolling $svc -> $TAG" dc up -d --no-build --no-deps "$svc" || return 1 "$@" || { echo "!!! $svc failed health check"; return 1; } echo "<<< $svc healthy" } rollback() { echo "########## ROLLBACK -> $PREV_TAG ##########" if [ "$PREV_TAG" = none ]; then echo "no previous tag (first deploy): cannot roll back; leaving the stack up for inspection." return fi use_tag "$PREV_TAG" dc up -d --no-build --remove-orphans echo "rolled back to $PREV_TAG." [ "$MIGRATION" = 1 ] && echo "NOTE: the DB is forward-migrated; a pre-deploy dump is in $DUMP_DIR — restore manually ONLY if the migration was destructive (see deploy/README.md, prod runbook)." } mkdir -p "$DUMP_DIR" echo "=== prod deploy: tag=$TAG prev=$PREV_TAG migration=$MIGRATION ===" use_tag "$TAG" dc pull # First deploy: nothing to roll from; bring the whole stack up and gate on health. if [ -z "$(docker ps -aq -f name=scrabble-backend)" ]; then echo "first deploy: bringing the whole stack up" dc up -d --no-build --remove-orphans || { echo "compose up failed"; exit 1; } health_backend || { echo "backend not ready"; exit 1; } health_landing || { echo "landing not ready"; exit 1; } echo "$TAG" > "$STATE_FILE" echo "first deploy healthy ($TAG)." exit 0 fi # Migration deploy: freeze writes and snapshot a consistent dump before migrating. if [ "$MIGRATION" = 1 ]; then echo "migration deploy: opening maintenance window (stopping the backend = the only writer)" dc stop backend dump="$DUMP_DIR/pre-$TAG-$(date +%Y%m%d-%H%M%S).sql" if ! docker exec scrabble-postgres pg_dump -U "$PG_USER" -d "$PG_DB" -n backend > "$dump"; then echo "pg_dump failed; restarting the old backend and aborting" dc start backend exit 1 fi echo "consistent dump: $dump" fi # Roll one service at a time, least -> most dependent; any failure rolls everything back. roll postgres health_postgres || { rollback; exit 1; } roll backend health_backend || { rollback; exit 1; } roll gateway health_running scrabble-gateway || { rollback; exit 1; } roll landing health_landing || { rollback; exit 1; } roll validator health_running scrabble-telegram-validator || { rollback; exit 1; } roll caddy health_running scrabble-caddy || { rollback; exit 1; } # Observability + node_exporter: bring up the remainder and pick up any config changes. dc up -d --no-build --remove-orphans || { rollback; exit 1; } # Final internal sanity before committing the new tag. health_backend || { rollback; exit 1; } echo "$TAG" > "$STATE_FILE" echo "=== deploy healthy ($TAG) ==="