feat(deploy): manual prod-deploy pipeline with rolling rollback (Stage 18)
A workflow_dispatch-only rollout from master (confirm=deploy): - .gitea/workflows/prod-deploy.yaml builds + pushes the images to the registry, ships the compose/config/certs/env over SSH, deploys the main host via prod-deploy.sh, then the bot host, then verifies the public site. - deploy/prod-deploy.sh rolls the main stack one service at a time in dependency order (postgres->backend->gateway->landing->validator->caddy), health-checking after each; any failure rolls the whole stack back to the previous tag. A schema migration adds a maintenance window: the backend (sole writer) is stopped for a consistent pg_dump before migrating; image rollback stays DB-safe (expand-contract), the dump is kept for a manual restore. - prod overlay: pull the four main images from the registry by tag. - Runtime secrets reach the host via a sourced env.sh (single-quoted values keep the bcrypt hash's literal $ intact, unlike a --env-file).
This commit is contained in:
Ilia Denisov
Executable
+133
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env bash
|
||||
# Production main-host deploy driver. Runs ON the main host, invoked over SSH by
|
||||
# .gitea/workflows/prod-deploy.yaml as the deploy user (which must already be
|
||||
# `docker login`ed to the registry). It pulls the images at the new tag and rolls
|
||||
# the stack ONE service at a time in dependency order (least -> most dependent),
|
||||
# health-checking after each; any failure rolls the whole stack back to the
|
||||
# previously deployed tag.
|
||||
#
|
||||
# A schema migration adds a maintenance window: the backend (the only writer) is
|
||||
# stopped so a consistent pg_dump is taken before the new backend migrates forward.
|
||||
# Image rollback alone is safe under the expand-contract migration rule, so the
|
||||
# automatic rollback never touches the database; the dump is kept for a MANUAL
|
||||
# restore if a migration turned out to be destructive (see deploy/prod/README.md).
|
||||
#
|
||||
# Required env (exported by the workflow over SSH):
|
||||
# REGISTRY registry namespace, e.g. docker.iliadenisov.ru/developer
|
||||
# TAG new image tag (the deployed git SHA)
|
||||
# PREV_TAG previously deployed tag, or "none" on the first deploy
|
||||
# MIGRATION "1" when the deploy carries a schema migration, else "0"
|
||||
# Optional: COMPOSE_DIR ENV_FILE DUMP_DIR STATE_FILE POSTGRES_USER POSTGRES_DB
|
||||
set -uo pipefail
|
||||
|
||||
# Runtime compose vars (POSTGRES_*, GM_*, GRAFANA_*, CADDY_*, TELEGRAM_*, REGISTRY,
|
||||
# SCRABBLE_CONFIG_DIR, ...) come from a shell-sourceable env file the workflow writes
|
||||
# with single-quoted values. Exporting them into the process environment lets compose
|
||||
# interpolate ${...} without re-parsing the value — a plain --env-file would mangle the
|
||||
# literal '$' in the bcrypt GM_BASICAUTH_HASH.
|
||||
ENV_FILE="${ENV_FILE:-/opt/scrabble/env.sh}"
|
||||
# shellcheck disable=SC1090
|
||||
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
|
||||
|
||||
REGISTRY="${REGISTRY:?REGISTRY required (env.sh)}"
|
||||
TAG="${TAG:?TAG required}"
|
||||
PREV_TAG="${PREV_TAG:-none}"
|
||||
MIGRATION="${MIGRATION:-0}"
|
||||
COMPOSE_DIR="${COMPOSE_DIR:-/opt/scrabble/compose}"
|
||||
DUMP_DIR="${DUMP_DIR:-/opt/scrabble/dumps}"
|
||||
STATE_FILE="${STATE_FILE:-/opt/scrabble/DEPLOYED_TAG}"
|
||||
PG_USER="${POSTGRES_USER:-scrabble}"
|
||||
PG_DB="${POSTGRES_DB:-scrabble}"
|
||||
|
||||
cd "$COMPOSE_DIR"
|
||||
export REGISTRY
|
||||
# otelcol joins the host docker group to read the socket; the GID varies per host.
|
||||
export DOCKER_GID="$(getent group docker | cut -d: -f3)"
|
||||
|
||||
dc() { docker compose -f docker-compose.yml -f docker-compose.prod.yml "$@"; }
|
||||
use_tag() { export TAG="$1"; }
|
||||
|
||||
# --- health probes (one-off containers on the contour networks, like CI) --------
|
||||
_probe() { docker run --rm --network "$1" alpine:3.20 wget -q -T 5 -O /dev/null "$2"; }
|
||||
health_backend() { for _ in $(seq 1 20); do _probe scrabble-internal http://backend:8080/readyz && return 0; sleep 3; done; return 1; }
|
||||
health_landing() { for _ in $(seq 1 20); do _probe scrabble-internal http://landing:80/ && return 0; sleep 3; done; return 1; }
|
||||
health_postgres() { for _ in $(seq 1 30); do [ "$(docker inspect -f '{{.State.Health.Status}}' scrabble-postgres 2>/dev/null)" = healthy ] && return 0; sleep 2; done; return 1; }
|
||||
health_running() { # health_running <container>: running, not restarting, stable restart count
|
||||
local n="$1" s r c1 c2
|
||||
for _ in $(seq 1 20); do
|
||||
s="$(docker inspect -f '{{.State.Status}}' "$n" 2>/dev/null || echo missing)"
|
||||
r="$(docker inspect -f '{{.State.Restarting}}' "$n" 2>/dev/null || echo true)"
|
||||
if [ "$s" = running ] && [ "$r" = false ]; then
|
||||
c1="$(docker inspect -f '{{.RestartCount}}' "$n")"; sleep 5
|
||||
c2="$(docker inspect -f '{{.RestartCount}}' "$n")"
|
||||
[ "$c1" = "$c2" ] && return 0
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
roll() { # roll <service> <health-cmd...>
|
||||
local svc="$1"; shift
|
||||
echo ">>> rolling $svc -> $TAG"
|
||||
dc up -d --no-build --no-deps "$svc" || return 1
|
||||
"$@" || { echo "!!! $svc failed health check"; return 1; }
|
||||
echo "<<< $svc healthy"
|
||||
}
|
||||
|
||||
rollback() {
|
||||
echo "########## ROLLBACK -> $PREV_TAG ##########"
|
||||
if [ "$PREV_TAG" = none ]; then
|
||||
echo "no previous tag (first deploy): cannot roll back; leaving the stack up for inspection."
|
||||
return
|
||||
fi
|
||||
use_tag "$PREV_TAG"
|
||||
dc up -d --no-build --remove-orphans
|
||||
echo "rolled back to $PREV_TAG."
|
||||
[ "$MIGRATION" = 1 ] && echo "NOTE: the DB is forward-migrated; a pre-deploy dump is in $DUMP_DIR — restore manually ONLY if the migration was destructive (see deploy/README.md, prod runbook)."
|
||||
}
|
||||
|
||||
mkdir -p "$DUMP_DIR"
|
||||
echo "=== prod deploy: tag=$TAG prev=$PREV_TAG migration=$MIGRATION ==="
|
||||
use_tag "$TAG"
|
||||
dc pull
|
||||
|
||||
# First deploy: nothing to roll from; bring the whole stack up and gate on health.
|
||||
if [ -z "$(docker ps -aq -f name=scrabble-backend)" ]; then
|
||||
echo "first deploy: bringing the whole stack up"
|
||||
dc up -d --no-build --remove-orphans || { echo "compose up failed"; exit 1; }
|
||||
health_backend || { echo "backend not ready"; exit 1; }
|
||||
health_landing || { echo "landing not ready"; exit 1; }
|
||||
echo "$TAG" > "$STATE_FILE"
|
||||
echo "first deploy healthy ($TAG)."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Migration deploy: freeze writes and snapshot a consistent dump before migrating.
|
||||
if [ "$MIGRATION" = 1 ]; then
|
||||
echo "migration deploy: opening maintenance window (stopping the backend = the only writer)"
|
||||
dc stop backend
|
||||
dump="$DUMP_DIR/pre-$TAG-$(date +%Y%m%d-%H%M%S).sql"
|
||||
if ! docker exec scrabble-postgres pg_dump -U "$PG_USER" -d "$PG_DB" -n backend > "$dump"; then
|
||||
echo "pg_dump failed; restarting the old backend and aborting"
|
||||
dc start backend
|
||||
exit 1
|
||||
fi
|
||||
echo "consistent dump: $dump"
|
||||
fi
|
||||
|
||||
# Roll one service at a time, least -> most dependent; any failure rolls everything back.
|
||||
roll postgres health_postgres || { rollback; exit 1; }
|
||||
roll backend health_backend || { rollback; exit 1; }
|
||||
roll gateway health_running scrabble-gateway || { rollback; exit 1; }
|
||||
roll landing health_landing || { rollback; exit 1; }
|
||||
roll validator health_running scrabble-telegram-validator || { rollback; exit 1; }
|
||||
roll caddy health_running scrabble-caddy || { rollback; exit 1; }
|
||||
|
||||
# Observability + node_exporter: bring up the remainder and pick up any config changes.
|
||||
dc up -d --no-build --remove-orphans || { rollback; exit 1; }
|
||||
|
||||
# Final internal sanity before committing the new tag.
|
||||
health_backend || { rollback; exit 1; }
|
||||
echo "$TAG" > "$STATE_FILE"
|
||||
echo "=== deploy healthy ($TAG) ==="
|
||||
Reference in New Issue
Block a user