From 649b90024f3e45289cf0e113f475f3248630f16e Mon Sep 17 00:00:00 2001 From: Ilia Denisov Date: Sat, 20 Jun 2026 20:49:14 +0200 Subject: [PATCH] ci(deploy): probe backend /readyz so a dead backend fails the deploy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The post-deploy health check probed only the static landing (`GET /`) and the gateway-served SPA shell (`GET /app/`) through the edge caddy. Neither needs the backend, so a crash-looping backend passed the deploy green — which masked the seed-drift crash until it was inspected by hand. Add a third probe, `http://backend:8080/readyz` on the internal network, to the same retry loop, and dump the backend logs on failure. A backend that cannot become ready now fails the deploy loudly instead of going green. --- .gitea/workflows/ci.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 047c483..8eb3d23 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -299,22 +299,28 @@ jobs: # pick up the fresh config. docker compose --ansi never up -d --force-recreate --no-deps caddy otelcol prometheus tempo grafana - - name: Probe the landing and the gateway through caddy + - name: Probe the landing, gateway and backend run: | set -u - # Two probes through the contour caddy: "/" is the static - # landing container, "/app/" is the gateway-served SPA shell. + # Three probes. "/" is the static landing container and "/app/" the + # gateway-served SPA shell (both through the contour caddy on the edge net). + # The backend /readyz is probed on the internal net as well: the caddy probes + # are blind to a crash-looping backend (the landing is static and the SPA + # shell is served without it), which let a bad deploy go green while the + # backend was down — so check it directly here. for i in $(seq 1 20); do if docker run --rm --network edge alpine:3.20 wget -q -T 5 -O /dev/null http://scrabble/ && - docker run --rm --network edge alpine:3.20 wget -q -T 5 -O /dev/null http://scrabble/app/; then - echo "healthy: GET http://scrabble/ (landing) + /app/ (gateway)" + docker run --rm --network edge alpine:3.20 wget -q -T 5 -O /dev/null http://scrabble/app/ && + docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then + echo "healthy: GET / (landing) + /app/ (gateway) + backend /readyz" exit 0 fi sleep 3 done - echo "probe failed; recent landing + gateway logs:" + echo "probe failed; recent landing + gateway + backend logs:" docker logs --tail 50 scrabble-landing || true docker logs --tail 50 scrabble-gateway || true + docker logs --tail 50 scrabble-backend || true exit 1 - name: Probe the Telegram connector liveness