Merge pull request 'Deploy v2 — release versioning + visible deploy jobs + manual rollback' (#105) from feature/release-versioning into development
CI / changes (push) Successful in 2s
CI / unit (push) Successful in 11s
CI / integration (push) Successful in 15s
CI / ui (push) Successful in 57s
CI / changes (pull_request) Successful in 2s
CI / gate (push) Successful in 0s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 16s
CI / ui (pull_request) Successful in 56s
CI / deploy (push) Successful in 1m19s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Has been skipped
CI / changes (push) Successful in 2s
CI / unit (push) Successful in 11s
CI / integration (push) Successful in 15s
CI / ui (push) Successful in 57s
CI / changes (pull_request) Successful in 2s
CI / gate (push) Successful in 0s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 16s
CI / ui (pull_request) Successful in 56s
CI / deploy (push) Successful in 1m19s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Has been skipped
This commit was merged in pull request #105.
This commit is contained in:
@@ -1,9 +1,11 @@
|
|||||||
# Manual production rollout. Runs ONLY from master, ONLY on an explicit
|
# Manual production rollout. Runs ONLY from master, ONLY on workflow_dispatch with
|
||||||
# workflow_dispatch with confirm=deploy (development->master is merged + green first;
|
# confirm=deploy (development->master is merged + green first; this is the separate,
|
||||||
# this is the separate, deliberate prod step). It builds and pushes the images to the
|
# deliberate prod step). Visible sequential jobs from most to least significant:
|
||||||
# registry, then deploys over SSH: the main host via deploy/prod-deploy.sh (rolling,
|
# build -> deploy-main -> deploy-bot -> verify
|
||||||
# health-gated, auto-rollback; a maintenance window + consistent dump on a migration),
|
# The per-service rolling (postgres->backend->gateway->landing->validator->caddy),
|
||||||
# then the bot host. See deploy/README.md (prod runbook) for operator steps.
|
# health-gating and auto-rollback live in deploy/prod-deploy.sh on the main host and
|
||||||
|
# show in the deploy-main log. Manual post-deploy rollback is prod-rollback.yaml.
|
||||||
|
# See deploy/README.md (prod runbook).
|
||||||
name: prod-deploy
|
name: prod-deploy
|
||||||
run-name: "prod deploy ${{ github.sha }}"
|
run-name: "prod deploy ${{ github.sha }}"
|
||||||
|
|
||||||
@@ -18,41 +20,71 @@ on:
|
|||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
|
env:
|
||||||
|
NO_COLOR: "1"
|
||||||
|
DOCKER_CLI_HINTS: "false"
|
||||||
|
REGISTRY: docker.iliadenisov.ru/developer
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
deploy:
|
build:
|
||||||
if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'deploy' }}
|
if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'deploy' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
defaults:
|
defaults:
|
||||||
run:
|
run:
|
||||||
shell: bash
|
shell: bash
|
||||||
|
outputs:
|
||||||
|
tag: ${{ steps.ver.outputs.tag }}
|
||||||
env:
|
env:
|
||||||
NO_COLOR: "1"
|
PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
|
||||||
DOCKER_CLI_HINTS: "false"
|
PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
|
||||||
REGISTRY: docker.iliadenisov.ru/developer
|
VITE_TELEGRAM_BOT_ID: ${{ vars.PROD_VITE_TELEGRAM_BOT_ID }}
|
||||||
# SSH + registry
|
VITE_TELEGRAM_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
|
||||||
|
VITE_TELEGRAM_GAME_CHANNEL_NAME: ${{ vars.PROD_VITE_TELEGRAM_GAME_CHANNEL_NAME }}
|
||||||
|
VITE_GATEWAY_URL: ${{ vars.PROD_VITE_GATEWAY_URL }}
|
||||||
|
POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }}
|
||||||
|
GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }}
|
||||||
|
TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }}
|
||||||
|
DICT_VERSION: ${{ vars.PROD_DICT_VERSION }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
- name: Compute version tag
|
||||||
|
id: ver
|
||||||
|
run: echo "tag=$(git describe --tags --always)" >> "$GITHUB_OUTPUT"
|
||||||
|
- name: Registry login
|
||||||
|
run: echo "$PROD_REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$PROD_REGISTRY_USER" --password-stdin
|
||||||
|
- name: Build and push images
|
||||||
|
working-directory: deploy
|
||||||
|
run: |
|
||||||
|
export TAG="${{ steps.ver.outputs.tag }}" APP_VERSION="${{ steps.ver.outputs.tag }}" SCRABBLE_CONFIG_DIR=.
|
||||||
|
# The four main-stack images via compose (reuses the build args, incl. VERSION);
|
||||||
|
# the bot separately, since it is profiled out of the prod compose.
|
||||||
|
docker compose -f docker-compose.yml -f docker-compose.prod.yml build
|
||||||
|
docker compose -f docker-compose.yml -f docker-compose.prod.yml push backend gateway landing validator
|
||||||
|
docker build -f ../platform/telegram/Dockerfile --target bot --build-arg VERSION="$TAG" -t "$REGISTRY/scrabble-telegram-bot:$TAG" ..
|
||||||
|
docker push "$REGISTRY/scrabble-telegram-bot:$TAG"
|
||||||
|
|
||||||
|
deploy-main:
|
||||||
|
needs: build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
TAG: ${{ needs.build.outputs.tag }}
|
||||||
PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
|
PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
|
||||||
PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
|
PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
|
||||||
PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
|
PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
|
||||||
PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
|
PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
|
||||||
MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
|
MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
|
||||||
TG_HOST: ${{ vars.PROD_TG_HOST }}
|
|
||||||
# SPA build args (baked into the gateway/landing images)
|
|
||||||
VITE_TELEGRAM_BOT_ID: ${{ vars.PROD_VITE_TELEGRAM_BOT_ID }}
|
|
||||||
VITE_TELEGRAM_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
|
|
||||||
VITE_TELEGRAM_GAME_CHANNEL_NAME: ${{ vars.PROD_VITE_TELEGRAM_GAME_CHANNEL_NAME }}
|
|
||||||
VITE_GATEWAY_URL: ${{ vars.PROD_VITE_GATEWAY_URL }}
|
|
||||||
# Runtime secrets
|
|
||||||
POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }}
|
POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }}
|
||||||
GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }}
|
GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }}
|
||||||
GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }}
|
GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }}
|
||||||
TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
|
TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
|
||||||
TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }}
|
|
||||||
PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
|
PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
|
||||||
PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }}
|
PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }}
|
||||||
PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }}
|
PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }}
|
||||||
PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }}
|
|
||||||
PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }}
|
|
||||||
# Runtime variables
|
|
||||||
GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }}
|
GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }}
|
||||||
GRAFANA_ROOT_URL: ${{ vars.PROD_GRAFANA_ROOT_URL }}
|
GRAFANA_ROOT_URL: ${{ vars.PROD_GRAFANA_ROOT_URL }}
|
||||||
CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
|
CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
|
||||||
@@ -61,41 +93,15 @@ jobs:
|
|||||||
POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }}
|
POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }}
|
||||||
POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }}
|
POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }}
|
||||||
TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }}
|
TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }}
|
||||||
TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }}
|
|
||||||
TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }}
|
|
||||||
TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }}
|
|
||||||
TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
- name: Compute image tag and app version
|
|
||||||
run: |
|
|
||||||
echo "TAG=$(git rev-parse --short=12 HEAD)" >> "$GITHUB_ENV"
|
|
||||||
echo "APP_VERSION=$(git describe --tags --always)" >> "$GITHUB_ENV"
|
|
||||||
|
|
||||||
- name: Registry login
|
|
||||||
run: echo "$PROD_REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$PROD_REGISTRY_USER" --password-stdin
|
|
||||||
|
|
||||||
- name: Build and push images
|
|
||||||
working-directory: deploy
|
|
||||||
run: |
|
|
||||||
export TAG SCRABBLE_CONFIG_DIR=.
|
|
||||||
# The four main-stack images via compose (reuses the compose build args); the
|
|
||||||
# bot separately, since it is profiled out of the prod compose.
|
|
||||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml build
|
|
||||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml push backend gateway landing validator
|
|
||||||
docker build -f ../platform/telegram/Dockerfile --target bot -t "$REGISTRY/scrabble-telegram-bot:$TAG" ..
|
|
||||||
docker push "$REGISTRY/scrabble-telegram-bot:$TAG"
|
|
||||||
|
|
||||||
- name: Set up SSH
|
- name: Set up SSH
|
||||||
run: |
|
run: |
|
||||||
mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||||
printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy
|
printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
|
||||||
chmod 600 ~/.ssh/id_deploy
|
|
||||||
printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
||||||
|
|
||||||
- name: Determine previous tag and migration
|
- name: Determine previous tag and migration
|
||||||
run: |
|
run: |
|
||||||
ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
|
ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
|
||||||
@@ -103,21 +109,17 @@ jobs:
|
|||||||
MIGRATION=0
|
MIGRATION=0
|
||||||
if [ "$PREV_TAG" != none ]; then
|
if [ "$PREV_TAG" != none ]; then
|
||||||
if ! git cat-file -e "$PREV_TAG^{commit}" 2>/dev/null; then
|
if ! git cat-file -e "$PREV_TAG^{commit}" 2>/dev/null; then
|
||||||
MIGRATION=1 # unknown previous tag: take the safe window + dump
|
MIGRATION=1
|
||||||
elif git diff --name-only "$PREV_TAG..$TAG" -- backend/internal/postgres/migrations/ | grep -q .; then
|
elif git diff --name-only "$PREV_TAG..$TAG" -- backend/internal/postgres/migrations/ | grep -q .; then
|
||||||
MIGRATION=1
|
MIGRATION=1
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
echo "PREV_TAG=$PREV_TAG" >> "$GITHUB_ENV"
|
{ echo "PREV_TAG=$PREV_TAG"; echo "MIGRATION=$MIGRATION"; } >> "$GITHUB_ENV"
|
||||||
echo "MIGRATION=$MIGRATION" >> "$GITHUB_ENV"
|
|
||||||
echo "prev=$PREV_TAG migration=$MIGRATION"
|
echo "prev=$PREV_TAG migration=$MIGRATION"
|
||||||
|
- name: Render main env + certs
|
||||||
- name: Render env files and certs
|
|
||||||
run: |
|
run: |
|
||||||
umask 077
|
umask 077
|
||||||
mkdir -p stage/certs-main stage/certs-bot
|
mkdir -p stage/certs-main
|
||||||
# Main-host runtime env (single-quoted so the literal '$' in the bcrypt hash
|
|
||||||
# survives; prod-deploy.sh sources this, compose reads it from the environment).
|
|
||||||
cat > stage/env.sh <<EOF
|
cat > stage/env.sh <<EOF
|
||||||
export REGISTRY='$REGISTRY'
|
export REGISTRY='$REGISTRY'
|
||||||
export SCRABBLE_CONFIG_DIR='/opt/scrabble'
|
export SCRABBLE_CONFIG_DIR='/opt/scrabble'
|
||||||
@@ -131,11 +133,65 @@ jobs:
|
|||||||
export CADDY_SITE_ADDRESS='$CADDY_SITE_ADDRESS'
|
export CADDY_SITE_ADDRESS='$CADDY_SITE_ADDRESS'
|
||||||
export LOG_LEVEL='${LOG_LEVEL:-info}'
|
export LOG_LEVEL='${LOG_LEVEL:-info}'
|
||||||
export DICT_VERSION='$DICT_VERSION'
|
export DICT_VERSION='$DICT_VERSION'
|
||||||
|
export APP_VERSION='$TAG'
|
||||||
export TELEGRAM_BOT_TOKEN='$TELEGRAM_BOT_TOKEN'
|
export TELEGRAM_BOT_TOKEN='$TELEGRAM_BOT_TOKEN'
|
||||||
export TELEGRAM_MINIAPP_URL='$TELEGRAM_MINIAPP_URL'
|
export TELEGRAM_MINIAPP_URL='$TELEGRAM_MINIAPP_URL'
|
||||||
export GATEWAY_ABUSE_BAN_ENABLED='true'
|
export GATEWAY_ABUSE_BAN_ENABLED='true'
|
||||||
EOF
|
EOF
|
||||||
# Bot-host runtime env.
|
printf '%s\n' "$PROD_BOTLINK_CA" > stage/certs-main/ca.crt
|
||||||
|
printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt
|
||||||
|
printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY" > stage/certs-main/gateway.key
|
||||||
|
chmod 644 stage/certs-main/*
|
||||||
|
- name: Deploy the main host
|
||||||
|
run: |
|
||||||
|
ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
|
||||||
|
ssh_main 'mkdir -p /opt/scrabble/compose'
|
||||||
|
tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \
|
||||||
|
| ssh_main 'tar -C /opt/scrabble/compose -xzf -'
|
||||||
|
tar -C deploy -czf - caddy otelcol prometheus tempo grafana \
|
||||||
|
| ssh_main 'tar -C /opt/scrabble -xzf -'
|
||||||
|
tar -C stage -czf - certs-main \
|
||||||
|
| ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'
|
||||||
|
scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh"
|
||||||
|
echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin"
|
||||||
|
ssh_main "TAG='$TAG' PREV_TAG='$PREV_TAG' MIGRATION='$MIGRATION' bash /opt/scrabble/compose/prod-deploy.sh"
|
||||||
|
|
||||||
|
deploy-bot:
|
||||||
|
needs: [build, deploy-main]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
TAG: ${{ needs.build.outputs.tag }}
|
||||||
|
PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
|
||||||
|
PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
|
||||||
|
PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
|
||||||
|
PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
|
||||||
|
TG_HOST: ${{ vars.PROD_TG_HOST }}
|
||||||
|
MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
|
||||||
|
TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
|
||||||
|
TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }}
|
||||||
|
PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
|
||||||
|
PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }}
|
||||||
|
PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }}
|
||||||
|
LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }}
|
||||||
|
TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }}
|
||||||
|
TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }}
|
||||||
|
TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }}
|
||||||
|
TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }}
|
||||||
|
TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Set up SSH
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||||
|
printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
|
||||||
|
printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
||||||
|
- name: Render bot env + certs
|
||||||
|
run: |
|
||||||
|
umask 077
|
||||||
|
mkdir -p stage/certs-bot
|
||||||
cat > stage/env.bot.sh <<EOF
|
cat > stage/env.bot.sh <<EOF
|
||||||
export SCRABBLE_CONFIG_DIR='/opt/scrabble'
|
export SCRABBLE_CONFIG_DIR='/opt/scrabble'
|
||||||
export BOT_IMAGE='$REGISTRY/scrabble-telegram-bot:$TAG'
|
export BOT_IMAGE='$REGISTRY/scrabble-telegram-bot:$TAG'
|
||||||
@@ -149,32 +205,10 @@ jobs:
|
|||||||
export TELEGRAM_BOT_LINK='$TELEGRAM_BOT_LINK'
|
export TELEGRAM_BOT_LINK='$TELEGRAM_BOT_LINK'
|
||||||
export LOG_LEVEL='${LOG_LEVEL:-info}'
|
export LOG_LEVEL='${LOG_LEVEL:-info}'
|
||||||
EOF
|
EOF
|
||||||
# Bot-link certs (world-readable: the distroless gateway/bot run as UID 65532
|
|
||||||
# and bind-mount the dir read-only).
|
|
||||||
printf '%s\n' "$PROD_BOTLINK_CA" > stage/certs-main/ca.crt
|
|
||||||
printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt
|
|
||||||
printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY" > stage/certs-main/gateway.key
|
|
||||||
printf '%s\n' "$PROD_BOTLINK_CA" > stage/certs-bot/ca.crt
|
printf '%s\n' "$PROD_BOTLINK_CA" > stage/certs-bot/ca.crt
|
||||||
printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt
|
printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt
|
||||||
printf '%s\n' "$PROD_BOTLINK_BOT_KEY" > stage/certs-bot/bot.key
|
printf '%s\n' "$PROD_BOTLINK_BOT_KEY" > stage/certs-bot/bot.key
|
||||||
chmod 644 stage/certs-main/* stage/certs-bot/*
|
chmod 644 stage/certs-bot/*
|
||||||
|
|
||||||
- name: Deploy the main host
|
|
||||||
run: |
|
|
||||||
ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
|
|
||||||
ssh_main 'mkdir -p /opt/scrabble/compose'
|
|
||||||
# Compose files + config + script.
|
|
||||||
tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \
|
|
||||||
| ssh_main 'tar -C /opt/scrabble/compose -xzf -'
|
|
||||||
tar -C deploy -czf - caddy otelcol prometheus tempo grafana \
|
|
||||||
| ssh_main 'tar -C /opt/scrabble -xzf -'
|
|
||||||
tar -C stage -czf - certs-main \
|
|
||||||
| ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'
|
|
||||||
scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh"
|
|
||||||
# Registry login on the host so compose can pull the private images.
|
|
||||||
echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin"
|
|
||||||
ssh_main "TAG='$TAG' PREV_TAG='$PREV_TAG' MIGRATION='$MIGRATION' bash /opt/scrabble/compose/prod-deploy.sh"
|
|
||||||
|
|
||||||
- name: Deploy the bot host
|
- name: Deploy the bot host
|
||||||
run: |
|
run: |
|
||||||
ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; }
|
ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; }
|
||||||
@@ -187,7 +221,6 @@ jobs:
|
|||||||
ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose;
|
ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose;
|
||||||
docker compose -f docker-compose.bot.yml pull;
|
docker compose -f docker-compose.bot.yml pull;
|
||||||
docker compose -f docker-compose.bot.yml up -d'
|
docker compose -f docker-compose.bot.yml up -d'
|
||||||
# Bot liveness: running, not restarting, stable restart count.
|
|
||||||
ssh_tg 'for i in $(seq 1 20); do
|
ssh_tg 'for i in $(seq 1 20); do
|
||||||
s=$(docker inspect -f "{{.State.Status}}" scrabble-telegram-bot 2>/dev/null || echo missing)
|
s=$(docker inspect -f "{{.State.Status}}" scrabble-telegram-bot 2>/dev/null || echo missing)
|
||||||
r=$(docker inspect -f "{{.State.Restarting}}" scrabble-telegram-bot 2>/dev/null || echo true)
|
r=$(docker inspect -f "{{.State.Restarting}}" scrabble-telegram-bot 2>/dev/null || echo true)
|
||||||
@@ -200,13 +233,27 @@ jobs:
|
|||||||
done
|
done
|
||||||
echo "bot not healthy:"; docker logs --tail 80 scrabble-telegram-bot; exit 1'
|
echo "bot not healthy:"; docker logs --tail 80 scrabble-telegram-bot; exit 1'
|
||||||
|
|
||||||
|
verify:
|
||||||
|
needs: [deploy-main, deploy-bot]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
|
||||||
|
PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
|
||||||
|
MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
|
||||||
|
CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
|
||||||
|
steps:
|
||||||
|
- name: Set up SSH
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||||
|
printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
|
||||||
|
printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
||||||
- name: Verify the public site
|
- name: Verify the public site
|
||||||
run: |
|
run: |
|
||||||
ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
|
domain="${CADDY_SITE_ADDRESS%% *}"
|
||||||
domain="${CADDY_SITE_ADDRESS%% *}" # first domain of "erudit-game.ru www..."
|
ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "for i in \$(seq 1 20); do
|
||||||
# Probe through caddy with the right vhost (force-resolved to the host); -k
|
|
||||||
# tolerates an ACME-pending cert. Also check the backend is actually ready.
|
|
||||||
ssh_main "for i in \$(seq 1 20); do
|
|
||||||
if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null &&
|
if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null &&
|
||||||
curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/app/ -o /dev/null &&
|
curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/app/ -o /dev/null &&
|
||||||
docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then
|
docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then
|
||||||
|
|||||||
@@ -0,0 +1,223 @@
|
|||||||
|
# Manual production rollback. Runs ONLY from master, ONLY on workflow_dispatch with
|
||||||
|
# confirm=rollback. Re-deploys an already-published image tag (no build): leave
|
||||||
|
# target_version blank to roll back to the previously deployed version (read from the
|
||||||
|
# main host), or set it to a specific release tag from the Releases page. The
|
||||||
|
# re-deploy is the same rolling, health-gated path as prod-deploy (TAG=target,
|
||||||
|
# MIGRATION=0 — rollback is image-only and never migrates the DB; image rollback is
|
||||||
|
# DB-safe under the expand-contract rule). See deploy/README.md (prod runbook).
|
||||||
|
name: prod-rollback
|
||||||
|
run-name: "prod rollback ${{ inputs.target_version || 'previous' }}"
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
confirm:
|
||||||
|
description: 'Type "rollback" to confirm a production rollback.'
|
||||||
|
required: true
|
||||||
|
default: ""
|
||||||
|
target_version:
|
||||||
|
description: "Release tag to roll back to (blank = the previous deployed version)."
|
||||||
|
required: false
|
||||||
|
default: ""
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
env:
|
||||||
|
NO_COLOR: "1"
|
||||||
|
DOCKER_CLI_HINTS: "false"
|
||||||
|
REGISTRY: docker.iliadenisov.ru/developer
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
rollback-main:
|
||||||
|
if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'rollback' }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: bash
|
||||||
|
outputs:
|
||||||
|
target: ${{ steps.resolve.outputs.target }}
|
||||||
|
env:
|
||||||
|
PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
|
||||||
|
PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
|
||||||
|
PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
|
||||||
|
PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
|
||||||
|
MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
|
||||||
|
POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }}
|
||||||
|
GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }}
|
||||||
|
GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }}
|
||||||
|
TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
|
||||||
|
PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
|
||||||
|
PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }}
|
||||||
|
PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }}
|
||||||
|
GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }}
|
||||||
|
GRAFANA_ROOT_URL: ${{ vars.PROD_GRAFANA_ROOT_URL }}
|
||||||
|
CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
|
||||||
|
LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }}
|
||||||
|
DICT_VERSION: ${{ vars.PROD_DICT_VERSION }}
|
||||||
|
POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }}
|
||||||
|
POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }}
|
||||||
|
TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }}
|
||||||
|
INPUT_TARGET: ${{ inputs.target_version }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Set up SSH
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||||
|
printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
|
||||||
|
printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
||||||
|
- name: Resolve rollback target
|
||||||
|
id: resolve
|
||||||
|
run: |
|
||||||
|
ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
|
||||||
|
CURRENT="$(ssh_main 'cat /opt/scrabble/DEPLOYED_TAG 2>/dev/null || echo none')"
|
||||||
|
if [ -n "$INPUT_TARGET" ]; then
|
||||||
|
TARGET="$INPUT_TARGET"
|
||||||
|
else
|
||||||
|
TARGET="$(ssh_main 'cat /opt/scrabble/PREVIOUS_TAG 2>/dev/null || echo none')"
|
||||||
|
fi
|
||||||
|
if [ -z "$TARGET" ] || [ "$TARGET" = none ]; then
|
||||||
|
echo "no rollback target (no PREVIOUS_TAG on the host and no target_version input)"; exit 1
|
||||||
|
fi
|
||||||
|
if [ "$TARGET" = "$CURRENT" ]; then
|
||||||
|
echo "target $TARGET is already the deployed version; nothing to do"; exit 1
|
||||||
|
fi
|
||||||
|
echo "rolling back: current=$CURRENT -> target=$TARGET"
|
||||||
|
echo "target=$TARGET" >> "$GITHUB_OUTPUT"
|
||||||
|
{ echo "TARGET=$TARGET"; echo "CURRENT=$CURRENT"; } >> "$GITHUB_ENV"
|
||||||
|
- name: Render main env + certs
|
||||||
|
run: |
|
||||||
|
umask 077
|
||||||
|
mkdir -p stage/certs-main
|
||||||
|
cat > stage/env.sh <<EOF
|
||||||
|
export REGISTRY='$REGISTRY'
|
||||||
|
export SCRABBLE_CONFIG_DIR='/opt/scrabble'
|
||||||
|
export POSTGRES_DB='${POSTGRES_DB:-scrabble}'
|
||||||
|
export POSTGRES_USER='${POSTGRES_USER:-scrabble}'
|
||||||
|
export POSTGRES_PASSWORD='$POSTGRES_PASSWORD'
|
||||||
|
export GM_BASICAUTH_USER='${GM_BASICAUTH_USER:-gm}'
|
||||||
|
export GM_BASICAUTH_HASH='$GM_BASICAUTH_HASH'
|
||||||
|
export GRAFANA_ADMIN_PASSWORD='$GRAFANA_ADMIN_PASSWORD'
|
||||||
|
export GRAFANA_ROOT_URL='$GRAFANA_ROOT_URL'
|
||||||
|
export CADDY_SITE_ADDRESS='$CADDY_SITE_ADDRESS'
|
||||||
|
export LOG_LEVEL='${LOG_LEVEL:-info}'
|
||||||
|
export DICT_VERSION='$DICT_VERSION'
|
||||||
|
export APP_VERSION='$TARGET'
|
||||||
|
export TELEGRAM_BOT_TOKEN='$TELEGRAM_BOT_TOKEN'
|
||||||
|
export TELEGRAM_MINIAPP_URL='$TELEGRAM_MINIAPP_URL'
|
||||||
|
export GATEWAY_ABUSE_BAN_ENABLED='true'
|
||||||
|
EOF
|
||||||
|
printf '%s\n' "$PROD_BOTLINK_CA" > stage/certs-main/ca.crt
|
||||||
|
printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt
|
||||||
|
printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY" > stage/certs-main/gateway.key
|
||||||
|
chmod 644 stage/certs-main/*
|
||||||
|
- name: Roll the main host back
|
||||||
|
run: |
|
||||||
|
ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; }
|
||||||
|
ssh_main 'mkdir -p /opt/scrabble/compose'
|
||||||
|
tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \
|
||||||
|
| ssh_main 'tar -C /opt/scrabble/compose -xzf -'
|
||||||
|
tar -C deploy -czf - caddy otelcol prometheus tempo grafana \
|
||||||
|
| ssh_main 'tar -C /opt/scrabble -xzf -'
|
||||||
|
tar -C stage -czf - certs-main \
|
||||||
|
| ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'
|
||||||
|
scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh"
|
||||||
|
echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin"
|
||||||
|
# Image-only rollback: no migration window (TAG=target, MIGRATION=0). A failed
|
||||||
|
# rollback's auto-revert returns to the current version (PREV_TAG=$CURRENT).
|
||||||
|
ssh_main "TAG='$TARGET' PREV_TAG='$CURRENT' MIGRATION=0 bash /opt/scrabble/compose/prod-deploy.sh"
|
||||||
|
|
||||||
|
rollback-bot:
|
||||||
|
needs: rollback-main
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
TARGET: ${{ needs.rollback-main.outputs.target }}
|
||||||
|
PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }}
|
||||||
|
PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }}
|
||||||
|
PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
|
||||||
|
PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
|
||||||
|
TG_HOST: ${{ vars.PROD_TG_HOST }}
|
||||||
|
MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
|
||||||
|
TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }}
|
||||||
|
TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }}
|
||||||
|
PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
|
||||||
|
PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }}
|
||||||
|
PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }}
|
||||||
|
LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }}
|
||||||
|
TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }}
|
||||||
|
TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }}
|
||||||
|
TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }}
|
||||||
|
TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }}
|
||||||
|
TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Set up SSH
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||||
|
printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
|
||||||
|
printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
||||||
|
- name: Render bot env + certs
|
||||||
|
run: |
|
||||||
|
umask 077
|
||||||
|
mkdir -p stage/certs-bot
|
||||||
|
cat > stage/env.bot.sh <<EOF
|
||||||
|
export SCRABBLE_CONFIG_DIR='/opt/scrabble'
|
||||||
|
export BOT_IMAGE='$REGISTRY/scrabble-telegram-bot:$TARGET'
|
||||||
|
export BOTLINK_GATEWAY_ADDR='$MAIN_HOST:9443'
|
||||||
|
export TELEGRAM_BOT_TOKEN='$TELEGRAM_BOT_TOKEN'
|
||||||
|
export TELEGRAM_MINIAPP_URL='$TELEGRAM_MINIAPP_URL'
|
||||||
|
export TELEGRAM_GAME_CHANNEL_ID='$TELEGRAM_GAME_CHANNEL_ID'
|
||||||
|
export TELEGRAM_CHAT_ID='$TELEGRAM_CHAT_ID'
|
||||||
|
export TELEGRAM_PROMO_BOT_TOKEN='$TELEGRAM_PROMO_BOT_TOKEN'
|
||||||
|
export TELEGRAM_BOT_USERNAME='$TELEGRAM_BOT_USERNAME'
|
||||||
|
export TELEGRAM_BOT_LINK='$TELEGRAM_BOT_LINK'
|
||||||
|
export LOG_LEVEL='${LOG_LEVEL:-info}'
|
||||||
|
EOF
|
||||||
|
printf '%s\n' "$PROD_BOTLINK_CA" > stage/certs-bot/ca.crt
|
||||||
|
printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt
|
||||||
|
printf '%s\n' "$PROD_BOTLINK_BOT_KEY" > stage/certs-bot/bot.key
|
||||||
|
chmod 644 stage/certs-bot/*
|
||||||
|
- name: Roll the bot host back
|
||||||
|
run: |
|
||||||
|
ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; }
|
||||||
|
ssh_tg 'mkdir -p /opt/scrabble/compose'
|
||||||
|
tar -C deploy -czf - docker-compose.bot.yml | ssh_tg 'tar -C /opt/scrabble/compose -xzf -'
|
||||||
|
tar -C stage -czf - certs-bot \
|
||||||
|
| ssh_tg 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'
|
||||||
|
scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.bot.sh "deploy@$TG_HOST:/opt/scrabble/env.bot.sh"
|
||||||
|
echo "$PROD_REGISTRY_PASSWORD" | ssh_tg "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin"
|
||||||
|
ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose;
|
||||||
|
docker compose -f docker-compose.bot.yml pull;
|
||||||
|
docker compose -f docker-compose.bot.yml up -d'
|
||||||
|
|
||||||
|
verify:
|
||||||
|
needs: [rollback-main, rollback-bot]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
|
||||||
|
PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }}
|
||||||
|
MAIN_HOST: ${{ vars.PROD_MAIN_HOST }}
|
||||||
|
CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }}
|
||||||
|
steps:
|
||||||
|
- name: Set up SSH
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||||
|
printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy
|
||||||
|
printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
||||||
|
- name: Verify the public site
|
||||||
|
run: |
|
||||||
|
domain="${CADDY_SITE_ADDRESS%% *}"
|
||||||
|
ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "for i in \$(seq 1 20); do
|
||||||
|
if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null &&
|
||||||
|
docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then
|
||||||
|
echo 'rolled-back site healthy'; exit 0
|
||||||
|
fi
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
echo 'verify failed'; docker logs --tail 40 scrabble-caddy; docker logs --tail 40 scrabble-backend; exit 1"
|
||||||
+3
-1
@@ -33,7 +33,9 @@ COPY backend ./backend
|
|||||||
# Reduce the workspace to what the backend needs: backend + pkg. loadtest and the
|
# Reduce the workspace to what the backend needs: backend + pkg. loadtest and the
|
||||||
# gateway replace it requires are not in this context, so drop both.
|
# gateway replace it requires are not in this context, so drop both.
|
||||||
RUN go work edit -dropuse=./gateway -dropuse=./platform/telegram -dropuse=./loadtest -dropreplace=scrabble/gateway@v0.0.0
|
RUN go work edit -dropuse=./gateway -dropuse=./platform/telegram -dropuse=./loadtest -dropreplace=scrabble/gateway@v0.0.0
|
||||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/backend ./backend/cmd/backend
|
# VERSION (the deploy passes the git tag) is stamped into the binary via the linker.
|
||||||
|
ARG VERSION=dev
|
||||||
|
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "-X scrabble/pkg/version.Version=${VERSION}" -o /out/backend ./backend/cmd/backend
|
||||||
|
|
||||||
# --- runtime -----------------------------------------------------------------
|
# --- runtime -----------------------------------------------------------------
|
||||||
FROM gcr.io/distroless/static-debian12:nonroot
|
FROM gcr.io/distroless/static-debian12:nonroot
|
||||||
|
|||||||
+19
-1
@@ -130,7 +130,25 @@ workflow manually (Gitea → Actions → prod-deploy → run from `master`, inpu
|
|||||||
compose/config/certs/env over SSH, deploys the main host with `prod-deploy.sh` (rolling,
|
compose/config/certs/env over SSH, deploys the main host with `prod-deploy.sh` (rolling,
|
||||||
health-gated, **auto-rollback to the previous tag**), then the bot host, then probes the
|
health-gated, **auto-rollback to the previous tag**), then the bot host, then probes the
|
||||||
public site. After `master` is green this workflow is the **only** thing that touches
|
public site. After `master` is green this workflow is the **only** thing that touches
|
||||||
prod — nothing auto-deploys there.
|
prod — nothing auto-deploys there. It runs four visible jobs: **build → deploy-main →
|
||||||
|
deploy-bot → verify** (the per-service rolling shows in the deploy-main log).
|
||||||
|
|
||||||
|
**Versioning.** Each release is a git tag `vX.Y.Z` on `master`; the deploy stamps
|
||||||
|
`git describe --tags` into every image tag, every binary (`-ldflags` → `pkg/version` →
|
||||||
|
the `service.version` telemetry attribute) and the SPA About screen. Tag the release
|
||||||
|
before running the deploy:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git tag -a v1.0.0 -m v1.0.0 && git push origin v1.0.0
|
||||||
|
```
|
||||||
|
|
||||||
|
**Manual rollback** (any time after a successful deploy). Run the **`prod-rollback`**
|
||||||
|
workflow (Gitea → Actions → prod-rollback, `confirm=rollback`). Leave `target_version`
|
||||||
|
blank to roll back to the previously deployed version (read from the host's
|
||||||
|
`PREVIOUS_TAG`), or set it to a release tag from the **Releases** page. It re-deploys
|
||||||
|
that already-published image rolling + health-gated — no rebuild, no DB migration
|
||||||
|
(image rollback is DB-safe under the expand-contract rule). The registry keeps every
|
||||||
|
release tag, so any prior release is reachable.
|
||||||
|
|
||||||
**Migrations** must be **expand-contract** (backward-compatible; goose is forward-only):
|
**Migrations** must be **expand-contract** (backward-compatible; goose is forward-only):
|
||||||
the automatic rollback is image-only and never restores the DB. A deploy that changes
|
the automatic rollback is image-only and never restores the DB. A deploy that changes
|
||||||
|
|||||||
@@ -71,6 +71,8 @@ services:
|
|||||||
# Seed dictionary for a FRESH volume; the per-contour value comes from the
|
# Seed dictionary for a FRESH volume; the per-contour value comes from the
|
||||||
# deploy env (Gitea TEST_/PROD_DICT_VERSION). See the volume note below.
|
# deploy env (Gitea TEST_/PROD_DICT_VERSION). See the volume note below.
|
||||||
DICT_VERSION: ${DICT_VERSION:-v1.2.1}
|
DICT_VERSION: ${DICT_VERSION:-v1.2.1}
|
||||||
|
# Build version stamped into the binary (git tag; see pkg/version).
|
||||||
|
VERSION: ${APP_VERSION:-dev}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
logging: *default-logging
|
logging: *default-logging
|
||||||
depends_on:
|
depends_on:
|
||||||
@@ -132,6 +134,8 @@ services:
|
|||||||
VITE_TELEGRAM_GAME_CHANNEL_NAME: ${VITE_TELEGRAM_GAME_CHANNEL_NAME:-}
|
VITE_TELEGRAM_GAME_CHANNEL_NAME: ${VITE_TELEGRAM_GAME_CHANNEL_NAME:-}
|
||||||
VITE_GATEWAY_URL: ${VITE_GATEWAY_URL:-}
|
VITE_GATEWAY_URL: ${VITE_GATEWAY_URL:-}
|
||||||
VITE_APP_VERSION: ${APP_VERSION:-dev}
|
VITE_APP_VERSION: ${APP_VERSION:-dev}
|
||||||
|
# Go binary version (the SPA's VITE_APP_VERSION is the same git tag).
|
||||||
|
VERSION: ${APP_VERSION:-dev}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
logging: *default-logging
|
logging: *default-logging
|
||||||
depends_on: [backend]
|
depends_on: [backend]
|
||||||
@@ -218,6 +222,8 @@ services:
|
|||||||
context: ..
|
context: ..
|
||||||
dockerfile: platform/telegram/Dockerfile
|
dockerfile: platform/telegram/Dockerfile
|
||||||
target: validator
|
target: validator
|
||||||
|
args:
|
||||||
|
VERSION: ${APP_VERSION:-dev}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
logging: *default-logging
|
logging: *default-logging
|
||||||
environment:
|
environment:
|
||||||
@@ -268,6 +274,8 @@ services:
|
|||||||
context: ..
|
context: ..
|
||||||
dockerfile: platform/telegram/Dockerfile
|
dockerfile: platform/telegram/Dockerfile
|
||||||
target: bot
|
target: bot
|
||||||
|
args:
|
||||||
|
VERSION: ${APP_VERSION:-dev}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
logging: *default-logging
|
logging: *default-logging
|
||||||
depends_on: [vpn]
|
depends_on: [vpn]
|
||||||
|
|||||||
+11
-2
@@ -36,6 +36,9 @@ MIGRATION="${MIGRATION:-0}"
|
|||||||
COMPOSE_DIR="${COMPOSE_DIR:-/opt/scrabble/compose}"
|
COMPOSE_DIR="${COMPOSE_DIR:-/opt/scrabble/compose}"
|
||||||
DUMP_DIR="${DUMP_DIR:-/opt/scrabble/dumps}"
|
DUMP_DIR="${DUMP_DIR:-/opt/scrabble/dumps}"
|
||||||
STATE_FILE="${STATE_FILE:-/opt/scrabble/DEPLOYED_TAG}"
|
STATE_FILE="${STATE_FILE:-/opt/scrabble/DEPLOYED_TAG}"
|
||||||
|
# The prior deployed tag, preserved on every successful deploy so prod-rollback can
|
||||||
|
# target "the previous version" with no operator input.
|
||||||
|
PREV_STATE_FILE="${PREV_STATE_FILE:-/opt/scrabble/PREVIOUS_TAG}"
|
||||||
PG_USER="${POSTGRES_USER:-scrabble}"
|
PG_USER="${POSTGRES_USER:-scrabble}"
|
||||||
PG_DB="${POSTGRES_DB:-scrabble}"
|
PG_DB="${POSTGRES_DB:-scrabble}"
|
||||||
|
|
||||||
@@ -88,6 +91,12 @@ rollback() {
|
|||||||
[ "$MIGRATION" = 1 ] && echo "NOTE: the DB is forward-migrated; a pre-deploy dump is in $DUMP_DIR — restore manually ONLY if the migration was destructive (see deploy/README.md, prod runbook)."
|
[ "$MIGRATION" = 1 ] && echo "NOTE: the DB is forward-migrated; a pre-deploy dump is in $DUMP_DIR — restore manually ONLY if the migration was destructive (see deploy/README.md, prod runbook)."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
commit_tag() {
|
||||||
|
# Record the just-deployed tag as current, preserving the prior one as previous.
|
||||||
|
[ -f "$STATE_FILE" ] && cp "$STATE_FILE" "$PREV_STATE_FILE"
|
||||||
|
echo "$TAG" > "$STATE_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
mkdir -p "$DUMP_DIR"
|
mkdir -p "$DUMP_DIR"
|
||||||
echo "=== prod deploy: tag=$TAG prev=$PREV_TAG migration=$MIGRATION ==="
|
echo "=== prod deploy: tag=$TAG prev=$PREV_TAG migration=$MIGRATION ==="
|
||||||
use_tag "$TAG"
|
use_tag "$TAG"
|
||||||
@@ -99,7 +108,7 @@ if [ -z "$(docker ps -aq -f name=scrabble-backend)" ]; then
|
|||||||
dc up -d --no-build --remove-orphans || { echo "compose up failed"; exit 1; }
|
dc up -d --no-build --remove-orphans || { echo "compose up failed"; exit 1; }
|
||||||
health_backend || { echo "backend not ready"; exit 1; }
|
health_backend || { echo "backend not ready"; exit 1; }
|
||||||
health_landing || { echo "landing not ready"; exit 1; }
|
health_landing || { echo "landing not ready"; exit 1; }
|
||||||
echo "$TAG" > "$STATE_FILE"
|
commit_tag
|
||||||
echo "first deploy healthy ($TAG)."
|
echo "first deploy healthy ($TAG)."
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
@@ -130,5 +139,5 @@ dc up -d --no-build --remove-orphans || { rollback; exit 1; }
|
|||||||
|
|
||||||
# Final internal sanity before committing the new tag.
|
# Final internal sanity before committing the new tag.
|
||||||
health_backend || { rollback; exit 1; }
|
health_backend || { rollback; exit 1; }
|
||||||
echo "$TAG" > "$STATE_FILE"
|
commit_tag
|
||||||
echo "=== deploy healthy ($TAG) ==="
|
echo "=== deploy healthy ($TAG) ==="
|
||||||
|
|||||||
@@ -1100,7 +1100,13 @@ Two contours, two secret/variable prefixes (`TEST_` / `PROD_`):
|
|||||||
any failure **rolls the whole stack back to the previous image tag**. A **schema migration**
|
any failure **rolls the whole stack back to the previous image tag**. A **schema migration**
|
||||||
adds a maintenance window: the backend (the sole writer) is stopped for a consistent
|
adds a maintenance window: the backend (the sole writer) is stopped for a consistent
|
||||||
`pg_dump` before the new backend migrates forward — image rollback stays DB-safe under the
|
`pg_dump` before the new backend migrates forward — image rollback stays DB-safe under the
|
||||||
expand-contract migration rule, and the dump is kept for a manual restore. The main host is
|
expand-contract migration rule, and the dump is kept for a manual restore. The workflow runs
|
||||||
|
four visible jobs (build → deploy-main → deploy-bot → verify). Releases are git tags
|
||||||
|
`vX.Y.Z`; the version is stamped into the image tag, every binary (`-ldflags` → `pkg/version`
|
||||||
|
→ the `service.version` telemetry attribute) and the SPA About screen. A separate manual
|
||||||
|
**`prod-rollback`** workflow re-deploys any prior release tag (blank input = the previous
|
||||||
|
deployed version, tracked on the host) over the same rolling, health-gated path — image-only,
|
||||||
|
no DB migration. The main host is
|
||||||
intentionally **launch-sized** (2 vCPU / 1.9 GiB): the prod overlay trims the R7 limits
|
intentionally **launch-sized** (2 vCPU / 1.9 GiB): the prod overlay trims the R7 limits
|
||||||
(`GOMAXPROCS=2`, smaller caps, 7d Prometheus retention) and a **node_exporter** feeds
|
(`GOMAXPROCS=2`, smaller caps, 7d Prometheus retention) and a **node_exporter** feeds
|
||||||
host-memory metrics to Grafana so it can be resized reactively as players arrive.
|
host-memory metrics to Grafana so it can be resized reactively as players arrive.
|
||||||
|
|||||||
+3
-1
@@ -70,7 +70,9 @@ RUN rm gateway/internal/webui/dist/landing.html
|
|||||||
# Reduce the workspace to what the gateway needs: gateway + pkg (loadtest is not in
|
# Reduce the workspace to what the gateway needs: gateway + pkg (loadtest is not in
|
||||||
# this context; its scrabble/gateway replace targets ./gateway, which is present here).
|
# this context; its scrabble/gateway replace targets ./gateway, which is present here).
|
||||||
RUN go work edit -dropuse=./backend -dropuse=./platform/telegram -dropuse=./loadtest
|
RUN go work edit -dropuse=./backend -dropuse=./platform/telegram -dropuse=./loadtest
|
||||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/gateway ./gateway/cmd/gateway
|
# VERSION (the deploy passes the git tag) is stamped into the binary via the linker.
|
||||||
|
ARG VERSION=dev
|
||||||
|
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "-X scrabble/pkg/version.Version=${VERSION}" -o /out/gateway ./gateway/cmd/gateway
|
||||||
|
|
||||||
# --- runtime -----------------------------------------------------------------
|
# --- runtime -----------------------------------------------------------------
|
||||||
FROM gcr.io/distroless/static-debian12:nonroot AS gateway
|
FROM gcr.io/distroless/static-debian12:nonroot AS gateway
|
||||||
|
|||||||
@@ -29,6 +29,8 @@ import (
|
|||||||
"go.opentelemetry.io/otel/sdk/resource"
|
"go.opentelemetry.io/otel/sdk/resource"
|
||||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||||
"go.opentelemetry.io/otel/trace"
|
"go.opentelemetry.io/otel/trace"
|
||||||
|
|
||||||
|
"scrabble/pkg/version"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Exporter selectors supported per signal.
|
// Exporter selectors supported per signal.
|
||||||
@@ -95,9 +97,7 @@ func New(ctx context.Context, cfg Config) (*Runtime, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := resource.New(ctx, resource.WithAttributes(
|
res, err := serviceResource(ctx, cfg)
|
||||||
attribute.String("service.name", cfg.ServiceName),
|
|
||||||
))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("telemetry: build resource: %w", err)
|
return nil, fmt.Errorf("telemetry: build resource: %w", err)
|
||||||
}
|
}
|
||||||
@@ -122,6 +122,16 @@ func New(ctx context.Context, cfg Config) (*Runtime, error) {
|
|||||||
return &Runtime{tracerProvider: tracerProvider, meterProvider: meterProvider}, nil
|
return &Runtime{tracerProvider: tracerProvider, meterProvider: meterProvider}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// serviceResource builds the OpenTelemetry resource describing this service: its
|
||||||
|
// service.name and the service.version stamped into the binary at build time
|
||||||
|
// (pkg/version, set from the git tag by the deploy).
|
||||||
|
func serviceResource(ctx context.Context, cfg Config) (*resource.Resource, error) {
|
||||||
|
return resource.New(ctx, resource.WithAttributes(
|
||||||
|
attribute.String("service.name", cfg.ServiceName),
|
||||||
|
attribute.String("service.version", version.Version),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
// TracerProvider returns the runtime tracer provider, or the global one when r is
|
// TracerProvider returns the runtime tracer provider, or the global one when r is
|
||||||
// not initialised.
|
// not initialised.
|
||||||
func (r *Runtime) TracerProvider() trace.TracerProvider {
|
func (r *Runtime) TracerProvider() trace.TracerProvider {
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"scrabble/pkg/version"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestConfigValidate covers the supported and rejected exporter selections.
|
// TestConfigValidate covers the supported and rejected exporter selections.
|
||||||
@@ -82,3 +84,22 @@ func TestNilRuntime(t *testing.T) {
|
|||||||
t.Errorf("nil runtime Shutdown: %v", err)
|
t.Errorf("nil runtime Shutdown: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestServiceResource checks the resource carries service.name and the embedded
|
||||||
|
// service.version (pkg/version, stamped at build time).
|
||||||
|
func TestServiceResource(t *testing.T) {
|
||||||
|
res, err := serviceResource(context.Background(), DefaultConfig("svc"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("serviceResource: %v", err)
|
||||||
|
}
|
||||||
|
attrs := map[string]string{}
|
||||||
|
for _, kv := range res.Attributes() {
|
||||||
|
attrs[string(kv.Key)] = kv.Value.AsString()
|
||||||
|
}
|
||||||
|
if attrs["service.name"] != "svc" {
|
||||||
|
t.Errorf("service.name = %q, want svc", attrs["service.name"])
|
||||||
|
}
|
||||||
|
if attrs["service.version"] != version.Version {
|
||||||
|
t.Errorf("service.version = %q, want %q", attrs["service.version"], version.Version)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
// Package version exposes the build version stamped into every Scrabble service
|
||||||
|
// binary. The default is "dev"; release builds override it through the linker
|
||||||
|
// (`go build -ldflags "-X scrabble/pkg/version.Version=<value>"`), wired from the
|
||||||
|
// VERSION build-arg in each service Dockerfile, which the deploy sets to the git
|
||||||
|
// tag (`git describe --tags`). It surfaces as the OpenTelemetry service.version
|
||||||
|
// resource attribute (see pkg/telemetry) and the SPA About screen.
|
||||||
|
package version
|
||||||
|
|
||||||
|
// Version is the build version, "dev" unless overridden at link time.
|
||||||
|
var Version = "dev"
|
||||||
@@ -19,8 +19,10 @@ COPY platform/telegram ./platform/telegram
|
|||||||
# Reduce the workspace to what the platform needs: only pkg + platform/telegram.
|
# Reduce the workspace to what the platform needs: only pkg + platform/telegram.
|
||||||
RUN go work edit -dropuse=./backend -dropuse=./gateway -dropuse=./loadtest -dropreplace=scrabble/gateway@v0.0.0 -dropreplace=scrabble-solver
|
RUN go work edit -dropuse=./backend -dropuse=./gateway -dropuse=./loadtest -dropreplace=scrabble/gateway@v0.0.0 -dropreplace=scrabble-solver
|
||||||
|
|
||||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/validator ./platform/telegram/cmd/validator
|
# VERSION (the deploy passes the git tag) is stamped into both binaries via the linker.
|
||||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/bot ./platform/telegram/cmd/bot
|
ARG VERSION=dev
|
||||||
|
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "-X scrabble/pkg/version.Version=${VERSION}" -o /out/validator ./platform/telegram/cmd/validator
|
||||||
|
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "-X scrabble/pkg/version.Version=${VERSION}" -o /out/bot ./platform/telegram/cmd/bot
|
||||||
|
|
||||||
# --- validator (home) --------------------------------------------------------
|
# --- validator (home) --------------------------------------------------------
|
||||||
FROM gcr.io/distroless/static-debian12:nonroot AS validator
|
FROM gcr.io/distroless/static-debian12:nonroot AS validator
|
||||||
|
|||||||
Reference in New Issue
Block a user