From 8d45ae6e3bf11bf07daec68204958e19caaff54b Mon Sep 17 00:00:00 2001 From: Ilia Denisov Date: Mon, 22 Jun 2026 07:28:27 +0200 Subject: [PATCH 1/2] feat: stamp the build version into every service pkg/version.Version (default "dev") is set at link time via -ldflags from each service Dockerfile's VERSION build-arg, which the deploy passes as the git tag (git describe --tags). It surfaces as the OpenTelemetry service.version resource attribute (so Grafana/Tempo are version-aware), alongside the SPA's existing About version. Adds the VERSION build-arg to the backend/gateway/validator/bot compose builds and a serviceResource test covering service.name + service.version. --- backend/Dockerfile | 4 +++- deploy/docker-compose.yml | 8 ++++++++ gateway/Dockerfile | 4 +++- pkg/telemetry/telemetry.go | 16 +++++++++++++--- pkg/telemetry/telemetry_test.go | 21 +++++++++++++++++++++ pkg/version/version.go | 10 ++++++++++ platform/telegram/Dockerfile | 6 ++++-- 7 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 pkg/version/version.go diff --git a/backend/Dockerfile b/backend/Dockerfile index 31cad95..a95c7e8 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -33,7 +33,9 @@ COPY backend ./backend # Reduce the workspace to what the backend needs: backend + pkg. loadtest and the # gateway replace it requires are not in this context, so drop both. RUN go work edit -dropuse=./gateway -dropuse=./platform/telegram -dropuse=./loadtest -dropreplace=scrabble/gateway@v0.0.0 -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/backend ./backend/cmd/backend +# VERSION (the deploy passes the git tag) is stamped into the binary via the linker. +ARG VERSION=dev +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "-X scrabble/pkg/version.Version=${VERSION}" -o /out/backend ./backend/cmd/backend # --- runtime ----------------------------------------------------------------- FROM gcr.io/distroless/static-debian12:nonroot diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index fce7650..309a12b 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -71,6 +71,8 @@ services: # Seed dictionary for a FRESH volume; the per-contour value comes from the # deploy env (Gitea TEST_/PROD_DICT_VERSION). See the volume note below. DICT_VERSION: ${DICT_VERSION:-v1.2.1} + # Build version stamped into the binary (git tag; see pkg/version). + VERSION: ${APP_VERSION:-dev} restart: unless-stopped logging: *default-logging depends_on: @@ -132,6 +134,8 @@ services: VITE_TELEGRAM_GAME_CHANNEL_NAME: ${VITE_TELEGRAM_GAME_CHANNEL_NAME:-} VITE_GATEWAY_URL: ${VITE_GATEWAY_URL:-} VITE_APP_VERSION: ${APP_VERSION:-dev} + # Go binary version (the SPA's VITE_APP_VERSION is the same git tag). + VERSION: ${APP_VERSION:-dev} restart: unless-stopped logging: *default-logging depends_on: [backend] @@ -218,6 +222,8 @@ services: context: .. dockerfile: platform/telegram/Dockerfile target: validator + args: + VERSION: ${APP_VERSION:-dev} restart: unless-stopped logging: *default-logging environment: @@ -268,6 +274,8 @@ services: context: .. dockerfile: platform/telegram/Dockerfile target: bot + args: + VERSION: ${APP_VERSION:-dev} restart: unless-stopped logging: *default-logging depends_on: [vpn] diff --git a/gateway/Dockerfile b/gateway/Dockerfile index bfb43c5..c8e5bf9 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -70,7 +70,9 @@ RUN rm gateway/internal/webui/dist/landing.html # Reduce the workspace to what the gateway needs: gateway + pkg (loadtest is not in # this context; its scrabble/gateway replace targets ./gateway, which is present here). RUN go work edit -dropuse=./backend -dropuse=./platform/telegram -dropuse=./loadtest -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/gateway ./gateway/cmd/gateway +# VERSION (the deploy passes the git tag) is stamped into the binary via the linker. +ARG VERSION=dev +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "-X scrabble/pkg/version.Version=${VERSION}" -o /out/gateway ./gateway/cmd/gateway # --- runtime ----------------------------------------------------------------- FROM gcr.io/distroless/static-debian12:nonroot AS gateway diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go index c7e9cd3..c4980c5 100644 --- a/pkg/telemetry/telemetry.go +++ b/pkg/telemetry/telemetry.go @@ -29,6 +29,8 @@ import ( "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/trace" + + "scrabble/pkg/version" ) // Exporter selectors supported per signal. @@ -95,9 +97,7 @@ func New(ctx context.Context, cfg Config) (*Runtime, error) { return nil, err } - res, err := resource.New(ctx, resource.WithAttributes( - attribute.String("service.name", cfg.ServiceName), - )) + res, err := serviceResource(ctx, cfg) if err != nil { return nil, fmt.Errorf("telemetry: build resource: %w", err) } @@ -122,6 +122,16 @@ func New(ctx context.Context, cfg Config) (*Runtime, error) { return &Runtime{tracerProvider: tracerProvider, meterProvider: meterProvider}, nil } +// serviceResource builds the OpenTelemetry resource describing this service: its +// service.name and the service.version stamped into the binary at build time +// (pkg/version, set from the git tag by the deploy). +func serviceResource(ctx context.Context, cfg Config) (*resource.Resource, error) { + return resource.New(ctx, resource.WithAttributes( + attribute.String("service.name", cfg.ServiceName), + attribute.String("service.version", version.Version), + )) +} + // TracerProvider returns the runtime tracer provider, or the global one when r is // not initialised. func (r *Runtime) TracerProvider() trace.TracerProvider { diff --git a/pkg/telemetry/telemetry_test.go b/pkg/telemetry/telemetry_test.go index d6edf23..6259521 100644 --- a/pkg/telemetry/telemetry_test.go +++ b/pkg/telemetry/telemetry_test.go @@ -4,6 +4,8 @@ import ( "context" "testing" "time" + + "scrabble/pkg/version" ) // TestConfigValidate covers the supported and rejected exporter selections. @@ -82,3 +84,22 @@ func TestNilRuntime(t *testing.T) { t.Errorf("nil runtime Shutdown: %v", err) } } + +// TestServiceResource checks the resource carries service.name and the embedded +// service.version (pkg/version, stamped at build time). +func TestServiceResource(t *testing.T) { + res, err := serviceResource(context.Background(), DefaultConfig("svc")) + if err != nil { + t.Fatalf("serviceResource: %v", err) + } + attrs := map[string]string{} + for _, kv := range res.Attributes() { + attrs[string(kv.Key)] = kv.Value.AsString() + } + if attrs["service.name"] != "svc" { + t.Errorf("service.name = %q, want svc", attrs["service.name"]) + } + if attrs["service.version"] != version.Version { + t.Errorf("service.version = %q, want %q", attrs["service.version"], version.Version) + } +} diff --git a/pkg/version/version.go b/pkg/version/version.go new file mode 100644 index 0000000..825e202 --- /dev/null +++ b/pkg/version/version.go @@ -0,0 +1,10 @@ +// Package version exposes the build version stamped into every Scrabble service +// binary. The default is "dev"; release builds override it through the linker +// (`go build -ldflags "-X scrabble/pkg/version.Version="`), wired from the +// VERSION build-arg in each service Dockerfile, which the deploy sets to the git +// tag (`git describe --tags`). It surfaces as the OpenTelemetry service.version +// resource attribute (see pkg/telemetry) and the SPA About screen. +package version + +// Version is the build version, "dev" unless overridden at link time. +var Version = "dev" diff --git a/platform/telegram/Dockerfile b/platform/telegram/Dockerfile index fa3e099..32e4dd4 100644 --- a/platform/telegram/Dockerfile +++ b/platform/telegram/Dockerfile @@ -19,8 +19,10 @@ COPY platform/telegram ./platform/telegram # Reduce the workspace to what the platform needs: only pkg + platform/telegram. RUN go work edit -dropuse=./backend -dropuse=./gateway -dropuse=./loadtest -dropreplace=scrabble/gateway@v0.0.0 -dropreplace=scrabble-solver -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/validator ./platform/telegram/cmd/validator -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/bot ./platform/telegram/cmd/bot +# VERSION (the deploy passes the git tag) is stamped into both binaries via the linker. +ARG VERSION=dev +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "-X scrabble/pkg/version.Version=${VERSION}" -o /out/validator ./platform/telegram/cmd/validator +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "-X scrabble/pkg/version.Version=${VERSION}" -o /out/bot ./platform/telegram/cmd/bot # --- validator (home) -------------------------------------------------------- FROM gcr.io/distroless/static-debian12:nonroot AS validator From c59e5227326a72700724488bf58a5db75bf72854 Mon Sep 17 00:00:00 2001 From: Ilia Denisov Date: Mon, 22 Jun 2026 07:37:08 +0200 Subject: [PATCH 2/2] feat(deploy): visible prod-deploy jobs + manual prod-rollback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - prod-deploy.yaml is now four visible sequential jobs (build -> deploy-main -> deploy-bot -> verify) so the rollout stages show in the Actions UI; the per-service rolling stays in the deploy-main log. - prod-rollback.yaml: a separate manual workflow_dispatch. Leave target_version blank to roll back to the previous deployed version (the host now tracks DEPLOYED_TAG + PREVIOUS_TAG), or pick a release tag. Re-deploys an already published image rolling + health-gated, image-only (no rebuild, no DB migration). - prod-deploy.sh tracks the previous tag (commit_tag) for the blank-input rollback. - Docs: ARCHITECTURE §13 + deploy/README runbook cover versioning + rollback. --- .gitea/workflows/prod-deploy.yaml | 227 +++++++++++++++++----------- .gitea/workflows/prod-rollback.yaml | 223 +++++++++++++++++++++++++++ deploy/README.md | 20 ++- deploy/prod-deploy.sh | 13 +- docs/ARCHITECTURE.md | 8 +- 5 files changed, 397 insertions(+), 94 deletions(-) create mode 100644 .gitea/workflows/prod-rollback.yaml diff --git a/.gitea/workflows/prod-deploy.yaml b/.gitea/workflows/prod-deploy.yaml index 8ebd502..72e5596 100644 --- a/.gitea/workflows/prod-deploy.yaml +++ b/.gitea/workflows/prod-deploy.yaml @@ -1,9 +1,11 @@ -# Manual production rollout. Runs ONLY from master, ONLY on an explicit -# workflow_dispatch with confirm=deploy (development->master is merged + green first; -# this is the separate, deliberate prod step). It builds and pushes the images to the -# registry, then deploys over SSH: the main host via deploy/prod-deploy.sh (rolling, -# health-gated, auto-rollback; a maintenance window + consistent dump on a migration), -# then the bot host. See deploy/README.md (prod runbook) for operator steps. +# Manual production rollout. Runs ONLY from master, ONLY on workflow_dispatch with +# confirm=deploy (development->master is merged + green first; this is the separate, +# deliberate prod step). Visible sequential jobs from most to least significant: +# build -> deploy-main -> deploy-bot -> verify +# The per-service rolling (postgres->backend->gateway->landing->validator->caddy), +# health-gating and auto-rollback live in deploy/prod-deploy.sh on the main host and +# show in the deploy-main log. Manual post-deploy rollback is prod-rollback.yaml. +# See deploy/README.md (prod runbook). name: prod-deploy run-name: "prod deploy ${{ github.sha }}" @@ -18,41 +20,71 @@ on: permissions: contents: read +env: + NO_COLOR: "1" + DOCKER_CLI_HINTS: "false" + REGISTRY: docker.iliadenisov.ru/developer + jobs: - deploy: + build: if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'deploy' }} runs-on: ubuntu-latest defaults: run: shell: bash + outputs: + tag: ${{ steps.ver.outputs.tag }} env: - NO_COLOR: "1" - DOCKER_CLI_HINTS: "false" - REGISTRY: docker.iliadenisov.ru/developer - # SSH + registry + PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} + PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} + VITE_TELEGRAM_BOT_ID: ${{ vars.PROD_VITE_TELEGRAM_BOT_ID }} + VITE_TELEGRAM_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }} + VITE_TELEGRAM_GAME_CHANNEL_NAME: ${{ vars.PROD_VITE_TELEGRAM_GAME_CHANNEL_NAME }} + VITE_GATEWAY_URL: ${{ vars.PROD_VITE_GATEWAY_URL }} + POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }} + GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }} + TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }} + DICT_VERSION: ${{ vars.PROD_DICT_VERSION }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Compute version tag + id: ver + run: echo "tag=$(git describe --tags --always)" >> "$GITHUB_OUTPUT" + - name: Registry login + run: echo "$PROD_REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$PROD_REGISTRY_USER" --password-stdin + - name: Build and push images + working-directory: deploy + run: | + export TAG="${{ steps.ver.outputs.tag }}" APP_VERSION="${{ steps.ver.outputs.tag }}" SCRABBLE_CONFIG_DIR=. + # The four main-stack images via compose (reuses the build args, incl. VERSION); + # the bot separately, since it is profiled out of the prod compose. + docker compose -f docker-compose.yml -f docker-compose.prod.yml build + docker compose -f docker-compose.yml -f docker-compose.prod.yml push backend gateway landing validator + docker build -f ../platform/telegram/Dockerfile --target bot --build-arg VERSION="$TAG" -t "$REGISTRY/scrabble-telegram-bot:$TAG" .. + docker push "$REGISTRY/scrabble-telegram-bot:$TAG" + + deploy-main: + needs: build + runs-on: ubuntu-latest + defaults: + run: + shell: bash + env: + TAG: ${{ needs.build.outputs.tag }} PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} - TG_HOST: ${{ vars.PROD_TG_HOST }} - # SPA build args (baked into the gateway/landing images) - VITE_TELEGRAM_BOT_ID: ${{ vars.PROD_VITE_TELEGRAM_BOT_ID }} - VITE_TELEGRAM_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }} - VITE_TELEGRAM_GAME_CHANNEL_NAME: ${{ vars.PROD_VITE_TELEGRAM_GAME_CHANNEL_NAME }} - VITE_GATEWAY_URL: ${{ vars.PROD_VITE_GATEWAY_URL }} - # Runtime secrets POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }} GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }} GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }} TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }} - TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }} PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }} PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }} PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }} - PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }} - PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }} - # Runtime variables GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }} GRAFANA_ROOT_URL: ${{ vars.PROD_GRAFANA_ROOT_URL }} CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }} @@ -61,41 +93,15 @@ jobs: POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }} POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }} TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }} - TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }} - TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }} - TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }} - TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - - name: Compute image tag and app version - run: | - echo "TAG=$(git rev-parse --short=12 HEAD)" >> "$GITHUB_ENV" - echo "APP_VERSION=$(git describe --tags --always)" >> "$GITHUB_ENV" - - - name: Registry login - run: echo "$PROD_REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$PROD_REGISTRY_USER" --password-stdin - - - name: Build and push images - working-directory: deploy - run: | - export TAG SCRABBLE_CONFIG_DIR=. - # The four main-stack images via compose (reuses the compose build args); the - # bot separately, since it is profiled out of the prod compose. - docker compose -f docker-compose.yml -f docker-compose.prod.yml build - docker compose -f docker-compose.yml -f docker-compose.prod.yml push backend gateway landing validator - docker build -f ../platform/telegram/Dockerfile --target bot -t "$REGISTRY/scrabble-telegram-bot:$TAG" .. - docker push "$REGISTRY/scrabble-telegram-bot:$TAG" - - name: Set up SSH run: | mkdir -p ~/.ssh && chmod 700 ~/.ssh - printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy - chmod 600 ~/.ssh/id_deploy + printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - - name: Determine previous tag and migration run: | ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } @@ -103,21 +109,17 @@ jobs: MIGRATION=0 if [ "$PREV_TAG" != none ]; then if ! git cat-file -e "$PREV_TAG^{commit}" 2>/dev/null; then - MIGRATION=1 # unknown previous tag: take the safe window + dump + MIGRATION=1 elif git diff --name-only "$PREV_TAG..$TAG" -- backend/internal/postgres/migrations/ | grep -q .; then MIGRATION=1 fi fi - echo "PREV_TAG=$PREV_TAG" >> "$GITHUB_ENV" - echo "MIGRATION=$MIGRATION" >> "$GITHUB_ENV" + { echo "PREV_TAG=$PREV_TAG"; echo "MIGRATION=$MIGRATION"; } >> "$GITHUB_ENV" echo "prev=$PREV_TAG migration=$MIGRATION" - - - name: Render env files and certs + - name: Render main env + certs run: | umask 077 - mkdir -p stage/certs-main stage/certs-bot - # Main-host runtime env (single-quoted so the literal '$' in the bcrypt hash - # survives; prod-deploy.sh sources this, compose reads it from the environment). + mkdir -p stage/certs-main cat > stage/env.sh < stage/certs-main/ca.crt + printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt + printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY" > stage/certs-main/gateway.key + chmod 644 stage/certs-main/* + - name: Deploy the main host + run: | + ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } + ssh_main 'mkdir -p /opt/scrabble/compose' + tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \ + | ssh_main 'tar -C /opt/scrabble/compose -xzf -' + tar -C deploy -czf - caddy otelcol prometheus tempo grafana \ + | ssh_main 'tar -C /opt/scrabble -xzf -' + tar -C stage -czf - certs-main \ + | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -' + scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh" + echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin" + ssh_main "TAG='$TAG' PREV_TAG='$PREV_TAG' MIGRATION='$MIGRATION' bash /opt/scrabble/compose/prod-deploy.sh" + + deploy-bot: + needs: [build, deploy-main] + runs-on: ubuntu-latest + defaults: + run: + shell: bash + env: + TAG: ${{ needs.build.outputs.tag }} + PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} + PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} + PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} + PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} + TG_HOST: ${{ vars.PROD_TG_HOST }} + MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} + TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }} + TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }} + PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }} + PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }} + PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }} + LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }} + TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }} + TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }} + TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }} + TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }} + TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }} + steps: + - uses: actions/checkout@v4 + - name: Set up SSH + run: | + mkdir -p ~/.ssh && chmod 700 ~/.ssh + printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy + printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts + - name: Render bot env + certs + run: | + umask 077 + mkdir -p stage/certs-bot cat > stage/env.bot.sh < stage/certs-main/ca.crt - printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt - printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY" > stage/certs-main/gateway.key - printf '%s\n' "$PROD_BOTLINK_CA" > stage/certs-bot/ca.crt - printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt - printf '%s\n' "$PROD_BOTLINK_BOT_KEY" > stage/certs-bot/bot.key - chmod 644 stage/certs-main/* stage/certs-bot/* - - - name: Deploy the main host - run: | - ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } - ssh_main 'mkdir -p /opt/scrabble/compose' - # Compose files + config + script. - tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \ - | ssh_main 'tar -C /opt/scrabble/compose -xzf -' - tar -C deploy -czf - caddy otelcol prometheus tempo grafana \ - | ssh_main 'tar -C /opt/scrabble -xzf -' - tar -C stage -czf - certs-main \ - | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -' - scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh" - # Registry login on the host so compose can pull the private images. - echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin" - ssh_main "TAG='$TAG' PREV_TAG='$PREV_TAG' MIGRATION='$MIGRATION' bash /opt/scrabble/compose/prod-deploy.sh" - + printf '%s\n' "$PROD_BOTLINK_CA" > stage/certs-bot/ca.crt + printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt + printf '%s\n' "$PROD_BOTLINK_BOT_KEY" > stage/certs-bot/bot.key + chmod 644 stage/certs-bot/* - name: Deploy the bot host run: | ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; } @@ -187,7 +221,6 @@ jobs: ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose; docker compose -f docker-compose.bot.yml pull; docker compose -f docker-compose.bot.yml up -d' - # Bot liveness: running, not restarting, stable restart count. ssh_tg 'for i in $(seq 1 20); do s=$(docker inspect -f "{{.State.Status}}" scrabble-telegram-bot 2>/dev/null || echo missing) r=$(docker inspect -f "{{.State.Restarting}}" scrabble-telegram-bot 2>/dev/null || echo true) @@ -200,13 +233,27 @@ jobs: done echo "bot not healthy:"; docker logs --tail 80 scrabble-telegram-bot; exit 1' + verify: + needs: [deploy-main, deploy-bot] + runs-on: ubuntu-latest + defaults: + run: + shell: bash + env: + PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} + PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} + MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} + CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }} + steps: + - name: Set up SSH + run: | + mkdir -p ~/.ssh && chmod 700 ~/.ssh + printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy + printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - name: Verify the public site run: | - ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } - domain="${CADDY_SITE_ADDRESS%% *}" # first domain of "erudit-game.ru www..." - # Probe through caddy with the right vhost (force-resolved to the host); -k - # tolerates an ACME-pending cert. Also check the backend is actually ready. - ssh_main "for i in \$(seq 1 20); do + domain="${CADDY_SITE_ADDRESS%% *}" + ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "for i in \$(seq 1 20); do if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null && curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/app/ -o /dev/null && docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then diff --git a/.gitea/workflows/prod-rollback.yaml b/.gitea/workflows/prod-rollback.yaml new file mode 100644 index 0000000..2e0a3fa --- /dev/null +++ b/.gitea/workflows/prod-rollback.yaml @@ -0,0 +1,223 @@ +# Manual production rollback. Runs ONLY from master, ONLY on workflow_dispatch with +# confirm=rollback. Re-deploys an already-published image tag (no build): leave +# target_version blank to roll back to the previously deployed version (read from the +# main host), or set it to a specific release tag from the Releases page. The +# re-deploy is the same rolling, health-gated path as prod-deploy (TAG=target, +# MIGRATION=0 — rollback is image-only and never migrates the DB; image rollback is +# DB-safe under the expand-contract rule). See deploy/README.md (prod runbook). +name: prod-rollback +run-name: "prod rollback ${{ inputs.target_version || 'previous' }}" + +on: + workflow_dispatch: + inputs: + confirm: + description: 'Type "rollback" to confirm a production rollback.' + required: true + default: "" + target_version: + description: "Release tag to roll back to (blank = the previous deployed version)." + required: false + default: "" + +permissions: + contents: read + +env: + NO_COLOR: "1" + DOCKER_CLI_HINTS: "false" + REGISTRY: docker.iliadenisov.ru/developer + +jobs: + rollback-main: + if: ${{ github.ref == 'refs/heads/master' && inputs.confirm == 'rollback' }} + runs-on: ubuntu-latest + defaults: + run: + shell: bash + outputs: + target: ${{ steps.resolve.outputs.target }} + env: + PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} + PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} + PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} + PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} + MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} + POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }} + GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }} + GRAFANA_ADMIN_PASSWORD: ${{ secrets.PROD_GRAFANA_ADMIN_PASSWORD }} + TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }} + PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }} + PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }} + PROD_BOTLINK_GATEWAY_KEY: ${{ secrets.PROD_BOTLINK_GATEWAY_KEY }} + GM_BASICAUTH_USER: ${{ vars.PROD_GM_BASICAUTH_USER }} + GRAFANA_ROOT_URL: ${{ vars.PROD_GRAFANA_ROOT_URL }} + CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }} + LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }} + DICT_VERSION: ${{ vars.PROD_DICT_VERSION }} + POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }} + POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }} + TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }} + INPUT_TARGET: ${{ inputs.target_version }} + steps: + - uses: actions/checkout@v4 + - name: Set up SSH + run: | + mkdir -p ~/.ssh && chmod 700 ~/.ssh + printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy + printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts + - name: Resolve rollback target + id: resolve + run: | + ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } + CURRENT="$(ssh_main 'cat /opt/scrabble/DEPLOYED_TAG 2>/dev/null || echo none')" + if [ -n "$INPUT_TARGET" ]; then + TARGET="$INPUT_TARGET" + else + TARGET="$(ssh_main 'cat /opt/scrabble/PREVIOUS_TAG 2>/dev/null || echo none')" + fi + if [ -z "$TARGET" ] || [ "$TARGET" = none ]; then + echo "no rollback target (no PREVIOUS_TAG on the host and no target_version input)"; exit 1 + fi + if [ "$TARGET" = "$CURRENT" ]; then + echo "target $TARGET is already the deployed version; nothing to do"; exit 1 + fi + echo "rolling back: current=$CURRENT -> target=$TARGET" + echo "target=$TARGET" >> "$GITHUB_OUTPUT" + { echo "TARGET=$TARGET"; echo "CURRENT=$CURRENT"; } >> "$GITHUB_ENV" + - name: Render main env + certs + run: | + umask 077 + mkdir -p stage/certs-main + cat > stage/env.sh < stage/certs-main/ca.crt + printf '%s\n' "$PROD_BOTLINK_GATEWAY_CERT" > stage/certs-main/gateway.crt + printf '%s\n' "$PROD_BOTLINK_GATEWAY_KEY" > stage/certs-main/gateway.key + chmod 644 stage/certs-main/* + - name: Roll the main host back + run: | + ssh_main() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "$@"; } + ssh_main 'mkdir -p /opt/scrabble/compose' + tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \ + | ssh_main 'tar -C /opt/scrabble/compose -xzf -' + tar -C deploy -czf - caddy otelcol prometheus tempo grafana \ + | ssh_main 'tar -C /opt/scrabble -xzf -' + tar -C stage -czf - certs-main \ + | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -' + scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.sh "deploy@$MAIN_HOST:/opt/scrabble/env.sh" + echo "$PROD_REGISTRY_PASSWORD" | ssh_main "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin" + # Image-only rollback: no migration window (TAG=target, MIGRATION=0). A failed + # rollback's auto-revert returns to the current version (PREV_TAG=$CURRENT). + ssh_main "TAG='$TARGET' PREV_TAG='$CURRENT' MIGRATION=0 bash /opt/scrabble/compose/prod-deploy.sh" + + rollback-bot: + needs: rollback-main + runs-on: ubuntu-latest + defaults: + run: + shell: bash + env: + TARGET: ${{ needs.rollback-main.outputs.target }} + PROD_REGISTRY_USER: ${{ vars.PROD_REGISTRY_USER }} + PROD_REGISTRY_PASSWORD: ${{ secrets.PROD_REGISTRY_PASSWORD }} + PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} + PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} + TG_HOST: ${{ vars.PROD_TG_HOST }} + MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} + TELEGRAM_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_BOT_TOKEN }} + TELEGRAM_PROMO_BOT_TOKEN: ${{ secrets.PROD_TELEGRAM_PROMO_BOT_TOKEN }} + PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }} + PROD_BOTLINK_BOT_CERT: ${{ secrets.PROD_BOTLINK_BOT_CERT }} + PROD_BOTLINK_BOT_KEY: ${{ secrets.PROD_BOTLINK_BOT_KEY }} + LOG_LEVEL: ${{ vars.PROD_LOG_LEVEL }} + TELEGRAM_MINIAPP_URL: ${{ vars.PROD_TELEGRAM_MINIAPP_URL }} + TELEGRAM_GAME_CHANNEL_ID: ${{ vars.PROD_TELEGRAM_GAME_CHANNEL_ID }} + TELEGRAM_CHAT_ID: ${{ vars.PROD_TELEGRAM_CHAT_ID }} + TELEGRAM_BOT_USERNAME: ${{ vars.PROD_TELEGRAM_BOT_USERNAME }} + TELEGRAM_BOT_LINK: ${{ vars.PROD_VITE_TELEGRAM_LINK }} + steps: + - uses: actions/checkout@v4 + - name: Set up SSH + run: | + mkdir -p ~/.ssh && chmod 700 ~/.ssh + printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy + printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts + - name: Render bot env + certs + run: | + umask 077 + mkdir -p stage/certs-bot + cat > stage/env.bot.sh < stage/certs-bot/ca.crt + printf '%s\n' "$PROD_BOTLINK_BOT_CERT" > stage/certs-bot/bot.crt + printf '%s\n' "$PROD_BOTLINK_BOT_KEY" > stage/certs-bot/bot.key + chmod 644 stage/certs-bot/* + - name: Roll the bot host back + run: | + ssh_tg() { ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$TG_HOST" "$@"; } + ssh_tg 'mkdir -p /opt/scrabble/compose' + tar -C deploy -czf - docker-compose.bot.yml | ssh_tg 'tar -C /opt/scrabble/compose -xzf -' + tar -C stage -czf - certs-bot \ + | ssh_tg 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -' + scp -i ~/.ssh/id_deploy -o BatchMode=yes stage/env.bot.sh "deploy@$TG_HOST:/opt/scrabble/env.bot.sh" + echo "$PROD_REGISTRY_PASSWORD" | ssh_tg "docker login ${REGISTRY%%/*} -u $PROD_REGISTRY_USER --password-stdin" + ssh_tg 'set -a; . /opt/scrabble/env.bot.sh; set +a; cd /opt/scrabble/compose; + docker compose -f docker-compose.bot.yml pull; + docker compose -f docker-compose.bot.yml up -d' + + verify: + needs: [rollback-main, rollback-bot] + runs-on: ubuntu-latest + defaults: + run: + shell: bash + env: + PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }} + PROD_SSH_KNOWN_HOSTS: ${{ secrets.PROD_SSH_KNOWN_HOSTS }} + MAIN_HOST: ${{ vars.PROD_MAIN_HOST }} + CADDY_SITE_ADDRESS: ${{ vars.PROD_CADDY_SITE_ADDRESS }} + steps: + - name: Set up SSH + run: | + mkdir -p ~/.ssh && chmod 700 ~/.ssh + printf '%s\n' "$PROD_SSH_KEY" > ~/.ssh/id_deploy && chmod 600 ~/.ssh/id_deploy + printf '%s\n' "$PROD_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts + - name: Verify the public site + run: | + domain="${CADDY_SITE_ADDRESS%% *}" + ssh -i ~/.ssh/id_deploy -o BatchMode=yes "deploy@$MAIN_HOST" "for i in \$(seq 1 20); do + if curl -fsS -k --resolve $domain:443:127.0.0.1 https://$domain/ -o /dev/null && + docker run --rm --network scrabble-internal alpine:3.20 wget -q -T 5 -O /dev/null http://backend:8080/readyz; then + echo 'rolled-back site healthy'; exit 0 + fi + sleep 5 + done + echo 'verify failed'; docker logs --tail 40 scrabble-caddy; docker logs --tail 40 scrabble-backend; exit 1" diff --git a/deploy/README.md b/deploy/README.md index c2d8d94..7007af3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -130,7 +130,25 @@ workflow manually (Gitea → Actions → prod-deploy → run from `master`, inpu compose/config/certs/env over SSH, deploys the main host with `prod-deploy.sh` (rolling, health-gated, **auto-rollback to the previous tag**), then the bot host, then probes the public site. After `master` is green this workflow is the **only** thing that touches -prod — nothing auto-deploys there. +prod — nothing auto-deploys there. It runs four visible jobs: **build → deploy-main → +deploy-bot → verify** (the per-service rolling shows in the deploy-main log). + +**Versioning.** Each release is a git tag `vX.Y.Z` on `master`; the deploy stamps +`git describe --tags` into every image tag, every binary (`-ldflags` → `pkg/version` → +the `service.version` telemetry attribute) and the SPA About screen. Tag the release +before running the deploy: + +```sh +git tag -a v1.0.0 -m v1.0.0 && git push origin v1.0.0 +``` + +**Manual rollback** (any time after a successful deploy). Run the **`prod-rollback`** +workflow (Gitea → Actions → prod-rollback, `confirm=rollback`). Leave `target_version` +blank to roll back to the previously deployed version (read from the host's +`PREVIOUS_TAG`), or set it to a release tag from the **Releases** page. It re-deploys +that already-published image rolling + health-gated — no rebuild, no DB migration +(image rollback is DB-safe under the expand-contract rule). The registry keeps every +release tag, so any prior release is reachable. **Migrations** must be **expand-contract** (backward-compatible; goose is forward-only): the automatic rollback is image-only and never restores the DB. A deploy that changes diff --git a/deploy/prod-deploy.sh b/deploy/prod-deploy.sh index 91a408e..c2fef68 100755 --- a/deploy/prod-deploy.sh +++ b/deploy/prod-deploy.sh @@ -36,6 +36,9 @@ MIGRATION="${MIGRATION:-0}" COMPOSE_DIR="${COMPOSE_DIR:-/opt/scrabble/compose}" DUMP_DIR="${DUMP_DIR:-/opt/scrabble/dumps}" STATE_FILE="${STATE_FILE:-/opt/scrabble/DEPLOYED_TAG}" +# The prior deployed tag, preserved on every successful deploy so prod-rollback can +# target "the previous version" with no operator input. +PREV_STATE_FILE="${PREV_STATE_FILE:-/opt/scrabble/PREVIOUS_TAG}" PG_USER="${POSTGRES_USER:-scrabble}" PG_DB="${POSTGRES_DB:-scrabble}" @@ -88,6 +91,12 @@ rollback() { [ "$MIGRATION" = 1 ] && echo "NOTE: the DB is forward-migrated; a pre-deploy dump is in $DUMP_DIR — restore manually ONLY if the migration was destructive (see deploy/README.md, prod runbook)." } +commit_tag() { + # Record the just-deployed tag as current, preserving the prior one as previous. + [ -f "$STATE_FILE" ] && cp "$STATE_FILE" "$PREV_STATE_FILE" + echo "$TAG" > "$STATE_FILE" +} + mkdir -p "$DUMP_DIR" echo "=== prod deploy: tag=$TAG prev=$PREV_TAG migration=$MIGRATION ===" use_tag "$TAG" @@ -99,7 +108,7 @@ if [ -z "$(docker ps -aq -f name=scrabble-backend)" ]; then dc up -d --no-build --remove-orphans || { echo "compose up failed"; exit 1; } health_backend || { echo "backend not ready"; exit 1; } health_landing || { echo "landing not ready"; exit 1; } - echo "$TAG" > "$STATE_FILE" + commit_tag echo "first deploy healthy ($TAG)." exit 0 fi @@ -130,5 +139,5 @@ dc up -d --no-build --remove-orphans || { rollback; exit 1; } # Final internal sanity before committing the new tag. health_backend || { rollback; exit 1; } -echo "$TAG" > "$STATE_FILE" +commit_tag echo "=== deploy healthy ($TAG) ===" diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f95b818..0d2ec07 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1100,7 +1100,13 @@ Two contours, two secret/variable prefixes (`TEST_` / `PROD_`): any failure **rolls the whole stack back to the previous image tag**. A **schema migration** adds a maintenance window: the backend (the sole writer) is stopped for a consistent `pg_dump` before the new backend migrates forward — image rollback stays DB-safe under the - expand-contract migration rule, and the dump is kept for a manual restore. The main host is + expand-contract migration rule, and the dump is kept for a manual restore. The workflow runs + four visible jobs (build → deploy-main → deploy-bot → verify). Releases are git tags + `vX.Y.Z`; the version is stamped into the image tag, every binary (`-ldflags` → `pkg/version` + → the `service.version` telemetry attribute) and the SPA About screen. A separate manual + **`prod-rollback`** workflow re-deploys any prior release tag (blank input = the previous + deployed version, tracked on the host) over the same rolling, health-gated path — image-only, + no DB migration. The main host is intentionally **launch-sized** (2 vCPU / 1.9 GiB): the prod overlay trims the R7 limits (`GOMAXPROCS=2`, smaller caps, 7d Prometheus retention) and a **node_exporter** feeds host-memory metrics to Grafana so it can be resized reactively as players arrive.