feat(deploy): prod compose split + host-memory monitoring (Stage 18)

Split the contour across the two prod hosts and retune for the small main host:

- Gate vpn+bot to the telegram-local profile. The CI test deploy now passes
  --profile telegram-local so the test contour still brings them; the prod main
  host omits both, and the prod bot runs standalone from docker-compose.bot.yml.
- docker-compose.prod.yml (main-host overlay): publish caddy 80/443 (no host
  caddy in prod; caddy owns ACME) and gateway 9443 (the remote bot dials in over
  mTLS); GOMAXPROCS=2, smaller memory caps and 7d Prometheus retention for the
  2 vCPU / 1.9 GiB host. It launches deliberately undersized; resize reactively.
- docker-compose.bot.yml: standalone bot for the tg host (no VPN, OTLP off since
  otelcol is unreachable from there, dials the main host's bot-link).
- Add node_exporter + a Prometheus scrape so host memory pressure (the OOM
  signal on the tight main host), not just per-container docker_stats, is visible.
- Soften AWG_CONF to a default: only the profiled vpn sidecar consumes it, and
  compose interpolates profiled-out services too, so prod must not require it.

.gitea/workflows/ci.yaml

@@ -301,8 +301,11 @@ jobs:
           # App version for the About screen: the git tag if present, else the short SHA
           # (the test checkout is shallow/untagged, so this is the SHA here — fine).
           export APP_VERSION="$(git -C "$GITHUB_WORKSPACE" describe --tags --always 2>/dev/null || echo dev)"
-          docker compose --ansi never build --progress plain
+          # The telegram-local profile brings the bot + its VPN sidecar; prod runs the
-          docker compose --ansi never up -d --remove-orphans
+          # bot on its own host instead (deploy/docker-compose.bot.yml), and the prod
+          # main host omits both. Without the profile they would not start here.
+          docker compose --ansi never --profile telegram-local build --progress plain
+          docker compose --ansi never --profile telegram-local up -d --remove-orphans
           # The config-only services bind-mount the reseeded config dir. A plain `up -d`
           # leaves them on the previous bind mount (the dir was rm'd + recreated), so a
           # changed Caddyfile or Grafana dashboard is ignored — force-recreate them to

deploy/docker-compose.bot.yml

@@ -0,0 +1,53 @@
+# Production Telegram bot host descriptor (standalone — NOT an overlay). Run only on
+# the bot host:
+#   docker compose -f docker-compose.bot.yml up -d
+#
+# The bot egresses to the Bot API directly (no VPN sidecar) and dials the main host's
+# published bot-link :9443 over mTLS. It exports no telemetry — otelcol lives on the
+# main host and is unreachable from here — so observe it via `docker logs` on this host.
+# Values come from the prod-deploy workflow (PROD_ secrets/variables); BOT_IMAGE is the
+# pushed registry tag and BOTLINK_GATEWAY_ADDR is the main host's <ip>:9443.
+name: scrabble-bot
+
+services:
+  bot:
+    container_name: scrabble-telegram-bot
+    image: ${BOT_IMAGE:?set BOT_IMAGE to the registry tag}
+    restart: unless-stopped
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+    environment:
+      TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:?set TELEGRAM_BOT_TOKEN}
+      TELEGRAM_GAME_CHANNEL_ID: ${TELEGRAM_GAME_CHANNEL_ID:-}
+      TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-}
+      TELEGRAM_PROMO_BOT_TOKEN: ${TELEGRAM_PROMO_BOT_TOKEN:-}
+      TELEGRAM_BOT_USERNAME: ${TELEGRAM_BOT_USERNAME:-}
+      TELEGRAM_BOT_LINK: ${TELEGRAM_BOT_LINK:-}
+      TELEGRAM_MINIAPP_URL: ${TELEGRAM_MINIAPP_URL:?set TELEGRAM_MINIAPP_URL}
+      # Real Bot API in prod (the test contour pins TELEGRAM_TEST_ENV=true instead).
+      TELEGRAM_TEST_ENV: "false"
+      TELEGRAM_API_BASE_URL: ${TELEGRAM_API_BASE_URL:-}
+      TELEGRAM_OWNS_UPDATES: "true"
+      # Dials the main host's published bot-link. ServerName stays `gateway` (the cert
+      # SAN), so TLS validation is independent of the dial address.
+      TELEGRAM_GATEWAY_ADDR: ${BOTLINK_GATEWAY_ADDR:?set BOTLINK_GATEWAY_ADDR (main:9443)}
+      TELEGRAM_BOTLINK_SERVER_NAME: gateway
+      TELEGRAM_BOTLINK_TLS_CERT: /certs/bot.crt
+      TELEGRAM_BOTLINK_TLS_KEY: /certs/bot.key
+      TELEGRAM_BOTLINK_TLS_CA: /certs/ca.crt
+      TELEGRAM_LOG_LEVEL: ${LOG_LEVEL:-info}
+      TELEGRAM_SERVICE_NAME: scrabble-telegram-bot
+      # No telemetry export: otelcol is on the main host, unreachable from here.
+      TELEGRAM_OTEL_TRACES_EXPORTER: none
+      TELEGRAM_OTEL_METRICS_EXPORTER: none
+      GOMAXPROCS: "1"
+    volumes:
+      - ${SCRABBLE_CONFIG_DIR:-.}/certs:/certs:ro
+    deploy:
+      resources:
+        limits:
+          cpus: "1.0"
+          memory: 256M

deploy/docker-compose.prod.yml

@@ -0,0 +1,92 @@
+# Production main-host overlay, applied on top of docker-compose.yml on the main host:
+#   docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
+#
+# It (1) publishes caddy 80/443 — there is no host caddy in prod, so the contour caddy
+# owns the edge and does its own ACME on CADDY_SITE_ADDRESS — and the gateway bot-link
+# :9443 the remote bot dials in over mTLS; and (2) retunes the R7 limits down for the
+# 2 vCPU / 1.9 GiB host (GOMAXPROCS=2, smaller memory caps, shorter Prometheus
+# retention). The contour launches deliberately undersized at zero players; the added
+# node_exporter + Grafana watch host memory so it can be resized at Selectel when
+# traffic arrives.
+#
+# The bot + its VPN sidecar are absent here (the telegram-local profile is not
+# activated); the prod bot runs on its own host from docker-compose.bot.yml.
+
+services:
+  caddy:
+    ports:
+      - "80:80"
+      - "443:443"
+    deploy:
+      resources:
+        limits:
+          memory: 96M
+
+  gateway:
+    ports:
+      - "9443:9443"
+    environment:
+      # 2 vCPU host: align the Go scheduler with the cgroup quota (R7's 3 needs 3 cores).
+      GOMAXPROCS: "2"
+    deploy:
+      resources:
+        limits:
+          cpus: "2.0"
+          memory: 384M
+
+  backend:
+    deploy:
+      resources:
+        limits:
+          memory: 384M
+
+  postgres:
+    deploy:
+      resources:
+        limits:
+          memory: 384M
+
+  validator:
+    deploy:
+      resources:
+        limits:
+          memory: 96M
+
+  landing:
+    deploy:
+      resources:
+        limits:
+          memory: 64M
+
+  otelcol:
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+
+  prometheus:
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.retention.time=7d
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+
+  tempo:
+    deploy:
+      resources:
+        limits:
+          memory: 384M
+
+  grafana:
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+
+  postgres_exporter:
+    deploy:
+      resources:
+        limits:
+          memory: 64M

deploy/docker-compose.yml

@@ -240,14 +240,22 @@ services:
     networks: [internal]
 
   # --- Telegram bot (egress via the VPN sidecar in test; dials the gateway) ---
+  # vpn + bot are gated to the `telegram-local` profile: the test contour runs them
+  # locally (CI passes --profile telegram-local), the prod main host omits them, and
+  # the prod bot runs on its own host from deploy/docker-compose.bot.yml.
   vpn:
     container_name: scrabble-telegram-vpn
     image: docker.iliadenisov.ru/developer/amneziawg-sidecar:latest
+    profiles: ["telegram-local"]
     restart: unless-stopped
     logging: *default-logging
     privileged: true
     environment:
-      AWG_CONF: ${AWG_CONF:?set AWG_CONF}
+      # Required by the vpn sidecar, which is gated to the telegram-local profile.
+      # Compose can't scope a `:?` guard to a profile (interpolation runs for
+      # profiled-out services too) and the prod main host has no VPN, so this is a soft
+      # default; the test contour always supplies TEST_AWG_CONF and the sidecar validates it.
+      AWG_CONF: ${AWG_CONF:-}
     networks:
       internal:
         aliases: [telegram]
@@ -255,6 +263,7 @@ services:
   bot:
     container_name: scrabble-telegram-bot
     image: scrabble-telegram-bot:latest
+    profiles: ["telegram-local"]
     build:
       context: ..
       dockerfile: platform/telegram/Dockerfile
@@ -444,6 +453,26 @@ services:
           memory: 128M
     networks: [internal]
 
+  # node_exporter exports host CPU/memory/disk metrics. The prod main host runs a tight
+  # 1.9 GiB budget, so host memory pressure — not just per-container docker_stats — is
+  # what warns before an OOM. Prometheus scrapes it at :9100 (see prometheus.yml).
+  node_exporter:
+    container_name: scrabble-node-exporter
+    image: quay.io/prometheus/node-exporter:v1.8.2
+    restart: unless-stopped
+    logging: *default-logging
+    command:
+      - --path.rootfs=/host
+      - --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host)($|/)
+    pid: host
+    volumes:
+      - /:/host:ro,rslave
+    deploy:
+      resources:
+        limits:
+          memory: 64M
+    networks: [internal]
+
 networks:
   internal:
     name: scrabble-internal

deploy/prometheus/prometheus.yml

@@ -18,3 +18,8 @@ scrape_configs:
   - job_name: postgres_exporter
     static_configs:
       - targets: ["postgres_exporter:9187"]
+  # Host-level metrics (memory/CPU/disk). Matters most on the prod main host's tight
+  # 1.9 GiB budget, where total host memory is the OOM-proximity signal.
+  - job_name: node
+    static_configs:
+      - targets: ["node_exporter:9100"]