feat(deploy): prod compose split + host-memory monitoring (Stage 18)
Split the contour across the two prod hosts and retune for the small main host: - Gate vpn+bot to the telegram-local profile. The CI test deploy now passes --profile telegram-local so the test contour still brings them; the prod main host omits both, and the prod bot runs standalone from docker-compose.bot.yml. - docker-compose.prod.yml (main-host overlay): publish caddy 80/443 (no host caddy in prod; caddy owns ACME) and gateway 9443 (the remote bot dials in over mTLS); GOMAXPROCS=2, smaller memory caps and 7d Prometheus retention for the 2 vCPU / 1.9 GiB host. It launches deliberately undersized; resize reactively. - docker-compose.bot.yml: standalone bot for the tg host (no VPN, OTLP off since otelcol is unreachable from there, dials the main host's bot-link). - Add node_exporter + a Prometheus scrape so host memory pressure (the OOM signal on the tight main host), not just per-container docker_stats, is visible. - Soften AWG_CONF to a default: only the profiled vpn sidecar consumes it, and compose interpolates profiled-out services too, so prod must not require it.
This commit is contained in:
Ilia Denisov
@@ -0,0 +1,53 @@
|
||||
# Production Telegram bot host descriptor (standalone — NOT an overlay). Run only on
|
||||
# the bot host:
|
||||
# docker compose -f docker-compose.bot.yml up -d
|
||||
#
|
||||
# The bot egresses to the Bot API directly (no VPN sidecar) and dials the main host's
|
||||
# published bot-link :9443 over mTLS. It exports no telemetry — otelcol lives on the
|
||||
# main host and is unreachable from here — so observe it via `docker logs` on this host.
|
||||
# Values come from the prod-deploy workflow (PROD_ secrets/variables); BOT_IMAGE is the
|
||||
# pushed registry tag and BOTLINK_GATEWAY_ADDR is the main host's <ip>:9443.
|
||||
name: scrabble-bot
|
||||
|
||||
services:
|
||||
bot:
|
||||
container_name: scrabble-telegram-bot
|
||||
image: ${BOT_IMAGE:?set BOT_IMAGE to the registry tag}
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
environment:
|
||||
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:?set TELEGRAM_BOT_TOKEN}
|
||||
TELEGRAM_GAME_CHANNEL_ID: ${TELEGRAM_GAME_CHANNEL_ID:-}
|
||||
TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-}
|
||||
TELEGRAM_PROMO_BOT_TOKEN: ${TELEGRAM_PROMO_BOT_TOKEN:-}
|
||||
TELEGRAM_BOT_USERNAME: ${TELEGRAM_BOT_USERNAME:-}
|
||||
TELEGRAM_BOT_LINK: ${TELEGRAM_BOT_LINK:-}
|
||||
TELEGRAM_MINIAPP_URL: ${TELEGRAM_MINIAPP_URL:?set TELEGRAM_MINIAPP_URL}
|
||||
# Real Bot API in prod (the test contour pins TELEGRAM_TEST_ENV=true instead).
|
||||
TELEGRAM_TEST_ENV: "false"
|
||||
TELEGRAM_API_BASE_URL: ${TELEGRAM_API_BASE_URL:-}
|
||||
TELEGRAM_OWNS_UPDATES: "true"
|
||||
# Dials the main host's published bot-link. ServerName stays `gateway` (the cert
|
||||
# SAN), so TLS validation is independent of the dial address.
|
||||
TELEGRAM_GATEWAY_ADDR: ${BOTLINK_GATEWAY_ADDR:?set BOTLINK_GATEWAY_ADDR (main:9443)}
|
||||
TELEGRAM_BOTLINK_SERVER_NAME: gateway
|
||||
TELEGRAM_BOTLINK_TLS_CERT: /certs/bot.crt
|
||||
TELEGRAM_BOTLINK_TLS_KEY: /certs/bot.key
|
||||
TELEGRAM_BOTLINK_TLS_CA: /certs/ca.crt
|
||||
TELEGRAM_LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
TELEGRAM_SERVICE_NAME: scrabble-telegram-bot
|
||||
# No telemetry export: otelcol is on the main host, unreachable from here.
|
||||
TELEGRAM_OTEL_TRACES_EXPORTER: none
|
||||
TELEGRAM_OTEL_METRICS_EXPORTER: none
|
||||
GOMAXPROCS: "1"
|
||||
volumes:
|
||||
- ${SCRABBLE_CONFIG_DIR:-.}/certs:/certs:ro
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 256M
|
||||
@@ -0,0 +1,92 @@
|
||||
# Production main-host overlay, applied on top of docker-compose.yml on the main host:
|
||||
# docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
|
||||
#
|
||||
# It (1) publishes caddy 80/443 — there is no host caddy in prod, so the contour caddy
|
||||
# owns the edge and does its own ACME on CADDY_SITE_ADDRESS — and the gateway bot-link
|
||||
# :9443 the remote bot dials in over mTLS; and (2) retunes the R7 limits down for the
|
||||
# 2 vCPU / 1.9 GiB host (GOMAXPROCS=2, smaller memory caps, shorter Prometheus
|
||||
# retention). The contour launches deliberately undersized at zero players; the added
|
||||
# node_exporter + Grafana watch host memory so it can be resized at Selectel when
|
||||
# traffic arrives.
|
||||
#
|
||||
# The bot + its VPN sidecar are absent here (the telegram-local profile is not
|
||||
# activated); the prod bot runs on its own host from docker-compose.bot.yml.
|
||||
|
||||
services:
|
||||
caddy:
|
||||
ports:
|
||||
- "80:80"
|
||||
- "443:443"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 96M
|
||||
|
||||
gateway:
|
||||
ports:
|
||||
- "9443:9443"
|
||||
environment:
|
||||
# 2 vCPU host: align the Go scheduler with the cgroup quota (R7's 3 needs 3 cores).
|
||||
GOMAXPROCS: "2"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "2.0"
|
||||
memory: 384M
|
||||
|
||||
backend:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 384M
|
||||
|
||||
postgres:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 384M
|
||||
|
||||
validator:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 96M
|
||||
|
||||
landing:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 64M
|
||||
|
||||
otelcol:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 256M
|
||||
|
||||
prometheus:
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.retention.time=7d
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 256M
|
||||
|
||||
tempo:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 384M
|
||||
|
||||
grafana:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 256M
|
||||
|
||||
postgres_exporter:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 64M
|
||||
@@ -240,14 +240,22 @@ services:
|
||||
networks: [internal]
|
||||
|
||||
# --- Telegram bot (egress via the VPN sidecar in test; dials the gateway) ---
|
||||
# vpn + bot are gated to the `telegram-local` profile: the test contour runs them
|
||||
# locally (CI passes --profile telegram-local), the prod main host omits them, and
|
||||
# the prod bot runs on its own host from deploy/docker-compose.bot.yml.
|
||||
vpn:
|
||||
container_name: scrabble-telegram-vpn
|
||||
image: docker.iliadenisov.ru/developer/amneziawg-sidecar:latest
|
||||
profiles: ["telegram-local"]
|
||||
restart: unless-stopped
|
||||
logging: *default-logging
|
||||
privileged: true
|
||||
environment:
|
||||
AWG_CONF: ${AWG_CONF:?set AWG_CONF}
|
||||
# Required by the vpn sidecar, which is gated to the telegram-local profile.
|
||||
# Compose can't scope a `:?` guard to a profile (interpolation runs for
|
||||
# profiled-out services too) and the prod main host has no VPN, so this is a soft
|
||||
# default; the test contour always supplies TEST_AWG_CONF and the sidecar validates it.
|
||||
AWG_CONF: ${AWG_CONF:-}
|
||||
networks:
|
||||
internal:
|
||||
aliases: [telegram]
|
||||
@@ -255,6 +263,7 @@ services:
|
||||
bot:
|
||||
container_name: scrabble-telegram-bot
|
||||
image: scrabble-telegram-bot:latest
|
||||
profiles: ["telegram-local"]
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: platform/telegram/Dockerfile
|
||||
@@ -444,6 +453,26 @@ services:
|
||||
memory: 128M
|
||||
networks: [internal]
|
||||
|
||||
# node_exporter exports host CPU/memory/disk metrics. The prod main host runs a tight
|
||||
# 1.9 GiB budget, so host memory pressure — not just per-container docker_stats — is
|
||||
# what warns before an OOM. Prometheus scrapes it at :9100 (see prometheus.yml).
|
||||
node_exporter:
|
||||
container_name: scrabble-node-exporter
|
||||
image: quay.io/prometheus/node-exporter:v1.8.2
|
||||
restart: unless-stopped
|
||||
logging: *default-logging
|
||||
command:
|
||||
- --path.rootfs=/host
|
||||
- --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host)($|/)
|
||||
pid: host
|
||||
volumes:
|
||||
- /:/host:ro,rslave
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 64M
|
||||
networks: [internal]
|
||||
|
||||
networks:
|
||||
internal:
|
||||
name: scrabble-internal
|
||||
|
||||
@@ -18,3 +18,8 @@ scrape_configs:
|
||||
- job_name: postgres_exporter
|
||||
static_configs:
|
||||
- targets: ["postgres_exporter:9187"]
|
||||
# Host-level metrics (memory/CPU/disk). Matters most on the prod main host's tight
|
||||
# 1.9 GiB budget, where total host memory is the OOM-proximity signal.
|
||||
- job_name: node
|
||||
static_configs:
|
||||
- targets: ["node_exporter:9100"]
|
||||
|
||||
Reference in New Issue
Block a user