Merge pull request 'feat(deploy): prod OOM swap cushion + edge maintenance page' (#170) from feature/prod-hardening into development

deploy/README.md

@@ -190,6 +190,16 @@ public site. After `master` is green this workflow is the **only** thing that to
 prod — nothing auto-deploys there. It runs four visible jobs: **build → deploy-main →
 deploy-bot → verify** (the per-service rolling shows in the deploy-main log).
 
+**Maintenance page.** During the roll (and any migration window) `prod-deploy.sh` raises a
+flag the edge caddy serves a static 503 "технические работы" page from
+(`deploy/caddy/maintenance.html`), for the user-facing routes only — `/_gm` (Grafana) stays
+reachable. The flag is cleared on any exit (success, a health failure + rollback, or an
+error) by a shell trap, so it can never stick on. It arms from this feature's own deploy
+onward (the caddy carrying the gate must be live first). This is **not** a zero-downtime
+deploy — the backend is a single stateful instance (in-memory game state + push hub, no
+Redis), so its swap still blips (clients auto-reconnect the live stream); the page just
+makes the window graceful instead of raw 502s.
+
 **Versioning.** Each release is a git tag `vX.Y.Z` on `master`; the deploy stamps
 `git describe --tags` into every image tag, every binary (`-ldflags` → `pkg/version` →
 the `service.version` telemetry attribute) and the SPA About screen. Tag the release
@@ -222,7 +232,11 @@ together with the fresh CA.
 **Sizing / monitoring:** the main host launches undersized (2 vCPU / 1.9 GiB); the prod
 overlay trims limits + `GOMAXPROCS=2` + 7d Prometheus retention, and `node_exporter` feeds
 host memory to Grafana (`/_gm/grafana/`). Watch host memory and resize at Selectel when
-players arrive.
+players arrive. The per-container memory caps are enforced (Compose v2 → cgroup), so no one
+service can eat all RAM, but they **overcommit** the host (~2.8 GiB of caps vs 1.9 GiB) — a
+**1 GiB swap file** (Ansible `common` role, `swap_size`, `vm.swappiness=10`) cushions a
+simultaneous spike into swap instead of the kernel OOM-killer. The `host_mem_low` Grafana
+alert fires under 10% available.
 
 **Shared Gitea set** (one unprefixed entry each, used by every contour) — secrets:
 `GATEWAY_VK_APP_SECRET, GATEWAY_VK_ID_CLIENT_SECRET, SMTP_RELAY_USER, SMTP_RELAY_PASS`;

deploy/ansible/group_vars/all.yml

@@ -19,3 +19,11 @@ scrabble_base_dir: /opt/scrabble
 # the host's own containers (and any ad-hoc runs) rotate identically.
 docker_log_max_size: "10m"
 docker_log_max_file: "3"
+
+# Swap file as an OOM cushion. The per-container memory caps (docker-compose.prod.yml)
+# sum to more than the tight main host's RAM (2 vCPU / 1.9 GiB), so a simultaneous
+# spike could otherwise hit the kernel OOM-killer and it might pick postgres. A small
+# swap absorbs the overshoot; swappiness stays low so swap is a cushion, not a hot
+# path (a slow service beats a killed one). Set swap_size to "0" to skip provisioning.
+swap_size: "1G"
+swap_swappiness: 10

deploy/ansible/roles/common/tasks/main.yml

@@ -150,6 +150,57 @@
     enabled: true
     state: started
 
+# --- Swap file (OOM cushion) ---------------------------------------------------
+# The prod main host is tight (1.9 GiB) and the per-container memory caps
+# (docker-compose.prod.yml) sum to more than RAM, so a simultaneous spike could hit
+# the kernel OOM-killer (which might pick postgres). A small swap absorbs the
+# overshoot. Idempotent; skipped when swap_size == "0". Builtin-only (no ansible.posix).
+
+- name: Check whether the swap file is already active
+  ansible.builtin.command: swapon --show=NAME --noheadings
+  register: swap_active
+  changed_when: false
+  when: swap_size != "0"
+
+- name: Allocate the swap file
+  ansible.builtin.command:
+    cmd: "fallocate -l {{ swap_size }} /swapfile"
+    creates: /swapfile
+  when: swap_size != "0" and '/swapfile' not in swap_active.stdout
+
+- name: Secure the swap file (0600)
+  ansible.builtin.file:
+    path: /swapfile
+    owner: root
+    group: root
+    mode: "0600"
+  when: swap_size != "0"
+
+- name: Format and enable the swap file
+  ansible.builtin.shell: "mkswap /swapfile && swapon /swapfile"
+  when: swap_size != "0" and '/swapfile' not in swap_active.stdout
+
+- name: Persist the swap file in /etc/fstab
+  ansible.builtin.lineinfile:
+    path: /etc/fstab
+    line: "/swapfile none swap sw 0 0"
+    regexp: '^/swapfile\s'
+    state: present
+  when: swap_size != "0"
+
+- name: Keep swap a cushion, not a hot path (low swappiness)
+  ansible.builtin.copy:
+    dest: /etc/sysctl.d/60-scrabble-swap.conf
+    mode: "0644"
+    content: "vm.swappiness = {{ swap_swappiness }}\n"
+  register: swappiness_conf
+  when: swap_size != "0"
+
+- name: Apply swappiness now
+  ansible.builtin.command: sysctl --system
+  changed_when: false
+  when: swap_size != "0" and swappiness_conf is changed
+
 # --- Deploy directories --------------------------------------------------------
 
 - name: Create the scrabble base directories

deploy/caddy/Caddyfile

@@ -33,6 +33,39 @@
 	# is the prod fix. Background + alternatives (incl. serving h3 for real): docs/EDGE_HTTP3.md.
 	header Alt-Svc clear
 
+	# Maintenance gate. While the deploy holds the flag file /srv/maint/on (touched by
+	# prod-deploy.sh around a rolling swap, removed on the script's exit), serve a static
+	# 503 "works in progress" page instead of proxying to a mid-recreate upstream —
+	# a graceful signal rather than raw 502s. Operator surfaces (/_gm) are excluded so
+	# Grafana stays reachable during the window. Caddy stat()s the flag per request, so
+	# toggling it needs no reload; the page + flag live in the caddy config dir bind-mounted
+	# read-only at /srv/maint (docker-compose.yml). The test contour never sets the flag
+	# (only prod-deploy.sh does), so this is inert there.
+	@maintenance {
+		not path /_gm /_gm/*
+		file {
+			root /srv/maint
+			try_files on
+		}
+	}
+	handle @maintenance {
+		error 503
+	}
+	handle_errors {
+		@maint503 expression {err.status_code} == 503
+		handle @maint503 {
+			root * /srv/maint
+			rewrite * /maintenance.html
+			header Retry-After 120
+			# Distinctive maintenance marker so the SPA can tell a planned window apart from
+			# a transient upstream 503 and show its own dimmed overlay (an in-session user
+			# never sees this static page — it only renders on a fresh load). The header
+			# rides every gated response, incl. the Connect/gRPC edge the app polls.
+			header X-Scrabble-Maintenance 1
+			file_server
+		}
+	}
+
 	# Operator surfaces under /_gm: a single shared Basic-Auth, then route.
 	@gm path /_gm /_gm/*
 	handle @gm {

deploy/caddy/maintenance.html

@@ -0,0 +1,44 @@
+<!doctype html>
+<!--
+  Served with 503 by the edge caddy while the deploy holds the maintenance flag
+  (/srv/maint/on, toggled by deploy/prod-deploy.sh around a rolling swap). Static and
+  self-contained (no upstream, no external assets) so it renders while the backend /
+  gateway are mid-recreate.
+-->
+<html lang="ru">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <meta name="robots" content="noindex">
+  <title>Эрудит — технические работы</title>
+  <style>
+    :root { color-scheme: light dark; }
+    html, body { height: 100%; margin: 0; }
+    body {
+      display: flex; align-items: center; justify-content: center;
+      font-family: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
+      background: #f4f1ea; color: #2b2b2b;
+      padding: 24px; box-sizing: border-box;
+    }
+    @media (prefers-color-scheme: dark) { body { background: #1d1b17; color: #e8e4da; } }
+    .card { max-width: 30rem; text-align: center; line-height: 1.5; }
+    .tile {
+      display: inline-block; width: 3.5rem; height: 3.5rem; line-height: 3.5rem;
+      font-size: 2rem; font-weight: 700; border-radius: 10px;
+      background: #d9b451; color: #2b2b2b; box-shadow: 0 2px 4px rgba(0,0,0,.25);
+      margin-bottom: 1.25rem;
+    }
+    h1 { font-size: 1.4rem; margin: 0 0 .5rem; }
+    p { margin: .35rem 0; }
+    .en { opacity: .7; font-size: .95rem; }
+  </style>
+</head>
+<body>
+  <main class="card">
+    <div class="tile">Э</div>
+    <h1>Технические работы</h1>
+    <p>Идёт короткое обновление игры. Мы скоро вернёмся — обновите страницу через пару минут.</p>
+    <p class="en">Scrabble is briefly down for an update. Please refresh in a couple of minutes.</p>
+  </main>
+</body>
+</html>

deploy/docker-compose.yml

@@ -428,6 +428,11 @@ services:
       GM_BASICAUTH_HASH: ${GM_BASICAUTH_HASH:?set GM_BASICAUTH_HASH}
     volumes:
       - ${SCRABBLE_CONFIG_DIR:-.}/caddy/Caddyfile:/etc/caddy/Caddyfile:ro
+      # Maintenance page + toggle flag: the caddy config dir holds maintenance.html and the
+      # `on` flag prod-deploy.sh touches around a rolling swap; the Caddyfile serves a 503
+      # from here while the flag exists (read-only mount — the deploy writes the flag on the
+      # host side). See deploy/caddy/Caddyfile and deploy/prod-deploy.sh.
+      - ${SCRABBLE_CONFIG_DIR:-.}/caddy:/srv/maint:ro
       - caddy-data:/data
     deploy:
       resources:

deploy/prod-deploy.sh

@@ -42,6 +42,15 @@ PREV_STATE_FILE="${PREV_STATE_FILE:-/opt/scrabble/PREVIOUS_TAG}"
 PG_USER="${POSTGRES_USER:-scrabble}"
 PG_DB="${POSTGRES_DB:-scrabble}"
 
+# Edge maintenance page. caddy serves a static 503 "works in progress" page for the
+# user-facing routes while this flag file exists (deploy/caddy/Caddyfile). We hold it
+# across the roll / migration window and clear it on ANY exit — success, a health
+# failure + rollback, or an unexpected error — via the trap, so users get a graceful
+# page instead of raw mid-swap 502s and the flag can never get stuck on.
+MAINT_FLAG="${MAINT_FLAG:-${SCRABBLE_CONFIG_DIR:-/opt/scrabble}/caddy/on}"
+maint_off() { rm -f "$MAINT_FLAG" 2>/dev/null || true; }
+trap maint_off EXIT
+
 cd "$COMPOSE_DIR" || { echo "compose dir $COMPOSE_DIR missing"; exit 1; }
 export REGISTRY
 # otelcol joins the host docker group to read the socket; the GID varies per host.
@@ -119,6 +128,13 @@ if [ -z "$(docker ps -aq -f name=scrabble-backend)" ]; then
   exit 0
 fi
 
+# Existing stack: raise the maintenance page for the whole roll (and any migration
+# window). It shows once caddy carries the gate (from this feature's own deploy
+# onward); the EXIT trap lowers it however this run ends.
+mkdir -p "$(dirname "$MAINT_FLAG")"
+: > "$MAINT_FLAG"
+echo "maintenance page raised ($MAINT_FLAG)"
+
 # Migration deploy: freeze writes and snapshot a consistent dump before migrating.
 if [ "$MIGRATION" = 1 ]; then
   echo "migration deploy: opening maintenance window (stopping the backend = the only writer)"