Merge pull request 'Release v1.8.0: prod build fix (re-promote)' (#175) from development into master

The first v1.8.0 prod-deploy build job failed: `docker compose build` interpolates the
WHOLE compose file, and the backend's :?-guarded EXPORT_SIGN_KEY — added with the
finished-game export after the last prod release (v1.7.0), so the prod build never
exercised it — was absent from the build job's env, tripping the guard before any image
built. Prod was untouched (build-only failure: no image pushed, no deploy, no migration).

Add EXPORT_SIGN_KEY (audited every :?-guarded compose var against the build job env — it
was the only genuinely-missing one; TELEGRAM_MINIAPP_URL is derived in the run step).
Verified by reproducing the build-job env + `docker compose config`.

.gitea/workflows/ci.yaml

@@ -399,6 +399,13 @@ jobs:
           mkdir -p "$conf"
           cp -r caddy otelcol prometheus tempo grafana blackbox "$conf"/
           export SCRABBLE_CONFIG_DIR="$conf"
+          # Maintenance page for the redeploy window, mirroring prod-deploy.sh so the SPA
+          # overlay is exercised on the test contour too (not only prod). Raised just before
+          # the recreate and lowered once caddy is back (below); the trap clears it if the
+          # step fails so the contour never sticks in maintenance (and the reseed above wipes a
+          # stale flag anyway). The caddy-routed probes run in the NEXT step, after it is lowered.
+          maint_flag="$conf/caddy/on"
+          trap 'rm -f "$maint_flag"' EXIT
           # Derive the public URLs from the one canonical origin instead of storing each as
           # its own variable (paths are structural SPA routes / the Caddy /_gm sub-path).
           # Exported before build so the VK ID redirect is baked into the SPA.
@@ -430,12 +437,23 @@ jobs:
           # bot on its own host instead (deploy/docker-compose.bot.yml), and the prod
           # main host omits both. Without the profile they would not start here.
           docker compose --ansi never --profile telegram-local build --progress plain
+          # Raise the maintenance page, THEN bring caddy onto the reseeded config mount so it
+          # actually carries the flag: the running caddy sits on the stale pre-reseed mount (the
+          # dir was rm'd + recreated — see the force-recreate note below), so a flag written to
+          # the new dir is invisible until caddy is recreated. With the fresh caddy up, an open
+          # SPA sees the 503 marker + overlay for the whole recreate window, not a bare reconnect.
+          : > "$maint_flag"
+          docker compose --ansi never up -d --force-recreate --no-deps caddy
           docker compose --ansi never --profile telegram-local up -d --remove-orphans
           # The config-only services bind-mount the reseeded config dir. A plain `up -d`
           # leaves them on the previous bind mount (the dir was rm'd + recreated), so a
-          # changed Caddyfile or Grafana dashboard is ignored — force-recreate them to
-          # pick up the fresh config.
-          docker compose --ansi never up -d --force-recreate --no-deps caddy otelcol prometheus tempo grafana
+          # changed Grafana dashboard is ignored — force-recreate them to pick up the fresh
+          # config. (Caddy was already recreated above so it would carry the maintenance flag.)
+          docker compose --ansi never up -d --force-recreate --no-deps otelcol prometheus tempo grafana
+          # Lower the maintenance page: services are back. An open SPA's poll now gets through
+          # (once the gateway finishes booting) and reloads into the fresh client; the caddy
+          # probes in the next step see 200. The EXIT trap is a backstop if we failed earlier.
+          rm -f "$maint_flag"
 
       - name: Probe the landing, gateway and backend
         run: |

.gitea/workflows/prod-deploy.yaml

@@ -46,6 +46,12 @@ jobs:
       # VITE_GATEWAY_URL omitted: the SPA is served same-origin (compose ":-" default).
       POSTGRES_PASSWORD: ${{ secrets.PROD_POSTGRES_PASSWORD }}
       GM_BASICAUTH_HASH: ${{ secrets.PROD_GM_BASICAUTH_HASH }}
+      # `docker compose build` interpolates the WHOLE compose file, so every :?-guarded
+      # runtime var must be present at build even though it is not a build-arg — incl. the
+      # backend's EXPORT_SIGN_KEY (added with the finished-game export after v1.7.0, which is
+      # why the first v1.8.0 build tripped on it). POSTGRES_PASSWORD/GM_BASICAUTH_HASH above
+      # are here for the same reason; DICT_VERSION + the derived Mini App URL cover the rest.
+      EXPORT_SIGN_KEY: ${{ secrets.PROD_EXPORT_SIGN_KEY }}
       # PUBLIC_BASE_URL drives the derived VK ID redirect (baked into the SPA) and the
       # Mini App URL; both are computed in the build step, not stored variables.
       PUBLIC_BASE_URL: ${{ vars.PROD_PUBLIC_BASE_URL }}

deploy/README.md

@@ -190,6 +190,16 @@ public site. After `master` is green this workflow is the **only** thing that to
 prod — nothing auto-deploys there. It runs four visible jobs: **build → deploy-main →
 deploy-bot → verify** (the per-service rolling shows in the deploy-main log).
 
+**Maintenance page.** During the roll (and any migration window) `prod-deploy.sh` raises a
+flag the edge caddy serves a static 503 "технические работы" page from
+(`deploy/caddy/maintenance.html`), for the user-facing routes only — `/_gm` (Grafana) stays
+reachable. The flag is cleared on any exit (success, a health failure + rollback, or an
+error) by a shell trap, so it can never stick on. It arms from this feature's own deploy
+onward (the caddy carrying the gate must be live first). This is **not** a zero-downtime
+deploy — the backend is a single stateful instance (in-memory game state + push hub, no
+Redis), so its swap still blips (clients auto-reconnect the live stream); the page just
+makes the window graceful instead of raw 502s.
+
 **Versioning.** Each release is a git tag `vX.Y.Z` on `master`; the deploy stamps
 `git describe --tags` into every image tag, every binary (`-ldflags` → `pkg/version` →
 the `service.version` telemetry attribute) and the SPA About screen. Tag the release
@@ -222,7 +232,11 @@ together with the fresh CA.
 **Sizing / monitoring:** the main host launches undersized (2 vCPU / 1.9 GiB); the prod
 overlay trims limits + `GOMAXPROCS=2` + 7d Prometheus retention, and `node_exporter` feeds
 host memory to Grafana (`/_gm/grafana/`). Watch host memory and resize at Selectel when
-players arrive.
+players arrive. The per-container memory caps are enforced (Compose v2 → cgroup), so no one
+service can eat all RAM, but they **overcommit** the host (~2.8 GiB of caps vs 1.9 GiB) — a
+**1 GiB swap file** (Ansible `common` role, `swap_size`, `vm.swappiness=10`) cushions a
+simultaneous spike into swap instead of the kernel OOM-killer. The `host_mem_low` Grafana
+alert fires under 10% available.
 
 **Shared Gitea set** (one unprefixed entry each, used by every contour) — secrets:
 `GATEWAY_VK_APP_SECRET, GATEWAY_VK_ID_CLIENT_SECRET, SMTP_RELAY_USER, SMTP_RELAY_PASS`;

deploy/ansible/group_vars/all.yml

@@ -19,3 +19,11 @@ scrabble_base_dir: /opt/scrabble
 # the host's own containers (and any ad-hoc runs) rotate identically.
 docker_log_max_size: "10m"
 docker_log_max_file: "3"
+
+# Swap file as an OOM cushion. The per-container memory caps (docker-compose.prod.yml)
+# sum to more than the tight main host's RAM (2 vCPU / 1.9 GiB), so a simultaneous
+# spike could otherwise hit the kernel OOM-killer and it might pick postgres. A small
+# swap absorbs the overshoot; swappiness stays low so swap is a cushion, not a hot
+# path (a slow service beats a killed one). Set swap_size to "0" to skip provisioning.
+swap_size: "1G"
+swap_swappiness: 10

deploy/ansible/roles/common/tasks/main.yml

@@ -150,6 +150,57 @@
     enabled: true
     state: started
 
+# --- Swap file (OOM cushion) ---------------------------------------------------
+# The prod main host is tight (1.9 GiB) and the per-container memory caps
+# (docker-compose.prod.yml) sum to more than RAM, so a simultaneous spike could hit
+# the kernel OOM-killer (which might pick postgres). A small swap absorbs the
+# overshoot. Idempotent; skipped when swap_size == "0". Builtin-only (no ansible.posix).
+
+- name: Check whether the swap file is already active
+  ansible.builtin.command: swapon --show=NAME --noheadings
+  register: swap_active
+  changed_when: false
+  when: swap_size != "0"
+
+- name: Allocate the swap file
+  ansible.builtin.command:
+    cmd: "fallocate -l {{ swap_size }} /swapfile"
+    creates: /swapfile
+  when: swap_size != "0" and '/swapfile' not in swap_active.stdout
+
+- name: Secure the swap file (0600)
+  ansible.builtin.file:
+    path: /swapfile
+    owner: root
+    group: root
+    mode: "0600"
+  when: swap_size != "0"
+
+- name: Format and enable the swap file
+  ansible.builtin.shell: "mkswap /swapfile && swapon /swapfile"
+  when: swap_size != "0" and '/swapfile' not in swap_active.stdout
+
+- name: Persist the swap file in /etc/fstab
+  ansible.builtin.lineinfile:
+    path: /etc/fstab
+    line: "/swapfile none swap sw 0 0"
+    regexp: '^/swapfile\s'
+    state: present
+  when: swap_size != "0"
+
+- name: Keep swap a cushion, not a hot path (low swappiness)
+  ansible.builtin.copy:
+    dest: /etc/sysctl.d/60-scrabble-swap.conf
+    mode: "0644"
+    content: "vm.swappiness = {{ swap_swappiness }}\n"
+  register: swappiness_conf
+  when: swap_size != "0"
+
+- name: Apply swappiness now
+  ansible.builtin.command: sysctl --system
+  changed_when: false
+  when: swap_size != "0" and swappiness_conf is changed
+
 # --- Deploy directories --------------------------------------------------------
 
 - name: Create the scrabble base directories

deploy/caddy/Caddyfile

@@ -33,6 +33,39 @@
 	# is the prod fix. Background + alternatives (incl. serving h3 for real): docs/EDGE_HTTP3.md.
 	header Alt-Svc clear
 
+	# Maintenance gate. While the deploy holds the flag file /srv/maint/on (touched by
+	# prod-deploy.sh around a rolling swap, removed on the script's exit), serve a static
+	# 503 "works in progress" page instead of proxying to a mid-recreate upstream —
+	# a graceful signal rather than raw 502s. Operator surfaces (/_gm) are excluded so
+	# Grafana stays reachable during the window. Caddy stat()s the flag per request, so
+	# toggling it needs no reload; the page + flag live in the caddy config dir bind-mounted
+	# read-only at /srv/maint (docker-compose.yml). The test contour never sets the flag
+	# (only prod-deploy.sh does), so this is inert there.
+	@maintenance {
+		not path /_gm /_gm/*
+		file {
+			root /srv/maint
+			try_files on
+		}
+	}
+	handle @maintenance {
+		error 503
+	}
+	handle_errors {
+		@maint503 expression {err.status_code} == 503
+		handle @maint503 {
+			root * /srv/maint
+			rewrite * /maintenance.html
+			header Retry-After 120
+			# Distinctive maintenance marker so the SPA can tell a planned window apart from
+			# a transient upstream 503 and show its own dimmed overlay (an in-session user
+			# never sees this static page — it only renders on a fresh load). The header
+			# rides every gated response, incl. the Connect/gRPC edge the app polls.
+			header X-Scrabble-Maintenance 1
+			file_server
+		}
+	}
+
 	# Operator surfaces under /_gm: a single shared Basic-Auth, then route.
 	@gm path /_gm /_gm/*
 	handle @gm {

deploy/caddy/maintenance.html

@@ -0,0 +1,44 @@
+<!doctype html>
+<!--
+  Served with 503 by the edge caddy while the deploy holds the maintenance flag
+  (/srv/maint/on, toggled by deploy/prod-deploy.sh around a rolling swap). Static and
+  self-contained (no upstream, no external assets) so it renders while the backend /
+  gateway are mid-recreate.
+-->
+<html lang="ru">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <meta name="robots" content="noindex">
+  <title>Эрудит — технические работы</title>
+  <style>
+    :root { color-scheme: light dark; }
+    html, body { height: 100%; margin: 0; }
+    body {
+      display: flex; align-items: center; justify-content: center;
+      font-family: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
+      background: #f4f1ea; color: #2b2b2b;
+      padding: 24px; box-sizing: border-box;
+    }
+    @media (prefers-color-scheme: dark) { body { background: #1d1b17; color: #e8e4da; } }
+    .card { max-width: 30rem; text-align: center; line-height: 1.5; }
+    .tile {
+      display: inline-block; width: 3.5rem; height: 3.5rem; line-height: 3.5rem;
+      font-size: 2rem; font-weight: 700; border-radius: 10px;
+      background: #d9b451; color: #2b2b2b; box-shadow: 0 2px 4px rgba(0,0,0,.25);
+      margin-bottom: 1.25rem;
+    }
+    h1 { font-size: 1.4rem; margin: 0 0 .5rem; }
+    p { margin: .35rem 0; }
+    .en { opacity: .7; font-size: .95rem; }
+  </style>
+</head>
+<body>
+  <main class="card">
+    <div class="tile">Э</div>
+    <h1>Технические работы</h1>
+    <p>Идёт короткое обновление игры. Мы скоро вернёмся — обновите страницу через пару минут.</p>
+    <p class="en">Scrabble is briefly down for an update. Please refresh in a couple of minutes.</p>
+  </main>
+</body>
+</html>

deploy/docker-compose.yml

@@ -428,6 +428,11 @@ services:
       GM_BASICAUTH_HASH: ${GM_BASICAUTH_HASH:?set GM_BASICAUTH_HASH}
     volumes:
       - ${SCRABBLE_CONFIG_DIR:-.}/caddy/Caddyfile:/etc/caddy/Caddyfile:ro
+      # Maintenance page + toggle flag: the caddy config dir holds maintenance.html and the
+      # `on` flag prod-deploy.sh touches around a rolling swap; the Caddyfile serves a 503
+      # from here while the flag exists (read-only mount — the deploy writes the flag on the
+      # host side). See deploy/caddy/Caddyfile and deploy/prod-deploy.sh.
+      - ${SCRABBLE_CONFIG_DIR:-.}/caddy:/srv/maint:ro
       - caddy-data:/data
     deploy:
       resources:

deploy/prod-deploy.sh

@@ -42,6 +42,15 @@ PREV_STATE_FILE="${PREV_STATE_FILE:-/opt/scrabble/PREVIOUS_TAG}"
 PG_USER="${POSTGRES_USER:-scrabble}"
 PG_DB="${POSTGRES_DB:-scrabble}"
 
+# Edge maintenance page. caddy serves a static 503 "works in progress" page for the
+# user-facing routes while this flag file exists (deploy/caddy/Caddyfile). We hold it
+# across the roll / migration window and clear it on ANY exit — success, a health
+# failure + rollback, or an unexpected error — via the trap, so users get a graceful
+# page instead of raw mid-swap 502s and the flag can never get stuck on.
+MAINT_FLAG="${MAINT_FLAG:-${SCRABBLE_CONFIG_DIR:-/opt/scrabble}/caddy/on}"
+maint_off() { rm -f "$MAINT_FLAG" 2>/dev/null || true; }
+trap maint_off EXIT
+
 cd "$COMPOSE_DIR" || { echo "compose dir $COMPOSE_DIR missing"; exit 1; }
 export REGISTRY
 # otelcol joins the host docker group to read the socket; the GID varies per host.
@@ -119,6 +128,13 @@ if [ -z "$(docker ps -aq -f name=scrabble-backend)" ]; then
   exit 0
 fi
 
+# Existing stack: raise the maintenance page for the whole roll (and any migration
+# window). It shows once caddy carries the gate (from this feature's own deploy
+# onward); the EXIT trap lowers it however this run ends.
+mkdir -p "$(dirname "$MAINT_FLAG")"
+: > "$MAINT_FLAG"
+echo "maintenance page raised ($MAINT_FLAG)"
+
 # Migration deploy: freeze writes and snapshot a consistent dump before migrating.
 if [ "$MIGRATION" = 1 ]; then
   echo "migration deploy: opening maintenance window (stopping the backend = the only writer)"

ui/e2e/maintenance.spec.ts

@@ -0,0 +1,38 @@
+import { expect, test } from './fixtures';
+
+// The maintenance overlay is raised in prod by the edge 503 marker (X-Scrabble-Maintenance);
+// the mock transport never produces one, so — like the offline indicator — the e2e drives it
+// through the window.__maint hook (gateway.ts, mock-only). It is app-global (mounted outside
+// the route blocks in App.svelte), so it shows without a session.
+test('maintenance overlay covers the app and lifts on recovery', async ({ page }) => {
+  await page.goto('/');
+  await expect(page.getByRole('alertdialog')).toHaveCount(0);
+
+  // A planned deploy window begins: a non-dismissable dimmed overlay covers the app.
+  await page.evaluate(() => (window as unknown as { __maint: { on(): void } }).__maint.on());
+  const overlay = page.getByRole('alertdialog');
+  await expect(overlay).toBeVisible();
+  // The retry button is present (EN "Try again" / RU "Повторить" depending on locale).
+  await expect(overlay.getByRole('button', { name: /again|Повторить/i })).toBeVisible();
+
+  // The window ends — in prod the store's poll lifts it on the first successful read; the mock
+  // has no probe, so it clears explicitly. The overlay disappears with no page reload.
+  await page.evaluate(() => (window as unknown as { __maint: { off(): void } }).__maint.off());
+  await expect(page.getByRole('alertdialog')).toHaveCount(0);
+});
+
+test('recovery reloads the SPA to pick up the fresh client', async ({ page }) => {
+  await page.goto('/');
+  await page.evaluate(() => (window as unknown as { __maint: { on(): void } }).__maint.on());
+  await expect(page.getByRole('alertdialog')).toBeVisible();
+
+  // Real recovery reloads the page (the deploy may have shipped an incompatible client, and the
+  // running bundle is the old one). Wait for the forced navigation, then confirm the app
+  // re-bootstrapped: the overlay is gone (the store reset by the reload) and the login is back.
+  await Promise.all([
+    page.waitForEvent('load'),
+    page.evaluate(() => (window as unknown as { __maint: { recover(): void } }).__maint.recover()),
+  ]);
+  await expect(page.getByRole('alertdialog')).toHaveCount(0);
+  await expect(page.getByRole('button', { name: /guest/i })).toBeVisible();
+});

ui/src/App.svelte

@@ -9,6 +9,7 @@
   import StaleInviteModal from './components/StaleInviteModal.svelte';
   import WelcomeRedeemModal from './components/WelcomeRedeemModal.svelte';
   import Coachmark from './components/Coachmark.svelte';
+  import MaintenanceOverlay from './components/MaintenanceOverlay.svelte';
   import DebugPanel from './components/DebugPanel.svelte';
   import Login from './screens/Login.svelte';
   import Lobby from './screens/Lobby.svelte';
@@ -128,6 +129,7 @@
 <StaleInviteModal />
 <WelcomeRedeemModal />
 <Coachmark />
+<MaintenanceOverlay />
 
 {#if routeIsLobby && !app.splashDone && !app.blocked && !app.bootError && !app.launchError}
   <Splash />

ui/src/components/MaintenanceOverlay.svelte

@@ -0,0 +1,83 @@
+<script lang="ts">
+  // Non-dismissable maintenance cover. Shown while the edge is in a planned deploy window
+  // (maintenance.svelte.ts, raised from a caddy 503 carrying X-Scrabble-Maintenance). It has
+  // no close affordance — an in-session user cannot dismiss it — but the store's poll lifts
+  // it automatically the moment the gateway answers again; the "retry" button just forces an
+  // immediate re-check. Mirrors the static caddy maintenance page (deploy/caddy/maintenance.html)
+  // so a fresh load and an in-session user see the same thing.
+  import { maintenance, retryNow } from '../lib/maintenance.svelte';
+  import { t } from '../lib/i18n/index.svelte';
+</script>
+
+{#if maintenance.active}
+  <div class="scrim" role="alertdialog" aria-modal="true" aria-labelledby="maint-title" aria-describedby="maint-body">
+    <div class="card">
+      <div class="tile" aria-hidden="true">Э</div>
+      <h1 id="maint-title">{t('maintenance.title')}</h1>
+      <p id="maint-body">{t('maintenance.body')}</p>
+      <button type="button" onclick={retryNow}>{t('maintenance.retry')}</button>
+    </div>
+  </div>
+{/if}
+
+<style>
+  .scrim {
+    position: fixed;
+    inset: 0;
+    /* Above the game (z 60) and toasts (z 50) — a planned window covers everything the user
+       could otherwise interact with; below the dev DebugPanel (z 10000). */
+    z-index: 100;
+    background: rgba(0, 0, 0, 0.55);
+    display: grid;
+    place-items: center;
+    padding: 24px;
+    box-sizing: border-box;
+  }
+  .card {
+    max-width: 22rem;
+    text-align: center;
+    background: var(--surface);
+    color: var(--text);
+    border-radius: var(--radius);
+    box-shadow: var(--shadow);
+    padding: 2rem 1.5rem;
+  }
+  /* Mirrors a placed board tile (as Splash.svelte / the caddy page do). */
+  .tile {
+    display: inline-grid;
+    place-items: center;
+    width: 3.5rem;
+    height: 3.5rem;
+    font-size: 2rem;
+    font-weight: 700;
+    border-radius: 4px;
+    background: var(--tile-bg);
+    color: var(--tile-text);
+    box-shadow:
+      inset 0 -2px 0 var(--tile-edge),
+      2px 0 3px -1px rgba(0, 0, 0, 0.4);
+    margin-bottom: 1.25rem;
+  }
+  h1 {
+    font-size: 1.25rem;
+    margin: 0 0 0.5rem;
+  }
+  p {
+    margin: 0 0 1.5rem;
+    color: var(--text-muted);
+    line-height: 1.5;
+  }
+  button {
+    font: inherit;
+    padding: 0.55rem 1.4rem;
+    border: 1px solid var(--border);
+    border-radius: var(--radius);
+    background: transparent;
+    color: var(--text);
+    cursor: pointer;
+  }
+  button:hover {
+    border-color: var(--accent);
+    color: var(--accent);
+  }
+</style>

ui/src/lib/gateway.ts

@@ -6,6 +6,7 @@ import type { GatewayClient } from './client';
 import { MockGateway } from './mock/client';
 import { createTransport } from './transport';
 import { reportOffline, reportOnline } from './connection.svelte';
+import { clearMaintenance, maintenanceRecovered, reportMaintenance } from './maintenance.svelte';
 
 const isMock = import.meta.env.MODE === 'mock';
 
@@ -20,6 +21,15 @@ if (isMock && typeof window !== 'undefined') {
     offline: reportOffline,
     online: reportOnline,
   };
+  // The mock never produces a real 503, so the e2e drives the maintenance overlay directly
+  // (the store's poll is inert in mock — no probe is registered — so `off` clears it).
+  (
+    window as unknown as { __maint?: { on(retryAfterMs?: number): void; off(): void; recover(): void } }
+  ).__maint = {
+    on: (retryAfterMs = 15000) => reportMaintenance(retryAfterMs),
+    off: clearMaintenance,
+    recover: maintenanceRecovered,
+  };
   // Drive the auto-match opponent join deterministically from the e2e (the mock otherwise
   // attaches a robot on a timer).
   (

ui/src/lib/i18n/en.ts

@@ -7,6 +7,9 @@ export const en = {
   'app.brand': 'Erudit',
   'dict.loading': 'Loading…',
   'connection.connecting': 'Connecting…',
+  'maintenance.title': 'Under maintenance',
+  'maintenance.body': 'Scrabble is briefly down for an update. It will be back in a moment — no need to reload the page.',
+  'maintenance.retry': 'Try again',
 
   'blocked.title': 'Account blocked',
   'blocked.permanent': 'Your account is blocked.',

ui/src/lib/i18n/ru.ts

@@ -8,6 +8,9 @@ export const ru: Record<MessageKey, string> = {
   'app.brand': 'Эрудит',
   'dict.loading': 'Загрузка…',
   'connection.connecting': 'Подключение…',
+  'maintenance.title': 'Технические работы',
+  'maintenance.body': 'Идёт короткое обновление игры. Мы скоро вернёмся — страницу перезагружать не нужно.',
+  'maintenance.retry': 'Повторить',
 
   'blocked.title': 'Учётная запись заблокирована',
   'blocked.permanent': 'Ваша учётная запись заблокирована.',

ui/src/lib/maintenance.svelte.ts

@@ -0,0 +1,99 @@
+// Global maintenance signal + self-clearing poll. `active` is true while the edge is in a
+// planned deploy window (a caddy 503 carried `X-Scrabble-Maintenance`; see maintenance.ts).
+// A non-dismissable overlay (MaintenanceOverlay.svelte) covers the app while active, and a
+// cheap read is retried on a capped-backoff cadence (mirroring connection.svelte.ts) — the
+// first success lifts the overlay, so it can never get stuck even with no other traffic.
+// Mock mode never reports maintenance, so it simply stays inactive.
+
+import { backoffMs } from './retry';
+
+let active = $state(false);
+let retryHintMs = $state(0);
+let pollTimer: ReturnType<typeof setTimeout> | null = null;
+let attempt = 0;
+let probing = false;
+let probe: (() => Promise<void>) | null = null;
+
+export const maintenance = {
+  /** active is true while the edge is in a planned maintenance window. */
+  get active(): boolean {
+    return active;
+  },
+  /** retryHintMs is the last Retry-After hint in milliseconds (0 when unknown). */
+  get retryHintMs(): number {
+    return retryHintMs;
+  },
+};
+
+/** registerMaintenanceProbe installs the reachability read the poll fires while active
+ *  (the transport wires a cheap authenticated read). */
+export function registerMaintenanceProbe(fn: () => Promise<void>): void {
+  probe = fn;
+}
+
+/** reportMaintenance raises the overlay and starts the self-clearing poll. Idempotent: a
+ *  repeat hit only refreshes the hint, it never restarts a running poll. */
+export function reportMaintenance(retryAfterMs: number): void {
+  retryHintMs = retryAfterMs;
+  if (active) return;
+  active = true;
+  attempt = 0;
+  if (probe) schedulePoll();
+}
+
+/** clearMaintenance lowers the overlay and stops the poll without reloading (a reset, or the
+ *  e2e hook). Prod recovery goes through maintenanceRecovered instead. */
+export function clearMaintenance(): void {
+  active = false;
+  if (pollTimer) {
+    clearTimeout(pollTimer);
+    pollTimer = null;
+  }
+}
+
+/** maintenanceRecovered runs when the edge answers again after a window we were showing.
+ *  The deploy that ended the window may have shipped a client-incompatible change (a wire /
+ *  schema bump), and the running bundle is the OLD one — so reload to pick up the fresh
+ *  client instead of merely hiding the overlay. A no-op unless the overlay was up; the cover
+ *  stays over the brief reload flash. In prod recovery is the only way the overlay clears. */
+export function maintenanceRecovered(): void {
+  if (!active) return;
+  active = false;
+  if (pollTimer) {
+    clearTimeout(pollTimer);
+    pollTimer = null;
+  }
+  if (typeof location !== 'undefined') location.reload();
+}
+
+/** retryNow forces an immediate re-check (the overlay's button). It never closes the
+ *  overlay itself — only a successful probe does. A no-op while a probe is in flight. */
+export function retryNow(): void {
+  if (!active || !probe || probing) return;
+  if (pollTimer) {
+    clearTimeout(pollTimer);
+    pollTimer = null;
+  }
+  attempt = 0;
+  runProbe();
+}
+
+function schedulePoll(): void {
+  pollTimer = setTimeout(runProbe, backoffMs(++attempt));
+}
+
+function runProbe(): void {
+  pollTimer = null;
+  if (!active || !probe || probing) return;
+  probing = true;
+  probe().then(
+    () => {
+      probing = false;
+      maintenanceRecovered();
+    },
+    () => {
+      probing = false;
+      if (active) schedulePoll();
+    },
+  );
+}

ui/src/lib/maintenance.test.ts

@@ -0,0 +1,44 @@
+import { describe, expect, it } from 'vitest';
+import { Code, ConnectError } from '@connectrpc/connect';
+import { maintenanceRetryMs, parseRetryAfterMs } from './maintenance';
+
+// A raw ConnectError as connect-web builds it from a non-Connect HTTP 503: the response
+// headers are preserved as `.metadata` (verified against @connectrpc/connect-web).
+const err503 = (headers: Record<string, string>): ConnectError =>
+  new ConnectError('HTTP 503', Code.Unavailable, new Headers(headers));
+
+describe('maintenanceRetryMs', () => {
+  it('detects the marker and parses Retry-After', () => {
+    expect(maintenanceRetryMs(err503({ 'x-scrabble-maintenance': '1', 'retry-after': '120' }))).toBe(120_000);
+  });
+
+  it('matches the header case-insensitively, defaulting the delay when Retry-After is absent', () => {
+    expect(maintenanceRetryMs(err503({ 'X-Scrabble-Maintenance': '1' }))).toBe(15_000);
+  });
+
+  it('returns null for a 503 without the marker (a plain transient failure)', () => {
+    expect(maintenanceRetryMs(err503({ 'retry-after': '30' }))).toBeNull();
+  });
+
+  it('returns null for non-Connect errors', () => {
+    expect(maintenanceRetryMs(new Error('boom'))).toBeNull();
+    expect(maintenanceRetryMs(undefined)).toBeNull();
+    expect(maintenanceRetryMs('nope')).toBeNull();
+  });
+});
+
+describe('parseRetryAfterMs', () => {
+  it('clamps into the 3s–120s band', () => {
+    expect(parseRetryAfterMs('120')).toBe(120_000);
+    expect(parseRetryAfterMs('1')).toBe(3_000); // floor
+    expect(parseRetryAfterMs('9999')).toBe(120_000); // ceiling
+  });
+
+  it('falls back to the default on absent / garbage / non-positive', () => {
+    expect(parseRetryAfterMs(null)).toBe(15_000);
+    expect(parseRetryAfterMs(undefined)).toBe(15_000);
+    expect(parseRetryAfterMs('soon')).toBe(15_000);
+    expect(parseRetryAfterMs('0')).toBe(15_000);
+    expect(parseRetryAfterMs('-5')).toBe(15_000);
+  });
+});

ui/src/lib/maintenance.ts

@@ -0,0 +1,37 @@
+// Detection of a planned edge maintenance window. During a prod deploy roll the edge
+// caddy returns HTTP 503 with an `X-Scrabble-Maintenance: 1` header (and a `Retry-After`)
+// for every gated route. The SPA keys STRICTLY on that marker — not on any transport
+// `unavailable`, which the "Connecting…" indicator already covers — so a planned window
+// raises the maintenance overlay while an ordinary blip stays a soft reconnect.
+//
+// These helpers are pure (no DOM, no runes) so they unit-test in the node vitest env. The
+// maintenance marker + Retry-After ride on the raw ConnectError's `metadata`, which
+// toGatewayError (retry.ts) discards — so detection must run on the raw error.
+
+import { ConnectError } from '@connectrpc/connect';
+
+const DEFAULT_RETRY_MS = 15_000;
+const MIN_RETRY_MS = 3_000;
+const MAX_RETRY_MS = 120_000;
+
+/**
+ * parseRetryAfterMs converts a `Retry-After` header (delta-seconds — the edge emits
+ * seconds, never an HTTP-date) into a millisecond hint clamped to a sane band. An absent
+ * or unparseable value falls back to a default.
+ */
+export function parseRetryAfterMs(header: string | null | undefined): number {
+  const secs = header == null ? NaN : Number(header);
+  if (!Number.isFinite(secs) || secs <= 0) return DEFAULT_RETRY_MS;
+  return Math.min(MAX_RETRY_MS, Math.max(MIN_RETRY_MS, Math.round(secs * 1000)));
+}
+
+/**
+ * maintenanceRetryMs returns the maintenance `Retry-After` hint in milliseconds when the
+ * thrown transport error is the edge maintenance 503 (carries `X-Scrabble-Maintenance: 1`),
+ * or null for any other error. Header lookup is case-insensitive (Headers.get).
+ */
+export function maintenanceRetryMs(e: unknown): number | null {
+  if (!(e instanceof ConnectError)) return null;
+  if (e.metadata?.get('x-scrabble-maintenance') !== '1') return null;
+  return parseRetryAfterMs(e.metadata.get('retry-after'));
+}

ui/src/lib/transport.ts

@@ -12,6 +12,8 @@ import { GatewayError, type GatewayClient } from './client';
 import * as codec from './codec';
 import { browserOffset } from './profileValidation';
 import { registerProbe, reportOffline, reportOnline } from './connection.svelte';
+import { maintenanceRecovered, registerMaintenanceProbe, reportMaintenance } from './maintenance.svelte';
+import { maintenanceRetryMs } from './maintenance';
 import { backoffMs, isConnectionCode, retryable, toGatewayError } from './retry';
 
 const MAX_RETRIES = 6;
@@ -28,10 +30,14 @@ export function createTransport(baseUrl: string): GatewayClient {
 
   // The reachability probe the connection watcher fires while offline: a cheap authenticated read
   // (it must reject when there is no session, so the watcher keeps waiting rather than reporting up).
-  registerProbe(async () => {
+  const reachabilityProbe = async (): Promise<void> => {
     if (!token) throw new Error('no session');
     await client.execute({ messageType: 'profile.get', payload: codec.empty(), requestId: '' }, { headers: headers() });
-  });
+  };
+  registerProbe(reachabilityProbe);
+  // The maintenance overlay reuses the same cheap read to poll for the end of a deploy
+  // window; the first success lifts the overlay (maintenance.svelte.ts).
+  registerMaintenanceProbe(reachabilityProbe);
 
   // exec runs one unary op, auto-retrying transient transport failures with capped backoff (so a
   // dropped connection or a rate-limit recovers seamlessly) and driving the global Connecting
@@ -43,6 +49,14 @@ export function createTransport(baseUrl: string): GatewayClient {
         res = await client.execute({ messageType, payload, requestId: '' }, { headers: headers(), signal });
       } catch (e) {
         if (signal?.aborted) throw e; // an intentional cancel (e.g. the tiles moved) — do not retry
+        const maintMs = maintenanceRetryMs(e);
+        if (maintMs !== null) {
+          // A planned deploy window (the edge 503 carried X-Scrabble-Maintenance): raise the
+          // overlay and fail fast — its own poll drives recovery — instead of burning the
+          // retry budget on every read for the length of the window.
+          reportMaintenance(maintMs);
+          throw toGatewayError(e);
+        }
         const err = toGatewayError(e);
         if (retryable(err.code, messageType) && attempt < MAX_RETRIES) {
           reportOffline();
@@ -53,6 +67,9 @@ export function createTransport(baseUrl: string): GatewayClient {
         throw err;
       }
       reportOnline();
+      // A read got through: if the maintenance overlay was up, the deploy window has ended —
+      // reload to pick up the (possibly incompatible) fresh client (maintenance.svelte.ts).
+      maintenanceRecovered();
       if (res.resultCode && res.resultCode !== 'ok') throw new GatewayError(res.resultCode);
       return res.payload;
     }
@@ -310,7 +327,11 @@ export function createTransport(baseUrl: string): GatewayClient {
             if (pe) onEvent(pe);
           }
         } catch (e) {
-          if (!ctrl.signal.aborted) onError?.(toGatewayError(e));
+          if (!ctrl.signal.aborted) {
+            const maintMs = maintenanceRetryMs(e);
+            if (maintMs !== null) reportMaintenance(maintMs);
+            onError?.(toGatewayError(e));
+          }
         }
       })();
       return () => ctrl.abort();