Merge pull request 'release v1.2.2: edge HTTP/3 stall fix + db-size dashboard threshold' (#121) from development into master

The "Database size" stat had no thresholds, so Grafana applied its stat
default (green base, red at >=80). The query is pg_database_size_bytes, so a
healthy ~9 MiB database (9.4M >> 80) rendered permanently RED on the
Scrabble - Resources dashboard (test + prod), reading as a false alert; the
neighbouring percentunit cache-hit stat stayed green only because its 0..1
values fall under 80.

Add absolute byte thresholds sized to the 40 GiB prod disk (4.6 GiB used,
observability bounded -- Tempo <=1 GiB, Prometheus 7d -- so the DB is the
only unbounded grower): green up to 8 GiB, yellow at 8 GiB (~20% of disk),
red at 16 GiB (~40%), an early warning with ample runway before the disk
tightens, not a panic line. Cosmetic panel coloring only; there are no
Grafana alert rules provisioned.

deploy/caddy/Caddyfile

@@ -21,6 +21,18 @@
 }
 
 {$CADDY_SITE_ADDRESS::80} {
+	# HTTP/3 is advertised by default whenever this caddy terminates TLS (prod:
+	# CADDY_SITE_ADDRESS is the domain). But UDP/443 is never reachable — the prod
+	# compose maps only "443:443" (TCP) and ufw opens 443/tcp — so a client that cached
+	# the `Alt-Svc: h3` advert (sticky for ma=2592000s) stalls on the dead QUIC path
+	# before falling back to h2, which surfaced as the Telegram Mini App intermittently
+	# hanging on load. `Alt-Svc: clear` actively drops any cached alternative and pins
+	# clients to h2/h1; it is applied site-wide so every route is covered. In the test
+	# contour this caddy serves plain :80 (no h3 to advertise) and the host caddy
+	# re-stamps its own Alt-Svc, so the live test fix lives in the host caddy — here it
+	# is the prod fix. Background + alternatives (incl. serving h3 for real): docs/EDGE_HTTP3.md.
+	header Alt-Svc clear
+
 	# Operator surfaces under /_gm: a single shared Basic-Auth, then route.
 	@gm path /_gm /_gm/*
 	handle @gm {

deploy/grafana/dashboards/resources.json

@@ -36,7 +36,21 @@
       "type": "stat",
       "title": "Database size",
       "gridPos": { "h": 5, "w": 6, "x": 18, "y": 0 },
-      "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 8589934592 },
+              { "color": "red", "value": 17179869184 }
+            ]
+          }
+        },
+        "overrides": []
+      },
       "datasource": { "type": "prometheus", "uid": "prometheus" },
       "targets": [{ "refId": "A", "expr": "max(pg_database_size_bytes{datname=\"scrabble\"})" }]
     },

docs/ARCHITECTURE.md

@@ -158,7 +158,12 @@ arrive from a platform rather than completing a mandatory registration).
   rendered in the recipient's **interface language** (`preferred_language`, en/ru), not in
   any bot-scoped language, and the friend-invite **share link** (and its caption) point at
   that one bot. First Telegram contact seeds the new account's `preferred_language` from the
-  launch `language_code` (§4); the interface language is otherwise edited in Settings.
+  launch `language_code` (§4), but the **interface language follows the device** — the system
+  guess, or an explicit Settings choice saved locally — and the bot never dictates the UI.
+  `preferred_language` is then **reconciled to the active interface locale on every session
+  adopt** (not only on a Settings change; a no-op for guests and when already equal), so the
+  server-rendered language surfaces — this push and the ad banner — always match the UI rather
+  than stranding a user who never opened Settings on the creation-time seed.
 - **Variant preferences (New Game gating).** Which variants a player may be matched into is a
   per-user **profile** setting — `variant_preferences`, a set of `engine.Variant` labels
   (`scrabble_en`, `scrabble_ru`, `erudit_ru`) edited on the Settings/Profile screen. New
@@ -640,7 +645,7 @@ in either direction (the enqueue excludes the caller's `BlockedWith` set);
   **floats games with any unread entry to the top** of the your-turn and opponent-turn
   sections (the finished section keeps its activity order). On each clear the publish-to-read
   latency is recorded; the read time itself is not retained.
-- **Profile**: `preferred_language` (en/ru, edited in Settings), display name, email
+- **Profile**: `preferred_language` (en/ru; tracks the interface language — §4), display name, email
   (confirm-code binding, see §4), **timezone**, the daily **away window**, the
   **variant preferences** (`variant_preferences`, the matchable-variant set that gates New
   Game — §3, defaulting to Erudit only, at least one enforced) and the
@@ -1093,7 +1098,10 @@ Two contours, two secret/variable prefixes (`TEST_` / `PROD_`):
   the **main host** runs the full stack (`docker-compose.yml` + `docker-compose.prod.yml`),
   the **bot host** runs only the bot (`docker-compose.bot.yml`, no VPN — native Bot API
   egress, telemetry off). There is no host caddy, so the contour caddy terminates TLS —
-  `CADDY_SITE_ADDRESS` is the domain and caddy does its own ACME. The gateway **publishes**
+  `CADDY_SITE_ADDRESS` is the domain and caddy does its own ACME. Caddy advertises HTTP/3 by default, but UDP/443 is not exposed (the
+  compose maps only TCP and ufw opens 443/tcp), so the edge emits `Alt-Svc: clear` to keep
+  clients on h2/h1 rather than stall on a dead QUIC path — see [`EDGE_HTTP3.md`](EDGE_HTTP3.md).
+  The gateway **publishes**
   the bot-link `:9443`; the remote bot dials it over mTLS (certs from `PROD_BOTLINK_*`,
   ServerName `gateway`, so TLS validation is independent of the public dial address), holds
   no inbound port, and login is unaffected if that host or the link is down.

docs/EDGE_HTTP3.md

@@ -0,0 +1,111 @@
+# Edge HTTP/3 (`Alt-Svc`) policy
+
+## TL;DR
+
+The edge **advertises HTTP/3 but does not actually serve it** (UDP/443 is not exposed),
+so we suppress the advert with `Alt-Svc: clear`. Advertising QUIC on `:443/udp` while
+that port is unreachable makes clients — notably the Telegram Mini App webview — stall
+on a dead QUIC connection before falling back to h2, which shows up as the app "hanging
+on load".
+
+## Symptom
+
+Opening the Mini App intermittently hangs on load: from a barely-noticeable pause to
+several seconds, sometimes a blank window that never finishes downloading `index.html`.
+Intermittent, worse after the first successful visit, reproduced on both the test
+contour and prod.
+
+## Root cause
+
+Caddy enables HTTP/3 by default on any TLS listener and emits
+`Alt-Svc: h3=":443"; ma=2592000` — telling every client "reach me over QUIC/UDP 443"
+and to cache that for 30 days. But UDP/443 is **never reachable end to end**:
+
+- **Test contour**: the host caddy publishes only `:443/tcp` (`docker port caddy` shows
+  no `udp`); QUIC packets from the internet are dropped.
+- **Prod**: `deploy/docker-compose.prod.yml` maps `"443:443"` (Docker = **TCP only**)
+  and `deploy/ansible/roles/main/tasks/main.yml` opens 443 `proto: tcp`. UDP/443 is
+  dropped at both the publish and the firewall.
+
+Caddy *does* bind `udp/443` inside the container and h3 works container-to-container
+(verified `http=3 code=200`), so the listener is healthy — it is simply not exposed.
+
+A client that cached the advert tries QUIC first on later opens, gets no response, and
+waits for the QUIC attempt to time out before falling back to TCP/h2. That wait is the
+stall. The very first visit (no cached `Alt-Svc`) uses h2 and is fast.
+
+The h2/TCP serving path itself is healthy: 30 fresh-TLS requests through the full path
+(host caddy -> contour caddy -> gateway) measured TTFB ~9.5 ms, total ~9.8 ms, no tail;
+`index.html` is ~1 KB.
+
+## Fix in place (option A — suppress the advert)
+
+Emit `Alt-Svc: clear`, which actively drops any cached alternative (better than merely
+deleting the header, which leaves the sticky 30-day cache in place):
+
+- **Prod / repo**: `deploy/caddy/Caddyfile` — a site-level `header Alt-Svc clear` (this
+  caddy terminates TLS in prod).
+- **Test contour**: the host caddy terminates TLS, so the fix lives there (homelab
+  config, outside this repo): `header Alt-Svc clear` on the `scrabble.*` site. The
+  in-compose caddy serves plain `:80` in test and never advertises h3, so the repo
+  directive is a harmless no-op there (the host caddy re-stamps the header).
+
+`header Alt-Svc clear` overrides Caddy's auto-advert (verified) and is site-scoped.
+
+### Verify
+
+The runner/prod host shell cannot reach the Docker bridge IPs directly, so probe from a
+container on the relevant network, using `--resolve` to hit the TLS-terminating caddy by
+its bridge IP (this also bypasses the public-IP NAT hairpin):
+
+```sh
+# <edge-ip> = the TLS-terminating caddy's IP on its network (docker inspect ... )
+docker run --rm --network edge curlimages/curl:latest -sS -D - -o /dev/null \
+  --resolve <host>:443:<edge-ip> https://<host>/telegram/ | grep -iE '^HTTP|^alt-svc'
+# expect: HTTP/2 200, and NO `alt-svc: h3=...` (the header is absent or `alt-svc: clear`)
+```
+
+## If it recurs — alternatives to try
+
+So we do not re-derive the diagnosis from scratch:
+
+1. **Re-confirm the advert is actually suppressed** with the verify command above. A
+   redeploy or a Caddy upgrade could regress it, or a client may still hold a cached
+   `h3` entry that has not yet been replaced by a `clear` (it needs one successful h2
+   response to receive the `clear`).
+2. **Option B — serve HTTP/3 for real** instead of suppressing it. Worth it only if we
+   actually want QUIC (the benefit is marginal for a ~1 KB shell plus hash-immutable
+   cached assets, and it adds UDP/QUIC attack surface):
+   - Publish UDP: add `"443:443/udp"` next to the TCP map in
+     `deploy/docker-compose.prod.yml` (and publish udp/443 on the test host caddy too).
+   - Open the firewall: add a `443 proto: udp` rule in
+     `deploy/ansible/roles/main/tasks/main.yml`.
+   - Drop the `header Alt-Svc clear` so Caddy advertises h3 again.
+   - Verify with an h3 client from inside the network:
+     `docker run --rm --network edge ymuski/curl-http3 curl --http3-only ...` should
+     return `http=3 code=200`.
+3. **Look past the edge** if the advert is suppressed and stalls persist. The h2 path is
+   fast server-side, so a remaining stall is most likely the client network / RTT / the
+   provider, not our stack. Re-run the timing loop (below) to confirm the server is
+   still <~10 ms TTFB before chasing the client side.
+
+## How this was diagnosed (method, to repeat)
+
+- The runner/prod host shell cannot reach the Docker bridge subnets, so all probing runs
+  from a throwaway container on the target network (`docker run --network <net>
+  curlimages/curl`), using `--resolve <host>:443:<edge-ip>` to bypass the public-IP NAT
+  hairpin and exercise the real TLS path.
+- Compare a fresh-connection timing loop (worst case, full TLS each time) against a
+  keepalive batch to separate handshake cost from serving cost:
+
+  ```sh
+  docker run --rm --network edge curlimages/curl:latest sh -c '
+    for i in $(seq 1 30); do
+      curl -sS -o /dev/null --resolve <host>:443:<edge-ip> \
+        -w "http=%{http_version} code=%{http_code} tls=%{time_appconnect} ttfb=%{time_starttransfer} total=%{time_total}\n" \
+        https://<host>/telegram/
+    done'
+  ```
+
+- `docker port <caddy>` shows whether `udp/443` is actually published; the response
+  `Alt-Svc` header shows what the edge advertises. The two disagreeing is the bug.

ui/src/lib/app.svelte.ts

@@ -68,7 +68,6 @@ export const app = $state<{
   locale: Locale;
   reduceMotion: boolean;
   boardLabels: BoardLabelMode;
-  localeLocked: boolean;
   /** Pending incoming friend requests, for the lobby ⚙️ badge and the Settings Friends tab. */
   notifications: number;
   /** Per-game flag: the player has at least one unread chat entry (message or nudge) in that
@@ -109,7 +108,6 @@ export const app = $state<{
   locale: 'en',
   reduceMotion: false,
   boardLabels: 'beginner',
-  localeLocked: false,
   notifications: 0,
   chatUnread: {},
   messageUnread: {},
@@ -451,18 +449,20 @@ async function adoptSession(s: Session): Promise<void> {
   await saveSession(s);
   try {
     app.profile = await gateway.profileGet();
-    // The live interface language follows the device — the explicit local choice (locked, saved
-    // in prefs) or the system guess made at bootstrap — and is no longer overridden from the
-    // account here. preferred_language stays the user's saved choice (written from Settings,
-    // and used for out-of-app push routing), but the Telegram bot a user signs in through must
-    // not dictate the UI: a ru-bot launch on an English system stays English.
+    // The live interface language follows the device — the explicit local choice (saved in
+    // prefs) or the system guess made at bootstrap — and is no longer overridden from the
+    // account here: the Telegram bot a user signs in through must not dictate the UI, so a
+    // ru-bot launch on an English system stays English.
     //
-    // But the banner and out-of-app push routing ARE resolved from preferred_language, so an
-    // explicit device choice the account has not recorded yet (picked while a guest, or
-    // differing from the Telegram system-language seed) would otherwise leave them in the wrong
-    // language until the next Settings change. Reconcile the account to the saved local choice
-    // here; persistLanguageToServer no-ops for guests and when already equal.
-    if (app.localeLocked) void persistLanguageToServer(app.locale);
+    // The banner and out-of-app push are resolved server-side from preferred_language, so it
+    // must track whatever language the UI actually shows — the explicit choice AND the system
+    // guess. Reconcile it to the active locale on every adopt, not only after an explicit
+    // Settings choice: a user who never opened Settings would otherwise be stuck on the
+    // creation-time seed — e.g. an English banner under a Russian UI. This keeps every
+    // server-rendered, language-dependent surface (banner, out-of-app push) aligned with the
+    // interface, not just one. persistLanguageToServer self-gates (a no-op for guests and when
+    // already equal), so there is no write in the steady state.
+    void persistLanguageToServer(app.locale);
   } catch (err) {
     handleError(err);
   }
@@ -483,9 +483,10 @@ export async function applyLinkResult(r: LinkResult): Promise<void> {
     return;
   }
   app.profile = await gateway.profileGet();
-  // A guest who chose a language and then linked in place now has a durable account: push the
-  // saved choice so the banner + push routing follow it (see adoptSession).
-  if (app.localeLocked) void persistLanguageToServer(app.locale);
+  // A guest who linked in place now has a durable account: push the active interface language
+  // so the banner + push routing follow it (see adoptSession — reconciled regardless of an
+  // explicit Settings choice).
+  void persistLanguageToServer(app.locale);
 }
 
 /**
@@ -538,7 +539,6 @@ export async function bootstrap(): Promise<void> {
   applyReduceMotion(app.reduceMotion);
   if (prefs.locale) {
     app.locale = prefs.locale;
-    app.localeLocked = true;
     setLocale(prefs.locale);
   } else {
     const guess = localeFrom(typeof navigator !== 'undefined' ? navigator.language : 'en');
@@ -754,7 +754,6 @@ export function setTheme(theme: ThemePref): void {
 
 export function setLocalePref(locale: Locale): void {
   app.locale = locale;
-  app.localeLocked = true;
   setLocale(locale);
   persistPrefs();
   void persistLanguageToServer(locale);