feat(bot): report Telegram bot Bot API health to the gateway over the bot-link

The bot runs on its own host and exports no telemetry (otelcol is unreachable
from there), so it was a monitoring blind spot. Observe its Bot API health
centrally by wrapping the HTTP client — one place, no per-call-site
instrumentation — and relay it up the existing bot-link as a periodic Health
message the gateway turns into its own metrics.

- proto: add Health (delta connect / api / 429 counters + a last-ok stamp) to
  the FromBot oneof (additive, backward-compatible).
- bot: platform/telegram/internal/health wraps the Bot API HTTP client — it
  stamps liveness on any 2xx (so the getUpdates long-poll keeps it fresh even
  when idle), classifies transport/5xx failures (getUpdates vs other) and 429s,
  and honours a 429's Retry-After (bounded) so the bot backs off; a 4xx other
  than 429 is a normal per-request outcome and is not counted.
- bot-link client: flush the reporter as a Health message every 30s over a
  single-sender loop (a gRPC stream forbids concurrent Send).
- gateway: fold each report into bot_tg_errors_total{kind} and the
  bot_tg_last_ok_unix liveness gauge.
- grafana: alerts (bot disconnected, bot not reaching the Bot API, sustained
  429s) routed to the operator email — which does not go through the bot — plus
  a Telegram-bot dashboard.
- docs (ARCHITECTURE, compose comments); unit tests (observer classification,
  Retry-After, snapshot/commit, hub last-ok monotonicity).

deploy/docker-compose.bot.yml

@@ -3,8 +3,10 @@
 #   docker compose -f docker-compose.bot.yml up -d
 #
 # The bot egresses to the Bot API directly (no VPN sidecar) and dials the main host's
-# published bot-link :9443 over mTLS. It exports no telemetry — otelcol lives on the
-# main host and is unreachable from here — so observe it via `docker logs` on this host.
+# published bot-link :9443 over mTLS. It exports no OTLP telemetry — otelcol lives on the
+# main host and is unreachable from here — but it reports its Bot API health up the bot-link,
+# which the gateway turns into metrics + alerts on the main host (docs/ARCHITECTURE.md); `docker
+# logs` on this host is the local detail view.
 # Values come from the prod-deploy workflow (PROD_ secrets/variables); BOT_IMAGE is the
 # pushed registry tag and BOTLINK_GATEWAY_ADDR is the main host's <ip>:9443.
 name: scrabble-bot
@@ -50,7 +52,8 @@ services:
       TELEGRAM_BOTLINK_TLS_CA: /certs/ca.crt
       TELEGRAM_LOG_LEVEL: ${LOG_LEVEL:-info}
       TELEGRAM_SERVICE_NAME: scrabble-telegram-bot
-      # No telemetry export: otelcol is on the main host, unreachable from here.
+      # No OTLP export: otelcol is on the main host, unreachable from here. Bot API health rides the
+      # bot-link instead (the gateway exposes it as metrics); see docs/ARCHITECTURE.md.
       TELEGRAM_OTEL_TRACES_EXPORTER: none
       TELEGRAM_OTEL_METRICS_EXPORTER: none
       GOMAXPROCS: "1"

deploy/grafana/dashboards/bot.json

@@ -0,0 +1,45 @@
+{
+  "uid": "scrabble-bot",
+  "title": "Scrabble — Telegram bot",
+  "tags": ["scrabble"],
+  "timezone": "",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "panels": [
+    {
+      "type": "stat",
+      "title": "Bot connected",
+      "description": "botlink_connected_bots: bots currently holding the gateway bot-link (1 = healthy).",
+      "gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [{ "refId": "A", "expr": "max(botlink_connected_bots)" }]
+    },
+    {
+      "type": "stat",
+      "title": "Last Bot API OK (age)",
+      "description": "Seconds since the bot's most recent successful Bot API call. Grows unbounded if the bot wedges.",
+      "gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 },
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [{ "refId": "A", "expr": "time() - max(bot_tg_last_ok_unix)" }]
+    },
+    {
+      "type": "timeseries",
+      "title": "Bot API errors/s by kind",
+      "description": "bot_tg_errors_total by kind: connect (getUpdates), api (other sends), rate_limited (429). 429 should stay ~0.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [{ "refId": "A", "expr": "sum by (kind) (rate(bot_tg_errors_total[5m]))", "legendFormat": "{{kind}}" }]
+    },
+    {
+      "type": "timeseries",
+      "title": "Bot-link commands/s by result",
+      "description": "botlink_commands_total by result: delivered / not_delivered / dropped / error. Sustained not_delivered or error means gateway sends are failing to reach Telegram.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [{ "refId": "A", "expr": "sum by (result) (rate(botlink_commands_total[5m]))", "legendFormat": "{{result}}" }]
+    }
+  ]
+}

deploy/grafana/provisioning/alerting/rules.yaml

@@ -277,3 +277,91 @@ groups:
                 - evaluator: { type: gt, params: [1800] }
         labels: { severity: critical }
         annotations: { summary: 'No WAL archived for over 30 minutes while pg_wal keeps growing — archiving is genuinely stalled; the PITR recovery point is frozen and pg_wal will fill the disk. Run `docker exec scrabble-postgres pgbackrest --stanza=scrabble --pg1-user=scrabble check`.' }
+
+  # Remote Telegram bot health. The bot runs on its own host and exports no telemetry of its own; the
+  # gateway observes it through the bot-link (botlink_connected_bots) and the health it reports over
+  # that stream (bot_tg_*). All absent/NaN-safe: before a bot ever connects the metrics are absent, so
+  # noDataState OK keeps them quiet on a fresh contour. The alert email does NOT go through the bot, so
+  # a bot-down alert is deliverable.
+  - orgId: 1
+    name: scrabble-bot
+    folder: Alerts
+    interval: 1m
+    rules:
+      - uid: bot_disconnected
+        title: Telegram bot disconnected
+        condition: C
+        for: 5m
+        noDataState: OK
+        execErrState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: botlink_connected_bots, instant: true }
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: lt, params: [1] }
+        labels: { severity: critical }
+        annotations: { summary: 'No Telegram bot is connected to the gateway bot-link — out-of-app push and admin sends are down. Check the bot host.' }
+
+      # Positive liveness: a bot is connected but its last successful Bot API call is over 5 minutes
+      # old — the getUpdates long-poll returns every ~minute even when idle, so a stale stamp means the
+      # bot is wedged (no errors, no traffic). Guarded by "and connected" so a mere disconnect (owned by
+      # bot_disconnected) does not double-fire.
+      - uid: bot_tg_stale
+        title: Telegram bot not reaching the Bot API
+        condition: C
+        for: 5m
+        noDataState: OK
+        execErrState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 600, to: 0 }
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: (time() - bot_tg_last_ok_unix) and on() (botlink_connected_bots >= 1)
+              instant: true
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: gt, params: [300] }
+        labels: { severity: critical }
+        annotations: { summary: 'A connected Telegram bot has not reached the Bot API in over 5 minutes — the update poll is wedged. Check the bot host and Telegram reachability.' }
+
+      # 429s should be ~never once the bot honours Retry-After; any sustained rate-limiting is a symptom
+      # (a send loop, a misbehaving path) worth investigating.
+      - uid: bot_tg_rate_limited
+        title: Telegram bot rate-limited (429)
+        condition: C
+        for: 5m
+        noDataState: OK
+        execErrState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: increase(bot_tg_errors_total{kind="rate_limited"}[15m])
+              instant: true
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: gt, params: [0] }
+        labels: { severity: warning }
+        annotations: { summary: 'The Telegram bot is being rate-limited (HTTP 429). It should honour Retry-After, so sustained 429s point to a send loop or a hot path.' }