feat(bot): report Telegram bot Bot API health to the gateway over the bot-link

The bot runs on its own host and exports no telemetry (otelcol is unreachable
from there), so it was a monitoring blind spot. Observe its Bot API health
centrally by wrapping the HTTP client — one place, no per-call-site
instrumentation — and relay it up the existing bot-link as a periodic Health
message the gateway turns into its own metrics.

- proto: add Health (delta connect / api / 429 counters + a last-ok stamp) to
  the FromBot oneof (additive, backward-compatible).
- bot: platform/telegram/internal/health wraps the Bot API HTTP client — it
  stamps liveness on any 2xx (so the getUpdates long-poll keeps it fresh even
  when idle), classifies transport/5xx failures (getUpdates vs other) and 429s,
  and honours a 429's Retry-After (bounded) so the bot backs off; a 4xx other
  than 429 is a normal per-request outcome and is not counted.
- bot-link client: flush the reporter as a Health message every 30s over a
  single-sender loop (a gRPC stream forbids concurrent Send).
- gateway: fold each report into bot_tg_errors_total{kind} and the
  bot_tg_last_ok_unix liveness gauge.
- grafana: alerts (bot disconnected, bot not reaching the Bot API, sustained
  429s) routed to the operator email — which does not go through the bot — plus
  a Telegram-bot dashboard.
- docs (ARCHITECTURE, compose comments); unit tests (observer classification,
  Retry-After, snapshot/commit, hub last-ok monotonicity).

platform/telegram/internal/health/health.go

@@ -0,0 +1,145 @@
+// Package health tracks the Telegram bot's Bot API health and reports it to the gateway over the
+// bot-link. The bot exports no telemetry of its own — otelcol lives on the main host, unreachable
+// from the bot host — so the gateway turns these reports into its own metrics (see
+// gateway/internal/botlink). Health is observed CENTRALLY by wrapping the Bot API HTTP client
+// ([Reporter.Wrap]); every request is classified there with no per-call-site instrumentation.
+package health
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	botlinkv1 "scrabble/pkg/proto/botlink/v1"
+)
+
+// maxBackoff caps how long a single 429 makes the bot wait, so a hostile or buggy Retry-After can
+// never wedge the bot.
+const maxBackoff = 30 * time.Second
+
+// Reporter accumulates the bot's Bot API health between reports: three delta counters and the last
+// successful-response stamp. It is safe for concurrent use. The bot-link client periodically
+// [Reporter.Snapshot]s it, sends the snapshot as a botlink Health, and [Reporter.Commit]s it on a
+// successful send.
+type Reporter struct {
+	connectFailures atomic.Int64
+	apiErrors       atomic.Int64
+	rateLimited     atomic.Int64
+	lastOKUnix      atomic.Int64
+}
+
+// NewReporter returns an empty Reporter.
+func NewReporter() *Reporter { return &Reporter{} }
+
+// Snapshot reads the current delta counters and the last-ok stamp into a Health message WITHOUT
+// resetting them. The counters are cleared only by [Reporter.Commit] after a successful send, so a
+// failed send loses nothing.
+func (r *Reporter) Snapshot() *botlinkv1.Health {
+	return &botlinkv1.Health{
+		ConnectFailures: uint64(max(r.connectFailures.Load(), 0)),
+		ApiErrors:       uint64(max(r.apiErrors.Load(), 0)),
+		RateLimited:     uint64(max(r.rateLimited.Load(), 0)),
+		LastOkUnix:      r.lastOKUnix.Load(),
+	}
+}
+
+// Commit subtracts a just-sent snapshot's delta counters, so counts accrued between the snapshot and
+// the send survive into the next report. last_ok is a stamp, not a delta, so it is left as is.
+func (r *Reporter) Commit(sent *botlinkv1.Health) {
+	r.connectFailures.Add(-int64(sent.GetConnectFailures()))
+	r.apiErrors.Add(-int64(sent.GetApiErrors()))
+	r.rateLimited.Add(-int64(sent.GetRateLimited()))
+}
+
+// Wrap returns an http.Client-shaped observer over base for tgbot.WithHTTPClient: it stamps liveness
+// on every successful response, counts transport and 5xx failures (split getUpdates vs other) and
+// 429s, and honours a 429's Retry-After (bounded by maxBackoff) so the bot backs off instead of
+// hammering. A 4xx other than 429 (a user blocked the bot, a malformed request) is a normal
+// per-request outcome, not a health signal, and is not counted.
+func (r *Reporter) Wrap(base Doer) Doer { return &observer{base: base, r: r} }
+
+// Doer is the minimal HTTP surface the Bot API client needs; both *http.Client and [observer]
+// satisfy it, and it matches the go-telegram/bot HttpClient interface.
+type Doer interface {
+	Do(*http.Request) (*http.Response, error)
+}
+
+// observer is the Bot API HTTP client wrapper (see [Reporter.Wrap]).
+type observer struct {
+	base Doer
+	r    *Reporter
+}
+
+// Do performs the request and records its health outcome.
+func (o *observer) Do(req *http.Request) (*http.Response, error) {
+	resp, err := o.base.Do(req)
+	poll := strings.HasSuffix(req.URL.Path, "/getUpdates")
+	if err != nil {
+		// A cancelled context is a shutdown, not a Bot API failure.
+		if req.Context().Err() == nil {
+			o.fail(poll)
+		}
+		return resp, err
+	}
+	switch {
+	case resp.StatusCode == http.StatusTooManyRequests:
+		o.r.rateLimited.Add(1)
+		o.backoff(req.Context(), resp)
+	case resp.StatusCode >= 200 && resp.StatusCode < 300:
+		o.r.lastOKUnix.Store(time.Now().Unix())
+	case resp.StatusCode >= 500:
+		o.fail(poll)
+	}
+	return resp, err
+}
+
+// fail records a connect failure on the getUpdates long-poll or an api error otherwise.
+func (o *observer) fail(poll bool) {
+	if poll {
+		o.r.connectFailures.Add(1)
+	} else {
+		o.r.apiErrors.Add(1)
+	}
+}
+
+// backoff waits out a 429's Retry-After (bounded, cancellable). It buffers the response body first so
+// the wait holds no connection open and the Bot API library can still parse the 429 afterwards.
+func (o *observer) backoff(ctx context.Context, resp *http.Response) {
+	body, _ := io.ReadAll(resp.Body)
+	resp.Body.Close()
+	resp.Body = io.NopCloser(bytes.NewReader(body))
+	d := retryAfter(resp.Header.Get("Retry-After"), body)
+	if d <= 0 {
+		return
+	}
+	if d > maxBackoff {
+		d = maxBackoff
+	}
+	select {
+	case <-ctx.Done():
+	case <-time.After(d):
+	}
+}
+
+// retryAfter resolves the 429 back-off from the Retry-After header, falling back to the Bot API
+// body's parameters.retry_after (both are seconds). It returns 0 when neither gives a positive value.
+func retryAfter(header string, body []byte) time.Duration {
+	if secs, err := strconv.Atoi(strings.TrimSpace(header)); err == nil && secs > 0 {
+		return time.Duration(secs) * time.Second
+	}
+	var payload struct {
+		Parameters struct {
+			RetryAfter int `json:"retry_after"`
+		} `json:"parameters"`
+	}
+	if err := json.Unmarshal(body, &payload); err == nil && payload.Parameters.RetryAfter > 0 {
+		return time.Duration(payload.Parameters.RetryAfter) * time.Second
+	}
+	return 0
+}