feat(bot): report Telegram bot Bot API health to the gateway over the bot-link

The bot runs on its own host and exports no telemetry (otelcol is unreachable from there), so it was a monitoring blind spot. Observe its Bot API health centrally by wrapping the HTTP client — one place, no per-call-site instrumentation — and relay it up the existing bot-link as a periodic Health message the gateway turns into its own metrics. - proto: add Health (delta connect / api / 429 counters + a last-ok stamp) to the FromBot oneof (additive, backward-compatible). - bot: platform/telegram/internal/health wraps the Bot API HTTP client — it stamps liveness on any 2xx (so the getUpdates long-poll keeps it fresh even when idle), classifies transport/5xx failures (getUpdates vs other) and 429s, and honours a 429's Retry-After (bounded) so the bot backs off; a 4xx other than 429 is a normal per-request outcome and is not counted. - bot-link client: flush the reporter as a Health message every 30s over a single-sender loop (a gRPC stream forbids concurrent Send). - gateway: fold each report into bot_tg_errors_total{kind} and the bot_tg_last_ok_unix liveness gauge. - grafana: alerts (bot disconnected, bot not reaching the Bot API, sustained 429s) routed to the operator email — which does not go through the bot — plus a Telegram-bot dashboard. - docs (ARCHITECTURE, compose comments); unit tests (observer classification, Retry-After, snapshot/commit, hub last-ok monotonicity).
2026-07-11 12:48:06 +02:00
parent 5c1f64c7d1
commit bb71e7b1c7
13 changed files with 774 additions and 98 deletions
@@ -0,0 +1,145 @@
+// Package health tracks the Telegram bot's Bot API health and reports it to the gateway over the
+// bot-link. The bot exports no telemetry of its own — otelcol lives on the main host, unreachable
+// from the bot host — so the gateway turns these reports into its own metrics (see
+// gateway/internal/botlink). Health is observed CENTRALLY by wrapping the Bot API HTTP client
+// ([Reporter.Wrap]); every request is classified there with no per-call-site instrumentation.
+package health
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	botlinkv1 "scrabble/pkg/proto/botlink/v1"
+)
+
+// maxBackoff caps how long a single 429 makes the bot wait, so a hostile or buggy Retry-After can
+// never wedge the bot.
+const maxBackoff = 30 * time.Second
+
+// Reporter accumulates the bot's Bot API health between reports: three delta counters and the last
+// successful-response stamp. It is safe for concurrent use. The bot-link client periodically
+// [Reporter.Snapshot]s it, sends the snapshot as a botlink Health, and [Reporter.Commit]s it on a
+// successful send.
+type Reporter struct {
+	connectFailures atomic.Int64
+	apiErrors       atomic.Int64
+	rateLimited     atomic.Int64
+	lastOKUnix      atomic.Int64
+}
+
+// NewReporter returns an empty Reporter.
+func NewReporter() *Reporter { return &Reporter{} }
+
+// Snapshot reads the current delta counters and the last-ok stamp into a Health message WITHOUT
+// resetting them. The counters are cleared only by [Reporter.Commit] after a successful send, so a
+// failed send loses nothing.
+func (r *Reporter) Snapshot() *botlinkv1.Health {
+	return &botlinkv1.Health{
+		ConnectFailures: uint64(max(r.connectFailures.Load(), 0)),
+		ApiErrors:       uint64(max(r.apiErrors.Load(), 0)),
+		RateLimited:     uint64(max(r.rateLimited.Load(), 0)),
+		LastOkUnix:      r.lastOKUnix.Load(),
+	}
+}
+
+// Commit subtracts a just-sent snapshot's delta counters, so counts accrued between the snapshot and
+// the send survive into the next report. last_ok is a stamp, not a delta, so it is left as is.
+func (r *Reporter) Commit(sent *botlinkv1.Health) {
+	r.connectFailures.Add(-int64(sent.GetConnectFailures()))
+	r.apiErrors.Add(-int64(sent.GetApiErrors()))
+	r.rateLimited.Add(-int64(sent.GetRateLimited()))
+}
+
+// Wrap returns an http.Client-shaped observer over base for tgbot.WithHTTPClient: it stamps liveness
+// on every successful response, counts transport and 5xx failures (split getUpdates vs other) and
+// 429s, and honours a 429's Retry-After (bounded by maxBackoff) so the bot backs off instead of
+// hammering. A 4xx other than 429 (a user blocked the bot, a malformed request) is a normal
+// per-request outcome, not a health signal, and is not counted.
+func (r *Reporter) Wrap(base Doer) Doer { return &observer{base: base, r: r} }
+
+// Doer is the minimal HTTP surface the Bot API client needs; both *http.Client and [observer]
+// satisfy it, and it matches the go-telegram/bot HttpClient interface.
+type Doer interface {
+	Do(*http.Request) (*http.Response, error)
+}
+
+// observer is the Bot API HTTP client wrapper (see [Reporter.Wrap]).
+type observer struct {
+	base Doer
+	r    *Reporter
+}
+
+// Do performs the request and records its health outcome.
+func (o *observer) Do(req *http.Request) (*http.Response, error) {
+	resp, err := o.base.Do(req)
+	poll := strings.HasSuffix(req.URL.Path, "/getUpdates")
+	if err != nil {
+		// A cancelled context is a shutdown, not a Bot API failure.
+		if req.Context().Err() == nil {
+			o.fail(poll)
+		}
+		return resp, err
+	}
+	switch {
+	case resp.StatusCode == http.StatusTooManyRequests:
+		o.r.rateLimited.Add(1)
+		o.backoff(req.Context(), resp)
+	case resp.StatusCode >= 200 && resp.StatusCode < 300:
+		o.r.lastOKUnix.Store(time.Now().Unix())
+	case resp.StatusCode >= 500:
+		o.fail(poll)
+	}
+	return resp, err
+}
+
+// fail records a connect failure on the getUpdates long-poll or an api error otherwise.
+func (o *observer) fail(poll bool) {
+	if poll {
+		o.r.connectFailures.Add(1)
+	} else {
+		o.r.apiErrors.Add(1)
+	}
+}
+
+// backoff waits out a 429's Retry-After (bounded, cancellable). It buffers the response body first so
+// the wait holds no connection open and the Bot API library can still parse the 429 afterwards.
+func (o *observer) backoff(ctx context.Context, resp *http.Response) {
+	body, _ := io.ReadAll(resp.Body)
+	resp.Body.Close()
+	resp.Body = io.NopCloser(bytes.NewReader(body))
+	d := retryAfter(resp.Header.Get("Retry-After"), body)
+	if d <= 0 {
+		return
+	}
+	if d > maxBackoff {
+		d = maxBackoff
+	}
+	select {
+	case <-ctx.Done():
+	case <-time.After(d):
+	}
+}
+
+// retryAfter resolves the 429 back-off from the Retry-After header, falling back to the Bot API
+// body's parameters.retry_after (both are seconds). It returns 0 when neither gives a positive value.
+func retryAfter(header string, body []byte) time.Duration {
+	if secs, err := strconv.Atoi(strings.TrimSpace(header)); err == nil && secs > 0 {
+		return time.Duration(secs) * time.Second
+	}
+	var payload struct {
+		Parameters struct {
+			RetryAfter int `json:"retry_after"`
+		} `json:"parameters"`
+	}
+	if err := json.Unmarshal(body, &payload); err == nil && payload.Parameters.RetryAfter > 0 {
+		return time.Duration(payload.Parameters.RetryAfter) * time.Second
+	}
+	return 0
+}
@@ -0,0 +1,121 @@
+package health
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+// fakeDoer returns a canned response and error for every request.
+type fakeDoer struct {
+	resp *http.Response
+	err  error
+}
+
+func (f *fakeDoer) Do(*http.Request) (*http.Response, error) { return f.resp, f.err }
+
+// mkResp builds a response with a status, optional Retry-After header and body.
+func mkResp(status int, retryAfter, body string) *http.Response {
+	h := http.Header{}
+	if retryAfter != "" {
+		h.Set("Retry-After", retryAfter)
+	}
+	return &http.Response{StatusCode: status, Header: h, Body: io.NopCloser(strings.NewReader(body))}
+}
+
+// run sends one request for the Bot API method through a fresh observer and returns the reporter's
+// resulting snapshot. The 429 cases carry no Retry-After, so the back-off returns before any wait.
+func run(method string, resp *http.Response, err error) *Reporter {
+	r := NewReporter()
+	req := httptest.NewRequest(http.MethodPost, "https://api.telegram.org/bot123/"+method, nil)
+	_, _ = r.Wrap(&fakeDoer{resp: resp, err: err}).Do(req)
+	return r
+}
+
+func TestObserverClassifies(t *testing.T) {
+	cases := []struct {
+		name          string
+		method        string
+		resp          *http.Response
+		err           error
+		wantConnect   uint64
+		wantAPI       uint64
+		wantRate      uint64
+		wantLastOKSet bool
+	}{
+		{"poll ok stamps liveness", "getUpdates", mkResp(200, "", `{"ok":true}`), nil, 0, 0, 0, true},
+		{"send ok stamps liveness", "sendMessage", mkResp(200, "", `{"ok":true}`), nil, 0, 0, 0, true},
+		{"poll 5xx is a connect failure", "getUpdates", mkResp(502, "", ""), nil, 1, 0, 0, false},
+		{"send 5xx is an api error", "sendMessage", mkResp(500, "", ""), nil, 0, 1, 0, false},
+		{"429 is rate limited, not an error", "sendMessage", mkResp(429, "", `{"ok":false}`), nil, 0, 0, 1, false},
+		{"4xx other than 429 is not counted", "sendMessage", mkResp(403, "", ""), nil, 0, 0, 0, false},
+		{"poll transport error is a connect failure", "getUpdates", nil, errors.New("dial"), 1, 0, 0, false},
+		{"send transport error is an api error", "sendMessage", nil, errors.New("dial"), 0, 1, 0, false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			s := run(tc.method, tc.resp, tc.err).Snapshot()
+			if s.GetConnectFailures() != tc.wantConnect || s.GetApiErrors() != tc.wantAPI || s.GetRateLimited() != tc.wantRate {
+				t.Errorf("counters = connect %d api %d rate %d, want %d/%d/%d",
+					s.GetConnectFailures(), s.GetApiErrors(), s.GetRateLimited(), tc.wantConnect, tc.wantAPI, tc.wantRate)
+			}
+			if (s.GetLastOkUnix() > 0) != tc.wantLastOKSet {
+				t.Errorf("last_ok set = %v, want %v", s.GetLastOkUnix() > 0, tc.wantLastOKSet)
+			}
+		})
+	}
+}
+
+// TestObserverIgnoresShutdownTransportError checks a transport error under a cancelled context (a
+// shutdown) is not counted as a Bot API failure.
+func TestObserverIgnoresShutdownTransportError(t *testing.T) {
+	r := NewReporter()
+	req := httptest.NewRequest(http.MethodPost, "https://api.telegram.org/bot123/getUpdates", nil)
+	ctx, cancel := context.WithCancel(req.Context())
+	cancel()
+	_, _ = r.Wrap(&fakeDoer{err: errors.New("dial")}).Do(req.WithContext(ctx))
+	if s := r.Snapshot(); s.GetConnectFailures() != 0 {
+		t.Errorf("shutdown transport error must not count, got connect=%d", s.GetConnectFailures())
+	}
+}
+
+func TestRetryAfter(t *testing.T) {
+	cases := []struct {
+		header string
+		body   string
+		want   time.Duration
+	}{
+		{"5", "", 5 * time.Second},
+		{"", `{"parameters":{"retry_after":7}}`, 7 * time.Second},
+		{"3", `{"parameters":{"retry_after":9}}`, 3 * time.Second}, // header wins
+		{"", `{"ok":false}`, 0},
+		{"0", "", 0},
+		{"", "not json", 0},
+	}
+	for _, tc := range cases {
+		if got := retryAfter(tc.header, []byte(tc.body)); got != tc.want {
+			t.Errorf("retryAfter(%q, %q) = %v, want %v", tc.header, tc.body, got, tc.want)
+		}
+	}
+}
+
+// TestSnapshotCommit checks Commit clears only the sent deltas, so counts accrued between the
+// snapshot and the send survive into the next report.
+func TestSnapshotCommit(t *testing.T) {
+	r := NewReporter()
+	r.apiErrors.Add(3)
+	snap := r.Snapshot()
+	if snap.GetApiErrors() != 3 {
+		t.Fatalf("snapshot api_errors = %d, want 3", snap.GetApiErrors())
+	}
+	r.apiErrors.Add(2) // accrues after the snapshot
+	r.Commit(snap)
+	if got := r.Snapshot().GetApiErrors(); got != 2 {
+		t.Errorf("after commit api_errors = %d, want 2 (the post-snapshot accrual)", got)
+	}
+}