bb71e7b1c7
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 12s
CI / integration (pull_request) Successful in 26s
CI / ui (pull_request) Successful in 1m11s
CI / conformance (pull_request) Successful in 10s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m56s
The bot runs on its own host and exports no telemetry (otelcol is unreachable
from there), so it was a monitoring blind spot. Observe its Bot API health
centrally by wrapping the HTTP client — one place, no per-call-site
instrumentation — and relay it up the existing bot-link as a periodic Health
message the gateway turns into its own metrics.
- proto: add Health (delta connect / api / 429 counters + a last-ok stamp) to
the FromBot oneof (additive, backward-compatible).
- bot: platform/telegram/internal/health wraps the Bot API HTTP client — it
stamps liveness on any 2xx (so the getUpdates long-poll keeps it fresh even
when idle), classifies transport/5xx failures (getUpdates vs other) and 429s,
and honours a 429's Retry-After (bounded) so the bot backs off; a 4xx other
than 429 is a normal per-request outcome and is not counted.
- bot-link client: flush the reporter as a Health message every 30s over a
single-sender loop (a gRPC stream forbids concurrent Send).
- gateway: fold each report into bot_tg_errors_total{kind} and the
bot_tg_last_ok_unix liveness gauge.
- grafana: alerts (bot disconnected, bot not reaching the Bot API, sustained
429s) routed to the operator email — which does not go through the bot — plus
a Telegram-bot dashboard.
- docs (ARCHITECTURE, compose comments); unit tests (observer classification,
Retry-After, snapshot/commit, hub last-ok monotonicity).
122 lines
4.3 KiB
Go
122 lines
4.3 KiB
Go
package health
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// fakeDoer returns a canned response and error for every request.
|
|
type fakeDoer struct {
|
|
resp *http.Response
|
|
err error
|
|
}
|
|
|
|
func (f *fakeDoer) Do(*http.Request) (*http.Response, error) { return f.resp, f.err }
|
|
|
|
// mkResp builds a response with a status, optional Retry-After header and body.
|
|
func mkResp(status int, retryAfter, body string) *http.Response {
|
|
h := http.Header{}
|
|
if retryAfter != "" {
|
|
h.Set("Retry-After", retryAfter)
|
|
}
|
|
return &http.Response{StatusCode: status, Header: h, Body: io.NopCloser(strings.NewReader(body))}
|
|
}
|
|
|
|
// run sends one request for the Bot API method through a fresh observer and returns the reporter's
|
|
// resulting snapshot. The 429 cases carry no Retry-After, so the back-off returns before any wait.
|
|
func run(method string, resp *http.Response, err error) *Reporter {
|
|
r := NewReporter()
|
|
req := httptest.NewRequest(http.MethodPost, "https://api.telegram.org/bot123/"+method, nil)
|
|
_, _ = r.Wrap(&fakeDoer{resp: resp, err: err}).Do(req)
|
|
return r
|
|
}
|
|
|
|
func TestObserverClassifies(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
method string
|
|
resp *http.Response
|
|
err error
|
|
wantConnect uint64
|
|
wantAPI uint64
|
|
wantRate uint64
|
|
wantLastOKSet bool
|
|
}{
|
|
{"poll ok stamps liveness", "getUpdates", mkResp(200, "", `{"ok":true}`), nil, 0, 0, 0, true},
|
|
{"send ok stamps liveness", "sendMessage", mkResp(200, "", `{"ok":true}`), nil, 0, 0, 0, true},
|
|
{"poll 5xx is a connect failure", "getUpdates", mkResp(502, "", ""), nil, 1, 0, 0, false},
|
|
{"send 5xx is an api error", "sendMessage", mkResp(500, "", ""), nil, 0, 1, 0, false},
|
|
{"429 is rate limited, not an error", "sendMessage", mkResp(429, "", `{"ok":false}`), nil, 0, 0, 1, false},
|
|
{"4xx other than 429 is not counted", "sendMessage", mkResp(403, "", ""), nil, 0, 0, 0, false},
|
|
{"poll transport error is a connect failure", "getUpdates", nil, errors.New("dial"), 1, 0, 0, false},
|
|
{"send transport error is an api error", "sendMessage", nil, errors.New("dial"), 0, 1, 0, false},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s := run(tc.method, tc.resp, tc.err).Snapshot()
|
|
if s.GetConnectFailures() != tc.wantConnect || s.GetApiErrors() != tc.wantAPI || s.GetRateLimited() != tc.wantRate {
|
|
t.Errorf("counters = connect %d api %d rate %d, want %d/%d/%d",
|
|
s.GetConnectFailures(), s.GetApiErrors(), s.GetRateLimited(), tc.wantConnect, tc.wantAPI, tc.wantRate)
|
|
}
|
|
if (s.GetLastOkUnix() > 0) != tc.wantLastOKSet {
|
|
t.Errorf("last_ok set = %v, want %v", s.GetLastOkUnix() > 0, tc.wantLastOKSet)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestObserverIgnoresShutdownTransportError checks a transport error under a cancelled context (a
|
|
// shutdown) is not counted as a Bot API failure.
|
|
func TestObserverIgnoresShutdownTransportError(t *testing.T) {
|
|
r := NewReporter()
|
|
req := httptest.NewRequest(http.MethodPost, "https://api.telegram.org/bot123/getUpdates", nil)
|
|
ctx, cancel := context.WithCancel(req.Context())
|
|
cancel()
|
|
_, _ = r.Wrap(&fakeDoer{err: errors.New("dial")}).Do(req.WithContext(ctx))
|
|
if s := r.Snapshot(); s.GetConnectFailures() != 0 {
|
|
t.Errorf("shutdown transport error must not count, got connect=%d", s.GetConnectFailures())
|
|
}
|
|
}
|
|
|
|
func TestRetryAfter(t *testing.T) {
|
|
cases := []struct {
|
|
header string
|
|
body string
|
|
want time.Duration
|
|
}{
|
|
{"5", "", 5 * time.Second},
|
|
{"", `{"parameters":{"retry_after":7}}`, 7 * time.Second},
|
|
{"3", `{"parameters":{"retry_after":9}}`, 3 * time.Second}, // header wins
|
|
{"", `{"ok":false}`, 0},
|
|
{"0", "", 0},
|
|
{"", "not json", 0},
|
|
}
|
|
for _, tc := range cases {
|
|
if got := retryAfter(tc.header, []byte(tc.body)); got != tc.want {
|
|
t.Errorf("retryAfter(%q, %q) = %v, want %v", tc.header, tc.body, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestSnapshotCommit checks Commit clears only the sent deltas, so counts accrued between the
|
|
// snapshot and the send survive into the next report.
|
|
func TestSnapshotCommit(t *testing.T) {
|
|
r := NewReporter()
|
|
r.apiErrors.Add(3)
|
|
snap := r.Snapshot()
|
|
if snap.GetApiErrors() != 3 {
|
|
t.Fatalf("snapshot api_errors = %d, want 3", snap.GetApiErrors())
|
|
}
|
|
r.apiErrors.Add(2) // accrues after the snapshot
|
|
r.Commit(snap)
|
|
if got := r.Snapshot().GetApiErrors(); got != 2 {
|
|
t.Errorf("after commit api_errors = %d, want 2 (the post-snapshot accrual)", got)
|
|
}
|
|
}
|