feat(bot): report Telegram bot Bot API health to the gateway over the bot-link
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 12s
CI / integration (pull_request) Successful in 26s
CI / ui (pull_request) Successful in 1m11s
CI / conformance (pull_request) Successful in 10s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m56s
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 12s
CI / integration (pull_request) Successful in 26s
CI / ui (pull_request) Successful in 1m11s
CI / conformance (pull_request) Successful in 10s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m56s
The bot runs on its own host and exports no telemetry (otelcol is unreachable
from there), so it was a monitoring blind spot. Observe its Bot API health
centrally by wrapping the HTTP client — one place, no per-call-site
instrumentation — and relay it up the existing bot-link as a periodic Health
message the gateway turns into its own metrics.
- proto: add Health (delta connect / api / 429 counters + a last-ok stamp) to
the FromBot oneof (additive, backward-compatible).
- bot: platform/telegram/internal/health wraps the Bot API HTTP client — it
stamps liveness on any 2xx (so the getUpdates long-poll keeps it fresh even
when idle), classifies transport/5xx failures (getUpdates vs other) and 429s,
and honours a 429's Retry-After (bounded) so the bot backs off; a 4xx other
than 429 is a normal per-request outcome and is not counted.
- bot-link client: flush the reporter as a Health message every 30s over a
single-sender loop (a gRPC stream forbids concurrent Send).
- gateway: fold each report into bot_tg_errors_total{kind} and the
bot_tg_last_ok_unix liveness gauge.
- grafana: alerts (bot disconnected, bot not reaching the Bot API, sustained
429s) routed to the operator email — which does not go through the bot — plus
a Telegram-bot dashboard.
- docs (ARCHITECTURE, compose comments); unit tests (observer classification,
Retry-After, snapshot/commit, hub last-ok monotonicity).
This commit is contained in:
@@ -76,6 +76,11 @@ type Hub struct {
|
||||
|
||||
connected metric.Int64UpDownCounter
|
||||
commands metric.Int64Counter
|
||||
// tgErrors counts the remote bot's Bot API failures it reports over the stream, by kind
|
||||
// (connect / api / rate_limited). lastOK holds the wall-clock second of the bot's most recent
|
||||
// successful Bot API call, read by the bot_tg_last_ok_unix observable gauge for a staleness alert.
|
||||
tgErrors metric.Int64Counter
|
||||
lastOK atomic.Int64
|
||||
}
|
||||
|
||||
// link is one connected bot's outbound queue and identity.
|
||||
@@ -103,6 +108,19 @@ func NewHub(log *zap.Logger, meter metric.Meter, resolve EligibilityResolver) *H
|
||||
metric.WithDescription("Number of Telegram bots currently connected to the gateway bot-link."))
|
||||
h.commands, _ = meter.Int64Counter("botlink_commands_total",
|
||||
metric.WithDescription("Bot-link send commands by result (delivered, not_delivered, dropped, error)."))
|
||||
h.tgErrors, _ = meter.Int64Counter("bot_tg_errors_total",
|
||||
metric.WithDescription("Telegram Bot API failures the remote bot reports over the bot-link, by kind (connect, api, rate_limited)."))
|
||||
// The bot's last successful Bot API second, reported over the stream. An observable gauge
|
||||
// reads the atomic on each collection; it observes nothing until a bot has reported, so a
|
||||
// never-connected gateway shows no data (the connected-bots alert covers that case).
|
||||
_, _ = meter.Int64ObservableGauge("bot_tg_last_ok_unix",
|
||||
metric.WithDescription("Unix second of the remote bot's most recent successful Bot API call (0/absent until reported)."),
|
||||
metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error {
|
||||
if v := h.lastOK.Load(); v > 0 {
|
||||
o.Observe(v)
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -149,6 +167,36 @@ func (h *Hub) Link(stream grpc.BidiStreamingServer[botlinkv1.FromBot, botlinkv1.
|
||||
if ack := msg.GetAck(); ack != nil {
|
||||
h.resolve(ack)
|
||||
}
|
||||
if hb := msg.GetHealth(); hb != nil {
|
||||
h.recordHealth(hb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// recordHealth folds one bot Health report into the gateway's cumulative Bot API metrics: the three
|
||||
// delta counters are added under their kind label, and the last-ok stamp (an absolute wall-clock
|
||||
// second) advances the liveness gauge (monotonically — a stale reordered report cannot rewind it).
|
||||
func (h *Hub) recordHealth(hb *botlinkv1.Health) {
|
||||
if h.tgErrors != nil {
|
||||
ctx := context.Background()
|
||||
if v := hb.GetConnectFailures(); v > 0 {
|
||||
h.tgErrors.Add(ctx, int64(v), metric.WithAttributes(attribute.String("kind", "connect")))
|
||||
}
|
||||
if v := hb.GetApiErrors(); v > 0 {
|
||||
h.tgErrors.Add(ctx, int64(v), metric.WithAttributes(attribute.String("kind", "api")))
|
||||
}
|
||||
if v := hb.GetRateLimited(); v > 0 {
|
||||
h.tgErrors.Add(ctx, int64(v), metric.WithAttributes(attribute.String("kind", "rate_limited")))
|
||||
}
|
||||
}
|
||||
for {
|
||||
cur := h.lastOK.Load()
|
||||
if hb.GetLastOkUnix() <= cur {
|
||||
break
|
||||
}
|
||||
if h.lastOK.CompareAndSwap(cur, hb.GetLastOkUnix()) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -233,3 +233,21 @@ func TestRelayServerNoBot(t *testing.T) {
|
||||
t.Fatal("expected an error with no bot connected")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecordHealthLastOKMonotonic checks a bot Health report advances the last-ok liveness stamp but
|
||||
// a stale or reordered report (an earlier second) never rewinds it.
|
||||
func TestRecordHealthLastOKMonotonic(t *testing.T) {
|
||||
hub := NewHub(nil, nil, nil)
|
||||
hub.recordHealth(&botlinkv1.Health{LastOkUnix: 100})
|
||||
if got := hub.lastOK.Load(); got != 100 {
|
||||
t.Fatalf("lastOK = %d, want 100", got)
|
||||
}
|
||||
hub.recordHealth(&botlinkv1.Health{LastOkUnix: 90}) // reordered/stale — must not rewind
|
||||
if got := hub.lastOK.Load(); got != 100 {
|
||||
t.Errorf("lastOK rewound to %d, want 100", got)
|
||||
}
|
||||
hub.recordHealth(&botlinkv1.Health{LastOkUnix: 150})
|
||||
if got := hub.lastOK.Load(); got != 150 {
|
||||
t.Errorf("lastOK = %d, want 150", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user