feat(bot): report Telegram bot Bot API health to the gateway over the bot-link
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 12s
CI / integration (pull_request) Successful in 26s
CI / ui (pull_request) Successful in 1m11s
CI / conformance (pull_request) Successful in 10s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m56s

The bot runs on its own host and exports no telemetry (otelcol is unreachable
from there), so it was a monitoring blind spot. Observe its Bot API health
centrally by wrapping the HTTP client — one place, no per-call-site
instrumentation — and relay it up the existing bot-link as a periodic Health
message the gateway turns into its own metrics.

- proto: add Health (delta connect / api / 429 counters + a last-ok stamp) to
  the FromBot oneof (additive, backward-compatible).
- bot: platform/telegram/internal/health wraps the Bot API HTTP client — it
  stamps liveness on any 2xx (so the getUpdates long-poll keeps it fresh even
  when idle), classifies transport/5xx failures (getUpdates vs other) and 429s,
  and honours a 429's Retry-After (bounded) so the bot backs off; a 4xx other
  than 429 is a normal per-request outcome and is not counted.
- bot-link client: flush the reporter as a Health message every 30s over a
  single-sender loop (a gRPC stream forbids concurrent Send).
- gateway: fold each report into bot_tg_errors_total{kind} and the
  bot_tg_last_ok_unix liveness gauge.
- grafana: alerts (bot disconnected, bot not reaching the Bot API, sustained
  429s) routed to the operator email — which does not go through the bot — plus
  a Telegram-bot dashboard.
- docs (ARCHITECTURE, compose comments); unit tests (observer classification,
  Retry-After, snapshot/commit, hub last-ok monotonicity).
This commit is contained in:
Ilia Denisov
2026-07-11 12:48:06 +02:00
parent 5c1f64c7d1
commit bb71e7b1c7
13 changed files with 774 additions and 98 deletions
+57 -14
View File
@@ -11,6 +11,7 @@ import (
"google.golang.org/grpc/keepalive"
botlinkv1 "scrabble/pkg/proto/botlink/v1"
"scrabble/platform/telegram/internal/health"
)
const (
@@ -34,6 +35,11 @@ type ClientConfig struct {
Creds credentials.TransportCredentials
// ReconnectDelay is the pause before re-dialing after the stream ends.
ReconnectDelay time.Duration
// Health, when set with a positive HealthInterval, is flushed to the gateway as a botlink Health
// message every HealthInterval while the stream is open. Nil disables health reporting.
Health *health.Reporter
// HealthInterval is the cadence of the Health flush; 0 disables it.
HealthInterval time.Duration
}
// Client maintains the long-lived bot-link to the gateway, executing the commands
@@ -140,22 +146,59 @@ func (c *Client) serve(ctx context.Context, client botlinkv1.BotLinkClient) erro
}
c.log.Info("bot-link connected", zap.String("gateway", c.cfg.GatewayAddr), zap.Bool("owns_updates", c.cfg.OwnsUpdates))
// One goroutine owns stream.Recv, feeding commands to the loop below; the loop owns every
// stream.Send (the Acks and the periodic Health) — a gRPC stream forbids concurrent Send.
rctx, cancel := context.WithCancel(ctx)
defer cancel()
cmds := make(chan *botlinkv1.Command)
recvErr := make(chan error, 1)
go func() {
for {
msg, err := stream.Recv()
if err != nil {
recvErr <- err
return
}
if cmd := msg.GetCommand(); cmd != nil {
select {
case cmds <- cmd:
case <-rctx.Done():
return
}
}
}
}()
var healthTick <-chan time.Time
if c.cfg.Health != nil && c.cfg.HealthInterval > 0 {
t := time.NewTicker(c.cfg.HealthInterval)
defer t.Stop()
healthTick = t.C
}
for {
msg, err := stream.Recv()
if err != nil {
return err
}
cmd := msg.GetCommand()
if cmd == nil {
continue
}
delivered, result, herr := c.exec.Handle(ctx, cmd)
ack := &botlinkv1.Ack{CommandId: cmd.GetCommandId(), Delivered: delivered, Result: result}
if herr != nil {
ack.Error = herr.Error()
}
if err := stream.Send(&botlinkv1.FromBot{Msg: &botlinkv1.FromBot_Ack{Ack: ack}}); err != nil {
select {
case <-ctx.Done():
return ctx.Err()
case err := <-recvErr:
return err
case cmd := <-cmds:
delivered, result, herr := c.exec.Handle(ctx, cmd)
ack := &botlinkv1.Ack{CommandId: cmd.GetCommandId(), Delivered: delivered, Result: result}
if herr != nil {
ack.Error = herr.Error()
}
if err := stream.Send(&botlinkv1.FromBot{Msg: &botlinkv1.FromBot_Ack{Ack: ack}}); err != nil {
return err
}
case <-healthTick:
// Snapshot without resetting; only commit (clear the deltas) after a successful send, so a
// failed flush loses nothing and the counts ride the next connection.
hb := c.cfg.Health.Snapshot()
if err := stream.Send(&botlinkv1.FromBot{Msg: &botlinkv1.FromBot_Health{Health: hb}}); err != nil {
return err
}
c.cfg.Health.Commit(hb)
}
}
}