feat(bot): report Telegram bot Bot API health to the gateway over the bot-link
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 12s
CI / integration (pull_request) Successful in 26s
CI / ui (pull_request) Successful in 1m11s
CI / conformance (pull_request) Successful in 10s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m56s
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 12s
CI / integration (pull_request) Successful in 26s
CI / ui (pull_request) Successful in 1m11s
CI / conformance (pull_request) Successful in 10s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m56s
The bot runs on its own host and exports no telemetry (otelcol is unreachable
from there), so it was a monitoring blind spot. Observe its Bot API health
centrally by wrapping the HTTP client — one place, no per-call-site
instrumentation — and relay it up the existing bot-link as a periodic Health
message the gateway turns into its own metrics.
- proto: add Health (delta connect / api / 429 counters + a last-ok stamp) to
the FromBot oneof (additive, backward-compatible).
- bot: platform/telegram/internal/health wraps the Bot API HTTP client — it
stamps liveness on any 2xx (so the getUpdates long-poll keeps it fresh even
when idle), classifies transport/5xx failures (getUpdates vs other) and 429s,
and honours a 429's Retry-After (bounded) so the bot backs off; a 4xx other
than 429 is a normal per-request outcome and is not counted.
- bot-link client: flush the reporter as a Health message every 30s over a
single-sender loop (a gRPC stream forbids concurrent Send).
- gateway: fold each report into bot_tg_errors_total{kind} and the
bot_tg_last_ok_unix liveness gauge.
- grafana: alerts (bot disconnected, bot not reaching the Bot API, sustained
429s) routed to the operator email — which does not go through the bot — plus
a Telegram-bot dashboard.
- docs (ARCHITECTURE, compose comments); unit tests (observer classification,
Retry-After, snapshot/commit, hub last-ok monotonicity).
This commit is contained in:
@@ -11,6 +11,7 @@ import (
|
||||
"google.golang.org/grpc/keepalive"
|
||||
|
||||
botlinkv1 "scrabble/pkg/proto/botlink/v1"
|
||||
"scrabble/platform/telegram/internal/health"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -34,6 +35,11 @@ type ClientConfig struct {
|
||||
Creds credentials.TransportCredentials
|
||||
// ReconnectDelay is the pause before re-dialing after the stream ends.
|
||||
ReconnectDelay time.Duration
|
||||
// Health, when set with a positive HealthInterval, is flushed to the gateway as a botlink Health
|
||||
// message every HealthInterval while the stream is open. Nil disables health reporting.
|
||||
Health *health.Reporter
|
||||
// HealthInterval is the cadence of the Health flush; 0 disables it.
|
||||
HealthInterval time.Duration
|
||||
}
|
||||
|
||||
// Client maintains the long-lived bot-link to the gateway, executing the commands
|
||||
@@ -140,22 +146,59 @@ func (c *Client) serve(ctx context.Context, client botlinkv1.BotLinkClient) erro
|
||||
}
|
||||
c.log.Info("bot-link connected", zap.String("gateway", c.cfg.GatewayAddr), zap.Bool("owns_updates", c.cfg.OwnsUpdates))
|
||||
|
||||
// One goroutine owns stream.Recv, feeding commands to the loop below; the loop owns every
|
||||
// stream.Send (the Acks and the periodic Health) — a gRPC stream forbids concurrent Send.
|
||||
rctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
cmds := make(chan *botlinkv1.Command)
|
||||
recvErr := make(chan error, 1)
|
||||
go func() {
|
||||
for {
|
||||
msg, err := stream.Recv()
|
||||
if err != nil {
|
||||
recvErr <- err
|
||||
return
|
||||
}
|
||||
if cmd := msg.GetCommand(); cmd != nil {
|
||||
select {
|
||||
case cmds <- cmd:
|
||||
case <-rctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
var healthTick <-chan time.Time
|
||||
if c.cfg.Health != nil && c.cfg.HealthInterval > 0 {
|
||||
t := time.NewTicker(c.cfg.HealthInterval)
|
||||
defer t.Stop()
|
||||
healthTick = t.C
|
||||
}
|
||||
|
||||
for {
|
||||
msg, err := stream.Recv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cmd := msg.GetCommand()
|
||||
if cmd == nil {
|
||||
continue
|
||||
}
|
||||
delivered, result, herr := c.exec.Handle(ctx, cmd)
|
||||
ack := &botlinkv1.Ack{CommandId: cmd.GetCommandId(), Delivered: delivered, Result: result}
|
||||
if herr != nil {
|
||||
ack.Error = herr.Error()
|
||||
}
|
||||
if err := stream.Send(&botlinkv1.FromBot{Msg: &botlinkv1.FromBot_Ack{Ack: ack}}); err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case err := <-recvErr:
|
||||
return err
|
||||
case cmd := <-cmds:
|
||||
delivered, result, herr := c.exec.Handle(ctx, cmd)
|
||||
ack := &botlinkv1.Ack{CommandId: cmd.GetCommandId(), Delivered: delivered, Result: result}
|
||||
if herr != nil {
|
||||
ack.Error = herr.Error()
|
||||
}
|
||||
if err := stream.Send(&botlinkv1.FromBot{Msg: &botlinkv1.FromBot_Ack{Ack: ack}}); err != nil {
|
||||
return err
|
||||
}
|
||||
case <-healthTick:
|
||||
// Snapshot without resetting; only commit (clear the deltas) after a successful send, so a
|
||||
// failed flush loses nothing and the counts ride the next connection.
|
||||
hb := c.cfg.Health.Snapshot()
|
||||
if err := stream.Send(&botlinkv1.FromBot{Msg: &botlinkv1.FromBot_Health{Health: hb}}); err != nil {
|
||||
return err
|
||||
}
|
||||
c.cfg.Health.Commit(hb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user