diff --git a/backend/internal/payments/catalog_admin.go b/backend/internal/payments/catalog_admin.go index 401ec98..e2d351c 100644 --- a/backend/internal/payments/catalog_admin.go +++ b/backend/internal/payments/catalog_admin.go @@ -1,9 +1,12 @@ package payments import ( + "cmp" "context" "errors" "fmt" + "math" + "slices" "strings" "github.com/google/uuid" @@ -147,6 +150,65 @@ func (s *Service) AdminCatalog(ctx context.Context) ([]AdminProduct, error) { return s.store.adminCatalog(ctx) } +// SortAdminCatalog orders an admin catalog list in place the same way the public offer lists products +// ([projectOfferPricing]): chip packs first, ascending by rouble price; then chip-priced values, +// grouped (hints only, no-ads only, no-ads + hints, tournament) and ascending by chip price within a +// group. It is stable, so equal-keyed products keep their incoming order, and it keys archived +// products the same as active ones (the admin list shows both). +func SortAdminCatalog(products []AdminProduct) { + slices.SortStableFunc(products, func(a, b AdminProduct) int { + if pa, pb := adminIsPack(a), adminIsPack(b); pa != pb { + if pa { + return -1 // packs (sales) before values (chip exchange) + } + return 1 + } else if pa { + return cmp.Compare(adminPriceAmount(a, string(SourceDirect), CurrencyRUB), adminPriceAmount(b, string(SourceDirect), CurrencyRUB)) + } + if d := cmp.Compare(adminValueGroup(a), adminValueGroup(b)); d != 0 { + return d + } + return cmp.Compare(adminPriceAmount(a, "", CurrencyChip), adminPriceAmount(b, "", CurrencyChip)) + }) +} + +// adminIsPack reports whether the product is a chip pack (it carries the chips atom). +func adminIsPack(p AdminProduct) bool { + for _, a := range p.Atoms { + if a.Atom == atomChips { + return true + } + } + return false +} + +// adminValueGroup ranks a chip-priced value into the shared listing groups (see [valueGroup]). +func adminValueGroup(p AdminProduct) int { + hasHints, hasNoAds, hasTournament := false, false, false + for _, a := range p.Atoms { + switch a.Atom { + case "hints": + hasHints = true + case "noads_days": + hasNoAds = true + case "tournament": + hasTournament = true + } + } + return valueGroup(hasHints, hasNoAds, hasTournament) +} + +// adminPriceAmount returns the minor-unit amount of the product's price for the method and currency, +// or math.MaxInt64 when it carries no such price (so a misconfigured product sorts last, not first). +func adminPriceAmount(p AdminProduct, method string, cur Currency) int64 { + for _, pr := range p.Prices { + if pr.Method == method && pr.Currency == cur { + return pr.Amount + } + } + return math.MaxInt64 +} + // CreateProduct validates and inserts a new product with its atoms and prices, returning its id. A // product created active must satisfy the sellable shape. func (s *Service) CreateProduct(ctx context.Context, in ProductInput, active bool) (uuid.UUID, error) { diff --git a/backend/internal/payments/catalog_test.go b/backend/internal/payments/catalog_test.go index a243a98..132fa9a 100644 --- a/backend/internal/payments/catalog_test.go +++ b/backend/internal/payments/catalog_test.go @@ -134,3 +134,47 @@ func TestProjectCatalog_Empty(t *testing.T) { t.Errorf("empty catalog projected %d products, want 0", len(got.Products)) } } + +// TestSortAdminCatalog checks the admin list is ordered like the public offer: chip packs first, +// ascending by rouble price; then values, grouped (hints only -> no-ads only -> no-ads + hints) and +// ascending by chip price within a group. Stable and applied to active + archived alike. +func TestSortAdminCatalog(t *testing.T) { + pack := func(title string, rub int64) AdminProduct { + return AdminProduct{ + Title: title, + Atoms: []AtomLine{{Atom: atomChips, Quantity: 1}}, + Prices: []PriceLine{{Method: string(SourceDirect), Currency: CurrencyRUB, Amount: rub}}, + } + } + value := func(title string, chips int64, atoms ...string) AdminProduct { + p := AdminProduct{Title: title, Prices: []PriceLine{{Currency: CurrencyChip, Amount: chips}}} + for _, a := range atoms { + p.Atoms = append(p.Atoms, AtomLine{Atom: a, Quantity: 1}) + } + return p + } + products := []AdminProduct{ + pack("packDear", 30000), + value("bundle", 500, "hints", "noads_days"), + pack("packCheap", 10000), + value("adsOnly", 150, "noads_days"), + value("hintsBig", 200, "hints"), + value("hintsSmall", 50, "hints"), + } + SortAdminCatalog(products) + want := []string{"packCheap", "packDear", "hintsSmall", "hintsBig", "adsOnly", "bundle"} + for i, p := range products { + if p.Title != want[i] { + t.Fatalf("order[%d] = %q, want %q (full: %v)", i, p.Title, want[i], titles(products)) + } + } +} + +// titles extracts the product titles for a failure message. +func titles(products []AdminProduct) []string { + out := make([]string, len(products)) + for i, p := range products { + out[i] = p.Title + } + return out +} diff --git a/backend/internal/payments/offer.go b/backend/internal/payments/offer.go index 0fc4cc0..f4c6392 100644 --- a/backend/internal/payments/offer.go +++ b/backend/internal/payments/offer.go @@ -109,11 +109,7 @@ func projectOfferPricing(entries []catalogEntry) string { return strings.TrimRight(b.String(), "\n") } -// offerValueGroup ranks a chip-priced value into the offer's usage groups, in listing order: hints -// only (0), no-ads only (1), no-ads + hints (2), then anything carrying the tournament atom (3). -// Tournament products are not sellable yet (validateProduct forbids an active one), so group 3 is -// empty today; the rank reserves their place for when the tournament economy lands. A value with no -// recognised benefit atom sorts after the known groups (defensive — the catalog shape forbids it). +// offerValueGroup ranks a chip-priced value into the usage groups (see [valueGroup]). func offerValueGroup(e catalogEntry) int { hasHints, hasNoAds, hasTournament := false, false, false for _, a := range e.atoms { @@ -126,6 +122,16 @@ func offerValueGroup(e catalogEntry) int { hasTournament = true } } + return valueGroup(hasHints, hasNoAds, hasTournament) +} + +// valueGroup ranks a chip-priced value into the listing groups shared by the public offer and the +// admin catalog, in listing order: hints only (0), no-ads only (1), no-ads + hints (2), then anything +// carrying the tournament atom (3). Tournament products are not sellable yet (validateProduct forbids +// an active one), so group 3 is empty today; the rank reserves their place for when the tournament +// economy lands. A value with no recognised benefit atom sorts after the known groups (defensive — +// the catalog shape forbids it). +func valueGroup(hasHints, hasNoAds, hasTournament bool) int { switch { case hasTournament: return 3 diff --git a/backend/internal/server/handlers_admin_catalog.go b/backend/internal/server/handlers_admin_catalog.go index aca135b..9912d08 100644 --- a/backend/internal/server/handlers_admin_catalog.go +++ b/backend/internal/server/handlers_admin_catalog.go @@ -41,6 +41,9 @@ func (s *Server) consoleCatalog(c *gin.Context) { s.consoleError(c, err) return } + // Order the list like the public offer: sales (chip packs) first, then the chip-exchange values, + // grouped and price-sorted the same way, so the console mirrors what a buyer sees. + payments.SortAdminCatalog(products) var view adminconsole.CatalogView for _, p := range products { view.Products = append(view.Products, catalogRow(p)) diff --git a/deploy/docker-compose.bot.yml b/deploy/docker-compose.bot.yml index e3e9ca9..2231aba 100644 --- a/deploy/docker-compose.bot.yml +++ b/deploy/docker-compose.bot.yml @@ -3,8 +3,10 @@ # docker compose -f docker-compose.bot.yml up -d # # The bot egresses to the Bot API directly (no VPN sidecar) and dials the main host's -# published bot-link :9443 over mTLS. It exports no telemetry — otelcol lives on the -# main host and is unreachable from here — so observe it via `docker logs` on this host. +# published bot-link :9443 over mTLS. It exports no OTLP telemetry — otelcol lives on the +# main host and is unreachable from here — but it reports its Bot API health up the bot-link, +# which the gateway turns into metrics + alerts on the main host (docs/ARCHITECTURE.md); `docker +# logs` on this host is the local detail view. # Values come from the prod-deploy workflow (PROD_ secrets/variables); BOT_IMAGE is the # pushed registry tag and BOTLINK_GATEWAY_ADDR is the main host's :9443. name: scrabble-bot @@ -50,7 +52,8 @@ services: TELEGRAM_BOTLINK_TLS_CA: /certs/ca.crt TELEGRAM_LOG_LEVEL: ${LOG_LEVEL:-info} TELEGRAM_SERVICE_NAME: scrabble-telegram-bot - # No telemetry export: otelcol is on the main host, unreachable from here. + # No OTLP export: otelcol is on the main host, unreachable from here. Bot API health rides the + # bot-link instead (the gateway exposes it as metrics); see docs/ARCHITECTURE.md. TELEGRAM_OTEL_TRACES_EXPORTER: none TELEGRAM_OTEL_METRICS_EXPORTER: none GOMAXPROCS: "1" diff --git a/deploy/grafana/dashboards/bot.json b/deploy/grafana/dashboards/bot.json new file mode 100644 index 0000000..a8b4503 --- /dev/null +++ b/deploy/grafana/dashboards/bot.json @@ -0,0 +1,45 @@ +{ + "uid": "scrabble-bot", + "title": "Scrabble — Telegram bot", + "tags": ["scrabble"], + "timezone": "", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { "from": "now-6h", "to": "now" }, + "panels": [ + { + "type": "stat", + "title": "Bot connected", + "description": "botlink_connected_bots: bots currently holding the gateway bot-link (1 = healthy).", + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "max(botlink_connected_bots)" }] + }, + { + "type": "stat", + "title": "Last Bot API OK (age)", + "description": "Seconds since the bot's most recent successful Bot API call. Grows unbounded if the bot wedges.", + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "time() - max(bot_tg_last_ok_unix)" }] + }, + { + "type": "timeseries", + "title": "Bot API errors/s by kind", + "description": "bot_tg_errors_total by kind: connect (getUpdates), api (other sends), rate_limited (429). 429 should stay ~0.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "sum by (kind) (rate(bot_tg_errors_total[5m]))", "legendFormat": "{{kind}}" }] + }, + { + "type": "timeseries", + "title": "Bot-link commands/s by result", + "description": "botlink_commands_total by result: delivered / not_delivered / dropped / error. Sustained not_delivered or error means gateway sends are failing to reach Telegram.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "sum by (result) (rate(botlink_commands_total[5m]))", "legendFormat": "{{result}}" }] + } + ] +} diff --git a/deploy/grafana/provisioning/alerting/rules.yaml b/deploy/grafana/provisioning/alerting/rules.yaml index 9657c23..fff25b2 100644 --- a/deploy/grafana/provisioning/alerting/rules.yaml +++ b/deploy/grafana/provisioning/alerting/rules.yaml @@ -277,3 +277,91 @@ groups: - evaluator: { type: gt, params: [1800] } labels: { severity: critical } annotations: { summary: 'No WAL archived for over 30 minutes while pg_wal keeps growing — archiving is genuinely stalled; the PITR recovery point is frozen and pg_wal will fill the disk. Run `docker exec scrabble-postgres pgbackrest --stanza=scrabble --pg1-user=scrabble check`.' } + + # Remote Telegram bot health. The bot runs on its own host and exports no telemetry of its own; the + # gateway observes it through the bot-link (botlink_connected_bots) and the health it reports over + # that stream (bot_tg_*). All absent/NaN-safe: before a bot ever connects the metrics are absent, so + # noDataState OK keeps them quiet on a fresh contour. The alert email does NOT go through the bot, so + # a bot-down alert is deliverable. + - orgId: 1 + name: scrabble-bot + folder: Alerts + interval: 1m + rules: + - uid: bot_disconnected + title: Telegram bot disconnected + condition: C + for: 5m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: { refId: A, expr: botlink_connected_bots, instant: true } + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: lt, params: [1] } + labels: { severity: critical } + annotations: { summary: 'No Telegram bot is connected to the gateway bot-link — out-of-app push and admin sends are down. Check the bot host.' } + + # Positive liveness: a bot is connected but its last successful Bot API call is over 5 minutes + # old — the getUpdates long-poll returns every ~minute even when idle, so a stale stamp means the + # bot is wedged (no errors, no traffic). Guarded by "and connected" so a mere disconnect (owned by + # bot_disconnected) does not double-fire. + - uid: bot_tg_stale + title: Telegram bot not reaching the Bot API + condition: C + for: 5m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 600, to: 0 } + datasourceUid: prometheus + model: + refId: A + expr: (time() - bot_tg_last_ok_unix) and on() (botlink_connected_bots >= 1) + instant: true + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: gt, params: [300] } + labels: { severity: critical } + annotations: { summary: 'A connected Telegram bot has not reached the Bot API in over 5 minutes — the update poll is wedged. Check the bot host and Telegram reachability.' } + + # 429s should be ~never once the bot honours Retry-After; any sustained rate-limiting is a symptom + # (a send loop, a misbehaving path) worth investigating. + - uid: bot_tg_rate_limited + title: Telegram bot rate-limited (429) + condition: C + for: 5m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 900, to: 0 } + datasourceUid: prometheus + model: + refId: A + expr: increase(bot_tg_errors_total{kind="rate_limited"}[15m]) + instant: true + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: gt, params: [0] } + labels: { severity: warning } + annotations: { summary: 'The Telegram bot is being rate-limited (HTTP 429). It should honour Retry-After, so sustained 429s point to a send loop or a hot path.' } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index fbd26d4..7fc916f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1015,6 +1015,20 @@ answering `/start` with a URL button into the **main** bot's Mini App (`?startap button would sign initData with the promo token); it is self-contained — no bot-link, no gateway. Session-revocation events and cursor-based stream resume stay deferred (single-instance MVP). +Because the bot **exports no telemetry of its own** (the OTel collector is on the main host, +unreachable from the bot host), it reports its **Bot API health** up the same stream: a periodic +`Health` message (`platform/telegram/internal/health`) carries delta counts of connect failures (the +getUpdates long-poll), other API errors and 429s, plus the wall-clock second of its last successful +Bot API call. The bot observes these **centrally by wrapping the Bot API HTTP client** — one place, +no per-call-site instrumentation — and there also honours a 429's `Retry-After` (bounded) so it backs +off rather than hammering (a 429 should therefore stay ~0). The gateway folds each report into its +own metrics (`bot_tg_errors_total{kind}`, the `bot_tg_last_ok_unix` liveness gauge). The remote bot +is thus monitored from the main host's Grafana with three layered signals: the **connection gauge** +(`botlink_connected_bots`) catches a link/host/process outage, the **liveness gauge** catches a +silently wedged bot (its stamp stays fresh even when idle, since getUpdates returns every poll), and +**`botlink_commands_total{result}`** catches gateway sends failing to reach Telegram. Alerts route to +the operator email, which does not depend on the bot. + A separate **advertising-banner** channel feeds the client's one-line strip (UI_DESIGN.md), server-driven by `internal/ads`. An operator manages **campaigns** (each one placement order) in the admin console (`/_gm/banners`): a campaign has a show **weight** (integer percent 1..100), an @@ -1073,7 +1087,9 @@ link — misses the event; while an add-email confirmation is pending the client `OTEL_EXPORTER_OTLP_*` environment) exports to a collector. The Postgres pool is instrumented with otelsql and `otelgrpc` traces the backend↔gateway push stream and the gateway↔validator and bot-link calls; the gateway also exports - `botlink_connected_bots` and `botlink_commands_total` (by result) for the bot-link. The OTLP **Collector** (OTLP/gRPC → Prometheus + `botlink_connected_bots` and `botlink_commands_total` (by result) for the bot-link, plus the remote + bot's own Bot API health it relays over the stream — `bot_tg_errors_total` (by kind) and the + `bot_tg_last_ok_unix` liveness gauge (see the bot-link section). The OTLP **Collector** (OTLP/gRPC → Prometheus metrics + Tempo traces), **Prometheus** (15d), **Tempo** (72h) and **Grafana** (provisioned datasources + dashboards, behind the caddy `/_gm/grafana` Basic-Auth) are stood up with the deploy (`deploy/`); the default exporter stays diff --git a/gateway/internal/botlink/hub.go b/gateway/internal/botlink/hub.go index 91e9b3a..7851026 100644 --- a/gateway/internal/botlink/hub.go +++ b/gateway/internal/botlink/hub.go @@ -76,6 +76,11 @@ type Hub struct { connected metric.Int64UpDownCounter commands metric.Int64Counter + // tgErrors counts the remote bot's Bot API failures it reports over the stream, by kind + // (connect / api / rate_limited). lastOK holds the wall-clock second of the bot's most recent + // successful Bot API call, read by the bot_tg_last_ok_unix observable gauge for a staleness alert. + tgErrors metric.Int64Counter + lastOK atomic.Int64 } // link is one connected bot's outbound queue and identity. @@ -103,6 +108,19 @@ func NewHub(log *zap.Logger, meter metric.Meter, resolve EligibilityResolver) *H metric.WithDescription("Number of Telegram bots currently connected to the gateway bot-link.")) h.commands, _ = meter.Int64Counter("botlink_commands_total", metric.WithDescription("Bot-link send commands by result (delivered, not_delivered, dropped, error).")) + h.tgErrors, _ = meter.Int64Counter("bot_tg_errors_total", + metric.WithDescription("Telegram Bot API failures the remote bot reports over the bot-link, by kind (connect, api, rate_limited).")) + // The bot's last successful Bot API second, reported over the stream. An observable gauge + // reads the atomic on each collection; it observes nothing until a bot has reported, so a + // never-connected gateway shows no data (the connected-bots alert covers that case). + _, _ = meter.Int64ObservableGauge("bot_tg_last_ok_unix", + metric.WithDescription("Unix second of the remote bot's most recent successful Bot API call (0/absent until reported)."), + metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error { + if v := h.lastOK.Load(); v > 0 { + o.Observe(v) + } + return nil + })) } return h } @@ -149,6 +167,36 @@ func (h *Hub) Link(stream grpc.BidiStreamingServer[botlinkv1.FromBot, botlinkv1. if ack := msg.GetAck(); ack != nil { h.resolve(ack) } + if hb := msg.GetHealth(); hb != nil { + h.recordHealth(hb) + } + } +} + +// recordHealth folds one bot Health report into the gateway's cumulative Bot API metrics: the three +// delta counters are added under their kind label, and the last-ok stamp (an absolute wall-clock +// second) advances the liveness gauge (monotonically — a stale reordered report cannot rewind it). +func (h *Hub) recordHealth(hb *botlinkv1.Health) { + if h.tgErrors != nil { + ctx := context.Background() + if v := hb.GetConnectFailures(); v > 0 { + h.tgErrors.Add(ctx, int64(v), metric.WithAttributes(attribute.String("kind", "connect"))) + } + if v := hb.GetApiErrors(); v > 0 { + h.tgErrors.Add(ctx, int64(v), metric.WithAttributes(attribute.String("kind", "api"))) + } + if v := hb.GetRateLimited(); v > 0 { + h.tgErrors.Add(ctx, int64(v), metric.WithAttributes(attribute.String("kind", "rate_limited"))) + } + } + for { + cur := h.lastOK.Load() + if hb.GetLastOkUnix() <= cur { + break + } + if h.lastOK.CompareAndSwap(cur, hb.GetLastOkUnix()) { + break + } } } diff --git a/gateway/internal/botlink/hub_test.go b/gateway/internal/botlink/hub_test.go index eae28fc..5fb4150 100644 --- a/gateway/internal/botlink/hub_test.go +++ b/gateway/internal/botlink/hub_test.go @@ -233,3 +233,21 @@ func TestRelayServerNoBot(t *testing.T) { t.Fatal("expected an error with no bot connected") } } + +// TestRecordHealthLastOKMonotonic checks a bot Health report advances the last-ok liveness stamp but +// a stale or reordered report (an earlier second) never rewinds it. +func TestRecordHealthLastOKMonotonic(t *testing.T) { + hub := NewHub(nil, nil, nil) + hub.recordHealth(&botlinkv1.Health{LastOkUnix: 100}) + if got := hub.lastOK.Load(); got != 100 { + t.Fatalf("lastOK = %d, want 100", got) + } + hub.recordHealth(&botlinkv1.Health{LastOkUnix: 90}) // reordered/stale — must not rewind + if got := hub.lastOK.Load(); got != 100 { + t.Errorf("lastOK rewound to %d, want 100", got) + } + hub.recordHealth(&botlinkv1.Health{LastOkUnix: 150}) + if got := hub.lastOK.Load(); got != 150 { + t.Errorf("lastOK = %d, want 150", got) + } +} diff --git a/pkg/proto/botlink/v1/botlink.pb.go b/pkg/proto/botlink/v1/botlink.pb.go index 61d1c52..70b91a2 100644 --- a/pkg/proto/botlink/v1/botlink.pb.go +++ b/pkg/proto/botlink/v1/botlink.pb.go @@ -30,13 +30,14 @@ const ( ) // FromBot is a message the bot sends to the gateway: the opening Hello, then one -// Ack per Command. +// Ack per Command and a periodic Health report. type FromBot struct { state protoimpl.MessageState `protogen:"open.v1"` // Types that are valid to be assigned to Msg: // // *FromBot_Hello // *FromBot_Ack + // *FromBot_Health Msg isFromBot_Msg `protobuf_oneof:"msg"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -97,6 +98,15 @@ func (x *FromBot) GetAck() *Ack { return nil } +func (x *FromBot) GetHealth() *Health { + if x != nil { + if x, ok := x.Msg.(*FromBot_Health); ok { + return x.Health + } + } + return nil +} + type isFromBot_Msg interface { isFromBot_Msg() } @@ -109,10 +119,92 @@ type FromBot_Ack struct { Ack *Ack `protobuf:"bytes,2,opt,name=ack,proto3,oneof"` } +type FromBot_Health struct { + Health *Health `protobuf:"bytes,3,opt,name=health,proto3,oneof"` +} + func (*FromBot_Hello) isFromBot_Msg() {} func (*FromBot_Ack) isFromBot_Msg() {} +func (*FromBot_Health) isFromBot_Msg() {} + +// Health is a periodic report the bot pushes up the stream so the gateway can expose the remote +// bot's Bot-API health as its own metrics — the bot exports no telemetry of its own (otelcol lives +// on the main host, unreachable from the bot), so the bot-link is its only channel out. The three +// counters are DELTAS since the previous Health: the gateway adds them to cumulative counters, so a +// report lost across a reconnect only undercounts, never corrupts. last_ok_unix is the wall-clock +// second of the bot's most recent successful Bot API response (any call, including the getUpdates +// long-poll that returns every poll cycle even when idle) — an absolute liveness stamp the gateway +// surfaces as a gauge, so a silently stalled bot (no errors, no traffic) is still caught. +type Health struct { + state protoimpl.MessageState `protogen:"open.v1"` + ConnectFailures uint64 `protobuf:"varint,1,opt,name=connect_failures,json=connectFailures,proto3" json:"connect_failures,omitempty"` // getUpdates long-poll failures (transport / 5xx) since the last report + ApiErrors uint64 `protobuf:"varint,2,opt,name=api_errors,json=apiErrors,proto3" json:"api_errors,omitempty"` // other Bot API failures (transport / 5xx) since the last report + RateLimited uint64 `protobuf:"varint,3,opt,name=rate_limited,json=rateLimited,proto3" json:"rate_limited,omitempty"` // Bot API 429 responses since the last report (should stay ~0) + LastOkUnix int64 `protobuf:"varint,4,opt,name=last_ok_unix,json=lastOkUnix,proto3" json:"last_ok_unix,omitempty"` // unix seconds of the last successful Bot API response (0 = none yet) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Health) Reset() { + *x = Health{} + mi := &file_botlink_v1_botlink_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Health) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Health) ProtoMessage() {} + +func (x *Health) ProtoReflect() protoreflect.Message { + mi := &file_botlink_v1_botlink_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Health.ProtoReflect.Descriptor instead. +func (*Health) Descriptor() ([]byte, []int) { + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{1} +} + +func (x *Health) GetConnectFailures() uint64 { + if x != nil { + return x.ConnectFailures + } + return 0 +} + +func (x *Health) GetApiErrors() uint64 { + if x != nil { + return x.ApiErrors + } + return 0 +} + +func (x *Health) GetRateLimited() uint64 { + if x != nil { + return x.RateLimited + } + return 0 +} + +func (x *Health) GetLastOkUnix() int64 { + if x != nil { + return x.LastOkUnix + } + return 0 +} + // ToBot is a message the gateway sends to the bot. Only Command is carried today. type ToBot struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -123,7 +215,7 @@ type ToBot struct { func (x *ToBot) Reset() { *x = ToBot{} - mi := &file_botlink_v1_botlink_proto_msgTypes[1] + mi := &file_botlink_v1_botlink_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -135,7 +227,7 @@ func (x *ToBot) String() string { func (*ToBot) ProtoMessage() {} func (x *ToBot) ProtoReflect() protoreflect.Message { - mi := &file_botlink_v1_botlink_proto_msgTypes[1] + mi := &file_botlink_v1_botlink_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -148,7 +240,7 @@ func (x *ToBot) ProtoReflect() protoreflect.Message { // Deprecated: Use ToBot.ProtoReflect.Descriptor instead. func (*ToBot) Descriptor() ([]byte, []int) { - return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{1} + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{2} } func (x *ToBot) GetCommand() *Command { @@ -171,7 +263,7 @@ type Hello struct { func (x *Hello) Reset() { *x = Hello{} - mi := &file_botlink_v1_botlink_proto_msgTypes[2] + mi := &file_botlink_v1_botlink_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -183,7 +275,7 @@ func (x *Hello) String() string { func (*Hello) ProtoMessage() {} func (x *Hello) ProtoReflect() protoreflect.Message { - mi := &file_botlink_v1_botlink_proto_msgTypes[2] + mi := &file_botlink_v1_botlink_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -196,7 +288,7 @@ func (x *Hello) ProtoReflect() protoreflect.Message { // Deprecated: Use Hello.ProtoReflect.Descriptor instead. func (*Hello) Descriptor() ([]byte, []int) { - return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{2} + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{3} } func (x *Hello) GetInstanceId() string { @@ -233,7 +325,7 @@ type Command struct { func (x *Command) Reset() { *x = Command{} - mi := &file_botlink_v1_botlink_proto_msgTypes[3] + mi := &file_botlink_v1_botlink_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -245,7 +337,7 @@ func (x *Command) String() string { func (*Command) ProtoMessage() {} func (x *Command) ProtoReflect() protoreflect.Message { - mi := &file_botlink_v1_botlink_proto_msgTypes[3] + mi := &file_botlink_v1_botlink_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -258,7 +350,7 @@ func (x *Command) ProtoReflect() protoreflect.Message { // Deprecated: Use Command.ProtoReflect.Descriptor instead. func (*Command) Descriptor() ([]byte, []int) { - return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{3} + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{4} } func (x *Command) GetCommandId() string { @@ -372,7 +464,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_botlink_v1_botlink_proto_msgTypes[4] + mi := &file_botlink_v1_botlink_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -384,7 +476,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_botlink_v1_botlink_proto_msgTypes[4] + mi := &file_botlink_v1_botlink_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -397,7 +489,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{4} + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{5} } func (x *Ack) GetCommandId() string { @@ -445,7 +537,7 @@ type ChatGateCommand struct { func (x *ChatGateCommand) Reset() { *x = ChatGateCommand{} - mi := &file_botlink_v1_botlink_proto_msgTypes[5] + mi := &file_botlink_v1_botlink_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -457,7 +549,7 @@ func (x *ChatGateCommand) String() string { func (*ChatGateCommand) ProtoMessage() {} func (x *ChatGateCommand) ProtoReflect() protoreflect.Message { - mi := &file_botlink_v1_botlink_proto_msgTypes[5] + mi := &file_botlink_v1_botlink_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -470,7 +562,7 @@ func (x *ChatGateCommand) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatGateCommand.ProtoReflect.Descriptor instead. func (*ChatGateCommand) Descriptor() ([]byte, []int) { - return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{5} + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{6} } func (x *ChatGateCommand) GetExternalId() string { @@ -498,7 +590,7 @@ type ChatEligibilityRequest struct { func (x *ChatEligibilityRequest) Reset() { *x = ChatEligibilityRequest{} - mi := &file_botlink_v1_botlink_proto_msgTypes[6] + mi := &file_botlink_v1_botlink_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -510,7 +602,7 @@ func (x *ChatEligibilityRequest) String() string { func (*ChatEligibilityRequest) ProtoMessage() {} func (x *ChatEligibilityRequest) ProtoReflect() protoreflect.Message { - mi := &file_botlink_v1_botlink_proto_msgTypes[6] + mi := &file_botlink_v1_botlink_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -523,7 +615,7 @@ func (x *ChatEligibilityRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatEligibilityRequest.ProtoReflect.Descriptor instead. func (*ChatEligibilityRequest) Descriptor() ([]byte, []int) { - return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{6} + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{7} } func (x *ChatEligibilityRequest) GetExternalId() string { @@ -546,7 +638,7 @@ type ChatEligibilityResponse struct { func (x *ChatEligibilityResponse) Reset() { *x = ChatEligibilityResponse{} - mi := &file_botlink_v1_botlink_proto_msgTypes[7] + mi := &file_botlink_v1_botlink_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -558,7 +650,7 @@ func (x *ChatEligibilityResponse) String() string { func (*ChatEligibilityResponse) ProtoMessage() {} func (x *ChatEligibilityResponse) ProtoReflect() protoreflect.Message { - mi := &file_botlink_v1_botlink_proto_msgTypes[7] + mi := &file_botlink_v1_botlink_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -571,7 +663,7 @@ func (x *ChatEligibilityResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatEligibilityResponse.ProtoReflect.Descriptor instead. func (*ChatEligibilityResponse) Descriptor() ([]byte, []int) { - return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{7} + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{8} } func (x *ChatEligibilityResponse) GetRegistered() bool { @@ -605,7 +697,7 @@ type CreateInvoiceCommand struct { func (x *CreateInvoiceCommand) Reset() { *x = CreateInvoiceCommand{} - mi := &file_botlink_v1_botlink_proto_msgTypes[8] + mi := &file_botlink_v1_botlink_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -617,7 +709,7 @@ func (x *CreateInvoiceCommand) String() string { func (*CreateInvoiceCommand) ProtoMessage() {} func (x *CreateInvoiceCommand) ProtoReflect() protoreflect.Message { - mi := &file_botlink_v1_botlink_proto_msgTypes[8] + mi := &file_botlink_v1_botlink_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -630,7 +722,7 @@ func (x *CreateInvoiceCommand) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateInvoiceCommand.ProtoReflect.Descriptor instead. func (*CreateInvoiceCommand) Descriptor() ([]byte, []int) { - return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{8} + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{9} } func (x *CreateInvoiceCommand) GetTitle() string { @@ -674,7 +766,7 @@ type PreCheckoutRequest struct { func (x *PreCheckoutRequest) Reset() { *x = PreCheckoutRequest{} - mi := &file_botlink_v1_botlink_proto_msgTypes[9] + mi := &file_botlink_v1_botlink_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -686,7 +778,7 @@ func (x *PreCheckoutRequest) String() string { func (*PreCheckoutRequest) ProtoMessage() {} func (x *PreCheckoutRequest) ProtoReflect() protoreflect.Message { - mi := &file_botlink_v1_botlink_proto_msgTypes[9] + mi := &file_botlink_v1_botlink_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -699,7 +791,7 @@ func (x *PreCheckoutRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PreCheckoutRequest.ProtoReflect.Descriptor instead. func (*PreCheckoutRequest) Descriptor() ([]byte, []int) { - return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{9} + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{10} } func (x *PreCheckoutRequest) GetOrderId() string { @@ -735,7 +827,7 @@ type PreCheckoutResponse struct { func (x *PreCheckoutResponse) Reset() { *x = PreCheckoutResponse{} - mi := &file_botlink_v1_botlink_proto_msgTypes[10] + mi := &file_botlink_v1_botlink_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -747,7 +839,7 @@ func (x *PreCheckoutResponse) String() string { func (*PreCheckoutResponse) ProtoMessage() {} func (x *PreCheckoutResponse) ProtoReflect() protoreflect.Message { - mi := &file_botlink_v1_botlink_proto_msgTypes[10] + mi := &file_botlink_v1_botlink_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -760,7 +852,7 @@ func (x *PreCheckoutResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PreCheckoutResponse.ProtoReflect.Descriptor instead. func (*PreCheckoutResponse) Descriptor() ([]byte, []int) { - return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{10} + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{11} } func (x *PreCheckoutResponse) GetOk() bool { @@ -792,7 +884,7 @@ type ForwardPaymentRequest struct { func (x *ForwardPaymentRequest) Reset() { *x = ForwardPaymentRequest{} - mi := &file_botlink_v1_botlink_proto_msgTypes[11] + mi := &file_botlink_v1_botlink_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -804,7 +896,7 @@ func (x *ForwardPaymentRequest) String() string { func (*ForwardPaymentRequest) ProtoMessage() {} func (x *ForwardPaymentRequest) ProtoReflect() protoreflect.Message { - mi := &file_botlink_v1_botlink_proto_msgTypes[11] + mi := &file_botlink_v1_botlink_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -817,7 +909,7 @@ func (x *ForwardPaymentRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ForwardPaymentRequest.ProtoReflect.Descriptor instead. func (*ForwardPaymentRequest) Descriptor() ([]byte, []int) { - return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{11} + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{12} } func (x *ForwardPaymentRequest) GetOrderId() string { @@ -861,7 +953,7 @@ type ForwardPaymentResponse struct { func (x *ForwardPaymentResponse) Reset() { *x = ForwardPaymentResponse{} - mi := &file_botlink_v1_botlink_proto_msgTypes[12] + mi := &file_botlink_v1_botlink_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -873,7 +965,7 @@ func (x *ForwardPaymentResponse) String() string { func (*ForwardPaymentResponse) ProtoMessage() {} func (x *ForwardPaymentResponse) ProtoReflect() protoreflect.Message { - mi := &file_botlink_v1_botlink_proto_msgTypes[12] + mi := &file_botlink_v1_botlink_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -886,7 +978,7 @@ func (x *ForwardPaymentResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ForwardPaymentResponse.ProtoReflect.Descriptor instead. func (*ForwardPaymentResponse) Descriptor() ([]byte, []int) { - return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{12} + return file_botlink_v1_botlink_proto_rawDescGZIP(), []int{13} } func (x *ForwardPaymentResponse) GetCredited() bool { @@ -900,11 +992,19 @@ var File_botlink_v1_botlink_proto protoreflect.FileDescriptor const file_botlink_v1_botlink_proto_rawDesc = "" + "\n" + - "\x18botlink/v1/botlink.proto\x12\x13scrabble.botlink.v1\x1a\x1atelegram/v1/telegram.proto\"r\n" + + "\x18botlink/v1/botlink.proto\x12\x13scrabble.botlink.v1\x1a\x1atelegram/v1/telegram.proto\"\xa9\x01\n" + "\aFromBot\x122\n" + "\x05hello\x18\x01 \x01(\v2\x1a.scrabble.botlink.v1.HelloH\x00R\x05hello\x12,\n" + - "\x03ack\x18\x02 \x01(\v2\x18.scrabble.botlink.v1.AckH\x00R\x03ackB\x05\n" + - "\x03msg\"?\n" + + "\x03ack\x18\x02 \x01(\v2\x18.scrabble.botlink.v1.AckH\x00R\x03ack\x125\n" + + "\x06health\x18\x03 \x01(\v2\x1b.scrabble.botlink.v1.HealthH\x00R\x06healthB\x05\n" + + "\x03msg\"\x97\x01\n" + + "\x06Health\x12)\n" + + "\x10connect_failures\x18\x01 \x01(\x04R\x0fconnectFailures\x12\x1d\n" + + "\n" + + "api_errors\x18\x02 \x01(\x04R\tapiErrors\x12!\n" + + "\frate_limited\x18\x03 \x01(\x04R\vrateLimited\x12 \n" + + "\flast_ok_unix\x18\x04 \x01(\x03R\n" + + "lastOkUnix\"?\n" + "\x05ToBot\x126\n" + "\acommand\x18\x01 \x01(\v2\x1c.scrabble.botlink.v1.CommandR\acommand\"K\n" + "\x05Hello\x12\x1f\n" + @@ -976,47 +1076,49 @@ func file_botlink_v1_botlink_proto_rawDescGZIP() []byte { return file_botlink_v1_botlink_proto_rawDescData } -var file_botlink_v1_botlink_proto_msgTypes = make([]protoimpl.MessageInfo, 13) +var file_botlink_v1_botlink_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_botlink_v1_botlink_proto_goTypes = []any{ (*FromBot)(nil), // 0: scrabble.botlink.v1.FromBot - (*ToBot)(nil), // 1: scrabble.botlink.v1.ToBot - (*Hello)(nil), // 2: scrabble.botlink.v1.Hello - (*Command)(nil), // 3: scrabble.botlink.v1.Command - (*Ack)(nil), // 4: scrabble.botlink.v1.Ack - (*ChatGateCommand)(nil), // 5: scrabble.botlink.v1.ChatGateCommand - (*ChatEligibilityRequest)(nil), // 6: scrabble.botlink.v1.ChatEligibilityRequest - (*ChatEligibilityResponse)(nil), // 7: scrabble.botlink.v1.ChatEligibilityResponse - (*CreateInvoiceCommand)(nil), // 8: scrabble.botlink.v1.CreateInvoiceCommand - (*PreCheckoutRequest)(nil), // 9: scrabble.botlink.v1.PreCheckoutRequest - (*PreCheckoutResponse)(nil), // 10: scrabble.botlink.v1.PreCheckoutResponse - (*ForwardPaymentRequest)(nil), // 11: scrabble.botlink.v1.ForwardPaymentRequest - (*ForwardPaymentResponse)(nil), // 12: scrabble.botlink.v1.ForwardPaymentResponse - (*v1.NotifyRequest)(nil), // 13: scrabble.telegram.v1.NotifyRequest - (*v1.SendToUserRequest)(nil), // 14: scrabble.telegram.v1.SendToUserRequest - (*v1.SendToGameChannelRequest)(nil), // 15: scrabble.telegram.v1.SendToGameChannelRequest + (*Health)(nil), // 1: scrabble.botlink.v1.Health + (*ToBot)(nil), // 2: scrabble.botlink.v1.ToBot + (*Hello)(nil), // 3: scrabble.botlink.v1.Hello + (*Command)(nil), // 4: scrabble.botlink.v1.Command + (*Ack)(nil), // 5: scrabble.botlink.v1.Ack + (*ChatGateCommand)(nil), // 6: scrabble.botlink.v1.ChatGateCommand + (*ChatEligibilityRequest)(nil), // 7: scrabble.botlink.v1.ChatEligibilityRequest + (*ChatEligibilityResponse)(nil), // 8: scrabble.botlink.v1.ChatEligibilityResponse + (*CreateInvoiceCommand)(nil), // 9: scrabble.botlink.v1.CreateInvoiceCommand + (*PreCheckoutRequest)(nil), // 10: scrabble.botlink.v1.PreCheckoutRequest + (*PreCheckoutResponse)(nil), // 11: scrabble.botlink.v1.PreCheckoutResponse + (*ForwardPaymentRequest)(nil), // 12: scrabble.botlink.v1.ForwardPaymentRequest + (*ForwardPaymentResponse)(nil), // 13: scrabble.botlink.v1.ForwardPaymentResponse + (*v1.NotifyRequest)(nil), // 14: scrabble.telegram.v1.NotifyRequest + (*v1.SendToUserRequest)(nil), // 15: scrabble.telegram.v1.SendToUserRequest + (*v1.SendToGameChannelRequest)(nil), // 16: scrabble.telegram.v1.SendToGameChannelRequest } var file_botlink_v1_botlink_proto_depIdxs = []int32{ - 2, // 0: scrabble.botlink.v1.FromBot.hello:type_name -> scrabble.botlink.v1.Hello - 4, // 1: scrabble.botlink.v1.FromBot.ack:type_name -> scrabble.botlink.v1.Ack - 3, // 2: scrabble.botlink.v1.ToBot.command:type_name -> scrabble.botlink.v1.Command - 13, // 3: scrabble.botlink.v1.Command.notify:type_name -> scrabble.telegram.v1.NotifyRequest - 14, // 4: scrabble.botlink.v1.Command.send_to_user:type_name -> scrabble.telegram.v1.SendToUserRequest - 15, // 5: scrabble.botlink.v1.Command.send_to_channel:type_name -> scrabble.telegram.v1.SendToGameChannelRequest - 5, // 6: scrabble.botlink.v1.Command.chat_gate:type_name -> scrabble.botlink.v1.ChatGateCommand - 8, // 7: scrabble.botlink.v1.Command.create_invoice:type_name -> scrabble.botlink.v1.CreateInvoiceCommand - 0, // 8: scrabble.botlink.v1.BotLink.Link:input_type -> scrabble.botlink.v1.FromBot - 6, // 9: scrabble.botlink.v1.BotLink.ResolveChatEligibility:input_type -> scrabble.botlink.v1.ChatEligibilityRequest - 9, // 10: scrabble.botlink.v1.BotLink.ValidatePreCheckout:input_type -> scrabble.botlink.v1.PreCheckoutRequest - 11, // 11: scrabble.botlink.v1.BotLink.ForwardPayment:input_type -> scrabble.botlink.v1.ForwardPaymentRequest - 1, // 12: scrabble.botlink.v1.BotLink.Link:output_type -> scrabble.botlink.v1.ToBot - 7, // 13: scrabble.botlink.v1.BotLink.ResolveChatEligibility:output_type -> scrabble.botlink.v1.ChatEligibilityResponse - 10, // 14: scrabble.botlink.v1.BotLink.ValidatePreCheckout:output_type -> scrabble.botlink.v1.PreCheckoutResponse - 12, // 15: scrabble.botlink.v1.BotLink.ForwardPayment:output_type -> scrabble.botlink.v1.ForwardPaymentResponse - 12, // [12:16] is the sub-list for method output_type - 8, // [8:12] is the sub-list for method input_type - 8, // [8:8] is the sub-list for extension type_name - 8, // [8:8] is the sub-list for extension extendee - 0, // [0:8] is the sub-list for field type_name + 3, // 0: scrabble.botlink.v1.FromBot.hello:type_name -> scrabble.botlink.v1.Hello + 5, // 1: scrabble.botlink.v1.FromBot.ack:type_name -> scrabble.botlink.v1.Ack + 1, // 2: scrabble.botlink.v1.FromBot.health:type_name -> scrabble.botlink.v1.Health + 4, // 3: scrabble.botlink.v1.ToBot.command:type_name -> scrabble.botlink.v1.Command + 14, // 4: scrabble.botlink.v1.Command.notify:type_name -> scrabble.telegram.v1.NotifyRequest + 15, // 5: scrabble.botlink.v1.Command.send_to_user:type_name -> scrabble.telegram.v1.SendToUserRequest + 16, // 6: scrabble.botlink.v1.Command.send_to_channel:type_name -> scrabble.telegram.v1.SendToGameChannelRequest + 6, // 7: scrabble.botlink.v1.Command.chat_gate:type_name -> scrabble.botlink.v1.ChatGateCommand + 9, // 8: scrabble.botlink.v1.Command.create_invoice:type_name -> scrabble.botlink.v1.CreateInvoiceCommand + 0, // 9: scrabble.botlink.v1.BotLink.Link:input_type -> scrabble.botlink.v1.FromBot + 7, // 10: scrabble.botlink.v1.BotLink.ResolveChatEligibility:input_type -> scrabble.botlink.v1.ChatEligibilityRequest + 10, // 11: scrabble.botlink.v1.BotLink.ValidatePreCheckout:input_type -> scrabble.botlink.v1.PreCheckoutRequest + 12, // 12: scrabble.botlink.v1.BotLink.ForwardPayment:input_type -> scrabble.botlink.v1.ForwardPaymentRequest + 2, // 13: scrabble.botlink.v1.BotLink.Link:output_type -> scrabble.botlink.v1.ToBot + 8, // 14: scrabble.botlink.v1.BotLink.ResolveChatEligibility:output_type -> scrabble.botlink.v1.ChatEligibilityResponse + 11, // 15: scrabble.botlink.v1.BotLink.ValidatePreCheckout:output_type -> scrabble.botlink.v1.PreCheckoutResponse + 13, // 16: scrabble.botlink.v1.BotLink.ForwardPayment:output_type -> scrabble.botlink.v1.ForwardPaymentResponse + 13, // [13:17] is the sub-list for method output_type + 9, // [9:13] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_botlink_v1_botlink_proto_init() } @@ -1027,8 +1129,9 @@ func file_botlink_v1_botlink_proto_init() { file_botlink_v1_botlink_proto_msgTypes[0].OneofWrappers = []any{ (*FromBot_Hello)(nil), (*FromBot_Ack)(nil), + (*FromBot_Health)(nil), } - file_botlink_v1_botlink_proto_msgTypes[3].OneofWrappers = []any{ + file_botlink_v1_botlink_proto_msgTypes[4].OneofWrappers = []any{ (*Command_Notify)(nil), (*Command_SendToUser)(nil), (*Command_SendToChannel)(nil), @@ -1041,7 +1144,7 @@ func file_botlink_v1_botlink_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_botlink_v1_botlink_proto_rawDesc), len(file_botlink_v1_botlink_proto_rawDesc)), NumEnums: 0, - NumMessages: 13, + NumMessages: 14, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/proto/botlink/v1/botlink.proto b/pkg/proto/botlink/v1/botlink.proto index 3f37723..b08ddf4 100644 --- a/pkg/proto/botlink/v1/botlink.proto +++ b/pkg/proto/botlink/v1/botlink.proto @@ -48,14 +48,30 @@ service BotLink { } // FromBot is a message the bot sends to the gateway: the opening Hello, then one -// Ack per Command. +// Ack per Command and a periodic Health report. message FromBot { oneof msg { Hello hello = 1; Ack ack = 2; + Health health = 3; } } +// Health is a periodic report the bot pushes up the stream so the gateway can expose the remote +// bot's Bot-API health as its own metrics — the bot exports no telemetry of its own (otelcol lives +// on the main host, unreachable from the bot), so the bot-link is its only channel out. The three +// counters are DELTAS since the previous Health: the gateway adds them to cumulative counters, so a +// report lost across a reconnect only undercounts, never corrupts. last_ok_unix is the wall-clock +// second of the bot's most recent successful Bot API response (any call, including the getUpdates +// long-poll that returns every poll cycle even when idle) — an absolute liveness stamp the gateway +// surfaces as a gauge, so a silently stalled bot (no errors, no traffic) is still caught. +message Health { + uint64 connect_failures = 1; // getUpdates long-poll failures (transport / 5xx) since the last report + uint64 api_errors = 2; // other Bot API failures (transport / 5xx) since the last report + uint64 rate_limited = 3; // Bot API 429 responses since the last report (should stay ~0) + int64 last_ok_unix = 4; // unix seconds of the last successful Bot API response (0 = none yet) +} + // ToBot is a message the gateway sends to the bot. Only Command is carried today. message ToBot { Command command = 1; diff --git a/platform/telegram/cmd/bot/main.go b/platform/telegram/cmd/bot/main.go index 152fad7..51eb62d 100644 --- a/platform/telegram/cmd/bot/main.go +++ b/platform/telegram/cmd/bot/main.go @@ -23,6 +23,7 @@ import ( "scrabble/platform/telegram/internal/bot" "scrabble/platform/telegram/internal/botlink" "scrabble/platform/telegram/internal/config" + "scrabble/platform/telegram/internal/health" "scrabble/platform/telegram/internal/outbox" "scrabble/platform/telegram/internal/promobot" "scrabble/platform/telegram/internal/support" @@ -31,6 +32,10 @@ import ( // telemetryShutdownTimeout bounds the OpenTelemetry flush during process exit. const telemetryShutdownTimeout = 5 * time.Second +// healthReportInterval is how often the bot flushes its accumulated Bot API health to the gateway +// over the bot-link. It bounds how stale a metric can be, not how often the bot talks to Telegram. +const healthReportInterval = 30 * time.Second + func main() { cfg, err := config.LoadBot() if err != nil { @@ -81,6 +86,12 @@ func run(ctx context.Context, cfg config.BotConfig, logger *zap.Logger) error { } } + // The bot exports no telemetry of its own (otelcol is on the main host, unreachable from here), + // so its Bot API health is observed centrally and reported to the gateway over the bot-link, + // which turns the reports into metrics. Shared between the bot (which feeds it) and the bot-link + // client (which flushes it). + healthReporter := health.NewReporter() + b, err := bot.New(bot.Config{ Token: cfg.Token, APIBaseURL: cfg.APIBaseURL, @@ -92,6 +103,7 @@ func run(ctx context.Context, cfg config.BotConfig, logger *zap.Logger) error { SupportChatID: cfg.SupportChatID, SupportStore: supportStore, AcceptPayments: cfg.StarsOutboxDir != "", + Health: healthReporter, }, logger) if err != nil { return err @@ -120,6 +132,8 @@ func run(ctx context.Context, cfg config.BotConfig, logger *zap.Logger) error { OwnsUpdates: cfg.OwnsUpdates, Creds: credentials.NewTLS(tlsCfg), ReconnectDelay: cfg.BotLink.ReconnectDelay, + Health: healthReporter, + HealthInterval: healthReportInterval, }, exec, logger) if err != nil { return err diff --git a/platform/telegram/internal/bot/bot.go b/platform/telegram/internal/bot/bot.go index 0ca666b..982e3f5 100644 --- a/platform/telegram/internal/bot/bot.go +++ b/platform/telegram/internal/bot/bot.go @@ -7,19 +7,26 @@ package bot import ( "context" + "net/http" "net/url" "strconv" "strings" + "time" tgbot "github.com/go-telegram/bot" "github.com/go-telegram/bot/models" "go.uber.org/zap" "golang.org/x/time/rate" + "scrabble/platform/telegram/internal/health" "scrabble/platform/telegram/internal/outbox" "scrabble/platform/telegram/internal/support" ) +// botPollTimeout is the getUpdates long-poll timeout. It matches the go-telegram/bot default; we set +// it explicitly only because wiring the health observer (WithHTTPClient) also sets the poll timeout. +const botPollTimeout = time.Minute + // Config configures the bot wrapper. type Config struct { // Token is the Bot API token. @@ -54,6 +61,9 @@ type Config struct { // pre_checkout_query updates and handles pre_checkout / successful_payment. The runtime // dependencies (the validator, forwarder and outbox) are wired with SetPaymentHandlers. AcceptPayments bool + // Health, when set, observes every Bot API request for health metrics (reported to the gateway + // over the bot-link) and honours a 429's Retry-After. Nil disables the observation. + Health *health.Reporter } // EligibilityResolver answers whether the Telegram user identified by externalID @@ -159,6 +169,12 @@ func New(cfg Config, log *zap.Logger) (*Bot, error) { if cfg.APIBaseURL != "" { opts = append(opts, tgbot.WithServerURL(cfg.APIBaseURL)) } + if cfg.Health != nil { + // Observe every Bot API request centrally for health metrics and the 429 back-off. The + // client timeout leaves slack over the long-poll hold (botPollTimeout - 1s server-side). + base := &http.Client{Timeout: botPollTimeout + 10*time.Second} + opts = append(opts, tgbot.WithHTTPClient(botPollTimeout, cfg.Health.Wrap(base))) + } api, err := tgbot.New(cfg.Token, opts...) if err != nil { return nil, err diff --git a/platform/telegram/internal/botlink/client.go b/platform/telegram/internal/botlink/client.go index 92fa84c..4e2ed5f 100644 --- a/platform/telegram/internal/botlink/client.go +++ b/platform/telegram/internal/botlink/client.go @@ -11,6 +11,7 @@ import ( "google.golang.org/grpc/keepalive" botlinkv1 "scrabble/pkg/proto/botlink/v1" + "scrabble/platform/telegram/internal/health" ) const ( @@ -34,6 +35,11 @@ type ClientConfig struct { Creds credentials.TransportCredentials // ReconnectDelay is the pause before re-dialing after the stream ends. ReconnectDelay time.Duration + // Health, when set with a positive HealthInterval, is flushed to the gateway as a botlink Health + // message every HealthInterval while the stream is open. Nil disables health reporting. + Health *health.Reporter + // HealthInterval is the cadence of the Health flush; 0 disables it. + HealthInterval time.Duration } // Client maintains the long-lived bot-link to the gateway, executing the commands @@ -140,22 +146,59 @@ func (c *Client) serve(ctx context.Context, client botlinkv1.BotLinkClient) erro } c.log.Info("bot-link connected", zap.String("gateway", c.cfg.GatewayAddr), zap.Bool("owns_updates", c.cfg.OwnsUpdates)) + // One goroutine owns stream.Recv, feeding commands to the loop below; the loop owns every + // stream.Send (the Acks and the periodic Health) — a gRPC stream forbids concurrent Send. + rctx, cancel := context.WithCancel(ctx) + defer cancel() + cmds := make(chan *botlinkv1.Command) + recvErr := make(chan error, 1) + go func() { + for { + msg, err := stream.Recv() + if err != nil { + recvErr <- err + return + } + if cmd := msg.GetCommand(); cmd != nil { + select { + case cmds <- cmd: + case <-rctx.Done(): + return + } + } + } + }() + + var healthTick <-chan time.Time + if c.cfg.Health != nil && c.cfg.HealthInterval > 0 { + t := time.NewTicker(c.cfg.HealthInterval) + defer t.Stop() + healthTick = t.C + } + for { - msg, err := stream.Recv() - if err != nil { - return err - } - cmd := msg.GetCommand() - if cmd == nil { - continue - } - delivered, result, herr := c.exec.Handle(ctx, cmd) - ack := &botlinkv1.Ack{CommandId: cmd.GetCommandId(), Delivered: delivered, Result: result} - if herr != nil { - ack.Error = herr.Error() - } - if err := stream.Send(&botlinkv1.FromBot{Msg: &botlinkv1.FromBot_Ack{Ack: ack}}); err != nil { + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-recvErr: return err + case cmd := <-cmds: + delivered, result, herr := c.exec.Handle(ctx, cmd) + ack := &botlinkv1.Ack{CommandId: cmd.GetCommandId(), Delivered: delivered, Result: result} + if herr != nil { + ack.Error = herr.Error() + } + if err := stream.Send(&botlinkv1.FromBot{Msg: &botlinkv1.FromBot_Ack{Ack: ack}}); err != nil { + return err + } + case <-healthTick: + // Snapshot without resetting; only commit (clear the deltas) after a successful send, so a + // failed flush loses nothing and the counts ride the next connection. + hb := c.cfg.Health.Snapshot() + if err := stream.Send(&botlinkv1.FromBot{Msg: &botlinkv1.FromBot_Health{Health: hb}}); err != nil { + return err + } + c.cfg.Health.Commit(hb) } } } diff --git a/platform/telegram/internal/health/health.go b/platform/telegram/internal/health/health.go new file mode 100644 index 0000000..b8351fb --- /dev/null +++ b/platform/telegram/internal/health/health.go @@ -0,0 +1,145 @@ +// Package health tracks the Telegram bot's Bot API health and reports it to the gateway over the +// bot-link. The bot exports no telemetry of its own — otelcol lives on the main host, unreachable +// from the bot host — so the gateway turns these reports into its own metrics (see +// gateway/internal/botlink). Health is observed CENTRALLY by wrapping the Bot API HTTP client +// ([Reporter.Wrap]); every request is classified there with no per-call-site instrumentation. +package health + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "strconv" + "strings" + "sync/atomic" + "time" + + botlinkv1 "scrabble/pkg/proto/botlink/v1" +) + +// maxBackoff caps how long a single 429 makes the bot wait, so a hostile or buggy Retry-After can +// never wedge the bot. +const maxBackoff = 30 * time.Second + +// Reporter accumulates the bot's Bot API health between reports: three delta counters and the last +// successful-response stamp. It is safe for concurrent use. The bot-link client periodically +// [Reporter.Snapshot]s it, sends the snapshot as a botlink Health, and [Reporter.Commit]s it on a +// successful send. +type Reporter struct { + connectFailures atomic.Int64 + apiErrors atomic.Int64 + rateLimited atomic.Int64 + lastOKUnix atomic.Int64 +} + +// NewReporter returns an empty Reporter. +func NewReporter() *Reporter { return &Reporter{} } + +// Snapshot reads the current delta counters and the last-ok stamp into a Health message WITHOUT +// resetting them. The counters are cleared only by [Reporter.Commit] after a successful send, so a +// failed send loses nothing. +func (r *Reporter) Snapshot() *botlinkv1.Health { + return &botlinkv1.Health{ + ConnectFailures: uint64(max(r.connectFailures.Load(), 0)), + ApiErrors: uint64(max(r.apiErrors.Load(), 0)), + RateLimited: uint64(max(r.rateLimited.Load(), 0)), + LastOkUnix: r.lastOKUnix.Load(), + } +} + +// Commit subtracts a just-sent snapshot's delta counters, so counts accrued between the snapshot and +// the send survive into the next report. last_ok is a stamp, not a delta, so it is left as is. +func (r *Reporter) Commit(sent *botlinkv1.Health) { + r.connectFailures.Add(-int64(sent.GetConnectFailures())) + r.apiErrors.Add(-int64(sent.GetApiErrors())) + r.rateLimited.Add(-int64(sent.GetRateLimited())) +} + +// Wrap returns an http.Client-shaped observer over base for tgbot.WithHTTPClient: it stamps liveness +// on every successful response, counts transport and 5xx failures (split getUpdates vs other) and +// 429s, and honours a 429's Retry-After (bounded by maxBackoff) so the bot backs off instead of +// hammering. A 4xx other than 429 (a user blocked the bot, a malformed request) is a normal +// per-request outcome, not a health signal, and is not counted. +func (r *Reporter) Wrap(base Doer) Doer { return &observer{base: base, r: r} } + +// Doer is the minimal HTTP surface the Bot API client needs; both *http.Client and [observer] +// satisfy it, and it matches the go-telegram/bot HttpClient interface. +type Doer interface { + Do(*http.Request) (*http.Response, error) +} + +// observer is the Bot API HTTP client wrapper (see [Reporter.Wrap]). +type observer struct { + base Doer + r *Reporter +} + +// Do performs the request and records its health outcome. +func (o *observer) Do(req *http.Request) (*http.Response, error) { + resp, err := o.base.Do(req) + poll := strings.HasSuffix(req.URL.Path, "/getUpdates") + if err != nil { + // A cancelled context is a shutdown, not a Bot API failure. + if req.Context().Err() == nil { + o.fail(poll) + } + return resp, err + } + switch { + case resp.StatusCode == http.StatusTooManyRequests: + o.r.rateLimited.Add(1) + o.backoff(req.Context(), resp) + case resp.StatusCode >= 200 && resp.StatusCode < 300: + o.r.lastOKUnix.Store(time.Now().Unix()) + case resp.StatusCode >= 500: + o.fail(poll) + } + return resp, err +} + +// fail records a connect failure on the getUpdates long-poll or an api error otherwise. +func (o *observer) fail(poll bool) { + if poll { + o.r.connectFailures.Add(1) + } else { + o.r.apiErrors.Add(1) + } +} + +// backoff waits out a 429's Retry-After (bounded, cancellable). It buffers the response body first so +// the wait holds no connection open and the Bot API library can still parse the 429 afterwards. +func (o *observer) backoff(ctx context.Context, resp *http.Response) { + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + resp.Body = io.NopCloser(bytes.NewReader(body)) + d := retryAfter(resp.Header.Get("Retry-After"), body) + if d <= 0 { + return + } + if d > maxBackoff { + d = maxBackoff + } + select { + case <-ctx.Done(): + case <-time.After(d): + } +} + +// retryAfter resolves the 429 back-off from the Retry-After header, falling back to the Bot API +// body's parameters.retry_after (both are seconds). It returns 0 when neither gives a positive value. +func retryAfter(header string, body []byte) time.Duration { + if secs, err := strconv.Atoi(strings.TrimSpace(header)); err == nil && secs > 0 { + return time.Duration(secs) * time.Second + } + var payload struct { + Parameters struct { + RetryAfter int `json:"retry_after"` + } `json:"parameters"` + } + if err := json.Unmarshal(body, &payload); err == nil && payload.Parameters.RetryAfter > 0 { + return time.Duration(payload.Parameters.RetryAfter) * time.Second + } + return 0 +} diff --git a/platform/telegram/internal/health/health_test.go b/platform/telegram/internal/health/health_test.go new file mode 100644 index 0000000..82e67fb --- /dev/null +++ b/platform/telegram/internal/health/health_test.go @@ -0,0 +1,121 @@ +package health + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// fakeDoer returns a canned response and error for every request. +type fakeDoer struct { + resp *http.Response + err error +} + +func (f *fakeDoer) Do(*http.Request) (*http.Response, error) { return f.resp, f.err } + +// mkResp builds a response with a status, optional Retry-After header and body. +func mkResp(status int, retryAfter, body string) *http.Response { + h := http.Header{} + if retryAfter != "" { + h.Set("Retry-After", retryAfter) + } + return &http.Response{StatusCode: status, Header: h, Body: io.NopCloser(strings.NewReader(body))} +} + +// run sends one request for the Bot API method through a fresh observer and returns the reporter's +// resulting snapshot. The 429 cases carry no Retry-After, so the back-off returns before any wait. +func run(method string, resp *http.Response, err error) *Reporter { + r := NewReporter() + req := httptest.NewRequest(http.MethodPost, "https://api.telegram.org/bot123/"+method, nil) + _, _ = r.Wrap(&fakeDoer{resp: resp, err: err}).Do(req) + return r +} + +func TestObserverClassifies(t *testing.T) { + cases := []struct { + name string + method string + resp *http.Response + err error + wantConnect uint64 + wantAPI uint64 + wantRate uint64 + wantLastOKSet bool + }{ + {"poll ok stamps liveness", "getUpdates", mkResp(200, "", `{"ok":true}`), nil, 0, 0, 0, true}, + {"send ok stamps liveness", "sendMessage", mkResp(200, "", `{"ok":true}`), nil, 0, 0, 0, true}, + {"poll 5xx is a connect failure", "getUpdates", mkResp(502, "", ""), nil, 1, 0, 0, false}, + {"send 5xx is an api error", "sendMessage", mkResp(500, "", ""), nil, 0, 1, 0, false}, + {"429 is rate limited, not an error", "sendMessage", mkResp(429, "", `{"ok":false}`), nil, 0, 0, 1, false}, + {"4xx other than 429 is not counted", "sendMessage", mkResp(403, "", ""), nil, 0, 0, 0, false}, + {"poll transport error is a connect failure", "getUpdates", nil, errors.New("dial"), 1, 0, 0, false}, + {"send transport error is an api error", "sendMessage", nil, errors.New("dial"), 0, 1, 0, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := run(tc.method, tc.resp, tc.err).Snapshot() + if s.GetConnectFailures() != tc.wantConnect || s.GetApiErrors() != tc.wantAPI || s.GetRateLimited() != tc.wantRate { + t.Errorf("counters = connect %d api %d rate %d, want %d/%d/%d", + s.GetConnectFailures(), s.GetApiErrors(), s.GetRateLimited(), tc.wantConnect, tc.wantAPI, tc.wantRate) + } + if (s.GetLastOkUnix() > 0) != tc.wantLastOKSet { + t.Errorf("last_ok set = %v, want %v", s.GetLastOkUnix() > 0, tc.wantLastOKSet) + } + }) + } +} + +// TestObserverIgnoresShutdownTransportError checks a transport error under a cancelled context (a +// shutdown) is not counted as a Bot API failure. +func TestObserverIgnoresShutdownTransportError(t *testing.T) { + r := NewReporter() + req := httptest.NewRequest(http.MethodPost, "https://api.telegram.org/bot123/getUpdates", nil) + ctx, cancel := context.WithCancel(req.Context()) + cancel() + _, _ = r.Wrap(&fakeDoer{err: errors.New("dial")}).Do(req.WithContext(ctx)) + if s := r.Snapshot(); s.GetConnectFailures() != 0 { + t.Errorf("shutdown transport error must not count, got connect=%d", s.GetConnectFailures()) + } +} + +func TestRetryAfter(t *testing.T) { + cases := []struct { + header string + body string + want time.Duration + }{ + {"5", "", 5 * time.Second}, + {"", `{"parameters":{"retry_after":7}}`, 7 * time.Second}, + {"3", `{"parameters":{"retry_after":9}}`, 3 * time.Second}, // header wins + {"", `{"ok":false}`, 0}, + {"0", "", 0}, + {"", "not json", 0}, + } + for _, tc := range cases { + if got := retryAfter(tc.header, []byte(tc.body)); got != tc.want { + t.Errorf("retryAfter(%q, %q) = %v, want %v", tc.header, tc.body, got, tc.want) + } + } +} + +// TestSnapshotCommit checks Commit clears only the sent deltas, so counts accrued between the +// snapshot and the send survive into the next report. +func TestSnapshotCommit(t *testing.T) { + r := NewReporter() + r.apiErrors.Add(3) + snap := r.Snapshot() + if snap.GetApiErrors() != 3 { + t.Fatalf("snapshot api_errors = %d, want 3", snap.GetApiErrors()) + } + r.apiErrors.Add(2) // accrues after the snapshot + r.Commit(snap) + if got := r.Snapshot().GetApiErrors(); got != 2 { + t.Errorf("after commit api_errors = %d, want 2 (the post-snapshot accrual)", got) + } +}