R3: backend rate-limit observability — ratewatch, auto-flag, admin throttled view

- accounts.flagged_high_rate_at baked into the R1 baseline (no prod data; the
  contour schema is wiped after merge); jet regenerated — the regen also picks
  up the previously missing game_drafts/game_hidden models.
- account.Store: FlagHighRate (set-once), ClearHighRateFlag, the flag in
  GetByID/ListUsers and a ListFlaggedHighRate review queue.
- New internal/ratewatch: ingests the gateway rejection reports, keeps a
  bounded in-memory episode window for the console and applies the
  conservative auto-flag (1000 rejected / 10 min, BACKEND_HIGHRATE_FLAG_*).
- POST /api/v1/internal/ratelimit/report (network-trusted, like
  sessions/resolve).
- Admin console: Throttled page (episodes + flagged accounts), a high-rate
  badge in the user list, the marker + operator clear action on the user card.
- Tests: ratewatch unit suite, report-route handler test, renderer cases,
  integration coverage for the store round-trip and the console flow.
This commit is contained in:
Ilia Denisov
2026-06-10 02:14:00 +02:00
parent 8878711cf3
commit ab58062565
27 changed files with 1081 additions and 33 deletions
@@ -19,6 +19,7 @@ import (
"scrabble/backend/internal/adminconsole"
"scrabble/backend/internal/engine"
"scrabble/backend/internal/game"
"scrabble/backend/internal/ratewatch"
"scrabble/backend/internal/robot"
"scrabble/backend/internal/social"
)
@@ -48,6 +49,8 @@ func (s *Server) registerConsole(router *gin.Engine) {
gm.GET("/users", s.consoleUsers)
gm.GET("/users/:id", s.consoleUserDetail)
gm.POST("/users/:id/message", s.consoleUserMessage)
gm.POST("/users/:id/clear-high-rate-flag", s.consoleClearHighRateFlag)
gm.GET("/throttled", s.consoleThrottled)
gm.GET("/games", s.consoleGames)
gm.GET("/games/:id", s.consoleGameDetail)
gm.GET("/complaints", s.consoleComplaints)
@@ -117,7 +120,8 @@ func (s *Server) consoleUsers(c *gin.Context) {
}
view.Items = append(view.Items, adminconsole.UserRow{
ID: it.ID.String(), DisplayName: it.DisplayName, Kind: kind,
Language: it.PreferredLanguage, Guest: it.IsGuest, CreatedAt: fmtTime(it.CreatedAt),
Language: it.PreferredLanguage, Guest: it.IsGuest,
FlaggedHighRate: !it.FlaggedHighRateAt.IsZero(), CreatedAt: fmtTime(it.CreatedAt),
})
ids = append(ids, it.ID)
}
@@ -257,6 +261,9 @@ func (s *Server) consoleUserDetail(c *gin.Context) {
if acc.MergedInto != uuid.Nil {
view.MergedInto = acc.MergedInto.String()
}
if !acc.FlaggedHighRateAt.IsZero() {
view.FlaggedHighRateAt = fmtTime(acc.FlaggedHighRateAt)
}
if view.HasStats {
if st, err := s.accounts.GetStats(ctx, id); err == nil {
view.Stats = adminconsole.StatsRow{Wins: st.Wins, Losses: st.Losses, Draws: st.Draws, MaxGamePoints: st.MaxGamePoints, MaxWordPoints: st.MaxWordPoints}
@@ -551,6 +558,56 @@ func (s *Server) consolePostBroadcast(c *gin.Context) {
}
}
// consoleThrottled renders the rate-limit observability page: the recent
// gateway-reported throttle episodes (in-memory, reset on a backend restart)
// and the accounts currently carrying the soft high-rate flag (R3).
func (s *Server) consoleThrottled(c *gin.Context) {
ctx := c.Request.Context()
var view adminconsole.ThrottledView
if s.ratewatch != nil {
cfg := s.ratewatch.Config()
view.FlagThreshold = cfg.FlagThreshold
view.FlagWindow = cfg.FlagWindow.String()
for _, ep := range s.ratewatch.Recent() {
row := adminconsole.ThrottleEpisodeRow{
Class: ep.Class, Key: ep.Key, Rejected: ep.Rejected,
FirstSeen: fmtTime(ep.FirstSeen), LastSeen: fmtTime(ep.LastSeen),
}
if ep.Class == ratewatch.ClassUser {
if id, err := uuid.Parse(ep.Key); err == nil {
row.UserID = id.String()
}
}
view.Episodes = append(view.Episodes, row)
}
}
flagged, err := s.accounts.ListFlaggedHighRate(ctx)
if err != nil {
s.consoleError(c, err)
return
}
for _, fa := range flagged {
view.Flagged = append(view.Flagged, adminconsole.FlaggedAccountRow{
ID: fa.ID.String(), DisplayName: fa.DisplayName, FlaggedAt: fmtTime(fa.FlaggedHighRateAt),
})
}
s.renderConsole(c, "throttled", "throttled", "Throttled", view)
}
// consoleClearHighRateFlag clears the soft high-rate marker — the operator's
// reversible review action (R3).
func (s *Server) consoleClearHighRateFlag(c *gin.Context) {
id, ok := s.consoleUUID(c, "/_gm/users")
if !ok {
return
}
if err := s.accounts.ClearHighRateFlag(c.Request.Context(), id); err != nil {
s.consoleError(c, err)
return
}
s.renderConsoleMessage(c, "Cleared", "high-rate flag cleared", "/_gm/users/"+id.String())
}
// variantVersions builds the per-variant resident-version summary from the registry.
func (s *Server) variantVersions() []adminconsole.VariantVersions {
out := make([]adminconsole.VariantVersions, 0, len(engine.Variants()))