R3: backend rate-limit observability — ratewatch, auto-flag, admin throttled view

- accounts.flagged_high_rate_at baked into the R1 baseline (no prod data; the contour schema is wiped after merge); jet regenerated — the regen also picks up the previously missing game_drafts/game_hidden models. - account.Store: FlagHighRate (set-once), ClearHighRateFlag, the flag in GetByID/ListUsers and a ListFlaggedHighRate review queue. - New internal/ratewatch: ingests the gateway rejection reports, keeps a bounded in-memory episode window for the console and applies the conservative auto-flag (1000 rejected / 10 min, BACKEND_HIGHRATE_FLAG_*). - POST /api/v1/internal/ratelimit/report (network-trusted, like sessions/resolve). - Admin console: Throttled page (episodes + flagged accounts), a high-rate badge in the user list, the marker + operator clear action on the user card. - Tests: ratewatch unit suite, report-route handler test, renderer cases, integration coverage for the store round-trip and the console flow.
2026-06-10 02:14:00 +02:00
parent 8878711cf3
commit ab58062565
27 changed files with 1081 additions and 33 deletions
@@ -21,8 +21,14 @@ func TestRendererRendersEveryPage(t *testing.T) {
 		want string
 	}{
 		{"dashboard", DashboardView{Accounts: 3, Variants: []VariantVersions{{Variant: "scrabble_en", Latest: "v1", Versions: []string{"v1"}}}}, "Dashboard"},
-		{"users", UsersView{Items: []UserRow{{ID: "a1", DisplayName: "Kaya"}}, Pager: NewPager(1, 50, 1)}, "Kaya"},
+		{"users", UsersView{Items: []UserRow{{ID: "a1", DisplayName: "Kaya", FlaggedHighRate: true}}, Pager: NewPager(1, 50, 1)}, "high-rate"},
 		{"user_detail", UserDetailView{ID: "a1", DisplayName: "Kaya", HasStats: true, Stats: StatsRow{Wins: 2}, TelegramID: "123", ConnectorEnabled: true}, "Send Telegram message"},
+		{"user_detail", UserDetailView{ID: "a1", DisplayName: "Kaya", FlaggedHighRateAt: "2026-06-10 12:00"}, "Clear high-rate flag"},
+		{"throttled", ThrottledView{
+			Episodes:      []ThrottleEpisodeRow{{Class: "user", Key: "a1", UserID: "a1", Rejected: 1234, FirstSeen: "2026-06-10 12:00", LastSeen: "2026-06-10 12:05"}},
+			Flagged:       []FlaggedAccountRow{{ID: "a1", DisplayName: "Kaya", FlaggedAt: "2026-06-10 12:05"}},
+			FlagThreshold: 1000, FlagWindow: "10m0s",
+		}, "Recent episodes"},
 		{"games", GamesView{Items: []GameRow{{ID: "g1", Variant: "scrabble_en", Status: "active"}}, Status: "active", Pager: NewPager(1, 50, 1)}, "g1"},
 		{"game_detail", GameDetailView{ID: "g1", Variant: "scrabble_en", Seats: []SeatRow{{Seat: 0, DisplayName: "Kaya"}}}, "Seats"},
 		{"complaints", ComplaintsView{Items: []ComplaintRow{{ID: "c1", Word: "qi", Status: "open"}}, Status: "open", Pager: NewPager(1, 50, 1)}, "qi"},
@@ -17,6 +17,7 @@
    <a href="/_gm/games"{{if eq .ActiveNav "games"}} class="active"{{end}}>Games</a>
    <a href="/_gm/complaints"{{if eq .ActiveNav "complaints"}} class="active"{{end}}>Complaints</a>
    <a href="/_gm/messages"{{if eq .ActiveNav "messages"}} class="active"{{end}}>Messages</a>
+    <a href="/_gm/throttled"{{if eq .ActiveNav "throttled"}} class="active"{{end}}>Throttled</a>
    <a href="/_gm/dictionary"{{if eq .ActiveNav "dictionary"}} class="active"{{end}}>Dictionary</a>
    <a href="/_gm/broadcast"{{if eq .ActiveNav "broadcast"}} class="active"{{end}}>Broadcast</a>
    <a href="/_gm/grafana/">Grafana ↗</a>
@@ -0,0 +1,39 @@
+{{define "content" -}}
+<h1>Throttled</h1>
+{{with .Data}}
+<p class="note">Rate-limiter rejections reported periodically by the gateway. The episode
+list is in-memory and resets on a backend restart. An account sustaining
+{{.FlagThreshold}}+ rejected calls within {{.FlagWindow}} is soft-flagged for review
+below — never banned automatically; clear the flag on the user card.</p>
+<section class="panel"><h2>Recent episodes</h2>
+<table class="list">
+<thead><tr><th>Class</th><th>Key</th><th class="num">Rejected</th><th>First seen</th><th>Last seen</th></tr></thead>
+<tbody>
+{{range .Episodes}}
+<tr>
+<td>{{.Class}}</td>
+<td>{{if .UserID}}<a href="/_gm/users/{{.UserID}}">{{.Key}}</a>{{else}}<code>{{.Key}}</code>{{end}}</td>
+<td class="num">{{.Rejected}}</td>
+<td>{{.FirstSeen}}</td>
+<td>{{.LastSeen}}</td>
+</tr>
+{{else}}
+<tr><td colspan="5"><span class="note">nothing throttled recently</span></td></tr>
+{{end}}
+</tbody>
+</table>
+</section>
+<section class="panel"><h2>Flagged accounts</h2>
+<table class="list">
+<thead><tr><th>Account</th><th>Display name</th><th>Flagged</th></tr></thead>
+<tbody>
+{{range .Flagged}}
+<tr><td><a href="/_gm/users/{{.ID}}">{{.ID}}</a></td><td>{{.DisplayName}}</td><td>{{.FlaggedAt}}</td></tr>
+{{else}}
+<tr><td colspan="3"><span class="note">no flagged accounts</span></td></tr>
+{{end}}
+</tbody>
+</table>
+</section>
+{{end}}
+{{- end}}
@@ -13,8 +13,14 @@
 <li><b>Paid</b> {{if .PaidAccount}}yes{{else}}no{{end}}</li>
 <li><b>Hint wallet</b> {{.HintBalance}}</li>
 {{if .MergedInto}}<li><b>Merged into</b> {{.MergedInto}}</li>{{end}}
+{{if .FlaggedHighRateAt}}<li><b>High-rate flag</b> <span class="warn">{{.FlaggedHighRateAt}}</span></li>{{end}}
 <li><b>Created</b> {{.CreatedAt}}</li>
 </ul>
+{{if .FlaggedHighRateAt}}
+<form class="form" method="post" action="/_gm/users/{{.ID}}/clear-high-rate-flag">
+<button type="submit">Clear high-rate flag</button>
+</form>
+{{end}}
 </section>
 <section class="panel"><h2>Statistics</h2>
 {{if .HasStats}}
@@ -17,7 +17,7 @@
 {{range .Items}}
 <tr>
 <td><a href="/_gm/users/{{.ID}}">{{.ID}}</a></td>
-<td>{{.DisplayName}}{{if .Guest}} <span class="pill">guest</span>{{end}}</td>
+<td>{{.DisplayName}}{{if .Guest}} <span class="pill">guest</span>{{end}}{{if .FlaggedHighRate}} <span class="pill">high-rate</span>{{end}}</td>
 <td>{{.Kind}}</td>
 <td>{{.Language}}</td>
 <td>{{.CreatedAt}}</td>
@@ -59,18 +59,20 @@ type UsersView struct {
 }

 // UserRow is one account row in the list. MoveMin/Avg/Max are the account's
-// pre-formatted move-duration summary (empty when it has no timed move).
+// pre-formatted move-duration summary (empty when it has no timed move);
+// FlaggedHighRate marks the soft high-rate badge (R3).
 type UserRow struct {
-	ID           string
-	DisplayName  string
-	Kind         string
-	Language     string
-	Guest        bool
-	CreatedAt    string
-	HasMoveStats bool
-	MoveMin      string
-	MoveAvg      string
-	MoveMax      string
+	ID              string
+	DisplayName     string
+	Kind            string
+	Language        string
+	Guest           bool
+	FlaggedHighRate bool
+	CreatedAt       string
+	HasMoveStats    bool
+	MoveMin         string
+	MoveAvg         string
+	MoveMax         string
 }

 // MessagesView is the paginated chat-message moderation list. NameMask/ExtMask are the
@@ -110,15 +112,18 @@ type UserDetailView struct {
 	PaidAccount            bool
 	// MergedInto is the primary account id when this account has been retired by a
 	// merge (Stage 11), or empty for a live account.
-	MergedInto       string
-	HintBalance      int
-	CreatedAt        string
-	HasStats         bool
-	Stats            StatsRow
-	Identities       []IdentityRow
-	Games            []GameRow
-	TelegramID       string
-	ConnectorEnabled bool
+	MergedInto string
+	// FlaggedHighRateAt is the pre-formatted soft high-rate marker timestamp,
+	// empty for an unflagged account; the card shows it with the Clear action (R3).
+	FlaggedHighRateAt string
+	HintBalance       int
+	CreatedAt         string
+	HasStats          bool
+	Stats             StatsRow
+	Identities        []IdentityRow
+	Games             []GameRow
+	TelegramID        string
+	ConnectorEnabled  bool
 	// MoveChart is the pre-rendered inline SVG of the account's per-move-number think
 	// time (min/mean/max), empty when the account has no timed move.
 	MoveChart template.HTML
@@ -247,6 +252,35 @@ type BroadcastView struct {
 	ConnectorEnabled bool
 }

+// ThrottledView is the rate-limit observability page: the recent gateway-reported
+// throttle episodes (in-memory, reset on restart) and the accounts currently
+// carrying the high-rate flag. FlagThreshold and FlagWindow caption the active
+// auto-flag tuning.
+type ThrottledView struct {
+	Episodes      []ThrottleEpisodeRow
+	Flagged       []FlaggedAccountRow
+	FlagThreshold int
+	FlagWindow    string
+}
+
+// ThrottleEpisodeRow is one recently throttled limiter key. UserID links to the
+// user card and is set only for the user class (the other classes key by IP).
+type ThrottleEpisodeRow struct {
+	Class     string
+	Key       string
+	UserID    string
+	Rejected  int
+	FirstSeen string
+	LastSeen  string
+}
+
+// FlaggedAccountRow is one account carrying the high-rate flag.
+type FlaggedAccountRow struct {
+	ID          string
+	DisplayName string
+	FlaggedAt   string
+}
+
 // MessageView is the result page shown after a POST action.
 type MessageView struct {
 	Heading string