R3: backend rate-limit observability — ratewatch, auto-flag, admin throttled view

- accounts.flagged_high_rate_at baked into the R1 baseline (no prod data; the
  contour schema is wiped after merge); jet regenerated — the regen also picks
  up the previously missing game_drafts/game_hidden models.
- account.Store: FlagHighRate (set-once), ClearHighRateFlag, the flag in
  GetByID/ListUsers and a ListFlaggedHighRate review queue.
- New internal/ratewatch: ingests the gateway rejection reports, keeps a
  bounded in-memory episode window for the console and applies the
  conservative auto-flag (1000 rejected / 10 min, BACKEND_HIGHRATE_FLAG_*).
- POST /api/v1/internal/ratelimit/report (network-trusted, like
  sessions/resolve).
- Admin console: Throttled page (episodes + flagged accounts), a high-rate
  badge in the user list, the marker + operator clear action on the user card.
- Tests: ratewatch unit suite, report-route handler test, renderer cases,
  integration coverage for the store round-trip and the console flow.
This commit is contained in:
Ilia Denisov
2026-06-10 02:14:00 +02:00
parent 8878711cf3
commit ab58062565
27 changed files with 1081 additions and 33 deletions
+57
View File
@@ -6,6 +6,7 @@ import (
"context"
"errors"
"testing"
"time"
"github.com/google/uuid"
@@ -195,6 +196,62 @@ func TestServiceLanguageRoundTrip(t *testing.T) {
}
}
// TestHighRateFlagRoundTrip covers the R3 soft high-rate marker: a fresh account
// is unflagged, FlagHighRate stamps it exactly once (a second sustained episode
// never moves the timestamp), ClearHighRateFlag reverses it, and a re-flag after
// the operator clear takes a fresh timestamp.
func TestHighRateFlagRoundTrip(t *testing.T) {
ctx := context.Background()
store := account.NewStore(testDB)
acc, err := store.ProvisionTelegram(ctx, "tg-"+uuid.NewString(), "en", "", "Player")
if err != nil {
t.Fatalf("provision telegram: %v", err)
}
if !acc.FlaggedHighRateAt.IsZero() {
t.Fatalf("fresh FlaggedHighRateAt = %v, want zero", acc.FlaggedHighRateAt)
}
first := time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC)
set, err := store.FlagHighRate(ctx, acc.ID, first)
if err != nil {
t.Fatalf("flag: %v", err)
}
if !set {
t.Fatal("first FlagHighRate reported not set")
}
if set, err = store.FlagHighRate(ctx, acc.ID, first.Add(time.Hour)); err != nil {
t.Fatalf("re-flag: %v", err)
} else if set {
t.Fatal("second FlagHighRate must not overwrite the marker")
}
got, err := store.GetByID(ctx, acc.ID)
if err != nil {
t.Fatalf("get by id: %v", err)
}
if !got.FlaggedHighRateAt.Equal(first) {
t.Errorf("FlaggedHighRateAt = %v, want %v", got.FlaggedHighRateAt, first)
}
if err := store.ClearHighRateFlag(ctx, acc.ID); err != nil {
t.Fatalf("clear: %v", err)
}
if got, err = store.GetByID(ctx, acc.ID); err != nil {
t.Fatalf("get by id: %v", err)
} else if !got.FlaggedHighRateAt.IsZero() {
t.Errorf("cleared FlaggedHighRateAt = %v, want zero", got.FlaggedHighRateAt)
}
second := first.Add(24 * time.Hour)
if set, err = store.FlagHighRate(ctx, acc.ID, second); err != nil || !set {
t.Fatalf("re-flag after clear = (%v, %v), want (true, nil)", set, err)
}
if got, err = store.GetByID(ctx, acc.ID); err != nil {
t.Fatalf("get by id: %v", err)
} else if !got.FlaggedHighRateAt.Equal(second) {
t.Errorf("re-flagged FlaggedHighRateAt = %v, want %v", got.FlaggedHighRateAt, second)
}
}
// TestIdentityExternalID covers the reverse identity lookup the push-target route
// uses: it returns the external_id for the matching kind and ErrNotFound otherwise,
// including for a guest that carries no identity.
+67
View File
@@ -16,6 +16,7 @@ import (
"scrabble/backend/internal/account"
"scrabble/backend/internal/engine"
"scrabble/backend/internal/game"
"scrabble/backend/internal/ratewatch"
"scrabble/backend/internal/server"
)
@@ -206,6 +207,72 @@ func TestConsoleGameDetailRobotSchedule(t *testing.T) {
}
}
// TestConsoleThrottledViewAndFlagClear drives the R3 rate-limit surface end to
// end against real stores: a gateway report past the threshold auto-flags the
// account, the throttled view shows the episode and the flagged account, the
// user card carries the marker, and the operator clear (a same-origin POST)
// reverses it.
func TestConsoleThrottledViewAndFlagClear(t *testing.T) {
ctx := context.Background()
accounts := account.NewStore(testDB)
acc, err := accounts.ProvisionTelegram(ctx, "tg-"+uuid.NewString(), "en", "", "Throttled Player")
if err != nil {
t.Fatalf("provision: %v", err)
}
watch := ratewatch.New(ratewatch.Config{FlagThreshold: 100, FlagWindow: 10 * time.Minute}, accounts, zap.NewNop())
srv := server.New(":0", server.Deps{
Logger: zap.NewNop(),
Accounts: accounts,
Games: newGameService(),
Registry: testRegistry,
DictDir: dictDir(),
RateWatch: watch,
})
h := srv.Handler()
report := `{"window_seconds":30,"entries":[` +
`{"class":"user","key":"` + acc.ID.String() + `","rejected":150},` +
`{"class":"public","key":"10.1.2.3","rejected":7}]}`
req := httptest.NewRequest(http.MethodPost, "http://admin.test/api/v1/internal/ratelimit/report", strings.NewReader(report))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != http.StatusNoContent {
t.Fatalf("report = %d, want 204", rec.Code)
}
got, err := accounts.GetByID(ctx, acc.ID)
if err != nil {
t.Fatalf("get by id: %v", err)
}
if got.FlaggedHighRateAt.IsZero() {
t.Fatal("account not auto-flagged past the threshold")
}
base := "http://admin.test/_gm"
code, body := consoleDo(h, http.MethodGet, base+"/throttled", "", "")
if code != http.StatusOK || !strings.Contains(body, acc.ID.String()) ||
!strings.Contains(body, "10.1.2.3") || !strings.Contains(body, "Throttled Player") {
t.Fatalf("throttled view = %d, episode/flag shown = %v/%v",
code, strings.Contains(body, "10.1.2.3"), strings.Contains(body, "Throttled Player"))
}
if code, body = consoleDo(h, http.MethodGet, base+"/users/"+acc.ID.String(), "", ""); code != http.StatusOK || !strings.Contains(body, "Clear high-rate flag") {
t.Fatalf("user card = %d, has clear action = %v", code, strings.Contains(body, "Clear high-rate flag"))
}
// The clear POST is CSRF-guarded like every console action.
if code, _ = consoleDo(h, http.MethodPost, base+"/users/"+acc.ID.String()+"/clear-high-rate-flag", "", ""); code != http.StatusForbidden {
t.Fatalf("clear without origin = %d, want 403", code)
}
if code, body = consoleDo(h, http.MethodPost, base+"/users/"+acc.ID.String()+"/clear-high-rate-flag", "x=1", "http://admin.test"); code != http.StatusOK || !strings.Contains(body, "Cleared") {
t.Fatalf("clear with origin = %d, has Cleared = %v", code, strings.Contains(body, "Cleared"))
}
if got, err = accounts.GetByID(ctx, acc.ID); err != nil || !got.FlaggedHighRateAt.IsZero() {
t.Fatalf("flag survived the clear: %v (err %v)", got.FlaggedHighRateAt, err)
}
}
// consoleDo issues a request to h, optionally with an Origin header, and returns
// the status and body. Form bodies are sent as application/x-www-form-urlencoded.
func consoleDo(h http.Handler, method, target, body, origin string) (int, string) {