R3: backend rate-limit observability — ratewatch, auto-flag, admin throttled view
- accounts.flagged_high_rate_at baked into the R1 baseline (no prod data; the contour schema is wiped after merge); jet regenerated — the regen also picks up the previously missing game_drafts/game_hidden models. - account.Store: FlagHighRate (set-once), ClearHighRateFlag, the flag in GetByID/ListUsers and a ListFlaggedHighRate review queue. - New internal/ratewatch: ingests the gateway rejection reports, keeps a bounded in-memory episode window for the console and applies the conservative auto-flag (1000 rejected / 10 min, BACKEND_HIGHRATE_FLAG_*). - POST /api/v1/internal/ratelimit/report (network-trusted, like sessions/resolve). - Admin console: Throttled page (episodes + flagged accounts), a high-rate badge in the user list, the marker + operator clear action on the user card. - Tests: ratewatch unit suite, report-route handler test, renderer cases, integration coverage for the store round-trip and the console flow.
This commit is contained in:
@@ -6,6 +6,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
@@ -195,6 +196,62 @@ func TestServiceLanguageRoundTrip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestHighRateFlagRoundTrip covers the R3 soft high-rate marker: a fresh account
|
||||
// is unflagged, FlagHighRate stamps it exactly once (a second sustained episode
|
||||
// never moves the timestamp), ClearHighRateFlag reverses it, and a re-flag after
|
||||
// the operator clear takes a fresh timestamp.
|
||||
func TestHighRateFlagRoundTrip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := account.NewStore(testDB)
|
||||
acc, err := store.ProvisionTelegram(ctx, "tg-"+uuid.NewString(), "en", "", "Player")
|
||||
if err != nil {
|
||||
t.Fatalf("provision telegram: %v", err)
|
||||
}
|
||||
if !acc.FlaggedHighRateAt.IsZero() {
|
||||
t.Fatalf("fresh FlaggedHighRateAt = %v, want zero", acc.FlaggedHighRateAt)
|
||||
}
|
||||
|
||||
first := time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC)
|
||||
set, err := store.FlagHighRate(ctx, acc.ID, first)
|
||||
if err != nil {
|
||||
t.Fatalf("flag: %v", err)
|
||||
}
|
||||
if !set {
|
||||
t.Fatal("first FlagHighRate reported not set")
|
||||
}
|
||||
if set, err = store.FlagHighRate(ctx, acc.ID, first.Add(time.Hour)); err != nil {
|
||||
t.Fatalf("re-flag: %v", err)
|
||||
} else if set {
|
||||
t.Fatal("second FlagHighRate must not overwrite the marker")
|
||||
}
|
||||
got, err := store.GetByID(ctx, acc.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get by id: %v", err)
|
||||
}
|
||||
if !got.FlaggedHighRateAt.Equal(first) {
|
||||
t.Errorf("FlaggedHighRateAt = %v, want %v", got.FlaggedHighRateAt, first)
|
||||
}
|
||||
|
||||
if err := store.ClearHighRateFlag(ctx, acc.ID); err != nil {
|
||||
t.Fatalf("clear: %v", err)
|
||||
}
|
||||
if got, err = store.GetByID(ctx, acc.ID); err != nil {
|
||||
t.Fatalf("get by id: %v", err)
|
||||
} else if !got.FlaggedHighRateAt.IsZero() {
|
||||
t.Errorf("cleared FlaggedHighRateAt = %v, want zero", got.FlaggedHighRateAt)
|
||||
}
|
||||
|
||||
second := first.Add(24 * time.Hour)
|
||||
if set, err = store.FlagHighRate(ctx, acc.ID, second); err != nil || !set {
|
||||
t.Fatalf("re-flag after clear = (%v, %v), want (true, nil)", set, err)
|
||||
}
|
||||
if got, err = store.GetByID(ctx, acc.ID); err != nil {
|
||||
t.Fatalf("get by id: %v", err)
|
||||
} else if !got.FlaggedHighRateAt.Equal(second) {
|
||||
t.Errorf("re-flagged FlaggedHighRateAt = %v, want %v", got.FlaggedHighRateAt, second)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIdentityExternalID covers the reverse identity lookup the push-target route
|
||||
// uses: it returns the external_id for the matching kind and ErrNotFound otherwise,
|
||||
// including for a guest that carries no identity.
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"scrabble/backend/internal/account"
|
||||
"scrabble/backend/internal/engine"
|
||||
"scrabble/backend/internal/game"
|
||||
"scrabble/backend/internal/ratewatch"
|
||||
"scrabble/backend/internal/server"
|
||||
)
|
||||
|
||||
@@ -206,6 +207,72 @@ func TestConsoleGameDetailRobotSchedule(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestConsoleThrottledViewAndFlagClear drives the R3 rate-limit surface end to
|
||||
// end against real stores: a gateway report past the threshold auto-flags the
|
||||
// account, the throttled view shows the episode and the flagged account, the
|
||||
// user card carries the marker, and the operator clear (a same-origin POST)
|
||||
// reverses it.
|
||||
func TestConsoleThrottledViewAndFlagClear(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
accounts := account.NewStore(testDB)
|
||||
acc, err := accounts.ProvisionTelegram(ctx, "tg-"+uuid.NewString(), "en", "", "Throttled Player")
|
||||
if err != nil {
|
||||
t.Fatalf("provision: %v", err)
|
||||
}
|
||||
|
||||
watch := ratewatch.New(ratewatch.Config{FlagThreshold: 100, FlagWindow: 10 * time.Minute}, accounts, zap.NewNop())
|
||||
srv := server.New(":0", server.Deps{
|
||||
Logger: zap.NewNop(),
|
||||
Accounts: accounts,
|
||||
Games: newGameService(),
|
||||
Registry: testRegistry,
|
||||
DictDir: dictDir(),
|
||||
RateWatch: watch,
|
||||
})
|
||||
h := srv.Handler()
|
||||
|
||||
report := `{"window_seconds":30,"entries":[` +
|
||||
`{"class":"user","key":"` + acc.ID.String() + `","rejected":150},` +
|
||||
`{"class":"public","key":"10.1.2.3","rejected":7}]}`
|
||||
req := httptest.NewRequest(http.MethodPost, "http://admin.test/api/v1/internal/ratelimit/report", strings.NewReader(report))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("report = %d, want 204", rec.Code)
|
||||
}
|
||||
|
||||
got, err := accounts.GetByID(ctx, acc.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get by id: %v", err)
|
||||
}
|
||||
if got.FlaggedHighRateAt.IsZero() {
|
||||
t.Fatal("account not auto-flagged past the threshold")
|
||||
}
|
||||
|
||||
base := "http://admin.test/_gm"
|
||||
code, body := consoleDo(h, http.MethodGet, base+"/throttled", "", "")
|
||||
if code != http.StatusOK || !strings.Contains(body, acc.ID.String()) ||
|
||||
!strings.Contains(body, "10.1.2.3") || !strings.Contains(body, "Throttled Player") {
|
||||
t.Fatalf("throttled view = %d, episode/flag shown = %v/%v",
|
||||
code, strings.Contains(body, "10.1.2.3"), strings.Contains(body, "Throttled Player"))
|
||||
}
|
||||
if code, body = consoleDo(h, http.MethodGet, base+"/users/"+acc.ID.String(), "", ""); code != http.StatusOK || !strings.Contains(body, "Clear high-rate flag") {
|
||||
t.Fatalf("user card = %d, has clear action = %v", code, strings.Contains(body, "Clear high-rate flag"))
|
||||
}
|
||||
|
||||
// The clear POST is CSRF-guarded like every console action.
|
||||
if code, _ = consoleDo(h, http.MethodPost, base+"/users/"+acc.ID.String()+"/clear-high-rate-flag", "", ""); code != http.StatusForbidden {
|
||||
t.Fatalf("clear without origin = %d, want 403", code)
|
||||
}
|
||||
if code, body = consoleDo(h, http.MethodPost, base+"/users/"+acc.ID.String()+"/clear-high-rate-flag", "x=1", "http://admin.test"); code != http.StatusOK || !strings.Contains(body, "Cleared") {
|
||||
t.Fatalf("clear with origin = %d, has Cleared = %v", code, strings.Contains(body, "Cleared"))
|
||||
}
|
||||
if got, err = accounts.GetByID(ctx, acc.ID); err != nil || !got.FlaggedHighRateAt.IsZero() {
|
||||
t.Fatalf("flag survived the clear: %v (err %v)", got.FlaggedHighRateAt, err)
|
||||
}
|
||||
}
|
||||
|
||||
// consoleDo issues a request to h, optionally with an Origin header, and returns
|
||||
// the status and body. Form bodies are sent as application/x-www-form-urlencoded.
|
||||
func consoleDo(h http.Handler, method, target, body, origin string) (int, string) {
|
||||
|
||||
Reference in New Issue
Block a user