From ab58062565d276ede394cf912079063a2a00c4e2 Mon Sep 17 00:00:00 2001 From: Ilia Denisov Date: Wed, 10 Jun 2026 02:14:00 +0200 Subject: [PATCH] =?UTF-8?q?R3:=20backend=20rate-limit=20observability=20?= =?UTF-8?q?=E2=80=94=20ratewatch,=20auto-flag,=20admin=20throttled=20view?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - accounts.flagged_high_rate_at baked into the R1 baseline (no prod data; the contour schema is wiped after merge); jet regenerated — the regen also picks up the previously missing game_drafts/game_hidden models. - account.Store: FlagHighRate (set-once), ClearHighRateFlag, the flag in GetByID/ListUsers and a ListFlaggedHighRate review queue. - New internal/ratewatch: ingests the gateway rejection reports, keeps a bounded in-memory episode window for the console and applies the conservative auto-flag (1000 rejected / 10 min, BACKEND_HIGHRATE_FLAG_*). - POST /api/v1/internal/ratelimit/report (network-trusted, like sessions/resolve). - Admin console: Throttled page (episodes + flagged accounts), a high-rate badge in the user list, the marker + operator clear action on the user card. - Tests: ratewatch unit suite, report-route handler test, renderer cases, integration coverage for the store round-trip and the console flow. --- backend/cmd/backend/main.go | 9 + backend/internal/account/account.go | 51 +++- backend/internal/account/userlist.go | 45 +++- backend/internal/adminconsole/render_test.go | 8 +- .../adminconsole/templates/layout.gohtml | 1 + .../templates/pages/throttled.gohtml | 39 +++ .../templates/pages/user_detail.gohtml | 6 + .../adminconsole/templates/pages/users.gohtml | 2 +- backend/internal/adminconsole/views.go | 74 ++++-- backend/internal/config/config.go | 16 ++ backend/internal/inttest/account_test.go | 57 +++++ backend/internal/inttest/admin_test.go | 67 +++++ .../postgres/jet/backend/model/accounts.go | 1 + .../postgres/jet/backend/model/game_drafts.go | 21 ++ .../postgres/jet/backend/model/game_hidden.go | 19 ++ .../postgres/jet/backend/table/accounts.go | 7 +- .../postgres/jet/backend/table/game_drafts.go | 90 +++++++ .../postgres/jet/backend/table/game_hidden.go | 84 +++++++ .../jet/backend/table/table_use_schema.go | 2 + .../postgres/migrations/00001_baseline.sql | 4 + backend/internal/ratewatch/ratewatch.go | 235 ++++++++++++++++++ backend/internal/ratewatch/ratewatch_test.go | 140 +++++++++++ backend/internal/server/handlers.go | 11 +- .../internal/server/handlers_admin_console.go | 59 ++++- backend/internal/server/handlers_ratelimit.go | 41 +++ backend/internal/server/handlers_test.go | 18 ++ backend/internal/server/server.go | 7 + 27 files changed, 1081 insertions(+), 33 deletions(-) create mode 100644 backend/internal/adminconsole/templates/pages/throttled.gohtml create mode 100644 backend/internal/postgres/jet/backend/model/game_drafts.go create mode 100644 backend/internal/postgres/jet/backend/model/game_hidden.go create mode 100644 backend/internal/postgres/jet/backend/table/game_drafts.go create mode 100644 backend/internal/postgres/jet/backend/table/game_hidden.go create mode 100644 backend/internal/ratewatch/ratewatch.go create mode 100644 backend/internal/ratewatch/ratewatch_test.go create mode 100644 backend/internal/server/handlers_ratelimit.go diff --git a/backend/cmd/backend/main.go b/backend/cmd/backend/main.go index 0fca27e..0854508 100644 --- a/backend/cmd/backend/main.go +++ b/backend/cmd/backend/main.go @@ -28,6 +28,7 @@ import ( "scrabble/backend/internal/notify" "scrabble/backend/internal/postgres" "scrabble/backend/internal/pushgrpc" + "scrabble/backend/internal/ratewatch" "scrabble/backend/internal/robot" "scrabble/backend/internal/server" "scrabble/backend/internal/session" @@ -177,6 +178,13 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error { invitations.SetNotifier(hub) logger.Info("lobby and social domains ready", zap.Duration("robot_wait", cfg.Lobby.RobotWait)) + // R3 rate-limit observability: ingest the gateway's rejection reports for the + // admin throttled view and the conservative high-rate auto-flag. + rateWatch := ratewatch.New(cfg.RateWatch, accounts, logger) + logger.Info("rate watch ready", + zap.Int("flag_threshold", cfg.RateWatch.FlagThreshold), + zap.Duration("flag_window", cfg.RateWatch.FlagWindow)) + srv := server.New(cfg.HTTPAddr, server.Deps{ Logger: logger, DB: db, @@ -193,6 +201,7 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error { Registry: registry, DictDir: cfg.Game.DictDir, Connector: conn, + RateWatch: rateWatch, }) pushSrv := pushgrpc.NewServer(cfg.GRPCAddr, hub, logger) diff --git a/backend/internal/account/account.go b/backend/internal/account/account.go index 8fee67a..975484a 100644 --- a/backend/internal/account/account.go +++ b/backend/internal/account/account.go @@ -76,8 +76,13 @@ type Account struct { // uuid.Nil for a live account. A tombstone keeps the row so the no-cascade // foreign keys of a shared finished game stay valid (Stage 11). MergedInto uuid.UUID - CreatedAt time.Time - UpdatedAt time.Time + // FlaggedHighRateAt is the soft, reversible "suspected high-rate" marker: the + // zero time for an unflagged account, otherwise when the gateway-reported + // rate-limiter rejections first crossed the sustained threshold (R3). An + // operator clears it in the admin console; it never gates any request. + FlaggedHighRateAt time.Time + CreatedAt time.Time + UpdatedAt time.Time } // Identity is one of an account's platform/email identities, surfaced on the @@ -422,6 +427,43 @@ func (s *Store) SpendHint(ctx context.Context, id uuid.UUID) (bool, error) { return n > 0, nil } +// FlagHighRate stamps the soft "suspected high-rate" marker with at, only when +// the account is not already flagged — the first sustained episode wins, and a +// re-flag after an operator clear starts a fresh timestamp. An infra marker, not +// a profile edit, so updated_at is untouched; it never gates any request (R3). +// It reports whether the flag was newly set. +func (s *Store) FlagHighRate(ctx context.Context, id uuid.UUID, at time.Time) (bool, error) { + stmt := table.Accounts. + UPDATE(table.Accounts.FlaggedHighRateAt). + SET(postgres.TimestampzT(at.UTC())). + WHERE( + table.Accounts.AccountID.EQ(postgres.UUID(id)). + AND(table.Accounts.FlaggedHighRateAt.IS_NULL()), + ) + res, err := stmt.ExecContext(ctx, s.db) + if err != nil { + return false, fmt.Errorf("account: flag high rate %s: %w", id, err) + } + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("account: flag high rate rows %s: %w", id, err) + } + return n > 0, nil +} + +// ClearHighRateFlag removes the high-rate marker — the operator's reversible +// action in the admin console. Clearing an unflagged account is a no-op. +func (s *Store) ClearHighRateFlag(ctx context.Context, id uuid.UUID) error { + stmt := table.Accounts. + UPDATE(table.Accounts.FlaggedHighRateAt). + SET(postgres.NULL). + WHERE(table.Accounts.AccountID.EQ(postgres.UUID(id))) + if _, err := stmt.ExecContext(ctx, s.db); err != nil { + return fmt.Errorf("account: clear high-rate flag %s: %w", id, err) + } + return nil +} + // SetServiceLanguage records the service language (en/ru) of the bot a Telegram // user authenticated through. It is called on every Telegram login — new and // existing accounts — so it tracks the bot the user last came through (last-login- @@ -452,6 +494,10 @@ func modelToAccount(row model.Accounts) Account { if row.ServiceLanguage != nil { serviceLanguage = *row.ServiceLanguage } + var flaggedHighRateAt time.Time + if row.FlaggedHighRateAt != nil { + flaggedHighRateAt = *row.FlaggedHighRateAt + } return Account{ ID: row.AccountID, DisplayName: row.DisplayName, @@ -467,6 +513,7 @@ func modelToAccount(row model.Accounts) Account { NotificationsInAppOnly: row.NotificationsInAppOnly, PaidAccount: row.PaidAccount, MergedInto: mergedInto, + FlaggedHighRateAt: flaggedHighRateAt, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt, } diff --git a/backend/internal/account/userlist.go b/backend/internal/account/userlist.go index b2bd372..6f2d877 100644 --- a/backend/internal/account/userlist.go +++ b/backend/internal/account/userlist.go @@ -2,6 +2,7 @@ package account import ( "context" + "database/sql" "fmt" "strings" "time" @@ -18,6 +19,9 @@ type UserListItem struct { PreferredLanguage string IsGuest bool IsRobot bool + // FlaggedHighRateAt is the soft high-rate marker (zero when unflagged), shown + // as a badge in the console list (R3). + FlaggedHighRateAt time.Time CreatedAt time.Time } @@ -65,7 +69,7 @@ func userListWhere(f UserFilter) (string, []any) { // ListUsers returns the filtered admin user list, newest first, paginated. func (s *Store) ListUsers(ctx context.Context, f UserFilter, limit, offset int) ([]UserListItem, error) { where, args := userListWhere(f) - q := `SELECT a.account_id, a.display_name, a.preferred_language, a.is_guest, a.created_at, ` + robotExists + ` AS is_robot + q := `SELECT a.account_id, a.display_name, a.preferred_language, a.is_guest, a.flagged_high_rate_at, a.created_at, ` + robotExists + ` AS is_robot FROM backend.accounts a WHERE ` + where + fmt.Sprintf(` ORDER BY a.created_at DESC LIMIT $%d OFFSET $%d`, len(args)+1, len(args)+2) args = append(args, limit, offset) @@ -77,14 +81,51 @@ FROM backend.accounts a WHERE ` + where + var out []UserListItem for rows.Next() { var it UserListItem - if err := rows.Scan(&it.ID, &it.DisplayName, &it.PreferredLanguage, &it.IsGuest, &it.CreatedAt, &it.IsRobot); err != nil { + var flagged sql.NullTime + if err := rows.Scan(&it.ID, &it.DisplayName, &it.PreferredLanguage, &it.IsGuest, &flagged, &it.CreatedAt, &it.IsRobot); err != nil { return nil, fmt.Errorf("account: scan user: %w", err) } + if flagged.Valid { + it.FlaggedHighRateAt = flagged.Time + } out = append(out, it) } return out, rows.Err() } +// FlaggedAccount is one row of the console's high-rate review queue. +type FlaggedAccount struct { + ID uuid.UUID + DisplayName string + FlaggedHighRateAt time.Time +} + +// flaggedListCap bounds the console's flagged-account list; the operator clears +// flags as they are reviewed, so the queue stays short in practice. +const flaggedListCap = 200 + +// ListFlaggedHighRate returns the accounts carrying the high-rate flag, most +// recently flagged first (R3). +func (s *Store) ListFlaggedHighRate(ctx context.Context) ([]FlaggedAccount, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT account_id, display_name, flagged_high_rate_at +FROM backend.accounts WHERE flagged_high_rate_at IS NOT NULL +ORDER BY flagged_high_rate_at DESC LIMIT $1`, flaggedListCap) + if err != nil { + return nil, fmt.Errorf("account: list flagged: %w", err) + } + defer rows.Close() + var out []FlaggedAccount + for rows.Next() { + var fa FlaggedAccount + if err := rows.Scan(&fa.ID, &fa.DisplayName, &fa.FlaggedHighRateAt); err != nil { + return nil, fmt.Errorf("account: scan flagged: %w", err) + } + out = append(out, fa) + } + return out, rows.Err() +} + // CountUsers counts the filtered admin user list, for pagination. func (s *Store) CountUsers(ctx context.Context, f UserFilter) (int, error) { where, args := userListWhere(f) diff --git a/backend/internal/adminconsole/render_test.go b/backend/internal/adminconsole/render_test.go index 4bdffad..eb6ab9c 100644 --- a/backend/internal/adminconsole/render_test.go +++ b/backend/internal/adminconsole/render_test.go @@ -21,8 +21,14 @@ func TestRendererRendersEveryPage(t *testing.T) { want string }{ {"dashboard", DashboardView{Accounts: 3, Variants: []VariantVersions{{Variant: "scrabble_en", Latest: "v1", Versions: []string{"v1"}}}}, "Dashboard"}, - {"users", UsersView{Items: []UserRow{{ID: "a1", DisplayName: "Kaya"}}, Pager: NewPager(1, 50, 1)}, "Kaya"}, + {"users", UsersView{Items: []UserRow{{ID: "a1", DisplayName: "Kaya", FlaggedHighRate: true}}, Pager: NewPager(1, 50, 1)}, "high-rate"}, {"user_detail", UserDetailView{ID: "a1", DisplayName: "Kaya", HasStats: true, Stats: StatsRow{Wins: 2}, TelegramID: "123", ConnectorEnabled: true}, "Send Telegram message"}, + {"user_detail", UserDetailView{ID: "a1", DisplayName: "Kaya", FlaggedHighRateAt: "2026-06-10 12:00"}, "Clear high-rate flag"}, + {"throttled", ThrottledView{ + Episodes: []ThrottleEpisodeRow{{Class: "user", Key: "a1", UserID: "a1", Rejected: 1234, FirstSeen: "2026-06-10 12:00", LastSeen: "2026-06-10 12:05"}}, + Flagged: []FlaggedAccountRow{{ID: "a1", DisplayName: "Kaya", FlaggedAt: "2026-06-10 12:05"}}, + FlagThreshold: 1000, FlagWindow: "10m0s", + }, "Recent episodes"}, {"games", GamesView{Items: []GameRow{{ID: "g1", Variant: "scrabble_en", Status: "active"}}, Status: "active", Pager: NewPager(1, 50, 1)}, "g1"}, {"game_detail", GameDetailView{ID: "g1", Variant: "scrabble_en", Seats: []SeatRow{{Seat: 0, DisplayName: "Kaya"}}}, "Seats"}, {"complaints", ComplaintsView{Items: []ComplaintRow{{ID: "c1", Word: "qi", Status: "open"}}, Status: "open", Pager: NewPager(1, 50, 1)}, "qi"}, diff --git a/backend/internal/adminconsole/templates/layout.gohtml b/backend/internal/adminconsole/templates/layout.gohtml index 970d876..14e09d9 100644 --- a/backend/internal/adminconsole/templates/layout.gohtml +++ b/backend/internal/adminconsole/templates/layout.gohtml @@ -17,6 +17,7 @@ Games Complaints Messages + Throttled Dictionary Broadcast Grafana ↗ diff --git a/backend/internal/adminconsole/templates/pages/throttled.gohtml b/backend/internal/adminconsole/templates/pages/throttled.gohtml new file mode 100644 index 0000000..696b339 --- /dev/null +++ b/backend/internal/adminconsole/templates/pages/throttled.gohtml @@ -0,0 +1,39 @@ +{{define "content" -}} +

Throttled

+{{with .Data}} +

Rate-limiter rejections reported periodically by the gateway. The episode +list is in-memory and resets on a backend restart. An account sustaining +{{.FlagThreshold}}+ rejected calls within {{.FlagWindow}} is soft-flagged for review +below — never banned automatically; clear the flag on the user card.

+

Recent episodes

+ + + +{{range .Episodes}} + + + + + + + +{{else}} + +{{end}} + +
ClassKeyRejectedFirst seenLast seen
{{.Class}}{{if .UserID}}{{.Key}}{{else}}{{.Key}}{{end}}{{.Rejected}}{{.FirstSeen}}{{.LastSeen}}
nothing throttled recently
+
+

Flagged accounts

+ + + +{{range .Flagged}} + +{{else}} + +{{end}} + +
AccountDisplay nameFlagged
{{.ID}}{{.DisplayName}}{{.FlaggedAt}}
no flagged accounts
+
+{{end}} +{{- end}} diff --git a/backend/internal/adminconsole/templates/pages/user_detail.gohtml b/backend/internal/adminconsole/templates/pages/user_detail.gohtml index 08b75b2..f4396d1 100644 --- a/backend/internal/adminconsole/templates/pages/user_detail.gohtml +++ b/backend/internal/adminconsole/templates/pages/user_detail.gohtml @@ -13,8 +13,14 @@
  • Paid {{if .PaidAccount}}yes{{else}}no{{end}}
  • Hint wallet {{.HintBalance}}
    • {{if .MergedInto}}
  • Merged into {{.MergedInto}}
    • {{end}} +{{if .FlaggedHighRateAt}}
  • High-rate flag {{.FlaggedHighRateAt}}
    • {{end}}
  • Created {{.CreatedAt}}
    • +{{if .FlaggedHighRateAt}} +
    + +
    +{{end}}

    Statistics

    {{if .HasStats}} diff --git a/backend/internal/adminconsole/templates/pages/users.gohtml b/backend/internal/adminconsole/templates/pages/users.gohtml index ef8cf21..9a116a0 100644 --- a/backend/internal/adminconsole/templates/pages/users.gohtml +++ b/backend/internal/adminconsole/templates/pages/users.gohtml @@ -17,7 +17,7 @@ {{range .Items}} {{.ID}} -{{.DisplayName}}{{if .Guest}} guest{{end}} +{{.DisplayName}}{{if .Guest}} guest{{end}}{{if .FlaggedHighRate}} high-rate{{end}} {{.Kind}} {{.Language}} {{.CreatedAt}} diff --git a/backend/internal/adminconsole/views.go b/backend/internal/adminconsole/views.go index 1ef4aed..8aef4c0 100644 --- a/backend/internal/adminconsole/views.go +++ b/backend/internal/adminconsole/views.go @@ -59,18 +59,20 @@ type UsersView struct { } // UserRow is one account row in the list. MoveMin/Avg/Max are the account's -// pre-formatted move-duration summary (empty when it has no timed move). +// pre-formatted move-duration summary (empty when it has no timed move); +// FlaggedHighRate marks the soft high-rate badge (R3). type UserRow struct { - ID string - DisplayName string - Kind string - Language string - Guest bool - CreatedAt string - HasMoveStats bool - MoveMin string - MoveAvg string - MoveMax string + ID string + DisplayName string + Kind string + Language string + Guest bool + FlaggedHighRate bool + CreatedAt string + HasMoveStats bool + MoveMin string + MoveAvg string + MoveMax string } // MessagesView is the paginated chat-message moderation list. NameMask/ExtMask are the @@ -110,15 +112,18 @@ type UserDetailView struct { PaidAccount bool // MergedInto is the primary account id when this account has been retired by a // merge (Stage 11), or empty for a live account. - MergedInto string - HintBalance int - CreatedAt string - HasStats bool - Stats StatsRow - Identities []IdentityRow - Games []GameRow - TelegramID string - ConnectorEnabled bool + MergedInto string + // FlaggedHighRateAt is the pre-formatted soft high-rate marker timestamp, + // empty for an unflagged account; the card shows it with the Clear action (R3). + FlaggedHighRateAt string + HintBalance int + CreatedAt string + HasStats bool + Stats StatsRow + Identities []IdentityRow + Games []GameRow + TelegramID string + ConnectorEnabled bool // MoveChart is the pre-rendered inline SVG of the account's per-move-number think // time (min/mean/max), empty when the account has no timed move. MoveChart template.HTML @@ -247,6 +252,35 @@ type BroadcastView struct { ConnectorEnabled bool } +// ThrottledView is the rate-limit observability page: the recent gateway-reported +// throttle episodes (in-memory, reset on restart) and the accounts currently +// carrying the high-rate flag. FlagThreshold and FlagWindow caption the active +// auto-flag tuning. +type ThrottledView struct { + Episodes []ThrottleEpisodeRow + Flagged []FlaggedAccountRow + FlagThreshold int + FlagWindow string +} + +// ThrottleEpisodeRow is one recently throttled limiter key. UserID links to the +// user card and is set only for the user class (the other classes key by IP). +type ThrottleEpisodeRow struct { + Class string + Key string + UserID string + Rejected int + FirstSeen string + LastSeen string +} + +// FlaggedAccountRow is one account carrying the high-rate flag. +type FlaggedAccountRow struct { + ID string + DisplayName string + FlaggedAt string +} + // MessageView is the result page shown after a POST action. type MessageView struct { Heading string diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 9cc69e0..063911f 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -12,6 +12,7 @@ import ( "scrabble/backend/internal/game" "scrabble/backend/internal/lobby" "scrabble/backend/internal/postgres" + "scrabble/backend/internal/ratewatch" "scrabble/backend/internal/robot" "scrabble/backend/internal/telemetry" ) @@ -35,6 +36,9 @@ type Config struct { Lobby lobby.Config // Robot configures the robot opponent driver (scan cadence). Robot robot.Config + // RateWatch tunes the conservative high-rate auto-flag applied to the + // gateway's rate-limiter rejection reports (R3). + RateWatch ratewatch.Config // SMTP configures the email relay used for confirm-codes. An empty Host // selects the development log mailer (the code is logged, not sent). SMTP account.SMTPConfig @@ -105,6 +109,14 @@ func Load() (Config, error) { return Config{}, err } + rw := ratewatch.DefaultConfig() + if rw.FlagThreshold, err = envInt("BACKEND_HIGHRATE_FLAG_THRESHOLD", rw.FlagThreshold); err != nil { + return Config{}, err + } + if rw.FlagWindow, err = envDuration("BACKEND_HIGHRATE_FLAG_WINDOW", rw.FlagWindow); err != nil { + return Config{}, err + } + guestReapInterval, err := envDuration("BACKEND_GUEST_REAP_INTERVAL", defaultGuestReapInterval) if err != nil { return Config{}, err @@ -131,6 +143,7 @@ func Load() (Config, error) { Game: gm, Lobby: lb, Robot: rb, + RateWatch: rw, SMTP: smtp, ConnectorAddr: os.Getenv("BACKEND_CONNECTOR_ADDR"), GuestReapInterval: guestReapInterval, @@ -170,6 +183,9 @@ func (c Config) validate() error { if err := c.Robot.Validate(); err != nil { return fmt.Errorf("config: %w", err) } + if err := c.RateWatch.Validate(); err != nil { + return fmt.Errorf("config: %w", err) + } if c.GuestReapInterval <= 0 { return fmt.Errorf("config: BACKEND_GUEST_REAP_INTERVAL must be positive") } diff --git a/backend/internal/inttest/account_test.go b/backend/internal/inttest/account_test.go index 1be5922..286b171 100644 --- a/backend/internal/inttest/account_test.go +++ b/backend/internal/inttest/account_test.go @@ -6,6 +6,7 @@ import ( "context" "errors" "testing" + "time" "github.com/google/uuid" @@ -195,6 +196,62 @@ func TestServiceLanguageRoundTrip(t *testing.T) { } } +// TestHighRateFlagRoundTrip covers the R3 soft high-rate marker: a fresh account +// is unflagged, FlagHighRate stamps it exactly once (a second sustained episode +// never moves the timestamp), ClearHighRateFlag reverses it, and a re-flag after +// the operator clear takes a fresh timestamp. +func TestHighRateFlagRoundTrip(t *testing.T) { + ctx := context.Background() + store := account.NewStore(testDB) + acc, err := store.ProvisionTelegram(ctx, "tg-"+uuid.NewString(), "en", "", "Player") + if err != nil { + t.Fatalf("provision telegram: %v", err) + } + if !acc.FlaggedHighRateAt.IsZero() { + t.Fatalf("fresh FlaggedHighRateAt = %v, want zero", acc.FlaggedHighRateAt) + } + + first := time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC) + set, err := store.FlagHighRate(ctx, acc.ID, first) + if err != nil { + t.Fatalf("flag: %v", err) + } + if !set { + t.Fatal("first FlagHighRate reported not set") + } + if set, err = store.FlagHighRate(ctx, acc.ID, first.Add(time.Hour)); err != nil { + t.Fatalf("re-flag: %v", err) + } else if set { + t.Fatal("second FlagHighRate must not overwrite the marker") + } + got, err := store.GetByID(ctx, acc.ID) + if err != nil { + t.Fatalf("get by id: %v", err) + } + if !got.FlaggedHighRateAt.Equal(first) { + t.Errorf("FlaggedHighRateAt = %v, want %v", got.FlaggedHighRateAt, first) + } + + if err := store.ClearHighRateFlag(ctx, acc.ID); err != nil { + t.Fatalf("clear: %v", err) + } + if got, err = store.GetByID(ctx, acc.ID); err != nil { + t.Fatalf("get by id: %v", err) + } else if !got.FlaggedHighRateAt.IsZero() { + t.Errorf("cleared FlaggedHighRateAt = %v, want zero", got.FlaggedHighRateAt) + } + + second := first.Add(24 * time.Hour) + if set, err = store.FlagHighRate(ctx, acc.ID, second); err != nil || !set { + t.Fatalf("re-flag after clear = (%v, %v), want (true, nil)", set, err) + } + if got, err = store.GetByID(ctx, acc.ID); err != nil { + t.Fatalf("get by id: %v", err) + } else if !got.FlaggedHighRateAt.Equal(second) { + t.Errorf("re-flagged FlaggedHighRateAt = %v, want %v", got.FlaggedHighRateAt, second) + } +} + // TestIdentityExternalID covers the reverse identity lookup the push-target route // uses: it returns the external_id for the matching kind and ErrNotFound otherwise, // including for a guest that carries no identity. diff --git a/backend/internal/inttest/admin_test.go b/backend/internal/inttest/admin_test.go index 6b77c97..0171c54 100644 --- a/backend/internal/inttest/admin_test.go +++ b/backend/internal/inttest/admin_test.go @@ -16,6 +16,7 @@ import ( "scrabble/backend/internal/account" "scrabble/backend/internal/engine" "scrabble/backend/internal/game" + "scrabble/backend/internal/ratewatch" "scrabble/backend/internal/server" ) @@ -206,6 +207,72 @@ func TestConsoleGameDetailRobotSchedule(t *testing.T) { } } +// TestConsoleThrottledViewAndFlagClear drives the R3 rate-limit surface end to +// end against real stores: a gateway report past the threshold auto-flags the +// account, the throttled view shows the episode and the flagged account, the +// user card carries the marker, and the operator clear (a same-origin POST) +// reverses it. +func TestConsoleThrottledViewAndFlagClear(t *testing.T) { + ctx := context.Background() + accounts := account.NewStore(testDB) + acc, err := accounts.ProvisionTelegram(ctx, "tg-"+uuid.NewString(), "en", "", "Throttled Player") + if err != nil { + t.Fatalf("provision: %v", err) + } + + watch := ratewatch.New(ratewatch.Config{FlagThreshold: 100, FlagWindow: 10 * time.Minute}, accounts, zap.NewNop()) + srv := server.New(":0", server.Deps{ + Logger: zap.NewNop(), + Accounts: accounts, + Games: newGameService(), + Registry: testRegistry, + DictDir: dictDir(), + RateWatch: watch, + }) + h := srv.Handler() + + report := `{"window_seconds":30,"entries":[` + + `{"class":"user","key":"` + acc.ID.String() + `","rejected":150},` + + `{"class":"public","key":"10.1.2.3","rejected":7}]}` + req := httptest.NewRequest(http.MethodPost, "http://admin.test/api/v1/internal/ratelimit/report", strings.NewReader(report)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("report = %d, want 204", rec.Code) + } + + got, err := accounts.GetByID(ctx, acc.ID) + if err != nil { + t.Fatalf("get by id: %v", err) + } + if got.FlaggedHighRateAt.IsZero() { + t.Fatal("account not auto-flagged past the threshold") + } + + base := "http://admin.test/_gm" + code, body := consoleDo(h, http.MethodGet, base+"/throttled", "", "") + if code != http.StatusOK || !strings.Contains(body, acc.ID.String()) || + !strings.Contains(body, "10.1.2.3") || !strings.Contains(body, "Throttled Player") { + t.Fatalf("throttled view = %d, episode/flag shown = %v/%v", + code, strings.Contains(body, "10.1.2.3"), strings.Contains(body, "Throttled Player")) + } + if code, body = consoleDo(h, http.MethodGet, base+"/users/"+acc.ID.String(), "", ""); code != http.StatusOK || !strings.Contains(body, "Clear high-rate flag") { + t.Fatalf("user card = %d, has clear action = %v", code, strings.Contains(body, "Clear high-rate flag")) + } + + // The clear POST is CSRF-guarded like every console action. + if code, _ = consoleDo(h, http.MethodPost, base+"/users/"+acc.ID.String()+"/clear-high-rate-flag", "", ""); code != http.StatusForbidden { + t.Fatalf("clear without origin = %d, want 403", code) + } + if code, body = consoleDo(h, http.MethodPost, base+"/users/"+acc.ID.String()+"/clear-high-rate-flag", "x=1", "http://admin.test"); code != http.StatusOK || !strings.Contains(body, "Cleared") { + t.Fatalf("clear with origin = %d, has Cleared = %v", code, strings.Contains(body, "Cleared")) + } + if got, err = accounts.GetByID(ctx, acc.ID); err != nil || !got.FlaggedHighRateAt.IsZero() { + t.Fatalf("flag survived the clear: %v (err %v)", got.FlaggedHighRateAt, err) + } +} + // consoleDo issues a request to h, optionally with an Origin header, and returns // the status and body. Form bodies are sent as application/x-www-form-urlencoded. func consoleDo(h http.Handler, method, target, body, origin string) (int, string) { diff --git a/backend/internal/postgres/jet/backend/model/accounts.go b/backend/internal/postgres/jet/backend/model/accounts.go index 2501d3a..ae26769 100644 --- a/backend/internal/postgres/jet/backend/model/accounts.go +++ b/backend/internal/postgres/jet/backend/model/accounts.go @@ -30,4 +30,5 @@ type Accounts struct { MergedInto *uuid.UUID MergedAt *time.Time ServiceLanguage *string + FlaggedHighRateAt *time.Time } diff --git a/backend/internal/postgres/jet/backend/model/game_drafts.go b/backend/internal/postgres/jet/backend/model/game_drafts.go new file mode 100644 index 0000000..c9c32c2 --- /dev/null +++ b/backend/internal/postgres/jet/backend/model/game_drafts.go @@ -0,0 +1,21 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +import ( + "github.com/google/uuid" + "time" +) + +type GameDrafts struct { + GameID uuid.UUID `sql:"primary_key"` + AccountID uuid.UUID `sql:"primary_key"` + RackOrder string + BoardTiles string + UpdatedAt time.Time +} diff --git a/backend/internal/postgres/jet/backend/model/game_hidden.go b/backend/internal/postgres/jet/backend/model/game_hidden.go new file mode 100644 index 0000000..d64f4a1 --- /dev/null +++ b/backend/internal/postgres/jet/backend/model/game_hidden.go @@ -0,0 +1,19 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +import ( + "github.com/google/uuid" + "time" +) + +type GameHidden struct { + AccountID uuid.UUID `sql:"primary_key"` + GameID uuid.UUID `sql:"primary_key"` + CreatedAt time.Time +} diff --git a/backend/internal/postgres/jet/backend/table/accounts.go b/backend/internal/postgres/jet/backend/table/accounts.go index 906a396..021e7a4 100644 --- a/backend/internal/postgres/jet/backend/table/accounts.go +++ b/backend/internal/postgres/jet/backend/table/accounts.go @@ -34,6 +34,7 @@ type accountsTable struct { MergedInto postgres.ColumnString MergedAt postgres.ColumnTimestampz ServiceLanguage postgres.ColumnString + FlaggedHighRateAt postgres.ColumnTimestampz AllColumns postgres.ColumnList MutableColumns postgres.ColumnList @@ -92,8 +93,9 @@ func newAccountsTableImpl(schemaName, tableName, alias string) accountsTable { MergedIntoColumn = postgres.StringColumn("merged_into") MergedAtColumn = postgres.TimestampzColumn("merged_at") ServiceLanguageColumn = postgres.StringColumn("service_language") - allColumns = postgres.ColumnList{AccountIDColumn, DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn, MergedIntoColumn, MergedAtColumn, ServiceLanguageColumn} - mutableColumns = postgres.ColumnList{DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn, MergedIntoColumn, MergedAtColumn, ServiceLanguageColumn} + FlaggedHighRateAtColumn = postgres.TimestampzColumn("flagged_high_rate_at") + allColumns = postgres.ColumnList{AccountIDColumn, DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn, MergedIntoColumn, MergedAtColumn, ServiceLanguageColumn, FlaggedHighRateAtColumn} + mutableColumns = postgres.ColumnList{DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn, MergedIntoColumn, MergedAtColumn, ServiceLanguageColumn, FlaggedHighRateAtColumn} defaultColumns = postgres.ColumnList{DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn} ) @@ -118,6 +120,7 @@ func newAccountsTableImpl(schemaName, tableName, alias string) accountsTable { MergedInto: MergedIntoColumn, MergedAt: MergedAtColumn, ServiceLanguage: ServiceLanguageColumn, + FlaggedHighRateAt: FlaggedHighRateAtColumn, AllColumns: allColumns, MutableColumns: mutableColumns, diff --git a/backend/internal/postgres/jet/backend/table/game_drafts.go b/backend/internal/postgres/jet/backend/table/game_drafts.go new file mode 100644 index 0000000..18f099e --- /dev/null +++ b/backend/internal/postgres/jet/backend/table/game_drafts.go @@ -0,0 +1,90 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var GameDrafts = newGameDraftsTable("backend", "game_drafts", "") + +type gameDraftsTable struct { + postgres.Table + + // Columns + GameID postgres.ColumnString + AccountID postgres.ColumnString + RackOrder postgres.ColumnString + BoardTiles postgres.ColumnString + UpdatedAt postgres.ColumnTimestampz + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList + DefaultColumns postgres.ColumnList +} + +type GameDraftsTable struct { + gameDraftsTable + + EXCLUDED gameDraftsTable +} + +// AS creates new GameDraftsTable with assigned alias +func (a GameDraftsTable) AS(alias string) *GameDraftsTable { + return newGameDraftsTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new GameDraftsTable with assigned schema name +func (a GameDraftsTable) FromSchema(schemaName string) *GameDraftsTable { + return newGameDraftsTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new GameDraftsTable with assigned table prefix +func (a GameDraftsTable) WithPrefix(prefix string) *GameDraftsTable { + return newGameDraftsTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new GameDraftsTable with assigned table suffix +func (a GameDraftsTable) WithSuffix(suffix string) *GameDraftsTable { + return newGameDraftsTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newGameDraftsTable(schemaName, tableName, alias string) *GameDraftsTable { + return &GameDraftsTable{ + gameDraftsTable: newGameDraftsTableImpl(schemaName, tableName, alias), + EXCLUDED: newGameDraftsTableImpl("", "excluded", ""), + } +} + +func newGameDraftsTableImpl(schemaName, tableName, alias string) gameDraftsTable { + var ( + GameIDColumn = postgres.StringColumn("game_id") + AccountIDColumn = postgres.StringColumn("account_id") + RackOrderColumn = postgres.StringColumn("rack_order") + BoardTilesColumn = postgres.StringColumn("board_tiles") + UpdatedAtColumn = postgres.TimestampzColumn("updated_at") + allColumns = postgres.ColumnList{GameIDColumn, AccountIDColumn, RackOrderColumn, BoardTilesColumn, UpdatedAtColumn} + mutableColumns = postgres.ColumnList{RackOrderColumn, BoardTilesColumn, UpdatedAtColumn} + defaultColumns = postgres.ColumnList{RackOrderColumn, BoardTilesColumn, UpdatedAtColumn} + ) + + return gameDraftsTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + GameID: GameIDColumn, + AccountID: AccountIDColumn, + RackOrder: RackOrderColumn, + BoardTiles: BoardTilesColumn, + UpdatedAt: UpdatedAtColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + DefaultColumns: defaultColumns, + } +} diff --git a/backend/internal/postgres/jet/backend/table/game_hidden.go b/backend/internal/postgres/jet/backend/table/game_hidden.go new file mode 100644 index 0000000..d55a2a9 --- /dev/null +++ b/backend/internal/postgres/jet/backend/table/game_hidden.go @@ -0,0 +1,84 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var GameHidden = newGameHiddenTable("backend", "game_hidden", "") + +type gameHiddenTable struct { + postgres.Table + + // Columns + AccountID postgres.ColumnString + GameID postgres.ColumnString + CreatedAt postgres.ColumnTimestampz + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList + DefaultColumns postgres.ColumnList +} + +type GameHiddenTable struct { + gameHiddenTable + + EXCLUDED gameHiddenTable +} + +// AS creates new GameHiddenTable with assigned alias +func (a GameHiddenTable) AS(alias string) *GameHiddenTable { + return newGameHiddenTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new GameHiddenTable with assigned schema name +func (a GameHiddenTable) FromSchema(schemaName string) *GameHiddenTable { + return newGameHiddenTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new GameHiddenTable with assigned table prefix +func (a GameHiddenTable) WithPrefix(prefix string) *GameHiddenTable { + return newGameHiddenTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new GameHiddenTable with assigned table suffix +func (a GameHiddenTable) WithSuffix(suffix string) *GameHiddenTable { + return newGameHiddenTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newGameHiddenTable(schemaName, tableName, alias string) *GameHiddenTable { + return &GameHiddenTable{ + gameHiddenTable: newGameHiddenTableImpl(schemaName, tableName, alias), + EXCLUDED: newGameHiddenTableImpl("", "excluded", ""), + } +} + +func newGameHiddenTableImpl(schemaName, tableName, alias string) gameHiddenTable { + var ( + AccountIDColumn = postgres.StringColumn("account_id") + GameIDColumn = postgres.StringColumn("game_id") + CreatedAtColumn = postgres.TimestampzColumn("created_at") + allColumns = postgres.ColumnList{AccountIDColumn, GameIDColumn, CreatedAtColumn} + mutableColumns = postgres.ColumnList{CreatedAtColumn} + defaultColumns = postgres.ColumnList{CreatedAtColumn} + ) + + return gameHiddenTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + AccountID: AccountIDColumn, + GameID: GameIDColumn, + CreatedAt: CreatedAtColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + DefaultColumns: defaultColumns, + } +} diff --git a/backend/internal/postgres/jet/backend/table/table_use_schema.go b/backend/internal/postgres/jet/backend/table/table_use_schema.go index edc7307..70d757d 100644 --- a/backend/internal/postgres/jet/backend/table/table_use_schema.go +++ b/backend/internal/postgres/jet/backend/table/table_use_schema.go @@ -18,6 +18,8 @@ func UseSchema(schema string) { EmailConfirmations = EmailConfirmations.FromSchema(schema) FriendCodes = FriendCodes.FromSchema(schema) Friendships = Friendships.FromSchema(schema) + GameDrafts = GameDrafts.FromSchema(schema) + GameHidden = GameHidden.FromSchema(schema) GameInvitationInvitees = GameInvitationInvitees.FromSchema(schema) GameInvitations = GameInvitations.FromSchema(schema) GameMoves = GameMoves.FromSchema(schema) diff --git a/backend/internal/postgres/migrations/00001_baseline.sql b/backend/internal/postgres/migrations/00001_baseline.sql index 7a63bef..f33592e 100644 --- a/backend/internal/postgres/migrations/00001_baseline.sql +++ b/backend/internal/postgres/migrations/00001_baseline.sql @@ -35,6 +35,10 @@ CREATE TABLE accounts ( merged_into uuid REFERENCES accounts (account_id) ON DELETE SET NULL, merged_at timestamptz, service_language text CHECK (service_language IN ('en', 'ru')), + -- Soft, reversible "suspected high-rate" marker (R3): set once when the gateway + -- reports sustained rate-limiter rejections past the threshold; an operator + -- clears it in the admin console. Never an automatic ban. + flagged_high_rate_at timestamptz, CONSTRAINT accounts_preferred_language_chk CHECK (preferred_language IN ('en', 'ru')), CONSTRAINT accounts_hint_balance_chk CHECK (hint_balance >= 0) ); diff --git a/backend/internal/ratewatch/ratewatch.go b/backend/internal/ratewatch/ratewatch.go new file mode 100644 index 0000000..f236f78 --- /dev/null +++ b/backend/internal/ratewatch/ratewatch.go @@ -0,0 +1,235 @@ +// Package ratewatch ingests the gateway's periodic rate-limiter rejection +// reports (R3). It keeps an in-memory window of recent throttle episodes for +// the admin console's view and applies the conservative high-rate auto-flag: +// when one account's rejections within the rolling window cross the threshold, +// the account store stamps the soft, reversible flagged_high_rate_at marker +// (set-once; an operator clears it; never an automatic ban). Like the gateway's +// active_users gauge it is single-instance and resets on restart by design — +// the durable part is the account flag, not the episode window. +package ratewatch + +import ( + "context" + "fmt" + "sort" + "sync" + "time" + + "github.com/google/uuid" + "go.uber.org/zap" +) + +// ClassUser is the limiter class whose keys are account ids — the only class +// the auto-flag applies to (the others are keyed by client IP). +const ClassUser = "user" + +const ( + // maxSeries bounds the distinct (class, key) series kept for the console + // view, so a key-spraying client cannot grow the map: past the bound the + // least-recently-throttled series is evicted. + maxSeries = 200 + // minRetention keeps an episode visible in the console for at least an hour + // after its last rejection (longer when the flag window is longer). + minRetention = time.Hour +) + +// Config tunes the conservative high-rate auto-flag. +type Config struct { + // FlagThreshold is the rejected-call count within FlagWindow past which a + // user account is flagged. + FlagThreshold int + // FlagWindow is the rolling window the rejections accumulate over. + FlagWindow time.Duration +} + +// DefaultConfig returns the agreed conservative defaults — 1000 rejected calls +// within a rolling 10 minutes (~1.7/s sustained, far above the client's +// capped-backoff retry noise yet a fraction of an abusive loop). +func DefaultConfig() Config { + return Config{FlagThreshold: 1000, FlagWindow: 10 * time.Minute} +} + +// Validate reports whether the configuration values are acceptable. +func (c Config) Validate() error { + if c.FlagThreshold <= 0 { + return fmt.Errorf("ratewatch: flag threshold must be positive") + } + if c.FlagWindow <= 0 { + return fmt.Errorf("ratewatch: flag window must be positive") + } + return nil +} + +// Flagger stamps the account-level high-rate marker; account.Store satisfies it. +type Flagger interface { + FlagHighRate(ctx context.Context, id uuid.UUID, at time.Time) (bool, error) +} + +// Entry is one reported aggregate: the rejections of one limiter key within one +// gateway report window (the wire mirror of the gateway's rejection summary). +type Entry struct { + Class string + Key string + Rejected int +} + +// Episode is one key's recent-throttle aggregate for the admin view. +type Episode struct { + Class string + Key string + Rejected int + FirstSeen time.Time + LastSeen time.Time +} + +// Watch accumulates reports and applies the auto-flag rule. +type Watch struct { + cfg Config + flagger Flagger + log *zap.Logger + now func() time.Time + + mu sync.Mutex + series map[seriesKey]*series +} + +type seriesKey struct{ class, key string } + +type series struct { + points []point // ascending by time +} + +type point struct { + at time.Time + n int +} + +// New constructs a Watch over flagger with cfg. A nil logger is replaced by a +// no-op one; a nil flagger disables the auto-flag (the view still works). +func New(cfg Config, flagger Flagger, log *zap.Logger) *Watch { + if log == nil { + log = zap.NewNop() + } + return &Watch{ + cfg: cfg, + flagger: flagger, + log: log, + now: time.Now, + series: make(map[seriesKey]*series), + } +} + +// Ingest records one gateway report. Entries with an empty class or key or a +// non-positive count are skipped. When a user-class series crosses the flag +// threshold within the flag window, the account is flagged (the store keeps it +// set-once, so a sustained episode costs one no-op UPDATE per report). +func (w *Watch) Ingest(ctx context.Context, entries []Entry) { + if len(entries) == 0 { + return + } + now := w.now() + var flag []uuid.UUID + w.mu.Lock() + for _, e := range entries { + if e.Class == "" || e.Key == "" || e.Rejected <= 0 { + continue + } + k := seriesKey{class: e.Class, key: e.Key} + s := w.series[k] + if s == nil { + s = &series{} + w.series[k] = s + } + s.points = append(s.points, point{at: now, n: e.Rejected}) + if e.Class == ClassUser && s.sumSince(now.Add(-w.cfg.FlagWindow)) >= w.cfg.FlagThreshold { + if id, err := uuid.Parse(e.Key); err == nil { + flag = append(flag, id) + } + } + } + w.pruneLocked(now) + w.mu.Unlock() + + if w.flagger == nil { + return + } + for _, id := range flag { + set, err := w.flagger.FlagHighRate(ctx, id, now) + switch { + case err != nil: + w.log.Warn("high-rate flag failed", zap.String("account_id", id.String()), zap.Error(err)) + case set: + w.log.Info("account flagged high-rate", + zap.String("account_id", id.String()), + zap.Int("threshold", w.cfg.FlagThreshold), + zap.Duration("window", w.cfg.FlagWindow)) + } + } +} + +// Config returns the active auto-flag tuning (the admin console captions it). +func (w *Watch) Config() Config { return w.cfg } + +// Recent returns the retained throttle episodes, most recently throttled first. +func (w *Watch) Recent() []Episode { + w.mu.Lock() + defer w.mu.Unlock() + out := make([]Episode, 0, len(w.series)) + for k, s := range w.series { + if len(s.points) == 0 { + continue + } + ep := Episode{ + Class: k.class, + Key: k.key, + FirstSeen: s.points[0].at, + LastSeen: s.points[len(s.points)-1].at, + } + for _, p := range s.points { + ep.Rejected += p.n + } + out = append(out, ep) + } + sort.Slice(out, func(i, j int) bool { return out[i].LastSeen.After(out[j].LastSeen) }) + return out +} + +// sumSince totals the points at or after cutoff. +func (s *series) sumSince(cutoff time.Time) int { + sum := 0 + for i := len(s.points) - 1; i >= 0; i-- { + if s.points[i].at.Before(cutoff) { + break + } + sum += s.points[i].n + } + return sum +} + +// pruneLocked drops points past retention, empty series, and — past maxSeries — +// the least-recently-throttled series. The caller holds w.mu. +func (w *Watch) pruneLocked(now time.Time) { + cutoff := now.Add(-max(minRetention, w.cfg.FlagWindow)) + for k, s := range w.series { + i := 0 + for i < len(s.points) && s.points[i].at.Before(cutoff) { + i++ + } + s.points = s.points[i:] + if len(s.points) == 0 { + delete(w.series, k) + } + } + for len(w.series) > maxSeries { + var oldest seriesKey + var oldestAt time.Time + first := true + for k, s := range w.series { + last := s.points[len(s.points)-1].at + if first || last.Before(oldestAt) { + oldest, oldestAt, first = k, last, false + } + } + delete(w.series, oldest) + } +} diff --git a/backend/internal/ratewatch/ratewatch_test.go b/backend/internal/ratewatch/ratewatch_test.go new file mode 100644 index 0000000..0bde791 --- /dev/null +++ b/backend/internal/ratewatch/ratewatch_test.go @@ -0,0 +1,140 @@ +package ratewatch + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/uuid" +) + +// fakeFlagger records flag calls and reports them as newly set. +type fakeFlagger struct { + calls []uuid.UUID +} + +func (f *fakeFlagger) FlagHighRate(_ context.Context, id uuid.UUID, _ time.Time) (bool, error) { + f.calls = append(f.calls, id) + return true, nil +} + +// watchAt returns a Watch with a controllable clock. +func watchAt(cfg Config, flagger Flagger, at *time.Time) *Watch { + w := New(cfg, flagger, nil) + w.now = func() time.Time { return *at } + return w +} + +// TestIngestAggregatesAndRecent verifies episodes accumulate per (class, key), +// invalid entries are skipped, and Recent orders by last rejection. +func TestIngestAggregatesAndRecent(t *testing.T) { + now := time.Date(2026, 6, 10, 12, 0, 0, 0, time.UTC) + w := watchAt(DefaultConfig(), nil, &now) + ctx := context.Background() + + w.Ingest(ctx, []Entry{ + {Class: "public", Key: "10.0.0.1", Rejected: 3}, + {Class: "user", Key: "u-1", Rejected: 5}, + {Class: "", Key: "x", Rejected: 1}, + {Class: "user", Key: "", Rejected: 1}, + {Class: "user", Key: "u-1", Rejected: 0}, + }) + now = now.Add(30 * time.Second) + w.Ingest(ctx, []Entry{{Class: "public", Key: "10.0.0.1", Rejected: 4}}) + + got := w.Recent() + if len(got) != 2 { + t.Fatalf("Recent returned %d episodes, want 2", len(got)) + } + if got[0].Class != "public" || got[0].Key != "10.0.0.1" || got[0].Rejected != 7 { + t.Errorf("first episode = %+v, want public/10.0.0.1 rejected=7", got[0]) + } + if !got[0].LastSeen.After(got[0].FirstSeen) { + t.Errorf("episode span = [%v, %v], want a positive span", got[0].FirstSeen, got[0].LastSeen) + } + if got[1].Class != "user" || got[1].Rejected != 5 { + t.Errorf("second episode = %+v, want user rejected=5", got[1]) + } +} + +// TestAutoFlagThreshold verifies the flag fires only for a user-class series +// crossing the threshold within the window, with a parseable account id. +func TestAutoFlagThreshold(t *testing.T) { + now := time.Date(2026, 6, 10, 12, 0, 0, 0, time.UTC) + flagged := &fakeFlagger{} + id := uuid.New() + w := watchAt(Config{FlagThreshold: 100, FlagWindow: 10 * time.Minute}, flagged, &now) + ctx := context.Background() + + w.Ingest(ctx, []Entry{ + {Class: "user", Key: id.String(), Rejected: 99}, + {Class: "public", Key: "10.0.0.1", Rejected: 1000}, + {Class: "user", Key: "not-a-uuid", Rejected: 1000}, + }) + if len(flagged.calls) != 0 { + t.Fatalf("flagged %v below the threshold", flagged.calls) + } + + now = now.Add(30 * time.Second) + w.Ingest(ctx, []Entry{{Class: "user", Key: id.String(), Rejected: 1}}) + if len(flagged.calls) != 1 || flagged.calls[0] != id { + t.Fatalf("flag calls = %v, want exactly [%s]", flagged.calls, id) + } +} + +// TestAutoFlagWindowExpiry verifies rejections age out of the rolling window. +func TestAutoFlagWindowExpiry(t *testing.T) { + now := time.Date(2026, 6, 10, 12, 0, 0, 0, time.UTC) + flagged := &fakeFlagger{} + id := uuid.New() + w := watchAt(Config{FlagThreshold: 100, FlagWindow: 10 * time.Minute}, flagged, &now) + ctx := context.Background() + + w.Ingest(ctx, []Entry{{Class: "user", Key: id.String(), Rejected: 60}}) + now = now.Add(11 * time.Minute) + w.Ingest(ctx, []Entry{{Class: "user", Key: id.String(), Rejected: 60}}) + if len(flagged.calls) != 0 { + t.Fatalf("flagged %v across an expired window", flagged.calls) + } + now = now.Add(time.Minute) + w.Ingest(ctx, []Entry{{Class: "user", Key: id.String(), Rejected: 50}}) + if len(flagged.calls) != 1 { + t.Fatalf("flag calls = %v, want one in-window crossing", flagged.calls) + } +} + +// TestSeriesBound verifies the episode map stays bounded by evicting the +// least-recently-throttled series. +func TestSeriesBound(t *testing.T) { + now := time.Date(2026, 6, 10, 12, 0, 0, 0, time.UTC) + w := watchAt(DefaultConfig(), nil, &now) + ctx := context.Background() + + for i := range maxSeries + 10 { + now = now.Add(time.Second) + w.Ingest(ctx, []Entry{{Class: "public", Key: fmt.Sprintf("10.0.%d.%d", i/256, i%256), Rejected: 1}}) + } + got := w.Recent() + if len(got) != maxSeries { + t.Fatalf("retained %d series, want %d", len(got), maxSeries) + } + for _, ep := range got { + if ep.Key == "10.0.0.0" { + t.Fatal("the least-recently-throttled series survived the bound") + } + } +} + +// TestConfigValidate covers the tuning guards. +func TestConfigValidate(t *testing.T) { + if err := DefaultConfig().Validate(); err != nil { + t.Errorf("default config invalid: %v", err) + } + if err := (Config{FlagThreshold: 0, FlagWindow: time.Minute}).Validate(); err == nil { + t.Error("zero threshold passed validation") + } + if err := (Config{FlagThreshold: 1, FlagWindow: 0}).Validate(); err == nil { + t.Error("zero window passed validation") + } +} diff --git a/backend/internal/server/handlers.go b/backend/internal/server/handlers.go index f760409..374df86 100644 --- a/backend/internal/server/handlers.go +++ b/backend/internal/server/handlers.go @@ -37,6 +37,11 @@ func (s *Server) registerRoutes() { // before delivering an out-of-app notification. in.POST("/push-target", s.handlePushTarget) } + if s.ratewatch != nil { + // The gateway's periodic rate-limiter rejection summary (R3): feeds the + // admin console's throttled view and the high-rate auto-flag. + s.internal.POST("/ratelimit/report", s.handleRateLimitReport) + } u := s.user if s.accounts != nil { u.GET("/profile", s.handleProfile) @@ -120,10 +125,8 @@ func gameIDParam(c *gin.Context) (uuid.UUID, bool) { // X-Forwarded-For (the first hop), falling back to the direct peer. func clientIP(c *gin.Context) string { if xff := c.GetHeader("X-Forwarded-For"); xff != "" { - if i := strings.IndexByte(xff, ','); i >= 0 { - return strings.TrimSpace(xff[:i]) - } - return strings.TrimSpace(xff) + first, _, _ := strings.Cut(xff, ",") + return strings.TrimSpace(first) } return c.ClientIP() } diff --git a/backend/internal/server/handlers_admin_console.go b/backend/internal/server/handlers_admin_console.go index bfa3758..e86302f 100644 --- a/backend/internal/server/handlers_admin_console.go +++ b/backend/internal/server/handlers_admin_console.go @@ -19,6 +19,7 @@ import ( "scrabble/backend/internal/adminconsole" "scrabble/backend/internal/engine" "scrabble/backend/internal/game" + "scrabble/backend/internal/ratewatch" "scrabble/backend/internal/robot" "scrabble/backend/internal/social" ) @@ -48,6 +49,8 @@ func (s *Server) registerConsole(router *gin.Engine) { gm.GET("/users", s.consoleUsers) gm.GET("/users/:id", s.consoleUserDetail) gm.POST("/users/:id/message", s.consoleUserMessage) + gm.POST("/users/:id/clear-high-rate-flag", s.consoleClearHighRateFlag) + gm.GET("/throttled", s.consoleThrottled) gm.GET("/games", s.consoleGames) gm.GET("/games/:id", s.consoleGameDetail) gm.GET("/complaints", s.consoleComplaints) @@ -117,7 +120,8 @@ func (s *Server) consoleUsers(c *gin.Context) { } view.Items = append(view.Items, adminconsole.UserRow{ ID: it.ID.String(), DisplayName: it.DisplayName, Kind: kind, - Language: it.PreferredLanguage, Guest: it.IsGuest, CreatedAt: fmtTime(it.CreatedAt), + Language: it.PreferredLanguage, Guest: it.IsGuest, + FlaggedHighRate: !it.FlaggedHighRateAt.IsZero(), CreatedAt: fmtTime(it.CreatedAt), }) ids = append(ids, it.ID) } @@ -257,6 +261,9 @@ func (s *Server) consoleUserDetail(c *gin.Context) { if acc.MergedInto != uuid.Nil { view.MergedInto = acc.MergedInto.String() } + if !acc.FlaggedHighRateAt.IsZero() { + view.FlaggedHighRateAt = fmtTime(acc.FlaggedHighRateAt) + } if view.HasStats { if st, err := s.accounts.GetStats(ctx, id); err == nil { view.Stats = adminconsole.StatsRow{Wins: st.Wins, Losses: st.Losses, Draws: st.Draws, MaxGamePoints: st.MaxGamePoints, MaxWordPoints: st.MaxWordPoints} @@ -551,6 +558,56 @@ func (s *Server) consolePostBroadcast(c *gin.Context) { } } +// consoleThrottled renders the rate-limit observability page: the recent +// gateway-reported throttle episodes (in-memory, reset on a backend restart) +// and the accounts currently carrying the soft high-rate flag (R3). +func (s *Server) consoleThrottled(c *gin.Context) { + ctx := c.Request.Context() + var view adminconsole.ThrottledView + if s.ratewatch != nil { + cfg := s.ratewatch.Config() + view.FlagThreshold = cfg.FlagThreshold + view.FlagWindow = cfg.FlagWindow.String() + for _, ep := range s.ratewatch.Recent() { + row := adminconsole.ThrottleEpisodeRow{ + Class: ep.Class, Key: ep.Key, Rejected: ep.Rejected, + FirstSeen: fmtTime(ep.FirstSeen), LastSeen: fmtTime(ep.LastSeen), + } + if ep.Class == ratewatch.ClassUser { + if id, err := uuid.Parse(ep.Key); err == nil { + row.UserID = id.String() + } + } + view.Episodes = append(view.Episodes, row) + } + } + flagged, err := s.accounts.ListFlaggedHighRate(ctx) + if err != nil { + s.consoleError(c, err) + return + } + for _, fa := range flagged { + view.Flagged = append(view.Flagged, adminconsole.FlaggedAccountRow{ + ID: fa.ID.String(), DisplayName: fa.DisplayName, FlaggedAt: fmtTime(fa.FlaggedHighRateAt), + }) + } + s.renderConsole(c, "throttled", "throttled", "Throttled", view) +} + +// consoleClearHighRateFlag clears the soft high-rate marker — the operator's +// reversible review action (R3). +func (s *Server) consoleClearHighRateFlag(c *gin.Context) { + id, ok := s.consoleUUID(c, "/_gm/users") + if !ok { + return + } + if err := s.accounts.ClearHighRateFlag(c.Request.Context(), id); err != nil { + s.consoleError(c, err) + return + } + s.renderConsoleMessage(c, "Cleared", "high-rate flag cleared", "/_gm/users/"+id.String()) +} + // variantVersions builds the per-variant resident-version summary from the registry. func (s *Server) variantVersions() []adminconsole.VariantVersions { out := make([]adminconsole.VariantVersions, 0, len(engine.Variants())) diff --git a/backend/internal/server/handlers_ratelimit.go b/backend/internal/server/handlers_ratelimit.go new file mode 100644 index 0000000..512a9ae --- /dev/null +++ b/backend/internal/server/handlers_ratelimit.go @@ -0,0 +1,41 @@ +package server + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "scrabble/backend/internal/ratewatch" +) + +// rateLimitReportRequest mirrors the gateway's periodic rejection summary: every +// entry aggregates one limiter key (class + key) over the report window. +type rateLimitReportRequest struct { + WindowSeconds int `json:"window_seconds"` + Entries []rateLimitReportEntry `json:"entries"` +} + +// rateLimitReportEntry is one (class, key) aggregate of the report. +type rateLimitReportEntry struct { + Class string `json:"class"` + Key string `json:"key"` + Rejected int `json:"rejected"` +} + +// handleRateLimitReport ingests one gateway rejection report into the rate +// watch — the admin console's throttled view and the high-rate auto-flag (R3). +// Internal, gateway-only: like sessions/resolve it trusts the network segment. +// Malformed individual entries are skipped by the watch itself. +func (s *Server) handleRateLimitReport(c *gin.Context) { + var req rateLimitReportRequest + if err := c.ShouldBindJSON(&req); err != nil { + abortBadRequest(c, "invalid rate-limit report") + return + } + entries := make([]ratewatch.Entry, 0, len(req.Entries)) + for _, e := range req.Entries { + entries = append(entries, ratewatch.Entry{Class: e.Class, Key: e.Key, Rejected: e.Rejected}) + } + s.ratewatch.Ingest(c.Request.Context(), entries) + c.Status(http.StatusNoContent) +} diff --git a/backend/internal/server/handlers_test.go b/backend/internal/server/handlers_test.go index d3257dd..c8e086a 100644 --- a/backend/internal/server/handlers_test.go +++ b/backend/internal/server/handlers_test.go @@ -10,6 +10,7 @@ import ( "scrabble/backend/internal/account" "scrabble/backend/internal/game" + "scrabble/backend/internal/ratewatch" "scrabble/backend/internal/session" ) @@ -57,6 +58,23 @@ func TestResolveSessionRejectsEmptyToken(t *testing.T) { } } +// TestRateLimitReportEndpoint covers the internal R3 report route: a malformed +// body is a 400, a valid report lands in the rate watch with 204. +func TestRateLimitReportEndpoint(t *testing.T) { + watch := ratewatch.New(ratewatch.DefaultConfig(), nil, nil) + s := New(":0", Deps{RateWatch: watch}) + if rec := do(t, s, http.MethodPost, "/api/v1/internal/ratelimit/report", `{bad`, nil); rec.Code != http.StatusBadRequest { + t.Fatalf("malformed report = %d, want 400", rec.Code) + } + body := `{"window_seconds":30,"entries":[{"class":"user","key":"` + uuid.NewString() + `","rejected":7}]}` + if rec := do(t, s, http.MethodPost, "/api/v1/internal/ratelimit/report", body, nil); rec.Code != http.StatusNoContent { + t.Fatalf("report = %d, want 204", rec.Code) + } + if eps := watch.Recent(); len(eps) != 1 || eps[0].Rejected != 7 { + t.Fatalf("watch episodes = %+v, want one entry with rejected=7", eps) + } +} + func TestSubmitPlayRejectsBadDirection(t *testing.T) { headers := map[string]string{"X-User-ID": uuid.New().String()} path := "/api/v1/user/games/" + uuid.New().String() + "/play" diff --git a/backend/internal/server/server.go b/backend/internal/server/server.go index 9c914f7..618efdc 100644 --- a/backend/internal/server/server.go +++ b/backend/internal/server/server.go @@ -24,6 +24,7 @@ import ( "scrabble/backend/internal/game" "scrabble/backend/internal/link" "scrabble/backend/internal/lobby" + "scrabble/backend/internal/ratewatch" "scrabble/backend/internal/session" "scrabble/backend/internal/social" "scrabble/backend/internal/telemetry" @@ -71,6 +72,10 @@ type Deps struct { // nil when BACKEND_CONNECTOR_ADDR is unset (broadcasts show a "not configured" // notice). Connector *connector.Client + // RateWatch ingests the gateway's rate-limiter rejection reports (R3): the + // admin console's throttled view + the high-rate auto-flag. A nil RateWatch + // disables the internal report endpoint and the console view. + RateWatch *ratewatch.Watch } // Server owns the gin engine, the underlying HTTP server and the readiness @@ -93,6 +98,7 @@ type Server struct { registry *engine.Registry dictDir string connector *connector.Client + ratewatch *ratewatch.Watch console *adminconsole.Renderer public *gin.RouterGroup @@ -133,6 +139,7 @@ func New(addr string, deps Deps) *Server { registry: deps.Registry, dictDir: deps.DictDir, connector: deps.Connector, + ratewatch: deps.RateWatch, http: &http.Server{Addr: addr, Handler: engine}, } s.registerProbes(engine)