R3: backend rate-limit observability — ratewatch, auto-flag, admin throttled view

- accounts.flagged_high_rate_at baked into the R1 baseline (no prod data; the
  contour schema is wiped after merge); jet regenerated — the regen also picks
  up the previously missing game_drafts/game_hidden models.
- account.Store: FlagHighRate (set-once), ClearHighRateFlag, the flag in
  GetByID/ListUsers and a ListFlaggedHighRate review queue.
- New internal/ratewatch: ingests the gateway rejection reports, keeps a
  bounded in-memory episode window for the console and applies the
  conservative auto-flag (1000 rejected / 10 min, BACKEND_HIGHRATE_FLAG_*).
- POST /api/v1/internal/ratelimit/report (network-trusted, like
  sessions/resolve).
- Admin console: Throttled page (episodes + flagged accounts), a high-rate
  badge in the user list, the marker + operator clear action on the user card.
- Tests: ratewatch unit suite, report-route handler test, renderer cases,
  integration coverage for the store round-trip and the console flow.
This commit is contained in:
Ilia Denisov
2026-06-10 02:14:00 +02:00
parent 8878711cf3
commit ab58062565
27 changed files with 1081 additions and 33 deletions
+9
View File
@@ -28,6 +28,7 @@ import (
"scrabble/backend/internal/notify"
"scrabble/backend/internal/postgres"
"scrabble/backend/internal/pushgrpc"
"scrabble/backend/internal/ratewatch"
"scrabble/backend/internal/robot"
"scrabble/backend/internal/server"
"scrabble/backend/internal/session"
@@ -177,6 +178,13 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
invitations.SetNotifier(hub)
logger.Info("lobby and social domains ready", zap.Duration("robot_wait", cfg.Lobby.RobotWait))
// R3 rate-limit observability: ingest the gateway's rejection reports for the
// admin throttled view and the conservative high-rate auto-flag.
rateWatch := ratewatch.New(cfg.RateWatch, accounts, logger)
logger.Info("rate watch ready",
zap.Int("flag_threshold", cfg.RateWatch.FlagThreshold),
zap.Duration("flag_window", cfg.RateWatch.FlagWindow))
srv := server.New(cfg.HTTPAddr, server.Deps{
Logger: logger,
DB: db,
@@ -193,6 +201,7 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
Registry: registry,
DictDir: cfg.Game.DictDir,
Connector: conn,
RateWatch: rateWatch,
})
pushSrv := pushgrpc.NewServer(cfg.GRPCAddr, hub, logger)
+49 -2
View File
@@ -76,8 +76,13 @@ type Account struct {
// uuid.Nil for a live account. A tombstone keeps the row so the no-cascade
// foreign keys of a shared finished game stay valid (Stage 11).
MergedInto uuid.UUID
CreatedAt time.Time
UpdatedAt time.Time
// FlaggedHighRateAt is the soft, reversible "suspected high-rate" marker: the
// zero time for an unflagged account, otherwise when the gateway-reported
// rate-limiter rejections first crossed the sustained threshold (R3). An
// operator clears it in the admin console; it never gates any request.
FlaggedHighRateAt time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
// Identity is one of an account's platform/email identities, surfaced on the
@@ -422,6 +427,43 @@ func (s *Store) SpendHint(ctx context.Context, id uuid.UUID) (bool, error) {
return n > 0, nil
}
// FlagHighRate stamps the soft "suspected high-rate" marker with at, only when
// the account is not already flagged — the first sustained episode wins, and a
// re-flag after an operator clear starts a fresh timestamp. An infra marker, not
// a profile edit, so updated_at is untouched; it never gates any request (R3).
// It reports whether the flag was newly set.
func (s *Store) FlagHighRate(ctx context.Context, id uuid.UUID, at time.Time) (bool, error) {
stmt := table.Accounts.
UPDATE(table.Accounts.FlaggedHighRateAt).
SET(postgres.TimestampzT(at.UTC())).
WHERE(
table.Accounts.AccountID.EQ(postgres.UUID(id)).
AND(table.Accounts.FlaggedHighRateAt.IS_NULL()),
)
res, err := stmt.ExecContext(ctx, s.db)
if err != nil {
return false, fmt.Errorf("account: flag high rate %s: %w", id, err)
}
n, err := res.RowsAffected()
if err != nil {
return false, fmt.Errorf("account: flag high rate rows %s: %w", id, err)
}
return n > 0, nil
}
// ClearHighRateFlag removes the high-rate marker — the operator's reversible
// action in the admin console. Clearing an unflagged account is a no-op.
func (s *Store) ClearHighRateFlag(ctx context.Context, id uuid.UUID) error {
stmt := table.Accounts.
UPDATE(table.Accounts.FlaggedHighRateAt).
SET(postgres.NULL).
WHERE(table.Accounts.AccountID.EQ(postgres.UUID(id)))
if _, err := stmt.ExecContext(ctx, s.db); err != nil {
return fmt.Errorf("account: clear high-rate flag %s: %w", id, err)
}
return nil
}
// SetServiceLanguage records the service language (en/ru) of the bot a Telegram
// user authenticated through. It is called on every Telegram login — new and
// existing accounts — so it tracks the bot the user last came through (last-login-
@@ -452,6 +494,10 @@ func modelToAccount(row model.Accounts) Account {
if row.ServiceLanguage != nil {
serviceLanguage = *row.ServiceLanguage
}
var flaggedHighRateAt time.Time
if row.FlaggedHighRateAt != nil {
flaggedHighRateAt = *row.FlaggedHighRateAt
}
return Account{
ID: row.AccountID,
DisplayName: row.DisplayName,
@@ -467,6 +513,7 @@ func modelToAccount(row model.Accounts) Account {
NotificationsInAppOnly: row.NotificationsInAppOnly,
PaidAccount: row.PaidAccount,
MergedInto: mergedInto,
FlaggedHighRateAt: flaggedHighRateAt,
CreatedAt: row.CreatedAt,
UpdatedAt: row.UpdatedAt,
}
+43 -2
View File
@@ -2,6 +2,7 @@ package account
import (
"context"
"database/sql"
"fmt"
"strings"
"time"
@@ -18,6 +19,9 @@ type UserListItem struct {
PreferredLanguage string
IsGuest bool
IsRobot bool
// FlaggedHighRateAt is the soft high-rate marker (zero when unflagged), shown
// as a badge in the console list (R3).
FlaggedHighRateAt time.Time
CreatedAt time.Time
}
@@ -65,7 +69,7 @@ func userListWhere(f UserFilter) (string, []any) {
// ListUsers returns the filtered admin user list, newest first, paginated.
func (s *Store) ListUsers(ctx context.Context, f UserFilter, limit, offset int) ([]UserListItem, error) {
where, args := userListWhere(f)
q := `SELECT a.account_id, a.display_name, a.preferred_language, a.is_guest, a.created_at, ` + robotExists + ` AS is_robot
q := `SELECT a.account_id, a.display_name, a.preferred_language, a.is_guest, a.flagged_high_rate_at, a.created_at, ` + robotExists + ` AS is_robot
FROM backend.accounts a WHERE ` + where +
fmt.Sprintf(` ORDER BY a.created_at DESC LIMIT $%d OFFSET $%d`, len(args)+1, len(args)+2)
args = append(args, limit, offset)
@@ -77,14 +81,51 @@ FROM backend.accounts a WHERE ` + where +
var out []UserListItem
for rows.Next() {
var it UserListItem
if err := rows.Scan(&it.ID, &it.DisplayName, &it.PreferredLanguage, &it.IsGuest, &it.CreatedAt, &it.IsRobot); err != nil {
var flagged sql.NullTime
if err := rows.Scan(&it.ID, &it.DisplayName, &it.PreferredLanguage, &it.IsGuest, &flagged, &it.CreatedAt, &it.IsRobot); err != nil {
return nil, fmt.Errorf("account: scan user: %w", err)
}
if flagged.Valid {
it.FlaggedHighRateAt = flagged.Time
}
out = append(out, it)
}
return out, rows.Err()
}
// FlaggedAccount is one row of the console's high-rate review queue.
type FlaggedAccount struct {
ID uuid.UUID
DisplayName string
FlaggedHighRateAt time.Time
}
// flaggedListCap bounds the console's flagged-account list; the operator clears
// flags as they are reviewed, so the queue stays short in practice.
const flaggedListCap = 200
// ListFlaggedHighRate returns the accounts carrying the high-rate flag, most
// recently flagged first (R3).
func (s *Store) ListFlaggedHighRate(ctx context.Context) ([]FlaggedAccount, error) {
rows, err := s.db.QueryContext(ctx,
`SELECT account_id, display_name, flagged_high_rate_at
FROM backend.accounts WHERE flagged_high_rate_at IS NOT NULL
ORDER BY flagged_high_rate_at DESC LIMIT $1`, flaggedListCap)
if err != nil {
return nil, fmt.Errorf("account: list flagged: %w", err)
}
defer rows.Close()
var out []FlaggedAccount
for rows.Next() {
var fa FlaggedAccount
if err := rows.Scan(&fa.ID, &fa.DisplayName, &fa.FlaggedHighRateAt); err != nil {
return nil, fmt.Errorf("account: scan flagged: %w", err)
}
out = append(out, fa)
}
return out, rows.Err()
}
// CountUsers counts the filtered admin user list, for pagination.
func (s *Store) CountUsers(ctx context.Context, f UserFilter) (int, error) {
where, args := userListWhere(f)
+7 -1
View File
@@ -21,8 +21,14 @@ func TestRendererRendersEveryPage(t *testing.T) {
want string
}{
{"dashboard", DashboardView{Accounts: 3, Variants: []VariantVersions{{Variant: "scrabble_en", Latest: "v1", Versions: []string{"v1"}}}}, "Dashboard"},
{"users", UsersView{Items: []UserRow{{ID: "a1", DisplayName: "Kaya"}}, Pager: NewPager(1, 50, 1)}, "Kaya"},
{"users", UsersView{Items: []UserRow{{ID: "a1", DisplayName: "Kaya", FlaggedHighRate: true}}, Pager: NewPager(1, 50, 1)}, "high-rate"},
{"user_detail", UserDetailView{ID: "a1", DisplayName: "Kaya", HasStats: true, Stats: StatsRow{Wins: 2}, TelegramID: "123", ConnectorEnabled: true}, "Send Telegram message"},
{"user_detail", UserDetailView{ID: "a1", DisplayName: "Kaya", FlaggedHighRateAt: "2026-06-10 12:00"}, "Clear high-rate flag"},
{"throttled", ThrottledView{
Episodes: []ThrottleEpisodeRow{{Class: "user", Key: "a1", UserID: "a1", Rejected: 1234, FirstSeen: "2026-06-10 12:00", LastSeen: "2026-06-10 12:05"}},
Flagged: []FlaggedAccountRow{{ID: "a1", DisplayName: "Kaya", FlaggedAt: "2026-06-10 12:05"}},
FlagThreshold: 1000, FlagWindow: "10m0s",
}, "Recent episodes"},
{"games", GamesView{Items: []GameRow{{ID: "g1", Variant: "scrabble_en", Status: "active"}}, Status: "active", Pager: NewPager(1, 50, 1)}, "g1"},
{"game_detail", GameDetailView{ID: "g1", Variant: "scrabble_en", Seats: []SeatRow{{Seat: 0, DisplayName: "Kaya"}}}, "Seats"},
{"complaints", ComplaintsView{Items: []ComplaintRow{{ID: "c1", Word: "qi", Status: "open"}}, Status: "open", Pager: NewPager(1, 50, 1)}, "qi"},
@@ -17,6 +17,7 @@
<a href="/_gm/games"{{if eq .ActiveNav "games"}} class="active"{{end}}>Games</a>
<a href="/_gm/complaints"{{if eq .ActiveNav "complaints"}} class="active"{{end}}>Complaints</a>
<a href="/_gm/messages"{{if eq .ActiveNav "messages"}} class="active"{{end}}>Messages</a>
<a href="/_gm/throttled"{{if eq .ActiveNav "throttled"}} class="active"{{end}}>Throttled</a>
<a href="/_gm/dictionary"{{if eq .ActiveNav "dictionary"}} class="active"{{end}}>Dictionary</a>
<a href="/_gm/broadcast"{{if eq .ActiveNav "broadcast"}} class="active"{{end}}>Broadcast</a>
<a href="/_gm/grafana/">Grafana ↗</a>
@@ -0,0 +1,39 @@
{{define "content" -}}
<h1>Throttled</h1>
{{with .Data}}
<p class="note">Rate-limiter rejections reported periodically by the gateway. The episode
list is in-memory and resets on a backend restart. An account sustaining
{{.FlagThreshold}}+ rejected calls within {{.FlagWindow}} is soft-flagged for review
below — never banned automatically; clear the flag on the user card.</p>
<section class="panel"><h2>Recent episodes</h2>
<table class="list">
<thead><tr><th>Class</th><th>Key</th><th class="num">Rejected</th><th>First seen</th><th>Last seen</th></tr></thead>
<tbody>
{{range .Episodes}}
<tr>
<td>{{.Class}}</td>
<td>{{if .UserID}}<a href="/_gm/users/{{.UserID}}">{{.Key}}</a>{{else}}<code>{{.Key}}</code>{{end}}</td>
<td class="num">{{.Rejected}}</td>
<td>{{.FirstSeen}}</td>
<td>{{.LastSeen}}</td>
</tr>
{{else}}
<tr><td colspan="5"><span class="note">nothing throttled recently</span></td></tr>
{{end}}
</tbody>
</table>
</section>
<section class="panel"><h2>Flagged accounts</h2>
<table class="list">
<thead><tr><th>Account</th><th>Display name</th><th>Flagged</th></tr></thead>
<tbody>
{{range .Flagged}}
<tr><td><a href="/_gm/users/{{.ID}}">{{.ID}}</a></td><td>{{.DisplayName}}</td><td>{{.FlaggedAt}}</td></tr>
{{else}}
<tr><td colspan="3"><span class="note">no flagged accounts</span></td></tr>
{{end}}
</tbody>
</table>
</section>
{{end}}
{{- end}}
@@ -13,8 +13,14 @@
<li><b>Paid</b> {{if .PaidAccount}}yes{{else}}no{{end}}</li>
<li><b>Hint wallet</b> {{.HintBalance}}</li>
{{if .MergedInto}}<li><b>Merged into</b> {{.MergedInto}}</li>{{end}}
{{if .FlaggedHighRateAt}}<li><b>High-rate flag</b> <span class="warn">{{.FlaggedHighRateAt}}</span></li>{{end}}
<li><b>Created</b> {{.CreatedAt}}</li>
</ul>
{{if .FlaggedHighRateAt}}
<form class="form" method="post" action="/_gm/users/{{.ID}}/clear-high-rate-flag">
<button type="submit">Clear high-rate flag</button>
</form>
{{end}}
</section>
<section class="panel"><h2>Statistics</h2>
{{if .HasStats}}
@@ -17,7 +17,7 @@
{{range .Items}}
<tr>
<td><a href="/_gm/users/{{.ID}}">{{.ID}}</a></td>
<td>{{.DisplayName}}{{if .Guest}} <span class="pill">guest</span>{{end}}</td>
<td>{{.DisplayName}}{{if .Guest}} <span class="pill">guest</span>{{end}}{{if .FlaggedHighRate}} <span class="pill">high-rate</span>{{end}}</td>
<td>{{.Kind}}</td>
<td>{{.Language}}</td>
<td>{{.CreatedAt}}</td>
+54 -20
View File
@@ -59,18 +59,20 @@ type UsersView struct {
}
// UserRow is one account row in the list. MoveMin/Avg/Max are the account's
// pre-formatted move-duration summary (empty when it has no timed move).
// pre-formatted move-duration summary (empty when it has no timed move);
// FlaggedHighRate marks the soft high-rate badge (R3).
type UserRow struct {
ID string
DisplayName string
Kind string
Language string
Guest bool
CreatedAt string
HasMoveStats bool
MoveMin string
MoveAvg string
MoveMax string
ID string
DisplayName string
Kind string
Language string
Guest bool
FlaggedHighRate bool
CreatedAt string
HasMoveStats bool
MoveMin string
MoveAvg string
MoveMax string
}
// MessagesView is the paginated chat-message moderation list. NameMask/ExtMask are the
@@ -110,15 +112,18 @@ type UserDetailView struct {
PaidAccount bool
// MergedInto is the primary account id when this account has been retired by a
// merge (Stage 11), or empty for a live account.
MergedInto string
HintBalance int
CreatedAt string
HasStats bool
Stats StatsRow
Identities []IdentityRow
Games []GameRow
TelegramID string
ConnectorEnabled bool
MergedInto string
// FlaggedHighRateAt is the pre-formatted soft high-rate marker timestamp,
// empty for an unflagged account; the card shows it with the Clear action (R3).
FlaggedHighRateAt string
HintBalance int
CreatedAt string
HasStats bool
Stats StatsRow
Identities []IdentityRow
Games []GameRow
TelegramID string
ConnectorEnabled bool
// MoveChart is the pre-rendered inline SVG of the account's per-move-number think
// time (min/mean/max), empty when the account has no timed move.
MoveChart template.HTML
@@ -247,6 +252,35 @@ type BroadcastView struct {
ConnectorEnabled bool
}
// ThrottledView is the rate-limit observability page: the recent gateway-reported
// throttle episodes (in-memory, reset on restart) and the accounts currently
// carrying the high-rate flag. FlagThreshold and FlagWindow caption the active
// auto-flag tuning.
type ThrottledView struct {
Episodes []ThrottleEpisodeRow
Flagged []FlaggedAccountRow
FlagThreshold int
FlagWindow string
}
// ThrottleEpisodeRow is one recently throttled limiter key. UserID links to the
// user card and is set only for the user class (the other classes key by IP).
type ThrottleEpisodeRow struct {
Class string
Key string
UserID string
Rejected int
FirstSeen string
LastSeen string
}
// FlaggedAccountRow is one account carrying the high-rate flag.
type FlaggedAccountRow struct {
ID string
DisplayName string
FlaggedAt string
}
// MessageView is the result page shown after a POST action.
type MessageView struct {
Heading string
+16
View File
@@ -12,6 +12,7 @@ import (
"scrabble/backend/internal/game"
"scrabble/backend/internal/lobby"
"scrabble/backend/internal/postgres"
"scrabble/backend/internal/ratewatch"
"scrabble/backend/internal/robot"
"scrabble/backend/internal/telemetry"
)
@@ -35,6 +36,9 @@ type Config struct {
Lobby lobby.Config
// Robot configures the robot opponent driver (scan cadence).
Robot robot.Config
// RateWatch tunes the conservative high-rate auto-flag applied to the
// gateway's rate-limiter rejection reports (R3).
RateWatch ratewatch.Config
// SMTP configures the email relay used for confirm-codes. An empty Host
// selects the development log mailer (the code is logged, not sent).
SMTP account.SMTPConfig
@@ -105,6 +109,14 @@ func Load() (Config, error) {
return Config{}, err
}
rw := ratewatch.DefaultConfig()
if rw.FlagThreshold, err = envInt("BACKEND_HIGHRATE_FLAG_THRESHOLD", rw.FlagThreshold); err != nil {
return Config{}, err
}
if rw.FlagWindow, err = envDuration("BACKEND_HIGHRATE_FLAG_WINDOW", rw.FlagWindow); err != nil {
return Config{}, err
}
guestReapInterval, err := envDuration("BACKEND_GUEST_REAP_INTERVAL", defaultGuestReapInterval)
if err != nil {
return Config{}, err
@@ -131,6 +143,7 @@ func Load() (Config, error) {
Game: gm,
Lobby: lb,
Robot: rb,
RateWatch: rw,
SMTP: smtp,
ConnectorAddr: os.Getenv("BACKEND_CONNECTOR_ADDR"),
GuestReapInterval: guestReapInterval,
@@ -170,6 +183,9 @@ func (c Config) validate() error {
if err := c.Robot.Validate(); err != nil {
return fmt.Errorf("config: %w", err)
}
if err := c.RateWatch.Validate(); err != nil {
return fmt.Errorf("config: %w", err)
}
if c.GuestReapInterval <= 0 {
return fmt.Errorf("config: BACKEND_GUEST_REAP_INTERVAL must be positive")
}
+57
View File
@@ -6,6 +6,7 @@ import (
"context"
"errors"
"testing"
"time"
"github.com/google/uuid"
@@ -195,6 +196,62 @@ func TestServiceLanguageRoundTrip(t *testing.T) {
}
}
// TestHighRateFlagRoundTrip covers the R3 soft high-rate marker: a fresh account
// is unflagged, FlagHighRate stamps it exactly once (a second sustained episode
// never moves the timestamp), ClearHighRateFlag reverses it, and a re-flag after
// the operator clear takes a fresh timestamp.
func TestHighRateFlagRoundTrip(t *testing.T) {
ctx := context.Background()
store := account.NewStore(testDB)
acc, err := store.ProvisionTelegram(ctx, "tg-"+uuid.NewString(), "en", "", "Player")
if err != nil {
t.Fatalf("provision telegram: %v", err)
}
if !acc.FlaggedHighRateAt.IsZero() {
t.Fatalf("fresh FlaggedHighRateAt = %v, want zero", acc.FlaggedHighRateAt)
}
first := time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC)
set, err := store.FlagHighRate(ctx, acc.ID, first)
if err != nil {
t.Fatalf("flag: %v", err)
}
if !set {
t.Fatal("first FlagHighRate reported not set")
}
if set, err = store.FlagHighRate(ctx, acc.ID, first.Add(time.Hour)); err != nil {
t.Fatalf("re-flag: %v", err)
} else if set {
t.Fatal("second FlagHighRate must not overwrite the marker")
}
got, err := store.GetByID(ctx, acc.ID)
if err != nil {
t.Fatalf("get by id: %v", err)
}
if !got.FlaggedHighRateAt.Equal(first) {
t.Errorf("FlaggedHighRateAt = %v, want %v", got.FlaggedHighRateAt, first)
}
if err := store.ClearHighRateFlag(ctx, acc.ID); err != nil {
t.Fatalf("clear: %v", err)
}
if got, err = store.GetByID(ctx, acc.ID); err != nil {
t.Fatalf("get by id: %v", err)
} else if !got.FlaggedHighRateAt.IsZero() {
t.Errorf("cleared FlaggedHighRateAt = %v, want zero", got.FlaggedHighRateAt)
}
second := first.Add(24 * time.Hour)
if set, err = store.FlagHighRate(ctx, acc.ID, second); err != nil || !set {
t.Fatalf("re-flag after clear = (%v, %v), want (true, nil)", set, err)
}
if got, err = store.GetByID(ctx, acc.ID); err != nil {
t.Fatalf("get by id: %v", err)
} else if !got.FlaggedHighRateAt.Equal(second) {
t.Errorf("re-flagged FlaggedHighRateAt = %v, want %v", got.FlaggedHighRateAt, second)
}
}
// TestIdentityExternalID covers the reverse identity lookup the push-target route
// uses: it returns the external_id for the matching kind and ErrNotFound otherwise,
// including for a guest that carries no identity.
+67
View File
@@ -16,6 +16,7 @@ import (
"scrabble/backend/internal/account"
"scrabble/backend/internal/engine"
"scrabble/backend/internal/game"
"scrabble/backend/internal/ratewatch"
"scrabble/backend/internal/server"
)
@@ -206,6 +207,72 @@ func TestConsoleGameDetailRobotSchedule(t *testing.T) {
}
}
// TestConsoleThrottledViewAndFlagClear drives the R3 rate-limit surface end to
// end against real stores: a gateway report past the threshold auto-flags the
// account, the throttled view shows the episode and the flagged account, the
// user card carries the marker, and the operator clear (a same-origin POST)
// reverses it.
func TestConsoleThrottledViewAndFlagClear(t *testing.T) {
ctx := context.Background()
accounts := account.NewStore(testDB)
acc, err := accounts.ProvisionTelegram(ctx, "tg-"+uuid.NewString(), "en", "", "Throttled Player")
if err != nil {
t.Fatalf("provision: %v", err)
}
watch := ratewatch.New(ratewatch.Config{FlagThreshold: 100, FlagWindow: 10 * time.Minute}, accounts, zap.NewNop())
srv := server.New(":0", server.Deps{
Logger: zap.NewNop(),
Accounts: accounts,
Games: newGameService(),
Registry: testRegistry,
DictDir: dictDir(),
RateWatch: watch,
})
h := srv.Handler()
report := `{"window_seconds":30,"entries":[` +
`{"class":"user","key":"` + acc.ID.String() + `","rejected":150},` +
`{"class":"public","key":"10.1.2.3","rejected":7}]}`
req := httptest.NewRequest(http.MethodPost, "http://admin.test/api/v1/internal/ratelimit/report", strings.NewReader(report))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != http.StatusNoContent {
t.Fatalf("report = %d, want 204", rec.Code)
}
got, err := accounts.GetByID(ctx, acc.ID)
if err != nil {
t.Fatalf("get by id: %v", err)
}
if got.FlaggedHighRateAt.IsZero() {
t.Fatal("account not auto-flagged past the threshold")
}
base := "http://admin.test/_gm"
code, body := consoleDo(h, http.MethodGet, base+"/throttled", "", "")
if code != http.StatusOK || !strings.Contains(body, acc.ID.String()) ||
!strings.Contains(body, "10.1.2.3") || !strings.Contains(body, "Throttled Player") {
t.Fatalf("throttled view = %d, episode/flag shown = %v/%v",
code, strings.Contains(body, "10.1.2.3"), strings.Contains(body, "Throttled Player"))
}
if code, body = consoleDo(h, http.MethodGet, base+"/users/"+acc.ID.String(), "", ""); code != http.StatusOK || !strings.Contains(body, "Clear high-rate flag") {
t.Fatalf("user card = %d, has clear action = %v", code, strings.Contains(body, "Clear high-rate flag"))
}
// The clear POST is CSRF-guarded like every console action.
if code, _ = consoleDo(h, http.MethodPost, base+"/users/"+acc.ID.String()+"/clear-high-rate-flag", "", ""); code != http.StatusForbidden {
t.Fatalf("clear without origin = %d, want 403", code)
}
if code, body = consoleDo(h, http.MethodPost, base+"/users/"+acc.ID.String()+"/clear-high-rate-flag", "x=1", "http://admin.test"); code != http.StatusOK || !strings.Contains(body, "Cleared") {
t.Fatalf("clear with origin = %d, has Cleared = %v", code, strings.Contains(body, "Cleared"))
}
if got, err = accounts.GetByID(ctx, acc.ID); err != nil || !got.FlaggedHighRateAt.IsZero() {
t.Fatalf("flag survived the clear: %v (err %v)", got.FlaggedHighRateAt, err)
}
}
// consoleDo issues a request to h, optionally with an Origin header, and returns
// the status and body. Form bodies are sent as application/x-www-form-urlencoded.
func consoleDo(h http.Handler, method, target, body, origin string) (int, string) {
@@ -30,4 +30,5 @@ type Accounts struct {
MergedInto *uuid.UUID
MergedAt *time.Time
ServiceLanguage *string
FlaggedHighRateAt *time.Time
}
@@ -0,0 +1,21 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package model
import (
"github.com/google/uuid"
"time"
)
type GameDrafts struct {
GameID uuid.UUID `sql:"primary_key"`
AccountID uuid.UUID `sql:"primary_key"`
RackOrder string
BoardTiles string
UpdatedAt time.Time
}
@@ -0,0 +1,19 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package model
import (
"github.com/google/uuid"
"time"
)
type GameHidden struct {
AccountID uuid.UUID `sql:"primary_key"`
GameID uuid.UUID `sql:"primary_key"`
CreatedAt time.Time
}
@@ -34,6 +34,7 @@ type accountsTable struct {
MergedInto postgres.ColumnString
MergedAt postgres.ColumnTimestampz
ServiceLanguage postgres.ColumnString
FlaggedHighRateAt postgres.ColumnTimestampz
AllColumns postgres.ColumnList
MutableColumns postgres.ColumnList
@@ -92,8 +93,9 @@ func newAccountsTableImpl(schemaName, tableName, alias string) accountsTable {
MergedIntoColumn = postgres.StringColumn("merged_into")
MergedAtColumn = postgres.TimestampzColumn("merged_at")
ServiceLanguageColumn = postgres.StringColumn("service_language")
allColumns = postgres.ColumnList{AccountIDColumn, DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn, MergedIntoColumn, MergedAtColumn, ServiceLanguageColumn}
mutableColumns = postgres.ColumnList{DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn, MergedIntoColumn, MergedAtColumn, ServiceLanguageColumn}
FlaggedHighRateAtColumn = postgres.TimestampzColumn("flagged_high_rate_at")
allColumns = postgres.ColumnList{AccountIDColumn, DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn, MergedIntoColumn, MergedAtColumn, ServiceLanguageColumn, FlaggedHighRateAtColumn}
mutableColumns = postgres.ColumnList{DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn, MergedIntoColumn, MergedAtColumn, ServiceLanguageColumn, FlaggedHighRateAtColumn}
defaultColumns = postgres.ColumnList{DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn}
)
@@ -118,6 +120,7 @@ func newAccountsTableImpl(schemaName, tableName, alias string) accountsTable {
MergedInto: MergedIntoColumn,
MergedAt: MergedAtColumn,
ServiceLanguage: ServiceLanguageColumn,
FlaggedHighRateAt: FlaggedHighRateAtColumn,
AllColumns: allColumns,
MutableColumns: mutableColumns,
@@ -0,0 +1,90 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package table
import (
"github.com/go-jet/jet/v2/postgres"
)
var GameDrafts = newGameDraftsTable("backend", "game_drafts", "")
type gameDraftsTable struct {
postgres.Table
// Columns
GameID postgres.ColumnString
AccountID postgres.ColumnString
RackOrder postgres.ColumnString
BoardTiles postgres.ColumnString
UpdatedAt postgres.ColumnTimestampz
AllColumns postgres.ColumnList
MutableColumns postgres.ColumnList
DefaultColumns postgres.ColumnList
}
type GameDraftsTable struct {
gameDraftsTable
EXCLUDED gameDraftsTable
}
// AS creates new GameDraftsTable with assigned alias
func (a GameDraftsTable) AS(alias string) *GameDraftsTable {
return newGameDraftsTable(a.SchemaName(), a.TableName(), alias)
}
// Schema creates new GameDraftsTable with assigned schema name
func (a GameDraftsTable) FromSchema(schemaName string) *GameDraftsTable {
return newGameDraftsTable(schemaName, a.TableName(), a.Alias())
}
// WithPrefix creates new GameDraftsTable with assigned table prefix
func (a GameDraftsTable) WithPrefix(prefix string) *GameDraftsTable {
return newGameDraftsTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
}
// WithSuffix creates new GameDraftsTable with assigned table suffix
func (a GameDraftsTable) WithSuffix(suffix string) *GameDraftsTable {
return newGameDraftsTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
}
func newGameDraftsTable(schemaName, tableName, alias string) *GameDraftsTable {
return &GameDraftsTable{
gameDraftsTable: newGameDraftsTableImpl(schemaName, tableName, alias),
EXCLUDED: newGameDraftsTableImpl("", "excluded", ""),
}
}
func newGameDraftsTableImpl(schemaName, tableName, alias string) gameDraftsTable {
var (
GameIDColumn = postgres.StringColumn("game_id")
AccountIDColumn = postgres.StringColumn("account_id")
RackOrderColumn = postgres.StringColumn("rack_order")
BoardTilesColumn = postgres.StringColumn("board_tiles")
UpdatedAtColumn = postgres.TimestampzColumn("updated_at")
allColumns = postgres.ColumnList{GameIDColumn, AccountIDColumn, RackOrderColumn, BoardTilesColumn, UpdatedAtColumn}
mutableColumns = postgres.ColumnList{RackOrderColumn, BoardTilesColumn, UpdatedAtColumn}
defaultColumns = postgres.ColumnList{RackOrderColumn, BoardTilesColumn, UpdatedAtColumn}
)
return gameDraftsTable{
Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
//Columns
GameID: GameIDColumn,
AccountID: AccountIDColumn,
RackOrder: RackOrderColumn,
BoardTiles: BoardTilesColumn,
UpdatedAt: UpdatedAtColumn,
AllColumns: allColumns,
MutableColumns: mutableColumns,
DefaultColumns: defaultColumns,
}
}
@@ -0,0 +1,84 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package table
import (
"github.com/go-jet/jet/v2/postgres"
)
var GameHidden = newGameHiddenTable("backend", "game_hidden", "")
type gameHiddenTable struct {
postgres.Table
// Columns
AccountID postgres.ColumnString
GameID postgres.ColumnString
CreatedAt postgres.ColumnTimestampz
AllColumns postgres.ColumnList
MutableColumns postgres.ColumnList
DefaultColumns postgres.ColumnList
}
type GameHiddenTable struct {
gameHiddenTable
EXCLUDED gameHiddenTable
}
// AS creates new GameHiddenTable with assigned alias
func (a GameHiddenTable) AS(alias string) *GameHiddenTable {
return newGameHiddenTable(a.SchemaName(), a.TableName(), alias)
}
// Schema creates new GameHiddenTable with assigned schema name
func (a GameHiddenTable) FromSchema(schemaName string) *GameHiddenTable {
return newGameHiddenTable(schemaName, a.TableName(), a.Alias())
}
// WithPrefix creates new GameHiddenTable with assigned table prefix
func (a GameHiddenTable) WithPrefix(prefix string) *GameHiddenTable {
return newGameHiddenTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
}
// WithSuffix creates new GameHiddenTable with assigned table suffix
func (a GameHiddenTable) WithSuffix(suffix string) *GameHiddenTable {
return newGameHiddenTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
}
func newGameHiddenTable(schemaName, tableName, alias string) *GameHiddenTable {
return &GameHiddenTable{
gameHiddenTable: newGameHiddenTableImpl(schemaName, tableName, alias),
EXCLUDED: newGameHiddenTableImpl("", "excluded", ""),
}
}
func newGameHiddenTableImpl(schemaName, tableName, alias string) gameHiddenTable {
var (
AccountIDColumn = postgres.StringColumn("account_id")
GameIDColumn = postgres.StringColumn("game_id")
CreatedAtColumn = postgres.TimestampzColumn("created_at")
allColumns = postgres.ColumnList{AccountIDColumn, GameIDColumn, CreatedAtColumn}
mutableColumns = postgres.ColumnList{CreatedAtColumn}
defaultColumns = postgres.ColumnList{CreatedAtColumn}
)
return gameHiddenTable{
Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
//Columns
AccountID: AccountIDColumn,
GameID: GameIDColumn,
CreatedAt: CreatedAtColumn,
AllColumns: allColumns,
MutableColumns: mutableColumns,
DefaultColumns: defaultColumns,
}
}
@@ -18,6 +18,8 @@ func UseSchema(schema string) {
EmailConfirmations = EmailConfirmations.FromSchema(schema)
FriendCodes = FriendCodes.FromSchema(schema)
Friendships = Friendships.FromSchema(schema)
GameDrafts = GameDrafts.FromSchema(schema)
GameHidden = GameHidden.FromSchema(schema)
GameInvitationInvitees = GameInvitationInvitees.FromSchema(schema)
GameInvitations = GameInvitations.FromSchema(schema)
GameMoves = GameMoves.FromSchema(schema)
@@ -35,6 +35,10 @@ CREATE TABLE accounts (
merged_into uuid REFERENCES accounts (account_id) ON DELETE SET NULL,
merged_at timestamptz,
service_language text CHECK (service_language IN ('en', 'ru')),
-- Soft, reversible "suspected high-rate" marker (R3): set once when the gateway
-- reports sustained rate-limiter rejections past the threshold; an operator
-- clears it in the admin console. Never an automatic ban.
flagged_high_rate_at timestamptz,
CONSTRAINT accounts_preferred_language_chk CHECK (preferred_language IN ('en', 'ru')),
CONSTRAINT accounts_hint_balance_chk CHECK (hint_balance >= 0)
);
+235
View File
@@ -0,0 +1,235 @@
// Package ratewatch ingests the gateway's periodic rate-limiter rejection
// reports (R3). It keeps an in-memory window of recent throttle episodes for
// the admin console's view and applies the conservative high-rate auto-flag:
// when one account's rejections within the rolling window cross the threshold,
// the account store stamps the soft, reversible flagged_high_rate_at marker
// (set-once; an operator clears it; never an automatic ban). Like the gateway's
// active_users gauge it is single-instance and resets on restart by design —
// the durable part is the account flag, not the episode window.
package ratewatch
import (
"context"
"fmt"
"sort"
"sync"
"time"
"github.com/google/uuid"
"go.uber.org/zap"
)
// ClassUser is the limiter class whose keys are account ids — the only class
// the auto-flag applies to (the others are keyed by client IP).
const ClassUser = "user"
const (
// maxSeries bounds the distinct (class, key) series kept for the console
// view, so a key-spraying client cannot grow the map: past the bound the
// least-recently-throttled series is evicted.
maxSeries = 200
// minRetention keeps an episode visible in the console for at least an hour
// after its last rejection (longer when the flag window is longer).
minRetention = time.Hour
)
// Config tunes the conservative high-rate auto-flag.
type Config struct {
// FlagThreshold is the rejected-call count within FlagWindow past which a
// user account is flagged.
FlagThreshold int
// FlagWindow is the rolling window the rejections accumulate over.
FlagWindow time.Duration
}
// DefaultConfig returns the agreed conservative defaults — 1000 rejected calls
// within a rolling 10 minutes (~1.7/s sustained, far above the client's
// capped-backoff retry noise yet a fraction of an abusive loop).
func DefaultConfig() Config {
return Config{FlagThreshold: 1000, FlagWindow: 10 * time.Minute}
}
// Validate reports whether the configuration values are acceptable.
func (c Config) Validate() error {
if c.FlagThreshold <= 0 {
return fmt.Errorf("ratewatch: flag threshold must be positive")
}
if c.FlagWindow <= 0 {
return fmt.Errorf("ratewatch: flag window must be positive")
}
return nil
}
// Flagger stamps the account-level high-rate marker; account.Store satisfies it.
type Flagger interface {
FlagHighRate(ctx context.Context, id uuid.UUID, at time.Time) (bool, error)
}
// Entry is one reported aggregate: the rejections of one limiter key within one
// gateway report window (the wire mirror of the gateway's rejection summary).
type Entry struct {
Class string
Key string
Rejected int
}
// Episode is one key's recent-throttle aggregate for the admin view.
type Episode struct {
Class string
Key string
Rejected int
FirstSeen time.Time
LastSeen time.Time
}
// Watch accumulates reports and applies the auto-flag rule.
type Watch struct {
cfg Config
flagger Flagger
log *zap.Logger
now func() time.Time
mu sync.Mutex
series map[seriesKey]*series
}
type seriesKey struct{ class, key string }
type series struct {
points []point // ascending by time
}
type point struct {
at time.Time
n int
}
// New constructs a Watch over flagger with cfg. A nil logger is replaced by a
// no-op one; a nil flagger disables the auto-flag (the view still works).
func New(cfg Config, flagger Flagger, log *zap.Logger) *Watch {
if log == nil {
log = zap.NewNop()
}
return &Watch{
cfg: cfg,
flagger: flagger,
log: log,
now: time.Now,
series: make(map[seriesKey]*series),
}
}
// Ingest records one gateway report. Entries with an empty class or key or a
// non-positive count are skipped. When a user-class series crosses the flag
// threshold within the flag window, the account is flagged (the store keeps it
// set-once, so a sustained episode costs one no-op UPDATE per report).
func (w *Watch) Ingest(ctx context.Context, entries []Entry) {
if len(entries) == 0 {
return
}
now := w.now()
var flag []uuid.UUID
w.mu.Lock()
for _, e := range entries {
if e.Class == "" || e.Key == "" || e.Rejected <= 0 {
continue
}
k := seriesKey{class: e.Class, key: e.Key}
s := w.series[k]
if s == nil {
s = &series{}
w.series[k] = s
}
s.points = append(s.points, point{at: now, n: e.Rejected})
if e.Class == ClassUser && s.sumSince(now.Add(-w.cfg.FlagWindow)) >= w.cfg.FlagThreshold {
if id, err := uuid.Parse(e.Key); err == nil {
flag = append(flag, id)
}
}
}
w.pruneLocked(now)
w.mu.Unlock()
if w.flagger == nil {
return
}
for _, id := range flag {
set, err := w.flagger.FlagHighRate(ctx, id, now)
switch {
case err != nil:
w.log.Warn("high-rate flag failed", zap.String("account_id", id.String()), zap.Error(err))
case set:
w.log.Info("account flagged high-rate",
zap.String("account_id", id.String()),
zap.Int("threshold", w.cfg.FlagThreshold),
zap.Duration("window", w.cfg.FlagWindow))
}
}
}
// Config returns the active auto-flag tuning (the admin console captions it).
func (w *Watch) Config() Config { return w.cfg }
// Recent returns the retained throttle episodes, most recently throttled first.
func (w *Watch) Recent() []Episode {
w.mu.Lock()
defer w.mu.Unlock()
out := make([]Episode, 0, len(w.series))
for k, s := range w.series {
if len(s.points) == 0 {
continue
}
ep := Episode{
Class: k.class,
Key: k.key,
FirstSeen: s.points[0].at,
LastSeen: s.points[len(s.points)-1].at,
}
for _, p := range s.points {
ep.Rejected += p.n
}
out = append(out, ep)
}
sort.Slice(out, func(i, j int) bool { return out[i].LastSeen.After(out[j].LastSeen) })
return out
}
// sumSince totals the points at or after cutoff.
func (s *series) sumSince(cutoff time.Time) int {
sum := 0
for i := len(s.points) - 1; i >= 0; i-- {
if s.points[i].at.Before(cutoff) {
break
}
sum += s.points[i].n
}
return sum
}
// pruneLocked drops points past retention, empty series, and — past maxSeries —
// the least-recently-throttled series. The caller holds w.mu.
func (w *Watch) pruneLocked(now time.Time) {
cutoff := now.Add(-max(minRetention, w.cfg.FlagWindow))
for k, s := range w.series {
i := 0
for i < len(s.points) && s.points[i].at.Before(cutoff) {
i++
}
s.points = s.points[i:]
if len(s.points) == 0 {
delete(w.series, k)
}
}
for len(w.series) > maxSeries {
var oldest seriesKey
var oldestAt time.Time
first := true
for k, s := range w.series {
last := s.points[len(s.points)-1].at
if first || last.Before(oldestAt) {
oldest, oldestAt, first = k, last, false
}
}
delete(w.series, oldest)
}
}
@@ -0,0 +1,140 @@
package ratewatch
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/uuid"
)
// fakeFlagger records flag calls and reports them as newly set.
type fakeFlagger struct {
calls []uuid.UUID
}
func (f *fakeFlagger) FlagHighRate(_ context.Context, id uuid.UUID, _ time.Time) (bool, error) {
f.calls = append(f.calls, id)
return true, nil
}
// watchAt returns a Watch with a controllable clock.
func watchAt(cfg Config, flagger Flagger, at *time.Time) *Watch {
w := New(cfg, flagger, nil)
w.now = func() time.Time { return *at }
return w
}
// TestIngestAggregatesAndRecent verifies episodes accumulate per (class, key),
// invalid entries are skipped, and Recent orders by last rejection.
func TestIngestAggregatesAndRecent(t *testing.T) {
now := time.Date(2026, 6, 10, 12, 0, 0, 0, time.UTC)
w := watchAt(DefaultConfig(), nil, &now)
ctx := context.Background()
w.Ingest(ctx, []Entry{
{Class: "public", Key: "10.0.0.1", Rejected: 3},
{Class: "user", Key: "u-1", Rejected: 5},
{Class: "", Key: "x", Rejected: 1},
{Class: "user", Key: "", Rejected: 1},
{Class: "user", Key: "u-1", Rejected: 0},
})
now = now.Add(30 * time.Second)
w.Ingest(ctx, []Entry{{Class: "public", Key: "10.0.0.1", Rejected: 4}})
got := w.Recent()
if len(got) != 2 {
t.Fatalf("Recent returned %d episodes, want 2", len(got))
}
if got[0].Class != "public" || got[0].Key != "10.0.0.1" || got[0].Rejected != 7 {
t.Errorf("first episode = %+v, want public/10.0.0.1 rejected=7", got[0])
}
if !got[0].LastSeen.After(got[0].FirstSeen) {
t.Errorf("episode span = [%v, %v], want a positive span", got[0].FirstSeen, got[0].LastSeen)
}
if got[1].Class != "user" || got[1].Rejected != 5 {
t.Errorf("second episode = %+v, want user rejected=5", got[1])
}
}
// TestAutoFlagThreshold verifies the flag fires only for a user-class series
// crossing the threshold within the window, with a parseable account id.
func TestAutoFlagThreshold(t *testing.T) {
now := time.Date(2026, 6, 10, 12, 0, 0, 0, time.UTC)
flagged := &fakeFlagger{}
id := uuid.New()
w := watchAt(Config{FlagThreshold: 100, FlagWindow: 10 * time.Minute}, flagged, &now)
ctx := context.Background()
w.Ingest(ctx, []Entry{
{Class: "user", Key: id.String(), Rejected: 99},
{Class: "public", Key: "10.0.0.1", Rejected: 1000},
{Class: "user", Key: "not-a-uuid", Rejected: 1000},
})
if len(flagged.calls) != 0 {
t.Fatalf("flagged %v below the threshold", flagged.calls)
}
now = now.Add(30 * time.Second)
w.Ingest(ctx, []Entry{{Class: "user", Key: id.String(), Rejected: 1}})
if len(flagged.calls) != 1 || flagged.calls[0] != id {
t.Fatalf("flag calls = %v, want exactly [%s]", flagged.calls, id)
}
}
// TestAutoFlagWindowExpiry verifies rejections age out of the rolling window.
func TestAutoFlagWindowExpiry(t *testing.T) {
now := time.Date(2026, 6, 10, 12, 0, 0, 0, time.UTC)
flagged := &fakeFlagger{}
id := uuid.New()
w := watchAt(Config{FlagThreshold: 100, FlagWindow: 10 * time.Minute}, flagged, &now)
ctx := context.Background()
w.Ingest(ctx, []Entry{{Class: "user", Key: id.String(), Rejected: 60}})
now = now.Add(11 * time.Minute)
w.Ingest(ctx, []Entry{{Class: "user", Key: id.String(), Rejected: 60}})
if len(flagged.calls) != 0 {
t.Fatalf("flagged %v across an expired window", flagged.calls)
}
now = now.Add(time.Minute)
w.Ingest(ctx, []Entry{{Class: "user", Key: id.String(), Rejected: 50}})
if len(flagged.calls) != 1 {
t.Fatalf("flag calls = %v, want one in-window crossing", flagged.calls)
}
}
// TestSeriesBound verifies the episode map stays bounded by evicting the
// least-recently-throttled series.
func TestSeriesBound(t *testing.T) {
now := time.Date(2026, 6, 10, 12, 0, 0, 0, time.UTC)
w := watchAt(DefaultConfig(), nil, &now)
ctx := context.Background()
for i := range maxSeries + 10 {
now = now.Add(time.Second)
w.Ingest(ctx, []Entry{{Class: "public", Key: fmt.Sprintf("10.0.%d.%d", i/256, i%256), Rejected: 1}})
}
got := w.Recent()
if len(got) != maxSeries {
t.Fatalf("retained %d series, want %d", len(got), maxSeries)
}
for _, ep := range got {
if ep.Key == "10.0.0.0" {
t.Fatal("the least-recently-throttled series survived the bound")
}
}
}
// TestConfigValidate covers the tuning guards.
func TestConfigValidate(t *testing.T) {
if err := DefaultConfig().Validate(); err != nil {
t.Errorf("default config invalid: %v", err)
}
if err := (Config{FlagThreshold: 0, FlagWindow: time.Minute}).Validate(); err == nil {
t.Error("zero threshold passed validation")
}
if err := (Config{FlagThreshold: 1, FlagWindow: 0}).Validate(); err == nil {
t.Error("zero window passed validation")
}
}
+7 -4
View File
@@ -37,6 +37,11 @@ func (s *Server) registerRoutes() {
// before delivering an out-of-app notification.
in.POST("/push-target", s.handlePushTarget)
}
if s.ratewatch != nil {
// The gateway's periodic rate-limiter rejection summary (R3): feeds the
// admin console's throttled view and the high-rate auto-flag.
s.internal.POST("/ratelimit/report", s.handleRateLimitReport)
}
u := s.user
if s.accounts != nil {
u.GET("/profile", s.handleProfile)
@@ -120,10 +125,8 @@ func gameIDParam(c *gin.Context) (uuid.UUID, bool) {
// X-Forwarded-For (the first hop), falling back to the direct peer.
func clientIP(c *gin.Context) string {
if xff := c.GetHeader("X-Forwarded-For"); xff != "" {
if i := strings.IndexByte(xff, ','); i >= 0 {
return strings.TrimSpace(xff[:i])
}
return strings.TrimSpace(xff)
first, _, _ := strings.Cut(xff, ",")
return strings.TrimSpace(first)
}
return c.ClientIP()
}
@@ -19,6 +19,7 @@ import (
"scrabble/backend/internal/adminconsole"
"scrabble/backend/internal/engine"
"scrabble/backend/internal/game"
"scrabble/backend/internal/ratewatch"
"scrabble/backend/internal/robot"
"scrabble/backend/internal/social"
)
@@ -48,6 +49,8 @@ func (s *Server) registerConsole(router *gin.Engine) {
gm.GET("/users", s.consoleUsers)
gm.GET("/users/:id", s.consoleUserDetail)
gm.POST("/users/:id/message", s.consoleUserMessage)
gm.POST("/users/:id/clear-high-rate-flag", s.consoleClearHighRateFlag)
gm.GET("/throttled", s.consoleThrottled)
gm.GET("/games", s.consoleGames)
gm.GET("/games/:id", s.consoleGameDetail)
gm.GET("/complaints", s.consoleComplaints)
@@ -117,7 +120,8 @@ func (s *Server) consoleUsers(c *gin.Context) {
}
view.Items = append(view.Items, adminconsole.UserRow{
ID: it.ID.String(), DisplayName: it.DisplayName, Kind: kind,
Language: it.PreferredLanguage, Guest: it.IsGuest, CreatedAt: fmtTime(it.CreatedAt),
Language: it.PreferredLanguage, Guest: it.IsGuest,
FlaggedHighRate: !it.FlaggedHighRateAt.IsZero(), CreatedAt: fmtTime(it.CreatedAt),
})
ids = append(ids, it.ID)
}
@@ -257,6 +261,9 @@ func (s *Server) consoleUserDetail(c *gin.Context) {
if acc.MergedInto != uuid.Nil {
view.MergedInto = acc.MergedInto.String()
}
if !acc.FlaggedHighRateAt.IsZero() {
view.FlaggedHighRateAt = fmtTime(acc.FlaggedHighRateAt)
}
if view.HasStats {
if st, err := s.accounts.GetStats(ctx, id); err == nil {
view.Stats = adminconsole.StatsRow{Wins: st.Wins, Losses: st.Losses, Draws: st.Draws, MaxGamePoints: st.MaxGamePoints, MaxWordPoints: st.MaxWordPoints}
@@ -551,6 +558,56 @@ func (s *Server) consolePostBroadcast(c *gin.Context) {
}
}
// consoleThrottled renders the rate-limit observability page: the recent
// gateway-reported throttle episodes (in-memory, reset on a backend restart)
// and the accounts currently carrying the soft high-rate flag (R3).
func (s *Server) consoleThrottled(c *gin.Context) {
ctx := c.Request.Context()
var view adminconsole.ThrottledView
if s.ratewatch != nil {
cfg := s.ratewatch.Config()
view.FlagThreshold = cfg.FlagThreshold
view.FlagWindow = cfg.FlagWindow.String()
for _, ep := range s.ratewatch.Recent() {
row := adminconsole.ThrottleEpisodeRow{
Class: ep.Class, Key: ep.Key, Rejected: ep.Rejected,
FirstSeen: fmtTime(ep.FirstSeen), LastSeen: fmtTime(ep.LastSeen),
}
if ep.Class == ratewatch.ClassUser {
if id, err := uuid.Parse(ep.Key); err == nil {
row.UserID = id.String()
}
}
view.Episodes = append(view.Episodes, row)
}
}
flagged, err := s.accounts.ListFlaggedHighRate(ctx)
if err != nil {
s.consoleError(c, err)
return
}
for _, fa := range flagged {
view.Flagged = append(view.Flagged, adminconsole.FlaggedAccountRow{
ID: fa.ID.String(), DisplayName: fa.DisplayName, FlaggedAt: fmtTime(fa.FlaggedHighRateAt),
})
}
s.renderConsole(c, "throttled", "throttled", "Throttled", view)
}
// consoleClearHighRateFlag clears the soft high-rate marker — the operator's
// reversible review action (R3).
func (s *Server) consoleClearHighRateFlag(c *gin.Context) {
id, ok := s.consoleUUID(c, "/_gm/users")
if !ok {
return
}
if err := s.accounts.ClearHighRateFlag(c.Request.Context(), id); err != nil {
s.consoleError(c, err)
return
}
s.renderConsoleMessage(c, "Cleared", "high-rate flag cleared", "/_gm/users/"+id.String())
}
// variantVersions builds the per-variant resident-version summary from the registry.
func (s *Server) variantVersions() []adminconsole.VariantVersions {
out := make([]adminconsole.VariantVersions, 0, len(engine.Variants()))
@@ -0,0 +1,41 @@
package server
import (
"net/http"
"github.com/gin-gonic/gin"
"scrabble/backend/internal/ratewatch"
)
// rateLimitReportRequest mirrors the gateway's periodic rejection summary: every
// entry aggregates one limiter key (class + key) over the report window.
type rateLimitReportRequest struct {
WindowSeconds int `json:"window_seconds"`
Entries []rateLimitReportEntry `json:"entries"`
}
// rateLimitReportEntry is one (class, key) aggregate of the report.
type rateLimitReportEntry struct {
Class string `json:"class"`
Key string `json:"key"`
Rejected int `json:"rejected"`
}
// handleRateLimitReport ingests one gateway rejection report into the rate
// watch — the admin console's throttled view and the high-rate auto-flag (R3).
// Internal, gateway-only: like sessions/resolve it trusts the network segment.
// Malformed individual entries are skipped by the watch itself.
func (s *Server) handleRateLimitReport(c *gin.Context) {
var req rateLimitReportRequest
if err := c.ShouldBindJSON(&req); err != nil {
abortBadRequest(c, "invalid rate-limit report")
return
}
entries := make([]ratewatch.Entry, 0, len(req.Entries))
for _, e := range req.Entries {
entries = append(entries, ratewatch.Entry{Class: e.Class, Key: e.Key, Rejected: e.Rejected})
}
s.ratewatch.Ingest(c.Request.Context(), entries)
c.Status(http.StatusNoContent)
}
+18
View File
@@ -10,6 +10,7 @@ import (
"scrabble/backend/internal/account"
"scrabble/backend/internal/game"
"scrabble/backend/internal/ratewatch"
"scrabble/backend/internal/session"
)
@@ -57,6 +58,23 @@ func TestResolveSessionRejectsEmptyToken(t *testing.T) {
}
}
// TestRateLimitReportEndpoint covers the internal R3 report route: a malformed
// body is a 400, a valid report lands in the rate watch with 204.
func TestRateLimitReportEndpoint(t *testing.T) {
watch := ratewatch.New(ratewatch.DefaultConfig(), nil, nil)
s := New(":0", Deps{RateWatch: watch})
if rec := do(t, s, http.MethodPost, "/api/v1/internal/ratelimit/report", `{bad`, nil); rec.Code != http.StatusBadRequest {
t.Fatalf("malformed report = %d, want 400", rec.Code)
}
body := `{"window_seconds":30,"entries":[{"class":"user","key":"` + uuid.NewString() + `","rejected":7}]}`
if rec := do(t, s, http.MethodPost, "/api/v1/internal/ratelimit/report", body, nil); rec.Code != http.StatusNoContent {
t.Fatalf("report = %d, want 204", rec.Code)
}
if eps := watch.Recent(); len(eps) != 1 || eps[0].Rejected != 7 {
t.Fatalf("watch episodes = %+v, want one entry with rejected=7", eps)
}
}
func TestSubmitPlayRejectsBadDirection(t *testing.T) {
headers := map[string]string{"X-User-ID": uuid.New().String()}
path := "/api/v1/user/games/" + uuid.New().String() + "/play"
+7
View File
@@ -24,6 +24,7 @@ import (
"scrabble/backend/internal/game"
"scrabble/backend/internal/link"
"scrabble/backend/internal/lobby"
"scrabble/backend/internal/ratewatch"
"scrabble/backend/internal/session"
"scrabble/backend/internal/social"
"scrabble/backend/internal/telemetry"
@@ -71,6 +72,10 @@ type Deps struct {
// nil when BACKEND_CONNECTOR_ADDR is unset (broadcasts show a "not configured"
// notice).
Connector *connector.Client
// RateWatch ingests the gateway's rate-limiter rejection reports (R3): the
// admin console's throttled view + the high-rate auto-flag. A nil RateWatch
// disables the internal report endpoint and the console view.
RateWatch *ratewatch.Watch
}
// Server owns the gin engine, the underlying HTTP server and the readiness
@@ -93,6 +98,7 @@ type Server struct {
registry *engine.Registry
dictDir string
connector *connector.Client
ratewatch *ratewatch.Watch
console *adminconsole.Renderer
public *gin.RouterGroup
@@ -133,6 +139,7 @@ func New(addr string, deps Deps) *Server {
registry: deps.Registry,
dictDir: deps.DictDir,
connector: deps.Connector,
ratewatch: deps.RateWatch,
http: &http.Server{Addr: addr, Handler: engine},
}
s.registerProbes(engine)