R3: backend rate-limit observability — ratewatch, auto-flag, admin throttled view

- accounts.flagged_high_rate_at baked into the R1 baseline (no prod data; the
  contour schema is wiped after merge); jet regenerated — the regen also picks
  up the previously missing game_drafts/game_hidden models.
- account.Store: FlagHighRate (set-once), ClearHighRateFlag, the flag in
  GetByID/ListUsers and a ListFlaggedHighRate review queue.
- New internal/ratewatch: ingests the gateway rejection reports, keeps a
  bounded in-memory episode window for the console and applies the
  conservative auto-flag (1000 rejected / 10 min, BACKEND_HIGHRATE_FLAG_*).
- POST /api/v1/internal/ratelimit/report (network-trusted, like
  sessions/resolve).
- Admin console: Throttled page (episodes + flagged accounts), a high-rate
  badge in the user list, the marker + operator clear action on the user card.
- Tests: ratewatch unit suite, report-route handler test, renderer cases,
  integration coverage for the store round-trip and the console flow.
This commit is contained in:
Ilia Denisov
2026-06-10 02:14:00 +02:00
parent 8878711cf3
commit ab58062565
27 changed files with 1081 additions and 33 deletions
@@ -30,4 +30,5 @@ type Accounts struct {
MergedInto *uuid.UUID
MergedAt *time.Time
ServiceLanguage *string
FlaggedHighRateAt *time.Time
}
@@ -0,0 +1,21 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package model
import (
"github.com/google/uuid"
"time"
)
type GameDrafts struct {
GameID uuid.UUID `sql:"primary_key"`
AccountID uuid.UUID `sql:"primary_key"`
RackOrder string
BoardTiles string
UpdatedAt time.Time
}
@@ -0,0 +1,19 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package model
import (
"github.com/google/uuid"
"time"
)
type GameHidden struct {
AccountID uuid.UUID `sql:"primary_key"`
GameID uuid.UUID `sql:"primary_key"`
CreatedAt time.Time
}
@@ -34,6 +34,7 @@ type accountsTable struct {
MergedInto postgres.ColumnString
MergedAt postgres.ColumnTimestampz
ServiceLanguage postgres.ColumnString
FlaggedHighRateAt postgres.ColumnTimestampz
AllColumns postgres.ColumnList
MutableColumns postgres.ColumnList
@@ -92,8 +93,9 @@ func newAccountsTableImpl(schemaName, tableName, alias string) accountsTable {
MergedIntoColumn = postgres.StringColumn("merged_into")
MergedAtColumn = postgres.TimestampzColumn("merged_at")
ServiceLanguageColumn = postgres.StringColumn("service_language")
allColumns = postgres.ColumnList{AccountIDColumn, DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn, MergedIntoColumn, MergedAtColumn, ServiceLanguageColumn}
mutableColumns = postgres.ColumnList{DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn, MergedIntoColumn, MergedAtColumn, ServiceLanguageColumn}
FlaggedHighRateAtColumn = postgres.TimestampzColumn("flagged_high_rate_at")
allColumns = postgres.ColumnList{AccountIDColumn, DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn, MergedIntoColumn, MergedAtColumn, ServiceLanguageColumn, FlaggedHighRateAtColumn}
mutableColumns = postgres.ColumnList{DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn, MergedIntoColumn, MergedAtColumn, ServiceLanguageColumn, FlaggedHighRateAtColumn}
defaultColumns = postgres.ColumnList{DisplayNameColumn, PreferredLanguageColumn, TimeZoneColumn, BlockChatColumn, BlockFriendRequestsColumn, CreatedAtColumn, UpdatedAtColumn, AwayStartColumn, AwayEndColumn, HintBalanceColumn, IsGuestColumn, NotificationsInAppOnlyColumn, PaidAccountColumn}
)
@@ -118,6 +120,7 @@ func newAccountsTableImpl(schemaName, tableName, alias string) accountsTable {
MergedInto: MergedIntoColumn,
MergedAt: MergedAtColumn,
ServiceLanguage: ServiceLanguageColumn,
FlaggedHighRateAt: FlaggedHighRateAtColumn,
AllColumns: allColumns,
MutableColumns: mutableColumns,
@@ -0,0 +1,90 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package table
import (
"github.com/go-jet/jet/v2/postgres"
)
var GameDrafts = newGameDraftsTable("backend", "game_drafts", "")
type gameDraftsTable struct {
postgres.Table
// Columns
GameID postgres.ColumnString
AccountID postgres.ColumnString
RackOrder postgres.ColumnString
BoardTiles postgres.ColumnString
UpdatedAt postgres.ColumnTimestampz
AllColumns postgres.ColumnList
MutableColumns postgres.ColumnList
DefaultColumns postgres.ColumnList
}
type GameDraftsTable struct {
gameDraftsTable
EXCLUDED gameDraftsTable
}
// AS creates new GameDraftsTable with assigned alias
func (a GameDraftsTable) AS(alias string) *GameDraftsTable {
return newGameDraftsTable(a.SchemaName(), a.TableName(), alias)
}
// Schema creates new GameDraftsTable with assigned schema name
func (a GameDraftsTable) FromSchema(schemaName string) *GameDraftsTable {
return newGameDraftsTable(schemaName, a.TableName(), a.Alias())
}
// WithPrefix creates new GameDraftsTable with assigned table prefix
func (a GameDraftsTable) WithPrefix(prefix string) *GameDraftsTable {
return newGameDraftsTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
}
// WithSuffix creates new GameDraftsTable with assigned table suffix
func (a GameDraftsTable) WithSuffix(suffix string) *GameDraftsTable {
return newGameDraftsTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
}
func newGameDraftsTable(schemaName, tableName, alias string) *GameDraftsTable {
return &GameDraftsTable{
gameDraftsTable: newGameDraftsTableImpl(schemaName, tableName, alias),
EXCLUDED: newGameDraftsTableImpl("", "excluded", ""),
}
}
func newGameDraftsTableImpl(schemaName, tableName, alias string) gameDraftsTable {
var (
GameIDColumn = postgres.StringColumn("game_id")
AccountIDColumn = postgres.StringColumn("account_id")
RackOrderColumn = postgres.StringColumn("rack_order")
BoardTilesColumn = postgres.StringColumn("board_tiles")
UpdatedAtColumn = postgres.TimestampzColumn("updated_at")
allColumns = postgres.ColumnList{GameIDColumn, AccountIDColumn, RackOrderColumn, BoardTilesColumn, UpdatedAtColumn}
mutableColumns = postgres.ColumnList{RackOrderColumn, BoardTilesColumn, UpdatedAtColumn}
defaultColumns = postgres.ColumnList{RackOrderColumn, BoardTilesColumn, UpdatedAtColumn}
)
return gameDraftsTable{
Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
//Columns
GameID: GameIDColumn,
AccountID: AccountIDColumn,
RackOrder: RackOrderColumn,
BoardTiles: BoardTilesColumn,
UpdatedAt: UpdatedAtColumn,
AllColumns: allColumns,
MutableColumns: mutableColumns,
DefaultColumns: defaultColumns,
}
}
@@ -0,0 +1,84 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package table
import (
"github.com/go-jet/jet/v2/postgres"
)
var GameHidden = newGameHiddenTable("backend", "game_hidden", "")
type gameHiddenTable struct {
postgres.Table
// Columns
AccountID postgres.ColumnString
GameID postgres.ColumnString
CreatedAt postgres.ColumnTimestampz
AllColumns postgres.ColumnList
MutableColumns postgres.ColumnList
DefaultColumns postgres.ColumnList
}
type GameHiddenTable struct {
gameHiddenTable
EXCLUDED gameHiddenTable
}
// AS creates new GameHiddenTable with assigned alias
func (a GameHiddenTable) AS(alias string) *GameHiddenTable {
return newGameHiddenTable(a.SchemaName(), a.TableName(), alias)
}
// Schema creates new GameHiddenTable with assigned schema name
func (a GameHiddenTable) FromSchema(schemaName string) *GameHiddenTable {
return newGameHiddenTable(schemaName, a.TableName(), a.Alias())
}
// WithPrefix creates new GameHiddenTable with assigned table prefix
func (a GameHiddenTable) WithPrefix(prefix string) *GameHiddenTable {
return newGameHiddenTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
}
// WithSuffix creates new GameHiddenTable with assigned table suffix
func (a GameHiddenTable) WithSuffix(suffix string) *GameHiddenTable {
return newGameHiddenTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
}
func newGameHiddenTable(schemaName, tableName, alias string) *GameHiddenTable {
return &GameHiddenTable{
gameHiddenTable: newGameHiddenTableImpl(schemaName, tableName, alias),
EXCLUDED: newGameHiddenTableImpl("", "excluded", ""),
}
}
func newGameHiddenTableImpl(schemaName, tableName, alias string) gameHiddenTable {
var (
AccountIDColumn = postgres.StringColumn("account_id")
GameIDColumn = postgres.StringColumn("game_id")
CreatedAtColumn = postgres.TimestampzColumn("created_at")
allColumns = postgres.ColumnList{AccountIDColumn, GameIDColumn, CreatedAtColumn}
mutableColumns = postgres.ColumnList{CreatedAtColumn}
defaultColumns = postgres.ColumnList{CreatedAtColumn}
)
return gameHiddenTable{
Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
//Columns
AccountID: AccountIDColumn,
GameID: GameIDColumn,
CreatedAt: CreatedAtColumn,
AllColumns: allColumns,
MutableColumns: mutableColumns,
DefaultColumns: defaultColumns,
}
}
@@ -18,6 +18,8 @@ func UseSchema(schema string) {
EmailConfirmations = EmailConfirmations.FromSchema(schema)
FriendCodes = FriendCodes.FromSchema(schema)
Friendships = Friendships.FromSchema(schema)
GameDrafts = GameDrafts.FromSchema(schema)
GameHidden = GameHidden.FromSchema(schema)
GameInvitationInvitees = GameInvitationInvitees.FromSchema(schema)
GameInvitations = GameInvitations.FromSchema(schema)
GameMoves = GameMoves.FromSchema(schema)
@@ -35,6 +35,10 @@ CREATE TABLE accounts (
merged_into uuid REFERENCES accounts (account_id) ON DELETE SET NULL,
merged_at timestamptz,
service_language text CHECK (service_language IN ('en', 'ru')),
-- Soft, reversible "suspected high-rate" marker (R3): set once when the gateway
-- reports sustained rate-limiter rejections past the threshold; an operator
-- clears it in the admin console. Never an automatic ban.
flagged_high_rate_at timestamptz,
CONSTRAINT accounts_preferred_language_chk CHECK (preferred_language IN ('en', 'ru')),
CONSTRAINT accounts_hint_balance_chk CHECK (hint_balance >= 0)
);