R2: load-test harness + contour resource observability
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 9s
CI / integration (pull_request) Successful in 11s
CI / ui (pull_request) Successful in 38s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Failing after 3s

New scrabble/loadtest module (the pre-release stress harness): seeds 1000 guest +
10000 durable accounts with pre-created sessions directly in Postgres (token hash
matches backend/internal/session), drives virtual players through the edge protocol
(real 2-4p games assembled via invitations, mid-ranked legal moves generated locally
by the embedded scrabble-solver — the edge carries no board, so the client replays
history), plus nudge/chat/check-word/draft/profile/stats and a gateway-hammer that
verifies the rate limiter. Prints a trip-report summary (per-op latency percentiles,
result codes, live-event tally). Go unit tests cover the pure pieces; the DAWG-backed
move test runs under BACKEND_DICT_DIR.

Contour: add cAdvisor + postgres_exporter + a 'Scrabble - Resources' Grafana
dashboard and the two Prometheus scrape jobs, for the R2/R7 stress-run resource
baseline.

CI: gate ./loadtest/... (path filter + vet/build/test). Docs: TESTING, ARCHITECTURE,
project CLAUDE repo layout.
This commit is contained in:
Ilia Denisov
2026-06-09 23:45:24 +02:00
parent bf3ee62711
commit aa137e3558
27 changed files with 2554 additions and 7 deletions
+136
View File
@@ -0,0 +1,136 @@
// Package edge is the load harness's client of the gateway edge protocol: the
// Connect Execute envelope carrying FlatBuffers payloads, plus the Subscribe live
// stream, over h2c. It exposes typed wrappers for the operations the driver
// exercises, decoding responses into plain Go structs so the scenario layer never
// touches FlatBuffers directly.
package edge
import (
"context"
"crypto/tls"
"net"
"net/http"
"time"
"connectrpc.com/connect"
"golang.org/x/net/http2"
edgev1 "scrabble/gateway/proto/edge/v1"
"scrabble/gateway/proto/edge/v1/edgev1connect"
)
// Message types the driver uses, mirroring gateway/internal/transcode's catalog.
const (
msgSubmitPlay = "game.submit_play"
msgPass = "game.pass"
msgExchange = "game.exchange"
msgState = "game.state"
msgHistory = "game.history"
msgGamesList = "games.list"
msgCheckWord = "game.check_word"
msgNudge = "chat.nudge"
msgChatPost = "chat.post"
msgDraftSave = "draft.save"
msgDraftGet = "draft.get"
msgProfileGet = "profile.get"
msgProfileUpd = "profile.update"
msgStatsGet = "stats.get"
msgInvCreate = "invitation.create"
msgInvAccept = "invitation.accept"
msgInvList = "invitation.list"
msgEnqueue = "lobby.enqueue"
)
// Client speaks the edge protocol to a single gateway base URL over h2c. It is safe
// for concurrent use by many virtual players (the underlying http2.Transport pools
// and multiplexes connections).
type Client struct {
rpc edgev1connect.GatewayClient
}
// New builds a Client for baseURL (for example http://gateway:8081). The transport
// speaks HTTP/2 cleartext (h2c) to match the gateway, dialling plaintext TCP rather
// than TLS.
func New(baseURL string) *Client {
hc := &http.Client{
Transport: &http2.Transport{
AllowHTTP: true,
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, network, addr)
},
},
}
return &Client{rpc: edgev1connect.NewGatewayClient(hc, baseURL)}
}
// Result is the decoded Execute envelope: Code is "ok" or a stable domain error
// code (a non-ok Code is a domain outcome, not a transport failure); Payload is the
// FlatBuffers response body (empty on error).
type Result struct {
Code string
Payload []byte
}
// execute runs one operation as token (empty for an unauthenticated op). A transport
// or connection error is returned as err; a domain rejection is reported in
// Result.Code with a nil err.
func (c *Client) execute(ctx context.Context, token, msgType string, payload []byte) (Result, error) {
req := connect.NewRequest(&edgev1.ExecuteRequest{MessageType: msgType, Payload: payload})
if token != "" {
req.Header().Set("Authorization", "Bearer "+token)
}
resp, err := c.rpc.Execute(ctx, req)
if err != nil {
return Result{Code: connectCode(err)}, err
}
return Result{Code: resp.Msg.ResultCode, Payload: resp.Msg.Payload}, nil
}
// connectCode renders a transport error as a short code for the report (e.g.
// "rate_limited" for HTTP 429, "unavailable", "deadline"), so the gateway-hammer can
// tally limiter rejections without inspecting full errors.
func connectCode(err error) string {
switch connect.CodeOf(err) {
case connect.CodeResourceExhausted:
return "rate_limited"
case connect.CodeUnauthenticated:
return "unauthenticated"
case connect.CodeUnavailable:
return "unavailable"
case connect.CodeDeadlineExceeded:
return "deadline"
default:
return "transport_error"
}
}
// Event is one decoded live event: its kind and raw FlatBuffers payload (the driver
// reacts to kind alone — your_turn / match_found drive a state fetch).
type Event struct {
Kind string
}
// Subscribe opens the live-event stream as token and invokes onEvent for each event
// until the context is cancelled or the stream ends. It blocks; run it in its own
// goroutine. Stream errors are returned for the caller to count and (optionally)
// reconnect.
func (c *Client) Subscribe(ctx context.Context, token string, onEvent func(Event)) error {
req := connect.NewRequest(&edgev1.SubscribeRequest{})
req.Header().Set("Authorization", "Bearer "+token)
stream, err := c.rpc.Subscribe(ctx, req)
if err != nil {
return err
}
defer stream.Close()
for stream.Receive() {
if onEvent != nil {
onEvent(Event{Kind: stream.Msg().Kind})
}
}
return stream.Err()
}
// pollInterval bounds how often a player re-checks one game's state; exported for the
// scenario's pacing math so a virtual player stays under the per-user rate limit.
const DefaultPollInterval = 3 * time.Second
+186
View File
@@ -0,0 +1,186 @@
package edge
import (
fb "scrabble/pkg/fbs/scrabblefb"
)
// Game is the decoded non-private game summary the driver needs to decide a turn.
type Game struct {
ID string
Variant string
DictVer string
Status string
Players int
ToMove int
MoveCount int
Seats []string // account ids in seat order
}
// Active reports whether the game is still in progress.
func (g Game) Active() bool { return g.Status == "active" }
// SeatOf returns the seat index of accountID, or -1 if it is not seated.
func (g Game) SeatOf(accountID string) int {
for i, id := range g.Seats {
if id == accountID {
return i
}
}
return -1
}
// State is a player's private view: the shared game plus their seat, rack (alphabet
// indices; 255 a blank) and bag size.
type State struct {
Game Game
Seat int
Rack []byte
BagLen int
}
// Tile is one placed tile from a decoded history record (concrete letter, blank flag).
type Tile struct {
Row, Col int
Letter string
Blank bool
}
// Move is one decoded history record (a committed play carries Tiles; pass/exchange
// carry only Action).
type Move struct {
Action string
Dir string
Tiles []Tile
}
// Invitation is the decoded subset the assembler matches on.
type Invitation struct {
ID string
InviterID string
Status string
GameID string
}
func decodeGameView(gv *fb.GameView) Game {
g := Game{
ID: string(gv.Id()),
Variant: string(gv.Variant()),
DictVer: string(gv.DictVersion()),
Status: string(gv.Status()),
Players: int(gv.Players()),
ToMove: int(gv.ToMove()),
MoveCount: int(gv.MoveCount()),
}
n := gv.SeatsLength()
g.Seats = make([]string, n)
var sv fb.SeatView
for j := 0; j < n; j++ {
if gv.Seats(&sv, j) {
g.Seats[sv.Seat()] = string(sv.AccountId())
}
}
return g
}
// decodeState reads a StateView payload.
func decodeState(payload []byte) State {
sv := fb.GetRootAsStateView(payload, 0)
var gv fb.GameView
st := State{
Seat: int(sv.Seat()),
BagLen: int(sv.BagLen()),
Rack: append([]byte(nil), sv.RackBytes()...),
}
if g := sv.Game(&gv); g != nil {
st.Game = decodeGameView(g)
}
return st
}
// decodeHistory reads a History payload into the decoded move journal.
func decodeHistory(payload []byte) []Move {
h := fb.GetRootAsHistory(payload, 0)
n := h.MovesLength()
moves := make([]Move, 0, n)
var mr fb.MoveRecord
for j := 0; j < n; j++ {
if !h.Moves(&mr, j) {
continue
}
m := Move{Action: string(mr.Action()), Dir: string(mr.Dir())}
tn := mr.TilesLength()
m.Tiles = make([]Tile, 0, tn)
var tr fb.TileRecord
for k := 0; k < tn; k++ {
if mr.Tiles(&tr, k) {
m.Tiles = append(m.Tiles, Tile{
Row: int(tr.Row()), Col: int(tr.Col()),
Letter: string(tr.Letter()), Blank: tr.Blank(),
})
}
}
moves = append(moves, m)
}
return moves
}
// decodeMoveResultGame reads a MoveResult payload and returns its post-move game.
func decodeMoveResultGame(payload []byte) Game {
mr := fb.GetRootAsMoveResult(payload, 0)
var gv fb.GameView
if g := mr.Game(&gv); g != nil {
return decodeGameView(g)
}
return Game{}
}
// decodeGameList reads a GameList payload.
func decodeGameList(payload []byte) []Game {
gl := fb.GetRootAsGameList(payload, 0)
n := gl.GamesLength()
games := make([]Game, 0, n)
var gv fb.GameView
for j := 0; j < n; j++ {
if gl.Games(&gv, j) {
games = append(games, decodeGameView(&gv))
}
}
return games
}
// decodeInvitationList reads an InvitationList payload into the matched subset.
func decodeInvitationList(payload []byte) []Invitation {
il := fb.GetRootAsInvitationList(payload, 0)
n := il.InvitationsLength()
out := make([]Invitation, 0, n)
var inv fb.Invitation
var ref fb.AccountRef
for j := 0; j < n; j++ {
if !il.Invitations(&inv, j) {
continue
}
iv := Invitation{
ID: string(inv.Id()),
Status: string(inv.Status()),
GameID: string(inv.GameId()),
}
if r := inv.Inviter(&ref); r != nil {
iv.InviterID = string(r.AccountId())
}
out = append(out, iv)
}
return out
}
// decodeMatch reads a MatchResult payload.
func decodeMatch(payload []byte) (matched bool, game Game) {
mr := fb.GetRootAsMatchResult(payload, 0)
if !mr.Matched() {
return false, Game{}
}
var gv fb.GameView
if g := mr.Game(&gv); g != nil {
return true, decodeGameView(g)
}
return true, Game{}
}
+184
View File
@@ -0,0 +1,184 @@
package edge
import (
flatbuffers "github.com/google/flatbuffers/go"
fb "scrabble/pkg/fbs/scrabblefb"
)
// PlayTile is one tile to place, addressed by alphabet index (255 marks a blank's
// carrier letter together with Blank=true), as the submit-play request carries it.
type PlayTile struct {
Row, Col int
Letter byte
Blank bool
}
// gameAction builds a GameActionRequest payload (just a game id): pass, nudge,
// history, draft.get.
func gameAction(gameID string) []byte {
b := flatbuffers.NewBuilder(64)
gid := b.CreateString(gameID)
fb.GameActionRequestStart(b)
fb.GameActionRequestAddGameId(b, gid)
b.Finish(fb.GameActionRequestEnd(b))
return b.FinishedBytes()
}
// stateReq builds a StateRequest payload. includeAlphabet asks the backend to embed
// the variant alphabet table (the driver sets it once per variant).
func stateReq(gameID string, includeAlphabet bool) []byte {
b := flatbuffers.NewBuilder(64)
gid := b.CreateString(gameID)
fb.StateRequestStart(b)
fb.StateRequestAddGameId(b, gid)
fb.StateRequestAddIncludeAlphabet(b, includeAlphabet)
b.Finish(fb.StateRequestEnd(b))
return b.FinishedBytes()
}
// submitPlay builds a SubmitPlayRequest payload. dir is "H" or "V"; tiles are the
// newly-placed tiles in main-word order.
func submitPlay(gameID, dir string, tiles []PlayTile) []byte {
b := flatbuffers.NewBuilder(256)
gid := b.CreateString(gameID)
d := b.CreateString(dir)
offs := make([]flatbuffers.UOffsetT, len(tiles))
for i, t := range tiles {
fb.PlayTileStart(b)
fb.PlayTileAddRow(b, int32(t.Row))
fb.PlayTileAddCol(b, int32(t.Col))
fb.PlayTileAddLetter(b, t.Letter)
fb.PlayTileAddBlank(b, t.Blank)
offs[i] = fb.PlayTileEnd(b)
}
fb.SubmitPlayRequestStartTilesVector(b, len(offs))
for i := len(offs) - 1; i >= 0; i-- {
b.PrependUOffsetT(offs[i])
}
tilesVec := b.EndVector(len(offs))
fb.SubmitPlayRequestStart(b)
fb.SubmitPlayRequestAddGameId(b, gid)
fb.SubmitPlayRequestAddDir(b, d)
fb.SubmitPlayRequestAddTiles(b, tilesVec)
b.Finish(fb.SubmitPlayRequestEnd(b))
return b.FinishedBytes()
}
// exchange builds an ExchangeRequest payload swapping the listed rack tiles (alphabet
// indices; 255 a blank).
func exchange(gameID string, tiles []byte) []byte {
b := flatbuffers.NewBuilder(64)
gid := b.CreateString(gameID)
vec := b.CreateByteVector(tiles)
fb.ExchangeRequestStart(b)
fb.ExchangeRequestAddGameId(b, gid)
fb.ExchangeRequestAddTiles(b, vec)
b.Finish(fb.ExchangeRequestEnd(b))
return b.FinishedBytes()
}
// checkWord builds a CheckWordRequest payload (alphabet indices for the word).
func checkWord(gameID string, word []byte) []byte {
b := flatbuffers.NewBuilder(64)
gid := b.CreateString(gameID)
vec := b.CreateByteVector(word)
fb.CheckWordRequestStart(b)
fb.CheckWordRequestAddGameId(b, gid)
fb.CheckWordRequestAddWord(b, vec)
b.Finish(fb.CheckWordRequestEnd(b))
return b.FinishedBytes()
}
// chatPost builds a ChatPostRequest payload.
func chatPost(gameID, body string) []byte {
b := flatbuffers.NewBuilder(128)
gid := b.CreateString(gameID)
bd := b.CreateString(body)
fb.ChatPostRequestStart(b)
fb.ChatPostRequestAddGameId(b, gid)
fb.ChatPostRequestAddBody(b, bd)
b.Finish(fb.ChatPostRequestEnd(b))
return b.FinishedBytes()
}
// draftSave builds a DraftRequest payload carrying the opaque composition JSON.
func draftSave(gameID, jsonStr string) []byte {
b := flatbuffers.NewBuilder(128)
gid := b.CreateString(gameID)
j := b.CreateString(jsonStr)
fb.DraftRequestStart(b)
fb.DraftRequestAddGameId(b, gid)
fb.DraftRequestAddJson(b, j)
b.Finish(fb.DraftRequestEnd(b))
return b.FinishedBytes()
}
// updateProfile builds an UpdateProfileRequest payload. It resends the marker display
// name and sane defaults so the account stays findable by the seeder's Cleanup.
func updateProfile(displayName, lang string) []byte {
b := flatbuffers.NewBuilder(192)
name := b.CreateString(displayName)
pl := b.CreateString(lang)
tz := b.CreateString("UTC")
as := b.CreateString("00:00")
ae := b.CreateString("07:00")
fb.UpdateProfileRequestStart(b)
fb.UpdateProfileRequestAddDisplayName(b, name)
fb.UpdateProfileRequestAddPreferredLanguage(b, pl)
fb.UpdateProfileRequestAddTimeZone(b, tz)
fb.UpdateProfileRequestAddAwayStart(b, as)
fb.UpdateProfileRequestAddAwayEnd(b, ae)
fb.UpdateProfileRequestAddBlockChat(b, false)
fb.UpdateProfileRequestAddBlockFriendRequests(b, false)
fb.UpdateProfileRequestAddNotificationsInAppOnly(b, true)
b.Finish(fb.UpdateProfileRequestEnd(b))
return b.FinishedBytes()
}
// createInvitation builds a CreateInvitationRequest payload. turnTimeoutSecs 0 asks
// the backend for its default; dropoutTiles "remove" is the standard policy.
func createInvitation(inviteeIDs []string, variant string, turnTimeoutSecs int) []byte {
b := flatbuffers.NewBuilder(256)
idOffs := make([]flatbuffers.UOffsetT, len(inviteeIDs))
for i, id := range inviteeIDs {
idOffs[i] = b.CreateString(id)
}
fb.CreateInvitationRequestStartInviteeIdsVector(b, len(idOffs))
for i := len(idOffs) - 1; i >= 0; i-- {
b.PrependUOffsetT(idOffs[i])
}
ids := b.EndVector(len(idOffs))
variantOff := b.CreateString(variant)
dropout := b.CreateString("remove")
fb.CreateInvitationRequestStart(b)
fb.CreateInvitationRequestAddInviteeIds(b, ids)
fb.CreateInvitationRequestAddVariant(b, variantOff)
fb.CreateInvitationRequestAddTurnTimeoutSecs(b, int32(turnTimeoutSecs))
fb.CreateInvitationRequestAddHintsAllowed(b, true)
fb.CreateInvitationRequestAddHintsPerPlayer(b, 1)
fb.CreateInvitationRequestAddDropoutTiles(b, dropout)
b.Finish(fb.CreateInvitationRequestEnd(b))
return b.FinishedBytes()
}
// invitationAction builds an InvitationActionRequest payload (accept / decline /
// cancel by id).
func invitationAction(invitationID string) []byte {
b := flatbuffers.NewBuilder(64)
id := b.CreateString(invitationID)
fb.InvitationActionRequestStart(b)
fb.InvitationActionRequestAddInvitationId(b, id)
b.Finish(fb.InvitationActionRequestEnd(b))
return b.FinishedBytes()
}
// enqueueReq builds an EnqueueRequest payload (join the per-variant auto-match pool).
func enqueueReq(variant string) []byte {
b := flatbuffers.NewBuilder(64)
v := b.CreateString(variant)
fb.EnqueueRequestStart(b)
fb.EnqueueRequestAddVariant(b, v)
b.Finish(fb.EnqueueRequestEnd(b))
return b.FinishedBytes()
}
+136
View File
@@ -0,0 +1,136 @@
package edge
import "context"
// The typed operations below each build a request, run Execute and decode the
// response. They return the decoded value (where any), the domain result code
// ("ok" or a stable error code) and a transport error. The scenario layer times the
// call and records the code; a non-"ok" code with a nil error is a domain rejection
// (for example "not_your_turn"), not a failure of the harness.
// State fetches the caller's private view of a game.
func (c *Client) State(ctx context.Context, token, gameID string) (State, string, error) {
r, err := c.execute(ctx, token, msgState, stateReq(gameID, false))
if err != nil || r.Code != "ok" {
return State{}, r.Code, err
}
return decodeState(r.Payload), r.Code, nil
}
// History fetches a game's decoded move journal (the board-replay source).
func (c *Client) History(ctx context.Context, token, gameID string) ([]Move, string, error) {
r, err := c.execute(ctx, token, msgHistory, gameAction(gameID))
if err != nil || r.Code != "ok" {
return nil, r.Code, err
}
return decodeHistory(r.Payload), r.Code, nil
}
// SubmitPlay commits a play and returns the post-move game.
func (c *Client) SubmitPlay(ctx context.Context, token, gameID, dir string, tiles []PlayTile) (Game, string, error) {
r, err := c.execute(ctx, token, msgSubmitPlay, submitPlay(gameID, dir, tiles))
if err != nil || r.Code != "ok" {
return Game{}, r.Code, err
}
return decodeMoveResultGame(r.Payload), r.Code, nil
}
// Pass forfeits the turn and returns the post-move game.
func (c *Client) Pass(ctx context.Context, token, gameID string) (Game, string, error) {
r, err := c.execute(ctx, token, msgPass, gameAction(gameID))
if err != nil || r.Code != "ok" {
return Game{}, r.Code, err
}
return decodeMoveResultGame(r.Payload), r.Code, nil
}
// Exchange swaps the listed rack tiles and returns the post-move game.
func (c *Client) Exchange(ctx context.Context, token, gameID string, tiles []byte) (Game, string, error) {
r, err := c.execute(ctx, token, msgExchange, exchange(gameID, tiles))
if err != nil || r.Code != "ok" {
return Game{}, r.Code, err
}
return decodeMoveResultGame(r.Payload), r.Code, nil
}
// Nudge prods the opponent whose turn it is.
func (c *Client) Nudge(ctx context.Context, token, gameID string) (string, error) {
r, err := c.execute(ctx, token, msgNudge, gameAction(gameID))
return r.Code, err
}
// ChatPost posts a per-game chat line.
func (c *Client) ChatPost(ctx context.Context, token, gameID, body string) (string, error) {
r, err := c.execute(ctx, token, msgChatPost, chatPost(gameID, body))
return r.Code, err
}
// CheckWord looks a word up in the game's pinned dictionary.
func (c *Client) CheckWord(ctx context.Context, token, gameID string, word []byte) (string, error) {
r, err := c.execute(ctx, token, msgCheckWord, checkWord(gameID, word))
return r.Code, err
}
// DraftSave stores the player's client-side composition.
func (c *Client) DraftSave(ctx context.Context, token, gameID, jsonStr string) (string, error) {
r, err := c.execute(ctx, token, msgDraftSave, draftSave(gameID, jsonStr))
return r.Code, err
}
// DraftGet retrieves the player's stored composition.
func (c *Client) DraftGet(ctx context.Context, token, gameID string) (string, error) {
r, err := c.execute(ctx, token, msgDraftGet, gameAction(gameID))
return r.Code, err
}
// ProfileUpdate overwrites the profile, resending the marker display name.
func (c *Client) ProfileUpdate(ctx context.Context, token, displayName, lang string) (string, error) {
r, err := c.execute(ctx, token, msgProfileUpd, updateProfile(displayName, lang))
return r.Code, err
}
// Stats reads the caller's lifetime statistics.
func (c *Client) Stats(ctx context.Context, token string) (string, error) {
r, err := c.execute(ctx, token, msgStatsGet, nil)
return r.Code, err
}
// GamesList lists the caller's games (active and finished).
func (c *Client) GamesList(ctx context.Context, token string) ([]Game, string, error) {
r, err := c.execute(ctx, token, msgGamesList, nil)
if err != nil || r.Code != "ok" {
return nil, r.Code, err
}
return decodeGameList(r.Payload), r.Code, nil
}
// CreateInvitation proposes a 2-4 player friend game to the named invitees.
func (c *Client) CreateInvitation(ctx context.Context, token string, inviteeIDs []string, variant string) (string, error) {
r, err := c.execute(ctx, token, msgInvCreate, createInvitation(inviteeIDs, variant, 0))
return r.Code, err
}
// AcceptInvitation accepts an invitation by id (the completing accept starts the game).
func (c *Client) AcceptInvitation(ctx context.Context, token, invitationID string) (string, error) {
r, err := c.execute(ctx, token, msgInvAccept, invitationAction(invitationID))
return r.Code, err
}
// ListInvitations lists the caller's open invitations.
func (c *Client) ListInvitations(ctx context.Context, token string) ([]Invitation, string, error) {
r, err := c.execute(ctx, token, msgInvList, nil)
if err != nil || r.Code != "ok" {
return nil, r.Code, err
}
return decodeInvitationList(r.Payload), r.Code, nil
}
// Enqueue joins the per-variant auto-match pool and reports any immediate pairing.
func (c *Client) Enqueue(ctx context.Context, token, variant string) (bool, Game, string, error) {
r, err := c.execute(ctx, token, msgEnqueue, enqueueReq(variant))
if err != nil || r.Code != "ok" {
return false, Game{}, r.Code, err
}
matched, game := decodeMatch(r.Payload)
return matched, game, r.Code, nil
}