Files
scrabble-game/loadtest/internal/scenario/scenario.go
T
Ilia Denisov e2771826fd
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 18s
CI / ui (pull_request) Has been skipped
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m5s
perf(gateway): pool backend conns; loadtest evaluate hot path
The loadtest harness never modelled game.evaluate — the debounced per-tile
play preview a real client fires several times per turn, the hottest gameplay
call. Model it (one evaluate per placed tile + reconsideration re-previews +
draft.save, human-paced; --eval / --eval-recon toggle it).

That realistic load surfaced the real bottleneck: the gateway's backend HTTP
client used the default transport (MaxIdleConnsPerHost=2), so every sync call
to the single backend host churned a fresh TCP connection — ~26500 TIME_WAIT
sockets at 500 players (near the ephemeral-port ceiling), burning ~1.75 gateway
cores while the backend sat near-idle. It was the unfixed root of the residual
transport_error the earlier passes chased on the client side.

Widen the keep-alive pool (backendMaxIdleConns=512, ~2x the observed 225-conn
peak). At 500 players the churn collapses to ~0 and peak gateway CPU drops ~7x
(~1.75 -> ~0.26 cores); postgres (~1.65 cores) becomes the busiest service.
This overturns the earlier "gateway is the binding constraint, scale it
horizontally" sizing — that was sizing around this bug, not a real floor.

Consolidate the loadtest trip reports into one loadtest/REPORT.md (drop the
R2/R7 split) and bake the finding into README / PRERELEASE / ARCHITECTURE /
TESTING.
2026-06-21 19:55:57 +02:00

340 lines
12 KiB
Go

// Package scenario drives virtual players against the gateway edge protocol: it
// assembles real games through the invitation flow, then runs each player's turn
// loop (poll state, replay history, generate a legal move with the embedded solver,
// submit it) plus a fraction of secondary operations. It exposes the moderate
// realistic ramp and a separate gateway-hammer.
package scenario
import (
"context"
"log/slog"
"math/rand"
"slices"
"sync"
"time"
"scrabble/loadtest/internal/edge"
"scrabble/loadtest/internal/moves"
"scrabble/loadtest/internal/report"
"scrabble/loadtest/internal/seed"
)
// Driver ties the gateway endpoint, the local move generator and the run recorder
// together. It builds one edge client per virtual player, so each player owns its
// h2c connection (its Subscribe stream and Execute calls share it) the way a real
// client does, rather than multiplexing every player over a single shared transport.
type Driver struct {
gateway string // gateway base URL, e.g. http://gateway:8081
moves *moves.Registry
rec *report.Recorder
log *slog.Logger
}
// NewDriver builds a Driver targeting the gateway base URL.
func NewDriver(gateway string, m *moves.Registry, rec *report.Recorder, log *slog.Logger) *Driver {
return &Driver{gateway: gateway, moves: m, rec: rec, log: log}
}
// RealisticConfig parameterises the under-the-limit ramp.
type RealisticConfig struct {
Steps []int // concurrent active players per step (cumulative)
StepDur time.Duration // hold time per step
GamesPerPlayer int // target concurrent games per player; 0 => random 3..5
Tick time.Duration // per-player operation cadence (keeps a player under the per-user limit)
SecondaryProb float64 // chance per tick of a non-move operation
Eval bool // model the per-tile evaluate preview (the gameplay hot path); false reproduces the pre-evaluate harness
EvalRecon int // extra full-composition evaluate re-previews per play, beyond one per placed tile
}
// DefaultRealistic returns the moderate ramp: 50 -> 200
// -> 500 concurrent players, ~12 minutes per step, ~1 op/s per player, with the
// per-tile evaluate preview modelled (the realistic hot path).
func DefaultRealistic() RealisticConfig {
return RealisticConfig{
Steps: []int{50, 200, 500},
StepDur: 12 * time.Minute,
Tick: 800 * time.Millisecond,
SecondaryProb: 0.08,
Eval: true,
EvalRecon: 1,
}
}
// evalGapBase and evalGapSpan bound the modelled pause between successive tile
// placements: the client's 250 ms debounce coalesces faster drags into a single
// evaluate, so a thoughtful player's previews are spaced by a gap drawn from
// [base, base+span] — wide enough that a normal composition stays under the per-user
// rate limit, the way a real one does (the limiter's cost is measured by the hammer,
// not by self-inflicted rejections here).
const (
evalGapBase = 250 * time.Millisecond
evalGapSpan = 500 * time.Millisecond
)
// RunRealistic runs the staged ramp. Each step activates more players (drawn from the
// seeded pool), assembles a cohort of games for them and starts their turn loops; the
// loops run until the whole ramp ends. Players from earlier steps keep playing, so
// load is cumulative.
func (d *Driver) RunRealistic(ctx context.Context, pool *seed.Pool, cfg RealisticConfig) error {
players := shuffledPool(pool)
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
var wg sync.WaitGroup
activated := 0
for si, target := range cfg.Steps {
if target > len(players) {
target = len(players)
}
cohort := players[activated:target]
activated = target
if len(cohort) >= 2 {
rng := rand.New(rand.NewSource(time.Now().UnixNano() + int64(si)))
games := d.assembleCohort(runCtx, cohort, cfg.GamesPerPlayer, rng)
byPlayer := gamesByPlayer(games)
d.log.Info("ramp step", "step", si+1, "active", activated, "cohort", len(cohort), "games", len(games))
for pi := range cohort {
p := cohort[pi]
wg.Add(1)
go func(p seed.Account, pg []*Game, sd int64) {
defer wg.Done()
d.playerLoop(runCtx, p, pg, cfg, rand.New(rand.NewSource(sd)))
}(p, byPlayer[p.ID.String()], time.Now().UnixNano()+int64(pi))
}
} else {
d.log.Warn("ramp step skipped: cohort too small", "step", si+1, "cohort", len(cohort))
}
select {
case <-time.After(cfg.StepDur):
case <-ctx.Done():
cancel()
wg.Wait()
return ctx.Err()
}
}
cancel()
wg.Wait()
return nil
}
// playerLoop runs one virtual player over its own edge client (its own h2c
// connection): a live-event subscription (loads the push hub, counts events) plus a
// round-robin turn loop over the player's games. A game that has finished is dropped
// from the rotation so secondary ops stop hitting an ended game; once no active game
// remains the player idles, still holding its stream, until the run ends.
func (d *Driver) playerLoop(ctx context.Context, p seed.Account, games []*Game, cfg RealisticConfig, rng *rand.Rand) {
c := edge.New(d.gateway)
go d.subscribeLoop(ctx, c, p)
active := games
if len(active) == 0 {
<-ctx.Done()
return
}
ticker := time.NewTicker(cfg.Tick)
defer ticker.Stop()
gi := 0
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
g := active[gi%len(active)]
gi++
if rng.Float64() < cfg.SecondaryProb {
d.secondaryOp(ctx, c, p, g, rng)
continue
}
if d.playTurn(ctx, c, p, g, cfg, rng) {
active = slices.DeleteFunc(active, func(x *Game) bool { return x == g })
gi = 0
if len(active) == 0 {
<-ctx.Done()
return
}
}
}
}
}
// subscribeLoop holds the player's live-event stream open on the player's client,
// counting events and reconnecting with a brief backoff after a drop, until the run
// ends.
func (d *Driver) subscribeLoop(ctx context.Context, c *edge.Client, p seed.Account) {
for ctx.Err() == nil {
err := c.Subscribe(ctx, p.Token, func(e edge.Event) { d.rec.Event(e.Kind) })
if ctx.Err() != nil {
return
}
if err != nil {
d.rec.StreamErr()
}
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
}
// playTurn plays one turn in g over the player's client when it is the player's
// move: fetch state, replay history, pick a legal move, compose it (the per-tile
// evaluate previews a real client fires) and submit it (or exchange / pass). It reports
// whether the game has finished, so the caller can drop it from the rotation.
func (d *Driver) playTurn(ctx context.Context, c *edge.Client, p seed.Account, g *Game, cfg RealisticConfig, rng *rand.Rand) (finished bool) {
seat := g.seatOf(p.ID.String())
if seat < 0 {
return false
}
t0 := time.Now()
st, code, err := c.State(ctx, p.Token, g.ID)
d.rec.Record("game.state", code, time.Since(t0))
if err != nil || code != "ok" {
return false
}
if !st.Game.Active() {
return true
}
if st.Game.ToMove != seat {
return false
}
t0 = time.Now()
hist, hc, err := c.History(ctx, p.Token, g.ID)
d.rec.Record("game.history", hc, time.Since(t0))
if err != nil || hc != "ok" {
return false
}
action, err := d.moves.Pick(g.Variant, hist, st.Rack, st.BagLen, rng)
if err != nil {
d.log.Debug("pick move", "variant", g.Variant, "err", err)
return false
}
switch action.Kind {
case "play":
d.composePlay(ctx, c, p, g, action.Tiles, cfg, rng)
t0 = time.Now()
_, code, _ := c.SubmitPlay(ctx, p.Token, g.ID, action.Tiles)
d.rec.Record("game.submit_play", code, time.Since(t0))
case "exchange":
t0 = time.Now()
_, code, _ := c.Exchange(ctx, p.Token, g.ID, action.Exchange)
d.rec.Record("game.exchange", code, time.Since(t0))
default:
t0 = time.Now()
_, code, _ := c.Pass(ctx, p.Token, g.ID)
d.rec.Record("game.pass", code, time.Since(t0))
}
return false
}
// composePlay models a player arranging the chosen play tile by tile before committing:
// the debounced evaluate preview the real client fires on each placement (a growing prefix
// of the tiles), a few full-composition re-previews for reconsideration (recall a tile, try
// another spot), and the single draft persistence the client debounces out. evaluate is the
// hottest gameplay request at scale, so omitting it (the pre-evaluate harness) understated
// the load; cfg.Eval false reproduces that baseline for an A/B comparison. Every step
// honours ctx, so end-of-run cancellation never blocks on a sleep or an in-flight preview.
func (d *Driver) composePlay(ctx context.Context, c *edge.Client, p seed.Account, g *Game, tiles []edge.PlayTile, cfg RealisticConfig, rng *rand.Rand) {
if !cfg.Eval || len(tiles) == 0 {
return
}
// One evaluate per landed tile: the growing prefix mirrors the client re-previewing
// after each placement (an early prefix is often illegal, which is still a successful
// "ok" round trip — exactly the backend work a real composition triggers).
for n := 1; n <= len(tiles); n++ {
if !jitterSleep(ctx, rng, evalGapBase, evalGapSpan) {
return
}
t0 := time.Now()
code, _ := c.Evaluate(ctx, p.Token, g.ID, tiles[:n])
d.rec.Record("game.evaluate", code, time.Since(t0))
}
for r := 0; r < cfg.EvalRecon; r++ {
if !jitterSleep(ctx, rng, evalGapBase, evalGapSpan) {
return
}
t0 := time.Now()
code, _ := c.Evaluate(ctx, p.Token, g.ID, tiles)
d.rec.Record("game.evaluate", code, time.Since(t0))
}
// The client persists the in-progress composition (debounced to one upsert). Its opaque
// JSON content does not affect the call's cost, so a minimal valid shape stands in.
t0 := time.Now()
code, _ := c.DraftSave(ctx, p.Token, g.ID, `{"rack_order":"","board_tiles":[]}`)
d.rec.Record("draft.save", code, time.Since(t0))
}
// jitterSleep pauses for a randomised gap in [base, base+span], modelling the human pause
// between tile placements that the client's debounce coalesces into one evaluate. It
// returns false if ctx is cancelled during the wait, so a composition unwinds promptly at
// end of run.
func jitterSleep(ctx context.Context, rng *rand.Rand, base, span time.Duration) bool {
d := base + time.Duration(rng.Int63n(int64(span)+1))
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
return false
case <-t.C:
return true
}
}
// secondaryOp exercises one of the non-move edge operations the plan calls out, so
// the run touches nudge / chat / check-word / draft / profile / stats too, over the
// player's own client.
func (d *Driver) secondaryOp(ctx context.Context, c *edge.Client, p seed.Account, g *Game, rng *rand.Rand) {
t0 := time.Now()
switch rng.Intn(7) {
case 0:
code, _ := c.Nudge(ctx, p.Token, g.ID)
d.rec.Record("chat.nudge", code, time.Since(t0))
case 1:
code, _ := c.ChatPost(ctx, p.Token, g.ID, "gg")
d.rec.Record("chat.post", code, time.Since(t0))
case 2:
code, _ := c.CheckWord(ctx, p.Token, g.ID, []byte{0, 1, 2})
d.rec.Record("game.check_word", code, time.Since(t0))
case 3:
// rack_order is an opaque string and board_tiles a (here empty) array, per the
// backend draft DTO; a malformed shape is rejected as bad_request.
code, _ := c.DraftSave(ctx, p.Token, g.ID, `{"rack_order":"","board_tiles":[]}`)
d.rec.Record("draft.save", code, time.Since(t0))
case 4:
code, _ := c.DraftGet(ctx, p.Token, g.ID)
d.rec.Record("draft.get", code, time.Since(t0))
case 5:
lang := "en"
if rng.Intn(2) == 1 {
lang = "ru"
}
code, _ := c.ProfileUpdate(ctx, p.Token, p.Name, lang)
d.rec.Record("profile.update", code, time.Since(t0))
default:
code, _ := c.Stats(ctx, p.Token)
d.rec.Record("stats.get", code, time.Since(t0))
}
}
// shuffledPool returns every seeded account in random order, so an active set is a
// representative mix of durable and guest accounts.
func shuffledPool(pool *seed.Pool) []seed.Account {
all := pool.All()
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
rng.Shuffle(len(all), func(i, j int) { all[i], all[j] = all[j], all[i] })
return all
}
// gamesByPlayer indexes the assembled games by each member's account id.
func gamesByPlayer(games []*Game) map[string][]*Game {
m := make(map[string][]*Game)
for _, g := range games {
for _, mem := range g.Members {
id := mem.ID.String()
m[id] = append(m[id], g)
}
}
return m
}