e2771826fd
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 18s
CI / ui (pull_request) Has been skipped
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m5s
The loadtest harness never modelled game.evaluate — the debounced per-tile play preview a real client fires several times per turn, the hottest gameplay call. Model it (one evaluate per placed tile + reconsideration re-previews + draft.save, human-paced; --eval / --eval-recon toggle it). That realistic load surfaced the real bottleneck: the gateway's backend HTTP client used the default transport (MaxIdleConnsPerHost=2), so every sync call to the single backend host churned a fresh TCP connection — ~26500 TIME_WAIT sockets at 500 players (near the ephemeral-port ceiling), burning ~1.75 gateway cores while the backend sat near-idle. It was the unfixed root of the residual transport_error the earlier passes chased on the client side. Widen the keep-alive pool (backendMaxIdleConns=512, ~2x the observed 225-conn peak). At 500 players the churn collapses to ~0 and peak gateway CPU drops ~7x (~1.75 -> ~0.26 cores); postgres (~1.65 cores) becomes the busiest service. This overturns the earlier "gateway is the binding constraint, scale it horizontally" sizing — that was sizing around this bug, not a real floor. Consolidate the loadtest trip reports into one loadtest/REPORT.md (drop the R2/R7 split) and bake the finding into README / PRERELEASE / ARCHITECTURE / TESTING.
340 lines
12 KiB
Go
340 lines
12 KiB
Go
// Package scenario drives virtual players against the gateway edge protocol: it
|
|
// assembles real games through the invitation flow, then runs each player's turn
|
|
// loop (poll state, replay history, generate a legal move with the embedded solver,
|
|
// submit it) plus a fraction of secondary operations. It exposes the moderate
|
|
// realistic ramp and a separate gateway-hammer.
|
|
package scenario
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"math/rand"
|
|
"slices"
|
|
"sync"
|
|
"time"
|
|
|
|
"scrabble/loadtest/internal/edge"
|
|
"scrabble/loadtest/internal/moves"
|
|
"scrabble/loadtest/internal/report"
|
|
"scrabble/loadtest/internal/seed"
|
|
)
|
|
|
|
// Driver ties the gateway endpoint, the local move generator and the run recorder
|
|
// together. It builds one edge client per virtual player, so each player owns its
|
|
// h2c connection (its Subscribe stream and Execute calls share it) the way a real
|
|
// client does, rather than multiplexing every player over a single shared transport.
|
|
type Driver struct {
|
|
gateway string // gateway base URL, e.g. http://gateway:8081
|
|
moves *moves.Registry
|
|
rec *report.Recorder
|
|
log *slog.Logger
|
|
}
|
|
|
|
// NewDriver builds a Driver targeting the gateway base URL.
|
|
func NewDriver(gateway string, m *moves.Registry, rec *report.Recorder, log *slog.Logger) *Driver {
|
|
return &Driver{gateway: gateway, moves: m, rec: rec, log: log}
|
|
}
|
|
|
|
// RealisticConfig parameterises the under-the-limit ramp.
|
|
type RealisticConfig struct {
|
|
Steps []int // concurrent active players per step (cumulative)
|
|
StepDur time.Duration // hold time per step
|
|
GamesPerPlayer int // target concurrent games per player; 0 => random 3..5
|
|
Tick time.Duration // per-player operation cadence (keeps a player under the per-user limit)
|
|
SecondaryProb float64 // chance per tick of a non-move operation
|
|
Eval bool // model the per-tile evaluate preview (the gameplay hot path); false reproduces the pre-evaluate harness
|
|
EvalRecon int // extra full-composition evaluate re-previews per play, beyond one per placed tile
|
|
}
|
|
|
|
// DefaultRealistic returns the moderate ramp: 50 -> 200
|
|
// -> 500 concurrent players, ~12 minutes per step, ~1 op/s per player, with the
|
|
// per-tile evaluate preview modelled (the realistic hot path).
|
|
func DefaultRealistic() RealisticConfig {
|
|
return RealisticConfig{
|
|
Steps: []int{50, 200, 500},
|
|
StepDur: 12 * time.Minute,
|
|
Tick: 800 * time.Millisecond,
|
|
SecondaryProb: 0.08,
|
|
Eval: true,
|
|
EvalRecon: 1,
|
|
}
|
|
}
|
|
|
|
// evalGapBase and evalGapSpan bound the modelled pause between successive tile
|
|
// placements: the client's 250 ms debounce coalesces faster drags into a single
|
|
// evaluate, so a thoughtful player's previews are spaced by a gap drawn from
|
|
// [base, base+span] — wide enough that a normal composition stays under the per-user
|
|
// rate limit, the way a real one does (the limiter's cost is measured by the hammer,
|
|
// not by self-inflicted rejections here).
|
|
const (
|
|
evalGapBase = 250 * time.Millisecond
|
|
evalGapSpan = 500 * time.Millisecond
|
|
)
|
|
|
|
// RunRealistic runs the staged ramp. Each step activates more players (drawn from the
|
|
// seeded pool), assembles a cohort of games for them and starts their turn loops; the
|
|
// loops run until the whole ramp ends. Players from earlier steps keep playing, so
|
|
// load is cumulative.
|
|
func (d *Driver) RunRealistic(ctx context.Context, pool *seed.Pool, cfg RealisticConfig) error {
|
|
players := shuffledPool(pool)
|
|
runCtx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
|
|
var wg sync.WaitGroup
|
|
activated := 0
|
|
for si, target := range cfg.Steps {
|
|
if target > len(players) {
|
|
target = len(players)
|
|
}
|
|
cohort := players[activated:target]
|
|
activated = target
|
|
if len(cohort) >= 2 {
|
|
rng := rand.New(rand.NewSource(time.Now().UnixNano() + int64(si)))
|
|
games := d.assembleCohort(runCtx, cohort, cfg.GamesPerPlayer, rng)
|
|
byPlayer := gamesByPlayer(games)
|
|
d.log.Info("ramp step", "step", si+1, "active", activated, "cohort", len(cohort), "games", len(games))
|
|
for pi := range cohort {
|
|
p := cohort[pi]
|
|
wg.Add(1)
|
|
go func(p seed.Account, pg []*Game, sd int64) {
|
|
defer wg.Done()
|
|
d.playerLoop(runCtx, p, pg, cfg, rand.New(rand.NewSource(sd)))
|
|
}(p, byPlayer[p.ID.String()], time.Now().UnixNano()+int64(pi))
|
|
}
|
|
} else {
|
|
d.log.Warn("ramp step skipped: cohort too small", "step", si+1, "cohort", len(cohort))
|
|
}
|
|
select {
|
|
case <-time.After(cfg.StepDur):
|
|
case <-ctx.Done():
|
|
cancel()
|
|
wg.Wait()
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
cancel()
|
|
wg.Wait()
|
|
return nil
|
|
}
|
|
|
|
// playerLoop runs one virtual player over its own edge client (its own h2c
|
|
// connection): a live-event subscription (loads the push hub, counts events) plus a
|
|
// round-robin turn loop over the player's games. A game that has finished is dropped
|
|
// from the rotation so secondary ops stop hitting an ended game; once no active game
|
|
// remains the player idles, still holding its stream, until the run ends.
|
|
func (d *Driver) playerLoop(ctx context.Context, p seed.Account, games []*Game, cfg RealisticConfig, rng *rand.Rand) {
|
|
c := edge.New(d.gateway)
|
|
go d.subscribeLoop(ctx, c, p)
|
|
active := games
|
|
if len(active) == 0 {
|
|
<-ctx.Done()
|
|
return
|
|
}
|
|
ticker := time.NewTicker(cfg.Tick)
|
|
defer ticker.Stop()
|
|
gi := 0
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
g := active[gi%len(active)]
|
|
gi++
|
|
if rng.Float64() < cfg.SecondaryProb {
|
|
d.secondaryOp(ctx, c, p, g, rng)
|
|
continue
|
|
}
|
|
if d.playTurn(ctx, c, p, g, cfg, rng) {
|
|
active = slices.DeleteFunc(active, func(x *Game) bool { return x == g })
|
|
gi = 0
|
|
if len(active) == 0 {
|
|
<-ctx.Done()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// subscribeLoop holds the player's live-event stream open on the player's client,
|
|
// counting events and reconnecting with a brief backoff after a drop, until the run
|
|
// ends.
|
|
func (d *Driver) subscribeLoop(ctx context.Context, c *edge.Client, p seed.Account) {
|
|
for ctx.Err() == nil {
|
|
err := c.Subscribe(ctx, p.Token, func(e edge.Event) { d.rec.Event(e.Kind) })
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
if err != nil {
|
|
d.rec.StreamErr()
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(time.Second):
|
|
}
|
|
}
|
|
}
|
|
|
|
// playTurn plays one turn in g over the player's client when it is the player's
|
|
// move: fetch state, replay history, pick a legal move, compose it (the per-tile
|
|
// evaluate previews a real client fires) and submit it (or exchange / pass). It reports
|
|
// whether the game has finished, so the caller can drop it from the rotation.
|
|
func (d *Driver) playTurn(ctx context.Context, c *edge.Client, p seed.Account, g *Game, cfg RealisticConfig, rng *rand.Rand) (finished bool) {
|
|
seat := g.seatOf(p.ID.String())
|
|
if seat < 0 {
|
|
return false
|
|
}
|
|
t0 := time.Now()
|
|
st, code, err := c.State(ctx, p.Token, g.ID)
|
|
d.rec.Record("game.state", code, time.Since(t0))
|
|
if err != nil || code != "ok" {
|
|
return false
|
|
}
|
|
if !st.Game.Active() {
|
|
return true
|
|
}
|
|
if st.Game.ToMove != seat {
|
|
return false
|
|
}
|
|
|
|
t0 = time.Now()
|
|
hist, hc, err := c.History(ctx, p.Token, g.ID)
|
|
d.rec.Record("game.history", hc, time.Since(t0))
|
|
if err != nil || hc != "ok" {
|
|
return false
|
|
}
|
|
|
|
action, err := d.moves.Pick(g.Variant, hist, st.Rack, st.BagLen, rng)
|
|
if err != nil {
|
|
d.log.Debug("pick move", "variant", g.Variant, "err", err)
|
|
return false
|
|
}
|
|
switch action.Kind {
|
|
case "play":
|
|
d.composePlay(ctx, c, p, g, action.Tiles, cfg, rng)
|
|
t0 = time.Now()
|
|
_, code, _ := c.SubmitPlay(ctx, p.Token, g.ID, action.Tiles)
|
|
d.rec.Record("game.submit_play", code, time.Since(t0))
|
|
case "exchange":
|
|
t0 = time.Now()
|
|
_, code, _ := c.Exchange(ctx, p.Token, g.ID, action.Exchange)
|
|
d.rec.Record("game.exchange", code, time.Since(t0))
|
|
default:
|
|
t0 = time.Now()
|
|
_, code, _ := c.Pass(ctx, p.Token, g.ID)
|
|
d.rec.Record("game.pass", code, time.Since(t0))
|
|
}
|
|
return false
|
|
}
|
|
|
|
// composePlay models a player arranging the chosen play tile by tile before committing:
|
|
// the debounced evaluate preview the real client fires on each placement (a growing prefix
|
|
// of the tiles), a few full-composition re-previews for reconsideration (recall a tile, try
|
|
// another spot), and the single draft persistence the client debounces out. evaluate is the
|
|
// hottest gameplay request at scale, so omitting it (the pre-evaluate harness) understated
|
|
// the load; cfg.Eval false reproduces that baseline for an A/B comparison. Every step
|
|
// honours ctx, so end-of-run cancellation never blocks on a sleep or an in-flight preview.
|
|
func (d *Driver) composePlay(ctx context.Context, c *edge.Client, p seed.Account, g *Game, tiles []edge.PlayTile, cfg RealisticConfig, rng *rand.Rand) {
|
|
if !cfg.Eval || len(tiles) == 0 {
|
|
return
|
|
}
|
|
// One evaluate per landed tile: the growing prefix mirrors the client re-previewing
|
|
// after each placement (an early prefix is often illegal, which is still a successful
|
|
// "ok" round trip — exactly the backend work a real composition triggers).
|
|
for n := 1; n <= len(tiles); n++ {
|
|
if !jitterSleep(ctx, rng, evalGapBase, evalGapSpan) {
|
|
return
|
|
}
|
|
t0 := time.Now()
|
|
code, _ := c.Evaluate(ctx, p.Token, g.ID, tiles[:n])
|
|
d.rec.Record("game.evaluate", code, time.Since(t0))
|
|
}
|
|
for r := 0; r < cfg.EvalRecon; r++ {
|
|
if !jitterSleep(ctx, rng, evalGapBase, evalGapSpan) {
|
|
return
|
|
}
|
|
t0 := time.Now()
|
|
code, _ := c.Evaluate(ctx, p.Token, g.ID, tiles)
|
|
d.rec.Record("game.evaluate", code, time.Since(t0))
|
|
}
|
|
// The client persists the in-progress composition (debounced to one upsert). Its opaque
|
|
// JSON content does not affect the call's cost, so a minimal valid shape stands in.
|
|
t0 := time.Now()
|
|
code, _ := c.DraftSave(ctx, p.Token, g.ID, `{"rack_order":"","board_tiles":[]}`)
|
|
d.rec.Record("draft.save", code, time.Since(t0))
|
|
}
|
|
|
|
// jitterSleep pauses for a randomised gap in [base, base+span], modelling the human pause
|
|
// between tile placements that the client's debounce coalesces into one evaluate. It
|
|
// returns false if ctx is cancelled during the wait, so a composition unwinds promptly at
|
|
// end of run.
|
|
func jitterSleep(ctx context.Context, rng *rand.Rand, base, span time.Duration) bool {
|
|
d := base + time.Duration(rng.Int63n(int64(span)+1))
|
|
t := time.NewTimer(d)
|
|
defer t.Stop()
|
|
select {
|
|
case <-ctx.Done():
|
|
return false
|
|
case <-t.C:
|
|
return true
|
|
}
|
|
}
|
|
|
|
// secondaryOp exercises one of the non-move edge operations the plan calls out, so
|
|
// the run touches nudge / chat / check-word / draft / profile / stats too, over the
|
|
// player's own client.
|
|
func (d *Driver) secondaryOp(ctx context.Context, c *edge.Client, p seed.Account, g *Game, rng *rand.Rand) {
|
|
t0 := time.Now()
|
|
switch rng.Intn(7) {
|
|
case 0:
|
|
code, _ := c.Nudge(ctx, p.Token, g.ID)
|
|
d.rec.Record("chat.nudge", code, time.Since(t0))
|
|
case 1:
|
|
code, _ := c.ChatPost(ctx, p.Token, g.ID, "gg")
|
|
d.rec.Record("chat.post", code, time.Since(t0))
|
|
case 2:
|
|
code, _ := c.CheckWord(ctx, p.Token, g.ID, []byte{0, 1, 2})
|
|
d.rec.Record("game.check_word", code, time.Since(t0))
|
|
case 3:
|
|
// rack_order is an opaque string and board_tiles a (here empty) array, per the
|
|
// backend draft DTO; a malformed shape is rejected as bad_request.
|
|
code, _ := c.DraftSave(ctx, p.Token, g.ID, `{"rack_order":"","board_tiles":[]}`)
|
|
d.rec.Record("draft.save", code, time.Since(t0))
|
|
case 4:
|
|
code, _ := c.DraftGet(ctx, p.Token, g.ID)
|
|
d.rec.Record("draft.get", code, time.Since(t0))
|
|
case 5:
|
|
lang := "en"
|
|
if rng.Intn(2) == 1 {
|
|
lang = "ru"
|
|
}
|
|
code, _ := c.ProfileUpdate(ctx, p.Token, p.Name, lang)
|
|
d.rec.Record("profile.update", code, time.Since(t0))
|
|
default:
|
|
code, _ := c.Stats(ctx, p.Token)
|
|
d.rec.Record("stats.get", code, time.Since(t0))
|
|
}
|
|
}
|
|
|
|
// shuffledPool returns every seeded account in random order, so an active set is a
|
|
// representative mix of durable and guest accounts.
|
|
func shuffledPool(pool *seed.Pool) []seed.Account {
|
|
all := pool.All()
|
|
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
|
|
rng.Shuffle(len(all), func(i, j int) { all[i], all[j] = all[j], all[i] })
|
|
return all
|
|
}
|
|
|
|
// gamesByPlayer indexes the assembled games by each member's account id.
|
|
func gamesByPlayer(games []*Game) map[string][]*Game {
|
|
m := make(map[string][]*Game)
|
|
for _, g := range games {
|
|
for _, mem := range g.Members {
|
|
id := mem.ID.String()
|
|
m[id] = append(m[id], g)
|
|
}
|
|
}
|
|
return m
|
|
}
|