Files
scrabble-game/loadtest/internal/scenario/scenario.go
T
Ilia Denisov 04263a17ca R7: per-player transports + drop finished games in the load harness
Each virtual player now builds its own edge.Client (its own h2c connection
carrying both the Subscribe stream and the Execute calls), instead of every
player multiplexing over a single shared http2.Transport. The R2 trip report
traced the ~14% transport_error on game.state at 500 players to that single
shared transport; per-player connections mirror real clients and isolate the
artifact. The assembly burst and the gateway-hammer each get their own client.

playTurn now reports when a game has finished so playerLoop drops it from the
rotation (slices.DeleteFunc); once no active game remains the player idles while
still holding its stream. This stops secondary ops from hammering game_finished
on already-ended games (the other R2 harness finding).
2026-06-10 18:53:07 +02:00

270 lines
8.6 KiB
Go

// Package scenario drives virtual players against the gateway edge protocol: it
// assembles real games through the invitation flow, then runs each player's turn
// loop (poll state, replay history, generate a legal move with the embedded solver,
// submit it) plus a fraction of secondary operations. It exposes the moderate
// realistic ramp and a separate gateway-hammer.
package scenario
import (
"context"
"log/slog"
"math/rand"
"slices"
"sync"
"time"
"scrabble/loadtest/internal/edge"
"scrabble/loadtest/internal/moves"
"scrabble/loadtest/internal/report"
"scrabble/loadtest/internal/seed"
)
// Driver ties the gateway endpoint, the local move generator and the run recorder
// together. It builds one edge client per virtual player, so each player owns its
// h2c connection (its Subscribe stream and Execute calls share it) the way a real
// client does, rather than multiplexing every player over a single shared transport.
type Driver struct {
gateway string // gateway base URL, e.g. http://gateway:8081
moves *moves.Registry
rec *report.Recorder
log *slog.Logger
}
// NewDriver builds a Driver targeting the gateway base URL.
func NewDriver(gateway string, m *moves.Registry, rec *report.Recorder, log *slog.Logger) *Driver {
return &Driver{gateway: gateway, moves: m, rec: rec, log: log}
}
// RealisticConfig parameterises the under-the-limit ramp.
type RealisticConfig struct {
Steps []int // concurrent active players per step (cumulative)
StepDur time.Duration // hold time per step
GamesPerPlayer int // target concurrent games per player; 0 => random 3..5
Tick time.Duration // per-player operation cadence (keeps a player under the per-user limit)
SecondaryProb float64 // chance per tick of a non-move operation
}
// DefaultRealistic returns the moderate ramp: 50 -> 200
// -> 500 concurrent players, ~12 minutes per step, ~1 op/s per player.
func DefaultRealistic() RealisticConfig {
return RealisticConfig{
Steps: []int{50, 200, 500},
StepDur: 12 * time.Minute,
Tick: 800 * time.Millisecond,
SecondaryProb: 0.08,
}
}
// RunRealistic runs the staged ramp. Each step activates more players (drawn from the
// seeded pool), assembles a cohort of games for them and starts their turn loops; the
// loops run until the whole ramp ends. Players from earlier steps keep playing, so
// load is cumulative.
func (d *Driver) RunRealistic(ctx context.Context, pool *seed.Pool, cfg RealisticConfig) error {
players := shuffledPool(pool)
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
var wg sync.WaitGroup
activated := 0
for si, target := range cfg.Steps {
if target > len(players) {
target = len(players)
}
cohort := players[activated:target]
activated = target
if len(cohort) >= 2 {
rng := rand.New(rand.NewSource(time.Now().UnixNano() + int64(si)))
games := d.assembleCohort(runCtx, cohort, cfg.GamesPerPlayer, rng)
byPlayer := gamesByPlayer(games)
d.log.Info("ramp step", "step", si+1, "active", activated, "cohort", len(cohort), "games", len(games))
for pi := range cohort {
p := cohort[pi]
wg.Add(1)
go func(p seed.Account, pg []*Game, sd int64) {
defer wg.Done()
d.playerLoop(runCtx, p, pg, cfg, rand.New(rand.NewSource(sd)))
}(p, byPlayer[p.ID.String()], time.Now().UnixNano()+int64(pi))
}
} else {
d.log.Warn("ramp step skipped: cohort too small", "step", si+1, "cohort", len(cohort))
}
select {
case <-time.After(cfg.StepDur):
case <-ctx.Done():
cancel()
wg.Wait()
return ctx.Err()
}
}
cancel()
wg.Wait()
return nil
}
// playerLoop runs one virtual player over its own edge client (its own h2c
// connection): a live-event subscription (loads the push hub, counts events) plus a
// round-robin turn loop over the player's games. A game that has finished is dropped
// from the rotation so secondary ops stop hitting an ended game; once no active game
// remains the player idles, still holding its stream, until the run ends.
func (d *Driver) playerLoop(ctx context.Context, p seed.Account, games []*Game, cfg RealisticConfig, rng *rand.Rand) {
c := edge.New(d.gateway)
go d.subscribeLoop(ctx, c, p)
active := games
if len(active) == 0 {
<-ctx.Done()
return
}
ticker := time.NewTicker(cfg.Tick)
defer ticker.Stop()
gi := 0
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
g := active[gi%len(active)]
gi++
if rng.Float64() < cfg.SecondaryProb {
d.secondaryOp(ctx, c, p, g, rng)
continue
}
if d.playTurn(ctx, c, p, g, rng) {
active = slices.DeleteFunc(active, func(x *Game) bool { return x == g })
gi = 0
if len(active) == 0 {
<-ctx.Done()
return
}
}
}
}
}
// subscribeLoop holds the player's live-event stream open on the player's client,
// counting events and reconnecting with a brief backoff after a drop, until the run
// ends.
func (d *Driver) subscribeLoop(ctx context.Context, c *edge.Client, p seed.Account) {
for ctx.Err() == nil {
err := c.Subscribe(ctx, p.Token, func(e edge.Event) { d.rec.Event(e.Kind) })
if ctx.Err() != nil {
return
}
if err != nil {
d.rec.StreamErr()
}
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
}
// playTurn plays one turn in g over the player's client when it is the player's
// move: fetch state, replay history, pick a legal move and submit it (or exchange /
// pass). It reports whether the game has finished, so the caller can drop it from the
// rotation.
func (d *Driver) playTurn(ctx context.Context, c *edge.Client, p seed.Account, g *Game, rng *rand.Rand) (finished bool) {
seat := g.seatOf(p.ID.String())
if seat < 0 {
return false
}
t0 := time.Now()
st, code, err := c.State(ctx, p.Token, g.ID)
d.rec.Record("game.state", code, time.Since(t0))
if err != nil || code != "ok" {
return false
}
if !st.Game.Active() {
return true
}
if st.Game.ToMove != seat {
return false
}
t0 = time.Now()
hist, hc, err := c.History(ctx, p.Token, g.ID)
d.rec.Record("game.history", hc, time.Since(t0))
if err != nil || hc != "ok" {
return false
}
action, err := d.moves.Pick(g.Variant, hist, st.Rack, st.BagLen, rng)
if err != nil {
d.log.Debug("pick move", "variant", g.Variant, "err", err)
return false
}
switch action.Kind {
case "play":
t0 = time.Now()
_, code, _ := c.SubmitPlay(ctx, p.Token, g.ID, action.Dir, action.Tiles)
d.rec.Record("game.submit_play", code, time.Since(t0))
case "exchange":
t0 = time.Now()
_, code, _ := c.Exchange(ctx, p.Token, g.ID, action.Exchange)
d.rec.Record("game.exchange", code, time.Since(t0))
default:
t0 = time.Now()
_, code, _ := c.Pass(ctx, p.Token, g.ID)
d.rec.Record("game.pass", code, time.Since(t0))
}
return false
}
// secondaryOp exercises one of the non-move edge operations the plan calls out, so
// the run touches nudge / chat / check-word / draft / profile / stats too, over the
// player's own client.
func (d *Driver) secondaryOp(ctx context.Context, c *edge.Client, p seed.Account, g *Game, rng *rand.Rand) {
t0 := time.Now()
switch rng.Intn(7) {
case 0:
code, _ := c.Nudge(ctx, p.Token, g.ID)
d.rec.Record("chat.nudge", code, time.Since(t0))
case 1:
code, _ := c.ChatPost(ctx, p.Token, g.ID, "gg")
d.rec.Record("chat.post", code, time.Since(t0))
case 2:
code, _ := c.CheckWord(ctx, p.Token, g.ID, []byte{0, 1, 2})
d.rec.Record("game.check_word", code, time.Since(t0))
case 3:
// rack_order is an opaque string and board_tiles a (here empty) array, per the
// backend draft DTO; a malformed shape is rejected as bad_request.
code, _ := c.DraftSave(ctx, p.Token, g.ID, `{"rack_order":"","board_tiles":[]}`)
d.rec.Record("draft.save", code, time.Since(t0))
case 4:
code, _ := c.DraftGet(ctx, p.Token, g.ID)
d.rec.Record("draft.get", code, time.Since(t0))
case 5:
lang := "en"
if rng.Intn(2) == 1 {
lang = "ru"
}
code, _ := c.ProfileUpdate(ctx, p.Token, p.Name, lang)
d.rec.Record("profile.update", code, time.Since(t0))
default:
code, _ := c.Stats(ctx, p.Token)
d.rec.Record("stats.get", code, time.Since(t0))
}
}
// shuffledPool returns every seeded account in random order, so an active set is a
// representative mix of durable and guest accounts.
func shuffledPool(pool *seed.Pool) []seed.Account {
all := pool.All()
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
rng.Shuffle(len(all), func(i, j int) { all[i], all[j] = all[j], all[i] })
return all
}
// gamesByPlayer indexes the assembled games by each member's account id.
func gamesByPlayer(games []*Game) map[string][]*Game {
m := make(map[string][]*Game)
for _, g := range games {
for _, mem := range g.Members {
id := mem.ID.String()
m[id] = append(m[id], g)
}
}
return m
}