perf(gateway): pool backend conns; loadtest evaluate hot path
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 18s
CI / ui (pull_request) Has been skipped
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m5s
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 18s
CI / ui (pull_request) Has been skipped
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m5s
The loadtest harness never modelled game.evaluate — the debounced per-tile play preview a real client fires several times per turn, the hottest gameplay call. Model it (one evaluate per placed tile + reconsideration re-previews + draft.save, human-paced; --eval / --eval-recon toggle it). That realistic load surfaced the real bottleneck: the gateway's backend HTTP client used the default transport (MaxIdleConnsPerHost=2), so every sync call to the single backend host churned a fresh TCP connection — ~26500 TIME_WAIT sockets at 500 players (near the ephemeral-port ceiling), burning ~1.75 gateway cores while the backend sat near-idle. It was the unfixed root of the residual transport_error the earlier passes chased on the client side. Widen the keep-alive pool (backendMaxIdleConns=512, ~2x the observed 225-conn peak). At 500 players the churn collapses to ~0 and peak gateway CPU drops ~7x (~1.75 -> ~0.26 cores); postgres (~1.65 cores) becomes the busiest service. This overturns the earlier "gateway is the binding constraint, scale it horizontally" sizing — that was sizing around this bug, not a real floor. Consolidate the loadtest trip reports into one loadtest/REPORT.md (drop the R2/R7 split) and bake the finding into README / PRERELEASE / ARCHITECTURE / TESTING.
This commit is contained in:
@@ -42,19 +42,35 @@ type RealisticConfig struct {
|
||||
GamesPerPlayer int // target concurrent games per player; 0 => random 3..5
|
||||
Tick time.Duration // per-player operation cadence (keeps a player under the per-user limit)
|
||||
SecondaryProb float64 // chance per tick of a non-move operation
|
||||
Eval bool // model the per-tile evaluate preview (the gameplay hot path); false reproduces the pre-evaluate harness
|
||||
EvalRecon int // extra full-composition evaluate re-previews per play, beyond one per placed tile
|
||||
}
|
||||
|
||||
// DefaultRealistic returns the moderate ramp: 50 -> 200
|
||||
// -> 500 concurrent players, ~12 minutes per step, ~1 op/s per player.
|
||||
// -> 500 concurrent players, ~12 minutes per step, ~1 op/s per player, with the
|
||||
// per-tile evaluate preview modelled (the realistic hot path).
|
||||
func DefaultRealistic() RealisticConfig {
|
||||
return RealisticConfig{
|
||||
Steps: []int{50, 200, 500},
|
||||
StepDur: 12 * time.Minute,
|
||||
Tick: 800 * time.Millisecond,
|
||||
SecondaryProb: 0.08,
|
||||
Eval: true,
|
||||
EvalRecon: 1,
|
||||
}
|
||||
}
|
||||
|
||||
// evalGapBase and evalGapSpan bound the modelled pause between successive tile
|
||||
// placements: the client's 250 ms debounce coalesces faster drags into a single
|
||||
// evaluate, so a thoughtful player's previews are spaced by a gap drawn from
|
||||
// [base, base+span] — wide enough that a normal composition stays under the per-user
|
||||
// rate limit, the way a real one does (the limiter's cost is measured by the hammer,
|
||||
// not by self-inflicted rejections here).
|
||||
const (
|
||||
evalGapBase = 250 * time.Millisecond
|
||||
evalGapSpan = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
// RunRealistic runs the staged ramp. Each step activates more players (drawn from the
|
||||
// seeded pool), assembles a cohort of games for them and starts their turn loops; the
|
||||
// loops run until the whole ramp ends. Players from earlier steps keep playing, so
|
||||
@@ -128,7 +144,7 @@ func (d *Driver) playerLoop(ctx context.Context, p seed.Account, games []*Game,
|
||||
d.secondaryOp(ctx, c, p, g, rng)
|
||||
continue
|
||||
}
|
||||
if d.playTurn(ctx, c, p, g, rng) {
|
||||
if d.playTurn(ctx, c, p, g, cfg, rng) {
|
||||
active = slices.DeleteFunc(active, func(x *Game) bool { return x == g })
|
||||
gi = 0
|
||||
if len(active) == 0 {
|
||||
@@ -161,10 +177,10 @@ func (d *Driver) subscribeLoop(ctx context.Context, c *edge.Client, p seed.Accou
|
||||
}
|
||||
|
||||
// playTurn plays one turn in g over the player's client when it is the player's
|
||||
// move: fetch state, replay history, pick a legal move and submit it (or exchange /
|
||||
// pass). It reports whether the game has finished, so the caller can drop it from the
|
||||
// rotation.
|
||||
func (d *Driver) playTurn(ctx context.Context, c *edge.Client, p seed.Account, g *Game, rng *rand.Rand) (finished bool) {
|
||||
// move: fetch state, replay history, pick a legal move, compose it (the per-tile
|
||||
// evaluate previews a real client fires) and submit it (or exchange / pass). It reports
|
||||
// whether the game has finished, so the caller can drop it from the rotation.
|
||||
func (d *Driver) playTurn(ctx context.Context, c *edge.Client, p seed.Account, g *Game, cfg RealisticConfig, rng *rand.Rand) (finished bool) {
|
||||
seat := g.seatOf(p.ID.String())
|
||||
if seat < 0 {
|
||||
return false
|
||||
@@ -196,6 +212,7 @@ func (d *Driver) playTurn(ctx context.Context, c *edge.Client, p seed.Account, g
|
||||
}
|
||||
switch action.Kind {
|
||||
case "play":
|
||||
d.composePlay(ctx, c, p, g, action.Tiles, cfg, rng)
|
||||
t0 = time.Now()
|
||||
_, code, _ := c.SubmitPlay(ctx, p.Token, g.ID, action.Tiles)
|
||||
d.rec.Record("game.submit_play", code, time.Since(t0))
|
||||
@@ -211,6 +228,59 @@ func (d *Driver) playTurn(ctx context.Context, c *edge.Client, p seed.Account, g
|
||||
return false
|
||||
}
|
||||
|
||||
// composePlay models a player arranging the chosen play tile by tile before committing:
|
||||
// the debounced evaluate preview the real client fires on each placement (a growing prefix
|
||||
// of the tiles), a few full-composition re-previews for reconsideration (recall a tile, try
|
||||
// another spot), and the single draft persistence the client debounces out. evaluate is the
|
||||
// hottest gameplay request at scale, so omitting it (the pre-evaluate harness) understated
|
||||
// the load; cfg.Eval false reproduces that baseline for an A/B comparison. Every step
|
||||
// honours ctx, so end-of-run cancellation never blocks on a sleep or an in-flight preview.
|
||||
func (d *Driver) composePlay(ctx context.Context, c *edge.Client, p seed.Account, g *Game, tiles []edge.PlayTile, cfg RealisticConfig, rng *rand.Rand) {
|
||||
if !cfg.Eval || len(tiles) == 0 {
|
||||
return
|
||||
}
|
||||
// One evaluate per landed tile: the growing prefix mirrors the client re-previewing
|
||||
// after each placement (an early prefix is often illegal, which is still a successful
|
||||
// "ok" round trip — exactly the backend work a real composition triggers).
|
||||
for n := 1; n <= len(tiles); n++ {
|
||||
if !jitterSleep(ctx, rng, evalGapBase, evalGapSpan) {
|
||||
return
|
||||
}
|
||||
t0 := time.Now()
|
||||
code, _ := c.Evaluate(ctx, p.Token, g.ID, tiles[:n])
|
||||
d.rec.Record("game.evaluate", code, time.Since(t0))
|
||||
}
|
||||
for r := 0; r < cfg.EvalRecon; r++ {
|
||||
if !jitterSleep(ctx, rng, evalGapBase, evalGapSpan) {
|
||||
return
|
||||
}
|
||||
t0 := time.Now()
|
||||
code, _ := c.Evaluate(ctx, p.Token, g.ID, tiles)
|
||||
d.rec.Record("game.evaluate", code, time.Since(t0))
|
||||
}
|
||||
// The client persists the in-progress composition (debounced to one upsert). Its opaque
|
||||
// JSON content does not affect the call's cost, so a minimal valid shape stands in.
|
||||
t0 := time.Now()
|
||||
code, _ := c.DraftSave(ctx, p.Token, g.ID, `{"rack_order":"","board_tiles":[]}`)
|
||||
d.rec.Record("draft.save", code, time.Since(t0))
|
||||
}
|
||||
|
||||
// jitterSleep pauses for a randomised gap in [base, base+span], modelling the human pause
|
||||
// between tile placements that the client's debounce coalesces into one evaluate. It
|
||||
// returns false if ctx is cancelled during the wait, so a composition unwinds promptly at
|
||||
// end of run.
|
||||
func jitterSleep(ctx context.Context, rng *rand.Rand, base, span time.Duration) bool {
|
||||
d := base + time.Duration(rng.Int63n(int64(span)+1))
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-t.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// secondaryOp exercises one of the non-move edge operations the plan calls out, so
|
||||
// the run touches nudge / chat / check-word / draft / profile / stats too, over the
|
||||
// player's own client.
|
||||
|
||||
Reference in New Issue
Block a user