Stage 12: observability & performance (OTel/OTLP, domain metrics, guest GC)

- pkg/telemetry: shared OTel provider bootstrap (none/stdout/otlp + W3C
  propagators + Go runtime metrics); backend/internal/telemetry becomes a thin
  facade keeping its gin middleware.
- Telemetry parity: gateway and the Telegram connector gain telemetry runtimes
  and config (GATEWAY_/TELEGRAM_ SERVICE_NAME + OTEL_*); otelgrpc instruments the
  backend push server, the gateway's backend+connector clients and the connector
  server. Default exporter stays none (collector/dashboards are Stage 14).
- Operational metrics (variant attribute on game-scoped ones): game_replay_duration,
  game_move_validate_duration, games_started_total, games_abandoned_total,
  game_cache_active, chat_messages_total{kind}, gateway edge_request_duration.
  Wired via the SetMetrics setter pattern (default no-op meter).
- TODO-3: account.GuestReaper deletes guests with no game seat past
  BACKEND_GUEST_RETENTION (default 30d, swept every BACKEND_GUEST_REAP_INTERVAL).
- Tests: pkg/telemetry exporter selection; game/social/edge metric recording via
  a manual reader; config (otlp accepted, guest knobs); inttest guest reaper.
- Docs: PLAN.md re-scopes Stage 12 and adds Stage 13 (alphabet-on-wire) + Stage 14
  (CI/deploy) with the agreed dictionary-versioning resolution; ARCHITECTURE 11/13,
  TESTING, the three READMEs and FUNCTIONAL(+ru) updated.

backend/internal/game/cache.go

@@ -63,6 +63,7 @@ type gameCache struct {
 
 type cachedGame struct {
 	game       *engine.Game
+	variant    string
 	lastAccess time.Time
 }
 
@@ -82,11 +83,12 @@ func (c *gameCache) get(id uuid.UUID) (*engine.Game, bool) {
 	return e.game, true
 }
 
-// put stores g as the live game for id.
-func (c *gameCache) put(id uuid.UUID, g *engine.Game) {
+// put stores g as the live game for id. variant labels the entry so the active-
+// games gauge can report counts by variant without inspecting engine internals.
+func (c *gameCache) put(id uuid.UUID, g *engine.Game, variant string) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.entries[id] = &cachedGame{game: g, lastAccess: c.now()}
+	c.entries[id] = &cachedGame{game: g, variant: variant, lastAccess: c.now()}
 }
 
 // remove drops id from the cache (used on a finished game and after a failed
@@ -119,3 +121,16 @@ func (c *gameCache) size() int {
 	defer c.mu.Unlock()
 	return len(c.entries)
 }
+
+// countByVariant tallies the resident games by their variant label. It backs the
+// game_cache_active observable gauge; the resident set is the bounded number of
+// live (active) games, so the scan under the lock is cheap.
+func (c *gameCache) countByVariant() map[string]int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	out := make(map[string]int, len(c.entries))
+	for _, e := range c.entries {
+		out[e.variant]++
+	}
+	return out
+}

backend/internal/game/helpers_test.go

@@ -94,7 +94,7 @@ func TestGameCacheEviction(t *testing.T) {
 	cur := time.Unix(1_700_000_000, 0)
 	cache := newGameCache(time.Hour, func() time.Time { return cur })
 	id := uuid.New()
-	cache.put(id, nil)
+	cache.put(id, nil, "english")
 	if _, ok := cache.get(id); !ok {
 		t.Fatal("game must be resident after put")
 	}

backend/internal/game/metrics.go

@@ -0,0 +1,109 @@
+package game
+
+import (
+	"context"
+	"time"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/noop"
+	"go.uber.org/zap"
+
+	"scrabble/backend/internal/engine"
+)
+
+// meterName scopes the game domain's OpenTelemetry instruments.
+const meterName = "scrabble/backend/game"
+
+// gameMetrics holds the game domain's operational instruments. Every game-scoped
+// measurement carries a "variant" attribute (english/russian/erudit). The
+// instruments default to no-ops (see defaultGameMetrics), so recording is always
+// safe; SetMetrics installs the real meter during startup wiring.
+type gameMetrics struct {
+	replay    metric.Float64Histogram
+	validate  metric.Float64Histogram
+	started   metric.Int64Counter
+	abandoned metric.Int64Counter
+}
+
+// defaultGameMetrics returns instruments backed by a no-op meter, recording
+// nothing until SetMetrics installs a real one.
+func defaultGameMetrics() *gameMetrics {
+	return newGameMetrics(noop.NewMeterProvider().Meter(meterName))
+}
+
+// newGameMetrics builds the instruments on meter, falling back to no-op
+// instruments on the (rare) construction error so the game domain never fails to
+// start over telemetry.
+func newGameMetrics(meter metric.Meter) *gameMetrics {
+	return &gameMetrics{
+		replay:    histogram(meter, "game_replay_duration", "Seconds to rebuild a live game from its journal on a cache miss."),
+		validate:  histogram(meter, "game_move_validate_duration", "Seconds to validate and score a tentative play (EvaluatePlay)."),
+		started:   counter(meter, "games_started_total", "Games created and started."),
+		abandoned: counter(meter, "games_abandoned_total", "Player seats dropped by the turn-timeout sweeper."),
+	}
+}
+
+// SetMetrics installs the meter the game domain records to and registers the
+// observable gauge reporting the live games resident in the cache by variant. It
+// must be called during startup wiring; the default is a no-op meter.
+func (svc *Service) SetMetrics(meter metric.Meter) {
+	if meter == nil {
+		return
+	}
+	svc.metrics = newGameMetrics(meter)
+	if _, err := meter.Int64ObservableGauge("game_cache_active",
+		metric.WithDescription("Live games currently resident in the in-memory cache, by variant."),
+		metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error {
+			for variant, n := range svc.cache.countByVariant() {
+				o.Observe(int64(n), metric.WithAttributes(attribute.String("variant", variant)))
+			}
+			return nil
+		}),
+	); err != nil {
+		svc.log.Warn("game: register cache gauge", zap.Error(err))
+	}
+}
+
+// recordReplay records the duration of a cache-miss journal replay for variant.
+func (m *gameMetrics) recordReplay(ctx context.Context, v engine.Variant, start time.Time) {
+	m.replay.Record(ctx, time.Since(start).Seconds(), variantAttr(v))
+}
+
+// recordValidate records the duration of one play validation for variant.
+func (m *gameMetrics) recordValidate(ctx context.Context, v engine.Variant, start time.Time) {
+	m.validate.Record(ctx, time.Since(start).Seconds(), variantAttr(v))
+}
+
+// recordStarted counts one started game of variant.
+func (m *gameMetrics) recordStarted(ctx context.Context, v engine.Variant) {
+	m.started.Add(ctx, 1, variantAttr(v))
+}
+
+// recordAbandoned counts one seat dropped by the turn-timeout sweeper in a game of
+// variant.
+func (m *gameMetrics) recordAbandoned(ctx context.Context, v engine.Variant) {
+	m.abandoned.Add(ctx, 1, variantAttr(v))
+}
+
+// variantAttr is the shared "variant" attribute option, usable for both Record and
+// Add measurements.
+func variantAttr(v engine.Variant) metric.MeasurementOption {
+	return metric.WithAttributes(attribute.String("variant", v.String()))
+}
+
+func histogram(m metric.Meter, name, desc string) metric.Float64Histogram {
+	h, err := m.Float64Histogram(name, metric.WithUnit("s"), metric.WithDescription(desc))
+	if err != nil {
+		h, _ = noop.NewMeterProvider().Meter(meterName).Float64Histogram(name)
+	}
+	return h
+}
+
+func counter(m metric.Meter, name, desc string) metric.Int64Counter {
+	c, err := m.Int64Counter(name, metric.WithDescription(desc))
+	if err != nil {
+		c, _ = noop.NewMeterProvider().Meter(meterName).Int64Counter(name)
+	}
+	return c
+}

backend/internal/game/metrics_test.go

@@ -0,0 +1,95 @@
+package game
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go.opentelemetry.io/otel/attribute"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+
+	"scrabble/backend/internal/engine"
+)
+
+// TestGameMetrics records each game instrument through a manual reader and asserts
+// the counters carry the right "variant" attribute and the histograms observe.
+func TestGameMetrics(t *testing.T) {
+	ctx := context.Background()
+	reader := sdkmetric.NewManualReader()
+	meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
+	m := newGameMetrics(meter)
+
+	m.recordStarted(ctx, engine.VariantEnglish)
+	m.recordStarted(ctx, engine.VariantEnglish)
+	m.recordStarted(ctx, engine.VariantRussianScrabble)
+	m.recordAbandoned(ctx, engine.VariantErudit)
+	m.recordReplay(ctx, engine.VariantEnglish, time.Now().Add(-time.Millisecond))
+	m.recordValidate(ctx, engine.VariantRussianScrabble, time.Now().Add(-time.Millisecond))
+
+	var rm metricdata.ResourceMetrics
+	if err := reader.Collect(ctx, &rm); err != nil {
+		t.Fatalf("collect: %v", err)
+	}
+
+	started := counterByAttr(t, rm, "games_started_total", "variant")
+	if started["english"] != 2 || started["russian_scrabble"] != 1 {
+		t.Errorf("games_started_total = %v, want english:2 russian_scrabble:1", started)
+	}
+	if abandoned := counterByAttr(t, rm, "games_abandoned_total", "variant"); abandoned["erudit"] != 1 {
+		t.Errorf("games_abandoned_total = %v, want erudit:1", abandoned)
+	}
+	if c := histogramCount(t, rm, "game_replay_duration"); c != 1 {
+		t.Errorf("game_replay_duration observations = %d, want 1", c)
+	}
+	if c := histogramCount(t, rm, "game_move_validate_duration"); c != 1 {
+		t.Errorf("game_move_validate_duration observations = %d, want 1", c)
+	}
+}
+
+// counterByAttr sums the int64 counter named name, grouped by the value of the
+// attribute key attr.
+func counterByAttr(t *testing.T, rm metricdata.ResourceMetrics, name, attr string) map[string]int64 {
+	t.Helper()
+	out := map[string]int64{}
+	for _, sm := range rm.ScopeMetrics {
+		for _, md := range sm.Metrics {
+			if md.Name != name {
+				continue
+			}
+			sum, ok := md.Data.(metricdata.Sum[int64])
+			if !ok {
+				t.Fatalf("%s is not an int64 sum", name)
+			}
+			for _, dp := range sum.DataPoints {
+				v, _ := dp.Attributes.Value(attribute.Key(attr))
+				out[v.AsString()] += dp.Value
+			}
+		}
+	}
+	return out
+}
+
+// histogramCount returns the total observation count of the float64 histogram
+// named name.
+func histogramCount(t *testing.T, rm metricdata.ResourceMetrics, name string) uint64 {
+	t.Helper()
+	for _, sm := range rm.ScopeMetrics {
+		for _, md := range sm.Metrics {
+			if md.Name != name {
+				continue
+			}
+			h, ok := md.Data.(metricdata.Histogram[float64])
+			if !ok {
+				t.Fatalf("%s is not a float64 histogram", name)
+			}
+			var n uint64
+			for _, dp := range h.DataPoints {
+				n += dp.Count
+			}
+			return n
+		}
+	}
+	t.Fatalf("%s not found", name)
+	return 0
+}

backend/internal/game/service.go

@@ -33,6 +33,7 @@ type Service struct {
 	clock    func() time.Time
 	rng      func() int64
 	pub      notify.Publisher
+	metrics  *gameMetrics
 	log      *zap.Logger
 }
 
@@ -51,6 +52,7 @@ func NewService(store *Store, accounts *account.Store, registry *engine.Registry
 		clock:    clock,
 		rng:      randomSeed,
 		pub:      notify.Nop{},
+		metrics:  defaultGameMetrics(),
 		log:      log,
 	}
 }
@@ -135,7 +137,8 @@ func (svc *Service) Create(ctx context.Context, params CreateParams) (Game, erro
 	if err := svc.store.CreateGame(ctx, ins, params.Seats); err != nil {
 		return Game{}, err
 	}
-	svc.cache.put(id, g)
+	svc.cache.put(id, g, params.Variant.String())
+	svc.metrics.recordStarted(ctx, params.Variant)
 	return svc.store.GetGame(ctx, id)
 }
 
@@ -350,6 +353,7 @@ func (svc *Service) timeoutGame(ctx context.Context, gameID uuid.UUID, now time.
 	if _, err := svc.commit(ctx, gameID, g, rec, "timeout", rackBefore, nil, cur.Seats); err != nil {
 		return false, err
 	}
+	svc.metrics.recordAbandoned(ctx, cur.Variant)
 	return true, nil
 }
 
@@ -373,7 +377,9 @@ func (svc *Service) EvaluatePlay(ctx context.Context, gameID, accountID uuid.UUI
 	if err != nil {
 		return EvalResult{}, err
 	}
+	validateStart := time.Now()
 	rec, err := g.EvaluatePlay(dir, tiles)
+	svc.metrics.recordValidate(ctx, pre.Variant, validateStart)
 	if err != nil {
 		if errors.Is(err, engine.ErrIllegalPlay) {
 			return EvalResult{Valid: false}, nil
@@ -704,7 +710,7 @@ func (svc *Service) liveGame(ctx context.Context, pre Game) (*engine.Game, error
 		return nil, err
 	}
 	if !g.Over() {
-		svc.cache.put(pre.ID, g)
+		svc.cache.put(pre.ID, g, pre.Variant.String())
 	}
 	return g, nil
 }
@@ -713,6 +719,7 @@ func (svc *Service) liveGame(ctx context.Context, pre Game) (*engine.Game, error
 // re-applying every journalled move in order. The deterministic bag makes the
 // reconstruction exact.
 func (svc *Service) replay(ctx context.Context, pre Game) (*engine.Game, error) {
+	defer svc.metrics.recordReplay(ctx, pre.Variant, time.Now())
 	seed, err := svc.store.GameSeed(ctx, pre.ID)
 	if err != nil {
 		return nil, err