Stage 12: observability & performance (OTel/OTLP, domain metrics, guest GC)
Tests · Go / test (push) Successful in 11s
Tests · Integration / integration (push) Successful in 12s
Tests · Go / test (pull_request) Successful in 10s
Tests · Integration / integration (pull_request) Successful in 11s

- pkg/telemetry: shared OTel provider bootstrap (none/stdout/otlp + W3C
  propagators + Go runtime metrics); backend/internal/telemetry becomes a thin
  facade keeping its gin middleware.
- Telemetry parity: gateway and the Telegram connector gain telemetry runtimes
  and config (GATEWAY_/TELEGRAM_ SERVICE_NAME + OTEL_*); otelgrpc instruments the
  backend push server, the gateway's backend+connector clients and the connector
  server. Default exporter stays none (collector/dashboards are Stage 14).
- Operational metrics (variant attribute on game-scoped ones): game_replay_duration,
  game_move_validate_duration, games_started_total, games_abandoned_total,
  game_cache_active, chat_messages_total{kind}, gateway edge_request_duration.
  Wired via the SetMetrics setter pattern (default no-op meter).
- TODO-3: account.GuestReaper deletes guests with no game seat past
  BACKEND_GUEST_RETENTION (default 30d, swept every BACKEND_GUEST_REAP_INTERVAL).
- Tests: pkg/telemetry exporter selection; game/social/edge metric recording via
  a manual reader; config (otlp accepted, guest knobs); inttest guest reaper.
- Docs: PLAN.md re-scopes Stage 12 and adds Stage 13 (alphabet-on-wire) + Stage 14
  (CI/deploy) with the agreed dictionary-versioning resolution; ARCHITECTURE 11/13,
  TESTING, the three READMEs and FUNCTIONAL(+ru) updated.
This commit is contained in:
Ilia Denisov
2026-06-04 14:22:15 +02:00
parent 01485d8fc6
commit dcd8de8b00
44 changed files with 1434 additions and 224 deletions
+4 -2
View File
@@ -132,8 +132,8 @@ internal/connector/ # backend gRPC client to the Telegram connector (operator b
| `BACKEND_POSTGRES_CONN_MAX_LIFETIME` | `30m` | Max connection lifetime. |
| `BACKEND_POSTGRES_OPERATION_TIMEOUT` | `5s` | Connect attempt + `/readyz` ping bound. |
| `BACKEND_SERVICE_NAME` | `scrabble-backend` | OpenTelemetry `service.name`. |
| `BACKEND_OTEL_TRACES_EXPORTER` | `none` | `none` or `stdout` (OTLP arrives later). |
| `BACKEND_OTEL_METRICS_EXPORTER` | `none` | `none` or `stdout`. |
| `BACKEND_OTEL_TRACES_EXPORTER` | `none` | `none`, `stdout` or `otlp` (gRPC; endpoint from the standard `OTEL_EXPORTER_OTLP_*`). |
| `BACKEND_OTEL_METRICS_EXPORTER` | `none` | `none`, `stdout` or `otlp`. |
| `BACKEND_DICT_DIR` | — | **Required.** Directory of committed `.dawg` dictionaries. |
| `BACKEND_DICT_VERSION` | `v1` | Dictionary version new games pin. |
| `BACKEND_GAME_TIMEOUT_SWEEP_INTERVAL` | `1m` | How often the turn-timeout sweeper runs. |
@@ -147,6 +147,8 @@ internal/connector/ # backend gRPC client to the Telegram connector (operator b
| `BACKEND_SMTP_PASSWORD` | — | SMTP password. |
| `BACKEND_SMTP_FROM` | `no-reply@localhost` | Envelope/From address for confirm-codes. |
| `BACKEND_CONNECTOR_ADDR` | — | Telegram connector gRPC address for admin-console operator broadcasts. Empty disables broadcasts. |
| `BACKEND_GUEST_REAP_INTERVAL` | `1h` | How often the abandoned-guest reaper sweeps. |
| `BACKEND_GUEST_RETENTION` | `720h` | Account age past which a guest with no game seat is deleted. |
## Run
+13
View File
@@ -80,6 +80,9 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
logger.Warn("telemetry shutdown", zap.Error(err))
}
}()
if err := tel.StartRuntimeMetrics(); err != nil {
logger.Warn("telemetry: start runtime metrics", zap.Error(err))
}
db, err := postgres.Open(ctx, cfg.Postgres,
postgres.WithTracerProvider(tel.TracerProvider()),
@@ -131,10 +134,19 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
accounts := account.NewStore(db)
games := game.NewService(game.NewStore(db), accounts, registry, cfg.Game, logger)
games.SetNotifier(hub)
games.SetMetrics(tel.MeterProvider().Meter("scrabble/backend/game"))
go games.RunSweeper(ctx, cfg.Game.TimeoutSweepInterval)
logger.Info("game turn-timeout sweeper started",
zap.Duration("interval", cfg.Game.TimeoutSweepInterval))
// Stage 12 TODO-3: reap abandoned guest accounts (no game seat, account age past
// the retention window). Dependent rows fall away via ON DELETE CASCADE.
guestReaper := account.NewGuestReaper(accounts, cfg.GuestRetention, logger)
go guestReaper.Run(ctx, cfg.GuestReapInterval)
logger.Info("guest reaper started",
zap.Duration("interval", cfg.GuestReapInterval),
zap.Duration("retention", cfg.GuestRetention))
// Stage 4 lobby & social domains. Their REST and stream surface is added with
// the gateway in Stage 6, so they are handed to the server (like the route
// groups) for the handlers to come.
@@ -145,6 +157,7 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
links := link.NewService(emails, accounts, accountmerge.NewMerger(db), sessions)
socialSvc := social.NewService(social.NewStore(db), accounts, games)
socialSvc.SetNotifier(hub)
socialSvc.SetMetrics(tel.MeterProvider().Meter("scrabble/backend/social"))
// Stage 5 robot opponent: provision its durable account pool (a hard startup
// dependency, like the dictionaries) and start its move driver. The matchmaker
+1
View File
@@ -101,6 +101,7 @@ require (
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.22.0 // indirect
+87
View File
@@ -0,0 +1,87 @@
package account
import (
"context"
"fmt"
"time"
"github.com/go-jet/jet/v2/postgres"
"go.uber.org/zap"
"scrabble/backend/internal/postgres/jet/backend/table"
)
// ReapAbandonedGuests deletes guest accounts created before olderThan that are
// not seated in any game. It returns the number deleted.
//
// Scope is deliberately "no game seat at all", not merely "no active game": a
// finished game belongs to the other players' history, and game_players carries no
// ON DELETE CASCADE to accounts (docs/ARCHITECTURE.md §4), so a guest with any seat
// is retained (and a delete would be blocked by that foreign key regardless). The
// dependent rows of a reaped guest — sessions, identities, account_stats — fall
// away through their own ON DELETE CASCADE foreign keys. Account age is the
// abandonment signal because sessions are revoke-only with no maintained
// last_seen_at, so a lingering session never expires on its own.
func (s *Store) ReapAbandonedGuests(ctx context.Context, olderThan time.Time) (int64, error) {
stmt := table.Accounts.DELETE().WHERE(
table.Accounts.IsGuest.EQ(postgres.Bool(true)).
AND(table.Accounts.CreatedAt.LT(postgres.TimestampzT(olderThan))).
AND(postgres.NOT(postgres.EXISTS(
postgres.SELECT(table.GamePlayers.AccountID).
FROM(table.GamePlayers).
WHERE(table.GamePlayers.AccountID.EQ(table.Accounts.AccountID)),
))),
)
res, err := stmt.ExecContext(ctx, s.db)
if err != nil {
return 0, fmt.Errorf("account: reap guests: %w", err)
}
n, err := res.RowsAffected()
if err != nil {
return 0, fmt.Errorf("account: reap guests rows affected: %w", err)
}
return n, nil
}
// GuestReaper periodically deletes abandoned guest accounts via
// Store.ReapAbandonedGuests. It mirrors the game turn-timeout sweeper and the
// matchmaker reaper: one background goroutine, started once from main.
type GuestReaper struct {
store *Store
retention time.Duration
clock func() time.Time
log *zap.Logger
}
// NewGuestReaper constructs a reaper deleting guests whose account age exceeds
// retention. log may be nil.
func NewGuestReaper(store *Store, retention time.Duration, log *zap.Logger) *GuestReaper {
if log == nil {
log = zap.NewNop()
}
return &GuestReaper{
store: store,
retention: retention,
clock: func() time.Time { return time.Now().UTC() },
log: log,
}
}
// Run reaps abandoned guests on each tick until ctx is cancelled.
func (r *GuestReaper) Run(ctx context.Context, interval time.Duration) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
n, err := r.store.ReapAbandonedGuests(ctx, r.clock().Add(-r.retention))
if err != nil {
r.log.Warn("guest reap failed", zap.Error(err))
} else if n > 0 {
r.log.Info("reaped abandoned guests", zap.Int64("count", n))
}
}
}
}
+37 -13
View File
@@ -42,13 +42,20 @@ type Config struct {
// side-service, used by the admin console to send operator broadcasts. Empty
// disables broadcasts (the admin broadcast actions report "not configured").
ConnectorAddr string
// GuestReapInterval is the cadence of the abandoned-guest reaper sweep.
GuestReapInterval time.Duration
// GuestRetention is the account age past which an unused guest (no game seat)
// is eligible for deletion by the reaper.
GuestRetention time.Duration
}
// Defaults applied when the corresponding environment variable is unset.
const (
defaultHTTPAddr = ":8080"
defaultGRPCAddr = ":9090"
defaultLogLevel = "info"
defaultHTTPAddr = ":8080"
defaultGRPCAddr = ":9090"
defaultLogLevel = "info"
defaultGuestReapInterval = time.Hour
defaultGuestRetention = 30 * 24 * time.Hour
)
// Load reads the configuration from the environment, applies defaults for unset
@@ -98,6 +105,15 @@ func Load() (Config, error) {
return Config{}, err
}
guestReapInterval, err := envDuration("BACKEND_GUEST_REAP_INTERVAL", defaultGuestReapInterval)
if err != nil {
return Config{}, err
}
guestRetention, err := envDuration("BACKEND_GUEST_RETENTION", defaultGuestRetention)
if err != nil {
return Config{}, err
}
smtp := account.SMTPConfig{
Host: os.Getenv("BACKEND_SMTP_HOST"),
Port: envOr("BACKEND_SMTP_PORT", "587"),
@@ -107,16 +123,18 @@ func Load() (Config, error) {
}
c := Config{
HTTPAddr: envOr("BACKEND_HTTP_ADDR", defaultHTTPAddr),
GRPCAddr: envOr("BACKEND_GRPC_ADDR", defaultGRPCAddr),
LogLevel: envOr("BACKEND_LOG_LEVEL", defaultLogLevel),
Postgres: pg,
Telemetry: tel,
Game: gm,
Lobby: lb,
Robot: rb,
SMTP: smtp,
ConnectorAddr: os.Getenv("BACKEND_CONNECTOR_ADDR"),
HTTPAddr: envOr("BACKEND_HTTP_ADDR", defaultHTTPAddr),
GRPCAddr: envOr("BACKEND_GRPC_ADDR", defaultGRPCAddr),
LogLevel: envOr("BACKEND_LOG_LEVEL", defaultLogLevel),
Postgres: pg,
Telemetry: tel,
Game: gm,
Lobby: lb,
Robot: rb,
SMTP: smtp,
ConnectorAddr: os.Getenv("BACKEND_CONNECTOR_ADDR"),
GuestReapInterval: guestReapInterval,
GuestRetention: guestRetention,
}
if err := c.validate(); err != nil {
return Config{}, err
@@ -152,6 +170,12 @@ func (c Config) validate() error {
if err := c.Robot.Validate(); err != nil {
return fmt.Errorf("config: %w", err)
}
if c.GuestReapInterval <= 0 {
return fmt.Errorf("config: BACKEND_GUEST_REAP_INTERVAL must be positive")
}
if c.GuestRetention <= 0 {
return fmt.Errorf("config: BACKEND_GUEST_RETENTION must be positive")
}
return nil
}
+45 -3
View File
@@ -151,12 +151,54 @@ func TestLoadRejectsMalformedDuration(t *testing.T) {
}
}
// TestLoadRejectsUnsupportedExporter verifies that an exporter outside the MVP
// set is rejected.
// TestLoadRejectsUnsupportedExporter verifies that an exporter outside the
// supported set is rejected.
func TestLoadRejectsUnsupportedExporter(t *testing.T) {
t.Setenv("BACKEND_POSTGRES_DSN", testDSN)
t.Setenv("BACKEND_OTEL_TRACES_EXPORTER", "otlp")
t.Setenv("BACKEND_OTEL_TRACES_EXPORTER", "prometheus")
if _, err := Load(); err == nil {
t.Fatal("Load: expected an error for an unsupported exporter, got nil")
}
}
// TestLoadAcceptsOTLPExporter verifies that the otlp exporter is now accepted
// (the collector is stood up with the deploy; the default stays none).
func TestLoadAcceptsOTLPExporter(t *testing.T) {
t.Setenv("BACKEND_POSTGRES_DSN", testDSN)
t.Setenv("BACKEND_DICT_DIR", "/dict")
t.Setenv("BACKEND_OTEL_TRACES_EXPORTER", "otlp")
t.Setenv("BACKEND_OTEL_METRICS_EXPORTER", "otlp")
if _, err := Load(); err != nil {
t.Fatalf("Load with otlp exporters: %v", err)
}
}
// TestLoadGuestReaperDefaultsAndOverride covers the guest-reaper knobs: defaults
// when unset, an override, and rejection of a non-positive value.
func TestLoadGuestReaperDefaultsAndOverride(t *testing.T) {
t.Setenv("BACKEND_POSTGRES_DSN", testDSN)
t.Setenv("BACKEND_DICT_DIR", "/dict")
c, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if c.GuestReapInterval != defaultGuestReapInterval {
t.Errorf("GuestReapInterval = %s, want %s", c.GuestReapInterval, defaultGuestReapInterval)
}
if c.GuestRetention != defaultGuestRetention {
t.Errorf("GuestRetention = %s, want %s", c.GuestRetention, defaultGuestRetention)
}
t.Setenv("BACKEND_GUEST_RETENTION", "168h")
if c, err = Load(); err != nil {
t.Fatalf("Load (override): %v", err)
} else if c.GuestRetention != 168*time.Hour {
t.Errorf("GuestRetention = %s, want 168h", c.GuestRetention)
}
t.Setenv("BACKEND_GUEST_REAP_INTERVAL", "0s")
if _, err := Load(); err == nil {
t.Fatal("Load: expected an error for a non-positive reap interval, got nil")
}
}
+18 -3
View File
@@ -63,6 +63,7 @@ type gameCache struct {
type cachedGame struct {
game *engine.Game
variant string
lastAccess time.Time
}
@@ -82,11 +83,12 @@ func (c *gameCache) get(id uuid.UUID) (*engine.Game, bool) {
return e.game, true
}
// put stores g as the live game for id.
func (c *gameCache) put(id uuid.UUID, g *engine.Game) {
// put stores g as the live game for id. variant labels the entry so the active-
// games gauge can report counts by variant without inspecting engine internals.
func (c *gameCache) put(id uuid.UUID, g *engine.Game, variant string) {
c.mu.Lock()
defer c.mu.Unlock()
c.entries[id] = &cachedGame{game: g, lastAccess: c.now()}
c.entries[id] = &cachedGame{game: g, variant: variant, lastAccess: c.now()}
}
// remove drops id from the cache (used on a finished game and after a failed
@@ -119,3 +121,16 @@ func (c *gameCache) size() int {
defer c.mu.Unlock()
return len(c.entries)
}
// countByVariant tallies the resident games by their variant label. It backs the
// game_cache_active observable gauge; the resident set is the bounded number of
// live (active) games, so the scan under the lock is cheap.
func (c *gameCache) countByVariant() map[string]int {
c.mu.Lock()
defer c.mu.Unlock()
out := make(map[string]int, len(c.entries))
for _, e := range c.entries {
out[e.variant]++
}
return out
}
+1 -1
View File
@@ -94,7 +94,7 @@ func TestGameCacheEviction(t *testing.T) {
cur := time.Unix(1_700_000_000, 0)
cache := newGameCache(time.Hour, func() time.Time { return cur })
id := uuid.New()
cache.put(id, nil)
cache.put(id, nil, "english")
if _, ok := cache.get(id); !ok {
t.Fatal("game must be resident after put")
}
+109
View File
@@ -0,0 +1,109 @@
package game
import (
"context"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/metric/noop"
"go.uber.org/zap"
"scrabble/backend/internal/engine"
)
// meterName scopes the game domain's OpenTelemetry instruments.
const meterName = "scrabble/backend/game"
// gameMetrics holds the game domain's operational instruments. Every game-scoped
// measurement carries a "variant" attribute (english/russian/erudit). The
// instruments default to no-ops (see defaultGameMetrics), so recording is always
// safe; SetMetrics installs the real meter during startup wiring.
type gameMetrics struct {
replay metric.Float64Histogram
validate metric.Float64Histogram
started metric.Int64Counter
abandoned metric.Int64Counter
}
// defaultGameMetrics returns instruments backed by a no-op meter, recording
// nothing until SetMetrics installs a real one.
func defaultGameMetrics() *gameMetrics {
return newGameMetrics(noop.NewMeterProvider().Meter(meterName))
}
// newGameMetrics builds the instruments on meter, falling back to no-op
// instruments on the (rare) construction error so the game domain never fails to
// start over telemetry.
func newGameMetrics(meter metric.Meter) *gameMetrics {
return &gameMetrics{
replay: histogram(meter, "game_replay_duration", "Seconds to rebuild a live game from its journal on a cache miss."),
validate: histogram(meter, "game_move_validate_duration", "Seconds to validate and score a tentative play (EvaluatePlay)."),
started: counter(meter, "games_started_total", "Games created and started."),
abandoned: counter(meter, "games_abandoned_total", "Player seats dropped by the turn-timeout sweeper."),
}
}
// SetMetrics installs the meter the game domain records to and registers the
// observable gauge reporting the live games resident in the cache by variant. It
// must be called during startup wiring; the default is a no-op meter.
func (svc *Service) SetMetrics(meter metric.Meter) {
if meter == nil {
return
}
svc.metrics = newGameMetrics(meter)
if _, err := meter.Int64ObservableGauge("game_cache_active",
metric.WithDescription("Live games currently resident in the in-memory cache, by variant."),
metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error {
for variant, n := range svc.cache.countByVariant() {
o.Observe(int64(n), metric.WithAttributes(attribute.String("variant", variant)))
}
return nil
}),
); err != nil {
svc.log.Warn("game: register cache gauge", zap.Error(err))
}
}
// recordReplay records the duration of a cache-miss journal replay for variant.
func (m *gameMetrics) recordReplay(ctx context.Context, v engine.Variant, start time.Time) {
m.replay.Record(ctx, time.Since(start).Seconds(), variantAttr(v))
}
// recordValidate records the duration of one play validation for variant.
func (m *gameMetrics) recordValidate(ctx context.Context, v engine.Variant, start time.Time) {
m.validate.Record(ctx, time.Since(start).Seconds(), variantAttr(v))
}
// recordStarted counts one started game of variant.
func (m *gameMetrics) recordStarted(ctx context.Context, v engine.Variant) {
m.started.Add(ctx, 1, variantAttr(v))
}
// recordAbandoned counts one seat dropped by the turn-timeout sweeper in a game of
// variant.
func (m *gameMetrics) recordAbandoned(ctx context.Context, v engine.Variant) {
m.abandoned.Add(ctx, 1, variantAttr(v))
}
// variantAttr is the shared "variant" attribute option, usable for both Record and
// Add measurements.
func variantAttr(v engine.Variant) metric.MeasurementOption {
return metric.WithAttributes(attribute.String("variant", v.String()))
}
func histogram(m metric.Meter, name, desc string) metric.Float64Histogram {
h, err := m.Float64Histogram(name, metric.WithUnit("s"), metric.WithDescription(desc))
if err != nil {
h, _ = noop.NewMeterProvider().Meter(meterName).Float64Histogram(name)
}
return h
}
func counter(m metric.Meter, name, desc string) metric.Int64Counter {
c, err := m.Int64Counter(name, metric.WithDescription(desc))
if err != nil {
c, _ = noop.NewMeterProvider().Meter(meterName).Int64Counter(name)
}
return c
}
+95
View File
@@ -0,0 +1,95 @@
package game
import (
"context"
"testing"
"time"
"go.opentelemetry.io/otel/attribute"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
"scrabble/backend/internal/engine"
)
// TestGameMetrics records each game instrument through a manual reader and asserts
// the counters carry the right "variant" attribute and the histograms observe.
func TestGameMetrics(t *testing.T) {
ctx := context.Background()
reader := sdkmetric.NewManualReader()
meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
m := newGameMetrics(meter)
m.recordStarted(ctx, engine.VariantEnglish)
m.recordStarted(ctx, engine.VariantEnglish)
m.recordStarted(ctx, engine.VariantRussianScrabble)
m.recordAbandoned(ctx, engine.VariantErudit)
m.recordReplay(ctx, engine.VariantEnglish, time.Now().Add(-time.Millisecond))
m.recordValidate(ctx, engine.VariantRussianScrabble, time.Now().Add(-time.Millisecond))
var rm metricdata.ResourceMetrics
if err := reader.Collect(ctx, &rm); err != nil {
t.Fatalf("collect: %v", err)
}
started := counterByAttr(t, rm, "games_started_total", "variant")
if started["english"] != 2 || started["russian_scrabble"] != 1 {
t.Errorf("games_started_total = %v, want english:2 russian_scrabble:1", started)
}
if abandoned := counterByAttr(t, rm, "games_abandoned_total", "variant"); abandoned["erudit"] != 1 {
t.Errorf("games_abandoned_total = %v, want erudit:1", abandoned)
}
if c := histogramCount(t, rm, "game_replay_duration"); c != 1 {
t.Errorf("game_replay_duration observations = %d, want 1", c)
}
if c := histogramCount(t, rm, "game_move_validate_duration"); c != 1 {
t.Errorf("game_move_validate_duration observations = %d, want 1", c)
}
}
// counterByAttr sums the int64 counter named name, grouped by the value of the
// attribute key attr.
func counterByAttr(t *testing.T, rm metricdata.ResourceMetrics, name, attr string) map[string]int64 {
t.Helper()
out := map[string]int64{}
for _, sm := range rm.ScopeMetrics {
for _, md := range sm.Metrics {
if md.Name != name {
continue
}
sum, ok := md.Data.(metricdata.Sum[int64])
if !ok {
t.Fatalf("%s is not an int64 sum", name)
}
for _, dp := range sum.DataPoints {
v, _ := dp.Attributes.Value(attribute.Key(attr))
out[v.AsString()] += dp.Value
}
}
}
return out
}
// histogramCount returns the total observation count of the float64 histogram
// named name.
func histogramCount(t *testing.T, rm metricdata.ResourceMetrics, name string) uint64 {
t.Helper()
for _, sm := range rm.ScopeMetrics {
for _, md := range sm.Metrics {
if md.Name != name {
continue
}
h, ok := md.Data.(metricdata.Histogram[float64])
if !ok {
t.Fatalf("%s is not a float64 histogram", name)
}
var n uint64
for _, dp := range h.DataPoints {
n += dp.Count
}
return n
}
}
t.Fatalf("%s not found", name)
return 0
}
+9 -2
View File
@@ -33,6 +33,7 @@ type Service struct {
clock func() time.Time
rng func() int64
pub notify.Publisher
metrics *gameMetrics
log *zap.Logger
}
@@ -51,6 +52,7 @@ func NewService(store *Store, accounts *account.Store, registry *engine.Registry
clock: clock,
rng: randomSeed,
pub: notify.Nop{},
metrics: defaultGameMetrics(),
log: log,
}
}
@@ -135,7 +137,8 @@ func (svc *Service) Create(ctx context.Context, params CreateParams) (Game, erro
if err := svc.store.CreateGame(ctx, ins, params.Seats); err != nil {
return Game{}, err
}
svc.cache.put(id, g)
svc.cache.put(id, g, params.Variant.String())
svc.metrics.recordStarted(ctx, params.Variant)
return svc.store.GetGame(ctx, id)
}
@@ -350,6 +353,7 @@ func (svc *Service) timeoutGame(ctx context.Context, gameID uuid.UUID, now time.
if _, err := svc.commit(ctx, gameID, g, rec, "timeout", rackBefore, nil, cur.Seats); err != nil {
return false, err
}
svc.metrics.recordAbandoned(ctx, cur.Variant)
return true, nil
}
@@ -373,7 +377,9 @@ func (svc *Service) EvaluatePlay(ctx context.Context, gameID, accountID uuid.UUI
if err != nil {
return EvalResult{}, err
}
validateStart := time.Now()
rec, err := g.EvaluatePlay(dir, tiles)
svc.metrics.recordValidate(ctx, pre.Variant, validateStart)
if err != nil {
if errors.Is(err, engine.ErrIllegalPlay) {
return EvalResult{Valid: false}, nil
@@ -704,7 +710,7 @@ func (svc *Service) liveGame(ctx context.Context, pre Game) (*engine.Game, error
return nil, err
}
if !g.Over() {
svc.cache.put(pre.ID, g)
svc.cache.put(pre.ID, g, pre.Variant.String())
}
return g, nil
}
@@ -713,6 +719,7 @@ func (svc *Service) liveGame(ctx context.Context, pre Game) (*engine.Game, error
// re-applying every journalled move in order. The deterministic bag makes the
// reconstruction exact.
func (svc *Service) replay(ctx context.Context, pre Game) (*engine.Game, error) {
defer svc.metrics.recordReplay(ctx, pre.Variant, time.Now())
seed, err := svc.store.GameSeed(ctx, pre.ID)
if err != nil {
return nil, err
@@ -0,0 +1,76 @@
//go:build integration
package inttest
import (
"context"
"errors"
"testing"
"time"
"github.com/google/uuid"
"scrabble/backend/internal/account"
"scrabble/backend/internal/engine"
"scrabble/backend/internal/game"
)
// TestGuestReaper verifies the abandoned-guest reaper: it deletes guests with no
// game seat once their account age is past the cutoff, while sparing guests that
// are too young, guests seated in a game (the FK-protected opponent history), and
// durable accounts.
func TestGuestReaper(t *testing.T) {
ctx := context.Background()
store := account.NewStore(testDB)
guestA := provisionGuest(t) // guest, no seat → reaped on a future cutoff
guestB := provisionGuest(t) // guest, no seat → reaped on a future cutoff
seated := provisionGuest(t) // guest seated in a game → kept
durable := provisionAccount(t)
// Seat the third guest in a game with a durable opponent (Create needs 2-4).
opp := provisionAccount(t)
if _, err := newGameService().Create(ctx, game.CreateParams{
Variant: engine.VariantEnglish, Seats: []uuid.UUID{seated, opp}, TurnTimeout: 24 * time.Hour, Seed: 1,
}); err != nil {
t.Fatalf("create game: %v", err)
}
// A cutoff in the past: every account is younger than the window, so the age
// gate spares them all.
if n, err := store.ReapAbandonedGuests(ctx, time.Now().Add(-time.Hour)); err != nil {
t.Fatalf("reap (past cutoff): %v", err)
} else if n != 0 {
t.Fatalf("reap with a past cutoff deleted %d, want 0", n)
}
assertAccount(t, store, guestA, true)
// A cutoff in the future: every account predates it, so the no-seat guests are
// reaped and the seated guest and the durable account survive.
if _, err := store.ReapAbandonedGuests(ctx, time.Now().Add(time.Hour)); err != nil {
t.Fatalf("reap (future cutoff): %v", err)
}
assertAccount(t, store, guestA, false)
assertAccount(t, store, guestB, false)
assertAccount(t, store, seated, true)
assertAccount(t, store, durable, true)
}
// assertAccount checks whether the account with id is present, failing the test
// when its presence differs from want.
func assertAccount(t *testing.T, store *account.Store, id uuid.UUID, want bool) {
t.Helper()
_, err := store.GetByID(context.Background(), id)
switch {
case err == nil:
if !want {
t.Errorf("account %s still exists, want reaped", id)
}
case errors.Is(err, account.ErrNotFound):
if want {
t.Errorf("account %s was reaped, want kept", id)
}
default:
t.Fatalf("get account %s: %v", id, err)
}
}
+2 -1
View File
@@ -11,6 +11,7 @@ import (
"fmt"
"net"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.uber.org/zap"
"google.golang.org/grpc"
@@ -78,7 +79,7 @@ func NewServer(addr string, hub *notify.Hub, log *zap.Logger) *Server {
if log == nil {
log = zap.NewNop()
}
gs := grpc.NewServer()
gs := grpc.NewServer(grpc.StatsHandler(otelgrpc.NewServerHandler()))
pushv1.RegisterPushServer(gs, NewService(hub, log))
return &Server{grpc: gs, addr: addr, log: log}
}
+2
View File
@@ -77,6 +77,7 @@ func (svc *Service) PostMessage(ctx context.Context, gameID, senderID uuid.UUID,
if err != nil {
return Message{}, err
}
svc.metrics.recordChat(ctx, kindMessage)
svc.emitChat(seats, senderID, msg)
return msg, nil
}
@@ -110,6 +111,7 @@ func (svc *Service) Nudge(ctx context.Context, gameID, senderID uuid.UUID) (Mess
if err != nil {
return Message{}, err
}
svc.metrics.recordChat(ctx, kindNudge)
if toMove >= 0 && toMove < len(seats) {
svc.pub.Publish(notify.Nudge(seats[toMove], gameID, senderID))
}
+49
View File
@@ -0,0 +1,49 @@
package social
import (
"context"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/metric/noop"
)
// meterName scopes the social domain's OpenTelemetry instruments.
const meterName = "scrabble/backend/social"
// socialMetrics holds the social domain's operational instruments. It defaults to
// no-ops (see defaultSocialMetrics); SetMetrics installs the real meter during
// startup wiring.
type socialMetrics struct {
messages metric.Int64Counter
}
// defaultSocialMetrics returns instruments backed by a no-op meter.
func defaultSocialMetrics() *socialMetrics {
return newSocialMetrics(noop.NewMeterProvider().Meter(meterName))
}
// newSocialMetrics builds the instruments on meter, falling back to a no-op
// counter on the (rare) construction error.
func newSocialMetrics(meter metric.Meter) *socialMetrics {
c, err := meter.Int64Counter("chat_messages_total",
metric.WithDescription("Per-game chat entries posted, labelled by kind (message/nudge)."))
if err != nil {
c, _ = noop.NewMeterProvider().Meter(meterName).Int64Counter("chat_messages_total")
}
return &socialMetrics{messages: c}
}
// SetMetrics installs the meter the social domain records to. It must be called
// during startup wiring; the default is a no-op meter.
func (svc *Service) SetMetrics(meter metric.Meter) {
if meter == nil {
return
}
svc.metrics = newSocialMetrics(meter)
}
// recordChat counts one posted chat entry of the given kind (message or nudge).
func (m *socialMetrics) recordChat(ctx context.Context, kind string) {
m.messages.Add(ctx, 1, metric.WithAttributes(attribute.String("kind", kind)))
}
+48
View File
@@ -0,0 +1,48 @@
package social
import (
"context"
"testing"
"go.opentelemetry.io/otel/attribute"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
)
// TestSocialMetrics records chat and nudge entries through a manual reader and
// asserts chat_messages_total splits by the "kind" attribute.
func TestSocialMetrics(t *testing.T) {
ctx := context.Background()
reader := sdkmetric.NewManualReader()
meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
m := newSocialMetrics(meter)
m.recordChat(ctx, kindMessage)
m.recordChat(ctx, kindMessage)
m.recordChat(ctx, kindNudge)
var rm metricdata.ResourceMetrics
if err := reader.Collect(ctx, &rm); err != nil {
t.Fatalf("collect: %v", err)
}
got := map[string]int64{}
for _, sm := range rm.ScopeMetrics {
for _, md := range sm.Metrics {
if md.Name != "chat_messages_total" {
continue
}
sum, ok := md.Data.(metricdata.Sum[int64])
if !ok {
t.Fatalf("chat_messages_total is not an int64 sum")
}
for _, dp := range sum.DataPoints {
v, _ := dp.Attributes.Value(attribute.Key("kind"))
got[v.AsString()] += dp.Value
}
}
}
if got[kindMessage] != 2 || got[kindNudge] != 1 {
t.Errorf("chat_messages_total = %v, want message:2 nudge:1", got)
}
}
+2
View File
@@ -76,6 +76,7 @@ type Service struct {
accounts *account.Store
games GameReader
pub notify.Publisher
metrics *socialMetrics
now func() time.Time
}
@@ -87,6 +88,7 @@ func NewService(store *Store, accounts *account.Store, games GameReader) *Servic
accounts: accounts,
games: games,
pub: notify.Nop{},
metrics: defaultSocialMetrics(),
now: func() time.Time { return time.Now().UTC() },
}
}
+32 -168
View File
@@ -1,158 +1,58 @@
// Package telemetry owns the OpenTelemetry runtime for the backend process.
//
// New constructs the configured tracer and meter providers, registers them as
// the OpenTelemetry globals, and exposes Shutdown for orderly exit. The MVP
// supports the `none` and `stdout` exporters; OTLP export and dashboards arrive
// in a later stage. The per-request timing middleware lives in middleware.go and
// uses the registered global tracer, so requests are timed and logged even when
// the exporter is `none`.
// Package telemetry owns the backend's OpenTelemetry wiring. The provider
// bootstrap (exporter selection, propagators, shutdown, Go runtime metrics) is
// shared across the Scrabble services in scrabble/pkg/telemetry; this package is a
// thin backend-flavoured facade over it (the "scrabble-backend" default service
// name) plus the backend-specific gin request-timing middleware (middleware.go),
// which uses the registered global tracer so requests are timed and logged even
// when the exporter is "none".
package telemetry
import (
"context"
"errors"
"fmt"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric"
"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/propagation"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
pkgtel "scrabble/pkg/telemetry"
)
// Exporter selectors supported by the backend.
// Exporter selectors, re-exported from scrabble/pkg/telemetry so the backend's
// config and tests need not import the shared package directly.
const (
ExporterNone = "none"
ExporterStdout = "stdout"
ExporterNone = pkgtel.ExporterNone
ExporterStdout = pkgtel.ExporterStdout
ExporterOTLP = pkgtel.ExporterOTLP
)
// DefaultServiceName labels traces and metrics when BACKEND_SERVICE_NAME is
// unset.
// DefaultServiceName labels traces and metrics when BACKEND_SERVICE_NAME is unset.
const DefaultServiceName = "scrabble-backend"
// Config selects the telemetry providers' service name and exporters.
type Config struct {
// ServiceName is reported as the OpenTelemetry service.name resource.
ServiceName string
// TracesExporter is one of ExporterNone or ExporterStdout.
TracesExporter string
// MetricsExporter is one of ExporterNone or ExporterStdout.
MetricsExporter string
}
// Config selects the telemetry providers' service name and exporters. It aliases
// the shared configuration type.
type Config = pkgtel.Config
// DefaultConfig returns the MVP telemetry configuration: named service, no
// exporters (so no collector is required locally or in CI).
// Runtime owns the shared OpenTelemetry providers. It aliases the shared runtime
// type, so callers keep using telemetry.Runtime.
type Runtime = pkgtel.Runtime
// DefaultConfig returns the backend's telemetry configuration: the
// "scrabble-backend" service name and both exporters off (so no collector is
// required locally or in CI).
func DefaultConfig() Config {
return Config{
ServiceName: DefaultServiceName,
TracesExporter: ExporterNone,
MetricsExporter: ExporterNone,
}
}
// Validate reports whether the configuration selects supported exporters.
func (c Config) Validate() error {
if c.ServiceName == "" {
return errors.New("telemetry: ServiceName must not be empty")
}
if err := validateExporter("traces", c.TracesExporter); err != nil {
return err
}
return validateExporter("metrics", c.MetricsExporter)
}
func validateExporter(kind, value string) error {
switch value {
case ExporterNone, ExporterStdout:
return nil
default:
return fmt.Errorf("telemetry: unsupported %s exporter %q", kind, value)
}
}
// Runtime owns the shared OpenTelemetry providers.
type Runtime struct {
tracerProvider *sdktrace.TracerProvider
meterProvider *sdkmetric.MeterProvider
return pkgtel.DefaultConfig(DefaultServiceName)
}
// New constructs the telemetry runtime, registers the global providers and the
// W3C trace-context/baggage propagators, and returns the Runtime. Callers must
// invoke Runtime.Shutdown during process exit.
// W3C propagators, and returns the Runtime. Callers must invoke Runtime.Shutdown
// during process exit.
func New(ctx context.Context, cfg Config) (*Runtime, error) {
if err := cfg.Validate(); err != nil {
return nil, err
}
res, err := resource.New(ctx, resource.WithAttributes(
attribute.String("service.name", cfg.ServiceName),
))
if err != nil {
return nil, fmt.Errorf("telemetry: build resource: %w", err)
}
tracerProvider, err := newTracerProvider(cfg, res)
if err != nil {
return nil, fmt.Errorf("telemetry: build tracer provider: %w", err)
}
meterProvider, err := newMeterProvider(cfg, res)
if err != nil {
_ = tracerProvider.Shutdown(ctx)
return nil, fmt.Errorf("telemetry: build meter provider: %w", err)
}
otel.SetTracerProvider(tracerProvider)
otel.SetMeterProvider(meterProvider)
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
))
return &Runtime{tracerProvider: tracerProvider, meterProvider: meterProvider}, nil
}
// TracerProvider returns the runtime tracer provider, or the global one when r
// is not initialised.
func (r *Runtime) TracerProvider() trace.TracerProvider {
if r == nil || r.tracerProvider == nil {
return otel.GetTracerProvider()
}
return r.tracerProvider
}
// MeterProvider returns the runtime meter provider, or the global one when r is
// not initialised.
func (r *Runtime) MeterProvider() metric.MeterProvider {
if r == nil || r.meterProvider == nil {
return otel.GetMeterProvider()
}
return r.meterProvider
}
// Shutdown flushes both providers within ctx.
func (r *Runtime) Shutdown(ctx context.Context) error {
if r == nil {
return nil
}
var err error
if r.meterProvider != nil {
err = errors.Join(err, r.meterProvider.Shutdown(ctx))
}
if r.tracerProvider != nil {
err = errors.Join(err, r.tracerProvider.Shutdown(ctx))
}
return err
return pkgtel.New(ctx, cfg)
}
// TraceFieldsFromContext returns zap fields identifying the active span, or nil
// when ctx carries no valid span context. Collocated here so callers do not
// import the OpenTelemetry API directly.
// when ctx carries no valid span context. Collocated here so callers (the
// request-timing middleware and the access log) do not import the OpenTelemetry
// API directly.
func TraceFieldsFromContext(ctx context.Context) []zap.Field {
if ctx == nil {
return nil
@@ -166,39 +66,3 @@ func TraceFieldsFromContext(ctx context.Context) []zap.Field {
zap.String("otel_span_id", sc.SpanID().String()),
}
}
func newTracerProvider(cfg Config, res *resource.Resource) (*sdktrace.TracerProvider, error) {
switch cfg.TracesExporter {
case ExporterNone:
return sdktrace.NewTracerProvider(sdktrace.WithResource(res)), nil
case ExporterStdout:
exporter, err := stdouttrace.New()
if err != nil {
return nil, fmt.Errorf("stdout trace exporter: %w", err)
}
return sdktrace.NewTracerProvider(
sdktrace.WithBatcher(exporter),
sdktrace.WithResource(res),
), nil
default:
return nil, fmt.Errorf("unsupported traces exporter %q", cfg.TracesExporter)
}
}
func newMeterProvider(cfg Config, res *resource.Resource) (*sdkmetric.MeterProvider, error) {
switch cfg.MetricsExporter {
case ExporterNone:
return sdkmetric.NewMeterProvider(sdkmetric.WithResource(res)), nil
case ExporterStdout:
exporter, err := stdoutmetric.New()
if err != nil {
return nil, fmt.Errorf("stdout metric exporter: %w", err)
}
return sdkmetric.NewMeterProvider(
sdkmetric.WithResource(res),
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter)),
), nil
default:
return nil, fmt.Errorf("unsupported metrics exporter %q", cfg.MetricsExporter)
}
}
+9 -3
View File
@@ -12,9 +12,15 @@ func TestConfigValidate(t *testing.T) {
}
otlp := DefaultConfig()
otlp.TracesExporter = "otlp"
if err := otlp.Validate(); err == nil {
t.Error("otlp exporter must be rejected in the MVP set")
otlp.TracesExporter = ExporterOTLP
if err := otlp.Validate(); err != nil {
t.Errorf("otlp exporter must be accepted: %v", err)
}
bad := DefaultConfig()
bad.MetricsExporter = "prometheus"
if err := bad.Validate(); err == nil {
t.Error("unsupported exporter must be rejected")
}
noName := DefaultConfig()