Stage 12: observability & performance (OTel/OTLP, domain metrics, guest GC)

- pkg/telemetry: shared OTel provider bootstrap (none/stdout/otlp + W3C
  propagators + Go runtime metrics); backend/internal/telemetry becomes a thin
  facade keeping its gin middleware.
- Telemetry parity: gateway and the Telegram connector gain telemetry runtimes
  and config (GATEWAY_/TELEGRAM_ SERVICE_NAME + OTEL_*); otelgrpc instruments the
  backend push server, the gateway's backend+connector clients and the connector
  server. Default exporter stays none (collector/dashboards are Stage 14).
- Operational metrics (variant attribute on game-scoped ones): game_replay_duration,
  game_move_validate_duration, games_started_total, games_abandoned_total,
  game_cache_active, chat_messages_total{kind}, gateway edge_request_duration.
  Wired via the SetMetrics setter pattern (default no-op meter).
- TODO-3: account.GuestReaper deletes guests with no game seat past
  BACKEND_GUEST_RETENTION (default 30d, swept every BACKEND_GUEST_REAP_INTERVAL).
- Tests: pkg/telemetry exporter selection; game/social/edge metric recording via
  a manual reader; config (otlp accepted, guest knobs); inttest guest reaper.
- Docs: PLAN.md re-scopes Stage 12 and adds Stage 13 (alphabet-on-wire) + Stage 14
  (CI/deploy) with the agreed dictionary-versioning resolution; ARCHITECTURE 11/13,
  TESTING, the three READMEs and FUNCTIONAL(+ru) updated.

backend/internal/telemetry/telemetry.go

@@ -1,158 +1,58 @@
-// Package telemetry owns the OpenTelemetry runtime for the backend process.
-//
-// New constructs the configured tracer and meter providers, registers them as
-// the OpenTelemetry globals, and exposes Shutdown for orderly exit. The MVP
-// supports the `none` and `stdout` exporters; OTLP export and dashboards arrive
-// in a later stage. The per-request timing middleware lives in middleware.go and
-// uses the registered global tracer, so requests are timed and logged even when
-// the exporter is `none`.
+// Package telemetry owns the backend's OpenTelemetry wiring. The provider
+// bootstrap (exporter selection, propagators, shutdown, Go runtime metrics) is
+// shared across the Scrabble services in scrabble/pkg/telemetry; this package is a
+// thin backend-flavoured facade over it (the "scrabble-backend" default service
+// name) plus the backend-specific gin request-timing middleware (middleware.go),
+// which uses the registered global tracer so requests are timed and logged even
+// when the exporter is "none".
 package telemetry
 
 import (
 	"context"
-	"errors"
-	"fmt"
 
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric"
-	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
-	"go.opentelemetry.io/otel/metric"
-	"go.opentelemetry.io/otel/propagation"
-	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
-	"go.opentelemetry.io/otel/sdk/resource"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
+
+	pkgtel "scrabble/pkg/telemetry"
 )
 
-// Exporter selectors supported by the backend.
+// Exporter selectors, re-exported from scrabble/pkg/telemetry so the backend's
+// config and tests need not import the shared package directly.
 const (
-	ExporterNone   = "none"
-	ExporterStdout = "stdout"
+	ExporterNone   = pkgtel.ExporterNone
+	ExporterStdout = pkgtel.ExporterStdout
+	ExporterOTLP   = pkgtel.ExporterOTLP
 )
 
-// DefaultServiceName labels traces and metrics when BACKEND_SERVICE_NAME is
-// unset.
+// DefaultServiceName labels traces and metrics when BACKEND_SERVICE_NAME is unset.
 const DefaultServiceName = "scrabble-backend"
 
-// Config selects the telemetry providers' service name and exporters.
-type Config struct {
-	// ServiceName is reported as the OpenTelemetry service.name resource.
-	ServiceName string
-	// TracesExporter is one of ExporterNone or ExporterStdout.
-	TracesExporter string
-	// MetricsExporter is one of ExporterNone or ExporterStdout.
-	MetricsExporter string
-}
+// Config selects the telemetry providers' service name and exporters. It aliases
+// the shared configuration type.
+type Config = pkgtel.Config
 
-// DefaultConfig returns the MVP telemetry configuration: named service, no
-// exporters (so no collector is required locally or in CI).
+// Runtime owns the shared OpenTelemetry providers. It aliases the shared runtime
+// type, so callers keep using telemetry.Runtime.
+type Runtime = pkgtel.Runtime
+
+// DefaultConfig returns the backend's telemetry configuration: the
+// "scrabble-backend" service name and both exporters off (so no collector is
+// required locally or in CI).
 func DefaultConfig() Config {
-	return Config{
-		ServiceName:     DefaultServiceName,
-		TracesExporter:  ExporterNone,
-		MetricsExporter: ExporterNone,
-	}
-}
-
-// Validate reports whether the configuration selects supported exporters.
-func (c Config) Validate() error {
-	if c.ServiceName == "" {
-		return errors.New("telemetry: ServiceName must not be empty")
-	}
-	if err := validateExporter("traces", c.TracesExporter); err != nil {
-		return err
-	}
-	return validateExporter("metrics", c.MetricsExporter)
-}
-
-func validateExporter(kind, value string) error {
-	switch value {
-	case ExporterNone, ExporterStdout:
-		return nil
-	default:
-		return fmt.Errorf("telemetry: unsupported %s exporter %q", kind, value)
-	}
-}
-
-// Runtime owns the shared OpenTelemetry providers.
-type Runtime struct {
-	tracerProvider *sdktrace.TracerProvider
-	meterProvider  *sdkmetric.MeterProvider
+	return pkgtel.DefaultConfig(DefaultServiceName)
 }
 
 // New constructs the telemetry runtime, registers the global providers and the
-// W3C trace-context/baggage propagators, and returns the Runtime. Callers must
-// invoke Runtime.Shutdown during process exit.
+// W3C propagators, and returns the Runtime. Callers must invoke Runtime.Shutdown
+// during process exit.
 func New(ctx context.Context, cfg Config) (*Runtime, error) {
-	if err := cfg.Validate(); err != nil {
-		return nil, err
-	}
-
-	res, err := resource.New(ctx, resource.WithAttributes(
-		attribute.String("service.name", cfg.ServiceName),
-	))
-	if err != nil {
-		return nil, fmt.Errorf("telemetry: build resource: %w", err)
-	}
-
-	tracerProvider, err := newTracerProvider(cfg, res)
-	if err != nil {
-		return nil, fmt.Errorf("telemetry: build tracer provider: %w", err)
-	}
-	meterProvider, err := newMeterProvider(cfg, res)
-	if err != nil {
-		_ = tracerProvider.Shutdown(ctx)
-		return nil, fmt.Errorf("telemetry: build meter provider: %w", err)
-	}
-
-	otel.SetTracerProvider(tracerProvider)
-	otel.SetMeterProvider(meterProvider)
-	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
-		propagation.TraceContext{},
-		propagation.Baggage{},
-	))
-
-	return &Runtime{tracerProvider: tracerProvider, meterProvider: meterProvider}, nil
-}
-
-// TracerProvider returns the runtime tracer provider, or the global one when r
-// is not initialised.
-func (r *Runtime) TracerProvider() trace.TracerProvider {
-	if r == nil || r.tracerProvider == nil {
-		return otel.GetTracerProvider()
-	}
-	return r.tracerProvider
-}
-
-// MeterProvider returns the runtime meter provider, or the global one when r is
-// not initialised.
-func (r *Runtime) MeterProvider() metric.MeterProvider {
-	if r == nil || r.meterProvider == nil {
-		return otel.GetMeterProvider()
-	}
-	return r.meterProvider
-}
-
-// Shutdown flushes both providers within ctx.
-func (r *Runtime) Shutdown(ctx context.Context) error {
-	if r == nil {
-		return nil
-	}
-	var err error
-	if r.meterProvider != nil {
-		err = errors.Join(err, r.meterProvider.Shutdown(ctx))
-	}
-	if r.tracerProvider != nil {
-		err = errors.Join(err, r.tracerProvider.Shutdown(ctx))
-	}
-	return err
+	return pkgtel.New(ctx, cfg)
 }
 
 // TraceFieldsFromContext returns zap fields identifying the active span, or nil
-// when ctx carries no valid span context. Collocated here so callers do not
-// import the OpenTelemetry API directly.
+// when ctx carries no valid span context. Collocated here so callers (the
+// request-timing middleware and the access log) do not import the OpenTelemetry
+// API directly.
 func TraceFieldsFromContext(ctx context.Context) []zap.Field {
 	if ctx == nil {
 		return nil
@@ -166,39 +66,3 @@ func TraceFieldsFromContext(ctx context.Context) []zap.Field {
 		zap.String("otel_span_id", sc.SpanID().String()),
 	}
 }
-
-func newTracerProvider(cfg Config, res *resource.Resource) (*sdktrace.TracerProvider, error) {
-	switch cfg.TracesExporter {
-	case ExporterNone:
-		return sdktrace.NewTracerProvider(sdktrace.WithResource(res)), nil
-	case ExporterStdout:
-		exporter, err := stdouttrace.New()
-		if err != nil {
-			return nil, fmt.Errorf("stdout trace exporter: %w", err)
-		}
-		return sdktrace.NewTracerProvider(
-			sdktrace.WithBatcher(exporter),
-			sdktrace.WithResource(res),
-		), nil
-	default:
-		return nil, fmt.Errorf("unsupported traces exporter %q", cfg.TracesExporter)
-	}
-}
-
-func newMeterProvider(cfg Config, res *resource.Resource) (*sdkmetric.MeterProvider, error) {
-	switch cfg.MetricsExporter {
-	case ExporterNone:
-		return sdkmetric.NewMeterProvider(sdkmetric.WithResource(res)), nil
-	case ExporterStdout:
-		exporter, err := stdoutmetric.New()
-		if err != nil {
-			return nil, fmt.Errorf("stdout metric exporter: %w", err)
-		}
-		return sdkmetric.NewMeterProvider(
-			sdkmetric.WithResource(res),
-			sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter)),
-		), nil
-	default:
-		return nil, fmt.Errorf("unsupported metrics exporter %q", cfg.MetricsExporter)
-	}
-}