Stage 12: observability & performance (OTel/OTLP, domain metrics, guest GC)

- pkg/telemetry: shared OTel provider bootstrap (none/stdout/otlp + W3C
  propagators + Go runtime metrics); backend/internal/telemetry becomes a thin
  facade keeping its gin middleware.
- Telemetry parity: gateway and the Telegram connector gain telemetry runtimes
  and config (GATEWAY_/TELEGRAM_ SERVICE_NAME + OTEL_*); otelgrpc instruments the
  backend push server, the gateway's backend+connector clients and the connector
  server. Default exporter stays none (collector/dashboards are Stage 14).
- Operational metrics (variant attribute on game-scoped ones): game_replay_duration,
  game_move_validate_duration, games_started_total, games_abandoned_total,
  game_cache_active, chat_messages_total{kind}, gateway edge_request_duration.
  Wired via the SetMetrics setter pattern (default no-op meter).
- TODO-3: account.GuestReaper deletes guests with no game seat past
  BACKEND_GUEST_RETENTION (default 30d, swept every BACKEND_GUEST_REAP_INTERVAL).
- Tests: pkg/telemetry exporter selection; game/social/edge metric recording via
  a manual reader; config (otlp accepted, guest knobs); inttest guest reaper.
- Docs: PLAN.md re-scopes Stage 12 and adds Stage 13 (alphabet-on-wire) + Stage 14
  (CI/deploy) with the agreed dictionary-versioning resolution; ARCHITECTURE 11/13,
  TESTING, the three READMEs and FUNCTIONAL(+ru) updated.

gateway/internal/backendclient/client.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 
@@ -32,7 +33,10 @@ type Client struct {
 // backend lives on a trusted network segment, so the gRPC connection uses
 // insecure (plaintext) transport credentials (ARCHITECTURE.md §12).
 func New(httpURL, grpcAddr string, timeout time.Duration) (*Client, error) {
-	conn, err := grpc.NewClient(grpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(grpcAddr,
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
+	)
 	if err != nil {
 		return nil, fmt.Errorf("backendclient: dial push %s: %w", grpcAddr, err)
 	}

gateway/internal/config/config.go

@@ -7,6 +7,8 @@ import (
 	"os"
 	"strconv"
 	"time"
+
+	pkgtel "scrabble/pkg/telemetry"
 )
 
 // Config holds the gateway's runtime configuration.
@@ -38,6 +40,8 @@ type Config struct {
 	PushHeartbeatInterval time.Duration
 	// RateLimit configures the in-memory anti-abuse limiter.
 	RateLimit RateLimitConfig
+	// Telemetry configures the OpenTelemetry providers (shared bootstrap).
+	Telemetry pkgtel.Config
 }
 
 // RateLimitConfig holds the token-bucket limits per class. Public and admin are
@@ -64,6 +68,7 @@ const (
 	defaultSessionTTL            = 10 * time.Minute
 	defaultSessionCacheMax       = 50000
 	defaultPushHeartbeatInterval = 15 * time.Second
+	defaultServiceName           = "scrabble-gateway"
 )
 
 // DefaultRateLimit returns the built-in anti-abuse limits.
@@ -91,6 +96,11 @@ func Load() (Config, error) {
 		SessionCacheMax: defaultSessionCacheMax,
 		RateLimit:       DefaultRateLimit(),
 	}
+	tel := pkgtel.DefaultConfig(defaultServiceName)
+	tel.ServiceName = envOr("GATEWAY_SERVICE_NAME", tel.ServiceName)
+	tel.TracesExporter = envOr("GATEWAY_OTEL_TRACES_EXPORTER", tel.TracesExporter)
+	tel.MetricsExporter = envOr("GATEWAY_OTEL_METRICS_EXPORTER", tel.MetricsExporter)
+	c.Telemetry = tel
 	if c.BackendTimeout, err = envDuration("GATEWAY_BACKEND_TIMEOUT", defaultBackendTimeout); err != nil {
 		return Config{}, err
 	}
@@ -131,6 +141,9 @@ func (c Config) validate() error {
 	if c.BackendGRPCAddr == "" {
 		return fmt.Errorf("config: GATEWAY_BACKEND_GRPC_ADDR must not be empty")
 	}
+	if err := c.Telemetry.Validate(); err != nil {
+		return fmt.Errorf("config: %w", err)
+	}
 	return nil
 }
 

gateway/internal/config/config_test.go

@@ -0,0 +1,31 @@
+package config
+
+import (
+	"testing"
+
+	pkgtel "scrabble/pkg/telemetry"
+)
+
+// TestLoadTelemetryDefaults verifies the gateway telemetry defaults: the
+// "scrabble-gateway" service name and both exporters off.
+func TestLoadTelemetryDefaults(t *testing.T) {
+	c, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if c.Telemetry.ServiceName != defaultServiceName {
+		t.Errorf("Telemetry.ServiceName = %q, want %q", c.Telemetry.ServiceName, defaultServiceName)
+	}
+	if c.Telemetry.TracesExporter != pkgtel.ExporterNone || c.Telemetry.MetricsExporter != pkgtel.ExporterNone {
+		t.Errorf("exporters = %q/%q, want none/none", c.Telemetry.TracesExporter, c.Telemetry.MetricsExporter)
+	}
+}
+
+// TestLoadRejectsUnsupportedExporter verifies an exporter outside the supported
+// set fails validation.
+func TestLoadRejectsUnsupportedExporter(t *testing.T) {
+	t.Setenv("GATEWAY_OTEL_METRICS_EXPORTER", "prometheus")
+	if _, err := Load(); err == nil {
+		t.Fatal("Load: expected an error for an unsupported exporter, got nil")
+	}
+}

gateway/internal/connector/client.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials/insecure"
@@ -42,7 +43,10 @@ type Client struct {
 
 // New dials the connector gRPC endpoint.
 func New(addr string) (*Client, error) {
-	conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(addr,
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
+	)
 	if err != nil {
 		return nil, fmt.Errorf("connector: dial %s: %w", addr, err)
 	}

gateway/internal/connectsrv/metrics.go

@@ -0,0 +1,43 @@
+package connectsrv
+
+import (
+	"context"
+	"time"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/noop"
+)
+
+// meterName scopes the gateway edge's OpenTelemetry instruments.
+const meterName = "scrabble/gateway/edge"
+
+// serverMetrics holds the edge's operational instruments. It defaults to no-ops;
+// NewServer installs the real meter when one is supplied in Deps.
+type serverMetrics struct {
+	edge metric.Float64Histogram
+}
+
+// newServerMetrics builds the instruments on meter (nil selects a no-op meter),
+// falling back to a no-op histogram on the (rare) construction error.
+func newServerMetrics(meter metric.Meter) *serverMetrics {
+	if meter == nil {
+		meter = noop.NewMeterProvider().Meter(meterName)
+	}
+	h, err := meter.Float64Histogram("edge_request_duration",
+		metric.WithUnit("s"),
+		metric.WithDescription("Seconds to serve one Connect Execute call, by message type and result."))
+	if err != nil {
+		h, _ = noop.NewMeterProvider().Meter(meterName).Float64Histogram("edge_request_duration")
+	}
+	return &serverMetrics{edge: h}
+}
+
+// recordEdge records the duration of one Execute call labelled by message type and
+// outcome (ok, domain, unauthenticated, rate_limited, unknown_type or internal).
+func (m *serverMetrics) recordEdge(ctx context.Context, msgType, result string, start time.Time) {
+	m.edge.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(
+		attribute.String("message_type", msgType),
+		attribute.String("result", result),
+	))
+}

gateway/internal/connectsrv/metrics_test.go

@@ -0,0 +1,54 @@
+package connectsrv
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go.opentelemetry.io/otel/attribute"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+)
+
+// TestEdgeMetric records Execute outcomes through a manual reader and asserts the
+// edge_request_duration histogram splits by message_type and result.
+func TestEdgeMetric(t *testing.T) {
+	ctx := context.Background()
+	reader := sdkmetric.NewManualReader()
+	meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
+	m := newServerMetrics(meter)
+
+	m.recordEdge(ctx, "game.submit_play", "ok", time.Now().Add(-time.Millisecond))
+	m.recordEdge(ctx, "game.submit_play", "ok", time.Now().Add(-time.Millisecond))
+	m.recordEdge(ctx, "auth.guest", "domain", time.Now().Add(-time.Millisecond))
+
+	var rm metricdata.ResourceMetrics
+	if err := reader.Collect(ctx, &rm); err != nil {
+		t.Fatalf("collect: %v", err)
+	}
+
+	type key struct{ messageType, result string }
+	counts := map[key]uint64{}
+	for _, sm := range rm.ScopeMetrics {
+		for _, md := range sm.Metrics {
+			if md.Name != "edge_request_duration" {
+				continue
+			}
+			h, ok := md.Data.(metricdata.Histogram[float64])
+			if !ok {
+				t.Fatalf("edge_request_duration is not a float64 histogram")
+			}
+			for _, dp := range h.DataPoints {
+				mt, _ := dp.Attributes.Value(attribute.Key("message_type"))
+				res, _ := dp.Attributes.Value(attribute.Key("result"))
+				counts[key{mt.AsString(), res.AsString()}] += dp.Count
+			}
+		}
+	}
+	if got := counts[key{"game.submit_play", "ok"}]; got != 2 {
+		t.Errorf("edge game.submit_play/ok = %d, want 2", got)
+	}
+	if got := counts[key{"auth.guest", "domain"}]; got != 1 {
+		t.Errorf("edge auth.guest/domain = %d, want 1", got)
+	}
+}

gateway/internal/connectsrv/server.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"connectrpc.com/connect"
+	"go.opentelemetry.io/otel/metric"
 	"go.uber.org/zap"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
@@ -39,6 +40,7 @@ type Server struct {
 	heartbeat  time.Duration
 	log        *zap.Logger
 	adminProxy http.Handler
+	metrics    *serverMetrics
 
 	publicPolicy ratelimit.Policy
 	userPolicy   ratelimit.Policy
@@ -55,6 +57,7 @@ type Deps struct {
 	Heartbeat  time.Duration
 	Logger     *zap.Logger
 	AdminProxy http.Handler
+	Meter      metric.Meter
 }
 
 // NewServer constructs the edge service.
@@ -71,6 +74,7 @@ func NewServer(d Deps) *Server {
 		heartbeat:    d.Heartbeat,
 		log:          log,
 		adminProxy:   d.AdminProxy,
+		metrics:      newServerMetrics(d.Meter),
 		publicPolicy: ratelimit.PerMinute(d.RateLimit.PublicPerMinute, d.RateLimit.PublicBurst),
 		userPolicy:   ratelimit.PerMinute(d.RateLimit.UserPerMinute, d.RateLimit.UserBurst),
 		emailPolicy:  ratelimit.Per(d.RateLimit.EmailPer10Min, 10*time.Minute, d.RateLimit.EmailBurst),
@@ -95,9 +99,14 @@ func (s *Server) HTTPHandler() http.Handler {
 // (result_code != "ok", HTTP 200); only edge failures (rate limit, missing
 // session, unknown type, internal) become Connect errors.
 func (s *Server) Execute(ctx context.Context, req *connect.Request[edgev1.ExecuteRequest]) (*connect.Response[edgev1.ExecuteResponse], error) {
+	start := time.Now()
 	msgType := req.Msg.GetMessageType()
+	result := "internal"
+	defer func() { s.metrics.recordEdge(ctx, msgType, result, start) }()
+
 	op, ok := s.registry.Lookup(msgType)
 	if !ok {
+		result = "unknown_type"
 		return nil, connect.NewError(connect.CodeNotFound, errUnknownMessageType(msgType))
 	}
 	clientIP := peerIP(req.Peer().Addr, req.Header())
@@ -106,17 +115,21 @@ func (s *Server) Execute(ctx context.Context, req *connect.Request[edgev1.Execut
 	if op.Auth {
 		uid, err := s.resolve(ctx, req.Header())
 		if err != nil {
+			result = "unauthenticated"
 			return nil, err
 		}
 		if !s.limiter.Allow("user:"+uid, s.userPolicy) {
+			result = "rate_limited"
 			return nil, connect.NewError(connect.CodeResourceExhausted, errRateLimited)
 		}
 		tr.UserID = uid
 	} else {
 		if !s.limiter.Allow("ip:"+clientIP, s.publicPolicy) {
+			result = "rate_limited"
 			return nil, connect.NewError(connect.CodeResourceExhausted, errRateLimited)
 		}
 		if op.Email && !s.limiter.Allow("email:"+clientIP, s.emailPolicy) {
+			result = "rate_limited"
 			return nil, connect.NewError(connect.CodeResourceExhausted, errRateLimited)
 		}
 	}
@@ -124,6 +137,7 @@ func (s *Server) Execute(ctx context.Context, req *connect.Request[edgev1.Execut
 	payload, err := op.Handler(ctx, tr)
 	if err != nil {
 		if code, domain := transcode.DomainCode(err); domain {
+			result = "domain"
 			return connect.NewResponse(&edgev1.ExecuteResponse{
 				RequestId:  req.Msg.GetRequestId(),
 				ResultCode: code,
@@ -132,6 +146,7 @@ func (s *Server) Execute(ctx context.Context, req *connect.Request[edgev1.Execut
 		s.log.Error("execute failed", zap.String("message_type", msgType), zap.Error(err))
 		return nil, connect.NewError(connect.CodeInternal, errInternal)
 	}
+	result = "ok"
 	return connect.NewResponse(&edgev1.ExecuteResponse{
 		RequestId:  req.Msg.GetRequestId(),
 		ResultCode: "ok",