feat: notification service

notification/internal/app/app.go

@@ -0,0 +1,168 @@
+// Package app wires the Notification Service process lifecycle and
+// coordinates component startup and graceful shutdown.
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+
+	"galaxy/notification/internal/config"
+)
+
+// Component is a long-lived Notification Service subsystem that participates
+// in coordinated startup and graceful shutdown.
+type Component interface {
+	// Run starts the component and blocks until it stops.
+	Run(context.Context) error
+
+	// Shutdown stops the component within the provided timeout-bounded context.
+	Shutdown(context.Context) error
+}
+
+// App owns the process-level lifecycle of Notification Service and its
+// registered components.
+type App struct {
+	cfg        config.Config
+	components []Component
+}
+
+// New constructs App with a defensive copy of the supplied components.
+func New(cfg config.Config, components ...Component) *App {
+	clonedComponents := append([]Component(nil), components...)
+
+	return &App{
+		cfg:        cfg,
+		components: clonedComponents,
+	}
+}
+
+// Run starts all configured components, waits for cancellation or the first
+// component failure, and then executes best-effort graceful shutdown.
+func (app *App) Run(ctx context.Context) error {
+	if ctx == nil {
+		return errors.New("run notification app: nil context")
+	}
+	if err := app.validate(); err != nil {
+		return err
+	}
+	if len(app.components) == 0 {
+		<-ctx.Done()
+		return nil
+	}
+
+	runCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	results := make(chan componentResult, len(app.components))
+	var runWaitGroup sync.WaitGroup
+
+	for index, component := range app.components {
+		runWaitGroup.Add(1)
+
+		go func(componentIndex int, component Component) {
+			defer runWaitGroup.Done()
+			results <- componentResult{
+				index: componentIndex,
+				err:   component.Run(runCtx),
+			}
+		}(index, component)
+	}
+
+	var runErr error
+
+	select {
+	case <-ctx.Done():
+	case result := <-results:
+		runErr = classifyComponentResult(ctx, result)
+	}
+
+	cancel()
+
+	shutdownErr := app.shutdownComponents()
+	waitErr := app.waitForComponents(&runWaitGroup)
+
+	return errors.Join(runErr, shutdownErr, waitErr)
+}
+
+type componentResult struct {
+	index int
+	err   error
+}
+
+func (app *App) validate() error {
+	if app.cfg.ShutdownTimeout <= 0 {
+		return fmt.Errorf("run notification app: shutdown timeout must be positive, got %s", app.cfg.ShutdownTimeout)
+	}
+
+	for index, component := range app.components {
+		if component == nil {
+			return fmt.Errorf("run notification app: component %d is nil", index)
+		}
+	}
+
+	return nil
+}
+
+func classifyComponentResult(parentCtx context.Context, result componentResult) error {
+	switch {
+	case result.err == nil:
+		if parentCtx.Err() != nil {
+			return nil
+		}
+		return fmt.Errorf("run notification app: component %d exited without error before shutdown", result.index)
+	case errors.Is(result.err, context.Canceled) && parentCtx.Err() != nil:
+		return nil
+	default:
+		return fmt.Errorf("run notification app: component %d: %w", result.index, result.err)
+	}
+}
+
+func (app *App) shutdownComponents() error {
+	var shutdownWaitGroup sync.WaitGroup
+	errs := make(chan error, len(app.components))
+
+	for index, component := range app.components {
+		shutdownWaitGroup.Add(1)
+
+		go func(componentIndex int, component Component) {
+			defer shutdownWaitGroup.Done()
+
+			shutdownCtx, cancel := context.WithTimeout(context.Background(), app.cfg.ShutdownTimeout)
+			defer cancel()
+
+			if err := component.Shutdown(shutdownCtx); err != nil {
+				errs <- fmt.Errorf("shutdown notification component %d: %w", componentIndex, err)
+			}
+		}(index, component)
+	}
+
+	shutdownWaitGroup.Wait()
+	close(errs)
+
+	var joined error
+	for err := range errs {
+		joined = errors.Join(joined, err)
+	}
+
+	return joined
+}
+
+func (app *App) waitForComponents(runWaitGroup *sync.WaitGroup) error {
+	done := make(chan struct{})
+	go func() {
+		runWaitGroup.Wait()
+		close(done)
+	}()
+
+	waitCtx, cancel := context.WithTimeout(context.Background(), app.cfg.ShutdownTimeout)
+	defer cancel()
+
+	select {
+	case <-done:
+		return nil
+	case <-waitCtx.Done():
+		return fmt.Errorf("wait for notification components: %w", waitCtx.Err())
+	}
+}

notification/internal/app/runtime.go

@@ -0,0 +1,229 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+
+	redisadapter "galaxy/notification/internal/adapters/redis"
+	"galaxy/notification/internal/adapters/redisstate"
+	userserviceadapter "galaxy/notification/internal/adapters/userservice"
+	"galaxy/notification/internal/api/internalhttp"
+	"galaxy/notification/internal/config"
+	"galaxy/notification/internal/service/acceptintent"
+	"galaxy/notification/internal/telemetry"
+	"galaxy/notification/internal/worker"
+
+	"github.com/redis/go-redis/v9"
+)
+
+// Runtime owns the runnable Notification Service process plus the cleanup
+// functions that release runtime resources after shutdown.
+type Runtime struct {
+	cfg config.Config
+
+	app *App
+
+	probeServer    *internalhttp.Server
+	telemetry      *telemetry.Runtime
+	intentConsumer *worker.IntentConsumer
+	pushPublisher  *worker.PushPublisher
+	emailPublisher *worker.EmailPublisher
+
+	cleanupFns []func() error
+}
+
+// NewRuntime constructs the runnable Notification Service process from cfg.
+func NewRuntime(ctx context.Context, cfg config.Config, logger *slog.Logger) (*Runtime, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("new notification runtime: nil context")
+	}
+	if err := cfg.Validate(); err != nil {
+		return nil, fmt.Errorf("new notification runtime: %w", err)
+	}
+	if logger == nil {
+		logger = slog.Default()
+	}
+
+	runtime := &Runtime{
+		cfg: cfg,
+	}
+	cleanupOnError := func(err error) (*Runtime, error) {
+		if cleanupErr := runtime.Close(); cleanupErr != nil {
+			return nil, fmt.Errorf("%w; cleanup: %w", err, cleanupErr)
+		}
+
+		return nil, err
+	}
+
+	telemetryRuntime, err := telemetry.NewProcess(ctx, telemetry.ProcessConfig{
+		ServiceName:          cfg.Telemetry.ServiceName,
+		TracesExporter:       cfg.Telemetry.TracesExporter,
+		MetricsExporter:      cfg.Telemetry.MetricsExporter,
+		TracesProtocol:       cfg.Telemetry.TracesProtocol,
+		MetricsProtocol:      cfg.Telemetry.MetricsProtocol,
+		StdoutTracesEnabled:  cfg.Telemetry.StdoutTracesEnabled,
+		StdoutMetricsEnabled: cfg.Telemetry.StdoutMetricsEnabled,
+	}, logger.With("component", "telemetry"))
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: telemetry: %w", err))
+	}
+	runtime.telemetry = telemetryRuntime
+	runtime.cleanupFns = append(runtime.cleanupFns, func() error {
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout)
+		defer cancel()
+		return telemetryRuntime.Shutdown(shutdownCtx)
+	})
+
+	redisClient := redisadapter.NewClient(cfg.Redis)
+	if err := redisadapter.InstrumentClient(redisClient, telemetryRuntime); err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: %w", err))
+	}
+	runtime.cleanupFns = append(runtime.cleanupFns, func() error {
+		err := redisClient.Close()
+		if errors.Is(err, redis.ErrClosed) {
+			return nil
+		}
+		return err
+	})
+	if err := redisadapter.Ping(ctx, cfg.Redis, redisClient); err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: %w", err))
+	}
+
+	acceptanceStore, err := redisstate.NewAcceptanceStore(redisClient, redisstate.AcceptanceConfig{
+		RecordTTL:      cfg.Retry.RecordTTL,
+		DeadLetterTTL:  cfg.Retry.DeadLetterTTL,
+		IdempotencyTTL: cfg.Retry.IdempotencyTTL,
+	})
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: acceptance store: %w", err))
+	}
+	malformedIntentStore, err := redisstate.NewMalformedIntentStore(redisClient, cfg.Retry.DeadLetterTTL)
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: malformed intent store: %w", err))
+	}
+	streamOffsetStore, err := redisstate.NewStreamOffsetStore(redisClient)
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: stream offset store: %w", err))
+	}
+	intentStreamLagReader, err := redisstate.NewIntentStreamLagReader(streamOffsetStore, cfg.Streams.Intents)
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: intent stream lag reader: %w", err))
+	}
+	telemetryRuntime.SetRouteScheduleSnapshotReader(acceptanceStore)
+	telemetryRuntime.SetIntentStreamLagSnapshotReader(intentStreamLagReader)
+	userDirectory, err := userserviceadapter.NewClient(userserviceadapter.Config{
+		BaseURL:        cfg.UserService.BaseURL,
+		RequestTimeout: cfg.UserService.Timeout,
+	})
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: user service client: %w", err))
+	}
+	runtime.cleanupFns = append(runtime.cleanupFns, userDirectory.Close)
+	acceptIntentService, err := acceptintent.New(acceptintent.Config{
+		Store:            acceptanceStore,
+		UserDirectory:    userDirectory,
+		Clock:            nil,
+		Logger:           logger,
+		Telemetry:        telemetryRuntime,
+		PushMaxAttempts:  cfg.Retry.PushMaxAttempts,
+		EmailMaxAttempts: cfg.Retry.EmailMaxAttempts,
+		IdempotencyTTL:   cfg.Retry.IdempotencyTTL,
+		AdminRouting:     cfg.AdminRouting,
+	})
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: accept intent service: %w", err))
+	}
+	intentConsumer, err := worker.NewIntentConsumer(worker.IntentConsumerConfig{
+		Client:            redisClient,
+		Stream:            cfg.Streams.Intents,
+		BlockTimeout:      cfg.IntentsReadBlockTimeout,
+		Acceptor:          acceptIntentService,
+		MalformedRecorder: malformedIntentStore,
+		OffsetStore:       streamOffsetStore,
+		Telemetry:         telemetryRuntime,
+	}, logger)
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: intent consumer: %w", err))
+	}
+	runtime.intentConsumer = intentConsumer
+	pushPublisher, err := worker.NewPushPublisher(worker.PushPublisherConfig{
+		Store:               acceptanceStore,
+		GatewayStream:       cfg.Streams.GatewayClientEvents,
+		GatewayStreamMaxLen: cfg.Streams.GatewayClientEventsStreamMaxLen,
+		RouteLeaseTTL:       cfg.Retry.RouteLeaseTTL,
+		RouteBackoffMin:     cfg.Retry.RouteBackoffMin,
+		RouteBackoffMax:     cfg.Retry.RouteBackoffMax,
+		Encoder:             nil,
+		Telemetry:           telemetryRuntime,
+		Clock:               nil,
+	}, logger)
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: push publisher: %w", err))
+	}
+	runtime.pushPublisher = pushPublisher
+	emailPublisher, err := worker.NewEmailPublisher(worker.EmailPublisherConfig{
+		Store:                      acceptanceStore,
+		MailDeliveryCommandsStream: cfg.Streams.MailDeliveryCommands,
+		RouteLeaseTTL:              cfg.Retry.RouteLeaseTTL,
+		RouteBackoffMin:            cfg.Retry.RouteBackoffMin,
+		RouteBackoffMax:            cfg.Retry.RouteBackoffMax,
+		Encoder:                    nil,
+		Telemetry:                  telemetryRuntime,
+		Clock:                      nil,
+	}, logger)
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: email publisher: %w", err))
+	}
+	runtime.emailPublisher = emailPublisher
+
+	probeServer, err := internalhttp.NewServer(internalhttp.Config{
+		Addr:              cfg.InternalHTTP.Addr,
+		ReadHeaderTimeout: cfg.InternalHTTP.ReadHeaderTimeout,
+		ReadTimeout:       cfg.InternalHTTP.ReadTimeout,
+		IdleTimeout:       cfg.InternalHTTP.IdleTimeout,
+	}, internalhttp.Dependencies{
+		Logger:    logger,
+		Telemetry: telemetryRuntime,
+	})
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: internal HTTP server: %w", err))
+	}
+	runtime.probeServer = probeServer
+	runtime.app = New(cfg, probeServer, intentConsumer, pushPublisher, emailPublisher)
+
+	return runtime, nil
+}
+
+// Run serves the private probe HTTP listener until ctx is canceled or one
+// component fails.
+func (runtime *Runtime) Run(ctx context.Context) error {
+	if ctx == nil {
+		return errors.New("run notification runtime: nil context")
+	}
+	if runtime == nil {
+		return errors.New("run notification runtime: nil runtime")
+	}
+	if runtime.app == nil {
+		return errors.New("run notification runtime: nil app")
+	}
+
+	return runtime.app.Run(ctx)
+}
+
+// Close releases every runtime dependency in reverse construction order.
+func (runtime *Runtime) Close() error {
+	if runtime == nil {
+		return nil
+	}
+
+	var joined error
+	for index := len(runtime.cleanupFns) - 1; index >= 0; index-- {
+		if err := runtime.cleanupFns[index](); err != nil {
+			joined = errors.Join(joined, err)
+		}
+	}
+
+	return joined
+}

notification/internal/app/runtime_smoke_test.go

@@ -0,0 +1,72 @@
+package app
+
+import (
+	"context"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"galaxy/notification/internal/config"
+
+	"github.com/stretchr/testify/require"
+	testcontainers "github.com/testcontainers/testcontainers-go"
+	rediscontainer "github.com/testcontainers/testcontainers-go/modules/redis"
+)
+
+const (
+	realRuntimeSmokeEnv   = "NOTIFICATION_REAL_RUNTIME_SMOKE"
+	realRuntimeRedisImage = "redis:7"
+)
+
+func TestRealRuntimeCompatibility(t *testing.T) {
+	if os.Getenv(realRuntimeSmokeEnv) != "1" {
+		t.Skipf("set %s=1 to run the real runtime smoke suite", realRuntimeSmokeEnv)
+	}
+
+	ctx := context.Background()
+
+	redisContainer, err := rediscontainer.Run(ctx, realRuntimeRedisImage)
+	require.NoError(t, err)
+	testcontainers.CleanupContainer(t, redisContainer)
+
+	redisAddr, err := redisContainer.Endpoint(ctx, "")
+	require.NoError(t, err)
+
+	cfg := config.DefaultConfig()
+	cfg.Redis.Addr = redisAddr
+	cfg.UserService.BaseURL = "http://user-service.internal"
+	cfg.InternalHTTP.Addr = mustFreeAddr(t)
+	cfg.ShutdownTimeout = 2 * time.Second
+	cfg.Telemetry.TracesExporter = "none"
+	cfg.Telemetry.MetricsExporter = "none"
+
+	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, runtime.Close())
+	}()
+
+	runCtx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	runErrCh := make(chan error, 1)
+	go func() {
+		runErrCh <- runtime.Run(runCtx)
+	}()
+
+	client := &http.Client{
+		Timeout: 500 * time.Millisecond,
+		Transport: &http.Transport{
+			DisableKeepAlives: true,
+		},
+	}
+	t.Cleanup(client.CloseIdleConnections)
+
+	waitForRuntimeReady(t, client, cfg.InternalHTTP.Addr)
+	assertHTTPStatus(t, client, "http://"+cfg.InternalHTTP.Addr+"/healthz", http.StatusOK)
+	assertHTTPStatus(t, client, "http://"+cfg.InternalHTTP.Addr+"/readyz", http.StatusOK)
+
+	cancel()
+	waitForRunResult(t, runErrCh, cfg.ShutdownTimeout+2*time.Second)
+}

notification/internal/app/runtime_test.go

@@ -0,0 +1,581 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+
+	redisstate "galaxy/notification/internal/adapters/redisstate"
+	"galaxy/notification/internal/config"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/redis/go-redis/v9"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewRuntimeStartsProbeListenerAndStopsCleanly(t *testing.T) {
+	t.Parallel()
+
+	redisServer := miniredis.RunT(t)
+	userService := newUserLookupServer(t, func(http.ResponseWriter, *http.Request) {})
+	defer userService.Close()
+
+	cfg := config.DefaultConfig()
+	cfg.Redis.Addr = redisServer.Addr()
+	cfg.UserService.BaseURL = userService.URL
+	cfg.InternalHTTP.Addr = mustFreeAddr(t)
+	cfg.ShutdownTimeout = 10 * time.Second
+	cfg.IntentsReadBlockTimeout = 25 * time.Millisecond
+	cfg.Telemetry.TracesExporter = "none"
+	cfg.Telemetry.MetricsExporter = "none"
+
+	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, runtime.Close())
+	}()
+
+	runCtx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	runErrCh := make(chan error, 1)
+	go func() {
+		runErrCh <- runtime.Run(runCtx)
+	}()
+
+	client := newTestHTTPClient(t)
+	waitForRuntimeReady(t, client, cfg.InternalHTTP.Addr)
+	assertHTTPStatus(t, client, "http://"+cfg.InternalHTTP.Addr+"/healthz", http.StatusOK)
+	assertHTTPStatus(t, client, "http://"+cfg.InternalHTTP.Addr+"/readyz", http.StatusOK)
+	assertHTTPStatus(t, client, "http://"+cfg.InternalHTTP.Addr+"/metrics", http.StatusNotFound)
+
+	cancel()
+	waitForRunResult(t, runErrCh, cfg.ShutdownTimeout+2*time.Second)
+}
+
+func TestNewRuntimeFailsFastWhenRedisPingCheckFails(t *testing.T) {
+	t.Parallel()
+
+	cfg := config.DefaultConfig()
+	cfg.Redis.Addr = mustFreeAddr(t)
+	cfg.UserService.BaseURL = "http://127.0.0.1:18080"
+	cfg.IntentsReadBlockTimeout = 25 * time.Millisecond
+	cfg.Telemetry.TracesExporter = "none"
+	cfg.Telemetry.MetricsExporter = "none"
+
+	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
+	require.Nil(t, runtime)
+	require.Error(t, err)
+	assert.ErrorContains(t, err, "ping redis")
+}
+
+func TestNewRuntimeAcceptsIntentThroughConsumer(t *testing.T) {
+	t.Parallel()
+
+	redisServer := miniredis.RunT(t)
+	redisClient := redis.NewClient(&redis.Options{
+		Addr:            redisServer.Addr(),
+		Protocol:        2,
+		DisableIdentity: true,
+	})
+	t.Cleanup(func() {
+		assert.NoError(t, redisClient.Close())
+	})
+	userService := newUserLookupServer(t, func(writer http.ResponseWriter, request *http.Request) {
+		writeJSON(t, writer, http.StatusOK, map[string]any{
+			"user": map[string]any{
+				"email":              "pilot@example.com",
+				"preferred_language": "en-US",
+			},
+		})
+	})
+	defer userService.Close()
+
+	cfg := config.DefaultConfig()
+	cfg.Redis.Addr = redisServer.Addr()
+	cfg.UserService.BaseURL = userService.URL
+	cfg.InternalHTTP.Addr = mustFreeAddr(t)
+	cfg.ShutdownTimeout = 10 * time.Second
+	cfg.IntentsReadBlockTimeout = 25 * time.Millisecond
+	cfg.Telemetry.TracesExporter = "none"
+	cfg.Telemetry.MetricsExporter = "none"
+
+	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, runtime.Close())
+	}()
+
+	runCtx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	runErrCh := make(chan error, 1)
+	go func() {
+		runErrCh <- runtime.Run(runCtx)
+	}()
+
+	client := newTestHTTPClient(t)
+	waitForRuntimeReady(t, client, cfg.InternalHTTP.Addr)
+
+	messageID, err := redisClient.XAdd(context.Background(), &redis.XAddArgs{
+		Stream: cfg.Streams.Intents,
+		Values: map[string]any{
+			"notification_type":       "game.turn.ready",
+			"producer":                "game_master",
+			"audience_kind":           "user",
+			"recipient_user_ids_json": `["user-1"]`,
+			"idempotency_key":         "game-123:turn-ready",
+			"occurred_at_ms":          "1775121700000",
+			"payload_json":            `{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54}`,
+		},
+	}).Result()
+	require.NoError(t, err)
+
+	require.Eventually(t, func() bool {
+		payload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "email:user:user-1")).Bytes()
+		if err != nil {
+			return false
+		}
+		route, err := redisstate.UnmarshalRoute(payload)
+		if err != nil {
+			return false
+		}
+		return route.ResolvedEmail == "pilot@example.com" && route.ResolvedLocale == "en"
+	}, time.Second, 10*time.Millisecond)
+
+	cancel()
+	waitForRunResult(t, runErrCh, cfg.ShutdownTimeout+2*time.Second)
+}
+
+func TestNewRuntimePublishesAcceptedPushAndEmailRoutes(t *testing.T) {
+	t.Parallel()
+
+	redisServer := miniredis.RunT(t)
+	redisClient := redis.NewClient(&redis.Options{
+		Addr:            redisServer.Addr(),
+		Protocol:        2,
+		DisableIdentity: true,
+	})
+	t.Cleanup(func() {
+		assert.NoError(t, redisClient.Close())
+	})
+	userService := newUserLookupServer(t, func(writer http.ResponseWriter, request *http.Request) {
+		writeJSON(t, writer, http.StatusOK, map[string]any{
+			"user": map[string]any{
+				"email":              "pilot@example.com",
+				"preferred_language": "en-US",
+			},
+		})
+	})
+	defer userService.Close()
+
+	cfg := config.DefaultConfig()
+	cfg.Redis.Addr = redisServer.Addr()
+	cfg.UserService.BaseURL = userService.URL
+	cfg.InternalHTTP.Addr = mustFreeAddr(t)
+	cfg.ShutdownTimeout = 10 * time.Second
+	cfg.IntentsReadBlockTimeout = 25 * time.Millisecond
+	cfg.Telemetry.TracesExporter = "none"
+	cfg.Telemetry.MetricsExporter = "none"
+
+	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, runtime.Close())
+	}()
+
+	runCtx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	runErrCh := make(chan error, 1)
+	go func() {
+		runErrCh <- runtime.Run(runCtx)
+	}()
+
+	client := newTestHTTPClient(t)
+	waitForRuntimeReady(t, client, cfg.InternalHTTP.Addr)
+
+	messageID, err := redisClient.XAdd(context.Background(), &redis.XAddArgs{
+		Stream: cfg.Streams.Intents,
+		Values: map[string]any{
+			"notification_type":       "game.turn.ready",
+			"producer":                "game_master",
+			"audience_kind":           "user",
+			"recipient_user_ids_json": `["user-1"]`,
+			"idempotency_key":         "game-123:turn-ready",
+			"occurred_at_ms":          "1775121700000",
+			"payload_json":            `{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54}`,
+			"request_id":              "request-1",
+			"trace_id":                "trace-1",
+		},
+	}).Result()
+	require.NoError(t, err)
+
+	require.Eventually(t, func() bool {
+		pushPayload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "push:user:user-1")).Bytes()
+		if err != nil {
+			return false
+		}
+		pushRoute, err := redisstate.UnmarshalRoute(pushPayload)
+		if err != nil {
+			return false
+		}
+
+		emailPayload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "email:user:user-1")).Bytes()
+		if err != nil {
+			return false
+		}
+		emailRoute, err := redisstate.UnmarshalRoute(emailPayload)
+		if err != nil {
+			return false
+		}
+
+		return pushRoute.Status == "published" && pushRoute.AttemptCount == 1 &&
+			emailRoute.Status == "published" && emailRoute.AttemptCount == 1
+	}, 2*time.Second, 10*time.Millisecond)
+
+	pushRoutePayload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "push:user:user-1")).Bytes()
+	require.NoError(t, err)
+	pushRoute, err := redisstate.UnmarshalRoute(pushRoutePayload)
+	require.NoError(t, err)
+	require.Equal(t, "published", string(pushRoute.Status))
+
+	notificationPayload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Notification(messageID)).Bytes()
+	require.NoError(t, err)
+	notificationRecord, err := redisstate.UnmarshalNotification(notificationPayload)
+	require.NoError(t, err)
+
+	emailRoutePayload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "email:user:user-1")).Bytes()
+	require.NoError(t, err)
+	emailRoute, err := redisstate.UnmarshalRoute(emailRoutePayload)
+	require.NoError(t, err)
+	require.Equal(t, "published", string(emailRoute.Status))
+
+	messages, err := redisClient.XRange(context.Background(), cfg.Streams.GatewayClientEvents, "-", "+").Result()
+	require.NoError(t, err)
+	require.Len(t, messages, 1)
+	require.Equal(t, "user-1", messages[0].Values["user_id"])
+	require.Equal(t, "game.turn.ready", messages[0].Values["event_type"])
+	require.Equal(t, messageID+"/push:user:user-1", messages[0].Values["event_id"])
+	require.Equal(t, "request-1", messages[0].Values["request_id"])
+	require.Equal(t, "trace-1", messages[0].Values["trace_id"])
+	require.NotContains(t, messages[0].Values, "device_session_id")
+	switch payload := messages[0].Values["payload_bytes"].(type) {
+	case string:
+		require.NotEmpty(t, payload)
+	case []byte:
+		require.NotEmpty(t, payload)
+	default:
+		require.Failf(t, "unexpected payload type", "payload_bytes has type %T", payload)
+	}
+
+	mailCommands, err := redisClient.XRange(context.Background(), cfg.Streams.MailDeliveryCommands, "-", "+").Result()
+	require.NoError(t, err)
+	require.Len(t, mailCommands, 1)
+	require.Equal(t, messageID+"/email:user:user-1", mailCommands[0].Values["delivery_id"])
+	require.Equal(t, "notification", mailCommands[0].Values["source"])
+	require.Equal(t, "template", mailCommands[0].Values["payload_mode"])
+	require.Equal(t, "notification:"+messageID+"/email:user:user-1", mailCommands[0].Values["idempotency_key"])
+	require.Equal(t, strconv.FormatInt(notificationRecord.AcceptedAt.UnixMilli(), 10), mailCommands[0].Values["requested_at_ms"])
+	require.Equal(t, "request-1", mailCommands[0].Values["request_id"])
+	require.Equal(t, "trace-1", mailCommands[0].Values["trace_id"])
+	require.JSONEq(t,
+		`{"to":["pilot@example.com"],"cc":[],"bcc":[],"reply_to":[],"template_id":"game.turn.ready","locale":"en","variables":{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54},"attachments":[]}`,
+		mailCommands[0].Values["payload_json"].(string),
+	)
+
+	cancel()
+	waitForRunResult(t, runErrCh, cfg.ShutdownTimeout+2*time.Second)
+}
+
+func TestNewRuntimePublishesAdminEmailRouteOnlyToMailService(t *testing.T) {
+	t.Parallel()
+
+	redisServer := miniredis.RunT(t)
+	redisClient := redis.NewClient(&redis.Options{
+		Addr:            redisServer.Addr(),
+		Protocol:        2,
+		DisableIdentity: true,
+	})
+	t.Cleanup(func() {
+		assert.NoError(t, redisClient.Close())
+	})
+	userService := newUserLookupServer(t, func(http.ResponseWriter, *http.Request) {})
+	defer userService.Close()
+
+	cfg := config.DefaultConfig()
+	cfg.Redis.Addr = redisServer.Addr()
+	cfg.UserService.BaseURL = userService.URL
+	cfg.AdminRouting.LobbyApplicationSubmitted = []string{"owner@example.com"}
+	cfg.InternalHTTP.Addr = mustFreeAddr(t)
+	cfg.ShutdownTimeout = 10 * time.Second
+	cfg.IntentsReadBlockTimeout = 25 * time.Millisecond
+	cfg.Telemetry.TracesExporter = "none"
+	cfg.Telemetry.MetricsExporter = "none"
+
+	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, runtime.Close())
+	}()
+
+	runCtx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	runErrCh := make(chan error, 1)
+	go func() {
+		runErrCh <- runtime.Run(runCtx)
+	}()
+
+	client := newTestHTTPClient(t)
+	waitForRuntimeReady(t, client, cfg.InternalHTTP.Addr)
+
+	messageID, err := redisClient.XAdd(context.Background(), &redis.XAddArgs{
+		Stream: cfg.Streams.Intents,
+		Values: map[string]any{
+			"notification_type": "lobby.application.submitted",
+			"producer":          "game_lobby",
+			"audience_kind":     "admin_email",
+			"idempotency_key":   "game-123:application-submitted:user-42",
+			"occurred_at_ms":    "1775121700000",
+			"payload_json":      `{"applicant_name":"Nova Pilot","applicant_user_id":"user-42","game_id":"game-123","game_name":"Nebula Clash"}`,
+		},
+	}).Result()
+	require.NoError(t, err)
+
+	require.Eventually(t, func() bool {
+		payload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "email:email:owner@example.com")).Bytes()
+		if err != nil {
+			return false
+		}
+		route, err := redisstate.UnmarshalRoute(payload)
+		if err != nil {
+			return false
+		}
+
+		return route.Status == "published" && route.AttemptCount == 1
+	}, 2*time.Second, 10*time.Millisecond)
+
+	pushRoutePayload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "push:email:owner@example.com")).Bytes()
+	require.NoError(t, err)
+	pushRoute, err := redisstate.UnmarshalRoute(pushRoutePayload)
+	require.NoError(t, err)
+	require.Equal(t, "skipped", string(pushRoute.Status))
+
+	mailCommands, err := redisClient.XRange(context.Background(), cfg.Streams.MailDeliveryCommands, "-", "+").Result()
+	require.NoError(t, err)
+	require.Len(t, mailCommands, 1)
+	require.Equal(t, messageID+"/email:email:owner@example.com", mailCommands[0].Values["delivery_id"])
+	require.JSONEq(t,
+		`{"to":["owner@example.com"],"cc":[],"bcc":[],"reply_to":[],"template_id":"lobby.application.submitted","locale":"en","variables":{"applicant_name":"Nova Pilot","applicant_user_id":"user-42","game_id":"game-123","game_name":"Nebula Clash"},"attachments":[]}`,
+		mailCommands[0].Values["payload_json"].(string),
+	)
+
+	gatewayMessages, err := redisClient.XRange(context.Background(), cfg.Streams.GatewayClientEvents, "-", "+").Result()
+	require.NoError(t, err)
+	require.Empty(t, gatewayMessages)
+
+	cancel()
+	waitForRunResult(t, runErrCh, cfg.ShutdownTimeout+2*time.Second)
+}
+
+func TestNewRuntimeUsesConfiguredUserServiceTimeout(t *testing.T) {
+	t.Parallel()
+
+	redisServer := miniredis.RunT(t)
+	redisClient := redis.NewClient(&redis.Options{
+		Addr:            redisServer.Addr(),
+		Protocol:        2,
+		DisableIdentity: true,
+	})
+	t.Cleanup(func() {
+		assert.NoError(t, redisClient.Close())
+	})
+	userService := newUserLookupServer(t, func(_ http.ResponseWriter, request *http.Request) {
+		<-request.Context().Done()
+	})
+	defer userService.Close()
+
+	cfg := config.DefaultConfig()
+	cfg.Redis.Addr = redisServer.Addr()
+	cfg.UserService.BaseURL = userService.URL
+	cfg.UserService.Timeout = 20 * time.Millisecond
+	cfg.InternalHTTP.Addr = mustFreeAddr(t)
+	cfg.ShutdownTimeout = 10 * time.Second
+	cfg.IntentsReadBlockTimeout = 25 * time.Millisecond
+	cfg.Telemetry.TracesExporter = "none"
+	cfg.Telemetry.MetricsExporter = "none"
+
+	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, runtime.Close())
+	}()
+
+	runCtx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	runErrCh := make(chan error, 1)
+	go func() {
+		runErrCh <- runtime.Run(runCtx)
+	}()
+
+	client := newTestHTTPClient(t)
+	waitForRuntimeReady(t, client, cfg.InternalHTTP.Addr)
+
+	messageID, err := redisClient.XAdd(context.Background(), &redis.XAddArgs{
+		Stream: cfg.Streams.Intents,
+		Values: map[string]any{
+			"notification_type":       "game.turn.ready",
+			"producer":                "game_master",
+			"audience_kind":           "user",
+			"recipient_user_ids_json": `["user-1"]`,
+			"idempotency_key":         "game-123:turn-ready",
+			"occurred_at_ms":          "1775121700000",
+			"payload_json":            `{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54}`,
+		},
+	}).Result()
+	require.NoError(t, err)
+
+	var runErr error
+	require.Eventually(t, func() bool {
+		select {
+		case runErr = <-runErrCh:
+			return true
+		default:
+			return false
+		}
+	}, time.Second, 10*time.Millisecond)
+
+	require.Error(t, runErr)
+	require.ErrorContains(t, runErr, "context deadline exceeded")
+
+	offsetStore, err := redisstate.NewStreamOffsetStore(redisClient)
+	require.NoError(t, err)
+	offset, found, err := offsetStore.Load(context.Background(), cfg.Streams.Intents)
+	require.NoError(t, err)
+	require.False(t, found)
+	require.Empty(t, offset)
+
+	_, err = redisClient.Get(context.Background(), redisstate.Keyspace{}.Notification(messageID)).Bytes()
+	require.Error(t, err)
+}
+
+func testLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, nil))
+}
+
+func newTestHTTPClient(t *testing.T) *http.Client {
+	t.Helper()
+
+	transport := &http.Transport{DisableKeepAlives: true}
+	t.Cleanup(transport.CloseIdleConnections)
+
+	return &http.Client{
+		Timeout:   500 * time.Millisecond,
+		Transport: transport,
+	}
+}
+
+func waitForRuntimeReady(t *testing.T, client *http.Client, addr string) {
+	t.Helper()
+
+	require.Eventually(t, func() bool {
+		request, err := http.NewRequest(http.MethodGet, "http://"+addr+"/readyz", nil)
+		if err != nil {
+			return false
+		}
+
+		response, err := client.Do(request)
+		if err != nil {
+			return false
+		}
+		defer response.Body.Close()
+		_, _ = io.Copy(io.Discard, response.Body)
+
+		return response.StatusCode == http.StatusOK
+	}, 5*time.Second, 25*time.Millisecond, "notification runtime did not become reachable")
+}
+
+func waitForRunResult(t *testing.T, runErrCh <-chan error, waitTimeout time.Duration) {
+	t.Helper()
+
+	var err error
+	require.Eventually(t, func() bool {
+		select {
+		case err = <-runErrCh:
+			return true
+		default:
+			return false
+		}
+	}, waitTimeout, 10*time.Millisecond, "notification runtime did not stop")
+	require.NoError(t, err)
+}
+
+func assertHTTPStatus(t *testing.T, client *http.Client, target string, want int) {
+	t.Helper()
+
+	request, err := http.NewRequest(http.MethodGet, target, nil)
+	require.NoError(t, err)
+
+	response, err := client.Do(request)
+	require.NoError(t, err)
+	defer response.Body.Close()
+	_, _ = io.Copy(io.Discard, response.Body)
+
+	require.Equal(t, want, response.StatusCode)
+}
+
+func mustFreeAddr(t *testing.T) string {
+	t.Helper()
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	defer func() {
+		assert.NoError(t, listener.Close())
+	}()
+
+	return listener.Addr().String()
+}
+
+func newUserLookupServer(t *testing.T, handler func(http.ResponseWriter, *http.Request)) *httptest.Server {
+	t.Helper()
+
+	return httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		if request.Method != http.MethodGet {
+			http.NotFound(writer, request)
+			return
+		}
+		if request.URL.Path != "/api/v1/internal/users/user-1" {
+			writeJSON(t, writer, http.StatusNotFound, map[string]any{
+				"error": map[string]any{
+					"code":    "subject_not_found",
+					"message": "subject not found",
+				},
+			})
+			return
+		}
+
+		handler(writer, request)
+	}))
+}
+
+func writeJSON(t *testing.T, writer http.ResponseWriter, statusCode int, payload any) {
+	t.Helper()
+
+	body, err := json.Marshal(payload)
+	require.NoError(t, err)
+
+	writer.Header().Set("Content-Type", "application/json")
+	writer.WriteHeader(statusCode)
+	_, err = writer.Write(body)
+	require.NoError(t, err)
+}