feat: use postgres

notification/internal/app/runtime.go

@@ -5,7 +5,11 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"time"
 
+	"galaxy/notification/internal/adapters/postgres/migrations"
+	"galaxy/notification/internal/adapters/postgres/notificationstore"
+	"galaxy/notification/internal/adapters/postgres/routepublisher"
 	redisadapter "galaxy/notification/internal/adapters/redis"
 	"galaxy/notification/internal/adapters/redisstate"
 	userserviceadapter "galaxy/notification/internal/adapters/userservice"
@@ -14,10 +18,16 @@ import (
 	"galaxy/notification/internal/service/acceptintent"
 	"galaxy/notification/internal/telemetry"
 	"galaxy/notification/internal/worker"
+	"galaxy/postgres"
 
 	"github.com/redis/go-redis/v9"
 )
 
+// systemClock satisfies the worker.Clock contract for runtime wiring.
+type systemClock struct{}
+
+func (systemClock) Now() time.Time { return time.Now() }
+
 // Runtime owns the runnable Notification Service process plus the cleanup
 // functions that release runtime resources after shutdown.
 type Runtime struct {
@@ -25,16 +35,24 @@ type Runtime struct {
 
 	app *App
 
-	probeServer    *internalhttp.Server
-	telemetry      *telemetry.Runtime
-	intentConsumer *worker.IntentConsumer
-	pushPublisher  *worker.PushPublisher
-	emailPublisher *worker.EmailPublisher
+	probeServer     *internalhttp.Server
+	telemetry       *telemetry.Runtime
+	intentConsumer  *worker.IntentConsumer
+	pushPublisher   *worker.PushPublisher
+	emailPublisher  *worker.EmailPublisher
+	retentionWorker *worker.SQLRetentionWorker
 
 	cleanupFns []func() error
 }
 
 // NewRuntime constructs the runnable Notification Service process from cfg.
+//
+// PostgreSQL migrations apply strictly before any HTTP listener becomes
+// ready. The runtime opens one shared `*redis.Client` consumed by the intent
+// consumer (XREAD), the publishers (outbound XADDs), the route lease store,
+// and the persisted stream offset store. Per PG_PLAN.md §5 the durable
+// notification state lives in PostgreSQL while the lease key, the consumer
+// offset, and the streams themselves remain on Redis.
 func NewRuntime(ctx context.Context, cfg config.Config, logger *slog.Logger) (*Runtime, error) {
 	if ctx == nil {
 		return nil, fmt.Errorf("new notification runtime: nil context")
@@ -91,17 +109,42 @@ func NewRuntime(ctx context.Context, cfg config.Config, logger *slog.Logger) (*R
 		return cleanupOnError(fmt.Errorf("new notification runtime: %w", err))
 	}
 
-	acceptanceStore, err := redisstate.NewAcceptanceStore(redisClient, redisstate.AcceptanceConfig{
-		RecordTTL:      cfg.Retry.RecordTTL,
-		DeadLetterTTL:  cfg.Retry.DeadLetterTTL,
-		IdempotencyTTL: cfg.Retry.IdempotencyTTL,
+	pgPool, err := postgres.OpenPrimary(ctx, cfg.Postgres.Conn,
+		postgres.WithTracerProvider(telemetryRuntime.TracerProvider()),
+		postgres.WithMeterProvider(telemetryRuntime.MeterProvider()),
+	)
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: open postgres: %w", err))
+	}
+	runtime.cleanupFns = append(runtime.cleanupFns, pgPool.Close)
+	unregisterPGStats, err := postgres.InstrumentDBStats(pgPool,
+		postgres.WithMeterProvider(telemetryRuntime.MeterProvider()),
+	)
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: instrument postgres: %w", err))
+	}
+	runtime.cleanupFns = append(runtime.cleanupFns, func() error {
+		unregisterPGStats()
+		return nil
+	})
+	if err := postgres.Ping(ctx, pgPool, cfg.Postgres.Conn.OperationTimeout); err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: ping postgres: %w", err))
+	}
+	if err := postgres.RunMigrations(ctx, pgPool, migrations.FS(), "."); err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: run postgres migrations: %w", err))
+	}
+
+	notificationStore, err := notificationstore.New(notificationstore.Config{
+		DB:               pgPool,
+		OperationTimeout: cfg.Postgres.Conn.OperationTimeout,
 	})
 	if err != nil {
-		return cleanupOnError(fmt.Errorf("new notification runtime: acceptance store: %w", err))
+		return cleanupOnError(fmt.Errorf("new notification runtime: notification store: %w", err))
 	}
-	malformedIntentStore, err := redisstate.NewMalformedIntentStore(redisClient, cfg.Retry.DeadLetterTTL)
+
+	leaseStore, err := redisstate.NewLeaseStore(redisClient)
 	if err != nil {
-		return cleanupOnError(fmt.Errorf("new notification runtime: malformed intent store: %w", err))
+		return cleanupOnError(fmt.Errorf("new notification runtime: lease store: %w", err))
 	}
 	streamOffsetStore, err := redisstate.NewStreamOffsetStore(redisClient)
 	if err != nil {
@@ -111,8 +154,14 @@ func NewRuntime(ctx context.Context, cfg config.Config, logger *slog.Logger) (*R
 	if err != nil {
 		return cleanupOnError(fmt.Errorf("new notification runtime: intent stream lag reader: %w", err))
 	}
-	telemetryRuntime.SetRouteScheduleSnapshotReader(acceptanceStore)
+	publisherStore, err := routepublisher.New(notificationStore, leaseStore)
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: route publisher store: %w", err))
+	}
+
+	telemetryRuntime.SetRouteScheduleSnapshotReader(notificationStore)
 	telemetryRuntime.SetIntentStreamLagSnapshotReader(intentStreamLagReader)
+
 	userDirectory, err := userserviceadapter.NewClient(userserviceadapter.Config{
 		BaseURL:        cfg.UserService.BaseURL,
 		RequestTimeout: cfg.UserService.Timeout,
@@ -121,8 +170,9 @@ func NewRuntime(ctx context.Context, cfg config.Config, logger *slog.Logger) (*R
 		return cleanupOnError(fmt.Errorf("new notification runtime: user service client: %w", err))
 	}
 	runtime.cleanupFns = append(runtime.cleanupFns, userDirectory.Close)
+
 	acceptIntentService, err := acceptintent.New(acceptintent.Config{
-		Store:            acceptanceStore,
+		Store:            notificationStore,
 		UserDirectory:    userDirectory,
 		Clock:            nil,
 		Logger:           logger,
@@ -140,7 +190,7 @@ func NewRuntime(ctx context.Context, cfg config.Config, logger *slog.Logger) (*R
 		Stream:            cfg.Streams.Intents,
 		BlockTimeout:      cfg.IntentsReadBlockTimeout,
 		Acceptor:          acceptIntentService,
-		MalformedRecorder: malformedIntentStore,
+		MalformedRecorder: notificationStore,
 		OffsetStore:       streamOffsetStore,
 		Telemetry:         telemetryRuntime,
 	}, logger)
@@ -149,7 +199,7 @@ func NewRuntime(ctx context.Context, cfg config.Config, logger *slog.Logger) (*R
 	}
 	runtime.intentConsumer = intentConsumer
 	pushPublisher, err := worker.NewPushPublisher(worker.PushPublisherConfig{
-		Store:               acceptanceStore,
+		Store:               publisherStore,
 		GatewayStream:       cfg.Streams.GatewayClientEvents,
 		GatewayStreamMaxLen: cfg.Streams.GatewayClientEventsStreamMaxLen,
 		RouteLeaseTTL:       cfg.Retry.RouteLeaseTTL,
@@ -158,13 +208,14 @@ func NewRuntime(ctx context.Context, cfg config.Config, logger *slog.Logger) (*R
 		Encoder:             nil,
 		Telemetry:           telemetryRuntime,
 		Clock:               nil,
+		StreamPublisher:     redisClient,
 	}, logger)
 	if err != nil {
 		return cleanupOnError(fmt.Errorf("new notification runtime: push publisher: %w", err))
 	}
 	runtime.pushPublisher = pushPublisher
 	emailPublisher, err := worker.NewEmailPublisher(worker.EmailPublisherConfig{
-		Store:                      acceptanceStore,
+		Store:                      publisherStore,
 		MailDeliveryCommandsStream: cfg.Streams.MailDeliveryCommands,
 		RouteLeaseTTL:              cfg.Retry.RouteLeaseTTL,
 		RouteBackoffMin:            cfg.Retry.RouteBackoffMin,
@@ -172,12 +223,25 @@ func NewRuntime(ctx context.Context, cfg config.Config, logger *slog.Logger) (*R
 		Encoder:                    nil,
 		Telemetry:                  telemetryRuntime,
 		Clock:                      nil,
+		StreamPublisher:            redisClient,
 	}, logger)
 	if err != nil {
 		return cleanupOnError(fmt.Errorf("new notification runtime: email publisher: %w", err))
 	}
 	runtime.emailPublisher = emailPublisher
 
+	retentionWorker, err := worker.NewSQLRetentionWorker(worker.SQLRetentionConfig{
+		Store:                    notificationStore,
+		RecordRetention:          cfg.Retention.RecordRetention,
+		MalformedIntentRetention: cfg.Retention.MalformedIntentRetention,
+		CleanupInterval:          cfg.Retention.CleanupInterval,
+		Clock:                    systemClock{},
+	}, logger)
+	if err != nil {
+		return cleanupOnError(fmt.Errorf("new notification runtime: sql retention worker: %w", err))
+	}
+	runtime.retentionWorker = retentionWorker
+
 	probeServer, err := internalhttp.NewServer(internalhttp.Config{
 		Addr:              cfg.InternalHTTP.Addr,
 		ReadHeaderTimeout: cfg.InternalHTTP.ReadHeaderTimeout,
@@ -191,7 +255,7 @@ func NewRuntime(ctx context.Context, cfg config.Config, logger *slog.Logger) (*R
 		return cleanupOnError(fmt.Errorf("new notification runtime: internal HTTP server: %w", err))
 	}
 	runtime.probeServer = probeServer
-	runtime.app = New(cfg, probeServer, intentConsumer, pushPublisher, emailPublisher)
+	runtime.app = New(cfg, probeServer, intentConsumer, pushPublisher, emailPublisher, retentionWorker)
 
 	return runtime, nil
 }

notification/internal/app/runtime_smoke_test.go

@@ -1,72 +0,0 @@
-package app
-
-import (
-	"context"
-	"net/http"
-	"os"
-	"testing"
-	"time"
-
-	"galaxy/notification/internal/config"
-
-	"github.com/stretchr/testify/require"
-	testcontainers "github.com/testcontainers/testcontainers-go"
-	rediscontainer "github.com/testcontainers/testcontainers-go/modules/redis"
-)
-
-const (
-	realRuntimeSmokeEnv   = "NOTIFICATION_REAL_RUNTIME_SMOKE"
-	realRuntimeRedisImage = "redis:7"
-)
-
-func TestRealRuntimeCompatibility(t *testing.T) {
-	if os.Getenv(realRuntimeSmokeEnv) != "1" {
-		t.Skipf("set %s=1 to run the real runtime smoke suite", realRuntimeSmokeEnv)
-	}
-
-	ctx := context.Background()
-
-	redisContainer, err := rediscontainer.Run(ctx, realRuntimeRedisImage)
-	require.NoError(t, err)
-	testcontainers.CleanupContainer(t, redisContainer)
-
-	redisAddr, err := redisContainer.Endpoint(ctx, "")
-	require.NoError(t, err)
-
-	cfg := config.DefaultConfig()
-	cfg.Redis.Addr = redisAddr
-	cfg.UserService.BaseURL = "http://user-service.internal"
-	cfg.InternalHTTP.Addr = mustFreeAddr(t)
-	cfg.ShutdownTimeout = 2 * time.Second
-	cfg.Telemetry.TracesExporter = "none"
-	cfg.Telemetry.MetricsExporter = "none"
-
-	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
-	require.NoError(t, err)
-	defer func() {
-		require.NoError(t, runtime.Close())
-	}()
-
-	runCtx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	runErrCh := make(chan error, 1)
-	go func() {
-		runErrCh <- runtime.Run(runCtx)
-	}()
-
-	client := &http.Client{
-		Timeout: 500 * time.Millisecond,
-		Transport: &http.Transport{
-			DisableKeepAlives: true,
-		},
-	}
-	t.Cleanup(client.CloseIdleConnections)
-
-	waitForRuntimeReady(t, client, cfg.InternalHTTP.Addr)
-	assertHTTPStatus(t, client, "http://"+cfg.InternalHTTP.Addr+"/healthz", http.StatusOK)
-	assertHTTPStatus(t, client, "http://"+cfg.InternalHTTP.Addr+"/readyz", http.StatusOK)
-
-	cancel()
-	waitForRunResult(t, runErrCh, cfg.ShutdownTimeout+2*time.Second)
-}

notification/internal/app/runtime_test.go

@@ -1,581 +0,0 @@
-package app
-
-import (
-	"context"
-	"encoding/json"
-	"io"
-	"log/slog"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"testing"
-	"time"
-
-	redisstate "galaxy/notification/internal/adapters/redisstate"
-	"galaxy/notification/internal/config"
-
-	"github.com/alicebob/miniredis/v2"
-	"github.com/redis/go-redis/v9"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestNewRuntimeStartsProbeListenerAndStopsCleanly(t *testing.T) {
-	t.Parallel()
-
-	redisServer := miniredis.RunT(t)
-	userService := newUserLookupServer(t, func(http.ResponseWriter, *http.Request) {})
-	defer userService.Close()
-
-	cfg := config.DefaultConfig()
-	cfg.Redis.Addr = redisServer.Addr()
-	cfg.UserService.BaseURL = userService.URL
-	cfg.InternalHTTP.Addr = mustFreeAddr(t)
-	cfg.ShutdownTimeout = 10 * time.Second
-	cfg.IntentsReadBlockTimeout = 25 * time.Millisecond
-	cfg.Telemetry.TracesExporter = "none"
-	cfg.Telemetry.MetricsExporter = "none"
-
-	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
-	require.NoError(t, err)
-	defer func() {
-		require.NoError(t, runtime.Close())
-	}()
-
-	runCtx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	runErrCh := make(chan error, 1)
-	go func() {
-		runErrCh <- runtime.Run(runCtx)
-	}()
-
-	client := newTestHTTPClient(t)
-	waitForRuntimeReady(t, client, cfg.InternalHTTP.Addr)
-	assertHTTPStatus(t, client, "http://"+cfg.InternalHTTP.Addr+"/healthz", http.StatusOK)
-	assertHTTPStatus(t, client, "http://"+cfg.InternalHTTP.Addr+"/readyz", http.StatusOK)
-	assertHTTPStatus(t, client, "http://"+cfg.InternalHTTP.Addr+"/metrics", http.StatusNotFound)
-
-	cancel()
-	waitForRunResult(t, runErrCh, cfg.ShutdownTimeout+2*time.Second)
-}
-
-func TestNewRuntimeFailsFastWhenRedisPingCheckFails(t *testing.T) {
-	t.Parallel()
-
-	cfg := config.DefaultConfig()
-	cfg.Redis.Addr = mustFreeAddr(t)
-	cfg.UserService.BaseURL = "http://127.0.0.1:18080"
-	cfg.IntentsReadBlockTimeout = 25 * time.Millisecond
-	cfg.Telemetry.TracesExporter = "none"
-	cfg.Telemetry.MetricsExporter = "none"
-
-	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
-	require.Nil(t, runtime)
-	require.Error(t, err)
-	assert.ErrorContains(t, err, "ping redis")
-}
-
-func TestNewRuntimeAcceptsIntentThroughConsumer(t *testing.T) {
-	t.Parallel()
-
-	redisServer := miniredis.RunT(t)
-	redisClient := redis.NewClient(&redis.Options{
-		Addr:            redisServer.Addr(),
-		Protocol:        2,
-		DisableIdentity: true,
-	})
-	t.Cleanup(func() {
-		assert.NoError(t, redisClient.Close())
-	})
-	userService := newUserLookupServer(t, func(writer http.ResponseWriter, request *http.Request) {
-		writeJSON(t, writer, http.StatusOK, map[string]any{
-			"user": map[string]any{
-				"email":              "pilot@example.com",
-				"preferred_language": "en-US",
-			},
-		})
-	})
-	defer userService.Close()
-
-	cfg := config.DefaultConfig()
-	cfg.Redis.Addr = redisServer.Addr()
-	cfg.UserService.BaseURL = userService.URL
-	cfg.InternalHTTP.Addr = mustFreeAddr(t)
-	cfg.ShutdownTimeout = 10 * time.Second
-	cfg.IntentsReadBlockTimeout = 25 * time.Millisecond
-	cfg.Telemetry.TracesExporter = "none"
-	cfg.Telemetry.MetricsExporter = "none"
-
-	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
-	require.NoError(t, err)
-	defer func() {
-		require.NoError(t, runtime.Close())
-	}()
-
-	runCtx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	runErrCh := make(chan error, 1)
-	go func() {
-		runErrCh <- runtime.Run(runCtx)
-	}()
-
-	client := newTestHTTPClient(t)
-	waitForRuntimeReady(t, client, cfg.InternalHTTP.Addr)
-
-	messageID, err := redisClient.XAdd(context.Background(), &redis.XAddArgs{
-		Stream: cfg.Streams.Intents,
-		Values: map[string]any{
-			"notification_type":       "game.turn.ready",
-			"producer":                "game_master",
-			"audience_kind":           "user",
-			"recipient_user_ids_json": `["user-1"]`,
-			"idempotency_key":         "game-123:turn-ready",
-			"occurred_at_ms":          "1775121700000",
-			"payload_json":            `{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54}`,
-		},
-	}).Result()
-	require.NoError(t, err)
-
-	require.Eventually(t, func() bool {
-		payload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "email:user:user-1")).Bytes()
-		if err != nil {
-			return false
-		}
-		route, err := redisstate.UnmarshalRoute(payload)
-		if err != nil {
-			return false
-		}
-		return route.ResolvedEmail == "pilot@example.com" && route.ResolvedLocale == "en"
-	}, time.Second, 10*time.Millisecond)
-
-	cancel()
-	waitForRunResult(t, runErrCh, cfg.ShutdownTimeout+2*time.Second)
-}
-
-func TestNewRuntimePublishesAcceptedPushAndEmailRoutes(t *testing.T) {
-	t.Parallel()
-
-	redisServer := miniredis.RunT(t)
-	redisClient := redis.NewClient(&redis.Options{
-		Addr:            redisServer.Addr(),
-		Protocol:        2,
-		DisableIdentity: true,
-	})
-	t.Cleanup(func() {
-		assert.NoError(t, redisClient.Close())
-	})
-	userService := newUserLookupServer(t, func(writer http.ResponseWriter, request *http.Request) {
-		writeJSON(t, writer, http.StatusOK, map[string]any{
-			"user": map[string]any{
-				"email":              "pilot@example.com",
-				"preferred_language": "en-US",
-			},
-		})
-	})
-	defer userService.Close()
-
-	cfg := config.DefaultConfig()
-	cfg.Redis.Addr = redisServer.Addr()
-	cfg.UserService.BaseURL = userService.URL
-	cfg.InternalHTTP.Addr = mustFreeAddr(t)
-	cfg.ShutdownTimeout = 10 * time.Second
-	cfg.IntentsReadBlockTimeout = 25 * time.Millisecond
-	cfg.Telemetry.TracesExporter = "none"
-	cfg.Telemetry.MetricsExporter = "none"
-
-	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
-	require.NoError(t, err)
-	defer func() {
-		require.NoError(t, runtime.Close())
-	}()
-
-	runCtx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	runErrCh := make(chan error, 1)
-	go func() {
-		runErrCh <- runtime.Run(runCtx)
-	}()
-
-	client := newTestHTTPClient(t)
-	waitForRuntimeReady(t, client, cfg.InternalHTTP.Addr)
-
-	messageID, err := redisClient.XAdd(context.Background(), &redis.XAddArgs{
-		Stream: cfg.Streams.Intents,
-		Values: map[string]any{
-			"notification_type":       "game.turn.ready",
-			"producer":                "game_master",
-			"audience_kind":           "user",
-			"recipient_user_ids_json": `["user-1"]`,
-			"idempotency_key":         "game-123:turn-ready",
-			"occurred_at_ms":          "1775121700000",
-			"payload_json":            `{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54}`,
-			"request_id":              "request-1",
-			"trace_id":                "trace-1",
-		},
-	}).Result()
-	require.NoError(t, err)
-
-	require.Eventually(t, func() bool {
-		pushPayload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "push:user:user-1")).Bytes()
-		if err != nil {
-			return false
-		}
-		pushRoute, err := redisstate.UnmarshalRoute(pushPayload)
-		if err != nil {
-			return false
-		}
-
-		emailPayload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "email:user:user-1")).Bytes()
-		if err != nil {
-			return false
-		}
-		emailRoute, err := redisstate.UnmarshalRoute(emailPayload)
-		if err != nil {
-			return false
-		}
-
-		return pushRoute.Status == "published" && pushRoute.AttemptCount == 1 &&
-			emailRoute.Status == "published" && emailRoute.AttemptCount == 1
-	}, 2*time.Second, 10*time.Millisecond)
-
-	pushRoutePayload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "push:user:user-1")).Bytes()
-	require.NoError(t, err)
-	pushRoute, err := redisstate.UnmarshalRoute(pushRoutePayload)
-	require.NoError(t, err)
-	require.Equal(t, "published", string(pushRoute.Status))
-
-	notificationPayload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Notification(messageID)).Bytes()
-	require.NoError(t, err)
-	notificationRecord, err := redisstate.UnmarshalNotification(notificationPayload)
-	require.NoError(t, err)
-
-	emailRoutePayload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "email:user:user-1")).Bytes()
-	require.NoError(t, err)
-	emailRoute, err := redisstate.UnmarshalRoute(emailRoutePayload)
-	require.NoError(t, err)
-	require.Equal(t, "published", string(emailRoute.Status))
-
-	messages, err := redisClient.XRange(context.Background(), cfg.Streams.GatewayClientEvents, "-", "+").Result()
-	require.NoError(t, err)
-	require.Len(t, messages, 1)
-	require.Equal(t, "user-1", messages[0].Values["user_id"])
-	require.Equal(t, "game.turn.ready", messages[0].Values["event_type"])
-	require.Equal(t, messageID+"/push:user:user-1", messages[0].Values["event_id"])
-	require.Equal(t, "request-1", messages[0].Values["request_id"])
-	require.Equal(t, "trace-1", messages[0].Values["trace_id"])
-	require.NotContains(t, messages[0].Values, "device_session_id")
-	switch payload := messages[0].Values["payload_bytes"].(type) {
-	case string:
-		require.NotEmpty(t, payload)
-	case []byte:
-		require.NotEmpty(t, payload)
-	default:
-		require.Failf(t, "unexpected payload type", "payload_bytes has type %T", payload)
-	}
-
-	mailCommands, err := redisClient.XRange(context.Background(), cfg.Streams.MailDeliveryCommands, "-", "+").Result()
-	require.NoError(t, err)
-	require.Len(t, mailCommands, 1)
-	require.Equal(t, messageID+"/email:user:user-1", mailCommands[0].Values["delivery_id"])
-	require.Equal(t, "notification", mailCommands[0].Values["source"])
-	require.Equal(t, "template", mailCommands[0].Values["payload_mode"])
-	require.Equal(t, "notification:"+messageID+"/email:user:user-1", mailCommands[0].Values["idempotency_key"])
-	require.Equal(t, strconv.FormatInt(notificationRecord.AcceptedAt.UnixMilli(), 10), mailCommands[0].Values["requested_at_ms"])
-	require.Equal(t, "request-1", mailCommands[0].Values["request_id"])
-	require.Equal(t, "trace-1", mailCommands[0].Values["trace_id"])
-	require.JSONEq(t,
-		`{"to":["pilot@example.com"],"cc":[],"bcc":[],"reply_to":[],"template_id":"game.turn.ready","locale":"en","variables":{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54},"attachments":[]}`,
-		mailCommands[0].Values["payload_json"].(string),
-	)
-
-	cancel()
-	waitForRunResult(t, runErrCh, cfg.ShutdownTimeout+2*time.Second)
-}
-
-func TestNewRuntimePublishesAdminEmailRouteOnlyToMailService(t *testing.T) {
-	t.Parallel()
-
-	redisServer := miniredis.RunT(t)
-	redisClient := redis.NewClient(&redis.Options{
-		Addr:            redisServer.Addr(),
-		Protocol:        2,
-		DisableIdentity: true,
-	})
-	t.Cleanup(func() {
-		assert.NoError(t, redisClient.Close())
-	})
-	userService := newUserLookupServer(t, func(http.ResponseWriter, *http.Request) {})
-	defer userService.Close()
-
-	cfg := config.DefaultConfig()
-	cfg.Redis.Addr = redisServer.Addr()
-	cfg.UserService.BaseURL = userService.URL
-	cfg.AdminRouting.LobbyApplicationSubmitted = []string{"owner@example.com"}
-	cfg.InternalHTTP.Addr = mustFreeAddr(t)
-	cfg.ShutdownTimeout = 10 * time.Second
-	cfg.IntentsReadBlockTimeout = 25 * time.Millisecond
-	cfg.Telemetry.TracesExporter = "none"
-	cfg.Telemetry.MetricsExporter = "none"
-
-	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
-	require.NoError(t, err)
-	defer func() {
-		require.NoError(t, runtime.Close())
-	}()
-
-	runCtx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	runErrCh := make(chan error, 1)
-	go func() {
-		runErrCh <- runtime.Run(runCtx)
-	}()
-
-	client := newTestHTTPClient(t)
-	waitForRuntimeReady(t, client, cfg.InternalHTTP.Addr)
-
-	messageID, err := redisClient.XAdd(context.Background(), &redis.XAddArgs{
-		Stream: cfg.Streams.Intents,
-		Values: map[string]any{
-			"notification_type": "lobby.application.submitted",
-			"producer":          "game_lobby",
-			"audience_kind":     "admin_email",
-			"idempotency_key":   "game-123:application-submitted:user-42",
-			"occurred_at_ms":    "1775121700000",
-			"payload_json":      `{"applicant_name":"Nova Pilot","applicant_user_id":"user-42","game_id":"game-123","game_name":"Nebula Clash"}`,
-		},
-	}).Result()
-	require.NoError(t, err)
-
-	require.Eventually(t, func() bool {
-		payload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "email:email:owner@example.com")).Bytes()
-		if err != nil {
-			return false
-		}
-		route, err := redisstate.UnmarshalRoute(payload)
-		if err != nil {
-			return false
-		}
-
-		return route.Status == "published" && route.AttemptCount == 1
-	}, 2*time.Second, 10*time.Millisecond)
-
-	pushRoutePayload, err := redisClient.Get(context.Background(), redisstate.Keyspace{}.Route(messageID, "push:email:owner@example.com")).Bytes()
-	require.NoError(t, err)
-	pushRoute, err := redisstate.UnmarshalRoute(pushRoutePayload)
-	require.NoError(t, err)
-	require.Equal(t, "skipped", string(pushRoute.Status))
-
-	mailCommands, err := redisClient.XRange(context.Background(), cfg.Streams.MailDeliveryCommands, "-", "+").Result()
-	require.NoError(t, err)
-	require.Len(t, mailCommands, 1)
-	require.Equal(t, messageID+"/email:email:owner@example.com", mailCommands[0].Values["delivery_id"])
-	require.JSONEq(t,
-		`{"to":["owner@example.com"],"cc":[],"bcc":[],"reply_to":[],"template_id":"lobby.application.submitted","locale":"en","variables":{"applicant_name":"Nova Pilot","applicant_user_id":"user-42","game_id":"game-123","game_name":"Nebula Clash"},"attachments":[]}`,
-		mailCommands[0].Values["payload_json"].(string),
-	)
-
-	gatewayMessages, err := redisClient.XRange(context.Background(), cfg.Streams.GatewayClientEvents, "-", "+").Result()
-	require.NoError(t, err)
-	require.Empty(t, gatewayMessages)
-
-	cancel()
-	waitForRunResult(t, runErrCh, cfg.ShutdownTimeout+2*time.Second)
-}
-
-func TestNewRuntimeUsesConfiguredUserServiceTimeout(t *testing.T) {
-	t.Parallel()
-
-	redisServer := miniredis.RunT(t)
-	redisClient := redis.NewClient(&redis.Options{
-		Addr:            redisServer.Addr(),
-		Protocol:        2,
-		DisableIdentity: true,
-	})
-	t.Cleanup(func() {
-		assert.NoError(t, redisClient.Close())
-	})
-	userService := newUserLookupServer(t, func(_ http.ResponseWriter, request *http.Request) {
-		<-request.Context().Done()
-	})
-	defer userService.Close()
-
-	cfg := config.DefaultConfig()
-	cfg.Redis.Addr = redisServer.Addr()
-	cfg.UserService.BaseURL = userService.URL
-	cfg.UserService.Timeout = 20 * time.Millisecond
-	cfg.InternalHTTP.Addr = mustFreeAddr(t)
-	cfg.ShutdownTimeout = 10 * time.Second
-	cfg.IntentsReadBlockTimeout = 25 * time.Millisecond
-	cfg.Telemetry.TracesExporter = "none"
-	cfg.Telemetry.MetricsExporter = "none"
-
-	runtime, err := NewRuntime(context.Background(), cfg, testLogger())
-	require.NoError(t, err)
-	defer func() {
-		require.NoError(t, runtime.Close())
-	}()
-
-	runCtx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	runErrCh := make(chan error, 1)
-	go func() {
-		runErrCh <- runtime.Run(runCtx)
-	}()
-
-	client := newTestHTTPClient(t)
-	waitForRuntimeReady(t, client, cfg.InternalHTTP.Addr)
-
-	messageID, err := redisClient.XAdd(context.Background(), &redis.XAddArgs{
-		Stream: cfg.Streams.Intents,
-		Values: map[string]any{
-			"notification_type":       "game.turn.ready",
-			"producer":                "game_master",
-			"audience_kind":           "user",
-			"recipient_user_ids_json": `["user-1"]`,
-			"idempotency_key":         "game-123:turn-ready",
-			"occurred_at_ms":          "1775121700000",
-			"payload_json":            `{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54}`,
-		},
-	}).Result()
-	require.NoError(t, err)
-
-	var runErr error
-	require.Eventually(t, func() bool {
-		select {
-		case runErr = <-runErrCh:
-			return true
-		default:
-			return false
-		}
-	}, time.Second, 10*time.Millisecond)
-
-	require.Error(t, runErr)
-	require.ErrorContains(t, runErr, "context deadline exceeded")
-
-	offsetStore, err := redisstate.NewStreamOffsetStore(redisClient)
-	require.NoError(t, err)
-	offset, found, err := offsetStore.Load(context.Background(), cfg.Streams.Intents)
-	require.NoError(t, err)
-	require.False(t, found)
-	require.Empty(t, offset)
-
-	_, err = redisClient.Get(context.Background(), redisstate.Keyspace{}.Notification(messageID)).Bytes()
-	require.Error(t, err)
-}
-
-func testLogger() *slog.Logger {
-	return slog.New(slog.NewTextHandler(io.Discard, nil))
-}
-
-func newTestHTTPClient(t *testing.T) *http.Client {
-	t.Helper()
-
-	transport := &http.Transport{DisableKeepAlives: true}
-	t.Cleanup(transport.CloseIdleConnections)
-
-	return &http.Client{
-		Timeout:   500 * time.Millisecond,
-		Transport: transport,
-	}
-}
-
-func waitForRuntimeReady(t *testing.T, client *http.Client, addr string) {
-	t.Helper()
-
-	require.Eventually(t, func() bool {
-		request, err := http.NewRequest(http.MethodGet, "http://"+addr+"/readyz", nil)
-		if err != nil {
-			return false
-		}
-
-		response, err := client.Do(request)
-		if err != nil {
-			return false
-		}
-		defer response.Body.Close()
-		_, _ = io.Copy(io.Discard, response.Body)
-
-		return response.StatusCode == http.StatusOK
-	}, 5*time.Second, 25*time.Millisecond, "notification runtime did not become reachable")
-}
-
-func waitForRunResult(t *testing.T, runErrCh <-chan error, waitTimeout time.Duration) {
-	t.Helper()
-
-	var err error
-	require.Eventually(t, func() bool {
-		select {
-		case err = <-runErrCh:
-			return true
-		default:
-			return false
-		}
-	}, waitTimeout, 10*time.Millisecond, "notification runtime did not stop")
-	require.NoError(t, err)
-}
-
-func assertHTTPStatus(t *testing.T, client *http.Client, target string, want int) {
-	t.Helper()
-
-	request, err := http.NewRequest(http.MethodGet, target, nil)
-	require.NoError(t, err)
-
-	response, err := client.Do(request)
-	require.NoError(t, err)
-	defer response.Body.Close()
-	_, _ = io.Copy(io.Discard, response.Body)
-
-	require.Equal(t, want, response.StatusCode)
-}
-
-func mustFreeAddr(t *testing.T) string {
-	t.Helper()
-
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	require.NoError(t, err)
-	defer func() {
-		assert.NoError(t, listener.Close())
-	}()
-
-	return listener.Addr().String()
-}
-
-func newUserLookupServer(t *testing.T, handler func(http.ResponseWriter, *http.Request)) *httptest.Server {
-	t.Helper()
-
-	return httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
-		if request.Method != http.MethodGet {
-			http.NotFound(writer, request)
-			return
-		}
-		if request.URL.Path != "/api/v1/internal/users/user-1" {
-			writeJSON(t, writer, http.StatusNotFound, map[string]any{
-				"error": map[string]any{
-					"code":    "subject_not_found",
-					"message": "subject not found",
-				},
-			})
-			return
-		}
-
-		handler(writer, request)
-	}))
-}
-
-func writeJSON(t *testing.T, writer http.ResponseWriter, statusCode int, payload any) {
-	t.Helper()
-
-	body, err := json.Marshal(payload)
-	require.NoError(t, err)
-
-	writer.Header().Set("Content-Type", "application/json")
-	writer.WriteHeader(statusCode)
-	_, err = writer.Write(body)
-	require.NoError(t, err)
-}