package redisstate import ( "context" "testing" "time" "galaxy/notification/internal/api/intentstream" "galaxy/notification/internal/service/acceptintent" "github.com/alicebob/miniredis/v2" "github.com/stretchr/testify/require" ) func TestAcceptanceStoreListDueRoutesLoadsScheduledMembers(t *testing.T) { t.Parallel() server := miniredis.RunT(t) client := newTestRedisClient(t, server) store, err := NewAcceptanceStore(client, AcceptanceConfig{ RecordTTL: 24 * time.Hour, DeadLetterTTL: 72 * time.Hour, IdempotencyTTL: 7 * 24 * time.Hour, }) require.NoError(t, err) now := time.UnixMilli(1775121700000).UTC() require.NoError(t, store.CreateAcceptance(context.Background(), validUserAcceptanceInput(now, 0))) routes, err := store.ListDueRoutes(context.Background(), now, 10) require.NoError(t, err) require.Len(t, routes, 2) require.ElementsMatch(t, []string{"push:user:user-1", "email:user:user-1"}, []string{routes[0].RouteID, routes[1].RouteID}) for _, route := range routes { require.NoError(t, route.Validate()) } } func TestAcceptanceStoreReadRouteScheduleSnapshot(t *testing.T) { t.Parallel() server := miniredis.RunT(t) client := newTestRedisClient(t, server) store, err := NewAcceptanceStore(client, AcceptanceConfig{ RecordTTL: 24 * time.Hour, DeadLetterTTL: 72 * time.Hour, IdempotencyTTL: 7 * 24 * time.Hour, }) require.NoError(t, err) now := time.UnixMilli(1775121700000).UTC() require.NoError(t, store.CreateAcceptance(context.Background(), validUserAcceptanceInput(now, 0))) snapshot, err := store.ReadRouteScheduleSnapshot(context.Background()) require.NoError(t, err) require.Equal(t, int64(2), snapshot.Depth) require.NotNil(t, snapshot.OldestScheduledFor) require.Equal(t, now, *snapshot.OldestScheduledFor) } func TestAcceptanceStoreRouteLeaseAcquireReleaseAndExpire(t *testing.T) { t.Parallel() server := miniredis.RunT(t) client := newTestRedisClient(t, server) store, err := NewAcceptanceStore(client, AcceptanceConfig{ RecordTTL: 24 * time.Hour, DeadLetterTTL: 72 * time.Hour, IdempotencyTTL: 7 * 24 * time.Hour, }) require.NoError(t, err) acquired, err := store.TryAcquireRouteLease(context.Background(), "1775121700000-0", "push:user:user-1", "token-1", 2*time.Second) require.NoError(t, err) require.True(t, acquired) acquired, err = store.TryAcquireRouteLease(context.Background(), "1775121700000-0", "push:user:user-1", "token-2", 2*time.Second) require.NoError(t, err) require.False(t, acquired) require.NoError(t, store.ReleaseRouteLease(context.Background(), "1775121700000-0", "push:user:user-1", "token-1")) acquired, err = store.TryAcquireRouteLease(context.Background(), "1775121700000-0", "push:user:user-1", "token-3", 2*time.Second) require.NoError(t, err) require.True(t, acquired) server.FastForward(3 * time.Second) acquired, err = store.TryAcquireRouteLease(context.Background(), "1775121700000-0", "push:user:user-1", "token-4", 2*time.Second) require.NoError(t, err) require.True(t, acquired) } func TestAcceptanceStoreCompleteRoutePublishedAppendsTrimmedStreamEntryAndMarksRoutePublished(t *testing.T) { t.Parallel() server := miniredis.RunT(t) client := newTestRedisClient(t, server) store, err := NewAcceptanceStore(client, AcceptanceConfig{ RecordTTL: 24 * time.Hour, DeadLetterTTL: 72 * time.Hour, IdempotencyTTL: 7 * 24 * time.Hour, }) require.NoError(t, err) now := time.UnixMilli(1775121700000).UTC() input := validUserAcceptanceInput(now, 0) require.NoError(t, store.CreateAcceptance(context.Background(), input)) acquired, err := store.TryAcquireRouteLease(context.Background(), input.Notification.NotificationID, "push:user:user-1", "token-1", 5*time.Second) require.NoError(t, err) require.True(t, acquired) route, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1") require.NoError(t, err) require.True(t, found) publishedAt := now.Add(time.Second).UTC().Truncate(time.Millisecond) require.NoError(t, store.CompleteRoutePublished(context.Background(), CompleteRoutePublishedInput{ ExpectedRoute: route, LeaseToken: "token-1", PublishedAt: publishedAt, Stream: "gateway:client-events", StreamMaxLen: 1024, StreamValues: map[string]any{ "user_id": "user-1", "event_type": "game.turn.ready", "event_id": input.Notification.NotificationID + "/push:user:user-1", "payload_bytes": []byte("payload-1"), "request_id": "request-1", "trace_id": "trace-1", }, })) updatedRoute, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1") require.NoError(t, err) require.True(t, found) require.Equal(t, acceptintent.RouteStatusPublished, updatedRoute.Status) require.Equal(t, 1, updatedRoute.AttemptCount) require.Equal(t, publishedAt, updatedRoute.PublishedAt) scheduled, err := client.ZRange(context.Background(), Keyspace{}.RouteSchedule(), 0, -1).Result() require.NoError(t, err) require.Equal(t, []string{Keyspace{}.Route(input.Notification.NotificationID, "email:user:user-1")}, scheduled) messages, err := client.XRange(context.Background(), "gateway:client-events", "-", "+").Result() require.NoError(t, err) require.Len(t, messages, 1) require.Equal(t, "user-1", messages[0].Values["user_id"]) require.Equal(t, "game.turn.ready", messages[0].Values["event_type"]) leaseKey := Keyspace{}.RouteLease(input.Notification.NotificationID, "push:user:user-1") _, err = client.Get(context.Background(), leaseKey).Result() require.Error(t, err) } func TestAcceptanceStoreCompleteRoutePublishedAppendsUntrimmedMailCommand(t *testing.T) { t.Parallel() server := miniredis.RunT(t) client := newTestRedisClient(t, server) store, err := NewAcceptanceStore(client, AcceptanceConfig{ RecordTTL: 24 * time.Hour, DeadLetterTTL: 72 * time.Hour, IdempotencyTTL: 7 * 24 * time.Hour, }) require.NoError(t, err) now := time.UnixMilli(1775121700000).UTC() input := validUserAcceptanceInput(now, 0) require.NoError(t, store.CreateAcceptance(context.Background(), input)) acquired, err := store.TryAcquireRouteLease(context.Background(), input.Notification.NotificationID, "email:user:user-1", "token-1", 5*time.Second) require.NoError(t, err) require.True(t, acquired) route, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "email:user:user-1") require.NoError(t, err) require.True(t, found) publishedAt := now.Add(time.Second).UTC().Truncate(time.Millisecond) require.NoError(t, store.CompleteRoutePublished(context.Background(), CompleteRoutePublishedInput{ ExpectedRoute: route, LeaseToken: "token-1", PublishedAt: publishedAt, Stream: "mail:delivery_commands", StreamMaxLen: 0, StreamValues: map[string]any{ "delivery_id": input.Notification.NotificationID + "/email:user:user-1", "source": "notification", "payload_mode": "template", "idempotency_key": "notification:" + input.Notification.NotificationID + "/email:user:user-1", "requested_at_ms": "1775121700000", "payload_json": `{"to":["pilot@example.com"],"cc":[],"bcc":[],"reply_to":[],"template_id":"game.turn.ready","locale":"en","variables":{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54},"attachments":[]}`, }, })) updatedRoute, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "email:user:user-1") require.NoError(t, err) require.True(t, found) require.Equal(t, acceptintent.RouteStatusPublished, updatedRoute.Status) require.Equal(t, 1, updatedRoute.AttemptCount) require.Equal(t, publishedAt, updatedRoute.PublishedAt) messages, err := client.XRange(context.Background(), "mail:delivery_commands", "-", "+").Result() require.NoError(t, err) require.Len(t, messages, 1) require.Equal(t, "notification", messages[0].Values["source"]) require.Equal(t, "template", messages[0].Values["payload_mode"]) require.Equal(t, "1775121700000-0/email:user:user-1", messages[0].Values["delivery_id"]) } func TestAcceptanceStoreCompleteRouteFailedReschedulesRoute(t *testing.T) { t.Parallel() server := miniredis.RunT(t) client := newTestRedisClient(t, server) store, err := NewAcceptanceStore(client, AcceptanceConfig{ RecordTTL: 24 * time.Hour, DeadLetterTTL: 72 * time.Hour, IdempotencyTTL: 7 * 24 * time.Hour, }) require.NoError(t, err) now := time.UnixMilli(1775121700000).UTC() input := validUserAcceptanceInput(now, 0) require.NoError(t, store.CreateAcceptance(context.Background(), input)) acquired, err := store.TryAcquireRouteLease(context.Background(), input.Notification.NotificationID, "push:user:user-1", "token-1", 5*time.Second) require.NoError(t, err) require.True(t, acquired) route, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1") require.NoError(t, err) require.True(t, found) failedAt := now.Add(time.Second).UTC().Truncate(time.Millisecond) nextAttemptAt := failedAt.Add(2 * time.Second).UTC().Truncate(time.Millisecond) require.NoError(t, store.CompleteRouteFailed(context.Background(), CompleteRouteFailedInput{ ExpectedRoute: route, LeaseToken: "token-1", FailedAt: failedAt, NextAttemptAt: nextAttemptAt, FailureClassification: "gateway_stream_publish_failed", FailureMessage: "temporary outage", })) updatedRoute, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1") require.NoError(t, err) require.True(t, found) require.Equal(t, acceptintent.RouteStatusFailed, updatedRoute.Status) require.Equal(t, 1, updatedRoute.AttemptCount) require.Equal(t, nextAttemptAt, updatedRoute.NextAttemptAt) require.Equal(t, "gateway_stream_publish_failed", updatedRoute.LastErrorClassification) scheduled, err := client.ZRangeWithScores(context.Background(), Keyspace{}.RouteSchedule(), 0, -1).Result() require.NoError(t, err) require.Len(t, scheduled, 2) require.Contains(t, []string{ scheduled[0].Member.(string), scheduled[1].Member.(string), }, Keyspace{}.Route(input.Notification.NotificationID, "push:user:user-1")) } func TestAcceptanceStoreCompleteRouteDeadLetterStoresTerminalFailure(t *testing.T) { t.Parallel() server := miniredis.RunT(t) client := newTestRedisClient(t, server) store, err := NewAcceptanceStore(client, AcceptanceConfig{ RecordTTL: 24 * time.Hour, DeadLetterTTL: 72 * time.Hour, IdempotencyTTL: 7 * 24 * time.Hour, }) require.NoError(t, err) now := time.UnixMilli(1775121700000).UTC() input := validUserAcceptanceInput(now, 2) require.NoError(t, store.CreateAcceptance(context.Background(), input)) acquired, err := store.TryAcquireRouteLease(context.Background(), input.Notification.NotificationID, "push:user:user-1", "token-1", 5*time.Second) require.NoError(t, err) require.True(t, acquired) route, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1") require.NoError(t, err) require.True(t, found) deadLetteredAt := now.Add(time.Second).UTC().Truncate(time.Millisecond) require.NoError(t, store.CompleteRouteDeadLetter(context.Background(), CompleteRouteDeadLetterInput{ ExpectedRoute: route, LeaseToken: "token-1", DeadLetteredAt: deadLetteredAt, FailureClassification: "payload_encoding_failed", FailureMessage: "payload is invalid", })) updatedRoute, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1") require.NoError(t, err) require.True(t, found) require.Equal(t, acceptintent.RouteStatusDeadLetter, updatedRoute.Status) require.Equal(t, 3, updatedRoute.AttemptCount) require.Equal(t, deadLetteredAt, updatedRoute.DeadLetteredAt) payload, err := client.Get(context.Background(), Keyspace{}.DeadLetter(input.Notification.NotificationID, "push:user:user-1")).Bytes() require.NoError(t, err) entry, err := UnmarshalDeadLetter(payload) require.NoError(t, err) require.Equal(t, "payload_encoding_failed", entry.FailureClassification) require.Equal(t, 3, entry.FinalAttemptCount) scheduled, err := client.ZRange(context.Background(), Keyspace{}.RouteSchedule(), 0, -1).Result() require.NoError(t, err) require.Equal(t, []string{Keyspace{}.Route(input.Notification.NotificationID, "email:user:user-1")}, scheduled) } func TestAcceptanceStoreDeadLetterIsIsolatedByChannelAndRecipient(t *testing.T) { t.Parallel() server := miniredis.RunT(t) client := newTestRedisClient(t, server) store, err := NewAcceptanceStore(client, AcceptanceConfig{ RecordTTL: 24 * time.Hour, DeadLetterTTL: 72 * time.Hour, IdempotencyTTL: 7 * 24 * time.Hour, }) require.NoError(t, err) now := time.UnixMilli(1775121700000).UTC() input := validUserAcceptanceInput(now, 2) input.Notification.RecipientUserIDs = []string{"user-1", "user-2"} input.Routes = append(input.Routes, acceptintent.NotificationRoute{ NotificationID: input.Notification.NotificationID, RouteID: "push:user:user-2", Channel: intentstream.ChannelPush, RecipientRef: "user:user-2", Status: acceptintent.RouteStatusPending, AttemptCount: 0, MaxAttempts: 3, NextAttemptAt: now, ResolvedEmail: "second@example.com", ResolvedLocale: "en", CreatedAt: now, UpdatedAt: now, }, acceptintent.NotificationRoute{ NotificationID: input.Notification.NotificationID, RouteID: "email:user:user-2", Channel: intentstream.ChannelEmail, RecipientRef: "user:user-2", Status: acceptintent.RouteStatusPending, AttemptCount: 0, MaxAttempts: 7, NextAttemptAt: now, ResolvedEmail: "second@example.com", ResolvedLocale: "en", CreatedAt: now, UpdatedAt: now, }, ) require.NoError(t, store.CreateAcceptance(context.Background(), input)) acquired, err := store.TryAcquireRouteLease(context.Background(), input.Notification.NotificationID, "push:user:user-1", "token-1", 5*time.Second) require.NoError(t, err) require.True(t, acquired) route, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1") require.NoError(t, err) require.True(t, found) deadLetteredAt := now.Add(time.Second).UTC().Truncate(time.Millisecond) require.NoError(t, store.CompleteRouteDeadLetter(context.Background(), CompleteRouteDeadLetterInput{ ExpectedRoute: route, LeaseToken: "token-1", DeadLetteredAt: deadLetteredAt, FailureClassification: "gateway_stream_publish_failed", FailureMessage: "gateway unavailable", })) deadLetterRoute, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1") require.NoError(t, err) require.True(t, found) require.Equal(t, acceptintent.RouteStatusDeadLetter, deadLetterRoute.Status) for _, routeID := range []string{"email:user:user-1", "push:user:user-2", "email:user:user-2"} { route, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, routeID) require.NoError(t, err) require.True(t, found, "route %s should remain stored", routeID) require.Equal(t, acceptintent.RouteStatusPending, route.Status, "route %s should remain pending", routeID) } scheduled, err := client.ZRange(context.Background(), Keyspace{}.RouteSchedule(), 0, -1).Result() require.NoError(t, err) require.ElementsMatch(t, []string{ Keyspace{}.Route(input.Notification.NotificationID, "email:user:user-1"), Keyspace{}.Route(input.Notification.NotificationID, "push:user:user-2"), Keyspace{}.Route(input.Notification.NotificationID, "email:user:user-2"), }, scheduled) } func validUserAcceptanceInput(now time.Time, pushAttemptCount int) acceptintent.CreateAcceptanceInput { return acceptintent.CreateAcceptanceInput{ Notification: acceptintent.NotificationRecord{ NotificationID: "1775121700000-0", NotificationType: intentstream.NotificationTypeGameTurnReady, Producer: intentstream.ProducerGameMaster, AudienceKind: intentstream.AudienceKindUser, RecipientUserIDs: []string{"user-1"}, PayloadJSON: `{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54}`, IdempotencyKey: "game-123:turn-54", RequestFingerprint: "sha256:deadbeef", RequestID: "request-1", TraceID: "trace-1", OccurredAt: now, AcceptedAt: now, UpdatedAt: now, }, Routes: []acceptintent.NotificationRoute{ { NotificationID: "1775121700000-0", RouteID: "push:user:user-1", Channel: intentstream.ChannelPush, RecipientRef: "user:user-1", Status: acceptintent.RouteStatusPending, AttemptCount: pushAttemptCount, MaxAttempts: 3, NextAttemptAt: now, ResolvedEmail: "pilot@example.com", ResolvedLocale: "en", CreatedAt: now, UpdatedAt: now, }, { NotificationID: "1775121700000-0", RouteID: "email:user:user-1", Channel: intentstream.ChannelEmail, RecipientRef: "user:user-1", Status: acceptintent.RouteStatusPending, AttemptCount: 0, MaxAttempts: 7, NextAttemptAt: now, ResolvedEmail: "pilot@example.com", ResolvedLocale: "en", CreatedAt: now, UpdatedAt: now, }, }, Idempotency: acceptintent.IdempotencyRecord{ Producer: intentstream.ProducerGameMaster, IdempotencyKey: "game-123:turn-54", NotificationID: "1775121700000-0", RequestFingerprint: "sha256:deadbeef", CreatedAt: now, ExpiresAt: now.Add(7 * 24 * time.Hour), }, } }