package worker import ( "context" "io" "log/slog" "sync" "testing" "time" redisstate "galaxy/notification/internal/adapters/redisstate" "galaxy/notification/internal/api/intentstream" "galaxy/notification/internal/service/acceptintent" "github.com/alicebob/miniredis/v2" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestPushPublisherPublishesDuePushRouteAndLeavesEmailRoutePending(t *testing.T) { t.Parallel() fixture := newPushPublisherFixture(t) require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validPushAcceptanceInput(fixture.now))) running := runPushPublisher(t, fixture.publisher) defer running.stop(t) require.Eventually(t, func() bool { route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1") return err == nil && found && route.Status == acceptintent.RouteStatusPublished }, time.Second, 10*time.Millisecond) emailRoute, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "email:user:user-1") require.NoError(t, err) require.True(t, found) require.Equal(t, acceptintent.RouteStatusPending, emailRoute.Status) messages, err := fixture.client.XRange(context.Background(), fixture.gatewayStream, "-", "+").Result() require.NoError(t, err) require.Len(t, messages, 1) require.Equal(t, "user-1", messages[0].Values["user_id"]) require.Equal(t, "game.turn.ready", messages[0].Values["event_type"]) require.Equal(t, "1775121700000-0/push:user:user-1", messages[0].Values["event_id"]) require.True(t, fixture.telemetry.hasRoutePublishAttempt("push", "published", "")) } func TestPushPublisherRetriesGatewayStreamPublicationFailures(t *testing.T) { t.Parallel() fixture := newPushPublisherFixture(t) require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validPushAcceptanceInput(fixture.now))) require.NoError(t, fixture.client.Set(context.Background(), fixture.gatewayStream, "wrong-type", 0).Err()) running := runPushPublisher(t, fixture.publisher) defer running.stop(t) require.Eventually(t, func() bool { route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1") return err == nil && found && route.Status == acceptintent.RouteStatusFailed && route.AttemptCount == 1 }, time.Second, 10*time.Millisecond) require.True(t, fixture.telemetry.hasRoutePublishAttempt("push", "retry", pushFailureClassificationGatewayStreamWrite)) require.True(t, fixture.telemetry.hasRouteRetry("push")) require.NoError(t, fixture.client.Del(context.Background(), fixture.gatewayStream).Err()) require.Eventually(t, func() bool { route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1") return err == nil && found && route.Status == acceptintent.RouteStatusPublished && route.AttemptCount == 2 }, 2*time.Second, 10*time.Millisecond) messages, err := fixture.client.XRange(context.Background(), fixture.gatewayStream, "-", "+").Result() require.NoError(t, err) require.Len(t, messages, 1) require.True(t, fixture.telemetry.hasRoutePublishAttempt("push", "published", "")) } func TestPushPublisherDeadLettersExhaustedRoute(t *testing.T) { t.Parallel() fixture := newPushPublisherFixture(t) input := validPushAcceptanceInput(fixture.now) for index := range input.Routes { if input.Routes[index].RouteID == "push:user:user-1" { input.Routes[index].AttemptCount = 2 input.Routes[index].MaxAttempts = 3 } } require.NoError(t, fixture.store.CreateAcceptance(context.Background(), input)) require.NoError(t, fixture.client.Set(context.Background(), fixture.gatewayStream, "wrong-type", 0).Err()) running := runPushPublisher(t, fixture.publisher) defer running.stop(t) require.Eventually(t, func() bool { route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1") return err == nil && found && route.Status == acceptintent.RouteStatusDeadLetter && route.AttemptCount == 3 }, time.Second, 10*time.Millisecond) deadLetterPayload, err := fixture.client.Get(context.Background(), redisstate.Keyspace{}.DeadLetter("1775121700000-0", "push:user:user-1")).Bytes() require.NoError(t, err) deadLetter, err := redisstate.UnmarshalDeadLetter(deadLetterPayload) require.NoError(t, err) require.Equal(t, pushFailureClassificationGatewayStreamWrite, deadLetter.FailureClassification) require.True(t, fixture.telemetry.hasRoutePublishAttempt("push", "dead_letter", pushFailureClassificationGatewayStreamWrite)) require.True(t, fixture.telemetry.hasRouteDeadLetter("push", pushFailureClassificationGatewayStreamWrite)) } func TestPushPublisherLeasePreventsDuplicatePublicationAcrossReplicas(t *testing.T) { t.Parallel() fixture := newPushPublisherFixture(t) require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validPushAcceptanceInput(fixture.now))) otherPublisher, err := NewPushPublisher(PushPublisherConfig{ Store: fixture.store, GatewayStream: fixture.gatewayStream, GatewayStreamMaxLen: 1024, RouteLeaseTTL: 200 * time.Millisecond, RouteBackoffMin: 20 * time.Millisecond, RouteBackoffMax: 20 * time.Millisecond, PollInterval: 10 * time.Millisecond, BatchSize: 16, Clock: newSteppingClock(fixture.now, time.Millisecond), }, testWorkerLogger()) require.NoError(t, err) first := runPushPublisher(t, fixture.publisher) defer first.stop(t) second := runPushPublisher(t, otherPublisher) defer second.stop(t) require.Eventually(t, func() bool { route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1") return err == nil && found && route.Status == acceptintent.RouteStatusPublished }, time.Second, 10*time.Millisecond) messages, err := fixture.client.XRange(context.Background(), fixture.gatewayStream, "-", "+").Result() require.NoError(t, err) require.Len(t, messages, 1) } type pushPublisherFixture struct { client *redis.Client store *redisstate.AcceptanceStore publisher *PushPublisher gatewayStream string now time.Time clock *steppingClock telemetry *recordingWorkerTelemetry } func newPushPublisherFixture(t *testing.T) pushPublisherFixture { t.Helper() server := miniredis.RunT(t) client := redis.NewClient(&redis.Options{ Addr: server.Addr(), Protocol: 2, DisableIdentity: true, }) t.Cleanup(func() { assert.NoError(t, client.Close()) }) store, err := redisstate.NewAcceptanceStore(client, redisstate.AcceptanceConfig{ RecordTTL: 24 * time.Hour, DeadLetterTTL: 72 * time.Hour, IdempotencyTTL: 7 * 24 * time.Hour, }) require.NoError(t, err) now := time.UnixMilli(1775121700000).UTC() clock := newSteppingClock(now, time.Millisecond) telemetry := &recordingWorkerTelemetry{} publisher, err := NewPushPublisher(PushPublisherConfig{ Store: store, GatewayStream: "gateway:client-events", GatewayStreamMaxLen: 1024, RouteLeaseTTL: 200 * time.Millisecond, RouteBackoffMin: 20 * time.Millisecond, RouteBackoffMax: 20 * time.Millisecond, PollInterval: 10 * time.Millisecond, BatchSize: 16, Telemetry: telemetry, Clock: clock, }, testWorkerLogger()) require.NoError(t, err) return pushPublisherFixture{ client: client, store: store, publisher: publisher, gatewayStream: "gateway:client-events", now: now, clock: clock, telemetry: telemetry, } } func validPushAcceptanceInput(now time.Time) acceptintent.CreateAcceptanceInput { return acceptintent.CreateAcceptanceInput{ Notification: acceptintent.NotificationRecord{ NotificationID: "1775121700000-0", NotificationType: intentstream.NotificationTypeGameTurnReady, Producer: intentstream.ProducerGameMaster, AudienceKind: intentstream.AudienceKindUser, RecipientUserIDs: []string{"user-1"}, PayloadJSON: `{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54}`, IdempotencyKey: "game-123:turn-54", RequestFingerprint: "sha256:deadbeef", RequestID: "request-1", TraceID: "trace-1", OccurredAt: now, AcceptedAt: now, UpdatedAt: now, }, Routes: []acceptintent.NotificationRoute{ { NotificationID: "1775121700000-0", RouteID: "push:user:user-1", Channel: intentstream.ChannelPush, RecipientRef: "user:user-1", Status: acceptintent.RouteStatusPending, AttemptCount: 0, MaxAttempts: 3, NextAttemptAt: now, ResolvedEmail: "pilot@example.com", ResolvedLocale: "en", CreatedAt: now, UpdatedAt: now, }, { NotificationID: "1775121700000-0", RouteID: "email:user:user-1", Channel: intentstream.ChannelEmail, RecipientRef: "user:user-1", Status: acceptintent.RouteStatusPending, AttemptCount: 0, MaxAttempts: 7, NextAttemptAt: now, ResolvedEmail: "pilot@example.com", ResolvedLocale: "en", CreatedAt: now, UpdatedAt: now, }, }, Idempotency: acceptintent.IdempotencyRecord{ Producer: intentstream.ProducerGameMaster, IdempotencyKey: "game-123:turn-54", NotificationID: "1775121700000-0", RequestFingerprint: "sha256:deadbeef", CreatedAt: now, ExpiresAt: now.Add(7 * 24 * time.Hour), }, } } type runningPushPublisher struct { cancel context.CancelFunc resultCh chan error } func runPushPublisher(t *testing.T, publisher *PushPublisher) runningPushPublisher { t.Helper() ctx, cancel := context.WithCancel(context.Background()) resultCh := make(chan error, 1) go func() { resultCh <- publisher.Run(ctx) }() return runningPushPublisher{ cancel: cancel, resultCh: resultCh, } } func (r runningPushPublisher) stop(t *testing.T) { t.Helper() r.cancel() select { case err := <-r.resultCh: require.ErrorIs(t, err, context.Canceled) case <-time.After(time.Second): require.FailNow(t, "push publisher did not stop") } } type steppingClock struct { mu sync.Mutex current time.Time step time.Duration } func newSteppingClock(start time.Time, step time.Duration) *steppingClock { return &steppingClock{ current: start.UTC().Truncate(time.Millisecond), step: step, } } func (clock *steppingClock) Now() time.Time { clock.mu.Lock() defer clock.mu.Unlock() now := clock.current clock.current = clock.current.Add(clock.step).UTC().Truncate(time.Millisecond) return now } func testWorkerLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) }