galaxy-game/notification/internal/worker/push_publisher_test.go

package worker

import (
	"context"
	"io"
	"log/slog"
	"sync"
	"testing"
	"time"

	redisstate "galaxy/notification/internal/adapters/redisstate"
	"galaxy/notification/internal/api/intentstream"
	"galaxy/notification/internal/service/acceptintent"

	"github.com/alicebob/miniredis/v2"
	"github.com/redis/go-redis/v9"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestPushPublisherPublishesDuePushRouteAndLeavesEmailRoutePending(t *testing.T) {
	t.Parallel()

	fixture := newPushPublisherFixture(t)
	require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validPushAcceptanceInput(fixture.now)))

	running := runPushPublisher(t, fixture.publisher)
	defer running.stop(t)

	require.Eventually(t, func() bool {
		route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1")
		return err == nil && found && route.Status == acceptintent.RouteStatusPublished
	}, time.Second, 10*time.Millisecond)

	emailRoute, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "email:user:user-1")
	require.NoError(t, err)
	require.True(t, found)
	require.Equal(t, acceptintent.RouteStatusPending, emailRoute.Status)

	messages, err := fixture.client.XRange(context.Background(), fixture.gatewayStream, "-", "+").Result()
	require.NoError(t, err)
	require.Len(t, messages, 1)
	require.Equal(t, "user-1", messages[0].Values["user_id"])
	require.Equal(t, "game.turn.ready", messages[0].Values["event_type"])
	require.Equal(t, "1775121700000-0/push:user:user-1", messages[0].Values["event_id"])
	require.True(t, fixture.telemetry.hasRoutePublishAttempt("push", "published", ""))
}

func TestPushPublisherRetriesGatewayStreamPublicationFailures(t *testing.T) {
	t.Parallel()

	fixture := newPushPublisherFixture(t)
	require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validPushAcceptanceInput(fixture.now)))
	require.NoError(t, fixture.client.Set(context.Background(), fixture.gatewayStream, "wrong-type", 0).Err())

	running := runPushPublisher(t, fixture.publisher)
	defer running.stop(t)

	require.Eventually(t, func() bool {
		route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1")
		return err == nil && found && route.Status == acceptintent.RouteStatusFailed && route.AttemptCount == 1
	}, time.Second, 10*time.Millisecond)
	require.True(t, fixture.telemetry.hasRoutePublishAttempt("push", "retry", pushFailureClassificationGatewayStreamWrite))
	require.True(t, fixture.telemetry.hasRouteRetry("push"))

	require.NoError(t, fixture.client.Del(context.Background(), fixture.gatewayStream).Err())

	require.Eventually(t, func() bool {
		route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1")
		return err == nil && found && route.Status == acceptintent.RouteStatusPublished && route.AttemptCount == 2
	}, 2*time.Second, 10*time.Millisecond)

	messages, err := fixture.client.XRange(context.Background(), fixture.gatewayStream, "-", "+").Result()
	require.NoError(t, err)
	require.Len(t, messages, 1)
	require.True(t, fixture.telemetry.hasRoutePublishAttempt("push", "published", ""))
}

func TestPushPublisherDeadLettersExhaustedRoute(t *testing.T) {
	t.Parallel()

	fixture := newPushPublisherFixture(t)
	input := validPushAcceptanceInput(fixture.now)
	for index := range input.Routes {
		if input.Routes[index].RouteID == "push:user:user-1" {
			input.Routes[index].AttemptCount = 2
			input.Routes[index].MaxAttempts = 3
		}
	}
	require.NoError(t, fixture.store.CreateAcceptance(context.Background(), input))
	require.NoError(t, fixture.client.Set(context.Background(), fixture.gatewayStream, "wrong-type", 0).Err())

	running := runPushPublisher(t, fixture.publisher)
	defer running.stop(t)

	require.Eventually(t, func() bool {
		route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1")
		return err == nil && found && route.Status == acceptintent.RouteStatusDeadLetter && route.AttemptCount == 3
	}, time.Second, 10*time.Millisecond)

	deadLetterPayload, err := fixture.client.Get(context.Background(), redisstate.Keyspace{}.DeadLetter("1775121700000-0", "push:user:user-1")).Bytes()
	require.NoError(t, err)
	deadLetter, err := redisstate.UnmarshalDeadLetter(deadLetterPayload)
	require.NoError(t, err)
	require.Equal(t, pushFailureClassificationGatewayStreamWrite, deadLetter.FailureClassification)
	require.True(t, fixture.telemetry.hasRoutePublishAttempt("push", "dead_letter", pushFailureClassificationGatewayStreamWrite))
	require.True(t, fixture.telemetry.hasRouteDeadLetter("push", pushFailureClassificationGatewayStreamWrite))
}

func TestPushPublisherLeasePreventsDuplicatePublicationAcrossReplicas(t *testing.T) {
	t.Parallel()

	fixture := newPushPublisherFixture(t)
	require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validPushAcceptanceInput(fixture.now)))

	otherPublisher, err := NewPushPublisher(PushPublisherConfig{
		Store:               fixture.store,
		GatewayStream:       fixture.gatewayStream,
		GatewayStreamMaxLen: 1024,
		RouteLeaseTTL:       200 * time.Millisecond,
		RouteBackoffMin:     20 * time.Millisecond,
		RouteBackoffMax:     20 * time.Millisecond,
		PollInterval:        10 * time.Millisecond,
		BatchSize:           16,
		Clock:               newSteppingClock(fixture.now, time.Millisecond),
	}, testWorkerLogger())
	require.NoError(t, err)

	first := runPushPublisher(t, fixture.publisher)
	defer first.stop(t)
	second := runPushPublisher(t, otherPublisher)
	defer second.stop(t)

	require.Eventually(t, func() bool {
		route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1")
		return err == nil && found && route.Status == acceptintent.RouteStatusPublished
	}, time.Second, 10*time.Millisecond)

	messages, err := fixture.client.XRange(context.Background(), fixture.gatewayStream, "-", "+").Result()
	require.NoError(t, err)
	require.Len(t, messages, 1)
}

type pushPublisherFixture struct {
	client        *redis.Client
	store         *redisstate.AcceptanceStore
	publisher     *PushPublisher
	gatewayStream string
	now           time.Time
	clock         *steppingClock
	telemetry     *recordingWorkerTelemetry
}

func newPushPublisherFixture(t *testing.T) pushPublisherFixture {
	t.Helper()

	server := miniredis.RunT(t)
	client := redis.NewClient(&redis.Options{
		Addr:            server.Addr(),
		Protocol:        2,
		DisableIdentity: true,
	})
	t.Cleanup(func() {
		assert.NoError(t, client.Close())
	})

	store, err := redisstate.NewAcceptanceStore(client, redisstate.AcceptanceConfig{
		RecordTTL:      24 * time.Hour,
		DeadLetterTTL:  72 * time.Hour,
		IdempotencyTTL: 7 * 24 * time.Hour,
	})
	require.NoError(t, err)

	now := time.UnixMilli(1775121700000).UTC()
	clock := newSteppingClock(now, time.Millisecond)
	telemetry := &recordingWorkerTelemetry{}
	publisher, err := NewPushPublisher(PushPublisherConfig{
		Store:               store,
		GatewayStream:       "gateway:client-events",
		GatewayStreamMaxLen: 1024,
		RouteLeaseTTL:       200 * time.Millisecond,
		RouteBackoffMin:     20 * time.Millisecond,
		RouteBackoffMax:     20 * time.Millisecond,
		PollInterval:        10 * time.Millisecond,
		BatchSize:           16,
		Telemetry:           telemetry,
		Clock:               clock,
	}, testWorkerLogger())
	require.NoError(t, err)

	return pushPublisherFixture{
		client:        client,
		store:         store,
		publisher:     publisher,
		gatewayStream: "gateway:client-events",
		now:           now,
		clock:         clock,
		telemetry:     telemetry,
	}
}

func validPushAcceptanceInput(now time.Time) acceptintent.CreateAcceptanceInput {
	return acceptintent.CreateAcceptanceInput{
		Notification: acceptintent.NotificationRecord{
			NotificationID:     "1775121700000-0",
			NotificationType:   intentstream.NotificationTypeGameTurnReady,
			Producer:           intentstream.ProducerGameMaster,
			AudienceKind:       intentstream.AudienceKindUser,
			RecipientUserIDs:   []string{"user-1"},
			PayloadJSON:        `{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54}`,
			IdempotencyKey:     "game-123:turn-54",
			RequestFingerprint: "sha256:deadbeef",
			RequestID:          "request-1",
			TraceID:            "trace-1",
			OccurredAt:         now,
			AcceptedAt:         now,
			UpdatedAt:          now,
		},
		Routes: []acceptintent.NotificationRoute{
			{
				NotificationID: "1775121700000-0",
				RouteID:        "push:user:user-1",
				Channel:        intentstream.ChannelPush,
				RecipientRef:   "user:user-1",
				Status:         acceptintent.RouteStatusPending,
				AttemptCount:   0,
				MaxAttempts:    3,
				NextAttemptAt:  now,
				ResolvedEmail:  "pilot@example.com",
				ResolvedLocale: "en",
				CreatedAt:      now,
				UpdatedAt:      now,
			},
			{
				NotificationID: "1775121700000-0",
				RouteID:        "email:user:user-1",
				Channel:        intentstream.ChannelEmail,
				RecipientRef:   "user:user-1",
				Status:         acceptintent.RouteStatusPending,
				AttemptCount:   0,
				MaxAttempts:    7,
				NextAttemptAt:  now,
				ResolvedEmail:  "pilot@example.com",
				ResolvedLocale: "en",
				CreatedAt:      now,
				UpdatedAt:      now,
			},
		},
		Idempotency: acceptintent.IdempotencyRecord{
			Producer:           intentstream.ProducerGameMaster,
			IdempotencyKey:     "game-123:turn-54",
			NotificationID:     "1775121700000-0",
			RequestFingerprint: "sha256:deadbeef",
			CreatedAt:          now,
			ExpiresAt:          now.Add(7 * 24 * time.Hour),
		},
	}
}

type runningPushPublisher struct {
	cancel   context.CancelFunc
	resultCh chan error
}

func runPushPublisher(t *testing.T, publisher *PushPublisher) runningPushPublisher {
	t.Helper()

	ctx, cancel := context.WithCancel(context.Background())
	resultCh := make(chan error, 1)
	go func() {
		resultCh <- publisher.Run(ctx)
	}()

	return runningPushPublisher{
		cancel:   cancel,
		resultCh: resultCh,
	}
}

func (r runningPushPublisher) stop(t *testing.T) {
	t.Helper()

	r.cancel()

	select {
	case err := <-r.resultCh:
		require.ErrorIs(t, err, context.Canceled)
	case <-time.After(time.Second):
		require.FailNow(t, "push publisher did not stop")
	}
}

type steppingClock struct {
	mu      sync.Mutex
	current time.Time
	step    time.Duration
}

func newSteppingClock(start time.Time, step time.Duration) *steppingClock {
	return &steppingClock{
		current: start.UTC().Truncate(time.Millisecond),
		step:    step,
	}
}

func (clock *steppingClock) Now() time.Time {
	clock.mu.Lock()
	defer clock.mu.Unlock()

	now := clock.current
	clock.current = clock.current.Add(clock.step).UTC().Truncate(time.Millisecond)

	return now
}

func testWorkerLogger() *slog.Logger {
	return slog.New(slog.NewTextHandler(io.Discard, nil))
}