feat: notification service
This commit is contained in:
Ilia Denisov
@@ -0,0 +1,318 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
redisstate "galaxy/notification/internal/adapters/redisstate"
|
||||
"galaxy/notification/internal/api/intentstream"
|
||||
"galaxy/notification/internal/service/acceptintent"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPushPublisherPublishesDuePushRouteAndLeavesEmailRoutePending(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newPushPublisherFixture(t)
|
||||
require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validPushAcceptanceInput(fixture.now)))
|
||||
|
||||
running := runPushPublisher(t, fixture.publisher)
|
||||
defer running.stop(t)
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1")
|
||||
return err == nil && found && route.Status == acceptintent.RouteStatusPublished
|
||||
}, time.Second, 10*time.Millisecond)
|
||||
|
||||
emailRoute, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "email:user:user-1")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.Equal(t, acceptintent.RouteStatusPending, emailRoute.Status)
|
||||
|
||||
messages, err := fixture.client.XRange(context.Background(), fixture.gatewayStream, "-", "+").Result()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, messages, 1)
|
||||
require.Equal(t, "user-1", messages[0].Values["user_id"])
|
||||
require.Equal(t, "game.turn.ready", messages[0].Values["event_type"])
|
||||
require.Equal(t, "1775121700000-0/push:user:user-1", messages[0].Values["event_id"])
|
||||
require.True(t, fixture.telemetry.hasRoutePublishAttempt("push", "published", ""))
|
||||
}
|
||||
|
||||
func TestPushPublisherRetriesGatewayStreamPublicationFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newPushPublisherFixture(t)
|
||||
require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validPushAcceptanceInput(fixture.now)))
|
||||
require.NoError(t, fixture.client.Set(context.Background(), fixture.gatewayStream, "wrong-type", 0).Err())
|
||||
|
||||
running := runPushPublisher(t, fixture.publisher)
|
||||
defer running.stop(t)
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1")
|
||||
return err == nil && found && route.Status == acceptintent.RouteStatusFailed && route.AttemptCount == 1
|
||||
}, time.Second, 10*time.Millisecond)
|
||||
require.True(t, fixture.telemetry.hasRoutePublishAttempt("push", "retry", pushFailureClassificationGatewayStreamWrite))
|
||||
require.True(t, fixture.telemetry.hasRouteRetry("push"))
|
||||
|
||||
require.NoError(t, fixture.client.Del(context.Background(), fixture.gatewayStream).Err())
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1")
|
||||
return err == nil && found && route.Status == acceptintent.RouteStatusPublished && route.AttemptCount == 2
|
||||
}, 2*time.Second, 10*time.Millisecond)
|
||||
|
||||
messages, err := fixture.client.XRange(context.Background(), fixture.gatewayStream, "-", "+").Result()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, messages, 1)
|
||||
require.True(t, fixture.telemetry.hasRoutePublishAttempt("push", "published", ""))
|
||||
}
|
||||
|
||||
func TestPushPublisherDeadLettersExhaustedRoute(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newPushPublisherFixture(t)
|
||||
input := validPushAcceptanceInput(fixture.now)
|
||||
for index := range input.Routes {
|
||||
if input.Routes[index].RouteID == "push:user:user-1" {
|
||||
input.Routes[index].AttemptCount = 2
|
||||
input.Routes[index].MaxAttempts = 3
|
||||
}
|
||||
}
|
||||
require.NoError(t, fixture.store.CreateAcceptance(context.Background(), input))
|
||||
require.NoError(t, fixture.client.Set(context.Background(), fixture.gatewayStream, "wrong-type", 0).Err())
|
||||
|
||||
running := runPushPublisher(t, fixture.publisher)
|
||||
defer running.stop(t)
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1")
|
||||
return err == nil && found && route.Status == acceptintent.RouteStatusDeadLetter && route.AttemptCount == 3
|
||||
}, time.Second, 10*time.Millisecond)
|
||||
|
||||
deadLetterPayload, err := fixture.client.Get(context.Background(), redisstate.Keyspace{}.DeadLetter("1775121700000-0", "push:user:user-1")).Bytes()
|
||||
require.NoError(t, err)
|
||||
deadLetter, err := redisstate.UnmarshalDeadLetter(deadLetterPayload)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, pushFailureClassificationGatewayStreamWrite, deadLetter.FailureClassification)
|
||||
require.True(t, fixture.telemetry.hasRoutePublishAttempt("push", "dead_letter", pushFailureClassificationGatewayStreamWrite))
|
||||
require.True(t, fixture.telemetry.hasRouteDeadLetter("push", pushFailureClassificationGatewayStreamWrite))
|
||||
}
|
||||
|
||||
func TestPushPublisherLeasePreventsDuplicatePublicationAcrossReplicas(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newPushPublisherFixture(t)
|
||||
require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validPushAcceptanceInput(fixture.now)))
|
||||
|
||||
otherPublisher, err := NewPushPublisher(PushPublisherConfig{
|
||||
Store: fixture.store,
|
||||
GatewayStream: fixture.gatewayStream,
|
||||
GatewayStreamMaxLen: 1024,
|
||||
RouteLeaseTTL: 200 * time.Millisecond,
|
||||
RouteBackoffMin: 20 * time.Millisecond,
|
||||
RouteBackoffMax: 20 * time.Millisecond,
|
||||
PollInterval: 10 * time.Millisecond,
|
||||
BatchSize: 16,
|
||||
Clock: newSteppingClock(fixture.now, time.Millisecond),
|
||||
}, testWorkerLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
first := runPushPublisher(t, fixture.publisher)
|
||||
defer first.stop(t)
|
||||
second := runPushPublisher(t, otherPublisher)
|
||||
defer second.stop(t)
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1")
|
||||
return err == nil && found && route.Status == acceptintent.RouteStatusPublished
|
||||
}, time.Second, 10*time.Millisecond)
|
||||
|
||||
messages, err := fixture.client.XRange(context.Background(), fixture.gatewayStream, "-", "+").Result()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, messages, 1)
|
||||
}
|
||||
|
||||
type pushPublisherFixture struct {
|
||||
client *redis.Client
|
||||
store *redisstate.AcceptanceStore
|
||||
publisher *PushPublisher
|
||||
gatewayStream string
|
||||
now time.Time
|
||||
clock *steppingClock
|
||||
telemetry *recordingWorkerTelemetry
|
||||
}
|
||||
|
||||
func newPushPublisherFixture(t *testing.T) pushPublisherFixture {
|
||||
t.Helper()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: server.Addr(),
|
||||
Protocol: 2,
|
||||
DisableIdentity: true,
|
||||
})
|
||||
t.Cleanup(func() {
|
||||
assert.NoError(t, client.Close())
|
||||
})
|
||||
|
||||
store, err := redisstate.NewAcceptanceStore(client, redisstate.AcceptanceConfig{
|
||||
RecordTTL: 24 * time.Hour,
|
||||
DeadLetterTTL: 72 * time.Hour,
|
||||
IdempotencyTTL: 7 * 24 * time.Hour,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
now := time.UnixMilli(1775121700000).UTC()
|
||||
clock := newSteppingClock(now, time.Millisecond)
|
||||
telemetry := &recordingWorkerTelemetry{}
|
||||
publisher, err := NewPushPublisher(PushPublisherConfig{
|
||||
Store: store,
|
||||
GatewayStream: "gateway:client-events",
|
||||
GatewayStreamMaxLen: 1024,
|
||||
RouteLeaseTTL: 200 * time.Millisecond,
|
||||
RouteBackoffMin: 20 * time.Millisecond,
|
||||
RouteBackoffMax: 20 * time.Millisecond,
|
||||
PollInterval: 10 * time.Millisecond,
|
||||
BatchSize: 16,
|
||||
Telemetry: telemetry,
|
||||
Clock: clock,
|
||||
}, testWorkerLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
return pushPublisherFixture{
|
||||
client: client,
|
||||
store: store,
|
||||
publisher: publisher,
|
||||
gatewayStream: "gateway:client-events",
|
||||
now: now,
|
||||
clock: clock,
|
||||
telemetry: telemetry,
|
||||
}
|
||||
}
|
||||
|
||||
func validPushAcceptanceInput(now time.Time) acceptintent.CreateAcceptanceInput {
|
||||
return acceptintent.CreateAcceptanceInput{
|
||||
Notification: acceptintent.NotificationRecord{
|
||||
NotificationID: "1775121700000-0",
|
||||
NotificationType: intentstream.NotificationTypeGameTurnReady,
|
||||
Producer: intentstream.ProducerGameMaster,
|
||||
AudienceKind: intentstream.AudienceKindUser,
|
||||
RecipientUserIDs: []string{"user-1"},
|
||||
PayloadJSON: `{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54}`,
|
||||
IdempotencyKey: "game-123:turn-54",
|
||||
RequestFingerprint: "sha256:deadbeef",
|
||||
RequestID: "request-1",
|
||||
TraceID: "trace-1",
|
||||
OccurredAt: now,
|
||||
AcceptedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
Routes: []acceptintent.NotificationRoute{
|
||||
{
|
||||
NotificationID: "1775121700000-0",
|
||||
RouteID: "push:user:user-1",
|
||||
Channel: intentstream.ChannelPush,
|
||||
RecipientRef: "user:user-1",
|
||||
Status: acceptintent.RouteStatusPending,
|
||||
AttemptCount: 0,
|
||||
MaxAttempts: 3,
|
||||
NextAttemptAt: now,
|
||||
ResolvedEmail: "pilot@example.com",
|
||||
ResolvedLocale: "en",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
NotificationID: "1775121700000-0",
|
||||
RouteID: "email:user:user-1",
|
||||
Channel: intentstream.ChannelEmail,
|
||||
RecipientRef: "user:user-1",
|
||||
Status: acceptintent.RouteStatusPending,
|
||||
AttemptCount: 0,
|
||||
MaxAttempts: 7,
|
||||
NextAttemptAt: now,
|
||||
ResolvedEmail: "pilot@example.com",
|
||||
ResolvedLocale: "en",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
},
|
||||
Idempotency: acceptintent.IdempotencyRecord{
|
||||
Producer: intentstream.ProducerGameMaster,
|
||||
IdempotencyKey: "game-123:turn-54",
|
||||
NotificationID: "1775121700000-0",
|
||||
RequestFingerprint: "sha256:deadbeef",
|
||||
CreatedAt: now,
|
||||
ExpiresAt: now.Add(7 * 24 * time.Hour),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type runningPushPublisher struct {
|
||||
cancel context.CancelFunc
|
||||
resultCh chan error
|
||||
}
|
||||
|
||||
func runPushPublisher(t *testing.T, publisher *PushPublisher) runningPushPublisher {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
resultCh := make(chan error, 1)
|
||||
go func() {
|
||||
resultCh <- publisher.Run(ctx)
|
||||
}()
|
||||
|
||||
return runningPushPublisher{
|
||||
cancel: cancel,
|
||||
resultCh: resultCh,
|
||||
}
|
||||
}
|
||||
|
||||
func (r runningPushPublisher) stop(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
r.cancel()
|
||||
|
||||
select {
|
||||
case err := <-r.resultCh:
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
case <-time.After(time.Second):
|
||||
require.FailNow(t, "push publisher did not stop")
|
||||
}
|
||||
}
|
||||
|
||||
type steppingClock struct {
|
||||
mu sync.Mutex
|
||||
current time.Time
|
||||
step time.Duration
|
||||
}
|
||||
|
||||
func newSteppingClock(start time.Time, step time.Duration) *steppingClock {
|
||||
return &steppingClock{
|
||||
current: start.UTC().Truncate(time.Millisecond),
|
||||
step: step,
|
||||
}
|
||||
}
|
||||
|
||||
func (clock *steppingClock) Now() time.Time {
|
||||
clock.mu.Lock()
|
||||
defer clock.mu.Unlock()
|
||||
|
||||
now := clock.current
|
||||
clock.current = clock.current.Add(clock.step).UTC().Truncate(time.Millisecond)
|
||||
|
||||
return now
|
||||
}
|
||||
|
||||
func testWorkerLogger() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
}
|
||||
Reference in New Issue
Block a user