Ilia Denisov
233 lines
8.2 KiB
Go
233 lines
8.2 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
redisstate "galaxy/notification/internal/adapters/redisstate"
|
|
"galaxy/notification/internal/service/acceptintent"
|
|
|
|
"github.com/alicebob/miniredis/v2"
|
|
"github.com/redis/go-redis/v9"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestEmailPublisherPublishesDueEmailRouteAndLeavesPushRoutePending(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
fixture := newEmailPublisherFixture(t)
|
|
require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validEmailAcceptanceInput(fixture.now, 0)))
|
|
|
|
running := runEmailPublisher(t, fixture.publisher)
|
|
defer running.stop(t)
|
|
|
|
require.Eventually(t, func() bool {
|
|
route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "email:user:user-1")
|
|
return err == nil && found && route.Status == acceptintent.RouteStatusPublished
|
|
}, time.Second, 10*time.Millisecond)
|
|
|
|
pushRoute, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "push:user:user-1")
|
|
require.NoError(t, err)
|
|
require.True(t, found)
|
|
require.Equal(t, acceptintent.RouteStatusPending, pushRoute.Status)
|
|
|
|
messages, err := fixture.client.XRange(context.Background(), fixture.mailStream, "-", "+").Result()
|
|
require.NoError(t, err)
|
|
require.Len(t, messages, 1)
|
|
require.Equal(t, "1775121700000-0/email:user:user-1", messages[0].Values["delivery_id"])
|
|
require.Equal(t, "notification", messages[0].Values["source"])
|
|
require.Equal(t, "template", messages[0].Values["payload_mode"])
|
|
require.True(t, fixture.telemetry.hasRoutePublishAttempt("email", "published", ""))
|
|
}
|
|
|
|
func TestEmailPublisherRetriesMailStreamPublicationFailures(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
fixture := newEmailPublisherFixture(t)
|
|
require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validEmailAcceptanceInput(fixture.now, 0)))
|
|
require.NoError(t, fixture.client.Set(context.Background(), fixture.mailStream, "wrong-type", 0).Err())
|
|
|
|
running := runEmailPublisher(t, fixture.publisher)
|
|
defer running.stop(t)
|
|
|
|
require.Eventually(t, func() bool {
|
|
route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "email:user:user-1")
|
|
return err == nil && found && route.Status == acceptintent.RouteStatusFailed && route.AttemptCount == 1
|
|
}, time.Second, 10*time.Millisecond)
|
|
require.True(t, fixture.telemetry.hasRoutePublishAttempt("email", "retry", emailFailureClassificationMailStreamWrite))
|
|
require.True(t, fixture.telemetry.hasRouteRetry("email"))
|
|
|
|
require.NoError(t, fixture.client.Del(context.Background(), fixture.mailStream).Err())
|
|
|
|
require.Eventually(t, func() bool {
|
|
route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "email:user:user-1")
|
|
return err == nil && found && route.Status == acceptintent.RouteStatusPublished && route.AttemptCount == 2
|
|
}, 2*time.Second, 10*time.Millisecond)
|
|
|
|
messages, err := fixture.client.XRange(context.Background(), fixture.mailStream, "-", "+").Result()
|
|
require.NoError(t, err)
|
|
require.Len(t, messages, 1)
|
|
require.True(t, fixture.telemetry.hasRoutePublishAttempt("email", "published", ""))
|
|
}
|
|
|
|
func TestEmailPublisherLeasePreventsDuplicatePublicationAcrossReplicas(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
fixture := newEmailPublisherFixture(t)
|
|
require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validEmailAcceptanceInput(fixture.now, 0)))
|
|
|
|
otherPublisher, err := NewEmailPublisher(EmailPublisherConfig{
|
|
Store: fixture.store,
|
|
MailDeliveryCommandsStream: fixture.mailStream,
|
|
RouteLeaseTTL: 200 * time.Millisecond,
|
|
RouteBackoffMin: 20 * time.Millisecond,
|
|
RouteBackoffMax: 20 * time.Millisecond,
|
|
PollInterval: 10 * time.Millisecond,
|
|
BatchSize: 16,
|
|
Clock: newSteppingClock(fixture.now, time.Millisecond),
|
|
}, testWorkerLogger())
|
|
require.NoError(t, err)
|
|
|
|
first := runEmailPublisher(t, fixture.publisher)
|
|
defer first.stop(t)
|
|
second := runEmailPublisher(t, otherPublisher)
|
|
defer second.stop(t)
|
|
|
|
require.Eventually(t, func() bool {
|
|
route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "email:user:user-1")
|
|
return err == nil && found && route.Status == acceptintent.RouteStatusPublished
|
|
}, time.Second, 10*time.Millisecond)
|
|
|
|
messages, err := fixture.client.XRange(context.Background(), fixture.mailStream, "-", "+").Result()
|
|
require.NoError(t, err)
|
|
require.Len(t, messages, 1)
|
|
}
|
|
|
|
func TestEmailPublisherDeadLettersExhaustedRoute(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
fixture := newEmailPublisherFixture(t)
|
|
require.NoError(t, fixture.store.CreateAcceptance(context.Background(), validEmailAcceptanceInput(fixture.now, 6)))
|
|
require.NoError(t, fixture.client.Set(context.Background(), fixture.mailStream, "wrong-type", 0).Err())
|
|
|
|
running := runEmailPublisher(t, fixture.publisher)
|
|
defer running.stop(t)
|
|
|
|
require.Eventually(t, func() bool {
|
|
route, found, err := fixture.store.GetRoute(context.Background(), "1775121700000-0", "email:user:user-1")
|
|
return err == nil && found && route.Status == acceptintent.RouteStatusDeadLetter && route.AttemptCount == 7
|
|
}, time.Second, 10*time.Millisecond)
|
|
|
|
deadLetterPayload, err := fixture.client.Get(context.Background(), redisstate.Keyspace{}.DeadLetter("1775121700000-0", "email:user:user-1")).Bytes()
|
|
require.NoError(t, err)
|
|
deadLetter, err := redisstate.UnmarshalDeadLetter(deadLetterPayload)
|
|
require.NoError(t, err)
|
|
require.Equal(t, emailFailureClassificationMailStreamWrite, deadLetter.FailureClassification)
|
|
require.True(t, fixture.telemetry.hasRoutePublishAttempt("email", "dead_letter", emailFailureClassificationMailStreamWrite))
|
|
require.True(t, fixture.telemetry.hasRouteDeadLetter("email", emailFailureClassificationMailStreamWrite))
|
|
}
|
|
|
|
type emailPublisherFixture struct {
|
|
client *redis.Client
|
|
store *redisstate.AcceptanceStore
|
|
publisher *EmailPublisher
|
|
mailStream string
|
|
now time.Time
|
|
clock *steppingClock
|
|
telemetry *recordingWorkerTelemetry
|
|
}
|
|
|
|
func newEmailPublisherFixture(t *testing.T) emailPublisherFixture {
|
|
t.Helper()
|
|
|
|
server := miniredis.RunT(t)
|
|
client := redis.NewClient(&redis.Options{
|
|
Addr: server.Addr(),
|
|
Protocol: 2,
|
|
DisableIdentity: true,
|
|
})
|
|
t.Cleanup(func() {
|
|
require.NoError(t, client.Close())
|
|
})
|
|
|
|
store, err := redisstate.NewAcceptanceStore(client, redisstate.AcceptanceConfig{
|
|
RecordTTL: 24 * time.Hour,
|
|
DeadLetterTTL: 72 * time.Hour,
|
|
IdempotencyTTL: 7 * 24 * time.Hour,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
now := time.UnixMilli(1775121700000).UTC()
|
|
clock := newSteppingClock(now, time.Millisecond)
|
|
telemetry := &recordingWorkerTelemetry{}
|
|
publisher, err := NewEmailPublisher(EmailPublisherConfig{
|
|
Store: store,
|
|
MailDeliveryCommandsStream: "mail:delivery_commands",
|
|
RouteLeaseTTL: 200 * time.Millisecond,
|
|
RouteBackoffMin: 20 * time.Millisecond,
|
|
RouteBackoffMax: 20 * time.Millisecond,
|
|
PollInterval: 10 * time.Millisecond,
|
|
BatchSize: 16,
|
|
Telemetry: telemetry,
|
|
Clock: clock,
|
|
}, testWorkerLogger())
|
|
require.NoError(t, err)
|
|
|
|
return emailPublisherFixture{
|
|
client: client,
|
|
store: store,
|
|
publisher: publisher,
|
|
mailStream: "mail:delivery_commands",
|
|
now: now,
|
|
clock: clock,
|
|
telemetry: telemetry,
|
|
}
|
|
}
|
|
|
|
func validEmailAcceptanceInput(now time.Time, emailAttemptCount int) acceptintent.CreateAcceptanceInput {
|
|
input := validPushAcceptanceInput(now)
|
|
for index := range input.Routes {
|
|
if input.Routes[index].RouteID != "email:user:user-1" {
|
|
continue
|
|
}
|
|
input.Routes[index].AttemptCount = emailAttemptCount
|
|
input.Routes[index].MaxAttempts = 7
|
|
}
|
|
|
|
return input
|
|
}
|
|
|
|
type runningEmailPublisher struct {
|
|
cancel context.CancelFunc
|
|
resultCh chan error
|
|
}
|
|
|
|
func runEmailPublisher(t *testing.T, publisher *EmailPublisher) runningEmailPublisher {
|
|
t.Helper()
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
resultCh := make(chan error, 1)
|
|
go func() {
|
|
resultCh <- publisher.Run(ctx)
|
|
}()
|
|
|
|
return runningEmailPublisher{
|
|
cancel: cancel,
|
|
resultCh: resultCh,
|
|
}
|
|
}
|
|
|
|
func (r runningEmailPublisher) stop(t *testing.T) {
|
|
t.Helper()
|
|
|
|
r.cancel()
|
|
|
|
select {
|
|
case err := <-r.resultCh:
|
|
require.ErrorIs(t, err, context.Canceled)
|
|
case <-time.After(time.Second):
|
|
require.FailNow(t, "email publisher did not stop")
|
|
}
|
|
}
|