package executeattempt import ( "bytes" "context" "log/slog" "testing" "time" "galaxy/mail/internal/domain/attempt" "galaxy/mail/internal/domain/common" deliverydomain "galaxy/mail/internal/domain/delivery" "galaxy/mail/internal/ports" "galaxy/mail/internal/service/acceptgenericdelivery" "galaxy/mail/internal/service/renderdelivery" "github.com/stretchr/testify/require" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" ) func TestServicePrepareRendersQueuedTemplateDelivery(t *testing.T) { t.Parallel() renderedDelivery := queuedTemplateWorkItem(t).Delivery renderedDelivery.Status = deliverydomain.StatusRendered renderedDelivery.Content = deliverydomain.Content{ Subject: "Turn 54", TextBody: "Hello Pilot", } renderedDelivery.UpdatedAt = renderedDelivery.CreatedAt.Add(time.Minute) require.NoError(t, renderedDelivery.Validate()) renderer := &stubRenderer{ result: renderdelivery.Result{ Outcome: renderdelivery.OutcomeRendered, Delivery: renderedDelivery, ResolvedLocale: common.Locale("en"), TemplateVersion: "sha256:template", LocaleFallbackUsed: false, }, } service := newTestService(t, Config{ Renderer: renderer, Provider: stubProvider{}, PayloadLoader: stubPayloadLoader{}, Store: &stubStore{}, Clock: stubClock{now: renderedDelivery.UpdatedAt}, AttemptTimeout: 15 * time.Second, }) ready, err := service.Prepare(context.Background(), queuedTemplateWorkItem(t)) require.NoError(t, err) require.True(t, ready) require.Len(t, renderer.inputs, 1) } func TestServiceExecuteAcceptedRenderedDelivery(t *testing.T) { t.Parallel() store := &stubStore{} service := newTestService(t, Config{ Renderer: &stubRenderer{}, Provider: stubProvider{ result: ports.Result{ Classification: ports.ClassificationAccepted, Summary: "provider=smtp result=accepted", }, }, PayloadLoader: stubPayloadLoader{}, Store: store, Clock: stubClock{now: fixedNow().Add(time.Minute)}, AttemptTimeout: 15 * time.Second, }) err := service.Execute(context.Background(), renderedWorkItem(t, 1)) require.NoError(t, err) require.Len(t, store.inputs, 1) require.Equal(t, deliverydomain.StatusSent, store.inputs[0].Delivery.Status) require.Equal(t, attempt.StatusProviderAccepted, store.inputs[0].Attempt.Status) require.Nil(t, store.inputs[0].NextAttempt) require.Nil(t, store.inputs[0].DeadLetter) } func TestServiceExecuteMapsSuppressedToProviderRejected(t *testing.T) { t.Parallel() store := &stubStore{} service := newTestService(t, Config{ Renderer: &stubRenderer{}, Provider: stubProvider{ result: ports.Result{ Classification: ports.ClassificationSuppressed, Summary: "provider=stub result=suppressed script=policy_skip", }, }, PayloadLoader: stubPayloadLoader{}, Store: store, Clock: stubClock{now: fixedNow().Add(time.Minute)}, AttemptTimeout: 15 * time.Second, }) err := service.Execute(context.Background(), renderedWorkItem(t, 1)) require.NoError(t, err) require.Len(t, store.inputs, 1) require.Equal(t, deliverydomain.StatusSuppressed, store.inputs[0].Delivery.Status) require.Equal(t, attempt.StatusProviderRejected, store.inputs[0].Attempt.Status) } func TestServiceExecuteMapsPermanentFailureToFailed(t *testing.T) { t.Parallel() store := &stubStore{} service := newTestService(t, Config{ Renderer: &stubRenderer{}, Provider: stubProvider{ result: ports.Result{ Classification: ports.ClassificationPermanentFailure, Summary: "provider=smtp result=permanent_failure phase=data smtp_code=550", }, }, PayloadLoader: stubPayloadLoader{}, Store: store, Clock: stubClock{now: fixedNow().Add(time.Minute)}, AttemptTimeout: 15 * time.Second, }) err := service.Execute(context.Background(), renderedWorkItem(t, 1)) require.NoError(t, err) require.Len(t, store.inputs, 1) require.Equal(t, deliverydomain.StatusFailed, store.inputs[0].Delivery.Status) require.Equal(t, attempt.StatusProviderRejected, store.inputs[0].Attempt.Status) require.Nil(t, store.inputs[0].DeadLetter) } func TestServiceExecuteBuildsRetryChainAndDeadLetter(t *testing.T) { t.Parallel() tests := []struct { name string attemptNo int wantStatus deliverydomain.Status wantAttemptStatus attempt.Status wantNextAttemptNo int wantNextDelay time.Duration wantDeadLetterEntry bool }{ { name: "attempt one schedules retry after one minute", attemptNo: 1, wantStatus: deliverydomain.StatusQueued, wantAttemptStatus: attempt.StatusTransportFailed, wantNextAttemptNo: 2, wantNextDelay: time.Minute, }, { name: "attempt two schedules retry after five minutes", attemptNo: 2, wantStatus: deliverydomain.StatusQueued, wantAttemptStatus: attempt.StatusTransportFailed, wantNextAttemptNo: 3, wantNextDelay: 5 * time.Minute, }, { name: "attempt three schedules retry after thirty minutes", attemptNo: 3, wantStatus: deliverydomain.StatusQueued, wantAttemptStatus: attempt.StatusTransportFailed, wantNextAttemptNo: 4, wantNextDelay: 30 * time.Minute, }, { name: "attempt four becomes dead letter", attemptNo: 4, wantStatus: deliverydomain.StatusDeadLetter, wantAttemptStatus: attempt.StatusTransportFailed, wantDeadLetterEntry: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() store := &stubStore{} service := newTestService(t, Config{ Renderer: &stubRenderer{}, Provider: stubProvider{ result: ports.Result{ Classification: ports.ClassificationTransientFailure, Summary: "provider=smtp result=transient_failure phase=data smtp_code=451", Details: map[string]string{ "phase": "data", }, }, }, PayloadLoader: stubPayloadLoader{}, Store: store, Clock: stubClock{now: fixedNow().Add(time.Minute)}, AttemptTimeout: 15 * time.Second, }) workItem := renderedWorkItem(t, tt.attemptNo) err := service.Execute(context.Background(), workItem) require.NoError(t, err) require.Len(t, store.inputs, 1) input := store.inputs[0] require.Equal(t, tt.wantStatus, input.Delivery.Status) require.Equal(t, tt.wantAttemptStatus, input.Attempt.Status) if tt.wantDeadLetterEntry { require.NotNil(t, input.DeadLetter) require.Nil(t, input.NextAttempt) require.Equal(t, "retry_exhausted", input.DeadLetter.FailureClassification) return } require.NotNil(t, input.NextAttempt) require.Nil(t, input.DeadLetter) require.Equal(t, tt.wantNextAttemptNo, input.NextAttempt.AttemptNo) require.Equal(t, input.Attempt.FinishedAt.Add(tt.wantNextDelay), input.NextAttempt.ScheduledFor) }) } } func TestServiceExecuteClassifiesDeadlineExceededAsTimedOut(t *testing.T) { t.Parallel() store := &stubStore{} service := newTestService(t, Config{ Renderer: &stubRenderer{}, Provider: stubProvider{ result: ports.Result{ Classification: ports.ClassificationTransientFailure, Summary: "provider=smtp result=transient_failure phase=context", Details: map[string]string{ "error": "deadline_exceeded", }, }, }, PayloadLoader: stubPayloadLoader{}, Store: store, Clock: stubClock{now: fixedNow().Add(time.Minute)}, AttemptTimeout: 15 * time.Second, }) err := service.Execute(context.Background(), renderedWorkItem(t, 1)) require.NoError(t, err) require.Len(t, store.inputs, 1) require.Equal(t, attempt.StatusTimedOut, store.inputs[0].Attempt.Status) require.Equal(t, "deadline_exceeded", store.inputs[0].Attempt.ProviderClassification) } func TestServiceRecoverExpiredSchedulesTimedOutRetry(t *testing.T) { t.Parallel() store := &stubStore{} service := newTestService(t, Config{ Renderer: &stubRenderer{}, Provider: stubProvider{}, PayloadLoader: stubPayloadLoader{}, Store: store, Clock: stubClock{now: fixedNow().Add(time.Minute)}, AttemptTimeout: 15 * time.Second, }) err := service.RecoverExpired(context.Background(), renderedWorkItem(t, 1)) require.NoError(t, err) require.Len(t, store.inputs, 1) require.Equal(t, attempt.StatusTimedOut, store.inputs[0].Attempt.Status) require.Equal(t, "claim_ttl_expired", store.inputs[0].Attempt.ProviderClassification) require.Equal(t, "attempt claim TTL expired", store.inputs[0].Attempt.ProviderSummary) require.NotNil(t, store.inputs[0].NextAttempt) } func TestServiceExecuteRecordsMetricsAndLogsProviderResult(t *testing.T) { t.Parallel() store := &stubStore{} telemetry := &stubTelemetry{} loggerBuffer := &bytes.Buffer{} recorder := tracetest.NewSpanRecorder() tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) service := newTestService(t, Config{ Renderer: &stubRenderer{}, Provider: stubProvider{ result: ports.Result{ Classification: ports.ClassificationAccepted, Summary: "provider=smtp result=accepted", }, }, PayloadLoader: stubPayloadLoader{}, Store: store, Clock: stubClock{now: fixedNow().Add(time.Minute)}, Telemetry: telemetry, TracerProvider: tracerProvider, Logger: slog.New(slog.NewJSONHandler(loggerBuffer, nil)), AttemptTimeout: 15 * time.Second, }) err := service.Execute(context.Background(), sendingTemplateWorkItem(t, 1)) require.NoError(t, err) require.Equal(t, []string{"notification:sent"}, telemetry.statuses) require.Equal(t, []string{"notification:provider_accepted"}, telemetry.attempts) require.Equal(t, []string{"smtp:accepted"}, telemetry.providerDurations) require.Contains(t, loggerBuffer.String(), "\"delivery_id\":\"delivery-template-sending\"") require.Contains(t, loggerBuffer.String(), "\"source\":\"notification\"") require.Contains(t, loggerBuffer.String(), "\"template_id\":\"game.turn.ready\"") require.Contains(t, loggerBuffer.String(), "\"attempt_no\":1") require.Contains(t, loggerBuffer.String(), "\"otel_trace_id\":") require.True(t, hasExecuteSpanNamed(recorder.Ended(), "mail.provider_send")) } func TestServiceExecuteReturnsServiceUnavailableOnMissingPayload(t *testing.T) { t.Parallel() service := newTestService(t, Config{ Renderer: &stubRenderer{}, Provider: stubProvider{ result: ports.Result{ Classification: ports.ClassificationAccepted, Summary: "provider=smtp result=accepted", }, }, PayloadLoader: stubPayloadLoader{}, Store: &stubStore{}, Clock: stubClock{now: fixedNow().Add(time.Minute)}, AttemptTimeout: 15 * time.Second, }) workItem := renderedWorkItem(t, 1) workItem.Delivery.Attachments = []common.AttachmentMetadata{ {Filename: "guide.txt", ContentType: "text/plain; charset=utf-8", SizeBytes: int64(len([]byte("read me")))}, } require.NoError(t, workItem.Delivery.Validate()) err := service.Execute(context.Background(), workItem) require.Error(t, err) require.ErrorIs(t, err, ErrServiceUnavailable) } type stubRenderer struct { result renderdelivery.Result err error inputs []renderdelivery.Input } func (renderer *stubRenderer) Execute(_ context.Context, input renderdelivery.Input) (renderdelivery.Result, error) { renderer.inputs = append(renderer.inputs, input) return renderer.result, renderer.err } type stubProvider struct { result ports.Result err error inputs []ports.Message } func (provider stubProvider) Send(_ context.Context, message ports.Message) (ports.Result, error) { provider.inputs = append(provider.inputs, message) return provider.result, provider.err } func (provider stubProvider) Close() error { return nil } type stubPayloadLoader struct { payload acceptgenericdelivery.DeliveryPayload found bool err error } func (loader stubPayloadLoader) LoadPayload(context.Context, common.DeliveryID) (acceptgenericdelivery.DeliveryPayload, bool, error) { return loader.payload, loader.found, loader.err } type stubStore struct { inputs []CommitStateInput err error } func (store *stubStore) Commit(_ context.Context, input CommitStateInput) error { store.inputs = append(store.inputs, input) return store.err } type stubClock struct { now time.Time } func (clock stubClock) Now() time.Time { return clock.now } type stubTelemetry struct { statuses []string attempts []string providerDurations []string } func (telemetry *stubTelemetry) RecordDeliveryStatusTransition(_ context.Context, status string, source string) { telemetry.statuses = append(telemetry.statuses, source+":"+status) } func (telemetry *stubTelemetry) RecordAttemptOutcome(_ context.Context, status string, source string) { telemetry.attempts = append(telemetry.attempts, source+":"+status) } func (telemetry *stubTelemetry) RecordProviderSendDuration(_ context.Context, provider string, outcome string, _ time.Duration) { telemetry.providerDurations = append(telemetry.providerDurations, provider+":"+outcome) } func newTestService(t *testing.T, cfg Config) *Service { t.Helper() service, err := New(cfg) require.NoError(t, err) return service } func queuedTemplateWorkItem(t *testing.T) WorkItem { t.Helper() createdAt := fixedNow().Add(-time.Minute) deliveryRecord := deliverydomain.Delivery{ DeliveryID: common.DeliveryID("delivery-template"), Source: deliverydomain.SourceNotification, PayloadMode: deliverydomain.PayloadModeTemplate, TemplateID: common.TemplateID("game.turn.ready"), Envelope: deliverydomain.Envelope{ To: []common.Email{common.Email("pilot@example.com")}, }, Locale: common.Locale("en"), TemplateVariables: map[string]any{ "player": map[string]any{ "name": "Pilot", }, "turn_number": float64(54), }, IdempotencyKey: common.IdempotencyKey("notification:delivery-template"), Status: deliverydomain.StatusQueued, AttemptCount: 1, CreatedAt: createdAt, UpdatedAt: createdAt, } require.NoError(t, deliveryRecord.Validate()) attemptRecord := attempt.Attempt{ DeliveryID: deliveryRecord.DeliveryID, AttemptNo: 1, ScheduledFor: createdAt, Status: attempt.StatusScheduled, } require.NoError(t, attemptRecord.Validate()) return WorkItem{ Delivery: deliveryRecord, Attempt: attemptRecord, } } func renderedWorkItem(t *testing.T, attemptNo int) WorkItem { t.Helper() createdAt := fixedNow().Add(-time.Duration(attemptNo) * time.Minute) deliveryRecord := deliverydomain.Delivery{ DeliveryID: common.DeliveryID("delivery-rendered"), Source: deliverydomain.SourceNotification, PayloadMode: deliverydomain.PayloadModeRendered, Envelope: deliverydomain.Envelope{ To: []common.Email{common.Email("pilot@example.com")}, }, Content: deliverydomain.Content{ Subject: "Turn ready", TextBody: "Turn 54 is ready.", }, IdempotencyKey: common.IdempotencyKey("notification:delivery-rendered"), Status: deliverydomain.StatusSending, AttemptCount: attemptNo, CreatedAt: createdAt, UpdatedAt: createdAt.Add(time.Second), } require.NoError(t, deliveryRecord.Validate()) scheduledFor := createdAt startedAt := scheduledFor.Add(5 * time.Second) attemptRecord := attempt.Attempt{ DeliveryID: deliveryRecord.DeliveryID, AttemptNo: attemptNo, ScheduledFor: scheduledFor, StartedAt: &startedAt, Status: attempt.StatusInProgress, } require.NoError(t, attemptRecord.Validate()) return WorkItem{ Delivery: deliveryRecord, Attempt: attemptRecord, } } func sendingTemplateWorkItem(t *testing.T, attemptNo int) WorkItem { t.Helper() createdAt := fixedNow().Add(-time.Duration(attemptNo) * time.Minute) deliveryRecord := deliverydomain.Delivery{ DeliveryID: common.DeliveryID("delivery-template-sending"), Source: deliverydomain.SourceNotification, PayloadMode: deliverydomain.PayloadModeTemplate, TemplateID: common.TemplateID("game.turn.ready"), Envelope: deliverydomain.Envelope{ To: []common.Email{common.Email("pilot@example.com")}, }, Content: deliverydomain.Content{ Subject: "Turn ready", TextBody: "Turn 54 is ready.", }, Locale: common.Locale("en"), TemplateVariables: map[string]any{ "turn_number": float64(54), }, IdempotencyKey: common.IdempotencyKey("notification:delivery-template-sending"), Status: deliverydomain.StatusSending, AttemptCount: attemptNo, CreatedAt: createdAt, UpdatedAt: createdAt.Add(time.Second), } require.NoError(t, deliveryRecord.Validate()) scheduledFor := createdAt startedAt := scheduledFor.Add(5 * time.Second) attemptRecord := attempt.Attempt{ DeliveryID: deliveryRecord.DeliveryID, AttemptNo: attemptNo, ScheduledFor: scheduledFor, StartedAt: &startedAt, Status: attempt.StatusInProgress, } require.NoError(t, attemptRecord.Validate()) return WorkItem{ Delivery: deliveryRecord, Attempt: attemptRecord, } } func fixedNow() time.Time { return time.Unix(1_775_121_700, 0).UTC() } var _ Renderer = (*stubRenderer)(nil) var _ ports.Provider = stubProvider{} var _ PayloadLoader = stubPayloadLoader{} var _ Store = (*stubStore)(nil) var _ Telemetry = (*stubTelemetry)(nil) func hasExecuteSpanNamed(spans []sdktrace.ReadOnlySpan, name string) bool { for _, span := range spans { if span.Name() == name { return true } } return false }