galaxy-game/mail/internal/service/executeattempt/service_test.go

package executeattempt

import (
	"bytes"
	"context"
	"log/slog"
	"testing"
	"time"

	"galaxy/mail/internal/domain/attempt"
	"galaxy/mail/internal/domain/common"
	deliverydomain "galaxy/mail/internal/domain/delivery"
	"galaxy/mail/internal/ports"
	"galaxy/mail/internal/service/acceptgenericdelivery"
	"galaxy/mail/internal/service/renderdelivery"

	"github.com/stretchr/testify/require"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	"go.opentelemetry.io/otel/sdk/trace/tracetest"
)

func TestServicePrepareRendersQueuedTemplateDelivery(t *testing.T) {
	t.Parallel()

	renderedDelivery := queuedTemplateWorkItem(t).Delivery
	renderedDelivery.Status = deliverydomain.StatusRendered
	renderedDelivery.Content = deliverydomain.Content{
		Subject:  "Turn 54",
		TextBody: "Hello Pilot",
	}
	renderedDelivery.UpdatedAt = renderedDelivery.CreatedAt.Add(time.Minute)
	require.NoError(t, renderedDelivery.Validate())

	renderer := &stubRenderer{
		result: renderdelivery.Result{
			Outcome:            renderdelivery.OutcomeRendered,
			Delivery:           renderedDelivery,
			ResolvedLocale:     common.Locale("en"),
			TemplateVersion:    "sha256:template",
			LocaleFallbackUsed: false,
		},
	}

	service := newTestService(t, Config{
		Renderer:       renderer,
		Provider:       stubProvider{},
		PayloadLoader:  stubPayloadLoader{},
		Store:          &stubStore{},
		Clock:          stubClock{now: renderedDelivery.UpdatedAt},
		AttemptTimeout: 15 * time.Second,
	})

	ready, err := service.Prepare(context.Background(), queuedTemplateWorkItem(t))
	require.NoError(t, err)
	require.True(t, ready)
	require.Len(t, renderer.inputs, 1)
}

func TestServiceExecuteAcceptedRenderedDelivery(t *testing.T) {
	t.Parallel()

	store := &stubStore{}
	service := newTestService(t, Config{
		Renderer: &stubRenderer{},
		Provider: stubProvider{
			result: ports.Result{
				Classification: ports.ClassificationAccepted,
				Summary:        "provider=smtp result=accepted",
			},
		},
		PayloadLoader:  stubPayloadLoader{},
		Store:          store,
		Clock:          stubClock{now: fixedNow().Add(time.Minute)},
		AttemptTimeout: 15 * time.Second,
	})

	err := service.Execute(context.Background(), renderedWorkItem(t, 1))
	require.NoError(t, err)
	require.Len(t, store.inputs, 1)
	require.Equal(t, deliverydomain.StatusSent, store.inputs[0].Delivery.Status)
	require.Equal(t, attempt.StatusProviderAccepted, store.inputs[0].Attempt.Status)
	require.Nil(t, store.inputs[0].NextAttempt)
	require.Nil(t, store.inputs[0].DeadLetter)
}

func TestServiceExecuteMapsSuppressedToProviderRejected(t *testing.T) {
	t.Parallel()

	store := &stubStore{}
	service := newTestService(t, Config{
		Renderer: &stubRenderer{},
		Provider: stubProvider{
			result: ports.Result{
				Classification: ports.ClassificationSuppressed,
				Summary:        "provider=stub result=suppressed script=policy_skip",
			},
		},
		PayloadLoader:  stubPayloadLoader{},
		Store:          store,
		Clock:          stubClock{now: fixedNow().Add(time.Minute)},
		AttemptTimeout: 15 * time.Second,
	})

	err := service.Execute(context.Background(), renderedWorkItem(t, 1))
	require.NoError(t, err)
	require.Len(t, store.inputs, 1)
	require.Equal(t, deliverydomain.StatusSuppressed, store.inputs[0].Delivery.Status)
	require.Equal(t, attempt.StatusProviderRejected, store.inputs[0].Attempt.Status)
}

func TestServiceExecuteMapsPermanentFailureToFailed(t *testing.T) {
	t.Parallel()

	store := &stubStore{}
	service := newTestService(t, Config{
		Renderer: &stubRenderer{},
		Provider: stubProvider{
			result: ports.Result{
				Classification: ports.ClassificationPermanentFailure,
				Summary:        "provider=smtp result=permanent_failure phase=data smtp_code=550",
			},
		},
		PayloadLoader:  stubPayloadLoader{},
		Store:          store,
		Clock:          stubClock{now: fixedNow().Add(time.Minute)},
		AttemptTimeout: 15 * time.Second,
	})

	err := service.Execute(context.Background(), renderedWorkItem(t, 1))
	require.NoError(t, err)
	require.Len(t, store.inputs, 1)
	require.Equal(t, deliverydomain.StatusFailed, store.inputs[0].Delivery.Status)
	require.Equal(t, attempt.StatusProviderRejected, store.inputs[0].Attempt.Status)
	require.Nil(t, store.inputs[0].DeadLetter)
}

func TestServiceExecuteBuildsRetryChainAndDeadLetter(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name                string
		attemptNo           int
		wantStatus          deliverydomain.Status
		wantAttemptStatus   attempt.Status
		wantNextAttemptNo   int
		wantNextDelay       time.Duration
		wantDeadLetterEntry bool
	}{
		{
			name:              "attempt one schedules retry after one minute",
			attemptNo:         1,
			wantStatus:        deliverydomain.StatusQueued,
			wantAttemptStatus: attempt.StatusTransportFailed,
			wantNextAttemptNo: 2,
			wantNextDelay:     time.Minute,
		},
		{
			name:              "attempt two schedules retry after five minutes",
			attemptNo:         2,
			wantStatus:        deliverydomain.StatusQueued,
			wantAttemptStatus: attempt.StatusTransportFailed,
			wantNextAttemptNo: 3,
			wantNextDelay:     5 * time.Minute,
		},
		{
			name:              "attempt three schedules retry after thirty minutes",
			attemptNo:         3,
			wantStatus:        deliverydomain.StatusQueued,
			wantAttemptStatus: attempt.StatusTransportFailed,
			wantNextAttemptNo: 4,
			wantNextDelay:     30 * time.Minute,
		},
		{
			name:                "attempt four becomes dead letter",
			attemptNo:           4,
			wantStatus:          deliverydomain.StatusDeadLetter,
			wantAttemptStatus:   attempt.StatusTransportFailed,
			wantDeadLetterEntry: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			store := &stubStore{}
			service := newTestService(t, Config{
				Renderer: &stubRenderer{},
				Provider: stubProvider{
					result: ports.Result{
						Classification: ports.ClassificationTransientFailure,
						Summary:        "provider=smtp result=transient_failure phase=data smtp_code=451",
						Details: map[string]string{
							"phase": "data",
						},
					},
				},
				PayloadLoader:  stubPayloadLoader{},
				Store:          store,
				Clock:          stubClock{now: fixedNow().Add(time.Minute)},
				AttemptTimeout: 15 * time.Second,
			})

			workItem := renderedWorkItem(t, tt.attemptNo)
			err := service.Execute(context.Background(), workItem)
			require.NoError(t, err)
			require.Len(t, store.inputs, 1)

			input := store.inputs[0]
			require.Equal(t, tt.wantStatus, input.Delivery.Status)
			require.Equal(t, tt.wantAttemptStatus, input.Attempt.Status)

			if tt.wantDeadLetterEntry {
				require.NotNil(t, input.DeadLetter)
				require.Nil(t, input.NextAttempt)
				require.Equal(t, "retry_exhausted", input.DeadLetter.FailureClassification)
				return
			}

			require.NotNil(t, input.NextAttempt)
			require.Nil(t, input.DeadLetter)
			require.Equal(t, tt.wantNextAttemptNo, input.NextAttempt.AttemptNo)
			require.Equal(t, input.Attempt.FinishedAt.Add(tt.wantNextDelay), input.NextAttempt.ScheduledFor)
		})
	}
}

func TestServiceExecuteClassifiesDeadlineExceededAsTimedOut(t *testing.T) {
	t.Parallel()

	store := &stubStore{}
	service := newTestService(t, Config{
		Renderer: &stubRenderer{},
		Provider: stubProvider{
			result: ports.Result{
				Classification: ports.ClassificationTransientFailure,
				Summary:        "provider=smtp result=transient_failure phase=context",
				Details: map[string]string{
					"error": "deadline_exceeded",
				},
			},
		},
		PayloadLoader:  stubPayloadLoader{},
		Store:          store,
		Clock:          stubClock{now: fixedNow().Add(time.Minute)},
		AttemptTimeout: 15 * time.Second,
	})

	err := service.Execute(context.Background(), renderedWorkItem(t, 1))
	require.NoError(t, err)
	require.Len(t, store.inputs, 1)
	require.Equal(t, attempt.StatusTimedOut, store.inputs[0].Attempt.Status)
	require.Equal(t, "deadline_exceeded", store.inputs[0].Attempt.ProviderClassification)
}

func TestServiceRecoverExpiredSchedulesTimedOutRetry(t *testing.T) {
	t.Parallel()

	store := &stubStore{}
	service := newTestService(t, Config{
		Renderer:       &stubRenderer{},
		Provider:       stubProvider{},
		PayloadLoader:  stubPayloadLoader{},
		Store:          store,
		Clock:          stubClock{now: fixedNow().Add(time.Minute)},
		AttemptTimeout: 15 * time.Second,
	})

	err := service.RecoverExpired(context.Background(), renderedWorkItem(t, 1))
	require.NoError(t, err)
	require.Len(t, store.inputs, 1)
	require.Equal(t, attempt.StatusTimedOut, store.inputs[0].Attempt.Status)
	require.Equal(t, "claim_ttl_expired", store.inputs[0].Attempt.ProviderClassification)
	require.Equal(t, "attempt claim TTL expired", store.inputs[0].Attempt.ProviderSummary)
	require.NotNil(t, store.inputs[0].NextAttempt)
}

func TestServiceExecuteRecordsMetricsAndLogsProviderResult(t *testing.T) {
	t.Parallel()

	store := &stubStore{}
	telemetry := &stubTelemetry{}
	loggerBuffer := &bytes.Buffer{}
	recorder := tracetest.NewSpanRecorder()
	tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))

	service := newTestService(t, Config{
		Renderer: &stubRenderer{},
		Provider: stubProvider{
			result: ports.Result{
				Classification: ports.ClassificationAccepted,
				Summary:        "provider=smtp result=accepted",
			},
		},
		PayloadLoader:  stubPayloadLoader{},
		Store:          store,
		Clock:          stubClock{now: fixedNow().Add(time.Minute)},
		Telemetry:      telemetry,
		TracerProvider: tracerProvider,
		Logger:         slog.New(slog.NewJSONHandler(loggerBuffer, nil)),
		AttemptTimeout: 15 * time.Second,
	})

	err := service.Execute(context.Background(), sendingTemplateWorkItem(t, 1))
	require.NoError(t, err)
	require.Equal(t, []string{"notification:sent"}, telemetry.statuses)
	require.Equal(t, []string{"notification:provider_accepted"}, telemetry.attempts)
	require.Equal(t, []string{"smtp:accepted"}, telemetry.providerDurations)
	require.Contains(t, loggerBuffer.String(), "\"delivery_id\":\"delivery-template-sending\"")
	require.Contains(t, loggerBuffer.String(), "\"source\":\"notification\"")
	require.Contains(t, loggerBuffer.String(), "\"template_id\":\"game.turn_ready\"")
	require.Contains(t, loggerBuffer.String(), "\"attempt_no\":1")
	require.Contains(t, loggerBuffer.String(), "\"otel_trace_id\":")
	require.True(t, hasExecuteSpanNamed(recorder.Ended(), "mail.provider_send"))
}

func TestServiceExecuteReturnsServiceUnavailableOnMissingPayload(t *testing.T) {
	t.Parallel()

	service := newTestService(t, Config{
		Renderer: &stubRenderer{},
		Provider: stubProvider{
			result: ports.Result{
				Classification: ports.ClassificationAccepted,
				Summary:        "provider=smtp result=accepted",
			},
		},
		PayloadLoader:  stubPayloadLoader{},
		Store:          &stubStore{},
		Clock:          stubClock{now: fixedNow().Add(time.Minute)},
		AttemptTimeout: 15 * time.Second,
	})

	workItem := renderedWorkItem(t, 1)
	workItem.Delivery.Attachments = []common.AttachmentMetadata{
		{Filename: "guide.txt", ContentType: "text/plain; charset=utf-8", SizeBytes: int64(len([]byte("read me")))},
	}
	require.NoError(t, workItem.Delivery.Validate())

	err := service.Execute(context.Background(), workItem)
	require.Error(t, err)
	require.ErrorIs(t, err, ErrServiceUnavailable)
}

type stubRenderer struct {
	result renderdelivery.Result
	err    error
	inputs []renderdelivery.Input
}

func (renderer *stubRenderer) Execute(_ context.Context, input renderdelivery.Input) (renderdelivery.Result, error) {
	renderer.inputs = append(renderer.inputs, input)
	return renderer.result, renderer.err
}

type stubProvider struct {
	result ports.Result
	err    error
	inputs []ports.Message
}

func (provider stubProvider) Send(_ context.Context, message ports.Message) (ports.Result, error) {
	provider.inputs = append(provider.inputs, message)
	return provider.result, provider.err
}

func (provider stubProvider) Close() error {
	return nil
}

type stubPayloadLoader struct {
	payload acceptgenericdelivery.DeliveryPayload
	found   bool
	err     error
}

func (loader stubPayloadLoader) LoadPayload(context.Context, common.DeliveryID) (acceptgenericdelivery.DeliveryPayload, bool, error) {
	return loader.payload, loader.found, loader.err
}

type stubStore struct {
	inputs []CommitStateInput
	err    error
}

func (store *stubStore) Commit(_ context.Context, input CommitStateInput) error {
	store.inputs = append(store.inputs, input)
	return store.err
}

type stubClock struct {
	now time.Time
}

func (clock stubClock) Now() time.Time {
	return clock.now
}

type stubTelemetry struct {
	statuses          []string
	attempts          []string
	providerDurations []string
}

func (telemetry *stubTelemetry) RecordDeliveryStatusTransition(_ context.Context, status string, source string) {
	telemetry.statuses = append(telemetry.statuses, source+":"+status)
}

func (telemetry *stubTelemetry) RecordAttemptOutcome(_ context.Context, status string, source string) {
	telemetry.attempts = append(telemetry.attempts, source+":"+status)
}

func (telemetry *stubTelemetry) RecordProviderSendDuration(_ context.Context, provider string, outcome string, _ time.Duration) {
	telemetry.providerDurations = append(telemetry.providerDurations, provider+":"+outcome)
}

func newTestService(t *testing.T, cfg Config) *Service {
	t.Helper()

	service, err := New(cfg)
	require.NoError(t, err)

	return service
}

func queuedTemplateWorkItem(t *testing.T) WorkItem {
	t.Helper()

	createdAt := fixedNow().Add(-time.Minute)
	deliveryRecord := deliverydomain.Delivery{
		DeliveryID:  common.DeliveryID("delivery-template"),
		Source:      deliverydomain.SourceNotification,
		PayloadMode: deliverydomain.PayloadModeTemplate,
		TemplateID:  common.TemplateID("game.turn_ready"),
		Envelope: deliverydomain.Envelope{
			To: []common.Email{common.Email("pilot@example.com")},
		},
		Locale: common.Locale("en"),
		TemplateVariables: map[string]any{
			"player": map[string]any{
				"name": "Pilot",
			},
			"turn_number": float64(54),
		},
		IdempotencyKey: common.IdempotencyKey("notification:delivery-template"),
		Status:         deliverydomain.StatusQueued,
		AttemptCount:   1,
		CreatedAt:      createdAt,
		UpdatedAt:      createdAt,
	}
	require.NoError(t, deliveryRecord.Validate())

	attemptRecord := attempt.Attempt{
		DeliveryID:   deliveryRecord.DeliveryID,
		AttemptNo:    1,
		ScheduledFor: createdAt,
		Status:       attempt.StatusScheduled,
	}
	require.NoError(t, attemptRecord.Validate())

	return WorkItem{
		Delivery: deliveryRecord,
		Attempt:  attemptRecord,
	}
}

func renderedWorkItem(t *testing.T, attemptNo int) WorkItem {
	t.Helper()

	createdAt := fixedNow().Add(-time.Duration(attemptNo) * time.Minute)
	deliveryRecord := deliverydomain.Delivery{
		DeliveryID:  common.DeliveryID("delivery-rendered"),
		Source:      deliverydomain.SourceNotification,
		PayloadMode: deliverydomain.PayloadModeRendered,
		Envelope: deliverydomain.Envelope{
			To: []common.Email{common.Email("pilot@example.com")},
		},
		Content: deliverydomain.Content{
			Subject:  "Turn ready",
			TextBody: "Turn 54 is ready.",
		},
		IdempotencyKey: common.IdempotencyKey("notification:delivery-rendered"),
		Status:         deliverydomain.StatusSending,
		AttemptCount:   attemptNo,
		CreatedAt:      createdAt,
		UpdatedAt:      createdAt.Add(time.Second),
	}
	require.NoError(t, deliveryRecord.Validate())

	scheduledFor := createdAt
	startedAt := scheduledFor.Add(5 * time.Second)
	attemptRecord := attempt.Attempt{
		DeliveryID:   deliveryRecord.DeliveryID,
		AttemptNo:    attemptNo,
		ScheduledFor: scheduledFor,
		StartedAt:    &startedAt,
		Status:       attempt.StatusInProgress,
	}
	require.NoError(t, attemptRecord.Validate())

	return WorkItem{
		Delivery: deliveryRecord,
		Attempt:  attemptRecord,
	}
}

func sendingTemplateWorkItem(t *testing.T, attemptNo int) WorkItem {
	t.Helper()

	createdAt := fixedNow().Add(-time.Duration(attemptNo) * time.Minute)
	deliveryRecord := deliverydomain.Delivery{
		DeliveryID:  common.DeliveryID("delivery-template-sending"),
		Source:      deliverydomain.SourceNotification,
		PayloadMode: deliverydomain.PayloadModeTemplate,
		TemplateID:  common.TemplateID("game.turn_ready"),
		Envelope: deliverydomain.Envelope{
			To: []common.Email{common.Email("pilot@example.com")},
		},
		Content: deliverydomain.Content{
			Subject:  "Turn ready",
			TextBody: "Turn 54 is ready.",
		},
		Locale: common.Locale("en"),
		TemplateVariables: map[string]any{
			"turn_number": float64(54),
		},
		IdempotencyKey: common.IdempotencyKey("notification:delivery-template-sending"),
		Status:         deliverydomain.StatusSending,
		AttemptCount:   attemptNo,
		CreatedAt:      createdAt,
		UpdatedAt:      createdAt.Add(time.Second),
	}
	require.NoError(t, deliveryRecord.Validate())

	scheduledFor := createdAt
	startedAt := scheduledFor.Add(5 * time.Second)
	attemptRecord := attempt.Attempt{
		DeliveryID:   deliveryRecord.DeliveryID,
		AttemptNo:    attemptNo,
		ScheduledFor: scheduledFor,
		StartedAt:    &startedAt,
		Status:       attempt.StatusInProgress,
	}
	require.NoError(t, attemptRecord.Validate())

	return WorkItem{
		Delivery: deliveryRecord,
		Attempt:  attemptRecord,
	}
}

func fixedNow() time.Time {
	return time.Unix(1_775_121_700, 0).UTC()
}

var _ Renderer = (*stubRenderer)(nil)
var _ ports.Provider = stubProvider{}
var _ PayloadLoader = stubPayloadLoader{}
var _ Store = (*stubStore)(nil)
var _ Telemetry = (*stubTelemetry)(nil)

func hasExecuteSpanNamed(spans []sdktrace.ReadOnlySpan, name string) bool {
	for _, span := range spans {
		if span.Name() == name {
			return true
		}
	}

	return false
}