feat: mail service

This commit is contained in:
Ilia Denisov
2026-04-17 18:39:16 +02:00
committed by GitHub
parent 23ffcb7535
commit 5b7593e6f6
183 changed files with 31215 additions and 248 deletions
@@ -0,0 +1,544 @@
// Package acceptauthdelivery implements synchronous durable acceptance of auth
// login-code deliveries.
package acceptauthdelivery
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log/slog"
"strings"
"time"
"galaxy/mail/internal/domain/attempt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"galaxy/mail/internal/domain/idempotency"
"galaxy/mail/internal/logging"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
oteltrace "go.opentelemetry.io/otel/trace"
)
var (
// ErrConflict reports that the idempotency scope already belongs to a
// different normalized auth request.
ErrConflict = errors.New("accept auth delivery conflict")
// ErrServiceUnavailable reports that durable acceptance could not be
// completed or recovered safely.
ErrServiceUnavailable = errors.New("accept auth delivery service unavailable")
)
const (
// AuthTemplateID is the dedicated template family used for auth login-code
// deliveries.
AuthTemplateID common.TemplateID = "auth.login_code"
maxCreateRetries = 3
tracerName = "galaxy/mail/acceptauthdelivery"
)
// Outcome identifies the stable auth-delivery acceptance outcome.
type Outcome string
const (
// OutcomeSent reports that the delivery was accepted into the durable
// internal pipeline.
OutcomeSent Outcome = "sent"
// OutcomeSuppressed reports that outward delivery was intentionally skipped
// while the auth flow remained success-shaped.
OutcomeSuppressed Outcome = "suppressed"
)
// IsKnown reports whether outcome belongs to the stable auth-delivery surface.
func (outcome Outcome) IsKnown() bool {
switch outcome {
case OutcomeSent, OutcomeSuppressed:
return true
default:
return false
}
}
// Result stores the coarse auth-delivery acceptance outcome.
type Result struct {
// Outcome stores the stable auth-delivery result.
Outcome Outcome
}
// Validate reports whether result contains a supported auth-delivery outcome.
func (result Result) Validate() error {
if !result.Outcome.IsKnown() {
return fmt.Errorf("accept auth delivery outcome %q is unsupported", result.Outcome)
}
return nil
}
// Input stores one normalized auth-delivery acceptance command.
type Input struct {
// IdempotencyKey stores the caller-owned stable deduplication key.
IdempotencyKey common.IdempotencyKey
// Email stores the normalized recipient mailbox.
Email common.Email
// Code stores the exact login code.
Code string
// Locale stores the canonical BCP 47 language tag selected upstream.
Locale common.Locale
}
// Validate reports whether input contains one valid auth-delivery command.
func (input Input) Validate() error {
if err := input.IdempotencyKey.Validate(); err != nil {
return fmt.Errorf("idempotency key: %w", err)
}
if err := input.Email.Validate(); err != nil {
return fmt.Errorf("email: %w", err)
}
if strings.TrimSpace(input.Code) == "" {
return errors.New("code must not be empty")
}
if strings.TrimSpace(input.Code) != input.Code {
return errors.New("code must not contain surrounding whitespace")
}
if err := input.Locale.Validate(); err != nil {
return fmt.Errorf("locale: %w", err)
}
return nil
}
// Fingerprint returns the stable idempotency fingerprint of input.
func (input Input) Fingerprint() (string, error) {
if err := input.Validate(); err != nil {
return "", err
}
normalized := struct {
IdempotencyKey string `json:"idempotency_key"`
Email string `json:"email"`
Code string `json:"code"`
Locale string `json:"locale"`
}{
IdempotencyKey: input.IdempotencyKey.String(),
Email: input.Email.String(),
Code: input.Code,
Locale: input.Locale.String(),
}
payload, err := json.Marshal(normalized)
if err != nil {
return "", fmt.Errorf("marshal auth-delivery fingerprint: %w", err)
}
sum := sha256.Sum256(payload)
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
// CreateAcceptanceInput stores the durable write set required for one
// auth-delivery acceptance attempt.
type CreateAcceptanceInput struct {
// Delivery stores the accepted delivery record.
Delivery deliverydomain.Delivery
// FirstAttempt stores the optional first scheduled attempt.
FirstAttempt *attempt.Attempt
// Idempotency stores the idempotency reservation bound to Delivery.
Idempotency idempotency.Record
}
// Validate reports whether input contains a consistent durable write set.
func (input CreateAcceptanceInput) Validate() error {
if err := input.Delivery.Validate(); err != nil {
return fmt.Errorf("delivery: %w", err)
}
if err := input.Idempotency.Validate(); err != nil {
return fmt.Errorf("idempotency: %w", err)
}
if input.Idempotency.DeliveryID != input.Delivery.DeliveryID {
return errors.New("idempotency delivery id must match delivery id")
}
if input.Idempotency.Source != input.Delivery.Source {
return errors.New("idempotency source must match delivery source")
}
if input.Idempotency.IdempotencyKey != input.Delivery.IdempotencyKey {
return errors.New("idempotency key must match delivery idempotency key")
}
switch {
case input.FirstAttempt == nil:
if input.Delivery.Status != deliverydomain.StatusSuppressed {
return errors.New("first attempt must not be nil unless delivery is suppressed")
}
case input.Delivery.Status == deliverydomain.StatusSuppressed:
return errors.New("suppressed delivery must not create first attempt")
default:
if err := input.FirstAttempt.Validate(); err != nil {
return fmt.Errorf("first attempt: %w", err)
}
if input.FirstAttempt.DeliveryID != input.Delivery.DeliveryID {
return errors.New("first attempt delivery id must match delivery id")
}
if input.FirstAttempt.Status != attempt.StatusScheduled {
return fmt.Errorf("first attempt status must be %q", attempt.StatusScheduled)
}
}
return nil
}
// Store describes the durable storage required by the auth-delivery use case.
type Store interface {
// CreateAcceptance stores the complete durable write set for one auth
// acceptance attempt. Implementations must wrap ErrConflict when the write
// set races with an already accepted idempotency scope.
CreateAcceptance(context.Context, CreateAcceptanceInput) error
// GetIdempotency loads the idempotency reservation for one auth-delivery
// scope.
GetIdempotency(context.Context, deliverydomain.Source, common.IdempotencyKey) (idempotency.Record, bool, error)
// GetDelivery loads one accepted delivery by its internal identifier.
GetDelivery(context.Context, common.DeliveryID) (deliverydomain.Delivery, bool, error)
}
// DeliveryIDGenerator describes the source of new internal delivery
// identifiers.
type DeliveryIDGenerator interface {
// NewDeliveryID returns one new internal delivery identifier.
NewDeliveryID() (common.DeliveryID, error)
}
// Clock provides the current wall-clock time.
type Clock interface {
// Now returns the current time.
Now() time.Time
}
// Telemetry records low-cardinality auth-delivery outcomes.
type Telemetry interface {
// RecordAuthDeliveryOutcome records one coarse auth-delivery outcome.
RecordAuthDeliveryOutcome(context.Context, string)
// RecordAcceptedAuthDelivery records one newly accepted auth delivery.
RecordAcceptedAuthDelivery(context.Context)
// RecordDeliveryStatusTransition records one durable delivery status
// transition.
RecordDeliveryStatusTransition(context.Context, string, string)
}
// Config stores the dependencies and policy switches used by Service.
type Config struct {
// Store owns the durable accepted state.
Store Store
// DeliveryIDGenerator builds internal delivery identifiers.
DeliveryIDGenerator DeliveryIDGenerator
// Clock provides wall-clock timestamps.
Clock Clock
// Telemetry records low-cardinality acceptance outcomes.
Telemetry Telemetry
// TracerProvider constructs the application span recorder used by the auth
// acceptance flow.
TracerProvider oteltrace.TracerProvider
// Logger writes structured auth acceptance logs.
Logger *slog.Logger
// IdempotencyTTL stores how long accepted idempotency scopes remain valid.
IdempotencyTTL time.Duration
// SuppressOutbound reports whether new auth-deliveries should be accepted
// directly as suppressed.
SuppressOutbound bool
}
// Service accepts auth login-code deliveries synchronously and durably.
type Service struct {
store Store
deliveryIDGenerator DeliveryIDGenerator
clock Clock
telemetry Telemetry
tracerProvider oteltrace.TracerProvider
logger *slog.Logger
idempotencyTTL time.Duration
suppressOutbound bool
}
// New constructs Service from cfg.
func New(cfg Config) (*Service, error) {
switch {
case cfg.Store == nil:
return nil, errors.New("new accept auth delivery service: nil store")
case cfg.DeliveryIDGenerator == nil:
return nil, errors.New("new accept auth delivery service: nil delivery id generator")
case cfg.Clock == nil:
return nil, errors.New("new accept auth delivery service: nil clock")
case cfg.IdempotencyTTL <= 0:
return nil, errors.New("new accept auth delivery service: non-positive idempotency ttl")
default:
tracerProvider := cfg.TracerProvider
if tracerProvider == nil {
tracerProvider = otel.GetTracerProvider()
}
logger := cfg.Logger
if logger == nil {
logger = slog.Default()
}
return &Service{
store: cfg.Store,
deliveryIDGenerator: cfg.DeliveryIDGenerator,
clock: cfg.Clock,
telemetry: cfg.Telemetry,
tracerProvider: tracerProvider,
logger: logger.With("component", "accept_auth_delivery"),
idempotencyTTL: cfg.IdempotencyTTL,
suppressOutbound: cfg.SuppressOutbound,
}, nil
}
}
// Execute accepts one auth login-code delivery command.
func (service *Service) Execute(ctx context.Context, input Input) (Result, error) {
if ctx == nil {
return Result{}, errors.New("accept auth delivery: nil context")
}
if service == nil {
return Result{}, errors.New("accept auth delivery: nil service")
}
if err := input.Validate(); err != nil {
return Result{}, fmt.Errorf("accept auth delivery: %w", err)
}
ctx, span := service.tracerProvider.Tracer(tracerName).Start(ctx, "mail.accept_auth_delivery")
defer span.End()
span.SetAttributes(
attribute.String("mail.locale", input.Locale.String()),
)
fingerprint, err := input.Fingerprint()
if err != nil {
return Result{}, fmt.Errorf("accept auth delivery: %w", err)
}
if result, handled, err := service.resolveReplay(ctx, input.IdempotencyKey, fingerprint); handled {
if err != nil {
service.recordOutcome(ctx, replayOutcomeForError(err))
return Result{}, err
}
service.recordOutcome(ctx, "duplicate")
return result, nil
}
for range maxCreateRetries {
createInput, result, err := service.buildCreateInput(input, fingerprint)
if err != nil {
return Result{}, fmt.Errorf("accept auth delivery: %w", err)
}
if err := service.store.CreateAcceptance(ctx, createInput); err != nil {
if !errors.Is(err, ErrConflict) {
service.recordOutcome(ctx, "service_unavailable")
return Result{}, fmt.Errorf("%w: create acceptance: %v", ErrServiceUnavailable, err)
}
if replayResult, handled, replayErr := service.resolveReplay(ctx, input.IdempotencyKey, fingerprint); handled {
if replayErr != nil {
service.recordOutcome(ctx, replayOutcomeForError(replayErr))
return Result{}, replayErr
}
service.recordOutcome(ctx, "duplicate")
return replayResult, nil
}
continue
}
service.recordOutcome(ctx, string(result.Outcome))
service.recordAcceptedDelivery(ctx)
service.recordStatusTransition(ctx, createInput.Delivery)
span.SetAttributes(
attribute.String("mail.delivery_id", createInput.Delivery.DeliveryID.String()),
attribute.String("mail.source", string(createInput.Delivery.Source)),
attribute.String("mail.status", string(createInput.Delivery.Status)),
)
logArgs := logging.DeliveryAttrs(createInput.Delivery)
logArgs = append(logArgs,
"status", string(createInput.Delivery.Status),
"outcome", string(result.Outcome),
"locale", input.Locale.String(),
)
logArgs = append(logArgs, logging.TraceAttrsFromContext(ctx)...)
service.logger.Info("auth delivery accepted", logArgs...)
return result, nil
}
service.recordOutcome(ctx, "service_unavailable")
return Result{}, fmt.Errorf("%w: delivery id conflict retry limit exceeded", ErrServiceUnavailable)
}
func (service *Service) buildCreateInput(input Input, fingerprint string) (CreateAcceptanceInput, Result, error) {
now := service.clock.Now().UTC().Truncate(time.Millisecond)
deliveryID, err := service.deliveryIDGenerator.NewDeliveryID()
if err != nil {
return CreateAcceptanceInput{}, Result{}, fmt.Errorf("%w: generate delivery id: %v", ErrServiceUnavailable, err)
}
deliveryRecord := deliverydomain.Delivery{
DeliveryID: deliveryID,
Source: deliverydomain.SourceAuthSession,
PayloadMode: deliverydomain.PayloadModeTemplate,
TemplateID: AuthTemplateID,
Envelope: deliverydomain.Envelope{To: []common.Email{input.Email}},
Locale: input.Locale,
TemplateVariables: map[string]any{
"code": input.Code,
},
IdempotencyKey: input.IdempotencyKey,
CreatedAt: now,
UpdatedAt: now,
}
result := Result{}
var firstAttempt *attempt.Attempt
if service.suppressOutbound {
deliveryRecord.Status = deliverydomain.StatusSuppressed
deliveryRecord.SuppressedAt = ptrTime(now)
result.Outcome = OutcomeSuppressed
} else {
deliveryRecord.Status = deliverydomain.StatusQueued
deliveryRecord.AttemptCount = 1
scheduledAttempt := attempt.Attempt{
DeliveryID: deliveryID,
AttemptNo: 1,
ScheduledFor: now,
Status: attempt.StatusScheduled,
}
firstAttempt = &scheduledAttempt
result.Outcome = OutcomeSent
}
if err := deliveryRecord.Validate(); err != nil {
return CreateAcceptanceInput{}, Result{}, fmt.Errorf("build auth delivery record: %w", err)
}
if err := result.Validate(); err != nil {
return CreateAcceptanceInput{}, Result{}, fmt.Errorf("build auth delivery result: %w", err)
}
createInput := CreateAcceptanceInput{
Delivery: deliveryRecord,
FirstAttempt: firstAttempt,
Idempotency: idempotency.Record{
Source: deliverydomain.SourceAuthSession,
IdempotencyKey: input.IdempotencyKey,
DeliveryID: deliveryID,
RequestFingerprint: fingerprint,
CreatedAt: now,
ExpiresAt: now.Add(service.idempotencyTTL),
},
}
if err := createInput.Validate(); err != nil {
return CreateAcceptanceInput{}, Result{}, fmt.Errorf("build auth create input: %w", err)
}
return createInput, result, nil
}
func (service *Service) recordAcceptedDelivery(ctx context.Context) {
if service == nil || service.telemetry == nil {
return
}
service.telemetry.RecordAcceptedAuthDelivery(ctx)
}
func (service *Service) recordStatusTransition(ctx context.Context, record deliverydomain.Delivery) {
if service == nil || service.telemetry == nil {
return
}
service.telemetry.RecordDeliveryStatusTransition(ctx, string(record.Status), string(record.Source))
}
func (service *Service) resolveReplay(ctx context.Context, key common.IdempotencyKey, fingerprint string) (Result, bool, error) {
record, found, err := service.store.GetIdempotency(ctx, deliverydomain.SourceAuthSession, key)
if err != nil {
return Result{}, true, fmt.Errorf("%w: load idempotency: %v", ErrServiceUnavailable, err)
}
if !found {
return Result{}, false, nil
}
if record.RequestFingerprint != fingerprint {
return Result{}, true, fmt.Errorf("%w: request conflicts with current state", ErrConflict)
}
deliveryRecord, found, err := service.store.GetDelivery(ctx, record.DeliveryID)
if err != nil {
return Result{}, true, fmt.Errorf("%w: load delivery: %v", ErrServiceUnavailable, err)
}
if !found {
return Result{}, true, fmt.Errorf("%w: delivery %q is missing for idempotency scope", ErrServiceUnavailable, record.DeliveryID)
}
return deriveReplayResult(deliveryRecord)
}
func deriveReplayResult(record deliverydomain.Delivery) (Result, bool, error) {
switch record.Status {
case deliverydomain.StatusSuppressed:
return Result{Outcome: OutcomeSuppressed}, true, nil
case deliverydomain.StatusAccepted,
deliverydomain.StatusQueued,
deliverydomain.StatusRendered,
deliverydomain.StatusSending,
deliverydomain.StatusSent,
deliverydomain.StatusFailed,
deliverydomain.StatusDeadLetter:
return Result{Outcome: OutcomeSent}, true, nil
default:
return Result{}, true, fmt.Errorf("%w: unsupported replay delivery status %q", ErrServiceUnavailable, record.Status)
}
}
func (service *Service) recordOutcome(ctx context.Context, outcome string) {
if service == nil || service.telemetry == nil || strings.TrimSpace(outcome) == "" {
return
}
service.telemetry.RecordAuthDeliveryOutcome(ctx, outcome)
}
func replayOutcomeForError(err error) string {
switch {
case errors.Is(err, ErrConflict):
return "conflict"
case errors.Is(err, ErrServiceUnavailable):
return "service_unavailable"
default:
return ""
}
}
func ptrTime(value time.Time) *time.Time {
return &value
}
@@ -0,0 +1,320 @@
package acceptauthdelivery
import (
"bytes"
"context"
"errors"
"log/slog"
"testing"
"time"
"galaxy/mail/internal/domain/attempt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"galaxy/mail/internal/domain/idempotency"
"github.com/stretchr/testify/require"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
)
func TestServiceExecuteAcceptsQueuedDelivery(t *testing.T) {
t.Parallel()
store := &stubStore{}
telemetry := &stubTelemetry{}
service := newTestService(t, Config{
Store: store,
DeliveryIDGenerator: stubIDGenerator{ids: []common.DeliveryID{"delivery-queued"}},
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
IdempotencyTTL: 7 * 24 * time.Hour,
})
result, err := service.Execute(context.Background(), validInput())
require.NoError(t, err)
require.Equal(t, Result{Outcome: OutcomeSent}, result)
require.Len(t, store.createInputs, 1)
require.NotNil(t, store.createInputs[0].FirstAttempt)
require.Equal(t, deliverydomain.StatusQueued, store.createInputs[0].Delivery.Status)
require.Equal(t, []string{"sent"}, telemetry.outcomes)
require.Equal(t, 1, telemetry.accepted)
require.Equal(t, []string{"authsession:queued"}, telemetry.statuses)
}
func TestServiceExecuteAcceptsSuppressedDelivery(t *testing.T) {
t.Parallel()
store := &stubStore{}
telemetry := &stubTelemetry{}
service := newTestService(t, Config{
Store: store,
DeliveryIDGenerator: stubIDGenerator{ids: []common.DeliveryID{"delivery-suppressed"}},
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
IdempotencyTTL: 7 * 24 * time.Hour,
SuppressOutbound: true,
})
result, err := service.Execute(context.Background(), validInput())
require.NoError(t, err)
require.Equal(t, Result{Outcome: OutcomeSuppressed}, result)
require.Len(t, store.createInputs, 1)
require.Nil(t, store.createInputs[0].FirstAttempt)
require.Equal(t, deliverydomain.StatusSuppressed, store.createInputs[0].Delivery.Status)
require.Equal(t, []string{"suppressed"}, telemetry.outcomes)
require.Equal(t, 1, telemetry.accepted)
require.Equal(t, []string{"authsession:suppressed"}, telemetry.statuses)
}
func TestServiceExecuteReturnsStableDuplicateResult(t *testing.T) {
t.Parallel()
input := validInput()
fingerprint, err := input.Fingerprint()
require.NoError(t, err)
store := &stubStore{
idempotencyRecord: &idempotency.Record{
Source: deliverydomain.SourceAuthSession,
IdempotencyKey: input.IdempotencyKey,
DeliveryID: common.DeliveryID("delivery-existing"),
RequestFingerprint: fingerprint,
CreatedAt: fixedNow(),
ExpiresAt: fixedNow().Add(7 * 24 * time.Hour),
},
deliveryRecord: &deliverydomain.Delivery{
DeliveryID: common.DeliveryID("delivery-existing"),
Source: deliverydomain.SourceAuthSession,
PayloadMode: deliverydomain.PayloadModeTemplate,
TemplateID: AuthTemplateID,
Envelope: deliverydomain.Envelope{
To: []common.Email{input.Email},
},
Locale: input.Locale,
TemplateVariables: map[string]any{
"code": input.Code,
},
IdempotencyKey: input.IdempotencyKey,
Status: deliverydomain.StatusSuppressed,
CreatedAt: fixedNow(),
UpdatedAt: fixedNow(),
SuppressedAt: ptrTime(fixedNow()),
},
}
require.NoError(t, store.idempotencyRecord.Validate())
require.NoError(t, store.deliveryRecord.Validate())
telemetry := &stubTelemetry{}
service := newTestService(t, Config{
Store: store,
DeliveryIDGenerator: stubIDGenerator{},
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
IdempotencyTTL: 7 * 24 * time.Hour,
})
result, err := service.Execute(context.Background(), input)
require.NoError(t, err)
require.Equal(t, Result{Outcome: OutcomeSuppressed}, result)
require.Empty(t, store.createInputs)
require.Equal(t, []string{"duplicate"}, telemetry.outcomes)
}
func TestServiceExecuteRejectsConflictingReplay(t *testing.T) {
t.Parallel()
input := validInput()
store := &stubStore{
idempotencyRecord: &idempotency.Record{
Source: deliverydomain.SourceAuthSession,
IdempotencyKey: input.IdempotencyKey,
DeliveryID: common.DeliveryID("delivery-existing"),
RequestFingerprint: "sha256:other",
CreatedAt: fixedNow(),
ExpiresAt: fixedNow().Add(7 * 24 * time.Hour),
},
}
require.NoError(t, store.idempotencyRecord.Validate())
telemetry := &stubTelemetry{}
service := newTestService(t, Config{
Store: store,
DeliveryIDGenerator: stubIDGenerator{},
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
IdempotencyTTL: 7 * 24 * time.Hour,
})
_, err := service.Execute(context.Background(), input)
require.Error(t, err)
require.ErrorIs(t, err, ErrConflict)
require.Equal(t, []string{"conflict"}, telemetry.outcomes)
}
func TestServiceExecuteReturnsServiceUnavailableOnCreateFailure(t *testing.T) {
t.Parallel()
telemetry := &stubTelemetry{}
service := newTestService(t, Config{
Store: &stubStore{
createErr: errors.New("redis unavailable"),
},
DeliveryIDGenerator: stubIDGenerator{ids: []common.DeliveryID{"delivery-queued"}},
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
IdempotencyTTL: 7 * 24 * time.Hour,
})
_, err := service.Execute(context.Background(), validInput())
require.Error(t, err)
require.ErrorIs(t, err, ErrServiceUnavailable)
require.Equal(t, []string{"service_unavailable"}, telemetry.outcomes)
}
func TestServiceExecuteLogsAcceptedDeliveryAndCreatesSpan(t *testing.T) {
t.Parallel()
store := &stubStore{}
telemetry := &stubTelemetry{}
loggerBuffer := &bytes.Buffer{}
recorder := tracetest.NewSpanRecorder()
tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
service := newTestService(t, Config{
Store: store,
DeliveryIDGenerator: stubIDGenerator{ids: []common.DeliveryID{"delivery-queued"}},
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
TracerProvider: tracerProvider,
Logger: slog.New(slog.NewJSONHandler(loggerBuffer, nil)),
IdempotencyTTL: 7 * 24 * time.Hour,
})
_, err := service.Execute(context.Background(), validInput())
require.NoError(t, err)
require.Contains(t, loggerBuffer.String(), "\"delivery_id\":\"delivery-queued\"")
require.Contains(t, loggerBuffer.String(), "\"source\":\"authsession\"")
require.Contains(t, loggerBuffer.String(), "\"template_id\":\"auth.login_code\"")
require.Contains(t, loggerBuffer.String(), "\"otel_trace_id\":")
require.True(t, hasSpanNamed(recorder.Ended(), "mail.accept_auth_delivery"))
}
func TestInputFingerprintStableForEquivalentInput(t *testing.T) {
t.Parallel()
first := validInput()
second := validInput()
firstFingerprint, err := first.Fingerprint()
require.NoError(t, err)
secondFingerprint, err := second.Fingerprint()
require.NoError(t, err)
require.Equal(t, firstFingerprint, secondFingerprint)
}
type stubStore struct {
createInputs []CreateAcceptanceInput
createErr error
idempotencyRecord *idempotency.Record
deliveryRecord *deliverydomain.Delivery
}
func (store *stubStore) CreateAcceptance(_ context.Context, input CreateAcceptanceInput) error {
store.createInputs = append(store.createInputs, input)
return store.createErr
}
func (store *stubStore) GetIdempotency(_ context.Context, _ deliverydomain.Source, _ common.IdempotencyKey) (idempotency.Record, bool, error) {
if store.idempotencyRecord == nil {
return idempotency.Record{}, false, nil
}
return *store.idempotencyRecord, true, nil
}
func (store *stubStore) GetDelivery(_ context.Context, _ common.DeliveryID) (deliverydomain.Delivery, bool, error) {
if store.deliveryRecord == nil {
return deliverydomain.Delivery{}, false, nil
}
return *store.deliveryRecord, true, nil
}
type stubIDGenerator struct {
ids []common.DeliveryID
}
func (generator stubIDGenerator) NewDeliveryID() (common.DeliveryID, error) {
if len(generator.ids) == 0 {
return "", errors.New("no delivery ids left")
}
return generator.ids[0], nil
}
type stubClock struct {
now time.Time
}
func (clock stubClock) Now() time.Time {
return clock.now
}
type stubTelemetry struct {
outcomes []string
accepted int
statuses []string
}
func (telemetry *stubTelemetry) RecordAuthDeliveryOutcome(_ context.Context, outcome string) {
telemetry.outcomes = append(telemetry.outcomes, outcome)
}
func (telemetry *stubTelemetry) RecordAcceptedAuthDelivery(context.Context) {
telemetry.accepted++
}
func (telemetry *stubTelemetry) RecordDeliveryStatusTransition(_ context.Context, status string, source string) {
telemetry.statuses = append(telemetry.statuses, source+":"+status)
}
func newTestService(t *testing.T, cfg Config) *Service {
t.Helper()
service, err := New(cfg)
require.NoError(t, err)
return service
}
func validInput() Input {
locale, err := common.ParseLocale("en")
if err != nil {
panic(err)
}
return Input{
IdempotencyKey: common.IdempotencyKey("challenge-123"),
Email: common.Email("pilot@example.com"),
Code: "123456",
Locale: locale,
}
}
func fixedNow() time.Time {
return time.Unix(1_775_121_700, 0).UTC()
}
func hasSpanNamed(spans []sdktrace.ReadOnlySpan, name string) bool {
for _, span := range spans {
if span.Name() == name {
return true
}
}
return false
}
var _ = attempt.Attempt{}
@@ -0,0 +1,598 @@
// Package acceptgenericdelivery implements durable asynchronous acceptance of
// generic delivery commands consumed from Redis Streams.
package acceptgenericdelivery
import (
"context"
"encoding/base64"
"errors"
"fmt"
"log/slog"
"strings"
"time"
"galaxy/mail/internal/api/streamcommand"
"galaxy/mail/internal/domain/attempt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"galaxy/mail/internal/domain/idempotency"
"galaxy/mail/internal/logging"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
oteltrace "go.opentelemetry.io/otel/trace"
)
var (
// ErrConflict reports that the idempotency scope already belongs to a
// different normalized generic request.
ErrConflict = errors.New("accept generic delivery conflict")
// ErrServiceUnavailable reports that durable generic acceptance could not
// be completed or recovered safely.
ErrServiceUnavailable = errors.New("accept generic delivery service unavailable")
)
const tracerName = "galaxy/mail/acceptgenericdelivery"
// Outcome identifies the coarse generic-delivery acceptance outcome.
type Outcome string
const (
// OutcomeAccepted reports that the command was durably accepted into the
// internal delivery pipeline.
OutcomeAccepted Outcome = "accepted"
// OutcomeDuplicate reports that the command matched an already accepted
// idempotent request and therefore became a no-op replay.
OutcomeDuplicate Outcome = "duplicate"
)
// IsKnown reports whether outcome belongs to the supported generic-acceptance
// outcome surface.
func (outcome Outcome) IsKnown() bool {
switch outcome {
case OutcomeAccepted, OutcomeDuplicate:
return true
default:
return false
}
}
// Result stores the coarse generic-delivery acceptance outcome.
type Result struct {
// Outcome stores the stable generic-acceptance result.
Outcome Outcome
}
// Validate reports whether result contains a supported generic-acceptance
// outcome.
func (result Result) Validate() error {
if !result.Outcome.IsKnown() {
return fmt.Errorf("accept generic delivery outcome %q is unsupported", result.Outcome)
}
return nil
}
// AttachmentPayload stores one durably persisted raw attachment payload owned
// by a generic delivery.
type AttachmentPayload struct {
// Filename stores the user-facing attachment filename.
Filename string
// ContentType stores the MIME media type used for SMTP body construction.
ContentType string
// ContentBase64 stores the exact accepted inline base64 payload.
ContentBase64 string
// SizeBytes stores the decoded attachment size in bytes.
SizeBytes int64
}
// Validate reports whether payload contains a complete attachment body.
func (payload AttachmentPayload) Validate() error {
metadata := common.AttachmentMetadata{
Filename: payload.Filename,
ContentType: payload.ContentType,
SizeBytes: payload.SizeBytes,
}
if err := metadata.Validate(); err != nil {
return err
}
decoded, err := base64.StdEncoding.DecodeString(payload.ContentBase64)
if err != nil {
return fmt.Errorf("attachment content_base64 must be valid base64: %w", err)
}
if int64(len(decoded)) != payload.SizeBytes {
return fmt.Errorf(
"attachment size bytes must match decoded content size: got %d, want %d",
payload.SizeBytes,
len(decoded),
)
}
return nil
}
// DeliveryPayload stores the raw attachment payloads that must survive stream
// offset advancement.
type DeliveryPayload struct {
// DeliveryID identifies the owning accepted delivery.
DeliveryID common.DeliveryID
// Attachments stores the raw inline attachment payloads.
Attachments []AttachmentPayload
}
// Validate reports whether payload contains a complete attachment bundle.
func (payload DeliveryPayload) Validate() error {
if err := payload.DeliveryID.Validate(); err != nil {
return fmt.Errorf("delivery payload delivery id: %w", err)
}
if len(payload.Attachments) == 0 {
return fmt.Errorf("delivery payload attachments must not be empty")
}
for index, attachment := range payload.Attachments {
if err := attachment.Validate(); err != nil {
return fmt.Errorf("delivery payload attachments[%d]: %w", index, err)
}
}
return nil
}
// CreateAcceptanceInput stores the durable write set required for one
// generic-delivery acceptance attempt.
type CreateAcceptanceInput struct {
// Delivery stores the accepted delivery record.
Delivery deliverydomain.Delivery
// FirstAttempt stores the first scheduled attempt.
FirstAttempt attempt.Attempt
// DeliveryPayload stores the optional raw attachment payload bundle.
DeliveryPayload *DeliveryPayload
// Idempotency stores the idempotency reservation bound to Delivery.
Idempotency idempotency.Record
}
// Validate reports whether input contains a consistent durable write set.
func (input CreateAcceptanceInput) Validate() error {
if err := input.Delivery.Validate(); err != nil {
return fmt.Errorf("delivery: %w", err)
}
if err := input.FirstAttempt.Validate(); err != nil {
return fmt.Errorf("first attempt: %w", err)
}
if input.FirstAttempt.DeliveryID != input.Delivery.DeliveryID {
return errors.New("first attempt delivery id must match delivery id")
}
if input.FirstAttempt.Status != attempt.StatusScheduled {
return fmt.Errorf("first attempt status must be %q", attempt.StatusScheduled)
}
if err := input.Idempotency.Validate(); err != nil {
return fmt.Errorf("idempotency: %w", err)
}
if input.Idempotency.DeliveryID != input.Delivery.DeliveryID {
return errors.New("idempotency delivery id must match delivery id")
}
if input.Idempotency.Source != input.Delivery.Source {
return errors.New("idempotency source must match delivery source")
}
if input.Idempotency.IdempotencyKey != input.Delivery.IdempotencyKey {
return errors.New("idempotency key must match delivery idempotency key")
}
if input.DeliveryPayload != nil {
if err := input.DeliveryPayload.Validate(); err != nil {
return fmt.Errorf("delivery payload: %w", err)
}
if input.DeliveryPayload.DeliveryID != input.Delivery.DeliveryID {
return errors.New("delivery payload delivery id must match delivery id")
}
}
return nil
}
// Store describes the durable storage required by the generic-delivery use
// case.
type Store interface {
// CreateAcceptance stores the complete durable write set for one generic
// acceptance attempt. Implementations must wrap ErrConflict when the write
// set races with an already accepted idempotency scope or delivery key.
CreateAcceptance(context.Context, CreateAcceptanceInput) error
// GetIdempotency loads the idempotency reservation for one generic-delivery
// scope.
GetIdempotency(context.Context, deliverydomain.Source, common.IdempotencyKey) (idempotency.Record, bool, error)
// GetDelivery loads one accepted delivery by its identifier.
GetDelivery(context.Context, common.DeliveryID) (deliverydomain.Delivery, bool, error)
}
// Clock provides the current wall-clock time.
type Clock interface {
// Now returns the current time.
Now() time.Time
}
// Telemetry records low-cardinality generic-delivery outcomes.
type Telemetry interface {
// RecordGenericDeliveryOutcome records one coarse generic-acceptance
// outcome.
RecordGenericDeliveryOutcome(context.Context, string)
// RecordAcceptedGenericDelivery records one newly accepted generic
// delivery.
RecordAcceptedGenericDelivery(context.Context)
// RecordDeliveryStatusTransition records one durable delivery status
// transition.
RecordDeliveryStatusTransition(context.Context, string, string)
}
// Config stores the dependencies and policy used by Service.
type Config struct {
// Store owns the durable accepted state.
Store Store
// Clock provides wall-clock timestamps.
Clock Clock
// Telemetry records low-cardinality acceptance outcomes.
Telemetry Telemetry
// TracerProvider constructs the application span recorder used by the
// generic acceptance flow.
TracerProvider oteltrace.TracerProvider
// Logger writes structured generic acceptance logs.
Logger *slog.Logger
// IdempotencyTTL stores how long accepted idempotency scopes remain valid.
IdempotencyTTL time.Duration
}
// Service durably accepts generic asynchronous delivery commands.
type Service struct {
store Store
clock Clock
telemetry Telemetry
tracerProvider oteltrace.TracerProvider
logger *slog.Logger
idempotencyTTL time.Duration
}
// New constructs Service from cfg.
func New(cfg Config) (*Service, error) {
switch {
case cfg.Store == nil:
return nil, errors.New("new accept generic delivery service: nil store")
case cfg.Clock == nil:
return nil, errors.New("new accept generic delivery service: nil clock")
case cfg.IdempotencyTTL <= 0:
return nil, errors.New("new accept generic delivery service: non-positive idempotency ttl")
default:
tracerProvider := cfg.TracerProvider
if tracerProvider == nil {
tracerProvider = otel.GetTracerProvider()
}
logger := cfg.Logger
if logger == nil {
logger = slog.Default()
}
return &Service{
store: cfg.Store,
clock: cfg.Clock,
telemetry: cfg.Telemetry,
tracerProvider: tracerProvider,
logger: logger.With("component", "accept_generic_delivery"),
idempotencyTTL: cfg.IdempotencyTTL,
}, nil
}
}
// Execute accepts one normalized generic-delivery command.
func (service *Service) Execute(ctx context.Context, command streamcommand.Command) (Result, error) {
if ctx == nil {
return Result{}, errors.New("accept generic delivery: nil context")
}
if service == nil {
return Result{}, errors.New("accept generic delivery: nil service")
}
if err := command.Validate(); err != nil {
return Result{}, fmt.Errorf("accept generic delivery: %w", err)
}
ctx, span := service.tracerProvider.Tracer(tracerName).Start(ctx, "mail.accept_generic_delivery")
defer span.End()
span.SetAttributes(
attribute.String("mail.delivery_id", command.DeliveryID.String()),
attribute.String("mail.source", string(command.Source)),
attribute.String("mail.payload_mode", string(command.PayloadMode)),
)
if strings.TrimSpace(command.TraceID) != "" {
span.SetAttributes(attribute.String("mail.command_trace_id", command.TraceID))
}
if !command.TemplateID.IsZero() {
span.SetAttributes(attribute.String("mail.template_id", command.TemplateID.String()))
}
fingerprint, err := command.Fingerprint()
if err != nil {
return Result{}, fmt.Errorf("accept generic delivery: %w", err)
}
if result, handled, err := service.resolveReplay(ctx, command, fingerprint); handled {
if err != nil {
service.recordOutcome(ctx, replayOutcomeForError(err))
return Result{}, err
}
service.recordOutcome(ctx, string(result.Outcome))
return result, nil
}
createInput, result, err := service.buildCreateInput(command, fingerprint)
if err != nil {
return Result{}, fmt.Errorf("accept generic delivery: %w", err)
}
if err := service.store.CreateAcceptance(ctx, createInput); err != nil {
if !errors.Is(err, ErrConflict) {
service.recordOutcome(ctx, "service_unavailable")
return Result{}, fmt.Errorf("%w: create acceptance: %v", ErrServiceUnavailable, err)
}
if replayResult, handled, replayErr := service.resolveReplay(ctx, command, fingerprint); handled {
if replayErr != nil {
service.recordOutcome(ctx, replayOutcomeForError(replayErr))
return Result{}, replayErr
}
service.recordOutcome(ctx, string(replayResult.Outcome))
return replayResult, nil
}
service.recordOutcome(ctx, "service_unavailable")
return Result{}, fmt.Errorf("%w: create acceptance conflict without replay state", ErrServiceUnavailable)
}
service.recordOutcome(ctx, string(result.Outcome))
service.recordAcceptedDelivery(ctx)
service.recordStatusTransition(ctx, createInput.Delivery)
span.SetAttributes(
attribute.String("mail.status", string(createInput.Delivery.Status)),
attribute.String("mail.outcome", string(result.Outcome)),
)
logArgs := logging.CommandAttrs(command)
logArgs = append(logArgs,
"status", string(createInput.Delivery.Status),
"outcome", string(result.Outcome),
"payload_mode", string(command.PayloadMode),
)
logArgs = append(logArgs, logging.TraceAttrsFromContext(ctx)...)
service.logger.Info("generic delivery accepted", logArgs...)
return result, nil
}
func (service *Service) buildCreateInput(command streamcommand.Command, fingerprint string) (CreateAcceptanceInput, Result, error) {
now := service.clock.Now().UTC().Truncate(time.Millisecond)
deliveryRecord := deliverydomain.Delivery{
DeliveryID: command.DeliveryID,
Source: command.Source,
PayloadMode: command.PayloadMode,
Envelope: command.Envelope,
Attachments: attachmentMetadata(command.Attachments),
IdempotencyKey: command.IdempotencyKey,
Status: deliverydomain.StatusQueued,
AttemptCount: 1,
CreatedAt: now,
UpdatedAt: now,
}
switch command.PayloadMode {
case deliverydomain.PayloadModeRendered:
deliveryRecord.Content = deliverydomain.Content{
Subject: command.Subject,
TextBody: command.TextBody,
HTMLBody: command.HTMLBody,
}
case deliverydomain.PayloadModeTemplate:
deliveryRecord.TemplateID = command.TemplateID
deliveryRecord.Locale = command.Locale
deliveryRecord.TemplateVariables = cloneJSONObject(command.Variables)
default:
return CreateAcceptanceInput{}, Result{}, fmt.Errorf("build generic delivery record: unsupported payload mode %q", command.PayloadMode)
}
if err := deliveryRecord.Validate(); err != nil {
return CreateAcceptanceInput{}, Result{}, fmt.Errorf("build generic delivery record: %w", err)
}
firstAttempt := attempt.Attempt{
DeliveryID: command.DeliveryID,
AttemptNo: 1,
ScheduledFor: now,
Status: attempt.StatusScheduled,
}
if err := firstAttempt.Validate(); err != nil {
return CreateAcceptanceInput{}, Result{}, fmt.Errorf("build generic first attempt: %w", err)
}
createInput := CreateAcceptanceInput{
Delivery: deliveryRecord,
FirstAttempt: firstAttempt,
Idempotency: idempotency.Record{
Source: command.Source,
IdempotencyKey: command.IdempotencyKey,
DeliveryID: command.DeliveryID,
RequestFingerprint: fingerprint,
CreatedAt: now,
ExpiresAt: now.Add(service.idempotencyTTL),
},
}
if len(command.Attachments) > 0 {
createInput.DeliveryPayload = &DeliveryPayload{
DeliveryID: command.DeliveryID,
Attachments: attachmentPayloads(command.Attachments),
}
}
if err := createInput.Validate(); err != nil {
return CreateAcceptanceInput{}, Result{}, fmt.Errorf("build generic create input: %w", err)
}
result := Result{Outcome: OutcomeAccepted}
if err := result.Validate(); err != nil {
return CreateAcceptanceInput{}, Result{}, fmt.Errorf("build generic delivery result: %w", err)
}
return createInput, result, nil
}
func (service *Service) resolveReplay(ctx context.Context, command streamcommand.Command, fingerprint string) (Result, bool, error) {
record, found, err := service.store.GetIdempotency(ctx, command.Source, command.IdempotencyKey)
if err != nil {
return Result{}, true, fmt.Errorf("%w: load idempotency: %v", ErrServiceUnavailable, err)
}
if !found {
return Result{}, false, nil
}
if record.RequestFingerprint != fingerprint {
return Result{}, true, fmt.Errorf("%w: request conflicts with current state", ErrConflict)
}
deliveryRecord, found, err := service.store.GetDelivery(ctx, record.DeliveryID)
if err != nil {
return Result{}, true, fmt.Errorf("%w: load delivery: %v", ErrServiceUnavailable, err)
}
if !found {
return Result{}, true, fmt.Errorf("%w: delivery %q is missing for idempotency scope", ErrServiceUnavailable, record.DeliveryID)
}
if deliveryRecord.DeliveryID != command.DeliveryID {
return Result{}, true, fmt.Errorf("%w: idempotency delivery %q mismatches command delivery %q", ErrServiceUnavailable, deliveryRecord.DeliveryID, command.DeliveryID)
}
return deriveReplayResult(deliveryRecord)
}
func deriveReplayResult(record deliverydomain.Delivery) (Result, bool, error) {
switch record.Status {
case deliverydomain.StatusAccepted,
deliverydomain.StatusQueued,
deliverydomain.StatusRendered,
deliverydomain.StatusSending,
deliverydomain.StatusSent,
deliverydomain.StatusSuppressed,
deliverydomain.StatusFailed,
deliverydomain.StatusDeadLetter:
return Result{Outcome: OutcomeDuplicate}, true, nil
default:
return Result{}, true, fmt.Errorf("%w: unsupported replay delivery status %q", ErrServiceUnavailable, record.Status)
}
}
func (service *Service) recordAcceptedDelivery(ctx context.Context) {
if service == nil || service.telemetry == nil {
return
}
service.telemetry.RecordAcceptedGenericDelivery(ctx)
}
func (service *Service) recordStatusTransition(ctx context.Context, record deliverydomain.Delivery) {
if service == nil || service.telemetry == nil {
return
}
service.telemetry.RecordDeliveryStatusTransition(ctx, string(record.Status), string(record.Source))
}
func (service *Service) recordOutcome(ctx context.Context, outcome string) {
if service == nil || service.telemetry == nil || strings.TrimSpace(outcome) == "" {
return
}
service.telemetry.RecordGenericDeliveryOutcome(ctx, outcome)
}
func replayOutcomeForError(err error) string {
switch {
case errors.Is(err, ErrConflict):
return "conflict"
case errors.Is(err, ErrServiceUnavailable):
return "service_unavailable"
default:
return ""
}
}
func attachmentMetadata(values []streamcommand.Attachment) []common.AttachmentMetadata {
if values == nil {
return nil
}
result := make([]common.AttachmentMetadata, len(values))
for index, value := range values {
result[index] = common.AttachmentMetadata{
Filename: value.Filename,
ContentType: value.ContentType,
SizeBytes: value.SizeBytes,
}
}
return result
}
func attachmentPayloads(values []streamcommand.Attachment) []AttachmentPayload {
result := make([]AttachmentPayload, len(values))
for index, value := range values {
result[index] = AttachmentPayload{
Filename: value.Filename,
ContentType: value.ContentType,
ContentBase64: value.ContentBase64,
SizeBytes: value.SizeBytes,
}
}
return result
}
func cloneJSONObject(value map[string]any) map[string]any {
if value == nil {
return nil
}
cloned := make(map[string]any, len(value))
for key, item := range value {
cloned[key] = cloneJSONValue(item)
}
return cloned
}
func cloneJSONValue(value any) any {
switch typed := value.(type) {
case map[string]any:
cloned := make(map[string]any, len(typed))
for key, item := range typed {
cloned[key] = cloneJSONValue(item)
}
return cloned
case []any:
cloned := make([]any, len(typed))
for index, item := range typed {
cloned[index] = cloneJSONValue(item)
}
return cloned
default:
return typed
}
}
@@ -0,0 +1,319 @@
package acceptgenericdelivery
import (
"bytes"
"context"
"encoding/base64"
"errors"
"log/slog"
"testing"
"time"
"galaxy/mail/internal/api/streamcommand"
"galaxy/mail/internal/domain/attempt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"galaxy/mail/internal/domain/idempotency"
"github.com/stretchr/testify/require"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
)
func TestServiceExecuteAcceptsRenderedDelivery(t *testing.T) {
t.Parallel()
store := &stubStore{}
telemetry := &stubTelemetry{}
service := newTestService(t, Config{
Store: store,
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
IdempotencyTTL: 7 * 24 * time.Hour,
})
result, err := service.Execute(context.Background(), validRenderedCommand(t))
require.NoError(t, err)
require.Equal(t, Result{Outcome: OutcomeAccepted}, result)
require.Len(t, store.createInputs, 1)
require.Equal(t, deliverydomain.StatusQueued, store.createInputs[0].Delivery.Status)
require.Equal(t, deliverydomain.PayloadModeRendered, store.createInputs[0].Delivery.PayloadMode)
require.Equal(t, "Turn ready", store.createInputs[0].Delivery.Content.Subject)
require.NotNil(t, store.createInputs[0].DeliveryPayload)
require.Equal(t, []string{"accepted"}, telemetry.outcomes)
require.Equal(t, 1, telemetry.accepted)
require.Equal(t, []string{"notification:queued"}, telemetry.statuses)
}
func TestServiceExecuteAcceptsTemplateDelivery(t *testing.T) {
t.Parallel()
store := &stubStore{}
telemetry := &stubTelemetry{}
service := newTestService(t, Config{
Store: store,
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
IdempotencyTTL: 7 * 24 * time.Hour,
})
result, err := service.Execute(context.Background(), validTemplateCommand(t))
require.NoError(t, err)
require.Equal(t, Result{Outcome: OutcomeAccepted}, result)
require.Len(t, store.createInputs, 1)
require.Nil(t, store.createInputs[0].DeliveryPayload)
require.Equal(t, common.TemplateID("game.turn_ready"), store.createInputs[0].Delivery.TemplateID)
require.Equal(t, map[string]any{
"turn_number": float64(54),
"player": map[string]any{
"name": "Pilot",
},
}, store.createInputs[0].Delivery.TemplateVariables)
require.Equal(t, []string{"accepted"}, telemetry.outcomes)
require.Equal(t, 1, telemetry.accepted)
require.Equal(t, []string{"notification:queued"}, telemetry.statuses)
}
func TestServiceExecuteReturnsStableDuplicateResult(t *testing.T) {
t.Parallel()
command := validTemplateCommand(t)
fingerprint, err := command.Fingerprint()
require.NoError(t, err)
store := &stubStore{
idempotencyRecord: &idempotency.Record{
Source: deliverydomain.SourceNotification,
IdempotencyKey: command.IdempotencyKey,
DeliveryID: command.DeliveryID,
RequestFingerprint: fingerprint,
CreatedAt: fixedNow(),
ExpiresAt: fixedNow().Add(7 * 24 * time.Hour),
},
deliveryRecord: &deliverydomain.Delivery{
DeliveryID: command.DeliveryID,
Source: deliverydomain.SourceNotification,
PayloadMode: deliverydomain.PayloadModeTemplate,
TemplateID: command.TemplateID,
Envelope: command.Envelope,
Locale: command.Locale,
TemplateVariables: map[string]any{
"turn_number": float64(54),
"player": map[string]any{
"name": "Pilot",
},
},
IdempotencyKey: command.IdempotencyKey,
Status: deliverydomain.StatusQueued,
AttemptCount: 1,
CreatedAt: fixedNow(),
UpdatedAt: fixedNow(),
},
}
require.NoError(t, store.idempotencyRecord.Validate())
require.NoError(t, store.deliveryRecord.Validate())
telemetry := &stubTelemetry{}
service := newTestService(t, Config{
Store: store,
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
IdempotencyTTL: 7 * 24 * time.Hour,
})
result, err := service.Execute(context.Background(), command)
require.NoError(t, err)
require.Equal(t, Result{Outcome: OutcomeDuplicate}, result)
require.Empty(t, store.createInputs)
require.Equal(t, []string{"duplicate"}, telemetry.outcomes)
}
func TestServiceExecuteRejectsConflictingReplay(t *testing.T) {
t.Parallel()
command := validRenderedCommand(t)
store := &stubStore{
idempotencyRecord: &idempotency.Record{
Source: deliverydomain.SourceNotification,
IdempotencyKey: command.IdempotencyKey,
DeliveryID: command.DeliveryID,
RequestFingerprint: "sha256:other",
CreatedAt: fixedNow(),
ExpiresAt: fixedNow().Add(7 * 24 * time.Hour),
},
}
require.NoError(t, store.idempotencyRecord.Validate())
telemetry := &stubTelemetry{}
service := newTestService(t, Config{
Store: store,
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
IdempotencyTTL: 7 * 24 * time.Hour,
})
_, err := service.Execute(context.Background(), command)
require.Error(t, err)
require.ErrorIs(t, err, ErrConflict)
require.Equal(t, []string{"conflict"}, telemetry.outcomes)
}
func TestServiceExecuteReturnsServiceUnavailableOnCreateFailure(t *testing.T) {
t.Parallel()
telemetry := &stubTelemetry{}
service := newTestService(t, Config{
Store: &stubStore{
createErr: errors.New("redis unavailable"),
},
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
IdempotencyTTL: 7 * 24 * time.Hour,
})
_, err := service.Execute(context.Background(), validRenderedCommand(t))
require.Error(t, err)
require.ErrorIs(t, err, ErrServiceUnavailable)
require.Equal(t, []string{"service_unavailable"}, telemetry.outcomes)
}
func TestServiceExecuteLogsAcceptedDeliveryAndCreatesSpan(t *testing.T) {
t.Parallel()
store := &stubStore{}
telemetry := &stubTelemetry{}
loggerBuffer := &bytes.Buffer{}
recorder := tracetest.NewSpanRecorder()
tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
command := validTemplateCommand(t)
command.TraceID = "trace-123"
service := newTestService(t, Config{
Store: store,
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
TracerProvider: tracerProvider,
Logger: slog.New(slog.NewJSONHandler(loggerBuffer, nil)),
IdempotencyTTL: 7 * 24 * time.Hour,
})
_, err := service.Execute(context.Background(), command)
require.NoError(t, err)
require.Contains(t, loggerBuffer.String(), "\"delivery_id\":\"mail-124\"")
require.Contains(t, loggerBuffer.String(), "\"source\":\"notification\"")
require.Contains(t, loggerBuffer.String(), "\"template_id\":\"game.turn_ready\"")
require.Contains(t, loggerBuffer.String(), "\"trace_id\":\"trace-123\"")
require.Contains(t, loggerBuffer.String(), "\"otel_trace_id\":")
require.True(t, hasSpanNamed(recorder.Ended(), "mail.accept_generic_delivery"))
}
type stubStore struct {
createInputs []CreateAcceptanceInput
createErr error
idempotencyRecord *idempotency.Record
deliveryRecord *deliverydomain.Delivery
}
func (store *stubStore) CreateAcceptance(_ context.Context, input CreateAcceptanceInput) error {
store.createInputs = append(store.createInputs, input)
return store.createErr
}
func (store *stubStore) GetIdempotency(_ context.Context, _ deliverydomain.Source, _ common.IdempotencyKey) (idempotency.Record, bool, error) {
if store.idempotencyRecord == nil {
return idempotency.Record{}, false, nil
}
return *store.idempotencyRecord, true, nil
}
func (store *stubStore) GetDelivery(_ context.Context, _ common.DeliveryID) (deliverydomain.Delivery, bool, error) {
if store.deliveryRecord == nil {
return deliverydomain.Delivery{}, false, nil
}
return *store.deliveryRecord, true, nil
}
type stubClock struct {
now time.Time
}
func (clock stubClock) Now() time.Time {
return clock.now
}
type stubTelemetry struct {
outcomes []string
accepted int
statuses []string
}
func (telemetry *stubTelemetry) RecordGenericDeliveryOutcome(_ context.Context, outcome string) {
telemetry.outcomes = append(telemetry.outcomes, outcome)
}
func (telemetry *stubTelemetry) RecordAcceptedGenericDelivery(context.Context) {
telemetry.accepted++
}
func (telemetry *stubTelemetry) RecordDeliveryStatusTransition(_ context.Context, status string, source string) {
telemetry.statuses = append(telemetry.statuses, source+":"+status)
}
func newTestService(t *testing.T, cfg Config) *Service {
t.Helper()
service, err := New(cfg)
require.NoError(t, err)
return service
}
func validRenderedCommand(t *testing.T) streamcommand.Command {
t.Helper()
command, err := streamcommand.DecodeCommand(map[string]any{
"delivery_id": "mail-123",
"source": "notification",
"payload_mode": "rendered",
"idempotency_key": "notification:mail-123",
"requested_at_ms": "1775121700000",
"payload_json": `{"to":["pilot@example.com"],"cc":[],"bcc":[],"reply_to":["noreply@example.com"],"subject":"Turn ready","text_body":"Turn 54 is ready.","html_body":"<p>Turn 54 is ready.</p>","attachments":[{"filename":"report.txt","content_type":"text/plain","content_base64":"` + base64.StdEncoding.EncodeToString([]byte("report")) + `"}]}`,
})
require.NoError(t, err)
return command
}
func validTemplateCommand(t *testing.T) streamcommand.Command {
t.Helper()
command, err := streamcommand.DecodeCommand(map[string]any{
"delivery_id": "mail-124",
"source": "notification",
"payload_mode": "template",
"idempotency_key": "notification:mail-124",
"requested_at_ms": "1775121700001",
"payload_json": `{"to":["pilot@example.com"],"cc":[],"bcc":[],"reply_to":[],"template_id":"game.turn_ready","locale":"fr-FR","variables":{"turn_number":54,"player":{"name":"Pilot"}},"attachments":[]}`,
})
require.NoError(t, err)
return command
}
func fixedNow() time.Time {
return time.Unix(1_775_121_700, 0).UTC()
}
func hasSpanNamed(spans []sdktrace.ReadOnlySpan, name string) bool {
for _, span := range spans {
if span.Name() == name {
return true
}
}
return false
}
var _ = attempt.Attempt{}
@@ -0,0 +1,781 @@
// Package executeattempt implements provider execution, retry planning, and
// terminal state handling for claimed delivery attempts.
package executeattempt
import (
"context"
"encoding/base64"
"errors"
"fmt"
"log/slog"
"strings"
"time"
"galaxy/mail/internal/domain/attempt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"galaxy/mail/internal/logging"
"galaxy/mail/internal/ports"
"galaxy/mail/internal/service/acceptgenericdelivery"
"galaxy/mail/internal/service/renderdelivery"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
oteltrace "go.opentelemetry.io/otel/trace"
)
var (
// ErrServiceUnavailable reports that attempt execution could not safely
// load or persist durable state.
ErrServiceUnavailable = errors.New("execute attempt service unavailable")
)
var retryDelays = [...]time.Duration{
time.Minute,
5 * time.Minute,
30 * time.Minute,
}
const (
retryExhaustedClassification = "retry_exhausted"
retryRecoveryHint = "check SMTP connectivity"
claimTTLClassification = "claim_ttl_expired"
claimTTLSummary = "attempt claim TTL expired"
deadlineExceededDetail = "deadline_exceeded"
tracerName = "galaxy/mail/executeattempt"
)
// WorkItem stores one delivery together with the concrete attempt that should
// be prepared, executed, or recovered.
type WorkItem struct {
// Delivery stores the owning logical delivery record.
Delivery deliverydomain.Delivery
// Attempt stores the concrete delivery attempt record.
Attempt attempt.Attempt
}
// ValidateForPreparation reports whether item can be prepared for claim-time
// rendering decisions.
func (item WorkItem) ValidateForPreparation() error {
if err := item.validateCommon(); err != nil {
return err
}
if item.Attempt.Status != attempt.StatusScheduled {
return fmt.Errorf("work attempt status must be %q", attempt.StatusScheduled)
}
switch item.Delivery.Status {
case deliverydomain.StatusQueued, deliverydomain.StatusRendered:
default:
return fmt.Errorf(
"work delivery status must be %q or %q",
deliverydomain.StatusQueued,
deliverydomain.StatusRendered,
)
}
return nil
}
// ValidateForExecution reports whether item represents one claimed in-flight
// provider execution.
func (item WorkItem) ValidateForExecution() error {
if err := item.validateCommon(); err != nil {
return err
}
if item.Delivery.Status != deliverydomain.StatusSending {
return fmt.Errorf("work delivery status must be %q", deliverydomain.StatusSending)
}
if item.Attempt.Status != attempt.StatusInProgress {
return fmt.Errorf("work attempt status must be %q", attempt.StatusInProgress)
}
return nil
}
func (item WorkItem) validateCommon() error {
if err := item.Delivery.Validate(); err != nil {
return fmt.Errorf("work delivery: %w", err)
}
if err := item.Attempt.Validate(); err != nil {
return fmt.Errorf("work attempt: %w", err)
}
if item.Attempt.DeliveryID != item.Delivery.DeliveryID {
return errors.New("work attempt delivery id must match delivery id")
}
if item.Delivery.AttemptCount != item.Attempt.AttemptNo {
return errors.New("work delivery attempt count must match attempt number")
}
return nil
}
// CommitStateInput stores one complete durable attempt outcome mutation.
type CommitStateInput struct {
// Delivery stores the mutated delivery record.
Delivery deliverydomain.Delivery
// Attempt stores the terminal current attempt record.
Attempt attempt.Attempt
// NextAttempt stores the optional next scheduled retry attempt.
NextAttempt *attempt.Attempt
// DeadLetter stores the optional dead-letter record when Delivery becomes
// `dead_letter`.
DeadLetter *deliverydomain.DeadLetterEntry
}
// Validate reports whether input stores one complete and internally
// consistent durable mutation.
func (input CommitStateInput) Validate() error {
if err := input.Delivery.Validate(); err != nil {
return fmt.Errorf("delivery: %w", err)
}
if err := input.Attempt.Validate(); err != nil {
return fmt.Errorf("attempt: %w", err)
}
if !input.Attempt.Status.IsTerminal() {
return errors.New("attempt status must be terminal")
}
if input.Attempt.DeliveryID != input.Delivery.DeliveryID {
return errors.New("attempt delivery id must match delivery id")
}
if input.Delivery.LastAttemptStatus != input.Attempt.Status {
return errors.New("delivery last attempt status must match attempt status")
}
if input.NextAttempt != nil {
if err := input.NextAttempt.Validate(); err != nil {
return fmt.Errorf("next attempt: %w", err)
}
if input.NextAttempt.DeliveryID != input.Delivery.DeliveryID {
return errors.New("next attempt delivery id must match delivery id")
}
if input.NextAttempt.Status != attempt.StatusScheduled {
return fmt.Errorf("next attempt status must be %q", attempt.StatusScheduled)
}
if input.Delivery.Status != deliverydomain.StatusQueued {
return fmt.Errorf("delivery status with next attempt must be %q", deliverydomain.StatusQueued)
}
if input.Delivery.AttemptCount != input.NextAttempt.AttemptNo {
return errors.New("delivery attempt count must match next attempt number")
}
if input.NextAttempt.AttemptNo != input.Attempt.AttemptNo+1 {
return errors.New("next attempt number must increment current attempt number")
}
if input.DeadLetter != nil {
return errors.New("next attempt and dead-letter entry are mutually exclusive")
}
} else if input.Delivery.AttemptCount != input.Attempt.AttemptNo {
return errors.New("delivery attempt count must match current attempt number without next attempt")
}
if err := deliverydomain.ValidateDeadLetterState(input.Delivery, input.DeadLetter); err != nil {
return fmt.Errorf("dead-letter state: %w", err)
}
switch input.Delivery.Status {
case deliverydomain.StatusSent:
if input.Attempt.Status != attempt.StatusProviderAccepted {
return fmt.Errorf("sent delivery requires attempt status %q", attempt.StatusProviderAccepted)
}
case deliverydomain.StatusSuppressed, deliverydomain.StatusFailed:
if input.Attempt.Status != attempt.StatusProviderRejected {
return fmt.Errorf(
"%s delivery requires attempt status %q",
input.Delivery.Status,
attempt.StatusProviderRejected,
)
}
case deliverydomain.StatusQueued:
if input.NextAttempt == nil {
return errors.New("queued delivery requires next attempt")
}
switch input.Attempt.Status {
case attempt.StatusTransportFailed, attempt.StatusTimedOut:
default:
return fmt.Errorf(
"queued delivery requires attempt status %q or %q",
attempt.StatusTransportFailed,
attempt.StatusTimedOut,
)
}
case deliverydomain.StatusDeadLetter:
switch input.Attempt.Status {
case attempt.StatusTransportFailed, attempt.StatusTimedOut:
default:
return fmt.Errorf(
"dead-letter delivery requires attempt status %q or %q",
attempt.StatusTransportFailed,
attempt.StatusTimedOut,
)
}
default:
return fmt.Errorf("unsupported delivery status %q for commit input", input.Delivery.Status)
}
return nil
}
// Renderer materializes template-mode deliveries before a scheduler claims an
// attempt for outbound execution.
type Renderer interface {
// Execute renders or terminally fails one queued template-mode delivery.
Execute(context.Context, renderdelivery.Input) (renderdelivery.Result, error)
}
// PayloadLoader loads raw attachment payloads for a delivery.
type PayloadLoader interface {
// LoadPayload returns the stored attachment payload bundle when one exists.
LoadPayload(context.Context, common.DeliveryID) (acceptgenericdelivery.DeliveryPayload, bool, error)
}
// Store persists durable attempt execution outcomes.
type Store interface {
// Commit applies one complete durable attempt outcome mutation.
Commit(context.Context, CommitStateInput) error
}
// Clock provides wall-clock time.
type Clock interface {
// Now returns the current time.
Now() time.Time
}
// Telemetry records low-cardinality attempt-execution metrics.
type Telemetry interface {
// RecordDeliveryStatusTransition records one durable delivery status
// transition.
RecordDeliveryStatusTransition(context.Context, string, string)
// RecordAttemptOutcome records one durable terminal attempt outcome.
RecordAttemptOutcome(context.Context, string, string)
// RecordProviderSendDuration records one provider-send latency sample.
RecordProviderSendDuration(context.Context, string, string, time.Duration)
}
// Config stores the dependencies used by Service.
type Config struct {
// Renderer stores the template renderer used during pre-claim preparation.
Renderer Renderer
// Provider stores the outbound provider adapter.
Provider ports.Provider
// PayloadLoader loads raw attachment payloads for SMTP construction.
PayloadLoader PayloadLoader
// Store persists durable attempt execution outcomes.
Store Store
// Clock provides wall-clock timestamps.
Clock Clock
// Telemetry records low-cardinality attempt-execution metrics.
Telemetry Telemetry
// TracerProvider constructs the application span recorder used by provider
// sends.
TracerProvider oteltrace.TracerProvider
// Logger writes structured attempt-execution logs.
Logger *slog.Logger
// AttemptTimeout bounds one provider execution budget.
AttemptTimeout time.Duration
}
// Service prepares template deliveries, executes claimed attempts, and
// applies retry policy.
type Service struct {
renderer Renderer
provider ports.Provider
payloadLoader PayloadLoader
store Store
clock Clock
telemetry Telemetry
tracerProvider oteltrace.TracerProvider
logger *slog.Logger
attemptTimeout time.Duration
}
// New constructs Service from cfg.
func New(cfg Config) (*Service, error) {
switch {
case cfg.Renderer == nil:
return nil, errors.New("new execute attempt service: nil renderer")
case cfg.Provider == nil:
return nil, errors.New("new execute attempt service: nil provider")
case cfg.PayloadLoader == nil:
return nil, errors.New("new execute attempt service: nil payload loader")
case cfg.Store == nil:
return nil, errors.New("new execute attempt service: nil store")
case cfg.Clock == nil:
return nil, errors.New("new execute attempt service: nil clock")
case cfg.AttemptTimeout <= 0:
return nil, errors.New("new execute attempt service: non-positive attempt timeout")
default:
tracerProvider := cfg.TracerProvider
if tracerProvider == nil {
tracerProvider = otel.GetTracerProvider()
}
logger := cfg.Logger
if logger == nil {
logger = slog.Default()
}
return &Service{
renderer: cfg.Renderer,
provider: cfg.Provider,
payloadLoader: cfg.PayloadLoader,
store: cfg.Store,
clock: cfg.Clock,
telemetry: cfg.Telemetry,
tracerProvider: tracerProvider,
logger: logger.With("component", "execute_attempt"),
attemptTimeout: cfg.AttemptTimeout,
}, nil
}
}
// Prepare renders one template-mode queued delivery when its content has not
// been materialized yet. The boolean result reports whether the scheduler may
// proceed to claim the attempt.
func (service *Service) Prepare(ctx context.Context, item WorkItem) (bool, error) {
if ctx == nil {
return false, errors.New("prepare execute attempt: nil context")
}
if service == nil {
return false, errors.New("prepare execute attempt: nil service")
}
if err := item.ValidateForPreparation(); err != nil {
return false, fmt.Errorf("prepare execute attempt: %w", err)
}
if item.Delivery.PayloadMode != deliverydomain.PayloadModeTemplate {
return true, nil
}
if item.Delivery.Status == deliverydomain.StatusRendered {
return true, nil
}
if err := item.Delivery.Content.ValidateMaterialized(); err == nil {
return true, nil
}
result, err := service.renderer.Execute(ctx, renderdelivery.Input{
Delivery: item.Delivery,
Attempt: item.Attempt,
})
if err != nil {
return false, fmt.Errorf("prepare execute attempt: %w", err)
}
if result.Outcome == renderdelivery.OutcomeFailed {
return false, nil
}
return true, nil
}
// Execute runs one claimed in-progress attempt through the provider and
// durably records the resulting outcome.
func (service *Service) Execute(ctx context.Context, item WorkItem) error {
if ctx == nil {
return errors.New("execute attempt: nil context")
}
if service == nil {
return errors.New("execute attempt: nil service")
}
if err := item.ValidateForExecution(); err != nil {
return fmt.Errorf("execute attempt: %w", err)
}
message, err := service.buildMessage(ctx, item.Delivery)
if err != nil {
return err
}
sendStartedAt := time.Now()
sendCtx, span := service.tracerProvider.Tracer(tracerName).Start(
ctx,
"mail.provider_send",
oteltrace.WithAttributes(
attribute.String("mail.delivery_id", item.Delivery.DeliveryID.String()),
attribute.String("mail.source", string(item.Delivery.Source)),
attribute.Int("mail.attempt_no", item.Attempt.AttemptNo),
),
)
if !item.Delivery.TemplateID.IsZero() {
span.SetAttributes(attribute.String("mail.template_id", item.Delivery.TemplateID.String()))
}
providerCtx, cancel := context.WithTimeout(sendCtx, service.attemptTimeout)
defer cancel()
defer span.End()
result, err := service.provider.Send(providerCtx, message)
if err != nil {
span.RecordError(err)
return fmt.Errorf("execute attempt: send provider message: %w", err)
}
if err := result.Validate(); err != nil {
span.RecordError(err)
return fmt.Errorf("execute attempt: provider result: %w", err)
}
providerName := providerNameFromSummary(result.Summary)
sendDuration := time.Since(sendStartedAt)
service.recordProviderSendDuration(sendCtx, providerName, string(result.Classification), sendDuration)
span.SetAttributes(
attribute.String("mail.provider", providerName),
attribute.String("mail.provider_outcome", string(result.Classification)),
attribute.String("mail.provider_summary", result.Summary),
)
commit, err := service.commitForProviderResult(item, result)
if err != nil {
return err
}
if err := service.store.Commit(ctx, commit); err != nil {
return fmt.Errorf("%w: commit attempt outcome: %v", ErrServiceUnavailable, err)
}
service.recordCommitMetrics(sendCtx, commit, item.Delivery.Source)
service.logProviderResult(sendCtx, item, result, commit, providerName, sendDuration)
return nil
}
// RecoverExpired marks one stale in-progress attempt as expired and applies
// the same retry policy used for runtime timeouts.
func (service *Service) RecoverExpired(ctx context.Context, item WorkItem) error {
if ctx == nil {
return errors.New("recover expired attempt: nil context")
}
if service == nil {
return errors.New("recover expired attempt: nil service")
}
if err := item.ValidateForExecution(); err != nil {
return fmt.Errorf("recover expired attempt: %w", err)
}
commit, err := service.commitForTimeout(item, claimTTLClassification, claimTTLSummary)
if err != nil {
return err
}
if err := service.store.Commit(ctx, commit); err != nil {
return fmt.Errorf("%w: commit recovered attempt outcome: %v", ErrServiceUnavailable, err)
}
service.recordCommitMetrics(ctx, commit, item.Delivery.Source)
return nil
}
func (service *Service) buildMessage(ctx context.Context, deliveryRecord deliverydomain.Delivery) (ports.Message, error) {
message := ports.Message{
Envelope: deliveryRecord.Envelope,
Content: deliveryRecord.Content,
}
if err := message.Content.ValidateMaterialized(); err != nil {
return ports.Message{}, fmt.Errorf("execute attempt: delivery content: %w", err)
}
if len(deliveryRecord.Attachments) == 0 {
if err := message.Validate(); err != nil {
return ports.Message{}, fmt.Errorf("execute attempt: provider message: %w", err)
}
return message, nil
}
payload, found, err := service.payloadLoader.LoadPayload(ctx, deliveryRecord.DeliveryID)
if err != nil {
return ports.Message{}, fmt.Errorf("%w: load delivery payload: %v", ErrServiceUnavailable, err)
}
if !found {
return ports.Message{}, fmt.Errorf("%w: delivery payload %q is missing", ErrServiceUnavailable, deliveryRecord.DeliveryID)
}
if len(payload.Attachments) != len(deliveryRecord.Attachments) {
return ports.Message{}, fmt.Errorf(
"%w: delivery payload attachment count %d mismatches delivery attachment count %d",
ErrServiceUnavailable,
len(payload.Attachments),
len(deliveryRecord.Attachments),
)
}
message.Attachments = make([]ports.Attachment, len(payload.Attachments))
for index, attachmentPayload := range payload.Attachments {
metadata := deliveryRecord.Attachments[index]
if metadata.Filename != attachmentPayload.Filename ||
metadata.ContentType != attachmentPayload.ContentType ||
metadata.SizeBytes != attachmentPayload.SizeBytes {
return ports.Message{}, fmt.Errorf(
"%w: delivery payload attachment %d metadata mismatches delivery audit metadata",
ErrServiceUnavailable,
index,
)
}
content, err := base64.StdEncoding.DecodeString(attachmentPayload.ContentBase64)
if err != nil {
return ports.Message{}, fmt.Errorf(
"%w: decode delivery payload attachment %d: %v",
ErrServiceUnavailable,
index,
err,
)
}
message.Attachments[index] = ports.Attachment{
Metadata: metadata,
Content: content,
}
}
if err := message.Validate(); err != nil {
return ports.Message{}, fmt.Errorf("execute attempt: provider message: %w", err)
}
return message, nil
}
func (service *Service) commitForProviderResult(item WorkItem, result ports.Result) (CommitStateInput, error) {
switch result.Classification {
case ports.ClassificationAccepted:
return service.commitTerminal(item, attempt.StatusProviderAccepted, deliverydomain.StatusSent, result.Summary, "")
case ports.ClassificationSuppressed:
return service.commitTerminal(item, attempt.StatusProviderRejected, deliverydomain.StatusSuppressed, result.Summary, "suppressed")
case ports.ClassificationPermanentFailure:
return service.commitTerminal(item, attempt.StatusProviderRejected, deliverydomain.StatusFailed, result.Summary, "permanent_failure")
case ports.ClassificationTransientFailure:
classification := attempt.StatusTransportFailed
providerClassification := "transient_failure"
if result.Details["error"] == deadlineExceededDetail {
classification = attempt.StatusTimedOut
providerClassification = deadlineExceededDetail
}
return service.commitForRetryableResult(item, classification, providerClassification, result.Summary)
default:
return CommitStateInput{}, fmt.Errorf("execute attempt: unsupported provider classification %q", result.Classification)
}
}
func (service *Service) commitForTimeout(item WorkItem, providerClassification string, providerSummary string) (CommitStateInput, error) {
return service.commitForRetryableResult(item, attempt.StatusTimedOut, providerClassification, providerSummary)
}
func (service *Service) commitForRetryableResult(
item WorkItem,
attemptStatus attempt.Status,
providerClassification string,
providerSummary string,
) (CommitStateInput, error) {
finishedAt := normalizedFinishedAt(service.clock.Now(), item.Attempt)
currentAttempt := item.Attempt
currentAttempt.Status = attemptStatus
currentAttempt.FinishedAt = ptrTime(finishedAt)
currentAttempt.ProviderClassification = providerClassification
currentAttempt.ProviderSummary = providerSummary
if err := currentAttempt.Validate(); err != nil {
return CommitStateInput{}, fmt.Errorf("execute attempt: build terminal attempt: %w", err)
}
nextDelay, ok := retryDelayForAttempt(currentAttempt.AttemptNo)
if ok {
nextScheduledFor := finishedAt.Add(nextDelay)
nextAttempt := attempt.Attempt{
DeliveryID: item.Delivery.DeliveryID,
AttemptNo: currentAttempt.AttemptNo + 1,
ScheduledFor: nextScheduledFor,
Status: attempt.StatusScheduled,
}
if err := nextAttempt.Validate(); err != nil {
return CommitStateInput{}, fmt.Errorf("execute attempt: build next attempt: %w", err)
}
deliveryRecord := item.Delivery
deliveryRecord.Status = deliverydomain.StatusQueued
deliveryRecord.AttemptCount = nextAttempt.AttemptNo
deliveryRecord.LastAttemptStatus = currentAttempt.Status
deliveryRecord.ProviderSummary = providerSummary
deliveryRecord.UpdatedAt = finishedAt
if err := deliveryRecord.Validate(); err != nil {
return CommitStateInput{}, fmt.Errorf("execute attempt: build queued delivery: %w", err)
}
input := CommitStateInput{
Delivery: deliveryRecord,
Attempt: currentAttempt,
NextAttempt: &nextAttempt,
}
if err := input.Validate(); err != nil {
return CommitStateInput{}, fmt.Errorf("execute attempt: build queued commit: %w", err)
}
return input, nil
}
deliveryRecord := item.Delivery
deliveryRecord.Status = deliverydomain.StatusDeadLetter
deliveryRecord.LastAttemptStatus = currentAttempt.Status
deliveryRecord.ProviderSummary = providerSummary
deliveryRecord.UpdatedAt = finishedAt
deliveryRecord.DeadLetteredAt = ptrTime(finishedAt)
if err := deliveryRecord.Validate(); err != nil {
return CommitStateInput{}, fmt.Errorf("execute attempt: build dead-letter delivery: %w", err)
}
deadLetter := &deliverydomain.DeadLetterEntry{
DeliveryID: deliveryRecord.DeliveryID,
FinalAttemptNo: currentAttempt.AttemptNo,
FailureClassification: retryExhaustedClassification,
ProviderSummary: providerSummary,
CreatedAt: finishedAt,
RecoveryHint: retryRecoveryHint,
}
input := CommitStateInput{
Delivery: deliveryRecord,
Attempt: currentAttempt,
DeadLetter: deadLetter,
}
if err := input.Validate(); err != nil {
return CommitStateInput{}, fmt.Errorf("execute attempt: build dead-letter commit: %w", err)
}
return input, nil
}
func (service *Service) commitTerminal(
item WorkItem,
attemptStatus attempt.Status,
deliveryStatus deliverydomain.Status,
providerSummary string,
providerClassification string,
) (CommitStateInput, error) {
finishedAt := normalizedFinishedAt(service.clock.Now(), item.Attempt)
currentAttempt := item.Attempt
currentAttempt.Status = attemptStatus
currentAttempt.FinishedAt = ptrTime(finishedAt)
currentAttempt.ProviderClassification = providerClassification
currentAttempt.ProviderSummary = providerSummary
if err := currentAttempt.Validate(); err != nil {
return CommitStateInput{}, fmt.Errorf("execute attempt: build terminal attempt: %w", err)
}
deliveryRecord := item.Delivery
deliveryRecord.Status = deliveryStatus
deliveryRecord.LastAttemptStatus = currentAttempt.Status
deliveryRecord.ProviderSummary = providerSummary
deliveryRecord.UpdatedAt = finishedAt
switch deliveryStatus {
case deliverydomain.StatusSent:
deliveryRecord.SentAt = ptrTime(finishedAt)
case deliverydomain.StatusSuppressed:
deliveryRecord.SuppressedAt = ptrTime(finishedAt)
case deliverydomain.StatusFailed:
deliveryRecord.FailedAt = ptrTime(finishedAt)
}
if err := deliveryRecord.Validate(); err != nil {
return CommitStateInput{}, fmt.Errorf("execute attempt: build terminal delivery: %w", err)
}
input := CommitStateInput{
Delivery: deliveryRecord,
Attempt: currentAttempt,
}
if err := input.Validate(); err != nil {
return CommitStateInput{}, fmt.Errorf("execute attempt: build terminal commit: %w", err)
}
return input, nil
}
func retryDelayForAttempt(attemptNo int) (time.Duration, bool) {
if attemptNo < 1 || attemptNo > len(retryDelays) {
return 0, false
}
return retryDelays[attemptNo-1], true
}
func normalizedFinishedAt(now time.Time, record attempt.Attempt) time.Time {
finishedAt := now.UTC().Truncate(time.Millisecond)
if record.StartedAt != nil && finishedAt.Before(*record.StartedAt) {
return *record.StartedAt
}
return finishedAt
}
func ptrTime(value time.Time) *time.Time {
return &value
}
func (service *Service) recordCommitMetrics(ctx context.Context, commit CommitStateInput, source deliverydomain.Source) {
if service == nil || service.telemetry == nil {
return
}
service.telemetry.RecordDeliveryStatusTransition(ctx, string(commit.Delivery.Status), string(source))
service.telemetry.RecordAttemptOutcome(ctx, string(commit.Attempt.Status), string(source))
}
func (service *Service) recordProviderSendDuration(ctx context.Context, provider string, outcome string, duration time.Duration) {
if service == nil || service.telemetry == nil {
return
}
service.telemetry.RecordProviderSendDuration(ctx, provider, outcome, duration)
}
func (service *Service) logProviderResult(
ctx context.Context,
item WorkItem,
result ports.Result,
commit CommitStateInput,
providerName string,
sendDuration time.Duration,
) {
logArgs := logging.DeliveryAttemptAttrs(item.Delivery, item.Attempt)
logArgs = append(logArgs,
"provider", providerName,
"provider_outcome", string(result.Classification),
"provider_summary", result.Summary,
"delivery_status", string(commit.Delivery.Status),
"attempt_status", string(commit.Attempt.Status),
"duration_ms", float64(sendDuration.Microseconds())/1000,
)
logArgs = append(logArgs, logging.TraceAttrsFromContext(ctx)...)
service.logger.Info("provider send completed", logArgs...)
if commit.NextAttempt != nil {
retryArgs := logging.DeliveryAttemptAttrs(item.Delivery, item.Attempt)
retryArgs = append(retryArgs,
"next_attempt_no", commit.NextAttempt.AttemptNo,
"next_scheduled_for", commit.NextAttempt.ScheduledFor,
"provider_summary", result.Summary,
)
retryArgs = append(retryArgs, logging.TraceAttrsFromContext(ctx)...)
service.logger.Info("delivery retry scheduled", retryArgs...)
}
if commit.DeadLetter != nil {
deadLetterArgs := logging.DeliveryAttemptAttrs(item.Delivery, item.Attempt)
deadLetterArgs = append(deadLetterArgs,
"failure_classification", commit.DeadLetter.FailureClassification,
"recovery_hint", commit.DeadLetter.RecoveryHint,
"provider_summary", commit.DeadLetter.ProviderSummary,
)
deadLetterArgs = append(deadLetterArgs, logging.TraceAttrsFromContext(ctx)...)
service.logger.Warn("delivery moved to dead letter", deadLetterArgs...)
}
}
func providerNameFromSummary(summary string) string {
for _, token := range strings.Split(strings.TrimSpace(summary), " ") {
key, value, ok := strings.Cut(token, "=")
if ok && key == "provider" && strings.TrimSpace(value) != "" {
return value
}
}
return "unknown"
}
@@ -0,0 +1,570 @@
package executeattempt
import (
"bytes"
"context"
"log/slog"
"testing"
"time"
"galaxy/mail/internal/domain/attempt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"galaxy/mail/internal/ports"
"galaxy/mail/internal/service/acceptgenericdelivery"
"galaxy/mail/internal/service/renderdelivery"
"github.com/stretchr/testify/require"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
)
func TestServicePrepareRendersQueuedTemplateDelivery(t *testing.T) {
t.Parallel()
renderedDelivery := queuedTemplateWorkItem(t).Delivery
renderedDelivery.Status = deliverydomain.StatusRendered
renderedDelivery.Content = deliverydomain.Content{
Subject: "Turn 54",
TextBody: "Hello Pilot",
}
renderedDelivery.UpdatedAt = renderedDelivery.CreatedAt.Add(time.Minute)
require.NoError(t, renderedDelivery.Validate())
renderer := &stubRenderer{
result: renderdelivery.Result{
Outcome: renderdelivery.OutcomeRendered,
Delivery: renderedDelivery,
ResolvedLocale: common.Locale("en"),
TemplateVersion: "sha256:template",
LocaleFallbackUsed: false,
},
}
service := newTestService(t, Config{
Renderer: renderer,
Provider: stubProvider{},
PayloadLoader: stubPayloadLoader{},
Store: &stubStore{},
Clock: stubClock{now: renderedDelivery.UpdatedAt},
AttemptTimeout: 15 * time.Second,
})
ready, err := service.Prepare(context.Background(), queuedTemplateWorkItem(t))
require.NoError(t, err)
require.True(t, ready)
require.Len(t, renderer.inputs, 1)
}
func TestServiceExecuteAcceptedRenderedDelivery(t *testing.T) {
t.Parallel()
store := &stubStore{}
service := newTestService(t, Config{
Renderer: &stubRenderer{},
Provider: stubProvider{
result: ports.Result{
Classification: ports.ClassificationAccepted,
Summary: "provider=smtp result=accepted",
},
},
PayloadLoader: stubPayloadLoader{},
Store: store,
Clock: stubClock{now: fixedNow().Add(time.Minute)},
AttemptTimeout: 15 * time.Second,
})
err := service.Execute(context.Background(), renderedWorkItem(t, 1))
require.NoError(t, err)
require.Len(t, store.inputs, 1)
require.Equal(t, deliverydomain.StatusSent, store.inputs[0].Delivery.Status)
require.Equal(t, attempt.StatusProviderAccepted, store.inputs[0].Attempt.Status)
require.Nil(t, store.inputs[0].NextAttempt)
require.Nil(t, store.inputs[0].DeadLetter)
}
func TestServiceExecuteMapsSuppressedToProviderRejected(t *testing.T) {
t.Parallel()
store := &stubStore{}
service := newTestService(t, Config{
Renderer: &stubRenderer{},
Provider: stubProvider{
result: ports.Result{
Classification: ports.ClassificationSuppressed,
Summary: "provider=stub result=suppressed script=policy_skip",
},
},
PayloadLoader: stubPayloadLoader{},
Store: store,
Clock: stubClock{now: fixedNow().Add(time.Minute)},
AttemptTimeout: 15 * time.Second,
})
err := service.Execute(context.Background(), renderedWorkItem(t, 1))
require.NoError(t, err)
require.Len(t, store.inputs, 1)
require.Equal(t, deliverydomain.StatusSuppressed, store.inputs[0].Delivery.Status)
require.Equal(t, attempt.StatusProviderRejected, store.inputs[0].Attempt.Status)
}
func TestServiceExecuteMapsPermanentFailureToFailed(t *testing.T) {
t.Parallel()
store := &stubStore{}
service := newTestService(t, Config{
Renderer: &stubRenderer{},
Provider: stubProvider{
result: ports.Result{
Classification: ports.ClassificationPermanentFailure,
Summary: "provider=smtp result=permanent_failure phase=data smtp_code=550",
},
},
PayloadLoader: stubPayloadLoader{},
Store: store,
Clock: stubClock{now: fixedNow().Add(time.Minute)},
AttemptTimeout: 15 * time.Second,
})
err := service.Execute(context.Background(), renderedWorkItem(t, 1))
require.NoError(t, err)
require.Len(t, store.inputs, 1)
require.Equal(t, deliverydomain.StatusFailed, store.inputs[0].Delivery.Status)
require.Equal(t, attempt.StatusProviderRejected, store.inputs[0].Attempt.Status)
require.Nil(t, store.inputs[0].DeadLetter)
}
func TestServiceExecuteBuildsRetryChainAndDeadLetter(t *testing.T) {
t.Parallel()
tests := []struct {
name string
attemptNo int
wantStatus deliverydomain.Status
wantAttemptStatus attempt.Status
wantNextAttemptNo int
wantNextDelay time.Duration
wantDeadLetterEntry bool
}{
{
name: "attempt one schedules retry after one minute",
attemptNo: 1,
wantStatus: deliverydomain.StatusQueued,
wantAttemptStatus: attempt.StatusTransportFailed,
wantNextAttemptNo: 2,
wantNextDelay: time.Minute,
},
{
name: "attempt two schedules retry after five minutes",
attemptNo: 2,
wantStatus: deliverydomain.StatusQueued,
wantAttemptStatus: attempt.StatusTransportFailed,
wantNextAttemptNo: 3,
wantNextDelay: 5 * time.Minute,
},
{
name: "attempt three schedules retry after thirty minutes",
attemptNo: 3,
wantStatus: deliverydomain.StatusQueued,
wantAttemptStatus: attempt.StatusTransportFailed,
wantNextAttemptNo: 4,
wantNextDelay: 30 * time.Minute,
},
{
name: "attempt four becomes dead letter",
attemptNo: 4,
wantStatus: deliverydomain.StatusDeadLetter,
wantAttemptStatus: attempt.StatusTransportFailed,
wantDeadLetterEntry: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
store := &stubStore{}
service := newTestService(t, Config{
Renderer: &stubRenderer{},
Provider: stubProvider{
result: ports.Result{
Classification: ports.ClassificationTransientFailure,
Summary: "provider=smtp result=transient_failure phase=data smtp_code=451",
Details: map[string]string{
"phase": "data",
},
},
},
PayloadLoader: stubPayloadLoader{},
Store: store,
Clock: stubClock{now: fixedNow().Add(time.Minute)},
AttemptTimeout: 15 * time.Second,
})
workItem := renderedWorkItem(t, tt.attemptNo)
err := service.Execute(context.Background(), workItem)
require.NoError(t, err)
require.Len(t, store.inputs, 1)
input := store.inputs[0]
require.Equal(t, tt.wantStatus, input.Delivery.Status)
require.Equal(t, tt.wantAttemptStatus, input.Attempt.Status)
if tt.wantDeadLetterEntry {
require.NotNil(t, input.DeadLetter)
require.Nil(t, input.NextAttempt)
require.Equal(t, "retry_exhausted", input.DeadLetter.FailureClassification)
return
}
require.NotNil(t, input.NextAttempt)
require.Nil(t, input.DeadLetter)
require.Equal(t, tt.wantNextAttemptNo, input.NextAttempt.AttemptNo)
require.Equal(t, input.Attempt.FinishedAt.Add(tt.wantNextDelay), input.NextAttempt.ScheduledFor)
})
}
}
func TestServiceExecuteClassifiesDeadlineExceededAsTimedOut(t *testing.T) {
t.Parallel()
store := &stubStore{}
service := newTestService(t, Config{
Renderer: &stubRenderer{},
Provider: stubProvider{
result: ports.Result{
Classification: ports.ClassificationTransientFailure,
Summary: "provider=smtp result=transient_failure phase=context",
Details: map[string]string{
"error": "deadline_exceeded",
},
},
},
PayloadLoader: stubPayloadLoader{},
Store: store,
Clock: stubClock{now: fixedNow().Add(time.Minute)},
AttemptTimeout: 15 * time.Second,
})
err := service.Execute(context.Background(), renderedWorkItem(t, 1))
require.NoError(t, err)
require.Len(t, store.inputs, 1)
require.Equal(t, attempt.StatusTimedOut, store.inputs[0].Attempt.Status)
require.Equal(t, "deadline_exceeded", store.inputs[0].Attempt.ProviderClassification)
}
func TestServiceRecoverExpiredSchedulesTimedOutRetry(t *testing.T) {
t.Parallel()
store := &stubStore{}
service := newTestService(t, Config{
Renderer: &stubRenderer{},
Provider: stubProvider{},
PayloadLoader: stubPayloadLoader{},
Store: store,
Clock: stubClock{now: fixedNow().Add(time.Minute)},
AttemptTimeout: 15 * time.Second,
})
err := service.RecoverExpired(context.Background(), renderedWorkItem(t, 1))
require.NoError(t, err)
require.Len(t, store.inputs, 1)
require.Equal(t, attempt.StatusTimedOut, store.inputs[0].Attempt.Status)
require.Equal(t, "claim_ttl_expired", store.inputs[0].Attempt.ProviderClassification)
require.Equal(t, "attempt claim TTL expired", store.inputs[0].Attempt.ProviderSummary)
require.NotNil(t, store.inputs[0].NextAttempt)
}
func TestServiceExecuteRecordsMetricsAndLogsProviderResult(t *testing.T) {
t.Parallel()
store := &stubStore{}
telemetry := &stubTelemetry{}
loggerBuffer := &bytes.Buffer{}
recorder := tracetest.NewSpanRecorder()
tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
service := newTestService(t, Config{
Renderer: &stubRenderer{},
Provider: stubProvider{
result: ports.Result{
Classification: ports.ClassificationAccepted,
Summary: "provider=smtp result=accepted",
},
},
PayloadLoader: stubPayloadLoader{},
Store: store,
Clock: stubClock{now: fixedNow().Add(time.Minute)},
Telemetry: telemetry,
TracerProvider: tracerProvider,
Logger: slog.New(slog.NewJSONHandler(loggerBuffer, nil)),
AttemptTimeout: 15 * time.Second,
})
err := service.Execute(context.Background(), sendingTemplateWorkItem(t, 1))
require.NoError(t, err)
require.Equal(t, []string{"notification:sent"}, telemetry.statuses)
require.Equal(t, []string{"notification:provider_accepted"}, telemetry.attempts)
require.Equal(t, []string{"smtp:accepted"}, telemetry.providerDurations)
require.Contains(t, loggerBuffer.String(), "\"delivery_id\":\"delivery-template-sending\"")
require.Contains(t, loggerBuffer.String(), "\"source\":\"notification\"")
require.Contains(t, loggerBuffer.String(), "\"template_id\":\"game.turn_ready\"")
require.Contains(t, loggerBuffer.String(), "\"attempt_no\":1")
require.Contains(t, loggerBuffer.String(), "\"otel_trace_id\":")
require.True(t, hasExecuteSpanNamed(recorder.Ended(), "mail.provider_send"))
}
func TestServiceExecuteReturnsServiceUnavailableOnMissingPayload(t *testing.T) {
t.Parallel()
service := newTestService(t, Config{
Renderer: &stubRenderer{},
Provider: stubProvider{
result: ports.Result{
Classification: ports.ClassificationAccepted,
Summary: "provider=smtp result=accepted",
},
},
PayloadLoader: stubPayloadLoader{},
Store: &stubStore{},
Clock: stubClock{now: fixedNow().Add(time.Minute)},
AttemptTimeout: 15 * time.Second,
})
workItem := renderedWorkItem(t, 1)
workItem.Delivery.Attachments = []common.AttachmentMetadata{
{Filename: "guide.txt", ContentType: "text/plain; charset=utf-8", SizeBytes: int64(len([]byte("read me")))},
}
require.NoError(t, workItem.Delivery.Validate())
err := service.Execute(context.Background(), workItem)
require.Error(t, err)
require.ErrorIs(t, err, ErrServiceUnavailable)
}
type stubRenderer struct {
result renderdelivery.Result
err error
inputs []renderdelivery.Input
}
func (renderer *stubRenderer) Execute(_ context.Context, input renderdelivery.Input) (renderdelivery.Result, error) {
renderer.inputs = append(renderer.inputs, input)
return renderer.result, renderer.err
}
type stubProvider struct {
result ports.Result
err error
inputs []ports.Message
}
func (provider stubProvider) Send(_ context.Context, message ports.Message) (ports.Result, error) {
provider.inputs = append(provider.inputs, message)
return provider.result, provider.err
}
func (provider stubProvider) Close() error {
return nil
}
type stubPayloadLoader struct {
payload acceptgenericdelivery.DeliveryPayload
found bool
err error
}
func (loader stubPayloadLoader) LoadPayload(context.Context, common.DeliveryID) (acceptgenericdelivery.DeliveryPayload, bool, error) {
return loader.payload, loader.found, loader.err
}
type stubStore struct {
inputs []CommitStateInput
err error
}
func (store *stubStore) Commit(_ context.Context, input CommitStateInput) error {
store.inputs = append(store.inputs, input)
return store.err
}
type stubClock struct {
now time.Time
}
func (clock stubClock) Now() time.Time {
return clock.now
}
type stubTelemetry struct {
statuses []string
attempts []string
providerDurations []string
}
func (telemetry *stubTelemetry) RecordDeliveryStatusTransition(_ context.Context, status string, source string) {
telemetry.statuses = append(telemetry.statuses, source+":"+status)
}
func (telemetry *stubTelemetry) RecordAttemptOutcome(_ context.Context, status string, source string) {
telemetry.attempts = append(telemetry.attempts, source+":"+status)
}
func (telemetry *stubTelemetry) RecordProviderSendDuration(_ context.Context, provider string, outcome string, _ time.Duration) {
telemetry.providerDurations = append(telemetry.providerDurations, provider+":"+outcome)
}
func newTestService(t *testing.T, cfg Config) *Service {
t.Helper()
service, err := New(cfg)
require.NoError(t, err)
return service
}
func queuedTemplateWorkItem(t *testing.T) WorkItem {
t.Helper()
createdAt := fixedNow().Add(-time.Minute)
deliveryRecord := deliverydomain.Delivery{
DeliveryID: common.DeliveryID("delivery-template"),
Source: deliverydomain.SourceNotification,
PayloadMode: deliverydomain.PayloadModeTemplate,
TemplateID: common.TemplateID("game.turn_ready"),
Envelope: deliverydomain.Envelope{
To: []common.Email{common.Email("pilot@example.com")},
},
Locale: common.Locale("en"),
TemplateVariables: map[string]any{
"player": map[string]any{
"name": "Pilot",
},
"turn_number": float64(54),
},
IdempotencyKey: common.IdempotencyKey("notification:delivery-template"),
Status: deliverydomain.StatusQueued,
AttemptCount: 1,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}
require.NoError(t, deliveryRecord.Validate())
attemptRecord := attempt.Attempt{
DeliveryID: deliveryRecord.DeliveryID,
AttemptNo: 1,
ScheduledFor: createdAt,
Status: attempt.StatusScheduled,
}
require.NoError(t, attemptRecord.Validate())
return WorkItem{
Delivery: deliveryRecord,
Attempt: attemptRecord,
}
}
func renderedWorkItem(t *testing.T, attemptNo int) WorkItem {
t.Helper()
createdAt := fixedNow().Add(-time.Duration(attemptNo) * time.Minute)
deliveryRecord := deliverydomain.Delivery{
DeliveryID: common.DeliveryID("delivery-rendered"),
Source: deliverydomain.SourceNotification,
PayloadMode: deliverydomain.PayloadModeRendered,
Envelope: deliverydomain.Envelope{
To: []common.Email{common.Email("pilot@example.com")},
},
Content: deliverydomain.Content{
Subject: "Turn ready",
TextBody: "Turn 54 is ready.",
},
IdempotencyKey: common.IdempotencyKey("notification:delivery-rendered"),
Status: deliverydomain.StatusSending,
AttemptCount: attemptNo,
CreatedAt: createdAt,
UpdatedAt: createdAt.Add(time.Second),
}
require.NoError(t, deliveryRecord.Validate())
scheduledFor := createdAt
startedAt := scheduledFor.Add(5 * time.Second)
attemptRecord := attempt.Attempt{
DeliveryID: deliveryRecord.DeliveryID,
AttemptNo: attemptNo,
ScheduledFor: scheduledFor,
StartedAt: &startedAt,
Status: attempt.StatusInProgress,
}
require.NoError(t, attemptRecord.Validate())
return WorkItem{
Delivery: deliveryRecord,
Attempt: attemptRecord,
}
}
func sendingTemplateWorkItem(t *testing.T, attemptNo int) WorkItem {
t.Helper()
createdAt := fixedNow().Add(-time.Duration(attemptNo) * time.Minute)
deliveryRecord := deliverydomain.Delivery{
DeliveryID: common.DeliveryID("delivery-template-sending"),
Source: deliverydomain.SourceNotification,
PayloadMode: deliverydomain.PayloadModeTemplate,
TemplateID: common.TemplateID("game.turn_ready"),
Envelope: deliverydomain.Envelope{
To: []common.Email{common.Email("pilot@example.com")},
},
Content: deliverydomain.Content{
Subject: "Turn ready",
TextBody: "Turn 54 is ready.",
},
Locale: common.Locale("en"),
TemplateVariables: map[string]any{
"turn_number": float64(54),
},
IdempotencyKey: common.IdempotencyKey("notification:delivery-template-sending"),
Status: deliverydomain.StatusSending,
AttemptCount: attemptNo,
CreatedAt: createdAt,
UpdatedAt: createdAt.Add(time.Second),
}
require.NoError(t, deliveryRecord.Validate())
scheduledFor := createdAt
startedAt := scheduledFor.Add(5 * time.Second)
attemptRecord := attempt.Attempt{
DeliveryID: deliveryRecord.DeliveryID,
AttemptNo: attemptNo,
ScheduledFor: scheduledFor,
StartedAt: &startedAt,
Status: attempt.StatusInProgress,
}
require.NoError(t, attemptRecord.Validate())
return WorkItem{
Delivery: deliveryRecord,
Attempt: attemptRecord,
}
}
func fixedNow() time.Time {
return time.Unix(1_775_121_700, 0).UTC()
}
var _ Renderer = (*stubRenderer)(nil)
var _ ports.Provider = stubProvider{}
var _ PayloadLoader = stubPayloadLoader{}
var _ Store = (*stubStore)(nil)
var _ Telemetry = (*stubTelemetry)(nil)
func hasExecuteSpanNamed(spans []sdktrace.ReadOnlySpan, name string) bool {
for _, span := range spans {
if span.Name() == name {
return true
}
}
return false
}
@@ -0,0 +1,128 @@
// Package getdelivery implements trusted operator lookup of one accepted mail
// delivery.
package getdelivery
import (
"context"
"errors"
"fmt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
)
var (
// ErrNotFound reports that the requested delivery does not exist.
ErrNotFound = errors.New("get delivery not found")
// ErrServiceUnavailable reports that trusted lookup could not load durable
// state safely.
ErrServiceUnavailable = errors.New("get delivery service unavailable")
)
// Input stores one exact trusted lookup by delivery identifier.
type Input struct {
// DeliveryID stores the exact accepted delivery identifier to resolve.
DeliveryID common.DeliveryID
}
// Validate reports whether input contains a complete lookup key.
func (input Input) Validate() error {
if err := input.DeliveryID.Validate(); err != nil {
return fmt.Errorf("delivery id: %w", err)
}
return nil
}
// Result stores one full delivery record and its optional dead-letter entry.
type Result struct {
// Delivery stores the resolved accepted delivery record.
Delivery deliverydomain.Delivery
// DeadLetter stores the optional dead-letter entry when Delivery is in the
// `dead_letter` terminal state.
DeadLetter *deliverydomain.DeadLetterEntry
}
// Validate reports whether result contains a consistent delivery view.
func (result Result) Validate() error {
if err := result.Delivery.Validate(); err != nil {
return fmt.Errorf("delivery: %w", err)
}
if err := deliverydomain.ValidateDeadLetterState(result.Delivery, result.DeadLetter); err != nil {
return fmt.Errorf("dead-letter state: %w", err)
}
return nil
}
// Store provides exact lookup of one accepted delivery and its dead-letter
// entry.
type Store interface {
// GetDelivery loads one accepted delivery by its identifier.
GetDelivery(context.Context, common.DeliveryID) (deliverydomain.Delivery, bool, error)
// GetDeadLetter loads the dead-letter entry associated with deliveryID when
// one exists.
GetDeadLetter(context.Context, common.DeliveryID) (deliverydomain.DeadLetterEntry, bool, error)
}
// Config stores the dependencies used by Service.
type Config struct {
// Store owns durable delivery and dead-letter state.
Store Store
}
// Service executes trusted exact delivery lookups.
type Service struct {
store Store
}
// New constructs Service from cfg.
func New(cfg Config) (*Service, error) {
if cfg.Store == nil {
return nil, errors.New("new get delivery service: nil store")
}
return &Service{store: cfg.Store}, nil
}
// Execute loads one accepted delivery and its optional dead-letter entry.
func (service *Service) Execute(ctx context.Context, input Input) (Result, error) {
if ctx == nil {
return Result{}, errors.New("execute get delivery: nil context")
}
if service == nil {
return Result{}, errors.New("execute get delivery: nil service")
}
if err := input.Validate(); err != nil {
return Result{}, fmt.Errorf("execute get delivery: %w", err)
}
record, found, err := service.store.GetDelivery(ctx, input.DeliveryID)
switch {
case err != nil:
return Result{}, fmt.Errorf("%w: load delivery: %v", ErrServiceUnavailable, err)
case !found:
return Result{}, ErrNotFound
}
result := Result{Delivery: record}
if record.Status == deliverydomain.StatusDeadLetter {
entry, found, err := service.store.GetDeadLetter(ctx, input.DeliveryID)
switch {
case err != nil:
return Result{}, fmt.Errorf("%w: load dead-letter entry: %v", ErrServiceUnavailable, err)
case !found:
return Result{}, fmt.Errorf("%w: missing dead-letter entry for delivery %q", ErrServiceUnavailable, input.DeliveryID)
default:
result.DeadLetter = &entry
}
}
if err := result.Validate(); err != nil {
return Result{}, fmt.Errorf("%w: invalid result: %v", ErrServiceUnavailable, err)
}
return result, nil
}
@@ -0,0 +1,154 @@
package getdelivery
import (
"context"
"errors"
"testing"
"time"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"github.com/stretchr/testify/require"
)
func TestServiceExecuteReturnsDeliveryWithoutDeadLetter(t *testing.T) {
t.Parallel()
store := &stubStore{
delivery: ptrDelivery(validSentDelivery()),
}
service := newTestService(t, Config{Store: store})
result, err := service.Execute(context.Background(), Input{DeliveryID: store.delivery.DeliveryID})
require.NoError(t, err)
require.Equal(t, *store.delivery, result.Delivery)
require.Nil(t, result.DeadLetter)
}
func TestServiceExecuteReturnsDeadLetterEntry(t *testing.T) {
t.Parallel()
record := validDeadLetterDelivery()
entry := validDeadLetterEntry(record.DeliveryID)
store := &stubStore{
delivery: &record,
deadLetter: &entry,
}
service := newTestService(t, Config{Store: store})
result, err := service.Execute(context.Background(), Input{DeliveryID: record.DeliveryID})
require.NoError(t, err)
require.Equal(t, record, result.Delivery)
require.NotNil(t, result.DeadLetter)
require.Equal(t, entry, *result.DeadLetter)
}
func TestServiceExecuteReturnsNotFound(t *testing.T) {
t.Parallel()
service := newTestService(t, Config{Store: &stubStore{}})
_, err := service.Execute(context.Background(), Input{DeliveryID: common.DeliveryID("missing")})
require.ErrorIs(t, err, ErrNotFound)
}
type stubStore struct {
delivery *deliverydomain.Delivery
deadLetter *deliverydomain.DeadLetterEntry
getDeliveryErr error
getDeadErr error
}
func (store *stubStore) GetDelivery(context.Context, common.DeliveryID) (deliverydomain.Delivery, bool, error) {
if store.getDeliveryErr != nil {
return deliverydomain.Delivery{}, false, store.getDeliveryErr
}
if store.delivery == nil {
return deliverydomain.Delivery{}, false, nil
}
return *store.delivery, true, nil
}
func (store *stubStore) GetDeadLetter(context.Context, common.DeliveryID) (deliverydomain.DeadLetterEntry, bool, error) {
if store.getDeadErr != nil {
return deliverydomain.DeadLetterEntry{}, false, store.getDeadErr
}
if store.deadLetter == nil {
return deliverydomain.DeadLetterEntry{}, false, nil
}
return *store.deadLetter, true, nil
}
func newTestService(t *testing.T, cfg Config) *Service {
t.Helper()
service, err := New(cfg)
require.NoError(t, err)
return service
}
func validSentDelivery() deliverydomain.Delivery {
createdAt := time.Unix(1_775_121_700, 0).UTC()
updatedAt := createdAt.Add(time.Minute)
sentAt := updatedAt.Add(time.Second)
record := deliverydomain.Delivery{
DeliveryID: common.DeliveryID("delivery-sent"),
Source: deliverydomain.SourceNotification,
PayloadMode: deliverydomain.PayloadModeRendered,
Envelope: deliverydomain.Envelope{To: []common.Email{common.Email("pilot@example.com")}},
Content: deliverydomain.Content{Subject: "Ready", TextBody: "Turn ready"},
IdempotencyKey: common.IdempotencyKey("notification:delivery-sent"),
Status: deliverydomain.StatusSent,
AttemptCount: 1,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
SentAt: &sentAt,
}
if err := record.Validate(); err != nil {
panic(err)
}
return record
}
func validDeadLetterDelivery() deliverydomain.Delivery {
record := validSentDelivery()
record.DeliveryID = common.DeliveryID("delivery-dead-letter")
record.IdempotencyKey = common.IdempotencyKey("notification:delivery-dead-letter")
record.Status = deliverydomain.StatusDeadLetter
record.UpdatedAt = record.CreatedAt.Add(2 * time.Minute)
record.SentAt = nil
deadLetteredAt := record.UpdatedAt
record.DeadLetteredAt = &deadLetteredAt
if err := record.Validate(); err != nil {
panic(err)
}
return record
}
func validDeadLetterEntry(deliveryID common.DeliveryID) deliverydomain.DeadLetterEntry {
entry := deliverydomain.DeadLetterEntry{
DeliveryID: deliveryID,
FinalAttemptNo: 1,
FailureClassification: "retry_exhausted",
ProviderSummary: "smtp timeout",
CreatedAt: time.Unix(1_775_121_900, 0).UTC(),
RecoveryHint: "check SMTP connectivity",
}
if err := entry.Validate(); err != nil {
panic(err)
}
return entry
}
func ptrDelivery(record deliverydomain.Delivery) *deliverydomain.Delivery {
return &record
}
var _ Store = (*stubStore)(nil)
var _ = errors.New
@@ -0,0 +1,137 @@
// Package listattempts implements trusted operator reads of delivery-attempt
// history.
package listattempts
import (
"context"
"errors"
"fmt"
"galaxy/mail/internal/domain/attempt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
)
var (
// ErrNotFound reports that the requested delivery does not exist.
ErrNotFound = errors.New("list attempts delivery not found")
// ErrServiceUnavailable reports that attempt history could not load durable
// state safely.
ErrServiceUnavailable = errors.New("list attempts service unavailable")
)
// Input stores one trusted attempt-history lookup request.
type Input struct {
// DeliveryID stores the exact accepted delivery identifier to inspect.
DeliveryID common.DeliveryID
}
// Validate reports whether input contains a complete lookup key.
func (input Input) Validate() error {
if err := input.DeliveryID.Validate(); err != nil {
return fmt.Errorf("delivery id: %w", err)
}
return nil
}
// Result stores the ordered attempt history of one accepted delivery.
type Result struct {
// Delivery stores the owning accepted delivery record.
Delivery deliverydomain.Delivery
// Attempts stores the concrete attempt history in `attempt_no ASC` order.
Attempts []attempt.Attempt
}
// Validate reports whether result contains a structurally valid attempt
// history.
func (result Result) Validate() error {
if err := result.Delivery.Validate(); err != nil {
return fmt.Errorf("delivery: %w", err)
}
if len(result.Attempts) != result.Delivery.AttemptCount {
return fmt.Errorf("attempt count %d mismatches delivery attempt count %d", len(result.Attempts), result.Delivery.AttemptCount)
}
for index, record := range result.Attempts {
if err := record.Validate(); err != nil {
return fmt.Errorf("attempts[%d]: %w", index, err)
}
if record.DeliveryID != result.Delivery.DeliveryID {
return fmt.Errorf("attempts[%d]: delivery id mismatch", index)
}
if record.AttemptNo != index+1 {
return fmt.Errorf("attempts[%d]: expected attempt number %d, got %d", index, index+1, record.AttemptNo)
}
}
return nil
}
// Store provides exact delivery lookup and ordered attempt-history reads.
type Store interface {
// GetDelivery loads one accepted delivery by its identifier.
GetDelivery(context.Context, common.DeliveryID) (deliverydomain.Delivery, bool, error)
// ListAttempts loads exactly expectedCount attempts in ascending attempt
// number order. Implementations must fail closed when the stored sequence
// contains a gap.
ListAttempts(context.Context, common.DeliveryID, int) ([]attempt.Attempt, error)
}
// Config stores the dependencies used by Service.
type Config struct {
// Store owns durable delivery and attempt state.
Store Store
}
// Service executes trusted attempt-history reads.
type Service struct {
store Store
}
// New constructs Service from cfg.
func New(cfg Config) (*Service, error) {
if cfg.Store == nil {
return nil, errors.New("new list attempts service: nil store")
}
return &Service{store: cfg.Store}, nil
}
// Execute loads one delivery and its complete attempt history.
func (service *Service) Execute(ctx context.Context, input Input) (Result, error) {
if ctx == nil {
return Result{}, errors.New("execute list attempts: nil context")
}
if service == nil {
return Result{}, errors.New("execute list attempts: nil service")
}
if err := input.Validate(); err != nil {
return Result{}, fmt.Errorf("execute list attempts: %w", err)
}
record, found, err := service.store.GetDelivery(ctx, input.DeliveryID)
switch {
case err != nil:
return Result{}, fmt.Errorf("%w: load delivery: %v", ErrServiceUnavailable, err)
case !found:
return Result{}, ErrNotFound
}
attempts, err := service.store.ListAttempts(ctx, input.DeliveryID, record.AttemptCount)
if err != nil {
return Result{}, fmt.Errorf("%w: load attempts: %v", ErrServiceUnavailable, err)
}
result := Result{
Delivery: record,
Attempts: attempts,
}
if err := result.Validate(); err != nil {
return Result{}, fmt.Errorf("%w: invalid result: %v", ErrServiceUnavailable, err)
}
return result, nil
}
@@ -0,0 +1,136 @@
package listattempts
import (
"context"
"testing"
"time"
"galaxy/mail/internal/domain/attempt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"github.com/stretchr/testify/require"
)
func TestServiceExecuteReturnsEmptyHistory(t *testing.T) {
t.Parallel()
record := validDelivery(0)
store := &stubStore{delivery: &record}
service := newTestService(t, Config{Store: store})
result, err := service.Execute(context.Background(), Input{DeliveryID: record.DeliveryID})
require.NoError(t, err)
require.Equal(t, record, result.Delivery)
require.Empty(t, result.Attempts)
}
func TestServiceExecuteReturnsOrderedHistory(t *testing.T) {
t.Parallel()
record := validDelivery(2)
store := &stubStore{
delivery: &record,
attempts: []attempt.Attempt{
validAttempt(record.DeliveryID, 1, attempt.StatusProviderRejected),
validAttempt(record.DeliveryID, 2, attempt.StatusProviderAccepted),
},
}
service := newTestService(t, Config{Store: store})
result, err := service.Execute(context.Background(), Input{DeliveryID: record.DeliveryID})
require.NoError(t, err)
require.Len(t, result.Attempts, 2)
require.Equal(t, 1, result.Attempts[0].AttemptNo)
require.Equal(t, 2, result.Attempts[1].AttemptNo)
}
func TestServiceExecuteFailsClosedOnGap(t *testing.T) {
t.Parallel()
record := validDelivery(2)
store := &stubStore{
delivery: &record,
attempts: []attempt.Attempt{
validAttempt(record.DeliveryID, 1, attempt.StatusProviderRejected),
validAttempt(record.DeliveryID, 3, attempt.StatusProviderAccepted),
},
}
service := newTestService(t, Config{Store: store})
_, err := service.Execute(context.Background(), Input{DeliveryID: record.DeliveryID})
require.ErrorIs(t, err, ErrServiceUnavailable)
}
type stubStore struct {
delivery *deliverydomain.Delivery
attempts []attempt.Attempt
}
func (store *stubStore) GetDelivery(context.Context, common.DeliveryID) (deliverydomain.Delivery, bool, error) {
if store.delivery == nil {
return deliverydomain.Delivery{}, false, nil
}
return *store.delivery, true, nil
}
func (store *stubStore) ListAttempts(context.Context, common.DeliveryID, int) ([]attempt.Attempt, error) {
return append([]attempt.Attempt(nil), store.attempts...), nil
}
func newTestService(t *testing.T, cfg Config) *Service {
t.Helper()
service, err := New(cfg)
require.NoError(t, err)
return service
}
func validDelivery(attemptCount int) deliverydomain.Delivery {
createdAt := time.Unix(1_775_121_700, 0).UTC()
updatedAt := createdAt.Add(time.Minute)
failedAt := updatedAt.Add(time.Second)
record := deliverydomain.Delivery{
DeliveryID: common.DeliveryID("delivery-attempts"),
Source: deliverydomain.SourceNotification,
PayloadMode: deliverydomain.PayloadModeRendered,
Envelope: deliverydomain.Envelope{To: []common.Email{common.Email("pilot@example.com")}},
Content: deliverydomain.Content{Subject: "Ready", TextBody: "Turn ready"},
IdempotencyKey: common.IdempotencyKey("notification:delivery-attempts"),
Status: deliverydomain.StatusFailed,
AttemptCount: attemptCount,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
FailedAt: &failedAt,
}
if err := record.Validate(); err != nil {
panic(err)
}
return record
}
func validAttempt(deliveryID common.DeliveryID, attemptNo int, status attempt.Status) attempt.Attempt {
scheduledFor := time.Unix(1_775_121_760+int64(attemptNo), 0).UTC()
startedAt := scheduledFor.Add(time.Second)
finishedAt := startedAt.Add(time.Second)
record := attempt.Attempt{
DeliveryID: deliveryID,
AttemptNo: attemptNo,
ScheduledFor: scheduledFor,
StartedAt: &startedAt,
FinishedAt: &finishedAt,
Status: status,
}
if err := record.Validate(); err != nil {
panic(err)
}
return record
}
var _ Store = (*stubStore)(nil)
@@ -0,0 +1,280 @@
// Package listdeliveries implements trusted operator listing of accepted mail
// deliveries.
package listdeliveries
import (
"context"
"errors"
"fmt"
"strings"
"time"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
)
var (
// ErrInvalidCursor reports that the supplied opaque pagination cursor is
// malformed or no longer matches durable state.
ErrInvalidCursor = errors.New("list deliveries invalid cursor")
// ErrServiceUnavailable reports that trusted listing could not load durable
// state safely.
ErrServiceUnavailable = errors.New("list deliveries service unavailable")
)
const (
// DefaultLimit stores the frozen default page size used by the operator
// listing surface.
DefaultLimit = 50
// MaxLimit stores the frozen maximum page size accepted by the operator
// listing surface.
MaxLimit = 200
)
// Cursor stores one deterministic continuation position in the delivery sort
// order `created_at_ms DESC, delivery_id DESC`.
type Cursor struct {
// CreatedAt stores the durable creation time of the last visible delivery.
CreatedAt time.Time
// DeliveryID stores the durable identifier of the last visible delivery.
DeliveryID common.DeliveryID
}
// Validate reports whether cursor contains a complete continuation tuple.
func (cursor Cursor) Validate() error {
if err := common.ValidateTimestamp("delivery list cursor created at", cursor.CreatedAt); err != nil {
return err
}
if err := cursor.DeliveryID.Validate(); err != nil {
return fmt.Errorf("delivery list cursor delivery id: %w", err)
}
return nil
}
// Filters stores the supported operator-listing filters.
type Filters struct {
// Recipient stores the optional recipient envelope filter covering `to`,
// `cc`, and `bcc`.
Recipient common.Email
// Status stores the optional delivery lifecycle filter.
Status deliverydomain.Status
// Source stores the optional delivery source filter.
Source deliverydomain.Source
// TemplateID stores the optional template family filter.
TemplateID common.TemplateID
// IdempotencyKey stores the optional idempotency-key filter.
IdempotencyKey common.IdempotencyKey
// FromCreatedAt stores the optional inclusive lower creation-time bound.
FromCreatedAt *time.Time
// ToCreatedAt stores the optional inclusive upper creation-time bound.
ToCreatedAt *time.Time
}
// Validate reports whether filters is structurally valid.
func (filters Filters) Validate() error {
if !filters.Recipient.IsZero() {
if err := filters.Recipient.Validate(); err != nil {
return fmt.Errorf("recipient: %w", err)
}
}
if filters.Status != "" && !filters.Status.IsKnown() {
return fmt.Errorf("status %q is unsupported", filters.Status)
}
if filters.Source != "" && !filters.Source.IsKnown() {
return fmt.Errorf("source %q is unsupported", filters.Source)
}
if !filters.TemplateID.IsZero() {
if err := filters.TemplateID.Validate(); err != nil {
return fmt.Errorf("template id: %w", err)
}
}
if !filters.IdempotencyKey.IsZero() {
if err := filters.IdempotencyKey.Validate(); err != nil {
return fmt.Errorf("idempotency key: %w", err)
}
}
if filters.FromCreatedAt != nil {
if err := common.ValidateTimestamp("from created at", *filters.FromCreatedAt); err != nil {
return err
}
}
if filters.ToCreatedAt != nil {
if err := common.ValidateTimestamp("to created at", *filters.ToCreatedAt); err != nil {
return err
}
}
if filters.FromCreatedAt != nil && filters.ToCreatedAt != nil && filters.FromCreatedAt.After(*filters.ToCreatedAt) {
return errors.New("from created at must not be after to created at")
}
return nil
}
// Input stores one trusted operator-listing request.
type Input struct {
// Limit stores the maximum number of returned deliveries. The zero value
// selects the frozen default limit.
Limit int
// Cursor stores the optional continuation cursor for the next page.
Cursor *Cursor
// Filters stores the normalized listing filters.
Filters Filters
}
// Validate reports whether input contains a complete supported listing
// request.
func (input Input) Validate() error {
switch {
case input.Limit < 0:
return errors.New("limit must not be negative")
case input.Limit > MaxLimit:
return fmt.Errorf("limit must be at most %d", MaxLimit)
}
if input.Cursor != nil {
if err := input.Cursor.Validate(); err != nil {
return fmt.Errorf("cursor: %w", err)
}
}
if err := input.Filters.Validate(); err != nil {
return fmt.Errorf("filters: %w", err)
}
return nil
}
// Result stores one deterministic ordered page of delivery records.
type Result struct {
// Items stores the returned deliveries in `created_at DESC, delivery_id
// DESC` order.
Items []deliverydomain.Delivery
// NextCursor stores the optional cursor for the next page.
NextCursor *Cursor
}
// Validate reports whether result contains valid delivery records and an
// optional next cursor.
func (result Result) Validate() error {
for index, record := range result.Items {
if err := record.Validate(); err != nil {
return fmt.Errorf("items[%d]: %w", index, err)
}
}
if result.NextCursor != nil {
if err := result.NextCursor.Validate(); err != nil {
return fmt.Errorf("next cursor: %w", err)
}
}
return nil
}
// Store provides deterministic ordered listing over durable delivery state.
type Store interface {
// List returns one filtered ordered page of delivery records.
List(context.Context, Input) (Result, error)
}
// Config stores the dependencies used by Service.
type Config struct {
// Store loads one deterministic ordered page of durable deliveries.
Store Store
}
// Service executes trusted operator delivery-list reads.
type Service struct {
store Store
}
// New constructs Service from cfg.
func New(cfg Config) (*Service, error) {
if cfg.Store == nil {
return nil, errors.New("new list deliveries service: nil store")
}
return &Service{store: cfg.Store}, nil
}
// Execute validates input, applies the default limit when omitted, and loads
// one deterministic page of deliveries.
func (service *Service) Execute(ctx context.Context, input Input) (Result, error) {
if ctx == nil {
return Result{}, errors.New("execute list deliveries: nil context")
}
if service == nil {
return Result{}, errors.New("execute list deliveries: nil service")
}
if input.Limit == 0 {
input.Limit = DefaultLimit
}
if err := input.Validate(); err != nil {
return Result{}, fmt.Errorf("execute list deliveries: %w", err)
}
result, err := service.store.List(ctx, input)
switch {
case errors.Is(err, ErrInvalidCursor):
return Result{}, err
case err != nil:
return Result{}, fmt.Errorf("%w: %v", ErrServiceUnavailable, err)
}
if err := result.Validate(); err != nil {
return Result{}, fmt.Errorf("%w: invalid result: %v", ErrServiceUnavailable, err)
}
if len(result.Items) > input.Limit {
return Result{}, fmt.Errorf("%w: invalid result: returned %d items for limit %d", ErrServiceUnavailable, len(result.Items), input.Limit)
}
return result, nil
}
// Matches reports whether record satisfies filters.
func (filters Filters) Matches(record deliverydomain.Delivery) bool {
if filters.Recipient != "" && !containsRecipient(record.Envelope, filters.Recipient) {
return false
}
if filters.Status != "" && record.Status != filters.Status {
return false
}
if filters.Source != "" && record.Source != filters.Source {
return false
}
if filters.TemplateID != "" && record.TemplateID != filters.TemplateID {
return false
}
if filters.IdempotencyKey != "" && record.IdempotencyKey != filters.IdempotencyKey {
return false
}
if filters.FromCreatedAt != nil && record.CreatedAt.Before(filters.FromCreatedAt.UTC()) {
return false
}
if filters.ToCreatedAt != nil && record.CreatedAt.After(filters.ToCreatedAt.UTC()) {
return false
}
return true
}
func containsRecipient(envelope deliverydomain.Envelope, email common.Email) bool {
for _, group := range [][]common.Email{envelope.To, envelope.Cc, envelope.Bcc} {
for _, candidate := range group {
if strings.EqualFold(candidate.String(), email.String()) {
return true
}
}
}
return false
}
@@ -0,0 +1,230 @@
package listdeliveries
import (
"context"
"errors"
"testing"
"time"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"github.com/stretchr/testify/require"
)
func TestServiceExecuteAppliesDefaultLimit(t *testing.T) {
t.Parallel()
store := &stubStore{
result: Result{
Items: []deliverydomain.Delivery{validDelivery("delivery-default", "notification:delivery-default")},
},
}
service := newTestService(t, Config{Store: store})
result, err := service.Execute(context.Background(), Input{})
require.NoError(t, err)
require.Len(t, result.Items, 1)
require.Equal(t, DefaultLimit, store.lastInput.Limit)
}
func TestInputValidateRejectsInvalidFiltersAndCursor(t *testing.T) {
t.Parallel()
validCursor := Cursor{
CreatedAt: time.Unix(1_775_121_700, 0).UTC(),
DeliveryID: common.DeliveryID("delivery-cursor"),
}
validFrom := time.Unix(1_775_121_700, 0).UTC()
validTo := validFrom.Add(time.Minute)
tests := []struct {
name string
input Input
wantErr string
}{
{
name: "invalid recipient",
input: Input{
Filters: Filters{Recipient: common.Email("not-an-email")},
},
wantErr: "recipient:",
},
{
name: "invalid status",
input: Input{
Filters: Filters{Status: deliverydomain.Status("bad")},
},
wantErr: `status "bad" is unsupported`,
},
{
name: "invalid source",
input: Input{
Filters: Filters{Source: deliverydomain.Source("bad")},
},
wantErr: `source "bad" is unsupported`,
},
{
name: "invalid template id",
input: Input{
Filters: Filters{TemplateID: common.TemplateID(" bad-template")},
},
wantErr: "template id:",
},
{
name: "invalid idempotency key",
input: Input{
Filters: Filters{IdempotencyKey: common.IdempotencyKey(" bad-key")},
},
wantErr: "idempotency key:",
},
{
name: "invalid created at range",
input: Input{
Filters: Filters{
FromCreatedAt: &validTo,
ToCreatedAt: &validFrom,
},
},
wantErr: "from created at must not be after to created at",
},
{
name: "invalid cursor",
input: Input{
Cursor: &Cursor{
CreatedAt: time.Time{},
DeliveryID: common.DeliveryID("delivery-cursor"),
},
},
wantErr: "cursor:",
},
{
name: "valid cursor and filters",
input: Input{
Limit: 1,
Cursor: &Cursor{
CreatedAt: validCursor.CreatedAt,
DeliveryID: validCursor.DeliveryID,
},
Filters: Filters{
Recipient: common.Email("pilot@example.com"),
Status: deliverydomain.StatusSent,
Source: deliverydomain.SourceNotification,
TemplateID: common.TemplateID("auth.login_code"),
IdempotencyKey: common.IdempotencyKey("notification:delivery-123"),
FromCreatedAt: &validFrom,
ToCreatedAt: &validTo,
},
},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
err := tt.input.Validate()
if tt.wantErr == "" {
require.NoError(t, err)
return
}
require.Error(t, err)
require.ErrorContains(t, err, tt.wantErr)
})
}
}
func TestServiceExecutePropagatesInvalidCursor(t *testing.T) {
t.Parallel()
service := newTestService(t, Config{
Store: &stubStore{listErr: ErrInvalidCursor},
})
_, err := service.Execute(context.Background(), Input{Limit: 1})
require.ErrorIs(t, err, ErrInvalidCursor)
}
func TestServiceExecuteWrapsServiceUnavailable(t *testing.T) {
t.Parallel()
service := newTestService(t, Config{
Store: &stubStore{listErr: errors.New("redis unavailable")},
})
_, err := service.Execute(context.Background(), Input{Limit: 1})
require.ErrorIs(t, err, ErrServiceUnavailable)
require.ErrorContains(t, err, "redis unavailable")
}
func TestServiceExecuteRejectsOversizedResult(t *testing.T) {
t.Parallel()
service := newTestService(t, Config{
Store: &stubStore{
result: Result{
Items: []deliverydomain.Delivery{
validDelivery("delivery-one", "notification:delivery-one"),
validDelivery("delivery-two", "notification:delivery-two"),
},
},
},
})
_, err := service.Execute(context.Background(), Input{Limit: 1})
require.ErrorIs(t, err, ErrServiceUnavailable)
require.ErrorContains(t, err, "returned 2 items for limit 1")
}
type stubStore struct {
lastInput Input
result Result
listErr error
}
func (store *stubStore) List(_ context.Context, input Input) (Result, error) {
store.lastInput = input
if store.listErr != nil {
return Result{}, store.listErr
}
return store.result, nil
}
func newTestService(t *testing.T, cfg Config) *Service {
t.Helper()
service, err := New(cfg)
require.NoError(t, err)
return service
}
func validDelivery(deliveryID string, idempotencyKey common.IdempotencyKey) deliverydomain.Delivery {
createdAt := time.Unix(1_775_121_700, 0).UTC()
updatedAt := createdAt.Add(time.Minute)
sentAt := updatedAt.Add(time.Second)
record := deliverydomain.Delivery{
DeliveryID: common.DeliveryID(deliveryID),
Source: deliverydomain.SourceNotification,
PayloadMode: deliverydomain.PayloadModeRendered,
Envelope: deliverydomain.Envelope{To: []common.Email{common.Email("pilot@example.com")}},
Content: deliverydomain.Content{Subject: "Ready", TextBody: "Turn ready"},
IdempotencyKey: idempotencyKey,
Status: deliverydomain.StatusSent,
AttemptCount: 1,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
SentAt: &sentAt,
}
if err := record.Validate(); err != nil {
panic(err)
}
return record
}
var _ Store = (*stubStore)(nil)
@@ -0,0 +1,695 @@
// Package renderdelivery implements deterministic rendering of template-mode
// deliveries.
package renderdelivery
import (
"context"
"errors"
"fmt"
"log/slog"
"slices"
"strings"
"time"
templatedir "galaxy/mail/internal/adapters/templates"
"galaxy/mail/internal/domain/attempt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"galaxy/mail/internal/logging"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
oteltrace "go.opentelemetry.io/otel/trace"
)
var (
// ErrServiceUnavailable reports that rendered or failed state could not be
// persisted durably.
ErrServiceUnavailable = errors.New("render delivery service unavailable")
)
const tracerName = "galaxy/mail/renderdelivery"
// FailureClassification identifies the stable render-failure classification
// surface.
type FailureClassification string
const (
// FailureTemplateNotFound reports that the requested template family does
// not exist in the catalog.
FailureTemplateNotFound FailureClassification = "template_not_found"
// FailureFallbackMissing reports that the requested locale is unavailable
// and the mandatory `en` fallback variant is also missing.
FailureFallbackMissing FailureClassification = "fallback_missing"
// FailureTemplateParseFailed reports that a template variant could not be
// parsed into a runnable form.
FailureTemplateParseFailed FailureClassification = "template_parse_failed"
// FailureMissingRequiredVariable reports that the accepted template
// variables do not provide one or more required dot-path values.
FailureMissingRequiredVariable FailureClassification = "missing_required_variable"
// FailureTemplateExecuteFailed reports that template execution failed after
// lookup and variable validation.
FailureTemplateExecuteFailed FailureClassification = "template_execute_failed"
)
// IsKnown reports whether classification belongs to the stable render-failure
// surface.
func (classification FailureClassification) IsKnown() bool {
switch classification {
case FailureTemplateNotFound,
FailureFallbackMissing,
FailureTemplateParseFailed,
FailureMissingRequiredVariable,
FailureTemplateExecuteFailed:
return true
default:
return false
}
}
// Outcome identifies the coarse result of one render-delivery execution.
type Outcome string
const (
// OutcomeRendered reports that template content was materialized and stored
// durably as `mail_delivery.status=rendered`.
OutcomeRendered Outcome = "rendered"
// OutcomeFailed reports that rendering reached a classified terminal
// failure and stored `mail_delivery.status=failed`.
OutcomeFailed Outcome = "failed"
)
// IsKnown reports whether outcome belongs to the supported render-delivery
// result surface.
func (outcome Outcome) IsKnown() bool {
switch outcome {
case OutcomeRendered, OutcomeFailed:
return true
default:
return false
}
}
// Input stores one queued template delivery together with its current
// scheduled attempt.
type Input struct {
// Delivery stores the queued template-mode delivery to render.
Delivery deliverydomain.Delivery
// Attempt stores the current scheduled attempt associated with Delivery.
Attempt attempt.Attempt
}
// Validate reports whether input contains one queued template delivery and
// its scheduled attempt.
func (input Input) Validate() error {
if err := input.Delivery.Validate(); err != nil {
return fmt.Errorf("delivery: %w", err)
}
if err := input.Attempt.Validate(); err != nil {
return fmt.Errorf("attempt: %w", err)
}
if input.Delivery.PayloadMode != deliverydomain.PayloadModeTemplate {
return fmt.Errorf("delivery payload mode must be %q", deliverydomain.PayloadModeTemplate)
}
if input.Delivery.Status != deliverydomain.StatusQueued {
return fmt.Errorf("delivery status must be %q", deliverydomain.StatusQueued)
}
if input.Attempt.DeliveryID != input.Delivery.DeliveryID {
return errors.New("attempt delivery id must match delivery id")
}
if input.Attempt.AttemptNo < 1 {
return errors.New("attempt number must be at least 1")
}
if input.Attempt.Status != attempt.StatusScheduled {
return fmt.Errorf("attempt status must be %q", attempt.StatusScheduled)
}
return nil
}
// Result stores the durable outcome of one render-delivery execution.
type Result struct {
// Outcome stores the coarse render-delivery result.
Outcome Outcome
// Delivery stores the durably persisted delivery record after rendering or
// render failure handling.
Delivery deliverydomain.Delivery
// Attempt stores the durably persisted terminal attempt when Outcome is
// failed. Successful rendering keeps the scheduled attempt unchanged and
// therefore leaves Attempt nil.
Attempt *attempt.Attempt
// ResolvedLocale stores the actual filesystem locale variant used by
// template lookup when available.
ResolvedLocale common.Locale
// LocaleFallbackUsed reports whether template lookup fell back from the
// requested locale to `en`.
LocaleFallbackUsed bool
// TemplateVersion stores the version marker of the resolved template
// variant when available.
TemplateVersion string
// FailureClassification stores the stable classified failure code when
// Outcome is failed.
FailureClassification FailureClassification
}
// Validate reports whether result contains a complete supported render
// outcome.
func (result Result) Validate() error {
if !result.Outcome.IsKnown() {
return fmt.Errorf("render delivery outcome %q is unsupported", result.Outcome)
}
if err := result.Delivery.Validate(); err != nil {
return fmt.Errorf("delivery: %w", err)
}
switch result.Outcome {
case OutcomeRendered:
if result.Attempt != nil {
return errors.New("rendered result must not contain terminal attempt")
}
if result.Delivery.Status != deliverydomain.StatusRendered {
return fmt.Errorf("rendered result delivery status must be %q", deliverydomain.StatusRendered)
}
if result.ResolvedLocale.IsZero() {
return errors.New("rendered result resolved locale must not be empty")
}
if err := result.ResolvedLocale.Validate(); err != nil {
return fmt.Errorf("resolved locale: %w", err)
}
if strings.TrimSpace(result.TemplateVersion) == "" {
return errors.New("rendered result template version must not be empty")
}
if result.FailureClassification != "" {
return errors.New("rendered result must not contain failure classification")
}
case OutcomeFailed:
if result.Attempt == nil {
return errors.New("failed result must contain terminal attempt")
}
if err := result.Attempt.Validate(); err != nil {
return fmt.Errorf("attempt: %w", err)
}
if result.Attempt.DeliveryID != result.Delivery.DeliveryID {
return errors.New("attempt delivery id must match delivery id")
}
if result.Delivery.Status != deliverydomain.StatusFailed {
return fmt.Errorf("failed result delivery status must be %q", deliverydomain.StatusFailed)
}
if result.Attempt.Status != attempt.StatusRenderFailed {
return fmt.Errorf("failed result attempt status must be %q", attempt.StatusRenderFailed)
}
if !result.FailureClassification.IsKnown() {
return fmt.Errorf("failed result classification %q is unsupported", result.FailureClassification)
}
if !result.ResolvedLocale.IsZero() {
if err := result.ResolvedLocale.Validate(); err != nil {
return fmt.Errorf("resolved locale: %w", err)
}
}
if result.Delivery.LastAttemptStatus != attempt.StatusRenderFailed {
return fmt.Errorf("failed result delivery last attempt status must be %q", attempt.StatusRenderFailed)
}
}
return nil
}
// MarkRenderedInput stores the durable mutation applied after successful
// template materialization.
type MarkRenderedInput struct {
// Delivery stores the rendered delivery record.
Delivery deliverydomain.Delivery
}
// Validate reports whether input contains one rendered delivery record.
func (input MarkRenderedInput) Validate() error {
if err := input.Delivery.Validate(); err != nil {
return fmt.Errorf("delivery: %w", err)
}
if input.Delivery.Status != deliverydomain.StatusRendered {
return fmt.Errorf("delivery status must be %q", deliverydomain.StatusRendered)
}
return nil
}
// MarkRenderFailedInput stores the durable mutation applied after classified
// render failure.
type MarkRenderFailedInput struct {
// Delivery stores the failed delivery record.
Delivery deliverydomain.Delivery
// Attempt stores the terminal render-failed attempt record.
Attempt attempt.Attempt
}
// Validate reports whether input contains one failed delivery record and its
// terminal render-failed attempt.
func (input MarkRenderFailedInput) Validate() error {
if err := input.Delivery.Validate(); err != nil {
return fmt.Errorf("delivery: %w", err)
}
if err := input.Attempt.Validate(); err != nil {
return fmt.Errorf("attempt: %w", err)
}
if input.Delivery.Status != deliverydomain.StatusFailed {
return fmt.Errorf("delivery status must be %q", deliverydomain.StatusFailed)
}
if input.Attempt.Status != attempt.StatusRenderFailed {
return fmt.Errorf("attempt status must be %q", attempt.StatusRenderFailed)
}
if input.Attempt.DeliveryID != input.Delivery.DeliveryID {
return errors.New("attempt delivery id must match delivery id")
}
if input.Delivery.LastAttemptStatus != attempt.StatusRenderFailed {
return fmt.Errorf("delivery last attempt status must be %q", attempt.StatusRenderFailed)
}
return nil
}
// Store describes the durable persistence required by the render-delivery
// use case.
type Store interface {
// MarkRendered stores the successful materialization result.
MarkRendered(context.Context, MarkRenderedInput) error
// MarkRenderFailed stores one classified terminal render failure.
MarkRenderFailed(context.Context, MarkRenderFailedInput) error
}
// TemplateCatalog describes the immutable in-memory template registry used by
// the renderer.
type TemplateCatalog interface {
// Lookup resolves one template family for locale using the frozen exact
// match followed by `en` fallback rule.
Lookup(common.TemplateID, common.Locale) (templatedir.ResolvedTemplate, error)
}
// Clock provides the current wall-clock time.
type Clock interface {
// Now returns the current time.
Now() time.Time
}
// Telemetry records low-cardinality render and delivery lifecycle metrics.
type Telemetry interface {
// RecordDeliveryStatusTransition records one durable delivery status
// transition.
RecordDeliveryStatusTransition(context.Context, string, string)
// RecordAttemptOutcome records one durable terminal attempt outcome.
RecordAttemptOutcome(context.Context, string, string)
// RecordLocaleFallback records one template locale fallback event.
RecordLocaleFallback(context.Context, string, string, string)
}
// Config stores the dependencies used by Service.
type Config struct {
// Catalog stores the immutable in-memory template registry.
Catalog TemplateCatalog
// Store owns the durable rendered and failed delivery state.
Store Store
// Clock provides the current time.
Clock Clock
// Telemetry records low-cardinality render and delivery lifecycle metrics.
Telemetry Telemetry
// TracerProvider constructs the application span recorder used by the
// render flow.
TracerProvider oteltrace.TracerProvider
// Logger writes structured render logs.
Logger *slog.Logger
}
// Service materializes queued template deliveries deterministically.
type Service struct {
catalog TemplateCatalog
store Store
clock Clock
telemetry Telemetry
tracerProvider oteltrace.TracerProvider
logger *slog.Logger
}
// New constructs Service from cfg.
func New(cfg Config) (*Service, error) {
switch {
case cfg.Catalog == nil:
return nil, errors.New("new render delivery service: nil catalog")
case cfg.Store == nil:
return nil, errors.New("new render delivery service: nil store")
case cfg.Clock == nil:
return nil, errors.New("new render delivery service: nil clock")
default:
tracerProvider := cfg.TracerProvider
if tracerProvider == nil {
tracerProvider = otel.GetTracerProvider()
}
logger := cfg.Logger
if logger == nil {
logger = slog.Default()
}
return &Service{
catalog: cfg.Catalog,
store: cfg.Store,
clock: cfg.Clock,
telemetry: cfg.Telemetry,
tracerProvider: tracerProvider,
logger: logger.With("component", "render_delivery"),
}, nil
}
}
// Execute resolves, validates, renders, and durably stores one template-mode
// delivery outcome.
func (service *Service) Execute(ctx context.Context, input Input) (Result, error) {
if ctx == nil {
return Result{}, errors.New("render delivery: nil context")
}
if service == nil {
return Result{}, errors.New("render delivery: nil service")
}
if err := input.Validate(); err != nil {
return Result{}, fmt.Errorf("render delivery: %w", err)
}
ctx, span := service.tracerProvider.Tracer(tracerName).Start(ctx, "mail.render_delivery")
defer span.End()
span.SetAttributes(
attribute.String("mail.delivery_id", input.Delivery.DeliveryID.String()),
attribute.String("mail.source", string(input.Delivery.Source)),
attribute.String("mail.template_id", input.Delivery.TemplateID.String()),
attribute.Int("mail.attempt_no", input.Attempt.AttemptNo),
attribute.String("mail.requested_locale", input.Delivery.Locale.String()),
)
resolved, err := service.catalog.Lookup(input.Delivery.TemplateID, input.Delivery.Locale)
if err != nil {
classification := classifyLookupError(err)
return service.fail(ctx, input, classification, failureSummaryForLookup(input.Delivery, classification), nil)
}
requiredPaths := resolved.RequiredVariablePaths()
missingPaths := collectMissingPaths(input.Delivery.TemplateVariables, requiredPaths)
if len(missingPaths) > 0 {
result, failErr := service.fail(
ctx,
input,
FailureMissingRequiredVariable,
failureSummaryForMissingVariables(missingPaths),
&resolved,
)
if failErr != nil {
return Result{}, failErr
}
return result, nil
}
content, err := renderContent(resolved, input.Delivery.TemplateVariables)
if err != nil {
result, failErr := service.fail(
ctx,
input,
FailureTemplateExecuteFailed,
"template execution failed",
&resolved,
)
if failErr != nil {
return Result{}, failErr
}
return result, nil
}
renderedDelivery := input.Delivery
renderedDelivery.Content = content
renderedDelivery.Status = deliverydomain.StatusRendered
renderedDelivery.LocaleFallbackUsed = resolved.LocaleFallbackUsed()
renderedDelivery.UpdatedAt = service.clock.Now().UTC().Truncate(time.Millisecond)
if err := renderedDelivery.Validate(); err != nil {
return Result{}, fmt.Errorf("render delivery: build rendered delivery: %w", err)
}
if err := service.store.MarkRendered(ctx, MarkRenderedInput{Delivery: renderedDelivery}); err != nil {
return Result{}, fmt.Errorf("%w: store rendered delivery: %v", ErrServiceUnavailable, err)
}
service.recordStatusTransition(ctx, renderedDelivery)
result := Result{
Outcome: OutcomeRendered,
Delivery: renderedDelivery,
ResolvedLocale: resolved.ResolvedLocale(),
LocaleFallbackUsed: resolved.LocaleFallbackUsed(),
TemplateVersion: resolved.Template().Version,
}
if err := result.Validate(); err != nil {
return Result{}, fmt.Errorf("render delivery: build rendered result: %w", err)
}
span.SetAttributes(
attribute.String("mail.resolved_locale", result.ResolvedLocale.String()),
attribute.Bool("mail.locale_fallback_used", result.LocaleFallbackUsed),
attribute.String("mail.status", string(renderedDelivery.Status)),
)
logArgs := logging.DeliveryAttemptAttrs(renderedDelivery, input.Attempt)
logArgs = append(logArgs,
"requested_locale", input.Delivery.Locale.String(),
"resolved_locale", result.ResolvedLocale.String(),
"locale_fallback_used", result.LocaleFallbackUsed,
"template_version", result.TemplateVersion,
)
logArgs = append(logArgs, logging.TraceAttrsFromContext(ctx)...)
if result.LocaleFallbackUsed {
service.recordLocaleFallback(ctx, renderedDelivery.TemplateID.String(), input.Delivery.Locale.String(), result.ResolvedLocale.String())
service.logger.Info("delivery rendered with locale fallback", logArgs...)
} else {
service.logger.Info("delivery rendered", logArgs...)
}
return result, nil
}
func (service *Service) fail(
ctx context.Context,
input Input,
classification FailureClassification,
summary string,
resolved *templatedir.ResolvedTemplate,
) (Result, error) {
failureAt := service.clock.Now().UTC().Truncate(time.Millisecond)
if failureAt.Before(input.Attempt.ScheduledFor) {
failureAt = input.Attempt.ScheduledFor
}
failedDelivery := input.Delivery
failedDelivery.Status = deliverydomain.StatusFailed
failedDelivery.LastAttemptStatus = attempt.StatusRenderFailed
failedDelivery.ProviderSummary = summary
failedDelivery.UpdatedAt = failureAt
failedDelivery.FailedAt = ptrTime(failureAt)
failedAttempt := input.Attempt
failedAttempt.Status = attempt.StatusRenderFailed
failedAttempt.StartedAt = ptrTime(failureAt)
failedAttempt.FinishedAt = ptrTime(failureAt)
failedAttempt.ProviderClassification = string(classification)
failedAttempt.ProviderSummary = summary
storeInput := MarkRenderFailedInput{
Delivery: failedDelivery,
Attempt: failedAttempt,
}
if err := storeInput.Validate(); err != nil {
return Result{}, fmt.Errorf("render delivery: build failed result: %w", err)
}
if err := service.store.MarkRenderFailed(ctx, storeInput); err != nil {
return Result{}, fmt.Errorf("%w: store failed delivery: %v", ErrServiceUnavailable, err)
}
service.recordStatusTransition(ctx, failedDelivery)
service.recordAttemptOutcome(ctx, failedAttempt.Status, failedDelivery.Source)
result := Result{
Outcome: OutcomeFailed,
Delivery: failedDelivery,
Attempt: &failedAttempt,
FailureClassification: classification,
}
if resolved != nil {
result.ResolvedLocale = resolved.ResolvedLocale()
result.LocaleFallbackUsed = resolved.LocaleFallbackUsed()
result.TemplateVersion = resolved.Template().Version
}
if err := result.Validate(); err != nil {
return Result{}, fmt.Errorf("render delivery: build failed result: %w", err)
}
spanAttrs := []attribute.KeyValue{
attribute.String("mail.status", string(failedDelivery.Status)),
attribute.String("mail.attempt_status", string(failedAttempt.Status)),
attribute.String("mail.failure_classification", string(classification)),
}
if resolved != nil {
spanAttrs = append(spanAttrs, attribute.String("mail.resolved_locale", resolved.ResolvedLocale().String()))
}
oteltrace.SpanFromContext(ctx).SetAttributes(spanAttrs...)
logArgs := logging.DeliveryAttemptAttrs(failedDelivery, failedAttempt)
logArgs = append(logArgs,
"failure_classification", string(classification),
"provider_summary", summary,
)
if resolved != nil {
logArgs = append(logArgs,
"requested_locale", input.Delivery.Locale.String(),
"resolved_locale", resolved.ResolvedLocale().String(),
"locale_fallback_used", resolved.LocaleFallbackUsed(),
)
}
logArgs = append(logArgs, logging.TraceAttrsFromContext(ctx)...)
service.logger.Warn("delivery rendering failed", logArgs...)
return result, nil
}
func (service *Service) recordStatusTransition(ctx context.Context, record deliverydomain.Delivery) {
if service == nil || service.telemetry == nil {
return
}
service.telemetry.RecordDeliveryStatusTransition(ctx, string(record.Status), string(record.Source))
}
func (service *Service) recordAttemptOutcome(ctx context.Context, status attempt.Status, source deliverydomain.Source) {
if service == nil || service.telemetry == nil {
return
}
service.telemetry.RecordAttemptOutcome(ctx, string(status), string(source))
}
func (service *Service) recordLocaleFallback(ctx context.Context, templateID string, requestedLocale string, resolvedLocale string) {
if service == nil || service.telemetry == nil {
return
}
service.telemetry.RecordLocaleFallback(ctx, templateID, requestedLocale, resolvedLocale)
}
func renderContent(resolved templatedir.ResolvedTemplate, variables map[string]any) (deliverydomain.Content, error) {
subject, err := resolved.ExecuteSubject(variables)
if err != nil {
return deliverydomain.Content{}, err
}
textBody, err := resolved.ExecuteText(variables)
if err != nil {
return deliverydomain.Content{}, err
}
htmlBody, ok, err := resolved.ExecuteHTML(variables)
if err != nil {
return deliverydomain.Content{}, err
}
if !ok {
htmlBody = ""
}
content := deliverydomain.Content{
Subject: subject,
TextBody: textBody,
HTMLBody: htmlBody,
}
if err := content.ValidateMaterialized(); err != nil {
return deliverydomain.Content{}, err
}
return content, nil
}
func collectMissingPaths(variables map[string]any, requiredPaths []string) []string {
missing := make([]string, 0)
for _, path := range requiredPaths {
if hasJSONPath(variables, path) {
continue
}
missing = append(missing, path)
}
return missing
}
func hasJSONPath(value map[string]any, path string) bool {
if len(value) == 0 || strings.TrimSpace(path) == "" {
return false
}
current := any(value)
for _, part := range strings.Split(path, ".") {
typed, ok := current.(map[string]any)
if !ok {
return false
}
next, ok := typed[part]
if !ok {
return false
}
current = next
}
return true
}
func classifyLookupError(err error) FailureClassification {
switch {
case errors.Is(err, templatedir.ErrFallbackMissing):
return FailureFallbackMissing
case errors.Is(err, templatedir.ErrTemplateParseFailed):
return FailureTemplateParseFailed
default:
return FailureTemplateNotFound
}
}
func failureSummaryForLookup(record deliverydomain.Delivery, classification FailureClassification) string {
switch classification {
case FailureFallbackMissing:
return fmt.Sprintf(
"template %q locale %q and fallback %q are unavailable",
record.TemplateID,
record.Locale,
common.Locale("en"),
)
case FailureTemplateParseFailed:
return "template parsing failed"
default:
return fmt.Sprintf("template %q is not available", record.TemplateID)
}
}
func failureSummaryForMissingVariables(missingPaths []string) string {
cloned := append([]string(nil), missingPaths...)
slices.Sort(cloned)
return "missing required variables: " + strings.Join(cloned, ", ")
}
func ptrTime(value time.Time) *time.Time {
return &value
}
@@ -0,0 +1,385 @@
package renderdelivery
import (
"bytes"
"context"
"errors"
"log/slog"
"os"
"path/filepath"
"testing"
"time"
templatedir "galaxy/mail/internal/adapters/templates"
"galaxy/mail/internal/domain/attempt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"github.com/stretchr/testify/require"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
)
func TestServiceExecuteRendersExactLocale(t *testing.T) {
t.Parallel()
catalog := newTestCatalog(t, map[string]string{
filepath.Join("auth.login_code", "en", "subject.tmpl"): "Your login code",
filepath.Join("auth.login_code", "en", "text.tmpl"): "Code: {{.code}}",
filepath.Join("game.turn_ready", "fr-fr", "subject.tmpl"): "Tour {{.turn_number}}",
filepath.Join("game.turn_ready", "fr-fr", "text.tmpl"): "Bonjour {{with .player}}{{.name}}{{end}}",
filepath.Join("game.turn_ready", "fr-fr", "html.tmpl"): "<p>{{.player.name}}</p>",
})
store := &stubStore{}
service := newTestService(t, Config{
Catalog: catalog,
Store: store,
Clock: stubClock{now: fixedNow()},
})
result, err := service.Execute(context.Background(), validInput(t, "fr-FR"))
require.NoError(t, err)
require.Equal(t, OutcomeRendered, result.Outcome)
require.Equal(t, common.Locale("fr-FR"), result.ResolvedLocale)
require.False(t, result.LocaleFallbackUsed)
require.NotEmpty(t, result.TemplateVersion)
require.Nil(t, result.Attempt)
require.Equal(t, deliverydomain.StatusRendered, result.Delivery.Status)
require.Equal(t, deliverydomain.Content{
Subject: "Tour 54",
TextBody: "Bonjour Pilot",
HTMLBody: "<p>Pilot</p>",
}, result.Delivery.Content)
require.Len(t, store.renderedInputs, 1)
require.Empty(t, store.failedInputs)
}
func TestServiceExecuteFallsBackToEnglish(t *testing.T) {
t.Parallel()
catalog := newTestCatalog(t, map[string]string{
filepath.Join("auth.login_code", "en", "subject.tmpl"): "Your login code",
filepath.Join("auth.login_code", "en", "text.tmpl"): "Code: {{.code}}",
filepath.Join("game.turn_ready", "en", "subject.tmpl"): "Turn {{.turn_number}}",
filepath.Join("game.turn_ready", "en", "text.tmpl"): "Hello {{.player.name}}",
})
store := &stubStore{}
service := newTestService(t, Config{
Catalog: catalog,
Store: store,
Clock: stubClock{now: fixedNow()},
})
result, err := service.Execute(context.Background(), validInput(t, "fr-FR"))
require.NoError(t, err)
require.Equal(t, OutcomeRendered, result.Outcome)
require.Equal(t, common.Locale("en"), result.ResolvedLocale)
require.True(t, result.LocaleFallbackUsed)
require.True(t, result.Delivery.LocaleFallbackUsed)
}
func TestServiceExecuteRecordsLocaleFallbackAndLogsFields(t *testing.T) {
t.Parallel()
catalog := newTestCatalog(t, map[string]string{
filepath.Join("auth.login_code", "en", "subject.tmpl"): "Your login code",
filepath.Join("auth.login_code", "en", "text.tmpl"): "Code: {{.code}}",
filepath.Join("game.turn_ready", "en", "subject.tmpl"): "Turn {{.turn_number}}",
filepath.Join("game.turn_ready", "en", "text.tmpl"): "Hello {{.player.name}}",
})
telemetry := &stubTelemetry{}
loggerBuffer := &bytes.Buffer{}
recorder := tracetest.NewSpanRecorder()
tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
service := newTestService(t, Config{
Catalog: catalog,
Store: &stubStore{},
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
TracerProvider: tracerProvider,
Logger: slog.New(slog.NewJSONHandler(loggerBuffer, nil)),
})
_, err := service.Execute(context.Background(), validInput(t, "fr-FR"))
require.NoError(t, err)
require.Equal(t, []string{"notification:rendered"}, telemetry.statuses)
require.Equal(t, []string{"game.turn_ready:fr-FR:en"}, telemetry.fallbacks)
require.Contains(t, loggerBuffer.String(), "\"delivery_id\":\"delivery-123\"")
require.Contains(t, loggerBuffer.String(), "\"source\":\"notification\"")
require.Contains(t, loggerBuffer.String(), "\"template_id\":\"game.turn_ready\"")
require.Contains(t, loggerBuffer.String(), "\"attempt_no\":1")
require.Contains(t, loggerBuffer.String(), "\"otel_trace_id\":")
require.True(t, hasRenderSpanNamed(recorder.Ended(), "mail.render_delivery"))
}
func TestServiceExecuteFailsOnMissingRequiredVariable(t *testing.T) {
t.Parallel()
catalog := newTestCatalog(t, map[string]string{
filepath.Join("auth.login_code", "en", "subject.tmpl"): "Your login code",
filepath.Join("auth.login_code", "en", "text.tmpl"): "Code: {{.code}}",
filepath.Join("game.turn_ready", "en", "subject.tmpl"): "Turn {{.turn_number}}",
filepath.Join("game.turn_ready", "en", "text.tmpl"): "Hello {{.player.name}}",
})
store := &stubStore{}
service := newTestService(t, Config{
Catalog: catalog,
Store: store,
Clock: stubClock{now: fixedNow()},
})
input := validInput(t, "en")
delete(input.Delivery.TemplateVariables, "player")
result, err := service.Execute(context.Background(), input)
require.NoError(t, err)
require.Equal(t, OutcomeFailed, result.Outcome)
require.Equal(t, FailureMissingRequiredVariable, result.FailureClassification)
require.NotNil(t, result.Attempt)
require.Equal(t, attempt.StatusRenderFailed, result.Attempt.Status)
require.Equal(t, "missing required variables: player.name", result.Attempt.ProviderSummary)
require.Len(t, store.failedInputs, 1)
require.Empty(t, store.renderedInputs)
}
func TestServiceExecuteFailsOnTemplateExecutionError(t *testing.T) {
t.Parallel()
catalog := newTestCatalog(t, map[string]string{
filepath.Join("auth.login_code", "en", "subject.tmpl"): "Your login code",
filepath.Join("auth.login_code", "en", "text.tmpl"): "Code: {{.code}}",
filepath.Join("game.turn_ready", "en", "subject.tmpl"): "{{call .callable}}",
filepath.Join("game.turn_ready", "en", "text.tmpl"): "Hello {{.player.name}}",
})
store := &stubStore{}
service := newTestService(t, Config{
Catalog: catalog,
Store: store,
Clock: stubClock{now: fixedNow()},
})
input := validInput(t, "en")
input.Delivery.TemplateVariables["callable"] = "not-a-func"
result, err := service.Execute(context.Background(), input)
require.NoError(t, err)
require.Equal(t, OutcomeFailed, result.Outcome)
require.Equal(t, FailureTemplateExecuteFailed, result.FailureClassification)
require.Equal(t, "template execution failed", result.Attempt.ProviderSummary)
}
func TestServiceExecuteClassifiesTemplateNotFound(t *testing.T) {
t.Parallel()
service := newTestService(t, Config{
Catalog: stubCatalog{
lookupErr: templatedir.ErrTemplateNotFound,
},
Store: &stubStore{},
Clock: stubClock{now: fixedNow()},
})
result, err := service.Execute(context.Background(), validInput(t, "en"))
require.NoError(t, err)
require.Equal(t, OutcomeFailed, result.Outcome)
require.Equal(t, FailureTemplateNotFound, result.FailureClassification)
}
func TestServiceExecuteClassifiesFallbackMissing(t *testing.T) {
t.Parallel()
service := newTestService(t, Config{
Catalog: stubCatalog{
lookupErr: templatedir.ErrFallbackMissing,
},
Store: &stubStore{},
Clock: stubClock{now: fixedNow()},
})
result, err := service.Execute(context.Background(), validInput(t, "fr-FR"))
require.NoError(t, err)
require.Equal(t, OutcomeFailed, result.Outcome)
require.Equal(t, FailureFallbackMissing, result.FailureClassification)
}
func TestServiceExecuteClassifiesTemplateParseFailure(t *testing.T) {
t.Parallel()
service := newTestService(t, Config{
Catalog: stubCatalog{
lookupErr: templatedir.ErrTemplateParseFailed,
},
Store: &stubStore{},
Clock: stubClock{now: fixedNow()},
})
result, err := service.Execute(context.Background(), validInput(t, "en"))
require.NoError(t, err)
require.Equal(t, OutcomeFailed, result.Outcome)
require.Equal(t, FailureTemplateParseFailed, result.FailureClassification)
}
func TestServiceExecuteReturnsServiceUnavailableOnStoreFailure(t *testing.T) {
t.Parallel()
catalog := newTestCatalog(t, map[string]string{
filepath.Join("auth.login_code", "en", "subject.tmpl"): "Your login code",
filepath.Join("auth.login_code", "en", "text.tmpl"): "Code: {{.code}}",
filepath.Join("game.turn_ready", "en", "subject.tmpl"): "Turn {{.turn_number}}",
filepath.Join("game.turn_ready", "en", "text.tmpl"): "Hello {{.player.name}}",
})
service := newTestService(t, Config{
Catalog: catalog,
Store: &stubStore{
markRenderedErr: errors.New("redis unavailable"),
},
Clock: stubClock{now: fixedNow()},
})
_, err := service.Execute(context.Background(), validInput(t, "en"))
require.Error(t, err)
require.ErrorIs(t, err, ErrServiceUnavailable)
}
type stubStore struct {
renderedInputs []MarkRenderedInput
failedInputs []MarkRenderFailedInput
markRenderedErr error
markFailedErr error
}
func (store *stubStore) MarkRendered(_ context.Context, input MarkRenderedInput) error {
store.renderedInputs = append(store.renderedInputs, input)
return store.markRenderedErr
}
func (store *stubStore) MarkRenderFailed(_ context.Context, input MarkRenderFailedInput) error {
store.failedInputs = append(store.failedInputs, input)
return store.markFailedErr
}
type stubCatalog struct {
lookupResult templatedir.ResolvedTemplate
lookupErr error
}
func (catalog stubCatalog) Lookup(common.TemplateID, common.Locale) (templatedir.ResolvedTemplate, error) {
return catalog.lookupResult, catalog.lookupErr
}
type stubClock struct {
now time.Time
}
func (clock stubClock) Now() time.Time {
return clock.now
}
func newTestService(t *testing.T, cfg Config) *Service {
t.Helper()
service, err := New(cfg)
require.NoError(t, err)
return service
}
func newTestCatalog(t *testing.T, files map[string]string) *templatedir.Catalog {
t.Helper()
rootDir := t.TempDir()
for path, contents := range files {
absolutePath := filepath.Join(rootDir, path)
require.NoError(t, os.MkdirAll(filepath.Dir(absolutePath), 0o755))
require.NoError(t, os.WriteFile(absolutePath, []byte(contents), 0o644))
}
catalog, err := templatedir.NewCatalog(rootDir)
require.NoError(t, err)
return catalog
}
type stubTelemetry struct {
statuses []string
attempts []string
fallbacks []string
}
func (telemetry *stubTelemetry) RecordDeliveryStatusTransition(_ context.Context, status string, source string) {
telemetry.statuses = append(telemetry.statuses, source+":"+status)
}
func (telemetry *stubTelemetry) RecordAttemptOutcome(_ context.Context, status string, source string) {
telemetry.attempts = append(telemetry.attempts, source+":"+status)
}
func (telemetry *stubTelemetry) RecordLocaleFallback(_ context.Context, templateID string, requestedLocale string, resolvedLocale string) {
telemetry.fallbacks = append(telemetry.fallbacks, templateID+":"+requestedLocale+":"+resolvedLocale)
}
func hasRenderSpanNamed(spans []sdktrace.ReadOnlySpan, name string) bool {
for _, span := range spans {
if span.Name() == name {
return true
}
}
return false
}
func validInput(t *testing.T, localeValue string) Input {
t.Helper()
locale, err := common.ParseLocale(localeValue)
require.NoError(t, err)
createdAt := fixedNow().Add(-time.Minute)
deliveryRecord := deliverydomain.Delivery{
DeliveryID: common.DeliveryID("delivery-123"),
Source: deliverydomain.SourceNotification,
PayloadMode: deliverydomain.PayloadModeTemplate,
TemplateID: common.TemplateID("game.turn_ready"),
Envelope: deliverydomain.Envelope{
To: []common.Email{common.Email("pilot@example.com")},
},
Locale: locale,
TemplateVariables: map[string]any{
"turn_number": float64(54),
"player": map[string]any{
"name": "Pilot",
},
},
IdempotencyKey: common.IdempotencyKey("notification:delivery-123"),
Status: deliverydomain.StatusQueued,
AttemptCount: 1,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}
require.NoError(t, deliveryRecord.Validate())
scheduledFor := createdAt
attemptRecord := attempt.Attempt{
DeliveryID: deliveryRecord.DeliveryID,
AttemptNo: 1,
ScheduledFor: scheduledFor,
Status: attempt.StatusScheduled,
}
require.NoError(t, attemptRecord.Validate())
return Input{
Delivery: deliveryRecord,
Attempt: attemptRecord,
}
}
func fixedNow() time.Time {
return time.Unix(1_775_121_700, 0).UTC()
}
@@ -0,0 +1,366 @@
// Package resenddelivery implements trusted operator resend by clone creation.
package resenddelivery
import (
"context"
"errors"
"fmt"
"log/slog"
"time"
"galaxy/mail/internal/domain/attempt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"galaxy/mail/internal/logging"
"galaxy/mail/internal/service/acceptgenericdelivery"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
oteltrace "go.opentelemetry.io/otel/trace"
)
var (
// ErrNotFound reports that the requested original delivery does not exist.
ErrNotFound = errors.New("resend delivery not found")
// ErrNotAllowed reports that the original delivery is not in a terminal
// state and therefore cannot be cloned for resend.
ErrNotAllowed = errors.New("resend delivery not allowed")
// ErrServiceUnavailable reports that clone creation could not load or
// persist durable state safely.
ErrServiceUnavailable = errors.New("resend delivery service unavailable")
)
const tracerName = "galaxy/mail/resenddelivery"
// Input stores one trusted resend request by original delivery identifier.
type Input struct {
// DeliveryID stores the original accepted delivery identifier to clone.
DeliveryID common.DeliveryID
}
// Validate reports whether input contains a complete resend target.
func (input Input) Validate() error {
if err := input.DeliveryID.Validate(); err != nil {
return fmt.Errorf("delivery id: %w", err)
}
return nil
}
// Result stores the new clone delivery identifier created by resend.
type Result struct {
// DeliveryID stores the identifier of the newly created clone delivery.
DeliveryID common.DeliveryID
}
// Validate reports whether result contains a usable clone delivery identifier.
func (result Result) Validate() error {
if err := result.DeliveryID.Validate(); err != nil {
return fmt.Errorf("delivery id: %w", err)
}
return nil
}
// CreateResendInput stores the durable write set required for one clone-only
// resend operation.
type CreateResendInput struct {
// Delivery stores the new cloned delivery record.
Delivery deliverydomain.Delivery
// FirstAttempt stores the initial scheduled attempt of the clone.
FirstAttempt attempt.Attempt
// DeliveryPayload stores the optional cloned raw attachment payload bundle.
DeliveryPayload *acceptgenericdelivery.DeliveryPayload
}
// Validate reports whether input contains a complete resend write set.
func (input CreateResendInput) Validate() error {
if err := input.Delivery.Validate(); err != nil {
return fmt.Errorf("delivery: %w", err)
}
if input.Delivery.Source != deliverydomain.SourceOperatorResend {
return fmt.Errorf("delivery source must be %q", deliverydomain.SourceOperatorResend)
}
if input.Delivery.Status != deliverydomain.StatusQueued {
return fmt.Errorf("delivery status must be %q", deliverydomain.StatusQueued)
}
if input.Delivery.AttemptCount != 1 {
return errors.New("delivery attempt count must equal 1")
}
if input.Delivery.LastAttemptStatus != "" {
return errors.New("delivery last attempt status must be empty")
}
if input.Delivery.ProviderSummary != "" {
return errors.New("delivery provider summary must be empty")
}
if input.Delivery.SentAt != nil || input.Delivery.SuppressedAt != nil || input.Delivery.FailedAt != nil || input.Delivery.DeadLetteredAt != nil {
return errors.New("delivery terminal timestamps must be empty")
}
if err := input.FirstAttempt.Validate(); err != nil {
return fmt.Errorf("first attempt: %w", err)
}
if input.FirstAttempt.DeliveryID != input.Delivery.DeliveryID {
return errors.New("first attempt delivery id must match delivery id")
}
if input.FirstAttempt.AttemptNo != 1 {
return errors.New("first attempt number must equal 1")
}
if input.FirstAttempt.Status != attempt.StatusScheduled {
return fmt.Errorf("first attempt status must be %q", attempt.StatusScheduled)
}
if input.DeliveryPayload != nil {
if err := input.DeliveryPayload.Validate(); err != nil {
return fmt.Errorf("delivery payload: %w", err)
}
if input.DeliveryPayload.DeliveryID != input.Delivery.DeliveryID {
return errors.New("delivery payload delivery id must match delivery id")
}
}
return nil
}
// Store provides the durable delivery state required by clone-only resend.
type Store interface {
// GetDelivery loads one accepted delivery by its identifier.
GetDelivery(context.Context, common.DeliveryID) (deliverydomain.Delivery, bool, error)
// GetDeliveryPayload loads the raw attachment payload bundle of deliveryID
// when one exists.
GetDeliveryPayload(context.Context, common.DeliveryID) (acceptgenericdelivery.DeliveryPayload, bool, error)
// CreateResend atomically creates the cloned delivery, its first attempt,
// the optional cloned delivery payload, and the related delivery indexes.
CreateResend(context.Context, CreateResendInput) error
}
// DeliveryIDGenerator describes the source of new internal delivery
// identifiers.
type DeliveryIDGenerator interface {
// NewDeliveryID returns one new internal delivery identifier.
NewDeliveryID() (common.DeliveryID, error)
}
// Clock provides the current wall-clock time.
type Clock interface {
// Now returns the current time.
Now() time.Time
}
// Telemetry records low-cardinality resend metrics.
type Telemetry interface {
// RecordDeliveryStatusTransition records one durable delivery status
// transition.
RecordDeliveryStatusTransition(context.Context, string, string)
}
// Config stores the dependencies used by Service.
type Config struct {
// Store owns durable resend state.
Store Store
// DeliveryIDGenerator builds internal clone identifiers.
DeliveryIDGenerator DeliveryIDGenerator
// Clock provides wall-clock timestamps.
Clock Clock
// Telemetry records low-cardinality resend metrics.
Telemetry Telemetry
// TracerProvider constructs the application span recorder used by resend.
TracerProvider oteltrace.TracerProvider
// Logger writes structured resend logs.
Logger *slog.Logger
}
// Service executes clone-only trusted resend requests.
type Service struct {
store Store
deliveryIDGenerator DeliveryIDGenerator
clock Clock
telemetry Telemetry
tracerProvider oteltrace.TracerProvider
logger *slog.Logger
}
// New constructs Service from cfg.
func New(cfg Config) (*Service, error) {
switch {
case cfg.Store == nil:
return nil, errors.New("new resend delivery service: nil store")
case cfg.DeliveryIDGenerator == nil:
return nil, errors.New("new resend delivery service: nil delivery id generator")
case cfg.Clock == nil:
return nil, errors.New("new resend delivery service: nil clock")
default:
tracerProvider := cfg.TracerProvider
if tracerProvider == nil {
tracerProvider = otel.GetTracerProvider()
}
logger := cfg.Logger
if logger == nil {
logger = slog.Default()
}
return &Service{
store: cfg.Store,
deliveryIDGenerator: cfg.DeliveryIDGenerator,
clock: cfg.Clock,
telemetry: cfg.Telemetry,
tracerProvider: tracerProvider,
logger: logger.With("component", "resend_delivery"),
}, nil
}
}
// Execute clones one terminal delivery into a new queued delivery with a
// fresh first attempt.
func (service *Service) Execute(ctx context.Context, input Input) (Result, error) {
if ctx == nil {
return Result{}, errors.New("execute resend delivery: nil context")
}
if service == nil {
return Result{}, errors.New("execute resend delivery: nil service")
}
if err := input.Validate(); err != nil {
return Result{}, fmt.Errorf("execute resend delivery: %w", err)
}
ctx, span := service.tracerProvider.Tracer(tracerName).Start(
ctx,
"mail.resend_delivery",
oteltrace.WithAttributes(attribute.String("mail.parent_delivery_id", input.DeliveryID.String())),
)
defer span.End()
original, found, err := service.store.GetDelivery(ctx, input.DeliveryID)
switch {
case err != nil:
return Result{}, fmt.Errorf("%w: load original delivery: %v", ErrServiceUnavailable, err)
case !found:
return Result{}, ErrNotFound
case !original.Status.AllowsResend():
return Result{}, ErrNotAllowed
}
now := service.clock.Now().UTC().Truncate(time.Millisecond)
cloneID, err := service.deliveryIDGenerator.NewDeliveryID()
if err != nil {
return Result{}, fmt.Errorf("%w: generate delivery id: %v", ErrServiceUnavailable, err)
}
clone := buildClonedDelivery(original, cloneID, now)
firstAttempt := attempt.Attempt{
DeliveryID: cloneID,
AttemptNo: 1,
ScheduledFor: now,
Status: attempt.StatusScheduled,
}
var clonedPayload *acceptgenericdelivery.DeliveryPayload
if len(original.Attachments) > 0 {
payload, found, err := service.store.GetDeliveryPayload(ctx, original.DeliveryID)
switch {
case err != nil:
return Result{}, fmt.Errorf("%w: load original delivery payload: %v", ErrServiceUnavailable, err)
case !found:
return Result{}, fmt.Errorf("%w: missing original delivery payload for %q", ErrServiceUnavailable, original.DeliveryID)
default:
cloned := cloneDeliveryPayload(payload, cloneID)
clonedPayload = &cloned
}
}
createInput := CreateResendInput{
Delivery: clone,
FirstAttempt: firstAttempt,
DeliveryPayload: clonedPayload,
}
if err := createInput.Validate(); err != nil {
return Result{}, fmt.Errorf("%w: build resend input: %v", ErrServiceUnavailable, err)
}
if err := service.store.CreateResend(ctx, createInput); err != nil {
return Result{}, fmt.Errorf("%w: create resend clone: %v", ErrServiceUnavailable, err)
}
service.recordStatusTransition(ctx, createInput.Delivery)
result := Result{DeliveryID: cloneID}
if err := result.Validate(); err != nil {
return Result{}, fmt.Errorf("%w: invalid result: %v", ErrServiceUnavailable, err)
}
span.SetAttributes(
attribute.String("mail.delivery_id", cloneID.String()),
attribute.String("mail.source", string(createInput.Delivery.Source)),
)
logArgs := logging.DeliveryAttrs(createInput.Delivery)
logArgs = append(logArgs,
"parent_delivery_id", original.DeliveryID.String(),
"status", string(createInput.Delivery.Status),
)
logArgs = append(logArgs, logging.TraceAttrsFromContext(ctx)...)
service.logger.Info("resend clone created", logArgs...)
return result, nil
}
func (service *Service) recordStatusTransition(ctx context.Context, record deliverydomain.Delivery) {
if service == nil || service.telemetry == nil {
return
}
service.telemetry.RecordDeliveryStatusTransition(ctx, string(record.Status), string(record.Source))
}
func buildClonedDelivery(original deliverydomain.Delivery, cloneID common.DeliveryID, now time.Time) deliverydomain.Delivery {
return deliverydomain.Delivery{
DeliveryID: cloneID,
ResendParentDeliveryID: original.DeliveryID,
Source: deliverydomain.SourceOperatorResend,
PayloadMode: original.PayloadMode,
TemplateID: original.TemplateID,
Envelope: deliverydomain.Envelope{
To: append([]common.Email(nil), original.Envelope.To...),
Cc: append([]common.Email(nil), original.Envelope.Cc...),
Bcc: append([]common.Email(nil), original.Envelope.Bcc...),
ReplyTo: append([]common.Email(nil), original.Envelope.ReplyTo...),
},
Content: original.Content,
Attachments: append([]common.AttachmentMetadata(nil), original.Attachments...),
Locale: original.Locale,
LocaleFallbackUsed: original.LocaleFallbackUsed,
TemplateVariables: cloneJSONObject(original.TemplateVariables),
IdempotencyKey: common.IdempotencyKey("operator:resend:" + original.DeliveryID.String()),
Status: deliverydomain.StatusQueued,
AttemptCount: 1,
CreatedAt: now,
UpdatedAt: now,
}
}
func cloneDeliveryPayload(payload acceptgenericdelivery.DeliveryPayload, cloneID common.DeliveryID) acceptgenericdelivery.DeliveryPayload {
cloned := acceptgenericdelivery.DeliveryPayload{
DeliveryID: cloneID,
Attachments: make([]acceptgenericdelivery.AttachmentPayload, len(payload.Attachments)),
}
copy(cloned.Attachments, payload.Attachments)
return cloned
}
func cloneJSONObject(value map[string]any) map[string]any {
if value == nil {
return nil
}
cloned := make(map[string]any, len(value))
for key, entry := range value {
cloned[key] = entry
}
return cloned
}
@@ -0,0 +1,273 @@
package resenddelivery
import (
"bytes"
"context"
"log/slog"
"testing"
"time"
"galaxy/mail/internal/domain/attempt"
"galaxy/mail/internal/domain/common"
deliverydomain "galaxy/mail/internal/domain/delivery"
"galaxy/mail/internal/service/acceptgenericdelivery"
"github.com/stretchr/testify/require"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
)
func TestServiceExecuteRejectsNonTerminalStatus(t *testing.T) {
t.Parallel()
tests := []deliverydomain.Status{
deliverydomain.StatusAccepted,
deliverydomain.StatusQueued,
deliverydomain.StatusRendered,
deliverydomain.StatusSending,
}
for _, status := range tests {
status := status
t.Run(string(status), func(t *testing.T) {
t.Parallel()
record := validOriginalDelivery()
record.Status = status
record.SentAt = nil
record.FailedAt = nil
record.DeadLetteredAt = nil
record.SuppressedAt = nil
require.NoError(t, record.Validate())
store := &stubStore{delivery: &record}
service := newTestService(t, Config{
Store: store,
DeliveryIDGenerator: &stubIDGenerator{ids: []common.DeliveryID{"clone-1"}},
Clock: stubClock{now: fixedNow()},
})
_, err := service.Execute(context.Background(), Input{DeliveryID: record.DeliveryID})
require.ErrorIs(t, err, ErrNotAllowed)
})
}
}
func TestServiceExecuteCreatesLinkedClone(t *testing.T) {
t.Parallel()
original := validOriginalDelivery()
originalCopy := original
payload := validPayload(original.DeliveryID)
store := &stubStore{
delivery: &original,
payload: &payload,
}
service := newTestService(t, Config{
Store: store,
DeliveryIDGenerator: &stubIDGenerator{ids: []common.DeliveryID{"clone-123"}},
Clock: stubClock{now: fixedNow()},
})
result, err := service.Execute(context.Background(), Input{DeliveryID: original.DeliveryID})
require.NoError(t, err)
require.Equal(t, Result{DeliveryID: common.DeliveryID("clone-123")}, result)
require.Len(t, store.createInputs, 1)
createInput := store.createInputs[0]
require.Equal(t, common.DeliveryID("clone-123"), createInput.Delivery.DeliveryID)
require.Equal(t, original.DeliveryID, createInput.Delivery.ResendParentDeliveryID)
require.Equal(t, deliverydomain.SourceOperatorResend, createInput.Delivery.Source)
require.Equal(t, common.IdempotencyKey("operator:resend:"+original.DeliveryID.String()), createInput.Delivery.IdempotencyKey)
require.Equal(t, deliverydomain.StatusQueued, createInput.Delivery.Status)
require.Equal(t, 1, createInput.Delivery.AttemptCount)
require.Empty(t, createInput.Delivery.LastAttemptStatus)
require.Nil(t, createInput.Delivery.SentAt)
require.Nil(t, createInput.Delivery.FailedAt)
require.Equal(t, attempt.StatusScheduled, createInput.FirstAttempt.Status)
require.Equal(t, 1, createInput.FirstAttempt.AttemptNo)
require.NotNil(t, createInput.DeliveryPayload)
require.Equal(t, common.DeliveryID("clone-123"), createInput.DeliveryPayload.DeliveryID)
require.Equal(t, payload.Attachments, createInput.DeliveryPayload.Attachments)
require.Equal(t, originalCopy, original)
}
func TestServiceExecuteLogsCloneCreationAndCreatesSpan(t *testing.T) {
t.Parallel()
original := validOriginalDelivery()
payload := validPayload(original.DeliveryID)
loggerBuffer := &bytes.Buffer{}
recorder := tracetest.NewSpanRecorder()
tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
telemetry := &stubTelemetry{}
store := &stubStore{
delivery: &original,
payload: &payload,
}
service := newTestService(t, Config{
Store: store,
DeliveryIDGenerator: &stubIDGenerator{ids: []common.DeliveryID{"clone-456"}},
Clock: stubClock{now: fixedNow()},
Telemetry: telemetry,
TracerProvider: tracerProvider,
Logger: slog.New(slog.NewJSONHandler(loggerBuffer, nil)),
})
_, err := service.Execute(context.Background(), Input{DeliveryID: original.DeliveryID})
require.NoError(t, err)
require.Equal(t, []string{"operator_resend:queued"}, telemetry.statuses)
require.Contains(t, loggerBuffer.String(), "\"delivery_id\":\"clone-456\"")
require.Contains(t, loggerBuffer.String(), "\"source\":\"operator_resend\"")
require.Contains(t, loggerBuffer.String(), "\"template_id\":\"game.turn_ready\"")
require.Contains(t, loggerBuffer.String(), "\"otel_trace_id\":")
require.True(t, hasResendSpanNamed(recorder.Ended(), "mail.resend_delivery"))
}
type stubStore struct {
delivery *deliverydomain.Delivery
payload *acceptgenericdelivery.DeliveryPayload
createInputs []CreateResendInput
}
func (store *stubStore) GetDelivery(context.Context, common.DeliveryID) (deliverydomain.Delivery, bool, error) {
if store.delivery == nil {
return deliverydomain.Delivery{}, false, nil
}
return *store.delivery, true, nil
}
func (store *stubStore) GetDeliveryPayload(context.Context, common.DeliveryID) (acceptgenericdelivery.DeliveryPayload, bool, error) {
if store.payload == nil {
return acceptgenericdelivery.DeliveryPayload{}, false, nil
}
return *store.payload, true, nil
}
func (store *stubStore) CreateResend(_ context.Context, input CreateResendInput) error {
store.createInputs = append(store.createInputs, input)
return nil
}
type stubIDGenerator struct {
ids []common.DeliveryID
}
func (generator *stubIDGenerator) NewDeliveryID() (common.DeliveryID, error) {
if len(generator.ids) == 0 {
return "", nil
}
next := generator.ids[0]
generator.ids = generator.ids[1:]
return next, nil
}
type stubClock struct {
now time.Time
}
func (clock stubClock) Now() time.Time {
return clock.now
}
type stubTelemetry struct {
statuses []string
}
func (telemetry *stubTelemetry) RecordDeliveryStatusTransition(_ context.Context, status string, source string) {
telemetry.statuses = append(telemetry.statuses, source+":"+status)
}
func newTestService(t *testing.T, cfg Config) *Service {
t.Helper()
service, err := New(cfg)
require.NoError(t, err)
return service
}
func fixedNow() time.Time {
return time.Unix(1_775_122_100, 0).UTC()
}
func validOriginalDelivery() deliverydomain.Delivery {
createdAt := time.Unix(1_775_121_700, 0).UTC()
updatedAt := createdAt.Add(time.Minute)
sentAt := updatedAt
record := deliverydomain.Delivery{
DeliveryID: common.DeliveryID("delivery-original"),
Source: deliverydomain.SourceNotification,
PayloadMode: deliverydomain.PayloadModeTemplate,
TemplateID: common.TemplateID("game.turn_ready"),
Envelope: deliverydomain.Envelope{
To: []common.Email{common.Email("pilot@example.com")},
Cc: []common.Email{common.Email("copilot@example.com")},
Bcc: []common.Email{common.Email("ops@example.com")},
ReplyTo: []common.Email{common.Email("noreply@example.com")},
},
Content: deliverydomain.Content{
Subject: "Turn ready",
TextBody: "Your next turn is ready",
},
Attachments: []common.AttachmentMetadata{
{Filename: "instructions.txt", ContentType: "text/plain; charset=utf-8", SizeBytes: 7},
},
Locale: common.Locale("en"),
TemplateVariables: map[string]any{"turn": 7},
LocaleFallbackUsed: true,
IdempotencyKey: common.IdempotencyKey("notification:delivery-original"),
Status: deliverydomain.StatusSent,
AttemptCount: 2,
LastAttemptStatus: attempt.StatusProviderAccepted,
ProviderSummary: "provider=smtp result=accepted",
CreatedAt: createdAt,
UpdatedAt: updatedAt,
SentAt: &sentAt,
}
if err := record.Validate(); err != nil {
panic(err)
}
return record
}
func validPayload(deliveryID common.DeliveryID) acceptgenericdelivery.DeliveryPayload {
payload := acceptgenericdelivery.DeliveryPayload{
DeliveryID: deliveryID,
Attachments: []acceptgenericdelivery.AttachmentPayload{
{
Filename: "instructions.txt",
ContentType: "text/plain; charset=utf-8",
ContentBase64: "cmVhZCBtZQ==",
SizeBytes: 7,
},
},
}
if err := payload.Validate(); err != nil {
panic(err)
}
return payload
}
var _ Store = (*stubStore)(nil)
var _ DeliveryIDGenerator = (*stubIDGenerator)(nil)
var _ Clock = stubClock{}
var _ Telemetry = (*stubTelemetry)(nil)
func hasResendSpanNamed(spans []sdktrace.ReadOnlySpan, name string) bool {
for _, span := range spans {
if span.Name() == name {
return true
}
}
return false
}