feat: use postgres
This commit is contained in:
Ilia Denisov
@@ -1,347 +0,0 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"galaxy/mail/internal/adapters/redisstate"
|
||||
"galaxy/mail/internal/adapters/stubprovider"
|
||||
"galaxy/mail/internal/domain/attempt"
|
||||
"galaxy/mail/internal/domain/common"
|
||||
deliverydomain "galaxy/mail/internal/domain/delivery"
|
||||
"galaxy/mail/internal/ports"
|
||||
"galaxy/mail/internal/service/executeattempt"
|
||||
"galaxy/mail/internal/service/renderdelivery"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAttemptWorkersSendImmediateFirstAttempt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newAttemptWorkerFixture(t, nil)
|
||||
createAcceptedRenderedDelivery(t, fixture.client, common.DeliveryID("delivery-immediate"), fixture.clock.Now())
|
||||
|
||||
cancel, wait := fixture.run(t)
|
||||
defer func() {
|
||||
cancel()
|
||||
wait()
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
deliveryRecord := loadDeliveryRecord(t, fixture.client, common.DeliveryID("delivery-immediate"))
|
||||
return deliveryRecord.Status == deliverydomain.StatusSent
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
require.Len(t, fixture.provider.Inputs(), 1)
|
||||
}
|
||||
|
||||
func TestAttemptWorkersRetryTransientFailuresUntilSuccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newAttemptWorkerFixture(t, []stubprovider.ScriptedOutcome{
|
||||
{
|
||||
Classification: ports.ClassificationTransientFailure,
|
||||
Script: "retry_1",
|
||||
},
|
||||
{
|
||||
Classification: ports.ClassificationTransientFailure,
|
||||
Script: "retry_2",
|
||||
},
|
||||
{
|
||||
Classification: ports.ClassificationAccepted,
|
||||
Script: "accepted",
|
||||
},
|
||||
})
|
||||
createAcceptedRenderedDelivery(t, fixture.client, common.DeliveryID("delivery-retry-success"), fixture.clock.Now())
|
||||
|
||||
cancel, wait := fixture.run(t)
|
||||
defer func() {
|
||||
cancel()
|
||||
wait()
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
deliveryRecord := loadDeliveryRecord(t, fixture.client, common.DeliveryID("delivery-retry-success"))
|
||||
return deliveryRecord.AttemptCount == 2 && deliveryRecord.Status == deliverydomain.StatusQueued
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
fixture.clock.Advance(time.Minute)
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
deliveryRecord := loadDeliveryRecord(t, fixture.client, common.DeliveryID("delivery-retry-success"))
|
||||
return deliveryRecord.AttemptCount == 3 && deliveryRecord.Status == deliverydomain.StatusQueued
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
fixture.clock.Advance(5 * time.Minute)
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
deliveryRecord := loadDeliveryRecord(t, fixture.client, common.DeliveryID("delivery-retry-success"))
|
||||
return deliveryRecord.Status == deliverydomain.StatusSent
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
require.Len(t, fixture.provider.Inputs(), 3)
|
||||
}
|
||||
|
||||
func TestAttemptWorkersDeadLetterAfterRetryExhaustion(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newAttemptWorkerFixture(t, []stubprovider.ScriptedOutcome{
|
||||
{Classification: ports.ClassificationTransientFailure, Script: "retry_1"},
|
||||
{Classification: ports.ClassificationTransientFailure, Script: "retry_2"},
|
||||
{Classification: ports.ClassificationTransientFailure, Script: "retry_3"},
|
||||
{Classification: ports.ClassificationTransientFailure, Script: "retry_4"},
|
||||
})
|
||||
deliveryID := common.DeliveryID("delivery-dead-letter")
|
||||
createAcceptedRenderedDelivery(t, fixture.client, deliveryID, fixture.clock.Now())
|
||||
|
||||
cancel, wait := fixture.run(t)
|
||||
defer func() {
|
||||
cancel()
|
||||
wait()
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
return loadDeliveryRecord(t, fixture.client, deliveryID).AttemptCount == 2
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
fixture.clock.Advance(time.Minute)
|
||||
require.Eventually(t, func() bool {
|
||||
return loadDeliveryRecord(t, fixture.client, deliveryID).AttemptCount == 3
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
fixture.clock.Advance(5 * time.Minute)
|
||||
require.Eventually(t, func() bool {
|
||||
return loadDeliveryRecord(t, fixture.client, deliveryID).AttemptCount == 4
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
fixture.clock.Advance(30 * time.Minute)
|
||||
require.Eventually(t, func() bool {
|
||||
return loadDeliveryRecord(t, fixture.client, deliveryID).Status == deliverydomain.StatusDeadLetter
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
deadLetter := loadDeadLetterRecord(t, fixture.client, deliveryID)
|
||||
require.Equal(t, "retry_exhausted", deadLetter.FailureClassification)
|
||||
require.Len(t, fixture.provider.Inputs(), 4)
|
||||
}
|
||||
|
||||
func TestAttemptWorkersRecoverExpiredClaimAfterCrash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newAttemptWorkerFixture(t, []stubprovider.ScriptedOutcome{
|
||||
{Classification: ports.ClassificationAccepted, Script: "accepted"},
|
||||
})
|
||||
deliveryID := common.DeliveryID("delivery-recovered")
|
||||
createAcceptedRenderedDelivery(t, fixture.client, deliveryID, fixture.clock.Now())
|
||||
|
||||
claimed, found, err := fixture.store.ClaimDueAttempt(context.Background(), deliveryID, fixture.clock.Now())
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.Equal(t, deliverydomain.StatusSending, claimed.Delivery.Status)
|
||||
|
||||
fixture.clock.Advance(20 * time.Millisecond)
|
||||
|
||||
cancel, wait := fixture.run(t)
|
||||
defer func() {
|
||||
cancel()
|
||||
wait()
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
deliveryRecord := loadDeliveryRecord(t, fixture.client, deliveryID)
|
||||
return deliveryRecord.Status == deliverydomain.StatusQueued && deliveryRecord.AttemptCount == 2
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
fixture.clock.Advance(time.Minute)
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
deliveryRecord := loadDeliveryRecord(t, fixture.client, deliveryID)
|
||||
return deliveryRecord.Status == deliverydomain.StatusSent
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
require.Len(t, fixture.provider.Inputs(), 1)
|
||||
}
|
||||
|
||||
type attemptWorkerFixture struct {
|
||||
client *redis.Client
|
||||
store *redisstate.AttemptExecutionStore
|
||||
service *executeattempt.Service
|
||||
scheduler *Scheduler
|
||||
pool *AttemptWorkerPool
|
||||
provider *stubprovider.Provider
|
||||
clock *schedulerTestClock
|
||||
}
|
||||
|
||||
func newAttemptWorkerFixture(t *testing.T, scripted []stubprovider.ScriptedOutcome) attemptWorkerFixture {
|
||||
t.Helper()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||
t.Cleanup(func() { require.NoError(t, client.Close()) })
|
||||
|
||||
store, err := redisstate.NewAttemptExecutionStore(client)
|
||||
require.NoError(t, err)
|
||||
|
||||
provider, err := stubprovider.New(scripted...)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, provider.Close()) })
|
||||
|
||||
clock := &schedulerTestClock{now: time.Unix(1_775_121_700, 0).UTC()}
|
||||
workQueue := make(chan executeattempt.WorkItem, 1)
|
||||
|
||||
service, err := executeattempt.New(executeattempt.Config{
|
||||
Renderer: noopRenderer{},
|
||||
Provider: provider,
|
||||
PayloadLoader: store,
|
||||
Store: store,
|
||||
Clock: clock,
|
||||
AttemptTimeout: 5 * time.Millisecond,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
scheduler, err := NewScheduler(SchedulerConfig{
|
||||
Store: store,
|
||||
Service: service,
|
||||
WorkQueue: workQueue,
|
||||
Clock: clock,
|
||||
AttemptTimeout: 5 * time.Millisecond,
|
||||
PollInterval: 10 * time.Millisecond,
|
||||
RecoveryInterval: 10 * time.Millisecond,
|
||||
RecoveryGrace: 5 * time.Millisecond,
|
||||
}, testWorkerLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
pool, err := NewAttemptWorkerPool(AttemptWorkerPoolConfig{
|
||||
Concurrency: 1,
|
||||
WorkQueue: workQueue,
|
||||
Service: service,
|
||||
}, testWorkerLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
return attemptWorkerFixture{
|
||||
client: client,
|
||||
store: store,
|
||||
service: service,
|
||||
scheduler: scheduler,
|
||||
pool: pool,
|
||||
provider: provider,
|
||||
clock: clock,
|
||||
}
|
||||
}
|
||||
|
||||
func (fixture attemptWorkerFixture) run(t *testing.T) (context.CancelFunc, func()) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
schedulerDone := make(chan error, 1)
|
||||
poolDone := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
schedulerDone <- fixture.scheduler.Run(ctx)
|
||||
}()
|
||||
go func() {
|
||||
poolDone <- fixture.pool.Run(ctx)
|
||||
}()
|
||||
|
||||
wait := func() {
|
||||
require.ErrorIs(t, <-schedulerDone, context.Canceled)
|
||||
require.ErrorIs(t, <-poolDone, context.Canceled)
|
||||
}
|
||||
|
||||
return cancel, wait
|
||||
}
|
||||
|
||||
type schedulerTestClock struct {
|
||||
mu sync.Mutex
|
||||
now time.Time
|
||||
}
|
||||
|
||||
func (clock *schedulerTestClock) Now() time.Time {
|
||||
clock.mu.Lock()
|
||||
defer clock.mu.Unlock()
|
||||
return clock.now
|
||||
}
|
||||
|
||||
func (clock *schedulerTestClock) Advance(delta time.Duration) {
|
||||
clock.mu.Lock()
|
||||
defer clock.mu.Unlock()
|
||||
clock.now = clock.now.Add(delta)
|
||||
}
|
||||
|
||||
type noopRenderer struct{}
|
||||
|
||||
func (noopRenderer) Execute(context.Context, renderdelivery.Input) (renderdelivery.Result, error) {
|
||||
return renderdelivery.Result{}, errors.New("unexpected render invocation")
|
||||
}
|
||||
|
||||
func createAcceptedRenderedDelivery(t *testing.T, client *redis.Client, deliveryID common.DeliveryID, createdAt time.Time) {
|
||||
t.Helper()
|
||||
|
||||
writer, err := redisstate.NewAtomicWriter(client)
|
||||
require.NoError(t, err)
|
||||
|
||||
deliveryRecord := deliverydomain.Delivery{
|
||||
DeliveryID: deliveryID,
|
||||
Source: deliverydomain.SourceNotification,
|
||||
PayloadMode: deliverydomain.PayloadModeRendered,
|
||||
Envelope: deliverydomain.Envelope{
|
||||
To: []common.Email{common.Email("pilot@example.com")},
|
||||
},
|
||||
Content: deliverydomain.Content{
|
||||
Subject: "Turn ready",
|
||||
TextBody: "Turn 54 is ready.",
|
||||
},
|
||||
IdempotencyKey: common.IdempotencyKey("notification:" + deliveryID.String()),
|
||||
Status: deliverydomain.StatusQueued,
|
||||
AttemptCount: 1,
|
||||
CreatedAt: createdAt.UTC().Truncate(time.Millisecond),
|
||||
UpdatedAt: createdAt.UTC().Truncate(time.Millisecond),
|
||||
}
|
||||
require.NoError(t, deliveryRecord.Validate())
|
||||
|
||||
firstAttempt := attempt.Attempt{
|
||||
DeliveryID: deliveryID,
|
||||
AttemptNo: 1,
|
||||
ScheduledFor: createdAt.UTC().Truncate(time.Millisecond),
|
||||
Status: attempt.StatusScheduled,
|
||||
}
|
||||
require.NoError(t, firstAttempt.Validate())
|
||||
|
||||
require.NoError(t, writer.CreateAcceptance(context.Background(), redisstate.CreateAcceptanceInput{
|
||||
Delivery: deliveryRecord,
|
||||
FirstAttempt: &firstAttempt,
|
||||
}))
|
||||
}
|
||||
|
||||
func loadDeliveryRecord(t *testing.T, client *redis.Client, deliveryID common.DeliveryID) deliverydomain.Delivery {
|
||||
t.Helper()
|
||||
|
||||
payload, err := client.Get(context.Background(), redisstate.Keyspace{}.Delivery(deliveryID)).Bytes()
|
||||
require.NoError(t, err)
|
||||
record, err := redisstate.UnmarshalDelivery(payload)
|
||||
require.NoError(t, err)
|
||||
|
||||
return record
|
||||
}
|
||||
|
||||
func loadDeadLetterRecord(t *testing.T, client *redis.Client, deliveryID common.DeliveryID) deliverydomain.DeadLetterEntry {
|
||||
t.Helper()
|
||||
|
||||
payload, err := client.Get(context.Background(), redisstate.Keyspace{}.DeadLetter(deliveryID)).Bytes()
|
||||
require.NoError(t, err)
|
||||
record, err := redisstate.UnmarshalDeadLetter(payload)
|
||||
require.NoError(t, err)
|
||||
|
||||
return record
|
||||
}
|
||||
|
||||
func testWorkerLogger() *slog.Logger {
|
||||
return slog.New(slog.NewJSONHandler(io.Discard, nil))
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"galaxy/mail/internal/adapters/redisstate"
|
||||
)
|
||||
|
||||
const cleanupInterval = time.Hour
|
||||
|
||||
// CleanupWorker stores the idle index cleanup worker used by the Stage 6
|
||||
// runtime skeleton.
|
||||
type CleanupWorker struct {
|
||||
cleaner *redisstate.IndexCleaner
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewCleanupWorker constructs the idle Stage 6 cleanup worker.
|
||||
func NewCleanupWorker(cleaner *redisstate.IndexCleaner, logger *slog.Logger) (*CleanupWorker, error) {
|
||||
if cleaner == nil {
|
||||
return nil, errors.New("new cleanup worker: nil index cleaner")
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
|
||||
return &CleanupWorker{
|
||||
cleaner: cleaner,
|
||||
logger: logger.With("component", "cleanup_worker"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Run starts the idle cleanup worker and blocks until ctx is canceled.
|
||||
func (worker *CleanupWorker) Run(ctx context.Context) error {
|
||||
if ctx == nil {
|
||||
return errors.New("run cleanup worker: nil context")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if worker == nil || worker.cleaner == nil {
|
||||
return errors.New("run cleanup worker: nil cleanup worker")
|
||||
}
|
||||
|
||||
worker.logger.Info("cleanup worker started", "interval", cleanupInterval.String())
|
||||
ticker := time.NewTicker(cleanupInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
worker.logger.Info("cleanup worker stopped")
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown stops the cleanup worker within ctx. The Stage 6 skeleton has no
|
||||
// additional resources to release.
|
||||
func (worker *CleanupWorker) Shutdown(ctx context.Context) error {
|
||||
if ctx == nil {
|
||||
return errors.New("shutdown cleanup worker: nil context")
|
||||
}
|
||||
if worker == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -304,9 +304,10 @@ func optionalRawString(values map[string]any, key string) string {
|
||||
return value
|
||||
}
|
||||
|
||||
// Shutdown stops the command consumer within ctx. The consumer uses the
|
||||
// shared process Redis client and therefore has no dedicated resources to
|
||||
// release here.
|
||||
// Shutdown stops the command consumer within ctx. The consumer borrows the
|
||||
// shared process Redis client and forcibly closes it during Shutdown so the
|
||||
// in-flight blocking XREAD returns immediately; the runtime owns the same
|
||||
// client and its cleanupFn is tolerant of ErrClosed.
|
||||
func (consumer *CommandConsumer) Shutdown(ctx context.Context) error {
|
||||
if ctx == nil {
|
||||
return errors.New("shutdown command consumer: nil context")
|
||||
@@ -318,9 +319,10 @@ func (consumer *CommandConsumer) Shutdown(ctx context.Context) error {
|
||||
var err error
|
||||
consumer.closeOnce.Do(func() {
|
||||
if consumer.client != nil {
|
||||
err = consumer.client.Close()
|
||||
if cerr := consumer.client.Close(); cerr != nil && !errors.Is(cerr, redis.ErrClosed) {
|
||||
err = cerr
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,391 +0,0 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"galaxy/mail/internal/adapters/redisstate"
|
||||
"galaxy/mail/internal/service/acceptgenericdelivery"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCommandConsumerAcceptsRenderedCommand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newCommandConsumerFixture(t)
|
||||
messageID := addRenderedCommand(t, fixture.client, "mail-123", "notification:mail-123")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- fixture.consumer.Run(ctx)
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
delivery, found, err := fixture.acceptanceStore.GetDelivery(context.Background(), "mail-123")
|
||||
if err != nil || !found {
|
||||
return false
|
||||
}
|
||||
entryID, found, err := fixture.offsetStore.Load(context.Background(), fixture.stream)
|
||||
return err == nil && found && entryID == messageID && delivery.DeliveryID == "mail-123"
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
cancel()
|
||||
require.ErrorIs(t, <-done, context.Canceled)
|
||||
}
|
||||
|
||||
func TestCommandConsumerAcceptsTemplateCommand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newCommandConsumerFixture(t)
|
||||
messageID := addTemplateCommand(t, fixture.client, "mail-124", "notification:mail-124")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- fixture.consumer.Run(ctx)
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
delivery, found, err := fixture.acceptanceStore.GetDelivery(context.Background(), "mail-124")
|
||||
if err != nil || !found {
|
||||
return false
|
||||
}
|
||||
entryID, found, err := fixture.offsetStore.Load(context.Background(), fixture.stream)
|
||||
return err == nil && found && entryID == messageID && delivery.TemplateID == "game.turn.ready"
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
cancel()
|
||||
require.ErrorIs(t, <-done, context.Canceled)
|
||||
}
|
||||
|
||||
func TestCommandConsumerRecordsMalformedCommandAndContinues(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newCommandConsumerFixture(t)
|
||||
malformedID := addMalformedRenderedCommand(t, fixture.client, "mail-bad", "notification:mail-bad")
|
||||
validID := addRenderedCommand(t, fixture.client, "mail-125", "notification:mail-125")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- fixture.consumer.Run(ctx)
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
_, deliveryFound, deliveryErr := fixture.acceptanceStore.GetDelivery(context.Background(), "mail-125")
|
||||
entry, malformedFound, malformedErr := fixture.malformedStore.Get(context.Background(), malformedID)
|
||||
entryID, offsetFound, offsetErr := fixture.offsetStore.Load(context.Background(), fixture.stream)
|
||||
return deliveryErr == nil &&
|
||||
malformedErr == nil &&
|
||||
offsetErr == nil &&
|
||||
deliveryFound &&
|
||||
malformedFound &&
|
||||
entry.FailureCode == "invalid_payload" &&
|
||||
offsetFound &&
|
||||
entryID == validID
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
cancel()
|
||||
require.ErrorIs(t, <-done, context.Canceled)
|
||||
}
|
||||
|
||||
func TestCommandConsumerRestartsFromSavedOffset(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newCommandConsumerFixture(t)
|
||||
firstID := addRenderedCommand(t, fixture.client, "mail-126", "notification:mail-126")
|
||||
|
||||
firstCtx, firstCancel := context.WithCancel(context.Background())
|
||||
firstDone := make(chan error, 1)
|
||||
go func() {
|
||||
firstDone <- fixture.consumer.Run(firstCtx)
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
entryID, found, err := fixture.offsetStore.Load(context.Background(), fixture.stream)
|
||||
return err == nil && found && entryID == firstID
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
firstCancel()
|
||||
require.ErrorIs(t, <-firstDone, context.Canceled)
|
||||
|
||||
secondID := addRenderedCommand(t, fixture.client, "mail-127", "notification:mail-127")
|
||||
|
||||
secondCtx, secondCancel := context.WithCancel(context.Background())
|
||||
secondDone := make(chan error, 1)
|
||||
go func() {
|
||||
secondDone <- fixture.consumer.Run(secondCtx)
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
_, firstFound, firstErr := fixture.acceptanceStore.GetDelivery(context.Background(), "mail-126")
|
||||
_, secondFound, secondErr := fixture.acceptanceStore.GetDelivery(context.Background(), "mail-127")
|
||||
entryID, offsetFound, offsetErr := fixture.offsetStore.Load(context.Background(), fixture.stream)
|
||||
return firstErr == nil &&
|
||||
secondErr == nil &&
|
||||
offsetErr == nil &&
|
||||
firstFound &&
|
||||
secondFound &&
|
||||
offsetFound &&
|
||||
entryID == secondID
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
secondCancel()
|
||||
require.ErrorIs(t, <-secondDone, context.Canceled)
|
||||
}
|
||||
|
||||
func TestCommandConsumerDoesNotDuplicateAcceptanceAfterOffsetSaveFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newCommandConsumerFixture(t)
|
||||
messageID := addRenderedCommand(t, fixture.client, "mail-128", "notification:mail-128")
|
||||
failingOffsetStore := &scriptedOffsetStore{
|
||||
saveErrs: []error{errors.New("offset unavailable")},
|
||||
}
|
||||
consumer := newCommandConsumerForTest(t, fixture.client, fixture.stream, fixture.acceptor, fixture.malformedStore, failingOffsetStore)
|
||||
|
||||
err := consumer.Run(context.Background())
|
||||
require.Error(t, err)
|
||||
require.ErrorContains(t, err, "save stream offset")
|
||||
|
||||
delivery, found, err := fixture.acceptanceStore.GetDelivery(context.Background(), "mail-128")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.Equal(t, "mail-128", delivery.DeliveryID.String())
|
||||
|
||||
indexCard, err := fixture.client.ZCard(context.Background(), redisstate.Keyspace{}.CreatedAtIndex()).Result()
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 1, indexCard)
|
||||
|
||||
replayConsumer := newCommandConsumerForTest(t, fixture.client, fixture.stream, fixture.acceptor, fixture.malformedStore, failingOffsetStore)
|
||||
replayCtx, replayCancel := context.WithCancel(context.Background())
|
||||
replayDone := make(chan error, 1)
|
||||
go func() {
|
||||
replayDone <- replayConsumer.Run(replayCtx)
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
return failingOffsetStore.lastEntryID == messageID
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
replayCancel()
|
||||
require.ErrorIs(t, <-replayDone, context.Canceled)
|
||||
|
||||
indexCard, err = fixture.client.ZCard(context.Background(), redisstate.Keyspace{}.CreatedAtIndex()).Result()
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 1, indexCard)
|
||||
|
||||
scheduleCard, err := fixture.client.ZCard(context.Background(), redisstate.Keyspace{}.AttemptSchedule()).Result()
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 1, scheduleCard)
|
||||
}
|
||||
|
||||
func TestCommandConsumerRecordsIdempotencyConflictAsMalformed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fixture := newCommandConsumerFixture(t)
|
||||
addRenderedCommand(t, fixture.client, "mail-129", "notification:shared")
|
||||
conflictID := addRenderedCommandWithSubject(t, fixture.client, "mail-130", "notification:shared", "Different subject")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- fixture.consumer.Run(ctx)
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
_, firstFound, firstErr := fixture.acceptanceStore.GetDelivery(context.Background(), "mail-129")
|
||||
_, secondFound, secondErr := fixture.acceptanceStore.GetDelivery(context.Background(), "mail-130")
|
||||
entry, malformedFound, malformedErr := fixture.malformedStore.Get(context.Background(), conflictID)
|
||||
return firstErr == nil &&
|
||||
secondErr == nil &&
|
||||
malformedErr == nil &&
|
||||
firstFound &&
|
||||
!secondFound &&
|
||||
malformedFound &&
|
||||
entry.FailureCode == "idempotency_conflict"
|
||||
}, 5*time.Second, 20*time.Millisecond)
|
||||
|
||||
cancel()
|
||||
require.ErrorIs(t, <-done, context.Canceled)
|
||||
}
|
||||
|
||||
type commandConsumerFixture struct {
|
||||
client *redis.Client
|
||||
stream string
|
||||
consumer *CommandConsumer
|
||||
acceptor *acceptgenericdelivery.Service
|
||||
acceptanceStore *redisstate.GenericAcceptanceStore
|
||||
malformedStore *redisstate.MalformedCommandStore
|
||||
offsetStore *redisstate.StreamOffsetStore
|
||||
}
|
||||
|
||||
func newCommandConsumerFixture(t *testing.T) commandConsumerFixture {
|
||||
t.Helper()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||
t.Cleanup(func() { require.NoError(t, client.Close()) })
|
||||
|
||||
acceptanceStore, err := redisstate.NewGenericAcceptanceStore(client)
|
||||
require.NoError(t, err)
|
||||
now := time.Now().UTC().Truncate(time.Millisecond)
|
||||
acceptor, err := acceptgenericdelivery.New(acceptgenericdelivery.Config{
|
||||
Store: acceptanceStore,
|
||||
Clock: testClock{now: now},
|
||||
IdempotencyTTL: redisstate.IdempotencyTTL,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
malformedStore, err := redisstate.NewMalformedCommandStore(client)
|
||||
require.NoError(t, err)
|
||||
offsetStore, err := redisstate.NewStreamOffsetStore(client)
|
||||
require.NoError(t, err)
|
||||
|
||||
stream := redisstate.Keyspace{}.DeliveryCommands()
|
||||
consumer := newCommandConsumerForTest(t, client, stream, acceptor, malformedStore, offsetStore)
|
||||
|
||||
return commandConsumerFixture{
|
||||
client: client,
|
||||
stream: stream,
|
||||
consumer: consumer,
|
||||
acceptor: acceptor,
|
||||
acceptanceStore: acceptanceStore,
|
||||
malformedStore: malformedStore,
|
||||
offsetStore: offsetStore,
|
||||
}
|
||||
}
|
||||
|
||||
func newCommandConsumerForTest(
|
||||
t *testing.T,
|
||||
client *redis.Client,
|
||||
stream string,
|
||||
acceptor AcceptGenericDeliveryUseCase,
|
||||
malformedRecorder MalformedCommandRecorder,
|
||||
offsetStore StreamOffsetStore,
|
||||
) *CommandConsumer {
|
||||
t.Helper()
|
||||
|
||||
consumer, err := NewCommandConsumer(CommandConsumerConfig{
|
||||
Client: client,
|
||||
Stream: stream,
|
||||
BlockTimeout: 20 * time.Millisecond,
|
||||
Acceptor: acceptor,
|
||||
MalformedRecorder: malformedRecorder,
|
||||
OffsetStore: offsetStore,
|
||||
Clock: testClock{now: time.Now().UTC().Truncate(time.Millisecond)},
|
||||
}, testLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
return consumer
|
||||
}
|
||||
|
||||
func addRenderedCommand(t *testing.T, client *redis.Client, deliveryID string, idempotencyKey string) string {
|
||||
t.Helper()
|
||||
|
||||
return addRenderedCommandWithSubject(t, client, deliveryID, idempotencyKey, "Turn ready")
|
||||
}
|
||||
|
||||
func addRenderedCommandWithSubject(t *testing.T, client *redis.Client, deliveryID string, idempotencyKey string, subject string) string {
|
||||
t.Helper()
|
||||
|
||||
messageID, err := client.XAdd(context.Background(), &redis.XAddArgs{
|
||||
Stream: redisstate.Keyspace{}.DeliveryCommands(),
|
||||
Values: map[string]any{
|
||||
"delivery_id": deliveryID,
|
||||
"source": "notification",
|
||||
"payload_mode": "rendered",
|
||||
"idempotency_key": idempotencyKey,
|
||||
"requested_at_ms": "1775121700000",
|
||||
"payload_json": `{"to":["pilot@example.com"],"cc":[],"bcc":[],"reply_to":["noreply@example.com"],"subject":"` + subject + `","text_body":"Turn 54 is ready.","html_body":"<p>Turn 54 is ready.</p>","attachments":[]}`,
|
||||
},
|
||||
}).Result()
|
||||
require.NoError(t, err)
|
||||
|
||||
return messageID
|
||||
}
|
||||
|
||||
func addTemplateCommand(t *testing.T, client *redis.Client, deliveryID string, idempotencyKey string) string {
|
||||
t.Helper()
|
||||
|
||||
messageID, err := client.XAdd(context.Background(), &redis.XAddArgs{
|
||||
Stream: redisstate.Keyspace{}.DeliveryCommands(),
|
||||
Values: map[string]any{
|
||||
"delivery_id": deliveryID,
|
||||
"source": "notification",
|
||||
"payload_mode": "template",
|
||||
"idempotency_key": idempotencyKey,
|
||||
"requested_at_ms": "1775121700001",
|
||||
"payload_json": `{"to":["pilot@example.com"],"cc":[],"bcc":[],"reply_to":[],"template_id":"game.turn.ready","locale":"fr-FR","variables":{"turn_number":54},"attachments":[]}`,
|
||||
},
|
||||
}).Result()
|
||||
require.NoError(t, err)
|
||||
|
||||
return messageID
|
||||
}
|
||||
|
||||
func addMalformedRenderedCommand(t *testing.T, client *redis.Client, deliveryID string, idempotencyKey string) string {
|
||||
t.Helper()
|
||||
|
||||
messageID, err := client.XAdd(context.Background(), &redis.XAddArgs{
|
||||
Stream: redisstate.Keyspace{}.DeliveryCommands(),
|
||||
Values: map[string]any{
|
||||
"delivery_id": deliveryID,
|
||||
"source": "notification",
|
||||
"payload_mode": "rendered",
|
||||
"idempotency_key": idempotencyKey,
|
||||
"requested_at_ms": "1775121700000",
|
||||
"payload_json": `{"to":["pilot@example.com"],"cc":[],"bcc":[],"reply_to":[],"text_body":"Turn 54 is ready.","attachments":[]}`,
|
||||
},
|
||||
}).Result()
|
||||
require.NoError(t, err)
|
||||
|
||||
return messageID
|
||||
}
|
||||
|
||||
type testClock struct {
|
||||
now time.Time
|
||||
}
|
||||
|
||||
func (clock testClock) Now() time.Time {
|
||||
return clock.now
|
||||
}
|
||||
|
||||
type scriptedOffsetStore struct {
|
||||
lastEntryID string
|
||||
found bool
|
||||
saveErrs []error
|
||||
saveCalls int
|
||||
}
|
||||
|
||||
func (store *scriptedOffsetStore) Load(context.Context, string) (string, bool, error) {
|
||||
if !store.found {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
return store.lastEntryID, true, nil
|
||||
}
|
||||
|
||||
func (store *scriptedOffsetStore) Save(_ context.Context, _ string, entryID string) error {
|
||||
if store.saveCalls < len(store.saveErrs) && store.saveErrs[store.saveCalls] != nil {
|
||||
store.saveCalls++
|
||||
return store.saveErrs[store.saveCalls-1]
|
||||
}
|
||||
|
||||
store.saveCalls++
|
||||
store.lastEntryID = entryID
|
||||
store.found = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func testLogger() *slog.Logger {
|
||||
return slog.New(slog.NewJSONHandler(io.Discard, nil))
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SQLRetentionStore performs the durable DELETE statements applied by the
|
||||
// retention worker. Implementations are typically the umbrella PostgreSQL
|
||||
// mail store; the interface keeps the worker decoupled from the store
|
||||
// package.
|
||||
type SQLRetentionStore interface {
|
||||
// DeleteDeliveriesOlderThan removes deliveries whose created_at predates
|
||||
// cutoff. Cascading FKs drop attempts, dead_letters, delivery_payloads,
|
||||
// and delivery_recipients owned by the deleted rows.
|
||||
DeleteDeliveriesOlderThan(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
|
||||
// DeleteMalformedCommandsOlderThan removes malformed-command rows whose
|
||||
// recorded_at predates cutoff.
|
||||
DeleteMalformedCommandsOlderThan(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
}
|
||||
|
||||
// SQLRetentionConfig stores the dependencies and policy used by
|
||||
// SQLRetentionWorker.
|
||||
type SQLRetentionConfig struct {
|
||||
// Store applies the durable DELETE statements.
|
||||
Store SQLRetentionStore
|
||||
|
||||
// DeliveryRetention bounds how long deliveries (and their cascaded
|
||||
// attempts/dead_letters/payloads/recipients) survive after creation.
|
||||
DeliveryRetention time.Duration
|
||||
|
||||
// MalformedCommandRetention bounds how long malformed-command rows
|
||||
// survive after recorded_at.
|
||||
MalformedCommandRetention time.Duration
|
||||
|
||||
// CleanupInterval stores the wall-clock period between two retention
|
||||
// passes.
|
||||
CleanupInterval time.Duration
|
||||
|
||||
// Clock provides the wall-clock used to compute cutoff timestamps.
|
||||
Clock Clock
|
||||
}
|
||||
|
||||
// SQLRetentionWorker periodically deletes deliveries and malformed-command
|
||||
// rows whose retention window has expired. The worker replaces the previous
|
||||
// Redis index_cleaner that maintained secondary index keys; PostgreSQL
|
||||
// indexes are maintained by the engine, so the worker only needs to enforce
|
||||
// retention.
|
||||
type SQLRetentionWorker struct {
|
||||
store SQLRetentionStore
|
||||
deliveryRetention time.Duration
|
||||
malformedCommandRetention time.Duration
|
||||
cleanupInterval time.Duration
|
||||
clock Clock
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewSQLRetentionWorker constructs the periodic retention worker.
|
||||
func NewSQLRetentionWorker(cfg SQLRetentionConfig, logger *slog.Logger) (*SQLRetentionWorker, error) {
|
||||
switch {
|
||||
case cfg.Store == nil:
|
||||
return nil, errors.New("new sql retention worker: nil store")
|
||||
case cfg.DeliveryRetention <= 0:
|
||||
return nil, errors.New("new sql retention worker: non-positive delivery retention")
|
||||
case cfg.MalformedCommandRetention <= 0:
|
||||
return nil, errors.New("new sql retention worker: non-positive malformed command retention")
|
||||
case cfg.CleanupInterval <= 0:
|
||||
return nil, errors.New("new sql retention worker: non-positive cleanup interval")
|
||||
case cfg.Clock == nil:
|
||||
return nil, errors.New("new sql retention worker: nil clock")
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
|
||||
return &SQLRetentionWorker{
|
||||
store: cfg.Store,
|
||||
deliveryRetention: cfg.DeliveryRetention,
|
||||
malformedCommandRetention: cfg.MalformedCommandRetention,
|
||||
cleanupInterval: cfg.CleanupInterval,
|
||||
clock: cfg.Clock,
|
||||
logger: logger.With("component", "sql_retention_worker"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Run starts the retention loop and blocks until ctx is canceled.
|
||||
func (worker *SQLRetentionWorker) Run(ctx context.Context) error {
|
||||
if ctx == nil {
|
||||
return errors.New("run sql retention worker: nil context")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if worker == nil {
|
||||
return errors.New("run sql retention worker: nil worker")
|
||||
}
|
||||
|
||||
worker.logger.Info("sql retention worker started",
|
||||
"delivery_retention", worker.deliveryRetention.String(),
|
||||
"malformed_command_retention", worker.malformedCommandRetention.String(),
|
||||
"cleanup_interval", worker.cleanupInterval.String(),
|
||||
)
|
||||
defer worker.logger.Info("sql retention worker stopped")
|
||||
|
||||
// First pass runs immediately so a freshly started service does not wait
|
||||
// one full interval before evicting stale rows.
|
||||
worker.runOnce(ctx)
|
||||
|
||||
ticker := time.NewTicker(worker.cleanupInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
worker.runOnce(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown stops the retention worker within ctx.
|
||||
func (worker *SQLRetentionWorker) Shutdown(ctx context.Context) error {
|
||||
if ctx == nil {
|
||||
return errors.New("shutdown sql retention worker: nil context")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (worker *SQLRetentionWorker) runOnce(ctx context.Context) {
|
||||
now := worker.clock.Now().UTC()
|
||||
|
||||
deliveryCutoff := now.Add(-worker.deliveryRetention)
|
||||
if deleted, err := worker.store.DeleteDeliveriesOlderThan(ctx, deliveryCutoff); err != nil {
|
||||
worker.logger.Warn("delete expired deliveries failed",
|
||||
"cutoff", deliveryCutoff,
|
||||
"error", fmt.Sprintf("%v", err),
|
||||
)
|
||||
} else if deleted > 0 {
|
||||
worker.logger.Info("expired deliveries deleted",
|
||||
"cutoff", deliveryCutoff,
|
||||
"deleted", deleted,
|
||||
)
|
||||
}
|
||||
|
||||
malformedCutoff := now.Add(-worker.malformedCommandRetention)
|
||||
if deleted, err := worker.store.DeleteMalformedCommandsOlderThan(ctx, malformedCutoff); err != nil {
|
||||
worker.logger.Warn("delete expired malformed commands failed",
|
||||
"cutoff", malformedCutoff,
|
||||
"error", fmt.Sprintf("%v", err),
|
||||
)
|
||||
} else if deleted > 0 {
|
||||
worker.logger.Info("expired malformed commands deleted",
|
||||
"cutoff", malformedCutoff,
|
||||
"deleted", deleted,
|
||||
)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user