feat: use postgres
This commit is contained in:
Ilia Denisov
@@ -1,140 +0,0 @@
|
||||
package redisstate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"galaxy/notification/internal/api/intentstream"
|
||||
"galaxy/notification/internal/service/acceptintent"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// AcceptanceStore provides the Redis-backed durable storage used by the
|
||||
// intent-acceptance use case.
|
||||
type AcceptanceStore struct {
|
||||
client *redis.Client
|
||||
writer *AtomicWriter
|
||||
keys Keyspace
|
||||
cfg AcceptanceConfig
|
||||
}
|
||||
|
||||
// NewAcceptanceStore constructs one Redis-backed acceptance store.
|
||||
func NewAcceptanceStore(client *redis.Client, cfg AcceptanceConfig) (*AcceptanceStore, error) {
|
||||
if client == nil {
|
||||
return nil, errors.New("new notification acceptance store: nil redis client")
|
||||
}
|
||||
|
||||
writer, err := NewAtomicWriter(client, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new notification acceptance store: %w", err)
|
||||
}
|
||||
|
||||
return &AcceptanceStore{
|
||||
client: client,
|
||||
writer: writer,
|
||||
keys: Keyspace{},
|
||||
cfg: cfg,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CreateAcceptance stores one complete accepted notification write set in
|
||||
// Redis.
|
||||
func (store *AcceptanceStore) CreateAcceptance(ctx context.Context, input acceptintent.CreateAcceptanceInput) error {
|
||||
if store == nil || store.client == nil || store.writer == nil {
|
||||
return errors.New("create notification acceptance: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return errors.New("create notification acceptance: nil context")
|
||||
}
|
||||
if err := input.Validate(); err != nil {
|
||||
return fmt.Errorf("create notification acceptance: %w", err)
|
||||
}
|
||||
|
||||
err := store.writer.CreateAcceptance(ctx, input)
|
||||
if errors.Is(err, ErrConflict) {
|
||||
return fmt.Errorf("create notification acceptance: %w", acceptintent.ErrConflict)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("create notification acceptance: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetIdempotency loads one accepted idempotency scope from Redis.
|
||||
func (store *AcceptanceStore) GetIdempotency(ctx context.Context, producer intentstream.Producer, idempotencyKey string) (acceptintent.IdempotencyRecord, bool, error) {
|
||||
if store == nil || store.client == nil {
|
||||
return acceptintent.IdempotencyRecord{}, false, errors.New("get notification idempotency: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return acceptintent.IdempotencyRecord{}, false, errors.New("get notification idempotency: nil context")
|
||||
}
|
||||
|
||||
payload, err := store.client.Get(ctx, store.keys.Idempotency(producer, idempotencyKey)).Bytes()
|
||||
switch {
|
||||
case errors.Is(err, redis.Nil):
|
||||
return acceptintent.IdempotencyRecord{}, false, nil
|
||||
case err != nil:
|
||||
return acceptintent.IdempotencyRecord{}, false, fmt.Errorf("get notification idempotency: %w", err)
|
||||
}
|
||||
|
||||
record, err := UnmarshalIdempotency(payload)
|
||||
if err != nil {
|
||||
return acceptintent.IdempotencyRecord{}, false, fmt.Errorf("get notification idempotency: %w", err)
|
||||
}
|
||||
|
||||
return record, true, nil
|
||||
}
|
||||
|
||||
// GetNotification loads one accepted notification record from Redis.
|
||||
func (store *AcceptanceStore) GetNotification(ctx context.Context, notificationID string) (acceptintent.NotificationRecord, bool, error) {
|
||||
if store == nil || store.client == nil {
|
||||
return acceptintent.NotificationRecord{}, false, errors.New("get notification record: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return acceptintent.NotificationRecord{}, false, errors.New("get notification record: nil context")
|
||||
}
|
||||
|
||||
payload, err := store.client.Get(ctx, store.keys.Notification(notificationID)).Bytes()
|
||||
switch {
|
||||
case errors.Is(err, redis.Nil):
|
||||
return acceptintent.NotificationRecord{}, false, nil
|
||||
case err != nil:
|
||||
return acceptintent.NotificationRecord{}, false, fmt.Errorf("get notification record: %w", err)
|
||||
}
|
||||
|
||||
record, err := UnmarshalNotification(payload)
|
||||
if err != nil {
|
||||
return acceptintent.NotificationRecord{}, false, fmt.Errorf("get notification record: %w", err)
|
||||
}
|
||||
|
||||
return record, true, nil
|
||||
}
|
||||
|
||||
// GetRoute loads one accepted notification route by NotificationID and
|
||||
// RouteID.
|
||||
func (store *AcceptanceStore) GetRoute(ctx context.Context, notificationID string, routeID string) (acceptintent.NotificationRoute, bool, error) {
|
||||
if store == nil || store.client == nil {
|
||||
return acceptintent.NotificationRoute{}, false, errors.New("get notification route: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return acceptintent.NotificationRoute{}, false, errors.New("get notification route: nil context")
|
||||
}
|
||||
|
||||
payload, err := store.client.Get(ctx, store.keys.Route(notificationID, routeID)).Bytes()
|
||||
switch {
|
||||
case errors.Is(err, redis.Nil):
|
||||
return acceptintent.NotificationRoute{}, false, nil
|
||||
case err != nil:
|
||||
return acceptintent.NotificationRoute{}, false, fmt.Errorf("get notification route: %w", err)
|
||||
}
|
||||
|
||||
record, err := UnmarshalRoute(payload)
|
||||
if err != nil {
|
||||
return acceptintent.NotificationRoute{}, false, fmt.Errorf("get notification route: %w", err)
|
||||
}
|
||||
|
||||
return record, true, nil
|
||||
}
|
||||
@@ -1,311 +0,0 @@
|
||||
package redisstate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"galaxy/notification/internal/api/intentstream"
|
||||
"galaxy/notification/internal/config"
|
||||
"galaxy/notification/internal/service/acceptintent"
|
||||
"galaxy/notification/internal/service/malformedintent"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAcceptanceStoreCreateAcceptancePersistsNotificationRoutesAndSchedule(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewAcceptanceStore(client, AcceptanceConfig{
|
||||
RecordTTL: 24 * time.Hour,
|
||||
DeadLetterTTL: 72 * time.Hour,
|
||||
IdempotencyTTL: 7 * 24 * time.Hour,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
now := time.UnixMilli(1775121700000).UTC()
|
||||
input := validAdminAcceptanceInput(now)
|
||||
|
||||
require.NoError(t, store.CreateAcceptance(context.Background(), input))
|
||||
|
||||
notificationRecord, found, err := store.GetNotification(context.Background(), input.Notification.NotificationID)
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.Equal(t, input.Notification.NotificationID, notificationRecord.NotificationID)
|
||||
|
||||
idempotencyRecord, found, err := store.GetIdempotency(context.Background(), input.Idempotency.Producer, input.Idempotency.IdempotencyKey)
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.Equal(t, input.Idempotency.RequestFingerprint, idempotencyRecord.RequestFingerprint)
|
||||
|
||||
pushRoutePayload, err := client.Get(context.Background(), Keyspace{}.Route(input.Notification.NotificationID, "push:email:owner@example.com")).Bytes()
|
||||
require.NoError(t, err)
|
||||
pushRoute, err := UnmarshalRoute(pushRoutePayload)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, acceptintent.RouteStatusSkipped, pushRoute.Status)
|
||||
|
||||
emailRouteKey := Keyspace{}.Route(input.Notification.NotificationID, "email:email:owner@example.com")
|
||||
emailRoutePayload, err := client.Get(context.Background(), emailRouteKey).Bytes()
|
||||
require.NoError(t, err)
|
||||
emailRoute, err := UnmarshalRoute(emailRoutePayload)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, acceptintent.RouteStatusPending, emailRoute.Status)
|
||||
|
||||
scheduled, err := client.ZRangeWithScores(context.Background(), Keyspace{}.RouteSchedule(), 0, -1).Result()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, scheduled, 1)
|
||||
require.Equal(t, emailRouteKey, scheduled[0].Member)
|
||||
require.Equal(t, float64(now.UnixMilli()), scheduled[0].Score)
|
||||
|
||||
notificationTTL, err := client.PTTL(context.Background(), Keyspace{}.Notification(input.Notification.NotificationID)).Result()
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, notificationTTL, 23*time.Hour)
|
||||
require.LessOrEqual(t, notificationTTL, 24*time.Hour)
|
||||
|
||||
routeTTL, err := client.PTTL(context.Background(), emailRouteKey).Result()
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, routeTTL, 23*time.Hour)
|
||||
require.LessOrEqual(t, routeTTL, 24*time.Hour)
|
||||
|
||||
idempotencyTTL, err := client.PTTL(context.Background(), Keyspace{}.Idempotency(input.Idempotency.Producer, input.Idempotency.IdempotencyKey)).Result()
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, idempotencyTTL, 6*24*time.Hour)
|
||||
require.LessOrEqual(t, idempotencyTTL, 7*24*time.Hour)
|
||||
}
|
||||
|
||||
func TestMalformedIntentStoreRecordPersistsEntry(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewMalformedIntentStore(client, 72*time.Hour)
|
||||
require.NoError(t, err)
|
||||
|
||||
entry := malformedintent.Entry{
|
||||
StreamEntryID: "1775121700000-0",
|
||||
NotificationType: "game.turn.ready",
|
||||
Producer: "game_master",
|
||||
IdempotencyKey: "game-123:turn-54",
|
||||
FailureCode: malformedintent.FailureCodeInvalidPayload,
|
||||
FailureMessage: "payload_json.turn_number is required",
|
||||
RawFields: map[string]any{
|
||||
"notification_type": "game.turn.ready",
|
||||
},
|
||||
RecordedAt: time.UnixMilli(1775121700000).UTC(),
|
||||
}
|
||||
|
||||
require.NoError(t, store.Record(context.Background(), entry))
|
||||
|
||||
payload, err := client.Get(context.Background(), Keyspace{}.MalformedIntent(entry.StreamEntryID)).Bytes()
|
||||
require.NoError(t, err)
|
||||
recordedEntry, err := UnmarshalMalformedIntent(payload)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, entry.StreamEntryID, recordedEntry.StreamEntryID)
|
||||
require.Equal(t, entry.FailureCode, recordedEntry.FailureCode)
|
||||
|
||||
ttl, err := client.PTTL(context.Background(), Keyspace{}.MalformedIntent(entry.StreamEntryID)).Result()
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, ttl, 71*time.Hour)
|
||||
require.LessOrEqual(t, ttl, 72*time.Hour)
|
||||
}
|
||||
|
||||
func TestStreamOffsetStoreLoadAndSave(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewStreamOffsetStore(client)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, found, err := store.Load(context.Background(), "notification:intents")
|
||||
require.NoError(t, err)
|
||||
require.False(t, found)
|
||||
|
||||
require.NoError(t, store.Save(context.Background(), "notification:intents", "1775121700000-0"))
|
||||
|
||||
entryID, found, err := store.Load(context.Background(), "notification:intents")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.Equal(t, "1775121700000-0", entryID)
|
||||
}
|
||||
|
||||
func TestIntentStreamLagReaderReadsOldestUnprocessedEntry(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewStreamOffsetStore(client)
|
||||
require.NoError(t, err)
|
||||
reader, err := NewIntentStreamLagReader(store, "notification:intents")
|
||||
require.NoError(t, err)
|
||||
|
||||
firstID, err := client.XAdd(context.Background(), &redis.XAddArgs{
|
||||
Stream: "notification:intents",
|
||||
ID: "1775121700000-0",
|
||||
Values: map[string]any{"payload": "first"},
|
||||
}).Result()
|
||||
require.NoError(t, err)
|
||||
secondID, err := client.XAdd(context.Background(), &redis.XAddArgs{
|
||||
Stream: "notification:intents",
|
||||
ID: "1775121701000-0",
|
||||
Values: map[string]any{"payload": "second"},
|
||||
}).Result()
|
||||
require.NoError(t, err)
|
||||
|
||||
snapshot, err := reader.ReadIntentStreamLagSnapshot(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, snapshot.OldestUnprocessedAt)
|
||||
require.Equal(t, time.UnixMilli(1775121700000).UTC(), *snapshot.OldestUnprocessedAt)
|
||||
|
||||
require.NoError(t, store.Save(context.Background(), "notification:intents", firstID))
|
||||
snapshot, err = reader.ReadIntentStreamLagSnapshot(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, snapshot.OldestUnprocessedAt)
|
||||
require.Equal(t, time.UnixMilli(1775121701000).UTC(), *snapshot.OldestUnprocessedAt)
|
||||
|
||||
require.NoError(t, store.Save(context.Background(), "notification:intents", secondID))
|
||||
snapshot, err = reader.ReadIntentStreamLagSnapshot(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, snapshot.OldestUnprocessedAt)
|
||||
}
|
||||
|
||||
func TestAcceptanceStoreWorksWithAcceptIntentService(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewAcceptanceStore(client, AcceptanceConfig{
|
||||
RecordTTL: 24 * time.Hour,
|
||||
DeadLetterTTL: 72 * time.Hour,
|
||||
IdempotencyTTL: 7 * 24 * time.Hour,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
service, err := acceptintent.New(acceptintent.Config{
|
||||
Store: store,
|
||||
UserDirectory: staticUserDirectory{},
|
||||
Clock: fixedClock{now: time.UnixMilli(1775121700000).UTC()},
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
PushMaxAttempts: 3,
|
||||
EmailMaxAttempts: 7,
|
||||
IdempotencyTTL: 7 * 24 * time.Hour,
|
||||
AdminRouting: config.AdminRoutingConfig{
|
||||
LobbyApplicationSubmitted: []string{"owner@example.com"},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
result, err := service.Execute(context.Background(), acceptintent.AcceptInput{
|
||||
NotificationID: "1775121700000-0",
|
||||
Intent: intentstream.Intent{
|
||||
NotificationType: intentstream.NotificationTypeLobbyApplicationSubmitted,
|
||||
Producer: intentstream.ProducerGameLobby,
|
||||
AudienceKind: intentstream.AudienceKindAdminEmail,
|
||||
IdempotencyKey: "game-456:application-submitted:user-42",
|
||||
OccurredAt: time.UnixMilli(1775121700002).UTC(),
|
||||
PayloadJSON: `{"applicant_name":"Nova Pilot","applicant_user_id":"user-42","game_id":"game-456","game_name":"Orion Front"}`,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, acceptintent.OutcomeAccepted, result.Outcome)
|
||||
|
||||
record, found, err := store.GetNotification(context.Background(), "1775121700000-0")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.Equal(t, "1775121700000-0", record.NotificationID)
|
||||
}
|
||||
|
||||
type fixedClock struct {
|
||||
now time.Time
|
||||
}
|
||||
|
||||
func (clock fixedClock) Now() time.Time {
|
||||
return clock.now
|
||||
}
|
||||
|
||||
func validAdminAcceptanceInput(now time.Time) acceptintent.CreateAcceptanceInput {
|
||||
return acceptintent.CreateAcceptanceInput{
|
||||
Notification: acceptintent.NotificationRecord{
|
||||
NotificationID: "1775121700000-0",
|
||||
NotificationType: intentstream.NotificationTypeLobbyApplicationSubmitted,
|
||||
Producer: intentstream.ProducerGameLobby,
|
||||
AudienceKind: intentstream.AudienceKindAdminEmail,
|
||||
PayloadJSON: `{"applicant_name":"Nova Pilot","applicant_user_id":"user-42","game_id":"game-456","game_name":"Orion Front"}`,
|
||||
IdempotencyKey: "game-456:application-submitted:user-42",
|
||||
RequestFingerprint: "sha256:deadbeef",
|
||||
OccurredAt: now,
|
||||
AcceptedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
Routes: []acceptintent.NotificationRoute{
|
||||
{
|
||||
NotificationID: "1775121700000-0",
|
||||
RouteID: "push:email:owner@example.com",
|
||||
Channel: intentstream.ChannelPush,
|
||||
RecipientRef: "email:owner@example.com",
|
||||
Status: acceptintent.RouteStatusSkipped,
|
||||
AttemptCount: 0,
|
||||
MaxAttempts: 3,
|
||||
ResolvedEmail: "owner@example.com",
|
||||
ResolvedLocale: "en",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
SkippedAt: now,
|
||||
},
|
||||
{
|
||||
NotificationID: "1775121700000-0",
|
||||
RouteID: "email:email:owner@example.com",
|
||||
Channel: intentstream.ChannelEmail,
|
||||
RecipientRef: "email:owner@example.com",
|
||||
Status: acceptintent.RouteStatusPending,
|
||||
AttemptCount: 0,
|
||||
MaxAttempts: 7,
|
||||
NextAttemptAt: now,
|
||||
ResolvedEmail: "owner@example.com",
|
||||
ResolvedLocale: "en",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
},
|
||||
Idempotency: acceptintent.IdempotencyRecord{
|
||||
Producer: intentstream.ProducerGameLobby,
|
||||
IdempotencyKey: "game-456:application-submitted:user-42",
|
||||
NotificationID: "1775121700000-0",
|
||||
RequestFingerprint: "sha256:deadbeef",
|
||||
CreatedAt: now,
|
||||
ExpiresAt: now.Add(7 * 24 * time.Hour),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newTestRedisClient(t *testing.T, server *miniredis.Miniredis) *redis.Client {
|
||||
t.Helper()
|
||||
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: server.Addr(),
|
||||
Protocol: 2,
|
||||
DisableIdentity: true,
|
||||
})
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, client.Close())
|
||||
})
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
type staticUserDirectory struct{}
|
||||
|
||||
func (staticUserDirectory) GetUserByID(context.Context, string) (acceptintent.UserRecord, error) {
|
||||
return acceptintent.UserRecord{}, acceptintent.ErrRecipientNotFound
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
package redisstate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"galaxy/notification/internal/service/acceptintent"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// AcceptanceConfig stores the retention settings applied to accepted durable
|
||||
// notification state.
|
||||
type AcceptanceConfig struct {
|
||||
// RecordTTL stores the retention period applied to notification and route
|
||||
// records.
|
||||
RecordTTL time.Duration
|
||||
|
||||
// DeadLetterTTL stores the retention period applied to route dead-letter
|
||||
// entries.
|
||||
DeadLetterTTL time.Duration
|
||||
|
||||
// IdempotencyTTL stores the retention period applied to idempotency
|
||||
// reservations.
|
||||
IdempotencyTTL time.Duration
|
||||
}
|
||||
|
||||
// Validate reports whether cfg contains usable retention settings.
|
||||
func (cfg AcceptanceConfig) Validate() error {
|
||||
switch {
|
||||
case cfg.RecordTTL <= 0:
|
||||
return fmt.Errorf("record ttl must be positive")
|
||||
case cfg.DeadLetterTTL <= 0:
|
||||
return fmt.Errorf("dead-letter ttl must be positive")
|
||||
case cfg.IdempotencyTTL <= 0:
|
||||
return fmt.Errorf("idempotency ttl must be positive")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// AtomicWriter performs the minimal multi-key Redis mutations required by
|
||||
// notification intent acceptance.
|
||||
type AtomicWriter struct {
|
||||
client *redis.Client
|
||||
keys Keyspace
|
||||
cfg AcceptanceConfig
|
||||
}
|
||||
|
||||
// NewAtomicWriter constructs a low-level Redis mutation helper.
|
||||
func NewAtomicWriter(client *redis.Client, cfg AcceptanceConfig) (*AtomicWriter, error) {
|
||||
if client == nil {
|
||||
return nil, errors.New("new notification redis atomic writer: nil client")
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("new notification redis atomic writer: %w", err)
|
||||
}
|
||||
|
||||
return &AtomicWriter{
|
||||
client: client,
|
||||
keys: Keyspace{},
|
||||
cfg: cfg,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CreateAcceptance stores one notification record, all derived routes, and
|
||||
// the matching idempotency reservation in one optimistic Redis transaction.
|
||||
func (writer *AtomicWriter) CreateAcceptance(ctx context.Context, input acceptintent.CreateAcceptanceInput) error {
|
||||
if writer == nil || writer.client == nil {
|
||||
return errors.New("create notification acceptance in redis: nil writer")
|
||||
}
|
||||
if ctx == nil {
|
||||
return errors.New("create notification acceptance in redis: nil context")
|
||||
}
|
||||
if err := input.Validate(); err != nil {
|
||||
return fmt.Errorf("create notification acceptance in redis: %w", err)
|
||||
}
|
||||
|
||||
notificationPayload, err := MarshalNotification(input.Notification)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create notification acceptance in redis: %w", err)
|
||||
}
|
||||
idempotencyPayload, err := MarshalIdempotency(input.Idempotency)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create notification acceptance in redis: %w", err)
|
||||
}
|
||||
|
||||
routePayloads := make([][]byte, len(input.Routes))
|
||||
routeKeys := make([]string, len(input.Routes))
|
||||
scheduledRouteKeys := make([]string, 0, len(input.Routes))
|
||||
scheduledRouteScores := make([]float64, 0, len(input.Routes))
|
||||
for index, route := range input.Routes {
|
||||
payload, err := MarshalRoute(route)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create notification acceptance in redis: route %d: %w", index, err)
|
||||
}
|
||||
routePayloads[index] = payload
|
||||
routeKeys[index] = writer.keys.Route(route.NotificationID, route.RouteID)
|
||||
if route.Status == acceptintent.RouteStatusPending {
|
||||
scheduledRouteKeys = append(scheduledRouteKeys, routeKeys[index])
|
||||
scheduledRouteScores = append(scheduledRouteScores, float64(route.NextAttemptAt.UTC().UnixMilli()))
|
||||
}
|
||||
}
|
||||
|
||||
notificationKey := writer.keys.Notification(input.Notification.NotificationID)
|
||||
idempotencyKey := writer.keys.Idempotency(input.Idempotency.Producer, input.Idempotency.IdempotencyKey)
|
||||
watchKeys := append([]string{notificationKey, idempotencyKey}, routeKeys...)
|
||||
|
||||
watchErr := writer.client.Watch(ctx, func(tx *redis.Tx) error {
|
||||
for _, key := range watchKeys {
|
||||
if err := ensureKeyAbsent(ctx, tx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
_, err := tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {
|
||||
pipe.Set(ctx, notificationKey, notificationPayload, writer.cfg.RecordTTL)
|
||||
pipe.Set(ctx, idempotencyKey, idempotencyPayload, writer.cfg.IdempotencyTTL)
|
||||
for index, routeKey := range routeKeys {
|
||||
pipe.Set(ctx, routeKey, routePayloads[index], writer.cfg.RecordTTL)
|
||||
}
|
||||
for index, routeKey := range scheduledRouteKeys {
|
||||
pipe.ZAdd(ctx, writer.keys.RouteSchedule(), redis.Z{
|
||||
Score: scheduledRouteScores[index],
|
||||
Member: routeKey,
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return err
|
||||
}, watchKeys...)
|
||||
|
||||
switch {
|
||||
case errors.Is(watchErr, ErrConflict), errors.Is(watchErr, redis.TxFailedErr):
|
||||
return ErrConflict
|
||||
case watchErr != nil:
|
||||
return fmt.Errorf("create notification acceptance in redis: %w", watchErr)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func ensureKeyAbsent(ctx context.Context, tx *redis.Tx, key string) error {
|
||||
exists, err := tx.Exists(ctx, key).Result()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists > 0 {
|
||||
return ErrConflict
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -6,10 +6,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"galaxy/notification/internal/api/intentstream"
|
||||
"galaxy/notification/internal/service/acceptintent"
|
||||
"galaxy/notification/internal/service/malformedintent"
|
||||
)
|
||||
|
||||
// StreamOffset stores the persisted progress of the plain-XREAD intent
|
||||
@@ -18,412 +14,14 @@ type StreamOffset struct {
|
||||
// Stream stores the Redis Stream name.
|
||||
Stream string
|
||||
|
||||
// LastProcessedEntryID stores the last durably processed Redis Stream entry
|
||||
// identifier.
|
||||
// LastProcessedEntryID stores the last durably processed Redis Stream
|
||||
// entry identifier.
|
||||
LastProcessedEntryID string
|
||||
|
||||
// UpdatedAt stores when the offset record was last updated.
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// DeadLetterEntry stores one terminal route-publication failure recorded for
|
||||
// later operator inspection.
|
||||
type DeadLetterEntry struct {
|
||||
// NotificationID stores the owning notification identifier.
|
||||
NotificationID string
|
||||
|
||||
// RouteID stores the exhausted route identifier.
|
||||
RouteID string
|
||||
|
||||
// Channel stores the failed route channel.
|
||||
Channel intentstream.Channel
|
||||
|
||||
// RecipientRef stores the stable failed recipient slot identifier.
|
||||
RecipientRef string
|
||||
|
||||
// FinalAttemptCount stores how many publication attempts were consumed.
|
||||
FinalAttemptCount int
|
||||
|
||||
// MaxAttempts stores the configured retry budget for Channel.
|
||||
MaxAttempts int
|
||||
|
||||
// FailureClassification stores the stable classified failure reason.
|
||||
FailureClassification string
|
||||
|
||||
// FailureMessage stores the last failure detail.
|
||||
FailureMessage string
|
||||
|
||||
// CreatedAt stores when the route moved to dead_letter.
|
||||
CreatedAt time.Time
|
||||
|
||||
// RecoveryHint stores the optional operator-facing recovery hint.
|
||||
RecoveryHint string
|
||||
}
|
||||
|
||||
type notificationRecordJSON struct {
|
||||
NotificationID string `json:"notification_id"`
|
||||
NotificationType intentstream.NotificationType `json:"notification_type"`
|
||||
Producer intentstream.Producer `json:"producer"`
|
||||
AudienceKind intentstream.AudienceKind `json:"audience_kind"`
|
||||
RecipientUserIDs []string `json:"recipient_user_ids,omitempty"`
|
||||
PayloadJSON string `json:"payload_json"`
|
||||
IdempotencyKey string `json:"idempotency_key"`
|
||||
RequestFingerprint string `json:"request_fingerprint"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
TraceID string `json:"trace_id,omitempty"`
|
||||
OccurredAtMS int64 `json:"occurred_at_ms"`
|
||||
AcceptedAtMS int64 `json:"accepted_at_ms"`
|
||||
UpdatedAtMS int64 `json:"updated_at_ms"`
|
||||
}
|
||||
|
||||
type notificationRouteJSON struct {
|
||||
NotificationID string `json:"notification_id"`
|
||||
RouteID string `json:"route_id"`
|
||||
Channel intentstream.Channel `json:"channel"`
|
||||
RecipientRef string `json:"recipient_ref"`
|
||||
Status acceptintent.RouteStatus `json:"status"`
|
||||
AttemptCount int `json:"attempt_count"`
|
||||
MaxAttempts int `json:"max_attempts"`
|
||||
NextAttemptAtMS *int64 `json:"next_attempt_at_ms,omitempty"`
|
||||
ResolvedEmail string `json:"resolved_email,omitempty"`
|
||||
ResolvedLocale string `json:"resolved_locale,omitempty"`
|
||||
LastErrorClassification string `json:"last_error_classification,omitempty"`
|
||||
LastErrorMessage string `json:"last_error_message,omitempty"`
|
||||
LastErrorAtMS *int64 `json:"last_error_at_ms,omitempty"`
|
||||
CreatedAtMS int64 `json:"created_at_ms"`
|
||||
UpdatedAtMS int64 `json:"updated_at_ms"`
|
||||
PublishedAtMS *int64 `json:"published_at_ms,omitempty"`
|
||||
DeadLetteredAtMS *int64 `json:"dead_lettered_at_ms,omitempty"`
|
||||
SkippedAtMS *int64 `json:"skipped_at_ms,omitempty"`
|
||||
}
|
||||
|
||||
type idempotencyRecordJSON struct {
|
||||
Producer intentstream.Producer `json:"producer"`
|
||||
IdempotencyKey string `json:"idempotency_key"`
|
||||
NotificationID string `json:"notification_id"`
|
||||
RequestFingerprint string `json:"request_fingerprint"`
|
||||
CreatedAtMS int64 `json:"created_at_ms"`
|
||||
ExpiresAtMS int64 `json:"expires_at_ms"`
|
||||
}
|
||||
|
||||
type malformedIntentJSON struct {
|
||||
StreamEntryID string `json:"stream_entry_id"`
|
||||
NotificationType string `json:"notification_type,omitempty"`
|
||||
Producer string `json:"producer,omitempty"`
|
||||
IdempotencyKey string `json:"idempotency_key,omitempty"`
|
||||
FailureCode malformedintent.FailureCode `json:"failure_code"`
|
||||
FailureMessage string `json:"failure_message"`
|
||||
RawFields map[string]any `json:"raw_fields_json"`
|
||||
RecordedAtMS int64 `json:"recorded_at_ms"`
|
||||
}
|
||||
|
||||
type streamOffsetJSON struct {
|
||||
Stream string `json:"stream"`
|
||||
LastProcessedEntryID string `json:"last_processed_entry_id"`
|
||||
UpdatedAtMS int64 `json:"updated_at_ms"`
|
||||
}
|
||||
|
||||
type deadLetterEntryJSON struct {
|
||||
NotificationID string `json:"notification_id"`
|
||||
RouteID string `json:"route_id"`
|
||||
Channel intentstream.Channel `json:"channel"`
|
||||
RecipientRef string `json:"recipient_ref"`
|
||||
FinalAttemptCount int `json:"final_attempt_count"`
|
||||
MaxAttempts int `json:"max_attempts"`
|
||||
FailureClassification string `json:"failure_classification"`
|
||||
FailureMessage string `json:"failure_message"`
|
||||
CreatedAtMS int64 `json:"created_at_ms"`
|
||||
RecoveryHint string `json:"recovery_hint,omitempty"`
|
||||
}
|
||||
|
||||
// MarshalNotification marshals one notification record into the strict JSON
|
||||
// representation owned by Notification Service.
|
||||
func MarshalNotification(record acceptintent.NotificationRecord) ([]byte, error) {
|
||||
if err := record.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("marshal notification record: %w", err)
|
||||
}
|
||||
|
||||
return marshalStrictJSON(notificationRecordJSON{
|
||||
NotificationID: record.NotificationID,
|
||||
NotificationType: record.NotificationType,
|
||||
Producer: record.Producer,
|
||||
AudienceKind: record.AudienceKind,
|
||||
RecipientUserIDs: append([]string(nil), record.RecipientUserIDs...),
|
||||
PayloadJSON: record.PayloadJSON,
|
||||
IdempotencyKey: record.IdempotencyKey,
|
||||
RequestFingerprint: record.RequestFingerprint,
|
||||
RequestID: record.RequestID,
|
||||
TraceID: record.TraceID,
|
||||
OccurredAtMS: unixMilli(record.OccurredAt),
|
||||
AcceptedAtMS: unixMilli(record.AcceptedAt),
|
||||
UpdatedAtMS: unixMilli(record.UpdatedAt),
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalNotification unmarshals one strict JSON notification record.
|
||||
func UnmarshalNotification(payload []byte) (acceptintent.NotificationRecord, error) {
|
||||
var wire notificationRecordJSON
|
||||
if err := unmarshalStrictJSON(payload, &wire); err != nil {
|
||||
return acceptintent.NotificationRecord{}, fmt.Errorf("unmarshal notification record: %w", err)
|
||||
}
|
||||
|
||||
record := acceptintent.NotificationRecord{
|
||||
NotificationID: wire.NotificationID,
|
||||
NotificationType: wire.NotificationType,
|
||||
Producer: wire.Producer,
|
||||
AudienceKind: wire.AudienceKind,
|
||||
RecipientUserIDs: append([]string(nil), wire.RecipientUserIDs...),
|
||||
PayloadJSON: wire.PayloadJSON,
|
||||
IdempotencyKey: wire.IdempotencyKey,
|
||||
RequestFingerprint: wire.RequestFingerprint,
|
||||
RequestID: wire.RequestID,
|
||||
TraceID: wire.TraceID,
|
||||
OccurredAt: time.UnixMilli(wire.OccurredAtMS).UTC(),
|
||||
AcceptedAt: time.UnixMilli(wire.AcceptedAtMS).UTC(),
|
||||
UpdatedAt: time.UnixMilli(wire.UpdatedAtMS).UTC(),
|
||||
}
|
||||
if err := record.Validate(); err != nil {
|
||||
return acceptintent.NotificationRecord{}, fmt.Errorf("unmarshal notification record: %w", err)
|
||||
}
|
||||
|
||||
return record, nil
|
||||
}
|
||||
|
||||
// MarshalRoute marshals one notification route into the strict JSON
|
||||
// representation owned by Notification Service.
|
||||
func MarshalRoute(route acceptintent.NotificationRoute) ([]byte, error) {
|
||||
if err := route.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("marshal notification route: %w", err)
|
||||
}
|
||||
|
||||
return marshalStrictJSON(notificationRouteJSON{
|
||||
NotificationID: route.NotificationID,
|
||||
RouteID: route.RouteID,
|
||||
Channel: route.Channel,
|
||||
RecipientRef: route.RecipientRef,
|
||||
Status: route.Status,
|
||||
AttemptCount: route.AttemptCount,
|
||||
MaxAttempts: route.MaxAttempts,
|
||||
NextAttemptAtMS: optionalUnixMilli(route.NextAttemptAt),
|
||||
ResolvedEmail: route.ResolvedEmail,
|
||||
ResolvedLocale: route.ResolvedLocale,
|
||||
LastErrorClassification: route.LastErrorClassification,
|
||||
LastErrorMessage: route.LastErrorMessage,
|
||||
LastErrorAtMS: optionalUnixMilli(route.LastErrorAt),
|
||||
CreatedAtMS: unixMilli(route.CreatedAt),
|
||||
UpdatedAtMS: unixMilli(route.UpdatedAt),
|
||||
PublishedAtMS: optionalUnixMilli(route.PublishedAt),
|
||||
DeadLetteredAtMS: optionalUnixMilli(route.DeadLetteredAt),
|
||||
SkippedAtMS: optionalUnixMilli(route.SkippedAt),
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalRoute unmarshals one strict JSON notification route.
|
||||
func UnmarshalRoute(payload []byte) (acceptintent.NotificationRoute, error) {
|
||||
var wire notificationRouteJSON
|
||||
if err := unmarshalStrictJSON(payload, &wire); err != nil {
|
||||
return acceptintent.NotificationRoute{}, fmt.Errorf("unmarshal notification route: %w", err)
|
||||
}
|
||||
|
||||
route := acceptintent.NotificationRoute{
|
||||
NotificationID: wire.NotificationID,
|
||||
RouteID: wire.RouteID,
|
||||
Channel: wire.Channel,
|
||||
RecipientRef: wire.RecipientRef,
|
||||
Status: wire.Status,
|
||||
AttemptCount: wire.AttemptCount,
|
||||
MaxAttempts: wire.MaxAttempts,
|
||||
ResolvedEmail: wire.ResolvedEmail,
|
||||
ResolvedLocale: wire.ResolvedLocale,
|
||||
LastErrorClassification: wire.LastErrorClassification,
|
||||
LastErrorMessage: wire.LastErrorMessage,
|
||||
CreatedAt: time.UnixMilli(wire.CreatedAtMS).UTC(),
|
||||
UpdatedAt: time.UnixMilli(wire.UpdatedAtMS).UTC(),
|
||||
}
|
||||
if wire.NextAttemptAtMS != nil {
|
||||
route.NextAttemptAt = time.UnixMilli(*wire.NextAttemptAtMS).UTC()
|
||||
}
|
||||
if wire.LastErrorAtMS != nil {
|
||||
route.LastErrorAt = time.UnixMilli(*wire.LastErrorAtMS).UTC()
|
||||
}
|
||||
if wire.PublishedAtMS != nil {
|
||||
route.PublishedAt = time.UnixMilli(*wire.PublishedAtMS).UTC()
|
||||
}
|
||||
if wire.DeadLetteredAtMS != nil {
|
||||
route.DeadLetteredAt = time.UnixMilli(*wire.DeadLetteredAtMS).UTC()
|
||||
}
|
||||
if wire.SkippedAtMS != nil {
|
||||
route.SkippedAt = time.UnixMilli(*wire.SkippedAtMS).UTC()
|
||||
}
|
||||
if err := route.Validate(); err != nil {
|
||||
return acceptintent.NotificationRoute{}, fmt.Errorf("unmarshal notification route: %w", err)
|
||||
}
|
||||
|
||||
return route, nil
|
||||
}
|
||||
|
||||
// MarshalIdempotency marshals one idempotency record into the strict JSON
|
||||
// representation owned by Notification Service.
|
||||
func MarshalIdempotency(record acceptintent.IdempotencyRecord) ([]byte, error) {
|
||||
if err := record.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("marshal notification idempotency record: %w", err)
|
||||
}
|
||||
|
||||
return marshalStrictJSON(idempotencyRecordJSON{
|
||||
Producer: record.Producer,
|
||||
IdempotencyKey: record.IdempotencyKey,
|
||||
NotificationID: record.NotificationID,
|
||||
RequestFingerprint: record.RequestFingerprint,
|
||||
CreatedAtMS: unixMilli(record.CreatedAt),
|
||||
ExpiresAtMS: unixMilli(record.ExpiresAt),
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalIdempotency unmarshals one strict JSON idempotency record.
|
||||
func UnmarshalIdempotency(payload []byte) (acceptintent.IdempotencyRecord, error) {
|
||||
var wire idempotencyRecordJSON
|
||||
if err := unmarshalStrictJSON(payload, &wire); err != nil {
|
||||
return acceptintent.IdempotencyRecord{}, fmt.Errorf("unmarshal notification idempotency record: %w", err)
|
||||
}
|
||||
|
||||
record := acceptintent.IdempotencyRecord{
|
||||
Producer: wire.Producer,
|
||||
IdempotencyKey: wire.IdempotencyKey,
|
||||
NotificationID: wire.NotificationID,
|
||||
RequestFingerprint: wire.RequestFingerprint,
|
||||
CreatedAt: time.UnixMilli(wire.CreatedAtMS).UTC(),
|
||||
ExpiresAt: time.UnixMilli(wire.ExpiresAtMS).UTC(),
|
||||
}
|
||||
if err := record.Validate(); err != nil {
|
||||
return acceptintent.IdempotencyRecord{}, fmt.Errorf("unmarshal notification idempotency record: %w", err)
|
||||
}
|
||||
|
||||
return record, nil
|
||||
}
|
||||
|
||||
// MarshalDeadLetter marshals one dead-letter entry into the strict JSON
|
||||
// representation owned by Notification Service.
|
||||
func MarshalDeadLetter(entry DeadLetterEntry) ([]byte, error) {
|
||||
if err := entry.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("marshal dead letter entry: %w", err)
|
||||
}
|
||||
|
||||
return marshalStrictJSON(deadLetterEntryJSON{
|
||||
NotificationID: entry.NotificationID,
|
||||
RouteID: entry.RouteID,
|
||||
Channel: entry.Channel,
|
||||
RecipientRef: entry.RecipientRef,
|
||||
FinalAttemptCount: entry.FinalAttemptCount,
|
||||
MaxAttempts: entry.MaxAttempts,
|
||||
FailureClassification: entry.FailureClassification,
|
||||
FailureMessage: entry.FailureMessage,
|
||||
CreatedAtMS: unixMilli(entry.CreatedAt),
|
||||
RecoveryHint: entry.RecoveryHint,
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalDeadLetter unmarshals one strict JSON dead-letter entry.
|
||||
func UnmarshalDeadLetter(payload []byte) (DeadLetterEntry, error) {
|
||||
var wire deadLetterEntryJSON
|
||||
if err := unmarshalStrictJSON(payload, &wire); err != nil {
|
||||
return DeadLetterEntry{}, fmt.Errorf("unmarshal dead letter entry: %w", err)
|
||||
}
|
||||
|
||||
entry := DeadLetterEntry{
|
||||
NotificationID: wire.NotificationID,
|
||||
RouteID: wire.RouteID,
|
||||
Channel: wire.Channel,
|
||||
RecipientRef: wire.RecipientRef,
|
||||
FinalAttemptCount: wire.FinalAttemptCount,
|
||||
MaxAttempts: wire.MaxAttempts,
|
||||
FailureClassification: wire.FailureClassification,
|
||||
FailureMessage: wire.FailureMessage,
|
||||
CreatedAt: time.UnixMilli(wire.CreatedAtMS).UTC(),
|
||||
RecoveryHint: wire.RecoveryHint,
|
||||
}
|
||||
if err := entry.Validate(); err != nil {
|
||||
return DeadLetterEntry{}, fmt.Errorf("unmarshal dead letter entry: %w", err)
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
// MarshalMalformedIntent marshals one malformed-intent entry into the strict
|
||||
// JSON representation owned by Notification Service.
|
||||
func MarshalMalformedIntent(entry malformedintent.Entry) ([]byte, error) {
|
||||
if err := entry.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("marshal malformed intent: %w", err)
|
||||
}
|
||||
|
||||
return marshalStrictJSON(malformedIntentJSON{
|
||||
StreamEntryID: entry.StreamEntryID,
|
||||
NotificationType: entry.NotificationType,
|
||||
Producer: entry.Producer,
|
||||
IdempotencyKey: entry.IdempotencyKey,
|
||||
FailureCode: entry.FailureCode,
|
||||
FailureMessage: entry.FailureMessage,
|
||||
RawFields: cloneJSONObject(entry.RawFields),
|
||||
RecordedAtMS: unixMilli(entry.RecordedAt),
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalMalformedIntent unmarshals one strict JSON malformed-intent entry.
|
||||
func UnmarshalMalformedIntent(payload []byte) (malformedintent.Entry, error) {
|
||||
var wire malformedIntentJSON
|
||||
if err := unmarshalStrictJSON(payload, &wire); err != nil {
|
||||
return malformedintent.Entry{}, fmt.Errorf("unmarshal malformed intent: %w", err)
|
||||
}
|
||||
|
||||
entry := malformedintent.Entry{
|
||||
StreamEntryID: wire.StreamEntryID,
|
||||
NotificationType: wire.NotificationType,
|
||||
Producer: wire.Producer,
|
||||
IdempotencyKey: wire.IdempotencyKey,
|
||||
FailureCode: wire.FailureCode,
|
||||
FailureMessage: wire.FailureMessage,
|
||||
RawFields: cloneJSONObject(wire.RawFields),
|
||||
RecordedAt: time.UnixMilli(wire.RecordedAtMS).UTC(),
|
||||
}
|
||||
if err := entry.Validate(); err != nil {
|
||||
return malformedintent.Entry{}, fmt.Errorf("unmarshal malformed intent: %w", err)
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
// MarshalStreamOffset marshals one stream-offset record into the strict JSON
|
||||
// representation owned by Notification Service.
|
||||
func MarshalStreamOffset(offset StreamOffset) ([]byte, error) {
|
||||
if err := offset.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("marshal stream offset: %w", err)
|
||||
}
|
||||
|
||||
return marshalStrictJSON(streamOffsetJSON{
|
||||
Stream: offset.Stream,
|
||||
LastProcessedEntryID: offset.LastProcessedEntryID,
|
||||
UpdatedAtMS: unixMilli(offset.UpdatedAt),
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalStreamOffset unmarshals one strict JSON stream-offset record.
|
||||
func UnmarshalStreamOffset(payload []byte) (StreamOffset, error) {
|
||||
var wire streamOffsetJSON
|
||||
if err := unmarshalStrictJSON(payload, &wire); err != nil {
|
||||
return StreamOffset{}, fmt.Errorf("unmarshal stream offset: %w", err)
|
||||
}
|
||||
|
||||
offset := StreamOffset{
|
||||
Stream: wire.Stream,
|
||||
LastProcessedEntryID: wire.LastProcessedEntryID,
|
||||
UpdatedAt: time.UnixMilli(wire.UpdatedAtMS).UTC(),
|
||||
}
|
||||
if err := offset.Validate(); err != nil {
|
||||
return StreamOffset{}, fmt.Errorf("unmarshal stream offset: %w", err)
|
||||
}
|
||||
|
||||
return offset, nil
|
||||
}
|
||||
|
||||
// Validate reports whether offset contains a complete persisted consumer
|
||||
// progress record.
|
||||
func (offset StreamOffset) Validate() error {
|
||||
@@ -446,43 +44,43 @@ func (offset StreamOffset) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate reports whether entry contains a complete dead-letter record.
|
||||
func (entry DeadLetterEntry) Validate() error {
|
||||
if entry.NotificationID == "" {
|
||||
return fmt.Errorf("dead letter entry notification id must not be empty")
|
||||
}
|
||||
if entry.RouteID == "" {
|
||||
return fmt.Errorf("dead letter entry route id must not be empty")
|
||||
}
|
||||
if !entry.Channel.IsKnown() {
|
||||
return fmt.Errorf("dead letter entry channel %q is unsupported", entry.Channel)
|
||||
}
|
||||
if entry.RecipientRef == "" {
|
||||
return fmt.Errorf("dead letter entry recipient ref must not be empty")
|
||||
}
|
||||
if entry.FinalAttemptCount <= 0 {
|
||||
return fmt.Errorf("dead letter entry final attempt count must be positive")
|
||||
}
|
||||
if entry.MaxAttempts <= 0 {
|
||||
return fmt.Errorf("dead letter entry max attempts must be positive")
|
||||
}
|
||||
if entry.FailureClassification == "" {
|
||||
return fmt.Errorf("dead letter entry failure classification must not be empty")
|
||||
}
|
||||
if entry.FailureMessage == "" {
|
||||
return fmt.Errorf("dead letter entry failure message must not be empty")
|
||||
}
|
||||
if entry.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("dead letter entry created at must not be zero")
|
||||
}
|
||||
if !entry.CreatedAt.Equal(entry.CreatedAt.UTC()) {
|
||||
return fmt.Errorf("dead letter entry created at must be UTC")
|
||||
}
|
||||
if !entry.CreatedAt.Equal(entry.CreatedAt.Truncate(time.Millisecond)) {
|
||||
return fmt.Errorf("dead letter entry created at must use millisecond precision")
|
||||
type streamOffsetJSON struct {
|
||||
Stream string `json:"stream"`
|
||||
LastProcessedEntryID string `json:"last_processed_entry_id"`
|
||||
UpdatedAtMS int64 `json:"updated_at_ms"`
|
||||
}
|
||||
|
||||
// MarshalStreamOffset marshals one stream-offset record into the strict JSON
|
||||
// representation owned by Notification Service.
|
||||
func MarshalStreamOffset(offset StreamOffset) ([]byte, error) {
|
||||
if err := offset.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("marshal stream offset: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return marshalStrictJSON(streamOffsetJSON{
|
||||
Stream: offset.Stream,
|
||||
LastProcessedEntryID: offset.LastProcessedEntryID,
|
||||
UpdatedAtMS: offset.UpdatedAt.UTC().UnixMilli(),
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalStreamOffset unmarshals one strict JSON stream-offset record.
|
||||
func UnmarshalStreamOffset(payload []byte) (StreamOffset, error) {
|
||||
var wire streamOffsetJSON
|
||||
if err := unmarshalStrictJSON(payload, &wire); err != nil {
|
||||
return StreamOffset{}, fmt.Errorf("unmarshal stream offset: %w", err)
|
||||
}
|
||||
|
||||
offset := StreamOffset{
|
||||
Stream: wire.Stream,
|
||||
LastProcessedEntryID: wire.LastProcessedEntryID,
|
||||
UpdatedAt: time.UnixMilli(wire.UpdatedAtMS).UTC(),
|
||||
}
|
||||
if err := offset.Validate(); err != nil {
|
||||
return StreamOffset{}, fmt.Errorf("unmarshal stream offset: %w", err)
|
||||
}
|
||||
|
||||
return offset, nil
|
||||
}
|
||||
|
||||
func marshalStrictJSON(value any) ([]byte, error) {
|
||||
@@ -505,43 +103,3 @@ func unmarshalStrictJSON(payload []byte, target any) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func unixMilli(value time.Time) int64 {
|
||||
return value.UTC().UnixMilli()
|
||||
}
|
||||
|
||||
func optionalUnixMilli(value time.Time) *int64 {
|
||||
if value.IsZero() {
|
||||
return nil
|
||||
}
|
||||
millis := unixMilli(value)
|
||||
return &millis
|
||||
}
|
||||
|
||||
func cloneJSONObject(value map[string]any) map[string]any {
|
||||
if value == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
|
||||
cloned := make(map[string]any, len(value))
|
||||
for key, raw := range value {
|
||||
cloned[key] = cloneJSONValue(raw)
|
||||
}
|
||||
|
||||
return cloned
|
||||
}
|
||||
|
||||
func cloneJSONValue(value any) any {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
return cloneJSONObject(typed)
|
||||
case []any:
|
||||
cloned := make([]any, len(typed))
|
||||
for index, item := range typed {
|
||||
cloned[index] = cloneJSONValue(item)
|
||||
}
|
||||
return cloned
|
||||
default:
|
||||
return typed
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package redisstate
|
||||
|
||||
import "errors"
|
||||
import "galaxy/notification/internal/service/routestate"
|
||||
|
||||
var (
|
||||
// ErrConflict reports that a Redis mutation could not be applied because
|
||||
// one of the watched or newly created keys already existed or changed
|
||||
// concurrently.
|
||||
ErrConflict = errors.New("redis state conflict")
|
||||
)
|
||||
// ErrConflict reports that a Redis mutation could not be applied because
|
||||
// one of the watched or newly created keys already existed or changed
|
||||
// concurrently. Aliased to routestate.ErrConflict so the publisher
|
||||
// boundary uses one stable sentinel regardless of which storage backend
|
||||
// drives the mutation.
|
||||
var ErrConflict = routestate.ErrConflict
|
||||
|
||||
@@ -2,79 +2,25 @@ package redisstate
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"galaxy/notification/internal/api/intentstream"
|
||||
)
|
||||
|
||||
const defaultPrefix = "notification:"
|
||||
|
||||
// Keyspace builds the frozen Notification Service Redis keys. All dynamic key
|
||||
// segments are encoded with base64url so raw key structure does not depend on
|
||||
// caller-provided characters.
|
||||
// Keyspace builds the Notification Service Redis keys retained after the
|
||||
// Stage 5 PostgreSQL migration: only the route lease, the persisted stream
|
||||
// offset, and the inbound intent stream key are managed here. Durable
|
||||
// notification state lives in the `notification` PostgreSQL schema.
|
||||
//
|
||||
// Dynamic key segments are encoded with base64url so raw key structure
|
||||
// does not depend on caller-provided characters.
|
||||
type Keyspace struct{}
|
||||
|
||||
// Notification returns the primary Redis key for one notification_record.
|
||||
func (Keyspace) Notification(notificationID string) string {
|
||||
return defaultPrefix + "records:" + encodeKeyComponent(notificationID)
|
||||
}
|
||||
|
||||
// Route returns the primary Redis key for one notification_route.
|
||||
func (Keyspace) Route(notificationID string, routeID string) string {
|
||||
return defaultPrefix + "routes:" + encodeKeyComponent(notificationID) + ":" + encodeKeyComponent(routeID)
|
||||
}
|
||||
|
||||
// ParseRoute returns the notification identifier and route identifier encoded
|
||||
// inside routeKey.
|
||||
func (Keyspace) ParseRoute(routeKey string) (string, string, error) {
|
||||
trimmedPrefix := defaultPrefix + "routes:"
|
||||
if !strings.HasPrefix(routeKey, trimmedPrefix) {
|
||||
return "", "", fmt.Errorf("parse route key: %q does not use %q prefix", routeKey, trimmedPrefix)
|
||||
}
|
||||
|
||||
encoded := strings.TrimPrefix(routeKey, trimmedPrefix)
|
||||
parts := strings.Split(encoded, ":")
|
||||
if len(parts) != 2 {
|
||||
return "", "", fmt.Errorf("parse route key: %q must contain exactly two encoded segments", routeKey)
|
||||
}
|
||||
|
||||
notificationID, err := decodeKeyComponent(parts[0])
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("parse route key: notification id: %w", err)
|
||||
}
|
||||
routeID, err := decodeKeyComponent(parts[1])
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("parse route key: route id: %w", err)
|
||||
}
|
||||
|
||||
return notificationID, routeID, nil
|
||||
}
|
||||
|
||||
// Idempotency returns the primary Redis key for one
|
||||
// notification_idempotency_record.
|
||||
func (Keyspace) Idempotency(producer intentstream.Producer, idempotencyKey string) string {
|
||||
return defaultPrefix + "idempotency:" + encodeKeyComponent(string(producer)) + ":" + encodeKeyComponent(idempotencyKey)
|
||||
}
|
||||
|
||||
// DeadLetter returns the primary Redis key for one
|
||||
// notification_dead_letter_entry.
|
||||
func (Keyspace) DeadLetter(notificationID string, routeID string) string {
|
||||
return defaultPrefix + "dead_letters:" + encodeKeyComponent(notificationID) + ":" + encodeKeyComponent(routeID)
|
||||
}
|
||||
|
||||
// RouteLease returns the temporary Redis key used to coordinate exclusive
|
||||
// publication of one notification_route across replicas.
|
||||
func (Keyspace) RouteLease(notificationID string, routeID string) string {
|
||||
return defaultPrefix + "route_leases:" + encodeKeyComponent(notificationID) + ":" + encodeKeyComponent(routeID)
|
||||
}
|
||||
|
||||
// MalformedIntent returns the primary Redis key for one malformed-intent
|
||||
// record.
|
||||
func (Keyspace) MalformedIntent(streamEntryID string) string {
|
||||
return defaultPrefix + "malformed_intents:" + encodeKeyComponent(streamEntryID)
|
||||
}
|
||||
|
||||
// StreamOffset returns the primary Redis key for one persisted intent-consumer
|
||||
// offset.
|
||||
func (Keyspace) StreamOffset(stream string) string {
|
||||
@@ -86,20 +32,6 @@ func (Keyspace) Intents() string {
|
||||
return defaultPrefix + "intents"
|
||||
}
|
||||
|
||||
// RouteSchedule returns the frozen route schedule sorted-set key.
|
||||
func (Keyspace) RouteSchedule() string {
|
||||
return defaultPrefix + "route_schedule"
|
||||
}
|
||||
|
||||
func encodeKeyComponent(value string) string {
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(value))
|
||||
}
|
||||
|
||||
func decodeKeyComponent(value string) (string, error) {
|
||||
decoded, err := base64.RawURLEncoding.DecodeString(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(decoded), nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package redisstate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// releaseRouteLeaseScript releases the route lease only when the supplied
|
||||
// token still owns it. The Lua script gates the DEL on the SET value match
|
||||
// so a publisher that lost the lease (TTL expiry, replica swap) cannot
|
||||
// clear another worker's claim.
|
||||
var releaseRouteLeaseScript = redis.NewScript(`
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("DEL", KEYS[1])
|
||||
end
|
||||
return 0
|
||||
`)
|
||||
|
||||
// LeaseStore owns the short-lived route lease keys that coordinate exclusive
|
||||
// route publication across replicas. The lease lives on Redis as a per-route
|
||||
// SETNX-with-TTL token; releasing it requires the same token via a Lua
|
||||
// script that compares the stored value before deleting it.
|
||||
//
|
||||
// LeaseStore is intentionally separate from the durable route-state storage
|
||||
// so the publishers can compose one storage-layer adapter (PostgreSQL since
|
||||
// Stage 5) with the runtime-coordination layer that stays on Redis per
|
||||
// `ARCHITECTURE.md §Persistence Backends`.
|
||||
type LeaseStore struct {
|
||||
client *redis.Client
|
||||
keys Keyspace
|
||||
}
|
||||
|
||||
// NewLeaseStore constructs one Redis-backed lease store.
|
||||
func NewLeaseStore(client *redis.Client) (*LeaseStore, error) {
|
||||
if client == nil {
|
||||
return nil, errors.New("new notification lease store: nil redis client")
|
||||
}
|
||||
|
||||
return &LeaseStore{client: client, keys: Keyspace{}}, nil
|
||||
}
|
||||
|
||||
// TryAcquireRouteLease attempts to acquire one temporary route lease owned
|
||||
// by token for ttl. The lease is stored at the route-lease keyspace key and
|
||||
// auto-expires; a publisher whose work outlives the TTL must accept that
|
||||
// another replica may pick the route up.
|
||||
func (store *LeaseStore) TryAcquireRouteLease(ctx context.Context, notificationID string, routeID string, token string, ttl time.Duration) (bool, error) {
|
||||
if store == nil || store.client == nil {
|
||||
return false, errors.New("try acquire route lease: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return false, errors.New("try acquire route lease: nil context")
|
||||
}
|
||||
if notificationID == "" {
|
||||
return false, errors.New("try acquire route lease: notification id must not be empty")
|
||||
}
|
||||
if routeID == "" {
|
||||
return false, errors.New("try acquire route lease: route id must not be empty")
|
||||
}
|
||||
if token == "" {
|
||||
return false, errors.New("try acquire route lease: token must not be empty")
|
||||
}
|
||||
if ttl <= 0 {
|
||||
return false, errors.New("try acquire route lease: ttl must be positive")
|
||||
}
|
||||
|
||||
acquired, err := store.client.SetNX(ctx, store.keys.RouteLease(notificationID, routeID), token, ttl).Result()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("try acquire route lease: %w", err)
|
||||
}
|
||||
|
||||
return acquired, nil
|
||||
}
|
||||
|
||||
// ReleaseRouteLease releases one temporary route lease only when token still
|
||||
// matches the stored owner value. Releasing a lease the caller no longer
|
||||
// owns is a silent no-op.
|
||||
func (store *LeaseStore) ReleaseRouteLease(ctx context.Context, notificationID string, routeID string, token string) error {
|
||||
if store == nil || store.client == nil {
|
||||
return errors.New("release route lease: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return errors.New("release route lease: nil context")
|
||||
}
|
||||
if notificationID == "" {
|
||||
return errors.New("release route lease: notification id must not be empty")
|
||||
}
|
||||
if routeID == "" {
|
||||
return errors.New("release route lease: route id must not be empty")
|
||||
}
|
||||
if token == "" {
|
||||
return errors.New("release route lease: token must not be empty")
|
||||
}
|
||||
|
||||
if err := releaseRouteLeaseScript.Run(
|
||||
ctx,
|
||||
store.client,
|
||||
[]string{store.keys.RouteLease(notificationID, routeID)},
|
||||
token,
|
||||
).Err(); err != nil {
|
||||
return fmt.Errorf("release route lease: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package redisstate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"galaxy/notification/internal/service/malformedintent"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// MalformedIntentStore provides the Redis-backed storage used for
|
||||
// operator-visible malformed-intent records.
|
||||
type MalformedIntentStore struct {
|
||||
client *redis.Client
|
||||
keys Keyspace
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
// NewMalformedIntentStore constructs one Redis-backed malformed-intent store.
|
||||
func NewMalformedIntentStore(client *redis.Client, ttl time.Duration) (*MalformedIntentStore, error) {
|
||||
if client == nil {
|
||||
return nil, errors.New("new malformed intent store: nil redis client")
|
||||
}
|
||||
if ttl <= 0 {
|
||||
return nil, errors.New("new malformed intent store: non-positive ttl")
|
||||
}
|
||||
|
||||
return &MalformedIntentStore{
|
||||
client: client,
|
||||
keys: Keyspace{},
|
||||
ttl: ttl,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Record stores entry idempotently by its Redis Stream entry identifier.
|
||||
func (store *MalformedIntentStore) Record(ctx context.Context, entry malformedintent.Entry) error {
|
||||
if store == nil || store.client == nil {
|
||||
return errors.New("record malformed intent: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return errors.New("record malformed intent: nil context")
|
||||
}
|
||||
if err := entry.Validate(); err != nil {
|
||||
return fmt.Errorf("record malformed intent: %w", err)
|
||||
}
|
||||
|
||||
payload, err := MarshalMalformedIntent(entry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record malformed intent: %w", err)
|
||||
}
|
||||
if err := store.client.Set(ctx, store.keys.MalformedIntent(entry.StreamEntryID), payload, store.ttl).Err(); err != nil {
|
||||
return fmt.Errorf("record malformed intent: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,657 +0,0 @@
|
||||
package redisstate
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"galaxy/notification/internal/service/acceptintent"
|
||||
"galaxy/notification/internal/telemetry"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
var releaseRouteLeaseScript = redis.NewScript(`
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("DEL", KEYS[1])
|
||||
end
|
||||
return 0
|
||||
`)
|
||||
|
||||
var completePublishedRouteScript = redis.NewScript(`
|
||||
if redis.call("GET", KEYS[1]) ~= ARGV[1] then
|
||||
return 0
|
||||
end
|
||||
if redis.call("GET", KEYS[2]) ~= ARGV[2] then
|
||||
return 0
|
||||
end
|
||||
local field_count = tonumber(ARGV[6])
|
||||
local values = {}
|
||||
local index = 7
|
||||
for _ = 1, field_count do
|
||||
table.insert(values, ARGV[index])
|
||||
table.insert(values, ARGV[index + 1])
|
||||
index = index + 2
|
||||
end
|
||||
if tonumber(ARGV[4]) > 0 then
|
||||
redis.call("XADD", ARGV[3], "MAXLEN", "~", ARGV[4], "*", unpack(values))
|
||||
else
|
||||
redis.call("XADD", ARGV[3], "*", unpack(values))
|
||||
end
|
||||
redis.call("SET", KEYS[1], ARGV[5], "KEEPTTL")
|
||||
redis.call("ZREM", KEYS[3], KEYS[1])
|
||||
redis.call("DEL", KEYS[2])
|
||||
return 1
|
||||
`)
|
||||
|
||||
// ScheduledRoute stores one due route reference loaded from
|
||||
// `notification:route_schedule`.
|
||||
type ScheduledRoute struct {
|
||||
// RouteKey stores the full Redis route key scheduled for processing.
|
||||
RouteKey string
|
||||
|
||||
// NotificationID stores the owning notification identifier.
|
||||
NotificationID string
|
||||
|
||||
// RouteID stores the scheduled route identifier.
|
||||
RouteID string
|
||||
}
|
||||
|
||||
// CompleteRoutePublishedInput stores the data required to mark one route as
|
||||
// published while atomically appending one outbound stream entry.
|
||||
type CompleteRoutePublishedInput struct {
|
||||
// ExpectedRoute stores the current route state previously loaded by the
|
||||
// caller.
|
||||
ExpectedRoute acceptintent.NotificationRoute
|
||||
|
||||
// LeaseToken stores the route-lease owner token that must still be held.
|
||||
LeaseToken string
|
||||
|
||||
// PublishedAt stores when the publication attempt succeeded.
|
||||
PublishedAt time.Time
|
||||
|
||||
// Stream stores the outbound Redis Stream name.
|
||||
Stream string
|
||||
|
||||
// StreamMaxLen bounds Stream with approximate trimming when positive. Zero
|
||||
// disables trimming.
|
||||
StreamMaxLen int64
|
||||
|
||||
// StreamValues stores the exact Redis Stream fields appended to Stream.
|
||||
StreamValues map[string]any
|
||||
}
|
||||
|
||||
// CompleteRouteFailedInput stores the data required to record one retryable
|
||||
// publication failure.
|
||||
type CompleteRouteFailedInput struct {
|
||||
// ExpectedRoute stores the current route state previously loaded by the
|
||||
// caller.
|
||||
ExpectedRoute acceptintent.NotificationRoute
|
||||
|
||||
// LeaseToken stores the route-lease owner token that must still be held.
|
||||
LeaseToken string
|
||||
|
||||
// FailedAt stores when the publication attempt failed.
|
||||
FailedAt time.Time
|
||||
|
||||
// NextAttemptAt stores the next scheduled retry time.
|
||||
NextAttemptAt time.Time
|
||||
|
||||
// FailureClassification stores the classified publication failure kind.
|
||||
FailureClassification string
|
||||
|
||||
// FailureMessage stores the detailed publication failure text.
|
||||
FailureMessage string
|
||||
}
|
||||
|
||||
// CompleteRouteDeadLetterInput stores the data required to record one
|
||||
// exhausted publication failure.
|
||||
type CompleteRouteDeadLetterInput struct {
|
||||
// ExpectedRoute stores the current route state previously loaded by the
|
||||
// caller.
|
||||
ExpectedRoute acceptintent.NotificationRoute
|
||||
|
||||
// LeaseToken stores the route-lease owner token that must still be held.
|
||||
LeaseToken string
|
||||
|
||||
// DeadLetteredAt stores when the route exhausted its retry budget.
|
||||
DeadLetteredAt time.Time
|
||||
|
||||
// FailureClassification stores the classified terminal failure kind.
|
||||
FailureClassification string
|
||||
|
||||
// FailureMessage stores the detailed terminal failure text.
|
||||
FailureMessage string
|
||||
|
||||
// RecoveryHint stores the optional operator-facing recovery guidance.
|
||||
RecoveryHint string
|
||||
}
|
||||
|
||||
// ListDueRoutes loads up to limit scheduled routes whose next-attempt score is
|
||||
// due at or before now.
|
||||
func (store *AcceptanceStore) ListDueRoutes(ctx context.Context, now time.Time, limit int64) ([]ScheduledRoute, error) {
|
||||
if store == nil || store.client == nil {
|
||||
return nil, errors.New("list due routes: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return nil, errors.New("list due routes: nil context")
|
||||
}
|
||||
if err := validateRouteStateTimestamp("list due routes now", now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if limit <= 0 {
|
||||
return nil, errors.New("list due routes: limit must be positive")
|
||||
}
|
||||
|
||||
members, err := store.client.ZRangeByScore(ctx, store.keys.RouteSchedule(), &redis.ZRangeBy{
|
||||
Min: "-inf",
|
||||
Max: strconv.FormatInt(now.UnixMilli(), 10),
|
||||
Count: limit,
|
||||
}).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list due routes: %w", err)
|
||||
}
|
||||
|
||||
routes := make([]ScheduledRoute, 0, len(members))
|
||||
for _, member := range members {
|
||||
notificationID, routeID, err := store.keys.ParseRoute(member)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list due routes: %w", err)
|
||||
}
|
||||
routes = append(routes, ScheduledRoute{
|
||||
RouteKey: member,
|
||||
NotificationID: notificationID,
|
||||
RouteID: routeID,
|
||||
})
|
||||
}
|
||||
|
||||
return routes, nil
|
||||
}
|
||||
|
||||
// ReadRouteScheduleSnapshot returns the current depth of the durable route
|
||||
// schedule together with its oldest scheduled timestamp when one exists.
|
||||
func (store *AcceptanceStore) ReadRouteScheduleSnapshot(ctx context.Context) (telemetry.RouteScheduleSnapshot, error) {
|
||||
if store == nil || store.client == nil {
|
||||
return telemetry.RouteScheduleSnapshot{}, errors.New("read route schedule snapshot: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return telemetry.RouteScheduleSnapshot{}, errors.New("read route schedule snapshot: nil context")
|
||||
}
|
||||
|
||||
depth, err := store.client.ZCard(ctx, store.keys.RouteSchedule()).Result()
|
||||
if err != nil {
|
||||
return telemetry.RouteScheduleSnapshot{}, fmt.Errorf("read route schedule snapshot: depth: %w", err)
|
||||
}
|
||||
|
||||
snapshot := telemetry.RouteScheduleSnapshot{
|
||||
Depth: depth,
|
||||
}
|
||||
if depth == 0 {
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
values, err := store.client.ZRangeWithScores(ctx, store.keys.RouteSchedule(), 0, 0).Result()
|
||||
if err != nil {
|
||||
return telemetry.RouteScheduleSnapshot{}, fmt.Errorf("read route schedule snapshot: oldest scheduled entry: %w", err)
|
||||
}
|
||||
if len(values) == 0 {
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
oldestScheduledFor := time.UnixMilli(int64(values[0].Score)).UTC()
|
||||
snapshot.OldestScheduledFor = &oldestScheduledFor
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
// TryAcquireRouteLease attempts to acquire one temporary route lease owned by
|
||||
// token for ttl.
|
||||
func (store *AcceptanceStore) TryAcquireRouteLease(ctx context.Context, notificationID string, routeID string, token string, ttl time.Duration) (bool, error) {
|
||||
if store == nil || store.client == nil {
|
||||
return false, errors.New("try acquire route lease: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return false, errors.New("try acquire route lease: nil context")
|
||||
}
|
||||
if notificationID == "" {
|
||||
return false, errors.New("try acquire route lease: notification id must not be empty")
|
||||
}
|
||||
if routeID == "" {
|
||||
return false, errors.New("try acquire route lease: route id must not be empty")
|
||||
}
|
||||
if token == "" {
|
||||
return false, errors.New("try acquire route lease: token must not be empty")
|
||||
}
|
||||
if ttl <= 0 {
|
||||
return false, errors.New("try acquire route lease: ttl must be positive")
|
||||
}
|
||||
|
||||
acquired, err := store.client.SetNX(ctx, store.keys.RouteLease(notificationID, routeID), token, ttl).Result()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("try acquire route lease: %w", err)
|
||||
}
|
||||
|
||||
return acquired, nil
|
||||
}
|
||||
|
||||
// ReleaseRouteLease releases one temporary route lease only when token still
|
||||
// matches the stored owner value.
|
||||
func (store *AcceptanceStore) ReleaseRouteLease(ctx context.Context, notificationID string, routeID string, token string) error {
|
||||
if store == nil || store.client == nil {
|
||||
return errors.New("release route lease: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return errors.New("release route lease: nil context")
|
||||
}
|
||||
if notificationID == "" {
|
||||
return errors.New("release route lease: notification id must not be empty")
|
||||
}
|
||||
if routeID == "" {
|
||||
return errors.New("release route lease: route id must not be empty")
|
||||
}
|
||||
if token == "" {
|
||||
return errors.New("release route lease: token must not be empty")
|
||||
}
|
||||
|
||||
if err := releaseRouteLeaseScript.Run(
|
||||
ctx,
|
||||
store.client,
|
||||
[]string{store.keys.RouteLease(notificationID, routeID)},
|
||||
token,
|
||||
).Err(); err != nil {
|
||||
return fmt.Errorf("release route lease: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CompleteRoutePublished atomically appends one outbound stream entry and
|
||||
// marks the corresponding route as published.
|
||||
func (store *AcceptanceStore) CompleteRoutePublished(ctx context.Context, input CompleteRoutePublishedInput) error {
|
||||
if store == nil || store.client == nil {
|
||||
return errors.New("complete route published: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return errors.New("complete route published: nil context")
|
||||
}
|
||||
if err := input.Validate(); err != nil {
|
||||
return fmt.Errorf("complete route published: %w", err)
|
||||
}
|
||||
|
||||
updatedRoute := input.ExpectedRoute
|
||||
updatedRoute.Status = acceptintent.RouteStatusPublished
|
||||
updatedRoute.AttemptCount++
|
||||
updatedRoute.NextAttemptAt = time.Time{}
|
||||
updatedRoute.LastErrorClassification = ""
|
||||
updatedRoute.LastErrorMessage = ""
|
||||
updatedRoute.LastErrorAt = time.Time{}
|
||||
updatedRoute.UpdatedAt = input.PublishedAt
|
||||
updatedRoute.PublishedAt = input.PublishedAt
|
||||
updatedRoute.DeadLetteredAt = time.Time{}
|
||||
payload, err := MarshalRoute(updatedRoute)
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete route published: %w", err)
|
||||
}
|
||||
expectedPayload, err := MarshalRoute(input.ExpectedRoute)
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete route published: %w", err)
|
||||
}
|
||||
streamArgs, err := flattenStreamValues(input.StreamValues)
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete route published: %w", err)
|
||||
}
|
||||
|
||||
result, err := completePublishedRouteScript.Run(
|
||||
ctx,
|
||||
store.client,
|
||||
[]string{
|
||||
store.keys.Route(updatedRoute.NotificationID, updatedRoute.RouteID),
|
||||
store.keys.RouteLease(updatedRoute.NotificationID, updatedRoute.RouteID),
|
||||
store.keys.RouteSchedule(),
|
||||
},
|
||||
append([]any{
|
||||
string(expectedPayload),
|
||||
input.LeaseToken,
|
||||
input.Stream,
|
||||
input.StreamMaxLen,
|
||||
string(payload),
|
||||
len(streamArgs) / 2,
|
||||
}, streamArgs...)...,
|
||||
).Int()
|
||||
switch {
|
||||
case errors.Is(err, redis.Nil):
|
||||
return ErrConflict
|
||||
case err != nil:
|
||||
return err
|
||||
case result != 1:
|
||||
return ErrConflict
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// CompleteRouteFailed atomically records one retryable publication failure and
|
||||
// reschedules the route.
|
||||
func (store *AcceptanceStore) CompleteRouteFailed(ctx context.Context, input CompleteRouteFailedInput) error {
|
||||
if store == nil || store.client == nil {
|
||||
return errors.New("complete route failed: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return errors.New("complete route failed: nil context")
|
||||
}
|
||||
if err := input.Validate(); err != nil {
|
||||
return fmt.Errorf("complete route failed: %w", err)
|
||||
}
|
||||
|
||||
updatedRoute := input.ExpectedRoute
|
||||
updatedRoute.Status = acceptintent.RouteStatusFailed
|
||||
updatedRoute.AttemptCount++
|
||||
updatedRoute.NextAttemptAt = input.NextAttemptAt
|
||||
updatedRoute.LastErrorClassification = input.FailureClassification
|
||||
updatedRoute.LastErrorMessage = input.FailureMessage
|
||||
updatedRoute.LastErrorAt = input.FailedAt
|
||||
updatedRoute.UpdatedAt = input.FailedAt
|
||||
payload, err := MarshalRoute(updatedRoute)
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete route failed: %w", err)
|
||||
}
|
||||
|
||||
return store.completeRouteMutation(ctx, input.ExpectedRoute, input.LeaseToken, func(pipe redis.Pipeliner) error {
|
||||
pipe.SetArgs(ctx, store.keys.Route(updatedRoute.NotificationID, updatedRoute.RouteID), payload, redis.SetArgs{KeepTTL: true})
|
||||
pipe.ZAdd(ctx, store.keys.RouteSchedule(), redis.Z{
|
||||
Score: float64(input.NextAttemptAt.UnixMilli()),
|
||||
Member: store.keys.Route(updatedRoute.NotificationID, updatedRoute.RouteID),
|
||||
})
|
||||
pipe.Del(ctx, store.keys.RouteLease(updatedRoute.NotificationID, updatedRoute.RouteID))
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// CompleteRouteDeadLetter atomically records one exhausted publication
|
||||
// failure, stores the dead-letter entry, and removes the route from the
|
||||
// retry schedule.
|
||||
func (store *AcceptanceStore) CompleteRouteDeadLetter(ctx context.Context, input CompleteRouteDeadLetterInput) error {
|
||||
if store == nil || store.client == nil {
|
||||
return errors.New("complete route dead letter: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return errors.New("complete route dead letter: nil context")
|
||||
}
|
||||
if err := input.Validate(); err != nil {
|
||||
return fmt.Errorf("complete route dead letter: %w", err)
|
||||
}
|
||||
|
||||
updatedRoute := input.ExpectedRoute
|
||||
updatedRoute.Status = acceptintent.RouteStatusDeadLetter
|
||||
updatedRoute.AttemptCount++
|
||||
updatedRoute.NextAttemptAt = time.Time{}
|
||||
updatedRoute.LastErrorClassification = input.FailureClassification
|
||||
updatedRoute.LastErrorMessage = input.FailureMessage
|
||||
updatedRoute.LastErrorAt = input.DeadLetteredAt
|
||||
updatedRoute.UpdatedAt = input.DeadLetteredAt
|
||||
updatedRoute.DeadLetteredAt = input.DeadLetteredAt
|
||||
if updatedRoute.AttemptCount < updatedRoute.MaxAttempts {
|
||||
return fmt.Errorf(
|
||||
"complete route dead letter: final attempt count %d is below max attempts %d",
|
||||
updatedRoute.AttemptCount,
|
||||
updatedRoute.MaxAttempts,
|
||||
)
|
||||
}
|
||||
|
||||
routePayload, err := MarshalRoute(updatedRoute)
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete route dead letter: %w", err)
|
||||
}
|
||||
deadLetterPayload, err := MarshalDeadLetter(DeadLetterEntry{
|
||||
NotificationID: updatedRoute.NotificationID,
|
||||
RouteID: updatedRoute.RouteID,
|
||||
Channel: updatedRoute.Channel,
|
||||
RecipientRef: updatedRoute.RecipientRef,
|
||||
FinalAttemptCount: updatedRoute.AttemptCount,
|
||||
MaxAttempts: updatedRoute.MaxAttempts,
|
||||
FailureClassification: input.FailureClassification,
|
||||
FailureMessage: input.FailureMessage,
|
||||
CreatedAt: input.DeadLetteredAt,
|
||||
RecoveryHint: input.RecoveryHint,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete route dead letter: %w", err)
|
||||
}
|
||||
|
||||
return store.completeRouteMutation(ctx, input.ExpectedRoute, input.LeaseToken, func(pipe redis.Pipeliner) error {
|
||||
pipe.SetArgs(ctx, store.keys.Route(updatedRoute.NotificationID, updatedRoute.RouteID), routePayload, redis.SetArgs{KeepTTL: true})
|
||||
pipe.Set(ctx, store.keys.DeadLetter(updatedRoute.NotificationID, updatedRoute.RouteID), deadLetterPayload, store.cfg.DeadLetterTTL)
|
||||
pipe.ZRem(ctx, store.keys.RouteSchedule(), store.keys.Route(updatedRoute.NotificationID, updatedRoute.RouteID))
|
||||
pipe.Del(ctx, store.keys.RouteLease(updatedRoute.NotificationID, updatedRoute.RouteID))
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (store *AcceptanceStore) completeRouteMutation(
|
||||
ctx context.Context,
|
||||
expectedRoute acceptintent.NotificationRoute,
|
||||
leaseToken string,
|
||||
mutate func(redis.Pipeliner) error,
|
||||
) error {
|
||||
routeKey := store.keys.Route(expectedRoute.NotificationID, expectedRoute.RouteID)
|
||||
leaseKey := store.keys.RouteLease(expectedRoute.NotificationID, expectedRoute.RouteID)
|
||||
|
||||
watchErr := store.client.Watch(ctx, func(tx *redis.Tx) error {
|
||||
currentRoute, err := loadWatchedRoute(ctx, tx, routeKey)
|
||||
switch {
|
||||
case errors.Is(err, redis.Nil):
|
||||
return ErrConflict
|
||||
case err != nil:
|
||||
return err
|
||||
}
|
||||
if err := ensureRoutesEqual(expectedRoute, currentRoute); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
leaseValue, err := tx.Get(ctx, leaseKey).Result()
|
||||
switch {
|
||||
case errors.Is(err, redis.Nil):
|
||||
return ErrConflict
|
||||
case err != nil:
|
||||
return err
|
||||
case leaseValue != leaseToken:
|
||||
return ErrConflict
|
||||
}
|
||||
|
||||
_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {
|
||||
return mutate(pipe)
|
||||
})
|
||||
|
||||
return err
|
||||
}, routeKey, leaseKey)
|
||||
|
||||
switch {
|
||||
case errors.Is(watchErr, ErrConflict), errors.Is(watchErr, redis.TxFailedErr):
|
||||
return ErrConflict
|
||||
case watchErr != nil:
|
||||
return watchErr
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func loadWatchedRoute(ctx context.Context, tx *redis.Tx, routeKey string) (acceptintent.NotificationRoute, error) {
|
||||
payload, err := tx.Get(ctx, routeKey).Bytes()
|
||||
if err != nil {
|
||||
return acceptintent.NotificationRoute{}, err
|
||||
}
|
||||
|
||||
return UnmarshalRoute(payload)
|
||||
}
|
||||
|
||||
func ensureRoutesEqual(expected acceptintent.NotificationRoute, actual acceptintent.NotificationRoute) error {
|
||||
expectedPayload, err := MarshalRoute(expected)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal expected route: %w", err)
|
||||
}
|
||||
actualPayload, err := MarshalRoute(actual)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal current route: %w", err)
|
||||
}
|
||||
if !bytes.Equal(expectedPayload, actualPayload) {
|
||||
return ErrConflict
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCompletionRoute(route acceptintent.NotificationRoute) error {
|
||||
if err := route.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
switch route.Status {
|
||||
case acceptintent.RouteStatusPending, acceptintent.RouteStatusFailed:
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("route status %q is not completable", route.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func validateStreamValues(values map[string]any) error {
|
||||
if len(values) == 0 {
|
||||
return fmt.Errorf("stream values must not be empty")
|
||||
}
|
||||
|
||||
for key, raw := range values {
|
||||
if key == "" {
|
||||
return fmt.Errorf("stream values key must not be empty")
|
||||
}
|
||||
switch typed := raw.(type) {
|
||||
case string:
|
||||
if typed == "" {
|
||||
return fmt.Errorf("stream values %q must not be empty", key)
|
||||
}
|
||||
case []byte:
|
||||
if len(typed) == 0 {
|
||||
return fmt.Errorf("stream values %q must not be empty", key)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("stream values %q must be string or []byte", key)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func flattenStreamValues(values map[string]any) ([]any, error) {
|
||||
keys := make([]string, 0, len(values))
|
||||
for key := range values {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
args := make([]any, 0, len(values)*2)
|
||||
for _, key := range keys {
|
||||
args = append(args, key, values[key])
|
||||
}
|
||||
|
||||
return args, nil
|
||||
}
|
||||
|
||||
func validateRouteStateTimestamp(name string, value time.Time) error {
|
||||
if value.IsZero() {
|
||||
return fmt.Errorf("%s must not be zero", name)
|
||||
}
|
||||
if !value.Equal(value.UTC()) {
|
||||
return fmt.Errorf("%s must be UTC", name)
|
||||
}
|
||||
if !value.Equal(value.Truncate(time.Millisecond)) {
|
||||
return fmt.Errorf("%s must use millisecond precision", name)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate reports whether route contains a complete due-route reference.
|
||||
func (route ScheduledRoute) Validate() error {
|
||||
if route.RouteKey == "" {
|
||||
return fmt.Errorf("scheduled route key must not be empty")
|
||||
}
|
||||
if route.NotificationID == "" {
|
||||
return fmt.Errorf("scheduled route notification id must not be empty")
|
||||
}
|
||||
if route.RouteID == "" {
|
||||
return fmt.Errorf("scheduled route route id must not be empty")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate reports whether input contains a complete published-route
|
||||
// transition.
|
||||
func (input CompleteRoutePublishedInput) Validate() error {
|
||||
if err := validateCompletionRoute(input.ExpectedRoute); err != nil {
|
||||
return err
|
||||
}
|
||||
if input.LeaseToken == "" {
|
||||
return fmt.Errorf("lease token must not be empty")
|
||||
}
|
||||
if err := validateRouteStateTimestamp("published at", input.PublishedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
if input.Stream == "" {
|
||||
return fmt.Errorf("stream must not be empty")
|
||||
}
|
||||
if input.StreamMaxLen < 0 {
|
||||
return fmt.Errorf("stream max len must not be negative")
|
||||
}
|
||||
if err := validateStreamValues(input.StreamValues); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate reports whether input contains a complete retryable failure
|
||||
// transition.
|
||||
func (input CompleteRouteFailedInput) Validate() error {
|
||||
if err := validateCompletionRoute(input.ExpectedRoute); err != nil {
|
||||
return err
|
||||
}
|
||||
if input.LeaseToken == "" {
|
||||
return fmt.Errorf("lease token must not be empty")
|
||||
}
|
||||
if err := validateRouteStateTimestamp("failed at", input.FailedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateRouteStateTimestamp("next attempt at", input.NextAttemptAt); err != nil {
|
||||
return err
|
||||
}
|
||||
if input.FailureClassification == "" {
|
||||
return fmt.Errorf("failure classification must not be empty")
|
||||
}
|
||||
if input.FailureMessage == "" {
|
||||
return fmt.Errorf("failure message must not be empty")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate reports whether input contains a complete dead-letter transition.
|
||||
func (input CompleteRouteDeadLetterInput) Validate() error {
|
||||
if err := validateCompletionRoute(input.ExpectedRoute); err != nil {
|
||||
return err
|
||||
}
|
||||
if input.LeaseToken == "" {
|
||||
return fmt.Errorf("lease token must not be empty")
|
||||
}
|
||||
if err := validateRouteStateTimestamp("dead lettered at", input.DeadLetteredAt); err != nil {
|
||||
return err
|
||||
}
|
||||
if input.FailureClassification == "" {
|
||||
return fmt.Errorf("failure classification must not be empty")
|
||||
}
|
||||
if input.FailureMessage == "" {
|
||||
return fmt.Errorf("failure message must not be empty")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,465 +0,0 @@
|
||||
package redisstate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"galaxy/notification/internal/api/intentstream"
|
||||
"galaxy/notification/internal/service/acceptintent"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAcceptanceStoreListDueRoutesLoadsScheduledMembers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewAcceptanceStore(client, AcceptanceConfig{
|
||||
RecordTTL: 24 * time.Hour,
|
||||
DeadLetterTTL: 72 * time.Hour,
|
||||
IdempotencyTTL: 7 * 24 * time.Hour,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
now := time.UnixMilli(1775121700000).UTC()
|
||||
require.NoError(t, store.CreateAcceptance(context.Background(), validUserAcceptanceInput(now, 0)))
|
||||
|
||||
routes, err := store.ListDueRoutes(context.Background(), now, 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, routes, 2)
|
||||
require.ElementsMatch(t, []string{"push:user:user-1", "email:user:user-1"}, []string{routes[0].RouteID, routes[1].RouteID})
|
||||
|
||||
for _, route := range routes {
|
||||
require.NoError(t, route.Validate())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptanceStoreReadRouteScheduleSnapshot(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewAcceptanceStore(client, AcceptanceConfig{
|
||||
RecordTTL: 24 * time.Hour,
|
||||
DeadLetterTTL: 72 * time.Hour,
|
||||
IdempotencyTTL: 7 * 24 * time.Hour,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
now := time.UnixMilli(1775121700000).UTC()
|
||||
require.NoError(t, store.CreateAcceptance(context.Background(), validUserAcceptanceInput(now, 0)))
|
||||
|
||||
snapshot, err := store.ReadRouteScheduleSnapshot(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(2), snapshot.Depth)
|
||||
require.NotNil(t, snapshot.OldestScheduledFor)
|
||||
require.Equal(t, now, *snapshot.OldestScheduledFor)
|
||||
}
|
||||
|
||||
func TestAcceptanceStoreRouteLeaseAcquireReleaseAndExpire(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewAcceptanceStore(client, AcceptanceConfig{
|
||||
RecordTTL: 24 * time.Hour,
|
||||
DeadLetterTTL: 72 * time.Hour,
|
||||
IdempotencyTTL: 7 * 24 * time.Hour,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
acquired, err := store.TryAcquireRouteLease(context.Background(), "1775121700000-0", "push:user:user-1", "token-1", 2*time.Second)
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
|
||||
acquired, err = store.TryAcquireRouteLease(context.Background(), "1775121700000-0", "push:user:user-1", "token-2", 2*time.Second)
|
||||
require.NoError(t, err)
|
||||
require.False(t, acquired)
|
||||
|
||||
require.NoError(t, store.ReleaseRouteLease(context.Background(), "1775121700000-0", "push:user:user-1", "token-1"))
|
||||
acquired, err = store.TryAcquireRouteLease(context.Background(), "1775121700000-0", "push:user:user-1", "token-3", 2*time.Second)
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
|
||||
server.FastForward(3 * time.Second)
|
||||
acquired, err = store.TryAcquireRouteLease(context.Background(), "1775121700000-0", "push:user:user-1", "token-4", 2*time.Second)
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
}
|
||||
|
||||
func TestAcceptanceStoreCompleteRoutePublishedAppendsTrimmedStreamEntryAndMarksRoutePublished(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewAcceptanceStore(client, AcceptanceConfig{
|
||||
RecordTTL: 24 * time.Hour,
|
||||
DeadLetterTTL: 72 * time.Hour,
|
||||
IdempotencyTTL: 7 * 24 * time.Hour,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
now := time.UnixMilli(1775121700000).UTC()
|
||||
input := validUserAcceptanceInput(now, 0)
|
||||
require.NoError(t, store.CreateAcceptance(context.Background(), input))
|
||||
|
||||
acquired, err := store.TryAcquireRouteLease(context.Background(), input.Notification.NotificationID, "push:user:user-1", "token-1", 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
|
||||
route, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
|
||||
publishedAt := now.Add(time.Second).UTC().Truncate(time.Millisecond)
|
||||
require.NoError(t, store.CompleteRoutePublished(context.Background(), CompleteRoutePublishedInput{
|
||||
ExpectedRoute: route,
|
||||
LeaseToken: "token-1",
|
||||
PublishedAt: publishedAt,
|
||||
Stream: "gateway:client-events",
|
||||
StreamMaxLen: 1024,
|
||||
StreamValues: map[string]any{
|
||||
"user_id": "user-1",
|
||||
"event_type": "game.turn.ready",
|
||||
"event_id": input.Notification.NotificationID + "/push:user:user-1",
|
||||
"payload_bytes": []byte("payload-1"),
|
||||
"request_id": "request-1",
|
||||
"trace_id": "trace-1",
|
||||
},
|
||||
}))
|
||||
|
||||
updatedRoute, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.Equal(t, acceptintent.RouteStatusPublished, updatedRoute.Status)
|
||||
require.Equal(t, 1, updatedRoute.AttemptCount)
|
||||
require.Equal(t, publishedAt, updatedRoute.PublishedAt)
|
||||
|
||||
scheduled, err := client.ZRange(context.Background(), Keyspace{}.RouteSchedule(), 0, -1).Result()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []string{Keyspace{}.Route(input.Notification.NotificationID, "email:user:user-1")}, scheduled)
|
||||
|
||||
messages, err := client.XRange(context.Background(), "gateway:client-events", "-", "+").Result()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, messages, 1)
|
||||
require.Equal(t, "user-1", messages[0].Values["user_id"])
|
||||
require.Equal(t, "game.turn.ready", messages[0].Values["event_type"])
|
||||
|
||||
leaseKey := Keyspace{}.RouteLease(input.Notification.NotificationID, "push:user:user-1")
|
||||
_, err = client.Get(context.Background(), leaseKey).Result()
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestAcceptanceStoreCompleteRoutePublishedAppendsUntrimmedMailCommand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewAcceptanceStore(client, AcceptanceConfig{
|
||||
RecordTTL: 24 * time.Hour,
|
||||
DeadLetterTTL: 72 * time.Hour,
|
||||
IdempotencyTTL: 7 * 24 * time.Hour,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
now := time.UnixMilli(1775121700000).UTC()
|
||||
input := validUserAcceptanceInput(now, 0)
|
||||
require.NoError(t, store.CreateAcceptance(context.Background(), input))
|
||||
|
||||
acquired, err := store.TryAcquireRouteLease(context.Background(), input.Notification.NotificationID, "email:user:user-1", "token-1", 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
|
||||
route, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "email:user:user-1")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
|
||||
publishedAt := now.Add(time.Second).UTC().Truncate(time.Millisecond)
|
||||
require.NoError(t, store.CompleteRoutePublished(context.Background(), CompleteRoutePublishedInput{
|
||||
ExpectedRoute: route,
|
||||
LeaseToken: "token-1",
|
||||
PublishedAt: publishedAt,
|
||||
Stream: "mail:delivery_commands",
|
||||
StreamMaxLen: 0,
|
||||
StreamValues: map[string]any{
|
||||
"delivery_id": input.Notification.NotificationID + "/email:user:user-1",
|
||||
"source": "notification",
|
||||
"payload_mode": "template",
|
||||
"idempotency_key": "notification:" + input.Notification.NotificationID + "/email:user:user-1",
|
||||
"requested_at_ms": "1775121700000",
|
||||
"payload_json": `{"to":["pilot@example.com"],"cc":[],"bcc":[],"reply_to":[],"template_id":"game.turn.ready","locale":"en","variables":{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54},"attachments":[]}`,
|
||||
},
|
||||
}))
|
||||
|
||||
updatedRoute, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "email:user:user-1")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.Equal(t, acceptintent.RouteStatusPublished, updatedRoute.Status)
|
||||
require.Equal(t, 1, updatedRoute.AttemptCount)
|
||||
require.Equal(t, publishedAt, updatedRoute.PublishedAt)
|
||||
|
||||
messages, err := client.XRange(context.Background(), "mail:delivery_commands", "-", "+").Result()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, messages, 1)
|
||||
require.Equal(t, "notification", messages[0].Values["source"])
|
||||
require.Equal(t, "template", messages[0].Values["payload_mode"])
|
||||
require.Equal(t, "1775121700000-0/email:user:user-1", messages[0].Values["delivery_id"])
|
||||
}
|
||||
|
||||
func TestAcceptanceStoreCompleteRouteFailedReschedulesRoute(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewAcceptanceStore(client, AcceptanceConfig{
|
||||
RecordTTL: 24 * time.Hour,
|
||||
DeadLetterTTL: 72 * time.Hour,
|
||||
IdempotencyTTL: 7 * 24 * time.Hour,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
now := time.UnixMilli(1775121700000).UTC()
|
||||
input := validUserAcceptanceInput(now, 0)
|
||||
require.NoError(t, store.CreateAcceptance(context.Background(), input))
|
||||
|
||||
acquired, err := store.TryAcquireRouteLease(context.Background(), input.Notification.NotificationID, "push:user:user-1", "token-1", 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
|
||||
route, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
|
||||
failedAt := now.Add(time.Second).UTC().Truncate(time.Millisecond)
|
||||
nextAttemptAt := failedAt.Add(2 * time.Second).UTC().Truncate(time.Millisecond)
|
||||
require.NoError(t, store.CompleteRouteFailed(context.Background(), CompleteRouteFailedInput{
|
||||
ExpectedRoute: route,
|
||||
LeaseToken: "token-1",
|
||||
FailedAt: failedAt,
|
||||
NextAttemptAt: nextAttemptAt,
|
||||
FailureClassification: "gateway_stream_publish_failed",
|
||||
FailureMessage: "temporary outage",
|
||||
}))
|
||||
|
||||
updatedRoute, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.Equal(t, acceptintent.RouteStatusFailed, updatedRoute.Status)
|
||||
require.Equal(t, 1, updatedRoute.AttemptCount)
|
||||
require.Equal(t, nextAttemptAt, updatedRoute.NextAttemptAt)
|
||||
require.Equal(t, "gateway_stream_publish_failed", updatedRoute.LastErrorClassification)
|
||||
|
||||
scheduled, err := client.ZRangeWithScores(context.Background(), Keyspace{}.RouteSchedule(), 0, -1).Result()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, scheduled, 2)
|
||||
require.Contains(t, []string{
|
||||
scheduled[0].Member.(string),
|
||||
scheduled[1].Member.(string),
|
||||
}, Keyspace{}.Route(input.Notification.NotificationID, "push:user:user-1"))
|
||||
}
|
||||
|
||||
func TestAcceptanceStoreCompleteRouteDeadLetterStoresTerminalFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewAcceptanceStore(client, AcceptanceConfig{
|
||||
RecordTTL: 24 * time.Hour,
|
||||
DeadLetterTTL: 72 * time.Hour,
|
||||
IdempotencyTTL: 7 * 24 * time.Hour,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
now := time.UnixMilli(1775121700000).UTC()
|
||||
input := validUserAcceptanceInput(now, 2)
|
||||
require.NoError(t, store.CreateAcceptance(context.Background(), input))
|
||||
|
||||
acquired, err := store.TryAcquireRouteLease(context.Background(), input.Notification.NotificationID, "push:user:user-1", "token-1", 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
|
||||
route, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
|
||||
deadLetteredAt := now.Add(time.Second).UTC().Truncate(time.Millisecond)
|
||||
require.NoError(t, store.CompleteRouteDeadLetter(context.Background(), CompleteRouteDeadLetterInput{
|
||||
ExpectedRoute: route,
|
||||
LeaseToken: "token-1",
|
||||
DeadLetteredAt: deadLetteredAt,
|
||||
FailureClassification: "payload_encoding_failed",
|
||||
FailureMessage: "payload is invalid",
|
||||
}))
|
||||
|
||||
updatedRoute, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.Equal(t, acceptintent.RouteStatusDeadLetter, updatedRoute.Status)
|
||||
require.Equal(t, 3, updatedRoute.AttemptCount)
|
||||
require.Equal(t, deadLetteredAt, updatedRoute.DeadLetteredAt)
|
||||
|
||||
payload, err := client.Get(context.Background(), Keyspace{}.DeadLetter(input.Notification.NotificationID, "push:user:user-1")).Bytes()
|
||||
require.NoError(t, err)
|
||||
entry, err := UnmarshalDeadLetter(payload)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "payload_encoding_failed", entry.FailureClassification)
|
||||
require.Equal(t, 3, entry.FinalAttemptCount)
|
||||
|
||||
scheduled, err := client.ZRange(context.Background(), Keyspace{}.RouteSchedule(), 0, -1).Result()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []string{Keyspace{}.Route(input.Notification.NotificationID, "email:user:user-1")}, scheduled)
|
||||
}
|
||||
|
||||
func TestAcceptanceStoreDeadLetterIsIsolatedByChannelAndRecipient(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := miniredis.RunT(t)
|
||||
client := newTestRedisClient(t, server)
|
||||
|
||||
store, err := NewAcceptanceStore(client, AcceptanceConfig{
|
||||
RecordTTL: 24 * time.Hour,
|
||||
DeadLetterTTL: 72 * time.Hour,
|
||||
IdempotencyTTL: 7 * 24 * time.Hour,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
now := time.UnixMilli(1775121700000).UTC()
|
||||
input := validUserAcceptanceInput(now, 2)
|
||||
input.Notification.RecipientUserIDs = []string{"user-1", "user-2"}
|
||||
input.Routes = append(input.Routes,
|
||||
acceptintent.NotificationRoute{
|
||||
NotificationID: input.Notification.NotificationID,
|
||||
RouteID: "push:user:user-2",
|
||||
Channel: intentstream.ChannelPush,
|
||||
RecipientRef: "user:user-2",
|
||||
Status: acceptintent.RouteStatusPending,
|
||||
AttemptCount: 0,
|
||||
MaxAttempts: 3,
|
||||
NextAttemptAt: now,
|
||||
ResolvedEmail: "second@example.com",
|
||||
ResolvedLocale: "en",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
acceptintent.NotificationRoute{
|
||||
NotificationID: input.Notification.NotificationID,
|
||||
RouteID: "email:user:user-2",
|
||||
Channel: intentstream.ChannelEmail,
|
||||
RecipientRef: "user:user-2",
|
||||
Status: acceptintent.RouteStatusPending,
|
||||
AttemptCount: 0,
|
||||
MaxAttempts: 7,
|
||||
NextAttemptAt: now,
|
||||
ResolvedEmail: "second@example.com",
|
||||
ResolvedLocale: "en",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
)
|
||||
require.NoError(t, store.CreateAcceptance(context.Background(), input))
|
||||
|
||||
acquired, err := store.TryAcquireRouteLease(context.Background(), input.Notification.NotificationID, "push:user:user-1", "token-1", 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
|
||||
route, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
|
||||
deadLetteredAt := now.Add(time.Second).UTC().Truncate(time.Millisecond)
|
||||
require.NoError(t, store.CompleteRouteDeadLetter(context.Background(), CompleteRouteDeadLetterInput{
|
||||
ExpectedRoute: route,
|
||||
LeaseToken: "token-1",
|
||||
DeadLetteredAt: deadLetteredAt,
|
||||
FailureClassification: "gateway_stream_publish_failed",
|
||||
FailureMessage: "gateway unavailable",
|
||||
}))
|
||||
|
||||
deadLetterRoute, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, "push:user:user-1")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.Equal(t, acceptintent.RouteStatusDeadLetter, deadLetterRoute.Status)
|
||||
|
||||
for _, routeID := range []string{"email:user:user-1", "push:user:user-2", "email:user:user-2"} {
|
||||
route, found, err := store.GetRoute(context.Background(), input.Notification.NotificationID, routeID)
|
||||
require.NoError(t, err)
|
||||
require.True(t, found, "route %s should remain stored", routeID)
|
||||
require.Equal(t, acceptintent.RouteStatusPending, route.Status, "route %s should remain pending", routeID)
|
||||
}
|
||||
|
||||
scheduled, err := client.ZRange(context.Background(), Keyspace{}.RouteSchedule(), 0, -1).Result()
|
||||
require.NoError(t, err)
|
||||
require.ElementsMatch(t, []string{
|
||||
Keyspace{}.Route(input.Notification.NotificationID, "email:user:user-1"),
|
||||
Keyspace{}.Route(input.Notification.NotificationID, "push:user:user-2"),
|
||||
Keyspace{}.Route(input.Notification.NotificationID, "email:user:user-2"),
|
||||
}, scheduled)
|
||||
}
|
||||
|
||||
func validUserAcceptanceInput(now time.Time, pushAttemptCount int) acceptintent.CreateAcceptanceInput {
|
||||
return acceptintent.CreateAcceptanceInput{
|
||||
Notification: acceptintent.NotificationRecord{
|
||||
NotificationID: "1775121700000-0",
|
||||
NotificationType: intentstream.NotificationTypeGameTurnReady,
|
||||
Producer: intentstream.ProducerGameMaster,
|
||||
AudienceKind: intentstream.AudienceKindUser,
|
||||
RecipientUserIDs: []string{"user-1"},
|
||||
PayloadJSON: `{"game_id":"game-123","game_name":"Nebula Clash","turn_number":54}`,
|
||||
IdempotencyKey: "game-123:turn-54",
|
||||
RequestFingerprint: "sha256:deadbeef",
|
||||
RequestID: "request-1",
|
||||
TraceID: "trace-1",
|
||||
OccurredAt: now,
|
||||
AcceptedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
Routes: []acceptintent.NotificationRoute{
|
||||
{
|
||||
NotificationID: "1775121700000-0",
|
||||
RouteID: "push:user:user-1",
|
||||
Channel: intentstream.ChannelPush,
|
||||
RecipientRef: "user:user-1",
|
||||
Status: acceptintent.RouteStatusPending,
|
||||
AttemptCount: pushAttemptCount,
|
||||
MaxAttempts: 3,
|
||||
NextAttemptAt: now,
|
||||
ResolvedEmail: "pilot@example.com",
|
||||
ResolvedLocale: "en",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
NotificationID: "1775121700000-0",
|
||||
RouteID: "email:user:user-1",
|
||||
Channel: intentstream.ChannelEmail,
|
||||
RecipientRef: "user:user-1",
|
||||
Status: acceptintent.RouteStatusPending,
|
||||
AttemptCount: 0,
|
||||
MaxAttempts: 7,
|
||||
NextAttemptAt: now,
|
||||
ResolvedEmail: "pilot@example.com",
|
||||
ResolvedLocale: "en",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
},
|
||||
Idempotency: acceptintent.IdempotencyRecord{
|
||||
Producer: intentstream.ProducerGameMaster,
|
||||
IdempotencyKey: "game-123:turn-54",
|
||||
NotificationID: "1775121700000-0",
|
||||
RequestFingerprint: "sha256:deadbeef",
|
||||
CreatedAt: now,
|
||||
ExpiresAt: now.Add(7 * 24 * time.Hour),
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user