Ilia Denisov
637 lines
23 KiB
Go
637 lines
23 KiB
Go
package healtheventsconsumer_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strconv"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"galaxy/gamemaster/internal/adapters/mocks"
|
|
"galaxy/gamemaster/internal/domain/runtime"
|
|
"galaxy/gamemaster/internal/ports"
|
|
"galaxy/gamemaster/internal/telemetry"
|
|
"galaxy/gamemaster/internal/worker/healtheventsconsumer"
|
|
|
|
"github.com/alicebob/miniredis/v2"
|
|
"github.com/redis/go-redis/v9"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/mock/gomock"
|
|
)
|
|
|
|
const (
|
|
testStream = "runtime:health_events"
|
|
testLabel = "health_events"
|
|
)
|
|
|
|
func newTestTelemetry(t *testing.T) *telemetry.Runtime {
|
|
t.Helper()
|
|
tm, err := telemetry.NewWithProviders(nil, nil)
|
|
require.NoError(t, err)
|
|
return tm
|
|
}
|
|
|
|
// runningRecord builds a runtime_records row in `running` with a known
|
|
// engine_health value. The seed simplifies expectations on Get reads.
|
|
func runningRecord(gameID, health string) runtime.RuntimeRecord {
|
|
created := time.Date(2026, time.May, 1, 12, 0, 0, 0, time.UTC)
|
|
startedAt := created.Add(time.Second)
|
|
nextGen := created.Add(time.Hour)
|
|
return runtime.RuntimeRecord{
|
|
GameID: gameID,
|
|
Status: runtime.StatusRunning,
|
|
EngineEndpoint: "http://galaxy-game-" + gameID + ":8080",
|
|
CurrentImageRef: "ghcr.io/galaxy/game:v1.2.3",
|
|
CurrentEngineVersion: "v1.2.3",
|
|
TurnSchedule: "0 18 * * *",
|
|
CurrentTurn: 5,
|
|
NextGenerationAt: &nextGen,
|
|
EngineHealth: health,
|
|
CreatedAt: created,
|
|
UpdatedAt: startedAt,
|
|
StartedAt: &startedAt,
|
|
}
|
|
}
|
|
|
|
func unreachableRecord(gameID, health string) runtime.RuntimeRecord {
|
|
rec := runningRecord(gameID, health)
|
|
rec.Status = runtime.StatusEngineUnreachable
|
|
return rec
|
|
}
|
|
|
|
// withSummary returns a copy of rec with EngineHealth replaced.
|
|
func withSummary(rec runtime.RuntimeRecord, summary string) runtime.RuntimeRecord {
|
|
rec.EngineHealth = summary
|
|
return rec
|
|
}
|
|
|
|
// withStatus returns a copy of rec with Status replaced.
|
|
func withStatus(rec runtime.RuntimeRecord, status runtime.Status) runtime.RuntimeRecord {
|
|
rec.Status = status
|
|
return rec
|
|
}
|
|
|
|
// xMessage builds a redis.XMessage with the wire field layout used by
|
|
// RTM's healtheventspublisher.
|
|
func xMessage(id, gameID, eventType string, occurredAt time.Time) redis.XMessage {
|
|
return redis.XMessage{
|
|
ID: id,
|
|
Values: map[string]any{
|
|
"game_id": gameID,
|
|
"event_type": eventType,
|
|
"occurred_at_ms": strconv.FormatInt(occurredAt.UnixMilli(), 10),
|
|
"details": "{}",
|
|
},
|
|
}
|
|
}
|
|
|
|
// newWorker constructs a worker with mocked dependencies. The returned
|
|
// pointers are mocks; gomock.Controller is owned by the test.
|
|
type harness struct {
|
|
worker *healtheventsconsumer.Worker
|
|
store *mocks.MockRuntimeRecordStore
|
|
publisher *mocks.MockLobbyEventsPublisher
|
|
offsetStore *mocks.MockStreamOffsetStore
|
|
now time.Time
|
|
}
|
|
|
|
func newHarness(t *testing.T, ctrl *gomock.Controller) *harness {
|
|
t.Helper()
|
|
now := time.Date(2026, time.May, 1, 13, 0, 0, 0, time.UTC)
|
|
store := mocks.NewMockRuntimeRecordStore(ctrl)
|
|
publisher := mocks.NewMockLobbyEventsPublisher(ctrl)
|
|
offsetStore := mocks.NewMockStreamOffsetStore(ctrl)
|
|
telem := newTestTelemetry(t)
|
|
worker, err := healtheventsconsumer.NewWorker(healtheventsconsumer.Dependencies{
|
|
Client: redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"}),
|
|
Stream: testStream,
|
|
StreamLabel: testLabel,
|
|
BlockTimeout: 100 * time.Millisecond,
|
|
OffsetStore: offsetStore,
|
|
RuntimeRecords: store,
|
|
LobbyEvents: publisher,
|
|
Telemetry: telem,
|
|
Clock: func() time.Time { return now },
|
|
})
|
|
require.NoError(t, err)
|
|
return &harness{
|
|
worker: worker,
|
|
store: store,
|
|
publisher: publisher,
|
|
offsetStore: offsetStore,
|
|
now: now,
|
|
}
|
|
}
|
|
|
|
// TestNewWorkerValidates exercises every required-dep branch.
|
|
func TestNewWorkerValidates(t *testing.T) {
|
|
telem := newTestTelemetry(t)
|
|
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
|
|
cases := []struct {
|
|
name string
|
|
mut func(*healtheventsconsumer.Dependencies)
|
|
}{
|
|
{"client", func(d *healtheventsconsumer.Dependencies) { d.Client = nil }},
|
|
{"stream", func(d *healtheventsconsumer.Dependencies) { d.Stream = " " }},
|
|
{"block timeout", func(d *healtheventsconsumer.Dependencies) { d.BlockTimeout = 0 }},
|
|
{"offset store", func(d *healtheventsconsumer.Dependencies) { d.OffsetStore = nil }},
|
|
{"runtime records", func(d *healtheventsconsumer.Dependencies) { d.RuntimeRecords = nil }},
|
|
{"lobby events", func(d *healtheventsconsumer.Dependencies) { d.LobbyEvents = nil }},
|
|
{"telemetry", func(d *healtheventsconsumer.Dependencies) { d.Telemetry = nil }},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
deps := healtheventsconsumer.Dependencies{
|
|
Client: client,
|
|
Stream: testStream,
|
|
StreamLabel: testLabel,
|
|
BlockTimeout: time.Second,
|
|
OffsetStore: mocks.NewMockStreamOffsetStore(ctrl),
|
|
RuntimeRecords: mocks.NewMockRuntimeRecordStore(ctrl),
|
|
LobbyEvents: mocks.NewMockLobbyEventsPublisher(ctrl),
|
|
Telemetry: telem,
|
|
}
|
|
tc.mut(&deps)
|
|
worker, err := healtheventsconsumer.NewWorker(deps)
|
|
require.Error(t, err)
|
|
require.Nil(t, worker)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNewWorkerDefaultsLabel(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
telem := newTestTelemetry(t)
|
|
worker, err := healtheventsconsumer.NewWorker(healtheventsconsumer.Dependencies{
|
|
Client: redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"}),
|
|
Stream: testStream,
|
|
StreamLabel: "",
|
|
BlockTimeout: time.Second,
|
|
OffsetStore: mocks.NewMockStreamOffsetStore(ctrl),
|
|
RuntimeRecords: mocks.NewMockRuntimeRecordStore(ctrl),
|
|
LobbyEvents: mocks.NewMockLobbyEventsPublisher(ctrl),
|
|
Telemetry: telem,
|
|
})
|
|
require.NoError(t, err)
|
|
require.NotNil(t, worker)
|
|
}
|
|
|
|
// TestHandleMessage_ContainerExited covers a terminal event from a
|
|
// healthy `running` record: status transitions to engine_unreachable
|
|
// and a snapshot is published.
|
|
func TestHandleMessage_ContainerExited(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
gameID := "game-001"
|
|
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(runningRecord(gameID, "healthy"), nil)
|
|
h.store.EXPECT().UpdateStatus(gomock.Any(), gomock.Any()).DoAndReturn(
|
|
func(_ context.Context, input ports.UpdateStatusInput) error {
|
|
require.Equal(t, runtime.StatusRunning, input.ExpectedFrom)
|
|
require.Equal(t, runtime.StatusEngineUnreachable, input.To)
|
|
require.NotNil(t, input.EngineHealthSummary)
|
|
require.Equal(t, "exited", *input.EngineHealthSummary)
|
|
return nil
|
|
},
|
|
)
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(
|
|
withStatus(withSummary(runningRecord(gameID, "healthy"), "exited"), runtime.StatusEngineUnreachable),
|
|
nil,
|
|
)
|
|
h.publisher.EXPECT().PublishSnapshotUpdate(gomock.Any(), gomock.Any()).DoAndReturn(
|
|
func(_ context.Context, snap ports.RuntimeSnapshotUpdate) error {
|
|
assert.Equal(t, gameID, snap.GameID)
|
|
assert.Equal(t, runtime.StatusEngineUnreachable, snap.RuntimeStatus)
|
|
assert.Equal(t, "exited", snap.EngineHealthSummary)
|
|
assert.Nil(t, snap.PlayerTurnStats)
|
|
assert.Equal(t, h.now, snap.OccurredAt)
|
|
return nil
|
|
},
|
|
)
|
|
|
|
advance := h.worker.HandleMessage(context.Background(), xMessage("0-1", gameID, "container_exited", h.now))
|
|
assert.True(t, advance)
|
|
}
|
|
|
|
// TestHandleMessage_ProbeRecovered_Recovers demonstrates the symmetric
|
|
// recovery: engine_unreachable → running, summary set to healthy.
|
|
func TestHandleMessage_ProbeRecovered_Recovers(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
gameID := "game-001"
|
|
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(unreachableRecord(gameID, "exited"), nil)
|
|
h.store.EXPECT().UpdateStatus(gomock.Any(), gomock.Any()).DoAndReturn(
|
|
func(_ context.Context, input ports.UpdateStatusInput) error {
|
|
require.Equal(t, runtime.StatusEngineUnreachable, input.ExpectedFrom)
|
|
require.Equal(t, runtime.StatusRunning, input.To)
|
|
require.NotNil(t, input.EngineHealthSummary)
|
|
require.Equal(t, "healthy", *input.EngineHealthSummary)
|
|
return nil
|
|
},
|
|
)
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(
|
|
withStatus(withSummary(unreachableRecord(gameID, "exited"), "healthy"), runtime.StatusRunning),
|
|
nil,
|
|
)
|
|
h.publisher.EXPECT().PublishSnapshotUpdate(gomock.Any(), gomock.Any()).DoAndReturn(
|
|
func(_ context.Context, snap ports.RuntimeSnapshotUpdate) error {
|
|
assert.Equal(t, runtime.StatusRunning, snap.RuntimeStatus)
|
|
assert.Equal(t, "healthy", snap.EngineHealthSummary)
|
|
return nil
|
|
},
|
|
)
|
|
|
|
advance := h.worker.HandleMessage(context.Background(), xMessage("0-1", gameID, "probe_recovered", h.now))
|
|
assert.True(t, advance)
|
|
}
|
|
|
|
// TestHandleMessage_ContainerStarted_NoTransition asserts that
|
|
// container_started writes summary `healthy` without status mutation.
|
|
func TestHandleMessage_ContainerStarted_NoTransition(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
gameID := "game-001"
|
|
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(runningRecord(gameID, ""), nil)
|
|
h.store.EXPECT().UpdateEngineHealth(gomock.Any(), gomock.Any()).DoAndReturn(
|
|
func(_ context.Context, input ports.UpdateEngineHealthInput) error {
|
|
assert.Equal(t, gameID, input.GameID)
|
|
assert.Equal(t, "healthy", input.EngineHealthSummary)
|
|
return nil
|
|
},
|
|
)
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(withSummary(runningRecord(gameID, ""), "healthy"), nil)
|
|
h.publisher.EXPECT().PublishSnapshotUpdate(gomock.Any(), gomock.Any()).Return(nil)
|
|
|
|
advance := h.worker.HandleMessage(context.Background(), xMessage("0-1", gameID, "container_started", h.now))
|
|
assert.True(t, advance)
|
|
}
|
|
|
|
// TestHandleMessage_ProbeFailed covers the non-transitional path:
|
|
// summary is updated; status stays running.
|
|
func TestHandleMessage_ProbeFailed(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
gameID := "game-001"
|
|
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(runningRecord(gameID, "healthy"), nil)
|
|
h.store.EXPECT().UpdateEngineHealth(gomock.Any(), gomock.Any()).Return(nil)
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(withSummary(runningRecord(gameID, "healthy"), "probe_failed"), nil)
|
|
h.publisher.EXPECT().PublishSnapshotUpdate(gomock.Any(), gomock.Any()).DoAndReturn(
|
|
func(_ context.Context, snap ports.RuntimeSnapshotUpdate) error {
|
|
assert.Equal(t, runtime.StatusRunning, snap.RuntimeStatus)
|
|
assert.Equal(t, "probe_failed", snap.EngineHealthSummary)
|
|
return nil
|
|
},
|
|
)
|
|
|
|
advance := h.worker.HandleMessage(context.Background(), xMessage("0-1", gameID, "probe_failed", h.now))
|
|
assert.True(t, advance)
|
|
}
|
|
|
|
// TestHandleMessage_FallsBackOnCASConflict — record is in
|
|
// generation_in_progress (not running); CAS rejects with ErrConflict and
|
|
// the worker falls back to UpdateEngineHealth + publishes a snapshot
|
|
// because the summary changed.
|
|
func TestHandleMessage_FallsBackOnCASConflict(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
gameID := "game-001"
|
|
|
|
current := withStatus(runningRecord(gameID, "healthy"), runtime.StatusGenerationInProgress)
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(current, nil)
|
|
h.store.EXPECT().UpdateStatus(gomock.Any(), gomock.Any()).Return(runtime.ErrConflict)
|
|
h.store.EXPECT().UpdateEngineHealth(gomock.Any(), gomock.Any()).DoAndReturn(
|
|
func(_ context.Context, input ports.UpdateEngineHealthInput) error {
|
|
assert.Equal(t, "oom", input.EngineHealthSummary)
|
|
return nil
|
|
},
|
|
)
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(withSummary(current, "oom"), nil)
|
|
h.publisher.EXPECT().PublishSnapshotUpdate(gomock.Any(), gomock.Any()).DoAndReturn(
|
|
func(_ context.Context, snap ports.RuntimeSnapshotUpdate) error {
|
|
assert.Equal(t, runtime.StatusGenerationInProgress, snap.RuntimeStatus,
|
|
"status must reflect the unchanged record after fallback")
|
|
assert.Equal(t, "oom", snap.EngineHealthSummary)
|
|
return nil
|
|
},
|
|
)
|
|
|
|
advance := h.worker.HandleMessage(context.Background(), xMessage("0-1", gameID, "container_oom", h.now))
|
|
assert.True(t, advance)
|
|
}
|
|
|
|
// TestHandleMessage_DebouncesUnchangedSummary — two consecutive
|
|
// probe_failed events for the same game yield exactly one snapshot
|
|
// publication.
|
|
func TestHandleMessage_DebouncesUnchangedSummary(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
gameID := "game-001"
|
|
|
|
// First event: store update + reload + publish.
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(runningRecord(gameID, "healthy"), nil)
|
|
h.store.EXPECT().UpdateEngineHealth(gomock.Any(), gomock.Any()).Return(nil)
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(withSummary(runningRecord(gameID, "healthy"), "probe_failed"), nil)
|
|
h.publisher.EXPECT().PublishSnapshotUpdate(gomock.Any(), gomock.Any()).Return(nil)
|
|
|
|
// Second event: store update happens, but no second Get and no
|
|
// publication since the summary is unchanged.
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(withSummary(runningRecord(gameID, "probe_failed"), "probe_failed"), nil)
|
|
h.store.EXPECT().UpdateEngineHealth(gomock.Any(), gomock.Any()).Return(nil)
|
|
|
|
ctx := context.Background()
|
|
require.True(t, h.worker.HandleMessage(ctx, xMessage("0-1", gameID, "probe_failed", h.now)))
|
|
require.True(t, h.worker.HandleMessage(ctx, xMessage("0-2", gameID, "probe_failed", h.now)))
|
|
}
|
|
|
|
// TestHandleMessage_OrphanGameID — Get returns ErrNotFound, no further
|
|
// store calls, no publish, offset advances.
|
|
func TestHandleMessage_OrphanGameID(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
gameID := "missing-001"
|
|
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(runtime.RuntimeRecord{}, runtime.ErrNotFound)
|
|
|
|
advance := h.worker.HandleMessage(context.Background(), xMessage("0-1", gameID, "probe_failed", h.now))
|
|
assert.True(t, advance)
|
|
}
|
|
|
|
// TestHandleMessage_UnknownEventType — unrecognised event type yields
|
|
// no store calls and no publication, but offset advances.
|
|
func TestHandleMessage_UnknownEventType(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
|
|
advance := h.worker.HandleMessage(context.Background(), xMessage("0-1", "game-001", "future_event", h.now))
|
|
assert.True(t, advance)
|
|
}
|
|
|
|
// TestHandleMessage_MalformedOccurredAtMS — malformed wire payload is
|
|
// logged + skipped without store calls.
|
|
func TestHandleMessage_MalformedOccurredAtMS(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
|
|
msg := redis.XMessage{
|
|
ID: "0-1",
|
|
Values: map[string]any{
|
|
"game_id": "game-001",
|
|
"event_type": "probe_failed",
|
|
"occurred_at_ms": "not-a-number",
|
|
},
|
|
}
|
|
advance := h.worker.HandleMessage(context.Background(), msg)
|
|
assert.True(t, advance)
|
|
}
|
|
|
|
// TestHandleMessage_MissingFields — missing required wire field is
|
|
// logged + skipped.
|
|
func TestHandleMessage_MissingFields(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
msg redis.XMessage
|
|
}{
|
|
{"missing game_id", redis.XMessage{ID: "0-1", Values: map[string]any{"event_type": "probe_failed", "occurred_at_ms": "1"}}},
|
|
{"missing event_type", redis.XMessage{ID: "0-1", Values: map[string]any{"game_id": "g", "occurred_at_ms": "1"}}},
|
|
{"missing occurred_at_ms", redis.XMessage{ID: "0-1", Values: map[string]any{"game_id": "g", "event_type": "probe_failed"}}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
advance := h.worker.HandleMessage(context.Background(), tc.msg)
|
|
assert.True(t, advance)
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestHandleMessage_PublishErrorAdvancesOffset — a publisher error is
|
|
// logged and absorbed; the offset still advances so a transient hiccup
|
|
// does not stall the consumer.
|
|
func TestHandleMessage_PublishErrorAdvancesOffset(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
gameID := "game-001"
|
|
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(runningRecord(gameID, "healthy"), nil)
|
|
h.store.EXPECT().UpdateEngineHealth(gomock.Any(), gomock.Any()).Return(nil)
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(withSummary(runningRecord(gameID, "healthy"), "probe_failed"), nil)
|
|
h.publisher.EXPECT().PublishSnapshotUpdate(gomock.Any(), gomock.Any()).Return(errors.New("redis down"))
|
|
|
|
advance := h.worker.HandleMessage(context.Background(), xMessage("0-1", gameID, "probe_failed", h.now))
|
|
assert.True(t, advance)
|
|
}
|
|
|
|
// TestHandleMessage_AllEventTypes_RouteSummaries asserts the event-type
|
|
// → summary mapping for the four non-CAS event types, plus that
|
|
// container_started is non-CAS too. The CAS variants are covered by
|
|
// dedicated tests above.
|
|
func TestHandleMessage_AllEventTypes_RouteSummaries(t *testing.T) {
|
|
type expectation struct {
|
|
eventType string
|
|
wantSummary string
|
|
wantsCASCall bool
|
|
}
|
|
cases := []expectation{
|
|
{"container_started", "healthy", false},
|
|
{"probe_failed", "probe_failed", false},
|
|
{"inspect_unhealthy", "inspect_unhealthy", false},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.eventType, func(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
gameID := "game-001"
|
|
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(runningRecord(gameID, ""), nil)
|
|
h.store.EXPECT().UpdateEngineHealth(gomock.Any(), gomock.Any()).DoAndReturn(
|
|
func(_ context.Context, input ports.UpdateEngineHealthInput) error {
|
|
assert.Equal(t, tc.wantSummary, input.EngineHealthSummary)
|
|
return nil
|
|
},
|
|
)
|
|
h.store.EXPECT().Get(gomock.Any(), gameID).Return(withSummary(runningRecord(gameID, ""), tc.wantSummary), nil)
|
|
h.publisher.EXPECT().PublishSnapshotUpdate(gomock.Any(), gomock.Any()).Return(nil)
|
|
|
|
advance := h.worker.HandleMessage(context.Background(), xMessage("0-1", gameID, tc.eventType, h.now))
|
|
assert.True(t, advance)
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestRun_LoadsOffsetAndAdvances drives a real XREAD loop against a
|
|
// miniredis instance. After XADD-ing one entry and observing the loop
|
|
// exit on context cancellation, the persisted offset must equal the
|
|
// consumed entry's ID.
|
|
func TestRun_LoadsOffsetAndAdvances(t *testing.T) {
|
|
server := miniredis.RunT(t)
|
|
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
|
t.Cleanup(func() { _ = client.Close() })
|
|
|
|
ctrl := gomock.NewController(t)
|
|
store := mocks.NewMockRuntimeRecordStore(ctrl)
|
|
publisher := mocks.NewMockLobbyEventsPublisher(ctrl)
|
|
telem := newTestTelemetry(t)
|
|
|
|
gameID := "game-001"
|
|
rec := runningRecord(gameID, "healthy")
|
|
|
|
var (
|
|
mu sync.Mutex
|
|
offset string
|
|
offsetSet bool
|
|
)
|
|
offsetStore := mocks.NewMockStreamOffsetStore(ctrl)
|
|
offsetStore.EXPECT().Load(gomock.Any(), testLabel).Return("", false, nil)
|
|
offsetStore.EXPECT().Save(gomock.Any(), testLabel, gomock.Any()).DoAndReturn(
|
|
func(_ context.Context, _ string, entryID string) error {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
offset = entryID
|
|
offsetSet = true
|
|
return nil
|
|
},
|
|
).MinTimes(1)
|
|
|
|
store.EXPECT().Get(gomock.Any(), gameID).Return(rec, nil)
|
|
store.EXPECT().UpdateEngineHealth(gomock.Any(), gomock.Any()).Return(nil)
|
|
store.EXPECT().Get(gomock.Any(), gameID).Return(withSummary(rec, "probe_failed"), nil)
|
|
publisher.EXPECT().PublishSnapshotUpdate(gomock.Any(), gomock.Any()).Return(nil)
|
|
|
|
worker, err := healtheventsconsumer.NewWorker(healtheventsconsumer.Dependencies{
|
|
Client: client,
|
|
Stream: testStream,
|
|
StreamLabel: testLabel,
|
|
BlockTimeout: 100 * time.Millisecond,
|
|
OffsetStore: offsetStore,
|
|
RuntimeRecords: store,
|
|
LobbyEvents: publisher,
|
|
Telemetry: telem,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
occurredMS := strconv.FormatInt(time.Date(2026, time.May, 1, 12, 0, 0, 0, time.UTC).UnixMilli(), 10)
|
|
entryID, err := client.XAdd(context.Background(), &redis.XAddArgs{
|
|
Stream: testStream,
|
|
Values: map[string]any{
|
|
"game_id": gameID,
|
|
"event_type": "probe_failed",
|
|
"occurred_at_ms": occurredMS,
|
|
"details": "{}",
|
|
},
|
|
}).Result()
|
|
require.NoError(t, err)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- worker.Run(ctx) }()
|
|
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
mu.Lock()
|
|
set := offsetSet
|
|
mu.Unlock()
|
|
if set {
|
|
break
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
|
|
cancel()
|
|
select {
|
|
case err := <-done:
|
|
assert.True(t, errors.Is(err, context.Canceled), "run must exit with context.Canceled, got %v", err)
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("worker did not exit within deadline")
|
|
}
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
require.True(t, offsetSet, "offset must be persisted at least once")
|
|
assert.Equal(t, entryID, offset)
|
|
}
|
|
|
|
// TestRun_ContextCancel — Run returns context.Canceled on cancel even
|
|
// when no stream entry is available.
|
|
func TestRun_ContextCancel(t *testing.T) {
|
|
server := miniredis.RunT(t)
|
|
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
|
t.Cleanup(func() { _ = client.Close() })
|
|
|
|
ctrl := gomock.NewController(t)
|
|
store := mocks.NewMockRuntimeRecordStore(ctrl)
|
|
publisher := mocks.NewMockLobbyEventsPublisher(ctrl)
|
|
offsetStore := mocks.NewMockStreamOffsetStore(ctrl)
|
|
offsetStore.EXPECT().Load(gomock.Any(), testLabel).Return("0-0", true, nil)
|
|
|
|
worker, err := healtheventsconsumer.NewWorker(healtheventsconsumer.Dependencies{
|
|
Client: client,
|
|
Stream: testStream,
|
|
StreamLabel: testLabel,
|
|
BlockTimeout: 50 * time.Millisecond,
|
|
OffsetStore: offsetStore,
|
|
RuntimeRecords: store,
|
|
LobbyEvents: publisher,
|
|
Telemetry: newTestTelemetry(t),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- worker.Run(ctx) }()
|
|
|
|
time.Sleep(150 * time.Millisecond)
|
|
cancel()
|
|
select {
|
|
case err := <-done:
|
|
assert.True(t, errors.Is(err, context.Canceled), "want context.Canceled, got %v", err)
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("worker did not exit within deadline")
|
|
}
|
|
}
|
|
|
|
// TestRun_FailsOnOffsetLoadError covers the bootstrap failure: a load
|
|
// error is fatal and surfaces from Run.
|
|
func TestRun_FailsOnOffsetLoadError(t *testing.T) {
|
|
server := miniredis.RunT(t)
|
|
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
|
t.Cleanup(func() { _ = client.Close() })
|
|
|
|
ctrl := gomock.NewController(t)
|
|
offsetStore := mocks.NewMockStreamOffsetStore(ctrl)
|
|
offsetStore.EXPECT().Load(gomock.Any(), testLabel).Return("", false, errors.New("redis down"))
|
|
|
|
worker, err := healtheventsconsumer.NewWorker(healtheventsconsumer.Dependencies{
|
|
Client: client,
|
|
Stream: testStream,
|
|
StreamLabel: testLabel,
|
|
BlockTimeout: 50 * time.Millisecond,
|
|
OffsetStore: offsetStore,
|
|
RuntimeRecords: mocks.NewMockRuntimeRecordStore(ctrl),
|
|
LobbyEvents: mocks.NewMockLobbyEventsPublisher(ctrl),
|
|
Telemetry: newTestTelemetry(t),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
err = worker.Run(context.Background())
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "load offset")
|
|
}
|
|
|
|
// TestShutdown_Noop confirms Shutdown returns nil for a non-nil ctx
|
|
// and rejects a nil one.
|
|
func TestShutdown_Noop(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
h := newHarness(t, ctrl)
|
|
require.NoError(t, h.worker.Shutdown(context.Background()))
|
|
|
|
//nolint:staticcheck // Deliberate nil context to verify guard.
|
|
require.Error(t, h.worker.Shutdown(nil))
|
|
}
|