Ilia Denisov
468 lines
15 KiB
Go
468 lines
15 KiB
Go
package runtimejobresult_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"log/slog"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"galaxy/lobby/internal/adapters/gameinmem"
|
|
"galaxy/lobby/internal/adapters/mocks"
|
|
"galaxy/lobby/internal/adapters/streamoffsetinmem"
|
|
"galaxy/lobby/internal/domain/common"
|
|
"galaxy/lobby/internal/domain/game"
|
|
"galaxy/lobby/internal/ports"
|
|
"galaxy/lobby/internal/worker/runtimejobresult"
|
|
"galaxy/notificationintent"
|
|
|
|
"github.com/alicebob/miniredis/v2"
|
|
"github.com/redis/go-redis/v9"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/mock/gomock"
|
|
)
|
|
|
|
func silentLogger() *slog.Logger {
|
|
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
|
}
|
|
|
|
// recorder captures every call passed through the mocks. The harness
|
|
// installs a default EXPECT().AnyTimes() that funnels every call into
|
|
// the recorder so individual tests can assert on observed calls.
|
|
// Per-test error injection uses recorder.gmErr/intentsErr.
|
|
type recorder struct {
|
|
mu sync.Mutex
|
|
stopGameIDs []string
|
|
stopReasons []ports.StopReason
|
|
gmRequests []ports.RegisterGameRequest
|
|
publishedIntents []notificationintent.Intent
|
|
gmErr error
|
|
intentsErr error
|
|
}
|
|
|
|
func (r *recorder) recordStop(_ context.Context, gameID string, reason ports.StopReason) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.stopGameIDs = append(r.stopGameIDs, gameID)
|
|
r.stopReasons = append(r.stopReasons, reason)
|
|
return nil
|
|
}
|
|
|
|
func (r *recorder) recordGM(_ context.Context, request ports.RegisterGameRequest) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if r.gmErr != nil {
|
|
return r.gmErr
|
|
}
|
|
r.gmRequests = append(r.gmRequests, request)
|
|
return nil
|
|
}
|
|
|
|
func (r *recorder) recordIntent(_ context.Context, intent notificationintent.Intent) (string, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if r.intentsErr != nil {
|
|
return "", r.intentsErr
|
|
}
|
|
r.publishedIntents = append(r.publishedIntents, intent)
|
|
return "1", nil
|
|
}
|
|
|
|
func (r *recorder) stopGameIDsSnapshot() []string {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return append([]string(nil), r.stopGameIDs...)
|
|
}
|
|
|
|
func (r *recorder) stopReasonsSnapshot() []ports.StopReason {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return append([]ports.StopReason(nil), r.stopReasons...)
|
|
}
|
|
|
|
func (r *recorder) gmRequestsSnapshot() []ports.RegisterGameRequest {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return append([]ports.RegisterGameRequest(nil), r.gmRequests...)
|
|
}
|
|
|
|
func (r *recorder) publishedSnapshot() []notificationintent.Intent {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return append([]notificationintent.Intent(nil), r.publishedIntents...)
|
|
}
|
|
|
|
func (r *recorder) setGMErr(err error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.gmErr = err
|
|
}
|
|
|
|
type harness struct {
|
|
games *gameinmem.Store
|
|
runtime *mocks.MockRuntimeManager
|
|
gm *mocks.MockGMClient
|
|
intents *mocks.MockIntentPublisher
|
|
rec *recorder
|
|
offsets *streamoffsetinmem.Store
|
|
consumer *runtimejobresult.Consumer
|
|
server *miniredis.Miniredis
|
|
clientRedis *redis.Client
|
|
stream string
|
|
at time.Time
|
|
gameRecord game.Game
|
|
}
|
|
|
|
func newHarness(t *testing.T) *harness {
|
|
t.Helper()
|
|
server := miniredis.RunT(t)
|
|
clientRedis := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
|
t.Cleanup(func() { _ = clientRedis.Close() })
|
|
|
|
ctrl := gomock.NewController(t)
|
|
rec := &recorder{}
|
|
|
|
games := gameinmem.NewStore()
|
|
runtime := mocks.NewMockRuntimeManager(ctrl)
|
|
runtime.EXPECT().PublishStartJob(gomock.Any(), gomock.Any(), gomock.Any()).
|
|
DoAndReturn(func(_ context.Context, _, _ string) error { return nil }).AnyTimes()
|
|
runtime.EXPECT().PublishStopJob(gomock.Any(), gomock.Any(), gomock.Any()).
|
|
DoAndReturn(rec.recordStop).AnyTimes()
|
|
|
|
gm := mocks.NewMockGMClient(ctrl)
|
|
gm.EXPECT().RegisterGame(gomock.Any(), gomock.Any()).
|
|
DoAndReturn(rec.recordGM).AnyTimes()
|
|
gm.EXPECT().Ping(gomock.Any()).Return(nil).AnyTimes()
|
|
|
|
intents := mocks.NewMockIntentPublisher(ctrl)
|
|
intents.EXPECT().Publish(gomock.Any(), gomock.Any()).
|
|
DoAndReturn(rec.recordIntent).AnyTimes()
|
|
|
|
offsets := streamoffsetinmem.NewStore()
|
|
at := time.Date(2026, 4, 25, 13, 0, 0, 0, time.UTC)
|
|
|
|
h := &harness{
|
|
games: games,
|
|
runtime: runtime,
|
|
gm: gm,
|
|
intents: intents,
|
|
rec: rec,
|
|
offsets: offsets,
|
|
server: server,
|
|
clientRedis: clientRedis,
|
|
stream: "runtime:job_results",
|
|
at: at,
|
|
}
|
|
|
|
now := at.Add(-time.Hour)
|
|
record, err := game.New(game.NewGameInput{
|
|
GameID: common.GameID("game-w"),
|
|
GameName: "test worker game",
|
|
GameType: game.GameTypePublic,
|
|
MinPlayers: 4,
|
|
MaxPlayers: 8,
|
|
StartGapHours: 12,
|
|
StartGapPlayers: 2,
|
|
EnrollmentEndsAt: now.Add(24 * time.Hour),
|
|
TurnSchedule: "0 18 * * *",
|
|
TargetEngineVersion: "v1.0.0",
|
|
Now: now,
|
|
})
|
|
require.NoError(t, err)
|
|
record.Status = game.StatusStarting
|
|
require.NoError(t, games.Save(context.Background(), record))
|
|
h.gameRecord = record
|
|
|
|
consumer, err := runtimejobresult.NewConsumer(runtimejobresult.Config{
|
|
Client: clientRedis,
|
|
Stream: h.stream,
|
|
BlockTimeout: 100 * time.Millisecond,
|
|
Games: games,
|
|
RuntimeManager: runtime,
|
|
GMClient: gm,
|
|
Intents: intents,
|
|
OffsetStore: offsets,
|
|
Clock: func() time.Time { return at },
|
|
Logger: silentLogger(),
|
|
})
|
|
require.NoError(t, err)
|
|
h.consumer = consumer
|
|
|
|
return h
|
|
}
|
|
|
|
func successMessage(t *testing.T, h *harness, id string) redis.XMessage {
|
|
t.Helper()
|
|
return redis.XMessage{
|
|
ID: id,
|
|
Values: map[string]any{
|
|
"game_id": h.gameRecord.GameID.String(),
|
|
"outcome": "success",
|
|
"container_id": "container-1",
|
|
"engine_endpoint": "engine.local:9000",
|
|
"completed_at_ms": "1745581200000",
|
|
},
|
|
}
|
|
}
|
|
|
|
func failureMessage(t *testing.T, h *harness, id string) redis.XMessage {
|
|
t.Helper()
|
|
return redis.XMessage{
|
|
ID: id,
|
|
Values: map[string]any{
|
|
"game_id": h.gameRecord.GameID.String(),
|
|
"outcome": "failure",
|
|
"error_code": "image_pull_failed",
|
|
"error_message": "registry unreachable",
|
|
"completed_at_ms": "1745581200000",
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestNewConsumerRejectsMissingDeps(t *testing.T) {
|
|
server := miniredis.RunT(t)
|
|
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
|
t.Cleanup(func() { _ = client.Close() })
|
|
|
|
_, err := runtimejobresult.NewConsumer(runtimejobresult.Config{
|
|
Stream: "runtime:job_results",
|
|
BlockTimeout: time.Second,
|
|
})
|
|
require.Error(t, err)
|
|
|
|
_, err = runtimejobresult.NewConsumer(runtimejobresult.Config{
|
|
Client: client,
|
|
BlockTimeout: time.Second,
|
|
})
|
|
require.Error(t, err)
|
|
}
|
|
|
|
func TestHandleSuccessTransitionsToRunning(t *testing.T) {
|
|
h := newHarness(t)
|
|
h.consumer.HandleMessage(context.Background(), successMessage(t, h, "1700000000000-0"))
|
|
|
|
got, err := h.games.Get(context.Background(), h.gameRecord.GameID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, game.StatusRunning, got.Status)
|
|
require.NotNil(t, got.RuntimeBinding)
|
|
assert.Equal(t, "container-1", got.RuntimeBinding.ContainerID)
|
|
assert.Equal(t, "engine.local:9000", got.RuntimeBinding.EngineEndpoint)
|
|
assert.Equal(t, "1700000000000-0", got.RuntimeBinding.RuntimeJobID)
|
|
require.NotNil(t, got.StartedAt)
|
|
assert.True(t, got.StartedAt.Equal(h.at))
|
|
|
|
gmRequests := h.rec.gmRequestsSnapshot()
|
|
require.Len(t, gmRequests, 1)
|
|
req := gmRequests[0]
|
|
assert.Equal(t, h.gameRecord.GameID, req.GameID)
|
|
assert.Equal(t, "container-1", req.ContainerID)
|
|
assert.Equal(t, "engine.local:9000", req.EngineEndpoint)
|
|
assert.Equal(t, h.gameRecord.TargetEngineVersion, req.TargetEngineVersion)
|
|
assert.Equal(t, h.gameRecord.TurnSchedule, req.TurnSchedule)
|
|
|
|
assert.Empty(t, h.rec.stopGameIDsSnapshot())
|
|
assert.Empty(t, h.rec.publishedSnapshot())
|
|
}
|
|
|
|
func TestHandleSuccessGMUnavailableMovesToPausedAndPublishesIntent(t *testing.T) {
|
|
h := newHarness(t)
|
|
h.rec.setGMErr(ports.ErrGMUnavailable)
|
|
|
|
h.consumer.HandleMessage(context.Background(), successMessage(t, h, "1700000000001-0"))
|
|
|
|
got, err := h.games.Get(context.Background(), h.gameRecord.GameID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, game.StatusPaused, got.Status)
|
|
require.NotNil(t, got.RuntimeBinding, "binding still persisted before paused")
|
|
|
|
published := h.rec.publishedSnapshot()
|
|
require.Len(t, published, 1)
|
|
assert.Equal(t, notificationintent.NotificationTypeLobbyRuntimePausedAfterStart, published[0].NotificationType)
|
|
assert.Empty(t, h.rec.stopGameIDsSnapshot())
|
|
}
|
|
|
|
func TestHandleFailureTransitionsToStartFailed(t *testing.T) {
|
|
h := newHarness(t)
|
|
h.consumer.HandleMessage(context.Background(), failureMessage(t, h, "1700000000002-0"))
|
|
|
|
got, err := h.games.Get(context.Background(), h.gameRecord.GameID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, game.StatusStartFailed, got.Status)
|
|
assert.Nil(t, got.RuntimeBinding)
|
|
assert.Empty(t, h.rec.stopGameIDsSnapshot())
|
|
assert.Empty(t, h.rec.gmRequestsSnapshot())
|
|
assert.Empty(t, h.rec.publishedSnapshot())
|
|
}
|
|
|
|
func TestHandleSuccessOrphanContainerWhenBindingFails(t *testing.T) {
|
|
h := newHarness(t)
|
|
// Force binding update to fail by removing the game record from
|
|
// the store before the message lands.
|
|
require.NoError(t, h.games.Save(context.Background(), h.gameRecord))
|
|
failingGames := &fakeBindingFailer{Store: h.games, err: errors.New("redis tx failed")}
|
|
|
|
consumer, err := runtimejobresult.NewConsumer(runtimejobresult.Config{
|
|
Client: h.clientRedis,
|
|
Stream: h.stream,
|
|
BlockTimeout: 100 * time.Millisecond,
|
|
Games: failingGames,
|
|
RuntimeManager: h.runtime,
|
|
GMClient: h.gm,
|
|
Intents: h.intents,
|
|
OffsetStore: h.offsets,
|
|
Clock: func() time.Time { return h.at },
|
|
Logger: silentLogger(),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
consumer.HandleMessage(context.Background(), successMessage(t, h, "1700000000003-0"))
|
|
|
|
got, err := h.games.Get(context.Background(), h.gameRecord.GameID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, game.StatusStartFailed, got.Status,
|
|
"orphan path must move game to start_failed")
|
|
assert.Nil(t, got.RuntimeBinding, "binding never persisted")
|
|
|
|
assert.Equal(t, []string{h.gameRecord.GameID.String()}, h.rec.stopGameIDsSnapshot())
|
|
assert.Equal(t,
|
|
[]ports.StopReason{ports.StopReasonOrphanCleanup},
|
|
h.rec.stopReasonsSnapshot(),
|
|
"orphan path must classify the stop job as orphan_cleanup",
|
|
)
|
|
assert.Empty(t, h.rec.gmRequestsSnapshot())
|
|
assert.Empty(t, h.rec.publishedSnapshot())
|
|
}
|
|
|
|
func TestHandleSuccessReplayIsNoOp(t *testing.T) {
|
|
h := newHarness(t)
|
|
h.consumer.HandleMessage(context.Background(), successMessage(t, h, "1700000000004-0"))
|
|
require.Len(t, h.rec.gmRequestsSnapshot(), 1)
|
|
|
|
got, err := h.games.Get(context.Background(), h.gameRecord.GameID)
|
|
require.NoError(t, err)
|
|
originalUpdatedAt := got.UpdatedAt
|
|
|
|
// Replay the same event: status is already running, so the early
|
|
// status check exits before any side-effect call (no binding
|
|
// overwrite, no GM call, no transition).
|
|
h.rec.setGMErr(errors.New("must not be called again"))
|
|
h.consumer.HandleMessage(context.Background(), successMessage(t, h, "1700000000004-0"))
|
|
|
|
require.Len(t, h.rec.gmRequestsSnapshot(), 1, "GM register-game is invoked once across replays")
|
|
|
|
got, err = h.games.Get(context.Background(), h.gameRecord.GameID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, game.StatusRunning, got.Status)
|
|
assert.True(t, got.UpdatedAt.Equal(originalUpdatedAt), "no further mutations on replay")
|
|
assert.Empty(t, h.rec.publishedSnapshot())
|
|
}
|
|
|
|
func TestHandleFailureReplayIsNoOp(t *testing.T) {
|
|
h := newHarness(t)
|
|
h.consumer.HandleMessage(context.Background(), failureMessage(t, h, "1700000000005-0"))
|
|
got, err := h.games.Get(context.Background(), h.gameRecord.GameID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, game.StatusStartFailed, got.Status)
|
|
originalUpdatedAt := got.UpdatedAt
|
|
|
|
h.consumer.HandleMessage(context.Background(), failureMessage(t, h, "1700000000005-0"))
|
|
|
|
got, err = h.games.Get(context.Background(), h.gameRecord.GameID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, game.StatusStartFailed, got.Status)
|
|
assert.True(t, got.UpdatedAt.Equal(originalUpdatedAt), "no further mutations on replay")
|
|
}
|
|
|
|
func TestHandleMalformedEvents(t *testing.T) {
|
|
h := newHarness(t)
|
|
|
|
cases := []redis.XMessage{
|
|
{ID: "1-0", Values: map[string]any{"outcome": "success"}}, // missing game_id
|
|
{ID: "1-1", Values: map[string]any{"game_id": "bogus", "outcome": "success"}}, // invalid game_id format
|
|
{ID: "1-2", Values: map[string]any{"game_id": h.gameRecord.GameID.String(), "outcome": "weird"}}, // bad outcome
|
|
{ID: "1-3", Values: map[string]any{"game_id": h.gameRecord.GameID.String(), "outcome": "success"}}, // missing container_id
|
|
{ID: "1-4", Values: map[string]any{"game_id": h.gameRecord.GameID.String(), "outcome": "success", "container_id": "c"}}, // missing engine_endpoint
|
|
}
|
|
for _, msg := range cases {
|
|
h.consumer.HandleMessage(context.Background(), msg)
|
|
}
|
|
|
|
got, err := h.games.Get(context.Background(), h.gameRecord.GameID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, game.StatusStarting, got.Status, "malformed events leave game untouched")
|
|
assert.Empty(t, h.rec.stopGameIDsSnapshot())
|
|
assert.Empty(t, h.rec.gmRequestsSnapshot())
|
|
}
|
|
|
|
// fakeBindingFailer wraps gameinmem.Store and forces UpdateRuntimeBinding
|
|
// to fail; everything else delegates to the embedded store.
|
|
type fakeBindingFailer struct {
|
|
*gameinmem.Store
|
|
err error
|
|
}
|
|
|
|
func (f *fakeBindingFailer) UpdateRuntimeBinding(_ context.Context, _ ports.UpdateRuntimeBindingInput) error {
|
|
return f.err
|
|
}
|
|
|
|
var _ ports.GameStore = (*fakeBindingFailer)(nil)
|
|
|
|
func TestRunDrainsStreamUntilCancelled(t *testing.T) {
|
|
h := newHarness(t)
|
|
|
|
// Pre-publish a success message into the real miniredis stream
|
|
// before Run starts.
|
|
_, err := h.clientRedis.XAdd(context.Background(), &redis.XAddArgs{
|
|
Stream: h.stream,
|
|
Values: map[string]any{
|
|
"game_id": h.gameRecord.GameID.String(),
|
|
"outcome": "success",
|
|
"container_id": "container-2",
|
|
"engine_endpoint": "engine.local:9001",
|
|
"completed_at_ms": "1745581200000",
|
|
},
|
|
}).Result()
|
|
require.NoError(t, err)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
done <- h.consumer.Run(ctx)
|
|
}()
|
|
|
|
// Poll for the running transition; once observed cancel context.
|
|
deadline := time.Now().Add(1500 * time.Millisecond)
|
|
for time.Now().Before(deadline) {
|
|
got, err := h.games.Get(context.Background(), h.gameRecord.GameID)
|
|
require.NoError(t, err)
|
|
if got.Status == game.StatusRunning {
|
|
break
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
|
|
cancel()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatalf("consumer did not stop")
|
|
}
|
|
|
|
got, err := h.games.Get(context.Background(), h.gameRecord.GameID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, game.StatusRunning, got.Status)
|
|
require.NotNil(t, got.RuntimeBinding)
|
|
assert.Equal(t, "container-2", got.RuntimeBinding.ContainerID)
|
|
|
|
// Offset must have been persisted at least once.
|
|
id, found, err := h.offsets.Load(context.Background(), "runtime_results")
|
|
require.NoError(t, err)
|
|
assert.True(t, found)
|
|
assert.NotEmpty(t, id)
|
|
}
|