358 lines
10 KiB
Go
358 lines
10 KiB
Go
package stopjobsconsumer_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"log/slog"
|
|
"strconv"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"galaxy/rtmanager/internal/domain/operation"
|
|
"galaxy/rtmanager/internal/domain/runtime"
|
|
"galaxy/rtmanager/internal/ports"
|
|
"galaxy/rtmanager/internal/service/startruntime"
|
|
"galaxy/rtmanager/internal/service/stopruntime"
|
|
"galaxy/rtmanager/internal/worker/stopjobsconsumer"
|
|
|
|
"github.com/alicebob/miniredis/v2"
|
|
"github.com/redis/go-redis/v9"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func silentLogger() *slog.Logger {
|
|
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
|
}
|
|
|
|
type fakeStopService struct {
|
|
mu sync.Mutex
|
|
inputs []stopruntime.Input
|
|
result stopruntime.Result
|
|
err error
|
|
}
|
|
|
|
func (s *fakeStopService) Handle(_ context.Context, input stopruntime.Input) (stopruntime.Result, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.inputs = append(s.inputs, input)
|
|
return s.result, s.err
|
|
}
|
|
|
|
func (s *fakeStopService) Inputs() []stopruntime.Input {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
out := make([]stopruntime.Input, len(s.inputs))
|
|
copy(out, s.inputs)
|
|
return out
|
|
}
|
|
|
|
type fakeJobResults struct {
|
|
mu sync.Mutex
|
|
published []ports.JobResult
|
|
publishErr error
|
|
}
|
|
|
|
func (s *fakeJobResults) Publish(_ context.Context, result ports.JobResult) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.publishErr != nil {
|
|
return s.publishErr
|
|
}
|
|
s.published = append(s.published, result)
|
|
return nil
|
|
}
|
|
|
|
func (s *fakeJobResults) Published() []ports.JobResult {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
out := make([]ports.JobResult, len(s.published))
|
|
copy(out, s.published)
|
|
return out
|
|
}
|
|
|
|
type fakeOffsetStore struct {
|
|
mu sync.Mutex
|
|
offsets map[string]string
|
|
}
|
|
|
|
func newFakeOffsetStore() *fakeOffsetStore {
|
|
return &fakeOffsetStore{offsets: map[string]string{}}
|
|
}
|
|
|
|
func (s *fakeOffsetStore) Load(_ context.Context, label string) (string, bool, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
value, ok := s.offsets[label]
|
|
return value, ok, nil
|
|
}
|
|
|
|
func (s *fakeOffsetStore) Save(_ context.Context, label, entryID string) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.offsets[label] = entryID
|
|
return nil
|
|
}
|
|
|
|
func (s *fakeOffsetStore) Get(label string) (string, bool) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
value, ok := s.offsets[label]
|
|
return value, ok
|
|
}
|
|
|
|
type harness struct {
|
|
consumer *stopjobsconsumer.Consumer
|
|
stops *fakeStopService
|
|
results *fakeJobResults
|
|
offsets *fakeOffsetStore
|
|
stream string
|
|
server *miniredis.Miniredis
|
|
client *redis.Client
|
|
}
|
|
|
|
func newHarness(t *testing.T) *harness {
|
|
t.Helper()
|
|
server := miniredis.RunT(t)
|
|
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
|
t.Cleanup(func() { _ = client.Close() })
|
|
|
|
stops := &fakeStopService{}
|
|
results := &fakeJobResults{}
|
|
offsets := newFakeOffsetStore()
|
|
stream := "runtime:stop_jobs"
|
|
|
|
consumer, err := stopjobsconsumer.NewConsumer(stopjobsconsumer.Config{
|
|
Client: client,
|
|
Stream: stream,
|
|
BlockTimeout: 50 * time.Millisecond,
|
|
StopService: stops,
|
|
JobResults: results,
|
|
OffsetStore: offsets,
|
|
Logger: silentLogger(),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
return &harness{
|
|
consumer: consumer,
|
|
stops: stops,
|
|
results: results,
|
|
offsets: offsets,
|
|
stream: stream,
|
|
server: server,
|
|
client: client,
|
|
}
|
|
}
|
|
|
|
func stopMessage(id, gameID, reason string, requestedAtMS int64) redis.XMessage {
|
|
return redis.XMessage{
|
|
ID: id,
|
|
Values: map[string]any{
|
|
"game_id": gameID,
|
|
"reason": reason,
|
|
"requested_at_ms": strconv.FormatInt(requestedAtMS, 10),
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestNewConsumerRejectsMissingDeps(t *testing.T) {
|
|
server := miniredis.RunT(t)
|
|
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
|
t.Cleanup(func() { _ = client.Close() })
|
|
|
|
cases := []stopjobsconsumer.Config{
|
|
{},
|
|
{Client: client},
|
|
{Client: client, Stream: "runtime:stop_jobs"},
|
|
{Client: client, Stream: "runtime:stop_jobs", BlockTimeout: time.Second},
|
|
{Client: client, Stream: "runtime:stop_jobs", BlockTimeout: time.Second, StopService: &fakeStopService{}},
|
|
{Client: client, Stream: "runtime:stop_jobs", BlockTimeout: time.Second, StopService: &fakeStopService{}, JobResults: &fakeJobResults{}},
|
|
}
|
|
for index, cfg := range cases {
|
|
_, err := stopjobsconsumer.NewConsumer(cfg)
|
|
require.Errorf(t, err, "case %d should fail", index)
|
|
}
|
|
}
|
|
|
|
func TestHandleMessageSuccessPublishesSuccessResult(t *testing.T) {
|
|
h := newHarness(t)
|
|
h.stops.result = stopruntime.Result{
|
|
Record: runtime.RuntimeRecord{
|
|
GameID: "game-1",
|
|
Status: runtime.StatusStopped,
|
|
CurrentContainerID: "c-1",
|
|
CurrentImageRef: "galaxy/game:1.0.0",
|
|
EngineEndpoint: "http://galaxy-game-game-1:8080",
|
|
},
|
|
Outcome: operation.OutcomeSuccess,
|
|
}
|
|
|
|
h.consumer.HandleMessage(context.Background(), stopMessage("100-0", "game-1", "cancelled", 1700))
|
|
|
|
inputs := h.stops.Inputs()
|
|
require.Len(t, inputs, 1)
|
|
assert.Equal(t, "game-1", inputs[0].GameID)
|
|
assert.Equal(t, stopruntime.StopReasonCancelled, inputs[0].Reason)
|
|
assert.Equal(t, operation.OpSourceLobbyStream, inputs[0].OpSource)
|
|
assert.Equal(t, "100-0", inputs[0].SourceRef)
|
|
|
|
published := h.results.Published()
|
|
require.Len(t, published, 1)
|
|
assert.Equal(t, ports.JobResult{
|
|
GameID: "game-1",
|
|
Outcome: ports.JobOutcomeSuccess,
|
|
ContainerID: "c-1",
|
|
EngineEndpoint: "http://galaxy-game-game-1:8080",
|
|
}, published[0])
|
|
}
|
|
|
|
func TestHandleMessageFailureNotFoundPublishesFailureResult(t *testing.T) {
|
|
h := newHarness(t)
|
|
h.stops.result = stopruntime.Result{
|
|
Outcome: operation.OutcomeFailure,
|
|
ErrorCode: startruntime.ErrorCodeNotFound,
|
|
ErrorMessage: "runtime record for game \"game-2\" does not exist",
|
|
}
|
|
|
|
h.consumer.HandleMessage(context.Background(), stopMessage("101-0", "game-2", "admin_request", 1700))
|
|
|
|
published := h.results.Published()
|
|
require.Len(t, published, 1)
|
|
assert.Equal(t, ports.JobResult{
|
|
GameID: "game-2",
|
|
Outcome: ports.JobOutcomeFailure,
|
|
ErrorCode: "not_found",
|
|
ErrorMessage: "runtime record for game \"game-2\" does not exist",
|
|
}, published[0])
|
|
}
|
|
|
|
func TestHandleMessageReplayNoOpForRemovedRecordHasEmptyContainerAndEndpoint(t *testing.T) {
|
|
h := newHarness(t)
|
|
h.stops.result = stopruntime.Result{
|
|
Record: runtime.RuntimeRecord{
|
|
GameID: "game-3",
|
|
Status: runtime.StatusRemoved,
|
|
CurrentContainerID: "",
|
|
EngineEndpoint: "http://galaxy-game-game-3:8080",
|
|
},
|
|
Outcome: operation.OutcomeSuccess,
|
|
ErrorCode: startruntime.ErrorCodeReplayNoOp,
|
|
}
|
|
|
|
h.consumer.HandleMessage(context.Background(), stopMessage("102-0", "game-3", "finished", 1700))
|
|
|
|
published := h.results.Published()
|
|
require.Len(t, published, 1)
|
|
assert.Equal(t, ports.JobResult{
|
|
GameID: "game-3",
|
|
Outcome: ports.JobOutcomeSuccess,
|
|
ContainerID: "",
|
|
EngineEndpoint: "http://galaxy-game-game-3:8080",
|
|
ErrorCode: "replay_no_op",
|
|
}, published[0])
|
|
}
|
|
|
|
func TestHandleMessageMalformedEnvelopesAreAbsorbed(t *testing.T) {
|
|
h := newHarness(t)
|
|
|
|
cases := []redis.XMessage{
|
|
{ID: "200-0", Values: map[string]any{"reason": "cancelled", "requested_at_ms": "1"}},
|
|
{ID: "200-1", Values: map[string]any{"game_id": "game-x", "requested_at_ms": "1"}},
|
|
{ID: "200-2", Values: map[string]any{"game_id": "game-x", "reason": " ", "requested_at_ms": "1"}},
|
|
{ID: "200-3", Values: map[string]any{"game_id": "game-x", "reason": "not_a_known_reason", "requested_at_ms": "1"}},
|
|
{ID: "200-4", Values: map[string]any{"game_id": "game-x", "reason": "cancelled", "requested_at_ms": "abc"}},
|
|
}
|
|
for _, msg := range cases {
|
|
h.consumer.HandleMessage(context.Background(), msg)
|
|
}
|
|
|
|
assert.Empty(t, h.stops.Inputs(), "malformed envelopes must not reach the stop service")
|
|
assert.Empty(t, h.results.Published(), "malformed envelopes must not produce job results")
|
|
}
|
|
|
|
func TestHandleMessagePublishFailureIsAbsorbed(t *testing.T) {
|
|
h := newHarness(t)
|
|
h.stops.result = stopruntime.Result{Outcome: operation.OutcomeFailure, ErrorCode: "internal_error"}
|
|
h.results.publishErr = errors.New("redis transient")
|
|
|
|
h.consumer.HandleMessage(context.Background(), stopMessage("300-0", "game-x", "cancelled", 1700))
|
|
|
|
require.Len(t, h.stops.Inputs(), 1, "service still runs even when publish fails")
|
|
}
|
|
|
|
func TestHandleMessageGoLevelErrorIsAbsorbed(t *testing.T) {
|
|
h := newHarness(t)
|
|
h.stops.err = errors.New("nil ctx")
|
|
|
|
h.consumer.HandleMessage(context.Background(), stopMessage("400-0", "game-y", "cancelled", 1700))
|
|
|
|
assert.Empty(t, h.results.Published(), "go-level service errors must not surface as job results")
|
|
}
|
|
|
|
func TestRunAdvancesOffsetPerMessage(t *testing.T) {
|
|
h := newHarness(t)
|
|
h.stops.result = stopruntime.Result{
|
|
Record: runtime.RuntimeRecord{
|
|
GameID: "game-5",
|
|
Status: runtime.StatusStopped,
|
|
CurrentContainerID: "c-5",
|
|
EngineEndpoint: "http://galaxy-game-game-5:8080",
|
|
},
|
|
Outcome: operation.OutcomeSuccess,
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
done := make(chan error, 1)
|
|
go func() { done <- h.consumer.Run(ctx) }()
|
|
|
|
mustXAdd(t, h.client, h.stream, "game-5", "cancelled", 1)
|
|
mustXAdd(t, h.client, h.stream, "game-5", "finished", 2)
|
|
|
|
require.Eventually(t, func() bool {
|
|
return len(h.results.Published()) == 2
|
|
}, time.Second, 10*time.Millisecond, "consumer must produce one job result per envelope")
|
|
|
|
cancel()
|
|
require.Eventually(t, func() bool {
|
|
select {
|
|
case <-done:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}, time.Second, 10*time.Millisecond, "Run must exit after context cancel")
|
|
|
|
id, ok := h.offsets.Get("stopjobs")
|
|
require.True(t, ok, "offset must be persisted after the run loop processed messages")
|
|
assert.NotEmpty(t, id, "offset entry id must not be empty")
|
|
}
|
|
|
|
func TestRunExitsImmediatelyOnAlreadyCancelledContext(t *testing.T) {
|
|
h := newHarness(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
err := h.consumer.Run(ctx)
|
|
require.ErrorIs(t, err, context.Canceled)
|
|
assert.Empty(t, h.stops.Inputs())
|
|
assert.Empty(t, h.results.Published())
|
|
}
|
|
|
|
func mustXAdd(t *testing.T, client *redis.Client, stream, gameID, reason string, requestedAtMS int64) string {
|
|
t.Helper()
|
|
id, err := client.XAdd(context.Background(), &redis.XAddArgs{
|
|
Stream: stream,
|
|
Values: map[string]any{
|
|
"game_id": gameID,
|
|
"reason": reason,
|
|
"requested_at_ms": strconv.FormatInt(requestedAtMS, 10),
|
|
},
|
|
}).Result()
|
|
require.NoError(t, err)
|
|
return id
|
|
}
|