feat: runtime manager

This commit is contained in:
Ilia Denisov
2026-04-28 20:39:18 +02:00
committed by GitHub
parent e0a99b346b
commit a7cee15115
289 changed files with 45660 additions and 2207 deletions
@@ -0,0 +1,231 @@
// Package health defines the technical-health domain types owned by
// Runtime Manager.
//
// EventType matches the `event_type` enum frozen in
// `galaxy/rtmanager/api/runtime-health-asyncapi.yaml`. SnapshotStatus
// matches the SQL CHECK on `health_snapshots.status` and is intentionally
// narrower than EventType (the snapshot table collapses
// `container_started → healthy` and drops `probe_recovered` per
// `galaxy/rtmanager/README.md §Health Monitoring`).
package health
import (
"encoding/json"
"fmt"
"strings"
"time"
)
// EventType identifies one entry on the `runtime:health_events` Redis
// Stream. Used by the health-event publishers and consumers.
type EventType string
const (
// EventTypeContainerStarted reports a successful container start.
EventTypeContainerStarted EventType = "container_started"
// EventTypeContainerExited reports a non-zero Docker `die` event.
EventTypeContainerExited EventType = "container_exited"
// EventTypeContainerOOM reports a Docker `oom` event.
EventTypeContainerOOM EventType = "container_oom"
// EventTypeContainerDisappeared reports that the listener observed
// a `destroy` event for a record Runtime Manager did not initiate.
EventTypeContainerDisappeared EventType = "container_disappeared"
// EventTypeInspectUnhealthy reports an unexpected outcome of the
// periodic Docker inspect (RestartCount growth, unexpected status,
// declared HEALTHCHECK reporting unhealthy).
EventTypeInspectUnhealthy EventType = "inspect_unhealthy"
// EventTypeProbeFailed reports that the active HTTP probe crossed
// the configured failure threshold.
EventTypeProbeFailed EventType = "probe_failed"
// EventTypeProbeRecovered reports the first probe success after a
// `probe_failed` event was published.
EventTypeProbeRecovered EventType = "probe_recovered"
)
// IsKnown reports whether eventType belongs to the frozen event-type
// vocabulary.
func (eventType EventType) IsKnown() bool {
switch eventType {
case EventTypeContainerStarted,
EventTypeContainerExited,
EventTypeContainerOOM,
EventTypeContainerDisappeared,
EventTypeInspectUnhealthy,
EventTypeProbeFailed,
EventTypeProbeRecovered:
return true
default:
return false
}
}
// AllEventTypes returns the frozen list of every event-type value.
func AllEventTypes() []EventType {
return []EventType{
EventTypeContainerStarted,
EventTypeContainerExited,
EventTypeContainerOOM,
EventTypeContainerDisappeared,
EventTypeInspectUnhealthy,
EventTypeProbeFailed,
EventTypeProbeRecovered,
}
}
// SnapshotStatus identifies one latest-observation status value stored
// in the `health_snapshots.status` column. Distinct from EventType: the
// table collapses `container_started → healthy` and never persists
// `probe_recovered` (it is conveyed only as a `runtime:health_events`
// entry with status=healthy in the next observation).
type SnapshotStatus string
const (
// SnapshotStatusHealthy reports that the most recent observation
// found the container live and the engine probe responsive.
SnapshotStatusHealthy SnapshotStatus = "healthy"
// SnapshotStatusProbeFailed reports that the active probe crossed
// the failure threshold.
SnapshotStatusProbeFailed SnapshotStatus = "probe_failed"
// SnapshotStatusExited reports that the container exited.
SnapshotStatusExited SnapshotStatus = "exited"
// SnapshotStatusOOM reports that the container was killed by the
// OOM killer.
SnapshotStatusOOM SnapshotStatus = "oom"
// SnapshotStatusInspectUnhealthy reports that the periodic inspect
// observed an unexpected state.
SnapshotStatusInspectUnhealthy SnapshotStatus = "inspect_unhealthy"
// SnapshotStatusContainerDisappeared reports that Docker no longer
// reports the container.
SnapshotStatusContainerDisappeared SnapshotStatus = "container_disappeared"
)
// IsKnown reports whether status belongs to the frozen snapshot-status
// vocabulary.
func (status SnapshotStatus) IsKnown() bool {
switch status {
case SnapshotStatusHealthy,
SnapshotStatusProbeFailed,
SnapshotStatusExited,
SnapshotStatusOOM,
SnapshotStatusInspectUnhealthy,
SnapshotStatusContainerDisappeared:
return true
default:
return false
}
}
// AllSnapshotStatuses returns the frozen list of every snapshot-status
// value.
func AllSnapshotStatuses() []SnapshotStatus {
return []SnapshotStatus{
SnapshotStatusHealthy,
SnapshotStatusProbeFailed,
SnapshotStatusExited,
SnapshotStatusOOM,
SnapshotStatusInspectUnhealthy,
SnapshotStatusContainerDisappeared,
}
}
// SnapshotSource identifies the observation source that produced one
// snapshot. Matches the SQL CHECK on `health_snapshots.source`.
type SnapshotSource string
const (
// SnapshotSourceDockerEvent reports that the latest observation
// arrived through the Docker events listener.
SnapshotSourceDockerEvent SnapshotSource = "docker_event"
// SnapshotSourceInspect reports that the latest observation arrived
// through the periodic Docker inspect worker.
SnapshotSourceInspect SnapshotSource = "inspect"
// SnapshotSourceProbe reports that the latest observation arrived
// through the active HTTP probe.
SnapshotSourceProbe SnapshotSource = "probe"
)
// IsKnown reports whether source belongs to the frozen snapshot-source
// vocabulary.
func (source SnapshotSource) IsKnown() bool {
switch source {
case SnapshotSourceDockerEvent,
SnapshotSourceInspect,
SnapshotSourceProbe:
return true
default:
return false
}
}
// AllSnapshotSources returns the frozen list of every snapshot-source
// value.
func AllSnapshotSources() []SnapshotSource {
return []SnapshotSource{
SnapshotSourceDockerEvent,
SnapshotSourceInspect,
SnapshotSourceProbe,
}
}
// HealthSnapshot stores the latest technical-health observation for one
// game. One row per game_id; later observations overwrite.
type HealthSnapshot struct {
// GameID identifies the platform game.
GameID string
// ContainerID stores the Docker container id observed by the
// snapshot source. Empty when the source could not associate a
// container (e.g., reconciler dispose for a record whose container
// is already gone).
ContainerID string
// Status stores the latest observed snapshot status.
Status SnapshotStatus
// Source stores the observation source that produced this entry.
Source SnapshotSource
// Details stores the source-specific JSON detail payload. Adapters
// store and retrieve it verbatim. Empty / nil values are persisted
// as the SQL default `{}`.
Details json.RawMessage
// ObservedAt stores the wall-clock at which the source captured the
// observation.
ObservedAt time.Time
}
// Validate reports whether snapshot satisfies the snapshot invariants
// implied by the SQL CHECK constraints.
func (snapshot HealthSnapshot) Validate() error {
if strings.TrimSpace(snapshot.GameID) == "" {
return fmt.Errorf("game id must not be empty")
}
if !snapshot.Status.IsKnown() {
return fmt.Errorf("status %q is unsupported", snapshot.Status)
}
if !snapshot.Source.IsKnown() {
return fmt.Errorf("source %q is unsupported", snapshot.Source)
}
if snapshot.ObservedAt.IsZero() {
return fmt.Errorf("observed at must not be zero")
}
if len(snapshot.Details) > 0 && !json.Valid(snapshot.Details) {
return fmt.Errorf("details must be valid JSON when non-empty")
}
return nil
}
@@ -0,0 +1,133 @@
package health
import (
"encoding/json"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestEventTypeIsKnown(t *testing.T) {
for _, eventType := range AllEventTypes() {
assert.Truef(t, eventType.IsKnown(), "expected %q known", eventType)
}
assert.False(t, EventType("").IsKnown())
assert.False(t, EventType("paused").IsKnown())
}
func TestAllEventTypesCoverFrozenSet(t *testing.T) {
assert.ElementsMatch(t,
[]EventType{
EventTypeContainerStarted,
EventTypeContainerExited,
EventTypeContainerOOM,
EventTypeContainerDisappeared,
EventTypeInspectUnhealthy,
EventTypeProbeFailed,
EventTypeProbeRecovered,
},
AllEventTypes(),
)
}
func TestSnapshotStatusIsKnown(t *testing.T) {
for _, status := range AllSnapshotStatuses() {
assert.Truef(t, status.IsKnown(), "expected %q known", status)
}
assert.False(t, SnapshotStatus("").IsKnown())
assert.False(t, SnapshotStatus("starting").IsKnown())
assert.False(t, SnapshotStatus("probe_recovered").IsKnown(),
"snapshot status must not include event-only values")
assert.False(t, SnapshotStatus("container_started").IsKnown(),
"snapshot status must not include event-only values")
}
func TestAllSnapshotStatusesCoverFrozenSet(t *testing.T) {
assert.ElementsMatch(t,
[]SnapshotStatus{
SnapshotStatusHealthy,
SnapshotStatusProbeFailed,
SnapshotStatusExited,
SnapshotStatusOOM,
SnapshotStatusInspectUnhealthy,
SnapshotStatusContainerDisappeared,
},
AllSnapshotStatuses(),
)
}
func TestSnapshotSourceIsKnown(t *testing.T) {
for _, source := range AllSnapshotSources() {
assert.Truef(t, source.IsKnown(), "expected %q known", source)
}
assert.False(t, SnapshotSource("").IsKnown())
assert.False(t, SnapshotSource("manual").IsKnown())
}
func TestAllSnapshotSourcesCoverFrozenSet(t *testing.T) {
assert.ElementsMatch(t,
[]SnapshotSource{
SnapshotSourceDockerEvent,
SnapshotSourceInspect,
SnapshotSourceProbe,
},
AllSnapshotSources(),
)
}
func sampleSnapshot() HealthSnapshot {
return HealthSnapshot{
GameID: "game-test",
ContainerID: "container-1",
Status: SnapshotStatusHealthy,
Source: SnapshotSourceProbe,
Details: json.RawMessage(`{"prior_failure_count":0}`),
ObservedAt: time.Date(2026, 4, 27, 12, 0, 0, 0, time.UTC),
}
}
func TestHealthSnapshotValidateHappy(t *testing.T) {
require.NoError(t, sampleSnapshot().Validate())
}
func TestHealthSnapshotValidateAcceptsEmptyDetails(t *testing.T) {
snapshot := sampleSnapshot()
snapshot.Details = nil
assert.NoError(t, snapshot.Validate())
}
func TestHealthSnapshotValidateAcceptsEmptyContainerID(t *testing.T) {
snapshot := sampleSnapshot()
snapshot.ContainerID = ""
assert.NoError(t, snapshot.Validate())
}
func TestHealthSnapshotValidateRejects(t *testing.T) {
tests := []struct {
name string
mutate func(*HealthSnapshot)
}{
{"empty game id", func(s *HealthSnapshot) { s.GameID = "" }},
{"unknown status", func(s *HealthSnapshot) { s.Status = "exotic" }},
{"unknown source", func(s *HealthSnapshot) { s.Source = "exotic" }},
{"zero observed at", func(s *HealthSnapshot) { s.ObservedAt = time.Time{} }},
{"invalid details json", func(s *HealthSnapshot) {
s.Details = json.RawMessage("not-json")
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
snapshot := sampleSnapshot()
tt.mutate(&snapshot)
assert.Error(t, snapshot.Validate())
})
}
}
+245
View File
@@ -0,0 +1,245 @@
// Package operation defines the runtime-operation audit-log domain types
// owned by Runtime Manager.
//
// One OperationEntry maps to one row of the `operation_log` PostgreSQL
// table (see
// `galaxy/rtmanager/internal/adapters/postgres/migrations/00001_init.sql`).
// The OpKind / OpSource / Outcome enums match the SQL CHECK constraints
// verbatim and feed the telemetry counters declared in
// `galaxy/rtmanager/README.md §Observability`.
package operation
import (
"fmt"
"strings"
"time"
)
// OpKind identifies the kind of operation Runtime Manager performed.
type OpKind string
const (
// OpKindStart records a start lifecycle operation.
OpKindStart OpKind = "start"
// OpKindStop records a stop lifecycle operation.
OpKindStop OpKind = "stop"
// OpKindRestart records a restart lifecycle operation
// (recreate with the same image_ref).
OpKindRestart OpKind = "restart"
// OpKindPatch records a semver-patch lifecycle operation
// (recreate with a new image_ref).
OpKindPatch OpKind = "patch"
// OpKindCleanupContainer records a container removal performed by
// the cleanup TTL worker or the admin DELETE endpoint.
OpKindCleanupContainer OpKind = "cleanup_container"
// OpKindReconcileAdopt records that the reconciler discovered an
// unrecorded container labelled `com.galaxy.owner=rtmanager` and
// inserted a runtime record for it.
OpKindReconcileAdopt OpKind = "reconcile_adopt"
// OpKindReconcileDispose records that the reconciler observed a
// running record whose container is missing in Docker and marked it
// as removed.
OpKindReconcileDispose OpKind = "reconcile_dispose"
)
// IsKnown reports whether kind belongs to the frozen op-kind vocabulary.
func (kind OpKind) IsKnown() bool {
switch kind {
case OpKindStart,
OpKindStop,
OpKindRestart,
OpKindPatch,
OpKindCleanupContainer,
OpKindReconcileAdopt,
OpKindReconcileDispose:
return true
default:
return false
}
}
// AllOpKinds returns the frozen list of every op-kind value. The slice
// order is stable across calls.
func AllOpKinds() []OpKind {
return []OpKind{
OpKindStart,
OpKindStop,
OpKindRestart,
OpKindPatch,
OpKindCleanupContainer,
OpKindReconcileAdopt,
OpKindReconcileDispose,
}
}
// OpSource identifies where one operation entered Runtime Manager.
type OpSource string
const (
// OpSourceLobbyStream identifies entries triggered by the
// `runtime:start_jobs` or `runtime:stop_jobs` Redis Stream consumer.
OpSourceLobbyStream OpSource = "lobby_stream"
// OpSourceGMRest identifies entries triggered by Game Master through
// the internal REST surface.
OpSourceGMRest OpSource = "gm_rest"
// OpSourceAdminRest identifies entries triggered by Admin Service
// through the internal REST surface.
OpSourceAdminRest OpSource = "admin_rest"
// OpSourceAutoTTL identifies entries triggered by the periodic
// container-cleanup worker.
OpSourceAutoTTL OpSource = "auto_ttl"
// OpSourceAutoReconcile identifies entries triggered by the
// reconciler at startup or on its periodic interval.
OpSourceAutoReconcile OpSource = "auto_reconcile"
)
// IsKnown reports whether source belongs to the frozen op-source
// vocabulary.
func (source OpSource) IsKnown() bool {
switch source {
case OpSourceLobbyStream,
OpSourceGMRest,
OpSourceAdminRest,
OpSourceAutoTTL,
OpSourceAutoReconcile:
return true
default:
return false
}
}
// AllOpSources returns the frozen list of every op-source value. The
// slice order is stable across calls.
func AllOpSources() []OpSource {
return []OpSource{
OpSourceLobbyStream,
OpSourceGMRest,
OpSourceAdminRest,
OpSourceAutoTTL,
OpSourceAutoReconcile,
}
}
// Outcome reports the high-level outcome of one operation.
type Outcome string
const (
// OutcomeSuccess reports that the operation completed without
// surfacing an error.
OutcomeSuccess Outcome = "success"
// OutcomeFailure reports that the operation surfaced a stable error
// code recorded in OperationEntry.ErrorCode.
OutcomeFailure Outcome = "failure"
)
// IsKnown reports whether outcome belongs to the frozen outcome
// vocabulary.
func (outcome Outcome) IsKnown() bool {
switch outcome {
case OutcomeSuccess, OutcomeFailure:
return true
default:
return false
}
}
// AllOutcomes returns the frozen list of every outcome value.
func AllOutcomes() []Outcome {
return []Outcome{OutcomeSuccess, OutcomeFailure}
}
// OperationEntry stores one append-only audit row of the `operation_log`
// table. ID is zero on records that have not been persisted yet; the
// store assigns it from the table's bigserial column. FinishedAt is a
// pointer because the column is nullable for in-flight rows even though
// the lifecycle services finalise the row in the same transaction.
type OperationEntry struct {
// ID identifies the persisted row. Zero before persistence.
ID int64
// GameID identifies the platform game this operation acted on.
GameID string
// OpKind classifies what the operation did.
OpKind OpKind
// OpSource classifies how the operation entered Runtime Manager.
OpSource OpSource
// SourceRef stores an opaque per-source reference such as a Redis
// Stream entry id, a REST request id, or an admin user id. Empty
// when the source does not provide one.
SourceRef string
// ImageRef stores the engine image reference associated with the
// operation, when applicable. Empty for operations that do not
// touch an image (e.g., cleanup_container).
ImageRef string
// ContainerID stores the Docker container id observed at the time
// of the operation, when applicable.
ContainerID string
// Outcome reports whether the operation succeeded or failed.
Outcome Outcome
// ErrorCode stores the stable error code on failure. Empty on
// success.
ErrorCode string
// ErrorMessage stores the operator-readable detail on failure.
// Empty on success.
ErrorMessage string
// StartedAt stores the wall-clock at which the operation began.
StartedAt time.Time
// FinishedAt stores the wall-clock at which the operation
// finalised. Nil for in-flight rows.
FinishedAt *time.Time
}
// Validate reports whether entry satisfies the operation-log invariants
// implied by the SQL CHECK constraints and the README §Persistence
// Layout.
func (entry OperationEntry) Validate() error {
if strings.TrimSpace(entry.GameID) == "" {
return fmt.Errorf("game id must not be empty")
}
if !entry.OpKind.IsKnown() {
return fmt.Errorf("op kind %q is unsupported", entry.OpKind)
}
if !entry.OpSource.IsKnown() {
return fmt.Errorf("op source %q is unsupported", entry.OpSource)
}
if !entry.Outcome.IsKnown() {
return fmt.Errorf("outcome %q is unsupported", entry.Outcome)
}
if entry.StartedAt.IsZero() {
return fmt.Errorf("started at must not be zero")
}
if entry.FinishedAt != nil {
if entry.FinishedAt.IsZero() {
return fmt.Errorf("finished at must not be zero when present")
}
if entry.FinishedAt.Before(entry.StartedAt) {
return fmt.Errorf("finished at must not be before started at")
}
}
if entry.Outcome == OutcomeFailure && strings.TrimSpace(entry.ErrorCode) == "" {
return fmt.Errorf("error code must not be empty for failure entries")
}
return nil
}
@@ -0,0 +1,130 @@
package operation
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestOpKindIsKnown(t *testing.T) {
for _, kind := range AllOpKinds() {
assert.Truef(t, kind.IsKnown(), "expected %q known", kind)
}
assert.False(t, OpKind("").IsKnown())
assert.False(t, OpKind("rollback").IsKnown())
}
func TestAllOpKindsCoverFrozenSet(t *testing.T) {
assert.ElementsMatch(t,
[]OpKind{
OpKindStart, OpKindStop, OpKindRestart, OpKindPatch,
OpKindCleanupContainer, OpKindReconcileAdopt, OpKindReconcileDispose,
},
AllOpKinds(),
)
}
func TestOpSourceIsKnown(t *testing.T) {
for _, source := range AllOpSources() {
assert.Truef(t, source.IsKnown(), "expected %q known", source)
}
assert.False(t, OpSource("").IsKnown())
assert.False(t, OpSource("manual").IsKnown())
}
func TestAllOpSourcesCoverFrozenSet(t *testing.T) {
assert.ElementsMatch(t,
[]OpSource{
OpSourceLobbyStream, OpSourceGMRest, OpSourceAdminRest,
OpSourceAutoTTL, OpSourceAutoReconcile,
},
AllOpSources(),
)
}
func TestOutcomeIsKnown(t *testing.T) {
for _, outcome := range AllOutcomes() {
assert.Truef(t, outcome.IsKnown(), "expected %q known", outcome)
}
assert.False(t, Outcome("").IsKnown())
assert.False(t, Outcome("partial").IsKnown())
}
func TestAllOutcomesCoverFrozenSet(t *testing.T) {
assert.ElementsMatch(t,
[]Outcome{OutcomeSuccess, OutcomeFailure},
AllOutcomes(),
)
}
func successEntry() OperationEntry {
started := time.Date(2026, 4, 27, 12, 0, 0, 0, time.UTC)
finished := started.Add(time.Second)
return OperationEntry{
GameID: "game-test",
OpKind: OpKindStart,
OpSource: OpSourceLobbyStream,
SourceRef: "1700000000000-0",
ImageRef: "galaxy/game:1.0.0",
ContainerID: "container-1",
Outcome: OutcomeSuccess,
StartedAt: started,
FinishedAt: &finished,
}
}
func TestOperationEntryValidateHappy(t *testing.T) {
require.NoError(t, successEntry().Validate())
}
func TestOperationEntryValidateAcceptsReplayNoOp(t *testing.T) {
entry := successEntry()
entry.ErrorCode = "replay_no_op"
assert.NoError(t, entry.Validate())
}
func TestOperationEntryValidateAcceptsInFlight(t *testing.T) {
entry := successEntry()
entry.FinishedAt = nil
assert.NoError(t, entry.Validate())
}
func TestOperationEntryValidateRejects(t *testing.T) {
tests := []struct {
name string
mutate func(*OperationEntry)
}{
{"empty game id", func(e *OperationEntry) { e.GameID = "" }},
{"unknown op kind", func(e *OperationEntry) { e.OpKind = "exotic" }},
{"unknown op source", func(e *OperationEntry) { e.OpSource = "exotic" }},
{"unknown outcome", func(e *OperationEntry) { e.Outcome = "partial" }},
{"zero started at", func(e *OperationEntry) { e.StartedAt = time.Time{} }},
{"zero finished at", func(e *OperationEntry) {
zero := time.Time{}
e.FinishedAt = &zero
}},
{"finished before started", func(e *OperationEntry) {
before := e.StartedAt.Add(-time.Second)
e.FinishedAt = &before
}},
{"failure without error code", func(e *OperationEntry) {
e.Outcome = OutcomeFailure
e.ErrorCode = ""
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
entry := successEntry()
tt.mutate(&entry)
assert.Error(t, entry.Validate())
})
}
}
@@ -0,0 +1,43 @@
package runtime
import (
"errors"
"fmt"
)
// ErrNotFound reports that a runtime record was requested but does not
// exist in the store.
var ErrNotFound = errors.New("runtime record not found")
// ErrConflict reports that a runtime mutation could not be applied
// because the record changed concurrently or failed a compare-and-swap
// guard.
var ErrConflict = errors.New("runtime record conflict")
// ErrInvalidTransition is the sentinel returned when Transition rejects
// a `(from, to)` pair.
var ErrInvalidTransition = errors.New("invalid runtime status transition")
// InvalidTransitionError stores the rejected `(from, to)` pair and wraps
// ErrInvalidTransition so callers can match it with errors.Is.
type InvalidTransitionError struct {
// From stores the source status that was attempted to leave.
From Status
// To stores the destination status that was attempted to enter.
To Status
}
// Error reports a human-readable summary of the rejected pair.
func (err *InvalidTransitionError) Error() string {
return fmt.Sprintf(
"invalid runtime status transition from %q to %q",
err.From, err.To,
)
}
// Unwrap returns ErrInvalidTransition so errors.Is recognizes the
// sentinel.
func (err *InvalidTransitionError) Unwrap() error {
return ErrInvalidTransition
}
+197
View File
@@ -0,0 +1,197 @@
// Package runtime defines the runtime-record domain model, status machine,
// and sentinel errors owned by Runtime Manager.
//
// The package mirrors the durable shape of the `runtime_records`
// PostgreSQL table (see
// `galaxy/rtmanager/internal/adapters/postgres/migrations/00001_init.sql`).
// Every status / transition / required-field rule already documented in
// `galaxy/rtmanager/README.md` lives here as code so adapter and service
// layers do not re-derive it.
package runtime
import (
"fmt"
"strings"
"time"
)
// Status identifies one runtime-record lifecycle state.
type Status string
const (
// StatusRunning reports that an engine container is live and bound to
// the record. The associated container id and image ref are non-empty
// and StartedAt is set.
StatusRunning Status = "running"
// StatusStopped reports that the engine container has exited (graceful
// stop, observed Docker exit, or reconciled exit). The container is
// still present in Docker until the cleanup worker removes it.
StatusStopped Status = "stopped"
// StatusRemoved reports that the container has been removed from
// Docker (admin cleanup or reconcile_dispose). The record stays in
// PostgreSQL for audit; there is no transition out of this state.
StatusRemoved Status = "removed"
)
// IsKnown reports whether status belongs to the frozen runtime status
// vocabulary.
func (status Status) IsKnown() bool {
switch status {
case StatusRunning, StatusStopped, StatusRemoved:
return true
default:
return false
}
}
// IsTerminal reports whether status can no longer accept lifecycle
// transitions.
func (status Status) IsTerminal() bool {
return status == StatusRemoved
}
// AllStatuses returns the frozen list of every runtime status value. The
// slice order is stable across calls and matches the README §Persistence
// Layout listing.
func AllStatuses() []Status {
return []Status{
StatusRunning,
StatusStopped,
StatusRemoved,
}
}
// RuntimeRecord stores one durable runtime record owned by Runtime
// Manager. It mirrors one row of the `runtime_records` table.
//
// CurrentContainerID and CurrentImageRef are stored as plain strings; an
// empty value represents SQL NULL and is bridged at the adapter layer.
// StartedAt, StoppedAt, and RemovedAt are *time.Time so a missing value
// is unambiguous and aligns with the jet-generated model.
type RuntimeRecord struct {
// GameID identifies the platform game owning this runtime record.
GameID string
// Status stores the current lifecycle state.
Status Status
// CurrentContainerID identifies the bound Docker container. Empty
// when status is removed and after a reconciler observes
// disappearance.
CurrentContainerID string
// CurrentImageRef stores the Docker reference of the currently-bound
// engine image. Non-empty when status is running or stopped.
CurrentImageRef string
// EngineEndpoint stores the stable URL Game Master uses to reach the
// engine container, in `http://galaxy-game-{game_id}:8080` form.
EngineEndpoint string
// StatePath stores the absolute host path of the bind-mounted engine
// state directory.
StatePath string
// DockerNetwork stores the Docker network the container was attached
// to at create time.
DockerNetwork string
// StartedAt stores the wall-clock at which the container became
// running. Non-nil when status is running or stopped.
StartedAt *time.Time
// StoppedAt stores the wall-clock at which the container exited.
// Non-nil when status is stopped or removed (when the record passed
// through stopped before removal).
StoppedAt *time.Time
// RemovedAt stores the wall-clock at which the container was removed
// from Docker. Non-nil when status is removed.
RemovedAt *time.Time
// LastOpAt stores the wall-clock of the most recent operation
// affecting this record. Drives the cleanup TTL.
LastOpAt time.Time
// CreatedAt stores the wall-clock at which Runtime Manager first saw
// this game.
CreatedAt time.Time
}
// Validate reports whether record satisfies the runtime-record invariants
// implied by README §Lifecycles and the SQL CHECK on `runtime_records`.
func (record RuntimeRecord) Validate() error {
if strings.TrimSpace(record.GameID) == "" {
return fmt.Errorf("game id must not be empty")
}
if !record.Status.IsKnown() {
return fmt.Errorf("status %q is unsupported", record.Status)
}
if strings.TrimSpace(record.EngineEndpoint) == "" {
return fmt.Errorf("engine endpoint must not be empty")
}
if strings.TrimSpace(record.StatePath) == "" {
return fmt.Errorf("state path must not be empty")
}
if strings.TrimSpace(record.DockerNetwork) == "" {
return fmt.Errorf("docker network must not be empty")
}
if record.LastOpAt.IsZero() {
return fmt.Errorf("last op at must not be zero")
}
if record.CreatedAt.IsZero() {
return fmt.Errorf("created at must not be zero")
}
if record.LastOpAt.Before(record.CreatedAt) {
return fmt.Errorf("last op at must not be before created at")
}
switch record.Status {
case StatusRunning:
if strings.TrimSpace(record.CurrentContainerID) == "" {
return fmt.Errorf("current container id must not be empty for running records")
}
if strings.TrimSpace(record.CurrentImageRef) == "" {
return fmt.Errorf("current image ref must not be empty for running records")
}
if record.StartedAt == nil {
return fmt.Errorf("started at must not be nil for running records")
}
if record.StartedAt.IsZero() {
return fmt.Errorf("started at must not be zero when present")
}
case StatusStopped:
if strings.TrimSpace(record.CurrentImageRef) == "" {
return fmt.Errorf("current image ref must not be empty for stopped records")
}
if record.StoppedAt == nil {
return fmt.Errorf("stopped at must not be nil for stopped records")
}
if record.StoppedAt.IsZero() {
return fmt.Errorf("stopped at must not be zero when present")
}
case StatusRemoved:
if record.RemovedAt == nil {
return fmt.Errorf("removed at must not be nil for removed records")
}
if record.RemovedAt.IsZero() {
return fmt.Errorf("removed at must not be zero when present")
}
}
if record.StartedAt != nil && record.StartedAt.Before(record.CreatedAt) {
return fmt.Errorf("started at must not be before created at")
}
if record.StoppedAt != nil && record.StartedAt != nil && record.StoppedAt.Before(*record.StartedAt) {
return fmt.Errorf("stopped at must not be before started at")
}
if record.RemovedAt != nil && record.RemovedAt.Before(record.CreatedAt) {
return fmt.Errorf("removed at must not be before created at")
}
return nil
}
@@ -0,0 +1,156 @@
package runtime
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestStatusIsKnown(t *testing.T) {
for _, status := range AllStatuses() {
assert.Truef(t, status.IsKnown(), "expected %q known", status)
}
assert.False(t, Status("").IsKnown())
assert.False(t, Status("unknown").IsKnown())
}
func TestStatusIsTerminal(t *testing.T) {
assert.True(t, StatusRemoved.IsTerminal())
for _, status := range []Status{StatusRunning, StatusStopped} {
assert.Falsef(t, status.IsTerminal(), "expected %q non-terminal", status)
}
}
func TestAllStatuses(t *testing.T) {
statuses := AllStatuses()
assert.ElementsMatch(t,
[]Status{StatusRunning, StatusStopped, StatusRemoved},
statuses,
)
statuses[0] = "tampered"
assert.Equal(t, StatusRunning, AllStatuses()[0],
"AllStatuses must return an independent slice")
}
func runningRecord() RuntimeRecord {
created := time.Date(2026, 4, 27, 12, 0, 0, 0, time.UTC)
started := created.Add(time.Second)
return RuntimeRecord{
GameID: "game-test",
Status: StatusRunning,
CurrentContainerID: "container-1",
CurrentImageRef: "galaxy/game:1.0.0",
EngineEndpoint: "http://galaxy-game-game-test:8080",
StatePath: "/var/lib/galaxy/games/game-test",
DockerNetwork: "galaxy-net",
StartedAt: &started,
LastOpAt: started,
CreatedAt: created,
}
}
func TestRuntimeRecordValidateRunningHappy(t *testing.T) {
require.NoError(t, runningRecord().Validate())
}
func TestRuntimeRecordValidateStoppedHappy(t *testing.T) {
record := runningRecord()
stopped := record.StartedAt.Add(time.Minute)
record.Status = StatusStopped
record.StoppedAt = &stopped
record.LastOpAt = stopped
require.NoError(t, record.Validate())
}
func TestRuntimeRecordValidateRemovedHappy(t *testing.T) {
record := runningRecord()
stopped := record.StartedAt.Add(time.Minute)
removed := stopped.Add(time.Minute)
record.Status = StatusRemoved
record.StoppedAt = &stopped
record.RemovedAt = &removed
record.CurrentContainerID = ""
record.LastOpAt = removed
require.NoError(t, record.Validate())
}
func TestRuntimeRecordValidateRejects(t *testing.T) {
tests := []struct {
name string
mutate func(*RuntimeRecord)
}{
{"empty game id", func(r *RuntimeRecord) { r.GameID = "" }},
{"unknown status", func(r *RuntimeRecord) { r.Status = "exotic" }},
{"empty engine endpoint", func(r *RuntimeRecord) { r.EngineEndpoint = "" }},
{"empty state path", func(r *RuntimeRecord) { r.StatePath = "" }},
{"empty docker network", func(r *RuntimeRecord) { r.DockerNetwork = "" }},
{"zero last op at", func(r *RuntimeRecord) { r.LastOpAt = time.Time{} }},
{"zero created at", func(r *RuntimeRecord) { r.CreatedAt = time.Time{} }},
{"last op at before created at", func(r *RuntimeRecord) {
r.LastOpAt = r.CreatedAt.Add(-time.Second)
}},
{"running without container id", func(r *RuntimeRecord) {
r.CurrentContainerID = ""
}},
{"running without image ref", func(r *RuntimeRecord) {
r.CurrentImageRef = ""
}},
{"running without started at", func(r *RuntimeRecord) {
r.StartedAt = nil
}},
{"started at before created at", func(r *RuntimeRecord) {
before := r.CreatedAt.Add(-time.Second)
r.StartedAt = &before
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
record := runningRecord()
tt.mutate(&record)
assert.Error(t, record.Validate())
})
}
}
func TestRuntimeRecordValidateRejectsStoppedWithoutStoppedAt(t *testing.T) {
record := runningRecord()
record.Status = StatusStopped
record.StoppedAt = nil
assert.Error(t, record.Validate())
}
func TestRuntimeRecordValidateRejectsStoppedBeforeStarted(t *testing.T) {
record := runningRecord()
stopped := record.StartedAt.Add(-time.Second)
record.Status = StatusStopped
record.StoppedAt = &stopped
assert.Error(t, record.Validate())
}
func TestRuntimeRecordValidateRejectsRemovedWithoutRemovedAt(t *testing.T) {
record := runningRecord()
record.Status = StatusRemoved
record.RemovedAt = nil
assert.Error(t, record.Validate())
}
func TestRuntimeRecordValidateRejectsRemovedBeforeCreated(t *testing.T) {
record := runningRecord()
before := record.CreatedAt.Add(-time.Second)
record.Status = StatusRemoved
record.RemovedAt = &before
assert.Error(t, record.Validate())
}
@@ -0,0 +1,51 @@
package runtime
// transitionKey stores one `(from, to)` pair in the allowed-transitions
// table.
type transitionKey struct {
from Status
to Status
}
// allowedTransitions stores the set of permitted `(from, to)` status
// pairs. The four pairs mirror the lifecycle flows frozen in
// `galaxy/rtmanager/README.md §Lifecycles`:
//
// - running → stopped: graceful stop, observed Docker exit, or
// reconcile observing an exited container.
// - running → removed: reconcile_dispose when Docker no longer reports
// the container at all.
// - stopped → running: restart and patch inner start steps.
// - stopped → removed: cleanup_container, both the periodic TTL worker
// and the admin DELETE endpoint.
var allowedTransitions = map[transitionKey]struct{}{
{StatusRunning, StatusStopped}: {},
{StatusRunning, StatusRemoved}: {},
{StatusStopped, StatusRunning}: {},
{StatusStopped, StatusRemoved}: {},
}
// AllowedTransitions returns a copy of the `(from, to)` allowed
// transitions table used by Transition. The returned map is safe to
// mutate; callers should not rely on iteration order.
func AllowedTransitions() map[Status][]Status {
result := make(map[Status][]Status)
for key := range allowedTransitions {
result[key.from] = append(result[key.from], key.to)
}
return result
}
// Transition reports whether from may transition to next. The function
// returns nil when the pair is permitted, and an *InvalidTransitionError
// wrapping ErrInvalidTransition otherwise. It does not touch any store
// and is safe to call from any layer.
func Transition(from Status, next Status) error {
if !from.IsKnown() || !next.IsKnown() {
return &InvalidTransitionError{From: from, To: next}
}
if _, ok := allowedTransitions[transitionKey{from: from, to: next}]; !ok {
return &InvalidTransitionError{From: from, To: next}
}
return nil
}
@@ -0,0 +1,88 @@
package runtime
import (
"errors"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestTransitionAllowed(t *testing.T) {
cases := []struct {
from Status
to Status
}{
{StatusRunning, StatusStopped},
{StatusRunning, StatusRemoved},
{StatusStopped, StatusRunning},
{StatusStopped, StatusRemoved},
}
for _, tc := range cases {
assert.NoErrorf(t, Transition(tc.from, tc.to),
"expected %q -> %q allowed", tc.from, tc.to)
}
}
func TestTransitionRejected(t *testing.T) {
cases := []struct {
from Status
to Status
}{
{StatusRemoved, StatusRunning},
{StatusRemoved, StatusStopped},
{StatusRemoved, StatusRemoved},
{StatusRunning, StatusRunning},
{StatusStopped, StatusStopped},
{Status("unknown"), StatusRunning},
{StatusRunning, Status("unknown")},
{Status(""), Status("")},
}
for _, tc := range cases {
err := Transition(tc.from, tc.to)
require.Errorf(t, err, "expected %q -> %q rejected", tc.from, tc.to)
assert.ErrorIs(t, err, ErrInvalidTransition)
var transitionErr *InvalidTransitionError
require.True(t, errors.As(err, &transitionErr),
"expected *InvalidTransitionError for %q -> %q", tc.from, tc.to)
assert.Equal(t, tc.from, transitionErr.From)
assert.Equal(t, tc.to, transitionErr.To)
}
}
func TestAllowedTransitionsReturnsCopy(t *testing.T) {
first := AllowedTransitions()
require.NotEmpty(t, first)
for from := range first {
first[from] = nil
}
second := AllowedTransitions()
assert.NotEmpty(t, second[StatusRunning],
"AllowedTransitions must return an independent map per call")
}
func TestAllowedTransitionsCoversFourPairs(t *testing.T) {
transitions := AllowedTransitions()
assert.ElementsMatch(t,
[]Status{StatusStopped, StatusRemoved},
transitions[StatusRunning],
)
assert.ElementsMatch(t,
[]Status{StatusRunning, StatusRemoved},
transitions[StatusStopped],
)
assert.Empty(t, transitions[StatusRemoved],
"removed has no outgoing transitions")
}
func TestInvalidTransitionErrorMessage(t *testing.T) {
err := &InvalidTransitionError{From: StatusRunning, To: Status("bogus")}
assert.Contains(t, err.Error(), "running")
assert.Contains(t, err.Error(), "bogus")
}