feat: runtime manager
This commit is contained in:
Ilia Denisov
@@ -0,0 +1,940 @@
|
||||
// Package startruntime implements the `start` lifecycle operation owned
|
||||
// by Runtime Manager. The service is the single orchestrator behind
|
||||
// both the asynchronous `runtime:start_jobs` consumer and the
|
||||
// synchronous `POST /api/v1/internal/runtimes/{game_id}/start` REST
|
||||
// handler; both callers obtain a deterministic Result with a stable
|
||||
// `Outcome` / `ErrorCode` pair.
|
||||
//
|
||||
// Lifecycle and failure-mode semantics follow `rtmanager/README.md
|
||||
// §Lifecycles → Start`. Design rationale is captured in
|
||||
// `rtmanager/docs/services.md`.
|
||||
package startruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"galaxy/notificationintent"
|
||||
"galaxy/rtmanager/internal/config"
|
||||
"galaxy/rtmanager/internal/domain/health"
|
||||
"galaxy/rtmanager/internal/domain/operation"
|
||||
"galaxy/rtmanager/internal/domain/runtime"
|
||||
"galaxy/rtmanager/internal/logging"
|
||||
"galaxy/rtmanager/internal/ports"
|
||||
"galaxy/rtmanager/internal/telemetry"
|
||||
|
||||
"github.com/distribution/reference"
|
||||
)
|
||||
|
||||
// Container labels applied to every engine container created by the
|
||||
// start service. Frozen by `rtmanager/README.md §Container Model`.
|
||||
const (
|
||||
LabelOwner = "com.galaxy.owner"
|
||||
LabelOwnerValue = "rtmanager"
|
||||
LabelKind = "com.galaxy.kind"
|
||||
LabelKindValue = "game-engine"
|
||||
LabelGameID = "com.galaxy.game_id"
|
||||
LabelEngineImageRef = "com.galaxy.engine_image_ref"
|
||||
LabelStartedAtMs = "com.galaxy.started_at_ms"
|
||||
|
||||
// Image labels read at start time to derive resource limits.
|
||||
imageLabelCPUQuota = "com.galaxy.cpu_quota"
|
||||
imageLabelMemory = "com.galaxy.memory"
|
||||
imageLabelPIDsLimit = "com.galaxy.pids_limit"
|
||||
|
||||
// HostnamePrefix is the constant prefix used to build the per-game
|
||||
// container hostname (`galaxy-game-{game_id}`). The full hostname
|
||||
// also forms the container name; restart and patch keep the same
|
||||
// value so the engine endpoint stays stable across container
|
||||
// recreates.
|
||||
HostnamePrefix = "galaxy-game-"
|
||||
|
||||
// EngineStateBackCompatEnvName is the secondary env var name v1
|
||||
// engines accept for the bind-mounted state directory. Always set
|
||||
// alongside the configured primary name to honour the v1 backward
|
||||
// compatibility commitment in `rtmanager/README.md §Container Model`.
|
||||
EngineStateBackCompatEnvName = "STORAGE_PATH"
|
||||
|
||||
// leaseReleaseTimeout bounds the deferred lease-release call. A
|
||||
// fresh background context is used so the release runs even when
|
||||
// the request context was already canceled.
|
||||
leaseReleaseTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
// Input stores the per-call arguments for one start operation.
|
||||
type Input struct {
|
||||
// GameID identifies the platform game to start.
|
||||
GameID string
|
||||
|
||||
// ImageRef stores the producer-resolved Docker reference of the
|
||||
// engine image. Validated against `distribution/reference` before
|
||||
// any Docker work.
|
||||
ImageRef string
|
||||
|
||||
// OpSource classifies how the request entered Runtime Manager.
|
||||
// Required: every operation_log entry carries an op_source.
|
||||
OpSource operation.OpSource
|
||||
|
||||
// SourceRef stores the optional opaque per-source reference
|
||||
// (Redis Stream entry id, REST request id, admin user id). Empty
|
||||
// when the caller does not provide one.
|
||||
SourceRef string
|
||||
}
|
||||
|
||||
// Validate reports whether input carries the structural invariants the
|
||||
// service requires.
|
||||
func (input Input) Validate() error {
|
||||
if strings.TrimSpace(input.GameID) == "" {
|
||||
return fmt.Errorf("game id must not be empty")
|
||||
}
|
||||
if strings.TrimSpace(input.ImageRef) == "" {
|
||||
return fmt.Errorf("image ref must not be empty")
|
||||
}
|
||||
if !input.OpSource.IsKnown() {
|
||||
return fmt.Errorf("op source %q is unsupported", input.OpSource)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Result stores the deterministic outcome of one Handle call.
|
||||
type Result struct {
|
||||
// Record carries the runtime record installed by the operation.
|
||||
// Populated on success and on idempotent replay (`replay_no_op`);
|
||||
// zero on failure.
|
||||
Record runtime.RuntimeRecord
|
||||
|
||||
// Outcome reports whether the operation completed (success) or
|
||||
// produced a stable failure code.
|
||||
Outcome operation.Outcome
|
||||
|
||||
// ErrorCode stores the stable error code on failure, or
|
||||
// `replay_no_op` on idempotent replay. Empty for fresh successes.
|
||||
ErrorCode string
|
||||
|
||||
// ErrorMessage stores the operator-readable detail on failure.
|
||||
// Empty for successes.
|
||||
ErrorMessage string
|
||||
}
|
||||
|
||||
// Dependencies groups the collaborators required by Service.
|
||||
type Dependencies struct {
|
||||
// RuntimeRecords reads and installs the durable runtime record.
|
||||
RuntimeRecords ports.RuntimeRecordStore
|
||||
|
||||
// OperationLogs records the success / failure audit entry.
|
||||
OperationLogs ports.OperationLogStore
|
||||
|
||||
// Docker drives the Docker daemon (network check, pull, inspect,
|
||||
// run, remove).
|
||||
Docker ports.DockerClient
|
||||
|
||||
// Leases serialises operations against the same game id.
|
||||
Leases ports.GameLeaseStore
|
||||
|
||||
// HealthEvents publishes `runtime:health_events` and upserts the
|
||||
// matching `health_snapshots` row.
|
||||
HealthEvents ports.HealthEventPublisher
|
||||
|
||||
// Notifications publishes admin-only failure intents.
|
||||
Notifications ports.NotificationIntentPublisher
|
||||
|
||||
// Lobby provides best-effort diagnostic context for the started
|
||||
// game. May be nil; the start operation does not depend on it.
|
||||
Lobby ports.LobbyInternalClient
|
||||
|
||||
// Container groups the per-container defaults and state-directory
|
||||
// settings consumed at start time.
|
||||
Container config.ContainerConfig
|
||||
|
||||
// Docker groups the Docker daemon settings (network, log driver,
|
||||
// pull policy) consumed at start time.
|
||||
DockerCfg config.DockerConfig
|
||||
|
||||
// Coordination supplies the per-game lease TTL.
|
||||
Coordination config.CoordinationConfig
|
||||
|
||||
// Telemetry records start outcomes, lease latency, and health
|
||||
// event counters. Required.
|
||||
Telemetry *telemetry.Runtime
|
||||
|
||||
// Logger records structured service-level events. Defaults to
|
||||
// `slog.Default()` when nil.
|
||||
Logger *slog.Logger
|
||||
|
||||
// Clock supplies the wall-clock used for operation timestamps.
|
||||
// Defaults to `time.Now` when nil.
|
||||
Clock func() time.Time
|
||||
|
||||
// NewToken supplies a unique opaque lease token. Defaults to a
|
||||
// 32-byte random base64url string when nil. Tests may override.
|
||||
NewToken func() string
|
||||
|
||||
// PrepareStateDir creates the per-game state directory and
|
||||
// returns its absolute host path. Defaults to a real-filesystem
|
||||
// implementation that honours Container.GameStateRoot,
|
||||
// Container.GameStateDirMode, and Container.GameStateOwner{UID,GID}.
|
||||
// Tests override to point at a temporary directory.
|
||||
PrepareStateDir func(gameID string) (string, error)
|
||||
}
|
||||
|
||||
// Service executes the start lifecycle operation.
|
||||
type Service struct {
|
||||
runtimeRecords ports.RuntimeRecordStore
|
||||
operationLogs ports.OperationLogStore
|
||||
docker ports.DockerClient
|
||||
leases ports.GameLeaseStore
|
||||
healthEvents ports.HealthEventPublisher
|
||||
notifications ports.NotificationIntentPublisher
|
||||
lobby ports.LobbyInternalClient
|
||||
|
||||
containerCfg config.ContainerConfig
|
||||
dockerCfg config.DockerConfig
|
||||
leaseTTL time.Duration
|
||||
|
||||
telemetry *telemetry.Runtime
|
||||
logger *slog.Logger
|
||||
|
||||
clock func() time.Time
|
||||
newToken func() string
|
||||
prepareStateDir func(gameID string) (string, error)
|
||||
}
|
||||
|
||||
// NewService constructs one Service from deps.
|
||||
func NewService(deps Dependencies) (*Service, error) {
|
||||
switch {
|
||||
case deps.RuntimeRecords == nil:
|
||||
return nil, errors.New("new start runtime service: nil runtime records")
|
||||
case deps.OperationLogs == nil:
|
||||
return nil, errors.New("new start runtime service: nil operation logs")
|
||||
case deps.Docker == nil:
|
||||
return nil, errors.New("new start runtime service: nil docker client")
|
||||
case deps.Leases == nil:
|
||||
return nil, errors.New("new start runtime service: nil lease store")
|
||||
case deps.HealthEvents == nil:
|
||||
return nil, errors.New("new start runtime service: nil health events publisher")
|
||||
case deps.Notifications == nil:
|
||||
return nil, errors.New("new start runtime service: nil notification publisher")
|
||||
case deps.Telemetry == nil:
|
||||
return nil, errors.New("new start runtime service: nil telemetry runtime")
|
||||
}
|
||||
if err := deps.Container.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("new start runtime service: container config: %w", err)
|
||||
}
|
||||
if err := deps.DockerCfg.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("new start runtime service: docker config: %w", err)
|
||||
}
|
||||
if err := deps.Coordination.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("new start runtime service: coordination config: %w", err)
|
||||
}
|
||||
|
||||
clock := deps.Clock
|
||||
if clock == nil {
|
||||
clock = time.Now
|
||||
}
|
||||
logger := deps.Logger
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
logger = logger.With("service", "rtmanager.startruntime")
|
||||
|
||||
newToken := deps.NewToken
|
||||
if newToken == nil {
|
||||
newToken = defaultTokenGenerator()
|
||||
}
|
||||
prepareStateDir := deps.PrepareStateDir
|
||||
if prepareStateDir == nil {
|
||||
prepareStateDir = newDefaultStateDirPreparer(deps.Container)
|
||||
}
|
||||
|
||||
return &Service{
|
||||
runtimeRecords: deps.RuntimeRecords,
|
||||
operationLogs: deps.OperationLogs,
|
||||
docker: deps.Docker,
|
||||
leases: deps.Leases,
|
||||
healthEvents: deps.HealthEvents,
|
||||
notifications: deps.Notifications,
|
||||
lobby: deps.Lobby,
|
||||
containerCfg: deps.Container,
|
||||
dockerCfg: deps.DockerCfg,
|
||||
leaseTTL: deps.Coordination.GameLeaseTTL,
|
||||
telemetry: deps.Telemetry,
|
||||
logger: logger,
|
||||
clock: clock,
|
||||
newToken: newToken,
|
||||
prepareStateDir: prepareStateDir,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Handle executes one start operation end-to-end. The Go-level error
|
||||
// return is reserved for non-business failures (nil context, nil
|
||||
// receiver). Every business outcome — fresh success, idempotent
|
||||
// replay, or any of the stable failure modes — flows through Result.
|
||||
func (service *Service) Handle(ctx context.Context, input Input) (Result, error) {
|
||||
if service == nil {
|
||||
return Result{}, errors.New("start runtime: nil service")
|
||||
}
|
||||
if ctx == nil {
|
||||
return Result{}, errors.New("start runtime: nil context")
|
||||
}
|
||||
|
||||
opStartedAt := service.clock().UTC()
|
||||
|
||||
if err := input.Validate(); err != nil {
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeStartConfigInvalid,
|
||||
errorMessage: err.Error(),
|
||||
notificationType: notificationintent.NotificationTypeRuntimeStartConfigInvalid,
|
||||
}), nil
|
||||
}
|
||||
|
||||
token := service.newToken()
|
||||
leaseStart := service.clock()
|
||||
acquired, err := service.leases.TryAcquire(ctx, input.GameID, token, service.leaseTTL)
|
||||
service.telemetry.RecordLeaseAcquireLatency(ctx, service.clock().Sub(leaseStart))
|
||||
if err != nil {
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeServiceUnavailable,
|
||||
errorMessage: fmt.Sprintf("acquire game lease: %s", err.Error()),
|
||||
}), nil
|
||||
}
|
||||
if !acquired {
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeConflict,
|
||||
errorMessage: "another lifecycle operation is in progress for this game",
|
||||
}), nil
|
||||
}
|
||||
defer service.releaseLease(ctx, input.GameID, token)
|
||||
|
||||
return service.runUnderLease(ctx, input, opStartedAt)
|
||||
}
|
||||
|
||||
// Run executes the start lifecycle assuming the per-game lease is
|
||||
// already held by the caller. The method is reserved for orchestrator
|
||||
// services in `internal/service/` that compose start with another
|
||||
// operation under a single outer lease (restart and patch). External
|
||||
// callers must use Handle, which acquires and releases the lease
|
||||
// itself.
|
||||
//
|
||||
// Run still validates input and reports business outcomes through
|
||||
// Result; the Go-level error return is reserved for non-business
|
||||
// failures (nil context, nil receiver). Operation log entries,
|
||||
// telemetry counters, health events and admin-only notification
|
||||
// intents fire identically to Handle.
|
||||
func (service *Service) Run(ctx context.Context, input Input) (Result, error) {
|
||||
if service == nil {
|
||||
return Result{}, errors.New("start runtime: nil service")
|
||||
}
|
||||
if ctx == nil {
|
||||
return Result{}, errors.New("start runtime: nil context")
|
||||
}
|
||||
|
||||
opStartedAt := service.clock().UTC()
|
||||
|
||||
if err := input.Validate(); err != nil {
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeStartConfigInvalid,
|
||||
errorMessage: err.Error(),
|
||||
notificationType: notificationintent.NotificationTypeRuntimeStartConfigInvalid,
|
||||
}), nil
|
||||
}
|
||||
|
||||
return service.runUnderLease(ctx, input, opStartedAt)
|
||||
}
|
||||
|
||||
// runUnderLease executes the post-validation, lease-protected start
|
||||
// steps shared by Handle and Run. Callers must validate input and
|
||||
// acquire the lease (when applicable) before invocation.
|
||||
func (service *Service) runUnderLease(ctx context.Context, input Input, opStartedAt time.Time) (Result, error) {
|
||||
existing, hasExisting, err := service.loadExisting(ctx, input.GameID)
|
||||
if err != nil {
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeInternal,
|
||||
errorMessage: fmt.Sprintf("load runtime record: %s", err.Error()),
|
||||
}), nil
|
||||
}
|
||||
if hasExisting && existing.Status == runtime.StatusRunning {
|
||||
if existing.CurrentImageRef == input.ImageRef {
|
||||
return service.recordReplayNoOp(ctx, opStartedAt, input, existing), nil
|
||||
}
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeConflict,
|
||||
errorMessage: fmt.Sprintf("runtime already running with image_ref %q", existing.CurrentImageRef),
|
||||
}), nil
|
||||
}
|
||||
|
||||
service.fetchLobbyDiagnostic(ctx, input.GameID)
|
||||
|
||||
if err := validateImageRef(input.ImageRef); err != nil {
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeStartConfigInvalid,
|
||||
errorMessage: fmt.Sprintf("invalid image_ref: %s", err.Error()),
|
||||
notificationType: notificationintent.NotificationTypeRuntimeStartConfigInvalid,
|
||||
}), nil
|
||||
}
|
||||
|
||||
if err := service.docker.EnsureNetwork(ctx, service.dockerCfg.Network); err != nil {
|
||||
if errors.Is(err, ports.ErrNetworkMissing) {
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeStartConfigInvalid,
|
||||
errorMessage: fmt.Sprintf("docker network %q is missing", service.dockerCfg.Network),
|
||||
notificationType: notificationintent.NotificationTypeRuntimeStartConfigInvalid,
|
||||
}), nil
|
||||
}
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeServiceUnavailable,
|
||||
errorMessage: fmt.Sprintf("ensure docker network: %s", err.Error()),
|
||||
}), nil
|
||||
}
|
||||
|
||||
if err := service.docker.PullImage(ctx, input.ImageRef, ports.PullPolicy(service.dockerCfg.PullPolicy)); err != nil {
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeImagePullFailed,
|
||||
errorMessage: err.Error(),
|
||||
notificationType: notificationintent.NotificationTypeRuntimeImagePullFailed,
|
||||
}), nil
|
||||
}
|
||||
|
||||
imageInspect, err := service.docker.InspectImage(ctx, input.ImageRef)
|
||||
if err != nil {
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeImagePullFailed,
|
||||
errorMessage: fmt.Sprintf("inspect image: %s", err.Error()),
|
||||
notificationType: notificationintent.NotificationTypeRuntimeImagePullFailed,
|
||||
}), nil
|
||||
}
|
||||
cpuQuota, memory, pidsLimit := service.resolveLimits(imageInspect.Labels)
|
||||
|
||||
statePath, err := service.prepareStateDir(input.GameID)
|
||||
if err != nil {
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeStartConfigInvalid,
|
||||
errorMessage: fmt.Sprintf("prepare state directory: %s", err.Error()),
|
||||
notificationType: notificationintent.NotificationTypeRuntimeStartConfigInvalid,
|
||||
}), nil
|
||||
}
|
||||
|
||||
hostname := containerHostname(input.GameID)
|
||||
spec := ports.RunSpec{
|
||||
Name: hostname,
|
||||
Image: input.ImageRef,
|
||||
Hostname: hostname,
|
||||
Network: service.dockerCfg.Network,
|
||||
Env: service.buildEnv(),
|
||||
Labels: service.buildLabels(input.GameID, input.ImageRef, opStartedAt),
|
||||
BindMounts: []ports.BindMount{{
|
||||
HostPath: statePath,
|
||||
MountPath: service.containerCfg.EngineStateMountPath,
|
||||
ReadOnly: false,
|
||||
}},
|
||||
LogDriver: service.dockerCfg.LogDriver,
|
||||
LogOpts: parseLogOpts(service.dockerCfg.LogOpts),
|
||||
CPUQuota: cpuQuota,
|
||||
Memory: memory,
|
||||
PIDsLimit: pidsLimit,
|
||||
}
|
||||
runResult, err := service.docker.Run(ctx, spec)
|
||||
if err != nil {
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeContainerStartFailed,
|
||||
errorMessage: err.Error(),
|
||||
notificationType: notificationintent.NotificationTypeRuntimeContainerStartFailed,
|
||||
}), nil
|
||||
}
|
||||
|
||||
createdAt := opStartedAt
|
||||
if hasExisting && !existing.CreatedAt.IsZero() {
|
||||
createdAt = existing.CreatedAt
|
||||
}
|
||||
startedAt := runResult.StartedAt
|
||||
record := runtime.RuntimeRecord{
|
||||
GameID: input.GameID,
|
||||
Status: runtime.StatusRunning,
|
||||
CurrentContainerID: runResult.ContainerID,
|
||||
CurrentImageRef: input.ImageRef,
|
||||
EngineEndpoint: runResult.EngineEndpoint,
|
||||
StatePath: statePath,
|
||||
DockerNetwork: service.dockerCfg.Network,
|
||||
StartedAt: &startedAt,
|
||||
LastOpAt: startedAt,
|
||||
CreatedAt: createdAt,
|
||||
}
|
||||
if err := service.runtimeRecords.Upsert(ctx, record); err != nil {
|
||||
service.bestEffortRemove(input.GameID, runResult.ContainerID)
|
||||
return service.recordFailure(ctx, failureCtx{
|
||||
opStartedAt: opStartedAt,
|
||||
input: input,
|
||||
errorCode: ErrorCodeContainerStartFailed,
|
||||
errorMessage: fmt.Sprintf("upsert runtime record: %s", err.Error()),
|
||||
containerID: runResult.ContainerID,
|
||||
notificationType: notificationintent.NotificationTypeRuntimeContainerStartFailed,
|
||||
}), nil
|
||||
}
|
||||
|
||||
finishedAt := service.clock().UTC()
|
||||
service.bestEffortAppend(ctx, operation.OperationEntry{
|
||||
GameID: input.GameID,
|
||||
OpKind: operation.OpKindStart,
|
||||
OpSource: input.OpSource,
|
||||
SourceRef: input.SourceRef,
|
||||
ImageRef: input.ImageRef,
|
||||
ContainerID: runResult.ContainerID,
|
||||
Outcome: operation.OutcomeSuccess,
|
||||
StartedAt: opStartedAt,
|
||||
FinishedAt: &finishedAt,
|
||||
})
|
||||
service.bestEffortPublishHealth(ctx, ports.HealthEventEnvelope{
|
||||
GameID: input.GameID,
|
||||
ContainerID: runResult.ContainerID,
|
||||
EventType: health.EventTypeContainerStarted,
|
||||
OccurredAt: startedAt,
|
||||
Details: containerStartedDetails(input.ImageRef),
|
||||
})
|
||||
|
||||
service.telemetry.RecordStartOutcome(ctx, string(operation.OutcomeSuccess), "", string(input.OpSource))
|
||||
service.telemetry.RecordHealthEvent(ctx, string(health.EventTypeContainerStarted))
|
||||
|
||||
logArgs := []any{
|
||||
"game_id", input.GameID,
|
||||
"container_id", runResult.ContainerID,
|
||||
"image_ref", input.ImageRef,
|
||||
"op_source", string(input.OpSource),
|
||||
}
|
||||
logArgs = append(logArgs, logging.ContextAttrs(ctx)...)
|
||||
service.logger.InfoContext(ctx, "runtime started", logArgs...)
|
||||
|
||||
return Result{
|
||||
Record: record,
|
||||
Outcome: operation.OutcomeSuccess,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// failureCtx groups the inputs to recordFailure so the Handle method
|
||||
// stays readable.
|
||||
type failureCtx struct {
|
||||
opStartedAt time.Time
|
||||
input Input
|
||||
errorCode string
|
||||
errorMessage string
|
||||
containerID string
|
||||
notificationType notificationintent.NotificationType
|
||||
}
|
||||
|
||||
// recordFailure records the failure operation_log entry, publishes the
|
||||
// matching admin-only notification intent (when applicable), and emits
|
||||
// telemetry. All side effects are best-effort; a downstream failure is
|
||||
// logged but does not change the returned Result.
|
||||
func (service *Service) recordFailure(ctx context.Context, fc failureCtx) Result {
|
||||
finishedAt := service.clock().UTC()
|
||||
service.bestEffortAppend(ctx, operation.OperationEntry{
|
||||
GameID: fc.input.GameID,
|
||||
OpKind: operation.OpKindStart,
|
||||
OpSource: fc.input.OpSource,
|
||||
SourceRef: fc.input.SourceRef,
|
||||
ImageRef: fc.input.ImageRef,
|
||||
ContainerID: fc.containerID,
|
||||
Outcome: operation.OutcomeFailure,
|
||||
ErrorCode: fc.errorCode,
|
||||
ErrorMessage: fc.errorMessage,
|
||||
StartedAt: fc.opStartedAt,
|
||||
FinishedAt: &finishedAt,
|
||||
})
|
||||
|
||||
if fc.notificationType != "" {
|
||||
service.bestEffortNotify(ctx, fc)
|
||||
}
|
||||
|
||||
service.telemetry.RecordStartOutcome(ctx, string(operation.OutcomeFailure), fc.errorCode, string(fc.input.OpSource))
|
||||
|
||||
logArgs := []any{
|
||||
"game_id", fc.input.GameID,
|
||||
"image_ref", fc.input.ImageRef,
|
||||
"op_source", string(fc.input.OpSource),
|
||||
"error_code", fc.errorCode,
|
||||
"error_message", fc.errorMessage,
|
||||
}
|
||||
logArgs = append(logArgs, logging.ContextAttrs(ctx)...)
|
||||
service.logger.WarnContext(ctx, "runtime start failed", logArgs...)
|
||||
|
||||
return Result{
|
||||
Outcome: operation.OutcomeFailure,
|
||||
ErrorCode: fc.errorCode,
|
||||
ErrorMessage: fc.errorMessage,
|
||||
}
|
||||
}
|
||||
|
||||
// recordReplayNoOp records the idempotent replay outcome and returns
|
||||
// the existing record. The operation_log entry is appended best-effort
|
||||
// so audit history captures the replay; telemetry counts the call as a
|
||||
// successful start with `error_code=replay_no_op`.
|
||||
func (service *Service) recordReplayNoOp(ctx context.Context, opStartedAt time.Time, input Input, existing runtime.RuntimeRecord) Result {
|
||||
finishedAt := service.clock().UTC()
|
||||
service.bestEffortAppend(ctx, operation.OperationEntry{
|
||||
GameID: input.GameID,
|
||||
OpKind: operation.OpKindStart,
|
||||
OpSource: input.OpSource,
|
||||
SourceRef: input.SourceRef,
|
||||
ImageRef: input.ImageRef,
|
||||
ContainerID: existing.CurrentContainerID,
|
||||
Outcome: operation.OutcomeSuccess,
|
||||
ErrorCode: ErrorCodeReplayNoOp,
|
||||
StartedAt: opStartedAt,
|
||||
FinishedAt: &finishedAt,
|
||||
})
|
||||
service.telemetry.RecordStartOutcome(ctx, string(operation.OutcomeSuccess), ErrorCodeReplayNoOp, string(input.OpSource))
|
||||
|
||||
logArgs := []any{
|
||||
"game_id", input.GameID,
|
||||
"container_id", existing.CurrentContainerID,
|
||||
"image_ref", input.ImageRef,
|
||||
"op_source", string(input.OpSource),
|
||||
}
|
||||
logArgs = append(logArgs, logging.ContextAttrs(ctx)...)
|
||||
service.logger.InfoContext(ctx, "runtime start replay no-op", logArgs...)
|
||||
|
||||
return Result{
|
||||
Record: existing,
|
||||
Outcome: operation.OutcomeSuccess,
|
||||
ErrorCode: ErrorCodeReplayNoOp,
|
||||
}
|
||||
}
|
||||
|
||||
// loadExisting reads the runtime record for gameID. The boolean return
|
||||
// reports whether a record exists; ErrNotFound is translated to
|
||||
// (zero, false, nil) so the caller does not branch on the sentinel
|
||||
// elsewhere.
|
||||
func (service *Service) loadExisting(ctx context.Context, gameID string) (runtime.RuntimeRecord, bool, error) {
|
||||
record, err := service.runtimeRecords.Get(ctx, gameID)
|
||||
switch {
|
||||
case errors.Is(err, runtime.ErrNotFound):
|
||||
return runtime.RuntimeRecord{}, false, nil
|
||||
case err != nil:
|
||||
return runtime.RuntimeRecord{}, false, err
|
||||
default:
|
||||
return record, true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// fetchLobbyDiagnostic best-effort enriches the request log with the
|
||||
// Lobby-side game record. A nil Lobby client or any transport failure
|
||||
// is logged and the start operation continues.
|
||||
func (service *Service) fetchLobbyDiagnostic(ctx context.Context, gameID string) {
|
||||
if service.lobby == nil {
|
||||
return
|
||||
}
|
||||
record, err := service.lobby.GetGame(ctx, gameID)
|
||||
if err != nil {
|
||||
service.logger.DebugContext(ctx, "lobby diagnostic fetch failed",
|
||||
"game_id", gameID,
|
||||
"err", err.Error(),
|
||||
)
|
||||
return
|
||||
}
|
||||
service.logger.DebugContext(ctx, "lobby diagnostic fetched",
|
||||
"game_id", gameID,
|
||||
"lobby_status", record.Status,
|
||||
"lobby_target_engine_version", record.TargetEngineVersion,
|
||||
)
|
||||
}
|
||||
|
||||
// resolveLimits derives the per-container resource limits from the
|
||||
// resolved image's labels with config-driven fallbacks. Unparseable
|
||||
// label values silently fall back to the configured default; operators
|
||||
// see the chosen value through `rtmanager.docker_op_latency` and start
|
||||
// logs.
|
||||
func (service *Service) resolveLimits(labels map[string]string) (cpuQuota float64, memory string, pidsLimit int) {
|
||||
cpuQuota = service.containerCfg.DefaultCPUQuota
|
||||
memory = service.containerCfg.DefaultMemory
|
||||
pidsLimit = service.containerCfg.DefaultPIDsLimit
|
||||
|
||||
if raw, ok := labels[imageLabelCPUQuota]; ok {
|
||||
if value, err := strconv.ParseFloat(raw, 64); err == nil && value > 0 {
|
||||
cpuQuota = value
|
||||
}
|
||||
}
|
||||
if raw, ok := labels[imageLabelMemory]; ok && strings.TrimSpace(raw) != "" {
|
||||
memory = raw
|
||||
}
|
||||
if raw, ok := labels[imageLabelPIDsLimit]; ok {
|
||||
if value, err := strconv.Atoi(raw); err == nil && value > 0 {
|
||||
pidsLimit = value
|
||||
}
|
||||
}
|
||||
return cpuQuota, memory, pidsLimit
|
||||
}
|
||||
|
||||
// buildEnv assembles the env-var map handed to the engine. Both the
|
||||
// configured primary name and `STORAGE_PATH` are set per
|
||||
// `rtmanager/README.md §Container Model` v1 backward compatibility.
|
||||
func (service *Service) buildEnv() map[string]string {
|
||||
mount := service.containerCfg.EngineStateMountPath
|
||||
env := map[string]string{
|
||||
service.containerCfg.EngineStateEnvName: mount,
|
||||
EngineStateBackCompatEnvName: mount,
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
// buildLabels assembles the container labels per
|
||||
// `rtmanager/README.md §Container Model`.
|
||||
func (service *Service) buildLabels(gameID, imageRef string, startedAt time.Time) map[string]string {
|
||||
return map[string]string{
|
||||
LabelOwner: LabelOwnerValue,
|
||||
LabelKind: LabelKindValue,
|
||||
LabelGameID: gameID,
|
||||
LabelEngineImageRef: imageRef,
|
||||
LabelStartedAtMs: strconv.FormatInt(startedAt.UTC().UnixMilli(), 10),
|
||||
}
|
||||
}
|
||||
|
||||
// releaseLease releases the per-game lease in a fresh background
|
||||
// context so a canceled request context does not leave the lease
|
||||
// pinned for its TTL.
|
||||
func (service *Service) releaseLease(ctx context.Context, gameID, token string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), leaseReleaseTimeout)
|
||||
defer cancel()
|
||||
if err := service.leases.Release(cleanupCtx, gameID, token); err != nil {
|
||||
service.logger.WarnContext(ctx, "release game lease",
|
||||
"game_id", gameID,
|
||||
"err", err.Error(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// bestEffortAppend writes one operation_log entry. A failure is logged
|
||||
// and discarded; the durable runtime record (or its absence) remains
|
||||
// the source of truth.
|
||||
func (service *Service) bestEffortAppend(ctx context.Context, entry operation.OperationEntry) {
|
||||
if _, err := service.operationLogs.Append(ctx, entry); err != nil {
|
||||
service.logger.ErrorContext(ctx, "append operation log",
|
||||
"game_id", entry.GameID,
|
||||
"op_kind", string(entry.OpKind),
|
||||
"outcome", string(entry.Outcome),
|
||||
"error_code", entry.ErrorCode,
|
||||
"err", err.Error(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// bestEffortPublishHealth emits one health event + snapshot upsert.
|
||||
// Failures degrade silently per `rtmanager/README.md §Notification
|
||||
// Contracts`; the runtime record remains the source of truth.
|
||||
func (service *Service) bestEffortPublishHealth(ctx context.Context, envelope ports.HealthEventEnvelope) {
|
||||
if err := service.healthEvents.Publish(ctx, envelope); err != nil {
|
||||
service.logger.ErrorContext(ctx, "publish health event",
|
||||
"game_id", envelope.GameID,
|
||||
"container_id", envelope.ContainerID,
|
||||
"event_type", string(envelope.EventType),
|
||||
"err", err.Error(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// bestEffortNotify publishes one admin-only failure intent. Failures
|
||||
// degrade silently because the source business state already reflects
|
||||
// the outcome.
|
||||
func (service *Service) bestEffortNotify(ctx context.Context, fc failureCtx) {
|
||||
intent, err := buildFailureIntent(fc, service.clock().UTC())
|
||||
if err != nil {
|
||||
service.logger.ErrorContext(ctx, "build notification intent",
|
||||
"game_id", fc.input.GameID,
|
||||
"notification_type", string(fc.notificationType),
|
||||
"err", err.Error(),
|
||||
)
|
||||
return
|
||||
}
|
||||
if err := service.notifications.Publish(ctx, intent); err != nil {
|
||||
service.logger.ErrorContext(ctx, "publish notification intent",
|
||||
"game_id", fc.input.GameID,
|
||||
"notification_type", string(fc.notificationType),
|
||||
"err", err.Error(),
|
||||
)
|
||||
return
|
||||
}
|
||||
service.telemetry.RecordNotificationIntent(ctx, string(fc.notificationType))
|
||||
}
|
||||
|
||||
// bestEffortRemove forces removal of a container left running by a
|
||||
// failed start that progressed past Run but failed to register the
|
||||
// runtime record. Failures degrade silently — the reconciler adopts
|
||||
// orphans the periodic pass observes.
|
||||
func (service *Service) bestEffortRemove(gameID, containerID string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), leaseReleaseTimeout)
|
||||
defer cancel()
|
||||
if err := service.docker.Remove(cleanupCtx, containerID); err != nil {
|
||||
service.logger.ErrorContext(cleanupCtx, "rollback container after upsert failure",
|
||||
"game_id", gameID,
|
||||
"container_id", containerID,
|
||||
"err", err.Error(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// containerHostname builds the per-game hostname that doubles as the
|
||||
// Docker container name.
|
||||
func containerHostname(gameID string) string {
|
||||
return HostnamePrefix + gameID
|
||||
}
|
||||
|
||||
// containerStartedDetails builds the `details` payload required by the
|
||||
// `container_started` AsyncAPI variant.
|
||||
func containerStartedDetails(imageRef string) json.RawMessage {
|
||||
payload := map[string]string{"image_ref": imageRef}
|
||||
encoded, _ := json.Marshal(payload)
|
||||
return encoded
|
||||
}
|
||||
|
||||
// validateImageRef rejects malformed Docker references before any
|
||||
// daemon round-trip. The validation surfaces as `start_config_invalid`;
|
||||
// daemon-side rejections after a valid parse are reported as
|
||||
// `image_pull_failed`.
|
||||
func validateImageRef(ref string) error {
|
||||
if strings.TrimSpace(ref) == "" {
|
||||
return fmt.Errorf("image ref must not be empty")
|
||||
}
|
||||
if _, err := reference.ParseNormalizedNamed(ref); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseLogOpts turns the `key=value,key2=value2` shape of the
|
||||
// `RTMANAGER_DOCKER_LOG_OPTS` config into a map suitable for the
|
||||
// Docker SDK. Empty input returns nil so the SDK uses driver defaults.
|
||||
func parseLogOpts(raw string) map[string]string {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string)
|
||||
for part := range strings.SplitSeq(raw, ",") {
|
||||
entry := strings.TrimSpace(part)
|
||||
if entry == "" {
|
||||
continue
|
||||
}
|
||||
index := strings.IndexByte(entry, '=')
|
||||
if index <= 0 {
|
||||
continue
|
||||
}
|
||||
out[entry[:index]] = entry[index+1:]
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildFailureIntent constructs the admin-only notification intent for
|
||||
// fc. The idempotency key is scoped per (notification_type, game_id,
|
||||
// image_ref, attempted_at_ms) so the same failure observed twice is
|
||||
// recognised as a duplicate by Notification Service.
|
||||
func buildFailureIntent(fc failureCtx, attemptedAt time.Time) (notificationintent.Intent, error) {
|
||||
attemptedAtMs := attemptedAt.UnixMilli()
|
||||
idempotencyKey := fmt.Sprintf("%s.%s.%d", fc.notificationType, fc.input.GameID, attemptedAtMs)
|
||||
metadata := notificationintent.Metadata{
|
||||
IdempotencyKey: idempotencyKey,
|
||||
OccurredAt: attemptedAt,
|
||||
}
|
||||
|
||||
switch fc.notificationType {
|
||||
case notificationintent.NotificationTypeRuntimeImagePullFailed:
|
||||
return notificationintent.NewRuntimeImagePullFailedIntent(metadata, notificationintent.RuntimeImagePullFailedPayload{
|
||||
GameID: fc.input.GameID,
|
||||
ImageRef: fc.input.ImageRef,
|
||||
ErrorCode: fc.errorCode,
|
||||
ErrorMessage: fc.errorMessage,
|
||||
AttemptedAtMs: attemptedAtMs,
|
||||
})
|
||||
case notificationintent.NotificationTypeRuntimeContainerStartFailed:
|
||||
return notificationintent.NewRuntimeContainerStartFailedIntent(metadata, notificationintent.RuntimeContainerStartFailedPayload{
|
||||
GameID: fc.input.GameID,
|
||||
ImageRef: fc.input.ImageRef,
|
||||
ErrorCode: fc.errorCode,
|
||||
ErrorMessage: fc.errorMessage,
|
||||
AttemptedAtMs: attemptedAtMs,
|
||||
})
|
||||
case notificationintent.NotificationTypeRuntimeStartConfigInvalid:
|
||||
return notificationintent.NewRuntimeStartConfigInvalidIntent(metadata, notificationintent.RuntimeStartConfigInvalidPayload{
|
||||
GameID: fc.input.GameID,
|
||||
ImageRef: fc.input.ImageRef,
|
||||
ErrorCode: fc.errorCode,
|
||||
ErrorMessage: fc.errorMessage,
|
||||
AttemptedAtMs: attemptedAtMs,
|
||||
})
|
||||
default:
|
||||
return notificationintent.Intent{}, fmt.Errorf("unsupported notification type %q", fc.notificationType)
|
||||
}
|
||||
}
|
||||
|
||||
// defaultTokenGenerator returns a function that produces 32-byte
|
||||
// base64url-encoded tokens. The randomness source is `crypto/rand`;
|
||||
// failures fall back to a deterministic-looking but invalid token so
|
||||
// the caller observes a TryAcquire collision rather than a panic on a
|
||||
// degraded entropy source.
|
||||
func defaultTokenGenerator() func() string {
|
||||
return func() string {
|
||||
var buf [32]byte
|
||||
if _, err := rand.Read(buf[:]); err != nil {
|
||||
return "rtmanager-fallback-token"
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(buf[:])
|
||||
}
|
||||
}
|
||||
|
||||
// newDefaultStateDirPreparer returns a function that creates the
|
||||
// per-game state directory under cfg.GameStateRoot with the configured
|
||||
// permissions and ownership. The function is overridable through
|
||||
// Dependencies.PrepareStateDir; tests inject a temporary-dir fake.
|
||||
func newDefaultStateDirPreparer(cfg config.ContainerConfig) func(gameID string) (string, error) {
|
||||
mode := os.FileMode(cfg.GameStateDirMode)
|
||||
uid := cfg.GameStateOwnerUID
|
||||
gid := cfg.GameStateOwnerGID
|
||||
root := cfg.GameStateRoot
|
||||
return func(gameID string) (string, error) {
|
||||
path := filepath.Join(root, gameID)
|
||||
if err := os.MkdirAll(path, mode); err != nil {
|
||||
return "", fmt.Errorf("create state dir %q: %w", path, err)
|
||||
}
|
||||
if err := os.Chmod(path, mode); err != nil {
|
||||
return "", fmt.Errorf("chmod state dir %q: %w", path, err)
|
||||
}
|
||||
if err := os.Chown(path, uid, gid); err != nil {
|
||||
return "", fmt.Errorf("chown state dir %q: %w", path, err)
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user