feat: runtime manager
This commit is contained in:
Ilia Denisov
@@ -0,0 +1,117 @@
|
||||
// Package gamelease implements the Redis-backed adapter for
|
||||
// `ports.GameLeaseStore`.
|
||||
//
|
||||
// The lease guards every lifecycle operation Runtime Manager runs
|
||||
// against one game (start, stop, restart, patch, cleanup, plus the
|
||||
// reconciler's drift mutations). Acquisition uses `SET NX PX <ttl>`
|
||||
// with a random caller token; release runs a Lua compare-and-delete
|
||||
// so a holder that lost the lease through TTL expiry cannot wipe
|
||||
// another caller's claim.
|
||||
package gamelease
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"galaxy/rtmanager/internal/adapters/redisstate"
|
||||
"galaxy/rtmanager/internal/ports"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// releaseScript removes the per-game lease only when the supplied token
|
||||
// still owns it. Compare-and-delete prevents a TTL-expired holder from
|
||||
// clearing another caller's claim.
|
||||
var releaseScript = redis.NewScript(`
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("DEL", KEYS[1])
|
||||
end
|
||||
return 0
|
||||
`)
|
||||
|
||||
// Config configures one Redis-backed game lease store instance. The
|
||||
// store does not own the redis client lifecycle; the caller (typically
|
||||
// the service runtime) opens and closes it.
|
||||
type Config struct {
|
||||
// Client stores the Redis client the store uses for every command.
|
||||
Client *redis.Client
|
||||
}
|
||||
|
||||
// Store persists the per-game lifecycle lease in Redis.
|
||||
type Store struct {
|
||||
client *redis.Client
|
||||
keys redisstate.Keyspace
|
||||
}
|
||||
|
||||
// New constructs one Redis-backed game lease store from cfg.
|
||||
func New(cfg Config) (*Store, error) {
|
||||
if cfg.Client == nil {
|
||||
return nil, errors.New("new rtmanager game lease store: nil redis client")
|
||||
}
|
||||
return &Store{
|
||||
client: cfg.Client,
|
||||
keys: redisstate.Keyspace{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TryAcquire attempts to acquire the per-game lease for gameID owned by
|
||||
// token for ttl. The acquired return is true on a successful claim and
|
||||
// false when another caller still owns the lease. A non-nil error
|
||||
// reports a transport failure and must not be confused with a missed
|
||||
// lease.
|
||||
func (store *Store) TryAcquire(ctx context.Context, gameID, token string, ttl time.Duration) (bool, error) {
|
||||
if store == nil || store.client == nil {
|
||||
return false, errors.New("try acquire game lease: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return false, errors.New("try acquire game lease: nil context")
|
||||
}
|
||||
if strings.TrimSpace(gameID) == "" {
|
||||
return false, errors.New("try acquire game lease: game id must not be empty")
|
||||
}
|
||||
if strings.TrimSpace(token) == "" {
|
||||
return false, errors.New("try acquire game lease: token must not be empty")
|
||||
}
|
||||
if ttl <= 0 {
|
||||
return false, errors.New("try acquire game lease: ttl must be positive")
|
||||
}
|
||||
|
||||
acquired, err := store.client.SetNX(ctx, store.keys.GameLease(gameID), token, ttl).Result()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("try acquire game lease: %w", err)
|
||||
}
|
||||
return acquired, nil
|
||||
}
|
||||
|
||||
// Release removes the per-game lease for gameID only when token still
|
||||
// matches the stored owner value. A token mismatch is a silent no-op.
|
||||
func (store *Store) Release(ctx context.Context, gameID, token string) error {
|
||||
if store == nil || store.client == nil {
|
||||
return errors.New("release game lease: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return errors.New("release game lease: nil context")
|
||||
}
|
||||
if strings.TrimSpace(gameID) == "" {
|
||||
return errors.New("release game lease: game id must not be empty")
|
||||
}
|
||||
if strings.TrimSpace(token) == "" {
|
||||
return errors.New("release game lease: token must not be empty")
|
||||
}
|
||||
|
||||
if err := releaseScript.Run(
|
||||
ctx,
|
||||
store.client,
|
||||
[]string{store.keys.GameLease(gameID)},
|
||||
token,
|
||||
).Err(); err != nil {
|
||||
return fmt.Errorf("release game lease: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Compile-time assertion: Store implements ports.GameLeaseStore.
|
||||
var _ ports.GameLeaseStore = (*Store)(nil)
|
||||
@@ -0,0 +1,133 @@
|
||||
package gamelease_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"galaxy/rtmanager/internal/adapters/redisstate"
|
||||
"galaxy/rtmanager/internal/adapters/redisstate/gamelease"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newLeaseStore(t *testing.T) (*gamelease.Store, *miniredis.Miniredis) {
|
||||
t.Helper()
|
||||
server := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
store, err := gamelease.New(gamelease.Config{Client: client})
|
||||
require.NoError(t, err)
|
||||
return store, server
|
||||
}
|
||||
|
||||
func TestNewRejectsNilClient(t *testing.T) {
|
||||
_, err := gamelease.New(gamelease.Config{})
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestTryAcquireSetsKeyAndTTL(t *testing.T) {
|
||||
store, server := newLeaseStore(t)
|
||||
|
||||
acquired, err := store.TryAcquire(context.Background(), "game-1", "token-A", time.Minute)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, acquired)
|
||||
|
||||
key := redisstate.Keyspace{}.GameLease("game-1")
|
||||
assert.True(t, server.Exists(key), "key %q must exist after TryAcquire", key)
|
||||
|
||||
stored, err := server.Get(key)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "token-A", stored)
|
||||
|
||||
// TTL must be positive (miniredis returns the remaining duration).
|
||||
ttl := server.TTL(key)
|
||||
assert.Greater(t, ttl, time.Duration(0))
|
||||
}
|
||||
|
||||
func TestTryAcquireReturnsFalseWhenAlreadyHeld(t *testing.T) {
|
||||
store, _ := newLeaseStore(t)
|
||||
|
||||
acquired, err := store.TryAcquire(context.Background(), "game-1", "token-A", time.Minute)
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
|
||||
acquired, err = store.TryAcquire(context.Background(), "game-1", "token-B", time.Minute)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, acquired)
|
||||
}
|
||||
|
||||
func TestReleaseRemovesKeyForOwnerToken(t *testing.T) {
|
||||
store, server := newLeaseStore(t)
|
||||
|
||||
_, err := store.TryAcquire(context.Background(), "game-1", "token-A", time.Minute)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, store.Release(context.Background(), "game-1", "token-A"))
|
||||
|
||||
key := redisstate.Keyspace{}.GameLease("game-1")
|
||||
assert.False(t, server.Exists(key), "key %q must be deleted after Release", key)
|
||||
}
|
||||
|
||||
func TestReleaseIsNoOpForForeignToken(t *testing.T) {
|
||||
store, server := newLeaseStore(t)
|
||||
|
||||
_, err := store.TryAcquire(context.Background(), "game-1", "token-A", time.Minute)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, store.Release(context.Background(), "game-1", "token-B"))
|
||||
|
||||
key := redisstate.Keyspace{}.GameLease("game-1")
|
||||
assert.True(t, server.Exists(key), "key %q must still exist when foreign token is released", key)
|
||||
|
||||
stored, err := server.Get(key)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "token-A", stored)
|
||||
}
|
||||
|
||||
func TestTryAcquireSucceedsAfterTTLExpiry(t *testing.T) {
|
||||
store, server := newLeaseStore(t)
|
||||
|
||||
acquired, err := store.TryAcquire(context.Background(), "game-1", "token-A", time.Minute)
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
|
||||
server.FastForward(2 * time.Minute)
|
||||
|
||||
acquired, err = store.TryAcquire(context.Background(), "game-1", "token-B", time.Minute)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, acquired)
|
||||
}
|
||||
|
||||
func TestTryAcquireRejectsInvalidArguments(t *testing.T) {
|
||||
store, _ := newLeaseStore(t)
|
||||
|
||||
_, err := store.TryAcquire(context.Background(), "", "token", time.Minute)
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = store.TryAcquire(context.Background(), "game-1", "", time.Minute)
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = store.TryAcquire(context.Background(), "game-1", "token", 0)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestReleaseRejectsInvalidArguments(t *testing.T) {
|
||||
store, _ := newLeaseStore(t)
|
||||
|
||||
require.Error(t, store.Release(context.Background(), "", "token"))
|
||||
require.Error(t, store.Release(context.Background(), "game-1", ""))
|
||||
}
|
||||
|
||||
func TestKeyspaceGameLeaseIsPrefixedAndEncoded(t *testing.T) {
|
||||
key := redisstate.Keyspace{}.GameLease("game with spaces")
|
||||
assert.NotEmpty(t, key)
|
||||
assert.Contains(t, key, "rtmanager:game_lease:")
|
||||
suffix := key[len("rtmanager:game_lease:"):]
|
||||
// base64url-encoded suffix must not contain the original spaces.
|
||||
assert.NotContains(t, suffix, " ")
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
// Package redisstate hosts the Runtime Manager Redis adapters that share
|
||||
// a single keyspace. Each sibling subpackage (e.g. `streamoffsets`)
|
||||
// implements one port and uses Keyspace to compose its keys, so the
|
||||
// Redis namespace stays under one document and one prefix.
|
||||
//
|
||||
// The package itself only declares the keyspace; concrete stores live in
|
||||
// nested packages so dependencies (testcontainers, miniredis) stay out
|
||||
// of consumer build graphs that do not need them.
|
||||
package redisstate
|
||||
|
||||
import "encoding/base64"
|
||||
|
||||
// defaultPrefix is the mandatory `rtmanager:` namespace prefix shared by
|
||||
// every Runtime Manager Redis key.
|
||||
const defaultPrefix = "rtmanager:"
|
||||
|
||||
// Keyspace builds the Runtime Manager Redis keys. The namespace covers
|
||||
// the stream consumer offsets and the per-game lifecycle lease in v1.
|
||||
//
|
||||
// Dynamic key segments are encoded with base64url so raw key structure
|
||||
// does not depend on caller-provided characters; this matches the
|
||||
// encoding chosen by `lobby/internal/adapters/redisstate.Keyspace`.
|
||||
type Keyspace struct{}
|
||||
|
||||
// StreamOffset returns the Redis key that stores the last successfully
|
||||
// processed entry id for one Redis Stream consumer. The streamLabel is
|
||||
// the short logical identifier of the consumer (e.g. `start_jobs`,
|
||||
// `stop_jobs`), not the full stream name; it stays stable when the
|
||||
// underlying stream key is renamed.
|
||||
func (Keyspace) StreamOffset(streamLabel string) string {
|
||||
return defaultPrefix + "stream_offsets:" + encodeKeyComponent(streamLabel)
|
||||
}
|
||||
|
||||
// GameLease returns the Redis key that stores the per-game lifecycle
|
||||
// lease guarding start / stop / restart / patch / cleanup operations
|
||||
// against the same game. The gameID is base64url-encoded so callers can
|
||||
// pass any opaque identifier without escaping raw key characters.
|
||||
func (Keyspace) GameLease(gameID string) string {
|
||||
return defaultPrefix + "game_lease:" + encodeKeyComponent(gameID)
|
||||
}
|
||||
|
||||
func encodeKeyComponent(value string) string {
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(value))
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
// Package streamoffsets implements the Redis-backed adapter for
|
||||
// `ports.StreamOffsetStore`.
|
||||
//
|
||||
// The start-jobs and stop-jobs consumers call Load on startup to
|
||||
// resume from the persisted offset and Save after every successful
|
||||
// message handling. Keys are produced by
|
||||
// `redisstate.Keyspace.StreamOffset`, mirroring the lobby pattern.
|
||||
package streamoffsets
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"galaxy/rtmanager/internal/adapters/redisstate"
|
||||
"galaxy/rtmanager/internal/ports"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// Config configures one Redis-backed stream-offset store instance. The
|
||||
// store does not own the redis client lifecycle; the caller (typically
|
||||
// the service runtime) opens and closes it.
|
||||
type Config struct {
|
||||
// Client stores the Redis client the store uses for every command.
|
||||
Client *redis.Client
|
||||
}
|
||||
|
||||
// Store persists Runtime Manager stream consumer offsets in Redis.
|
||||
type Store struct {
|
||||
client *redis.Client
|
||||
keys redisstate.Keyspace
|
||||
}
|
||||
|
||||
// New constructs one Redis-backed stream-offset store from cfg.
|
||||
func New(cfg Config) (*Store, error) {
|
||||
if cfg.Client == nil {
|
||||
return nil, errors.New("new rtmanager stream offset store: nil redis client")
|
||||
}
|
||||
return &Store{
|
||||
client: cfg.Client,
|
||||
keys: redisstate.Keyspace{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Load returns the last processed entry id for streamLabel when one is
|
||||
// stored. A missing key returns ("", false, nil).
|
||||
func (store *Store) Load(ctx context.Context, streamLabel string) (string, bool, error) {
|
||||
if store == nil || store.client == nil {
|
||||
return "", false, errors.New("load rtmanager stream offset: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return "", false, errors.New("load rtmanager stream offset: nil context")
|
||||
}
|
||||
if strings.TrimSpace(streamLabel) == "" {
|
||||
return "", false, errors.New("load rtmanager stream offset: stream label must not be empty")
|
||||
}
|
||||
|
||||
value, err := store.client.Get(ctx, store.keys.StreamOffset(streamLabel)).Result()
|
||||
switch {
|
||||
case errors.Is(err, redis.Nil):
|
||||
return "", false, nil
|
||||
case err != nil:
|
||||
return "", false, fmt.Errorf("load rtmanager stream offset: %w", err)
|
||||
}
|
||||
return value, true, nil
|
||||
}
|
||||
|
||||
// Save stores entryID as the new offset for streamLabel. The key has no
|
||||
// TTL — offsets are durable and only overwritten by subsequent Saves.
|
||||
func (store *Store) Save(ctx context.Context, streamLabel, entryID string) error {
|
||||
if store == nil || store.client == nil {
|
||||
return errors.New("save rtmanager stream offset: nil store")
|
||||
}
|
||||
if ctx == nil {
|
||||
return errors.New("save rtmanager stream offset: nil context")
|
||||
}
|
||||
if strings.TrimSpace(streamLabel) == "" {
|
||||
return errors.New("save rtmanager stream offset: stream label must not be empty")
|
||||
}
|
||||
if strings.TrimSpace(entryID) == "" {
|
||||
return errors.New("save rtmanager stream offset: entry id must not be empty")
|
||||
}
|
||||
|
||||
if err := store.client.Set(ctx, store.keys.StreamOffset(streamLabel), entryID, 0).Err(); err != nil {
|
||||
return fmt.Errorf("save rtmanager stream offset: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ensure Store satisfies the ports.StreamOffsetStore interface at
|
||||
// compile time.
|
||||
var _ ports.StreamOffsetStore = (*Store)(nil)
|
||||
@@ -0,0 +1,86 @@
|
||||
package streamoffsets_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"galaxy/rtmanager/internal/adapters/redisstate"
|
||||
"galaxy/rtmanager/internal/adapters/redisstate/streamoffsets"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newOffsetStore(t *testing.T) (*streamoffsets.Store, *miniredis.Miniredis) {
|
||||
t.Helper()
|
||||
server := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
store, err := streamoffsets.New(streamoffsets.Config{Client: client})
|
||||
require.NoError(t, err)
|
||||
return store, server
|
||||
}
|
||||
|
||||
func TestNewRejectsNilClient(t *testing.T) {
|
||||
_, err := streamoffsets.New(streamoffsets.Config{})
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestLoadMissingReturnsNotFound(t *testing.T) {
|
||||
store, _ := newOffsetStore(t)
|
||||
|
||||
id, found, err := store.Load(context.Background(), "start_jobs")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, found)
|
||||
assert.Empty(t, id)
|
||||
}
|
||||
|
||||
func TestSaveLoadRoundTrip(t *testing.T) {
|
||||
store, server := newOffsetStore(t)
|
||||
|
||||
require.NoError(t, store.Save(context.Background(), "start_jobs", "1700000000000-0"))
|
||||
|
||||
id, found, err := store.Load(context.Background(), "start_jobs")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, "1700000000000-0", id)
|
||||
|
||||
// The persisted key must follow the rtmanager keyspace prefix.
|
||||
expectedKey := redisstate.Keyspace{}.StreamOffset("start_jobs")
|
||||
assert.True(t, server.Exists(expectedKey),
|
||||
"key %q must exist after Save", expectedKey)
|
||||
}
|
||||
|
||||
func TestSaveOverwritesPriorValue(t *testing.T) {
|
||||
store, _ := newOffsetStore(t)
|
||||
|
||||
require.NoError(t, store.Save(context.Background(), "start_jobs", "100-0"))
|
||||
require.NoError(t, store.Save(context.Background(), "start_jobs", "200-0"))
|
||||
|
||||
id, found, err := store.Load(context.Background(), "start_jobs")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, "200-0", id)
|
||||
}
|
||||
|
||||
func TestLoadAndSaveRejectInvalidArguments(t *testing.T) {
|
||||
store, _ := newOffsetStore(t)
|
||||
|
||||
require.Error(t, store.Save(context.Background(), "", "100-0"))
|
||||
require.Error(t, store.Save(context.Background(), "start_jobs", ""))
|
||||
|
||||
_, _, err := store.Load(context.Background(), "")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestKeyspaceStreamOffsetIsPrefixed(t *testing.T) {
|
||||
key := redisstate.Keyspace{}.StreamOffset("start_jobs")
|
||||
assert.NotEmpty(t, key)
|
||||
assert.Contains(t, key, "rtmanager:stream_offsets:")
|
||||
// base64url-encoded label must not contain raw colons or spaces.
|
||||
suffix := key[len("rtmanager:stream_offsets:"):]
|
||||
assert.NotContains(t, suffix, ":")
|
||||
}
|
||||
Reference in New Issue
Block a user