14b65389ef
Tests · UI / test (push) Successful in 2m35s
Tests · Go / test (push) Successful in 1m56s
Tests · UI / test (pull_request) Has been cancelled
Tests · Integration / integration (pull_request) Successful in 1m42s
Tests · Go / test (pull_request) Successful in 2m0s
Browser fetch-streaming layers close response bodies they consider
idle after roughly 15-30 s without incoming bytes. Safari is the
most aggressive, but the symptom matters everywhere: a quiet
SubscribeEvents stream (lobby, between turns, mailbox empty) gets
torn down by the browser, the EventStream singleton reconnects with
backoff, and any push event that fires inside the reconnect window
is lost because `push.Hub` queues are not persisted across
subscription closes. The user-visible failure mode is the
intermittent "Fetch API cannot load … due to access control checks"
console error (a misleading WebKit symptom — CORS headers are
actually present) plus missed turn-ready / mail-received toasts.
Server-side fix: a silence-based heartbeat at the
`authenticatedPushStreamService` wrapper layer. After the signed
`gateway.server_time` bootstrap event, gateway wraps the bound
stream with `heartbeatingStream`. Every tail Send (fan-out, future
variants) resets the silence timer; when the timer elapses, a
goroutine emits `gateway.heartbeat` with only `EventType` set —
everything else stays at proto3 defaults, so the wire frame is
~45 bytes amortised. A `sendMu` serialises the heartbeat goroutine
with tail Sends because grpc.ServerStream.Send is not goroutine-safe.
The heartbeat is intentionally UNSIGNED: heartbeats carry no
payload, dispatch to no handler on the client, and an injected
heartbeat trivially causes no user-visible state change. TLS still
protects the wire and real events keep the signed envelope
unchanged. Documented in `docs/ARCHITECTURE.md` § 15 alongside the
per-scale bandwidth projection (100…100 000 clients × 15…60 s).
Config: new `GATEWAY_PUSH_HEARTBEAT_INTERVAL` (default `15s`,
`0s` disables). Telemetry: new
`gateway.push.heartbeats_sent{outcome}` counter so operators can
budget bandwidth and spot a sudden `outcome=error` bump as an
upstream-failing-before-flush signal.
Client (`ui/frontend/src/api/events.svelte.ts`): early `continue`
on `event.eventType === "gateway.heartbeat"` before `verifyEvent`,
`verifyPayloadHash`, or dispatch — empty signature would otherwise
trip SignatureError and reconnect. A leading heartbeat still flips
`connectionStatus` to `connected` and resets backoff, because
receiving one is proof the stream is healthy.
Tests:
- `push_heartbeat_test.go`: unit tests for the wrapper — zero
interval returns nil, heartbeat fires after silence, real Send
resets the timer, Stop / context-cancel halt the goroutine,
Send errors propagate.
- `server_test.go`: integration tests through the full gateway
pipeline — heartbeat fires after the configured silence window,
zero interval keeps the stream silent.
- `config_test.go`: default applied, env-override parsed,
negative value rejected.
- `events.test.ts`: heartbeat skipped before verification + not
dispatched to handlers; leading heartbeat still flips
`connectionStatus` to `connected`.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
186 lines
5.1 KiB
Go
186 lines
5.1 KiB
Go
package grpcapi
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
gatewayv1 "galaxy/gateway/proto/galaxy/gateway/v1"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/metadata"
|
|
)
|
|
|
|
func TestNewHeartbeatingStreamZeroIntervalReturnsNil(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
stream := newHeartbeatingStream(newCapturingStream(t), 0, nil)
|
|
assert.Nil(t, stream, "zero interval must not allocate a wrapper")
|
|
|
|
negative := newHeartbeatingStream(newCapturingStream(t), -time.Second, nil)
|
|
assert.Nil(t, negative, "negative interval must not allocate a wrapper")
|
|
}
|
|
|
|
func TestHeartbeatingStreamSendsHeartbeatAfterSilence(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
inner := newCapturingStream(t)
|
|
hb := newHeartbeatingStream(inner, 30*time.Millisecond, nil)
|
|
require.NotNil(t, hb)
|
|
defer hb.Stop()
|
|
|
|
go func() { _ = hb.Run(t.Context()) }()
|
|
|
|
event := inner.recv(t, 200*time.Millisecond)
|
|
assert.Equal(t, gatewayHeartbeatEventType, event.GetEventType())
|
|
// Heartbeat envelope: only the event type travels. Every other
|
|
// field stays at proto3 default so the wire frame stays minimal.
|
|
assert.Empty(t, event.GetEventId())
|
|
assert.Zero(t, event.GetTimestampMs())
|
|
assert.Empty(t, event.GetPayloadBytes())
|
|
assert.Empty(t, event.GetPayloadHash())
|
|
assert.Empty(t, event.GetSignature())
|
|
assert.Empty(t, event.GetRequestId())
|
|
assert.Empty(t, event.GetTraceId())
|
|
}
|
|
|
|
func TestHeartbeatingStreamRealSendResetsSilenceTimer(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
inner := newCapturingStream(t)
|
|
hb := newHeartbeatingStream(inner, 50*time.Millisecond, nil)
|
|
require.NotNil(t, hb)
|
|
defer hb.Stop()
|
|
|
|
go func() { _ = hb.Run(t.Context()) }()
|
|
|
|
// Reset the timer every 20ms for 120ms — the silence window never
|
|
// elapses, so the heartbeat goroutine must stay quiet and the
|
|
// channel must only carry the manual real-event Sends.
|
|
go func() {
|
|
ticker := time.NewTicker(20 * time.Millisecond)
|
|
defer ticker.Stop()
|
|
for range 6 {
|
|
<-ticker.C
|
|
if err := hb.Send(&gatewayv1.GatewayEvent{EventType: "real.event"}); err != nil {
|
|
t.Errorf("real Send failed: %v", err)
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
|
|
for range 6 {
|
|
event := inner.recv(t, 100*time.Millisecond)
|
|
assert.Equal(t, "real.event", event.GetEventType(), "only real events should appear while Send keeps resetting the silence window")
|
|
}
|
|
}
|
|
|
|
func TestHeartbeatingStreamStopHaltsRun(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
inner := newCapturingStream(t)
|
|
hb := newHeartbeatingStream(inner, 20*time.Millisecond, nil)
|
|
require.NotNil(t, hb)
|
|
|
|
runDone := make(chan error, 1)
|
|
go func() { runDone <- hb.Run(context.Background()) }()
|
|
|
|
hb.Stop()
|
|
select {
|
|
case err := <-runDone:
|
|
require.NoError(t, err)
|
|
case <-time.After(200 * time.Millisecond):
|
|
t.Fatal("Run did not exit after Stop")
|
|
}
|
|
|
|
// Stop is idempotent; the second call must not panic on the
|
|
// already-closed done channel.
|
|
assert.NotPanics(t, hb.Stop)
|
|
}
|
|
|
|
func TestHeartbeatingStreamContextCancelHaltsRun(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
inner := newCapturingStream(t)
|
|
hb := newHeartbeatingStream(inner, 20*time.Millisecond, nil)
|
|
require.NotNil(t, hb)
|
|
defer hb.Stop()
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
runDone := make(chan error, 1)
|
|
go func() { runDone <- hb.Run(ctx) }()
|
|
|
|
cancel()
|
|
select {
|
|
case err := <-runDone:
|
|
require.NoError(t, err)
|
|
case <-time.After(200 * time.Millisecond):
|
|
t.Fatal("Run did not exit after context cancel")
|
|
}
|
|
}
|
|
|
|
func TestHeartbeatingStreamSendErrorPropagates(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
wantErr := errors.New("send failed")
|
|
inner := newCapturingStream(t)
|
|
inner.sendErr.Store(&errorBox{err: wantErr})
|
|
|
|
hb := newHeartbeatingStream(inner, time.Minute, nil)
|
|
require.NotNil(t, hb)
|
|
defer hb.Stop()
|
|
|
|
err := hb.Send(&gatewayv1.GatewayEvent{EventType: "real.event"})
|
|
require.ErrorIs(t, err, wantErr)
|
|
}
|
|
|
|
// capturingStream is a minimal grpc.ServerStreamingServer that pushes
|
|
// every Send into a channel so tests can assert on the wire frame.
|
|
type capturingStream struct {
|
|
grpc.ServerStreamingServer[gatewayv1.GatewayEvent]
|
|
|
|
events chan *gatewayv1.GatewayEvent
|
|
sendErr atomic.Pointer[errorBox]
|
|
}
|
|
|
|
type errorBox struct{ err error }
|
|
|
|
func newCapturingStream(t *testing.T) *capturingStream {
|
|
t.Helper()
|
|
|
|
return &capturingStream{events: make(chan *gatewayv1.GatewayEvent, 16)}
|
|
}
|
|
|
|
func (s *capturingStream) Send(event *gatewayv1.GatewayEvent) error {
|
|
if box := s.sendErr.Load(); box != nil {
|
|
return box.err
|
|
}
|
|
s.events <- event
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *capturingStream) Context() context.Context { return context.Background() }
|
|
|
|
func (s *capturingStream) SetHeader(metadata.MD) error { return nil }
|
|
func (s *capturingStream) SendHeader(metadata.MD) error { return nil }
|
|
func (s *capturingStream) SetTrailer(metadata.MD) {}
|
|
func (s *capturingStream) SendMsg(any) error { return errors.New("capturingStream.SendMsg: unused") }
|
|
func (s *capturingStream) RecvMsg(any) error { return errors.New("capturingStream.RecvMsg: unused") }
|
|
|
|
func (s *capturingStream) recv(t *testing.T, timeout time.Duration) *gatewayv1.GatewayEvent {
|
|
t.Helper()
|
|
|
|
select {
|
|
case event := <-s.events:
|
|
return event
|
|
case <-time.After(timeout):
|
|
t.Fatalf("no event captured within %s", timeout)
|
|
return nil
|
|
}
|
|
}
|