feat(telegram): split connector into home validator + remote bot
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 15s
CI / ui (pull_request) Successful in 53s
CI / gate (pull_request) Successful in 1s
CI / deploy (pull_request) Failing after 2m6s
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 15s
CI / ui (pull_request) Successful in 53s
CI / gate (pull_request) Successful in 1s
CI / deploy (pull_request) Failing after 2m6s
Move all Telegram egress off the main host. The single connector held the bot token, long-polled Telegram and answered the gateway/backend over the trusted internal network, so the whole component (including login validation) shared fate with its VPN sidecar. Split it into two binaries that share the token: - cmd/validator (home, no VPN): Mini App initData + Login Widget HMAC only, never calls the Bot API. The gateway dials it for Telegram auth, so game login is now independent of Telegram reachability. - cmd/bot (remote): Bot API long-poll + sendMessage, the only component reaching Telegram. It holds no inbound port — it dials the gateway over a new reverse mTLS bot-link (pkg/proto/botlink/v1) and executes the send commands the gateway pushes. The gateway funnels sends to the bot-link: out-of-app push is fire-and-forget (at-most-once, dropped if no bot is connected); the backend admin broadcasts reach a gateway-served relay that forwards them and awaits the bot's ack (SendToUser/SendToGameChannel contract preserved). mTLS (pkg/mtls) is the one inter-service link that leaves the trusted segment; validator<->gateway and the relay stay plaintext internal. The bot is Telegram-rate-limited. One bot now; the gateway bot registry, an owns_updates flag and per-command ids leave seams for N later. Webhook rejected (one URL per token, adds inbound + a static address). The unified test contour runs the split (the bot keeps its VPN sidecar and dials the gateway by its internal name; bot-link certs from deploy/gen-certs.sh, generated in CI). The prod wiring — the bot on a separate host (no VPN), the gateway bot-link port published, PROD_ certs with scheduled rotation, an SSH deploy of both hosts together — is the deferred final stage (PRERELEASE.md TX, Stage 18). Docs: ARCHITECTURE, PRERELEASE (phase TX), platform/telegram + gateway + backend + deploy READMEs, FUNCTIONAL(+ru), CLAUDE.md, .env.example.
This commit is contained in:
+89
-18
@@ -9,16 +9,23 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
"google.golang.org/grpc/keepalive"
|
||||
|
||||
"scrabble/gateway/internal/admin"
|
||||
"scrabble/gateway/internal/backendclient"
|
||||
"scrabble/gateway/internal/botlink"
|
||||
"scrabble/gateway/internal/config"
|
||||
"scrabble/gateway/internal/connector"
|
||||
"scrabble/gateway/internal/connectsrv"
|
||||
@@ -26,9 +33,23 @@ import (
|
||||
"scrabble/gateway/internal/ratelimit"
|
||||
"scrabble/gateway/internal/session"
|
||||
"scrabble/gateway/internal/transcode"
|
||||
"scrabble/pkg/mtls"
|
||||
botlinkv1 "scrabble/pkg/proto/botlink/v1"
|
||||
telegramv1 "scrabble/pkg/proto/telegram/v1"
|
||||
pkgtel "scrabble/pkg/telemetry"
|
||||
)
|
||||
|
||||
const (
|
||||
// botLinkKeepaliveTime is how often the gateway pings an idle bot-link stream
|
||||
// to hold the WAN connection open and detect a dead bot.
|
||||
botLinkKeepaliveTime = 30 * time.Second
|
||||
// botLinkKeepaliveTimeout bounds the wait for a keepalive ping reply.
|
||||
botLinkKeepaliveTimeout = 10 * time.Second
|
||||
// botLinkMinPingInterval is the smallest client ping interval the gateway
|
||||
// tolerates before treating it as abuse.
|
||||
botLinkMinPingInterval = 10 * time.Second
|
||||
)
|
||||
|
||||
const (
|
||||
// shutdownTimeout bounds the graceful HTTP shutdown.
|
||||
shutdownTimeout = 10 * time.Second
|
||||
@@ -100,17 +121,47 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
|
||||
tracker := ratelimit.NewTracker()
|
||||
hub := push.NewHub(0)
|
||||
|
||||
var conn *connector.Client
|
||||
var validator transcode.TelegramValidator
|
||||
if cfg.ConnectorAddr != "" {
|
||||
conn, err = connector.New(cfg.ConnectorAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
if cfg.ValidatorAddr != "" {
|
||||
conn, cerr := connector.New(cfg.ValidatorAddr)
|
||||
if cerr != nil {
|
||||
return cerr
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
validator = conn
|
||||
} else {
|
||||
logger.Warn("telegram disabled (GATEWAY_CONNECTOR_ADDR unset)")
|
||||
logger.Warn("telegram auth disabled (GATEWAY_VALIDATOR_ADDR unset)")
|
||||
}
|
||||
|
||||
// The reverse bot-link: the remote Telegram bot dials this gateway over mTLS and
|
||||
// the gateway pushes send commands down the stream. Out-of-app push is
|
||||
// fire-and-forget; the backend admin relay (plaintext, internal) awaits the Ack.
|
||||
var botHub *botlink.Hub
|
||||
if cfg.BotLinkEnabled() {
|
||||
botHub = botlink.NewHub(logger, tel.MeterProvider().Meter("scrabble/gateway/botlink"))
|
||||
tlsCfg, terr := mtls.ServerConfig(cfg.BotLink.CertFile, cfg.BotLink.KeyFile, cfg.BotLink.CAFile)
|
||||
if terr != nil {
|
||||
return terr
|
||||
}
|
||||
botSrv := grpc.NewServer(
|
||||
grpc.Creds(credentials.NewTLS(tlsCfg)),
|
||||
grpc.StatsHandler(otelgrpc.NewServerHandler()),
|
||||
grpc.KeepaliveParams(keepalive.ServerParameters{Time: botLinkKeepaliveTime, Timeout: botLinkKeepaliveTimeout}),
|
||||
grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{MinTime: botLinkMinPingInterval, PermitWithoutStream: true}),
|
||||
)
|
||||
botlinkv1.RegisterBotLinkServer(botSrv, botHub)
|
||||
if serr := serveGRPC(ctx, "botlink", cfg.BotLink.Addr, botSrv, logger); serr != nil {
|
||||
return serr
|
||||
}
|
||||
if cfg.BotLink.RelayAddr != "" {
|
||||
relaySrv := grpc.NewServer(grpc.StatsHandler(otelgrpc.NewServerHandler()))
|
||||
telegramv1.RegisterTelegramServer(relaySrv, botlink.NewRelayServer(botHub, cfg.BotLink.SendTimeout))
|
||||
if serr := serveGRPC(ctx, "botlink-relay", cfg.BotLink.RelayAddr, relaySrv, logger); serr != nil {
|
||||
return serr
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.Warn("telegram bot channel disabled (GATEWAY_BOTLINK_ADDR unset)")
|
||||
}
|
||||
|
||||
// The admin console (backend /_gm) is fronted on the public listener behind
|
||||
@@ -142,8 +193,8 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
|
||||
})
|
||||
|
||||
// Bridge the backend push stream into the fan-out hub (and the out-of-app
|
||||
// channel via the connector).
|
||||
go runPushPump(ctx, backend, hub, conn, logger)
|
||||
// channel via the bot-link).
|
||||
go runPushPump(ctx, backend, hub, botHub, logger)
|
||||
// Periodically summarise rate-limiter rejections (Warn log + backend report).
|
||||
go runThrottleReporter(ctx, tracker, backend, logger)
|
||||
|
||||
@@ -195,6 +246,27 @@ func runServers(ctx context.Context, cancel context.CancelFunc, servers []*named
|
||||
return first
|
||||
}
|
||||
|
||||
// serveGRPC starts a gRPC server on addr in the background and gracefully stops it
|
||||
// when ctx is cancelled. It returns synchronously on a bind error so a
|
||||
// misconfigured listener fails startup fast; a later Serve error is logged.
|
||||
func serveGRPC(ctx context.Context, name, addr string, srv *grpc.Server, logger *zap.Logger) error {
|
||||
lis, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("gateway: listen %s (%s): %w", addr, name, err)
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
srv.GracefulStop()
|
||||
}()
|
||||
go func() {
|
||||
logger.Info("listener starting", zap.String("server", name), zap.String("addr", addr))
|
||||
if err := srv.Serve(lis); err != nil && !errors.Is(err, grpc.ErrServerStopped) {
|
||||
logger.Error("grpc listener failed", zap.String("server", name), zap.Error(err))
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// runThrottleReporter drains the rate-limiter rejection tracker on a fixed
|
||||
// cadence, emits one Warn summary per throttled key and forwards the report to
|
||||
// the backend (which feeds the admin throttled view and the high-rate
|
||||
@@ -230,7 +302,7 @@ func runThrottleReporter(ctx context.Context, tracker *ratelimit.Tracker, backen
|
||||
// the hub and re-subscribing after the stream ends, until the context is done. For
|
||||
// the out-of-app push kinds it also routes events whose recipient has no live
|
||||
// in-app stream to the platform connector (a nil connector disables that channel).
|
||||
func runPushPump(ctx context.Context, backend *backendclient.Client, hub *push.Hub, conn *connector.Client, logger *zap.Logger) {
|
||||
func runPushPump(ctx context.Context, backend *backendclient.Client, hub *push.Hub, bot *botlink.Hub, logger *zap.Logger) {
|
||||
for ctx.Err() == nil {
|
||||
stream, err := backend.SubscribePush(ctx, gatewayID)
|
||||
if err != nil {
|
||||
@@ -255,10 +327,10 @@ func runPushPump(ctx context.Context, backend *backendclient.Client, hub *push.H
|
||||
EventID: ev.GetEventId(),
|
||||
})
|
||||
// Out-of-app fallback: when the recipient has no live in-app stream,
|
||||
// deliver the event over the platform push channel. Done in a goroutine
|
||||
// so a slow connector never stalls the in-app firehose.
|
||||
if conn != nil && connector.OutOfAppKind(ev.GetKind()) && !hub.HasSubscribers(ev.GetUserId()) {
|
||||
go deliverOutOfApp(ctx, backend, conn, ev.GetUserId(), ev.GetKind(), ev.GetPayload(), logger)
|
||||
// deliver the event over the bot-link. Done in a goroutine so a slow
|
||||
// target lookup never stalls the in-app firehose.
|
||||
if bot != nil && connector.OutOfAppKind(ev.GetKind()) && !hub.HasSubscribers(ev.GetUserId()) {
|
||||
go deliverOutOfApp(ctx, backend, bot, ev.GetUserId(), ev.GetKind(), ev.GetPayload(), logger)
|
||||
}
|
||||
}
|
||||
if !sleep(ctx, pushReconnectDelay) {
|
||||
@@ -271,7 +343,7 @@ func runPushPump(ctx context.Context, backend *backendclient.Client, hub *push.H
|
||||
// Telegram identity and have not confined notifications to the app, asks the
|
||||
// connector to deliver the event. It is best-effort: every failure is logged and
|
||||
// dropped (the in-app stream remains the primary channel).
|
||||
func deliverOutOfApp(ctx context.Context, backend *backendclient.Client, conn *connector.Client, userID, kind string, payload []byte, logger *zap.Logger) {
|
||||
func deliverOutOfApp(ctx context.Context, backend *backendclient.Client, bot *botlink.Hub, userID, kind string, payload []byte, logger *zap.Logger) {
|
||||
target, err := backend.PushTarget(ctx, userID)
|
||||
if err != nil {
|
||||
logger.Warn("push target lookup failed", zap.String("user_id", userID), zap.Error(err))
|
||||
@@ -280,10 +352,9 @@ func deliverOutOfApp(ctx context.Context, backend *backendclient.Client, conn *c
|
||||
if !connector.DeliverToTarget(target.ExternalID, target.NotificationsInAppOnly) {
|
||||
return
|
||||
}
|
||||
// The single bot renders the message in the recipient's interface language.
|
||||
if _, err := conn.Notify(ctx, target.ExternalID, kind, payload, target.Language); err != nil {
|
||||
logger.Warn("out-of-app notify failed", zap.String("kind", kind), zap.Error(err))
|
||||
}
|
||||
// Fire-and-forget down the bot-link; the bot renders the message in the
|
||||
// recipient's interface language and is dropped (best-effort) if no bot is up.
|
||||
bot.Send(botlink.NotifyCommand(target.ExternalID, kind, payload, target.Language))
|
||||
}
|
||||
|
||||
// sleep waits for d or until ctx is cancelled, reporting whether it waited the
|
||||
|
||||
Reference in New Issue
Block a user