feat(telegram): split connector into home validator + remote bot
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 15s
CI / ui (pull_request) Successful in 53s
CI / gate (pull_request) Successful in 1s
CI / deploy (pull_request) Failing after 2m6s

Move all Telegram egress off the main host. The single connector held the
bot token, long-polled Telegram and answered the gateway/backend over the
trusted internal network, so the whole component (including login validation)
shared fate with its VPN sidecar. Split it into two binaries that share the
token:

- cmd/validator (home, no VPN): Mini App initData + Login Widget HMAC only,
  never calls the Bot API. The gateway dials it for Telegram auth, so game
  login is now independent of Telegram reachability.
- cmd/bot (remote): Bot API long-poll + sendMessage, the only component
  reaching Telegram. It holds no inbound port — it dials the gateway over a
  new reverse mTLS bot-link (pkg/proto/botlink/v1) and executes the send
  commands the gateway pushes.

The gateway funnels sends to the bot-link: out-of-app push is fire-and-forget
(at-most-once, dropped if no bot is connected); the backend admin broadcasts
reach a gateway-served relay that forwards them and awaits the bot's ack
(SendToUser/SendToGameChannel contract preserved). mTLS (pkg/mtls) is the one
inter-service link that leaves the trusted segment; validator<->gateway and
the relay stay plaintext internal. The bot is Telegram-rate-limited.

One bot now; the gateway bot registry, an owns_updates flag and per-command
ids leave seams for N later. Webhook rejected (one URL per token, adds inbound
+ a static address).

The unified test contour runs the split (the bot keeps its VPN sidecar and
dials the gateway by its internal name; bot-link certs from deploy/gen-certs.sh,
generated in CI). The prod wiring — the bot on a separate host (no VPN), the
gateway bot-link port published, PROD_ certs with scheduled rotation, an SSH
deploy of both hosts together — is the deferred final stage (PRERELEASE.md TX,
Stage 18).

Docs: ARCHITECTURE, PRERELEASE (phase TX), platform/telegram + gateway +
backend + deploy READMEs, FUNCTIONAL(+ru), CLAUDE.md, .env.example.
This commit is contained in:
Ilia Denisov
2026-06-21 00:19:07 +02:00
parent 2a8717c930
commit 6aeb529f13
42 changed files with 3073 additions and 714 deletions
+89 -18
View File
@@ -9,16 +9,23 @@ package main
import (
"context"
"errors"
"fmt"
"log"
"net"
"net/http"
"os/signal"
"syscall"
"time"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/keepalive"
"scrabble/gateway/internal/admin"
"scrabble/gateway/internal/backendclient"
"scrabble/gateway/internal/botlink"
"scrabble/gateway/internal/config"
"scrabble/gateway/internal/connector"
"scrabble/gateway/internal/connectsrv"
@@ -26,9 +33,23 @@ import (
"scrabble/gateway/internal/ratelimit"
"scrabble/gateway/internal/session"
"scrabble/gateway/internal/transcode"
"scrabble/pkg/mtls"
botlinkv1 "scrabble/pkg/proto/botlink/v1"
telegramv1 "scrabble/pkg/proto/telegram/v1"
pkgtel "scrabble/pkg/telemetry"
)
const (
// botLinkKeepaliveTime is how often the gateway pings an idle bot-link stream
// to hold the WAN connection open and detect a dead bot.
botLinkKeepaliveTime = 30 * time.Second
// botLinkKeepaliveTimeout bounds the wait for a keepalive ping reply.
botLinkKeepaliveTimeout = 10 * time.Second
// botLinkMinPingInterval is the smallest client ping interval the gateway
// tolerates before treating it as abuse.
botLinkMinPingInterval = 10 * time.Second
)
const (
// shutdownTimeout bounds the graceful HTTP shutdown.
shutdownTimeout = 10 * time.Second
@@ -100,17 +121,47 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
tracker := ratelimit.NewTracker()
hub := push.NewHub(0)
var conn *connector.Client
var validator transcode.TelegramValidator
if cfg.ConnectorAddr != "" {
conn, err = connector.New(cfg.ConnectorAddr)
if err != nil {
return err
if cfg.ValidatorAddr != "" {
conn, cerr := connector.New(cfg.ValidatorAddr)
if cerr != nil {
return cerr
}
defer func() { _ = conn.Close() }()
validator = conn
} else {
logger.Warn("telegram disabled (GATEWAY_CONNECTOR_ADDR unset)")
logger.Warn("telegram auth disabled (GATEWAY_VALIDATOR_ADDR unset)")
}
// The reverse bot-link: the remote Telegram bot dials this gateway over mTLS and
// the gateway pushes send commands down the stream. Out-of-app push is
// fire-and-forget; the backend admin relay (plaintext, internal) awaits the Ack.
var botHub *botlink.Hub
if cfg.BotLinkEnabled() {
botHub = botlink.NewHub(logger, tel.MeterProvider().Meter("scrabble/gateway/botlink"))
tlsCfg, terr := mtls.ServerConfig(cfg.BotLink.CertFile, cfg.BotLink.KeyFile, cfg.BotLink.CAFile)
if terr != nil {
return terr
}
botSrv := grpc.NewServer(
grpc.Creds(credentials.NewTLS(tlsCfg)),
grpc.StatsHandler(otelgrpc.NewServerHandler()),
grpc.KeepaliveParams(keepalive.ServerParameters{Time: botLinkKeepaliveTime, Timeout: botLinkKeepaliveTimeout}),
grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{MinTime: botLinkMinPingInterval, PermitWithoutStream: true}),
)
botlinkv1.RegisterBotLinkServer(botSrv, botHub)
if serr := serveGRPC(ctx, "botlink", cfg.BotLink.Addr, botSrv, logger); serr != nil {
return serr
}
if cfg.BotLink.RelayAddr != "" {
relaySrv := grpc.NewServer(grpc.StatsHandler(otelgrpc.NewServerHandler()))
telegramv1.RegisterTelegramServer(relaySrv, botlink.NewRelayServer(botHub, cfg.BotLink.SendTimeout))
if serr := serveGRPC(ctx, "botlink-relay", cfg.BotLink.RelayAddr, relaySrv, logger); serr != nil {
return serr
}
}
} else {
logger.Warn("telegram bot channel disabled (GATEWAY_BOTLINK_ADDR unset)")
}
// The admin console (backend /_gm) is fronted on the public listener behind
@@ -142,8 +193,8 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
})
// Bridge the backend push stream into the fan-out hub (and the out-of-app
// channel via the connector).
go runPushPump(ctx, backend, hub, conn, logger)
// channel via the bot-link).
go runPushPump(ctx, backend, hub, botHub, logger)
// Periodically summarise rate-limiter rejections (Warn log + backend report).
go runThrottleReporter(ctx, tracker, backend, logger)
@@ -195,6 +246,27 @@ func runServers(ctx context.Context, cancel context.CancelFunc, servers []*named
return first
}
// serveGRPC starts a gRPC server on addr in the background and gracefully stops it
// when ctx is cancelled. It returns synchronously on a bind error so a
// misconfigured listener fails startup fast; a later Serve error is logged.
func serveGRPC(ctx context.Context, name, addr string, srv *grpc.Server, logger *zap.Logger) error {
lis, err := net.Listen("tcp", addr)
if err != nil {
return fmt.Errorf("gateway: listen %s (%s): %w", addr, name, err)
}
go func() {
<-ctx.Done()
srv.GracefulStop()
}()
go func() {
logger.Info("listener starting", zap.String("server", name), zap.String("addr", addr))
if err := srv.Serve(lis); err != nil && !errors.Is(err, grpc.ErrServerStopped) {
logger.Error("grpc listener failed", zap.String("server", name), zap.Error(err))
}
}()
return nil
}
// runThrottleReporter drains the rate-limiter rejection tracker on a fixed
// cadence, emits one Warn summary per throttled key and forwards the report to
// the backend (which feeds the admin throttled view and the high-rate
@@ -230,7 +302,7 @@ func runThrottleReporter(ctx context.Context, tracker *ratelimit.Tracker, backen
// the hub and re-subscribing after the stream ends, until the context is done. For
// the out-of-app push kinds it also routes events whose recipient has no live
// in-app stream to the platform connector (a nil connector disables that channel).
func runPushPump(ctx context.Context, backend *backendclient.Client, hub *push.Hub, conn *connector.Client, logger *zap.Logger) {
func runPushPump(ctx context.Context, backend *backendclient.Client, hub *push.Hub, bot *botlink.Hub, logger *zap.Logger) {
for ctx.Err() == nil {
stream, err := backend.SubscribePush(ctx, gatewayID)
if err != nil {
@@ -255,10 +327,10 @@ func runPushPump(ctx context.Context, backend *backendclient.Client, hub *push.H
EventID: ev.GetEventId(),
})
// Out-of-app fallback: when the recipient has no live in-app stream,
// deliver the event over the platform push channel. Done in a goroutine
// so a slow connector never stalls the in-app firehose.
if conn != nil && connector.OutOfAppKind(ev.GetKind()) && !hub.HasSubscribers(ev.GetUserId()) {
go deliverOutOfApp(ctx, backend, conn, ev.GetUserId(), ev.GetKind(), ev.GetPayload(), logger)
// deliver the event over the bot-link. Done in a goroutine so a slow
// target lookup never stalls the in-app firehose.
if bot != nil && connector.OutOfAppKind(ev.GetKind()) && !hub.HasSubscribers(ev.GetUserId()) {
go deliverOutOfApp(ctx, backend, bot, ev.GetUserId(), ev.GetKind(), ev.GetPayload(), logger)
}
}
if !sleep(ctx, pushReconnectDelay) {
@@ -271,7 +343,7 @@ func runPushPump(ctx context.Context, backend *backendclient.Client, hub *push.H
// Telegram identity and have not confined notifications to the app, asks the
// connector to deliver the event. It is best-effort: every failure is logged and
// dropped (the in-app stream remains the primary channel).
func deliverOutOfApp(ctx context.Context, backend *backendclient.Client, conn *connector.Client, userID, kind string, payload []byte, logger *zap.Logger) {
func deliverOutOfApp(ctx context.Context, backend *backendclient.Client, bot *botlink.Hub, userID, kind string, payload []byte, logger *zap.Logger) {
target, err := backend.PushTarget(ctx, userID)
if err != nil {
logger.Warn("push target lookup failed", zap.String("user_id", userID), zap.Error(err))
@@ -280,10 +352,9 @@ func deliverOutOfApp(ctx context.Context, backend *backendclient.Client, conn *c
if !connector.DeliverToTarget(target.ExternalID, target.NotificationsInAppOnly) {
return
}
// The single bot renders the message in the recipient's interface language.
if _, err := conn.Notify(ctx, target.ExternalID, kind, payload, target.Language); err != nil {
logger.Warn("out-of-app notify failed", zap.String("kind", kind), zap.Error(err))
}
// Fire-and-forget down the bot-link; the bot renders the message in the
// recipient's interface language and is dropped (best-effort) if no bot is up.
bot.Send(botlink.NotifyCommand(target.ExternalID, kind, payload, target.Language))
}
// sleep waits for d or until ctx is cancelled, reporting whether it waited the