feat(gateway): temporary IP ban (fail2ban) fed by rejections + honeypot/honeytoken
CI / changes (pull_request) Successful in 3s
CI / unit (pull_request) Successful in 11s
CI / integration (pull_request) Successful in 17s
CI / ui (pull_request) Successful in 53s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m11s

Add a prod-only, in-memory IP ban enforced at the edge, fed by three signals:
sustained rate-limiter rejections (the IP-keyed public/email/admin classes — the
user class stays the backend soft-flag's concern), a honeypot decoy-path hit (the
contour caddy tags decoys with X-Scrabble-Honeypot and routes them to the gateway),
and a honeytoken (a planted bearer, GATEWAY_HONEYTOKEN). A banned IP is refused with
429 by the abuseGuard middleware before any work — covering the Connect edge, the
live stream and the static SPA/landing the per-op limiter never gated.

The ban is off by default: it keys by the real client IP the shared-NAT test contour
does not expose, so a ban there would be self-inflicted; detection still logs in the
contour, only the ban action is gated (GATEWAY_ABUSE_BAN_ENABLED). Rejection bans last
GATEWAY_ABUSE_BAN_DURATION; tripwire/honeytoken hits are near-zero-false-positive and
earn longer fixed bans. Each ban increments gateway_abuse_banned_total{reason}.

Operators see and lift active bans on the admin console's Throttled page; the gateway
syncs its active set to the backend every 30s (POST /api/v1/internal/bans/sync,
backend/internal/banview) and applies the operator unbans the response returns.

PRERELEASE phase AG. Docs baked into ARCHITECTURE / FUNCTIONAL (+ru) / both READMEs.
This commit is contained in:
Ilia Denisov
2026-06-21 08:54:20 +02:00
parent 3fffee7817
commit 041106d623
28 changed files with 1295 additions and 19 deletions
+72 -7
View File
@@ -8,6 +8,7 @@ package connectsrv
import (
"context"
"crypto/subtle"
"errors"
"net"
"net/http"
@@ -34,6 +35,12 @@ import (
// heartbeatKind is the live-stream keep-alive event kind.
const heartbeatKind = "heartbeat"
// honeypotHeader marks a request the edge proxy routed from a honeypot decoy path;
// any request carrying it is treated as a scanner hit. The proxy strips any
// client-supplied value before setting its own, and a spoofed value only bans the
// spoofer, so trusting it is safe.
const honeypotHeader = "X-Scrabble-Honeypot"
// Limiter classes, the `class` attribute of gateway_rate_limited_total and the
// class field of the periodic rejection report.
const (
@@ -63,6 +70,8 @@ type Server struct {
sessions *session.Cache
limiter *ratelimit.Limiter
tracker *ratelimit.Tracker
banlist *ratelimit.Banlist
honeytoken string
hub *push.Hub
heartbeat time.Duration
log *zap.Logger
@@ -86,7 +95,13 @@ type Deps struct {
// Tracker accumulates limiter rejections for the periodic report; nil
// selects a private tracker (rejections are then only counted, never
// reported).
Tracker *ratelimit.Tracker
Tracker *ratelimit.Tracker
// Banlist enforces temporary IP bans on the hot path; nil selects a disabled
// (inert) banlist.
Banlist *ratelimit.Banlist
// Honeytoken, when non-empty, is the planted bearer value whose presentation
// bans the caller and raises a high-severity alarm.
Honeytoken string
Hub *push.Hub
RateLimit config.RateLimitConfig
Heartbeat time.Duration
@@ -116,6 +131,10 @@ func NewServer(d Deps) *Server {
if limiter == nil {
limiter = ratelimit.New()
}
banlist := d.Banlist
if banlist == nil {
banlist = ratelimit.NewBanlist(ratelimit.BanConfig{})
}
rl := d.RateLimit
if rl == (config.RateLimitConfig{}) {
rl = config.DefaultRateLimit()
@@ -125,6 +144,8 @@ func NewServer(d Deps) *Server {
sessions: d.Sessions,
limiter: limiter,
tracker: tracker,
banlist: banlist,
honeytoken: d.Honeytoken,
hub: d.Hub,
heartbeat: d.Heartbeat,
log: log,
@@ -172,9 +193,11 @@ func (s *Server) HTTPHandler() http.Handler {
mux.Handle("/telegram/", webui.Handler("/telegram/", "index.html"))
mux.Handle("/app/", webui.Handler("/app/", "index.html"))
mux.Handle("/", http.RedirectHandler("/app/", http.StatusPermanentRedirect))
// Every request body on the public listener is capped (the admin proxy POSTs
// included); the h2c server carries explicit stream/idle sizing.
return h2c.NewHandler(maxBodyHandler(s.maxBodyBytes, mux), &http2.Server{
// abuseGuard is the outermost wrap (right under h2c) so a banned IP or a
// honeypot hit is turned away before the body cap and the mux. Every request
// body on the public listener is then capped (the admin proxy POSTs included);
// the h2c server carries explicit stream/idle sizing.
return h2c.NewHandler(s.abuseGuard(maxBodyHandler(s.maxBodyBytes, mux)), &http2.Server{
MaxConcurrentStreams: h2cMaxConcurrentStreams,
IdleTimeout: h2cIdleTimeout,
})
@@ -189,6 +212,32 @@ func maxBodyHandler(limit int, next http.Handler) http.Handler {
})
}
// abuseGuard refuses a banned client IP with 429 before any work, and turns a
// honeypot decoy hit (the proxy-set honeypotHeader) into an instant ban plus a
// bland 404 that is indistinguishable from an ordinary miss. The ban check and the
// tripwire ban are inert on a disabled banlist (the prod-only gate); the tripwire
// hit is logged either way as scanner telemetry.
func (s *Server) abuseGuard(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ip := peerIP(r.RemoteAddr, r.Header)
if s.banlist.Banned(ip) {
http.Error(w, "banned", http.StatusTooManyRequests)
return
}
if r.Header.Get(honeypotHeader) != "" {
s.log.Warn("honeypot tripwire",
zap.String("path", r.URL.Path),
zap.String("client_ip", ip))
if s.banlist.BanNow(ip, ratelimit.ReasonTripwire) {
s.metrics.recordBan(r.Context(), string(ratelimit.ReasonTripwire))
}
http.NotFound(w, r)
return
}
next.ServeHTTP(w, r)
})
}
// Execute runs one unary operation. Domain failures are returned in the envelope
// (result_code != "ok", HTTP 200); only edge failures (rate limit, missing
// session, unknown type, internal) become Connect errors.
@@ -207,7 +256,7 @@ func (s *Server) Execute(ctx context.Context, req *connect.Request[edgev1.Execut
tr := transcode.Request{Payload: req.Msg.GetPayload(), ClientIP: clientIP}
if op.Auth {
uid, isGuest, err := s.resolve(ctx, req.Header())
uid, isGuest, err := s.resolve(ctx, req.Header(), clientIP)
if err != nil {
result = "unauthenticated"
return nil, err
@@ -263,7 +312,7 @@ func (s *Server) Execute(ctx context.Context, req *connect.Request[edgev1.Execut
// Subscribe streams the authenticated user's live events with a keep-alive
// heartbeat until the client disconnects.
func (s *Server) Subscribe(ctx context.Context, req *connect.Request[edgev1.SubscribeRequest], stream *connect.ServerStream[edgev1.Event]) error {
uid, _, err := s.resolve(ctx, req.Header())
uid, _, err := s.resolve(ctx, req.Header(), peerIP(req.Peer().Addr, req.Header()))
if err != nil {
return err
}
@@ -311,6 +360,12 @@ func (s *Server) Subscribe(ctx context.Context, req *connect.Request[edgev1.Subs
func (s *Server) noteRateLimited(ctx context.Context, class, key, msgType string) {
s.metrics.recordRateLimited(ctx, class)
s.tracker.Add(class, key)
// IP-keyed rejections (public, email, admin — the key is the client IP) feed
// the ban; the user class is keyed by account id and is the backend soft-flag's
// concern, not the IP ban's.
if class != classUser && s.banlist.Strike(key) {
s.metrics.recordBan(ctx, string(ratelimit.ReasonRejections))
}
s.log.Debug("rate limited",
zap.String("class", class),
zap.String("key", key),
@@ -344,11 +399,21 @@ func (s *Server) limitAdmin(next http.Handler) http.Handler {
// resolve extracts and resolves the Authorization bearer token to an account id
// and its guest flag, returning a Connect Unauthenticated error when it is missing
// or unknown.
func (s *Server) resolve(ctx context.Context, h http.Header) (string, bool, error) {
func (s *Server) resolve(ctx context.Context, h http.Header, clientIP string) (string, bool, error) {
token := bearerToken(h.Get("Authorization"))
if token == "" {
return "", false, connect.NewError(connect.CodeUnauthenticated, errMissingToken)
}
// The honeytoken is a planted value no real client holds: presenting it is a
// high-confidence intrusion signal, so ban the caller and raise the alarm, then
// return the ordinary invalid-session error so the trap stays indistinguishable.
if s.honeytoken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(s.honeytoken)) == 1 {
s.log.Warn("honeytoken presented", zap.String("client_ip", clientIP))
if s.banlist.BanNow(clientIP, ratelimit.ReasonHoneytoken) {
s.metrics.recordBan(ctx, string(ratelimit.ReasonHoneytoken))
}
return "", false, connect.NewError(connect.CodeUnauthenticated, errInvalidSession)
}
uid, isGuest, err := s.sessions.Resolve(ctx, token)
if err != nil {
// An unknown or expired token (a backend 4xx) is the client's problem and