feat(gateway): temporary IP ban (fail2ban) fed by rejections + honeypot/honeytoken
CI / changes (pull_request) Successful in 3s
CI / unit (pull_request) Successful in 11s
CI / integration (pull_request) Successful in 17s
CI / ui (pull_request) Successful in 53s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m11s
CI / changes (pull_request) Successful in 3s
CI / unit (pull_request) Successful in 11s
CI / integration (pull_request) Successful in 17s
CI / ui (pull_request) Successful in 53s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m11s
Add a prod-only, in-memory IP ban enforced at the edge, fed by three signals:
sustained rate-limiter rejections (the IP-keyed public/email/admin classes — the
user class stays the backend soft-flag's concern), a honeypot decoy-path hit (the
contour caddy tags decoys with X-Scrabble-Honeypot and routes them to the gateway),
and a honeytoken (a planted bearer, GATEWAY_HONEYTOKEN). A banned IP is refused with
429 by the abuseGuard middleware before any work — covering the Connect edge, the
live stream and the static SPA/landing the per-op limiter never gated.
The ban is off by default: it keys by the real client IP the shared-NAT test contour
does not expose, so a ban there would be self-inflicted; detection still logs in the
contour, only the ban action is gated (GATEWAY_ABUSE_BAN_ENABLED). Rejection bans last
GATEWAY_ABUSE_BAN_DURATION; tripwire/honeytoken hits are near-zero-false-positive and
earn longer fixed bans. Each ban increments gateway_abuse_banned_total{reason}.
Operators see and lift active bans on the admin console's Throttled page; the gateway
syncs its active set to the backend every 30s (POST /api/v1/internal/bans/sync,
backend/internal/banview) and applies the operator unbans the response returns.
PRERELEASE phase AG. Docs baked into ARCHITECTURE / FUNCTIONAL (+ru) / both READMEs.
This commit is contained in:
@@ -8,6 +8,7 @@ package connectsrv
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -34,6 +35,12 @@ import (
|
||||
// heartbeatKind is the live-stream keep-alive event kind.
|
||||
const heartbeatKind = "heartbeat"
|
||||
|
||||
// honeypotHeader marks a request the edge proxy routed from a honeypot decoy path;
|
||||
// any request carrying it is treated as a scanner hit. The proxy strips any
|
||||
// client-supplied value before setting its own, and a spoofed value only bans the
|
||||
// spoofer, so trusting it is safe.
|
||||
const honeypotHeader = "X-Scrabble-Honeypot"
|
||||
|
||||
// Limiter classes, the `class` attribute of gateway_rate_limited_total and the
|
||||
// class field of the periodic rejection report.
|
||||
const (
|
||||
@@ -63,6 +70,8 @@ type Server struct {
|
||||
sessions *session.Cache
|
||||
limiter *ratelimit.Limiter
|
||||
tracker *ratelimit.Tracker
|
||||
banlist *ratelimit.Banlist
|
||||
honeytoken string
|
||||
hub *push.Hub
|
||||
heartbeat time.Duration
|
||||
log *zap.Logger
|
||||
@@ -86,7 +95,13 @@ type Deps struct {
|
||||
// Tracker accumulates limiter rejections for the periodic report; nil
|
||||
// selects a private tracker (rejections are then only counted, never
|
||||
// reported).
|
||||
Tracker *ratelimit.Tracker
|
||||
Tracker *ratelimit.Tracker
|
||||
// Banlist enforces temporary IP bans on the hot path; nil selects a disabled
|
||||
// (inert) banlist.
|
||||
Banlist *ratelimit.Banlist
|
||||
// Honeytoken, when non-empty, is the planted bearer value whose presentation
|
||||
// bans the caller and raises a high-severity alarm.
|
||||
Honeytoken string
|
||||
Hub *push.Hub
|
||||
RateLimit config.RateLimitConfig
|
||||
Heartbeat time.Duration
|
||||
@@ -116,6 +131,10 @@ func NewServer(d Deps) *Server {
|
||||
if limiter == nil {
|
||||
limiter = ratelimit.New()
|
||||
}
|
||||
banlist := d.Banlist
|
||||
if banlist == nil {
|
||||
banlist = ratelimit.NewBanlist(ratelimit.BanConfig{})
|
||||
}
|
||||
rl := d.RateLimit
|
||||
if rl == (config.RateLimitConfig{}) {
|
||||
rl = config.DefaultRateLimit()
|
||||
@@ -125,6 +144,8 @@ func NewServer(d Deps) *Server {
|
||||
sessions: d.Sessions,
|
||||
limiter: limiter,
|
||||
tracker: tracker,
|
||||
banlist: banlist,
|
||||
honeytoken: d.Honeytoken,
|
||||
hub: d.Hub,
|
||||
heartbeat: d.Heartbeat,
|
||||
log: log,
|
||||
@@ -172,9 +193,11 @@ func (s *Server) HTTPHandler() http.Handler {
|
||||
mux.Handle("/telegram/", webui.Handler("/telegram/", "index.html"))
|
||||
mux.Handle("/app/", webui.Handler("/app/", "index.html"))
|
||||
mux.Handle("/", http.RedirectHandler("/app/", http.StatusPermanentRedirect))
|
||||
// Every request body on the public listener is capped (the admin proxy POSTs
|
||||
// included); the h2c server carries explicit stream/idle sizing.
|
||||
return h2c.NewHandler(maxBodyHandler(s.maxBodyBytes, mux), &http2.Server{
|
||||
// abuseGuard is the outermost wrap (right under h2c) so a banned IP or a
|
||||
// honeypot hit is turned away before the body cap and the mux. Every request
|
||||
// body on the public listener is then capped (the admin proxy POSTs included);
|
||||
// the h2c server carries explicit stream/idle sizing.
|
||||
return h2c.NewHandler(s.abuseGuard(maxBodyHandler(s.maxBodyBytes, mux)), &http2.Server{
|
||||
MaxConcurrentStreams: h2cMaxConcurrentStreams,
|
||||
IdleTimeout: h2cIdleTimeout,
|
||||
})
|
||||
@@ -189,6 +212,32 @@ func maxBodyHandler(limit int, next http.Handler) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
// abuseGuard refuses a banned client IP with 429 before any work, and turns a
|
||||
// honeypot decoy hit (the proxy-set honeypotHeader) into an instant ban plus a
|
||||
// bland 404 that is indistinguishable from an ordinary miss. The ban check and the
|
||||
// tripwire ban are inert on a disabled banlist (the prod-only gate); the tripwire
|
||||
// hit is logged either way as scanner telemetry.
|
||||
func (s *Server) abuseGuard(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ip := peerIP(r.RemoteAddr, r.Header)
|
||||
if s.banlist.Banned(ip) {
|
||||
http.Error(w, "banned", http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
if r.Header.Get(honeypotHeader) != "" {
|
||||
s.log.Warn("honeypot tripwire",
|
||||
zap.String("path", r.URL.Path),
|
||||
zap.String("client_ip", ip))
|
||||
if s.banlist.BanNow(ip, ratelimit.ReasonTripwire) {
|
||||
s.metrics.recordBan(r.Context(), string(ratelimit.ReasonTripwire))
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// Execute runs one unary operation. Domain failures are returned in the envelope
|
||||
// (result_code != "ok", HTTP 200); only edge failures (rate limit, missing
|
||||
// session, unknown type, internal) become Connect errors.
|
||||
@@ -207,7 +256,7 @@ func (s *Server) Execute(ctx context.Context, req *connect.Request[edgev1.Execut
|
||||
|
||||
tr := transcode.Request{Payload: req.Msg.GetPayload(), ClientIP: clientIP}
|
||||
if op.Auth {
|
||||
uid, isGuest, err := s.resolve(ctx, req.Header())
|
||||
uid, isGuest, err := s.resolve(ctx, req.Header(), clientIP)
|
||||
if err != nil {
|
||||
result = "unauthenticated"
|
||||
return nil, err
|
||||
@@ -263,7 +312,7 @@ func (s *Server) Execute(ctx context.Context, req *connect.Request[edgev1.Execut
|
||||
// Subscribe streams the authenticated user's live events with a keep-alive
|
||||
// heartbeat until the client disconnects.
|
||||
func (s *Server) Subscribe(ctx context.Context, req *connect.Request[edgev1.SubscribeRequest], stream *connect.ServerStream[edgev1.Event]) error {
|
||||
uid, _, err := s.resolve(ctx, req.Header())
|
||||
uid, _, err := s.resolve(ctx, req.Header(), peerIP(req.Peer().Addr, req.Header()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -311,6 +360,12 @@ func (s *Server) Subscribe(ctx context.Context, req *connect.Request[edgev1.Subs
|
||||
func (s *Server) noteRateLimited(ctx context.Context, class, key, msgType string) {
|
||||
s.metrics.recordRateLimited(ctx, class)
|
||||
s.tracker.Add(class, key)
|
||||
// IP-keyed rejections (public, email, admin — the key is the client IP) feed
|
||||
// the ban; the user class is keyed by account id and is the backend soft-flag's
|
||||
// concern, not the IP ban's.
|
||||
if class != classUser && s.banlist.Strike(key) {
|
||||
s.metrics.recordBan(ctx, string(ratelimit.ReasonRejections))
|
||||
}
|
||||
s.log.Debug("rate limited",
|
||||
zap.String("class", class),
|
||||
zap.String("key", key),
|
||||
@@ -344,11 +399,21 @@ func (s *Server) limitAdmin(next http.Handler) http.Handler {
|
||||
// resolve extracts and resolves the Authorization bearer token to an account id
|
||||
// and its guest flag, returning a Connect Unauthenticated error when it is missing
|
||||
// or unknown.
|
||||
func (s *Server) resolve(ctx context.Context, h http.Header) (string, bool, error) {
|
||||
func (s *Server) resolve(ctx context.Context, h http.Header, clientIP string) (string, bool, error) {
|
||||
token := bearerToken(h.Get("Authorization"))
|
||||
if token == "" {
|
||||
return "", false, connect.NewError(connect.CodeUnauthenticated, errMissingToken)
|
||||
}
|
||||
// The honeytoken is a planted value no real client holds: presenting it is a
|
||||
// high-confidence intrusion signal, so ban the caller and raise the alarm, then
|
||||
// return the ordinary invalid-session error so the trap stays indistinguishable.
|
||||
if s.honeytoken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(s.honeytoken)) == 1 {
|
||||
s.log.Warn("honeytoken presented", zap.String("client_ip", clientIP))
|
||||
if s.banlist.BanNow(clientIP, ratelimit.ReasonHoneytoken) {
|
||||
s.metrics.recordBan(ctx, string(ratelimit.ReasonHoneytoken))
|
||||
}
|
||||
return "", false, connect.NewError(connect.CodeUnauthenticated, errInvalidSession)
|
||||
}
|
||||
uid, isGuest, err := s.sessions.Resolve(ctx, token)
|
||||
if err != nil {
|
||||
// An unknown or expired token (a backend 4xx) is the client's problem and
|
||||
|
||||
Reference in New Issue
Block a user