feat(gateway): temporary IP ban (fail2ban) fed by rejections + honeypot/honeytoken

Add a prod-only, in-memory IP ban enforced at the edge, fed by three signals:
sustained rate-limiter rejections (the IP-keyed public/email/admin classes — the
user class stays the backend soft-flag's concern), a honeypot decoy-path hit (the
contour caddy tags decoys with X-Scrabble-Honeypot and routes them to the gateway),
and a honeytoken (a planted bearer, GATEWAY_HONEYTOKEN). A banned IP is refused with
429 by the abuseGuard middleware before any work — covering the Connect edge, the
live stream and the static SPA/landing the per-op limiter never gated.

The ban is off by default: it keys by the real client IP the shared-NAT test contour
does not expose, so a ban there would be self-inflicted; detection still logs in the
contour, only the ban action is gated (GATEWAY_ABUSE_BAN_ENABLED). Rejection bans last
GATEWAY_ABUSE_BAN_DURATION; tripwire/honeytoken hits are near-zero-false-positive and
earn longer fixed bans. Each ban increments gateway_abuse_banned_total{reason}.

Operators see and lift active bans on the admin console's Throttled page; the gateway
syncs its active set to the backend every 30s (POST /api/v1/internal/bans/sync,
backend/internal/banview) and applies the operator unbans the response returns.

PRERELEASE phase AG. Docs baked into ARCHITECTURE / FUNCTIONAL (+ru) / both READMEs.

gateway/internal/connectsrv/abuse_test.go

@@ -0,0 +1,219 @@
+package connectsrv_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"connectrpc.com/connect"
+
+	"scrabble/gateway/internal/backendclient"
+	"scrabble/gateway/internal/config"
+	"scrabble/gateway/internal/connectsrv"
+	"scrabble/gateway/internal/push"
+	"scrabble/gateway/internal/ratelimit"
+	"scrabble/gateway/internal/session"
+	"scrabble/gateway/internal/transcode"
+	edgev1 "scrabble/gateway/proto/edge/v1"
+	"scrabble/gateway/proto/edge/v1/edgev1connect"
+)
+
+const honeypotHeader = "X-Scrabble-Honeypot"
+
+// guardedEdge wires an edge with an explicit banlist and honeytoken over a fake
+// backend, returning the front URL, a Connect client and a cleanup func.
+func guardedEdge(t *testing.T, bl *ratelimit.Banlist, honeytoken string, limits config.RateLimitConfig, backendHandler http.HandlerFunc) (string, edgev1connect.GatewayClient, func()) {
+	t.Helper()
+	backendSrv := httptest.NewServer(backendHandler)
+	backend, err := backendclient.New(backendSrv.URL, "localhost:9090", 2*time.Second)
+	if err != nil {
+		t.Fatalf("backendclient: %v", err)
+	}
+	edge := connectsrv.NewServer(connectsrv.Deps{
+		Registry:   transcode.NewRegistry(backend, nil),
+		Sessions:   session.NewCache(backend, time.Minute, 100),
+		Limiter:    ratelimit.New(),
+		Banlist:    bl,
+		Honeytoken: honeytoken,
+		Hub:        push.NewHub(0),
+		RateLimit:  limits,
+		Heartbeat:  15 * time.Second,
+	})
+	edgeSrv := httptest.NewServer(edge.HTTPHandler())
+	client := edgev1connect.NewGatewayClient(http.DefaultClient, edgeSrv.URL)
+	return edgeSrv.URL, client, func() {
+		edgeSrv.Close()
+		_ = backend.Close()
+		backendSrv.Close()
+	}
+}
+
+// noRedirect is an HTTP client that surfaces a redirect instead of following it,
+// so the test can tell a 308 (passed the guard) from a 429 (blocked).
+func noRedirect() *http.Client {
+	return &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error {
+		return http.ErrUseLastResponse
+	}}
+}
+
+func enabledBanlist(threshold int) *ratelimit.Banlist {
+	return ratelimit.NewBanlist(ratelimit.BanConfig{
+		Enabled: true, Threshold: threshold, Window: time.Minute, Duration: time.Hour,
+	})
+}
+
+// TestAbuseGuardBlocksBannedIP verifies a banned client IP is refused with 429 at
+// the HTTP layer, before any handler runs.
+func TestAbuseGuardBlocksBannedIP(t *testing.T) {
+	bl := enabledBanlist(100)
+	bl.BanNow("127.0.0.1", ratelimit.ReasonTripwire)
+	url, _, cleanup := guardedEdge(t, bl, "", config.DefaultRateLimit(), func(w http.ResponseWriter, r *http.Request) {})
+	defer cleanup()
+
+	resp, err := noRedirect().Get(url + "/")
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusTooManyRequests {
+		t.Fatalf("banned GET / = %d, want 429", resp.StatusCode)
+	}
+}
+
+// TestHoneypotHeaderTrips verifies a request carrying the honeypot header is 404'd
+// and bans the client IP, so the next request is blocked.
+func TestHoneypotHeaderTrips(t *testing.T) {
+	bl := enabledBanlist(100)
+	url, _, cleanup := guardedEdge(t, bl, "", config.DefaultRateLimit(), func(w http.ResponseWriter, r *http.Request) {
+		t.Error("backend must not be called for a honeypot hit")
+	})
+	defer cleanup()
+
+	req, _ := http.NewRequest(http.MethodGet, url+"/.env", nil)
+	req.Header.Set(honeypotHeader, "1")
+	resp, err := noRedirect().Do(req)
+	if err != nil {
+		t.Fatalf("honeypot get: %v", err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusNotFound {
+		t.Fatalf("honeypot hit = %d, want 404", resp.StatusCode)
+	}
+	if !bl.Banned("127.0.0.1") {
+		t.Fatal("a honeypot hit must ban the client IP")
+	}
+	follow, err := noRedirect().Get(url + "/")
+	if err != nil {
+		t.Fatalf("follow-up get: %v", err)
+	}
+	_ = follow.Body.Close()
+	if follow.StatusCode != http.StatusTooManyRequests {
+		t.Fatalf("post-trip GET / = %d, want 429", follow.StatusCode)
+	}
+}
+
+// TestHoneypotDetectsWithoutBanWhenDisabled verifies the prod-only gate: a disabled
+// banlist still 404s the decoy (detection/logging) but bans nothing.
+func TestHoneypotDetectsWithoutBanWhenDisabled(t *testing.T) {
+	bl := ratelimit.NewBanlist(ratelimit.BanConfig{}) // disabled
+	url, _, cleanup := guardedEdge(t, bl, "", config.DefaultRateLimit(), func(w http.ResponseWriter, r *http.Request) {})
+	defer cleanup()
+
+	req, _ := http.NewRequest(http.MethodGet, url+"/.env", nil)
+	req.Header.Set(honeypotHeader, "1")
+	resp, err := noRedirect().Do(req)
+	if err != nil {
+		t.Fatalf("honeypot get: %v", err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusNotFound {
+		t.Fatalf("decoy = %d, want 404", resp.StatusCode)
+	}
+	if bl.Banned("127.0.0.1") {
+		t.Fatal("a disabled banlist must not ban")
+	}
+	follow, err := noRedirect().Get(url + "/")
+	if err != nil {
+		t.Fatalf("follow-up: %v", err)
+	}
+	_ = follow.Body.Close()
+	if follow.StatusCode != http.StatusPermanentRedirect {
+		t.Fatalf("post-trip GET / = %d, want 308 (not banned)", follow.StatusCode)
+	}
+}
+
+// TestPublicRejectionStrikesBan verifies a public-class limiter rejection feeds the
+// banlist: with a one-strike threshold the rejected IP is then banned.
+func TestPublicRejectionStrikesBan(t *testing.T) {
+	bl := enabledBanlist(1)
+	limits := config.DefaultRateLimit()
+	limits.PublicPerMinute, limits.PublicBurst = 1, 1
+	_, client, cleanup := guardedEdge(t, bl, "", limits, func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"token":"tok","user_id":"u-1","is_guest":true,"display_name":"Guest"}`))
+	})
+	defer cleanup()
+
+	if _, err := client.Execute(context.Background(), connect.NewRequest(&edgev1.ExecuteRequest{MessageType: transcode.MsgAuthGuest})); err != nil {
+		t.Fatalf("first execute: %v", err)
+	}
+	_, err := client.Execute(context.Background(), connect.NewRequest(&edgev1.ExecuteRequest{MessageType: transcode.MsgAuthGuest}))
+	if connect.CodeOf(err) != connect.CodeResourceExhausted {
+		t.Fatalf("second execute code = %v, want ResourceExhausted", connect.CodeOf(err))
+	}
+	if !bl.Banned("127.0.0.1") {
+		t.Fatal("a public rejection must strike the banlist")
+	}
+}
+
+// TestUserRejectionDoesNotBan verifies the user limiter class (keyed by account id,
+// not IP) does not feed the IP banlist — that path is the backend's soft flag.
+func TestUserRejectionDoesNotBan(t *testing.T) {
+	bl := enabledBanlist(1)
+	limits := config.DefaultRateLimit()
+	limits.UserPerMinute, limits.UserBurst = 1, 1
+	_, client, cleanup := guardedEdge(t, bl, "", limits, func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/v1/internal/sessions/resolve":
+			_, _ = w.Write([]byte(`{"user_id":"u-1","is_guest":false}`))
+		case "/api/v1/user/feedback/unread":
+			_, _ = w.Write([]byte(`{"reply_unread":false}`))
+		default:
+			t.Errorf("unexpected backend path %s", r.URL.Path)
+		}
+	})
+	defer cleanup()
+
+	for i := range 2 {
+		req := connect.NewRequest(&edgev1.ExecuteRequest{MessageType: transcode.MsgFeedbackUnread})
+		req.Header().Set("Authorization", "Bearer tok")
+		_, err := client.Execute(context.Background(), req)
+		if i == 1 && connect.CodeOf(err) != connect.CodeResourceExhausted {
+			t.Fatalf("second execute code = %v, want ResourceExhausted", connect.CodeOf(err))
+		}
+	}
+	if len(bl.Active()) != 0 {
+		t.Fatalf("user-class rejection must not ban; active = %v", bl.Active())
+	}
+}
+
+// TestHoneytokenBansAndRejects verifies presenting the planted honeytoken bans the
+// caller and returns the ordinary invalid-session error without a backend call.
+func TestHoneytokenBansAndRejects(t *testing.T) {
+	bl := enabledBanlist(100)
+	_, client, cleanup := guardedEdge(t, bl, "s3cr3t-trap", config.DefaultRateLimit(), func(w http.ResponseWriter, r *http.Request) {
+		t.Error("backend must not be called for the honeytoken")
+	})
+	defer cleanup()
+
+	req := connect.NewRequest(&edgev1.ExecuteRequest{MessageType: transcode.MsgProfileGet})
+	req.Header().Set("Authorization", "Bearer s3cr3t-trap")
+	_, err := client.Execute(context.Background(), req)
+	if connect.CodeOf(err) != connect.CodeUnauthenticated {
+		t.Fatalf("honeytoken code = %v, want Unauthenticated", connect.CodeOf(err))
+	}
+	if !bl.Banned("127.0.0.1") {
+		t.Fatal("the honeytoken must ban the caller")
+	}
+}

gateway/internal/connectsrv/metrics.go

@@ -26,6 +26,7 @@ var activeUserWindows = []struct {
 type serverMetrics struct {
 	edge        metric.Float64Histogram
 	rateLimited metric.Int64Counter
+	banned      metric.Int64Counter
 	active      *activeUsers
 }
 
@@ -48,7 +49,12 @@ func newServerMetrics(meter metric.Meter) *serverMetrics {
 	if err != nil {
 		c, _ = noop.NewMeterProvider().Meter(meterName).Int64Counter("gateway_rate_limited_total")
 	}
-	m := &serverMetrics{edge: h, rateLimited: c, active: newActiveUsers()}
+	b, err := meter.Int64Counter("gateway_abuse_banned_total",
+		metric.WithDescription("Temporary IP bans applied at the edge, by reason (rejections, tripwire or honeytoken)."))
+	if err != nil {
+		b, _ = noop.NewMeterProvider().Meter(meterName).Int64Counter("gateway_abuse_banned_total")
+	}
+	m := &serverMetrics{edge: h, rateLimited: c, banned: b, active: newActiveUsers()}
 
 	gauge, err := meter.Int64ObservableGauge("active_users",
 		metric.WithDescription("Distinct accounts that performed an authenticated action within the window (in-memory, single gateway instance)."))
@@ -86,3 +92,9 @@ func (m *serverMetrics) recordActive(uid string) {
 func (m *serverMetrics) recordRateLimited(ctx context.Context, class string) {
 	m.rateLimited.Add(ctx, 1, metric.WithAttributes(attribute.String("class", class)))
 }
+
+// recordBan counts one temporary IP ban under reason (rejections, tripwire or
+// honeytoken).
+func (m *serverMetrics) recordBan(ctx context.Context, reason string) {
+	m.banned.Add(ctx, 1, metric.WithAttributes(attribute.String("reason", reason)))
+}

gateway/internal/connectsrv/metrics_test.go

@@ -90,3 +90,41 @@ func TestRateLimitedMetric(t *testing.T) {
 		t.Errorf("rate_limited counts = %v, want user=2 public=1", counts)
 	}
 }
+
+// TestBannedMetric records ban events through a manual reader and asserts
+// gateway_abuse_banned_total splits by reason.
+func TestBannedMetric(t *testing.T) {
+	ctx := context.Background()
+	reader := sdkmetric.NewManualReader()
+	meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
+	m := newServerMetrics(meter)
+
+	m.recordBan(ctx, "tripwire")
+	m.recordBan(ctx, "tripwire")
+	m.recordBan(ctx, "honeytoken")
+
+	var rm metricdata.ResourceMetrics
+	if err := reader.Collect(ctx, &rm); err != nil {
+		t.Fatalf("collect: %v", err)
+	}
+
+	counts := map[string]int64{}
+	for _, sm := range rm.ScopeMetrics {
+		for _, md := range sm.Metrics {
+			if md.Name != "gateway_abuse_banned_total" {
+				continue
+			}
+			sum, ok := md.Data.(metricdata.Sum[int64])
+			if !ok {
+				t.Fatalf("gateway_abuse_banned_total is not an int64 sum")
+			}
+			for _, dp := range sum.DataPoints {
+				reason, _ := dp.Attributes.Value(attribute.Key("reason"))
+				counts[reason.AsString()] += dp.Value
+			}
+		}
+	}
+	if counts["tripwire"] != 2 || counts["honeytoken"] != 1 {
+		t.Errorf("banned counts = %v, want tripwire=2 honeytoken=1", counts)
+	}
+}

gateway/internal/connectsrv/server.go

@@ -8,6 +8,7 @@ package connectsrv
 
 import (
 	"context"
+	"crypto/subtle"
 	"errors"
 	"net"
 	"net/http"
@@ -34,6 +35,12 @@ import (
 // heartbeatKind is the live-stream keep-alive event kind.
 const heartbeatKind = "heartbeat"
 
+// honeypotHeader marks a request the edge proxy routed from a honeypot decoy path;
+// any request carrying it is treated as a scanner hit. The proxy strips any
+// client-supplied value before setting its own, and a spoofed value only bans the
+// spoofer, so trusting it is safe.
+const honeypotHeader = "X-Scrabble-Honeypot"
+
 // Limiter classes, the `class` attribute of gateway_rate_limited_total and the
 // class field of the periodic rejection report.
 const (
@@ -63,6 +70,8 @@ type Server struct {
 	sessions   *session.Cache
 	limiter    *ratelimit.Limiter
 	tracker    *ratelimit.Tracker
+	banlist    *ratelimit.Banlist
+	honeytoken string
 	hub        *push.Hub
 	heartbeat  time.Duration
 	log        *zap.Logger
@@ -86,7 +95,13 @@ type Deps struct {
 	// Tracker accumulates limiter rejections for the periodic report; nil
 	// selects a private tracker (rejections are then only counted, never
 	// reported).
-	Tracker    *ratelimit.Tracker
+	Tracker *ratelimit.Tracker
+	// Banlist enforces temporary IP bans on the hot path; nil selects a disabled
+	// (inert) banlist.
+	Banlist *ratelimit.Banlist
+	// Honeytoken, when non-empty, is the planted bearer value whose presentation
+	// bans the caller and raises a high-severity alarm.
+	Honeytoken string
 	Hub        *push.Hub
 	RateLimit  config.RateLimitConfig
 	Heartbeat  time.Duration
@@ -116,6 +131,10 @@ func NewServer(d Deps) *Server {
 	if limiter == nil {
 		limiter = ratelimit.New()
 	}
+	banlist := d.Banlist
+	if banlist == nil {
+		banlist = ratelimit.NewBanlist(ratelimit.BanConfig{})
+	}
 	rl := d.RateLimit
 	if rl == (config.RateLimitConfig{}) {
 		rl = config.DefaultRateLimit()
@@ -125,6 +144,8 @@ func NewServer(d Deps) *Server {
 		sessions:     d.Sessions,
 		limiter:      limiter,
 		tracker:      tracker,
+		banlist:      banlist,
+		honeytoken:   d.Honeytoken,
 		hub:          d.Hub,
 		heartbeat:    d.Heartbeat,
 		log:          log,
@@ -172,9 +193,11 @@ func (s *Server) HTTPHandler() http.Handler {
 	mux.Handle("/telegram/", webui.Handler("/telegram/", "index.html"))
 	mux.Handle("/app/", webui.Handler("/app/", "index.html"))
 	mux.Handle("/", http.RedirectHandler("/app/", http.StatusPermanentRedirect))
-	// Every request body on the public listener is capped (the admin proxy POSTs
-	// included); the h2c server carries explicit stream/idle sizing.
-	return h2c.NewHandler(maxBodyHandler(s.maxBodyBytes, mux), &http2.Server{
+	// abuseGuard is the outermost wrap (right under h2c) so a banned IP or a
+	// honeypot hit is turned away before the body cap and the mux. Every request
+	// body on the public listener is then capped (the admin proxy POSTs included);
+	// the h2c server carries explicit stream/idle sizing.
+	return h2c.NewHandler(s.abuseGuard(maxBodyHandler(s.maxBodyBytes, mux)), &http2.Server{
 		MaxConcurrentStreams: h2cMaxConcurrentStreams,
 		IdleTimeout:          h2cIdleTimeout,
 	})
@@ -189,6 +212,32 @@ func maxBodyHandler(limit int, next http.Handler) http.Handler {
 	})
 }
 
+// abuseGuard refuses a banned client IP with 429 before any work, and turns a
+// honeypot decoy hit (the proxy-set honeypotHeader) into an instant ban plus a
+// bland 404 that is indistinguishable from an ordinary miss. The ban check and the
+// tripwire ban are inert on a disabled banlist (the prod-only gate); the tripwire
+// hit is logged either way as scanner telemetry.
+func (s *Server) abuseGuard(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ip := peerIP(r.RemoteAddr, r.Header)
+		if s.banlist.Banned(ip) {
+			http.Error(w, "banned", http.StatusTooManyRequests)
+			return
+		}
+		if r.Header.Get(honeypotHeader) != "" {
+			s.log.Warn("honeypot tripwire",
+				zap.String("path", r.URL.Path),
+				zap.String("client_ip", ip))
+			if s.banlist.BanNow(ip, ratelimit.ReasonTripwire) {
+				s.metrics.recordBan(r.Context(), string(ratelimit.ReasonTripwire))
+			}
+			http.NotFound(w, r)
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
 // Execute runs one unary operation. Domain failures are returned in the envelope
 // (result_code != "ok", HTTP 200); only edge failures (rate limit, missing
 // session, unknown type, internal) become Connect errors.
@@ -207,7 +256,7 @@ func (s *Server) Execute(ctx context.Context, req *connect.Request[edgev1.Execut
 
 	tr := transcode.Request{Payload: req.Msg.GetPayload(), ClientIP: clientIP}
 	if op.Auth {
-		uid, isGuest, err := s.resolve(ctx, req.Header())
+		uid, isGuest, err := s.resolve(ctx, req.Header(), clientIP)
 		if err != nil {
 			result = "unauthenticated"
 			return nil, err
@@ -263,7 +312,7 @@ func (s *Server) Execute(ctx context.Context, req *connect.Request[edgev1.Execut
 // Subscribe streams the authenticated user's live events with a keep-alive
 // heartbeat until the client disconnects.
 func (s *Server) Subscribe(ctx context.Context, req *connect.Request[edgev1.SubscribeRequest], stream *connect.ServerStream[edgev1.Event]) error {
-	uid, _, err := s.resolve(ctx, req.Header())
+	uid, _, err := s.resolve(ctx, req.Header(), peerIP(req.Peer().Addr, req.Header()))
 	if err != nil {
 		return err
 	}
@@ -311,6 +360,12 @@ func (s *Server) Subscribe(ctx context.Context, req *connect.Request[edgev1.Subs
 func (s *Server) noteRateLimited(ctx context.Context, class, key, msgType string) {
 	s.metrics.recordRateLimited(ctx, class)
 	s.tracker.Add(class, key)
+	// IP-keyed rejections (public, email, admin — the key is the client IP) feed
+	// the ban; the user class is keyed by account id and is the backend soft-flag's
+	// concern, not the IP ban's.
+	if class != classUser && s.banlist.Strike(key) {
+		s.metrics.recordBan(ctx, string(ratelimit.ReasonRejections))
+	}
 	s.log.Debug("rate limited",
 		zap.String("class", class),
 		zap.String("key", key),
@@ -344,11 +399,21 @@ func (s *Server) limitAdmin(next http.Handler) http.Handler {
 // resolve extracts and resolves the Authorization bearer token to an account id
 // and its guest flag, returning a Connect Unauthenticated error when it is missing
 // or unknown.
-func (s *Server) resolve(ctx context.Context, h http.Header) (string, bool, error) {
+func (s *Server) resolve(ctx context.Context, h http.Header, clientIP string) (string, bool, error) {
 	token := bearerToken(h.Get("Authorization"))
 	if token == "" {
 		return "", false, connect.NewError(connect.CodeUnauthenticated, errMissingToken)
 	}
+	// The honeytoken is a planted value no real client holds: presenting it is a
+	// high-confidence intrusion signal, so ban the caller and raise the alarm, then
+	// return the ordinary invalid-session error so the trap stays indistinguishable.
+	if s.honeytoken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(s.honeytoken)) == 1 {
+		s.log.Warn("honeytoken presented", zap.String("client_ip", clientIP))
+		if s.banlist.BanNow(clientIP, ratelimit.ReasonHoneytoken) {
+			s.metrics.recordBan(ctx, string(ratelimit.ReasonHoneytoken))
+		}
+		return "", false, connect.NewError(connect.CodeUnauthenticated, errInvalidSession)
+	}
 	uid, isGuest, err := s.sessions.Resolve(ctx, token)
 	if err != nil {
 		// An unknown or expired token (a backend 4xx) is the client's problem and