feat(edge): community IP blocklist (Spamhaus DROP) at the gateway
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 24s
CI / ui (pull_request) Successful in 1m11s
CI / conformance (pull_request) Successful in 10s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m49s
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 24s
CI / ui (pull_request) Successful in 1m11s
CI / conformance (pull_request) Successful in 10s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m49s
Refuse a client whose IP is in a curated CIDR feed with 403 in the same
abuseGuard, before the fail2ban ban. Prod-only (keys by real client IP), off by
default.
- ratelimit.Blocklist: a sorted-range IPv4 matcher (binary search) with an
allowlist checked first; ParseDROP reads the feed; ApplyRefresh keeps the
last-good feed on a transient fetch failure and drops it fail-open once stale
(better to under-block than block a legitimate client on a frozen feed). A
separate static CIDR set, not the per-IP fail2ban store.
- gateway: a refresher goroutine re-fetches every few hours (bounded fetch + size
cap); config GATEWAY_BLOCKLIST_{ENABLED,URL,ALLOW,REFRESH,MAX_STALENESS}.
IPv6 is not matched (a v6 client is still covered by fail2ban / the honeypot).
- observability: gateway_blocklist_blocked_total + entries/age gauges; a Grafana
alert warns before the feed is dropped; service-overview panels.
- deploy: compose env + write-prod-env.sh + prod-deploy/rollback wiring (opt-in
via PROD_ vars).
- docs (ARCHITECTURE, deploy/README); unit tests (match, allowlist, IPv6 skip,
parser, fault-tolerance) + a 403 abuse-guard integration test.
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"scrabble/gateway/internal/config"
|
||||
"scrabble/gateway/internal/ratelimit"
|
||||
)
|
||||
|
||||
const (
|
||||
// blocklistFetchTimeout bounds one feed fetch.
|
||||
blocklistFetchTimeout = 30 * time.Second
|
||||
// maxBlocklistBytes caps the fetched feed (Spamhaus DROP is well under 1 MiB; the cap stops a
|
||||
// hostile or misconfigured URL from streaming unbounded data into memory).
|
||||
maxBlocklistBytes = 8 << 20
|
||||
)
|
||||
|
||||
// parseAllowlist parses the never-block entries (CIDRs or bare IPs, a bare IP read as a /32) into
|
||||
// networks. A malformed entry is a fatal config error — the operator must fix it, not silently lose
|
||||
// a protected range.
|
||||
func parseAllowlist(entries []string) ([]*net.IPNet, error) {
|
||||
out := make([]*net.IPNet, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
s := strings.TrimSpace(e)
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(s, "/") {
|
||||
s += "/32"
|
||||
}
|
||||
_, n, err := net.ParseCIDR(s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gateway: GATEWAY_BLOCKLIST_ALLOW: %q: %w", e, err)
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// runBlocklistRefresher keeps the community IP blocklist feed current: it fetches immediately, then
|
||||
// every cfg.Refresh, and applies each outcome through ratelimit.ApplyRefresh (keep-last-good on a
|
||||
// transient failure; drop the feed once it goes stale, fail-open). It returns when ctx is cancelled.
|
||||
func runBlocklistRefresher(ctx context.Context, bl *ratelimit.Blocklist, cfg config.BlocklistConfig, log *zap.Logger) {
|
||||
client := &http.Client{Timeout: blocklistFetchTimeout}
|
||||
for {
|
||||
cidrs, err := fetchBlocklist(ctx, client, cfg.URL)
|
||||
switch ratelimit.ApplyRefresh(bl, cidrs, err, time.Now(), cfg.MaxStaleness) {
|
||||
case ratelimit.RefreshUpdated:
|
||||
log.Info("blocklist refreshed", zap.String("url", cfg.URL), zap.Int("entries", bl.Len()))
|
||||
case ratelimit.RefreshKept:
|
||||
log.Warn("blocklist refresh failed; keeping the last-good feed", zap.Error(err))
|
||||
case ratelimit.RefreshDropped:
|
||||
log.Warn("blocklist refresh failed and the feed is stale; dropped it (fail-open)", zap.Error(err))
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(cfg.Refresh):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fetchBlocklist GETs the feed and parses it, bounded by the fetch timeout and the size cap.
|
||||
func fetchBlocklist(ctx context.Context, client *http.Client, url string) ([]*net.IPNet, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("gateway: blocklist fetch %s: status %d", url, resp.StatusCode)
|
||||
}
|
||||
return ratelimit.ParseDROP(io.LimitReader(resp.Body, maxBlocklistBytes))
|
||||
}
|
||||
@@ -129,6 +129,11 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
|
||||
Window: cfg.Abuse.BanWindow,
|
||||
Duration: cfg.Abuse.BanDuration,
|
||||
})
|
||||
allow, err := parseAllowlist(cfg.Blocklist.Allow)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blocklist := ratelimit.NewBlocklist(cfg.Blocklist.Enabled, allow)
|
||||
hub := push.NewHub(0)
|
||||
|
||||
var validator transcode.TelegramValidator
|
||||
@@ -222,6 +227,7 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
|
||||
Limiter: limiter,
|
||||
Tracker: tracker,
|
||||
Banlist: banlist,
|
||||
Blocklist: blocklist,
|
||||
Honeytoken: cfg.Abuse.Honeytoken,
|
||||
VKAppSecret: cfg.VKAppSecret,
|
||||
Hub: hub,
|
||||
@@ -243,6 +249,10 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
|
||||
if cfg.Abuse.BanEnabled {
|
||||
go runBanSync(ctx, banlist, backend, logger)
|
||||
}
|
||||
// When the edge blocklist is enabled (prod), keep the community feed refreshed.
|
||||
if cfg.Blocklist.Enabled {
|
||||
go runBlocklistRefresher(ctx, blocklist, cfg.Blocklist, logger)
|
||||
}
|
||||
|
||||
public := &http.Server{Addr: cfg.HTTPAddr, Handler: edge.HTTPHandler(), ReadHeaderTimeout: readHeaderTimeout}
|
||||
servers := []*namedServer{{name: "public", srv: public}}
|
||||
|
||||
Reference in New Issue
Block a user