4ba9da6721
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 24s
CI / ui (pull_request) Successful in 1m11s
CI / conformance (pull_request) Successful in 10s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m49s
Refuse a client whose IP is in a curated CIDR feed with 403 in the same
abuseGuard, before the fail2ban ban. Prod-only (keys by real client IP), off by
default.
- ratelimit.Blocklist: a sorted-range IPv4 matcher (binary search) with an
allowlist checked first; ParseDROP reads the feed; ApplyRefresh keeps the
last-good feed on a transient fetch failure and drops it fail-open once stale
(better to under-block than block a legitimate client on a frozen feed). A
separate static CIDR set, not the per-IP fail2ban store.
- gateway: a refresher goroutine re-fetches every few hours (bounded fetch + size
cap); config GATEWAY_BLOCKLIST_{ENABLED,URL,ALLOW,REFRESH,MAX_STALENESS}.
IPv6 is not matched (a v6 client is still covered by fail2ban / the honeypot).
- observability: gateway_blocklist_blocked_total + entries/age gauges; a Grafana
alert warns before the feed is dropped; service-overview panels.
- deploy: compose env + write-prod-env.sh + prod-deploy/rollback wiring (opt-in
via PROD_ vars).
- docs (ARCHITECTURE, deploy/README); unit tests (match, allowlist, IPv6 skip,
parser, fault-tolerance) + a 403 abuse-guard integration test.
189 lines
5.8 KiB
Go
189 lines
5.8 KiB
Go
package ratelimit
|
|
|
|
import (
|
|
"bufio"
|
|
"encoding/binary"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// ipRange is an inclusive IPv4 range [lo, hi] as uint32 — the form the blocklist matches against.
|
|
type ipRange struct{ lo, hi uint32 }
|
|
|
|
// Blocklist is a static IPv4 CIDR blocklist (a curated community feed such as Spamhaus DROP) enforced
|
|
// on the hot path alongside the fail2ban [Banlist]. It is refreshed periodically ([ApplyRefresh]); an
|
|
// allowlist (never blocked) protects known-good infrastructure and is checked first. Only IPv4 is
|
|
// matched — an IPv6 client is never blocked here (the fail2ban list and the honeypot still cover it).
|
|
// Disabled by default (prod-only, like the ban): while disabled, Blocked is always false.
|
|
type Blocklist struct {
|
|
enabled bool
|
|
allow []ipRange // sorted, non-overlapping; from config, immutable after construction
|
|
|
|
mu sync.RWMutex
|
|
ranges []ipRange // sorted, non-overlapping; the current feed
|
|
fetchedAt time.Time // last successful SetCIDRs; zero = never loaded
|
|
}
|
|
|
|
// NewBlocklist builds a Blocklist. enabled gates the whole mechanism; allow is the never-block set
|
|
// (CIDRs / bare IPs already parsed) — a client in it is never blocked even if the feed lists it.
|
|
func NewBlocklist(enabled bool, allow []*net.IPNet) *Blocklist {
|
|
return &Blocklist{enabled: enabled, allow: toRanges(allow)}
|
|
}
|
|
|
|
// Blocked reports whether ip (a textual address) is in the current feed and not allowlisted. It is
|
|
// false on a disabled or empty blocklist, and false for any non-IPv4 address.
|
|
func (b *Blocklist) Blocked(ip string) bool {
|
|
if !b.enabled {
|
|
return false
|
|
}
|
|
v, ok := ipv4ToUint32(ip)
|
|
if !ok {
|
|
return false
|
|
}
|
|
b.mu.RLock()
|
|
defer b.mu.RUnlock()
|
|
if len(b.ranges) == 0 || rangesContain(b.allow, v) {
|
|
return false
|
|
}
|
|
return rangesContain(b.ranges, v)
|
|
}
|
|
|
|
// SetCIDRs swaps in a freshly fetched feed, recording the fetch time. Non-IPv4 CIDRs are ignored.
|
|
func (b *Blocklist) SetCIDRs(cidrs []*net.IPNet, at time.Time) {
|
|
r := toRanges(cidrs)
|
|
b.mu.Lock()
|
|
b.ranges = r
|
|
b.fetchedAt = at
|
|
b.mu.Unlock()
|
|
}
|
|
|
|
// Clear drops the current feed (fail-open) but keeps the last-fetch time, so a staleness gauge keeps
|
|
// climbing. Blocked then returns false until a fresh feed loads.
|
|
func (b *Blocklist) Clear() {
|
|
b.mu.Lock()
|
|
b.ranges = nil
|
|
b.mu.Unlock()
|
|
}
|
|
|
|
// Len returns the number of ranges currently enforced.
|
|
func (b *Blocklist) Len() int {
|
|
b.mu.RLock()
|
|
defer b.mu.RUnlock()
|
|
return len(b.ranges)
|
|
}
|
|
|
|
// LastFetch returns the time of the last successful feed load (zero if never).
|
|
func (b *Blocklist) LastFetch() time.Time {
|
|
b.mu.RLock()
|
|
defer b.mu.RUnlock()
|
|
return b.fetchedAt
|
|
}
|
|
|
|
// RefreshOutcome is the result of one refresh attempt, for the caller's logging and metrics.
|
|
type RefreshOutcome int
|
|
|
|
const (
|
|
// RefreshUpdated: a new feed was fetched and applied.
|
|
RefreshUpdated RefreshOutcome = iota
|
|
// RefreshKept: the fetch failed but the last-good feed is still fresh, so it was kept.
|
|
RefreshKept
|
|
// RefreshDropped: the fetch failed and the feed went stale, so it was dropped (fail-open).
|
|
RefreshDropped
|
|
)
|
|
|
|
// ApplyRefresh updates bl from one fetch outcome: on success it applies the new feed; on failure it
|
|
// keeps the last-good feed unless it is older than maxStaleness, in which case it drops it (fail-open
|
|
// — better to under-block than to block a legitimate client on a frozen feed). now is the wall clock.
|
|
func ApplyRefresh(bl *Blocklist, cidrs []*net.IPNet, fetchErr error, now time.Time, maxStaleness time.Duration) RefreshOutcome {
|
|
if fetchErr == nil {
|
|
bl.SetCIDRs(cidrs, now)
|
|
return RefreshUpdated
|
|
}
|
|
last := bl.LastFetch()
|
|
if last.IsZero() || now.Sub(last) > maxStaleness {
|
|
bl.Clear()
|
|
return RefreshDropped
|
|
}
|
|
return RefreshKept
|
|
}
|
|
|
|
// ParseDROP parses a Spamhaus DROP-style feed: one CIDR per line with an optional "; comment" tail,
|
|
// plus blank and comment lines. It returns the IPv4 networks; a bare IP is read as a /32, and
|
|
// non-IPv4 or malformed entries are skipped so one bad line never fails the whole feed.
|
|
func ParseDROP(r io.Reader) ([]*net.IPNet, error) {
|
|
var out []*net.IPNet
|
|
sc := bufio.NewScanner(r)
|
|
sc.Buffer(make([]byte, 0, 64*1024), 1<<20)
|
|
for sc.Scan() {
|
|
line := sc.Text()
|
|
if i := strings.IndexByte(line, ';'); i >= 0 {
|
|
line = line[:i]
|
|
}
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
if !strings.Contains(line, "/") {
|
|
line += "/32"
|
|
}
|
|
_, ipnet, err := net.ParseCIDR(line)
|
|
if err != nil || ipnet.IP.To4() == nil {
|
|
continue
|
|
}
|
|
out = append(out, ipnet)
|
|
}
|
|
if err := sc.Err(); err != nil {
|
|
return nil, fmt.Errorf("ratelimit: read blocklist: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// toRanges converts IPv4 CIDRs to sorted, uint32 inclusive ranges (the match form); non-IPv4 CIDRs
|
|
// are dropped. Sorting by lo lets [rangesContain] binary-search.
|
|
func toRanges(cidrs []*net.IPNet) []ipRange {
|
|
out := make([]ipRange, 0, len(cidrs))
|
|
for _, n := range cidrs {
|
|
if n == nil {
|
|
continue
|
|
}
|
|
ip4 := n.IP.To4()
|
|
if ip4 == nil {
|
|
continue
|
|
}
|
|
ones, bits := n.Mask.Size()
|
|
if bits != 32 {
|
|
continue
|
|
}
|
|
lo := binary.BigEndian.Uint32(ip4)
|
|
hi := lo | (uint32(0xffffffff) >> uint(ones))
|
|
out = append(out, ipRange{lo: lo, hi: hi})
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].lo < out[j].lo })
|
|
return out
|
|
}
|
|
|
|
// rangesContain reports whether v falls in any range of the sorted, non-overlapping slice, via a
|
|
// binary search for the last range whose lo is at most v.
|
|
func rangesContain(ranges []ipRange, v uint32) bool {
|
|
i := sort.Search(len(ranges), func(i int) bool { return ranges[i].lo > v })
|
|
return i > 0 && ranges[i-1].hi >= v
|
|
}
|
|
|
|
// ipv4ToUint32 parses a textual address to a big-endian uint32, reporting whether it was IPv4.
|
|
func ipv4ToUint32(s string) (uint32, bool) {
|
|
ip := net.ParseIP(s)
|
|
if ip == nil {
|
|
return 0, false
|
|
}
|
|
ip4 := ip.To4()
|
|
if ip4 == nil {
|
|
return 0, false
|
|
}
|
|
return binary.BigEndian.Uint32(ip4), true
|
|
}
|