feat(deploy): Grafana infra alerts + blackbox cert probe + admin-alert wiring
Grafana: GF_SMTP from the shared relay (STARTTLS host:port) + alerting provisioning (contact point → SERVICE_EMAIL, route-all policy, and rules for scrape-target down, gateway internal-error rate + p99 latency, host mem/disk/cpu, postgres connections, and TLS cert < 20 days). All rules noDataState=OK so an absent metric never false-alerts. blackbox_exporter probes the edge caddy's TLS (probe_ssl_earliest_cert_expiry) — effective on prod (caddy terminates TLS; contour caddy is HTTP-only so the metric is absent). Wires the new env through compose (backend admin From/To, Grafana SMTP), ci.yaml (TEST_), prod-deploy (PROD_ + env.sh), .env.example and the README var table.
This commit is contained in:
Ilia Denisov
@@ -0,0 +1,13 @@
|
||||
# Grafana alerting contact point: the operator's alert mailbox, read from the
|
||||
# SERVICE_EMAIL container env (see docker-compose.yml grafana), so it stays per-contour.
|
||||
# SERVICE_EMAIL may hold several comma-separated addresses.
|
||||
apiVersion: 1
|
||||
contactPoints:
|
||||
- orgId: 1
|
||||
name: ops-email
|
||||
receivers:
|
||||
- uid: ops_email
|
||||
type: email
|
||||
settings:
|
||||
addresses: $__env{SERVICE_EMAIL}
|
||||
singleEmail: true
|
||||
@@ -0,0 +1,10 @@
|
||||
# Notification policy: every alert routes to the operator email, grouped so a burst is one
|
||||
# message, with a 4-hour re-notify while still firing.
|
||||
apiVersion: 1
|
||||
policies:
|
||||
- orgId: 1
|
||||
receiver: ops-email
|
||||
group_by: ['alertname']
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
@@ -0,0 +1,214 @@
|
||||
# Grafana provisioned alert rules for the Scrabble contour. Each rule is a Prometheus
|
||||
# instant query (refId A) fed into a threshold expression (refId C). noDataState/execErrState
|
||||
# are OK so an absent metric never raises a false alert — notably cert-expiry, whose blackbox
|
||||
# probe has no target on the test contour (caddy is HTTP-only there). Metric names are the
|
||||
# real ones Prometheus exposes (edge_request_* from the gateway via the collector,
|
||||
# node_*/pg_*/probe_ssl_* from the exporters). All route to the ops-email contact point.
|
||||
apiVersion: 1
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: scrabble-service
|
||||
folder: Alerts
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: svc_target_down
|
||||
title: Scrape target down
|
||||
condition: C
|
||||
for: 3m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: up, instant: true }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator: { type: lt, params: [1] }
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: 'A Prometheus scrape target is down (up < 1).' }
|
||||
|
||||
- uid: edge_error_rate
|
||||
title: Gateway internal-error rate high
|
||||
condition: C
|
||||
for: 10m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 600, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
refId: A
|
||||
expr: sum by (service_name) (rate(edge_request_duration_count{result="internal"}[5m]))
|
||||
instant: true
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator: { type: gt, params: [0.05] }
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: 'Sustained internal (5xx-equivalent) errors at the edge.' }
|
||||
|
||||
- uid: edge_latency_p99
|
||||
title: Gateway request latency p99 high
|
||||
condition: C
|
||||
for: 10m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 600, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
refId: A
|
||||
expr: histogram_quantile(0.99, sum by (le, service_name) (rate(edge_request_duration_bucket[5m])))
|
||||
instant: true
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator: { type: gt, params: [1] }
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: 'Edge request p99 latency above 1s.' }
|
||||
|
||||
- uid: tls_cert_expiry
|
||||
title: TLS certificate nearing expiry
|
||||
condition: C
|
||||
for: 15m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
refId: A
|
||||
expr: (probe_ssl_earliest_cert_expiry - time()) / 86400
|
||||
instant: true
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator: { type: lt, params: [20] }
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: 'Edge TLS cert has under 20 days left — Caddy ACME renewal may have failed.' }
|
||||
|
||||
- orgId: 1
|
||||
name: scrabble-host
|
||||
folder: Alerts
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: host_mem_low
|
||||
title: Host memory low
|
||||
condition: C
|
||||
for: 5m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
refId: A
|
||||
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes
|
||||
instant: true
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator: { type: lt, params: [0.1] }
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: 'Under 10% host memory available.' }
|
||||
|
||||
- uid: host_disk_low
|
||||
title: Host disk low
|
||||
condition: C
|
||||
for: 10m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
refId: A
|
||||
expr: min(node_filesystem_avail_bytes / node_filesystem_size_bytes)
|
||||
instant: true
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator: { type: lt, params: [0.1] }
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: 'A host filesystem is under 10% free.' }
|
||||
|
||||
- uid: host_cpu_high
|
||||
title: Host CPU saturated
|
||||
condition: C
|
||||
for: 10m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 600, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
refId: A
|
||||
expr: 1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m]))
|
||||
instant: true
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator: { type: gt, params: [0.9] }
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: 'Host CPU above 90% for 10 minutes.' }
|
||||
|
||||
- uid: pg_connections_high
|
||||
title: Postgres connections high
|
||||
condition: C
|
||||
for: 5m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
refId: A
|
||||
expr: sum(pg_stat_activity_count) / max(pg_settings_max_connections)
|
||||
instant: true
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator: { type: gt, params: [0.8] }
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: 'Postgres using over 80% of max_connections.' }
|
||||
Reference in New Issue
Block a user