scrabble-game/deploy/grafana/provisioning/alerting/rules.yaml

# Grafana provisioned alert rules for the Scrabble contour. Each rule is a Prometheus
# instant query (refId A) fed into a threshold expression (refId C). noDataState/execErrState
# are OK so an absent metric never raises a false alert — notably cert-expiry, whose blackbox
# probe has no target on the test contour (caddy is HTTP-only there). Metric names are the
# real ones Prometheus exposes (edge_request_* from the gateway via the collector,
# node_*/pg_*/probe_ssl_* from the exporters). All route to the ops-email contact point.
apiVersion: 1
groups:
  - orgId: 1
    name: scrabble-service
    folder: Alerts
    interval: 1m
    rules:
      - uid: svc_target_down
        title: Scrape target down
        condition: C
        for: 3m
        noDataState: OK
        execErrState: OK
        data:
          - refId: A
            relativeTimeRange: { from: 300, to: 0 }
            datasourceUid: prometheus
            model: { refId: A, expr: up, instant: true }
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions:
                - evaluator: { type: lt, params: [1] }
        labels: { severity: critical }
        annotations: { summary: 'A Prometheus scrape target is down (up < 1).' }

      - uid: edge_error_rate
        title: Gateway internal-error rate high
        condition: C
        for: 10m
        noDataState: OK
        execErrState: OK
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: prometheus
            model:
              refId: A
              expr: sum by (service_name) (rate(edge_request_duration_count{result="internal"}[5m]))
              instant: true
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions:
                - evaluator: { type: gt, params: [0.05] }
        labels: { severity: warning }
        annotations: { summary: 'Sustained internal (5xx-equivalent) errors at the edge.' }

      - uid: edge_latency_p99
        title: Gateway request latency p99 high
        condition: C
        for: 10m
        noDataState: OK
        execErrState: OK
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: prometheus
            model:
              refId: A
              expr: histogram_quantile(0.99, sum by (le, service_name) (rate(edge_request_duration_bucket[5m])))
              instant: true
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions:
                - evaluator: { type: gt, params: [1] }
        labels: { severity: warning }
        annotations: { summary: 'Edge request p99 latency above 1s.' }

      - uid: tls_cert_expiry
        title: TLS certificate nearing expiry
        condition: C
        for: 15m
        noDataState: OK
        execErrState: OK
        data:
          - refId: A
            relativeTimeRange: { from: 300, to: 0 }
            datasourceUid: prometheus
            model:
              refId: A
              expr: (probe_ssl_earliest_cert_expiry - time()) / 86400
              instant: true
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions:
                - evaluator: { type: lt, params: [20] }
        labels: { severity: critical }
        annotations: { summary: 'Edge TLS cert has under 20 days left — Caddy ACME renewal may have failed.' }

  - orgId: 1
    name: scrabble-host
    folder: Alerts
    interval: 1m
    rules:
      - uid: host_mem_low
        title: Host memory low
        condition: C
        for: 5m
        noDataState: OK
        execErrState: OK
        data:
          - refId: A
            relativeTimeRange: { from: 300, to: 0 }
            datasourceUid: prometheus
            model:
              refId: A
              expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes
              instant: true
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions:
                - evaluator: { type: lt, params: [0.1] }
        labels: { severity: critical }
        annotations: { summary: 'Under 10% host memory available.' }

      - uid: host_disk_low
        title: Host disk low
        condition: C
        for: 10m
        noDataState: OK
        execErrState: OK
        data:
          - refId: A
            relativeTimeRange: { from: 300, to: 0 }
            datasourceUid: prometheus
            model:
              refId: A
              expr: min(node_filesystem_avail_bytes / node_filesystem_size_bytes)
              instant: true
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions:
                - evaluator: { type: lt, params: [0.1] }
        labels: { severity: critical }
        annotations: { summary: 'A host filesystem is under 10% free.' }

      - uid: host_cpu_high
        title: Host CPU saturated
        condition: C
        for: 10m
        noDataState: OK
        execErrState: OK
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: prometheus
            model:
              refId: A
              expr: 1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m]))
              instant: true
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions:
                - evaluator: { type: gt, params: [0.9] }
        labels: { severity: warning }
        annotations: { summary: 'Host CPU above 90% for 10 minutes.' }

      - uid: pg_connections_high
        title: Postgres connections high
        condition: C
        for: 5m
        noDataState: OK
        execErrState: OK
        data:
          - refId: A
            relativeTimeRange: { from: 300, to: 0 }
            datasourceUid: prometheus
            model:
              refId: A
              expr: sum(pg_stat_activity_count) / max(pg_settings_max_connections)
              instant: true
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions:
                - evaluator: { type: gt, params: [0.8] }
        labels: { severity: warning }
        annotations: { summary: 'Postgres using over 80% of max_connections.' }