diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 7527852..ae8aaf0 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -340,6 +340,15 @@ jobs: SMTP_RELAY_PORT: ${{ vars.TEST_SMTP_RELAY_PORT }} SMTP_RELAY_TLS: ${{ vars.TEST_SMTP_RELAY_TLS }} SMTP_RELAY_FROM: ${{ vars.TEST_SMTP_RELAY_FROM }} + # Operator alerts: backend admin emails (new feedback / complaints) + Grafana + # infra alerts. Distinct senders + recipients; Grafana uses the relay's STARTTLS + # host:port. Empty leaves the alert worker off and Grafana SMTP disabled. + SMTP_RELAY_ADMIN_FROM: ${{ vars.TEST_SMTP_RELAY_ADMIN_FROM }} + ADMIN_EMAIL: ${{ vars.TEST_ADMIN_EMAIL }} + SMTP_RELAY_SERVICE_FROM: ${{ vars.TEST_SMTP_RELAY_SERVICE_FROM }} + SERVICE_EMAIL: ${{ vars.TEST_SERVICE_EMAIL }} + GRAFANA_SMTP_HOST: ${{ vars.TEST_GRAFANA_SMTP_HOST }} + GF_SMTP_ENABLED: ${{ vars.TEST_GF_SMTP_ENABLED }} # Canonical public origin for links in the email (this contour's URL); # required by the backend whenever SMTP_RELAY_HOST is set. PUBLIC_BASE_URL: ${{ vars.TEST_PUBLIC_BASE_URL }} diff --git a/.gitea/workflows/prod-deploy.yaml b/.gitea/workflows/prod-deploy.yaml index 139ecde..0a9aac1 100644 --- a/.gitea/workflows/prod-deploy.yaml +++ b/.gitea/workflows/prod-deploy.yaml @@ -93,6 +93,14 @@ jobs: SMTP_RELAY_PORT: ${{ vars.PROD_SMTP_RELAY_PORT }} SMTP_RELAY_TLS: ${{ vars.PROD_SMTP_RELAY_TLS }} SMTP_RELAY_FROM: ${{ vars.PROD_SMTP_RELAY_FROM }} + # Operator alerts: backend admin emails + Grafana infra alerts (distinct senders + + # recipients; Grafana uses the relay's STARTTLS host:port). + SMTP_RELAY_ADMIN_FROM: ${{ vars.PROD_SMTP_RELAY_ADMIN_FROM }} + ADMIN_EMAIL: ${{ vars.PROD_ADMIN_EMAIL }} + SMTP_RELAY_SERVICE_FROM: ${{ vars.PROD_SMTP_RELAY_SERVICE_FROM }} + SERVICE_EMAIL: ${{ vars.PROD_SERVICE_EMAIL }} + GRAFANA_SMTP_HOST: ${{ vars.PROD_GRAFANA_SMTP_HOST }} + GF_SMTP_ENABLED: ${{ vars.PROD_GF_SMTP_ENABLED }} PUBLIC_BASE_URL: ${{ vars.PROD_PUBLIC_BASE_URL }} PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }} PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }} @@ -156,6 +164,12 @@ jobs: export SMTP_RELAY_USER='$SMTP_RELAY_USER' export SMTP_RELAY_PASS='$SMTP_RELAY_PASS' export SMTP_RELAY_FROM='$SMTP_RELAY_FROM' + export SMTP_RELAY_ADMIN_FROM='$SMTP_RELAY_ADMIN_FROM' + export ADMIN_EMAIL='$ADMIN_EMAIL' + export SMTP_RELAY_SERVICE_FROM='$SMTP_RELAY_SERVICE_FROM' + export SERVICE_EMAIL='$SERVICE_EMAIL' + export GRAFANA_SMTP_HOST='$GRAFANA_SMTP_HOST' + export GF_SMTP_ENABLED='$GF_SMTP_ENABLED' export PUBLIC_BASE_URL='$PUBLIC_BASE_URL' export GATEWAY_ABUSE_BAN_ENABLED='true' EOF diff --git a/deploy/.env.example b/deploy/.env.example index 12d9fdc..74a3fbf 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -45,6 +45,17 @@ SMTP_RELAY_PASS= # secret SMTP_RELAY_FROM=no-reply@erudit-game.ru PUBLIC_BASE_URL= # required when SMTP_RELAY_HOST is set (e.g. https://erudit-game.ru) +# Operator alerts (email). The backend emails the admin on new feedback / word complaints +# (coalesced), and Grafana emails infra alerts. Distinct senders; recipients may be several +# comma-separated addresses. Grafana speaks STARTTLS, so it needs the relay's STARTTLS +# host:port (GRAFANA_SMTP_HOST) and reuses SMTP_RELAY_USER/PASS. All empty = alerts off. +SMTP_RELAY_ADMIN_FROM= # backend admin-alert From (e.g. alerts@erudit-game.ru) +ADMIN_EMAIL= # backend admin-alert recipient(s), comma-separated +SMTP_RELAY_SERVICE_FROM= # Grafana alert From (e.g. grafana@erudit-game.ru) +SERVICE_EMAIL= # Grafana alert recipient(s), comma-separated +GRAFANA_SMTP_HOST= # relay host:STARTTLS-port for Grafana (e.g. smtp.mail.selcloud.ru:1126) +GF_SMTP_ENABLED=false # set true to enable Grafana alert emails + # --- Edge / caddy ----------------------------------------------------------- # Test: ":80" (the host caddy terminates TLS and forwards to scrabble:80 on the # external `edge` network). Prod: a domain so caddy does its own ACME. diff --git a/deploy/README.md b/deploy/README.md index 47e981b..e259d9c 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -103,6 +103,12 @@ without it Docker's resolver handles `otelcol`, `gateway` and `api.telegram.org` | `SMTP_RELAY_PASS` | secret | _(empty)_ | Relay SMTP AUTH password. | | `SMTP_RELAY_FROM` | variable | `no-reply@localhost` | Sender address. **Must use the prod domain** (`no-reply@erudit-game.ru`) — Selectel only accepts the verified sender domain — so it is the same on every contour. | | `PUBLIC_BASE_URL` | variable | _(empty)_ | Canonical public https origin for links in the email (the contour's own URL, e.g. `https://erudit-game.ru`). **Required by the backend whenever `SMTP_RELAY_HOST` is set** — it is never derived from a request Host header (anti-injection). | +| `SMTP_RELAY_ADMIN_FROM` | variable | _(empty)_ | Backend operator-alert sender (new feedback / word complaints), distinct from the confirm-code From. Empty (with `ADMIN_EMAIL`) disables the alert worker. | +| `ADMIN_EMAIL` | variable | _(empty)_ | Backend operator-alert recipient(s); several comma-separated addresses allowed. | +| `SMTP_RELAY_SERVICE_FROM` | variable | _(empty)_ | Grafana infra-alert sender address. | +| `SERVICE_EMAIL` | variable | _(empty)_ | Grafana infra-alert recipient(s); comma-separated allowed (read by the provisioned contact point via `$__env{SERVICE_EMAIL}`). | +| `GRAFANA_SMTP_HOST` | variable | _(empty)_ | Relay `host:port` on the **STARTTLS** port for Grafana (its client speaks STARTTLS, not the backend's implicit-TLS port), e.g. `smtp.mail.selcloud.ru:1126`. Reuses `SMTP_RELAY_USER`/`PASS`. | +| `GF_SMTP_ENABLED` | variable | `false` | Set `true` to enable Grafana alert emails. | The six `VITE_*` are **build-args** baked into the gateway and landing images at build time (both targets share one UI build stage — keep the args identical so it is @@ -211,7 +217,9 @@ SMTP_RELAY_USER, SMTP_RELAY_PASS}`; variables: GRAFANA_ROOT_URL, LOG_LEVEL, DICT_VERSION, TELEGRAM_MINIAPP_URL, TELEGRAM_GAME_CHANNEL_ID, TELEGRAM_CHAT_ID, TELEGRAM_BOT_USERNAME, VITE_TELEGRAM_BOT_ID, VITE_TELEGRAM_LINK, VITE_TELEGRAM_GAME_CHANNEL_NAME, VITE_VK_APP_LINK, -SMTP_RELAY_HOST, SMTP_RELAY_PORT, SMTP_RELAY_TLS, SMTP_RELAY_FROM, PUBLIC_BASE_URL}`. +SMTP_RELAY_HOST, SMTP_RELAY_PORT, SMTP_RELAY_TLS, SMTP_RELAY_FROM, PUBLIC_BASE_URL, +SMTP_RELAY_ADMIN_FROM, ADMIN_EMAIL, SMTP_RELAY_SERVICE_FROM, SERVICE_EMAIL, GRAFANA_SMTP_HOST, +GF_SMTP_ENABLED}`. ## Host-side setup (outside this repo) diff --git a/deploy/blackbox/blackbox.yml b/deploy/blackbox/blackbox.yml new file mode 100644 index 0000000..c757979 --- /dev/null +++ b/deploy/blackbox/blackbox.yml @@ -0,0 +1,14 @@ +# blackbox_exporter probe modules. tls_cert opens a verified TLS connection to the edge +# caddy so Prometheus can read probe_ssl_earliest_cert_expiry — the signal behind the +# certificate-renewal-failure alert. The SNI is the production public host, whose cert the +# edge caddy serves once it does its own ACME; on the test contour caddy is HTTP-only +# (behind the host caddy), so the probe finds nothing on :443 and the cert metric is absent. +modules: + tls_cert: + prober: tcp + timeout: 5s + tcp: + tls: true + tls_config: + server_name: erudit-game.ru + insecure_skip_verify: false diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index d88221d..ef116e2 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -151,6 +151,10 @@ services: BACKEND_SMTP_PASSWORD: ${SMTP_RELAY_PASS:-} BACKEND_SMTP_FROM: ${SMTP_RELAY_FROM:-no-reply@localhost} BACKEND_PUBLIC_BASE_URL: ${PUBLIC_BASE_URL:-} + # Operator alert emails (new feedback / word complaints): a distinct sender and the + # recipient(s) (comma-separated allowed). Both empty disables the alert worker. + BACKEND_SMTP_ADMIN_FROM: ${SMTP_RELAY_ADMIN_FROM:-} + BACKEND_ADMIN_EMAIL: ${ADMIN_EMAIL:-} # The dictionary lives on a named volume seeded from the image on first boot # (the image's /opt/dawg is owned by the nonroot UID, which the fresh volume # inherits). The admin console writes new version subdirectories here, and the @@ -497,6 +501,18 @@ services: # caddy's Basic-Auth and re-prompts for the password on every dashboard; the # dashboards poll and do not need Live. GF_LIVE_MAX_CONNECTIONS: "0" + # SMTP for alert emails, reusing the shared relay credentials on its STARTTLS host:port + # (GRAFANA_SMTP_HOST — Grafana's client speaks STARTTLS, not the backend's implicit-TLS + # port), the SERVICE From, and the SERVICE recipient. Disabled unless GF_SMTP_ENABLED. + GF_SMTP_ENABLED: ${GF_SMTP_ENABLED:-false} + GF_SMTP_HOST: ${GRAFANA_SMTP_HOST:-} + GF_SMTP_USER: ${SMTP_RELAY_USER:-} + GF_SMTP_PASSWORD: ${SMTP_RELAY_PASS:-} + GF_SMTP_FROM_ADDRESS: ${SMTP_RELAY_SERVICE_FROM:-} + GF_SMTP_FROM_NAME: ${GRAFANA_SMTP_FROM_NAME:-Erudit Alerts} + GF_SMTP_STARTTLS_POLICY: MandatoryStartTLS + # The alert recipient(s), read by the provisioned contact point via $__env{SERVICE_EMAIL}. + SERVICE_EMAIL: ${SERVICE_EMAIL:-} volumes: - ${SCRABBLE_CONFIG_DIR:-.}/grafana/provisioning:/etc/grafana/provisioning:ro # Dashboards live under /etc/grafana (NOT /var/lib/grafana, which the @@ -547,6 +563,24 @@ services: memory: 64M networks: [internal] + # blackbox_exporter lets Prometheus alert on TLS certificate expiry (a Caddy ACME + # renewal failure) via probe_ssl_earliest_cert_expiry. It probes the edge caddy that + # terminates TLS (prod: the published scrabble-caddy; the test contour has no compose + # caddy, so the probe simply finds no target and the cert metric is absent — the rule is + # absent-safe). See prometheus.yml and grafana alerting rules. + blackbox_exporter: + container_name: scrabble-blackbox-exporter + image: prom/blackbox-exporter:v0.25.0 + restart: unless-stopped + logging: *default-logging + volumes: + - ${SCRABBLE_CONFIG_DIR:-.}/blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro + deploy: + resources: + limits: + memory: 64M + networks: [internal, edge] + networks: internal: name: scrabble-internal diff --git a/deploy/grafana/provisioning/alerting/contactpoints.yaml b/deploy/grafana/provisioning/alerting/contactpoints.yaml new file mode 100644 index 0000000..9d398a9 --- /dev/null +++ b/deploy/grafana/provisioning/alerting/contactpoints.yaml @@ -0,0 +1,13 @@ +# Grafana alerting contact point: the operator's alert mailbox, read from the +# SERVICE_EMAIL container env (see docker-compose.yml grafana), so it stays per-contour. +# SERVICE_EMAIL may hold several comma-separated addresses. +apiVersion: 1 +contactPoints: + - orgId: 1 + name: ops-email + receivers: + - uid: ops_email + type: email + settings: + addresses: $__env{SERVICE_EMAIL} + singleEmail: true diff --git a/deploy/grafana/provisioning/alerting/policies.yaml b/deploy/grafana/provisioning/alerting/policies.yaml new file mode 100644 index 0000000..6d2dc0e --- /dev/null +++ b/deploy/grafana/provisioning/alerting/policies.yaml @@ -0,0 +1,10 @@ +# Notification policy: every alert routes to the operator email, grouped so a burst is one +# message, with a 4-hour re-notify while still firing. +apiVersion: 1 +policies: + - orgId: 1 + receiver: ops-email + group_by: ['alertname'] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h diff --git a/deploy/grafana/provisioning/alerting/rules.yaml b/deploy/grafana/provisioning/alerting/rules.yaml new file mode 100644 index 0000000..a63c62b --- /dev/null +++ b/deploy/grafana/provisioning/alerting/rules.yaml @@ -0,0 +1,214 @@ +# Grafana provisioned alert rules for the Scrabble contour. Each rule is a Prometheus +# instant query (refId A) fed into a threshold expression (refId C). noDataState/execErrState +# are OK so an absent metric never raises a false alert — notably cert-expiry, whose blackbox +# probe has no target on the test contour (caddy is HTTP-only there). Metric names are the +# real ones Prometheus exposes (edge_request_* from the gateway via the collector, +# node_*/pg_*/probe_ssl_* from the exporters). All route to the ops-email contact point. +apiVersion: 1 +groups: + - orgId: 1 + name: scrabble-service + folder: Alerts + interval: 1m + rules: + - uid: svc_target_down + title: Scrape target down + condition: C + for: 3m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: { refId: A, expr: up, instant: true } + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: lt, params: [1] } + labels: { severity: critical } + annotations: { summary: 'A Prometheus scrape target is down (up < 1).' } + + - uid: edge_error_rate + title: Gateway internal-error rate high + condition: C + for: 10m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 600, to: 0 } + datasourceUid: prometheus + model: + refId: A + expr: sum by (service_name) (rate(edge_request_duration_count{result="internal"}[5m])) + instant: true + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: gt, params: [0.05] } + labels: { severity: warning } + annotations: { summary: 'Sustained internal (5xx-equivalent) errors at the edge.' } + + - uid: edge_latency_p99 + title: Gateway request latency p99 high + condition: C + for: 10m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 600, to: 0 } + datasourceUid: prometheus + model: + refId: A + expr: histogram_quantile(0.99, sum by (le, service_name) (rate(edge_request_duration_bucket[5m]))) + instant: true + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: gt, params: [1] } + labels: { severity: warning } + annotations: { summary: 'Edge request p99 latency above 1s.' } + + - uid: tls_cert_expiry + title: TLS certificate nearing expiry + condition: C + for: 15m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: + refId: A + expr: (probe_ssl_earliest_cert_expiry - time()) / 86400 + instant: true + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: lt, params: [20] } + labels: { severity: critical } + annotations: { summary: 'Edge TLS cert has under 20 days left — Caddy ACME renewal may have failed.' } + + - orgId: 1 + name: scrabble-host + folder: Alerts + interval: 1m + rules: + - uid: host_mem_low + title: Host memory low + condition: C + for: 5m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: + refId: A + expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes + instant: true + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: lt, params: [0.1] } + labels: { severity: critical } + annotations: { summary: 'Under 10% host memory available.' } + + - uid: host_disk_low + title: Host disk low + condition: C + for: 10m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: + refId: A + expr: min(node_filesystem_avail_bytes / node_filesystem_size_bytes) + instant: true + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: lt, params: [0.1] } + labels: { severity: critical } + annotations: { summary: 'A host filesystem is under 10% free.' } + + - uid: host_cpu_high + title: Host CPU saturated + condition: C + for: 10m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 600, to: 0 } + datasourceUid: prometheus + model: + refId: A + expr: 1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) + instant: true + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: gt, params: [0.9] } + labels: { severity: warning } + annotations: { summary: 'Host CPU above 90% for 10 minutes.' } + + - uid: pg_connections_high + title: Postgres connections high + condition: C + for: 5m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: + refId: A + expr: sum(pg_stat_activity_count) / max(pg_settings_max_connections) + instant: true + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: gt, params: [0.8] } + labels: { severity: warning } + annotations: { summary: 'Postgres using over 80% of max_connections.' } diff --git a/deploy/prometheus/prometheus.yml b/deploy/prometheus/prometheus.yml index 6dc3d43..0645427 100644 --- a/deploy/prometheus/prometheus.yml +++ b/deploy/prometheus/prometheus.yml @@ -23,3 +23,21 @@ scrape_configs: - job_name: node static_configs: - targets: ["node_exporter:9100"] + # TLS certificate expiry of the edge caddy (probe_ssl_earliest_cert_expiry), for the + # certificate-renewal-failure alert. Effective on prod, where caddy terminates TLS on + # :443; on the test contour caddy is HTTP-only, so the probe finds nothing and the metric + # is absent (the alert rule is absent-safe). The exporter probes the target passed as a + # scrape parameter and answers on its own :9115. + - job_name: blackbox_tls + metrics_path: /probe + params: + module: [tls_cert] + static_configs: + - targets: ["caddy:443"] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox_exporter:9115