Merge pull request 'feat(observability): Grafana infra alerts + admin email notifications (PR4)' (#165) from feature/email-relay-pr4 into development

.gitea/workflows/ci.yaml

@@ -340,6 +340,15 @@ jobs:
           SMTP_RELAY_PORT: ${{ vars.TEST_SMTP_RELAY_PORT }}
           SMTP_RELAY_TLS: ${{ vars.TEST_SMTP_RELAY_TLS }}
           SMTP_RELAY_FROM: ${{ vars.TEST_SMTP_RELAY_FROM }}
+          # Operator alerts: backend admin emails (new feedback / complaints) + Grafana
+          # infra alerts. Distinct senders + recipients; Grafana uses the relay's STARTTLS
+          # host:port. Empty leaves the alert worker off and Grafana SMTP disabled.
+          SMTP_RELAY_ADMIN_FROM: ${{ vars.TEST_SMTP_RELAY_ADMIN_FROM }}
+          ADMIN_EMAIL: ${{ vars.TEST_ADMIN_EMAIL }}
+          SMTP_RELAY_SERVICE_FROM: ${{ vars.TEST_SMTP_RELAY_SERVICE_FROM }}
+          SERVICE_EMAIL: ${{ vars.TEST_SERVICE_EMAIL }}
+          GRAFANA_SMTP_PORT: ${{ vars.TEST_GRAFANA_SMTP_PORT }}
+          GF_SMTP_ENABLED: ${{ vars.TEST_GF_SMTP_ENABLED }}
           # Canonical public origin for links in the email (this contour's URL);
           # required by the backend whenever SMTP_RELAY_HOST is set.
           PUBLIC_BASE_URL: ${{ vars.TEST_PUBLIC_BASE_URL }}
@@ -375,8 +384,20 @@ jobs:
           conf="$HOME/.scrabble-deploy"
           rm -rf "$conf"
           mkdir -p "$conf"
-          cp -r caddy otelcol prometheus tempo grafana "$conf"/
+          cp -r caddy otelcol prometheus tempo grafana blackbox "$conf"/
           export SCRABBLE_CONFIG_DIR="$conf"
+          # Grafana's SMTP from_address must be a BARE address (it rejects the "Name" <addr>
+          # form the backend go-mail accepts) and validates it even when SMTP is disabled — a
+          # bad value crash-loops Grafana. Split the display-format SERVICE From into a bare
+          # address + name for Grafana; the backend keeps the full form.
+          svc_from="${SMTP_RELAY_SERVICE_FROM:-}"
+          case "$svc_from" in
+            *"<"*">"*)
+              export GRAFANA_SMTP_FROM_ADDRESS="$(printf '%s' "$svc_from" | sed -E 's/.*<([^>]+)>.*/\1/')"
+              export GRAFANA_SMTP_FROM_NAME="$(printf '%s' "$svc_from" | sed -E 's/[[:space:]]*<[^>]*>.*$//; s/^"//; s/"$//')" ;;
+            *)
+              export GRAFANA_SMTP_FROM_ADDRESS="$svc_from" ;;
+          esac
           # Bot-link mTLS material for the test contour: a private CA + gateway/bot
           # leaves (CN=gateway, the service name the bot dials). Prod supplies these
           # from PROD_ secrets instead. Regenerated each deploy; both ends redeploy

.gitea/workflows/prod-deploy.yaml

@@ -93,6 +93,14 @@ jobs:
       SMTP_RELAY_PORT: ${{ vars.PROD_SMTP_RELAY_PORT }}
       SMTP_RELAY_TLS: ${{ vars.PROD_SMTP_RELAY_TLS }}
       SMTP_RELAY_FROM: ${{ vars.PROD_SMTP_RELAY_FROM }}
+      # Operator alerts: backend admin emails + Grafana infra alerts (distinct senders +
+      # recipients; Grafana uses the relay's STARTTLS host:port).
+      SMTP_RELAY_ADMIN_FROM: ${{ vars.PROD_SMTP_RELAY_ADMIN_FROM }}
+      ADMIN_EMAIL: ${{ vars.PROD_ADMIN_EMAIL }}
+      SMTP_RELAY_SERVICE_FROM: ${{ vars.PROD_SMTP_RELAY_SERVICE_FROM }}
+      SERVICE_EMAIL: ${{ vars.PROD_SERVICE_EMAIL }}
+      GRAFANA_SMTP_PORT: ${{ vars.PROD_GRAFANA_SMTP_PORT }}
+      GF_SMTP_ENABLED: ${{ vars.PROD_GF_SMTP_ENABLED }}
       PUBLIC_BASE_URL: ${{ vars.PROD_PUBLIC_BASE_URL }}
       PROD_BOTLINK_CA: ${{ secrets.PROD_BOTLINK_CA }}
       PROD_BOTLINK_GATEWAY_CERT: ${{ secrets.PROD_BOTLINK_GATEWAY_CERT }}
@@ -132,6 +140,16 @@ jobs:
         run: |
           umask 077
           mkdir -p stage/certs-main
+          # Grafana needs a BARE from-address (it rejects the "Name" <addr> form the backend
+          # accepts, and validates it even when disabled); split the display-format here.
+          svc_from="${SMTP_RELAY_SERVICE_FROM:-}"
+          GRAFANA_SMTP_FROM_NAME=''
+          case "$svc_from" in
+            *"<"*">"*)
+              GRAFANA_SMTP_FROM_ADDRESS="$(printf '%s' "$svc_from" | sed -E 's/.*<([^>]+)>.*/\1/')"
+              GRAFANA_SMTP_FROM_NAME="$(printf '%s' "$svc_from" | sed -E 's/[[:space:]]*<[^>]*>.*$//; s/^"//; s/"$//')" ;;
+            *) GRAFANA_SMTP_FROM_ADDRESS="$svc_from" ;;
+          esac
           cat > stage/env.sh <<EOF
           export REGISTRY='$REGISTRY'
           export SCRABBLE_CONFIG_DIR='/opt/scrabble'
@@ -156,6 +174,14 @@ jobs:
           export SMTP_RELAY_USER='$SMTP_RELAY_USER'
           export SMTP_RELAY_PASS='$SMTP_RELAY_PASS'
           export SMTP_RELAY_FROM='$SMTP_RELAY_FROM'
+          export SMTP_RELAY_ADMIN_FROM='$SMTP_RELAY_ADMIN_FROM'
+          export ADMIN_EMAIL='$ADMIN_EMAIL'
+          export SMTP_RELAY_SERVICE_FROM='$SMTP_RELAY_SERVICE_FROM'
+          export GRAFANA_SMTP_FROM_ADDRESS='$GRAFANA_SMTP_FROM_ADDRESS'
+          export GRAFANA_SMTP_FROM_NAME='$GRAFANA_SMTP_FROM_NAME'
+          export SERVICE_EMAIL='$SERVICE_EMAIL'
+          export GRAFANA_SMTP_PORT='$GRAFANA_SMTP_PORT'
+          export GF_SMTP_ENABLED='$GF_SMTP_ENABLED'
           export PUBLIC_BASE_URL='$PUBLIC_BASE_URL'
           export GATEWAY_ABUSE_BAN_ENABLED='true'
           EOF
@@ -169,7 +195,7 @@ jobs:
           ssh_main 'mkdir -p /opt/scrabble/compose'
           tar -C deploy -czf - docker-compose.yml docker-compose.prod.yml prod-deploy.sh \
             | ssh_main 'tar -C /opt/scrabble/compose -xzf -'
-          tar -C deploy -czf - caddy otelcol prometheus tempo grafana \
+          tar -C deploy -czf - caddy otelcol prometheus tempo grafana blackbox \
             | ssh_main 'tar -C /opt/scrabble -xzf -'
           tar -C stage -czf - certs-main \
             | ssh_main 'rm -rf /opt/scrabble/certs && mkdir -p /opt/scrabble/certs && tar -C /opt/scrabble/certs --strip-components=1 -xzf -'

backend/cmd/backend/main.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"log"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 
@@ -20,6 +21,7 @@ import (
 
 	"scrabble/backend/internal/account"
 	"scrabble/backend/internal/accountmerge"
+	"scrabble/backend/internal/adminalert"
 	"scrabble/backend/internal/ads"
 	"scrabble/backend/internal/banview"
 	"scrabble/backend/internal/config"
@@ -44,6 +46,10 @@ import (
 // telemetryShutdownTimeout bounds the OpenTelemetry flush during process exit.
 const telemetryShutdownTimeout = 5 * time.Second
 
+// adminAlertInterval is how often the operator-alert worker checks for new feedback /
+// complaints; a burst within one interval coalesces into a single digest email.
+const adminAlertInterval = 5 * time.Minute
+
 func main() {
 	cfg, err := config.Load()
 	if err != nil {
@@ -207,6 +213,18 @@ func run(ctx context.Context, cfg config.Config, logger *zap.Logger) error {
 	feedbackSvc := feedback.NewService(feedback.NewStore(db), accounts)
 	feedbackSvc.SetNotifier(hub)
 
+	// Operator alert emails on new feedback / word complaints, coalesced into one digest
+	// per interval. Inert unless a distinct admin sender and recipient are configured.
+	if cfg.SMTP.AdminFrom != "" && cfg.SMTP.AdminTo != "" {
+		consoleURL := ""
+		if cfg.PublicBaseURL != "" {
+			consoleURL = strings.TrimRight(cfg.PublicBaseURL, "/") + "/_gm"
+		}
+		alerts := adminalert.New(mailer, feedbackSvc, games, cfg.SMTP.AdminFrom, cfg.SMTP.AdminTo, consoleURL, logger)
+		go alerts.Run(ctx, adminAlertInterval)
+		logger.Info("admin alert worker started", zap.Duration("interval", adminAlertInterval))
+	}
+
 	// Robot opponent: provision its durable account pool (a hard startup
 	// dependency, like the dictionaries) and start its move driver. The matchmaker
 	// substitutes a pooled robot for a missing human after the wait window.

backend/internal/account/mailer.go

@@ -15,7 +15,11 @@ import (
 // required plain-text body and doubles as the multipart/alternative fallback;
 // HTML, when non-empty, is the preferred body a capable client renders instead.
 type Message struct {
-	To      string
+	// To is the recipient address, or several comma-separated (all get the one message).
+	To string
+	// From, when non-empty, overrides the configured sender for this message — the admin
+	// alert path uses a distinct From from the user-facing confirm-code sender.
+	From    string
 	Subject string
 	Text    string
 	HTML    string
@@ -29,6 +33,18 @@ type Mailer interface {
 	Send(ctx context.Context, msg Message) error
 }
 
+// splitAddrs splits a comma-separated recipient list into trimmed, non-empty addresses.
+func splitAddrs(list string) []string {
+	parts := strings.Split(list, ",")
+	out := make([]string, 0, len(parts))
+	for _, p := range parts {
+		if a := strings.TrimSpace(p); a != "" {
+			out = append(out, a)
+		}
+	}
+	return out
+}
+
 // SMTPConfig configures the SMTP relay. An empty Host selects the LogMailer
 // instead, so a deployment without a relay still runs (the code lands in the log).
 // TLS is always used and no client certificate is required — only the server
@@ -44,6 +60,11 @@ type SMTPConfig struct {
 	// port (implicit TLS on 465, STARTTLS otherwise); set it explicitly for a relay on
 	// a non-standard port (e.g. Selectel's 1127 = SSL, 1126 = STARTTLS).
 	TLS string
+	// AdminFrom / AdminTo drive the operator alert emails (new feedback / word complaints),
+	// distinct from the user-facing confirm-code sender. AdminTo may be several
+	// comma-separated addresses. Both empty disables the alert worker.
+	AdminFrom string
+	AdminTo   string
 }
 
 const (
@@ -110,10 +131,16 @@ func (m SMTPMailer) Send(ctx context.Context, msg Message) error {
 		return fmt.Errorf("account: build mail client: %w", err)
 	}
 	out := mail.NewMsg()
-	if err := out.From(m.cfg.From); err != nil {
+	from := m.cfg.From
-		return fmt.Errorf("account: set From %q: %w", m.cfg.From, err)
+	if msg.From != "" {
+		from = msg.From
 	}
-	if err := out.To(msg.To); err != nil {
+	if err := out.From(from); err != nil {
+		return fmt.Errorf("account: set From %q: %w", from, err)
+	}
+	// To may carry several comma-separated recipients; go-mail wants them as separate
+	// arguments (a single joined string parses as one malformed address).
+	if err := out.To(splitAddrs(msg.To)...); err != nil {
 		return fmt.Errorf("account: set To %q: %w", msg.To, err)
 	}
 	out.Subject(msg.Subject)

backend/internal/account/mailer_test.go

@@ -1,6 +1,28 @@
 package account
 
-import "testing"
+import (
+	"slices"
+	"testing"
+)
+
+// TestSplitAddrs covers the comma-separated recipient parsing used for the admin alert
+// To (several operator mailboxes in one message), including trimming and empty entries.
+func TestSplitAddrs(t *testing.T) {
+	cases := []struct {
+		in   string
+		want []string
+	}{
+		{"a@x.ru", []string{"a@x.ru"}},
+		{"a@x.ru, b@y.ru", []string{"a@x.ru", "b@y.ru"}},
+		{" a@x.ru ,, b@y.ru ,", []string{"a@x.ru", "b@y.ru"}},
+		{"", nil},
+	}
+	for _, c := range cases {
+		if got := splitAddrs(c.in); !slices.Equal(got, c.want) {
+			t.Errorf("splitAddrs(%q) = %v, want %v", c.in, got, c.want)
+		}
+	}
+}
 
 // TestSMTPTLSMode covers the explicit TLS mode and the port-based fallback, including
 // the non-standard Selectel ports (1127 = SSL, 1126 = STARTTLS) that the 465 heuristic

backend/internal/adminalert/notify.go

@@ -0,0 +1,116 @@
+// Package adminalert emails the operator when new player feedback or word complaints
+// arrive, coalescing a burst into a single digest per interval so a flood is one email,
+// not N. It is inert unless an admin sender and recipient are configured. The sender is
+// distinct from the user-facing confirm-code From, and the recipient may be several
+// comma-separated addresses (the mailer splits them).
+package adminalert
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"go.uber.org/zap"
+
+	"scrabble/backend/internal/account"
+)
+
+// FeedbackCounter counts feedback created since a time (satisfied by feedback.Service).
+type FeedbackCounter interface {
+	CountSince(ctx context.Context, since time.Time) (int, error)
+}
+
+// ComplaintCounter counts word complaints filed since a time (satisfied by game.Service).
+type ComplaintCounter interface {
+	CountComplaintsSince(ctx context.Context, since time.Time) (int, error)
+}
+
+// Notifier polls for new feedback and complaints and emails the operator a digest.
+type Notifier struct {
+	mailer     account.Mailer
+	feedback   FeedbackCounter
+	complaints ComplaintCounter
+	from       string
+	to         string
+	consoleURL string
+	clock      func() time.Time
+	log        *zap.Logger
+	last       time.Time
+}
+
+// New constructs a Notifier. from and to are the alert sender and recipient(s); consoleURL,
+// when non-empty, is the admin-console link included in the email. log may be nil. The
+// watermark starts at "now", so only items arriving after start-up are reported.
+func New(mailer account.Mailer, fb FeedbackCounter, cp ComplaintCounter, from, to, consoleURL string, log *zap.Logger) *Notifier {
+	if log == nil {
+		log = zap.NewNop()
+	}
+	return &Notifier{
+		mailer: mailer, feedback: fb, complaints: cp, from: from, to: to, consoleURL: consoleURL,
+		clock: func() time.Time { return time.Now().UTC() }, log: log, last: time.Now().UTC(),
+	}
+}
+
+// Run polls on each tick until ctx is cancelled.
+func (n *Notifier) Run(ctx context.Context, interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			n.tick(ctx)
+		}
+	}
+}
+
+// tick counts what arrived since the last watermark and, if anything did, emails one
+// digest. The watermark only advances after a successful send (or a quiet tick), so a
+// transient send failure is retried on the next tick — the counts simply grow.
+func (n *Notifier) tick(ctx context.Context) {
+	now := n.clock()
+	fb, err := n.feedback.CountSince(ctx, n.last)
+	if err != nil {
+		n.log.Warn("admin alert: count feedback failed", zap.Error(err))
+		return
+	}
+	cp, err := n.complaints.CountComplaintsSince(ctx, n.last)
+	if err != nil {
+		n.log.Warn("admin alert: count complaints failed", zap.Error(err))
+		return
+	}
+	if fb == 0 && cp == 0 {
+		n.last = now
+		return
+	}
+	if err := n.mailer.Send(ctx, n.digest(fb, cp)); err != nil {
+		n.log.Warn("admin alert: send failed", zap.Error(err))
+		return
+	}
+	n.log.Info("admin alert sent", zap.Int("feedback", fb), zap.Int("complaints", cp))
+	n.last = now
+}
+
+// digest builds the operator alert email for fb new feedback and cp new complaints.
+func (n *Notifier) digest(fb, cp int) account.Message {
+	var parts []string
+	if fb > 0 {
+		parts = append(parts, fmt.Sprintf("%d new feedback message(s)", fb))
+	}
+	if cp > 0 {
+		parts = append(parts, fmt.Sprintf("%d new word complaint(s)", cp))
+	}
+	summary := strings.Join(parts, ", ")
+	text := summary + "."
+	if n.consoleURL != "" {
+		text += "\n\nOpen the admin console: " + n.consoleURL
+	}
+	return account.Message{
+		From:    n.from,
+		To:      n.to,
+		Subject: "Erudit — " + summary,
+		Text:    text,
+	}
+}

backend/internal/adminalert/notify_test.go

@@ -0,0 +1,55 @@
+package adminalert
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"scrabble/backend/internal/account"
+)
+
+// The fakes ignore the watermark and return a fixed count, which is all the digest logic
+// needs.
+type fbCounter struct{ n int }
+
+func (f fbCounter) CountSince(context.Context, time.Time) (int, error) { return f.n, nil }
+
+type cpCounter struct{ n int }
+
+func (c cpCounter) CountComplaintsSince(context.Context, time.Time) (int, error) { return c.n, nil }
+
+type recordingMailer struct{ sent []account.Message }
+
+func (m *recordingMailer) Send(_ context.Context, msg account.Message) error {
+	m.sent = append(m.sent, msg)
+	return nil
+}
+
+func TestNotifierSkipsWhenNothingNew(t *testing.T) {
+	mailer := &recordingMailer{}
+	n := New(mailer, fbCounter{0}, cpCounter{0}, "alerts@erudit-game.ru", "op@x.ru", "", nil)
+	n.tick(context.Background())
+	if len(mailer.sent) != 0 {
+		t.Fatalf("sent %d emails, want 0 when nothing is new", len(mailer.sent))
+	}
+}
+
+func TestNotifierDigestsNewItems(t *testing.T) {
+	mailer := &recordingMailer{}
+	n := New(mailer, fbCounter{2}, cpCounter{1}, "alerts@erudit-game.ru", "op@x.ru, two@x.ru", "https://erudit-game.ru/_gm", nil)
+	n.tick(context.Background())
+	if len(mailer.sent) != 1 {
+		t.Fatalf("sent %d emails, want 1 digest", len(mailer.sent))
+	}
+	msg := mailer.sent[0]
+	if msg.From != "alerts@erudit-game.ru" || msg.To != "op@x.ru, two@x.ru" {
+		t.Errorf("digest addressing = From %q To %q", msg.From, msg.To)
+	}
+	if !strings.Contains(msg.Subject, "2 new feedback") || !strings.Contains(msg.Subject, "1 new word complaint") {
+		t.Errorf("digest subject = %q, want the feedback + complaint counts", msg.Subject)
+	}
+	if !strings.Contains(msg.Text, "/_gm") {
+		t.Errorf("digest body = %q, want the console link", msg.Text)
+	}
+}

backend/internal/config/config.go

@@ -143,12 +143,14 @@ func Load() (Config, error) {
 	}
 
 	smtp := account.SMTPConfig{
-		Host:     os.Getenv("BACKEND_SMTP_HOST"),
+		Host:      os.Getenv("BACKEND_SMTP_HOST"),
-		Port:     envOr("BACKEND_SMTP_PORT", "587"),
+		Port:      envOr("BACKEND_SMTP_PORT", "587"),
-		Username: os.Getenv("BACKEND_SMTP_USERNAME"),
+		Username:  os.Getenv("BACKEND_SMTP_USERNAME"),
-		Password: os.Getenv("BACKEND_SMTP_PASSWORD"),
+		Password:  os.Getenv("BACKEND_SMTP_PASSWORD"),
-		From:     envOr("BACKEND_SMTP_FROM", "no-reply@localhost"),
+		From:      envOr("BACKEND_SMTP_FROM", "no-reply@localhost"),
-		TLS:      os.Getenv("BACKEND_SMTP_TLS"),
+		TLS:       os.Getenv("BACKEND_SMTP_TLS"),
+		AdminFrom: os.Getenv("BACKEND_SMTP_ADMIN_FROM"),
+		AdminTo:   os.Getenv("BACKEND_ADMIN_EMAIL"),
 	}
 
 	c := Config{

backend/internal/feedback/service.go

@@ -195,6 +195,11 @@ func (svc *Service) CountUnread(ctx context.Context) (int, error) {
 	return svc.store.CountUnread(ctx)
 }
 
+// CountSince counts feedback created after since, for the operator alert worker.
+func (svc *Service) CountSince(ctx context.Context, since time.Time) (int, error) {
+	return svc.store.CountSince(ctx, since)
+}
+
 // Attachment returns a message's file name and bytes, reporting false when absent.
 func (svc *Service) Attachment(ctx context.Context, id uuid.UUID) (string, []byte, bool, error) {
 	return svc.store.Attachment(ctx, id)

backend/internal/feedback/store.go

@@ -386,3 +386,15 @@ func (s *Store) CountUnread(ctx context.Context) (int, error) {
 	}
 	return n, nil
 }
+
+// CountSince counts feedback messages created strictly after since — the operator alert
+// worker's "new since the last check" signal.
+func (s *Store) CountSince(ctx context.Context, since time.Time) (int, error) {
+	var n int
+	if err := s.db.QueryRowContext(ctx,
+		`SELECT COUNT(*) FROM backend.feedback_messages WHERE created_at > $1`, since,
+	).Scan(&n); err != nil {
+		return 0, fmt.Errorf("feedback: count since: %w", err)
+	}
+	return n, nil
+}

backend/internal/game/service.go

@@ -1008,6 +1008,12 @@ func (svc *Service) CountComplaints(ctx context.Context, status string) (int, er
 	return svc.store.CountComplaints(ctx, status)
 }
 
+// CountComplaintsSince counts word complaints filed after since, for the operator alert
+// worker.
+func (svc *Service) CountComplaintsSince(ctx context.Context, since time.Time) (int, error) {
+	return svc.store.CountComplaintsSince(ctx, since)
+}
+
 // ResolveComplaint closes a complaint with an operator disposition (reject /
 // accept_add / accept_remove) and an optional note. An accepted complaint then
 // appears in DictionaryChanges until a rebuilt dictionary is loaded and the

backend/internal/game/store.go

@@ -1040,6 +1040,19 @@ func (s *Store) CountComplaints(ctx context.Context, status string) (int, error)
 	return int(dest.Count), nil
 }
 
+// CountComplaintsSince counts word complaints filed strictly after since — the operator
+// alert worker's "new since the last check" signal.
+func (s *Store) CountComplaintsSince(ctx context.Context, since time.Time) (int, error) {
+	stmt := postgres.SELECT(postgres.COUNT(table.Complaints.ComplaintID).AS("count")).
+		FROM(table.Complaints).
+		WHERE(table.Complaints.CreatedAt.GT(postgres.TimestampzT(since)))
+	var dest struct{ Count int64 }
+	if err := stmt.QueryContext(ctx, s.db, &dest); err != nil {
+		return 0, fmt.Errorf("game: count complaints since: %w", err)
+	}
+	return int(dest.Count), nil
+}
+
 // ActiveGames returns the turn clocks of every in-progress game; the sweeper
 // filters them against the per-move deadline and the player's away window.
 func (s *Store) ActiveGames(ctx context.Context) ([]activeGame, error) {

deploy/.env.example

@@ -45,6 +45,17 @@ SMTP_RELAY_PASS=                       # secret
 SMTP_RELAY_FROM=no-reply@erudit-game.ru
 PUBLIC_BASE_URL=                       # required when SMTP_RELAY_HOST is set (e.g. https://erudit-game.ru)
 
+# Operator alerts (email). The backend emails the admin on new feedback / word complaints
+# (coalesced), and Grafana emails infra alerts. Distinct senders; recipients may be several
+# comma-separated addresses. Grafana reuses SMTP_RELAY_HOST/USER/PASS but dials the STARTTLS
+# port (it can't do the backend's implicit TLS), GRAFANA_SMTP_PORT. All empty = off.
+SMTP_RELAY_ADMIN_FROM=                 # backend admin-alert From (e.g. alerts@erudit-game.ru)
+ADMIN_EMAIL=                           # backend admin-alert recipient(s), comma-separated
+SMTP_RELAY_SERVICE_FROM=               # Grafana alert From; the deploy derives a bare address for Grafana (it rejects "Name" <addr>)
+SERVICE_EMAIL=                         # Grafana alert recipient(s), comma-separated
+GRAFANA_SMTP_PORT=                     # Grafana STARTTLS port on SMTP_RELAY_HOST (Selectel: 1126)
+GF_SMTP_ENABLED=false                  # set true to enable Grafana alert emails
+
 # --- Edge / caddy -----------------------------------------------------------
 # Test: ":80" (the host caddy terminates TLS and forwards to scrabble:80 on the
 # external `edge` network). Prod: a domain so caddy does its own ACME.

deploy/README.md

@@ -103,6 +103,12 @@ without it Docker's resolver handles `otelcol`, `gateway` and `api.telegram.org`
 | `SMTP_RELAY_PASS` | secret | _(empty)_ | Relay SMTP AUTH password. |
 | `SMTP_RELAY_FROM` | variable | `no-reply@localhost` | Sender address. **Must use the prod domain** (`no-reply@erudit-game.ru`) — Selectel only accepts the verified sender domain — so it is the same on every contour. |
 | `PUBLIC_BASE_URL` | variable | _(empty)_ | Canonical public https origin for links in the email (the contour's own URL, e.g. `https://erudit-game.ru`). **Required by the backend whenever `SMTP_RELAY_HOST` is set** — it is never derived from a request Host header (anti-injection). |
+| `SMTP_RELAY_ADMIN_FROM` | variable | _(empty)_ | Backend operator-alert sender (new feedback / word complaints), distinct from the confirm-code From. Empty (with `ADMIN_EMAIL`) disables the alert worker. |
+| `ADMIN_EMAIL` | variable | _(empty)_ | Backend operator-alert recipient(s); several comma-separated addresses allowed. |
+| `SMTP_RELAY_SERVICE_FROM` | variable | _(empty)_ | Grafana infra-alert sender address. |
+| `SERVICE_EMAIL` | variable | _(empty)_ | Grafana infra-alert recipient(s); comma-separated allowed (read by the provisioned contact point via `$__env{SERVICE_EMAIL}`). |
+| `GRAFANA_SMTP_PORT` | variable | _(empty)_ | STARTTLS port Grafana dials on `SMTP_RELAY_HOST` (its client can't do the backend's implicit TLS), e.g. Selectel's `1126`. Reuses `SMTP_RELAY_HOST`/`USER`/`PASS`. |
+| `GF_SMTP_ENABLED` | variable | `false` | Set `true` to enable Grafana alert emails. |
 
 The six `VITE_*` are **build-args** baked into the gateway and landing images at
 build time (both targets share one UI build stage — keep the args identical so it is
@@ -211,7 +217,9 @@ SMTP_RELAY_USER, SMTP_RELAY_PASS}`; variables:
 GRAFANA_ROOT_URL, LOG_LEVEL, DICT_VERSION, TELEGRAM_MINIAPP_URL, TELEGRAM_GAME_CHANNEL_ID,
 TELEGRAM_CHAT_ID, TELEGRAM_BOT_USERNAME, VITE_TELEGRAM_BOT_ID, VITE_TELEGRAM_LINK,
 VITE_TELEGRAM_GAME_CHANNEL_NAME, VITE_VK_APP_LINK,
-SMTP_RELAY_HOST, SMTP_RELAY_PORT, SMTP_RELAY_TLS, SMTP_RELAY_FROM, PUBLIC_BASE_URL}`.
+SMTP_RELAY_HOST, SMTP_RELAY_PORT, SMTP_RELAY_TLS, SMTP_RELAY_FROM, PUBLIC_BASE_URL,
+SMTP_RELAY_ADMIN_FROM, ADMIN_EMAIL, SMTP_RELAY_SERVICE_FROM, SERVICE_EMAIL,
+GRAFANA_SMTP_PORT, GF_SMTP_ENABLED}`.
 
 ## Host-side setup (outside this repo)
 

deploy/blackbox/blackbox.yml

@@ -0,0 +1,14 @@
+# blackbox_exporter probe modules. tls_cert opens a verified TLS connection to the edge
+# caddy so Prometheus can read probe_ssl_earliest_cert_expiry — the signal behind the
+# certificate-renewal-failure alert. The SNI is the production public host, whose cert the
+# edge caddy serves once it does its own ACME; on the test contour caddy is HTTP-only
+# (behind the host caddy), so the probe finds nothing on :443 and the cert metric is absent.
+modules:
+  tls_cert:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      tls: true
+      tls_config:
+        server_name: erudit-game.ru
+        insecure_skip_verify: false

deploy/docker-compose.yml

@@ -151,6 +151,10 @@ services:
       BACKEND_SMTP_PASSWORD: ${SMTP_RELAY_PASS:-}
       BACKEND_SMTP_FROM: ${SMTP_RELAY_FROM:-no-reply@localhost}
       BACKEND_PUBLIC_BASE_URL: ${PUBLIC_BASE_URL:-}
+      # Operator alert emails (new feedback / word complaints): a distinct sender and the
+      # recipient(s) (comma-separated allowed). Both empty disables the alert worker.
+      BACKEND_SMTP_ADMIN_FROM: ${SMTP_RELAY_ADMIN_FROM:-}
+      BACKEND_ADMIN_EMAIL: ${ADMIN_EMAIL:-}
     # The dictionary lives on a named volume seeded from the image on first boot
     # (the image's /opt/dawg is owned by the nonroot UID, which the fresh volume
     # inherits). The admin console writes new version subdirectories here, and the
@@ -497,6 +501,21 @@ services:
       # caddy's Basic-Auth and re-prompts for the password on every dashboard; the
       # dashboards poll and do not need Live.
       GF_LIVE_MAX_CONNECTIONS: "0"
+      # SMTP for alert emails, reusing the shared relay host + credentials + the SERVICE
+      # From/recipient. Grafana's client speaks STARTTLS (not the backend's implicit-TLS
+      # port), so it dials the relay host on its STARTTLS port (GRAFANA_SMTP_PORT).
+      # Disabled unless GF_SMTP_ENABLED.
+      GF_SMTP_ENABLED: ${GF_SMTP_ENABLED:-false}
+      GF_SMTP_HOST: ${SMTP_RELAY_HOST:-}:${GRAFANA_SMTP_PORT:-}
+      GF_SMTP_USER: ${SMTP_RELAY_USER:-}
+      GF_SMTP_PASSWORD: ${SMTP_RELAY_PASS:-}
+      # A BARE address (Grafana rejects the "Name" <addr> form); the deploy derives it from
+      # SMTP_RELAY_SERVICE_FROM, splitting off the display name into GRAFANA_SMTP_FROM_NAME.
+      GF_SMTP_FROM_ADDRESS: ${GRAFANA_SMTP_FROM_ADDRESS:-}
+      GF_SMTP_FROM_NAME: ${GRAFANA_SMTP_FROM_NAME:-Erudit Alerts}
+      GF_SMTP_STARTTLS_POLICY: MandatoryStartTLS
+      # The alert recipient(s), read by the provisioned contact point via $__env{SERVICE_EMAIL}.
+      SERVICE_EMAIL: ${SERVICE_EMAIL:-}
     volumes:
       - ${SCRABBLE_CONFIG_DIR:-.}/grafana/provisioning:/etc/grafana/provisioning:ro
       # Dashboards live under /etc/grafana (NOT /var/lib/grafana, which the
@@ -547,6 +566,24 @@ services:
           memory: 64M
     networks: [internal]
 
+  # blackbox_exporter lets Prometheus alert on TLS certificate expiry (a Caddy ACME
+  # renewal failure) via probe_ssl_earliest_cert_expiry. It probes the edge caddy that
+  # terminates TLS (prod: the published scrabble-caddy; the test contour has no compose
+  # caddy, so the probe simply finds no target and the cert metric is absent — the rule is
+  # absent-safe). See prometheus.yml and grafana alerting rules.
+  blackbox_exporter:
+    container_name: scrabble-blackbox-exporter
+    image: prom/blackbox-exporter:v0.25.0
+    restart: unless-stopped
+    logging: *default-logging
+    volumes:
+      - ${SCRABBLE_CONFIG_DIR:-.}/blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
+    deploy:
+      resources:
+        limits:
+          memory: 64M
+    networks: [internal, edge]
+
 networks:
   internal:
     name: scrabble-internal

deploy/grafana/provisioning/alerting/contactpoints.yaml

@@ -0,0 +1,13 @@
+# Grafana alerting contact point: the operator's alert mailbox, read from the
+# SERVICE_EMAIL container env (see docker-compose.yml grafana), so it stays per-contour.
+# SERVICE_EMAIL may hold several comma-separated addresses.
+apiVersion: 1
+contactPoints:
+  - orgId: 1
+    name: ops-email
+    receivers:
+      - uid: ops_email
+        type: email
+        settings:
+          addresses: $__env{SERVICE_EMAIL}
+          singleEmail: true

deploy/grafana/provisioning/alerting/policies.yaml

@@ -0,0 +1,10 @@
+# Notification policy: every alert routes to the operator email, grouped so a burst is one
+# message, with a 4-hour re-notify while still firing.
+apiVersion: 1
+policies:
+  - orgId: 1
+    receiver: ops-email
+    group_by: ['alertname']
+    group_wait: 30s
+    group_interval: 5m
+    repeat_interval: 4h

deploy/grafana/provisioning/alerting/rules.yaml

@@ -0,0 +1,214 @@
+# Grafana provisioned alert rules for the Scrabble contour. Each rule is a Prometheus
+# instant query (refId A) fed into a threshold expression (refId C). noDataState/execErrState
+# are OK so an absent metric never raises a false alert — notably cert-expiry, whose blackbox
+# probe has no target on the test contour (caddy is HTTP-only there). Metric names are the
+# real ones Prometheus exposes (edge_request_* from the gateway via the collector,
+# node_*/pg_*/probe_ssl_* from the exporters). All route to the ops-email contact point.
+apiVersion: 1
+groups:
+  - orgId: 1
+    name: scrabble-service
+    folder: Alerts
+    interval: 1m
+    rules:
+      - uid: svc_target_down
+        title: Scrape target down
+        condition: C
+        for: 3m
+        noDataState: OK
+        execErrState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: up, instant: true }
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: lt, params: [1] }
+        labels: { severity: critical }
+        annotations: { summary: 'A Prometheus scrape target is down (up < 1).' }
+
+      - uid: edge_error_rate
+        title: Gateway internal-error rate high
+        condition: C
+        for: 10m
+        noDataState: OK
+        execErrState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 600, to: 0 }
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: sum by (service_name) (rate(edge_request_duration_count{result="internal"}[5m]))
+              instant: true
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: gt, params: [0.05] }
+        labels: { severity: warning }
+        annotations: { summary: 'Sustained internal (5xx-equivalent) errors at the edge.' }
+
+      - uid: edge_latency_p99
+        title: Gateway request latency p99 high
+        condition: C
+        for: 10m
+        noDataState: OK
+        execErrState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 600, to: 0 }
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: histogram_quantile(0.99, sum by (le, service_name) (rate(edge_request_duration_bucket[5m])))
+              instant: true
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: gt, params: [1] }
+        labels: { severity: warning }
+        annotations: { summary: 'Edge request p99 latency above 1s.' }
+
+      - uid: tls_cert_expiry
+        title: TLS certificate nearing expiry
+        condition: C
+        for: 15m
+        noDataState: OK
+        execErrState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: (probe_ssl_earliest_cert_expiry - time()) / 86400
+              instant: true
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: lt, params: [20] }
+        labels: { severity: critical }
+        annotations: { summary: 'Edge TLS cert has under 20 days left — Caddy ACME renewal may have failed.' }
+
+  - orgId: 1
+    name: scrabble-host
+    folder: Alerts
+    interval: 1m
+    rules:
+      - uid: host_mem_low
+        title: Host memory low
+        condition: C
+        for: 5m
+        noDataState: OK
+        execErrState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes
+              instant: true
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: lt, params: [0.1] }
+        labels: { severity: critical }
+        annotations: { summary: 'Under 10% host memory available.' }
+
+      - uid: host_disk_low
+        title: Host disk low
+        condition: C
+        for: 10m
+        noDataState: OK
+        execErrState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: min(node_filesystem_avail_bytes / node_filesystem_size_bytes)
+              instant: true
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: lt, params: [0.1] }
+        labels: { severity: critical }
+        annotations: { summary: 'A host filesystem is under 10% free.' }
+
+      - uid: host_cpu_high
+        title: Host CPU saturated
+        condition: C
+        for: 10m
+        noDataState: OK
+        execErrState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 600, to: 0 }
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: 1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m]))
+              instant: true
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: gt, params: [0.9] }
+        labels: { severity: warning }
+        annotations: { summary: 'Host CPU above 90% for 10 minutes.' }
+
+      - uid: pg_connections_high
+        title: Postgres connections high
+        condition: C
+        for: 5m
+        noDataState: OK
+        execErrState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: sum(pg_stat_activity_count) / max(pg_settings_max_connections)
+              instant: true
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: gt, params: [0.8] }
+        labels: { severity: warning }
+        annotations: { summary: 'Postgres using over 80% of max_connections.' }

deploy/prometheus/prometheus.yml

@@ -23,3 +23,21 @@ scrape_configs:
   - job_name: node
     static_configs:
       - targets: ["node_exporter:9100"]
+  # TLS certificate expiry of the edge caddy (probe_ssl_earliest_cert_expiry), for the
+  # certificate-renewal-failure alert. Effective on prod, where caddy terminates TLS on
+  # :443; on the test contour caddy is HTTP-only, so the probe finds nothing and the metric
+  # is absent (the alert rule is absent-safe). The exporter probes the target passed as a
+  # scrape parameter and answers on its own :9115.
+  - job_name: blackbox_tls
+    metrics_path: /probe
+    params:
+      module: [tls_cert]
+    static_configs:
+      - targets: ["caddy:443"]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox_exporter:9115

docs/ARCHITECTURE.md

@@ -1003,6 +1003,19 @@ browser), so an open in-app session reflects it at once.
   both are surfaced on the **Scrabble — Resources** Grafana dashboard, which captures the
   stress-run resource profile. (`docker_stats` replaced cAdvisor, which on the contour
   host resolved only the root cgroup — a separate-XFS `/var/lib/docker`.)
+- **Alerting.** Grafana emails infra alerts through the shared relay (its own SMTP on the
+  STARTTLS port) to `SERVICE_EMAIL`, from provisioned rules
+  (`deploy/grafana/provisioning/alerting/`): a scrape-target down, the gateway's
+  internal-error rate and p99 latency (`edge_request_*`), host memory/disk/CPU
+  (node_exporter), Postgres connection saturation (postgres_exporter), and **TLS certificate
+  expiry < 20 days** — a Caddy ACME-renewal-failure signal from a **`blackbox_exporter`**
+  probe of the edge caddy (`probe_ssl_earliest_cert_expiry`). Every rule is `noDataState=OK`,
+  so an absent metric never false-alerts — notably the cert probe, which has no TLS target on
+  the HTTP-only contour caddy. Separately, the backend's **admin-alert worker**
+  (`internal/adminalert`, started from `main`) emails the operator (`ADMIN_EMAIL`, from a
+  distinct `SMTP_RELAY_ADMIN_FROM`) when new player feedback or word complaints arrive,
+  **coalescing** a burst into one digest per interval; both recipients may be several
+  comma-separated addresses. Both paths are inert unless configured.
 - Per-request server-side timing via gin middleware from day one (the access log
   carries method, route, status, latency and the active trace id). A
   client-measured RTT piggybacked on the next request is a later enhancement.