diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 706c1a0..038c156 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -39,6 +39,13 @@ services: retries: 30 volumes: - postgres-data:/var/lib/postgresql/data + # R7 starting limits: 512M leaves headroom over the default 128 MB shared_buffers + + # per-connection memory (R2 peaked at 28 backends / 69 MiB RSS); tighten after the run. + deploy: + resources: + limits: + cpus: "2.0" + memory: 512M networks: [internal] backend: @@ -65,8 +72,19 @@ services: BACKEND_OTEL_METRICS_EXPORTER: otlp OTEL_EXPORTER_OTLP_ENDPOINT: http://otelcol:4317 OTEL_EXPORTER_OTLP_INSECURE: "true" + # GOMAXPROCS matches the CPU limit below so the Go scheduler aligns with the + # cgroup quota (the runtime otherwise sees all of the host's cores). + GOMAXPROCS: "2" # No container healthcheck: the distroless image has no shell/wget. Readiness # is covered by the CI post-deploy probe (GET / through caddy). + # R7 starting limits (generous over the R2 ~1-core / <=100 MiB peak); tightened to + # the agreed prod values after the final stress run. deploy.resources.limits is + # honoured by `docker compose up` (Compose v2), not only by swarm. + deploy: + resources: + limits: + cpus: "2.0" + memory: 512M networks: [internal] gateway: @@ -97,8 +115,17 @@ services: GATEWAY_OTEL_METRICS_EXPORTER: otlp OTEL_EXPORTER_OTLP_ENDPOINT: http://otelcol:4317 OTEL_EXPORTER_OTLP_INSECURE: "true" + # GOMAXPROCS matches the CPU limit below (see backend). + GOMAXPROCS: "2" # GATEWAY_ADMIN_* intentionally unset: in the deployed contour the front # caddy owns the /_gm Basic-Auth and routes /_gm to the backend directly. + # R7 starting limits (generous over the R2 ~1-core / <=100 MiB peak); tighten after + # the final stress run. + deploy: + resources: + limits: + cpus: "2.0" + memory: 512M networks: [internal] # --- Landing (static) ------------------------------------------------------- @@ -121,6 +148,10 @@ services: VITE_GATEWAY_URL: ${VITE_GATEWAY_URL:-} VITE_APP_VERSION: ${APP_VERSION:-dev} restart: unless-stopped + deploy: + resources: + limits: + memory: 128M networks: [internal] # --- Telegram connector (egress via the VPN sidecar) ----------------------- @@ -167,6 +198,13 @@ services: TELEGRAM_OTEL_METRICS_EXPORTER: otlp OTEL_EXPORTER_OTLP_ENDPOINT: http://otelcol:4317 OTEL_EXPORTER_OTLP_INSECURE: "true" + # The connector is light (the stress run does not drive Telegram); one P suffices. + GOMAXPROCS: "1" + deploy: + resources: + limits: + cpus: "1.0" + memory: 256M # --- Edge reverse proxy (single /_gm Basic-Auth; SPA + Connect -> gateway; # the catch-all incl. the landing -> the static landing container) ------- @@ -183,6 +221,10 @@ services: volumes: - ${SCRABBLE_CONFIG_DIR:-.}/caddy/Caddyfile:/etc/caddy/Caddyfile:ro - caddy-data:/data + deploy: + resources: + limits: + memory: 128M networks: internal: {} edge: @@ -194,8 +236,19 @@ services: image: otel/opentelemetry-collector-contrib:0.119.0 restart: unless-stopped command: ["--config=/etc/otelcol/config.yaml"] + # The docker_stats receiver reads per-container metrics from the Docker API, so the + # collector (image UID 10001) joins the host's docker group to read the socket — + # DOCKER_GID defaults to the contour host's 989; set it for other hosts (prod). The + # socket is mounted read-only. This replaces cAdvisor, whose per-container metrics + # are empty on this host (separate-XFS /var/lib/docker). + group_add: ["${DOCKER_GID:-989}"] volumes: - ${SCRABBLE_CONFIG_DIR:-.}/otelcol/config.yaml:/etc/otelcol/config.yaml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + deploy: + resources: + limits: + memory: 512M networks: [internal] prometheus: @@ -208,6 +261,10 @@ services: volumes: - ${SCRABBLE_CONFIG_DIR:-.}/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro - prometheus-data:/prometheus + deploy: + resources: + limits: + memory: 512M networks: [internal] tempo: @@ -218,6 +275,11 @@ services: volumes: - ${SCRABBLE_CONFIG_DIR:-.}/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro - tempo-data:/var/tempo + # tempo peaked at ~446 MiB in R2; 1G leaves headroom for the final run. + deploy: + resources: + limits: + memory: 1G networks: [internal] grafana: @@ -247,26 +309,10 @@ services: # provider logs "no such file or directory"). - ${SCRABBLE_CONFIG_DIR:-.}/grafana/dashboards:/etc/grafana/dashboards:ro - grafana-data:/var/lib/grafana - networks: [internal] - - # cAdvisor exports per-container resource metrics (CPU / memory / network / disk) - # for the R2/R7 stress runs' resource baseline. Prometheus scrapes it at :8080 - # over the internal network. It needs read access to the host's cgroup and - # container state; --docker_only trims non-container cgroup series. - cadvisor: - container_name: scrabble-cadvisor - image: gcr.io/cadvisor/cadvisor:v0.49.1 - restart: unless-stopped - privileged: true - command: ["--docker_only=true", "--housekeeping_interval=15s"] - devices: - - /dev/kmsg - volumes: - - /:/rootfs:ro - - /var/run:/var/run:ro - - /sys:/sys:ro - - /var/lib/docker/:/var/lib/docker:ro - - /dev/disk/:/dev/disk:ro + deploy: + resources: + limits: + memory: 512M networks: [internal] # postgres_exporter exports Postgres server metrics (connections, cache hit ratio, @@ -279,6 +325,10 @@ services: depends_on: [postgres] environment: DATA_SOURCE_NAME: postgresql://${POSTGRES_USER:-scrabble}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-scrabble}?sslmode=disable + deploy: + resources: + limits: + memory: 128M networks: [internal] networks: diff --git a/deploy/grafana/dashboards/resources.json b/deploy/grafana/dashboards/resources.json index 425f88d..a1d503e 100644 --- a/deploy/grafana/dashboards/resources.json +++ b/deploy/grafana/dashboards/resources.json @@ -4,7 +4,7 @@ "tags": ["scrabble"], "timezone": "", "schemaVersion": 39, - "version": 1, + "version": 2, "refresh": "30s", "time": { "from": "now-1h", "to": "now" }, "panels": [ @@ -43,30 +43,30 @@ { "type": "timeseries", "title": "Container CPU (cores) by container", - "description": "cAdvisor container_cpu_usage_seconds_total rate, per scrabble-* container (the load harness appears when run as --name scrabble-loadtest). Verify the metric name against live Prometheus if empty.", + "description": "docker_stats container.cpu.utilization (a gauge where 100 == one core) / 100, per scrabble-* container; the load harness appears when run as --name scrabble-loadtest. Verify the scaling against live Prometheus.", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "refId": "A", "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"scrabble-.+\"}[5m])) by (name)", "legendFormat": "{{name}}" }] + "targets": [{ "refId": "A", "expr": "max(container_cpu_utilization{container_name=~\"scrabble-.+\"}) by (container_name) / 100", "legendFormat": "{{container_name}}" }] }, { "type": "timeseries", - "title": "Container memory (working set) by container", - "description": "cAdvisor container_memory_working_set_bytes, per scrabble-* container.", + "title": "Container memory (usage) by container", + "description": "docker_stats container.memory.usage.total bytes, per scrabble-* container.", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 }, "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "refId": "A", "expr": "max(container_memory_working_set_bytes{name=~\"scrabble-.+\"}) by (name)", "legendFormat": "{{name}}" }] + "targets": [{ "refId": "A", "expr": "max(container_memory_usage_total{container_name=~\"scrabble-.+\"}) by (container_name)", "legendFormat": "{{container_name}}" }] }, { "type": "timeseries", "title": "Container network I/O by container", - "description": "cAdvisor receive (+) and transmit (-) byte rates per scrabble-* container.", + "description": "docker_stats receive (+) and transmit (-) byte rates per scrabble-* container (summed across interfaces).", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [ - { "refId": "A", "expr": "sum(rate(container_network_receive_bytes_total{name=~\"scrabble-.+\"}[5m])) by (name)", "legendFormat": "rx {{name}}" }, - { "refId": "B", "expr": "-sum(rate(container_network_transmit_bytes_total{name=~\"scrabble-.+\"}[5m])) by (name)", "legendFormat": "tx {{name}}" } + { "refId": "A", "expr": "sum(rate(container_network_io_usage_rx_bytes{container_name=~\"scrabble-.+\"}[5m])) by (container_name)", "legendFormat": "rx {{container_name}}" }, + { "refId": "B", "expr": "-sum(rate(container_network_io_usage_tx_bytes{container_name=~\"scrabble-.+\"}[5m])) by (container_name)", "legendFormat": "tx {{container_name}}" } ] }, { diff --git a/deploy/otelcol/config.yaml b/deploy/otelcol/config.yaml index 8a0e1f4..3e814d4 100644 --- a/deploy/otelcol/config.yaml +++ b/deploy/otelcol/config.yaml @@ -6,6 +6,18 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + # Per-container resource metrics (CPU / memory / network) read straight from the + # Docker API. This replaces cAdvisor, which on the contour host resolves only the + # root cgroup (its /var/lib/docker is a separate XFS mount), and works the same in + # prod. The collector reaches the socket via group_add in docker-compose.yml. + # collection_interval matches Prometheus' 30s scrape. container.cpu.utilization is a + # gauge where 100 == one core (it mirrors `docker stats` CPU%). + docker_stats: + endpoint: unix:///var/run/docker.sock + collection_interval: 30s + metrics: + container.cpu.utilization: + enabled: true processors: batch: {} @@ -33,6 +45,6 @@ service: processors: [batch] exporters: [otlp/tempo] metrics: - receivers: [otlp] + receivers: [otlp, docker_stats] processors: [batch] exporters: [prometheus] diff --git a/deploy/prometheus/prometheus.yml b/deploy/prometheus/prometheus.yml index 1cecf13..71d6cd8 100644 --- a/deploy/prometheus/prometheus.yml +++ b/deploy/prometheus/prometheus.yml @@ -6,17 +6,14 @@ global: evaluation_interval: 30s scrape_configs: + # otelcol exposes both the services' OTLP metrics and the docker_stats receiver's + # per-container resource metrics (CPU/memory/network) on one endpoint. - job_name: otelcol static_configs: - targets: ["otelcol:9464"] - job_name: prometheus static_configs: - targets: ["localhost:9090"] - # Container resource metrics (CPU/memory/network/disk) for every contour - # container, for the R2/R7 stress runs' resource baseline. - - job_name: cadvisor - static_configs: - - targets: ["cadvisor:8080"] # Postgres server metrics (connections, cache hit ratio, transactions, db size). - job_name: postgres_exporter static_configs: