Ilia Denisov
46d569720c
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 17s
CI / ui (pull_request) Successful in 54s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m18s
The "Database size" stat had no thresholds, so Grafana applied its stat default (green base, red at >=80). The query is pg_database_size_bytes, so a healthy ~9 MiB database (9.4M >> 80) rendered permanently RED on the Scrabble - Resources dashboard (test + prod), reading as a false alert; the neighbouring percentunit cache-hit stat stayed green only because its 0..1 values fall under 80. Add absolute byte thresholds sized to the 40 GiB prod disk (4.6 GiB used, observability bounded -- Tempo <=1 GiB, Prometheus 7d -- so the DB is the only unbounded grower): green up to 8 GiB, yellow at 8 GiB (~20% of disk), red at 16 GiB (~40%), an early warning with ample runway before the disk tightens, not a panic line. Cosmetic panel coloring only; there are no Grafana alert rules provisioned.
99 lines
4.7 KiB
JSON
99 lines
4.7 KiB
JSON
{
|
|
"uid": "scrabble-resources",
|
|
"title": "Scrabble — Resources",
|
|
"tags": ["scrabble"],
|
|
"timezone": "",
|
|
"schemaVersion": 39,
|
|
"version": 2,
|
|
"refresh": "30s",
|
|
"time": { "from": "now-1h", "to": "now" },
|
|
"panels": [
|
|
{
|
|
"type": "stat",
|
|
"title": "Postgres connections",
|
|
"description": "Backends connected to the scrabble database (postgres_exporter).",
|
|
"gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "refId": "A", "expr": "sum(pg_stat_database_numbackends{datname=\"scrabble\"})" }]
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "Postgres cache hit ratio",
|
|
"description": "blks_hit / (blks_hit + blks_read) over 5m.",
|
|
"gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 },
|
|
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, "overrides": [] },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "refId": "A", "expr": "sum(rate(pg_stat_database_blks_hit{datname=\"scrabble\"}[5m])) / clamp_min(sum(rate(pg_stat_database_blks_hit{datname=\"scrabble\"}[5m])) + sum(rate(pg_stat_database_blks_read{datname=\"scrabble\"}[5m])), 1)" }]
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "Postgres commits/s",
|
|
"gridPos": { "h": 5, "w": 6, "x": 12, "y": 0 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "refId": "A", "expr": "sum(rate(pg_stat_database_xact_commit{datname=\"scrabble\"}[5m]))" }]
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "Database size",
|
|
"gridPos": { "h": 5, "w": 6, "x": 18, "y": 0 },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"color": { "mode": "thresholds" },
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "yellow", "value": 8589934592 },
|
|
{ "color": "red", "value": 17179869184 }
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "refId": "A", "expr": "max(pg_database_size_bytes{datname=\"scrabble\"})" }]
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Container CPU (cores) by container",
|
|
"description": "docker_stats container.cpu.utilization (a gauge where 100 == one core) / 100, per scrabble-* container; the load harness appears when run as --name scrabble-loadtest. Verify the scaling against live Prometheus.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "refId": "A", "expr": "max(container_cpu_utilization{container_name=~\"scrabble-.+\"}) by (container_name) / 100", "legendFormat": "{{container_name}}" }]
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Container memory (usage) by container",
|
|
"description": "docker_stats container.memory.usage.total bytes, per scrabble-* container.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
|
|
"fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "refId": "A", "expr": "max(container_memory_usage_total{container_name=~\"scrabble-.+\"}) by (container_name)", "legendFormat": "{{container_name}}" }]
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Container network I/O by container",
|
|
"description": "docker_stats receive (+) and transmit (-) byte rates per scrabble-* container (summed across interfaces).",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
|
|
"fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [
|
|
{ "refId": "A", "expr": "sum(rate(container_network_io_usage_rx_bytes{container_name=~\"scrabble-.+\"}[5m])) by (container_name)", "legendFormat": "rx {{container_name}}" },
|
|
{ "refId": "B", "expr": "-sum(rate(container_network_io_usage_tx_bytes{container_name=~\"scrabble-.+\"}[5m])) by (container_name)", "legendFormat": "tx {{container_name}}" }
|
|
]
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Postgres transactions/s",
|
|
"description": "Commit and rollback rates on the scrabble database (postgres_exporter).",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [
|
|
{ "refId": "A", "expr": "sum(rate(pg_stat_database_xact_commit{datname=\"scrabble\"}[5m]))", "legendFormat": "commit" },
|
|
{ "refId": "B", "expr": "sum(rate(pg_stat_database_xact_rollback{datname=\"scrabble\"}[5m]))", "legendFormat": "rollback" }
|
|
]
|
|
}
|
|
]
|
|
}
|