Files
scrabble-game/gateway/internal/connectsrv/metrics_test.go
T
Ilia Denisov 00bb66ba0f
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 14s
CI / ui (pull_request) Has been skipped
CI / conformance (pull_request) Successful in 8s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m44s
fix(metrics): second-scale buckets for edge_request_duration
The edge request-latency histogram records seconds but used the OTel SDK's
default millisecond-calibrated bucket boundaries (first boundary 5), so every
sub-5s request fell into one bucket and histogram_quantile(0.99) interpolated
to ~4.95s for every message type — regardless of the real (millisecond)
latency. That tripped the >1s "Gateway request latency p99 high" alert on
essentially every request, flapping fire/resolve on each app open (seen on the
test and prod contours).

Set explicit second-scale bucket boundaries (0.005 … 10s) straddling the 1s SLO
so the p99 reflects real latency and the alert fires only on genuine slowness.
Regression test asserts the histogram carries a sub-second boundary.
2026-07-05 22:49:04 +02:00

213 lines
6.9 KiB
Go

package connectsrv
import (
"context"
"testing"
"time"
"go.opentelemetry.io/otel/attribute"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
)
// TestEdgeMetric records Execute outcomes through a manual reader and asserts the
// edge_request_duration histogram splits by message_type and result.
func TestEdgeMetric(t *testing.T) {
ctx := context.Background()
reader := sdkmetric.NewManualReader()
meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
m := newServerMetrics(meter)
m.recordEdge(ctx, "game.submit_play", "ok", time.Now().Add(-time.Millisecond))
m.recordEdge(ctx, "game.submit_play", "ok", time.Now().Add(-time.Millisecond))
m.recordEdge(ctx, "auth.guest", "domain", time.Now().Add(-time.Millisecond))
var rm metricdata.ResourceMetrics
if err := reader.Collect(ctx, &rm); err != nil {
t.Fatalf("collect: %v", err)
}
type key struct{ messageType, result string }
counts := map[key]uint64{}
var bounds []float64
for _, sm := range rm.ScopeMetrics {
for _, md := range sm.Metrics {
if md.Name != "edge_request_duration" {
continue
}
h, ok := md.Data.(metricdata.Histogram[float64])
if !ok {
t.Fatalf("edge_request_duration is not a float64 histogram")
}
for _, dp := range h.DataPoints {
mt, _ := dp.Attributes.Value(attribute.Key("message_type"))
res, _ := dp.Attributes.Value(attribute.Key("result"))
counts[key{mt.AsString(), res.AsString()}] += dp.Count
bounds = dp.Bounds
}
}
}
if got := counts[key{"game.submit_play", "ok"}]; got != 2 {
t.Errorf("edge game.submit_play/ok = %d, want 2", got)
}
if got := counts[key{"auth.guest", "domain"}]; got != 1 {
t.Errorf("edge auth.guest/domain = %d, want 1", got)
}
// The buckets must be second-scaled. The default (millisecond-calibrated) boundaries have no
// boundary between 0 and 5, so every sub-5s request bins into one bucket and p99 interpolates
// to ~4.95s, flapping the >1s alert. Require at least one sub-second boundary.
subSecond := false
for _, b := range bounds {
if b > 0 && b < 1 {
subSecond = true
break
}
}
if !subSecond {
t.Errorf("edge_request_duration bounds = %v, want sub-second boundaries (seconds-scaled)", bounds)
}
}
// TestRateLimitedMetric records limiter rejections through a manual reader and
// asserts gateway_rate_limited_total splits by class.
func TestRateLimitedMetric(t *testing.T) {
ctx := context.Background()
reader := sdkmetric.NewManualReader()
meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
m := newServerMetrics(meter)
m.recordRateLimited(ctx, "user")
m.recordRateLimited(ctx, "user")
m.recordRateLimited(ctx, "public")
var rm metricdata.ResourceMetrics
if err := reader.Collect(ctx, &rm); err != nil {
t.Fatalf("collect: %v", err)
}
counts := map[string]int64{}
for _, sm := range rm.ScopeMetrics {
for _, md := range sm.Metrics {
if md.Name != "gateway_rate_limited_total" {
continue
}
sum, ok := md.Data.(metricdata.Sum[int64])
if !ok {
t.Fatalf("gateway_rate_limited_total is not an int64 sum")
}
for _, dp := range sum.DataPoints {
class, _ := dp.Attributes.Value(attribute.Key("class"))
counts[class.AsString()] += dp.Value
}
}
}
if counts["user"] != 2 || counts["public"] != 1 {
t.Errorf("rate_limited counts = %v, want user=2 public=1", counts)
}
}
// TestBannedMetric records ban events through a manual reader and asserts
// gateway_abuse_banned_total splits by reason.
func TestBannedMetric(t *testing.T) {
ctx := context.Background()
reader := sdkmetric.NewManualReader()
meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
m := newServerMetrics(meter)
m.recordBan(ctx, "tripwire")
m.recordBan(ctx, "tripwire")
m.recordBan(ctx, "honeytoken")
var rm metricdata.ResourceMetrics
if err := reader.Collect(ctx, &rm); err != nil {
t.Fatalf("collect: %v", err)
}
counts := map[string]int64{}
for _, sm := range rm.ScopeMetrics {
for _, md := range sm.Metrics {
if md.Name != "gateway_abuse_banned_total" {
continue
}
sum, ok := md.Data.(metricdata.Sum[int64])
if !ok {
t.Fatalf("gateway_abuse_banned_total is not an int64 sum")
}
for _, dp := range sum.DataPoints {
reason, _ := dp.Attributes.Value(attribute.Key("reason"))
counts[reason.AsString()] += dp.Value
}
}
}
if counts["tripwire"] != 2 || counts["honeytoken"] != 1 {
t.Errorf("banned counts = %v, want tripwire=2 honeytoken=1", counts)
}
}
// TestUnsupportedEngineMetric records unsupported-engine beacons through a manual reader and asserts
// unsupported_engine_total splits by reason and Chromium major.
func TestUnsupportedEngineMetric(t *testing.T) {
ctx := context.Background()
reader := sdkmetric.NewManualReader()
meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
m := newServerMetrics(meter)
m.recordUnsupportedEngine(ctx, "no_bigint", "66")
m.recordUnsupportedEngine(ctx, "no_bigint", "66")
m.recordUnsupportedEngine(ctx, "boot_error", "74")
var rm metricdata.ResourceMetrics
if err := reader.Collect(ctx, &rm); err != nil {
t.Fatalf("collect: %v", err)
}
type key struct{ reason, chromium string }
counts := map[key]int64{}
for _, sm := range rm.ScopeMetrics {
for _, md := range sm.Metrics {
if md.Name != "unsupported_engine_total" {
continue
}
sum, ok := md.Data.(metricdata.Sum[int64])
if !ok {
t.Fatalf("unsupported_engine_total is not an int64 sum")
}
for _, dp := range sum.DataPoints {
reason, _ := dp.Attributes.Value(attribute.Key("reason"))
chromium, _ := dp.Attributes.Value(attribute.Key("chromium"))
counts[key{reason.AsString(), chromium.AsString()}] += dp.Value
}
}
}
if got := counts[key{"no_bigint", "66"}]; got != 2 {
t.Errorf("unsupported no_bigint/66 = %d, want 2", got)
}
if got := counts[key{"boot_error", "74"}]; got != 1 {
t.Errorf("unsupported boot_error/74 = %d, want 1", got)
}
}
// TestNormalizeUnsupported checks that a beacon's reason and Chromium are reduced to bounded label
// values, so a spoofed beacon cannot explode the metric cardinality.
func TestNormalizeUnsupported(t *testing.T) {
cases := []struct {
reason, chromium string
wantReason, wantChromium string
}{
{"no_bigint", "66", "no_bigint", "66"},
{"no_proxy", " 74 ", "no_proxy", "74"},
{"boot_error", "105", "boot_error", "105"},
{"garbage", "66", "other", "66"},
{"", "", "other", "other"},
{"no_bigint", "not-a-number", "no_bigint", "other"},
{"no_bigint", "9999", "no_bigint", "other"},
{"no_bigint", "0", "no_bigint", "other"},
{"no_bigint", "-5", "no_bigint", "other"},
}
for _, c := range cases {
gotR, gotC := normalizeUnsupported(c.reason, c.chromium)
if gotR != c.wantReason || gotC != c.wantChromium {
t.Errorf("normalizeUnsupported(%q,%q) = %q,%q; want %q,%q", c.reason, c.chromium, gotR, gotC, c.wantReason, c.wantChromium)
}
}
}