00bb66ba0f
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 14s
CI / ui (pull_request) Has been skipped
CI / conformance (pull_request) Successful in 8s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m44s
The edge request-latency histogram records seconds but used the OTel SDK's default millisecond-calibrated bucket boundaries (first boundary 5), so every sub-5s request fell into one bucket and histogram_quantile(0.99) interpolated to ~4.95s for every message type — regardless of the real (millisecond) latency. That tripped the >1s "Gateway request latency p99 high" alert on essentially every request, flapping fire/resolve on each app open (seen on the test and prod contours). Set explicit second-scale bucket boundaries (0.005 … 10s) straddling the 1s SLO so the p99 reflects real latency and the alert fires only on genuine slowness. Regression test asserts the histogram carries a sub-second boundary.
213 lines
6.9 KiB
Go
213 lines
6.9 KiB
Go
package connectsrv
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel/attribute"
|
|
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
|
"go.opentelemetry.io/otel/sdk/metric/metricdata"
|
|
)
|
|
|
|
// TestEdgeMetric records Execute outcomes through a manual reader and asserts the
|
|
// edge_request_duration histogram splits by message_type and result.
|
|
func TestEdgeMetric(t *testing.T) {
|
|
ctx := context.Background()
|
|
reader := sdkmetric.NewManualReader()
|
|
meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
|
|
m := newServerMetrics(meter)
|
|
|
|
m.recordEdge(ctx, "game.submit_play", "ok", time.Now().Add(-time.Millisecond))
|
|
m.recordEdge(ctx, "game.submit_play", "ok", time.Now().Add(-time.Millisecond))
|
|
m.recordEdge(ctx, "auth.guest", "domain", time.Now().Add(-time.Millisecond))
|
|
|
|
var rm metricdata.ResourceMetrics
|
|
if err := reader.Collect(ctx, &rm); err != nil {
|
|
t.Fatalf("collect: %v", err)
|
|
}
|
|
|
|
type key struct{ messageType, result string }
|
|
counts := map[key]uint64{}
|
|
var bounds []float64
|
|
for _, sm := range rm.ScopeMetrics {
|
|
for _, md := range sm.Metrics {
|
|
if md.Name != "edge_request_duration" {
|
|
continue
|
|
}
|
|
h, ok := md.Data.(metricdata.Histogram[float64])
|
|
if !ok {
|
|
t.Fatalf("edge_request_duration is not a float64 histogram")
|
|
}
|
|
for _, dp := range h.DataPoints {
|
|
mt, _ := dp.Attributes.Value(attribute.Key("message_type"))
|
|
res, _ := dp.Attributes.Value(attribute.Key("result"))
|
|
counts[key{mt.AsString(), res.AsString()}] += dp.Count
|
|
bounds = dp.Bounds
|
|
}
|
|
}
|
|
}
|
|
if got := counts[key{"game.submit_play", "ok"}]; got != 2 {
|
|
t.Errorf("edge game.submit_play/ok = %d, want 2", got)
|
|
}
|
|
if got := counts[key{"auth.guest", "domain"}]; got != 1 {
|
|
t.Errorf("edge auth.guest/domain = %d, want 1", got)
|
|
}
|
|
// The buckets must be second-scaled. The default (millisecond-calibrated) boundaries have no
|
|
// boundary between 0 and 5, so every sub-5s request bins into one bucket and p99 interpolates
|
|
// to ~4.95s, flapping the >1s alert. Require at least one sub-second boundary.
|
|
subSecond := false
|
|
for _, b := range bounds {
|
|
if b > 0 && b < 1 {
|
|
subSecond = true
|
|
break
|
|
}
|
|
}
|
|
if !subSecond {
|
|
t.Errorf("edge_request_duration bounds = %v, want sub-second boundaries (seconds-scaled)", bounds)
|
|
}
|
|
}
|
|
|
|
// TestRateLimitedMetric records limiter rejections through a manual reader and
|
|
// asserts gateway_rate_limited_total splits by class.
|
|
func TestRateLimitedMetric(t *testing.T) {
|
|
ctx := context.Background()
|
|
reader := sdkmetric.NewManualReader()
|
|
meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
|
|
m := newServerMetrics(meter)
|
|
|
|
m.recordRateLimited(ctx, "user")
|
|
m.recordRateLimited(ctx, "user")
|
|
m.recordRateLimited(ctx, "public")
|
|
|
|
var rm metricdata.ResourceMetrics
|
|
if err := reader.Collect(ctx, &rm); err != nil {
|
|
t.Fatalf("collect: %v", err)
|
|
}
|
|
|
|
counts := map[string]int64{}
|
|
for _, sm := range rm.ScopeMetrics {
|
|
for _, md := range sm.Metrics {
|
|
if md.Name != "gateway_rate_limited_total" {
|
|
continue
|
|
}
|
|
sum, ok := md.Data.(metricdata.Sum[int64])
|
|
if !ok {
|
|
t.Fatalf("gateway_rate_limited_total is not an int64 sum")
|
|
}
|
|
for _, dp := range sum.DataPoints {
|
|
class, _ := dp.Attributes.Value(attribute.Key("class"))
|
|
counts[class.AsString()] += dp.Value
|
|
}
|
|
}
|
|
}
|
|
if counts["user"] != 2 || counts["public"] != 1 {
|
|
t.Errorf("rate_limited counts = %v, want user=2 public=1", counts)
|
|
}
|
|
}
|
|
|
|
// TestBannedMetric records ban events through a manual reader and asserts
|
|
// gateway_abuse_banned_total splits by reason.
|
|
func TestBannedMetric(t *testing.T) {
|
|
ctx := context.Background()
|
|
reader := sdkmetric.NewManualReader()
|
|
meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
|
|
m := newServerMetrics(meter)
|
|
|
|
m.recordBan(ctx, "tripwire")
|
|
m.recordBan(ctx, "tripwire")
|
|
m.recordBan(ctx, "honeytoken")
|
|
|
|
var rm metricdata.ResourceMetrics
|
|
if err := reader.Collect(ctx, &rm); err != nil {
|
|
t.Fatalf("collect: %v", err)
|
|
}
|
|
|
|
counts := map[string]int64{}
|
|
for _, sm := range rm.ScopeMetrics {
|
|
for _, md := range sm.Metrics {
|
|
if md.Name != "gateway_abuse_banned_total" {
|
|
continue
|
|
}
|
|
sum, ok := md.Data.(metricdata.Sum[int64])
|
|
if !ok {
|
|
t.Fatalf("gateway_abuse_banned_total is not an int64 sum")
|
|
}
|
|
for _, dp := range sum.DataPoints {
|
|
reason, _ := dp.Attributes.Value(attribute.Key("reason"))
|
|
counts[reason.AsString()] += dp.Value
|
|
}
|
|
}
|
|
}
|
|
if counts["tripwire"] != 2 || counts["honeytoken"] != 1 {
|
|
t.Errorf("banned counts = %v, want tripwire=2 honeytoken=1", counts)
|
|
}
|
|
}
|
|
|
|
// TestUnsupportedEngineMetric records unsupported-engine beacons through a manual reader and asserts
|
|
// unsupported_engine_total splits by reason and Chromium major.
|
|
func TestUnsupportedEngineMetric(t *testing.T) {
|
|
ctx := context.Background()
|
|
reader := sdkmetric.NewManualReader()
|
|
meter := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")
|
|
m := newServerMetrics(meter)
|
|
|
|
m.recordUnsupportedEngine(ctx, "no_bigint", "66")
|
|
m.recordUnsupportedEngine(ctx, "no_bigint", "66")
|
|
m.recordUnsupportedEngine(ctx, "boot_error", "74")
|
|
|
|
var rm metricdata.ResourceMetrics
|
|
if err := reader.Collect(ctx, &rm); err != nil {
|
|
t.Fatalf("collect: %v", err)
|
|
}
|
|
type key struct{ reason, chromium string }
|
|
counts := map[key]int64{}
|
|
for _, sm := range rm.ScopeMetrics {
|
|
for _, md := range sm.Metrics {
|
|
if md.Name != "unsupported_engine_total" {
|
|
continue
|
|
}
|
|
sum, ok := md.Data.(metricdata.Sum[int64])
|
|
if !ok {
|
|
t.Fatalf("unsupported_engine_total is not an int64 sum")
|
|
}
|
|
for _, dp := range sum.DataPoints {
|
|
reason, _ := dp.Attributes.Value(attribute.Key("reason"))
|
|
chromium, _ := dp.Attributes.Value(attribute.Key("chromium"))
|
|
counts[key{reason.AsString(), chromium.AsString()}] += dp.Value
|
|
}
|
|
}
|
|
}
|
|
if got := counts[key{"no_bigint", "66"}]; got != 2 {
|
|
t.Errorf("unsupported no_bigint/66 = %d, want 2", got)
|
|
}
|
|
if got := counts[key{"boot_error", "74"}]; got != 1 {
|
|
t.Errorf("unsupported boot_error/74 = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
// TestNormalizeUnsupported checks that a beacon's reason and Chromium are reduced to bounded label
|
|
// values, so a spoofed beacon cannot explode the metric cardinality.
|
|
func TestNormalizeUnsupported(t *testing.T) {
|
|
cases := []struct {
|
|
reason, chromium string
|
|
wantReason, wantChromium string
|
|
}{
|
|
{"no_bigint", "66", "no_bigint", "66"},
|
|
{"no_proxy", " 74 ", "no_proxy", "74"},
|
|
{"boot_error", "105", "boot_error", "105"},
|
|
{"garbage", "66", "other", "66"},
|
|
{"", "", "other", "other"},
|
|
{"no_bigint", "not-a-number", "no_bigint", "other"},
|
|
{"no_bigint", "9999", "no_bigint", "other"},
|
|
{"no_bigint", "0", "no_bigint", "other"},
|
|
{"no_bigint", "-5", "no_bigint", "other"},
|
|
}
|
|
for _, c := range cases {
|
|
gotR, gotC := normalizeUnsupported(c.reason, c.chromium)
|
|
if gotR != c.wantReason || gotC != c.wantChromium {
|
|
t.Errorf("normalizeUnsupported(%q,%q) = %q,%q; want %q,%q", c.reason, c.chromium, gotR, gotC, c.wantReason, c.wantChromium)
|
|
}
|
|
}
|
|
}
|