fix(metrics): second-scale buckets for edge_request_duration
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 14s
CI / ui (pull_request) Has been skipped
CI / conformance (pull_request) Successful in 8s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m44s

The edge request-latency histogram records seconds but used the OTel SDK's
default millisecond-calibrated bucket boundaries (first boundary 5), so every
sub-5s request fell into one bucket and histogram_quantile(0.99) interpolated
to ~4.95s for every message type — regardless of the real (millisecond)
latency. That tripped the >1s "Gateway request latency p99 high" alert on
essentially every request, flapping fire/resolve on each app open (seen on the
test and prod contours).

Set explicit second-scale bucket boundaries (0.005 … 10s) straddling the 1s SLO
so the p99 reflects real latency and the alert fires only on genuine slowness.
Regression test asserts the histogram carries a sub-second boundary.
This commit is contained in:
Ilia Denisov
2026-07-05 22:49:04 +02:00
parent a2c79c149a
commit 00bb66ba0f
2 changed files with 22 additions and 1 deletions
+7 -1
View File
@@ -46,7 +46,13 @@ func newServerMetrics(meter metric.Meter) *serverMetrics {
} }
h, err := meter.Float64Histogram("edge_request_duration", h, err := meter.Float64Histogram("edge_request_duration",
metric.WithUnit("s"), metric.WithUnit("s"),
metric.WithDescription("Seconds to serve one Connect Execute call, by message type and result.")) metric.WithDescription("Seconds to serve one Connect Execute call, by message type and result."),
// Explicit second-scale buckets. The durations are recorded in seconds, so the SDK's
// default millisecond-calibrated boundaries (first boundary 5) would bin every sub-5s
// request into one bucket — making histogram_quantile(0.99) interpolate to ~4.95s
// regardless of the real latency, which flapped the >1s edge-latency alert. These
// boundaries straddle the 1s SLO so the p99 reflects real (mostly sub-second) latency.
metric.WithExplicitBucketBoundaries(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10))
if err != nil { if err != nil {
h, _ = noop.NewMeterProvider().Meter(meterName).Float64Histogram("edge_request_duration") h, _ = noop.NewMeterProvider().Meter(meterName).Float64Histogram("edge_request_duration")
} }
@@ -29,6 +29,7 @@ func TestEdgeMetric(t *testing.T) {
type key struct{ messageType, result string } type key struct{ messageType, result string }
counts := map[key]uint64{} counts := map[key]uint64{}
var bounds []float64
for _, sm := range rm.ScopeMetrics { for _, sm := range rm.ScopeMetrics {
for _, md := range sm.Metrics { for _, md := range sm.Metrics {
if md.Name != "edge_request_duration" { if md.Name != "edge_request_duration" {
@@ -42,6 +43,7 @@ func TestEdgeMetric(t *testing.T) {
mt, _ := dp.Attributes.Value(attribute.Key("message_type")) mt, _ := dp.Attributes.Value(attribute.Key("message_type"))
res, _ := dp.Attributes.Value(attribute.Key("result")) res, _ := dp.Attributes.Value(attribute.Key("result"))
counts[key{mt.AsString(), res.AsString()}] += dp.Count counts[key{mt.AsString(), res.AsString()}] += dp.Count
bounds = dp.Bounds
} }
} }
} }
@@ -51,6 +53,19 @@ func TestEdgeMetric(t *testing.T) {
if got := counts[key{"auth.guest", "domain"}]; got != 1 { if got := counts[key{"auth.guest", "domain"}]; got != 1 {
t.Errorf("edge auth.guest/domain = %d, want 1", got) t.Errorf("edge auth.guest/domain = %d, want 1", got)
} }
// The buckets must be second-scaled. The default (millisecond-calibrated) boundaries have no
// boundary between 0 and 5, so every sub-5s request bins into one bucket and p99 interpolates
// to ~4.95s, flapping the >1s alert. Require at least one sub-second boundary.
subSecond := false
for _, b := range bounds {
if b > 0 && b < 1 {
subSecond = true
break
}
}
if !subSecond {
t.Errorf("edge_request_duration bounds = %v, want sub-second boundaries (seconds-scaled)", bounds)
}
} }
// TestRateLimitedMetric records limiter rejections through a manual reader and // TestRateLimitedMetric records limiter rejections through a manual reader and