fix(metrics): second-scale buckets for edge_request_duration
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 14s
CI / ui (pull_request) Has been skipped
CI / conformance (pull_request) Successful in 8s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m44s
CI / changes (pull_request) Successful in 2s
CI / unit (pull_request) Successful in 10s
CI / integration (pull_request) Successful in 14s
CI / ui (pull_request) Has been skipped
CI / conformance (pull_request) Successful in 8s
CI / gate (pull_request) Successful in 0s
CI / deploy (pull_request) Successful in 1m44s
The edge request-latency histogram records seconds but used the OTel SDK's default millisecond-calibrated bucket boundaries (first boundary 5), so every sub-5s request fell into one bucket and histogram_quantile(0.99) interpolated to ~4.95s for every message type — regardless of the real (millisecond) latency. That tripped the >1s "Gateway request latency p99 high" alert on essentially every request, flapping fire/resolve on each app open (seen on the test and prod contours). Set explicit second-scale bucket boundaries (0.005 … 10s) straddling the 1s SLO so the p99 reflects real latency and the alert fires only on genuine slowness. Regression test asserts the histogram carries a sub-second boundary.
This commit is contained in:
@@ -29,6 +29,7 @@ func TestEdgeMetric(t *testing.T) {
|
||||
|
||||
type key struct{ messageType, result string }
|
||||
counts := map[key]uint64{}
|
||||
var bounds []float64
|
||||
for _, sm := range rm.ScopeMetrics {
|
||||
for _, md := range sm.Metrics {
|
||||
if md.Name != "edge_request_duration" {
|
||||
@@ -42,6 +43,7 @@ func TestEdgeMetric(t *testing.T) {
|
||||
mt, _ := dp.Attributes.Value(attribute.Key("message_type"))
|
||||
res, _ := dp.Attributes.Value(attribute.Key("result"))
|
||||
counts[key{mt.AsString(), res.AsString()}] += dp.Count
|
||||
bounds = dp.Bounds
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -51,6 +53,19 @@ func TestEdgeMetric(t *testing.T) {
|
||||
if got := counts[key{"auth.guest", "domain"}]; got != 1 {
|
||||
t.Errorf("edge auth.guest/domain = %d, want 1", got)
|
||||
}
|
||||
// The buckets must be second-scaled. The default (millisecond-calibrated) boundaries have no
|
||||
// boundary between 0 and 5, so every sub-5s request bins into one bucket and p99 interpolates
|
||||
// to ~4.95s, flapping the >1s alert. Require at least one sub-second boundary.
|
||||
subSecond := false
|
||||
for _, b := range bounds {
|
||||
if b > 0 && b < 1 {
|
||||
subSecond = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !subSecond {
|
||||
t.Errorf("edge_request_duration bounds = %v, want sub-second boundaries (seconds-scaled)", bounds)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRateLimitedMetric records limiter rejections through a manual reader and
|
||||
|
||||
Reference in New Issue
Block a user