From 00bb66ba0f0529cbb37aa6ff3dc6b1f3a1876ffb Mon Sep 17 00:00:00 2001 From: Ilia Denisov Date: Sun, 5 Jul 2026 22:49:04 +0200 Subject: [PATCH] fix(metrics): second-scale buckets for edge_request_duration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The edge request-latency histogram records seconds but used the OTel SDK's default millisecond-calibrated bucket boundaries (first boundary 5), so every sub-5s request fell into one bucket and histogram_quantile(0.99) interpolated to ~4.95s for every message type — regardless of the real (millisecond) latency. That tripped the >1s "Gateway request latency p99 high" alert on essentially every request, flapping fire/resolve on each app open (seen on the test and prod contours). Set explicit second-scale bucket boundaries (0.005 … 10s) straddling the 1s SLO so the p99 reflects real latency and the alert fires only on genuine slowness. Regression test asserts the histogram carries a sub-second boundary. --- gateway/internal/connectsrv/metrics.go | 8 +++++++- gateway/internal/connectsrv/metrics_test.go | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/gateway/internal/connectsrv/metrics.go b/gateway/internal/connectsrv/metrics.go index c531e83..ff35ece 100644 --- a/gateway/internal/connectsrv/metrics.go +++ b/gateway/internal/connectsrv/metrics.go @@ -46,7 +46,13 @@ func newServerMetrics(meter metric.Meter) *serverMetrics { } h, err := meter.Float64Histogram("edge_request_duration", metric.WithUnit("s"), - metric.WithDescription("Seconds to serve one Connect Execute call, by message type and result.")) + metric.WithDescription("Seconds to serve one Connect Execute call, by message type and result."), + // Explicit second-scale buckets. The durations are recorded in seconds, so the SDK's + // default millisecond-calibrated boundaries (first boundary 5) would bin every sub-5s + // request into one bucket — making histogram_quantile(0.99) interpolate to ~4.95s + // regardless of the real latency, which flapped the >1s edge-latency alert. These + // boundaries straddle the 1s SLO so the p99 reflects real (mostly sub-second) latency. + metric.WithExplicitBucketBoundaries(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)) if err != nil { h, _ = noop.NewMeterProvider().Meter(meterName).Float64Histogram("edge_request_duration") } diff --git a/gateway/internal/connectsrv/metrics_test.go b/gateway/internal/connectsrv/metrics_test.go index c2cf8c8..d776798 100644 --- a/gateway/internal/connectsrv/metrics_test.go +++ b/gateway/internal/connectsrv/metrics_test.go @@ -29,6 +29,7 @@ func TestEdgeMetric(t *testing.T) { type key struct{ messageType, result string } counts := map[key]uint64{} + var bounds []float64 for _, sm := range rm.ScopeMetrics { for _, md := range sm.Metrics { if md.Name != "edge_request_duration" { @@ -42,6 +43,7 @@ func TestEdgeMetric(t *testing.T) { mt, _ := dp.Attributes.Value(attribute.Key("message_type")) res, _ := dp.Attributes.Value(attribute.Key("result")) counts[key{mt.AsString(), res.AsString()}] += dp.Count + bounds = dp.Bounds } } } @@ -51,6 +53,19 @@ func TestEdgeMetric(t *testing.T) { if got := counts[key{"auth.guest", "domain"}]; got != 1 { t.Errorf("edge auth.guest/domain = %d, want 1", got) } + // The buckets must be second-scaled. The default (millisecond-calibrated) boundaries have no + // boundary between 0 and 5, so every sub-5s request bins into one bucket and p99 interpolates + // to ~4.95s, flapping the >1s alert. Require at least one sub-second boundary. + subSecond := false + for _, b := range bounds { + if b > 0 && b < 1 { + subSecond = true + break + } + } + if !subSecond { + t.Errorf("edge_request_duration bounds = %v, want sub-second boundaries (seconds-scaled)", bounds) + } } // TestRateLimitedMetric records limiter rejections through a manual reader and -- 2.52.0