Merge pull request 'fix(metrics): second-scale buckets for edge_request_duration' (#186) from fix/edge-latency-histogram-buckets into development

gateway/internal/connectsrv/metrics.go

@@ -46,7 +46,13 @@ func newServerMetrics(meter metric.Meter) *serverMetrics {
 	}
 	h, err := meter.Float64Histogram("edge_request_duration",
 		metric.WithUnit("s"),
-		metric.WithDescription("Seconds to serve one Connect Execute call, by message type and result."))
+		metric.WithDescription("Seconds to serve one Connect Execute call, by message type and result."),
+		// Explicit second-scale buckets. The durations are recorded in seconds, so the SDK's
+		// default millisecond-calibrated boundaries (first boundary 5) would bin every sub-5s
+		// request into one bucket — making histogram_quantile(0.99) interpolate to ~4.95s
+		// regardless of the real latency, which flapped the >1s edge-latency alert. These
+		// boundaries straddle the 1s SLO so the p99 reflects real (mostly sub-second) latency.
+		metric.WithExplicitBucketBoundaries(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10))
 	if err != nil {
 		h, _ = noop.NewMeterProvider().Meter(meterName).Float64Histogram("edge_request_duration")
 	}

gateway/internal/connectsrv/metrics_test.go

@@ -29,6 +29,7 @@ func TestEdgeMetric(t *testing.T) {
 
 	type key struct{ messageType, result string }
 	counts := map[key]uint64{}
+	var bounds []float64
 	for _, sm := range rm.ScopeMetrics {
 		for _, md := range sm.Metrics {
 			if md.Name != "edge_request_duration" {
@@ -42,6 +43,7 @@ func TestEdgeMetric(t *testing.T) {
 				mt, _ := dp.Attributes.Value(attribute.Key("message_type"))
 				res, _ := dp.Attributes.Value(attribute.Key("result"))
 				counts[key{mt.AsString(), res.AsString()}] += dp.Count
+				bounds = dp.Bounds
 			}
 		}
 	}
@@ -51,6 +53,19 @@ func TestEdgeMetric(t *testing.T) {
 	if got := counts[key{"auth.guest", "domain"}]; got != 1 {
 		t.Errorf("edge auth.guest/domain = %d, want 1", got)
 	}
+	// The buckets must be second-scaled. The default (millisecond-calibrated) boundaries have no
+	// boundary between 0 and 5, so every sub-5s request bins into one bucket and p99 interpolates
+	// to ~4.95s, flapping the >1s alert. Require at least one sub-second boundary.
+	subSecond := false
+	for _, b := range bounds {
+		if b > 0 && b < 1 {
+			subSecond = true
+			break
+		}
+	}
+	if !subSecond {
+		t.Errorf("edge_request_duration bounds = %v, want sub-second boundaries (seconds-scaled)", bounds)
+	}
 }
 
 // TestRateLimitedMetric records limiter rejections through a manual reader and