galaxy-game/notification/internal/telemetry/runtime_test.go

package telemetry

import (
	"context"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"go.opentelemetry.io/otel/attribute"
	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	"go.opentelemetry.io/otel/sdk/metric/metricdata"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
)

func TestRuntimeRecordsMetrics(t *testing.T) {
	t.Parallel()

	reader := sdkmetric.NewManualReader()
	meterProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
	tracerProvider := sdktrace.NewTracerProvider()

	runtime, err := NewWithProviders(meterProvider, tracerProvider)
	require.NoError(t, err)

	runtime.RecordInternalHTTPRequest(context.Background(), []attribute.KeyValue{
		attribute.String("route", "/healthz"),
		attribute.String("method", "GET"),
		attribute.String("edge_outcome", "success"),
	}, 5*time.Millisecond)
	runtime.RecordInternalHTTPEvent(context.Background(), "started")
	runtime.RecordIntentOutcome(context.Background(), "game.turn.ready", "game_master", "user", "accepted")
	runtime.RecordIntentOutcome(context.Background(), "game.turn.ready", "game_master", "user", "duplicate")
	runtime.RecordMalformedIntent(context.Background(), "idempotency_conflict", "game.turn.ready", "game_master")
	runtime.RecordUserEnrichmentAttempt(context.Background(), "game.turn.ready", "success")
	runtime.RecordUserEnrichmentAttempt(context.Background(), "game.turn.ready", "recipient_not_found")
	runtime.RecordRoutePublishAttempt(context.Background(), "push", "game.turn.ready", "published", "")
	runtime.RecordRoutePublishAttempt(context.Background(), "email", "game.turn.ready", "retry", "mail_stream_publish_failed")
	runtime.RecordRouteRetry(context.Background(), "email", "game.turn.ready")
	runtime.RecordRouteDeadLetter(context.Background(), "email", "game.turn.ready", "mail_stream_publish_failed")
	scheduledAt := time.Now().Add(-time.Second).UTC()
	unprocessedAt := time.Now().Add(-2 * time.Second).UTC()
	runtime.SetRouteScheduleSnapshotReader(stubRouteScheduleSnapshotReader{
		snapshot: RouteScheduleSnapshot{
			Depth:              3,
			OldestScheduledFor: &scheduledAt,
		},
	})
	runtime.SetIntentStreamLagSnapshotReader(stubIntentStreamLagSnapshotReader{
		snapshot: IntentStreamLagSnapshot{
			OldestUnprocessedAt: &unprocessedAt,
		},
	})

	assertMetricCount(t, reader, "notification.internal_http.requests", map[string]string{
		"route":        "/healthz",
		"method":       "GET",
		"edge_outcome": "success",
	}, 1)
	assertMetricCount(t, reader, "notification.internal_http.lifecycle", map[string]string{
		"event": "started",
	}, 1)
	assertMetricCount(t, reader, "notification.intent.outcomes", map[string]string{
		"notification_type": "game.turn.ready",
		"producer":          "game_master",
		"audience_kind":     "user",
		"outcome":           "accepted",
	}, 1)
	assertMetricCount(t, reader, "notification.intent.outcomes", map[string]string{
		"notification_type": "game.turn.ready",
		"producer":          "game_master",
		"audience_kind":     "user",
		"outcome":           "duplicate",
	}, 1)
	assertMetricCount(t, reader, "notification.intent.malformed", map[string]string{
		"failure_code":      "idempotency_conflict",
		"notification_type": "game.turn.ready",
		"producer":          "game_master",
	}, 1)
	assertMetricCount(t, reader, "notification.user_enrichment.attempts", map[string]string{
		"notification_type": "game.turn.ready",
		"result":            "success",
	}, 1)
	assertMetricCount(t, reader, "notification.user_enrichment.attempts", map[string]string{
		"notification_type": "game.turn.ready",
		"result":            "recipient_not_found",
	}, 1)
	assertMetricCount(t, reader, "notification.route.publish_attempts", map[string]string{
		"channel":                "push",
		"notification_type":      "game.turn.ready",
		"result":                 "published",
		"failure_classification": "none",
	}, 1)
	assertMetricCount(t, reader, "notification.route.publish_attempts", map[string]string{
		"channel":                "email",
		"notification_type":      "game.turn.ready",
		"result":                 "retry",
		"failure_classification": "mail_stream_publish_failed",
	}, 1)
	assertMetricCount(t, reader, "notification.route.retries", map[string]string{
		"channel":           "email",
		"notification_type": "game.turn.ready",
	}, 1)
	assertMetricCount(t, reader, "notification.route.dead_letters", map[string]string{
		"channel":                "email",
		"notification_type":      "game.turn.ready",
		"failure_classification": "mail_stream_publish_failed",
	}, 1)
	assertGaugeValue(t, reader, "notification.route_schedule.depth", nil, 3)
	assertGaugePositive(t, reader, "notification.route_schedule.oldest_age_ms", nil)
	assertGaugePositive(t, reader, "notification.intent_stream.oldest_unprocessed_age_ms", nil)
}

func assertMetricCount(t *testing.T, reader *sdkmetric.ManualReader, metricName string, wantAttrs map[string]string, wantValue int64) {
	t.Helper()

	var resourceMetrics metricdata.ResourceMetrics
	require.NoError(t, reader.Collect(context.Background(), &resourceMetrics))

	for _, scopeMetrics := range resourceMetrics.ScopeMetrics {
		for _, metric := range scopeMetrics.Metrics {
			if metric.Name != metricName {
				continue
			}

			sum, ok := metric.Data.(metricdata.Sum[int64])
			require.True(t, ok)

			for _, point := range sum.DataPoints {
				if hasMetricAttributes(point.Attributes.ToSlice(), wantAttrs) {
					assert.Equal(t, wantValue, point.Value)
					return
				}
			}
		}
	}

	require.Failf(t, "test failed", "metric %q with attrs %v not found", metricName, wantAttrs)
}

func assertGaugeValue(t *testing.T, reader *sdkmetric.ManualReader, metricName string, wantAttrs map[string]string, wantValue int64) {
	t.Helper()

	var resourceMetrics metricdata.ResourceMetrics
	require.NoError(t, reader.Collect(context.Background(), &resourceMetrics))

	for _, scopeMetrics := range resourceMetrics.ScopeMetrics {
		for _, metric := range scopeMetrics.Metrics {
			if metric.Name != metricName {
				continue
			}

			gauge, ok := metric.Data.(metricdata.Gauge[int64])
			require.True(t, ok)

			for _, point := range gauge.DataPoints {
				if hasMetricAttributes(point.Attributes.ToSlice(), wantAttrs) {
					assert.Equal(t, wantValue, point.Value)
					return
				}
			}
		}
	}

	require.Failf(t, "test failed", "gauge %q with attrs %v not found", metricName, wantAttrs)
}

func assertGaugePositive(t *testing.T, reader *sdkmetric.ManualReader, metricName string, wantAttrs map[string]string) {
	t.Helper()

	var resourceMetrics metricdata.ResourceMetrics
	require.NoError(t, reader.Collect(context.Background(), &resourceMetrics))

	for _, scopeMetrics := range resourceMetrics.ScopeMetrics {
		for _, metric := range scopeMetrics.Metrics {
			if metric.Name != metricName {
				continue
			}

			gauge, ok := metric.Data.(metricdata.Gauge[int64])
			require.True(t, ok)

			for _, point := range gauge.DataPoints {
				if hasMetricAttributes(point.Attributes.ToSlice(), wantAttrs) {
					assert.Greater(t, point.Value, int64(0))
					return
				}
			}
		}
	}

	require.Failf(t, "test failed", "gauge %q with attrs %v not found", metricName, wantAttrs)
}

func hasMetricAttributes(values []attribute.KeyValue, want map[string]string) bool {
	if len(want) == 0 {
		return len(values) == 0
	}
	if len(values) != len(want) {
		return false
	}

	for _, value := range values {
		if want[string(value.Key)] != value.Value.AsString() {
			return false
		}
	}

	return true
}

type stubRouteScheduleSnapshotReader struct {
	snapshot RouteScheduleSnapshot
	err      error
}

func (reader stubRouteScheduleSnapshotReader) ReadRouteScheduleSnapshot(context.Context) (RouteScheduleSnapshot, error) {
	return reader.snapshot, reader.err
}

type stubIntentStreamLagSnapshotReader struct {
	snapshot IntentStreamLagSnapshot
	err      error
}

func (reader stubIntentStreamLagSnapshotReader) ReadIntentStreamLagSnapshot(context.Context) (IntentStreamLagSnapshot, error) {
	return reader.snapshot, reader.err
}