lingniu-vehicle-ingest/go/vehicle-gateway/internal/capacity/check_test.go

package capacity

import (
	"fmt"
	"math"
	"strconv"
	"strings"
	"testing"
	"time"
)

func TestEvaluateReportsOKWhenCriticalBacklogsAreZero(t *testing.T) {
	report := Evaluate(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_active_connections{protocol="JT808"} 50000
vehicle_gateway_connection_rejections_total{protocol="JT808",reason="max_connections"} 0
vehicle_gateway_identity_snapshot_ready 1
vehicle_gateway_identity_snapshot_last_success_unix_seconds 99999999999
vehicle_async_sink_queue_depth{sink="nats"} 0`,
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_nats_consumer_ack_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 0
vehicle_fast_writer_nats_consumer_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 0
vehicle_fast_writer_tdengine_enabled 0
vehicle_fast_writer_batch_pending_messages 0`,
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_batch_pending_messages 0
vehicle_history_batch_pending_rows 0
vehicle_history_retry_pending_messages 0
vehicle_history_kafka_lag{topic="vehicle.raw.go.jt808.v1",partition="0"} 0`,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if report.Totals.ActiveConnections != 50000 {
		t.Fatalf("active connections = %d, want 50000", report.Totals.ActiveConnections)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedForPendingAndRejects(t *testing.T) {
	report := Evaluate(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_active_connections{protocol="JT808"} 120000
vehicle_gateway_connection_rejections_total{protocol="JT808",reason="max_connections"} 3
vehicle_async_sink_queue_depth{sink="nats"} 25000`,
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_nats_consumer_ack_pending{consumer="vehicle-kafka-bridge",stream="VEHICLE_INGEST"} 101
vehicle_bridge_nats_consumer_pending{consumer="vehicle-kafka-bridge",stream="VEHICLE_INGEST"} 12001
vehicle_bridge_batch_pending_messages 1001`,
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_nats_consumer_ack_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 101
vehicle_fast_writer_nats_consumer_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 12001
vehicle_fast_writer_tdengine_enabled 1
vehicle_fast_writer_batch_pending_messages 1009`,
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_batch_pending_messages 1006
vehicle_history_batch_pending_rows 5007
vehicle_history_retry_pending_messages 2
vehicle_history_kafka_lag{topic="vehicle.raw.go.jt808.v1",partition="0"} 1042`,
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_batch_pending_messages 1008
vehicle_stat_retry_pending_messages 3`,
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_batch_pending_messages 1009
vehicle_realtime_retry_pending_messages 4
vehicle_realtime_async_queue_depth{protocol="JT808",store="mysql"} 10001`,
		"identity": `vehicle_service_info{service="vehicle-identity-writer"} 1
vehicle_kafka_consumer_info{service="vehicle-identity-writer",group="go-identity-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_identity_writer_retry_pending_messages 5`,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway connection rejections",
		"async sink queue depth",
		"bridge ack pending",
		"bridge consumer pending",
		"bridge batch pending",
		"fast writer ack pending",
		"fast writer consumer pending",
		"fast writer batch pending",
		"history batch pending",
		"history rows pending",
		"stat batch pending",
		"realtime batch pending",
		"realtime async queue depth",
		"history retry pending 2 exceeds 0",
		"realtime retry pending 4 exceeds 0",
		"stat retry pending 3 exceeds 0",
		"identity retry pending 5 exceeds 0",
		"kafka lag 1042 exceeds 1000",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
	if report.Totals.BridgeAckPending != 101 {
		t.Fatalf("bridge ack pending = %v, want 101", report.Totals.BridgeAckPending)
	}
	if report.Totals.BridgeConsumerPending != 12001 {
		t.Fatalf("bridge consumer pending = %v, want 12001", report.Totals.BridgeConsumerPending)
	}
	if report.Totals.FastWriterAckPending != 101 {
		t.Fatalf("fast writer ack pending = %v, want 101", report.Totals.FastWriterAckPending)
	}
	if report.Totals.FastWriterBatchPending != 1009 {
		t.Fatalf("fast writer batch pending = %v, want 1009", report.Totals.FastWriterBatchPending)
	}
	if report.Totals.HistoryBatchPending != 1006 {
		t.Fatalf("history batch pending = %v, want 1006", report.Totals.HistoryBatchPending)
	}
	if report.Totals.HistoryRowsPending != 5007 {
		t.Fatalf("history rows pending = %v, want 5007", report.Totals.HistoryRowsPending)
	}
	if report.Totals.RealtimeBatchPending != 1009 {
		t.Fatalf("realtime batch pending = %v, want 1009", report.Totals.RealtimeBatchPending)
	}
	if report.Totals.RealtimeAsyncQueuePending != 10001 {
		t.Fatalf("realtime async queue pending = %v, want 10001", report.Totals.RealtimeAsyncQueuePending)
	}
	if report.Totals.HistoryRetryPending != 2 || report.Totals.RealtimeRetryPending != 4 || report.Totals.StatRetryPending != 3 || report.Totals.IdentityRetryPending != 5 {
		t.Fatalf("retry pending totals = history:%v realtime:%v stat:%v identity:%v", report.Totals.HistoryRetryPending, report.Totals.RealtimeRetryPending, report.Totals.StatRetryPending, report.Totals.IdentityRetryPending)
	}
}

func TestEvaluateCanDisableConsumerRetryPendingLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_retry_pending_messages 9`,
	}, Thresholds{ConsumerRetryPendingMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateDoesNotDegradeForTransientBatchPending(t *testing.T) {
	report := Evaluate(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_batch_pending_messages 470`,
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_batch_pending_messages 470`,
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_batch_pending_messages 470`,
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_batch_pending_messages 470
vehicle_history_batch_pending_rows 470`,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
	if report.Totals.FastWriterBatchPending != 470 {
		t.Fatalf("fast writer batch pending = %v, want 470", report.Totals.FastWriterBatchPending)
	}
	if report.Totals.HistoryBatchPending != 470 {
		t.Fatalf("history batch pending = %v, want 470", report.Totals.HistoryBatchPending)
	}
	if report.Totals.HistoryRowsPending != 470 {
		t.Fatalf("history rows pending = %v, want 470", report.Totals.HistoryRowsPending)
	}
	if report.Totals.RealtimeBatchPending != 470 {
		t.Fatalf("realtime batch pending = %v, want 470", report.Totals.RealtimeBatchPending)
	}
}

func TestEvaluateReportsFastWriterTDengineStageEnabled(t *testing.T) {
	report := Evaluate(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_tdengine_enabled 1`,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "fast writer tdengine enabled 1 exceeds 0") {
		t.Fatalf("finding missing in:\n%s", joined)
	}
}

func TestEvaluateCanDisableFastWriterTDengineStageEnabledLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_tdengine_enabled 1`,
	}, Thresholds{FastWriterTDengineEnabledMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsRecentlyStartedProcess(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_process_uptime_seconds 12`,
	}, Thresholds{ProcessUptimeMinSec: 60})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "gateway process uptime 12s below 60s") {
		t.Fatalf("uptime finding missing in:\n%s", joined)
	}
}

func TestEvaluateReportsMissingProcessUptimeWhenRequired(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1`,
	}, Thresholds{ProcessUptimeMinSec: 60})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "gateway process uptime metric missing") {
		t.Fatalf("missing uptime finding not found in:\n%s", joined)
	}
}

func TestEvaluateCanDisableProcessUptimeLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_process_uptime_seconds 1`,
	}, Thresholds{ProcessUptimeMinSec: 0})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsProcessResourceLimits(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_process_heap_alloc_bytes 2048
vehicle_process_heap_sys_bytes 4096
vehicle_process_goroutines 301`,
	}, Thresholds{
		ProcessHeapAllocMaxBytes: 1024,
		ProcessHeapSysMaxBytes:   2048,
		ProcessGoroutinesMax:     300,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway process heap alloc 2048 bytes exceeds 1024",
		"gateway process heap sys 4096 bytes exceeds 2048",
		"gateway process goroutines 301 exceeds 300",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsMissingProcessResourceMetricsWhenRequired(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1`,
	}, Thresholds{
		ProcessHeapAllocMaxBytes: 1024,
		ProcessHeapSysMaxBytes:   2048,
		ProcessGoroutinesMax:     300,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway process heap alloc metric missing",
		"gateway process heap sys metric missing",
		"gateway process goroutines metric missing",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateCanDisableProcessResourceLimits(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_process_heap_alloc_bytes 2048
vehicle_process_heap_sys_bytes 4096
vehicle_process_goroutines 301`,
	}, Thresholds{
		ProcessHeapAllocMaxBytes: 0,
		ProcessHeapSysMaxBytes:   0,
		ProcessGoroutinesMax:     0,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateUsesConfigurableAsyncSinkQueueDepthLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_async_sink_queue_depth{sink="nats"} 1500`,
	}, Thresholds{AsyncSinkQueueDepthMax: 1000})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "async sink queue depth 1500 exceeds 1000") {
		t.Fatalf("finding missing in:\n%s", joined)
	}
}

func TestEvaluateCanDisableAsyncSinkQueueDepthLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_async_sink_queue_depth{sink="nats"} 999999`,
	}, Thresholds{AsyncSinkQueueDepthMax: 0})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsSlowAsyncSinkQueueWaitRecentP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_async_sink_queue_wait_recent_p99_ms{sink="nats",queue="raw",kind="raw"} 125
vehicle_async_sink_queue_wait_recent_samples{sink="nats",queue="raw",kind="raw"} 512`,
	}, Thresholds{
		AsyncSinkQueueWaitRecentP99MS: 100,
		MinHistogramSamples:           100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"async sink queue wait recent p99 125ms exceeds 100ms",
		"kind=raw,queue=raw,sink=nats",
		"samples 512",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateCanDisableAsyncSinkQueueWaitRecentP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_async_sink_queue_wait_recent_p99_ms{sink="nats",queue="raw",kind="raw"} 999
vehicle_async_sink_queue_wait_recent_samples{sink="nats",queue="raw",kind="raw"} 512`,
	}, Thresholds{
		AsyncSinkQueueWaitRecentP99MS: 0,
		MinHistogramSamples:           100,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
}

func TestEvaluateReportsAsyncSinkEnqueueAndPublishErrors(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_async_sink_enqueue_total{sink="nats",kind="raw",status="queued"} 100
vehicle_async_sink_enqueue_total{sink="nats",kind="fields",status="timeout"} 2
vehicle_async_sink_enqueue_total{sink="kafka",kind="raw",status="closed"} 1
vehicle_async_sink_publish_total{sink="nats",kind="raw",status="error"} 3
vehicle_async_sink_publish_total{sink="nats",kind="fields",status="ok"} 100`,
	}, Thresholds{AsyncSinkEnqueueErrorMax: 0, AsyncSinkPublishErrorMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway async sink enqueue timeout 2 exceeds 0 for sink nats kind fields",
		"gateway async sink enqueue closed 1 exceeds 0 for sink kafka kind raw",
		"gateway async sink publish errors 3 exceeds 0 for sink nats kind raw",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
	if strings.Contains(joined, "queued") || strings.Contains(joined, "publish errors 100") {
		t.Fatalf("successful enqueue/publish samples should not be reported:\n%s", joined)
	}
}

func TestEvaluateCanDisableAsyncSinkErrorLimits(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_async_sink_enqueue_total{sink="nats",kind="fields",status="timeout"} 2
vehicle_async_sink_publish_total{sink="nats",kind="raw",status="error"} 3`,
	}, Thresholds{AsyncSinkEnqueueErrorMax: -1, AsyncSinkPublishErrorMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsRuntimeConfigDrift(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 1000
vehicle_bridge_config{setting="workers"} 1`,
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 100
vehicle_fast_writer_config{setting="workers"} 2`,
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 1
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1`,
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 1
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1`,
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_config{setting="workers"} 1`,
		"identity": `vehicle_service_info{service="vehicle-identity-writer"} 1
vehicle_identity_writer_config{setting="workers"} 1
vehicle_kafka_consumer_info{service="vehicle-identity-writer",group="go-identity-writer",topic="vehicle.raw.go.jt808.v1"} 1`,
	}, Thresholds{
		BridgeFetchWaitMaxMS:     50,
		BridgeWorkersMin:         2,
		FastWriterFetchWaitMaxMS: 50,
		FastWriterWorkersMin:     4,
		HistoryWorkersMin:        3,
		RealtimeWorkersMin:       3,
		StatWorkersMin:           3,
		IdentityWorkersMin:       3,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"bridge fetch wait 1000ms exceeds 50ms",
		"bridge workers 1 below 2",
		"fast writer fetch wait 100ms exceeds 50ms",
		"fast writer workers 2 below 4",
		"history workers 1 below 3",
		"realtime workers 1 below 3",
		"stat workers 1 below 3",
		"identity workers 1 below 3",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateAcceptsRuntimeConfigWithinLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_jt808_registration_gateway_writes_enabled 0`,
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4`,
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8`,
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1`,
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1`,
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`,
		"identity": `vehicle_service_info{service="vehicle-identity-writer"} 1
vehicle_identity_writer_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-identity-writer",group="go-identity-writer",topic="vehicle.raw.go.jt808.v1"} 1`,
	}, Thresholds{
		BridgeFetchWaitMaxMS:     50,
		BridgeWorkersMin:         2,
		FastWriterFetchWaitMaxMS: 50,
		FastWriterWorkersMin:     4,
		HistoryWorkersMin:        3,
		RealtimeWorkersMin:       3,
		StatWorkersMin:           3,
		IdentityWorkersMin:       3,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsMissingRuntimeConfigMetrics(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge":      `vehicle_service_info{service="bridge-test"} 1`,
		"fast-writer": `vehicle_service_info{service="fast-writer-test"} 1`,
		"history": `vehicle_service_info{service="history-test"} 1
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1`,
		"realtime": `vehicle_service_info{service="realtime-test"} 1
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1`,
		"stat": `vehicle_service_info{service="stat-test"} 1
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`,
		"identity": `vehicle_service_info{service="identity-test"} 1
vehicle_kafka_consumer_info{service="vehicle-identity-writer",group="go-identity-writer",topic="vehicle.raw.go.jt808.v1"} 1`,
	}, Thresholds{
		BridgeFetchWaitMaxMS:     50,
		BridgeWorkersMin:         2,
		FastWriterFetchWaitMaxMS: 50,
		FastWriterWorkersMin:     4,
		HistoryWorkersMin:        3,
		RealtimeWorkersMin:       3,
		StatWorkersMin:           3,
		IdentityWorkersMin:       3,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"bridge fetch wait config metric missing (vehicle_bridge_config setting=fetch_wait_ms)",
		"bridge workers config metric missing (vehicle_bridge_config setting=workers)",
		"fast writer fetch wait config metric missing (vehicle_fast_writer_config setting=fetch_wait_ms)",
		"fast writer workers config metric missing (vehicle_fast_writer_config setting=workers)",
		"history workers config metric missing (vehicle_history_config setting=workers)",
		"realtime workers config metric missing (vehicle_realtime_config setting=workers)",
		"stat workers config metric missing (vehicle_stat_config setting=workers)",
		"identity workers config metric missing (vehicle_identity_writer_config setting=workers)",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateCanDisableRuntimeConfigMetricRequirements(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4`,
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8`,
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1`,
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1`,
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`,
	}, Thresholds{
		BridgeFetchWaitMaxMS:     0,
		BridgeWorkersMin:         0,
		FastWriterFetchWaitMaxMS: 0,
		FastWriterWorkersMin:     0,
		HistoryWorkersMin:        0,
		RealtimeWorkersMin:       0,
		StatWorkersMin:           0,
		IdentityWorkersMin:       0,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsAsyncSinkQueueUsageOverLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_async_sink_queue_depth{sink="nats",queue="raw"} 90
vehicle_async_sink_queue_capacity{sink="nats",queue="raw"} 100
vehicle_async_sink_queue_depth{sink="nats",queue="derived"} 10
vehicle_async_sink_queue_capacity{sink="nats",queue="derived"} 100`,
	}, Thresholds{AsyncSinkQueueMaxUsageRatio: 0.80})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway async sink queue usage 0.90 exceeds 0.80",
		"queue=raw",
		"sink=nats",
		"depth 90",
		"capacity 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
	if strings.Contains(joined, "queue=derived") {
		t.Fatalf("derived queue should not be reported:\n%s", joined)
	}
}

func TestEvaluateAllowsAsyncSinkQueueUsageUnderLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_async_sink_queue_depth{sink="kafka",queue="raw"} 79
vehicle_async_sink_queue_capacity{sink="kafka",queue="raw"} 100`,
	}, Thresholds{AsyncSinkQueueMaxUsageRatio: 0.80})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateCanDisableAsyncSinkQueueUsageLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_async_sink_queue_depth{sink="nats",queue="derived"} 100
vehicle_async_sink_queue_capacity{sink="nats",queue="derived"} 100`,
	}, Thresholds{AsyncSinkQueueMaxUsageRatio: 0})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDurableSpoolBacklogAndErrors(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_durable_spool_backlog_files{name="nats"} 10001
vehicle_durable_spool_oldest_age_seconds{name="nats"} 601
vehicle_durable_spool_records_total{kind="raw",name="nats",status="error"} 2
vehicle_durable_spool_replay_total{name="nats",status="publish_error"} 1`,
	}, Thresholds{
		DurableSpoolBacklogMax:      10000,
		DurableSpoolOldestAgeMaxSec: 300,
		DurableSpoolErrorMax:        0,
		DurableSpoolReplayErrorMax:  0,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"durable spool backlog 10001 exceeds 10000 for name nats",
		"durable spool oldest age 601s exceeds 300s for name nats",
		"durable spool record errors 2 exceeds 0 for name nats kind raw",
		"durable spool replay publish_error 1 exceeds 0 for name nats",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateCanDisableDurableSpoolLimits(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_durable_spool_backlog_files{name="kafka"} 999999
vehicle_durable_spool_oldest_age_seconds{name="kafka"} 999999
vehicle_durable_spool_records_total{kind="fields",name="kafka",status="error"} 99
vehicle_durable_spool_replay_total{name="kafka",status="delete_error"} 99`,
	}, Thresholds{
		DurableSpoolBacklogMax:      0,
		DurableSpoolOldestAgeMaxSec: 0,
		DurableSpoolErrorMax:        -1,
		DurableSpoolReplayErrorMax:  -1,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDurableOutboxInflightAndErrors(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_durable_outbox_inflight{name="nats-outbox"} 10001
vehicle_durable_outbox_publish_total{kind="raw",name="nats-outbox",status="submitted"} 50000
vehicle_durable_outbox_publish_total{kind="raw",name="nats-outbox",status="acked"} 49998
vehicle_durable_outbox_publish_total{kind="raw",name="nats-outbox",status="ack_error"} 2
vehicle_durable_outbox_publish_total{kind="all",name="nats-outbox",status="close_timeout"} 1`,
	}, Thresholds{
		DurableOutboxInflightMax: 10000,
		DurableOutboxErrorMax:    0,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"durable outbox inflight 10001 exceeds 10000 for name nats-outbox",
		"durable outbox ack_error 2 exceeds 0 for name nats-outbox kind raw",
		"durable outbox close_timeout 1 exceeds 0 for name nats-outbox kind all",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
	if strings.Contains(joined, "submitted") || strings.Contains(joined, "acked") {
		t.Fatalf("successful outbox statuses must not create findings:\n%s", joined)
	}
}

func TestEvaluateCanDisableDurableOutboxLimits(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_durable_outbox_inflight{name="nats-outbox"} 999999
vehicle_durable_outbox_publish_total{kind="raw",name="nats-outbox",status="submit_error"} 99`,
	}, Thresholds{
		DurableOutboxInflightMax: 0,
		DurableOutboxErrorMax:    -1,
	})

	if report.Status != StatusOK || len(report.Findings) != 0 {
		t.Fatalf("report = %#v, want ok without findings", report)
	}
}

func TestEvaluateCanDisableStatBatchPendingLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_batch_pending_messages 999999`,
	}, Thresholds{StatBatchPendingMax: 0})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateCanDisableRealtimeBatchPendingLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_batch_pending_messages 999999`,
	}, Thresholds{RealtimeBatchPendingMax: 0})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsBridgeRouteErrors(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_messages_total{status="route_error",subject="vehicle.unconfigured.v1"} 1`,
	}, Thresholds{BridgeRouteErrorMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "bridge route errors 1 exceeds 0 for subject vehicle.unconfigured.v1") {
		t.Fatalf("route error finding missing in:\n%s", joined)
	}
}

func TestEvaluateCanDisableBridgeRouteErrorLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_messages_total{status="route_error",subject="vehicle.unconfigured.v1"} 999`,
	}, Thresholds{BridgeRouteErrorMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsBridgeKafkaWriteErrors(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_writes_total{status="error",topic="vehicle.fields.go.jt808.v1"} 2`,
	}, Thresholds{BridgeKafkaWriteErrorMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "bridge kafka write errors 2 exceeds 0 for topic vehicle.fields.go.jt808.v1") {
		t.Fatalf("kafka write error finding missing in:\n%s", joined)
	}
}

func TestEvaluateCanDisableBridgeKafkaWriteErrorLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_writes_total{status="error",topic="vehicle.raw.go.gb32960.v1"} 99`,
	}, Thresholds{BridgeKafkaWriteErrorMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsBridgeUnackedMessageRatio(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 1000
vehicle_bridge_nats_acks_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 700
vehicle_bridge_nats_acks_total{status="dropped_route_error",subject="vehicle.raw.go.jt808.v1"} 50`,
	}, Thresholds{
		BridgeMessageMinReceived:     100,
		BridgeMessageMaxUnackedRatio: 0.20,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"bridge unacked message ratio 0.25 exceeds 0.20 for subject vehicle.raw.go.jt808.v1",
		"received 1000",
		"ack ok 700",
		"dropped route error 50",
		"unacked 250",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateAllowsBridgeCompletedAndLowSampleMessages(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_messages_total{status="received",subject="vehicle.raw.go.gb32960.v1"} 100
vehicle_bridge_nats_acks_total{status="ok",subject="vehicle.raw.go.gb32960.v1"} 80
vehicle_bridge_nats_acks_total{status="dropped_route_error",subject="vehicle.raw.go.gb32960.v1"} 20
vehicle_bridge_messages_total{status="received",subject="vehicle.raw.go.yutong-mqtt.v1"} 99`,
	}, Thresholds{
		BridgeMessageMinReceived:     100,
		BridgeMessageMaxUnackedRatio: 0.20,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsKafkaConsumerUncommittedMessageRatio(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_kafka_messages_total{status="received",topic="vehicle.fields.go.jt808.v1"} 1000
vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 700
vehicle_stat_kafka_commits_total{status="error",topic="vehicle.fields.go.jt808.v1"} 10`,
	}, Thresholds{
		KafkaConsumerMessageMinReceived:  100,
		KafkaConsumerMaxUncommittedRatio: 0.20,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat kafka consumer uncommitted message ratio 0.30 exceeds 0.20 for topic vehicle.fields.go.jt808.v1",
		"received 1000",
		"commit ok 700",
		"commit errors 10",
		"uncommitted 300",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsKafkaConsumerCommitErrors(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.raw.go.gb32960.v1"} 3
vehicle_realtime_kafka_commits_total{status="error",topic="vehicle.raw.go.gb32960.v1"} 1`,
	}, Thresholds{
		KafkaConsumerCommitErrorMax:      0,
		KafkaConsumerMessageMinReceived:  100,
		KafkaConsumerMaxUncommittedRatio: 0.20,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "realtime kafka consumer commit errors 1 exceeds 0 for topic vehicle.raw.go.gb32960.v1") {
		t.Fatalf("commit error finding missing in:\n%s", joined)
	}
	if strings.Contains(joined, "uncommitted message ratio") {
		t.Fatalf("low-sample uncommitted ratio should not be enforced:\n%s", joined)
	}
}

func TestEvaluateCanDisableKafkaConsumerCommitErrorLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.raw.go.gb32960.v1"} 3
vehicle_realtime_kafka_commits_total{status="error",topic="vehicle.raw.go.gb32960.v1"} 1`,
	}, Thresholds{
		KafkaConsumerCommitErrorMax:      -1,
		KafkaConsumerMessageMinReceived:  100,
		KafkaConsumerMaxUncommittedRatio: 0.20,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateAllowsKafkaConsumerCompletedAndLowSampleMessages(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_history_kafka_messages_total{status="received",topic="vehicle.raw.go.gb32960.v1"} 100
vehicle_history_kafka_commits_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 80
vehicle_history_kafka_messages_total{status="received",topic="vehicle.raw.go.yutong-mqtt.v1"} 99`,
	}, Thresholds{
		KafkaConsumerMessageMinReceived:  100,
		KafkaConsumerMaxUncommittedRatio: 0.20,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsFastWriterInvalidJSON(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 1`,
	}, Thresholds{FastWriterInvalidJSONMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "fast writer invalid json 1 exceeds 0 for subject vehicle.raw.go.jt808.v1") {
		t.Fatalf("invalid json finding missing in:\n%s", joined)
	}
}

func TestEvaluateCanDisableFastWriterInvalidJSONLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 99`,
	}, Thresholds{FastWriterInvalidJSONMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsFastWriterBatchFallbackErrors(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_batch_fallback_total{stage="redis",status="error"} 2`,
	}, Thresholds{FastWriterFallbackErrorMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "fast writer batch fallback errors 2 exceeds 0 for stage redis") {
		t.Fatalf("fallback finding missing in:\n%s", joined)
	}
}

func TestEvaluateCanDisableFastWriterBatchFallbackErrorLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="error"} 99`,
	}, Thresholds{FastWriterFallbackErrorMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsFastWriterDecoupledUpdateErrors(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="ok"} 5
vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="error"} 1`,
	}, Thresholds{FastWriterDecoupledUpdateErrorMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "fast writer decoupled update errors 1 exceeds 0 for reason tdengine_transient") {
		t.Fatalf("decoupled update finding missing in:\n%s", joined)
	}
}

func TestEvaluateCanDisableFastWriterDecoupledUpdateErrorLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="error"} 99`,
	}, Thresholds{FastWriterDecoupledUpdateErrorMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsHistoryInvalidJSON(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.yutong-mqtt.v1"} 1
vehicle_history_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.jt808.v1"} 1`,
	}, Thresholds{HistoryInvalidJSONMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "history invalid json 1 exceeds 0 for topic vehicle.raw.go.jt808.v1") {
		t.Fatalf("invalid json finding missing in:\n%s", joined)
	}
}

func TestEvaluateCanDisableHistoryInvalidJSONLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.yutong-mqtt.v1"} 1
vehicle_history_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.jt808.v1"} 99`,
	}, Thresholds{HistoryInvalidJSONMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsHistoryLocationWriteErrors(t *testing.T) {
	recentError := time.Now().Add(-time.Minute).Unix()
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 2
vehicle_history_last_location_write_unix_seconds{status="error",topic="vehicle.raw.go.jt808.v1"} ` + strconv.FormatInt(recentError, 10),
	}, Thresholds{HistoryLocationErrorMax: 0, HistoryLocationErrorRecentSec: 300})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"history location writes error 2 exceeds 0",
		"vehicle.raw.go.jt808.v1",
		"last error",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateIgnoresRecoveredHistoryLocationWriteErrors(t *testing.T) {
	staleError := time.Now().Add(-10 * time.Minute).Unix()
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 2
vehicle_history_last_location_write_unix_seconds{status="error",topic="vehicle.raw.go.jt808.v1"} ` + strconv.FormatInt(staleError, 10),
	}, Thresholds{HistoryLocationErrorMax: 0, HistoryLocationErrorRecentSec: 300})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok after the error window recovered: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none after the error window recovered", report.Findings)
	}
}

func TestEvaluateKeepsHistoryLocationErrorWhenLastActivityIsUnavailable(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 2`,
	}, Thresholds{HistoryLocationErrorMax: 0, HistoryLocationErrorRecentSec: 300})

	if report.Status != StatusDegraded || !strings.Contains(strings.Join(report.Findings, "\n"), "last error activity unavailable") {
		t.Fatalf("missing timestamp must remain degraded, got %#v", report)
	}
}

func TestEvaluateCanDisableHistoryLocationWriteErrorLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 99`,
	}, Thresholds{HistoryLocationErrorMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsHistoryLocationAccountingMismatch(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_history_location_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 70
vehicle_history_location_writes_total{status="skipped_non_realtime",topic="vehicle.raw.go.jt808.v1"} 10`,
	}, Thresholds{HistoryLocationAccountingMaxRatio: 0.02})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"history location accounting mismatch ratio 0.20 exceeds 0.02",
		"history writes ok 100, location statuses 80, diff 20",
		"vehicle.raw.go.jt808.v1",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateAcceptsHistoryLocationAccountingWhenStatusesMatchWrites(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_history_location_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 70
vehicle_history_location_writes_total{status="skipped_non_realtime",topic="vehicle.raw.go.jt808.v1"} 20
vehicle_history_location_writes_total{status="skipped_missing_coordinates",topic="vehicle.raw.go.jt808.v1"} 10`,
	}, Thresholds{HistoryLocationAccountingMaxRatio: 0.02})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateCanDisableHistoryLocationAccountingLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_history_location_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`,
	}, Thresholds{HistoryLocationAccountingMaxRatio: 0})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsHistoryBatchFallbackErrors(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_batch_fallback_total{status="error"} 2`,
	}, Thresholds{HistoryFallbackErrorMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "history batch fallback errors 2 exceeds 0") {
		t.Fatalf("fallback finding missing in:\n%s", joined)
	}
}

func TestEvaluateCanDisableHistoryBatchFallbackErrorLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_batch_fallback_total{status="error"} 99`,
	}, Thresholds{HistoryFallbackErrorMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsWriteRetriesOverLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_write_retries_total{operation="batch",status="retry"} 11
vehicle_history_write_retries_total{operation="batch",status="exhausted"} 1`,
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_write_retries_total{operation="single",status="retry"} 12
vehicle_stat_write_retries_total{operation="single",status="exhausted"} 1`,
	}, Thresholds{
		HistoryRetryMax:          10,
		HistoryRetryExhaustedMax: 0,
		StatRetryMax:             10,
		StatRetryExhaustedMax:    0,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"history write retries 11 exceeds 10 for operation batch",
		"history write retry exhausted 1 exceeds 0 for operation batch",
		"stat write retries 12 exceeds 10 for operation single",
		"stat write retry exhausted 1 exceeds 0 for operation single",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateCanDisableWriteRetryLimits(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_write_retries_total{operation="batch",status="retry"} 99
vehicle_history_write_retries_total{operation="batch",status="exhausted"} 99`,
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_write_retries_total{operation="single",status="retry"} 99
vehicle_stat_write_retries_total{operation="single",status="exhausted"} 99`,
	}, Thresholds{
		HistoryRetryMax:          -1,
		HistoryRetryExhaustedMax: -1,
		StatRetryMax:             -1,
		StatRetryExhaustedMax:    -1,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsRealtimeInvalidJSON(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.yutong-mqtt.v1"} 1
vehicle_realtime_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.gb32960.v1"} 1`,
	}, Thresholds{RealtimeInvalidJSONMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "realtime invalid json 1 exceeds 0 for topic vehicle.raw.go.gb32960.v1") {
		t.Fatalf("invalid json finding missing in:\n%s", joined)
	}
}

func TestEvaluateCanDisableRealtimeInvalidJSONLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.yutong-mqtt.v1"} 1
vehicle_realtime_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.gb32960.v1"} 99`,
	}, Thresholds{RealtimeInvalidJSONMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsStatInvalidJSON(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.yutong-mqtt.v1"} 1
vehicle_stat_kafka_messages_total{status="invalid_json",topic="vehicle.fields.go.jt808.v1"} 1`,
	}, Thresholds{StatInvalidJSONMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "stat invalid json 1 exceeds 0 for topic vehicle.fields.go.jt808.v1") {
		t.Fatalf("invalid json finding missing in:\n%s", joined)
	}
}

func TestEvaluateCanDisableStatInvalidJSONLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.yutong-mqtt.v1"} 1
vehicle_stat_kafka_messages_total{status="invalid_json",topic="vehicle.fields.go.jt808.v1"} 99`,
	}, Thresholds{StatInvalidJSONMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsStatProtocolTopicMismatches(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_stat_kafka_messages_total{status="protocol_topic_mismatch",topic="vehicle.fields.go.gb32960.v1"} 1`,
	}, Thresholds{StatProtocolTopicMismatchMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "stat protocol topic mismatches 1 exceeds 0 for topic vehicle.fields.go.gb32960.v1") {
		t.Fatalf("protocol topic mismatch finding missing in:\n%s", joined)
	}
}

func TestEvaluateReportsMessageContractMismatches(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_kafka_messages_total{status="event_kind_mismatch",topic="vehicle.raw.go.jt808.v1"} 1`,
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_realtime_kafka_messages_total{status="protocol_topic_mismatch",topic="vehicle.raw.go.gb32960.v1"} 1`,
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_messages_total{status="event_kind_mismatch",subject="vehicle.raw.go.yutong-mqtt.v1"} 1`,
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_kafka_messages_total{status="event_kind_mismatch",topic="vehicle.fields.go.jt808.v1"} 1`,
	}, Thresholds{MessageContractMismatchMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"history message contract event_kind_mismatch 1 exceeds 0 for topic vehicle.raw.go.jt808.v1",
		"realtime message contract protocol_topic_mismatch 1 exceeds 0 for topic vehicle.raw.go.gb32960.v1",
		"fast writer message contract event_kind_mismatch 1 exceeds 0 for subject vehicle.raw.go.yutong-mqtt.v1",
		"stat message contract event_kind_mismatch 1 exceeds 0 for topic vehicle.fields.go.jt808.v1",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateCanDisableMessageContractMismatchLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_kafka_messages_total{status="event_kind_mismatch",topic="vehicle.raw.go.jt808.v1"} 99`,
	}, Thresholds{MessageContractMismatchMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateCanDisableStatProtocolTopicMismatchLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_stat_kafka_messages_total{status="protocol_topic_mismatch",topic="vehicle.fields.go.gb32960.v1"} 1`,
	}, Thresholds{StatProtocolTopicMismatchMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsStatMetricProtocolTopicMismatches(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.jt808.v1"} 12
vehicle_stat_sources_total{protocol="YUTONG_MQTT",status="written",topic="vehicle.fields.go.gb32960.v1"} 7
vehicle_stat_projections_total{protocol="JT808",status="written",topic="vehicle.fields.go.yutong-mqtt.v1"} 5`,
	}, Thresholds{StatProtocolTopicMismatchMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat metric protocol topic mismatches 12 exceeds 0 for vehicle_stat_samples_total topic vehicle.fields.go.jt808.v1: expected JT808, got GB32960 (status found)",
		"stat metric protocol topic mismatches 7 exceeds 0 for vehicle_stat_sources_total topic vehicle.fields.go.gb32960.v1: expected GB32960, got YUTONG_MQTT (status written)",
		"stat metric protocol topic mismatches 5 exceeds 0 for vehicle_stat_projections_total topic vehicle.fields.go.yutong-mqtt.v1: expected YUTONG_MQTT, got JT808 (status written)",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsStatMetricProtocolLabelMissing(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_samples_total{status="found",topic="vehicle.fields.go.jt808.v1"} 12`,
	}, Thresholds{StatProtocolTopicMismatchMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "stat metric protocol label missing 12 exceeds 0 for vehicle_stat_samples_total topic vehicle.fields.go.jt808.v1 (status found)") {
		t.Fatalf("protocol label missing finding missing in:\n%s", joined)
	}
}

func TestEvaluateReportsDegradedWhenServiceInfoMetricIsMissing(t *testing.T) {
	report := Evaluate(map[string]string{
		"realtime": ``,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "realtime service info metric missing") {
		t.Fatalf("service info finding missing in:\n%s", joined)
	}
}

func TestEvaluateAggregatesLastActivityTimestampsByMax(t *testing.T) {
	report := Evaluate(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_last_publish_unix_seconds{kind="raw",protocol="GB32960",status="ok"} 100
vehicle_gateway_last_publish_unix_seconds{kind="fields",protocol="GB32960",status="ok"} 120
vehicle_gateway_publish_total{kind="raw",protocol="GB32960",status="ok"} 3
vehicle_gateway_publish_total{kind="fields",protocol="GB32960",status="ok"} 4`,
	})

	if got := report.Services[0].Metrics["vehicle_gateway_last_publish_unix_seconds"]; got != 120 {
		t.Fatalf("last publish aggregate = %v, want max timestamp 120", got)
	}
	if got := report.Services[0].Metrics["vehicle_gateway_last_publish_age_seconds"]; got <= 0 {
		t.Fatalf("last publish age = %v, want positive age", got)
	}
	if got := report.Services[0].Metrics["vehicle_gateway_publish_total"]; got != 7 {
		t.Fatalf("publish total aggregate = %v, want sum 7", got)
	}
}

func TestEvaluateReportsDegradedWhenObservedLastActivityIsStale(t *testing.T) {
	stale := time.Now().UTC().Add(-10 * time.Minute).Unix()
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_last_publish_unix_seconds{kind="raw",protocol="JT808",status="ok"} ` + strconv.FormatInt(stale, 10),
	}, Thresholds{LastActivityStaleSec: 300})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway last publish stale",
		"exceeds 300s",
		"kind=raw,protocol=JT808,status=ok",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotDegradeForFreshOrErrorLastActivity(t *testing.T) {
	fresh := time.Now().UTC().Unix()
	staleError := time.Now().UTC().Add(-10 * time.Minute).Unix()
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_last_publish_unix_seconds{kind="raw",protocol="JT808",status="ok"} ` + strconv.FormatInt(fresh, 10) + `
vehicle_gateway_last_publish_unix_seconds{kind="raw",protocol="JT808",status="error"} ` + strconv.FormatInt(staleError, 10),
	}, Thresholds{LastActivityStaleSec: 300})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenFastWriterLastActivityIsStale(t *testing.T) {
	stale := time.Now().UTC().Add(-10 * time.Minute).Unix()
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_last_message_unix_seconds{status="received",subject="vehicle.raw.go.jt808.v1"} ` + strconv.FormatInt(stale, 10) + `
vehicle_fast_writer_last_stage_unix_seconds{stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} ` + strconv.FormatInt(stale, 10),
	}, Thresholds{LastActivityStaleSec: 300})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"fast writer last message stale",
		"fast writer last stage stale",
		"subject=vehicle.raw.go.jt808.v1",
		"stage=redis",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateSkipsRollupLastActivitySubjects(t *testing.T) {
	stale := time.Now().UTC().Add(-10 * time.Minute).Unix()
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_last_stage_unix_seconds{stage="redis",status="ok",subject="mixed"} ` + strconv.FormatInt(stale, 10) + `
vehicle_fast_writer_last_stage_unix_seconds{stage="redis",status="ok",subject="unknown"} ` + strconv.FormatInt(stale, 10),
	}, Thresholds{LastActivityStaleSec: 300})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsRealtimeStoreLastActivityStale(t *testing.T) {
	stale := time.Now().UTC().Add(-10 * time.Minute).Unix()
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_realtime_last_store_update_unix_seconds{protocol="JT808",status="ok",store="mysql"} ` + strconv.FormatInt(stale, 10),
	}, Thresholds{LastActivityStaleSec: 300})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"realtime last store update stale",
		"exceeds 300s",
		"protocol=JT808,status=ok,store=mysql",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsDegradedWhenKafkaConsumerInfoIsMissing(t *testing.T) {
	report := Evaluate(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1`,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "stat kafka consumer info metric missing") {
		t.Fatalf("kafka consumer info finding missing in:\n%s", joined)
	}
}

func TestEvaluateDoesNotDegradeForSmallKafkaLag(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_kafka_lag{topic="vehicle.fields.go.jt808.v1",partition="0"} 8`,
	}, Thresholds{KafkaLagMax: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
	if report.Totals.KafkaLag != 8 {
		t.Fatalf("kafka lag total = %v, want 8", report.Totals.KafkaLag)
	}
}

func TestEvaluateReportsDegradedWhenRealtimeRedisProjectorIsEnabled(t *testing.T) {
	report := Evaluate(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_redis_projector_enabled 1
vehicle_realtime_kafka_lag{topic="vehicle.raw.go.jt808.v1",partition="0"} 0`,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "realtime redis projector enabled") {
		t.Fatalf("realtime redis projector finding missing in:\n%s", joined)
	}
}

func TestEvaluateReportsDegradedWhenGatewayRawHasNoFieldsPublish(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 120`,
	}, Thresholds{GatewayFieldMinRaw: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway fields missing for protocol JT808",
		"raw ok 120",
		"realtime candidates 120",
		"fields ok 0",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotRequireFieldsForNonRealtimeGatewayRaw(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="GB32960",status="ok"} 120
vehicle_gateway_fields_total{protocol="GB32960",status="skipped_non_realtime"} 120`,
	}, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMaxMissingRatio: 0.2})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateAcceptsFieldsDelegatedToBridge(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 120
vehicle_gateway_publish_total{kind="fields",protocol="JT808",status="delegated"} 120
vehicle_gateway_fields_total{protocol="JT808",status="delegated_to_bridge"} 120
vehicle_gateway_fields_count{protocol="JT808",status="delegated_to_bridge"} 31`,
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 120
vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 120`,
	}, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMaxMissingRatio: 0.2, GatewayFieldMinCount: 1, GatewayBridgeMinPublish: 100})

	if report.Status != StatusOK || len(report.Findings) != 0 {
		t.Fatalf("report = %#v, want delegated fields flow ok", report)
	}
	if report.GatewayFields == nil || len(report.GatewayFields.Protocols) != 1 {
		t.Fatalf("gateway fields diagnostics = %#v", report.GatewayFields)
	}
	row := report.GatewayFields.Protocols[0]
	if row.FieldsOK != 120 || row.FieldsPublished != 0 || row.FieldsDelegated != 120 {
		t.Fatalf("delegated fields diagnostics = %#v", row)
	}
}

func TestEvaluateReportsDegradedWhenGatewayFieldsMissingRatioIsHigh(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 200
vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="ok"} 130
vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="published"} 130
vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="skipped_non_realtime"} 20
vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="skipped_missing_fields"} 50`,
	}, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMaxMissingRatio: 0.2})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway fields missing ratio 0.28 exceeds 0.20 for protocol YUTONG_MQTT",
		"realtime candidates 180",
		"skipped missing fields 50",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotDegradeWhenGatewayPublishesSomeFields(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 120
vehicle_gateway_publish_total{kind="fields",protocol="JT808",status="ok"} 1`,
	}, Thresholds{GatewayFieldMinRaw: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateIncludesGatewayFieldsDiagnostics(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 200
vehicle_gateway_publish_total{kind="fields",protocol="JT808",status="ok"} 170
vehicle_gateway_fields_total{protocol="JT808",status="published"} 170
vehicle_gateway_fields_total{protocol="JT808",status="skipped_non_realtime"} 20
vehicle_gateway_fields_total{protocol="JT808",status="skipped_missing_fields"} 10
vehicle_gateway_fields_count{protocol="JT808",status="published"} 31`,
	}, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMaxMissingRatio: 0.2, GatewayFieldMinCount: 1})

	if report.GatewayFields == nil {
		t.Fatal("gateway fields diagnostics = nil")
	}
	if got, want := len(report.GatewayFields.Protocols), 1; got != want {
		t.Fatalf("gateway fields protocols = %d, want %d: %#v", got, want, report.GatewayFields.Protocols)
	}
	row := report.GatewayFields.Protocols[0]
	if row.Protocol != "JT808" || row.Status != "ok" || row.Reason != "" {
		t.Fatalf("gateway fields row identity/status = %#v", row)
	}
	if row.RawOK != 200 || row.RealtimeCandidates != 180 || row.FieldsOK != 170 || row.FieldsPublished != 170 {
		t.Fatalf("gateway fields counters = %#v", row)
	}
	if row.FieldsSkippedNonRealtime != 20 || row.FieldsSkippedMissing != 10 || row.FieldsPublishError != 0 {
		t.Fatalf("gateway fields skip counters = %#v", row)
	}
	if math.Abs(row.MissingRatio-(10.0/180.0)) > 0.0001 {
		t.Fatalf("missing ratio = %v, want %v", row.MissingRatio, 10.0/180.0)
	}
	if row.LatestFieldCount == nil || *row.LatestFieldCount != 31 {
		t.Fatalf("latest field count = %#v, want 31", row.LatestFieldCount)
	}
}

func TestEvaluateIncludesProtocolFlowDiagnostics(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 200
vehicle_gateway_publish_total{kind="fields",protocol="JT808",status="ok"} 170
vehicle_gateway_fields_total{protocol="JT808",status="published"} 170
vehicle_gateway_fields_total{protocol="JT808",status="skipped_non_realtime"} 20
vehicle_gateway_fields_count{protocol="JT808",status="published"} 31
vehicle_gateway_identity_skips_total{protocol="JT808",reason="non_vehicle_frame"} 7
vehicle_gateway_identity_total{protocol="JT808",status="resolved"} 160
vehicle_gateway_identity_total{protocol="JT808",status="unresolved"} 40`,
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 170
vehicle_stat_samples_total{protocol="JT808",status="found",topic="vehicle.fields.go.jt808.v1"} 160
vehicle_stat_samples_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 140
vehicle_stat_samples_total{protocol="JT808",status="skipped_missing_mileage",topic="vehicle.fields.go.jt808.v1"} 10
vehicle_stat_sources_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_sources_total{protocol="JT808",status="skipped_throttled",topic="vehicle.fields.go.jt808.v1"} 159
vehicle_stat_projections_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_projections_total{protocol="JT808",status="skipped_throttled",topic="vehicle.fields.go.jt808.v1"} 139`,
	}, Thresholds{
		GatewayFieldMinRaw:                100,
		GatewayFieldMaxMissingRatio:       0.2,
		GatewayFieldMinCount:              1,
		GatewayIdentityMinSamples:         100,
		GatewayIdentityMaxUnresolvedRatio: 0.25,
		StatSampleMinWrites:               100,
		StatSampleMaxActionableSkipRatio:  0.20,
		StatSourceMinSamples:              100,
		StatSourceMaxMissingRatio:         0.60,
		StatProjectionMinSampleWrites:     100,
	})

	if report.ProtocolFlow == nil {
		t.Fatal("protocol flow diagnostics = nil")
	}
	if got, want := len(report.ProtocolFlow.Protocols), 1; got != want {
		t.Fatalf("protocol flow rows = %d, want %d: %#v", got, want, report.ProtocolFlow.Protocols)
	}
	row := report.ProtocolFlow.Protocols[0]
	if row.Protocol != "JT808" || row.Status != "ok" || row.Reason != "" {
		t.Fatalf("protocol flow row identity/status = %#v", row)
	}
	if row.RawOK != 200 || row.RealtimeCandidates != 180 || row.FieldsPublished != 170 {
		t.Fatalf("gateway counters = %#v", row)
	}
	if row.LatestFieldCount == nil || *row.LatestFieldCount != 31 {
		t.Fatalf("latest field count = %#v, want 31", row.LatestFieldCount)
	}
	if row.IdentityTotal != 200 || row.IdentitySkipped != 7 || row.IdentityResolved != 160 || row.IdentityNonResolved != 40 {
		t.Fatalf("identity counters = %#v", row)
	}
	if math.Abs(row.IdentityNonResolvedRatio-0.2) > 0.0001 {
		t.Fatalf("identity ratio = %v, want 0.2", row.IdentityNonResolvedRatio)
	}
	if row.StatWritesOK != 170 || row.StatMileageCandidateWrites != 170 || row.StatSamplesFound != 160 || row.StatSamplesWritten != 140 {
		t.Fatalf("stat sample counters = %#v", row)
	}
	if row.StatSamplesActionableSkipped != 10 || row.StatSkippedMissingMileage != 10 {
		t.Fatalf("stat skip counters = %#v", row)
	}
	if row.StatSourceWritten != 1 || row.StatSourceThrottled != 159 {
		t.Fatalf("stat source counters = %#v", row)
	}
	if row.StatProjectionWritten != 1 || row.StatProjectionThrottled != 139 {
		t.Fatalf("stat projection counters = %#v", row)
	}
}

func TestProtocolFlowDiagnosticsHighlightsStatProjectionBreak(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 120
vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.gb32960.v1"} 120
vehicle_stat_samples_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 120
vehicle_stat_sources_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_stat_sources_total{protocol="GB32960",status="skipped_throttled",topic="vehicle.fields.go.gb32960.v1"} 119
vehicle_stat_projections_total{protocol="GB32960",status="attempted",topic="vehicle.fields.go.gb32960.v1"} 120`,
	}, Thresholds{
		StatSampleMinWrites:           100,
		StatSourceMinSamples:          100,
		StatProjectionMinSampleWrites: 100,
	})

	if report.ProtocolFlow == nil || len(report.ProtocolFlow.Protocols) != 1 {
		t.Fatalf("protocol flow diagnostics missing: %#v", report.ProtocolFlow)
	}
	row := report.ProtocolFlow.Protocols[0]
	if row.Protocol != "GB32960" || row.Status != "degraded" || row.Reason != "stat_projection_missing" {
		t.Fatalf("protocol flow row = %#v", row)
	}
	if row.StatProjectionAttempted != 120 || row.StatProjectionWritten != 0 {
		t.Fatalf("projection counters = %#v", row)
	}
}

func TestEvaluateReportsDegradedWhenGatewayPublishedFieldsAreEmpty(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 120
vehicle_gateway_publish_total{kind="fields",protocol="JT808",status="ok"} 120
vehicle_gateway_fields_total{protocol="JT808",status="published"} 120
vehicle_gateway_fields_count{protocol="JT808",status="published"} 0`,
	}, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMinCount: 1})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway fields count 0 below 1 for protocol JT808",
		"published fields events",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
	if report.GatewayFields == nil || len(report.GatewayFields.Protocols) != 1 {
		t.Fatalf("gateway fields diagnostics missing: %#v", report.GatewayFields)
	}
	row := report.GatewayFields.Protocols[0]
	if row.Status != "degraded" || row.Reason != "low_field_count" {
		t.Fatalf("gateway fields diagnostic row = %#v", row)
	}
}

func TestEvaluateReportsDegradedWhenGatewayFieldsCountMetricIsMissing(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="GB32960",status="ok"} 120
vehicle_gateway_publish_total{kind="fields",protocol="GB32960",status="ok"} 120
vehicle_gateway_fields_total{protocol="GB32960",status="published"} 120`,
	}, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMinCount: 1})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "gateway fields count metric missing for protocol GB32960 after 120 published fields events") {
		t.Fatalf("finding missing in:\n%s", joined)
	}
}

func TestEvaluateSkipsGatewayFieldCountWhenPublishedSamplesAreLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 2
vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="ok"} 2
vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="published"} 2
vehicle_gateway_fields_count{protocol="YUTONG_MQTT",status="published"} 0`,
	}, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMinCount: 1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenGatewayRawPublishDoesNotReachBridge(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 120`,
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4`,
	}, Thresholds{GatewayBridgeMinPublish: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"bridge writes missing for gateway raw publish protocol JT808",
		"gateway ok 120",
		"expected topic vehicle.raw.go.jt808.v1",
		"bridge writes ok 0",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsDegradedWhenGatewayFieldsPublishDoesNotReachBridge(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="fields",protocol="GB32960",status="ok"} 120`,
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4`,
	}, Thresholds{GatewayBridgeMinPublish: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"bridge writes missing for gateway fields publish protocol GB32960",
		"expected topic vehicle.fields.go.gb32960.v1",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotDegradeWhenGatewayPublishReachesBridge(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 120
vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="ok"} 120`,
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.yutong-mqtt.v1"} 1
vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.fields.go.yutong-mqtt.v1"} 1`,
	}, Thresholds{GatewayBridgeMinPublish: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateSkipsGatewayBridgeFlowWhenSamplesAreLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 99`,
	}, Thresholds{GatewayBridgeMinPublish: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenGatewayPublishTopicMappingIsMissing(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="UNKNOWN_PROTO",status="ok"} 120`,
	}, Thresholds{GatewayBridgeMinPublish: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "gateway raw publish topic mapping missing for protocol UNKNOWN_PROTO") {
		t.Fatalf("mapping finding missing in:\n%s", joined)
	}
}

func TestEvaluateReportsDegradedWhenGatewayRawPublishDoesNotReachFastWriter(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="GB32960",status="ok"} 120`,
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8`,
	}, Thresholds{GatewayFastWriterMinRawPublish: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"fast writer received messages missing for gateway raw publish protocol GB32960",
		"gateway ok 120",
		"expected subject vehicle.raw.go.gb32960.v1",
		"fast writer received 0",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotDegradeWhenGatewayRawPublishReachesFastWriter(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 120`,
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 1`,
	}, Thresholds{GatewayFastWriterMinRawPublish: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsFastWriterUnackedMessageRatio(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.gb32960.v1"} 1000
vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.gb32960.v1"} 700
vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.gb32960.v1"} 50`,
	}, Thresholds{
		FastWriterMessageMinReceived:     100,
		FastWriterMessageMaxUnackedRatio: 0.20,
		FastWriterInvalidJSONMax:         -1,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"fast writer unacked message ratio 0.25 exceeds 0.20 for subject vehicle.raw.go.gb32960.v1",
		"received 1000",
		"ok 700",
		"invalid_json 50",
		"unacked 250",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateAllowsFastWriterCompletedAndLowSampleMessages(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 100
vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 90
vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 10
vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.yutong-mqtt.v1"} 99`,
	}, Thresholds{
		FastWriterMessageMinReceived:     100,
		FastWriterMessageMaxUnackedRatio: 0.20,
		FastWriterInvalidJSONMax:         -1,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateDoesNotRequireFastWriterForGatewayFieldsPublish(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="ok"} 120`,
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8`,
	}, Thresholds{GatewayFastWriterMinRawPublish: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenFastWriterRedisStaleFieldRatioIsHigh(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.gb32960.v1",status="seen"} 2000
vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.gb32960.v1",status="written"} 1500
vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.gb32960.v1",status="skipped_stale"} 500`,
	}, Thresholds{
		FastWriterRedisFieldMinSeen:       1000,
		FastWriterRedisFieldMaxStaleRatio: 0.20,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"fast writer redis stale field ratio 0.25 exceeds 0.20 for subject vehicle.raw.go.gb32960.v1",
		"seen 2000",
		"written 1500",
		"skipped_stale 500",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsDegradedWhenFastWriterRedisEnvelopeUpdatesAreMissing(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.gb32960.v1",status="seen"} 120
vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.gb32960.v1",status="skipped_non_realtime"} 120`,
	}, Thresholds{
		FastWriterRedisEnvelopeMinSeen:     100,
		FastWriterRedisEnvelopeMaxBadRatio: 0.50,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"fast writer redis realtime updates missing for subject vehicle.raw.go.gb32960.v1",
		"envelopes seen 120",
		"updated 0",
		"skipped non-realtime 120",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsDegradedWhenFastWriterRedisEnvelopeIdentitySkipsAreHigh(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.jt808.v1",status="seen"} 200
vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.jt808.v1",status="updated"} 70
vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.jt808.v1",status="skipped_missing_vin"} 130`,
	}, Thresholds{
		FastWriterRedisEnvelopeMinSeen:     100,
		FastWriterRedisEnvelopeMaxBadRatio: 0.50,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"fast writer redis envelope skip ratio 0.65 exceeds 0.50 for subject vehicle.raw.go.jt808.v1",
		"envelopes seen 200",
		"updated 70",
		"missing vin 130",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsDegradedWhenFastWriterRedisParsedFieldSkipsAreHigh(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.gb32960.v1",status="seen"} 200
vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.gb32960.v1",status="updated"} 70
vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.gb32960.v1",status="skipped_missing_fields"} 130`,
	}, Thresholds{
		FastWriterRedisEnvelopeMinSeen:     100,
		FastWriterRedisEnvelopeMaxBadRatio: 0.50,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"fast writer redis envelope skip ratio 0.65 exceeds 0.50 for subject vehicle.raw.go.gb32960.v1",
		"missing parsed fields 130",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotDegradeWhenFastWriterRedisEnvelopeSkipsAreLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.yutong-mqtt.v1",status="seen"} 200
vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.yutong-mqtt.v1",status="updated"} 180
vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.yutong-mqtt.v1",status="skipped_missing_vin"} 20`,
	}, Thresholds{
		FastWriterRedisEnvelopeMinSeen:     100,
		FastWriterRedisEnvelopeMaxBadRatio: 0.50,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateDoesNotDegradeWhenFastWriterRedisStaleFieldRatioIsLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.jt808.v1",status="seen"} 2000
vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.jt808.v1",status="written"} 1950
vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.jt808.v1",status="skipped_stale"} 50`,
	}, Thresholds{
		FastWriterRedisFieldMinSeen:       1000,
		FastWriterRedisFieldMaxStaleRatio: 0.20,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateSkipsFastWriterRedisStaleFieldRatioWhenSamplesAreLowOrRollup(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.yutong-mqtt.v1",status="seen"} 999
vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.yutong-mqtt.v1",status="written"} 1
vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.yutong-mqtt.v1",status="skipped_stale"} 998
vehicle_fast_writer_redis_fields_total{subject="mixed",status="seen"} 2000
vehicle_fast_writer_redis_fields_total{subject="mixed",status="written"} 1
vehicle_fast_writer_redis_fields_total{subject="mixed",status="skipped_stale"} 1999`,
	}, Thresholds{
		FastWriterRedisFieldMinSeen:       1000,
		FastWriterRedisFieldMaxStaleRatio: 0.20,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateSkipsGatewayFastWriterFlowWhenSamplesAreLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 99`,
	}, Thresholds{GatewayFastWriterMinRawPublish: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenHistoryParsedFieldsAreMissing(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_history_parsed_fields_total{protocol="GB32960",status="present",topic="vehicle.raw.go.gb32960.v1"} 90
vehicle_history_parsed_fields_total{protocol="GB32960",status="missing",topic="vehicle.raw.go.gb32960.v1"} 10`,
	}, Thresholds{
		HistoryParsedFieldMinFrames:       100,
		HistoryParsedFieldMaxMissingRatio: 0.05,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"history parsed fields missing ratio 0.10 exceeds 0.05 for topic vehicle.raw.go.gb32960.v1 protocol GB32960",
		"total 100",
		"present 90",
		"missing 10",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotDegradeWhenHistoryParsedFieldMissingRatioIsLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_parsed_fields_total{protocol="JT808",status="present",topic="vehicle.raw.go.jt808.v1"} 98
vehicle_history_parsed_fields_total{protocol="JT808",status="missing",topic="vehicle.raw.go.jt808.v1"} 2`,
	}, Thresholds{
		HistoryParsedFieldMinFrames:       100,
		HistoryParsedFieldMaxMissingRatio: 0.05,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateSkipsHistoryParsedFieldCheckWhenSamplesAreLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.yutong-mqtt.v1"} 1
vehicle_history_parsed_fields_total{protocol="YUTONG_MQTT",status="present",topic="vehicle.raw.go.yutong-mqtt.v1"} 1
vehicle_history_parsed_fields_total{protocol="YUTONG_MQTT",status="missing",topic="vehicle.raw.go.yutong-mqtt.v1"} 98`,
	}, Thresholds{
		HistoryParsedFieldMinFrames:       100,
		HistoryParsedFieldMaxMissingRatio: 0.05,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenGatewayParseBadRatioIsHigh(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_frames_total{protocol="JT808",status="OK"} 90
vehicle_gateway_frames_total{protocol="JT808",status="PARTIAL"} 5
vehicle_gateway_frames_total{protocol="JT808",status="BAD_FRAME"} 10`,
	}, Thresholds{
		GatewayParseMinFrames:   100,
		GatewayParseMaxBadRatio: 0.05,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway parse bad ratio 0.14 exceeds 0.05 for protocol JT808",
		"total 105",
		"ok 90",
		"bad 15",
		"partial 5",
		"bad_frame 10",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotDegradeWhenGatewayParseBadRatioIsLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_frames_total{protocol="GB32960",status="OK"} 99
vehicle_gateway_frames_total{protocol="GB32960",status="BAD_FRAME"} 1`,
	}, Thresholds{
		GatewayParseMinFrames:   100,
		GatewayParseMaxBadRatio: 0.05,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateSkipsGatewayParseQualityWhenSampleCountIsLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="OK"} 1
vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="BAD_FRAME"} 9`,
	}, Thresholds{
		GatewayParseMinFrames:   100,
		GatewayParseMaxBadRatio: 0.05,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenGatewayResponseErrorRatioIsHigh(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_response_total{message_id="0x0100",protocol="JT808",status="ok"} 90
vehicle_gateway_response_total{message_id="0x0100",protocol="JT808",status="build_error"} 5
vehicle_gateway_response_total{message_id="0x0200",protocol="JT808",status="write_error"} 10
vehicle_gateway_response_total{message_id="0x0001",protocol="JT808",status="skipped"} 1000`,
	}, Thresholds{
		GatewayResponseMinSamples:    100,
		GatewayResponseMaxErrorRatio: 0.05,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway response error ratio 0.14 exceeds 0.05 for protocol JT808",
		"total 105",
		"ok 90",
		"errors 15",
		"build_error 5",
		"write_error 10",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotDegradeWhenGatewayResponseErrorRatioIsLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_response_total{message_id="0x05",protocol="GB32960",status="ok"} 99
vehicle_gateway_response_total{message_id="0x05",protocol="GB32960",status="write_error"} 1`,
	}, Thresholds{
		GatewayResponseMinSamples:    100,
		GatewayResponseMaxErrorRatio: 0.05,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateSkipsGatewayResponseQualityWhenSampleCountIsLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_response_total{message_id="0x0200",protocol="JT808",status="write_error"} 9
vehicle_gateway_response_total{message_id="0x0001",protocol="JT808",status="skipped"} 1000`,
	}, Thresholds{
		GatewayResponseMinSamples:    100,
		GatewayResponseMaxErrorRatio: 0.05,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenGatewayIdentityUnresolvedRatioIsHigh(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_total{protocol="JT808",status="resolved"} 60
vehicle_gateway_identity_total{protocol="JT808",status="unresolved"} 30
vehicle_gateway_identity_total{protocol="JT808",status="timeout"} 10`,
	}, Thresholds{
		GatewayIdentityMinSamples:         100,
		GatewayIdentityMaxUnresolvedRatio: 0.20,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway identity unresolved ratio 0.40 exceeds 0.20 for protocol JT808",
		"total 100",
		"unresolved 40",
		"resolved 60",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateIncludesIdentityResolutionDiagnostics(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_total{protocol="JT808",status="resolved"} 60
vehicle_gateway_identity_total{protocol="JT808",status="unresolved"} 30
vehicle_gateway_identity_total{protocol="JT808",status="timeout"} 7
vehicle_gateway_identity_total{protocol="JT808",status="error"} 3
vehicle_gateway_identity_issues_total{protocol="JT808",status="unresolved",message_id="0x0200",reason="no_binding"} 30
vehicle_gateway_identity_issues_total{protocol="JT808",status="timeout",message_id="0x0200",reason="timeout"} 7
vehicle_gateway_identity_issues_total{protocol="JT808",status="error",message_id="0x0100",reason="resolver_error"} 3
vehicle_gateway_identity_cache_total{protocol="JT808",cache_status="hit"} 80
vehicle_gateway_identity_cache_total{protocol="JT808",cache_status="stale"} 5
vehicle_gateway_identity_cache_entries{cache="lookup"} 123
vehicle_gateway_identity_cache_entries{cache="registration"} 45
vehicle_gateway_identity_skips_total{protocol="GB32960",reason="non_vehicle_frame"} 12
vehicle_gateway_identity_total{protocol="GB32960",status="resolved"} 100`,
	}, Thresholds{})

	if report.IdentityResolution == nil {
		t.Fatal("identity resolution diagnostics = nil")
	}
	got := report.IdentityResolution
	if got.Total != 200 || got.ResolvedCount != 160 || got.NonResolvedCount != 40 {
		t.Fatalf("identity totals = %#v", got)
	}
	if got.SkippedCount != 12 {
		t.Fatalf("identity skipped = %d, want 12", got.SkippedCount)
	}
	if got.UnresolvedCount != 30 || got.TimeoutCount != 7 || got.ErrorCount != 3 {
		t.Fatalf("identity error split = %#v", got)
	}
	if len(got.Issues) != 3 {
		t.Fatalf("identity issues = %#v, want 3 rows", got.Issues)
	}
	if got.Issues[0] != (IdentityIssueCount{Protocol: "JT808", Status: "error", MessageID: "0x0100", Reason: "resolver_error", Count: 3}) {
		t.Fatalf("first identity issue = %#v", got.Issues[0])
	}
	if got.Issues[1] != (IdentityIssueCount{Protocol: "JT808", Status: "timeout", MessageID: "0x0200", Reason: "timeout", Count: 7}) {
		t.Fatalf("second identity issue = %#v", got.Issues[1])
	}
	if got.Issues[2] != (IdentityIssueCount{Protocol: "JT808", Status: "unresolved", MessageID: "0x0200", Reason: "no_binding", Count: 30}) {
		t.Fatalf("third identity issue = %#v", got.Issues[2])
	}
	if got.CacheEntries["lookup"] != 123 || got.CacheEntries["registration"] != 45 {
		t.Fatalf("cache entries = %#v", got.CacheEntries)
	}
	if len(got.Protocols) != 2 || got.Protocols[0].Protocol != "GB32960" || got.Protocols[1].Protocol != "JT808" {
		t.Fatalf("protocol rows = %#v", got.Protocols)
	}
	if got.Protocols[0].SkippedCount != 12 || got.Protocols[0].NonResolvedCount != 0 {
		t.Fatalf("gb32960 skipped row = %#v", got.Protocols[0])
	}
	jt808 := got.Protocols[1]
	if jt808.Total != 100 || jt808.NonResolvedCount != 40 || jt808.NonResolvedRatio != 0.40 {
		t.Fatalf("jt808 row = %#v", jt808)
	}
	if jt808.StaleCacheCount != 5 || jt808.StaleCacheRatio != 0.05 || jt808.CacheStatusCounts["stale"] != 5 {
		t.Fatalf("jt808 cache row = %#v", jt808)
	}
}

func TestEvaluateDoesNotDegradeWhenGatewayIdentityUnresolvedRatioIsLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_total{protocol="GB32960",status="resolved"} 98
vehicle_gateway_identity_total{protocol="GB32960",status="unresolved"} 2`,
	}, Thresholds{
		GatewayIdentityMinSamples:         100,
		GatewayIdentityMaxUnresolvedRatio: 0.20,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateSkipsGatewayIdentityQualityWhenSampleCountIsLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_total{protocol="YUTONG_MQTT",status="resolved"} 2
vehicle_gateway_identity_total{protocol="YUTONG_MQTT",status="unresolved"} 8`,
	}, Thresholds{
		GatewayIdentityMinSamples:         100,
		GatewayIdentityMaxUnresolvedRatio: 0.20,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsGatewayIdentityCacheEntriesOverLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_cache_entries{cache="lookup"} 300001`,
	}, Thresholds{GatewayIdentityCacheEntriesMax: 300000})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "gateway identity cache entries 300001 exceeds 300000 for cache lookup") {
		t.Fatalf("finding missing in:\n%s", joined)
	}
}

func TestEvaluateReportsGatewayIdentitySnapshotNotReady(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_snapshot_ready 0`,
	}, Thresholds{
		GatewayIdentitySnapshotRequired: 1,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "gateway identity snapshot ready 0 below 1") {
		t.Fatalf("finding missing in:\n%s", joined)
	}
}

func TestEvaluateReportsGatewayIdentitySnapshotTooOld(t *testing.T) {
	lastSuccess := time.Now().Add(-10 * time.Minute).Unix()
	report := EvaluateWithThresholds(map[string]string{
		"gateway": fmt.Sprintf(`vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_snapshot_ready 1
vehicle_gateway_identity_snapshot_last_success_unix_seconds %d`, lastSuccess),
	}, Thresholds{
		GatewayIdentitySnapshotRequired:  1,
		GatewayIdentitySnapshotMaxAgeSec: 180,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "gateway identity snapshot age") || !strings.Contains(joined, "exceeds 180s") {
		t.Fatalf("finding missing in:\n%s", joined)
	}
}

func TestEvaluateAllowsFreshGatewayIdentitySnapshot(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": fmt.Sprintf(`vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_snapshot_ready 1
vehicle_gateway_identity_snapshot_last_success_unix_seconds %d`, time.Now().Unix()),
	}, Thresholds{
		GatewayIdentitySnapshotRequired:  1,
		GatewayIdentitySnapshotMaxAgeSec: 180,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
}

func TestEvaluateReportsGatewayIdentityLocationTouchFailures(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_cache_entries{cache="location_touch_failure"} 2`,
	}, Thresholds{GatewayIdentityLocationTouchFailureMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "gateway identity location touch failure cache 2 exceeds 0") {
		t.Fatalf("finding missing in:\n%s", joined)
	}
	if !strings.Contains(joined, "jt808 registration writes are failing or in protective backoff") {
		t.Fatalf("finding should explain registration write backoff in:\n%s", joined)
	}
}

func TestEvaluateReportsGatewayRegistrationWriteQueueDepth(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_cache_entries{cache="lookup"} 500000
vehicle_gateway_identity_cache_entries{cache="registration_write_queue"} 12001
vehicle_gateway_identity_cache_max_entries{cache="registration_write_queue"} 100000`,
	}, Thresholds{
		GatewayIdentityRegistrationWriteQueueDepthMax:      10000,
		GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0.80,
		GatewayIdentityCacheEntriesMax:                     0,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	if report.Totals.JT808RegistrationWriteQueuePending != 12001 {
		t.Fatalf("registration write queue pending = %v, want 12001", report.Totals.JT808RegistrationWriteQueuePending)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "gateway jt808 registration write queue depth 12001 exceeds 10000") {
		t.Fatalf("depth finding missing in:\n%s", joined)
	}
}

func TestEvaluateReportsGatewayRegistrationWriteQueueUsage(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_cache_entries{cache="registration_write_queue"} 90
vehicle_gateway_identity_cache_max_entries{cache="registration_write_queue"} 100`,
	}, Thresholds{
		GatewayIdentityRegistrationWriteQueueDepthMax:      0,
		GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0.80,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway jt808 registration write queue usage 0.90 exceeds 0.80",
		"depth 90",
		"capacity 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateCanDisableGatewayRegistrationWriteQueueLimits(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_cache_entries{cache="registration_write_queue"} 100
vehicle_gateway_identity_cache_max_entries{cache="registration_write_queue"} 100`,
	}, Thresholds{
		GatewayIdentityRegistrationWriteQueueDepthMax:      0,
		GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if report.Totals.JT808RegistrationWriteQueuePending != 100 {
		t.Fatalf("registration write queue pending = %v, want 100", report.Totals.JT808RegistrationWriteQueuePending)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsGatewayRegistrationWriteErrors(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_jt808_registration_write_total{mode="async_enqueue",status="ok"} 100
vehicle_gateway_jt808_registration_write_total{mode="async_background",status="error"} 2
vehicle_gateway_jt808_registration_write_total{mode="sync",status="error"} 1`,
	}, Thresholds{
		GatewayIdentityRegistrationWriteErrorMax:           0,
		GatewayIdentityRegistrationWriteQueueDepthMax:      0,
		GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	if report.Totals.JT808RegistrationWriteErrors != 3 {
		t.Fatalf("registration write errors = %v, want 3", report.Totals.JT808RegistrationWriteErrors)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway jt808 registration write error count 2 exceeds 0 for mode async_background",
		"gateway jt808 registration write error count 1 exceeds 0 for mode sync",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
	if strings.Contains(joined, "async_enqueue") {
		t.Fatalf("ok status should not produce finding:\n%s", joined)
	}
}

func TestEvaluateCanDisableGatewayRegistrationWriteErrorLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_jt808_registration_write_total{mode="async_background",status="error"} 2`,
	}, Thresholds{
		GatewayIdentityRegistrationWriteErrorMax:           -1,
		GatewayIdentityRegistrationWriteQueueDepthMax:      0,
		GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if report.Totals.JT808RegistrationWriteErrors != 2 {
		t.Fatalf("registration write errors = %v, want 2", report.Totals.JT808RegistrationWriteErrors)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestGatewayIdentityCacheLimitIgnoresRegistrationWriteQueue(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_cache_entries{cache="registration_write_queue"} 500000`,
	}, Thresholds{
		GatewayIdentityCacheEntriesMax:                     300000,
		GatewayIdentityRegistrationWriteQueueDepthMax:      0,
		GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateCanDisableGatewayIdentityLocationTouchFailureLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_cache_entries{cache="location_touch_failure"} 99`,
	}, Thresholds{GatewayIdentityLocationTouchFailureMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateAllowsGatewayIdentityCacheEntriesUnderLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_cache_entries{cache="registration"} 299999`,
	}, Thresholds{GatewayIdentityCacheEntriesMax: 300000})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateCanDisableGatewayIdentityCacheEntryLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_cache_entries{cache="lookup"} 9999999`,
	}, Thresholds{GatewayIdentityCacheEntriesMax: 0})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsGatewayIdentityStaleCacheRatioOverLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_total{protocol="JT808",status="resolved"} 90
vehicle_gateway_identity_total{protocol="JT808",status="unresolved"} 10
vehicle_gateway_identity_cache_total{protocol="JT808",cache_status="stale"} 9`,
	}, Thresholds{
		GatewayIdentityStaleMinSamples:    100,
		GatewayIdentityMaxStaleCacheRatio: 0.05,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "gateway identity stale cache ratio 0.09 exceeds 0.05 for protocol JT808") {
		t.Fatalf("finding missing in:\n%s", joined)
	}
}

func TestEvaluateAllowsGatewayIdentityStaleCacheRatioUnderLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_total{protocol="GB32960",status="resolved"} 100
vehicle_gateway_identity_cache_total{protocol="GB32960",cache_status="stale"} 3`,
	}, Thresholds{
		GatewayIdentityStaleMinSamples:    100,
		GatewayIdentityMaxStaleCacheRatio: 0.05,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateCanDisableGatewayIdentityStaleCacheRatio(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_total{protocol="JT808",status="resolved"} 100
vehicle_gateway_identity_cache_total{protocol="JT808",cache_status="stale"} 100`,
	}, Thresholds{
		GatewayIdentityStaleMinSamples:    0,
		GatewayIdentityMaxStaleCacheRatio: 0,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenRequiredConsumerTopicIsMissing(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1`,
	}, Thresholds{
		RequiredConsumerTopics: map[string][]string{
			"history": {
				"vehicle.raw.go.gb32960.v1",
				"vehicle.raw.go.jt808.v1",
			},
		},
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "history required kafka consumer topic missing: vehicle.raw.go.jt808.v1") {
		t.Fatalf("required topic finding missing in:\n%s", joined)
	}
}

func TestEvaluateDoesNotDegradeWhenRequiredConsumerTopicsArePresent(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`,
	}, Thresholds{
		RequiredConsumerTopics: map[string][]string{
			"stat": {
				"vehicle.fields.go.gb32960.v1",
				"vehicle.fields.go.jt808.v1",
			},
		},
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestDefaultRequiredConsumerTopicsIncludesIdentityWriter(t *testing.T) {
	required := DefaultRequiredConsumerTopics()
	if got := strings.Join(required["identity"], ","); got != "vehicle.raw.go.jt808.v1" {
		t.Fatalf("identity required topics = %q, want JT808 raw", got)
	}
}

func TestEvaluateRejectsDuplicateJT808RegistrationWriters(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_jt808_registration_gateway_writes_enabled 1`,
		"identity": `vehicle_service_info{service="vehicle-identity-writer"} 1
vehicle_kafka_consumer_info{service="vehicle-identity-writer",group="go-identity-writer",topic="vehicle.raw.go.jt808.v1"} 1`,
	}, Thresholds{})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	if joined := strings.Join(report.Findings, "\n"); !strings.Contains(joined, "duplicate writers") {
		t.Fatalf("duplicate writer finding missing in:\n%s", joined)
	}
}

func TestEvaluateAcceptsDelegatedJT808RegistrationWriter(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_jt808_registration_gateway_writes_enabled 0`,
		"identity": `vehicle_service_info{service="vehicle-identity-writer"} 1
vehicle_kafka_consumer_info{service="vehicle-identity-writer",group="go-identity-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_identity_writer_batch_pending_messages 0
vehicle_identity_writer_batch_pending_facts 0
vehicle_identity_writer_kafka_lag{topic="vehicle.raw.go.jt808.v1",partition="0"} 0`,
	}, Thresholds{})

	if report.Status != StatusOK || len(report.Findings) != 0 {
		t.Fatalf("report = %#v, want ok without findings", report)
	}
}

func TestEvaluateReportsDegradedWhenRawFanoutDoesNotReachHistoryOrRealtime(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 120`,
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1`,
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1`,
	}, Thresholds{RawFanoutMinBridge: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"history messages missing for vehicle.raw.go.jt808.v1",
		"realtime messages missing for vehicle.raw.go.jt808.v1",
		"bridge raw writes ok 120",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsDegradedWhenRawFanoutIsConsumedButNotWritten(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 120`,
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_history_kafka_messages_total{status="received",topic="vehicle.raw.go.gb32960.v1"} 120`,
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.raw.go.gb32960.v1"} 120`,
	}, Thresholds{RawFanoutMinBridge: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"history writes missing for vehicle.raw.go.gb32960.v1",
		"realtime updates missing for vehicle.raw.go.gb32960.v1",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotDegradeWhenRawFanoutIsComplete(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.yutong-mqtt.v1"} 120`,
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.yutong-mqtt.v1"} 1
vehicle_history_kafka_messages_total{status="received",topic="vehicle.raw.go.yutong-mqtt.v1"} 120
vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.yutong-mqtt.v1"} 120`,
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.yutong-mqtt.v1"} 1
vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.raw.go.yutong-mqtt.v1"} 120
vehicle_realtime_updates_total{status="ok",topic="vehicle.raw.go.yutong-mqtt.v1"} 120`,
	}, Thresholds{RawFanoutMinBridge: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateSkipsRawFanoutWhenBridgeSamplesAreLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 99`,
	}, Thresholds{RawFanoutMinBridge: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenBridgeFieldsTopicIsNotConsumedByStat(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 120`,
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`,
	}, Thresholds{StatTopicMinBridge: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat messages missing for vehicle.fields.go.jt808.v1",
		"bridge writes ok 120",
		"stat received 0",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsDegradedWhenBridgeFieldsTopicIsNotConfiguredByStat(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 120`,
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`,
	}, Thresholds{StatTopicMinBridge: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat consumer topic missing for vehicle.fields.go.gb32960.v1",
		"bridge writes ok 120",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsDegradedWhenStatWritesHaveNoExtractedSamples(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 120
vehicle_stat_samples_total{protocol="JT808",status="skipped_missing_fields",topic="vehicle.fields.go.jt808.v1"} 120`,
	}, Thresholds{StatSampleMinWrites: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat samples missing for vehicle.fields.go.jt808.v1",
		"writes ok 120",
		"missing fields 120",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotDegradeWhenStatWritesAreOnlyNonMileageFrames(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.yutong-mqtt.v1"} 1
vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.yutong-mqtt.v1"} 120
vehicle_stat_samples_total{protocol="YUTONG_MQTT",status="skipped_non_mileage_frame",topic="vehicle.fields.go.yutong-mqtt.v1"} 120`,
	}, Thresholds{StatSampleMinWrites: 100, StatSampleMaxActionableSkipRatio: 0.20})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
	if report.ProtocolFlow == nil || len(report.ProtocolFlow.Protocols) != 1 {
		t.Fatalf("protocol flow diagnostics missing: %#v", report.ProtocolFlow)
	}
	row := report.ProtocolFlow.Protocols[0]
	if row.Protocol != "YUTONG_MQTT" || row.Status != "ok" || row.StatMileageCandidateWrites != 0 || row.StatSkippedNonMileageFrame != 120 || row.StatSamplesActionableSkipped != 0 {
		t.Fatalf("protocol flow row = %#v", row)
	}
}

func TestEvaluateDoesNotDegradeWhenStatWritesOnlyMissSourceMileage(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.yutong-mqtt.v1"} 1
vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.yutong-mqtt.v1"} 120
vehicle_stat_samples_total{protocol="YUTONG_MQTT",status="skipped_missing_mileage",topic="vehicle.fields.go.yutong-mqtt.v1"} 120`,
	}, Thresholds{StatSampleMinWrites: 100, StatSampleMaxActionableSkipRatio: 0.20})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
	if report.ProtocolFlow == nil || len(report.ProtocolFlow.Protocols) != 1 {
		t.Fatalf("protocol flow diagnostics missing: %#v", report.ProtocolFlow)
	}
	row := report.ProtocolFlow.Protocols[0]
	if row.Protocol != "YUTONG_MQTT" || row.Status != "ok" || row.StatMileageCandidateWrites != 120 || row.StatSkippedMissingMileage != 120 || row.StatSamplesActionableSkipped != 120 {
		t.Fatalf("protocol flow row = %#v", row)
	}
}

func TestEvaluateUsesMileageCandidatesForStatActionableSkipRatio(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 1000
vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.gb32960.v1"} 50
vehicle_stat_samples_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 40
vehicle_stat_samples_total{protocol="GB32960",status="skipped_non_mileage_frame",topic="vehicle.fields.go.gb32960.v1"} 900
vehicle_stat_samples_total{protocol="GB32960",status="skipped_missing_mileage",topic="vehicle.fields.go.gb32960.v1"} 50`,
	}, Thresholds{
		StatSampleMinWrites:              100,
		StatSampleMaxActionableSkipRatio: 0.40,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat actionable skip ratio 0.50 exceeds 0.40 for vehicle.fields.go.gb32960.v1",
		"writes ok 1000",
		"mileage candidates 100",
		"non-mileage frames 900",
		"missing mileage 50",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
	if report.ProtocolFlow == nil || len(report.ProtocolFlow.Protocols) != 1 {
		t.Fatalf("protocol flow diagnostics missing: %#v", report.ProtocolFlow)
	}
	row := report.ProtocolFlow.Protocols[0]
	if row.StatMileageCandidateWrites != 100 || row.StatSamplesActionableSkipped != 50 || row.Status != "degraded" || row.Reason != "stat_actionable_skip_ratio" {
		t.Fatalf("protocol flow row = %#v", row)
	}
}

func TestEvaluateDoesNotDegradeWhenStatSamplesAreOnlyDuplicateMileage(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.yutong-mqtt.v1"} 1
vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.yutong-mqtt.v1"} 120
vehicle_stat_samples_total{protocol="YUTONG_MQTT",status="found",topic="vehicle.fields.go.yutong-mqtt.v1"} 120
vehicle_stat_samples_total{protocol="YUTONG_MQTT",status="skipped_same_mileage",topic="vehicle.fields.go.yutong-mqtt.v1"} 120`,
	}, Thresholds{StatSampleMinWrites: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenStatActionableSkipRatioIsHigh(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 200
vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.gb32960.v1"} 80
vehicle_stat_samples_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 70
vehicle_stat_samples_total{protocol="GB32960",status="skipped_missing_mileage",topic="vehicle.fields.go.gb32960.v1"} 100
vehicle_stat_samples_total{protocol="GB32960",status="skipped_non_positive_mileage",topic="vehicle.fields.go.gb32960.v1"} 20`,
	}, Thresholds{
		StatSampleMinWrites:              100,
		StatSampleMaxActionableSkipRatio: 0.50,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat actionable skip ratio 0.60 exceeds 0.50 for vehicle.fields.go.gb32960.v1",
		"writes ok 200",
		"missing mileage 100",
		"non-positive mileage 20",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateTreatsMissingTimeAsActionableStatSkip(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 200
vehicle_stat_samples_total{protocol="JT808",status="found",topic="vehicle.fields.go.jt808.v1"} 80
vehicle_stat_samples_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 70
vehicle_stat_samples_total{protocol="JT808",status="skipped_missing_time",topic="vehicle.fields.go.jt808.v1"} 120`,
	}, Thresholds{
		StatSampleMinWrites:              100,
		StatSampleMaxActionableSkipRatio: 0.50,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat actionable skip ratio 0.60 exceeds 0.50 for vehicle.fields.go.jt808.v1",
		"missing time 120",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsFutureEventTimeAdjustments(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 200
vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.gb32960.v1"} 190
vehicle_stat_samples_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 180
vehicle_stat_samples_total{protocol="GB32960",status="event_time_future_adjusted",topic="vehicle.fields.go.gb32960.v1"} 3`,
	}, Thresholds{
		StatSampleMinWrites:           100,
		StatSampleFutureAdjustmentMax: 0,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "stat future event time adjustments 3 exceeds 0 for vehicle.fields.go.gb32960.v1") {
		t.Fatalf("future adjustment finding missing in:\n%s", joined)
	}
	if report.ProtocolFlow == nil || len(report.ProtocolFlow.Protocols) != 1 {
		t.Fatalf("protocol flow diagnostics missing: %#v", report.ProtocolFlow)
	}
	row := report.ProtocolFlow.Protocols[0]
	if row.Protocol != "GB32960" || row.StatEventTimeFutureAdjusted != 3 || row.Status != "degraded" || row.Reason != "stat_future_event_time_adjusted" {
		t.Fatalf("protocol flow row = %#v", row)
	}
}

func TestEvaluateDoesNotDegradeWhenStatActionableSkipRatioIsLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 200
vehicle_stat_samples_total{protocol="JT808",status="found",topic="vehicle.fields.go.jt808.v1"} 180
vehicle_stat_samples_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 150
vehicle_stat_samples_total{protocol="JT808",status="skipped_missing_mileage",topic="vehicle.fields.go.jt808.v1"} 20`,
	}, Thresholds{
		StatSampleMinWrites:              100,
		StatSampleMaxActionableSkipRatio: 0.50,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenStatSamplesHaveNoSource(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 120
vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.gb32960.v1"} 120
vehicle_stat_samples_total{protocol="GB32960",status="skipped_missing_source",topic="vehicle.fields.go.gb32960.v1"} 120`,
	}, Thresholds{StatSampleMinWrites: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat samples not written for vehicle.fields.go.gb32960.v1",
		"found 120",
		"skipped missing source 120",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsDegradedWhenStatSourceTrackingIsMissing(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_stat_samples_total{protocol="JT808",status="found",topic="vehicle.fields.go.jt808.v1"} 120`,
	}, Thresholds{StatSourceMinSamples: 100, StatSourceMaxMissingRatio: 0.60})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat source tracking missing for vehicle.fields.go.jt808.v1",
		"samples found 120",
		"source written 0",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsDegradedWhenStatSourceMissingEndpointRatioIsHigh(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_stat_samples_total{protocol="YUTONG_MQTT",status="found",topic="vehicle.fields.go.yutong-mqtt.v1"} 200
vehicle_stat_sources_total{protocol="YUTONG_MQTT",status="written",topic="vehicle.fields.go.yutong-mqtt.v1"} 10
vehicle_stat_sources_total{protocol="YUTONG_MQTT",status="skipped_throttled",topic="vehicle.fields.go.yutong-mqtt.v1"} 50
vehicle_stat_sources_total{protocol="YUTONG_MQTT",status="skipped_missing_endpoint",topic="vehicle.fields.go.yutong-mqtt.v1"} 140`,
	}, Thresholds{StatSourceMinSamples: 100, StatSourceMaxMissingRatio: 0.60})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat source missing endpoint ratio 0.70 exceeds 0.60 for vehicle.fields.go.yutong-mqtt.v1",
		"samples found 200",
		"missing endpoint 140",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotDegradeWhenStatSourceTrackingIsHealthy(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.gb32960.v1"} 200
vehicle_stat_sources_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_stat_sources_total{protocol="GB32960",status="skipped_throttled",topic="vehicle.fields.go.gb32960.v1"} 199`,
	}, Thresholds{StatSourceMinSamples: 100, StatSourceMaxMissingRatio: 0.60})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedWhenStatFinalProjectionIsMissing(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_samples_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 120
vehicle_stat_projections_total{protocol="JT808",status="attempted",topic="vehicle.fields.go.jt808.v1"} 120
vehicle_stat_projections_total{protocol="JT808",status="skipped_throttled",topic="vehicle.fields.go.jt808.v1"} 30`,
	}, Thresholds{StatProjectionMinSampleWrites: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat final projection missing for vehicle.fields.go.jt808.v1",
		"samples written 120",
		"projections attempted 120",
		"written 0",
		"skipped throttled 30",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateDoesNotDegradeWhenStatFinalProjectionExists(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_stat_samples_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 120
vehicle_stat_projections_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_stat_projections_total{protocol="GB32960",status="skipped_throttled",topic="vehicle.fields.go.gb32960.v1"} 119`,
	}, Thresholds{StatProjectionMinSampleWrites: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateSkipsStatFinalProjectionCheckWhenSamplesAreLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.yutong-mqtt.v1"} 1
vehicle_stat_samples_total{protocol="YUTONG_MQTT",status="written",topic="vehicle.fields.go.yutong-mqtt.v1"} 99
vehicle_stat_projections_total{protocol="YUTONG_MQTT",status="attempted",topic="vehicle.fields.go.yutong-mqtt.v1"} 99`,
	}, Thresholds{StatProjectionMinSampleWrites: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsStatCacheEntriesOverLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_cache_entries{cache="last_total_mileage"} 1200
vehicle_stat_cache_entries{cache="source_seen"} 12`,
	}, Thresholds{StatCacheEntriesMax: 1000})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat cache entries 1200 exceeds 1000 for cache last_total_mileage",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateAllowsStatCacheEntriesUnderLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_cache_entries{cache="last_total_mileage"} 999
vehicle_stat_cache_entries{cache="source_seen"} 12`,
	}, Thresholds{StatCacheEntriesMax: 1000})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateCanDisableStatCacheEntryLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_cache_entries{cache="baseline"} 9999999`,
	}, Thresholds{StatCacheEntriesMax: 0})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateAllowsRealtimeThrottleWhenSomeWritesPass(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_throttle_total{protocol="JT808",status="passed",store="mysql"} 20
vehicle_realtime_throttle_total{protocol="JT808",status="skipped_interval",store="mysql"} 180`,
	}, Thresholds{RealtimeThrottleMinSamples: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsRealtimeThrottleWhenWritesNeverPass(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_realtime_throttle_total{protocol="GB32960",status="skipped_interval",store="mysql"} 120`,
	}, Thresholds{RealtimeThrottleMinSamples: 100})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"realtime throttle passing updates missing for store=mysql,protocol=GB32960",
		"total 120",
		"passed 0",
		"skipped interval 120",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateSkipsRealtimeThrottleWhenSamplesAreLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.yutong-mqtt.v1"} 1
vehicle_realtime_throttle_total{protocol="YUTONG_MQTT",status="skipped_interval",store="mysql"} 99`,
	}, Thresholds{RealtimeThrottleMinSamples: 100})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsRealtimeThrottleCacheEntriesOverLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_throttle_cache_entries{store="mysql"} 1200`,
	}, Thresholds{RealtimeThrottleCacheEntriesMax: 1000})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "realtime throttle cache entries 1200 exceeds 1000 for store mysql") {
		t.Fatalf("finding missing in:\n%s", joined)
	}
}

func TestEvaluateAllowsRealtimeThrottleCacheEntriesUnderLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_throttle_cache_entries{store="mysql"} 999`,
	}, Thresholds{RealtimeThrottleCacheEntriesMax: 1000})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateCanDisableRealtimeThrottleCacheEntryLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_throttle_cache_entries{store="mysql"} 9999999`,
	}, Thresholds{RealtimeThrottleCacheEntriesMax: 0})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsRealtimeStoreUpdateErrors(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_store_updates_total{protocol="JT808",status="error",store="mysql"} 2`,
	}, Thresholds{RealtimeStoreErrorMax: 0})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "realtime store update errors 2 exceeds 0 for store mysql protocol JT808") {
		t.Fatalf("finding missing in:\n%s", joined)
	}
}

func TestEvaluateCanDisableRealtimeStoreUpdateErrorLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_store_updates_total{protocol="JT808",status="error",store="mysql"} 99`,
	}, Thresholds{RealtimeStoreErrorMax: -1})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsRealtimeAsyncQueueDroppedAndUsage(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_async_queue_total{protocol="JT808",status="dropped",store="mysql"} 1
vehicle_realtime_async_queue_total{protocol="JT808",status="closed",store="mysql"} 1
vehicle_realtime_async_queue_depth{protocol="JT808",store="mysql"} 90
vehicle_realtime_async_queue_capacity{store="mysql"} 100`,
	}, Thresholds{
		RealtimeAsyncQueueDroppedMax:    0,
		RealtimeAsyncQueueMaxUsageRatio: 0.80,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"realtime async queue dropped 1 exceeds 0 for store mysql protocol JT808",
		"realtime async queue closed 1 exceeds 0 for store mysql protocol JT808",
		"realtime async queue usage 0.90 exceeds 0.80",
		"protocol=JT808,store=mysql",
		"depth 90",
		"capacity 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateCanDisableRealtimeAsyncQueueLimits(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_async_queue_total{protocol="JT808",status="dropped",store="mysql"} 99
vehicle_realtime_async_queue_depth{protocol="JT808",store="mysql"} 999
vehicle_realtime_async_queue_capacity{store="mysql"} 100`,
	}, Thresholds{
		RealtimeAsyncQueueDepthMax:      0,
		RealtimeAsyncQueueDroppedMax:    -1,
		RealtimeAsyncQueueMaxUsageRatio: 0,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsRealtimePlateCacheEntriesOverLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_plate_cache_entries 250000`,
	}, Thresholds{RealtimePlateCacheEntriesMax: 200000})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	if !strings.Contains(joined, "realtime plate cache entries 250000 exceeds 200000") {
		t.Fatalf("finding missing in:\n%s", joined)
	}
}

func TestEvaluateAllowsRealtimePlateCacheEntriesUnderLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_plate_cache_entries 199999`,
	}, Thresholds{RealtimePlateCacheEntriesMax: 200000})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateCanDisableRealtimePlateCacheEntryLimit(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_plate_cache_entries 9999999`,
	}, Thresholds{RealtimePlateCacheEntriesMax: 0})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Findings) != 0 {
		t.Fatalf("findings = %#v, want none", report.Findings)
	}
}

func TestEvaluateReportsDegradedForSlowGatewayIdentityP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_identity_duration_ms_histogram_bucket{le="1",protocol="JT808",status="resolved"} 10
vehicle_gateway_identity_duration_ms_histogram_bucket{le="5",protocol="JT808",status="resolved"} 80
vehicle_gateway_identity_duration_ms_histogram_bucket{le="100",protocol="JT808",status="resolved"} 98
vehicle_gateway_identity_duration_ms_histogram_bucket{le="250",protocol="JT808",status="resolved"} 100
vehicle_gateway_identity_duration_ms_histogram_bucket{le="+Inf",protocol="JT808",status="resolved"} 100
vehicle_gateway_identity_duration_ms_histogram_count{protocol="JT808",status="resolved"} 100
vehicle_gateway_identity_duration_ms_histogram_sum{protocol="JT808",status="resolved"} 350`,
	}, Thresholds{
		GatewayIdentityP99MS: 100,
		MinHistogramSamples:  100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway identity p99 250ms exceeds 100ms",
		"protocol=JT808,status=resolved",
		"samples 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateSkipsLatencyFindingsWhenHistogramSamplesAreLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_frame_duration_ms_histogram_bucket{le="250",protocol="GB32960",status="OK"} 1
vehicle_gateway_frame_duration_ms_histogram_bucket{le="+Inf",protocol="GB32960",status="OK"} 1
vehicle_gateway_frame_duration_ms_histogram_count{protocol="GB32960",status="OK"} 1`,
	}, Thresholds{
		GatewayFrameP99MS:   100,
		MinHistogramSamples: 10,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
}

func TestEvaluateReportsSlowFastWriterRedisE2EP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="100",subject="vehicle.raw.go.jt808.v1"} 90
vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="1000",subject="vehicle.raw.go.jt808.v1"} 100
vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="+Inf",subject="vehicle.raw.go.jt808.v1"} 100
vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.jt808.v1"} 100`,
	}, Thresholds{
		FastWriterRedisE2EP99MS: 300,
		MinHistogramSamples:     100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"fast writer redis e2e p99 1000ms exceeds 300ms",
		"subject=vehicle.raw.go.jt808.v1",
		"samples 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsSlowFastWriterRedisRecentE2EP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.jt808.v1"} 180
vehicle_fast_writer_redis_e2e_recent_samples{subject="vehicle.raw.go.jt808.v1"} 100`,
	}, Thresholds{
		FastWriterRedisRecentE2EP99MS: 100,
		MinHistogramSamples:           100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"fast writer redis e2e recent p99 180ms exceeds 100ms",
		"subject=vehicle.raw.go.jt808.v1",
		"samples 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsSlowGatewayResponseRecentP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"gateway": `vehicle_service_info{service="vehicle-gateway"} 1
vehicle_gateway_response_e2e_recent_p99_ms{protocol="JT808"} 180
vehicle_gateway_response_e2e_recent_samples{protocol="JT808"} 100`,
	}, Thresholds{
		GatewayResponseRecentP99MS: 100,
		MinHistogramSamples:        100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"gateway response e2e recent p99 180ms exceeds 100ms",
		"protocol=JT808",
		"samples 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
	if report.PipelineLatency == nil || len(report.PipelineLatency.Items) != 1 {
		t.Fatalf("gateway response pipeline diagnostics missing: %#v", report.PipelineLatency)
	}
	if got := report.PipelineLatency.Items[0].Stage; got != "gateway_protocol_response_e2e" {
		t.Fatalf("pipeline stage = %q", got)
	}
}

func TestEvaluateServiceSummaryAggregatesP99GaugesByMax(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_e2e_recent_p99_ms{topic="vehicle.fields.go.gb32960.v1"} 49
vehicle_bridge_kafka_e2e_recent_p99_ms{topic="vehicle.raw.go.jt808.v1"} 106
vehicle_bridge_kafka_e2e_recent_samples{topic="vehicle.fields.go.gb32960.v1"} 512
vehicle_bridge_kafka_e2e_recent_samples{topic="vehicle.raw.go.jt808.v1"} 512`,
	}, Thresholds{
		BridgeKafkaRecentE2EP99MS: 300,
		MinHistogramSamples:       100,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if len(report.Services) != 1 {
		t.Fatalf("services = %d, want 1", len(report.Services))
	}
	if got, want := report.Services[0].Metrics["vehicle_bridge_kafka_e2e_recent_p99_ms"], 106.0; got != want {
		t.Fatalf("summary p99 = %v, want max %v instead of label sum", got, want)
	}
}

func TestEvaluateServiceSummaryDoesNotCreateP99AgeMetric(t *testing.T) {
	parsed := parseMetricSet(`vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_last_kafka_e2e_unix_seconds{topic="vehicle.raw.go.jt808.v1"} 1000
vehicle_bridge_kafka_e2e_recent_p99_ms{topic="vehicle.raw.go.jt808.v1"} 106
vehicle_bridge_kafka_e2e_recent_samples{topic="vehicle.raw.go.jt808.v1"} 512`)
	addLastActivityAges(parsed.totals, time.Unix(1010, 0).UTC())

	metrics := parsed.totals
	if got, want := metrics["vehicle_bridge_last_kafka_e2e_age_seconds"], 10.0; got != want {
		t.Fatalf("last activity age = %v, want %v", got, want)
	}
	if _, ok := metrics["vehicle_bridge_kafka_e2e_recent_p99_ms_age_seconds"]; ok {
		t.Fatalf("p99 gauge should not produce an age metric: %#v", metrics)
	}
}

func TestEvaluateIncludesPipelineLatencyDiagnostics(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_e2e_recent_p99_ms{topic="vehicle.raw.go.gb32960.v1"} 80
vehicle_bridge_kafka_e2e_recent_samples{topic="vehicle.raw.go.gb32960.v1"} 128`,
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_tdengine_enabled 0
vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.gb32960.v1"} 42
vehicle_fast_writer_redis_e2e_recent_samples{subject="vehicle.raw.go.gb32960.v1"} 128`,
		"history": `vehicle_service_info{service="history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1
vehicle_history_write_e2e_recent_p99_ms{topic="vehicle.raw.go.gb32960.v1"} 1500
vehicle_history_write_e2e_recent_samples{topic="vehicle.raw.go.gb32960.v1"} 128`,
		"stat": `vehicle_service_info{service="stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1
vehicle_stat_write_e2e_recent_p99_ms{topic="vehicle.fields.go.gb32960.v1"} 900
vehicle_stat_write_e2e_recent_samples{topic="vehicle.fields.go.gb32960.v1"} 8`,
	}, Thresholds{
		BridgeKafkaRecentE2EP99MS:     300,
		FastWriterRedisRecentE2EP99MS: 100,
		HistoryWriteRecentE2EP99MS:    1000,
		StatWriteRecentE2EP99MS:       1000,
		MinHistogramSamples:           100,
	})

	if report.PipelineLatency == nil {
		t.Fatal("pipeline latency diagnostics missing")
	}
	if got, want := len(report.PipelineLatency.Items), 4; got != want {
		t.Fatalf("pipeline latency items = %d, want %d", got, want)
	}
	if got, want := report.PipelineLatency.TotalSamples, int64(392); got != want {
		t.Fatalf("total samples = %d, want %d", got, want)
	}
	if got, want := report.PipelineLatency.MaxP99MS, 1500.0; got != want {
		t.Fatalf("max p99 = %v, want %v", got, want)
	}
	statusByStage := map[string]string{}
	for _, item := range report.PipelineLatency.Items {
		statusByStage[item.Stage] = item.Status
	}
	if statusByStage["bridge_kafka_e2e"] != "ok" {
		t.Fatalf("bridge status = %q", statusByStage["bridge_kafka_e2e"])
	}
	if statusByStage["fast_writer_redis_e2e"] != "ok" {
		t.Fatalf("fast writer status = %q", statusByStage["fast_writer_redis_e2e"])
	}
	if statusByStage["history_tdengine_write_e2e"] != "degraded" {
		t.Fatalf("history status = %q", statusByStage["history_tdengine_write_e2e"])
	}
	if statusByStage["stat_mysql_write_e2e"] != "insufficient_samples" {
		t.Fatalf("stat status = %q", statusByStage["stat_mysql_write_e2e"])
	}
}

func TestEvaluateReportsSlowBridgeKafkaWriteP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_write_duration_ms_histogram_bucket{le="100",status="ok",topic="vehicle.raw.go.jt808.v1"} 90
vehicle_bridge_kafka_write_duration_ms_histogram_bucket{le="1000",status="ok",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_bridge_kafka_write_duration_ms_histogram_bucket{le="+Inf",status="ok",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_bridge_kafka_write_duration_ms_histogram_count{status="ok",topic="vehicle.raw.go.jt808.v1"} 100`,
	}, Thresholds{
		BridgeKafkaWriteP99MS: 300,
		MinHistogramSamples:   100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"bridge kafka write p99 1000ms exceeds 300ms",
		"topic=vehicle.raw.go.jt808.v1",
		"samples 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsSlowBridgeKafkaE2EP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_e2e_duration_ms_histogram_bucket{le="100",topic="vehicle.raw.go.jt808.v1"} 90
vehicle_bridge_kafka_e2e_duration_ms_histogram_bucket{le="1000",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_bridge_kafka_e2e_duration_ms_histogram_bucket{le="+Inf",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_bridge_kafka_e2e_duration_ms_histogram_count{topic="vehicle.raw.go.jt808.v1"} 100`,
	}, Thresholds{
		BridgeKafkaE2EP99MS: 300,
		MinHistogramSamples: 100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"bridge kafka e2e p99 1000ms exceeds 300ms",
		"topic=vehicle.raw.go.jt808.v1",
		"samples 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsSlowBridgeKafkaRecentE2EP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_e2e_recent_p99_ms{topic="vehicle.raw.go.jt808.v1"} 450
vehicle_bridge_kafka_e2e_recent_samples{topic="vehicle.raw.go.jt808.v1"} 100`,
	}, Thresholds{
		BridgeKafkaRecentE2EP99MS: 300,
		MinHistogramSamples:       100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"bridge kafka e2e recent p99 450ms exceeds 300ms",
		"topic=vehicle.raw.go.jt808.v1",
		"samples 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsSlowHistoryWriteE2EP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_history_write_e2e_duration_ms_histogram_bucket{le="100",topic="vehicle.raw.go.jt808.v1"} 90
vehicle_history_write_e2e_duration_ms_histogram_bucket{le="1000",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_history_write_e2e_duration_ms_histogram_bucket{le="+Inf",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_history_write_e2e_duration_ms_histogram_count{topic="vehicle.raw.go.jt808.v1"} 100`,
	}, Thresholds{
		HistoryWriteE2EP99MS: 300,
		MinHistogramSamples:  100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"history write e2e p99 1000ms exceeds 300ms",
		"topic=vehicle.raw.go.jt808.v1",
		"samples 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateReportsSlowStatWriteE2EP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_stat_write_e2e_duration_ms_histogram_bucket{le="100",topic="vehicle.fields.go.jt808.v1"} 90
vehicle_stat_write_e2e_duration_ms_histogram_bucket{le="1000",topic="vehicle.fields.go.jt808.v1"} 100
vehicle_stat_write_e2e_duration_ms_histogram_bucket{le="+Inf",topic="vehicle.fields.go.jt808.v1"} 100
vehicle_stat_write_e2e_duration_ms_histogram_count{topic="vehicle.fields.go.jt808.v1"} 100`,
	}, Thresholds{
		StatWriteE2EP99MS:   300,
		MinHistogramSamples: 100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat write e2e p99 1000ms exceeds 300ms",
		"topic=vehicle.fields.go.jt808.v1",
		"samples 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestDefaultThresholdsDoNotGateCumulativeE2EHistograms(t *testing.T) {
	report := Evaluate(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_kafka_e2e_duration_ms_histogram_bucket{le="100",topic="vehicle.raw.go.jt808.v1"} 90
vehicle_bridge_kafka_e2e_duration_ms_histogram_bucket{le="10000",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_bridge_kafka_e2e_duration_ms_histogram_bucket{le="+Inf",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_bridge_kafka_e2e_duration_ms_histogram_count{topic="vehicle.raw.go.jt808.v1"} 100`,
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_tdengine_enabled 0
vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="100",subject="vehicle.raw.go.jt808.v1"} 90
vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="10000",subject="vehicle.raw.go.jt808.v1"} 100
vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="+Inf",subject="vehicle.raw.go.jt808.v1"} 100
vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.jt808.v1"} 100`,
		"history": `vehicle_service_info{service="history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_write_e2e_duration_ms_histogram_bucket{le="100",topic="vehicle.raw.go.jt808.v1"} 90
vehicle_history_write_e2e_duration_ms_histogram_bucket{le="10000",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_history_write_e2e_duration_ms_histogram_bucket{le="+Inf",topic="vehicle.raw.go.jt808.v1"} 100
vehicle_history_write_e2e_duration_ms_histogram_count{topic="vehicle.raw.go.jt808.v1"} 100`,
		"stat": `vehicle_service_info{service="stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_write_e2e_duration_ms_histogram_bucket{le="100",topic="vehicle.fields.go.jt808.v1"} 90
vehicle_stat_write_e2e_duration_ms_histogram_bucket{le="10000",topic="vehicle.fields.go.jt808.v1"} 100
vehicle_stat_write_e2e_duration_ms_histogram_bucket{le="+Inf",topic="vehicle.fields.go.jt808.v1"} 100
vehicle_stat_write_e2e_duration_ms_histogram_count{topic="vehicle.fields.go.jt808.v1"} 100`,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
	if joined := strings.Join(report.Findings, "\n"); strings.Contains(joined, "e2e p99") {
		t.Fatalf("cumulative e2e p99 should be observe-only by default:\n%s", joined)
	}
}

func TestEvaluateReportsSlowHistoryWriteRecentE2EP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_write_e2e_recent_p99_ms{topic="vehicle.raw.go.jt808.v1"} 2500
vehicle_history_write_e2e_recent_samples{topic="vehicle.raw.go.jt808.v1"} 100`,
	}, Thresholds{
		HistoryWriteRecentE2EP99MS: 1000,
		MinHistogramSamples:        100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"history write e2e recent p99 2500ms exceeds 1000ms",
		"topic=vehicle.raw.go.jt808.v1",
		"samples 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestEvaluateSkipsHistoryWriteRecentE2EWhenSamplesAreLow(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"history": `vehicle_service_info{service="history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_write_e2e_recent_p99_ms{topic="vehicle.raw.go.jt808.v1"} 2500
vehicle_history_write_e2e_recent_samples{topic="vehicle.raw.go.jt808.v1"} 99`,
	}, Thresholds{
		HistoryWriteRecentE2EP99MS: 1000,
		MinHistogramSamples:        100,
	})

	if report.Status != StatusOK {
		t.Fatalf("status = %s, want ok: %#v", report.Status, report)
	}
}

func TestEvaluateReportsSlowStatWriteRecentE2EP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"stat": `vehicle_service_info{service="stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_write_e2e_recent_p99_ms{topic="vehicle.fields.go.jt808.v1"} 1500
vehicle_stat_write_e2e_recent_samples{topic="vehicle.fields.go.jt808.v1"} 100`,
	}, Thresholds{
		StatWriteRecentE2EP99MS: 1000,
		MinHistogramSamples:     100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"stat write e2e recent p99 1500ms exceeds 1000ms",
		"topic=vehicle.fields.go.jt808.v1",
		"samples 100",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}

func TestAnnotateDailyMileageDiagnosticsSplitsPipelineAndSourceDataIssues(t *testing.T) {
	items, pipelineIssues, sourceDataIssues := AnnotateDailyMileageDiagnostics([]DailyMileageDiagnosisReasonItem{
		{Diagnosis: "OK", Reason: "daily_metric_exists", Count: 234},
		{Diagnosis: "NO_SOURCE_SAMPLE", Reason: "realtime_location_has_total_mileage_but_no_stat_sample", Count: 7},
		{Diagnosis: "MISSING_DAILY", Reason: "source_samples_exist_but_final_metric_missing", Count: 1},
		{Diagnosis: "NO_TOTAL_MILEAGE", Reason: "realtime_total_mileage_time_missing", Count: 2},
		{Diagnosis: "NO_TOTAL_MILEAGE", Reason: "realtime_total_mileage_non_positive", Count: 3},
		{Diagnosis: "NO_TOTAL_MILEAGE", Reason: "realtime_total_mileage_missing", Count: 4},
	})

	if pipelineIssues != 10 || sourceDataIssues != 7 {
		t.Fatalf("pipeline=%d source=%d", pipelineIssues, sourceDataIssues)
	}
	severities := make([]string, 0, len(items))
	for _, item := range items {
		severities = append(severities, item.Severity)
	}
	got := strings.Join(severities, ",")
	want := "ok,pipeline,pipeline,pipeline,source_data,source_data"
	if got != want {
		t.Fatalf("severities = %q, want %q", got, want)
	}
	if !strings.Contains(items[4].RecommendedOperatorAction, "总里程小于等于 0") {
		t.Fatalf("recommended action = %q", items[4].RecommendedOperatorAction)
	}
	if !strings.Contains(items[5].RecommendedOperatorAction, "缺少总里程字段") {
		t.Fatalf("recommended action = %q", items[5].RecommendedOperatorAction)
	}
}

func TestAnnotateDailyMileageDiagnosticsPreservesAPISeverityAndClassifiesInvalidSamples(t *testing.T) {
	items, pipelineIssues, sourceDataIssues := AnnotateDailyMileageDiagnostics([]DailyMileageDiagnosisReasonItem{
		{Diagnosis: "MISSING_DAILY", Reason: "source_samples_all_invalid", Severity: DailyMileageSeveritySourceData, Count: 25},
		{Diagnosis: "MISSING_DAILY", Reason: "source_samples_all_invalid", Count: 17},
		{Diagnosis: "NO_SOURCE_SAMPLE", Reason: "realtime_location_has_total_mileage_but_no_stat_sample", Severity: DailyMileageSeverityPipeline, Count: 2},
	})

	if pipelineIssues != 2 || sourceDataIssues != 42 {
		t.Fatalf("pipeline=%d source=%d", pipelineIssues, sourceDataIssues)
	}
	if items[0].Severity != DailyMileageSeveritySourceData || items[1].Severity != DailyMileageSeveritySourceData {
		t.Fatalf("severities = %#v", []string{items[0].Severity, items[1].Severity})
	}
}

func TestEvaluateReportsDegradedForSlowDownstreamP99(t *testing.T) {
	report := EvaluateWithThresholds(map[string]string{
		"bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1
vehicle_bridge_config{setting="fetch_wait_ms"} 20
vehicle_bridge_config{setting="workers"} 4
vehicle_bridge_batch_duration_ms_histogram_bucket{le="1000",status="ok"} 98
vehicle_bridge_batch_duration_ms_histogram_bucket{le="5000",status="ok"} 100
vehicle_bridge_batch_duration_ms_histogram_bucket{le="+Inf",status="ok"} 100
vehicle_bridge_batch_duration_ms_histogram_count{status="ok"} 100`,
		"fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1
vehicle_fast_writer_config{setting="fetch_wait_ms"} 20
vehicle_fast_writer_config{setting="workers"} 8
vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="25",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 90
vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="250",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 100
vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 100
vehicle_fast_writer_stage_duration_ms_histogram_count{stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 100`,
		"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_batch_flush_duration_ms_histogram_bucket{le="100",status="ok"} 95
vehicle_history_batch_flush_duration_ms_histogram_bucket{le="1000",status="ok"} 100
vehicle_history_batch_flush_duration_ms_histogram_bucket{le="+Inf",status="ok"} 100
vehicle_history_batch_flush_duration_ms_histogram_count{status="ok"} 100`,
		"realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1
vehicle_realtime_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_realtime_batch_flush_duration_ms_histogram_bucket{le="50",status="ok"} 50
vehicle_realtime_batch_flush_duration_ms_histogram_bucket{le="250",status="ok"} 100
vehicle_realtime_batch_flush_duration_ms_histogram_bucket{le="+Inf",status="ok"} 100
vehicle_realtime_batch_flush_duration_ms_histogram_count{status="ok"} 100
vehicle_realtime_store_update_duration_ms_histogram_bucket{le="10",protocol="JT808",status="ok",store="mysql"} 50
vehicle_realtime_store_update_duration_ms_histogram_bucket{le="250",protocol="JT808",status="ok",store="mysql"} 100
vehicle_realtime_store_update_duration_ms_histogram_bucket{le="+Inf",protocol="JT808",status="ok",store="mysql"} 100
vehicle_realtime_store_update_duration_ms_histogram_count{protocol="JT808",status="ok",store="mysql"} 100`,
		"stat": `vehicle_service_info{service="vehicle-stat-writer"} 1
vehicle_stat_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1
vehicle_stat_write_duration_ms_histogram_bucket{le="5",status="ok",topic="vehicle.fields.go.jt808.v1"} 98
vehicle_stat_write_duration_ms_histogram_bucket{le="250",status="ok",topic="vehicle.fields.go.jt808.v1"} 100
vehicle_stat_write_duration_ms_histogram_bucket{le="+Inf",status="ok",topic="vehicle.fields.go.jt808.v1"} 100
vehicle_stat_write_duration_ms_histogram_count{status="ok",topic="vehicle.fields.go.jt808.v1"} 100`,
	}, Thresholds{
		BridgeBatchP99MS:     1000,
		FastWriterStageP99MS: 100,
		HistoryFlushP99MS:    500,
		RealtimeBatchP99MS:   100,
		RealtimeStoreP99MS:   100,
		StatWriteP99MS:       100,
		MinHistogramSamples:  100,
	})

	if report.Status != StatusDegraded {
		t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
	}
	joined := strings.Join(report.Findings, "\n")
	for _, want := range []string{
		"bridge batch p99 5000ms exceeds 1000ms",
		"fast writer stage p99 250ms exceeds 100ms",
		"history flush p99 1000ms exceeds 500ms",
		"realtime batch flush p99 250ms exceeds 100ms",
		"realtime store update p99 250ms exceeds 100ms",
		"stat write p99 250ms exceeds 100ms",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("finding missing %q in:\n%s", want, joined)
		}
	}
}