shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ec3071d59b292d556a3cc182b88f33f1d99782ed
lingniu-vehicle-ingest/go/vehicle-gateway/internal/capacity/check.go
lingniu ec3071d59b fix(ops): recover stale history error health
2026-07-16 06:48:47 +08:00

4949 lines
185 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package capacity
import (
"bufio"
"fmt"
"math"
"sort"
"strconv"
"strings"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
)
type Status string
const (
StatusOK Status = "ok"
StatusDegraded Status = "degraded"
)
type Report struct {
Status Status `json:"status"`
CheckedAt time.Time `json:"checked_at"`
Totals Totals `json:"totals"`
Services []ServiceCheck `json:"services"`
GatewayFields *GatewayFieldsDiagnostics `json:"gateway_fields,omitempty"`
ProtocolFlow *ProtocolFlowDiagnostics `json:"protocol_flow,omitempty"`
IdentityResolution *IdentityResolutionDiagnostics `json:"identity_resolution,omitempty"`
PipelineLatency *PipelineLatencyDiagnostics `json:"pipeline_latency,omitempty"`
DailyMileageDiagnostics *DailyMileageDiagnostics `json:"daily_mileage_diagnostics,omitempty"`
DailyMileageSourceQuality *DailyMileageSourceQualityDiagnostics `json:"daily_mileage_source_quality,omitempty"`
DailyMileageSourceSelection *DailyMileageSourceSelectionDiagnostics `json:"daily_mileage_source_selection,omitempty"`
DataSourceDiagnostics *DataSourceDiagnostics `json:"data_source_diagnostics,omitempty"`
DataSourceKindDiagnostics *DataSourceDiagnostics `json:"data_source_kind_diagnostics,omitempty"`
JT808IdentityGaps *JT808IdentityGapDiagnostics `json:"jt808_identity_gaps,omitempty"`
Findings []string `json:"findings"`
}
type Totals struct {
ActiveConnections int64 `json:"active_connections"`
KafkaLag float64 `json:"kafka_lag"`
BridgeConsumerPending float64 `json:"bridge_consumer_pending"`
BridgeAckPending float64 `json:"bridge_ack_pending"`
BridgeBatchPendingMessages float64 `json:"bridge_batch_pending_messages"`
FieldsProjectorKafkaLag float64 `json:"fields_projector_kafka_lag"`
FieldsProjectorBatchPending float64 `json:"fields_projector_batch_pending_messages"`
FieldsProjectorRetryPending float64 `json:"fields_projector_retry_pending_messages"`
FastWriterConsumerPending float64 `json:"fast_writer_consumer_pending"`
FastWriterAckPending float64 `json:"fast_writer_ack_pending"`
FastWriterBatchPending float64 `json:"fast_writer_batch_pending_messages"`
HistoryBatchPending float64 `json:"history_batch_pending_messages"`
HistoryRowsPending float64 `json:"history_rows_pending"`
HistoryRetryPending float64 `json:"history_retry_pending_messages"`
RealtimeBatchPending float64 `json:"realtime_batch_pending_messages"`
RealtimeAsyncQueuePending float64 `json:"realtime_async_queue_pending"`
RealtimeRetryPending float64 `json:"realtime_retry_pending_messages"`
StatRetryPending float64 `json:"stat_retry_pending_messages"`
IdentityBatchPending float64 `json:"identity_batch_pending_messages"`
IdentityFactsPending float64 `json:"identity_batch_pending_facts"`
IdentityRetryPending float64 `json:"identity_retry_pending_messages"`
JT808RegistrationWriteQueuePending float64 `json:"jt808_registration_write_queue_pending"`
JT808RegistrationWriteErrors float64 `json:"jt808_registration_write_errors"`
}
type ServiceCheck struct {
Name string `json:"name"`
Metrics map[string]float64 `json:"metrics"`
}
type GatewayFieldsDiagnostics struct {
Protocols []GatewayProtocolFields `json:"protocols,omitempty"`
}
type GatewayProtocolFields struct {
Protocol string `json:"protocol"`
RawOK float64 `json:"raw_ok"`
RealtimeCandidates float64 `json:"realtime_candidates"`
FieldsOK float64 `json:"fields_ok"`
FieldsPublished float64 `json:"fields_published"`
FieldsDelegated float64 `json:"fields_delegated"`
FieldsSkippedNonRealtime float64 `json:"fields_skipped_non_realtime"`
FieldsSkippedMissing float64 `json:"fields_skipped_missing"`
FieldsPublishError float64 `json:"fields_publish_error"`
MissingRatio float64 `json:"missing_ratio"`
LatestFieldCount *float64 `json:"latest_field_count,omitempty"`
Status string `json:"status"`
Reason string `json:"reason,omitempty"`
}
type ProtocolFlowDiagnostics struct {
Protocols []ProtocolFlowProtocol `json:"protocols,omitempty"`
}
type ProtocolFlowProtocol struct {
Protocol string `json:"protocol"`
RawOK float64 `json:"raw_ok"`
RealtimeCandidates float64 `json:"realtime_candidates"`
FieldsPublished float64 `json:"fields_published"`
LatestFieldCount *float64 `json:"latest_field_count,omitempty"`
IdentityTotal int64 `json:"identity_total"`
IdentitySkipped int64 `json:"identity_skipped"`
IdentityResolved int64 `json:"identity_resolved"`
IdentityNonResolved int64 `json:"identity_non_resolved"`
IdentityNonResolvedRatio float64 `json:"identity_non_resolved_ratio"`
StatWritesOK float64 `json:"stat_writes_ok"`
StatMileageCandidateWrites float64 `json:"stat_mileage_candidate_writes"`
StatSamplesFound float64 `json:"stat_samples_found"`
StatSamplesWritten float64 `json:"stat_samples_written"`
StatSamplesActionableSkipped float64 `json:"stat_samples_actionable_skipped"`
StatSkippedMissingFields float64 `json:"stat_skipped_missing_fields"`
StatSkippedMissingVIN float64 `json:"stat_skipped_missing_vin"`
StatSkippedMissingMileage float64 `json:"stat_skipped_missing_mileage"`
StatSkippedNonMileageFrame float64 `json:"stat_skipped_non_mileage_frame"`
StatSkippedNonPositiveMileage float64 `json:"stat_skipped_non_positive_mileage"`
StatSkippedMissingTime float64 `json:"stat_skipped_missing_time"`
StatEventTimeFutureAdjusted float64 `json:"stat_event_time_future_adjusted"`
StatSkippedMissingSource float64 `json:"stat_skipped_missing_source"`
StatSkippedSameMileage float64 `json:"stat_skipped_same_mileage"`
StatSourceWritten float64 `json:"stat_source_written"`
StatSourceThrottled float64 `json:"stat_source_throttled"`
StatSourceMissingEndpoint float64 `json:"stat_source_missing_endpoint"`
StatSourceSkippedUnmanaged float64 `json:"stat_source_skipped_unmanaged"`
StatProjectionAttempted float64 `json:"stat_projection_attempted"`
StatProjectionWritten float64 `json:"stat_projection_written"`
StatProjectionThrottled float64 `json:"stat_projection_throttled"`
Status string `json:"status"`
Reason string `json:"reason,omitempty"`
}
type IdentityResolutionDiagnostics struct {
Total int64 `json:"total"`
SkippedCount int64 `json:"skipped_count"`
ResolvedCount int64 `json:"resolved_count"`
NonResolvedCount int64 `json:"non_resolved_count"`
UnresolvedCount int64 `json:"unresolved_count"`
TimeoutCount int64 `json:"timeout_count"`
ErrorCount int64 `json:"error_count"`
OtherNonResolvedCount int64 `json:"other_non_resolved_count"`
NonResolvedRatio float64 `json:"non_resolved_ratio"`
StaleCacheCount int64 `json:"stale_cache_count"`
StaleCacheRatio float64 `json:"stale_cache_ratio"`
CacheEntries map[string]int64 `json:"cache_entries,omitempty"`
Issues []IdentityIssueCount `json:"issues,omitempty"`
Protocols []IdentityProtocolResolution `json:"protocols,omitempty"`
}
type IdentityIssueCount struct {
Protocol string `json:"protocol"`
Status string `json:"status"`
MessageID string `json:"message_id"`
Reason string `json:"reason"`
Count int64 `json:"count"`
}
type IdentityProtocolResolution struct {
Protocol string `json:"protocol"`
Total int64 `json:"total"`
SkippedCount int64 `json:"skipped_count"`
ResolvedCount int64 `json:"resolved_count"`
NonResolvedCount int64 `json:"non_resolved_count"`
UnresolvedCount int64 `json:"unresolved_count"`
TimeoutCount int64 `json:"timeout_count"`
ErrorCount int64 `json:"error_count"`
OtherNonResolvedCount int64 `json:"other_non_resolved_count"`
NonResolvedRatio float64 `json:"non_resolved_ratio"`
StaleCacheCount int64 `json:"stale_cache_count"`
StaleCacheRatio float64 `json:"stale_cache_ratio"`
CacheStatusCounts map[string]int64 `json:"cache_status_counts,omitempty"`
}
type PipelineLatencyDiagnostics struct {
TotalSamples int64 `json:"total_samples"`
MaxP99MS float64 `json:"max_p99_ms"`
Items []PipelineLatencyItem `json:"items,omitempty"`
}
type PipelineLatencyItem struct {
Stage string `json:"stage"`
Service string `json:"service"`
Topic string `json:"topic,omitempty"`
Subject string `json:"subject,omitempty"`
P99MS float64 `json:"p99_ms"`
Samples int64 `json:"samples"`
ThresholdMS float64 `json:"threshold_ms,omitempty"`
Status string `json:"status"`
}
type DailyMileageDiagnostics struct {
URL string `json:"url,omitempty"`
VehicleTotal int64 `json:"vehicle_total"`
ActionableIssueTotal int64 `json:"actionable_issue_total"`
PipelineIssueTotal int64 `json:"pipeline_issue_total"`
SourceDataIssueTotal int64 `json:"source_data_issue_total"`
Items []DailyMileageDiagnosisReasonItem `json:"items,omitempty"`
Samples []DailyMileageDiagnosisSample `json:"samples,omitempty"`
FieldStatus *DailyMileageFieldStatusDiagnostics `json:"field_status,omitempty"`
}
type DailyMileageDiagnosisReasonItem struct {
StatDate string `json:"stat_date"`
Protocol string `json:"protocol"`
Diagnosis string `json:"diagnosis"`
Reason string `json:"reason"`
Count int64 `json:"count"`
Severity string `json:"severity,omitempty"`
RecommendedOperatorAction string `json:"recommended_operator_action,omitempty"`
}
type DailyMileageFieldStatusDiagnostics struct {
URL string `json:"url,omitempty"`
SummaryTotal int64 `json:"summary_total"`
VehicleTotal int64 `json:"vehicle_total"`
ActionableIssueTotal int64 `json:"actionable_issue_total"`
PipelineIssueTotal int64 `json:"pipeline_issue_total"`
SourceDataIssueTotal int64 `json:"source_data_issue_total"`
Items []DailyMileageFieldStatusSummaryItem `json:"items,omitempty"`
}
type DailyMileageFieldStatusSummaryItem struct {
StatDate string `json:"stat_date"`
Protocol string `json:"protocol"`
Diagnosis string `json:"diagnosis"`
Reason string `json:"reason"`
RealtimeMileageFieldStatus string `json:"realtime_mileage_field_status"`
Count int64 `json:"count"`
Severity string `json:"severity,omitempty"`
FieldStatusSeverity string `json:"field_status_severity,omitempty"`
RecommendedOperatorAction string `json:"recommended_operator_action,omitempty"`
FieldStatusAction string `json:"field_status_action,omitempty"`
}
type DailyMileageDiagnosisSample struct {
VIN string `json:"vin"`
StatDate string `json:"stat_date"`
Protocol string `json:"protocol"`
Plate string `json:"plate,omitempty"`
PlatformName string `json:"platform_name,omitempty"`
Peer string `json:"peer,omitempty"`
SnapshotEventTime string `json:"snapshot_event_time,omitempty"`
LocationEventTime string `json:"location_event_time,omitempty"`
SnapshotUpdatedAt string `json:"snapshot_updated_at,omitempty"`
LocationUpdatedAt string `json:"location_updated_at,omitempty"`
RealtimeTotalMileageKM *float64 `json:"realtime_total_mileage_km,omitempty"`
RealtimeTotalMileageAt string `json:"realtime_total_mileage_event_time,omitempty"`
DailyMileageKM *float64 `json:"daily_mileage_km,omitempty"`
DailyLatestTotalMileageKM *float64 `json:"daily_latest_total_mileage_km,omitempty"`
SourceSampleCount int64 `json:"source_sample_count"`
OKSourceCount int64 `json:"ok_source_count"`
SelectableSourceCount int64 `json:"selectable_source_count"`
RealtimeFieldCount int64 `json:"realtime_field_count,omitempty"`
RealtimeSampleFields []string `json:"realtime_sample_fields,omitempty"`
RealtimeMileageFieldStatus string `json:"realtime_mileage_field_status,omitempty"`
RealtimeMileageEvidence string `json:"realtime_mileage_evidence,omitempty"`
LatestStatEventTime string `json:"latest_stat_event_time,omitempty"`
QualityStatuses []string `json:"quality_statuses,omitempty"`
RawFrameQueryPath string `json:"raw_frame_query_path,omitempty"`
SourceSampleQueryPath string `json:"source_sample_query_path,omitempty"`
Diagnosis string `json:"diagnosis"`
Reason string `json:"reason"`
Severity string `json:"severity,omitempty"`
RecommendedOperatorAction string `json:"recommended_operator_action,omitempty"`
}
type DailyMileageSourceQualityDiagnostics struct {
URL string `json:"url,omitempty"`
SummaryTotal int64 `json:"summary_total"`
SourceCount int64 `json:"source_count"`
VehicleCount int64 `json:"vehicle_count"`
SampleCount int64 `json:"sample_count"`
OKSourceCount int64 `json:"ok_source_count"`
InvalidDeltaSourceCount int64 `json:"invalid_delta_source_count"`
NoBaselineSourceCount int64 `json:"no_baseline_source_count"`
FallbackAfterInvalidBaselineSourceCount int64 `json:"fallback_after_invalid_baseline_source_count"`
RealtimeLocationFallbackSourceCount int64 `json:"realtime_location_fallback_source_count"`
Items []DailyMileageSourceQualityItem `json:"items,omitempty"`
ItemsTruncated bool `json:"items_truncated,omitempty"`
}
type DailyMileageSourceQualityItem struct {
StatDate string `json:"stat_date"`
Protocol string `json:"protocol"`
QualityStatus string `json:"quality_status"`
QualityReason string `json:"quality_reason,omitempty"`
SourceCount int64 `json:"source_count"`
VehicleCount int64 `json:"vehicle_count"`
SelectedCount int64 `json:"selected_count"`
SampleCount int64 `json:"sample_count"`
DailyMileageKM float64 `json:"daily_mileage_km"`
}
type DailyMileageSourceSelectionDiagnostics struct {
URL string `json:"url,omitempty"`
SummaryTotal int64 `json:"summary_total"`
SourceCount int64 `json:"source_count"`
VehicleCount int64 `json:"vehicle_count"`
SampleCount int64 `json:"sample_count"`
SelectedSourceCount int64 `json:"selected_source_count"`
NotSelectedSourceCount int64 `json:"not_selected_source_count"`
NotSelectedMileageKM float64 `json:"not_selected_mileage_km"`
ExcludedSourceCount int64 `json:"excluded_source_count"`
UnmanagedSourceCount int64 `json:"unmanaged_source_count"`
UnknownKindSourceCount int64 `json:"unknown_kind_source_count"`
Protocols []DailyMileageSourceSelectionProtocol `json:"protocols,omitempty"`
Items []DailyMileageSourceSelectionItem `json:"items,omitempty"`
Samples []DailyMileageSourceSelectionSample `json:"samples,omitempty"`
ItemsTruncated bool `json:"items_truncated,omitempty"`
}
type DailyMileageSourceSelectionProtocol struct {
Protocol string `json:"protocol"`
SourceCount int64 `json:"source_count"`
VehicleCount int64 `json:"vehicle_count"`
SampleCount int64 `json:"sample_count"`
SelectedSourceCount int64 `json:"selected_source_count"`
NotSelectedSourceCount int64 `json:"not_selected_source_count"`
NotSelectedMileageKM float64 `json:"not_selected_mileage_km"`
ExcludedSourceCount int64 `json:"excluded_source_count"`
UnmanagedSourceCount int64 `json:"unmanaged_source_count"`
UnknownKindSourceCount int64 `json:"unknown_kind_source_count"`
}
type DailyMileageSourceSelectionItem struct {
StatDate string `json:"stat_date"`
Protocol string `json:"protocol"`
SelectionStatus string `json:"selection_status"`
SelectionReason string `json:"selection_reason,omitempty"`
SelectionAction string `json:"selection_action,omitempty"`
SourceCount int64 `json:"source_count"`
VehicleCount int64 `json:"vehicle_count"`
SelectedCount int64 `json:"selected_count"`
SampleCount int64 `json:"sample_count"`
DailyMileageKM float64 `json:"daily_mileage_km"`
}
type DailyMileageSourceSelectionSample struct {
VIN string `json:"vin"`
StatDate string `json:"stat_date"`
Protocol string `json:"protocol"`
SourceKey string `json:"source_key"`
SourceIP string `json:"source_ip,omitempty"`
SourceEndpoint string `json:"source_endpoint,omitempty"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
PlatformName string `json:"platform_name,omitempty"`
SourceID *int64 `json:"source_id,omitempty"`
SourceCode string `json:"source_code,omitempty"`
SourceKind string `json:"source_kind,omitempty"`
SourceEnabled *bool `json:"source_enabled,omitempty"`
TrustPriority *int `json:"trust_priority,omitempty"`
LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"`
DailyMileageKM float64 `json:"daily_mileage_km"`
SampleCount int64 `json:"sample_count"`
QualityStatus string `json:"quality_status,omitempty"`
QualityReason string `json:"quality_reason,omitempty"`
SelectionStatus string `json:"selection_status,omitempty"`
SelectionReason string `json:"selection_reason,omitempty"`
SelectionAction string `json:"selection_action,omitempty"`
SelectedSource *DailyMileageSourceSelectionSample `json:"selected_source,omitempty"`
SelectedSourceQueryPath string `json:"selected_source_query_path,omitempty"`
UpdatedAt string `json:"updated_at,omitempty"`
}
type DataSourceDiagnostics struct {
URL string `json:"url,omitempty"`
MissingSourceCodeTotal int64 `json:"missing_source_code_total"`
RecentMissingSourceCodeTotal int64 `json:"recent_missing_source_code_total"`
UnknownSourceKindTotal int64 `json:"unknown_source_kind_total,omitempty"`
RecentUnknownSourceKindTotal int64 `json:"recent_unknown_source_kind_total,omitempty"`
PlatformKindCandidateTotal int64 `json:"platform_kind_candidate_total,omitempty"`
RecentPlatformKindCandidateTotal int64 `json:"recent_platform_kind_candidate_total,omitempty"`
RecentAgeThresholdSeconds int64 `json:"recent_age_threshold_seconds,omitempty"`
ActionableCandidateTotal int64 `json:"actionable_candidate_total"`
MappingIssueTotal int64 `json:"mapping_issue_total,omitempty"`
RecentMappingIssueTotal int64 `json:"recent_mapping_issue_total,omitempty"`
ReasonCounts map[string]int64 `json:"reason_counts,omitempty"`
MappingIssueReasonCounts map[string]int64 `json:"mapping_issue_reason_counts,omitempty"`
ReasonCountsTruncated bool `json:"reason_counts_truncated,omitempty"`
MappingIssueReasonCountsTruncated bool `json:"mapping_issue_reason_counts_truncated,omitempty"`
Samples []DataSourceDiagnosticSample `json:"samples,omitempty"`
MappingIssueSamples []DataSourceDiagnosticSample `json:"mapping_issue_samples,omitempty"`
}
type DataSourceDiagnosticSample struct {
ID int64 `json:"id"`
Protocol string `json:"protocol"`
SourceIP string `json:"source_ip"`
LatestSourceEndpoint string `json:"latest_source_endpoint,omitempty"`
PlatformName string `json:"platform_name,omitempty"`
SourceCode string `json:"source_code,omitempty"`
SourceKind string `json:"source_kind"`
FirstSeenAt string `json:"first_seen_at,omitempty"`
LatestSeenAt string `json:"latest_seen_at,omitempty"`
ActiveSpanSeconds int64 `json:"active_span_seconds"`
LatestSeenAgeSeconds int64 `json:"latest_seen_age_seconds"`
RegistrationRows int64 `json:"registration_rows"`
PhoneCount int64 `json:"phone_count"`
UnknownVINRows int64 `json:"unknown_vin_rows"`
IdentifierMatchedPhones int64 `json:"identifier_matched_phones"`
UnmappedPhoneCount int64 `json:"unmapped_phone_count"`
IdentifierMatchRatio float64 `json:"identifier_match_ratio"`
ConfiguredSourceCodeMatchedPhones int64 `json:"configured_source_code_matched_phones"`
ConfiguredSourceCodePlatformName string `json:"configured_source_code_platform_name,omitempty"`
ConfiguredSourceCodeConflict bool `json:"configured_source_code_conflict"`
SourcePlatformNameMismatch bool `json:"source_platform_name_mismatch"`
MatchedSourceCodeCount int64 `json:"matched_source_code_count"`
CandidateSourceCode string `json:"candidate_source_code,omitempty"`
CandidatePlatformName string `json:"candidate_platform_name,omitempty"`
MatchedSourceCodes []string `json:"matched_source_codes,omitempty"`
MatchedPlatformNames []string `json:"matched_platform_names,omitempty"`
SamplePhones []string `json:"sample_phones,omitempty"`
Reason string `json:"reason"`
RecommendedOperatorAction string `json:"recommended_operator_action"`
SuggestedSourceKind string `json:"suggested_source_kind"`
SuggestionConfidence string `json:"suggestion_confidence"`
SuggestionReason string `json:"suggestion_reason"`
}
type JT808IdentityGapDiagnostics struct {
URL string `json:"url,omitempty"`
Total int64 `json:"total"`
RecentSeconds int64 `json:"recent_seconds"`
Samples []JT808IdentityGapSample `json:"samples,omitempty"`
}
type JT808IdentityGapSample struct {
Phone string `json:"phone"`
DeviceID string `json:"device_id,omitempty"`
Plate string `json:"plate,omitempty"`
VIN string `json:"vin,omitempty"`
SourceIP string `json:"source_ip,omitempty"`
SourceEndpoint string `json:"source_endpoint,omitempty"`
SourceCode string `json:"source_code,omitempty"`
PlatformName string `json:"platform_name,omitempty"`
SourceKind string `json:"source_kind,omitempty"`
FirstRegisteredAt string `json:"first_registered_at,omitempty"`
LatestRegisteredAt string `json:"latest_registered_at,omitempty"`
LatestAuthenticatedAt string `json:"latest_authenticated_at,omitempty"`
LatestSeenAt string `json:"latest_seen_at,omitempty"`
LatestSeenAgeSeconds int64 `json:"latest_seen_age_seconds"`
Reason string `json:"reason"`
RecommendedOperatorAction string `json:"recommended_operator_action"`
RawFrameQueryPath string `json:"raw_frame_query_path,omitempty"`
DataSourceQueryPath string `json:"data_source_query_path,omitempty"`
}
const (
DailyMileageSeverityOK = "ok"
DailyMileageSeverityPipeline = "pipeline"
DailyMileageSeveritySourceData = "source_data"
)
func AnnotateDailyMileageDiagnostics(items []DailyMileageDiagnosisReasonItem) (annotated []DailyMileageDiagnosisReasonItem, pipelineIssues int64, sourceDataIssues int64) {
annotated = make([]DailyMileageDiagnosisReasonItem, 0, len(items))
for _, item := range items {
severity := strings.ToLower(strings.TrimSpace(item.Severity))
if severity != DailyMileageSeverityOK && severity != DailyMileageSeverityPipeline && severity != DailyMileageSeveritySourceData {
severity = DailyMileageIssueSeverity(item)
}
item.Severity = severity
if strings.TrimSpace(item.RecommendedOperatorAction) == "" {
item.RecommendedOperatorAction = DailyMileageRecommendedOperatorAction(item.Diagnosis, item.Reason)
}
switch item.Severity {
case DailyMileageSeverityPipeline:
pipelineIssues += item.Count
case DailyMileageSeveritySourceData:
sourceDataIssues += item.Count
}
annotated = append(annotated, item)
}
return annotated, pipelineIssues, sourceDataIssues
}
func DailyMileageIssueSeverity(item DailyMileageDiagnosisReasonItem) string {
diagnosis := strings.ToUpper(strings.TrimSpace(item.Diagnosis))
reason := strings.ToLower(strings.TrimSpace(item.Reason))
switch diagnosis {
case "", "OK":
return DailyMileageSeverityOK
case "MISSING_DAILY":
if reason == "source_samples_all_invalid" {
return DailyMileageSeveritySourceData
}
return DailyMileageSeverityPipeline
case "NO_SOURCE_SAMPLE":
return DailyMileageSeverityPipeline
case "NO_TOTAL_MILEAGE":
switch reason {
case "realtime_total_mileage_time_missing":
return DailyMileageSeverityPipeline
default:
return DailyMileageSeveritySourceData
}
default:
return DailyMileageSeverityPipeline
}
}
func DailyMileageRecommendedOperatorAction(diagnosis, reason string) string {
diagnosis = strings.ToUpper(strings.TrimSpace(diagnosis))
reason = strings.ToLower(strings.TrimSpace(reason))
switch diagnosis {
case "", "OK":
return "无需处理，车辆当日里程已生成"
case "MISSING_DAILY":
return "统计源样本已存在但最终日表缺失，优先排查 stat-writer 投影任务、MySQL 写入和 source 选举"
case "NO_SOURCE_SAMPLE":
return "实时位置已有有效总里程但统计字段样本缺失，优先排查字段流 topic、stat-writer 消费和协议字段映射"
case "NO_TOTAL_MILEAGE":
switch reason {
case "realtime_total_mileage_time_missing":
return "总里程值存在但事件时间缺失，优先排查解析器和 history/realtime 写入的时间字段"
case "realtime_total_mileage_missing":
return "当日实时数据缺少总里程字段，核对平台是否上报该字段以及协议字段映射是否覆盖"
case "realtime_total_mileage_non_positive":
return "终端上报总里程小于等于 0，不参与差值统计，需核对设备或平台里程字段来源"
case "realtime_total_mileage_not_reported_on_stat_date":
return "最新总里程不是统计日期内上报，等待当日含总里程数据或检查平台是否只上报部分字段"
default:
return "车辆当日在线但没有可用于里程统计的总里程，优先核对源平台报文字段和解析映射"
}
default:
return "诊断状态未知，检查统计诊断 SQL 和上游字段流"
}
}
type Thresholds struct {
GatewayFrameP99MS float64
GatewayIdentityP99MS float64
GatewayResponseRecentP99MS float64
AsyncSinkP99MS float64
AsyncSinkQueueWaitRecentP99MS float64
BridgeBatchP99MS float64
BridgeKafkaWriteP99MS float64
BridgeKafkaE2EP99MS float64
BridgeKafkaRecentE2EP99MS float64
BridgeFetchWaitMaxMS float64
BridgeWorkersMin float64
BridgeRouteErrorMax float64
BridgeKafkaWriteErrorMax float64
BridgeMessageMinReceived float64
BridgeMessageMaxUnackedRatio float64
FieldsProjectorWorkersPerProtocolMin float64
FieldsProjectorInvalidJSONMax float64
FieldsProjectorWriteErrorMax float64
FieldsProjectorBatchPendingMax float64
FastWriterStageP99MS float64
FastWriterRedisE2EP99MS float64
FastWriterRedisRecentE2EP99MS float64
FastWriterFetchWaitMaxMS float64
FastWriterWorkersMin float64
FastWriterInvalidJSONMax float64
FastWriterFallbackErrorMax float64
FastWriterDecoupledUpdateErrorMax float64
FastWriterTDengineEnabledMax float64
FastWriterMessageMinReceived float64
FastWriterMessageMaxUnackedRatio float64
HistoryFlushP99MS float64
HistoryWriteE2EP99MS float64
HistoryWriteRecentE2EP99MS float64
HistoryWorkersMin float64
HistoryInvalidJSONMax float64
HistoryLocationErrorMax float64
HistoryLocationErrorRecentSec float64
HistoryLocationAccountingMaxRatio float64
HistoryFallbackErrorMax float64
HistoryRetryMax float64
HistoryRetryExhaustedMax float64
RealtimeBatchP99MS float64
RealtimeStoreP99MS float64
RealtimeWorkersMin float64
RealtimeStoreErrorMax float64
RealtimeInvalidJSONMax float64
StatWriteP99MS float64
StatWriteE2EP99MS float64
StatWriteRecentE2EP99MS float64
StatWorkersMin float64
IdentityWorkersMin float64
StatInvalidJSONMax float64
StatProtocolTopicMismatchMax float64
MessageContractMismatchMax float64
StatRetryMax float64
StatRetryExhaustedMax float64
ProcessUptimeMinSec float64
ProcessHeapAllocMaxBytes float64
ProcessHeapSysMaxBytes float64
ProcessGoroutinesMax float64
KafkaLagMax float64
KafkaConsumerCommitErrorMax float64
KafkaConsumerMessageMinReceived float64
KafkaConsumerMaxUncommittedRatio float64
DurableSpoolBacklogMax float64
DurableSpoolOldestAgeMaxSec float64
DurableSpoolErrorMax float64
DurableSpoolReplayErrorMax float64
DurableOutboxInflightMax float64
DurableOutboxErrorMax float64
AsyncSinkEnqueueErrorMax float64
AsyncSinkPublishErrorMax float64
AsyncSinkQueueDepthMax float64
AsyncSinkQueueMaxUsageRatio float64
FastWriterBatchPendingMax float64
HistoryBatchPendingMax float64
HistoryRowsPendingMax float64
RealtimeBatchPendingMax float64
RealtimeAsyncQueueDepthMax float64
RealtimeAsyncQueueMaxUsageRatio float64
RealtimeAsyncQueueDroppedMax float64
HistoryParsedFieldMinFrames float64
HistoryParsedFieldMaxMissingRatio float64
MinHistogramSamples float64
StatSampleMinWrites float64
StatSampleMaxActionableSkipRatio float64
StatSampleFutureAdjustmentMax float64
StatSourceMinSamples float64
StatSourceMaxMissingRatio float64
StatProjectionMinSampleWrites float64
StatBatchPendingMax float64
ConsumerRetryPendingMax float64
StatCacheEntriesMax float64
RealtimeThrottleMinSamples float64
RealtimeThrottleCacheEntriesMax float64
RealtimePlateCacheEntriesMax float64
FastWriterRedisEnvelopeMinSeen float64
FastWriterRedisEnvelopeMaxBadRatio float64
FastWriterRedisFieldMinSeen float64
FastWriterRedisFieldMaxStaleRatio float64
GatewayFieldMinRaw float64
GatewayFieldMaxMissingRatio float64
GatewayFieldMinCount float64
GatewayBridgeMinPublish float64
GatewayFastWriterMinRawPublish float64
GatewayParseMinFrames float64
GatewayParseMaxBadRatio float64
GatewayResponseMinSamples float64
GatewayResponseMaxErrorRatio float64
GatewayIdentityMinSamples float64
GatewayIdentityMaxUnresolvedRatio float64
GatewayIdentityCacheEntriesMax float64
GatewayIdentitySnapshotRequired float64
GatewayIdentitySnapshotMaxAgeSec float64
GatewayIdentityLocationTouchFailureMax float64
GatewayIdentityRegistrationWriteQueueDepthMax float64
GatewayIdentityRegistrationWriteQueueMaxUsageRatio float64
GatewayIdentityRegistrationWriteErrorMax float64
GatewayIdentityStaleMinSamples float64
GatewayIdentityMaxStaleCacheRatio float64
StatTopicMinBridge float64
RawFanoutMinBridge float64
LastActivityStaleSec float64
RequiredConsumerTopics map[string][]string
}
func DefaultThresholds() Thresholds {
return Thresholds{
GatewayFrameP99MS: 250,
GatewayIdentityP99MS: 100,
GatewayResponseRecentP99MS: 100,
AsyncSinkP99MS: 100,
AsyncSinkQueueWaitRecentP99MS: 100,
BridgeBatchP99MS: 5000,
BridgeKafkaWriteP99MS: 500,
BridgeKafkaE2EP99MS: 0,
BridgeKafkaRecentE2EP99MS: 300,
BridgeFetchWaitMaxMS: 50,
BridgeWorkersMin: 2,
BridgeRouteErrorMax: 0,
BridgeKafkaWriteErrorMax: 0,
BridgeMessageMinReceived: 5000,
BridgeMessageMaxUnackedRatio: 0.20,
FieldsProjectorWorkersPerProtocolMin: 1,
FieldsProjectorInvalidJSONMax: 0,
FieldsProjectorWriteErrorMax: 0,
FieldsProjectorBatchPendingMax: 1000,
FastWriterStageP99MS: 100,
FastWriterRedisE2EP99MS: 0,
FastWriterRedisRecentE2EP99MS: 100,
FastWriterFetchWaitMaxMS: 50,
FastWriterWorkersMin: 4,
FastWriterInvalidJSONMax: 0,
FastWriterFallbackErrorMax: 0,
FastWriterDecoupledUpdateErrorMax: 0,
FastWriterTDengineEnabledMax: 0,
FastWriterMessageMinReceived: 5000,
FastWriterMessageMaxUnackedRatio: 0.20,
HistoryFlushP99MS: 1000,
HistoryWriteE2EP99MS: 0,
HistoryWriteRecentE2EP99MS: 2000,
HistoryWorkersMin: 3,
HistoryInvalidJSONMax: 0,
HistoryLocationErrorMax: 0,
HistoryLocationErrorRecentSec: 300,
HistoryLocationAccountingMaxRatio: 0.02,
HistoryFallbackErrorMax: 0,
HistoryRetryMax: 100,
HistoryRetryExhaustedMax: 0,
RealtimeBatchP99MS: 500,
RealtimeStoreP99MS: 100,
RealtimeWorkersMin: 3,
RealtimeStoreErrorMax: 0,
RealtimeInvalidJSONMax: 0,
StatWriteP99MS: 100,
StatWriteE2EP99MS: 0,
StatWriteRecentE2EP99MS: 1000,
StatWorkersMin: 3,
IdentityWorkersMin: 3,
StatInvalidJSONMax: 0,
StatProtocolTopicMismatchMax: 0,
MessageContractMismatchMax: 0,
StatRetryMax: 100,
StatRetryExhaustedMax: 0,
ProcessUptimeMinSec: 0,
ProcessHeapAllocMaxBytes: 0,
ProcessHeapSysMaxBytes: 0,
ProcessGoroutinesMax: 0,
KafkaLagMax: 1000,
KafkaConsumerCommitErrorMax: 0,
KafkaConsumerMessageMinReceived: 100,
KafkaConsumerMaxUncommittedRatio: 0.20,
DurableSpoolBacklogMax: 10000,
DurableSpoolOldestAgeMaxSec: 300,
DurableSpoolErrorMax: 0,
DurableSpoolReplayErrorMax: 0,
DurableOutboxInflightMax: 10000,
DurableOutboxErrorMax: 0,
AsyncSinkEnqueueErrorMax: 0,
AsyncSinkPublishErrorMax: 0,
AsyncSinkQueueDepthMax: 10000,
AsyncSinkQueueMaxUsageRatio: 0.80,
FastWriterBatchPendingMax: 1000,
HistoryBatchPendingMax: 1000,
HistoryRowsPendingMax: 5000,
RealtimeBatchPendingMax: 1000,
RealtimeAsyncQueueDepthMax: 10000,
RealtimeAsyncQueueMaxUsageRatio: 0.80,
RealtimeAsyncQueueDroppedMax: 0,
HistoryParsedFieldMinFrames: 100,
HistoryParsedFieldMaxMissingRatio: 0.05,
MinHistogramSamples: 100,
StatSampleMinWrites: 100,
StatSampleMaxActionableSkipRatio: 0.60,
StatSampleFutureAdjustmentMax: 0,
StatSourceMinSamples: 100,
StatSourceMaxMissingRatio: 0.60,
StatProjectionMinSampleWrites: 100,
StatBatchPendingMax: 1000,
ConsumerRetryPendingMax: 0,
StatCacheEntriesMax: 1000000,
RealtimeThrottleMinSamples: 100,
RealtimeThrottleCacheEntriesMax: 1000000,
RealtimePlateCacheEntriesMax: 200000,
FastWriterRedisEnvelopeMinSeen: 100,
FastWriterRedisEnvelopeMaxBadRatio: 0.50,
FastWriterRedisFieldMinSeen: 1000,
FastWriterRedisFieldMaxStaleRatio: 0.20,
GatewayFieldMinRaw: 100,
GatewayFieldMaxMissingRatio: 0.20,
GatewayFieldMinCount: 1,
GatewayBridgeMinPublish: 1000,
GatewayFastWriterMinRawPublish: 1000,
GatewayParseMinFrames: 100,
GatewayParseMaxBadRatio: 0.05,
GatewayResponseMinSamples: 100,
GatewayResponseMaxErrorRatio: 0.05,
GatewayIdentityMinSamples: 100,
GatewayIdentityMaxUnresolvedRatio: 0.20,
GatewayIdentityCacheEntriesMax: 300000,
GatewayIdentitySnapshotRequired: 1,
GatewayIdentitySnapshotMaxAgeSec: 180,
GatewayIdentityLocationTouchFailureMax: 0,
GatewayIdentityRegistrationWriteQueueDepthMax: 10000,
GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0.80,
GatewayIdentityRegistrationWriteErrorMax: 0,
GatewayIdentityStaleMinSamples: 100,
GatewayIdentityMaxStaleCacheRatio: 0.05,
StatTopicMinBridge: 1000,
RawFanoutMinBridge: 1000,
LastActivityStaleSec: 300,
}
}
func DefaultRequiredConsumerTopics() map[string][]string {
return map[string][]string{
"fields-projector": {
topics.RawGB32960,
topics.RawJT808,
topics.RawYutongMQTT,
},
"history": {
topics.RawGB32960,
topics.RawJT808,
topics.RawYutongMQTT,
},
"realtime": {
topics.RawGB32960,
topics.RawJT808,
topics.RawYutongMQTT,
},
"stat": {
topics.FieldsGB32960,
topics.FieldsJT808,
topics.FieldsYutongMQTT,
},
"identity": {
topics.RawJT808,
},
}
}
func Evaluate(metricsByService map[string]string) Report {
return EvaluateWithThresholds(metricsByService, DefaultThresholds())
}
func EvaluateWithThresholds(metricsByService map[string]string, thresholds Thresholds) Report {
report := Report{
Status: StatusOK,
CheckedAt: time.Now().UTC(),
Services: make([]ServiceCheck, 0, len(metricsByService)),
}
parsedByService := make(map[string]parsedMetricSet, len(metricsByService))
var names []string
for name := range metricsByService {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
parsed := parseMetricSet(metricsByService[name])
parsedByService[name] = parsed
metrics := parsed.totals
addLastActivityAges(metrics, report.CheckedAt)
report.Services = append(report.Services, ServiceCheck{Name: name, Metrics: metrics})
report.Totals.ActiveConnections += int64(metrics["vehicle_gateway_active_connections"])
report.Totals.KafkaLag += metrics["vehicle_history_kafka_lag"]
report.Totals.KafkaLag += metrics["vehicle_stat_kafka_lag"]
report.Totals.KafkaLag += metrics["vehicle_realtime_kafka_lag"]
report.Totals.KafkaLag += metrics["vehicle_identity_writer_kafka_lag"]
report.Totals.BridgeConsumerPending += metrics["vehicle_bridge_nats_consumer_pending"]
report.Totals.BridgeAckPending += metrics["vehicle_bridge_nats_consumer_ack_pending"]
report.Totals.BridgeBatchPendingMessages += metrics["vehicle_bridge_batch_pending_messages"]
report.Totals.FieldsProjectorKafkaLag += metrics["vehicle_fields_projector_kafka_lag"]
report.Totals.FieldsProjectorBatchPending += metrics["vehicle_fields_projector_batch_pending_messages"]
report.Totals.FieldsProjectorRetryPending += metrics["vehicle_fields_projector_retry_pending_messages"]
report.Totals.FastWriterConsumerPending += metrics["vehicle_fast_writer_nats_consumer_pending"]
report.Totals.FastWriterAckPending += metrics["vehicle_fast_writer_nats_consumer_ack_pending"]
report.Totals.FastWriterBatchPending += metrics["vehicle_fast_writer_batch_pending_messages"]
report.Totals.HistoryBatchPending += metrics["vehicle_history_batch_pending_messages"]
report.Totals.HistoryRowsPending += metrics["vehicle_history_batch_pending_rows"]
report.Totals.HistoryRetryPending += metrics["vehicle_history_retry_pending_messages"]
report.Totals.RealtimeBatchPending += metrics["vehicle_realtime_batch_pending_messages"]
report.Totals.RealtimeAsyncQueuePending += metrics["vehicle_realtime_async_queue_depth"]
report.Totals.RealtimeRetryPending += metrics["vehicle_realtime_retry_pending_messages"]
report.Totals.StatRetryPending += metrics["vehicle_stat_retry_pending_messages"]
report.Totals.IdentityBatchPending += metrics["vehicle_identity_writer_batch_pending_messages"]
report.Totals.IdentityFactsPending += metrics["vehicle_identity_writer_batch_pending_facts"]
report.Totals.IdentityRetryPending += metrics["vehicle_identity_writer_retry_pending_messages"]
report.Totals.JT808RegistrationWriteQueuePending += gatewayIdentityCacheMetric(parsed, "registration_write_queue", "vehicle_gateway_identity_cache_entries")
report.Totals.JT808RegistrationWriteErrors += jt808RegistrationWriteErrors(parsed)
report.Findings = append(report.Findings, findingsForService(name, metrics)...)
report.Findings = append(report.Findings, findingsForMetrics(name, metrics, thresholds)...)
report.Findings = append(report.Findings, findingsForRuntimeConfig(name, parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForAsyncSinkQueueUsage(name, parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForAsyncSinkErrors(name, parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForDurableSpool(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForDurableOutbox(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistograms(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForKafkaConsumerCompletion(name, parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForMessageContractMismatches(name, parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForBridgeRouteErrors(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForBridgeKafkaWriteErrors(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForBridgeMessageCompletion(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForFieldsProjector(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistoryInvalidJSON(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistoryLocationErrors(parsed, thresholds, report.CheckedAt)...)
report.Findings = append(report.Findings, findingsForHistoryLocationAccounting(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistoryFallbackErrors(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForWriteRetries(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistoryParsedFields(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForStatInvalidJSON(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForStatProtocolTopicMismatches(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForStatSamples(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForStatSources(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForStatProjections(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForStatCache(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForFastWriterRedisEnvelopes(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForFastWriterRedisFields(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForFastWriterInvalidJSON(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForFastWriterFallbackErrors(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForFastWriterDecoupledUpdateErrors(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForFastWriterMessageCompletion(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForRealtimeInvalidJSON(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForRealtimeStoreErrors(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForRealtimeAsyncQueue(parsed, thresholds)...)
}
report.Findings = append(report.Findings, findingsForGatewayIdentityQuality(parsedByService["gateway"], thresholds)...)
report.Findings = append(report.Findings, findingsForGatewayIdentityCache(parsedByService["gateway"], thresholds)...)
report.Findings = append(report.Findings, findingsForGatewayIdentitySnapshot(parsedByService["gateway"], thresholds, report.CheckedAt)...)
report.Findings = append(report.Findings, findingsForGatewayIdentityLocationTouchFailures(parsedByService["gateway"], thresholds)...)
report.Findings = append(report.Findings, findingsForGatewayRegistrationWriteQueue(parsedByService["gateway"], thresholds)...)
report.Findings = append(report.Findings, findingsForGatewayRegistrationWriteErrors(parsedByService["gateway"], thresholds)...)
report.Findings = append(report.Findings, findingsForGatewayIdentityStaleCache(parsedByService["gateway"], thresholds)...)
report.Findings = append(report.Findings, findingsForIdentityWriterOwnership(parsedByService["gateway"], parsedByService["identity"])...)
report.GatewayFields = buildGatewayFieldsDiagnostics(parsedByService["gateway"], thresholds)
report.IdentityResolution = buildIdentityResolutionDiagnostics(parsedByService["gateway"])
report.ProtocolFlow = buildProtocolFlowDiagnostics(parsedByService, thresholds)
report.PipelineLatency = buildPipelineLatencyDiagnostics(parsedByService, thresholds)
report.Findings = append(report.Findings, findingsForGatewayParseQuality(parsedByService["gateway"], thresholds)...)
report.Findings = append(report.Findings, findingsForGatewayResponseQuality(parsedByService["gateway"], thresholds)...)
report.Findings = append(report.Findings, findingsForRealtimeThrottle(parsedByService["realtime"], thresholds)...)
report.Findings = append(report.Findings, findingsForRealtimeThrottleCache(parsedByService["realtime"], thresholds)...)
report.Findings = append(report.Findings, findingsForRealtimePlateCache(parsedByService["realtime"], thresholds)...)
report.Findings = append(report.Findings, findingsForRequiredConsumerTopics(parsedByService, thresholds)...)
report.Findings = append(report.Findings, findingsForTopicFlow(parsedByService, thresholds)...)
report.Findings = append(report.Findings, findingsForLastActivity(parsedByService, thresholds, report.CheckedAt)...)
if len(report.Findings) > 0 {
report.Status = StatusDegraded
}
return report
}
func findingsForService(name string, metrics map[string]float64) []string {
if metrics["vehicle_service_info"] <= 0 {
return []string{fmt.Sprintf("%s service info metric missing", name)}
}
if requiresKafkaConsumerInfo(name) && metrics["vehicle_kafka_consumer_info"] <= 0 {
return []string{fmt.Sprintf("%s kafka consumer info metric missing", name)}
}
return nil
}
func requiresKafkaConsumerInfo(name string) bool {
switch name {
case "history", "stat", "realtime", "fields-projector":
return true
default:
return false
}
}
func findingsForMetrics(service string, metrics map[string]float64, thresholds Thresholds) []string {
var findings []string
if thresholds.ProcessUptimeMinSec > 0 {
value, ok := metrics["vehicle_process_uptime_seconds"]
if !ok {
findings = append(findings, fmt.Sprintf("%s process uptime metric missing", service))
} else if value < thresholds.ProcessUptimeMinSec {
findings = append(findings, fmt.Sprintf("%s process uptime %.0fs below %.0fs", service, value, thresholds.ProcessUptimeMinSec))
}
}
if thresholds.ProcessHeapAllocMaxBytes > 0 {
value, ok := metrics["vehicle_process_heap_alloc_bytes"]
if !ok {
findings = append(findings, fmt.Sprintf("%s process heap alloc metric missing", service))
} else if value > thresholds.ProcessHeapAllocMaxBytes {
findings = append(findings, fmt.Sprintf("%s process heap alloc %.0f bytes exceeds %.0f", service, value, thresholds.ProcessHeapAllocMaxBytes))
}
}
if thresholds.ProcessHeapSysMaxBytes > 0 {
value, ok := metrics["vehicle_process_heap_sys_bytes"]
if !ok {
findings = append(findings, fmt.Sprintf("%s process heap sys metric missing", service))
} else if value > thresholds.ProcessHeapSysMaxBytes {
findings = append(findings, fmt.Sprintf("%s process heap sys %.0f bytes exceeds %.0f", service, value, thresholds.ProcessHeapSysMaxBytes))
}
}
if thresholds.ProcessGoroutinesMax > 0 {
value, ok := metrics["vehicle_process_goroutines"]
if !ok {
findings = append(findings, fmt.Sprintf("%s process goroutines metric missing", service))
} else if value > thresholds.ProcessGoroutinesMax {
findings = append(findings, fmt.Sprintf("%s process goroutines %.0f exceeds %.0f", service, value, thresholds.ProcessGoroutinesMax))
}
}
if value := metrics["vehicle_gateway_connection_rejections_total"]; value > 0 {
findings = append(findings, fmt.Sprintf("gateway connection rejections %.0f", value))
}
if thresholds.AsyncSinkQueueDepthMax > 0 {
if value := metrics["vehicle_async_sink_queue_depth"]; value > thresholds.AsyncSinkQueueDepthMax {
findings = append(findings, fmt.Sprintf("async sink queue depth %.0f exceeds %.0f", value, thresholds.AsyncSinkQueueDepthMax))
}
}
if value := metrics["vehicle_bridge_nats_consumer_ack_pending"]; value > 100 {
findings = append(findings, fmt.Sprintf("bridge ack pending %.0f exceeds 100", value))
}
if value := metrics["vehicle_bridge_nats_consumer_pending"]; value > 10_000 {
findings = append(findings, fmt.Sprintf("bridge consumer pending %.0f exceeds 10000", value))
}
if value := metrics["vehicle_bridge_batch_pending_messages"]; value > 1000 {
findings = append(findings, fmt.Sprintf("bridge batch pending %.0f exceeds 1000", value))
}
if value := metrics["vehicle_fast_writer_nats_consumer_ack_pending"]; value > 100 {
findings = append(findings, fmt.Sprintf("fast writer ack pending %.0f exceeds 100", value))
}
if value := metrics["vehicle_fast_writer_nats_consumer_pending"]; value > 10_000 {
findings = append(findings, fmt.Sprintf("fast writer consumer pending %.0f exceeds 10000", value))
}
if value := metrics["vehicle_fast_writer_batch_pending_messages"]; thresholds.FastWriterBatchPendingMax > 0 && value > thresholds.FastWriterBatchPendingMax {
findings = append(findings, fmt.Sprintf("fast writer batch pending %.0f exceeds %.0f", value, thresholds.FastWriterBatchPendingMax))
}
if value := metrics["vehicle_fast_writer_tdengine_enabled"]; thresholds.FastWriterTDengineEnabledMax >= 0 && value > thresholds.FastWriterTDengineEnabledMax {
findings = append(findings, fmt.Sprintf("fast writer tdengine enabled %.0f exceeds %.0f", value, thresholds.FastWriterTDengineEnabledMax))
}
if value := metrics["vehicle_realtime_redis_projector_enabled"]; value > 0 {
findings = append(findings, fmt.Sprintf("realtime redis projector enabled %.0f", value))
}
if value := metrics["vehicle_history_batch_pending_messages"]; thresholds.HistoryBatchPendingMax > 0 && value > thresholds.HistoryBatchPendingMax {
findings = append(findings, fmt.Sprintf("history batch pending %.0f exceeds %.0f", value, thresholds.HistoryBatchPendingMax))
}
if value := metrics["vehicle_history_batch_pending_rows"]; thresholds.HistoryRowsPendingMax > 0 && value > thresholds.HistoryRowsPendingMax {
findings = append(findings, fmt.Sprintf("history rows pending %.0f exceeds %.0f", value, thresholds.HistoryRowsPendingMax))
}
if value := metrics["vehicle_stat_batch_pending_messages"]; thresholds.StatBatchPendingMax > 0 && value > thresholds.StatBatchPendingMax {
findings = append(findings, fmt.Sprintf("stat batch pending %.0f exceeds %.0f", value, thresholds.StatBatchPendingMax))
}
if value := metrics["vehicle_realtime_batch_pending_messages"]; thresholds.RealtimeBatchPendingMax > 0 && value > thresholds.RealtimeBatchPendingMax {
findings = append(findings, fmt.Sprintf("realtime batch pending %.0f exceeds %.0f", value, thresholds.RealtimeBatchPendingMax))
}
if value := metrics["vehicle_realtime_async_queue_depth"]; thresholds.RealtimeAsyncQueueDepthMax > 0 && value > thresholds.RealtimeAsyncQueueDepthMax {
findings = append(findings, fmt.Sprintf("realtime async queue depth %.0f exceeds %.0f", value, thresholds.RealtimeAsyncQueueDepthMax))
}
for _, retry := range []struct {
name string
metric string
}{
{name: "history", metric: "vehicle_history_retry_pending_messages"},
{name: "realtime", metric: "vehicle_realtime_retry_pending_messages"},
{name: "stat", metric: "vehicle_stat_retry_pending_messages"},
{name: "identity", metric: "vehicle_identity_writer_retry_pending_messages"},
{name: "fields projector", metric: "vehicle_fields_projector_retry_pending_messages"},
} {
if value := metrics[retry.metric]; thresholds.ConsumerRetryPendingMax >= 0 && value > thresholds.ConsumerRetryPendingMax {
findings = append(findings, fmt.Sprintf("%s retry pending %.0f exceeds %.0f", retry.name, value, thresholds.ConsumerRetryPendingMax))
}
}
if value := metrics["vehicle_history_kafka_lag"] + metrics["vehicle_stat_kafka_lag"] + metrics["vehicle_realtime_kafka_lag"] + metrics["vehicle_identity_writer_kafka_lag"] + metrics["vehicle_fields_projector_kafka_lag"]; thresholds.KafkaLagMax > 0 && value > thresholds.KafkaLagMax {
findings = append(findings, fmt.Sprintf("kafka lag %.0f exceeds %.0f", value, thresholds.KafkaLagMax))
}
if value := metrics["vehicle_identity_writer_batch_pending_messages"]; thresholds.RealtimeBatchPendingMax > 0 && value > thresholds.RealtimeBatchPendingMax {
findings = append(findings, fmt.Sprintf("identity batch pending %.0f exceeds %.0f", value, thresholds.RealtimeBatchPendingMax))
}
if value := metrics["vehicle_fields_projector_batch_pending_messages"]; thresholds.FieldsProjectorBatchPendingMax > 0 && value > thresholds.FieldsProjectorBatchPendingMax {
findings = append(findings, fmt.Sprintf("fields projector batch pending %.0f exceeds %.0f", value, thresholds.FieldsProjectorBatchPendingMax))
}
return findings
}
func findingsForFieldsProjector(parsed parsedMetricSet, thresholds Thresholds) []string {
var invalidJSON float64
var writeErrors float64
for _, sample := range parsed.samples {
switch {
case sample.name == "vehicle_fields_projector_kafka_messages_total" && sample.labels["status"] == "invalid_json":
invalidJSON += sample.value
case sample.name == "vehicle_fields_projector_kafka_writes_total" && sample.labels["status"] == "error":
writeErrors += sample.value
}
}
var findings []string
if thresholds.FieldsProjectorInvalidJSONMax >= 0 && invalidJSON > thresholds.FieldsProjectorInvalidJSONMax {
findings = append(findings, fmt.Sprintf("fields projector invalid json %.0f exceeds %.0f", invalidJSON, thresholds.FieldsProjectorInvalidJSONMax))
}
if thresholds.FieldsProjectorWriteErrorMax >= 0 && writeErrors > thresholds.FieldsProjectorWriteErrorMax {
findings = append(findings, fmt.Sprintf("fields projector kafka write errors %.0f exceeds %.0f", writeErrors, thresholds.FieldsProjectorWriteErrorMax))
}
return findings
}
func findingsForRuntimeConfig(service string, parsed parsedMetricSet, thresholds Thresholds) []string {
var findings []string
if service == "bridge" || hasServiceInfo(parsed, "nats-kafka-bridge") {
findings = append(findings, configMaxFinding(parsed, "vehicle_bridge_config", "fetch_wait_ms", "bridge fetch wait", "ms", thresholds.BridgeFetchWaitMaxMS)...)
findings = append(findings, configMinFinding(parsed, "vehicle_bridge_config", "workers", "bridge workers", "", thresholds.BridgeWorkersMin)...)
}
if service == "fields-projector" || hasServiceInfo(parsed, "vehicle-fields-projector") {
findings = append(findings, configMinFinding(parsed, "vehicle_fields_projector_config", "workers_per_protocol", "fields projector workers per protocol", "", thresholds.FieldsProjectorWorkersPerProtocolMin)...)
}
if service == "fast-writer" || hasServiceInfo(parsed, "nats-fast-writer") {
findings = append(findings, configMaxFinding(parsed, "vehicle_fast_writer_config", "fetch_wait_ms", "fast writer fetch wait", "ms", thresholds.FastWriterFetchWaitMaxMS)...)
findings = append(findings, configMinFinding(parsed, "vehicle_fast_writer_config", "workers", "fast writer workers", "", thresholds.FastWriterWorkersMin)...)
}
if service == "history" || hasServiceInfo(parsed, "vehicle-history-writer") {
findings = append(findings, configMinFinding(parsed, "vehicle_history_config", "workers", "history workers", "", thresholds.HistoryWorkersMin)...)
}
if service == "realtime" || hasServiceInfo(parsed, "vehicle-realtime-writer") {
findings = append(findings, configMinFinding(parsed, "vehicle_realtime_config", "workers", "realtime workers", "", thresholds.RealtimeWorkersMin)...)
}
if service == "stat" || hasServiceInfo(parsed, "vehicle-stat-writer") {
findings = append(findings, configMinFinding(parsed, "vehicle_stat_config", "workers", "stat workers", "", thresholds.StatWorkersMin)...)
}
if service == "identity" || hasServiceInfo(parsed, "vehicle-identity-writer") {
findings = append(findings, configMinFinding(parsed, "vehicle_identity_writer_config", "workers", "identity workers", "", thresholds.IdentityWorkersMin)...)
}
sort.Strings(findings)
return findings
}
func configMaxFinding(parsed parsedMetricSet, metricName string, setting string, label string, unit string, threshold float64) []string {
if threshold <= 0 {
return nil
}
value, ok := maxConfigGaugeValue(parsed, metricName, setting)
if !ok {
return []string{fmt.Sprintf("%s config metric missing (%s setting=%s)", label, metricName, setting)}
}
if value <= threshold {
return nil
}
return []string{fmt.Sprintf("%s %.0f%s exceeds %.0f%s", label, value, unit, threshold, unit)}
}
func configMinFinding(parsed parsedMetricSet, metricName string, setting string, label string, unit string, threshold float64) []string {
if threshold <= 0 {
return nil
}
value, ok := minConfigGaugeValue(parsed, metricName, setting)
if !ok {
return []string{fmt.Sprintf("%s config metric missing (%s setting=%s)", label, metricName, setting)}
}
if value >= threshold {
return nil
}
return []string{fmt.Sprintf("%s %.0f%s below %.0f%s", label, value, unit, threshold, unit)}
}
func hasServiceInfo(parsed parsedMetricSet, service string) bool {
service = strings.TrimSpace(service)
if service == "" {
return false
}
for _, sample := range parsed.samples {
if sample.name == "vehicle_service_info" && sample.labels["service"] == service && sample.value > 0 {
return true
}
}
return false
}
func maxConfigGaugeValue(parsed parsedMetricSet, metricName string, setting string) (float64, bool) {
var out float64
found := false
for _, sample := range parsed.samples {
if sample.name != metricName || sample.labels["setting"] != setting {
continue
}
if !found || sample.value > out {
out = sample.value
found = true
}
}
return out, found
}
func minConfigGaugeValue(parsed parsedMetricSet, metricName string, setting string) (float64, bool) {
var out float64
found := false
for _, sample := range parsed.samples {
if sample.name != metricName || sample.labels["setting"] != setting {
continue
}
if !found || sample.value < out {
out = sample.value
found = true
}
}
return out, found
}
func findingsForAsyncSinkQueueUsage(service string, parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.AsyncSinkQueueMaxUsageRatio <= 0 {
return nil
}
type queueSample struct {
labels map[string]string
depth float64
capacity float64
}
queues := map[string]*queueSample{}
for _, sample := range parsed.samples {
switch sample.name {
case "vehicle_async_sink_queue_depth":
key := metricLabelsKey(sample.labels)
item := queues[key]
if item == nil {
item = &queueSample{labels: sample.labels}
queues[key] = item
}
item.depth += sample.value
case "vehicle_async_sink_queue_capacity":
key := metricLabelsKey(sample.labels)
item := queues[key]
if item == nil {
item = &queueSample{labels: sample.labels}
queues[key] = item
}
item.capacity += sample.value
}
}
var keys []string
for key := range queues {
keys = append(keys, key)
}
sort.Strings(keys)
var findings []string
for _, key := range keys {
item := queues[key]
if item.capacity <= 0 || item.depth <= 0 {
continue
}
ratio := item.depth / item.capacity
if ratio <= thresholds.AsyncSinkQueueMaxUsageRatio {
continue
}
findings = append(findings, fmt.Sprintf(
"%s async sink queue usage %.2f exceeds %.2f (%s, depth %.0f, capacity %.0f)",
service,
ratio,
thresholds.AsyncSinkQueueMaxUsageRatio,
key,
item.depth,
item.capacity,
))
}
return findings
}
func findingsForAsyncSinkErrors(service string, parsed parsedMetricSet, thresholds Thresholds) []string {
var findings []string
if thresholds.AsyncSinkEnqueueErrorMax >= 0 {
for _, sample := range parsed.samples {
if sample.name != "vehicle_async_sink_enqueue_total" {
continue
}
status := strings.TrimSpace(sample.labels["status"])
if status == "" || status == "queued" {
continue
}
if sample.value <= thresholds.AsyncSinkEnqueueErrorMax {
continue
}
sink := strings.TrimSpace(sample.labels["sink"])
if sink == "" {
sink = "unknown"
}
kind := strings.TrimSpace(sample.labels["kind"])
if kind == "" {
kind = "unknown"
}
findings = append(findings, fmt.Sprintf(
"%s async sink enqueue %s %.0f exceeds %.0f for sink %s kind %s",
service,
status,
sample.value,
thresholds.AsyncSinkEnqueueErrorMax,
sink,
kind,
))
}
}
if thresholds.AsyncSinkPublishErrorMax >= 0 {
for _, sample := range parsed.samples {
if sample.name != "vehicle_async_sink_publish_total" || sample.labels["status"] != "error" {
continue
}
if sample.value <= thresholds.AsyncSinkPublishErrorMax {
continue
}
sink := strings.TrimSpace(sample.labels["sink"])
if sink == "" {
sink = "unknown"
}
kind := strings.TrimSpace(sample.labels["kind"])
if kind == "" {
kind = "unknown"
}
findings = append(findings, fmt.Sprintf(
"%s async sink publish errors %.0f exceeds %.0f for sink %s kind %s",
service,
sample.value,
thresholds.AsyncSinkPublishErrorMax,
sink,
kind,
))
}
}
sort.Strings(findings)
return findings
}
func findingsForDurableSpool(parsed parsedMetricSet, thresholds Thresholds) []string {
var findings []string
if thresholds.DurableSpoolBacklogMax > 0 {
for _, sample := range parsed.samples {
if sample.name != "vehicle_durable_spool_backlog_files" || sample.value <= thresholds.DurableSpoolBacklogMax {
continue
}
name := strings.TrimSpace(sample.labels["name"])
if name == "" {
name = "unknown"
}
findings = append(findings, fmt.Sprintf(
"durable spool backlog %.0f exceeds %.0f for name %s",
sample.value,
thresholds.DurableSpoolBacklogMax,
name,
))
}
}
if thresholds.DurableSpoolOldestAgeMaxSec > 0 {
for _, sample := range parsed.samples {
if sample.name != "vehicle_durable_spool_oldest_age_seconds" || sample.value <= thresholds.DurableSpoolOldestAgeMaxSec {
continue
}
name := strings.TrimSpace(sample.labels["name"])
if name == "" {
name = "unknown"
}
findings = append(findings, fmt.Sprintf(
"durable spool oldest age %.0fs exceeds %.0fs for name %s",
sample.value,
thresholds.DurableSpoolOldestAgeMaxSec,
name,
))
}
}
if thresholds.DurableSpoolErrorMax >= 0 {
for _, sample := range parsed.samples {
if sample.name != "vehicle_durable_spool_records_total" || sample.labels["status"] != "error" || sample.value <= thresholds.DurableSpoolErrorMax {
continue
}
name := strings.TrimSpace(sample.labels["name"])
if name == "" {
name = "unknown"
}
kind := strings.TrimSpace(sample.labels["kind"])
if kind == "" {
kind = "unknown"
}
findings = append(findings, fmt.Sprintf(
"durable spool record errors %.0f exceeds %.0f for name %s kind %s",
sample.value,
thresholds.DurableSpoolErrorMax,
name,
kind,
))
}
}
if thresholds.DurableSpoolReplayErrorMax >= 0 {
for _, sample := range parsed.samples {
if sample.name != "vehicle_durable_spool_replay_total" || !strings.HasSuffix(sample.labels["status"], "_error") || sample.value <= thresholds.DurableSpoolReplayErrorMax {
continue
}
name := strings.TrimSpace(sample.labels["name"])
if name == "" {
name = "unknown"
}
status := strings.TrimSpace(sample.labels["status"])
if status == "" {
status = "unknown"
}
findings = append(findings, fmt.Sprintf(
"durable spool replay %s %.0f exceeds %.0f for name %s",
status,
sample.value,
thresholds.DurableSpoolReplayErrorMax,
name,
))
}
}
sort.Strings(findings)
return findings
}
func findingsForDurableOutbox(parsed parsedMetricSet, thresholds Thresholds) []string {
var findings []string
if thresholds.DurableOutboxInflightMax > 0 {
for _, sample := range parsed.samples {
if sample.name != "vehicle_durable_outbox_inflight" || sample.value <= thresholds.DurableOutboxInflightMax {
continue
}
name := strings.TrimSpace(sample.labels["name"])
if name == "" {
name = "unknown"
}
findings = append(findings, fmt.Sprintf(
"durable outbox inflight %.0f exceeds %.0f for name %s",
sample.value,
thresholds.DurableOutboxInflightMax,
name,
))
}
}
if thresholds.DurableOutboxErrorMax >= 0 {
for _, sample := range parsed.samples {
if sample.name != "vehicle_durable_outbox_publish_total" || !durableOutboxErrorStatus(sample.labels["status"]) || sample.value <= thresholds.DurableOutboxErrorMax {
continue
}
name := strings.TrimSpace(sample.labels["name"])
if name == "" {
name = "unknown"
}
kind := strings.TrimSpace(sample.labels["kind"])
if kind == "" {
kind = "unknown"
}
status := strings.TrimSpace(sample.labels["status"])
findings = append(findings, fmt.Sprintf(
"durable outbox %s %.0f exceeds %.0f for name %s kind %s",
status,
sample.value,
thresholds.DurableOutboxErrorMax,
name,
kind,
))
}
}
sort.Strings(findings)
return findings
}
func durableOutboxErrorStatus(status string) bool {
switch strings.TrimSpace(status) {
case "validation_error", "submit_error", "ack_error", "remove_error", "close_timeout", "quarantined":
return true
default:
return false
}
}
func findingsForHistograms(parsed parsedMetricSet, thresholds Thresholds) []string {
var findings []string
findings = append(findings, histogramP99Findings(parsed, "vehicle_gateway_frame_duration_ms_histogram", "gateway frame", thresholds.GatewayFrameP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_gateway_identity_duration_ms_histogram", "gateway identity", thresholds.GatewayIdentityP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_gateway_response_e2e_recent_p99_ms", "vehicle_gateway_response_e2e_recent_samples", "gateway response e2e", thresholds.GatewayResponseRecentP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_async_sink_publish_duration_ms_histogram", "async sink publish", thresholds.AsyncSinkP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_async_sink_queue_wait_recent_p99_ms", "vehicle_async_sink_queue_wait_recent_samples", "async sink queue wait", thresholds.AsyncSinkQueueWaitRecentP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_bridge_batch_duration_ms_histogram", "bridge batch", thresholds.BridgeBatchP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_bridge_kafka_write_duration_ms_histogram", "bridge kafka write", thresholds.BridgeKafkaWriteP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_bridge_kafka_e2e_duration_ms_histogram", "bridge kafka e2e", thresholds.BridgeKafkaE2EP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_bridge_kafka_e2e_recent_p99_ms", "vehicle_bridge_kafka_e2e_recent_samples", "bridge kafka e2e", thresholds.BridgeKafkaRecentE2EP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_fast_writer_stage_duration_ms_histogram", "fast writer stage", thresholds.FastWriterStageP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_fast_writer_redis_e2e_duration_ms_histogram", "fast writer redis e2e", thresholds.FastWriterRedisE2EP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_fast_writer_redis_e2e_recent_p99_ms", "vehicle_fast_writer_redis_e2e_recent_samples", "fast writer redis e2e", thresholds.FastWriterRedisRecentE2EP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_history_batch_flush_duration_ms_histogram", "history flush", thresholds.HistoryFlushP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_history_write_e2e_duration_ms_histogram", "history write e2e", thresholds.HistoryWriteE2EP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_history_write_e2e_recent_p99_ms", "vehicle_history_write_e2e_recent_samples", "history write e2e", thresholds.HistoryWriteRecentE2EP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_realtime_batch_flush_duration_ms_histogram", "realtime batch flush", thresholds.RealtimeBatchP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_realtime_store_update_duration_ms_histogram", "realtime store update", thresholds.RealtimeStoreP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_stat_write_duration_ms_histogram", "stat write", thresholds.StatWriteP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, histogramP99Findings(parsed, "vehicle_stat_write_e2e_duration_ms_histogram", "stat write e2e", thresholds.StatWriteE2EP99MS, thresholds.MinHistogramSamples)...)
findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_stat_write_e2e_recent_p99_ms", "vehicle_stat_write_e2e_recent_samples", "stat write e2e", thresholds.StatWriteRecentE2EP99MS, thresholds.MinHistogramSamples)...)
return findings
}
type gatewayPublishCounters struct {
rawOK float64
fieldsOK float64
fieldsPublished float64
fieldsDelegated float64
fieldsSkippedNonRealtime float64
fieldsSkippedMissing float64
fieldsPublishError float64
}
type gatewayFieldFlowGroup struct {
gatewayPublishCounters
latestCount float64
hasLatestCount bool
}
type gatewayIdentityCounters struct {
total float64
resolved float64
unresolved float64
}
type identityResolutionGroup struct {
protocol string
total int64
skipped int64
resolved int64
unresolved int64
timeout int64
errorCount int64
otherNonResolved int64
staleCache int64
cacheStatusCounts map[string]int64
}
type pipelineLatencySpec struct {
service string
stage string
metricName string
samplesMetricName string
keyLabel string
thresholdMS float64
}
type gatewayParseCounters struct {
total float64
ok float64
partial float64
badFrame float64
other float64
}
type gatewayResponseCounters struct {
total float64
ok float64
buildError float64
writeError float64
}
type historyParsedFieldCounters struct {
present float64
missing float64
}
type kafkaConsumerCompletionCounters struct {
received float64
committedOK float64
commitError float64
}
type kafkaConsumerCompletionSpec struct {
serviceName string
messageMetric string
commitMetric string
}
func findingsForKafkaConsumerCompletion(service string, parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.KafkaConsumerMessageMinReceived <= 0 || thresholds.KafkaConsumerMaxUncommittedRatio <= 0 {
return nil
}
spec, ok := kafkaConsumerCompletionSpecForService(service)
if !ok {
return nil
}
groups := map[string]*kafkaConsumerCompletionCounters{}
for _, sample := range parsed.samples {
switch sample.name {
case spec.messageMetric, spec.commitMetric:
default:
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if topic == "" || topic == "mixed" || topic == "unknown" {
continue
}
group := groups[topic]
if group == nil {
group = &kafkaConsumerCompletionCounters{}
groups[topic] = group
}
switch {
case sample.name == spec.messageMetric && sample.labels["status"] == "received":
group.received += sample.value
case sample.name == spec.commitMetric && sample.labels["status"] == "ok":
group.committedOK += sample.value
case sample.name == spec.commitMetric && sample.labels["status"] == "error":
group.commitError += sample.value
}
}
var topics []string
for topic := range groups {
topics = append(topics, topic)
}
sort.Strings(topics)
var findings []string
for _, topic := range topics {
group := groups[topic]
if thresholds.KafkaConsumerCommitErrorMax >= 0 && group.commitError > thresholds.KafkaConsumerCommitErrorMax {
findings = append(findings, fmt.Sprintf(
"%s kafka consumer commit errors %.0f exceeds %.0f for topic %s",
spec.serviceName,
group.commitError,
thresholds.KafkaConsumerCommitErrorMax,
topic,
))
}
if group.received < thresholds.KafkaConsumerMessageMinReceived {
continue
}
uncommitted := group.received - group.committedOK
if uncommitted < 0 {
uncommitted = 0
}
ratio := uncommitted / group.received
if ratio <= thresholds.KafkaConsumerMaxUncommittedRatio {
continue
}
findings = append(findings, fmt.Sprintf(
"%s kafka consumer uncommitted message ratio %.2f exceeds %.2f for topic %s: received %.0f, commit ok %.0f, commit errors %.0f, uncommitted %.0f",
spec.serviceName,
ratio,
thresholds.KafkaConsumerMaxUncommittedRatio,
topic,
group.received,
group.committedOK,
group.commitError,
uncommitted,
))
}
return findings
}
func kafkaConsumerCompletionSpecForService(service string) (kafkaConsumerCompletionSpec, bool) {
switch service {
case "history":
return kafkaConsumerCompletionSpec{
serviceName: "history",
messageMetric: "vehicle_history_kafka_messages_total",
commitMetric: "vehicle_history_kafka_commits_total",
}, true
case "realtime":
return kafkaConsumerCompletionSpec{
serviceName: "realtime",
messageMetric: "vehicle_realtime_kafka_messages_total",
commitMetric: "vehicle_realtime_kafka_commits_total",
}, true
case "stat":
return kafkaConsumerCompletionSpec{
serviceName: "stat",
messageMetric: "vehicle_stat_kafka_messages_total",
commitMetric: "vehicle_stat_kafka_commits_total",
}, true
case "identity":
return kafkaConsumerCompletionSpec{
serviceName: "identity",
messageMetric: "vehicle_identity_writer_kafka_messages_total",
commitMetric: "vehicle_identity_writer_kafka_commits_total",
}, true
default:
return kafkaConsumerCompletionSpec{}, false
}
}
func findingsForMessageContractMismatches(service string, parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.MessageContractMismatchMax < 0 {
return nil
}
specs := messageContractMismatchSpecs(service)
if len(specs) == 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
status := strings.TrimSpace(sample.labels["status"])
if status == "" {
continue
}
for _, spec := range specs {
if sample.name != spec.metric || status != spec.status {
continue
}
if sample.value <= thresholds.MessageContractMismatchMax {
continue
}
key := strings.TrimSpace(sample.labels[spec.keyLabel])
if key == "" {
key = "unknown"
}
findings = append(findings, fmt.Sprintf(
"%s message contract %s %.0f exceeds %.0f for %s %s",
spec.serviceLabel,
status,
sample.value,
thresholds.MessageContractMismatchMax,
spec.keyLabel,
key,
))
}
}
sort.Strings(findings)
return findings
}
type messageContractMismatchSpec struct {
serviceLabel string
metric string
status string
keyLabel string
}
func messageContractMismatchSpecs(service string) []messageContractMismatchSpec {
switch service {
case "history":
return []messageContractMismatchSpec{
{serviceLabel: "history", metric: "vehicle_history_kafka_messages_total", status: "event_kind_mismatch", keyLabel: "topic"},
{serviceLabel: "history", metric: "vehicle_history_kafka_messages_total", status: "protocol_topic_mismatch", keyLabel: "topic"},
}
case "realtime":
return []messageContractMismatchSpec{
{serviceLabel: "realtime", metric: "vehicle_realtime_kafka_messages_total", status: "event_kind_mismatch", keyLabel: "topic"},
{serviceLabel: "realtime", metric: "vehicle_realtime_kafka_messages_total", status: "protocol_topic_mismatch", keyLabel: "topic"},
}
case "stat":
return []messageContractMismatchSpec{
{serviceLabel: "stat", metric: "vehicle_stat_kafka_messages_total", status: "event_kind_mismatch", keyLabel: "topic"},
}
case "identity":
return []messageContractMismatchSpec{
{serviceLabel: "identity", metric: "vehicle_identity_writer_kafka_messages_total", status: "event_kind_mismatch", keyLabel: "topic"},
{serviceLabel: "identity", metric: "vehicle_identity_writer_kafka_messages_total", status: "protocol_topic_mismatch", keyLabel: "topic"},
}
case "fast-writer":
return []messageContractMismatchSpec{
{serviceLabel: "fast writer", metric: "vehicle_fast_writer_messages_total", status: "event_kind_mismatch", keyLabel: "subject"},
{serviceLabel: "fast writer", metric: "vehicle_fast_writer_messages_total", status: "protocol_topic_mismatch", keyLabel: "subject"},
}
default:
return nil
}
}
func findingsForIdentityWriterOwnership(gateway parsedMetricSet, identityWriter parsedMetricSet) []string {
if len(identityWriter.samples) == 0 {
return nil
}
for _, sample := range gateway.samples {
if sample.name != "vehicle_gateway_jt808_registration_gateway_writes_enabled" {
continue
}
if sample.value > 0 {
return []string{"jt808 registration has duplicate writers: gateway writes remain enabled while identity writer is active"}
}
return nil
}
return []string{"gateway jt808 registration ownership metric missing while identity writer is active"}
}
func findingsForHistoryParsedFields(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.HistoryParsedFieldMinFrames <= 0 || thresholds.HistoryParsedFieldMaxMissingRatio <= 0 {
return nil
}
groups := map[string]*historyParsedFieldCounters{}
for _, sample := range parsed.samples {
if sample.name != "vehicle_history_parsed_fields_total" {
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if topic == "" {
topic = "unknown"
}
protocol := strings.TrimSpace(sample.labels["protocol"])
if protocol == "" {
protocol = "UNKNOWN"
}
key := topic + "\x00" + protocol
group := groups[key]
if group == nil {
group = &historyParsedFieldCounters{}
groups[key] = group
}
switch sample.labels["status"] {
case "present":
group.present += sample.value
case "missing":
group.missing += sample.value
}
}
var keys []string
for key := range groups {
keys = append(keys, key)
}
sort.Strings(keys)
var findings []string
for _, key := range keys {
group := groups[key]
total := group.present + group.missing
if total < thresholds.HistoryParsedFieldMinFrames || total <= 0 {
continue
}
ratio := group.missing / total
if ratio <= thresholds.HistoryParsedFieldMaxMissingRatio {
continue
}
topic, protocol, _ := strings.Cut(key, "\x00")
findings = append(findings, fmt.Sprintf(
"history parsed fields missing ratio %.2f exceeds %.2f for topic %s protocol %s (total %.0f, present %.0f, missing %.0f)",
ratio,
thresholds.HistoryParsedFieldMaxMissingRatio,
topic,
protocol,
total,
group.present,
group.missing,
))
}
return findings
}
func findingsForHistoryInvalidJSON(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.HistoryInvalidJSONMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_history_kafka_messages_total" || sample.labels["status"] != "invalid_json" {
continue
}
if sample.value <= thresholds.HistoryInvalidJSONMax {
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if topic == "" {
topic = "unknown"
}
findings = append(findings, fmt.Sprintf(
"history invalid json %.0f exceeds %.0f for topic %s",
sample.value,
thresholds.HistoryInvalidJSONMax,
topic,
))
}
sort.Strings(findings)
return findings
}
func findingsForRealtimeInvalidJSON(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.RealtimeInvalidJSONMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_realtime_kafka_messages_total" || sample.labels["status"] != "invalid_json" {
continue
}
if sample.value <= thresholds.RealtimeInvalidJSONMax {
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if topic == "" {
topic = "unknown"
}
findings = append(findings, fmt.Sprintf(
"realtime invalid json %.0f exceeds %.0f for topic %s",
sample.value,
thresholds.RealtimeInvalidJSONMax,
topic,
))
}
sort.Strings(findings)
return findings
}
func findingsForRealtimeStoreErrors(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.RealtimeStoreErrorMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_realtime_store_updates_total" || sample.labels["status"] != "error" {
continue
}
if sample.value <= thresholds.RealtimeStoreErrorMax {
continue
}
store := strings.TrimSpace(sample.labels["store"])
if store == "" {
store = "unknown"
}
protocol := strings.TrimSpace(sample.labels["protocol"])
if protocol == "" {
protocol = "UNKNOWN"
}
findings = append(findings, fmt.Sprintf(
"realtime store update errors %.0f exceeds %.0f for store %s protocol %s",
sample.value,
thresholds.RealtimeStoreErrorMax,
store,
protocol,
))
}
sort.Strings(findings)
return findings
}
func findingsForRealtimeAsyncQueue(parsed parsedMetricSet, thresholds Thresholds) []string {
var findings []string
if thresholds.RealtimeAsyncQueueDroppedMax >= 0 {
for _, sample := range parsed.samples {
if sample.name != "vehicle_realtime_async_queue_total" {
continue
}
status := sample.labels["status"]
if status != "dropped" && status != "closed" {
continue
}
if sample.value <= thresholds.RealtimeAsyncQueueDroppedMax {
continue
}
store := strings.TrimSpace(sample.labels["store"])
if store == "" {
store = "unknown"
}
protocol := strings.TrimSpace(sample.labels["protocol"])
if protocol == "" {
protocol = "UNKNOWN"
}
findings = append(findings, fmt.Sprintf(
"realtime async queue %s %.0f exceeds %.0f for store %s protocol %s",
status,
sample.value,
thresholds.RealtimeAsyncQueueDroppedMax,
store,
protocol,
))
}
}
if thresholds.RealtimeAsyncQueueMaxUsageRatio <= 0 {
sort.Strings(findings)
return findings
}
type asyncQueueSample struct {
labels map[string]string
depth float64
}
queues := map[string]*asyncQueueSample{}
capacityByKey := map[string]float64{}
capacityByStore := map[string]float64{}
for _, sample := range parsed.samples {
switch sample.name {
case "vehicle_realtime_async_queue_depth":
key := metricLabelsKey(sample.labels)
item := queues[key]
if item == nil {
item = &asyncQueueSample{labels: sample.labels}
queues[key] = item
}
item.depth += sample.value
case "vehicle_realtime_async_queue_capacity":
key := metricLabelsKey(sample.labels)
capacityByKey[key] += sample.value
store := strings.TrimSpace(sample.labels["store"])
if store != "" {
capacityByStore[store] += sample.value
}
}
}
var keys []string
for key := range queues {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
item := queues[key]
capacity := capacityByKey[key]
if capacity <= 0 {
capacity = capacityByStore[strings.TrimSpace(item.labels["store"])]
}
if capacity <= 0 || item.depth <= 0 {
continue
}
ratio := item.depth / capacity
if ratio <= thresholds.RealtimeAsyncQueueMaxUsageRatio {
continue
}
findings = append(findings, fmt.Sprintf(
"realtime async queue usage %.2f exceeds %.2f (%s, depth %.0f, capacity %.0f)",
ratio,
thresholds.RealtimeAsyncQueueMaxUsageRatio,
key,
item.depth,
capacity,
))
}
sort.Strings(findings)
return findings
}
func findingsForGatewayResponseQuality(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.GatewayResponseMinSamples <= 0 || thresholds.GatewayResponseMaxErrorRatio <= 0 {
return nil
}
groups := map[string]*gatewayResponseCounters{}
for _, sample := range parsed.samples {
if sample.name != "vehicle_gateway_response_total" {
continue
}
protocol := strings.TrimSpace(sample.labels["protocol"])
if protocol == "" {
protocol = "unknown"
}
group := groups[protocol]
if group == nil {
group = &gatewayResponseCounters{}
groups[protocol] = group
}
switch sample.labels["status"] {
case "ok":
group.ok += sample.value
group.total += sample.value
case "build_error":
group.buildError += sample.value
group.total += sample.value
case "write_error":
group.writeError += sample.value
group.total += sample.value
}
}
var protocols []string
for protocol := range groups {
protocols = append(protocols, protocol)
}
sort.Strings(protocols)
var findings []string
for _, protocol := range protocols {
group := groups[protocol]
if group.total < thresholds.GatewayResponseMinSamples || group.total <= 0 {
continue
}
errors := group.buildError + group.writeError
ratio := errors / group.total
if ratio <= thresholds.GatewayResponseMaxErrorRatio {
continue
}
findings = append(findings, fmt.Sprintf(
"gateway response error ratio %.2f exceeds %.2f for protocol %s (total %.0f, ok %.0f, errors %.0f, build_error %.0f, write_error %.0f)",
ratio,
thresholds.GatewayResponseMaxErrorRatio,
protocol,
group.total,
group.ok,
errors,
group.buildError,
group.writeError,
))
}
return findings
}
func findingsForGatewayParseQuality(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.GatewayParseMinFrames <= 0 || thresholds.GatewayParseMaxBadRatio <= 0 {
return nil
}
groups := map[string]*gatewayParseCounters{}
for _, sample := range parsed.samples {
if sample.name != "vehicle_gateway_frames_total" {
continue
}
protocol := strings.TrimSpace(sample.labels["protocol"])
if protocol == "" {
protocol = "unknown"
}
group := groups[protocol]
if group == nil {
group = &gatewayParseCounters{}
groups[protocol] = group
}
group.total += sample.value
switch envelope.ParseStatus(sample.labels["status"]) {
case envelope.ParseOK:
group.ok += sample.value
case envelope.ParsePartial:
group.partial += sample.value
case envelope.ParseBadFrame:
group.badFrame += sample.value
default:
group.other += sample.value
}
}
var protocols []string
for protocol := range groups {
protocols = append(protocols, protocol)
}
sort.Strings(protocols)
var findings []string
for _, protocol := range protocols {
group := groups[protocol]
if group.total < thresholds.GatewayParseMinFrames || group.total <= 0 {
continue
}
bad := group.partial + group.badFrame + group.other
ratio := bad / group.total
if ratio <= thresholds.GatewayParseMaxBadRatio {
continue
}
findings = append(findings, fmt.Sprintf(
"gateway parse bad ratio %.2f exceeds %.2f for protocol %s (total %.0f, ok %.0f, bad %.0f, partial %.0f, bad_frame %.0f, other %.0f)",
ratio,
thresholds.GatewayParseMaxBadRatio,
protocol,
group.total,
group.ok,
bad,
group.partial,
group.badFrame,
group.other,
))
}
return findings
}
func findingsForGatewayIdentityQuality(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.GatewayIdentityMinSamples <= 0 || thresholds.GatewayIdentityMaxUnresolvedRatio <= 0 {
return nil
}
groups := map[string]*gatewayIdentityCounters{}
for _, sample := range parsed.samples {
if sample.name != "vehicle_gateway_identity_total" {
continue
}
protocol := strings.TrimSpace(sample.labels["protocol"])
if protocol == "" {
protocol = "unknown"
}
group := groups[protocol]
if group == nil {
group = &gatewayIdentityCounters{}
groups[protocol] = group
}
group.total += sample.value
if sample.labels["status"] == "resolved" {
group.resolved += sample.value
} else {
group.unresolved += sample.value
}
}
var protocols []string
for protocol := range groups {
protocols = append(protocols, protocol)
}
sort.Strings(protocols)
var findings []string
for _, protocol := range protocols {
group := groups[protocol]
if group.total < thresholds.GatewayIdentityMinSamples || group.total <= 0 {
continue
}
ratio := group.unresolved / group.total
if ratio <= thresholds.GatewayIdentityMaxUnresolvedRatio {
continue
}
findings = append(findings, fmt.Sprintf(
"gateway identity unresolved ratio %.2f exceeds %.2f for protocol %s (total %.0f, unresolved %.0f, resolved %.0f)",
ratio,
thresholds.GatewayIdentityMaxUnresolvedRatio,
protocol,
group.total,
group.unresolved,
group.resolved,
))
}
return findings
}
func buildIdentityResolutionDiagnostics(parsed parsedMetricSet) *IdentityResolutionDiagnostics {
groups := map[string]*identityResolutionGroup{}
cacheEntries := map[string]int64{}
issues := map[string]IdentityIssueCount{}
for _, sample := range parsed.samples {
switch sample.name {
case "vehicle_gateway_identity_total":
protocol := normalizedMetricLabel(sample.labels, "protocol")
status := normalizedMetricLabel(sample.labels, "status")
group := identityResolutionGroupForProtocol(groups, protocol)
value := metricCount(sample.value)
group.total += value
switch status {
case "resolved":
group.resolved += value
case "unresolved":
group.unresolved += value
case "timeout":
group.timeout += value
case "error":
group.errorCount += value
default:
group.otherNonResolved += value
}
case "vehicle_gateway_identity_skips_total":
protocol := normalizedMetricLabel(sample.labels, "protocol")
group := identityResolutionGroupForProtocol(groups, protocol)
group.skipped += metricCount(sample.value)
case "vehicle_gateway_identity_issues_total":
issue := IdentityIssueCount{
Protocol: normalizedMetricLabel(sample.labels, "protocol"),
Status: normalizedMetricLabel(sample.labels, "status"),
MessageID: normalizedMetricLabel(sample.labels, "message_id"),
Reason: normalizedMetricLabel(sample.labels, "reason"),
Count: metricCount(sample.value),
}
if issue.Count <= 0 {
continue
}
key := strings.Join([]string{issue.Protocol, issue.Status, issue.MessageID, issue.Reason}, "\x00")
existing := issues[key]
if existing.Count == 0 {
existing = issue
} else {
existing.Count += issue.Count
}
issues[key] = existing
case "vehicle_gateway_identity_cache_total":
protocol := normalizedMetricLabel(sample.labels, "protocol")
cacheStatus := normalizedMetricLabel(sample.labels, "cache_status")
group := identityResolutionGroupForProtocol(groups, protocol)
value := metricCount(sample.value)
if group.cacheStatusCounts == nil {
group.cacheStatusCounts = map[string]int64{}
}
group.cacheStatusCounts[cacheStatus] += value
if cacheStatus == "stale" {
group.staleCache += value
}
case "vehicle_gateway_identity_cache_entries":
cacheName := normalizedMetricLabel(sample.labels, "cache")
cacheEntries[cacheName] += metricCount(sample.value)
}
}
if len(groups) == 0 && len(cacheEntries) == 0 && len(issues) == 0 {
return nil
}
diagnostics := &IdentityResolutionDiagnostics{}
if len(cacheEntries) > 0 {
diagnostics.CacheEntries = cacheEntries
}
if len(issues) > 0 {
diagnostics.Issues = sortedIdentityIssues(issues)
}
var protocols []string
for protocol := range groups {
protocols = append(protocols, protocol)
}
sort.Strings(protocols)
for _, protocol := range protocols {
group := groups[protocol]
row := identityProtocolResolution(group)
diagnostics.Total += row.Total
diagnostics.SkippedCount += row.SkippedCount
diagnostics.ResolvedCount += row.ResolvedCount
diagnostics.NonResolvedCount += row.NonResolvedCount
diagnostics.UnresolvedCount += row.UnresolvedCount
diagnostics.TimeoutCount += row.TimeoutCount
diagnostics.ErrorCount += row.ErrorCount
diagnostics.OtherNonResolvedCount += row.OtherNonResolvedCount
diagnostics.StaleCacheCount += row.StaleCacheCount
diagnostics.Protocols = append(diagnostics.Protocols, row)
}
diagnostics.NonResolvedRatio = ratioInt64(diagnostics.NonResolvedCount, diagnostics.Total)
diagnostics.StaleCacheRatio = ratioInt64(diagnostics.StaleCacheCount, diagnostics.Total)
return diagnostics
}
func sortedIdentityIssues(issues map[string]IdentityIssueCount) []IdentityIssueCount {
rows := make([]IdentityIssueCount, 0, len(issues))
for _, row := range issues {
rows = append(rows, row)
}
sort.Slice(rows, func(i, j int) bool {
if rows[i].Protocol != rows[j].Protocol {
return rows[i].Protocol < rows[j].Protocol
}
if rows[i].Status != rows[j].Status {
return rows[i].Status < rows[j].Status
}
if rows[i].MessageID != rows[j].MessageID {
return rows[i].MessageID < rows[j].MessageID
}
return rows[i].Reason < rows[j].Reason
})
return rows
}
func identityProtocolResolution(group *identityResolutionGroup) IdentityProtocolResolution {
if group == nil {
return IdentityProtocolResolution{Protocol: "unknown"}
}
nonResolved := group.unresolved + group.timeout + group.errorCount + group.otherNonResolved
row := IdentityProtocolResolution{
Protocol: group.protocol,
Total: group.total,
SkippedCount: group.skipped,
ResolvedCount: group.resolved,
NonResolvedCount: nonResolved,
UnresolvedCount: group.unresolved,
TimeoutCount: group.timeout,
ErrorCount: group.errorCount,
OtherNonResolvedCount: group.otherNonResolved,
NonResolvedRatio: ratioInt64(nonResolved, group.total),
StaleCacheCount: group.staleCache,
StaleCacheRatio: ratioInt64(group.staleCache, group.total),
CacheStatusCounts: group.cacheStatusCounts,
}
return row
}
func identityResolutionGroupForProtocol(groups map[string]*identityResolutionGroup, protocol string) *identityResolutionGroup {
if protocol == "" {
protocol = "unknown"
}
group := groups[protocol]
if group == nil {
group = &identityResolutionGroup{protocol: protocol}
groups[protocol] = group
}
return group
}
func normalizedMetricLabel(labels map[string]string, key string) string {
value := strings.TrimSpace(labels[key])
if value == "" {
return "unknown"
}
return value
}
func metricCount(value float64) int64 {
if value <= 0 {
return 0
}
return int64(math.Round(value))
}
func ratioInt64(numerator int64, denominator int64) float64 {
if denominator <= 0 {
return 0
}
return float64(numerator) / float64(denominator)
}
func buildPipelineLatencyDiagnostics(parsedByService map[string]parsedMetricSet, thresholds Thresholds) *PipelineLatencyDiagnostics {
specs := []pipelineLatencySpec{
{
service: "gateway",
stage: "gateway_protocol_response_e2e",
metricName: "vehicle_gateway_response_e2e_recent_p99_ms",
samplesMetricName: "vehicle_gateway_response_e2e_recent_samples",
keyLabel: "protocol",
thresholdMS: thresholds.GatewayResponseRecentP99MS,
},
{
service: "bridge",
stage: "bridge_kafka_e2e",
metricName: "vehicle_bridge_kafka_e2e_recent_p99_ms",
samplesMetricName: "vehicle_bridge_kafka_e2e_recent_samples",
keyLabel: "topic",
thresholdMS: thresholds.BridgeKafkaRecentE2EP99MS,
},
{
service: "fast-writer",
stage: "fast_writer_redis_e2e",
metricName: "vehicle_fast_writer_redis_e2e_recent_p99_ms",
samplesMetricName: "vehicle_fast_writer_redis_e2e_recent_samples",
keyLabel: "subject",
thresholdMS: thresholds.FastWriterRedisRecentE2EP99MS,
},
{
service: "history",
stage: "history_tdengine_write_e2e",
metricName: "vehicle_history_write_e2e_recent_p99_ms",
samplesMetricName: "vehicle_history_write_e2e_recent_samples",
keyLabel: "topic",
thresholdMS: thresholds.HistoryWriteRecentE2EP99MS,
},
{
service: "stat",
stage: "stat_mysql_write_e2e",
metricName: "vehicle_stat_write_e2e_recent_p99_ms",
samplesMetricName: "vehicle_stat_write_e2e_recent_samples",
keyLabel: "topic",
thresholdMS: thresholds.StatWriteRecentE2EP99MS,
},
}
var items []PipelineLatencyItem
for _, spec := range specs {
items = append(items, pipelineLatencyItemsForSpec(parsedByService[spec.service], spec, thresholds.MinHistogramSamples)...)
}
if len(items) == 0 {
return nil
}
sort.Slice(items, func(i, j int) bool {
if items[i].Stage != items[j].Stage {
return items[i].Stage < items[j].Stage
}
if items[i].Service != items[j].Service {
return items[i].Service < items[j].Service
}
if items[i].Topic != items[j].Topic {
return items[i].Topic < items[j].Topic
}
return items[i].Subject < items[j].Subject
})
diagnostics := &PipelineLatencyDiagnostics{Items: items}
for _, item := range items {
diagnostics.TotalSamples += item.Samples
if item.P99MS > diagnostics.MaxP99MS {
diagnostics.MaxP99MS = item.P99MS
}
}
return diagnostics
}
func pipelineLatencyItemsForSpec(parsed parsedMetricSet, spec pipelineLatencySpec, minSamples float64) []PipelineLatencyItem {
samplesByKey := map[string]float64{}
for _, sample := range parsed.samples {
if sample.name != spec.samplesMetricName {
continue
}
samplesByKey[metricLabelsKey(sample.labels)] = sample.value
}
var items []PipelineLatencyItem
for _, sample := range parsed.samples {
if sample.name != spec.metricName {
continue
}
key := metricLabelsKey(sample.labels)
row := PipelineLatencyItem{
Stage: spec.stage,
Service: spec.service,
P99MS: sample.value,
Samples: metricCount(samplesByKey[key]),
ThresholdMS: spec.thresholdMS,
}
switch spec.keyLabel {
case "subject":
row.Subject = normalizedMetricLabel(sample.labels, spec.keyLabel)
default:
row.Topic = normalizedMetricLabel(sample.labels, spec.keyLabel)
}
row.Status = pipelineLatencyStatus(row.P99MS, float64(row.Samples), spec.thresholdMS, minSamples)
items = append(items, row)
}
return items
}
func pipelineLatencyStatus(p99MS float64, samples float64, thresholdMS float64, minSamples float64) string {
if minSamples > 0 && samples < minSamples {
return "insufficient_samples"
}
if thresholdMS > 0 && p99MS > thresholdMS {
return "degraded"
}
return "ok"
}
func findingsForGatewayIdentityCache(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.GatewayIdentityCacheEntriesMax <= 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_gateway_identity_cache_entries" || sample.value <= thresholds.GatewayIdentityCacheEntriesMax {
continue
}
cacheName := strings.TrimSpace(sample.labels["cache"])
if cacheName == "" {
cacheName = "unknown"
}
if cacheName == "registration_write_queue" {
continue
}
findings = append(findings, fmt.Sprintf(
"gateway identity cache entries %.0f exceeds %.0f for cache %s",
sample.value,
thresholds.GatewayIdentityCacheEntriesMax,
cacheName,
))
}
sort.Strings(findings)
return findings
}
func findingsForGatewayIdentitySnapshot(parsed parsedMetricSet, thresholds Thresholds, checkedAt time.Time) []string {
if thresholds.GatewayIdentitySnapshotRequired <= 0 && thresholds.GatewayIdentitySnapshotMaxAgeSec <= 0 {
return nil
}
if !hasServiceInfo(parsed, "vehicle-gateway") {
return nil
}
ready, readyFound := parsed.totals["vehicle_gateway_identity_snapshot_ready"]
var findings []string
if thresholds.GatewayIdentitySnapshotRequired > 0 {
switch {
case !readyFound:
findings = append(findings, "gateway identity snapshot readiness metric missing")
case ready < thresholds.GatewayIdentitySnapshotRequired:
findings = append(findings, fmt.Sprintf("gateway identity snapshot ready %.0f below %.0f", ready, thresholds.GatewayIdentitySnapshotRequired))
}
}
if thresholds.GatewayIdentitySnapshotMaxAgeSec <= 0 || !readyFound || ready <= 0 {
return findings
}
lastSuccess, ok := parsed.totals["vehicle_gateway_identity_snapshot_last_success_unix_seconds"]
if !ok || lastSuccess <= 0 {
return append(findings, "gateway identity snapshot last-success metric missing")
}
age := checkedAt.Unix() - int64(lastSuccess)
if age < 0 {
age = 0
}
if float64(age) > thresholds.GatewayIdentitySnapshotMaxAgeSec {
findings = append(findings, fmt.Sprintf(
"gateway identity snapshot age %ds exceeds %.0fs",
age,
thresholds.GatewayIdentitySnapshotMaxAgeSec,
))
}
return findings
}
func findingsForGatewayRegistrationWriteQueue(parsed parsedMetricSet, thresholds Thresholds) []string {
depth := gatewayIdentityCacheMetric(parsed, "registration_write_queue", "vehicle_gateway_identity_cache_entries")
capacity := gatewayIdentityCacheMetric(parsed, "registration_write_queue", "vehicle_gateway_identity_cache_max_entries")
var findings []string
if thresholds.GatewayIdentityRegistrationWriteQueueDepthMax > 0 && depth > thresholds.GatewayIdentityRegistrationWriteQueueDepthMax {
findings = append(findings, fmt.Sprintf(
"gateway jt808 registration write queue depth %.0f exceeds %.0f",
depth,
thresholds.GatewayIdentityRegistrationWriteQueueDepthMax,
))
}
if thresholds.GatewayIdentityRegistrationWriteQueueMaxUsageRatio > 0 && depth > 0 && capacity > 0 {
ratio := depth / capacity
if ratio > thresholds.GatewayIdentityRegistrationWriteQueueMaxUsageRatio {
findings = append(findings, fmt.Sprintf(
"gateway jt808 registration write queue usage %.2f exceeds %.2f (depth %.0f, capacity %.0f)",
ratio,
thresholds.GatewayIdentityRegistrationWriteQueueMaxUsageRatio,
depth,
capacity,
))
}
}
sort.Strings(findings)
return findings
}
func gatewayIdentityCacheMetric(parsed parsedMetricSet, cacheName string, metricName string) float64 {
cacheName = strings.TrimSpace(cacheName)
if cacheName == "" || metricName == "" {
return 0
}
var total float64
for _, sample := range parsed.samples {
if sample.name != metricName || strings.TrimSpace(sample.labels["cache"]) != cacheName {
continue
}
total += sample.value
}
return total
}
func findingsForGatewayRegistrationWriteErrors(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.GatewayIdentityRegistrationWriteErrorMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_gateway_jt808_registration_write_total" {
continue
}
status := strings.TrimSpace(sample.labels["status"])
if status == "" || status == "ok" {
continue
}
if sample.value <= thresholds.GatewayIdentityRegistrationWriteErrorMax {
continue
}
mode := strings.TrimSpace(sample.labels["mode"])
if mode == "" {
mode = "unknown"
}
findings = append(findings, fmt.Sprintf(
"gateway jt808 registration write %s count %.0f exceeds %.0f for mode %s",
status,
sample.value,
thresholds.GatewayIdentityRegistrationWriteErrorMax,
mode,
))
}
sort.Strings(findings)
return findings
}
func jt808RegistrationWriteErrors(parsed parsedMetricSet) float64 {
var total float64
for _, sample := range parsed.samples {
if sample.name != "vehicle_gateway_jt808_registration_write_total" {
continue
}
status := strings.TrimSpace(sample.labels["status"])
if status == "" || status == "ok" {
continue
}
total += sample.value
}
return total
}
func findingsForGatewayIdentityLocationTouchFailures(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.GatewayIdentityLocationTouchFailureMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_gateway_identity_cache_entries" || strings.TrimSpace(sample.labels["cache"]) != "location_touch_failure" {
continue
}
if sample.value <= thresholds.GatewayIdentityLocationTouchFailureMax {
continue
}
findings = append(findings, fmt.Sprintf(
"gateway identity location touch failure cache %.0f exceeds %.0f; jt808 registration writes are failing or in protective backoff",
sample.value,
thresholds.GatewayIdentityLocationTouchFailureMax,
))
}
sort.Strings(findings)
return findings
}
func findingsForGatewayIdentityStaleCache(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.GatewayIdentityStaleMinSamples <= 0 || thresholds.GatewayIdentityMaxStaleCacheRatio <= 0 {
return nil
}
totalsByProtocol := map[string]float64{}
staleByProtocol := map[string]float64{}
for _, sample := range parsed.samples {
protocol := strings.TrimSpace(sample.labels["protocol"])
if protocol == "" {
protocol = "unknown"
}
switch sample.name {
case "vehicle_gateway_identity_total":
totalsByProtocol[protocol] += sample.value
case "vehicle_gateway_identity_cache_total":
if sample.labels["cache_status"] == "stale" {
staleByProtocol[protocol] += sample.value
}
}
}
var protocols []string
for protocol := range totalsByProtocol {
protocols = append(protocols, protocol)
}
sort.Strings(protocols)
var findings []string
for _, protocol := range protocols {
total := totalsByProtocol[protocol]
if total < thresholds.GatewayIdentityStaleMinSamples || total <= 0 {
continue
}
stale := staleByProtocol[protocol]
ratio := stale / total
if ratio <= thresholds.GatewayIdentityMaxStaleCacheRatio {
continue
}
findings = append(findings, fmt.Sprintf(
"gateway identity stale cache ratio %.2f exceeds %.2f for protocol %s (total %.0f, stale %.0f)",
ratio,
thresholds.GatewayIdentityMaxStaleCacheRatio,
protocol,
total,
stale,
))
}
return findings
}
func findingsForBridgeRouteErrors(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.BridgeRouteErrorMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_bridge_messages_total" || sample.labels["status"] != "route_error" {
continue
}
if sample.value <= thresholds.BridgeRouteErrorMax {
continue
}
subject := strings.TrimSpace(sample.labels["subject"])
if subject == "" {
subject = "unknown"
}
findings = append(findings, fmt.Sprintf(
"bridge route errors %.0f exceeds %.0f for subject %s",
sample.value,
thresholds.BridgeRouteErrorMax,
subject,
))
}
sort.Strings(findings)
return findings
}
func findingsForBridgeKafkaWriteErrors(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.BridgeKafkaWriteErrorMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_bridge_kafka_writes_total" || sample.labels["status"] != "error" {
continue
}
if sample.value <= thresholds.BridgeKafkaWriteErrorMax {
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if topic == "" {
topic = "unknown"
}
findings = append(findings, fmt.Sprintf(
"bridge kafka write errors %.0f exceeds %.0f for topic %s",
sample.value,
thresholds.BridgeKafkaWriteErrorMax,
topic,
))
}
sort.Strings(findings)
return findings
}
type bridgeMessageCounters struct {
received float64
ackOK float64
droppedRouteError float64
}
func findingsForBridgeMessageCompletion(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.BridgeMessageMinReceived <= 0 || thresholds.BridgeMessageMaxUnackedRatio <= 0 {
return nil
}
groups := map[string]*bridgeMessageCounters{}
for _, sample := range parsed.samples {
switch sample.name {
case "vehicle_bridge_messages_total", "vehicle_bridge_nats_acks_total":
default:
continue
}
subject := strings.TrimSpace(sample.labels["subject"])
if subject == "" || subject == "mixed" || subject == "unknown" {
continue
}
group := groups[subject]
if group == nil {
group = &bridgeMessageCounters{}
groups[subject] = group
}
switch {
case sample.name == "vehicle_bridge_messages_total" && sample.labels["status"] == "received":
group.received += sample.value
case sample.name == "vehicle_bridge_nats_acks_total" && sample.labels["status"] == "ok":
group.ackOK += sample.value
case sample.name == "vehicle_bridge_nats_acks_total" && sample.labels["status"] == "dropped_route_error":
group.droppedRouteError += sample.value
}
}
var subjects []string
for subject := range groups {
subjects = append(subjects, subject)
}
sort.Strings(subjects)
var findings []string
for _, subject := range subjects {
group := groups[subject]
if group.received < thresholds.BridgeMessageMinReceived {
continue
}
completed := group.ackOK + group.droppedRouteError
unacked := group.received - completed
if unacked < 0 {
unacked = 0
}
ratio := unacked / group.received
if ratio <= thresholds.BridgeMessageMaxUnackedRatio {
continue
}
findings = append(findings, fmt.Sprintf(
"bridge unacked message ratio %.2f exceeds %.2f for subject %s: received %.0f, ack ok %.0f, dropped route error %.0f, unacked %.0f",
ratio,
thresholds.BridgeMessageMaxUnackedRatio,
subject,
group.received,
group.ackOK,
group.droppedRouteError,
unacked,
))
}
return findings
}
func findingsForTopicFlow(parsedByService map[string]parsedMetricSet, thresholds Thresholds) []string {
var findings []string
findings = append(findings, findingsForGatewayFieldFlow(parsedByService["gateway"], thresholds)...)
findings = append(findings, findingsForGatewayFieldCounts(parsedByService["gateway"], thresholds)...)
findings = append(findings, findingsForGatewayToBridgeFlow(parsedByService["gateway"], parsedByService["bridge"], thresholds)...)
findings = append(findings, findingsForGatewayToFastWriterFlow(parsedByService["gateway"], parsedByService["fast-writer"], thresholds)...)
findings = append(findings, findingsForBridgeRawFanout(parsedByService["bridge"], parsedByService["history"], parsedByService["realtime"], thresholds)...)
findings = append(findings, findingsForBridgeToStatFieldFlow(parsedByService["bridge"], parsedByService["stat"], thresholds)...)
return findings
}
func findingsForGatewayFieldFlow(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.GatewayFieldMinRaw <= 0 {
return nil
}
groups := gatewayFieldFlowGroups(parsed)
protocols := gatewayFieldFlowProtocols(groups)
var findings []string
for _, protocol := range protocols {
group := groups[protocol]
realtimeCandidates := gatewayRealtimeCandidates(group.gatewayPublishCounters)
if realtimeCandidates < thresholds.GatewayFieldMinRaw {
continue
}
if group.fieldsOK <= 0 {
findings = append(findings, fmt.Sprintf(
"gateway fields missing for protocol %s: raw ok %.0f, realtime candidates %.0f, fields ok %.0f, skipped non-realtime %.0f, skipped missing fields %.0f, publish errors %.0f",
protocol,
group.rawOK,
realtimeCandidates,
group.fieldsOK,
group.fieldsSkippedNonRealtime,
group.fieldsSkippedMissing,
group.fieldsPublishError,
))
continue
}
missingRatio := (group.fieldsSkippedMissing + group.fieldsPublishError) / realtimeCandidates
if thresholds.GatewayFieldMaxMissingRatio > 0 && missingRatio > thresholds.GatewayFieldMaxMissingRatio {
findings = append(findings, fmt.Sprintf(
"gateway fields missing ratio %.2f exceeds %.2f for protocol %s: realtime candidates %.0f, fields ok %.0f, skipped missing fields %.0f, publish errors %.0f",
missingRatio,
thresholds.GatewayFieldMaxMissingRatio,
protocol,
realtimeCandidates,
group.fieldsOK,
group.fieldsSkippedMissing,
group.fieldsPublishError,
))
}
}
return findings
}
func findingsForGatewayFieldCounts(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.GatewayFieldMinCount <= 0 {
return nil
}
minPublished := thresholds.GatewayFieldMinRaw
if minPublished <= 0 {
minPublished = 1
}
groups := gatewayFieldFlowGroups(parsed)
protocols := gatewayFieldFlowProtocols(groups)
var findings []string
for _, protocol := range protocols {
group := groups[protocol]
fieldsAccepted := group.fieldsPublished + group.fieldsDelegated
if fieldsAccepted < minPublished {
continue
}
if !group.hasLatestCount {
findings = append(findings, fmt.Sprintf(
"gateway fields count metric missing for protocol %s after %.0f published fields events",
protocol,
fieldsAccepted,
))
continue
}
if group.latestCount < thresholds.GatewayFieldMinCount {
findings = append(findings, fmt.Sprintf(
"gateway fields count %.0f below %.0f for protocol %s after %.0f published fields events",
group.latestCount,
thresholds.GatewayFieldMinCount,
protocol,
fieldsAccepted,
))
}
}
return findings
}
func buildGatewayFieldsDiagnostics(parsed parsedMetricSet, thresholds Thresholds) *GatewayFieldsDiagnostics {
groups := gatewayFieldFlowGroups(parsed)
protocols := gatewayFieldFlowProtocols(groups)
if len(protocols) == 0 {
return nil
}
out := &GatewayFieldsDiagnostics{Protocols: make([]GatewayProtocolFields, 0, len(protocols))}
for _, protocol := range protocols {
group := groups[protocol]
realtimeCandidates := gatewayRealtimeCandidates(group.gatewayPublishCounters)
missingRatio := 0.0
if realtimeCandidates > 0 {
missingRatio = (group.fieldsSkippedMissing + group.fieldsPublishError) / realtimeCandidates
}
status, reason := gatewayFieldDiagnosticsStatus(group, realtimeCandidates, thresholds)
var latestCount *float64
if group.hasLatestCount {
value := group.latestCount
latestCount = &value
}
out.Protocols = append(out.Protocols, GatewayProtocolFields{
Protocol: protocol,
RawOK: group.rawOK,
RealtimeCandidates: realtimeCandidates,
FieldsOK: group.fieldsOK,
FieldsPublished: group.fieldsPublished,
FieldsDelegated: group.fieldsDelegated,
FieldsSkippedNonRealtime: group.fieldsSkippedNonRealtime,
FieldsSkippedMissing: group.fieldsSkippedMissing,
FieldsPublishError: group.fieldsPublishError,
MissingRatio: missingRatio,
LatestFieldCount: latestCount,
Status: status,
Reason: reason,
})
}
return out
}
func gatewayFieldDiagnosticsStatus(group *gatewayFieldFlowGroup, realtimeCandidates float64, thresholds Thresholds) (string, string) {
if thresholds.GatewayFieldMinRaw > 0 && realtimeCandidates >= thresholds.GatewayFieldMinRaw {
if group.fieldsOK <= 0 {
return "degraded", "missing_fields_publish"
}
missingRatio := 0.0
if realtimeCandidates > 0 {
missingRatio = (group.fieldsSkippedMissing + group.fieldsPublishError) / realtimeCandidates
}
if thresholds.GatewayFieldMaxMissingRatio > 0 && missingRatio > thresholds.GatewayFieldMaxMissingRatio {
return "degraded", "high_missing_ratio"
}
}
minPublished := thresholds.GatewayFieldMinRaw
if minPublished <= 0 {
minPublished = 1
}
if thresholds.GatewayFieldMinCount > 0 && group.fieldsPublished+group.fieldsDelegated >= minPublished {
if !group.hasLatestCount {
return "degraded", "missing_field_count_metric"
}
if group.latestCount < thresholds.GatewayFieldMinCount {
return "degraded", "low_field_count"
}
}
return "ok", ""
}
func gatewayFieldFlowGroups(parsed parsedMetricSet) map[string]*gatewayFieldFlowGroup {
groups := map[string]*gatewayFieldFlowGroup{}
groupForProtocol := func(protocol string) *gatewayFieldFlowGroup {
protocol = strings.TrimSpace(protocol)
if protocol == "" {
protocol = "unknown"
}
group := groups[protocol]
if group == nil {
group = &gatewayFieldFlowGroup{}
groups[protocol] = group
}
return group
}
for _, sample := range parsed.samples {
switch sample.name {
case "vehicle_gateway_publish_total":
if sample.labels["status"] != "ok" && sample.labels["status"] != "delegated" {
continue
}
group := groupForProtocol(sample.labels["protocol"])
switch sample.labels["kind"] {
case "raw":
group.rawOK += sample.value
case "fields":
group.fieldsOK += sample.value
}
case "vehicle_gateway_fields_total":
group := groupForProtocol(sample.labels["protocol"])
switch sample.labels["status"] {
case "published":
group.fieldsPublished += sample.value
case "delegated_to_bridge":
group.fieldsDelegated += sample.value
case "skipped_non_realtime":
group.fieldsSkippedNonRealtime += sample.value
case "skipped_missing_fields":
group.fieldsSkippedMissing += sample.value
case "publish_error":
group.fieldsPublishError += sample.value
}
case "vehicle_gateway_fields_count":
if sample.labels["status"] != "published" && sample.labels["status"] != "delegated_to_bridge" {
continue
}
group := groupForProtocol(sample.labels["protocol"])
group.latestCount = sample.value
group.hasLatestCount = true
}
}
return groups
}
func gatewayFieldFlowProtocols(groups map[string]*gatewayFieldFlowGroup) []string {
var protocols []string
for protocol := range groups {
protocols = append(protocols, protocol)
}
sort.Strings(protocols)
return protocols
}
func gatewayRealtimeCandidates(group gatewayPublishCounters) float64 {
realtimeCandidates := group.rawOK - group.fieldsSkippedNonRealtime
if realtimeCandidates < 0 {
return 0
}
return realtimeCandidates
}
func findingsForGatewayToFastWriterFlow(gateway parsedMetricSet, fastWriter parsedMetricSet, thresholds Thresholds) []string {
if thresholds.GatewayFastWriterMinRawPublish <= 0 {
return nil
}
gatewayPublishes := gatewayPublishGroups(gateway.samples)
fastWriterMessages := metricValuesBySubject(fastWriter.samples, "vehicle_fast_writer_messages_total", "received", "vehicle.raw.go.")
var findings []string
for _, group := range gatewayPublishes {
if group.kind != "raw" || group.count < thresholds.GatewayFastWriterMinRawPublish {
continue
}
subject, ok := expectedKafkaTopicForGatewayPublish(group.protocol, group.kind)
if !ok {
continue
}
if fastWriterMessages[subject] <= 0 {
findings = append(findings, fmt.Sprintf(
"fast writer received messages missing for gateway raw publish protocol %s: gateway ok %.0f, expected subject %s, fast writer received %.0f",
group.protocol,
group.count,
subject,
fastWriterMessages[subject],
))
}
}
return findings
}
type gatewayPublishGroup struct {
protocol string
kind string
count float64
}
func findingsForGatewayToBridgeFlow(gateway parsedMetricSet, bridge parsedMetricSet, thresholds Thresholds) []string {
if thresholds.GatewayBridgeMinPublish <= 0 {
return nil
}
gatewayPublishes := gatewayPublishGroups(gateway.samples)
bridgeWrites := metricValuesByTopic(bridge.samples, "vehicle_bridge_kafka_writes_total", "ok", "vehicle.")
var findings []string
for _, group := range gatewayPublishes {
if group.count < thresholds.GatewayBridgeMinPublish {
continue
}
topic, ok := expectedKafkaTopicForGatewayPublish(group.protocol, group.kind)
if !ok {
findings = append(findings, fmt.Sprintf(
"gateway %s publish topic mapping missing for protocol %s: gateway ok %.0f",
group.kind,
group.protocol,
group.count,
))
continue
}
if bridgeWrites[topic] <= 0 {
findings = append(findings, fmt.Sprintf(
"bridge writes missing for gateway %s publish protocol %s: gateway ok %.0f, expected topic %s, bridge writes ok %.0f",
group.kind,
group.protocol,
group.count,
topic,
bridgeWrites[topic],
))
}
}
return findings
}
func gatewayPublishGroups(samples []metricSample) []gatewayPublishGroup {
groups := map[string]*gatewayPublishGroup{}
for _, sample := range samples {
if sample.name != "vehicle_gateway_publish_total" || (sample.labels["status"] != "ok" && sample.labels["status"] != "delegated") {
continue
}
protocol := strings.TrimSpace(sample.labels["protocol"])
if protocol == "" {
protocol = "unknown"
}
kind := strings.TrimSpace(sample.labels["kind"])
if kind == "" {
kind = "unknown"
}
key := protocol + "\x00" + kind
group := groups[key]
if group == nil {
group = &gatewayPublishGroup{protocol: protocol, kind: kind}
groups[key] = group
}
group.count += sample.value
}
var keys []string
for key := range groups {
keys = append(keys, key)
}
sort.Strings(keys)
out := make([]gatewayPublishGroup, 0, len(keys))
for _, key := range keys {
out = append(out, *groups[key])
}
return out
}
func expectedKafkaTopicForGatewayPublish(protocol string, kind string) (string, bool) {
switch kind {
case "raw":
switch envelope.Protocol(protocol) {
case envelope.ProtocolGB32960:
return topics.RawGB32960, true
case envelope.ProtocolJT808:
return topics.RawJT808, true
case envelope.ProtocolYutongMQTT:
return topics.RawYutongMQTT, true
default:
return "", false
}
case "fields":
switch envelope.Protocol(protocol) {
case envelope.ProtocolGB32960:
return topics.FieldsGB32960, true
case envelope.ProtocolJT808:
return topics.FieldsJT808, true
case envelope.ProtocolYutongMQTT:
return topics.FieldsYutongMQTT, true
default:
return "", false
}
default:
return "", false
}
}
func findingsForBridgeRawFanout(bridge parsedMetricSet, history parsedMetricSet, realtime parsedMetricSet, thresholds Thresholds) []string {
if thresholds.RawFanoutMinBridge <= 0 {
return nil
}
bridgeWrites := metricValuesByTopic(bridge.samples, "vehicle_bridge_kafka_writes_total", "ok", "vehicle.raw.go.")
historyConsumers := metricValuesByTopic(history.samples, "vehicle_kafka_consumer_info", "", "vehicle.raw.go.")
historyReceived := metricValuesByTopic(history.samples, "vehicle_history_kafka_messages_total", "received", "vehicle.raw.go.")
historyWrites := metricValuesByTopic(history.samples, "vehicle_history_writes_total", "ok", "vehicle.raw.go.")
realtimeConsumers := metricValuesByTopic(realtime.samples, "vehicle_kafka_consumer_info", "", "vehicle.raw.go.")
realtimeReceived := metricValuesByTopic(realtime.samples, "vehicle_realtime_kafka_messages_total", "received", "vehicle.raw.go.")
realtimeUpdates := metricValuesByTopic(realtime.samples, "vehicle_realtime_updates_total", "ok", "vehicle.raw.go.")
var topics []string
for topic := range bridgeWrites {
topics = append(topics, topic)
}
sort.Strings(topics)
var findings []string
for _, topic := range topics {
writes := bridgeWrites[topic]
if writes < thresholds.RawFanoutMinBridge {
continue
}
if historyConsumers[topic] <= 0 {
findings = append(findings, fmt.Sprintf("history consumer topic missing for %s: bridge raw writes ok %.0f", topic, writes))
} else if historyReceived[topic] <= 0 {
findings = append(findings, fmt.Sprintf("history messages missing for %s: bridge raw writes ok %.0f, history received %.0f", topic, writes, historyReceived[topic]))
} else if historyWrites[topic] <= 0 {
findings = append(findings, fmt.Sprintf("history writes missing for %s: bridge raw writes ok %.0f, history received %.0f, history writes ok %.0f", topic, writes, historyReceived[topic], historyWrites[topic]))
}
if realtimeConsumers[topic] <= 0 {
findings = append(findings, fmt.Sprintf("realtime consumer topic missing for %s: bridge raw writes ok %.0f", topic, writes))
} else if realtimeReceived[topic] <= 0 {
findings = append(findings, fmt.Sprintf("realtime messages missing for %s: bridge raw writes ok %.0f, realtime received %.0f", topic, writes, realtimeReceived[topic]))
} else if realtimeUpdates[topic] <= 0 {
findings = append(findings, fmt.Sprintf("realtime updates missing for %s: bridge raw writes ok %.0f, realtime received %.0f, realtime updates ok %.0f", topic, writes, realtimeReceived[topic], realtimeUpdates[topic]))
}
}
return findings
}
func findingsForBridgeToStatFieldFlow(bridge parsedMetricSet, stat parsedMetricSet, thresholds Thresholds) []string {
if thresholds.StatTopicMinBridge <= 0 {
return nil
}
bridgeWrites := metricValuesByTopic(bridge.samples, "vehicle_bridge_kafka_writes_total", "ok", "vehicle.fields.go.")
statReceived := metricValuesByTopic(stat.samples, "vehicle_stat_kafka_messages_total", "received", "vehicle.fields.go.")
statConsumers := metricValuesByTopic(stat.samples, "vehicle_kafka_consumer_info", "", "vehicle.fields.go.")
var topics []string
for topic := range bridgeWrites {
topics = append(topics, topic)
}
sort.Strings(topics)
var findings []string
for _, topic := range topics {
writes := bridgeWrites[topic]
if writes < thresholds.StatTopicMinBridge {
continue
}
if statConsumers[topic] <= 0 {
findings = append(findings, fmt.Sprintf("stat consumer topic missing for %s: bridge writes ok %.0f", topic, writes))
continue
}
if statReceived[topic] <= 0 {
findings = append(findings, fmt.Sprintf("stat messages missing for %s: bridge writes ok %.0f, stat received %.0f", topic, writes, statReceived[topic]))
}
}
return findings
}
func findingsForRequiredConsumerTopics(parsedByService map[string]parsedMetricSet, thresholds Thresholds) []string {
if len(thresholds.RequiredConsumerTopics) == 0 {
return nil
}
var services []string
for service := range thresholds.RequiredConsumerTopics {
services = append(services, service)
}
sort.Strings(services)
var findings []string
for _, service := range services {
required := normalizeTopicSet(thresholds.RequiredConsumerTopics[service])
if len(required) == 0 {
continue
}
seen := kafkaConsumerTopicSet(parsedByService[service])
var topics []string
for topic := range required {
topics = append(topics, topic)
}
sort.Strings(topics)
for _, topic := range topics {
if seen[topic] > 0 {
continue
}
findings = append(findings, fmt.Sprintf("%s required kafka consumer topic missing: %s", service, topic))
}
}
return findings
}
func normalizeTopicSet(values []string) map[string]struct{} {
out := map[string]struct{}{}
for _, value := range values {
value = strings.TrimSpace(value)
if value == "" {
continue
}
out[value] = struct{}{}
}
return out
}
func kafkaConsumerTopicSet(parsed parsedMetricSet) map[string]float64 {
out := map[string]float64{}
for _, sample := range parsed.samples {
if sample.name != "vehicle_kafka_consumer_info" || sample.value <= 0 {
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if topic == "" {
continue
}
out[topic] += sample.value
}
return out
}
func metricValuesByTopic(samples []metricSample, metricName string, status string, topicPrefix string) map[string]float64 {
out := map[string]float64{}
for _, sample := range samples {
if sample.name != metricName {
continue
}
if status != "" && sample.labels["status"] != status {
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if topic == "" || (topicPrefix != "" && !strings.HasPrefix(topic, topicPrefix)) {
continue
}
out[topic] += sample.value
}
return out
}
func metricValuesBySubject(samples []metricSample, metricName string, status string, subjectPrefix string) map[string]float64 {
out := map[string]float64{}
for _, sample := range samples {
if sample.name != metricName {
continue
}
if status != "" && sample.labels["status"] != status {
continue
}
subject := strings.TrimSpace(sample.labels["subject"])
if subject == "" || (subjectPrefix != "" && !strings.HasPrefix(subject, subjectPrefix)) {
continue
}
out[subject] += sample.value
}
return out
}
func findingsForLastActivity(parsedByService map[string]parsedMetricSet, thresholds Thresholds, checkedAt time.Time) []string {
if thresholds.LastActivityStaleSec <= 0 {
return nil
}
specs := []lastActivitySpec{
{service: "gateway", metric: "vehicle_gateway_last_frame_unix_seconds", label: "gateway last frame", status: "OK"},
{service: "gateway", metric: "vehicle_gateway_last_publish_unix_seconds", label: "gateway last publish", status: "ok"},
{service: "bridge", metric: "vehicle_bridge_last_message_unix_seconds", label: "bridge last message", status: "received"},
{service: "bridge", metric: "vehicle_bridge_last_kafka_write_unix_seconds", label: "bridge last kafka write", status: "ok"},
{service: "bridge", metric: "vehicle_bridge_last_ack_unix_seconds", label: "bridge last ack", status: "ok"},
{service: "fast-writer", metric: "vehicle_fast_writer_last_message_unix_seconds", label: "fast writer last message", status: "received"},
{service: "fast-writer", metric: "vehicle_fast_writer_last_stage_unix_seconds", label: "fast writer last stage", status: "ok"},
{service: "history", metric: "vehicle_history_last_message_unix_seconds", label: "history last message", status: "received"},
{service: "history", metric: "vehicle_history_last_write_unix_seconds", label: "history last write", status: "ok"},
{service: "history", metric: "vehicle_history_last_commit_unix_seconds", label: "history last commit", status: "ok"},
{service: "stat", metric: "vehicle_stat_last_message_unix_seconds", label: "stat last message", status: "received"},
{service: "stat", metric: "vehicle_stat_last_write_unix_seconds", label: "stat last write", status: "ok"},
{service: "stat", metric: "vehicle_stat_last_commit_unix_seconds", label: "stat last commit", status: "ok"},
{service: "realtime", metric: "vehicle_realtime_last_message_unix_seconds", label: "realtime last message", status: "received"},
{service: "realtime", metric: "vehicle_realtime_last_update_unix_seconds", label: "realtime last update", status: "ok"},
{service: "realtime", metric: "vehicle_realtime_last_store_update_unix_seconds", label: "realtime last store update", status: "ok"},
{service: "realtime", metric: "vehicle_realtime_last_commit_unix_seconds", label: "realtime last commit", status: "ok"},
{service: "identity", metric: "vehicle_identity_writer_last_message_unix_seconds", label: "identity last message", status: "received"},
{service: "identity", metric: "vehicle_identity_writer_last_write_unix_seconds", label: "identity last write", status: "ok"},
{service: "identity", metric: "vehicle_identity_writer_last_commit_unix_seconds", label: "identity last commit", status: "ok"},
}
var findings []string
for _, spec := range specs {
findings = append(findings, staleActivityFindings(parsedByService[spec.service], spec, thresholds.LastActivityStaleSec, checkedAt)...)
}
sort.Strings(findings)
return findings
}
type lastActivitySpec struct {
service string
metric string
label string
status string
}
func staleActivityFindings(parsed parsedMetricSet, spec lastActivitySpec, thresholdSec float64, checkedAt time.Time) []string {
var findings []string
for _, sample := range parsed.samples {
if sample.name != spec.metric {
continue
}
if spec.status != "" && sample.labels["status"] != spec.status {
continue
}
if sample.value <= 0 {
continue
}
if isRollupLastActivity(sample) {
continue
}
ageSec := checkedAt.Sub(time.Unix(int64(sample.value), 0).UTC()).Seconds()
if ageSec <= thresholdSec {
continue
}
findings = append(findings, fmt.Sprintf("%s stale %.0fs exceeds %.0fs (%s)", spec.label, ageSec, thresholdSec, metricLabelsKey(sample.labels)))
}
return findings
}
func isRollupLastActivity(sample metricSample) bool {
for _, label := range []string{"subject", "topic"} {
switch strings.TrimSpace(sample.labels[label]) {
case "mixed", "unknown":
return true
}
}
return false
}
type statSampleCounters struct {
writesOK float64
found float64
written float64
skippedMissingFields float64
skippedMissingVIN float64
skippedMissingMileage float64
skippedNonMileageFrame float64
skippedNonPositiveMileage float64
skippedMissingTime float64
futureAdjusted float64
skippedSameMileage float64
skippedMissingSource float64
}
type statSourceCounters struct {
found float64
attempted float64
written float64
skippedThrottled float64
skippedMissingEndpoint float64
skippedUnmanaged float64
}
type statProjectionCounters struct {
samplesWritten float64
attempted float64
written float64
skippedThrottled float64
}
type protocolFlowStatCounters struct {
samples statSampleCounters
sources statSourceCounters
projections statProjectionCounters
}
func buildProtocolFlowDiagnostics(parsedByService map[string]parsedMetricSet, thresholds Thresholds) *ProtocolFlowDiagnostics {
gatewayGroups := gatewayFieldFlowGroups(parsedByService["gateway"])
identityDiagnostics := buildIdentityResolutionDiagnostics(parsedByService["gateway"])
identityByProtocol := map[string]IdentityProtocolResolution{}
if identityDiagnostics != nil {
for _, row := range identityDiagnostics.Protocols {
identityByProtocol[row.Protocol] = row
}
}
statGroups := protocolFlowStatGroups(parsedByService["stat"])
protocolSet := map[string]struct{}{}
for protocol := range gatewayGroups {
protocolSet[protocol] = struct{}{}
}
for protocol := range identityByProtocol {
protocolSet[protocol] = struct{}{}
}
for protocol := range statGroups {
protocolSet[protocol] = struct{}{}
}
if len(protocolSet) == 0 {
return nil
}
protocols := make([]string, 0, len(protocolSet))
for protocol := range protocolSet {
protocols = append(protocols, protocol)
}
sort.Strings(protocols)
out := &ProtocolFlowDiagnostics{Protocols: make([]ProtocolFlowProtocol, 0, len(protocols))}
for _, protocol := range protocols {
gatewayGroup := gatewayGroups[protocol]
statGroup := statGroups[protocol]
identityRow := identityByProtocol[protocol]
row := ProtocolFlowProtocol{Protocol: protocol}
if gatewayGroup != nil {
row.RawOK = gatewayGroup.rawOK
row.RealtimeCandidates = gatewayRealtimeCandidates(gatewayGroup.gatewayPublishCounters)
row.FieldsPublished = gatewayGroup.fieldsPublished
if gatewayGroup.hasLatestCount {
value := gatewayGroup.latestCount
row.LatestFieldCount = &value
}
}
row.IdentityTotal = identityRow.Total
row.IdentitySkipped = identityRow.SkippedCount
row.IdentityResolved = identityRow.ResolvedCount
row.IdentityNonResolved = identityRow.NonResolvedCount
row.IdentityNonResolvedRatio = identityRow.NonResolvedRatio
if statGroup != nil {
row.StatWritesOK = statGroup.samples.writesOK
row.StatMileageCandidateWrites = statMileageCandidateWrites(statGroup.samples)
row.StatSamplesFound = statGroup.samples.found
row.StatSamplesWritten = statGroup.samples.written
row.StatSamplesActionableSkipped = statActionableSkips(statGroup.samples)
row.StatSkippedMissingFields = statGroup.samples.skippedMissingFields
row.StatSkippedMissingVIN = statGroup.samples.skippedMissingVIN
row.StatSkippedMissingMileage = statGroup.samples.skippedMissingMileage
row.StatSkippedNonMileageFrame = statGroup.samples.skippedNonMileageFrame
row.StatSkippedNonPositiveMileage = statGroup.samples.skippedNonPositiveMileage
row.StatSkippedMissingTime = statGroup.samples.skippedMissingTime
row.StatEventTimeFutureAdjusted = statGroup.samples.futureAdjusted
row.StatSkippedMissingSource = statGroup.samples.skippedMissingSource
row.StatSkippedSameMileage = statGroup.samples.skippedSameMileage
row.StatSourceWritten = statGroup.sources.written
row.StatSourceThrottled = statGroup.sources.skippedThrottled
row.StatSourceMissingEndpoint = statGroup.sources.skippedMissingEndpoint
row.StatSourceSkippedUnmanaged = statGroup.sources.skippedUnmanaged
row.StatProjectionAttempted = statGroup.projections.attempted
row.StatProjectionWritten = statGroup.projections.written
row.StatProjectionThrottled = statGroup.projections.skippedThrottled
}
row.Status, row.Reason = protocolFlowStatus(gatewayGroup, identityRow, statGroup, thresholds)
out.Protocols = append(out.Protocols, row)
}
return out
}
func protocolFlowStatGroups(parsed parsedMetricSet) map[string]*protocolFlowStatCounters {
groups := map[string]*protocolFlowStatCounters{}
groupForSample := func(sample metricSample) *protocolFlowStatCounters {
protocol := protocolForFieldsMetricSample(sample)
group := groups[protocol]
if group == nil {
group = &protocolFlowStatCounters{}
groups[protocol] = group
}
return group
}
for _, sample := range parsed.samples {
switch sample.name {
case "vehicle_stat_writes_total":
if sample.labels["status"] == "ok" {
groupForSample(sample).samples.writesOK += sample.value
}
case "vehicle_stat_samples_total":
group := groupForSample(sample)
switch sample.labels["status"] {
case "found":
group.samples.found += sample.value
group.sources.found += sample.value
case "written":
group.samples.written += sample.value
group.projections.samplesWritten += sample.value
case "skipped_missing_fields":
group.samples.skippedMissingFields += sample.value
case "skipped_missing_vin":
group.samples.skippedMissingVIN += sample.value
case "skipped_missing_mileage":
group.samples.skippedMissingMileage += sample.value
case "skipped_non_mileage_frame":
group.samples.skippedNonMileageFrame += sample.value
case "skipped_non_positive_mileage":
group.samples.skippedNonPositiveMileage += sample.value
case "skipped_missing_time":
group.samples.skippedMissingTime += sample.value
case "event_time_future_adjusted":
group.samples.futureAdjusted += sample.value
case "skipped_same_mileage":
group.samples.skippedSameMileage += sample.value
case "skipped_missing_source":
group.samples.skippedMissingSource += sample.value
}
case "vehicle_stat_sources_total":
group := groupForSample(sample)
switch sample.labels["status"] {
case "attempted":
group.sources.attempted += sample.value
case "written":
group.sources.written += sample.value
case "skipped_throttled":
group.sources.skippedThrottled += sample.value
case "skipped_missing_endpoint":
group.sources.skippedMissingEndpoint += sample.value
case "skipped_unmanaged":
group.sources.skippedUnmanaged += sample.value
}
case "vehicle_stat_projections_total":
group := groupForSample(sample)
switch sample.labels["status"] {
case "attempted":
group.projections.attempted += sample.value
case "written":
group.projections.written += sample.value
case "skipped_throttled":
group.projections.skippedThrottled += sample.value
}
}
}
return groups
}
func protocolForFieldsMetricSample(sample metricSample) string {
if protocol := strings.TrimSpace(sample.labels["protocol"]); protocol != "" {
return protocol
}
if protocol, ok := topics.ProtocolForKnownFieldsTopic(sample.labels["topic"]); ok {
return protocol
}
return "unknown"
}
func statActionableSkips(group statSampleCounters) float64 {
return group.skippedMissingFields +
group.skippedMissingVIN +
group.skippedMissingMileage +
group.skippedNonPositiveMileage +
group.skippedMissingTime +
group.skippedMissingSource
}
func statOnlyMissingMileageSkips(group statSampleCounters) bool {
return group.skippedMissingMileage > 0 &&
group.skippedMissingFields == 0 &&
group.skippedMissingVIN == 0 &&
group.skippedNonPositiveMileage == 0 &&
group.skippedMissingTime == 0 &&
group.skippedMissingSource == 0
}
func statMileageCandidateWrites(group statSampleCounters) float64 {
candidates := group.writesOK - group.skippedNonMileageFrame
if candidates < 0 {
return 0
}
return candidates
}
func protocolFlowStatus(gatewayGroup *gatewayFieldFlowGroup, identityRow IdentityProtocolResolution, statGroup *protocolFlowStatCounters, thresholds Thresholds) (string, string) {
if gatewayGroup != nil {
realtimeCandidates := gatewayRealtimeCandidates(gatewayGroup.gatewayPublishCounters)
status, reason := gatewayFieldDiagnosticsStatus(gatewayGroup, realtimeCandidates, thresholds)
if status != "ok" {
return status, "gateway_fields_" + reason
}
}
if thresholds.GatewayIdentityMinSamples > 0 &&
thresholds.GatewayIdentityMaxUnresolvedRatio > 0 &&
float64(identityRow.Total) >= thresholds.GatewayIdentityMinSamples &&
identityRow.NonResolvedRatio > thresholds.GatewayIdentityMaxUnresolvedRatio {
return "degraded", "identity_unresolved_ratio"
}
if statGroup == nil {
return "ok", ""
}
if thresholds.StatSampleMinWrites > 0 && statMileageCandidateWrites(statGroup.samples) >= thresholds.StatSampleMinWrites {
if thresholds.StatSampleFutureAdjustmentMax >= 0 &&
statGroup.samples.futureAdjusted > thresholds.StatSampleFutureAdjustmentMax {
return "degraded", "stat_future_event_time_adjusted"
}
if statGroup.samples.found <= 0 {
if statActionableSkips(statGroup.samples) <= 0 && statGroup.samples.skippedNonMileageFrame > 0 {
return "ok", ""
}
if statOnlyMissingMileageSkips(statGroup.samples) {
return "ok", ""
}
return "degraded", "stat_samples_missing"
}
if thresholds.StatSampleMaxActionableSkipRatio > 0 {
ratio := statActionableSkips(statGroup.samples) / statMileageCandidateWrites(statGroup.samples)
if ratio > thresholds.StatSampleMaxActionableSkipRatio {
return "degraded", "stat_actionable_skip_ratio"
}
}
if statGroup.samples.written <= 0 && statGroup.samples.skippedMissingSource > 0 {
return "degraded", "stat_samples_missing_source"
}
}
if thresholds.StatSourceMinSamples > 0 && statGroup.sources.found >= thresholds.StatSourceMinSamples {
activeSourceTouches := statGroup.sources.written + statGroup.sources.skippedThrottled + statGroup.sources.skippedUnmanaged
if activeSourceTouches <= 0 && statGroup.sources.skippedMissingEndpoint <= 0 {
return "degraded", "stat_source_tracking_missing"
}
if thresholds.StatSourceMaxMissingRatio > 0 {
ratio := statGroup.sources.skippedMissingEndpoint / statGroup.sources.found
if ratio > thresholds.StatSourceMaxMissingRatio {
return "degraded", "stat_source_missing_endpoint_ratio"
}
}
}
if thresholds.StatProjectionMinSampleWrites > 0 &&
statGroup.projections.samplesWritten >= thresholds.StatProjectionMinSampleWrites &&
statGroup.projections.written <= 0 {
return "degraded", "stat_projection_missing"
}
return "ok", ""
}
func findingsForStatProjections(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.StatProjectionMinSampleWrites <= 0 {
return nil
}
groups := map[string]*statProjectionCounters{}
for _, sample := range parsed.samples {
switch sample.name {
case "vehicle_stat_samples_total":
if sample.labels["status"] == "written" {
statProjectionGroup(groups, sample.labels["topic"]).samplesWritten += sample.value
}
case "vehicle_stat_projections_total":
group := statProjectionGroup(groups, sample.labels["topic"])
switch sample.labels["status"] {
case "attempted":
group.attempted += sample.value
case "written":
group.written += sample.value
case "skipped_throttled":
group.skippedThrottled += sample.value
}
}
}
var topics []string
for topic := range groups {
topics = append(topics, topic)
}
sort.Strings(topics)
var findings []string
for _, topic := range topics {
group := groups[topic]
if group.samplesWritten < thresholds.StatProjectionMinSampleWrites {
continue
}
if group.written > 0 {
continue
}
findings = append(findings, fmt.Sprintf(
"stat final projection missing for %s: samples written %.0f, projections attempted %.0f, written %.0f, skipped throttled %.0f",
topic,
group.samplesWritten,
group.attempted,
group.written,
group.skippedThrottled,
))
}
return findings
}
func statProjectionGroup(groups map[string]*statProjectionCounters, topic string) *statProjectionCounters {
topic = strings.TrimSpace(topic)
if topic == "" {
topic = "unknown"
}
group := groups[topic]
if group == nil {
group = &statProjectionCounters{}
groups[topic] = group
}
return group
}
func findingsForStatCache(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.StatCacheEntriesMax <= 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_stat_cache_entries" || sample.value <= thresholds.StatCacheEntriesMax {
continue
}
cache := strings.TrimSpace(sample.labels["cache"])
if cache == "" {
cache = "unknown"
}
findings = append(findings, fmt.Sprintf(
"stat cache entries %.0f exceeds %.0f for cache %s",
sample.value,
thresholds.StatCacheEntriesMax,
cache,
))
}
sort.Strings(findings)
return findings
}
type realtimeThrottleCounters struct {
passed float64
skippedInterval float64
skippedNonRealtime float64
skippedMissingVIN float64
}
func findingsForRealtimeThrottle(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.RealtimeThrottleMinSamples <= 0 {
return nil
}
groups := map[string]*realtimeThrottleCounters{}
for _, sample := range parsed.samples {
if sample.name != "vehicle_realtime_throttle_total" {
continue
}
group := realtimeThrottleGroup(groups, sample.labels["store"], sample.labels["protocol"])
switch sample.labels["status"] {
case "passed":
group.passed += sample.value
case "skipped_interval":
group.skippedInterval += sample.value
case "skipped_non_realtime":
group.skippedNonRealtime += sample.value
case "skipped_missing_vin":
group.skippedMissingVIN += sample.value
}
}
var keys []string
for key := range groups {
keys = append(keys, key)
}
sort.Strings(keys)
var findings []string
for _, key := range keys {
group := groups[key]
total := group.passed + group.skippedInterval + group.skippedNonRealtime + group.skippedMissingVIN
if total < thresholds.RealtimeThrottleMinSamples || group.skippedInterval <= 0 || group.passed > 0 {
continue
}
findings = append(findings, fmt.Sprintf(
"realtime throttle passing updates missing for %s: total %.0f, passed %.0f, skipped interval %.0f, skipped non-realtime %.0f, skipped missing vin %.0f",
key,
total,
group.passed,
group.skippedInterval,
group.skippedNonRealtime,
group.skippedMissingVIN,
))
}
return findings
}
func findingsForRealtimeThrottleCache(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.RealtimeThrottleCacheEntriesMax <= 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_realtime_throttle_cache_entries" || sample.value <= thresholds.RealtimeThrottleCacheEntriesMax {
continue
}
store := strings.TrimSpace(sample.labels["store"])
if store == "" {
store = "unknown"
}
findings = append(findings, fmt.Sprintf(
"realtime throttle cache entries %.0f exceeds %.0f for store %s",
sample.value,
thresholds.RealtimeThrottleCacheEntriesMax,
store,
))
}
sort.Strings(findings)
return findings
}
func findingsForRealtimePlateCache(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.RealtimePlateCacheEntriesMax <= 0 {
return nil
}
value := parsed.totals["vehicle_realtime_plate_cache_entries"]
if value <= thresholds.RealtimePlateCacheEntriesMax {
return nil
}
return []string{fmt.Sprintf(
"realtime plate cache entries %.0f exceeds %.0f",
value,
thresholds.RealtimePlateCacheEntriesMax,
)}
}
func realtimeThrottleGroup(groups map[string]*realtimeThrottleCounters, store string, protocol string) *realtimeThrottleCounters {
store = strings.TrimSpace(store)
if store == "" {
store = "unknown"
}
protocol = strings.TrimSpace(protocol)
if protocol == "" {
protocol = "unknown"
}
key := "store=" + store + ",protocol=" + protocol
group := groups[key]
if group == nil {
group = &realtimeThrottleCounters{}
groups[key] = group
}
return group
}
type fastWriterRedisFieldCounters struct {
seen float64
written float64
skippedStale float64
}
type fastWriterRedisEnvelopeCounters struct {
seen float64
updated float64
skippedNonRealtime float64
skippedMissingVIN float64
skippedMissingVehicleKey float64
skippedMissingFields float64
}
type fastWriterMessageCounters struct {
received float64
ok float64
invalidJSON float64
}
func findingsForFastWriterMessageCompletion(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.FastWriterMessageMinReceived <= 0 || thresholds.FastWriterMessageMaxUnackedRatio <= 0 {
return nil
}
groups := map[string]*fastWriterMessageCounters{}
for _, sample := range parsed.samples {
if sample.name != "vehicle_fast_writer_messages_total" {
continue
}
subject := strings.TrimSpace(sample.labels["subject"])
if subject == "" || subject == "mixed" || subject == "unknown" {
continue
}
group := groups[subject]
if group == nil {
group = &fastWriterMessageCounters{}
groups[subject] = group
}
switch sample.labels["status"] {
case "received":
group.received += sample.value
case "ok":
group.ok += sample.value
case "invalid_json":
group.invalidJSON += sample.value
}
}
var subjects []string
for subject := range groups {
subjects = append(subjects, subject)
}
sort.Strings(subjects)
var findings []string
for _, subject := range subjects {
group := groups[subject]
if group.received < thresholds.FastWriterMessageMinReceived {
continue
}
completed := group.ok + group.invalidJSON
unacked := group.received - completed
if unacked < 0 {
unacked = 0
}
ratio := unacked / group.received
if ratio <= thresholds.FastWriterMessageMaxUnackedRatio {
continue
}
findings = append(findings, fmt.Sprintf(
"fast writer unacked message ratio %.2f exceeds %.2f for subject %s: received %.0f, ok %.0f, invalid_json %.0f, unacked %.0f",
ratio,
thresholds.FastWriterMessageMaxUnackedRatio,
subject,
group.received,
group.ok,
group.invalidJSON,
unacked,
))
}
return findings
}
func findingsForFastWriterRedisEnvelopes(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.FastWriterRedisEnvelopeMinSeen <= 0 {
return nil
}
groups := map[string]*fastWriterRedisEnvelopeCounters{}
for _, sample := range parsed.samples {
if sample.name != "vehicle_fast_writer_redis_envelopes_total" {
continue
}
subject := strings.TrimSpace(sample.labels["subject"])
if subject == "" || subject == "mixed" || subject == "unknown" {
continue
}
group := groups[subject]
if group == nil {
group = &fastWriterRedisEnvelopeCounters{}
groups[subject] = group
}
switch sample.labels["status"] {
case "seen":
group.seen += sample.value
case "updated":
group.updated += sample.value
case "skipped_non_realtime":
group.skippedNonRealtime += sample.value
case "skipped_missing_vin":
group.skippedMissingVIN += sample.value
case "skipped_missing_vehicle_key":
group.skippedMissingVehicleKey += sample.value
case "skipped_missing_fields":
group.skippedMissingFields += sample.value
}
}
var subjects []string
for subject := range groups {
subjects = append(subjects, subject)
}
sort.Strings(subjects)
var findings []string
for _, subject := range subjects {
group := groups[subject]
if group.seen < thresholds.FastWriterRedisEnvelopeMinSeen {
continue
}
if group.updated <= 0 {
findings = append(findings, fmt.Sprintf(
"fast writer redis realtime updates missing for subject %s: envelopes seen %.0f, updated %.0f, skipped non-realtime %.0f, missing vin %.0f, missing vehicle key %.0f, missing parsed fields %.0f",
subject,
group.seen,
group.updated,
group.skippedNonRealtime,
group.skippedMissingVIN,
group.skippedMissingVehicleKey,
group.skippedMissingFields,
))
continue
}
if thresholds.FastWriterRedisEnvelopeMaxBadRatio <= 0 {
continue
}
actionableSkips := group.skippedMissingVIN + group.skippedMissingVehicleKey + group.skippedMissingFields
ratio := actionableSkips / group.seen
if ratio <= thresholds.FastWriterRedisEnvelopeMaxBadRatio {
continue
}
findings = append(findings, fmt.Sprintf(
"fast writer redis envelope skip ratio %.2f exceeds %.2f for subject %s: envelopes seen %.0f, updated %.0f, missing vin %.0f, missing vehicle key %.0f, missing parsed fields %.0f",
ratio,
thresholds.FastWriterRedisEnvelopeMaxBadRatio,
subject,
group.seen,
group.updated,
group.skippedMissingVIN,
group.skippedMissingVehicleKey,
group.skippedMissingFields,
))
}
return findings
}
func findingsForFastWriterRedisFields(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.FastWriterRedisFieldMinSeen <= 0 || thresholds.FastWriterRedisFieldMaxStaleRatio <= 0 {
return nil
}
groups := map[string]*fastWriterRedisFieldCounters{}
for _, sample := range parsed.samples {
if sample.name != "vehicle_fast_writer_redis_fields_total" {
continue
}
subject := strings.TrimSpace(sample.labels["subject"])
if subject == "" || subject == "mixed" || subject == "unknown" {
continue
}
group := groups[subject]
if group == nil {
group = &fastWriterRedisFieldCounters{}
groups[subject] = group
}
switch sample.labels["status"] {
case "seen":
group.seen += sample.value
case "written":
group.written += sample.value
case "skipped_stale":
group.skippedStale += sample.value
}
}
var subjects []string
for subject := range groups {
subjects = append(subjects, subject)
}
sort.Strings(subjects)
var findings []string
for _, subject := range subjects {
group := groups[subject]
if group.seen < thresholds.FastWriterRedisFieldMinSeen || group.seen <= 0 {
continue
}
ratio := group.skippedStale / group.seen
if ratio <= thresholds.FastWriterRedisFieldMaxStaleRatio {
continue
}
findings = append(findings, fmt.Sprintf(
"fast writer redis stale field ratio %.2f exceeds %.2f for subject %s (seen %.0f, written %.0f, skipped_stale %.0f)",
ratio,
thresholds.FastWriterRedisFieldMaxStaleRatio,
subject,
group.seen,
group.written,
group.skippedStale,
))
}
return findings
}
func findingsForHistoryLocationErrors(parsed parsedMetricSet, thresholds Thresholds, checkedAt time.Time) []string {
if thresholds.HistoryLocationErrorMax < 0 {
return nil
}
lastErrorByTopic := map[string]float64{}
if thresholds.HistoryLocationErrorRecentSec > 0 {
for _, sample := range parsed.samples {
if sample.name != "vehicle_history_last_location_write_unix_seconds" || sample.labels["status"] != "error" || sample.value <= 0 {
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if sample.value > lastErrorByTopic[topic] {
lastErrorByTopic[topic] = sample.value
}
}
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_history_location_writes_total" || sample.labels["status"] != "error" {
continue
}
if sample.value <= thresholds.HistoryLocationErrorMax {
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if topic == "" {
topic = "unknown"
}
recency := ""
if thresholds.HistoryLocationErrorRecentSec > 0 {
if lastError, ok := lastErrorByTopic[topic]; ok {
ageSec := checkedAt.Sub(time.Unix(int64(lastError), 0).UTC()).Seconds()
if ageSec < 0 {
ageSec = 0
}
if ageSec > thresholds.HistoryLocationErrorRecentSec {
continue
}
recency = fmt.Sprintf(", last error %.0fs ago", ageSec)
} else {
recency = ", last error activity unavailable"
}
}
findings = append(findings, fmt.Sprintf(
"history location writes error %.0f exceeds %.0f for topic %s%s",
sample.value,
thresholds.HistoryLocationErrorMax,
topic,
recency,
))
}
sort.Strings(findings)
return findings
}
func findingsForHistoryLocationAccounting(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.HistoryLocationAccountingMaxRatio <= 0 {
return nil
}
writesOKByTopic := map[string]float64{}
locationStatusByTopic := map[string]float64{}
for _, sample := range parsed.samples {
topic := strings.TrimSpace(sample.labels["topic"])
if topic == "" || topic == "mixed" || topic == "unknown" {
continue
}
switch {
case sample.name == "vehicle_history_writes_total" && sample.labels["status"] == "ok":
writesOKByTopic[topic] += sample.value
case sample.name == "vehicle_history_location_writes_total":
locationStatusByTopic[topic] += sample.value
}
}
var topics []string
for topic := range writesOKByTopic {
topics = append(topics, topic)
}
sort.Strings(topics)
var findings []string
for _, topic := range topics {
writesOK := writesOKByTopic[topic]
if writesOK < thresholds.KafkaConsumerMessageMinReceived {
continue
}
accounted := locationStatusByTopic[topic]
diff := math.Abs(writesOK - accounted)
ratio := diff / writesOK
if ratio <= thresholds.HistoryLocationAccountingMaxRatio {
continue
}
findings = append(findings, fmt.Sprintf(
"history location accounting mismatch ratio %.2f exceeds %.2f for topic %s: history writes ok %.0f, location statuses %.0f, diff %.0f",
ratio,
thresholds.HistoryLocationAccountingMaxRatio,
topic,
writesOK,
accounted,
diff,
))
}
return findings
}
func findingsForHistoryFallbackErrors(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.HistoryFallbackErrorMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_history_batch_fallback_total" || sample.labels["status"] != "error" {
continue
}
if sample.value <= thresholds.HistoryFallbackErrorMax {
continue
}
findings = append(findings, fmt.Sprintf(
"history batch fallback errors %.0f exceeds %.0f",
sample.value,
thresholds.HistoryFallbackErrorMax,
))
}
sort.Strings(findings)
return findings
}
func findingsForFastWriterFallbackErrors(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.FastWriterFallbackErrorMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_fast_writer_batch_fallback_total" || sample.labels["status"] != "error" {
continue
}
if sample.value <= thresholds.FastWriterFallbackErrorMax {
continue
}
stage := strings.TrimSpace(sample.labels["stage"])
if stage == "" {
stage = "unknown"
}
findings = append(findings, fmt.Sprintf(
"fast writer batch fallback errors %.0f exceeds %.0f for stage %s",
sample.value,
thresholds.FastWriterFallbackErrorMax,
stage,
))
}
sort.Strings(findings)
return findings
}
func findingsForFastWriterDecoupledUpdateErrors(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.FastWriterDecoupledUpdateErrorMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_fast_writer_decoupled_updates_total" || sample.labels["status"] != "error" {
continue
}
if sample.value <= thresholds.FastWriterDecoupledUpdateErrorMax {
continue
}
reason := strings.TrimSpace(sample.labels["reason"])
if reason == "" {
reason = "unknown"
}
findings = append(findings, fmt.Sprintf(
"fast writer decoupled update errors %.0f exceeds %.0f for reason %s",
sample.value,
thresholds.FastWriterDecoupledUpdateErrorMax,
reason,
))
}
sort.Strings(findings)
return findings
}
func findingsForWriteRetries(parsed parsedMetricSet, thresholds Thresholds) []string {
var findings []string
findings = append(findings, writeRetryFindings(parsed, "vehicle_history_write_retries_total", "history", thresholds.HistoryRetryMax, thresholds.HistoryRetryExhaustedMax)...)
findings = append(findings, writeRetryFindings(parsed, "vehicle_stat_write_retries_total", "stat", thresholds.StatRetryMax, thresholds.StatRetryExhaustedMax)...)
sort.Strings(findings)
return findings
}
func writeRetryFindings(parsed parsedMetricSet, metricName string, serviceLabel string, retryMax float64, exhaustedMax float64) []string {
var findings []string
for _, sample := range parsed.samples {
if sample.name != metricName {
continue
}
status := strings.TrimSpace(sample.labels["status"])
operation := strings.TrimSpace(sample.labels["operation"])
if operation == "" {
operation = "unknown"
}
switch status {
case "retry":
if retryMax < 0 || sample.value <= retryMax {
continue
}
findings = append(findings, fmt.Sprintf(
"%s write retries %.0f exceeds %.0f for operation %s",
serviceLabel,
sample.value,
retryMax,
operation,
))
case "exhausted":
if exhaustedMax < 0 || sample.value <= exhaustedMax {
continue
}
findings = append(findings, fmt.Sprintf(
"%s write retry exhausted %.0f exceeds %.0f for operation %s",
serviceLabel,
sample.value,
exhaustedMax,
operation,
))
}
}
return findings
}
func findingsForStatInvalidJSON(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.StatInvalidJSONMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_stat_kafka_messages_total" || sample.labels["status"] != "invalid_json" {
continue
}
if sample.value <= thresholds.StatInvalidJSONMax {
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if topic == "" {
topic = "unknown"
}
findings = append(findings, fmt.Sprintf(
"stat invalid json %.0f exceeds %.0f for topic %s",
sample.value,
thresholds.StatInvalidJSONMax,
topic,
))
}
sort.Strings(findings)
return findings
}
func findingsForStatProtocolTopicMismatches(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.StatProtocolTopicMismatchMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_stat_kafka_messages_total" || sample.labels["status"] != "protocol_topic_mismatch" {
continue
}
if sample.value <= thresholds.StatProtocolTopicMismatchMax {
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if topic == "" {
topic = "unknown"
}
findings = append(findings, fmt.Sprintf(
"stat protocol topic mismatches %.0f exceeds %.0f for topic %s",
sample.value,
thresholds.StatProtocolTopicMismatchMax,
topic,
))
}
findings = append(findings, findingsForStatMetricProtocolTopicMismatches(parsed, thresholds)...)
sort.Strings(findings)
return findings
}
func findingsForStatMetricProtocolTopicMismatches(parsed parsedMetricSet, thresholds Thresholds) []string {
var findings []string
for _, sample := range parsed.samples {
switch sample.name {
case "vehicle_stat_samples_total", "vehicle_stat_sources_total", "vehicle_stat_projections_total":
default:
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
expectedProtocol, ok := topics.ProtocolForKnownFieldsTopic(topic)
if !ok {
continue
}
protocol := strings.TrimSpace(sample.labels["protocol"])
if sample.value <= thresholds.StatProtocolTopicMismatchMax {
continue
}
status := strings.TrimSpace(sample.labels["status"])
if status == "" {
status = "unknown"
}
if protocol == "" {
findings = append(findings, fmt.Sprintf(
"stat metric protocol label missing %.0f exceeds %.0f for %s topic %s (status %s)",
sample.value,
thresholds.StatProtocolTopicMismatchMax,
sample.name,
topic,
status,
))
continue
}
if protocol == expectedProtocol {
continue
}
findings = append(findings, fmt.Sprintf(
"stat metric protocol topic mismatches %.0f exceeds %.0f for %s topic %s: expected %s, got %s (status %s)",
sample.value,
thresholds.StatProtocolTopicMismatchMax,
sample.name,
topic,
expectedProtocol,
protocol,
status,
))
}
return findings
}
func findingsForFastWriterInvalidJSON(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.FastWriterInvalidJSONMax < 0 {
return nil
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_fast_writer_messages_total" || sample.labels["status"] != "invalid_json" {
continue
}
if sample.value <= thresholds.FastWriterInvalidJSONMax {
continue
}
subject := strings.TrimSpace(sample.labels["subject"])
if subject == "" {
subject = "unknown"
}
findings = append(findings, fmt.Sprintf(
"fast writer invalid json %.0f exceeds %.0f for subject %s",
sample.value,
thresholds.FastWriterInvalidJSONMax,
subject,
))
}
sort.Strings(findings)
return findings
}
func findingsForStatSamples(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.StatSampleMinWrites <= 0 {
return nil
}
groups := map[string]*statSampleCounters{}
for _, sample := range parsed.samples {
switch sample.name {
case "vehicle_stat_writes_total":
if sample.labels["status"] != "ok" {
continue
}
statSampleGroup(groups, sample.labels["topic"]).writesOK += sample.value
case "vehicle_stat_samples_total":
group := statSampleGroup(groups, sample.labels["topic"])
switch sample.labels["status"] {
case "found":
group.found += sample.value
case "written":
group.written += sample.value
case "skipped_missing_fields":
group.skippedMissingFields += sample.value
case "skipped_missing_vin":
group.skippedMissingVIN += sample.value
case "skipped_missing_mileage":
group.skippedMissingMileage += sample.value
case "skipped_non_mileage_frame":
group.skippedNonMileageFrame += sample.value
case "skipped_non_positive_mileage":
group.skippedNonPositiveMileage += sample.value
case "skipped_missing_time":
group.skippedMissingTime += sample.value
case "event_time_future_adjusted":
group.futureAdjusted += sample.value
case "skipped_same_mileage":
group.skippedSameMileage += sample.value
case "skipped_missing_source":
group.skippedMissingSource += sample.value
}
}
}
var topics []string
for topic := range groups {
topics = append(topics, topic)
}
sort.Strings(topics)
var findings []string
for _, topic := range topics {
group := groups[topic]
mileageCandidates := statMileageCandidateWrites(*group)
if mileageCandidates < thresholds.StatSampleMinWrites {
continue
}
if thresholds.StatSampleFutureAdjustmentMax >= 0 &&
group.futureAdjusted > thresholds.StatSampleFutureAdjustmentMax {
findings = append(findings, fmt.Sprintf(
"stat future event time adjustments %.0f exceeds %.0f for %s: writes ok %.0f, mileage candidates %.0f, found %.0f, written %.0f",
group.futureAdjusted,
thresholds.StatSampleFutureAdjustmentMax,
topic,
group.writesOK,
mileageCandidates,
group.found,
group.written,
))
}
actionableSkips := statActionableSkips(*group)
if group.found <= 0 {
if actionableSkips <= 0 && group.skippedNonMileageFrame > 0 {
continue
}
if statOnlyMissingMileageSkips(*group) {
continue
}
findings = append(findings, fmt.Sprintf(
"stat samples missing for %s: writes ok %.0f, mileage candidates %.0f, missing fields %.0f, skipped missing vin %.0f, missing mileage %.0f, non-mileage frames %.0f, non-positive mileage %.0f, missing time %.0f, missing source %.0f",
topic,
group.writesOK,
mileageCandidates,
group.skippedMissingFields,
group.skippedMissingVIN,
group.skippedMissingMileage,
group.skippedNonMileageFrame,
group.skippedNonPositiveMileage,
group.skippedMissingTime,
group.skippedMissingSource,
))
continue
}
if thresholds.StatSampleMaxActionableSkipRatio > 0 {
ratio := actionableSkips / mileageCandidates
if ratio > thresholds.StatSampleMaxActionableSkipRatio {
findings = append(findings, fmt.Sprintf(
"stat actionable skip ratio %.2f exceeds %.2f for %s: writes ok %.0f, mileage candidates %.0f, found %.0f, written %.0f, missing fields %.0f, skipped missing vin %.0f, missing mileage %.0f, non-mileage frames %.0f, non-positive mileage %.0f, missing time %.0f, missing source %.0f, same mileage %.0f",
ratio,
thresholds.StatSampleMaxActionableSkipRatio,
topic,
group.writesOK,
mileageCandidates,
group.found,
group.written,
group.skippedMissingFields,
group.skippedMissingVIN,
group.skippedMissingMileage,
group.skippedNonMileageFrame,
group.skippedNonPositiveMileage,
group.skippedMissingTime,
group.skippedMissingSource,
group.skippedSameMileage,
))
}
}
if group.written <= 0 && group.skippedMissingSource > 0 {
findings = append(findings, fmt.Sprintf(
"stat samples not written for %s: found %.0f, skipped missing source %.0f, same mileage %.0f",
topic,
group.found,
group.skippedMissingSource,
group.skippedSameMileage,
))
}
}
return findings
}
func findingsForStatSources(parsed parsedMetricSet, thresholds Thresholds) []string {
if thresholds.StatSourceMinSamples <= 0 {
return nil
}
groups := map[string]*statSourceCounters{}
for _, sample := range parsed.samples {
switch sample.name {
case "vehicle_stat_samples_total":
if sample.labels["status"] == "found" {
statSourceGroup(groups, sample.labels["topic"]).found += sample.value
}
case "vehicle_stat_sources_total":
group := statSourceGroup(groups, sample.labels["topic"])
switch sample.labels["status"] {
case "attempted":
group.attempted += sample.value
case "written":
group.written += sample.value
case "skipped_throttled":
group.skippedThrottled += sample.value
case "skipped_missing_endpoint":
group.skippedMissingEndpoint += sample.value
case "skipped_unmanaged":
group.skippedUnmanaged += sample.value
}
}
}
var topics []string
for topic := range groups {
topics = append(topics, topic)
}
sort.Strings(topics)
var findings []string
for _, topic := range topics {
group := groups[topic]
if group.found < thresholds.StatSourceMinSamples || group.found <= 0 {
continue
}
activeSourceTouches := group.written + group.skippedThrottled + group.skippedUnmanaged
if activeSourceTouches <= 0 && group.skippedMissingEndpoint <= 0 {
findings = append(findings, fmt.Sprintf(
"stat source tracking missing for %s: samples found %.0f, source written %.0f, source throttled %.0f, source unmanaged %.0f, missing endpoint %.0f",
topic,
group.found,
group.written,
group.skippedThrottled,
group.skippedUnmanaged,
group.skippedMissingEndpoint,
))
continue
}
if thresholds.StatSourceMaxMissingRatio <= 0 {
continue
}
ratio := group.skippedMissingEndpoint / group.found
if ratio > thresholds.StatSourceMaxMissingRatio {
findings = append(findings, fmt.Sprintf(
"stat source missing endpoint ratio %.2f exceeds %.2f for %s: samples found %.0f, source written %.0f, source throttled %.0f, source unmanaged %.0f, missing endpoint %.0f",
ratio,
thresholds.StatSourceMaxMissingRatio,
topic,
group.found,
group.written,
group.skippedThrottled,
group.skippedUnmanaged,
group.skippedMissingEndpoint,
))
}
}
return findings
}
func statSampleGroup(groups map[string]*statSampleCounters, topic string) *statSampleCounters {
topic = strings.TrimSpace(topic)
if topic == "" {
topic = "unknown"
}
group := groups[topic]
if group == nil {
group = &statSampleCounters{}
groups[topic] = group
}
return group
}
func statSourceGroup(groups map[string]*statSourceCounters, topic string) *statSourceCounters {
topic = strings.TrimSpace(topic)
if topic == "" {
topic = "unknown"
}
group := groups[topic]
if group == nil {
group = &statSourceCounters{}
groups[topic] = group
}
return group
}
func histogramP99Findings(parsed parsedMetricSet, histogramName string, label string, thresholdMS float64, minSamples float64) []string {
if thresholdMS <= 0 {
return nil
}
groups := histogramGroups(parsed.samples, histogramName)
var findings []string
var keys []string
for key := range groups {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
estimate, count, ok := histogramQuantile(groups[key], 0.99)
if !ok || count < minSamples || estimate <= thresholdMS {
continue
}
findings = append(findings, fmt.Sprintf("%s p99 %.0fms exceeds %.0fms (%s, samples %.0f)", label, estimate, thresholdMS, key, count))
}
return findings
}
func recentP99GaugeFindings(parsed parsedMetricSet, metricName string, samplesMetricName string, label string, thresholdMS float64, minSamples float64) []string {
if thresholdMS <= 0 {
return nil
}
samplesByKey := map[string]float64{}
for _, sample := range parsed.samples {
if sample.name != samplesMetricName {
continue
}
samplesByKey[metricLabelsKey(sample.labels)] = sample.value
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != metricName {
continue
}
key := metricLabelsKey(sample.labels)
samples := samplesByKey[key]
if samples < minSamples || sample.value <= thresholdMS {
continue
}
findings = append(findings, fmt.Sprintf("%s recent p99 %.0fms exceeds %.0fms (%s, samples %.0f)", label, sample.value, thresholdMS, key, samples))
}
sort.Strings(findings)
return findings
}
func parseMetrics(text string) map[string]float64 {
return parseMetricSet(text).totals
}
type parsedMetricSet struct {
totals map[string]float64
samples []metricSample
}
type metricSample struct {
name string
labels map[string]string
value float64
}
func parseMetricSet(text string) parsedMetricSet {
out := map[string]float64{}
var samples []metricSample
scanner := bufio.NewScanner(strings.NewReader(text))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
sample, ok := parseMetricSample(line)
if !ok {
continue
}
if aggregateMetricByMax(sample.name) {
if current, ok := out[sample.name]; !ok || sample.value > current {
out[sample.name] = sample.value
}
} else {
out[sample.name] += sample.value
}
samples = append(samples, sample)
}
return parsedMetricSet{totals: out, samples: samples}
}
func aggregateMetricByMax(name string) bool {
return isLastActivityUnixMetric(name) ||
strings.HasSuffix(name, "_p99_ms")
}
func isLastActivityUnixMetric(name string) bool {
return strings.Contains(name, "_last_") && strings.HasSuffix(name, "_unix_seconds")
}
func addLastActivityAges(metrics map[string]float64, checkedAt time.Time) {
if len(metrics) == 0 {
return
}
for name, value := range metrics {
if !isLastActivityUnixMetric(name) || value <= 0 {
continue
}
age := checkedAt.Sub(time.Unix(int64(value), 0).UTC()).Seconds()
if age < 0 {
age = 0
}
metrics[strings.TrimSuffix(name, "_unix_seconds")+"_age_seconds"] = age
}
}
func parseMetricLine(line string) (string, float64, bool) {
sample, ok := parseMetricSample(line)
return sample.name, sample.value, ok
}
func parseMetricSample(line string) (metricSample, bool) {
parts := strings.Fields(line)
if len(parts) < 2 {
return metricSample{}, false
}
name, labels := parseMetricNameAndLabels(parts[0])
value, err := strconv.ParseFloat(parts[1], 64)
if err != nil {
return metricSample{}, false
}
return metricSample{name: name, labels: labels, value: value}, true
}
func parseMetricNameAndLabels(value string) (string, map[string]string) {
index := strings.IndexByte(value, '{')
if index < 0 {
return value, nil
}
end := strings.LastIndexByte(value, '}')
if end < index {
return value[:index], nil
}
return value[:index], parseMetricLabels(value[index+1 : end])
}
func parseMetricLabels(value string) map[string]string {
labels := map[string]string{}
for len(value) > 0 {
value = strings.TrimLeft(value, " \t,")
if value == "" {
break
}
index := strings.IndexByte(value, '=')
if index <= 0 {
break
}
key := strings.TrimSpace(value[:index])
value = strings.TrimLeft(value[index+1:], " \t")
if key == "" || value == "" {
break
}
if value[0] != '"' {
next := strings.IndexByte(value, ',')
if next < 0 {
labels[key] = strings.TrimSpace(value)
break
}
labels[key] = strings.TrimSpace(value[:next])
value = value[next+1:]
continue
}
labelValue, rest, ok := readQuotedLabelValue(value)
if !ok {
break
}
labels[key] = labelValue
value = rest
}
if len(labels) == 0 {
return nil
}
return labels
}
func readQuotedLabelValue(value string) (string, string, bool) {
if value == "" || value[0] != '"' {
return "", value, false
}
var b strings.Builder
escaped := false
for index := 1; index < len(value); index++ {
ch := value[index]
if escaped {
switch ch {
case 'n':
b.WriteByte('\n')
default:
b.WriteByte(ch)
}
escaped = false
continue
}
if ch == '\\' {
escaped = true
continue
}
if ch == '"' {
return b.String(), value[index+1:], true
}
b.WriteByte(ch)
}
return "", value, false
}
type histogramBucket struct {
upperBound float64
count float64
}
func histogramGroups(samples []metricSample, histogramName string) map[string][]histogramBucket {
groups := map[string][]histogramBucket{}
for _, sample := range samples {
if sample.name != histogramName+"_bucket" {
continue
}
upperBound, ok := parseBucketUpperBound(sample.labels["le"])
if !ok {
continue
}
key := metricLabelsKey(sample.labels, "le")
groups[key] = append(groups[key], histogramBucket{upperBound: upperBound, count: sample.value})
}
return groups
}
func parseBucketUpperBound(value string) (float64, bool) {
if value == "+Inf" {
return math.Inf(1), true
}
parsed, err := strconv.ParseFloat(value, 64)
return parsed, err == nil
}
func histogramQuantile(buckets []histogramBucket, quantile float64) (float64, float64, bool) {
if len(buckets) == 0 || quantile <= 0 || quantile > 1 {
return 0, 0, false
}
sort.Slice(buckets, func(i, j int) bool {
return buckets[i].upperBound < buckets[j].upperBound
})
total := buckets[len(buckets)-1].count
if total <= 0 {
return 0, 0, false
}
target := total * quantile
for _, bucket := range buckets {
if bucket.count >= target {
return bucket.upperBound, total, true
}
}
return buckets[len(buckets)-1].upperBound, total, true
}
func metricLabelsKey(labels map[string]string, exclude ...string) string {
if len(labels) == 0 {
return "all"
}
excluded := map[string]struct{}{}
for _, key := range exclude {
excluded[key] = struct{}{}
}
keys := make([]string, 0, len(labels))
for key := range labels {
if _, skip := excluded[key]; skip {
continue
}
keys = append(keys, key)
}
if len(keys) == 0 {
return "all"
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, key := range keys {
parts = append(parts, key+"="+labels[key])
}
return strings.Join(parts, ",")
}
Reference in New Issue View Git Blame Copy Permalink