package capacity import ( "bufio" "fmt" "math" "sort" "strconv" "strings" "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics" ) type Status string const ( StatusOK Status = "ok" StatusDegraded Status = "degraded" ) type Report struct { Status Status `json:"status"` CheckedAt time.Time `json:"checked_at"` Totals Totals `json:"totals"` Services []ServiceCheck `json:"services"` GatewayFields *GatewayFieldsDiagnostics `json:"gateway_fields,omitempty"` ProtocolFlow *ProtocolFlowDiagnostics `json:"protocol_flow,omitempty"` IdentityResolution *IdentityResolutionDiagnostics `json:"identity_resolution,omitempty"` PipelineLatency *PipelineLatencyDiagnostics `json:"pipeline_latency,omitempty"` DailyMileageDiagnostics *DailyMileageDiagnostics `json:"daily_mileage_diagnostics,omitempty"` DailyMileageSourceQuality *DailyMileageSourceQualityDiagnostics `json:"daily_mileage_source_quality,omitempty"` DailyMileageSourceSelection *DailyMileageSourceSelectionDiagnostics `json:"daily_mileage_source_selection,omitempty"` DataSourceDiagnostics *DataSourceDiagnostics `json:"data_source_diagnostics,omitempty"` DataSourceKindDiagnostics *DataSourceDiagnostics `json:"data_source_kind_diagnostics,omitempty"` JT808IdentityGaps *JT808IdentityGapDiagnostics `json:"jt808_identity_gaps,omitempty"` Findings []string `json:"findings"` } type Totals struct { ActiveConnections int64 `json:"active_connections"` KafkaLag float64 `json:"kafka_lag"` BridgeConsumerPending float64 `json:"bridge_consumer_pending"` BridgeAckPending float64 `json:"bridge_ack_pending"` BridgeBatchPendingMessages float64 `json:"bridge_batch_pending_messages"` FieldsProjectorKafkaLag float64 `json:"fields_projector_kafka_lag"` FieldsProjectorBatchPending float64 `json:"fields_projector_batch_pending_messages"` FieldsProjectorRetryPending float64 `json:"fields_projector_retry_pending_messages"` FastWriterConsumerPending float64 `json:"fast_writer_consumer_pending"` FastWriterAckPending float64 `json:"fast_writer_ack_pending"` FastWriterBatchPending float64 `json:"fast_writer_batch_pending_messages"` HistoryBatchPending float64 `json:"history_batch_pending_messages"` HistoryRowsPending float64 `json:"history_rows_pending"` HistoryRetryPending float64 `json:"history_retry_pending_messages"` RealtimeBatchPending float64 `json:"realtime_batch_pending_messages"` RealtimeAsyncQueuePending float64 `json:"realtime_async_queue_pending"` RealtimeRetryPending float64 `json:"realtime_retry_pending_messages"` StatRetryPending float64 `json:"stat_retry_pending_messages"` IdentityBatchPending float64 `json:"identity_batch_pending_messages"` IdentityFactsPending float64 `json:"identity_batch_pending_facts"` IdentityRetryPending float64 `json:"identity_retry_pending_messages"` JT808RegistrationWriteQueuePending float64 `json:"jt808_registration_write_queue_pending"` JT808RegistrationWriteErrors float64 `json:"jt808_registration_write_errors"` } type ServiceCheck struct { Name string `json:"name"` Metrics map[string]float64 `json:"metrics"` } type GatewayFieldsDiagnostics struct { Protocols []GatewayProtocolFields `json:"protocols,omitempty"` } type GatewayProtocolFields struct { Protocol string `json:"protocol"` RawOK float64 `json:"raw_ok"` RealtimeCandidates float64 `json:"realtime_candidates"` FieldsOK float64 `json:"fields_ok"` FieldsPublished float64 `json:"fields_published"` FieldsDelegated float64 `json:"fields_delegated"` FieldsSkippedNonRealtime float64 `json:"fields_skipped_non_realtime"` FieldsSkippedMissing float64 `json:"fields_skipped_missing"` FieldsPublishError float64 `json:"fields_publish_error"` MissingRatio float64 `json:"missing_ratio"` LatestFieldCount *float64 `json:"latest_field_count,omitempty"` Status string `json:"status"` Reason string `json:"reason,omitempty"` } type ProtocolFlowDiagnostics struct { Protocols []ProtocolFlowProtocol `json:"protocols,omitempty"` } type ProtocolFlowProtocol struct { Protocol string `json:"protocol"` RawOK float64 `json:"raw_ok"` RealtimeCandidates float64 `json:"realtime_candidates"` FieldsPublished float64 `json:"fields_published"` LatestFieldCount *float64 `json:"latest_field_count,omitempty"` IdentityTotal int64 `json:"identity_total"` IdentitySkipped int64 `json:"identity_skipped"` IdentityResolved int64 `json:"identity_resolved"` IdentityNonResolved int64 `json:"identity_non_resolved"` IdentityNonResolvedRatio float64 `json:"identity_non_resolved_ratio"` StatWritesOK float64 `json:"stat_writes_ok"` StatMileageCandidateWrites float64 `json:"stat_mileage_candidate_writes"` StatSamplesFound float64 `json:"stat_samples_found"` StatSamplesWritten float64 `json:"stat_samples_written"` StatSamplesActionableSkipped float64 `json:"stat_samples_actionable_skipped"` StatSkippedMissingFields float64 `json:"stat_skipped_missing_fields"` StatSkippedMissingVIN float64 `json:"stat_skipped_missing_vin"` StatSkippedMissingMileage float64 `json:"stat_skipped_missing_mileage"` StatSkippedNonMileageFrame float64 `json:"stat_skipped_non_mileage_frame"` StatSkippedNonPositiveMileage float64 `json:"stat_skipped_non_positive_mileage"` StatSkippedMissingTime float64 `json:"stat_skipped_missing_time"` StatEventTimeFutureAdjusted float64 `json:"stat_event_time_future_adjusted"` StatSkippedMissingSource float64 `json:"stat_skipped_missing_source"` StatSkippedSameMileage float64 `json:"stat_skipped_same_mileage"` StatSourceWritten float64 `json:"stat_source_written"` StatSourceThrottled float64 `json:"stat_source_throttled"` StatSourceMissingEndpoint float64 `json:"stat_source_missing_endpoint"` StatSourceSkippedUnmanaged float64 `json:"stat_source_skipped_unmanaged"` StatProjectionAttempted float64 `json:"stat_projection_attempted"` StatProjectionWritten float64 `json:"stat_projection_written"` StatProjectionThrottled float64 `json:"stat_projection_throttled"` Status string `json:"status"` Reason string `json:"reason,omitempty"` } type IdentityResolutionDiagnostics struct { Total int64 `json:"total"` SkippedCount int64 `json:"skipped_count"` ResolvedCount int64 `json:"resolved_count"` NonResolvedCount int64 `json:"non_resolved_count"` UnresolvedCount int64 `json:"unresolved_count"` TimeoutCount int64 `json:"timeout_count"` ErrorCount int64 `json:"error_count"` OtherNonResolvedCount int64 `json:"other_non_resolved_count"` NonResolvedRatio float64 `json:"non_resolved_ratio"` StaleCacheCount int64 `json:"stale_cache_count"` StaleCacheRatio float64 `json:"stale_cache_ratio"` CacheEntries map[string]int64 `json:"cache_entries,omitempty"` Issues []IdentityIssueCount `json:"issues,omitempty"` Protocols []IdentityProtocolResolution `json:"protocols,omitempty"` } type IdentityIssueCount struct { Protocol string `json:"protocol"` Status string `json:"status"` MessageID string `json:"message_id"` Reason string `json:"reason"` Count int64 `json:"count"` } type IdentityProtocolResolution struct { Protocol string `json:"protocol"` Total int64 `json:"total"` SkippedCount int64 `json:"skipped_count"` ResolvedCount int64 `json:"resolved_count"` NonResolvedCount int64 `json:"non_resolved_count"` UnresolvedCount int64 `json:"unresolved_count"` TimeoutCount int64 `json:"timeout_count"` ErrorCount int64 `json:"error_count"` OtherNonResolvedCount int64 `json:"other_non_resolved_count"` NonResolvedRatio float64 `json:"non_resolved_ratio"` StaleCacheCount int64 `json:"stale_cache_count"` StaleCacheRatio float64 `json:"stale_cache_ratio"` CacheStatusCounts map[string]int64 `json:"cache_status_counts,omitempty"` } type PipelineLatencyDiagnostics struct { TotalSamples int64 `json:"total_samples"` MaxP99MS float64 `json:"max_p99_ms"` Items []PipelineLatencyItem `json:"items,omitempty"` } type PipelineLatencyItem struct { Stage string `json:"stage"` Service string `json:"service"` Topic string `json:"topic,omitempty"` Subject string `json:"subject,omitempty"` P99MS float64 `json:"p99_ms"` Samples int64 `json:"samples"` ThresholdMS float64 `json:"threshold_ms,omitempty"` Status string `json:"status"` } type DailyMileageDiagnostics struct { URL string `json:"url,omitempty"` VehicleTotal int64 `json:"vehicle_total"` ActionableIssueTotal int64 `json:"actionable_issue_total"` PipelineIssueTotal int64 `json:"pipeline_issue_total"` SourceDataIssueTotal int64 `json:"source_data_issue_total"` Items []DailyMileageDiagnosisReasonItem `json:"items,omitempty"` Samples []DailyMileageDiagnosisSample `json:"samples,omitempty"` FieldStatus *DailyMileageFieldStatusDiagnostics `json:"field_status,omitempty"` } type DailyMileageDiagnosisReasonItem struct { StatDate string `json:"stat_date"` Protocol string `json:"protocol"` Diagnosis string `json:"diagnosis"` Reason string `json:"reason"` Count int64 `json:"count"` Severity string `json:"severity,omitempty"` RecommendedOperatorAction string `json:"recommended_operator_action,omitempty"` } type DailyMileageFieldStatusDiagnostics struct { URL string `json:"url,omitempty"` SummaryTotal int64 `json:"summary_total"` VehicleTotal int64 `json:"vehicle_total"` ActionableIssueTotal int64 `json:"actionable_issue_total"` PipelineIssueTotal int64 `json:"pipeline_issue_total"` SourceDataIssueTotal int64 `json:"source_data_issue_total"` Items []DailyMileageFieldStatusSummaryItem `json:"items,omitempty"` } type DailyMileageFieldStatusSummaryItem struct { StatDate string `json:"stat_date"` Protocol string `json:"protocol"` Diagnosis string `json:"diagnosis"` Reason string `json:"reason"` RealtimeMileageFieldStatus string `json:"realtime_mileage_field_status"` Count int64 `json:"count"` Severity string `json:"severity,omitempty"` FieldStatusSeverity string `json:"field_status_severity,omitempty"` RecommendedOperatorAction string `json:"recommended_operator_action,omitempty"` FieldStatusAction string `json:"field_status_action,omitempty"` } type DailyMileageDiagnosisSample struct { VIN string `json:"vin"` StatDate string `json:"stat_date"` Protocol string `json:"protocol"` Plate string `json:"plate,omitempty"` PlatformName string `json:"platform_name,omitempty"` Peer string `json:"peer,omitempty"` SnapshotEventTime string `json:"snapshot_event_time,omitempty"` LocationEventTime string `json:"location_event_time,omitempty"` SnapshotUpdatedAt string `json:"snapshot_updated_at,omitempty"` LocationUpdatedAt string `json:"location_updated_at,omitempty"` RealtimeTotalMileageKM *float64 `json:"realtime_total_mileage_km,omitempty"` RealtimeTotalMileageAt string `json:"realtime_total_mileage_event_time,omitempty"` DailyMileageKM *float64 `json:"daily_mileage_km,omitempty"` DailyLatestTotalMileageKM *float64 `json:"daily_latest_total_mileage_km,omitempty"` SourceSampleCount int64 `json:"source_sample_count"` OKSourceCount int64 `json:"ok_source_count"` SelectableSourceCount int64 `json:"selectable_source_count"` RealtimeFieldCount int64 `json:"realtime_field_count,omitempty"` RealtimeSampleFields []string `json:"realtime_sample_fields,omitempty"` RealtimeMileageFieldStatus string `json:"realtime_mileage_field_status,omitempty"` RealtimeMileageEvidence string `json:"realtime_mileage_evidence,omitempty"` LatestStatEventTime string `json:"latest_stat_event_time,omitempty"` QualityStatuses []string `json:"quality_statuses,omitempty"` RawFrameQueryPath string `json:"raw_frame_query_path,omitempty"` SourceSampleQueryPath string `json:"source_sample_query_path,omitempty"` Diagnosis string `json:"diagnosis"` Reason string `json:"reason"` Severity string `json:"severity,omitempty"` RecommendedOperatorAction string `json:"recommended_operator_action,omitempty"` } type DailyMileageSourceQualityDiagnostics struct { URL string `json:"url,omitempty"` SummaryTotal int64 `json:"summary_total"` SourceCount int64 `json:"source_count"` VehicleCount int64 `json:"vehicle_count"` SampleCount int64 `json:"sample_count"` OKSourceCount int64 `json:"ok_source_count"` InvalidDeltaSourceCount int64 `json:"invalid_delta_source_count"` NoBaselineSourceCount int64 `json:"no_baseline_source_count"` FallbackAfterInvalidBaselineSourceCount int64 `json:"fallback_after_invalid_baseline_source_count"` RealtimeLocationFallbackSourceCount int64 `json:"realtime_location_fallback_source_count"` Items []DailyMileageSourceQualityItem `json:"items,omitempty"` ItemsTruncated bool `json:"items_truncated,omitempty"` } type DailyMileageSourceQualityItem struct { StatDate string `json:"stat_date"` Protocol string `json:"protocol"` QualityStatus string `json:"quality_status"` QualityReason string `json:"quality_reason,omitempty"` SourceCount int64 `json:"source_count"` VehicleCount int64 `json:"vehicle_count"` SelectedCount int64 `json:"selected_count"` SampleCount int64 `json:"sample_count"` DailyMileageKM float64 `json:"daily_mileage_km"` } type DailyMileageSourceSelectionDiagnostics struct { URL string `json:"url,omitempty"` SummaryTotal int64 `json:"summary_total"` SourceCount int64 `json:"source_count"` VehicleCount int64 `json:"vehicle_count"` SampleCount int64 `json:"sample_count"` SelectedSourceCount int64 `json:"selected_source_count"` NotSelectedSourceCount int64 `json:"not_selected_source_count"` NotSelectedMileageKM float64 `json:"not_selected_mileage_km"` ExcludedSourceCount int64 `json:"excluded_source_count"` UnmanagedSourceCount int64 `json:"unmanaged_source_count"` UnknownKindSourceCount int64 `json:"unknown_kind_source_count"` Protocols []DailyMileageSourceSelectionProtocol `json:"protocols,omitempty"` Items []DailyMileageSourceSelectionItem `json:"items,omitempty"` Samples []DailyMileageSourceSelectionSample `json:"samples,omitempty"` ItemsTruncated bool `json:"items_truncated,omitempty"` } type DailyMileageSourceSelectionProtocol struct { Protocol string `json:"protocol"` SourceCount int64 `json:"source_count"` VehicleCount int64 `json:"vehicle_count"` SampleCount int64 `json:"sample_count"` SelectedSourceCount int64 `json:"selected_source_count"` NotSelectedSourceCount int64 `json:"not_selected_source_count"` NotSelectedMileageKM float64 `json:"not_selected_mileage_km"` ExcludedSourceCount int64 `json:"excluded_source_count"` UnmanagedSourceCount int64 `json:"unmanaged_source_count"` UnknownKindSourceCount int64 `json:"unknown_kind_source_count"` } type DailyMileageSourceSelectionItem struct { StatDate string `json:"stat_date"` Protocol string `json:"protocol"` SelectionStatus string `json:"selection_status"` SelectionReason string `json:"selection_reason,omitempty"` SelectionAction string `json:"selection_action,omitempty"` SourceCount int64 `json:"source_count"` VehicleCount int64 `json:"vehicle_count"` SelectedCount int64 `json:"selected_count"` SampleCount int64 `json:"sample_count"` DailyMileageKM float64 `json:"daily_mileage_km"` } type DailyMileageSourceSelectionSample struct { VIN string `json:"vin"` StatDate string `json:"stat_date"` Protocol string `json:"protocol"` SourceKey string `json:"source_key"` SourceIP string `json:"source_ip,omitempty"` SourceEndpoint string `json:"source_endpoint,omitempty"` Phone string `json:"phone,omitempty"` DeviceID string `json:"device_id,omitempty"` PlatformName string `json:"platform_name,omitempty"` SourceID *int64 `json:"source_id,omitempty"` SourceCode string `json:"source_code,omitempty"` SourceKind string `json:"source_kind,omitempty"` SourceEnabled *bool `json:"source_enabled,omitempty"` TrustPriority *int `json:"trust_priority,omitempty"` LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"` DailyMileageKM float64 `json:"daily_mileage_km"` SampleCount int64 `json:"sample_count"` QualityStatus string `json:"quality_status,omitempty"` QualityReason string `json:"quality_reason,omitempty"` SelectionStatus string `json:"selection_status,omitempty"` SelectionReason string `json:"selection_reason,omitempty"` SelectionAction string `json:"selection_action,omitempty"` SelectedSource *DailyMileageSourceSelectionSample `json:"selected_source,omitempty"` SelectedSourceQueryPath string `json:"selected_source_query_path,omitempty"` UpdatedAt string `json:"updated_at,omitempty"` } type DataSourceDiagnostics struct { URL string `json:"url,omitempty"` MissingSourceCodeTotal int64 `json:"missing_source_code_total"` RecentMissingSourceCodeTotal int64 `json:"recent_missing_source_code_total"` UnknownSourceKindTotal int64 `json:"unknown_source_kind_total,omitempty"` RecentUnknownSourceKindTotal int64 `json:"recent_unknown_source_kind_total,omitempty"` PlatformKindCandidateTotal int64 `json:"platform_kind_candidate_total,omitempty"` RecentPlatformKindCandidateTotal int64 `json:"recent_platform_kind_candidate_total,omitempty"` RecentAgeThresholdSeconds int64 `json:"recent_age_threshold_seconds,omitempty"` ActionableCandidateTotal int64 `json:"actionable_candidate_total"` MappingIssueTotal int64 `json:"mapping_issue_total,omitempty"` RecentMappingIssueTotal int64 `json:"recent_mapping_issue_total,omitempty"` ReasonCounts map[string]int64 `json:"reason_counts,omitempty"` MappingIssueReasonCounts map[string]int64 `json:"mapping_issue_reason_counts,omitempty"` ReasonCountsTruncated bool `json:"reason_counts_truncated,omitempty"` MappingIssueReasonCountsTruncated bool `json:"mapping_issue_reason_counts_truncated,omitempty"` Samples []DataSourceDiagnosticSample `json:"samples,omitempty"` MappingIssueSamples []DataSourceDiagnosticSample `json:"mapping_issue_samples,omitempty"` } type DataSourceDiagnosticSample struct { ID int64 `json:"id"` Protocol string `json:"protocol"` SourceIP string `json:"source_ip"` LatestSourceEndpoint string `json:"latest_source_endpoint,omitempty"` PlatformName string `json:"platform_name,omitempty"` SourceCode string `json:"source_code,omitempty"` SourceKind string `json:"source_kind"` FirstSeenAt string `json:"first_seen_at,omitempty"` LatestSeenAt string `json:"latest_seen_at,omitempty"` ActiveSpanSeconds int64 `json:"active_span_seconds"` LatestSeenAgeSeconds int64 `json:"latest_seen_age_seconds"` RegistrationRows int64 `json:"registration_rows"` PhoneCount int64 `json:"phone_count"` UnknownVINRows int64 `json:"unknown_vin_rows"` IdentifierMatchedPhones int64 `json:"identifier_matched_phones"` UnmappedPhoneCount int64 `json:"unmapped_phone_count"` IdentifierMatchRatio float64 `json:"identifier_match_ratio"` ConfiguredSourceCodeMatchedPhones int64 `json:"configured_source_code_matched_phones"` ConfiguredSourceCodePlatformName string `json:"configured_source_code_platform_name,omitempty"` ConfiguredSourceCodeConflict bool `json:"configured_source_code_conflict"` SourcePlatformNameMismatch bool `json:"source_platform_name_mismatch"` MatchedSourceCodeCount int64 `json:"matched_source_code_count"` CandidateSourceCode string `json:"candidate_source_code,omitempty"` CandidatePlatformName string `json:"candidate_platform_name,omitempty"` MatchedSourceCodes []string `json:"matched_source_codes,omitempty"` MatchedPlatformNames []string `json:"matched_platform_names,omitempty"` SamplePhones []string `json:"sample_phones,omitempty"` Reason string `json:"reason"` RecommendedOperatorAction string `json:"recommended_operator_action"` SuggestedSourceKind string `json:"suggested_source_kind"` SuggestionConfidence string `json:"suggestion_confidence"` SuggestionReason string `json:"suggestion_reason"` } type JT808IdentityGapDiagnostics struct { URL string `json:"url,omitempty"` Total int64 `json:"total"` RecentSeconds int64 `json:"recent_seconds"` Samples []JT808IdentityGapSample `json:"samples,omitempty"` } type JT808IdentityGapSample struct { Phone string `json:"phone"` DeviceID string `json:"device_id,omitempty"` Plate string `json:"plate,omitempty"` VIN string `json:"vin,omitempty"` SourceIP string `json:"source_ip,omitempty"` SourceEndpoint string `json:"source_endpoint,omitempty"` SourceCode string `json:"source_code,omitempty"` PlatformName string `json:"platform_name,omitempty"` SourceKind string `json:"source_kind,omitempty"` FirstRegisteredAt string `json:"first_registered_at,omitempty"` LatestRegisteredAt string `json:"latest_registered_at,omitempty"` LatestAuthenticatedAt string `json:"latest_authenticated_at,omitempty"` LatestSeenAt string `json:"latest_seen_at,omitempty"` LatestSeenAgeSeconds int64 `json:"latest_seen_age_seconds"` Reason string `json:"reason"` RecommendedOperatorAction string `json:"recommended_operator_action"` RawFrameQueryPath string `json:"raw_frame_query_path,omitempty"` DataSourceQueryPath string `json:"data_source_query_path,omitempty"` } const ( DailyMileageSeverityOK = "ok" DailyMileageSeverityPipeline = "pipeline" DailyMileageSeveritySourceData = "source_data" ) func AnnotateDailyMileageDiagnostics(items []DailyMileageDiagnosisReasonItem) (annotated []DailyMileageDiagnosisReasonItem, pipelineIssues int64, sourceDataIssues int64) { annotated = make([]DailyMileageDiagnosisReasonItem, 0, len(items)) for _, item := range items { severity := strings.ToLower(strings.TrimSpace(item.Severity)) if severity != DailyMileageSeverityOK && severity != DailyMileageSeverityPipeline && severity != DailyMileageSeveritySourceData { severity = DailyMileageIssueSeverity(item) } item.Severity = severity if strings.TrimSpace(item.RecommendedOperatorAction) == "" { item.RecommendedOperatorAction = DailyMileageRecommendedOperatorAction(item.Diagnosis, item.Reason) } switch item.Severity { case DailyMileageSeverityPipeline: pipelineIssues += item.Count case DailyMileageSeveritySourceData: sourceDataIssues += item.Count } annotated = append(annotated, item) } return annotated, pipelineIssues, sourceDataIssues } func DailyMileageIssueSeverity(item DailyMileageDiagnosisReasonItem) string { diagnosis := strings.ToUpper(strings.TrimSpace(item.Diagnosis)) reason := strings.ToLower(strings.TrimSpace(item.Reason)) switch diagnosis { case "", "OK": return DailyMileageSeverityOK case "MISSING_DAILY": if reason == "source_samples_all_invalid" { return DailyMileageSeveritySourceData } return DailyMileageSeverityPipeline case "NO_SOURCE_SAMPLE": return DailyMileageSeverityPipeline case "NO_TOTAL_MILEAGE": switch reason { case "realtime_total_mileage_time_missing": return DailyMileageSeverityPipeline default: return DailyMileageSeveritySourceData } default: return DailyMileageSeverityPipeline } } func DailyMileageRecommendedOperatorAction(diagnosis, reason string) string { diagnosis = strings.ToUpper(strings.TrimSpace(diagnosis)) reason = strings.ToLower(strings.TrimSpace(reason)) switch diagnosis { case "", "OK": return "无需处理,车辆当日里程已生成" case "MISSING_DAILY": return "统计源样本已存在但最终日表缺失,优先排查 stat-writer 投影任务、MySQL 写入和 source 选举" case "NO_SOURCE_SAMPLE": return "实时位置已有有效总里程但统计字段样本缺失,优先排查字段流 topic、stat-writer 消费和协议字段映射" case "NO_TOTAL_MILEAGE": switch reason { case "realtime_total_mileage_time_missing": return "总里程值存在但事件时间缺失,优先排查解析器和 history/realtime 写入的时间字段" case "realtime_total_mileage_missing": return "当日实时数据缺少总里程字段,核对平台是否上报该字段以及协议字段映射是否覆盖" case "realtime_total_mileage_non_positive": return "终端上报总里程小于等于 0,不参与差值统计,需核对设备或平台里程字段来源" case "realtime_total_mileage_not_reported_on_stat_date": return "最新总里程不是统计日期内上报,等待当日含总里程数据或检查平台是否只上报部分字段" default: return "车辆当日在线但没有可用于里程统计的总里程,优先核对源平台报文字段和解析映射" } default: return "诊断状态未知,检查统计诊断 SQL 和上游字段流" } } type Thresholds struct { GatewayFrameP99MS float64 GatewayIdentityP99MS float64 GatewayResponseRecentP99MS float64 AsyncSinkP99MS float64 AsyncSinkQueueWaitRecentP99MS float64 BridgeBatchP99MS float64 BridgeKafkaWriteP99MS float64 BridgeKafkaE2EP99MS float64 BridgeKafkaRecentE2EP99MS float64 BridgeFetchWaitMaxMS float64 BridgeWorkersMin float64 BridgeRouteErrorMax float64 BridgeKafkaWriteErrorMax float64 BridgeMessageMinReceived float64 BridgeMessageMaxUnackedRatio float64 FieldsProjectorWorkersPerProtocolMin float64 FieldsProjectorInvalidJSONMax float64 FieldsProjectorWriteErrorMax float64 FieldsProjectorBatchPendingMax float64 FastWriterStageP99MS float64 FastWriterRedisE2EP99MS float64 FastWriterRedisRecentE2EP99MS float64 FastWriterFetchWaitMaxMS float64 FastWriterWorkersMin float64 FastWriterInvalidJSONMax float64 FastWriterFallbackErrorMax float64 FastWriterDecoupledUpdateErrorMax float64 FastWriterTDengineEnabledMax float64 FastWriterMessageMinReceived float64 FastWriterMessageMaxUnackedRatio float64 HistoryFlushP99MS float64 HistoryWriteE2EP99MS float64 HistoryWriteRecentE2EP99MS float64 HistoryWorkersMin float64 HistoryInvalidJSONMax float64 HistoryLocationErrorMax float64 HistoryLocationErrorRecentSec float64 HistoryLocationAccountingMaxRatio float64 HistoryFallbackErrorMax float64 HistoryRetryMax float64 HistoryRetryExhaustedMax float64 RealtimeBatchP99MS float64 RealtimeStoreP99MS float64 RealtimeWorkersMin float64 RealtimeStoreErrorMax float64 RealtimeInvalidJSONMax float64 StatWriteP99MS float64 StatWriteE2EP99MS float64 StatWriteRecentE2EP99MS float64 StatWorkersMin float64 IdentityWorkersMin float64 StatInvalidJSONMax float64 StatProtocolTopicMismatchMax float64 MessageContractMismatchMax float64 StatRetryMax float64 StatRetryExhaustedMax float64 ProcessUptimeMinSec float64 ProcessHeapAllocMaxBytes float64 ProcessHeapSysMaxBytes float64 ProcessGoroutinesMax float64 KafkaLagMax float64 KafkaConsumerCommitErrorMax float64 KafkaConsumerMessageMinReceived float64 KafkaConsumerMaxUncommittedRatio float64 DurableSpoolBacklogMax float64 DurableSpoolOldestAgeMaxSec float64 DurableSpoolErrorMax float64 DurableSpoolReplayErrorMax float64 DurableOutboxInflightMax float64 DurableOutboxErrorMax float64 AsyncSinkEnqueueErrorMax float64 AsyncSinkPublishErrorMax float64 AsyncSinkQueueDepthMax float64 AsyncSinkQueueMaxUsageRatio float64 FastWriterBatchPendingMax float64 HistoryBatchPendingMax float64 HistoryRowsPendingMax float64 RealtimeBatchPendingMax float64 RealtimeAsyncQueueDepthMax float64 RealtimeAsyncQueueMaxUsageRatio float64 RealtimeAsyncQueueDroppedMax float64 HistoryParsedFieldMinFrames float64 HistoryParsedFieldMaxMissingRatio float64 MinHistogramSamples float64 StatSampleMinWrites float64 StatSampleMaxActionableSkipRatio float64 StatSampleFutureAdjustmentMax float64 StatSourceMinSamples float64 StatSourceMaxMissingRatio float64 StatProjectionMinSampleWrites float64 StatBatchPendingMax float64 ConsumerRetryPendingMax float64 StatCacheEntriesMax float64 RealtimeThrottleMinSamples float64 RealtimeThrottleCacheEntriesMax float64 RealtimePlateCacheEntriesMax float64 FastWriterRedisEnvelopeMinSeen float64 FastWriterRedisEnvelopeMaxBadRatio float64 FastWriterRedisFieldMinSeen float64 FastWriterRedisFieldMaxStaleRatio float64 GatewayFieldMinRaw float64 GatewayFieldMaxMissingRatio float64 GatewayFieldMinCount float64 GatewayBridgeMinPublish float64 GatewayFastWriterMinRawPublish float64 GatewayParseMinFrames float64 GatewayParseMaxBadRatio float64 GatewayResponseMinSamples float64 GatewayResponseMaxErrorRatio float64 GatewayIdentityMinSamples float64 GatewayIdentityMaxUnresolvedRatio float64 GatewayIdentityCacheEntriesMax float64 GatewayIdentitySnapshotRequired float64 GatewayIdentitySnapshotMaxAgeSec float64 GatewayIdentityLocationTouchFailureMax float64 GatewayIdentityRegistrationWriteQueueDepthMax float64 GatewayIdentityRegistrationWriteQueueMaxUsageRatio float64 GatewayIdentityRegistrationWriteErrorMax float64 GatewayIdentityStaleMinSamples float64 GatewayIdentityMaxStaleCacheRatio float64 StatTopicMinBridge float64 RawFanoutMinBridge float64 LastActivityStaleSec float64 RequiredConsumerTopics map[string][]string } func DefaultThresholds() Thresholds { return Thresholds{ GatewayFrameP99MS: 250, GatewayIdentityP99MS: 100, GatewayResponseRecentP99MS: 100, AsyncSinkP99MS: 100, AsyncSinkQueueWaitRecentP99MS: 100, BridgeBatchP99MS: 5000, BridgeKafkaWriteP99MS: 500, BridgeKafkaE2EP99MS: 0, BridgeKafkaRecentE2EP99MS: 300, BridgeFetchWaitMaxMS: 50, BridgeWorkersMin: 2, BridgeRouteErrorMax: 0, BridgeKafkaWriteErrorMax: 0, BridgeMessageMinReceived: 5000, BridgeMessageMaxUnackedRatio: 0.20, FieldsProjectorWorkersPerProtocolMin: 1, FieldsProjectorInvalidJSONMax: 0, FieldsProjectorWriteErrorMax: 0, FieldsProjectorBatchPendingMax: 1000, FastWriterStageP99MS: 100, FastWriterRedisE2EP99MS: 0, FastWriterRedisRecentE2EP99MS: 100, FastWriterFetchWaitMaxMS: 50, FastWriterWorkersMin: 4, FastWriterInvalidJSONMax: 0, FastWriterFallbackErrorMax: 0, FastWriterDecoupledUpdateErrorMax: 0, FastWriterTDengineEnabledMax: 0, FastWriterMessageMinReceived: 5000, FastWriterMessageMaxUnackedRatio: 0.20, HistoryFlushP99MS: 1000, HistoryWriteE2EP99MS: 0, HistoryWriteRecentE2EP99MS: 2000, HistoryWorkersMin: 3, HistoryInvalidJSONMax: 0, HistoryLocationErrorMax: 0, HistoryLocationErrorRecentSec: 300, HistoryLocationAccountingMaxRatio: 0.02, HistoryFallbackErrorMax: 0, HistoryRetryMax: 100, HistoryRetryExhaustedMax: 0, RealtimeBatchP99MS: 500, RealtimeStoreP99MS: 100, RealtimeWorkersMin: 3, RealtimeStoreErrorMax: 0, RealtimeInvalidJSONMax: 0, StatWriteP99MS: 100, StatWriteE2EP99MS: 0, StatWriteRecentE2EP99MS: 1000, StatWorkersMin: 3, IdentityWorkersMin: 3, StatInvalidJSONMax: 0, StatProtocolTopicMismatchMax: 0, MessageContractMismatchMax: 0, StatRetryMax: 100, StatRetryExhaustedMax: 0, ProcessUptimeMinSec: 0, ProcessHeapAllocMaxBytes: 0, ProcessHeapSysMaxBytes: 0, ProcessGoroutinesMax: 0, KafkaLagMax: 1000, KafkaConsumerCommitErrorMax: 0, KafkaConsumerMessageMinReceived: 100, KafkaConsumerMaxUncommittedRatio: 0.20, DurableSpoolBacklogMax: 10000, DurableSpoolOldestAgeMaxSec: 300, DurableSpoolErrorMax: 0, DurableSpoolReplayErrorMax: 0, DurableOutboxInflightMax: 10000, DurableOutboxErrorMax: 0, AsyncSinkEnqueueErrorMax: 0, AsyncSinkPublishErrorMax: 0, AsyncSinkQueueDepthMax: 10000, AsyncSinkQueueMaxUsageRatio: 0.80, FastWriterBatchPendingMax: 1000, HistoryBatchPendingMax: 1000, HistoryRowsPendingMax: 5000, RealtimeBatchPendingMax: 1000, RealtimeAsyncQueueDepthMax: 10000, RealtimeAsyncQueueMaxUsageRatio: 0.80, RealtimeAsyncQueueDroppedMax: 0, HistoryParsedFieldMinFrames: 100, HistoryParsedFieldMaxMissingRatio: 0.05, MinHistogramSamples: 100, StatSampleMinWrites: 100, StatSampleMaxActionableSkipRatio: 0.60, StatSampleFutureAdjustmentMax: 0, StatSourceMinSamples: 100, StatSourceMaxMissingRatio: 0.60, StatProjectionMinSampleWrites: 100, StatBatchPendingMax: 1000, ConsumerRetryPendingMax: 0, StatCacheEntriesMax: 1000000, RealtimeThrottleMinSamples: 100, RealtimeThrottleCacheEntriesMax: 1000000, RealtimePlateCacheEntriesMax: 200000, FastWriterRedisEnvelopeMinSeen: 100, FastWriterRedisEnvelopeMaxBadRatio: 0.50, FastWriterRedisFieldMinSeen: 1000, FastWriterRedisFieldMaxStaleRatio: 0.20, GatewayFieldMinRaw: 100, GatewayFieldMaxMissingRatio: 0.20, GatewayFieldMinCount: 1, GatewayBridgeMinPublish: 1000, GatewayFastWriterMinRawPublish: 1000, GatewayParseMinFrames: 100, GatewayParseMaxBadRatio: 0.05, GatewayResponseMinSamples: 100, GatewayResponseMaxErrorRatio: 0.05, GatewayIdentityMinSamples: 100, GatewayIdentityMaxUnresolvedRatio: 0.20, GatewayIdentityCacheEntriesMax: 300000, GatewayIdentitySnapshotRequired: 1, GatewayIdentitySnapshotMaxAgeSec: 180, GatewayIdentityLocationTouchFailureMax: 0, GatewayIdentityRegistrationWriteQueueDepthMax: 10000, GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0.80, GatewayIdentityRegistrationWriteErrorMax: 0, GatewayIdentityStaleMinSamples: 100, GatewayIdentityMaxStaleCacheRatio: 0.05, StatTopicMinBridge: 1000, RawFanoutMinBridge: 1000, LastActivityStaleSec: 300, } } func DefaultRequiredConsumerTopics() map[string][]string { return map[string][]string{ "fields-projector": { topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT, }, "history": { topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT, }, "realtime": { topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT, }, "stat": { topics.FieldsGB32960, topics.FieldsJT808, topics.FieldsYutongMQTT, }, "identity": { topics.RawJT808, }, } } func Evaluate(metricsByService map[string]string) Report { return EvaluateWithThresholds(metricsByService, DefaultThresholds()) } func EvaluateWithThresholds(metricsByService map[string]string, thresholds Thresholds) Report { report := Report{ Status: StatusOK, CheckedAt: time.Now().UTC(), Services: make([]ServiceCheck, 0, len(metricsByService)), } parsedByService := make(map[string]parsedMetricSet, len(metricsByService)) var names []string for name := range metricsByService { names = append(names, name) } sort.Strings(names) for _, name := range names { parsed := parseMetricSet(metricsByService[name]) parsedByService[name] = parsed metrics := parsed.totals addLastActivityAges(metrics, report.CheckedAt) report.Services = append(report.Services, ServiceCheck{Name: name, Metrics: metrics}) report.Totals.ActiveConnections += int64(metrics["vehicle_gateway_active_connections"]) report.Totals.KafkaLag += metrics["vehicle_history_kafka_lag"] report.Totals.KafkaLag += metrics["vehicle_stat_kafka_lag"] report.Totals.KafkaLag += metrics["vehicle_realtime_kafka_lag"] report.Totals.KafkaLag += metrics["vehicle_identity_writer_kafka_lag"] report.Totals.BridgeConsumerPending += metrics["vehicle_bridge_nats_consumer_pending"] report.Totals.BridgeAckPending += metrics["vehicle_bridge_nats_consumer_ack_pending"] report.Totals.BridgeBatchPendingMessages += metrics["vehicle_bridge_batch_pending_messages"] report.Totals.FieldsProjectorKafkaLag += metrics["vehicle_fields_projector_kafka_lag"] report.Totals.FieldsProjectorBatchPending += metrics["vehicle_fields_projector_batch_pending_messages"] report.Totals.FieldsProjectorRetryPending += metrics["vehicle_fields_projector_retry_pending_messages"] report.Totals.FastWriterConsumerPending += metrics["vehicle_fast_writer_nats_consumer_pending"] report.Totals.FastWriterAckPending += metrics["vehicle_fast_writer_nats_consumer_ack_pending"] report.Totals.FastWriterBatchPending += metrics["vehicle_fast_writer_batch_pending_messages"] report.Totals.HistoryBatchPending += metrics["vehicle_history_batch_pending_messages"] report.Totals.HistoryRowsPending += metrics["vehicle_history_batch_pending_rows"] report.Totals.HistoryRetryPending += metrics["vehicle_history_retry_pending_messages"] report.Totals.RealtimeBatchPending += metrics["vehicle_realtime_batch_pending_messages"] report.Totals.RealtimeAsyncQueuePending += metrics["vehicle_realtime_async_queue_depth"] report.Totals.RealtimeRetryPending += metrics["vehicle_realtime_retry_pending_messages"] report.Totals.StatRetryPending += metrics["vehicle_stat_retry_pending_messages"] report.Totals.IdentityBatchPending += metrics["vehicle_identity_writer_batch_pending_messages"] report.Totals.IdentityFactsPending += metrics["vehicle_identity_writer_batch_pending_facts"] report.Totals.IdentityRetryPending += metrics["vehicle_identity_writer_retry_pending_messages"] report.Totals.JT808RegistrationWriteQueuePending += gatewayIdentityCacheMetric(parsed, "registration_write_queue", "vehicle_gateway_identity_cache_entries") report.Totals.JT808RegistrationWriteErrors += jt808RegistrationWriteErrors(parsed) report.Findings = append(report.Findings, findingsForService(name, metrics)...) report.Findings = append(report.Findings, findingsForMetrics(name, metrics, thresholds)...) report.Findings = append(report.Findings, findingsForRuntimeConfig(name, parsed, thresholds)...) report.Findings = append(report.Findings, findingsForAsyncSinkQueueUsage(name, parsed, thresholds)...) report.Findings = append(report.Findings, findingsForAsyncSinkErrors(name, parsed, thresholds)...) report.Findings = append(report.Findings, findingsForDurableSpool(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForDurableOutbox(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForHistograms(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForKafkaConsumerCompletion(name, parsed, thresholds)...) report.Findings = append(report.Findings, findingsForMessageContractMismatches(name, parsed, thresholds)...) report.Findings = append(report.Findings, findingsForBridgeRouteErrors(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForBridgeKafkaWriteErrors(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForBridgeMessageCompletion(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForFieldsProjector(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForHistoryInvalidJSON(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForHistoryLocationErrors(parsed, thresholds, report.CheckedAt)...) report.Findings = append(report.Findings, findingsForHistoryLocationAccounting(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForHistoryFallbackErrors(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForWriteRetries(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForHistoryParsedFields(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForStatInvalidJSON(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForStatProtocolTopicMismatches(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForStatSamples(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForStatSources(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForStatProjections(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForStatCache(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForFastWriterRedisEnvelopes(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForFastWriterRedisFields(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForFastWriterInvalidJSON(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForFastWriterFallbackErrors(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForFastWriterDecoupledUpdateErrors(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForFastWriterMessageCompletion(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForRealtimeInvalidJSON(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForRealtimeStoreErrors(parsed, thresholds)...) report.Findings = append(report.Findings, findingsForRealtimeAsyncQueue(parsed, thresholds)...) } report.Findings = append(report.Findings, findingsForGatewayIdentityQuality(parsedByService["gateway"], thresholds)...) report.Findings = append(report.Findings, findingsForGatewayIdentityCache(parsedByService["gateway"], thresholds)...) report.Findings = append(report.Findings, findingsForGatewayIdentitySnapshot(parsedByService["gateway"], thresholds, report.CheckedAt)...) report.Findings = append(report.Findings, findingsForGatewayIdentityLocationTouchFailures(parsedByService["gateway"], thresholds)...) report.Findings = append(report.Findings, findingsForGatewayRegistrationWriteQueue(parsedByService["gateway"], thresholds)...) report.Findings = append(report.Findings, findingsForGatewayRegistrationWriteErrors(parsedByService["gateway"], thresholds)...) report.Findings = append(report.Findings, findingsForGatewayIdentityStaleCache(parsedByService["gateway"], thresholds)...) report.Findings = append(report.Findings, findingsForIdentityWriterOwnership(parsedByService["gateway"], parsedByService["identity"])...) report.GatewayFields = buildGatewayFieldsDiagnostics(parsedByService["gateway"], thresholds) report.IdentityResolution = buildIdentityResolutionDiagnostics(parsedByService["gateway"]) report.ProtocolFlow = buildProtocolFlowDiagnostics(parsedByService, thresholds) report.PipelineLatency = buildPipelineLatencyDiagnostics(parsedByService, thresholds) report.Findings = append(report.Findings, findingsForGatewayParseQuality(parsedByService["gateway"], thresholds)...) report.Findings = append(report.Findings, findingsForGatewayResponseQuality(parsedByService["gateway"], thresholds)...) report.Findings = append(report.Findings, findingsForRealtimeThrottle(parsedByService["realtime"], thresholds)...) report.Findings = append(report.Findings, findingsForRealtimeThrottleCache(parsedByService["realtime"], thresholds)...) report.Findings = append(report.Findings, findingsForRealtimePlateCache(parsedByService["realtime"], thresholds)...) report.Findings = append(report.Findings, findingsForRequiredConsumerTopics(parsedByService, thresholds)...) report.Findings = append(report.Findings, findingsForTopicFlow(parsedByService, thresholds)...) report.Findings = append(report.Findings, findingsForLastActivity(parsedByService, thresholds, report.CheckedAt)...) if len(report.Findings) > 0 { report.Status = StatusDegraded } return report } func findingsForService(name string, metrics map[string]float64) []string { if metrics["vehicle_service_info"] <= 0 { return []string{fmt.Sprintf("%s service info metric missing", name)} } if requiresKafkaConsumerInfo(name) && metrics["vehicle_kafka_consumer_info"] <= 0 { return []string{fmt.Sprintf("%s kafka consumer info metric missing", name)} } return nil } func requiresKafkaConsumerInfo(name string) bool { switch name { case "history", "stat", "realtime", "fields-projector": return true default: return false } } func findingsForMetrics(service string, metrics map[string]float64, thresholds Thresholds) []string { var findings []string if thresholds.ProcessUptimeMinSec > 0 { value, ok := metrics["vehicle_process_uptime_seconds"] if !ok { findings = append(findings, fmt.Sprintf("%s process uptime metric missing", service)) } else if value < thresholds.ProcessUptimeMinSec { findings = append(findings, fmt.Sprintf("%s process uptime %.0fs below %.0fs", service, value, thresholds.ProcessUptimeMinSec)) } } if thresholds.ProcessHeapAllocMaxBytes > 0 { value, ok := metrics["vehicle_process_heap_alloc_bytes"] if !ok { findings = append(findings, fmt.Sprintf("%s process heap alloc metric missing", service)) } else if value > thresholds.ProcessHeapAllocMaxBytes { findings = append(findings, fmt.Sprintf("%s process heap alloc %.0f bytes exceeds %.0f", service, value, thresholds.ProcessHeapAllocMaxBytes)) } } if thresholds.ProcessHeapSysMaxBytes > 0 { value, ok := metrics["vehicle_process_heap_sys_bytes"] if !ok { findings = append(findings, fmt.Sprintf("%s process heap sys metric missing", service)) } else if value > thresholds.ProcessHeapSysMaxBytes { findings = append(findings, fmt.Sprintf("%s process heap sys %.0f bytes exceeds %.0f", service, value, thresholds.ProcessHeapSysMaxBytes)) } } if thresholds.ProcessGoroutinesMax > 0 { value, ok := metrics["vehicle_process_goroutines"] if !ok { findings = append(findings, fmt.Sprintf("%s process goroutines metric missing", service)) } else if value > thresholds.ProcessGoroutinesMax { findings = append(findings, fmt.Sprintf("%s process goroutines %.0f exceeds %.0f", service, value, thresholds.ProcessGoroutinesMax)) } } if value := metrics["vehicle_gateway_connection_rejections_total"]; value > 0 { findings = append(findings, fmt.Sprintf("gateway connection rejections %.0f", value)) } if thresholds.AsyncSinkQueueDepthMax > 0 { if value := metrics["vehicle_async_sink_queue_depth"]; value > thresholds.AsyncSinkQueueDepthMax { findings = append(findings, fmt.Sprintf("async sink queue depth %.0f exceeds %.0f", value, thresholds.AsyncSinkQueueDepthMax)) } } if value := metrics["vehicle_bridge_nats_consumer_ack_pending"]; value > 100 { findings = append(findings, fmt.Sprintf("bridge ack pending %.0f exceeds 100", value)) } if value := metrics["vehicle_bridge_nats_consumer_pending"]; value > 10_000 { findings = append(findings, fmt.Sprintf("bridge consumer pending %.0f exceeds 10000", value)) } if value := metrics["vehicle_bridge_batch_pending_messages"]; value > 1000 { findings = append(findings, fmt.Sprintf("bridge batch pending %.0f exceeds 1000", value)) } if value := metrics["vehicle_fast_writer_nats_consumer_ack_pending"]; value > 100 { findings = append(findings, fmt.Sprintf("fast writer ack pending %.0f exceeds 100", value)) } if value := metrics["vehicle_fast_writer_nats_consumer_pending"]; value > 10_000 { findings = append(findings, fmt.Sprintf("fast writer consumer pending %.0f exceeds 10000", value)) } if value := metrics["vehicle_fast_writer_batch_pending_messages"]; thresholds.FastWriterBatchPendingMax > 0 && value > thresholds.FastWriterBatchPendingMax { findings = append(findings, fmt.Sprintf("fast writer batch pending %.0f exceeds %.0f", value, thresholds.FastWriterBatchPendingMax)) } if value := metrics["vehicle_fast_writer_tdengine_enabled"]; thresholds.FastWriterTDengineEnabledMax >= 0 && value > thresholds.FastWriterTDengineEnabledMax { findings = append(findings, fmt.Sprintf("fast writer tdengine enabled %.0f exceeds %.0f", value, thresholds.FastWriterTDengineEnabledMax)) } if value := metrics["vehicle_realtime_redis_projector_enabled"]; value > 0 { findings = append(findings, fmt.Sprintf("realtime redis projector enabled %.0f", value)) } if value := metrics["vehicle_history_batch_pending_messages"]; thresholds.HistoryBatchPendingMax > 0 && value > thresholds.HistoryBatchPendingMax { findings = append(findings, fmt.Sprintf("history batch pending %.0f exceeds %.0f", value, thresholds.HistoryBatchPendingMax)) } if value := metrics["vehicle_history_batch_pending_rows"]; thresholds.HistoryRowsPendingMax > 0 && value > thresholds.HistoryRowsPendingMax { findings = append(findings, fmt.Sprintf("history rows pending %.0f exceeds %.0f", value, thresholds.HistoryRowsPendingMax)) } if value := metrics["vehicle_stat_batch_pending_messages"]; thresholds.StatBatchPendingMax > 0 && value > thresholds.StatBatchPendingMax { findings = append(findings, fmt.Sprintf("stat batch pending %.0f exceeds %.0f", value, thresholds.StatBatchPendingMax)) } if value := metrics["vehicle_realtime_batch_pending_messages"]; thresholds.RealtimeBatchPendingMax > 0 && value > thresholds.RealtimeBatchPendingMax { findings = append(findings, fmt.Sprintf("realtime batch pending %.0f exceeds %.0f", value, thresholds.RealtimeBatchPendingMax)) } if value := metrics["vehicle_realtime_async_queue_depth"]; thresholds.RealtimeAsyncQueueDepthMax > 0 && value > thresholds.RealtimeAsyncQueueDepthMax { findings = append(findings, fmt.Sprintf("realtime async queue depth %.0f exceeds %.0f", value, thresholds.RealtimeAsyncQueueDepthMax)) } for _, retry := range []struct { name string metric string }{ {name: "history", metric: "vehicle_history_retry_pending_messages"}, {name: "realtime", metric: "vehicle_realtime_retry_pending_messages"}, {name: "stat", metric: "vehicle_stat_retry_pending_messages"}, {name: "identity", metric: "vehicle_identity_writer_retry_pending_messages"}, {name: "fields projector", metric: "vehicle_fields_projector_retry_pending_messages"}, } { if value := metrics[retry.metric]; thresholds.ConsumerRetryPendingMax >= 0 && value > thresholds.ConsumerRetryPendingMax { findings = append(findings, fmt.Sprintf("%s retry pending %.0f exceeds %.0f", retry.name, value, thresholds.ConsumerRetryPendingMax)) } } if value := metrics["vehicle_history_kafka_lag"] + metrics["vehicle_stat_kafka_lag"] + metrics["vehicle_realtime_kafka_lag"] + metrics["vehicle_identity_writer_kafka_lag"] + metrics["vehicle_fields_projector_kafka_lag"]; thresholds.KafkaLagMax > 0 && value > thresholds.KafkaLagMax { findings = append(findings, fmt.Sprintf("kafka lag %.0f exceeds %.0f", value, thresholds.KafkaLagMax)) } if value := metrics["vehicle_identity_writer_batch_pending_messages"]; thresholds.RealtimeBatchPendingMax > 0 && value > thresholds.RealtimeBatchPendingMax { findings = append(findings, fmt.Sprintf("identity batch pending %.0f exceeds %.0f", value, thresholds.RealtimeBatchPendingMax)) } if value := metrics["vehicle_fields_projector_batch_pending_messages"]; thresholds.FieldsProjectorBatchPendingMax > 0 && value > thresholds.FieldsProjectorBatchPendingMax { findings = append(findings, fmt.Sprintf("fields projector batch pending %.0f exceeds %.0f", value, thresholds.FieldsProjectorBatchPendingMax)) } return findings } func findingsForFieldsProjector(parsed parsedMetricSet, thresholds Thresholds) []string { var invalidJSON float64 var writeErrors float64 for _, sample := range parsed.samples { switch { case sample.name == "vehicle_fields_projector_kafka_messages_total" && sample.labels["status"] == "invalid_json": invalidJSON += sample.value case sample.name == "vehicle_fields_projector_kafka_writes_total" && sample.labels["status"] == "error": writeErrors += sample.value } } var findings []string if thresholds.FieldsProjectorInvalidJSONMax >= 0 && invalidJSON > thresholds.FieldsProjectorInvalidJSONMax { findings = append(findings, fmt.Sprintf("fields projector invalid json %.0f exceeds %.0f", invalidJSON, thresholds.FieldsProjectorInvalidJSONMax)) } if thresholds.FieldsProjectorWriteErrorMax >= 0 && writeErrors > thresholds.FieldsProjectorWriteErrorMax { findings = append(findings, fmt.Sprintf("fields projector kafka write errors %.0f exceeds %.0f", writeErrors, thresholds.FieldsProjectorWriteErrorMax)) } return findings } func findingsForRuntimeConfig(service string, parsed parsedMetricSet, thresholds Thresholds) []string { var findings []string if service == "bridge" || hasServiceInfo(parsed, "nats-kafka-bridge") { findings = append(findings, configMaxFinding(parsed, "vehicle_bridge_config", "fetch_wait_ms", "bridge fetch wait", "ms", thresholds.BridgeFetchWaitMaxMS)...) findings = append(findings, configMinFinding(parsed, "vehicle_bridge_config", "workers", "bridge workers", "", thresholds.BridgeWorkersMin)...) } if service == "fields-projector" || hasServiceInfo(parsed, "vehicle-fields-projector") { findings = append(findings, configMinFinding(parsed, "vehicle_fields_projector_config", "workers_per_protocol", "fields projector workers per protocol", "", thresholds.FieldsProjectorWorkersPerProtocolMin)...) } if service == "fast-writer" || hasServiceInfo(parsed, "nats-fast-writer") { findings = append(findings, configMaxFinding(parsed, "vehicle_fast_writer_config", "fetch_wait_ms", "fast writer fetch wait", "ms", thresholds.FastWriterFetchWaitMaxMS)...) findings = append(findings, configMinFinding(parsed, "vehicle_fast_writer_config", "workers", "fast writer workers", "", thresholds.FastWriterWorkersMin)...) } if service == "history" || hasServiceInfo(parsed, "vehicle-history-writer") { findings = append(findings, configMinFinding(parsed, "vehicle_history_config", "workers", "history workers", "", thresholds.HistoryWorkersMin)...) } if service == "realtime" || hasServiceInfo(parsed, "vehicle-realtime-writer") { findings = append(findings, configMinFinding(parsed, "vehicle_realtime_config", "workers", "realtime workers", "", thresholds.RealtimeWorkersMin)...) } if service == "stat" || hasServiceInfo(parsed, "vehicle-stat-writer") { findings = append(findings, configMinFinding(parsed, "vehicle_stat_config", "workers", "stat workers", "", thresholds.StatWorkersMin)...) } if service == "identity" || hasServiceInfo(parsed, "vehicle-identity-writer") { findings = append(findings, configMinFinding(parsed, "vehicle_identity_writer_config", "workers", "identity workers", "", thresholds.IdentityWorkersMin)...) } sort.Strings(findings) return findings } func configMaxFinding(parsed parsedMetricSet, metricName string, setting string, label string, unit string, threshold float64) []string { if threshold <= 0 { return nil } value, ok := maxConfigGaugeValue(parsed, metricName, setting) if !ok { return []string{fmt.Sprintf("%s config metric missing (%s setting=%s)", label, metricName, setting)} } if value <= threshold { return nil } return []string{fmt.Sprintf("%s %.0f%s exceeds %.0f%s", label, value, unit, threshold, unit)} } func configMinFinding(parsed parsedMetricSet, metricName string, setting string, label string, unit string, threshold float64) []string { if threshold <= 0 { return nil } value, ok := minConfigGaugeValue(parsed, metricName, setting) if !ok { return []string{fmt.Sprintf("%s config metric missing (%s setting=%s)", label, metricName, setting)} } if value >= threshold { return nil } return []string{fmt.Sprintf("%s %.0f%s below %.0f%s", label, value, unit, threshold, unit)} } func hasServiceInfo(parsed parsedMetricSet, service string) bool { service = strings.TrimSpace(service) if service == "" { return false } for _, sample := range parsed.samples { if sample.name == "vehicle_service_info" && sample.labels["service"] == service && sample.value > 0 { return true } } return false } func maxConfigGaugeValue(parsed parsedMetricSet, metricName string, setting string) (float64, bool) { var out float64 found := false for _, sample := range parsed.samples { if sample.name != metricName || sample.labels["setting"] != setting { continue } if !found || sample.value > out { out = sample.value found = true } } return out, found } func minConfigGaugeValue(parsed parsedMetricSet, metricName string, setting string) (float64, bool) { var out float64 found := false for _, sample := range parsed.samples { if sample.name != metricName || sample.labels["setting"] != setting { continue } if !found || sample.value < out { out = sample.value found = true } } return out, found } func findingsForAsyncSinkQueueUsage(service string, parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.AsyncSinkQueueMaxUsageRatio <= 0 { return nil } type queueSample struct { labels map[string]string depth float64 capacity float64 } queues := map[string]*queueSample{} for _, sample := range parsed.samples { switch sample.name { case "vehicle_async_sink_queue_depth": key := metricLabelsKey(sample.labels) item := queues[key] if item == nil { item = &queueSample{labels: sample.labels} queues[key] = item } item.depth += sample.value case "vehicle_async_sink_queue_capacity": key := metricLabelsKey(sample.labels) item := queues[key] if item == nil { item = &queueSample{labels: sample.labels} queues[key] = item } item.capacity += sample.value } } var keys []string for key := range queues { keys = append(keys, key) } sort.Strings(keys) var findings []string for _, key := range keys { item := queues[key] if item.capacity <= 0 || item.depth <= 0 { continue } ratio := item.depth / item.capacity if ratio <= thresholds.AsyncSinkQueueMaxUsageRatio { continue } findings = append(findings, fmt.Sprintf( "%s async sink queue usage %.2f exceeds %.2f (%s, depth %.0f, capacity %.0f)", service, ratio, thresholds.AsyncSinkQueueMaxUsageRatio, key, item.depth, item.capacity, )) } return findings } func findingsForAsyncSinkErrors(service string, parsed parsedMetricSet, thresholds Thresholds) []string { var findings []string if thresholds.AsyncSinkEnqueueErrorMax >= 0 { for _, sample := range parsed.samples { if sample.name != "vehicle_async_sink_enqueue_total" { continue } status := strings.TrimSpace(sample.labels["status"]) if status == "" || status == "queued" { continue } if sample.value <= thresholds.AsyncSinkEnqueueErrorMax { continue } sink := strings.TrimSpace(sample.labels["sink"]) if sink == "" { sink = "unknown" } kind := strings.TrimSpace(sample.labels["kind"]) if kind == "" { kind = "unknown" } findings = append(findings, fmt.Sprintf( "%s async sink enqueue %s %.0f exceeds %.0f for sink %s kind %s", service, status, sample.value, thresholds.AsyncSinkEnqueueErrorMax, sink, kind, )) } } if thresholds.AsyncSinkPublishErrorMax >= 0 { for _, sample := range parsed.samples { if sample.name != "vehicle_async_sink_publish_total" || sample.labels["status"] != "error" { continue } if sample.value <= thresholds.AsyncSinkPublishErrorMax { continue } sink := strings.TrimSpace(sample.labels["sink"]) if sink == "" { sink = "unknown" } kind := strings.TrimSpace(sample.labels["kind"]) if kind == "" { kind = "unknown" } findings = append(findings, fmt.Sprintf( "%s async sink publish errors %.0f exceeds %.0f for sink %s kind %s", service, sample.value, thresholds.AsyncSinkPublishErrorMax, sink, kind, )) } } sort.Strings(findings) return findings } func findingsForDurableSpool(parsed parsedMetricSet, thresholds Thresholds) []string { var findings []string if thresholds.DurableSpoolBacklogMax > 0 { for _, sample := range parsed.samples { if sample.name != "vehicle_durable_spool_backlog_files" || sample.value <= thresholds.DurableSpoolBacklogMax { continue } name := strings.TrimSpace(sample.labels["name"]) if name == "" { name = "unknown" } findings = append(findings, fmt.Sprintf( "durable spool backlog %.0f exceeds %.0f for name %s", sample.value, thresholds.DurableSpoolBacklogMax, name, )) } } if thresholds.DurableSpoolOldestAgeMaxSec > 0 { for _, sample := range parsed.samples { if sample.name != "vehicle_durable_spool_oldest_age_seconds" || sample.value <= thresholds.DurableSpoolOldestAgeMaxSec { continue } name := strings.TrimSpace(sample.labels["name"]) if name == "" { name = "unknown" } findings = append(findings, fmt.Sprintf( "durable spool oldest age %.0fs exceeds %.0fs for name %s", sample.value, thresholds.DurableSpoolOldestAgeMaxSec, name, )) } } if thresholds.DurableSpoolErrorMax >= 0 { for _, sample := range parsed.samples { if sample.name != "vehicle_durable_spool_records_total" || sample.labels["status"] != "error" || sample.value <= thresholds.DurableSpoolErrorMax { continue } name := strings.TrimSpace(sample.labels["name"]) if name == "" { name = "unknown" } kind := strings.TrimSpace(sample.labels["kind"]) if kind == "" { kind = "unknown" } findings = append(findings, fmt.Sprintf( "durable spool record errors %.0f exceeds %.0f for name %s kind %s", sample.value, thresholds.DurableSpoolErrorMax, name, kind, )) } } if thresholds.DurableSpoolReplayErrorMax >= 0 { for _, sample := range parsed.samples { if sample.name != "vehicle_durable_spool_replay_total" || !strings.HasSuffix(sample.labels["status"], "_error") || sample.value <= thresholds.DurableSpoolReplayErrorMax { continue } name := strings.TrimSpace(sample.labels["name"]) if name == "" { name = "unknown" } status := strings.TrimSpace(sample.labels["status"]) if status == "" { status = "unknown" } findings = append(findings, fmt.Sprintf( "durable spool replay %s %.0f exceeds %.0f for name %s", status, sample.value, thresholds.DurableSpoolReplayErrorMax, name, )) } } sort.Strings(findings) return findings } func findingsForDurableOutbox(parsed parsedMetricSet, thresholds Thresholds) []string { var findings []string if thresholds.DurableOutboxInflightMax > 0 { for _, sample := range parsed.samples { if sample.name != "vehicle_durable_outbox_inflight" || sample.value <= thresholds.DurableOutboxInflightMax { continue } name := strings.TrimSpace(sample.labels["name"]) if name == "" { name = "unknown" } findings = append(findings, fmt.Sprintf( "durable outbox inflight %.0f exceeds %.0f for name %s", sample.value, thresholds.DurableOutboxInflightMax, name, )) } } if thresholds.DurableOutboxErrorMax >= 0 { for _, sample := range parsed.samples { if sample.name != "vehicle_durable_outbox_publish_total" || !durableOutboxErrorStatus(sample.labels["status"]) || sample.value <= thresholds.DurableOutboxErrorMax { continue } name := strings.TrimSpace(sample.labels["name"]) if name == "" { name = "unknown" } kind := strings.TrimSpace(sample.labels["kind"]) if kind == "" { kind = "unknown" } status := strings.TrimSpace(sample.labels["status"]) findings = append(findings, fmt.Sprintf( "durable outbox %s %.0f exceeds %.0f for name %s kind %s", status, sample.value, thresholds.DurableOutboxErrorMax, name, kind, )) } } sort.Strings(findings) return findings } func durableOutboxErrorStatus(status string) bool { switch strings.TrimSpace(status) { case "validation_error", "submit_error", "ack_error", "remove_error", "close_timeout", "quarantined": return true default: return false } } func findingsForHistograms(parsed parsedMetricSet, thresholds Thresholds) []string { var findings []string findings = append(findings, histogramP99Findings(parsed, "vehicle_gateway_frame_duration_ms_histogram", "gateway frame", thresholds.GatewayFrameP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_gateway_identity_duration_ms_histogram", "gateway identity", thresholds.GatewayIdentityP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_gateway_response_e2e_recent_p99_ms", "vehicle_gateway_response_e2e_recent_samples", "gateway response e2e", thresholds.GatewayResponseRecentP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_async_sink_publish_duration_ms_histogram", "async sink publish", thresholds.AsyncSinkP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_async_sink_queue_wait_recent_p99_ms", "vehicle_async_sink_queue_wait_recent_samples", "async sink queue wait", thresholds.AsyncSinkQueueWaitRecentP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_bridge_batch_duration_ms_histogram", "bridge batch", thresholds.BridgeBatchP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_bridge_kafka_write_duration_ms_histogram", "bridge kafka write", thresholds.BridgeKafkaWriteP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_bridge_kafka_e2e_duration_ms_histogram", "bridge kafka e2e", thresholds.BridgeKafkaE2EP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_bridge_kafka_e2e_recent_p99_ms", "vehicle_bridge_kafka_e2e_recent_samples", "bridge kafka e2e", thresholds.BridgeKafkaRecentE2EP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_fast_writer_stage_duration_ms_histogram", "fast writer stage", thresholds.FastWriterStageP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_fast_writer_redis_e2e_duration_ms_histogram", "fast writer redis e2e", thresholds.FastWriterRedisE2EP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_fast_writer_redis_e2e_recent_p99_ms", "vehicle_fast_writer_redis_e2e_recent_samples", "fast writer redis e2e", thresholds.FastWriterRedisRecentE2EP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_history_batch_flush_duration_ms_histogram", "history flush", thresholds.HistoryFlushP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_history_write_e2e_duration_ms_histogram", "history write e2e", thresholds.HistoryWriteE2EP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_history_write_e2e_recent_p99_ms", "vehicle_history_write_e2e_recent_samples", "history write e2e", thresholds.HistoryWriteRecentE2EP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_realtime_batch_flush_duration_ms_histogram", "realtime batch flush", thresholds.RealtimeBatchP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_realtime_store_update_duration_ms_histogram", "realtime store update", thresholds.RealtimeStoreP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_stat_write_duration_ms_histogram", "stat write", thresholds.StatWriteP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, histogramP99Findings(parsed, "vehicle_stat_write_e2e_duration_ms_histogram", "stat write e2e", thresholds.StatWriteE2EP99MS, thresholds.MinHistogramSamples)...) findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_stat_write_e2e_recent_p99_ms", "vehicle_stat_write_e2e_recent_samples", "stat write e2e", thresholds.StatWriteRecentE2EP99MS, thresholds.MinHistogramSamples)...) return findings } type gatewayPublishCounters struct { rawOK float64 fieldsOK float64 fieldsPublished float64 fieldsDelegated float64 fieldsSkippedNonRealtime float64 fieldsSkippedMissing float64 fieldsPublishError float64 } type gatewayFieldFlowGroup struct { gatewayPublishCounters latestCount float64 hasLatestCount bool } type gatewayIdentityCounters struct { total float64 resolved float64 unresolved float64 } type identityResolutionGroup struct { protocol string total int64 skipped int64 resolved int64 unresolved int64 timeout int64 errorCount int64 otherNonResolved int64 staleCache int64 cacheStatusCounts map[string]int64 } type pipelineLatencySpec struct { service string stage string metricName string samplesMetricName string keyLabel string thresholdMS float64 } type gatewayParseCounters struct { total float64 ok float64 partial float64 badFrame float64 other float64 } type gatewayResponseCounters struct { total float64 ok float64 buildError float64 writeError float64 } type historyParsedFieldCounters struct { present float64 missing float64 } type kafkaConsumerCompletionCounters struct { received float64 committedOK float64 commitError float64 } type kafkaConsumerCompletionSpec struct { serviceName string messageMetric string commitMetric string } func findingsForKafkaConsumerCompletion(service string, parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.KafkaConsumerMessageMinReceived <= 0 || thresholds.KafkaConsumerMaxUncommittedRatio <= 0 { return nil } spec, ok := kafkaConsumerCompletionSpecForService(service) if !ok { return nil } groups := map[string]*kafkaConsumerCompletionCounters{} for _, sample := range parsed.samples { switch sample.name { case spec.messageMetric, spec.commitMetric: default: continue } topic := strings.TrimSpace(sample.labels["topic"]) if topic == "" || topic == "mixed" || topic == "unknown" { continue } group := groups[topic] if group == nil { group = &kafkaConsumerCompletionCounters{} groups[topic] = group } switch { case sample.name == spec.messageMetric && sample.labels["status"] == "received": group.received += sample.value case sample.name == spec.commitMetric && sample.labels["status"] == "ok": group.committedOK += sample.value case sample.name == spec.commitMetric && sample.labels["status"] == "error": group.commitError += sample.value } } var topics []string for topic := range groups { topics = append(topics, topic) } sort.Strings(topics) var findings []string for _, topic := range topics { group := groups[topic] if thresholds.KafkaConsumerCommitErrorMax >= 0 && group.commitError > thresholds.KafkaConsumerCommitErrorMax { findings = append(findings, fmt.Sprintf( "%s kafka consumer commit errors %.0f exceeds %.0f for topic %s", spec.serviceName, group.commitError, thresholds.KafkaConsumerCommitErrorMax, topic, )) } if group.received < thresholds.KafkaConsumerMessageMinReceived { continue } uncommitted := group.received - group.committedOK if uncommitted < 0 { uncommitted = 0 } ratio := uncommitted / group.received if ratio <= thresholds.KafkaConsumerMaxUncommittedRatio { continue } findings = append(findings, fmt.Sprintf( "%s kafka consumer uncommitted message ratio %.2f exceeds %.2f for topic %s: received %.0f, commit ok %.0f, commit errors %.0f, uncommitted %.0f", spec.serviceName, ratio, thresholds.KafkaConsumerMaxUncommittedRatio, topic, group.received, group.committedOK, group.commitError, uncommitted, )) } return findings } func kafkaConsumerCompletionSpecForService(service string) (kafkaConsumerCompletionSpec, bool) { switch service { case "history": return kafkaConsumerCompletionSpec{ serviceName: "history", messageMetric: "vehicle_history_kafka_messages_total", commitMetric: "vehicle_history_kafka_commits_total", }, true case "realtime": return kafkaConsumerCompletionSpec{ serviceName: "realtime", messageMetric: "vehicle_realtime_kafka_messages_total", commitMetric: "vehicle_realtime_kafka_commits_total", }, true case "stat": return kafkaConsumerCompletionSpec{ serviceName: "stat", messageMetric: "vehicle_stat_kafka_messages_total", commitMetric: "vehicle_stat_kafka_commits_total", }, true case "identity": return kafkaConsumerCompletionSpec{ serviceName: "identity", messageMetric: "vehicle_identity_writer_kafka_messages_total", commitMetric: "vehicle_identity_writer_kafka_commits_total", }, true default: return kafkaConsumerCompletionSpec{}, false } } func findingsForMessageContractMismatches(service string, parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.MessageContractMismatchMax < 0 { return nil } specs := messageContractMismatchSpecs(service) if len(specs) == 0 { return nil } var findings []string for _, sample := range parsed.samples { status := strings.TrimSpace(sample.labels["status"]) if status == "" { continue } for _, spec := range specs { if sample.name != spec.metric || status != spec.status { continue } if sample.value <= thresholds.MessageContractMismatchMax { continue } key := strings.TrimSpace(sample.labels[spec.keyLabel]) if key == "" { key = "unknown" } findings = append(findings, fmt.Sprintf( "%s message contract %s %.0f exceeds %.0f for %s %s", spec.serviceLabel, status, sample.value, thresholds.MessageContractMismatchMax, spec.keyLabel, key, )) } } sort.Strings(findings) return findings } type messageContractMismatchSpec struct { serviceLabel string metric string status string keyLabel string } func messageContractMismatchSpecs(service string) []messageContractMismatchSpec { switch service { case "history": return []messageContractMismatchSpec{ {serviceLabel: "history", metric: "vehicle_history_kafka_messages_total", status: "event_kind_mismatch", keyLabel: "topic"}, {serviceLabel: "history", metric: "vehicle_history_kafka_messages_total", status: "protocol_topic_mismatch", keyLabel: "topic"}, } case "realtime": return []messageContractMismatchSpec{ {serviceLabel: "realtime", metric: "vehicle_realtime_kafka_messages_total", status: "event_kind_mismatch", keyLabel: "topic"}, {serviceLabel: "realtime", metric: "vehicle_realtime_kafka_messages_total", status: "protocol_topic_mismatch", keyLabel: "topic"}, } case "stat": return []messageContractMismatchSpec{ {serviceLabel: "stat", metric: "vehicle_stat_kafka_messages_total", status: "event_kind_mismatch", keyLabel: "topic"}, } case "identity": return []messageContractMismatchSpec{ {serviceLabel: "identity", metric: "vehicle_identity_writer_kafka_messages_total", status: "event_kind_mismatch", keyLabel: "topic"}, {serviceLabel: "identity", metric: "vehicle_identity_writer_kafka_messages_total", status: "protocol_topic_mismatch", keyLabel: "topic"}, } case "fast-writer": return []messageContractMismatchSpec{ {serviceLabel: "fast writer", metric: "vehicle_fast_writer_messages_total", status: "event_kind_mismatch", keyLabel: "subject"}, {serviceLabel: "fast writer", metric: "vehicle_fast_writer_messages_total", status: "protocol_topic_mismatch", keyLabel: "subject"}, } default: return nil } } func findingsForIdentityWriterOwnership(gateway parsedMetricSet, identityWriter parsedMetricSet) []string { if len(identityWriter.samples) == 0 { return nil } for _, sample := range gateway.samples { if sample.name != "vehicle_gateway_jt808_registration_gateway_writes_enabled" { continue } if sample.value > 0 { return []string{"jt808 registration has duplicate writers: gateway writes remain enabled while identity writer is active"} } return nil } return []string{"gateway jt808 registration ownership metric missing while identity writer is active"} } func findingsForHistoryParsedFields(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.HistoryParsedFieldMinFrames <= 0 || thresholds.HistoryParsedFieldMaxMissingRatio <= 0 { return nil } groups := map[string]*historyParsedFieldCounters{} for _, sample := range parsed.samples { if sample.name != "vehicle_history_parsed_fields_total" { continue } topic := strings.TrimSpace(sample.labels["topic"]) if topic == "" { topic = "unknown" } protocol := strings.TrimSpace(sample.labels["protocol"]) if protocol == "" { protocol = "UNKNOWN" } key := topic + "\x00" + protocol group := groups[key] if group == nil { group = &historyParsedFieldCounters{} groups[key] = group } switch sample.labels["status"] { case "present": group.present += sample.value case "missing": group.missing += sample.value } } var keys []string for key := range groups { keys = append(keys, key) } sort.Strings(keys) var findings []string for _, key := range keys { group := groups[key] total := group.present + group.missing if total < thresholds.HistoryParsedFieldMinFrames || total <= 0 { continue } ratio := group.missing / total if ratio <= thresholds.HistoryParsedFieldMaxMissingRatio { continue } topic, protocol, _ := strings.Cut(key, "\x00") findings = append(findings, fmt.Sprintf( "history parsed fields missing ratio %.2f exceeds %.2f for topic %s protocol %s (total %.0f, present %.0f, missing %.0f)", ratio, thresholds.HistoryParsedFieldMaxMissingRatio, topic, protocol, total, group.present, group.missing, )) } return findings } func findingsForHistoryInvalidJSON(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.HistoryInvalidJSONMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_history_kafka_messages_total" || sample.labels["status"] != "invalid_json" { continue } if sample.value <= thresholds.HistoryInvalidJSONMax { continue } topic := strings.TrimSpace(sample.labels["topic"]) if topic == "" { topic = "unknown" } findings = append(findings, fmt.Sprintf( "history invalid json %.0f exceeds %.0f for topic %s", sample.value, thresholds.HistoryInvalidJSONMax, topic, )) } sort.Strings(findings) return findings } func findingsForRealtimeInvalidJSON(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.RealtimeInvalidJSONMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_realtime_kafka_messages_total" || sample.labels["status"] != "invalid_json" { continue } if sample.value <= thresholds.RealtimeInvalidJSONMax { continue } topic := strings.TrimSpace(sample.labels["topic"]) if topic == "" { topic = "unknown" } findings = append(findings, fmt.Sprintf( "realtime invalid json %.0f exceeds %.0f for topic %s", sample.value, thresholds.RealtimeInvalidJSONMax, topic, )) } sort.Strings(findings) return findings } func findingsForRealtimeStoreErrors(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.RealtimeStoreErrorMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_realtime_store_updates_total" || sample.labels["status"] != "error" { continue } if sample.value <= thresholds.RealtimeStoreErrorMax { continue } store := strings.TrimSpace(sample.labels["store"]) if store == "" { store = "unknown" } protocol := strings.TrimSpace(sample.labels["protocol"]) if protocol == "" { protocol = "UNKNOWN" } findings = append(findings, fmt.Sprintf( "realtime store update errors %.0f exceeds %.0f for store %s protocol %s", sample.value, thresholds.RealtimeStoreErrorMax, store, protocol, )) } sort.Strings(findings) return findings } func findingsForRealtimeAsyncQueue(parsed parsedMetricSet, thresholds Thresholds) []string { var findings []string if thresholds.RealtimeAsyncQueueDroppedMax >= 0 { for _, sample := range parsed.samples { if sample.name != "vehicle_realtime_async_queue_total" { continue } status := sample.labels["status"] if status != "dropped" && status != "closed" { continue } if sample.value <= thresholds.RealtimeAsyncQueueDroppedMax { continue } store := strings.TrimSpace(sample.labels["store"]) if store == "" { store = "unknown" } protocol := strings.TrimSpace(sample.labels["protocol"]) if protocol == "" { protocol = "UNKNOWN" } findings = append(findings, fmt.Sprintf( "realtime async queue %s %.0f exceeds %.0f for store %s protocol %s", status, sample.value, thresholds.RealtimeAsyncQueueDroppedMax, store, protocol, )) } } if thresholds.RealtimeAsyncQueueMaxUsageRatio <= 0 { sort.Strings(findings) return findings } type asyncQueueSample struct { labels map[string]string depth float64 } queues := map[string]*asyncQueueSample{} capacityByKey := map[string]float64{} capacityByStore := map[string]float64{} for _, sample := range parsed.samples { switch sample.name { case "vehicle_realtime_async_queue_depth": key := metricLabelsKey(sample.labels) item := queues[key] if item == nil { item = &asyncQueueSample{labels: sample.labels} queues[key] = item } item.depth += sample.value case "vehicle_realtime_async_queue_capacity": key := metricLabelsKey(sample.labels) capacityByKey[key] += sample.value store := strings.TrimSpace(sample.labels["store"]) if store != "" { capacityByStore[store] += sample.value } } } var keys []string for key := range queues { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { item := queues[key] capacity := capacityByKey[key] if capacity <= 0 { capacity = capacityByStore[strings.TrimSpace(item.labels["store"])] } if capacity <= 0 || item.depth <= 0 { continue } ratio := item.depth / capacity if ratio <= thresholds.RealtimeAsyncQueueMaxUsageRatio { continue } findings = append(findings, fmt.Sprintf( "realtime async queue usage %.2f exceeds %.2f (%s, depth %.0f, capacity %.0f)", ratio, thresholds.RealtimeAsyncQueueMaxUsageRatio, key, item.depth, capacity, )) } sort.Strings(findings) return findings } func findingsForGatewayResponseQuality(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.GatewayResponseMinSamples <= 0 || thresholds.GatewayResponseMaxErrorRatio <= 0 { return nil } groups := map[string]*gatewayResponseCounters{} for _, sample := range parsed.samples { if sample.name != "vehicle_gateway_response_total" { continue } protocol := strings.TrimSpace(sample.labels["protocol"]) if protocol == "" { protocol = "unknown" } group := groups[protocol] if group == nil { group = &gatewayResponseCounters{} groups[protocol] = group } switch sample.labels["status"] { case "ok": group.ok += sample.value group.total += sample.value case "build_error": group.buildError += sample.value group.total += sample.value case "write_error": group.writeError += sample.value group.total += sample.value } } var protocols []string for protocol := range groups { protocols = append(protocols, protocol) } sort.Strings(protocols) var findings []string for _, protocol := range protocols { group := groups[protocol] if group.total < thresholds.GatewayResponseMinSamples || group.total <= 0 { continue } errors := group.buildError + group.writeError ratio := errors / group.total if ratio <= thresholds.GatewayResponseMaxErrorRatio { continue } findings = append(findings, fmt.Sprintf( "gateway response error ratio %.2f exceeds %.2f for protocol %s (total %.0f, ok %.0f, errors %.0f, build_error %.0f, write_error %.0f)", ratio, thresholds.GatewayResponseMaxErrorRatio, protocol, group.total, group.ok, errors, group.buildError, group.writeError, )) } return findings } func findingsForGatewayParseQuality(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.GatewayParseMinFrames <= 0 || thresholds.GatewayParseMaxBadRatio <= 0 { return nil } groups := map[string]*gatewayParseCounters{} for _, sample := range parsed.samples { if sample.name != "vehicle_gateway_frames_total" { continue } protocol := strings.TrimSpace(sample.labels["protocol"]) if protocol == "" { protocol = "unknown" } group := groups[protocol] if group == nil { group = &gatewayParseCounters{} groups[protocol] = group } group.total += sample.value switch envelope.ParseStatus(sample.labels["status"]) { case envelope.ParseOK: group.ok += sample.value case envelope.ParsePartial: group.partial += sample.value case envelope.ParseBadFrame: group.badFrame += sample.value default: group.other += sample.value } } var protocols []string for protocol := range groups { protocols = append(protocols, protocol) } sort.Strings(protocols) var findings []string for _, protocol := range protocols { group := groups[protocol] if group.total < thresholds.GatewayParseMinFrames || group.total <= 0 { continue } bad := group.partial + group.badFrame + group.other ratio := bad / group.total if ratio <= thresholds.GatewayParseMaxBadRatio { continue } findings = append(findings, fmt.Sprintf( "gateway parse bad ratio %.2f exceeds %.2f for protocol %s (total %.0f, ok %.0f, bad %.0f, partial %.0f, bad_frame %.0f, other %.0f)", ratio, thresholds.GatewayParseMaxBadRatio, protocol, group.total, group.ok, bad, group.partial, group.badFrame, group.other, )) } return findings } func findingsForGatewayIdentityQuality(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.GatewayIdentityMinSamples <= 0 || thresholds.GatewayIdentityMaxUnresolvedRatio <= 0 { return nil } groups := map[string]*gatewayIdentityCounters{} for _, sample := range parsed.samples { if sample.name != "vehicle_gateway_identity_total" { continue } protocol := strings.TrimSpace(sample.labels["protocol"]) if protocol == "" { protocol = "unknown" } group := groups[protocol] if group == nil { group = &gatewayIdentityCounters{} groups[protocol] = group } group.total += sample.value if sample.labels["status"] == "resolved" { group.resolved += sample.value } else { group.unresolved += sample.value } } var protocols []string for protocol := range groups { protocols = append(protocols, protocol) } sort.Strings(protocols) var findings []string for _, protocol := range protocols { group := groups[protocol] if group.total < thresholds.GatewayIdentityMinSamples || group.total <= 0 { continue } ratio := group.unresolved / group.total if ratio <= thresholds.GatewayIdentityMaxUnresolvedRatio { continue } findings = append(findings, fmt.Sprintf( "gateway identity unresolved ratio %.2f exceeds %.2f for protocol %s (total %.0f, unresolved %.0f, resolved %.0f)", ratio, thresholds.GatewayIdentityMaxUnresolvedRatio, protocol, group.total, group.unresolved, group.resolved, )) } return findings } func buildIdentityResolutionDiagnostics(parsed parsedMetricSet) *IdentityResolutionDiagnostics { groups := map[string]*identityResolutionGroup{} cacheEntries := map[string]int64{} issues := map[string]IdentityIssueCount{} for _, sample := range parsed.samples { switch sample.name { case "vehicle_gateway_identity_total": protocol := normalizedMetricLabel(sample.labels, "protocol") status := normalizedMetricLabel(sample.labels, "status") group := identityResolutionGroupForProtocol(groups, protocol) value := metricCount(sample.value) group.total += value switch status { case "resolved": group.resolved += value case "unresolved": group.unresolved += value case "timeout": group.timeout += value case "error": group.errorCount += value default: group.otherNonResolved += value } case "vehicle_gateway_identity_skips_total": protocol := normalizedMetricLabel(sample.labels, "protocol") group := identityResolutionGroupForProtocol(groups, protocol) group.skipped += metricCount(sample.value) case "vehicle_gateway_identity_issues_total": issue := IdentityIssueCount{ Protocol: normalizedMetricLabel(sample.labels, "protocol"), Status: normalizedMetricLabel(sample.labels, "status"), MessageID: normalizedMetricLabel(sample.labels, "message_id"), Reason: normalizedMetricLabel(sample.labels, "reason"), Count: metricCount(sample.value), } if issue.Count <= 0 { continue } key := strings.Join([]string{issue.Protocol, issue.Status, issue.MessageID, issue.Reason}, "\x00") existing := issues[key] if existing.Count == 0 { existing = issue } else { existing.Count += issue.Count } issues[key] = existing case "vehicle_gateway_identity_cache_total": protocol := normalizedMetricLabel(sample.labels, "protocol") cacheStatus := normalizedMetricLabel(sample.labels, "cache_status") group := identityResolutionGroupForProtocol(groups, protocol) value := metricCount(sample.value) if group.cacheStatusCounts == nil { group.cacheStatusCounts = map[string]int64{} } group.cacheStatusCounts[cacheStatus] += value if cacheStatus == "stale" { group.staleCache += value } case "vehicle_gateway_identity_cache_entries": cacheName := normalizedMetricLabel(sample.labels, "cache") cacheEntries[cacheName] += metricCount(sample.value) } } if len(groups) == 0 && len(cacheEntries) == 0 && len(issues) == 0 { return nil } diagnostics := &IdentityResolutionDiagnostics{} if len(cacheEntries) > 0 { diagnostics.CacheEntries = cacheEntries } if len(issues) > 0 { diagnostics.Issues = sortedIdentityIssues(issues) } var protocols []string for protocol := range groups { protocols = append(protocols, protocol) } sort.Strings(protocols) for _, protocol := range protocols { group := groups[protocol] row := identityProtocolResolution(group) diagnostics.Total += row.Total diagnostics.SkippedCount += row.SkippedCount diagnostics.ResolvedCount += row.ResolvedCount diagnostics.NonResolvedCount += row.NonResolvedCount diagnostics.UnresolvedCount += row.UnresolvedCount diagnostics.TimeoutCount += row.TimeoutCount diagnostics.ErrorCount += row.ErrorCount diagnostics.OtherNonResolvedCount += row.OtherNonResolvedCount diagnostics.StaleCacheCount += row.StaleCacheCount diagnostics.Protocols = append(diagnostics.Protocols, row) } diagnostics.NonResolvedRatio = ratioInt64(diagnostics.NonResolvedCount, diagnostics.Total) diagnostics.StaleCacheRatio = ratioInt64(diagnostics.StaleCacheCount, diagnostics.Total) return diagnostics } func sortedIdentityIssues(issues map[string]IdentityIssueCount) []IdentityIssueCount { rows := make([]IdentityIssueCount, 0, len(issues)) for _, row := range issues { rows = append(rows, row) } sort.Slice(rows, func(i, j int) bool { if rows[i].Protocol != rows[j].Protocol { return rows[i].Protocol < rows[j].Protocol } if rows[i].Status != rows[j].Status { return rows[i].Status < rows[j].Status } if rows[i].MessageID != rows[j].MessageID { return rows[i].MessageID < rows[j].MessageID } return rows[i].Reason < rows[j].Reason }) return rows } func identityProtocolResolution(group *identityResolutionGroup) IdentityProtocolResolution { if group == nil { return IdentityProtocolResolution{Protocol: "unknown"} } nonResolved := group.unresolved + group.timeout + group.errorCount + group.otherNonResolved row := IdentityProtocolResolution{ Protocol: group.protocol, Total: group.total, SkippedCount: group.skipped, ResolvedCount: group.resolved, NonResolvedCount: nonResolved, UnresolvedCount: group.unresolved, TimeoutCount: group.timeout, ErrorCount: group.errorCount, OtherNonResolvedCount: group.otherNonResolved, NonResolvedRatio: ratioInt64(nonResolved, group.total), StaleCacheCount: group.staleCache, StaleCacheRatio: ratioInt64(group.staleCache, group.total), CacheStatusCounts: group.cacheStatusCounts, } return row } func identityResolutionGroupForProtocol(groups map[string]*identityResolutionGroup, protocol string) *identityResolutionGroup { if protocol == "" { protocol = "unknown" } group := groups[protocol] if group == nil { group = &identityResolutionGroup{protocol: protocol} groups[protocol] = group } return group } func normalizedMetricLabel(labels map[string]string, key string) string { value := strings.TrimSpace(labels[key]) if value == "" { return "unknown" } return value } func metricCount(value float64) int64 { if value <= 0 { return 0 } return int64(math.Round(value)) } func ratioInt64(numerator int64, denominator int64) float64 { if denominator <= 0 { return 0 } return float64(numerator) / float64(denominator) } func buildPipelineLatencyDiagnostics(parsedByService map[string]parsedMetricSet, thresholds Thresholds) *PipelineLatencyDiagnostics { specs := []pipelineLatencySpec{ { service: "gateway", stage: "gateway_protocol_response_e2e", metricName: "vehicle_gateway_response_e2e_recent_p99_ms", samplesMetricName: "vehicle_gateway_response_e2e_recent_samples", keyLabel: "protocol", thresholdMS: thresholds.GatewayResponseRecentP99MS, }, { service: "bridge", stage: "bridge_kafka_e2e", metricName: "vehicle_bridge_kafka_e2e_recent_p99_ms", samplesMetricName: "vehicle_bridge_kafka_e2e_recent_samples", keyLabel: "topic", thresholdMS: thresholds.BridgeKafkaRecentE2EP99MS, }, { service: "fast-writer", stage: "fast_writer_redis_e2e", metricName: "vehicle_fast_writer_redis_e2e_recent_p99_ms", samplesMetricName: "vehicle_fast_writer_redis_e2e_recent_samples", keyLabel: "subject", thresholdMS: thresholds.FastWriterRedisRecentE2EP99MS, }, { service: "history", stage: "history_tdengine_write_e2e", metricName: "vehicle_history_write_e2e_recent_p99_ms", samplesMetricName: "vehicle_history_write_e2e_recent_samples", keyLabel: "topic", thresholdMS: thresholds.HistoryWriteRecentE2EP99MS, }, { service: "stat", stage: "stat_mysql_write_e2e", metricName: "vehicle_stat_write_e2e_recent_p99_ms", samplesMetricName: "vehicle_stat_write_e2e_recent_samples", keyLabel: "topic", thresholdMS: thresholds.StatWriteRecentE2EP99MS, }, } var items []PipelineLatencyItem for _, spec := range specs { items = append(items, pipelineLatencyItemsForSpec(parsedByService[spec.service], spec, thresholds.MinHistogramSamples)...) } if len(items) == 0 { return nil } sort.Slice(items, func(i, j int) bool { if items[i].Stage != items[j].Stage { return items[i].Stage < items[j].Stage } if items[i].Service != items[j].Service { return items[i].Service < items[j].Service } if items[i].Topic != items[j].Topic { return items[i].Topic < items[j].Topic } return items[i].Subject < items[j].Subject }) diagnostics := &PipelineLatencyDiagnostics{Items: items} for _, item := range items { diagnostics.TotalSamples += item.Samples if item.P99MS > diagnostics.MaxP99MS { diagnostics.MaxP99MS = item.P99MS } } return diagnostics } func pipelineLatencyItemsForSpec(parsed parsedMetricSet, spec pipelineLatencySpec, minSamples float64) []PipelineLatencyItem { samplesByKey := map[string]float64{} for _, sample := range parsed.samples { if sample.name != spec.samplesMetricName { continue } samplesByKey[metricLabelsKey(sample.labels)] = sample.value } var items []PipelineLatencyItem for _, sample := range parsed.samples { if sample.name != spec.metricName { continue } key := metricLabelsKey(sample.labels) row := PipelineLatencyItem{ Stage: spec.stage, Service: spec.service, P99MS: sample.value, Samples: metricCount(samplesByKey[key]), ThresholdMS: spec.thresholdMS, } switch spec.keyLabel { case "subject": row.Subject = normalizedMetricLabel(sample.labels, spec.keyLabel) default: row.Topic = normalizedMetricLabel(sample.labels, spec.keyLabel) } row.Status = pipelineLatencyStatus(row.P99MS, float64(row.Samples), spec.thresholdMS, minSamples) items = append(items, row) } return items } func pipelineLatencyStatus(p99MS float64, samples float64, thresholdMS float64, minSamples float64) string { if minSamples > 0 && samples < minSamples { return "insufficient_samples" } if thresholdMS > 0 && p99MS > thresholdMS { return "degraded" } return "ok" } func findingsForGatewayIdentityCache(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.GatewayIdentityCacheEntriesMax <= 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_gateway_identity_cache_entries" || sample.value <= thresholds.GatewayIdentityCacheEntriesMax { continue } cacheName := strings.TrimSpace(sample.labels["cache"]) if cacheName == "" { cacheName = "unknown" } if cacheName == "registration_write_queue" { continue } findings = append(findings, fmt.Sprintf( "gateway identity cache entries %.0f exceeds %.0f for cache %s", sample.value, thresholds.GatewayIdentityCacheEntriesMax, cacheName, )) } sort.Strings(findings) return findings } func findingsForGatewayIdentitySnapshot(parsed parsedMetricSet, thresholds Thresholds, checkedAt time.Time) []string { if thresholds.GatewayIdentitySnapshotRequired <= 0 && thresholds.GatewayIdentitySnapshotMaxAgeSec <= 0 { return nil } if !hasServiceInfo(parsed, "vehicle-gateway") { return nil } ready, readyFound := parsed.totals["vehicle_gateway_identity_snapshot_ready"] var findings []string if thresholds.GatewayIdentitySnapshotRequired > 0 { switch { case !readyFound: findings = append(findings, "gateway identity snapshot readiness metric missing") case ready < thresholds.GatewayIdentitySnapshotRequired: findings = append(findings, fmt.Sprintf("gateway identity snapshot ready %.0f below %.0f", ready, thresholds.GatewayIdentitySnapshotRequired)) } } if thresholds.GatewayIdentitySnapshotMaxAgeSec <= 0 || !readyFound || ready <= 0 { return findings } lastSuccess, ok := parsed.totals["vehicle_gateway_identity_snapshot_last_success_unix_seconds"] if !ok || lastSuccess <= 0 { return append(findings, "gateway identity snapshot last-success metric missing") } age := checkedAt.Unix() - int64(lastSuccess) if age < 0 { age = 0 } if float64(age) > thresholds.GatewayIdentitySnapshotMaxAgeSec { findings = append(findings, fmt.Sprintf( "gateway identity snapshot age %ds exceeds %.0fs", age, thresholds.GatewayIdentitySnapshotMaxAgeSec, )) } return findings } func findingsForGatewayRegistrationWriteQueue(parsed parsedMetricSet, thresholds Thresholds) []string { depth := gatewayIdentityCacheMetric(parsed, "registration_write_queue", "vehicle_gateway_identity_cache_entries") capacity := gatewayIdentityCacheMetric(parsed, "registration_write_queue", "vehicle_gateway_identity_cache_max_entries") var findings []string if thresholds.GatewayIdentityRegistrationWriteQueueDepthMax > 0 && depth > thresholds.GatewayIdentityRegistrationWriteQueueDepthMax { findings = append(findings, fmt.Sprintf( "gateway jt808 registration write queue depth %.0f exceeds %.0f", depth, thresholds.GatewayIdentityRegistrationWriteQueueDepthMax, )) } if thresholds.GatewayIdentityRegistrationWriteQueueMaxUsageRatio > 0 && depth > 0 && capacity > 0 { ratio := depth / capacity if ratio > thresholds.GatewayIdentityRegistrationWriteQueueMaxUsageRatio { findings = append(findings, fmt.Sprintf( "gateway jt808 registration write queue usage %.2f exceeds %.2f (depth %.0f, capacity %.0f)", ratio, thresholds.GatewayIdentityRegistrationWriteQueueMaxUsageRatio, depth, capacity, )) } } sort.Strings(findings) return findings } func gatewayIdentityCacheMetric(parsed parsedMetricSet, cacheName string, metricName string) float64 { cacheName = strings.TrimSpace(cacheName) if cacheName == "" || metricName == "" { return 0 } var total float64 for _, sample := range parsed.samples { if sample.name != metricName || strings.TrimSpace(sample.labels["cache"]) != cacheName { continue } total += sample.value } return total } func findingsForGatewayRegistrationWriteErrors(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.GatewayIdentityRegistrationWriteErrorMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_gateway_jt808_registration_write_total" { continue } status := strings.TrimSpace(sample.labels["status"]) if status == "" || status == "ok" { continue } if sample.value <= thresholds.GatewayIdentityRegistrationWriteErrorMax { continue } mode := strings.TrimSpace(sample.labels["mode"]) if mode == "" { mode = "unknown" } findings = append(findings, fmt.Sprintf( "gateway jt808 registration write %s count %.0f exceeds %.0f for mode %s", status, sample.value, thresholds.GatewayIdentityRegistrationWriteErrorMax, mode, )) } sort.Strings(findings) return findings } func jt808RegistrationWriteErrors(parsed parsedMetricSet) float64 { var total float64 for _, sample := range parsed.samples { if sample.name != "vehicle_gateway_jt808_registration_write_total" { continue } status := strings.TrimSpace(sample.labels["status"]) if status == "" || status == "ok" { continue } total += sample.value } return total } func findingsForGatewayIdentityLocationTouchFailures(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.GatewayIdentityLocationTouchFailureMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_gateway_identity_cache_entries" || strings.TrimSpace(sample.labels["cache"]) != "location_touch_failure" { continue } if sample.value <= thresholds.GatewayIdentityLocationTouchFailureMax { continue } findings = append(findings, fmt.Sprintf( "gateway identity location touch failure cache %.0f exceeds %.0f; jt808 registration writes are failing or in protective backoff", sample.value, thresholds.GatewayIdentityLocationTouchFailureMax, )) } sort.Strings(findings) return findings } func findingsForGatewayIdentityStaleCache(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.GatewayIdentityStaleMinSamples <= 0 || thresholds.GatewayIdentityMaxStaleCacheRatio <= 0 { return nil } totalsByProtocol := map[string]float64{} staleByProtocol := map[string]float64{} for _, sample := range parsed.samples { protocol := strings.TrimSpace(sample.labels["protocol"]) if protocol == "" { protocol = "unknown" } switch sample.name { case "vehicle_gateway_identity_total": totalsByProtocol[protocol] += sample.value case "vehicle_gateway_identity_cache_total": if sample.labels["cache_status"] == "stale" { staleByProtocol[protocol] += sample.value } } } var protocols []string for protocol := range totalsByProtocol { protocols = append(protocols, protocol) } sort.Strings(protocols) var findings []string for _, protocol := range protocols { total := totalsByProtocol[protocol] if total < thresholds.GatewayIdentityStaleMinSamples || total <= 0 { continue } stale := staleByProtocol[protocol] ratio := stale / total if ratio <= thresholds.GatewayIdentityMaxStaleCacheRatio { continue } findings = append(findings, fmt.Sprintf( "gateway identity stale cache ratio %.2f exceeds %.2f for protocol %s (total %.0f, stale %.0f)", ratio, thresholds.GatewayIdentityMaxStaleCacheRatio, protocol, total, stale, )) } return findings } func findingsForBridgeRouteErrors(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.BridgeRouteErrorMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_bridge_messages_total" || sample.labels["status"] != "route_error" { continue } if sample.value <= thresholds.BridgeRouteErrorMax { continue } subject := strings.TrimSpace(sample.labels["subject"]) if subject == "" { subject = "unknown" } findings = append(findings, fmt.Sprintf( "bridge route errors %.0f exceeds %.0f for subject %s", sample.value, thresholds.BridgeRouteErrorMax, subject, )) } sort.Strings(findings) return findings } func findingsForBridgeKafkaWriteErrors(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.BridgeKafkaWriteErrorMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_bridge_kafka_writes_total" || sample.labels["status"] != "error" { continue } if sample.value <= thresholds.BridgeKafkaWriteErrorMax { continue } topic := strings.TrimSpace(sample.labels["topic"]) if topic == "" { topic = "unknown" } findings = append(findings, fmt.Sprintf( "bridge kafka write errors %.0f exceeds %.0f for topic %s", sample.value, thresholds.BridgeKafkaWriteErrorMax, topic, )) } sort.Strings(findings) return findings } type bridgeMessageCounters struct { received float64 ackOK float64 droppedRouteError float64 } func findingsForBridgeMessageCompletion(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.BridgeMessageMinReceived <= 0 || thresholds.BridgeMessageMaxUnackedRatio <= 0 { return nil } groups := map[string]*bridgeMessageCounters{} for _, sample := range parsed.samples { switch sample.name { case "vehicle_bridge_messages_total", "vehicle_bridge_nats_acks_total": default: continue } subject := strings.TrimSpace(sample.labels["subject"]) if subject == "" || subject == "mixed" || subject == "unknown" { continue } group := groups[subject] if group == nil { group = &bridgeMessageCounters{} groups[subject] = group } switch { case sample.name == "vehicle_bridge_messages_total" && sample.labels["status"] == "received": group.received += sample.value case sample.name == "vehicle_bridge_nats_acks_total" && sample.labels["status"] == "ok": group.ackOK += sample.value case sample.name == "vehicle_bridge_nats_acks_total" && sample.labels["status"] == "dropped_route_error": group.droppedRouteError += sample.value } } var subjects []string for subject := range groups { subjects = append(subjects, subject) } sort.Strings(subjects) var findings []string for _, subject := range subjects { group := groups[subject] if group.received < thresholds.BridgeMessageMinReceived { continue } completed := group.ackOK + group.droppedRouteError unacked := group.received - completed if unacked < 0 { unacked = 0 } ratio := unacked / group.received if ratio <= thresholds.BridgeMessageMaxUnackedRatio { continue } findings = append(findings, fmt.Sprintf( "bridge unacked message ratio %.2f exceeds %.2f for subject %s: received %.0f, ack ok %.0f, dropped route error %.0f, unacked %.0f", ratio, thresholds.BridgeMessageMaxUnackedRatio, subject, group.received, group.ackOK, group.droppedRouteError, unacked, )) } return findings } func findingsForTopicFlow(parsedByService map[string]parsedMetricSet, thresholds Thresholds) []string { var findings []string findings = append(findings, findingsForGatewayFieldFlow(parsedByService["gateway"], thresholds)...) findings = append(findings, findingsForGatewayFieldCounts(parsedByService["gateway"], thresholds)...) findings = append(findings, findingsForGatewayToBridgeFlow(parsedByService["gateway"], parsedByService["bridge"], thresholds)...) findings = append(findings, findingsForGatewayToFastWriterFlow(parsedByService["gateway"], parsedByService["fast-writer"], thresholds)...) findings = append(findings, findingsForBridgeRawFanout(parsedByService["bridge"], parsedByService["history"], parsedByService["realtime"], thresholds)...) findings = append(findings, findingsForBridgeToStatFieldFlow(parsedByService["bridge"], parsedByService["stat"], thresholds)...) return findings } func findingsForGatewayFieldFlow(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.GatewayFieldMinRaw <= 0 { return nil } groups := gatewayFieldFlowGroups(parsed) protocols := gatewayFieldFlowProtocols(groups) var findings []string for _, protocol := range protocols { group := groups[protocol] realtimeCandidates := gatewayRealtimeCandidates(group.gatewayPublishCounters) if realtimeCandidates < thresholds.GatewayFieldMinRaw { continue } if group.fieldsOK <= 0 { findings = append(findings, fmt.Sprintf( "gateway fields missing for protocol %s: raw ok %.0f, realtime candidates %.0f, fields ok %.0f, skipped non-realtime %.0f, skipped missing fields %.0f, publish errors %.0f", protocol, group.rawOK, realtimeCandidates, group.fieldsOK, group.fieldsSkippedNonRealtime, group.fieldsSkippedMissing, group.fieldsPublishError, )) continue } missingRatio := (group.fieldsSkippedMissing + group.fieldsPublishError) / realtimeCandidates if thresholds.GatewayFieldMaxMissingRatio > 0 && missingRatio > thresholds.GatewayFieldMaxMissingRatio { findings = append(findings, fmt.Sprintf( "gateway fields missing ratio %.2f exceeds %.2f for protocol %s: realtime candidates %.0f, fields ok %.0f, skipped missing fields %.0f, publish errors %.0f", missingRatio, thresholds.GatewayFieldMaxMissingRatio, protocol, realtimeCandidates, group.fieldsOK, group.fieldsSkippedMissing, group.fieldsPublishError, )) } } return findings } func findingsForGatewayFieldCounts(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.GatewayFieldMinCount <= 0 { return nil } minPublished := thresholds.GatewayFieldMinRaw if minPublished <= 0 { minPublished = 1 } groups := gatewayFieldFlowGroups(parsed) protocols := gatewayFieldFlowProtocols(groups) var findings []string for _, protocol := range protocols { group := groups[protocol] fieldsAccepted := group.fieldsPublished + group.fieldsDelegated if fieldsAccepted < minPublished { continue } if !group.hasLatestCount { findings = append(findings, fmt.Sprintf( "gateway fields count metric missing for protocol %s after %.0f published fields events", protocol, fieldsAccepted, )) continue } if group.latestCount < thresholds.GatewayFieldMinCount { findings = append(findings, fmt.Sprintf( "gateway fields count %.0f below %.0f for protocol %s after %.0f published fields events", group.latestCount, thresholds.GatewayFieldMinCount, protocol, fieldsAccepted, )) } } return findings } func buildGatewayFieldsDiagnostics(parsed parsedMetricSet, thresholds Thresholds) *GatewayFieldsDiagnostics { groups := gatewayFieldFlowGroups(parsed) protocols := gatewayFieldFlowProtocols(groups) if len(protocols) == 0 { return nil } out := &GatewayFieldsDiagnostics{Protocols: make([]GatewayProtocolFields, 0, len(protocols))} for _, protocol := range protocols { group := groups[protocol] realtimeCandidates := gatewayRealtimeCandidates(group.gatewayPublishCounters) missingRatio := 0.0 if realtimeCandidates > 0 { missingRatio = (group.fieldsSkippedMissing + group.fieldsPublishError) / realtimeCandidates } status, reason := gatewayFieldDiagnosticsStatus(group, realtimeCandidates, thresholds) var latestCount *float64 if group.hasLatestCount { value := group.latestCount latestCount = &value } out.Protocols = append(out.Protocols, GatewayProtocolFields{ Protocol: protocol, RawOK: group.rawOK, RealtimeCandidates: realtimeCandidates, FieldsOK: group.fieldsOK, FieldsPublished: group.fieldsPublished, FieldsDelegated: group.fieldsDelegated, FieldsSkippedNonRealtime: group.fieldsSkippedNonRealtime, FieldsSkippedMissing: group.fieldsSkippedMissing, FieldsPublishError: group.fieldsPublishError, MissingRatio: missingRatio, LatestFieldCount: latestCount, Status: status, Reason: reason, }) } return out } func gatewayFieldDiagnosticsStatus(group *gatewayFieldFlowGroup, realtimeCandidates float64, thresholds Thresholds) (string, string) { if thresholds.GatewayFieldMinRaw > 0 && realtimeCandidates >= thresholds.GatewayFieldMinRaw { if group.fieldsOK <= 0 { return "degraded", "missing_fields_publish" } missingRatio := 0.0 if realtimeCandidates > 0 { missingRatio = (group.fieldsSkippedMissing + group.fieldsPublishError) / realtimeCandidates } if thresholds.GatewayFieldMaxMissingRatio > 0 && missingRatio > thresholds.GatewayFieldMaxMissingRatio { return "degraded", "high_missing_ratio" } } minPublished := thresholds.GatewayFieldMinRaw if minPublished <= 0 { minPublished = 1 } if thresholds.GatewayFieldMinCount > 0 && group.fieldsPublished+group.fieldsDelegated >= minPublished { if !group.hasLatestCount { return "degraded", "missing_field_count_metric" } if group.latestCount < thresholds.GatewayFieldMinCount { return "degraded", "low_field_count" } } return "ok", "" } func gatewayFieldFlowGroups(parsed parsedMetricSet) map[string]*gatewayFieldFlowGroup { groups := map[string]*gatewayFieldFlowGroup{} groupForProtocol := func(protocol string) *gatewayFieldFlowGroup { protocol = strings.TrimSpace(protocol) if protocol == "" { protocol = "unknown" } group := groups[protocol] if group == nil { group = &gatewayFieldFlowGroup{} groups[protocol] = group } return group } for _, sample := range parsed.samples { switch sample.name { case "vehicle_gateway_publish_total": if sample.labels["status"] != "ok" && sample.labels["status"] != "delegated" { continue } group := groupForProtocol(sample.labels["protocol"]) switch sample.labels["kind"] { case "raw": group.rawOK += sample.value case "fields": group.fieldsOK += sample.value } case "vehicle_gateway_fields_total": group := groupForProtocol(sample.labels["protocol"]) switch sample.labels["status"] { case "published": group.fieldsPublished += sample.value case "delegated_to_bridge": group.fieldsDelegated += sample.value case "skipped_non_realtime": group.fieldsSkippedNonRealtime += sample.value case "skipped_missing_fields": group.fieldsSkippedMissing += sample.value case "publish_error": group.fieldsPublishError += sample.value } case "vehicle_gateway_fields_count": if sample.labels["status"] != "published" && sample.labels["status"] != "delegated_to_bridge" { continue } group := groupForProtocol(sample.labels["protocol"]) group.latestCount = sample.value group.hasLatestCount = true } } return groups } func gatewayFieldFlowProtocols(groups map[string]*gatewayFieldFlowGroup) []string { var protocols []string for protocol := range groups { protocols = append(protocols, protocol) } sort.Strings(protocols) return protocols } func gatewayRealtimeCandidates(group gatewayPublishCounters) float64 { realtimeCandidates := group.rawOK - group.fieldsSkippedNonRealtime if realtimeCandidates < 0 { return 0 } return realtimeCandidates } func findingsForGatewayToFastWriterFlow(gateway parsedMetricSet, fastWriter parsedMetricSet, thresholds Thresholds) []string { if thresholds.GatewayFastWriterMinRawPublish <= 0 { return nil } gatewayPublishes := gatewayPublishGroups(gateway.samples) fastWriterMessages := metricValuesBySubject(fastWriter.samples, "vehicle_fast_writer_messages_total", "received", "vehicle.raw.go.") var findings []string for _, group := range gatewayPublishes { if group.kind != "raw" || group.count < thresholds.GatewayFastWriterMinRawPublish { continue } subject, ok := expectedKafkaTopicForGatewayPublish(group.protocol, group.kind) if !ok { continue } if fastWriterMessages[subject] <= 0 { findings = append(findings, fmt.Sprintf( "fast writer received messages missing for gateway raw publish protocol %s: gateway ok %.0f, expected subject %s, fast writer received %.0f", group.protocol, group.count, subject, fastWriterMessages[subject], )) } } return findings } type gatewayPublishGroup struct { protocol string kind string count float64 } func findingsForGatewayToBridgeFlow(gateway parsedMetricSet, bridge parsedMetricSet, thresholds Thresholds) []string { if thresholds.GatewayBridgeMinPublish <= 0 { return nil } gatewayPublishes := gatewayPublishGroups(gateway.samples) bridgeWrites := metricValuesByTopic(bridge.samples, "vehicle_bridge_kafka_writes_total", "ok", "vehicle.") var findings []string for _, group := range gatewayPublishes { if group.count < thresholds.GatewayBridgeMinPublish { continue } topic, ok := expectedKafkaTopicForGatewayPublish(group.protocol, group.kind) if !ok { findings = append(findings, fmt.Sprintf( "gateway %s publish topic mapping missing for protocol %s: gateway ok %.0f", group.kind, group.protocol, group.count, )) continue } if bridgeWrites[topic] <= 0 { findings = append(findings, fmt.Sprintf( "bridge writes missing for gateway %s publish protocol %s: gateway ok %.0f, expected topic %s, bridge writes ok %.0f", group.kind, group.protocol, group.count, topic, bridgeWrites[topic], )) } } return findings } func gatewayPublishGroups(samples []metricSample) []gatewayPublishGroup { groups := map[string]*gatewayPublishGroup{} for _, sample := range samples { if sample.name != "vehicle_gateway_publish_total" || (sample.labels["status"] != "ok" && sample.labels["status"] != "delegated") { continue } protocol := strings.TrimSpace(sample.labels["protocol"]) if protocol == "" { protocol = "unknown" } kind := strings.TrimSpace(sample.labels["kind"]) if kind == "" { kind = "unknown" } key := protocol + "\x00" + kind group := groups[key] if group == nil { group = &gatewayPublishGroup{protocol: protocol, kind: kind} groups[key] = group } group.count += sample.value } var keys []string for key := range groups { keys = append(keys, key) } sort.Strings(keys) out := make([]gatewayPublishGroup, 0, len(keys)) for _, key := range keys { out = append(out, *groups[key]) } return out } func expectedKafkaTopicForGatewayPublish(protocol string, kind string) (string, bool) { switch kind { case "raw": switch envelope.Protocol(protocol) { case envelope.ProtocolGB32960: return topics.RawGB32960, true case envelope.ProtocolJT808: return topics.RawJT808, true case envelope.ProtocolYutongMQTT: return topics.RawYutongMQTT, true default: return "", false } case "fields": switch envelope.Protocol(protocol) { case envelope.ProtocolGB32960: return topics.FieldsGB32960, true case envelope.ProtocolJT808: return topics.FieldsJT808, true case envelope.ProtocolYutongMQTT: return topics.FieldsYutongMQTT, true default: return "", false } default: return "", false } } func findingsForBridgeRawFanout(bridge parsedMetricSet, history parsedMetricSet, realtime parsedMetricSet, thresholds Thresholds) []string { if thresholds.RawFanoutMinBridge <= 0 { return nil } bridgeWrites := metricValuesByTopic(bridge.samples, "vehicle_bridge_kafka_writes_total", "ok", "vehicle.raw.go.") historyConsumers := metricValuesByTopic(history.samples, "vehicle_kafka_consumer_info", "", "vehicle.raw.go.") historyReceived := metricValuesByTopic(history.samples, "vehicle_history_kafka_messages_total", "received", "vehicle.raw.go.") historyWrites := metricValuesByTopic(history.samples, "vehicle_history_writes_total", "ok", "vehicle.raw.go.") realtimeConsumers := metricValuesByTopic(realtime.samples, "vehicle_kafka_consumer_info", "", "vehicle.raw.go.") realtimeReceived := metricValuesByTopic(realtime.samples, "vehicle_realtime_kafka_messages_total", "received", "vehicle.raw.go.") realtimeUpdates := metricValuesByTopic(realtime.samples, "vehicle_realtime_updates_total", "ok", "vehicle.raw.go.") var topics []string for topic := range bridgeWrites { topics = append(topics, topic) } sort.Strings(topics) var findings []string for _, topic := range topics { writes := bridgeWrites[topic] if writes < thresholds.RawFanoutMinBridge { continue } if historyConsumers[topic] <= 0 { findings = append(findings, fmt.Sprintf("history consumer topic missing for %s: bridge raw writes ok %.0f", topic, writes)) } else if historyReceived[topic] <= 0 { findings = append(findings, fmt.Sprintf("history messages missing for %s: bridge raw writes ok %.0f, history received %.0f", topic, writes, historyReceived[topic])) } else if historyWrites[topic] <= 0 { findings = append(findings, fmt.Sprintf("history writes missing for %s: bridge raw writes ok %.0f, history received %.0f, history writes ok %.0f", topic, writes, historyReceived[topic], historyWrites[topic])) } if realtimeConsumers[topic] <= 0 { findings = append(findings, fmt.Sprintf("realtime consumer topic missing for %s: bridge raw writes ok %.0f", topic, writes)) } else if realtimeReceived[topic] <= 0 { findings = append(findings, fmt.Sprintf("realtime messages missing for %s: bridge raw writes ok %.0f, realtime received %.0f", topic, writes, realtimeReceived[topic])) } else if realtimeUpdates[topic] <= 0 { findings = append(findings, fmt.Sprintf("realtime updates missing for %s: bridge raw writes ok %.0f, realtime received %.0f, realtime updates ok %.0f", topic, writes, realtimeReceived[topic], realtimeUpdates[topic])) } } return findings } func findingsForBridgeToStatFieldFlow(bridge parsedMetricSet, stat parsedMetricSet, thresholds Thresholds) []string { if thresholds.StatTopicMinBridge <= 0 { return nil } bridgeWrites := metricValuesByTopic(bridge.samples, "vehicle_bridge_kafka_writes_total", "ok", "vehicle.fields.go.") statReceived := metricValuesByTopic(stat.samples, "vehicle_stat_kafka_messages_total", "received", "vehicle.fields.go.") statConsumers := metricValuesByTopic(stat.samples, "vehicle_kafka_consumer_info", "", "vehicle.fields.go.") var topics []string for topic := range bridgeWrites { topics = append(topics, topic) } sort.Strings(topics) var findings []string for _, topic := range topics { writes := bridgeWrites[topic] if writes < thresholds.StatTopicMinBridge { continue } if statConsumers[topic] <= 0 { findings = append(findings, fmt.Sprintf("stat consumer topic missing for %s: bridge writes ok %.0f", topic, writes)) continue } if statReceived[topic] <= 0 { findings = append(findings, fmt.Sprintf("stat messages missing for %s: bridge writes ok %.0f, stat received %.0f", topic, writes, statReceived[topic])) } } return findings } func findingsForRequiredConsumerTopics(parsedByService map[string]parsedMetricSet, thresholds Thresholds) []string { if len(thresholds.RequiredConsumerTopics) == 0 { return nil } var services []string for service := range thresholds.RequiredConsumerTopics { services = append(services, service) } sort.Strings(services) var findings []string for _, service := range services { required := normalizeTopicSet(thresholds.RequiredConsumerTopics[service]) if len(required) == 0 { continue } seen := kafkaConsumerTopicSet(parsedByService[service]) var topics []string for topic := range required { topics = append(topics, topic) } sort.Strings(topics) for _, topic := range topics { if seen[topic] > 0 { continue } findings = append(findings, fmt.Sprintf("%s required kafka consumer topic missing: %s", service, topic)) } } return findings } func normalizeTopicSet(values []string) map[string]struct{} { out := map[string]struct{}{} for _, value := range values { value = strings.TrimSpace(value) if value == "" { continue } out[value] = struct{}{} } return out } func kafkaConsumerTopicSet(parsed parsedMetricSet) map[string]float64 { out := map[string]float64{} for _, sample := range parsed.samples { if sample.name != "vehicle_kafka_consumer_info" || sample.value <= 0 { continue } topic := strings.TrimSpace(sample.labels["topic"]) if topic == "" { continue } out[topic] += sample.value } return out } func metricValuesByTopic(samples []metricSample, metricName string, status string, topicPrefix string) map[string]float64 { out := map[string]float64{} for _, sample := range samples { if sample.name != metricName { continue } if status != "" && sample.labels["status"] != status { continue } topic := strings.TrimSpace(sample.labels["topic"]) if topic == "" || (topicPrefix != "" && !strings.HasPrefix(topic, topicPrefix)) { continue } out[topic] += sample.value } return out } func metricValuesBySubject(samples []metricSample, metricName string, status string, subjectPrefix string) map[string]float64 { out := map[string]float64{} for _, sample := range samples { if sample.name != metricName { continue } if status != "" && sample.labels["status"] != status { continue } subject := strings.TrimSpace(sample.labels["subject"]) if subject == "" || (subjectPrefix != "" && !strings.HasPrefix(subject, subjectPrefix)) { continue } out[subject] += sample.value } return out } func findingsForLastActivity(parsedByService map[string]parsedMetricSet, thresholds Thresholds, checkedAt time.Time) []string { if thresholds.LastActivityStaleSec <= 0 { return nil } specs := []lastActivitySpec{ {service: "gateway", metric: "vehicle_gateway_last_frame_unix_seconds", label: "gateway last frame", status: "OK"}, {service: "gateway", metric: "vehicle_gateway_last_publish_unix_seconds", label: "gateway last publish", status: "ok"}, {service: "bridge", metric: "vehicle_bridge_last_message_unix_seconds", label: "bridge last message", status: "received"}, {service: "bridge", metric: "vehicle_bridge_last_kafka_write_unix_seconds", label: "bridge last kafka write", status: "ok"}, {service: "bridge", metric: "vehicle_bridge_last_ack_unix_seconds", label: "bridge last ack", status: "ok"}, {service: "fast-writer", metric: "vehicle_fast_writer_last_message_unix_seconds", label: "fast writer last message", status: "received"}, {service: "fast-writer", metric: "vehicle_fast_writer_last_stage_unix_seconds", label: "fast writer last stage", status: "ok"}, {service: "history", metric: "vehicle_history_last_message_unix_seconds", label: "history last message", status: "received"}, {service: "history", metric: "vehicle_history_last_write_unix_seconds", label: "history last write", status: "ok"}, {service: "history", metric: "vehicle_history_last_commit_unix_seconds", label: "history last commit", status: "ok"}, {service: "stat", metric: "vehicle_stat_last_message_unix_seconds", label: "stat last message", status: "received"}, {service: "stat", metric: "vehicle_stat_last_write_unix_seconds", label: "stat last write", status: "ok"}, {service: "stat", metric: "vehicle_stat_last_commit_unix_seconds", label: "stat last commit", status: "ok"}, {service: "realtime", metric: "vehicle_realtime_last_message_unix_seconds", label: "realtime last message", status: "received"}, {service: "realtime", metric: "vehicle_realtime_last_update_unix_seconds", label: "realtime last update", status: "ok"}, {service: "realtime", metric: "vehicle_realtime_last_store_update_unix_seconds", label: "realtime last store update", status: "ok"}, {service: "realtime", metric: "vehicle_realtime_last_commit_unix_seconds", label: "realtime last commit", status: "ok"}, {service: "identity", metric: "vehicle_identity_writer_last_message_unix_seconds", label: "identity last message", status: "received"}, {service: "identity", metric: "vehicle_identity_writer_last_write_unix_seconds", label: "identity last write", status: "ok"}, {service: "identity", metric: "vehicle_identity_writer_last_commit_unix_seconds", label: "identity last commit", status: "ok"}, } var findings []string for _, spec := range specs { findings = append(findings, staleActivityFindings(parsedByService[spec.service], spec, thresholds.LastActivityStaleSec, checkedAt)...) } sort.Strings(findings) return findings } type lastActivitySpec struct { service string metric string label string status string } func staleActivityFindings(parsed parsedMetricSet, spec lastActivitySpec, thresholdSec float64, checkedAt time.Time) []string { var findings []string for _, sample := range parsed.samples { if sample.name != spec.metric { continue } if spec.status != "" && sample.labels["status"] != spec.status { continue } if sample.value <= 0 { continue } if isRollupLastActivity(sample) { continue } ageSec := checkedAt.Sub(time.Unix(int64(sample.value), 0).UTC()).Seconds() if ageSec <= thresholdSec { continue } findings = append(findings, fmt.Sprintf("%s stale %.0fs exceeds %.0fs (%s)", spec.label, ageSec, thresholdSec, metricLabelsKey(sample.labels))) } return findings } func isRollupLastActivity(sample metricSample) bool { for _, label := range []string{"subject", "topic"} { switch strings.TrimSpace(sample.labels[label]) { case "mixed", "unknown": return true } } return false } type statSampleCounters struct { writesOK float64 found float64 written float64 skippedMissingFields float64 skippedMissingVIN float64 skippedMissingMileage float64 skippedNonMileageFrame float64 skippedNonPositiveMileage float64 skippedMissingTime float64 futureAdjusted float64 skippedSameMileage float64 skippedMissingSource float64 } type statSourceCounters struct { found float64 attempted float64 written float64 skippedThrottled float64 skippedMissingEndpoint float64 skippedUnmanaged float64 } type statProjectionCounters struct { samplesWritten float64 attempted float64 written float64 skippedThrottled float64 } type protocolFlowStatCounters struct { samples statSampleCounters sources statSourceCounters projections statProjectionCounters } func buildProtocolFlowDiagnostics(parsedByService map[string]parsedMetricSet, thresholds Thresholds) *ProtocolFlowDiagnostics { gatewayGroups := gatewayFieldFlowGroups(parsedByService["gateway"]) identityDiagnostics := buildIdentityResolutionDiagnostics(parsedByService["gateway"]) identityByProtocol := map[string]IdentityProtocolResolution{} if identityDiagnostics != nil { for _, row := range identityDiagnostics.Protocols { identityByProtocol[row.Protocol] = row } } statGroups := protocolFlowStatGroups(parsedByService["stat"]) protocolSet := map[string]struct{}{} for protocol := range gatewayGroups { protocolSet[protocol] = struct{}{} } for protocol := range identityByProtocol { protocolSet[protocol] = struct{}{} } for protocol := range statGroups { protocolSet[protocol] = struct{}{} } if len(protocolSet) == 0 { return nil } protocols := make([]string, 0, len(protocolSet)) for protocol := range protocolSet { protocols = append(protocols, protocol) } sort.Strings(protocols) out := &ProtocolFlowDiagnostics{Protocols: make([]ProtocolFlowProtocol, 0, len(protocols))} for _, protocol := range protocols { gatewayGroup := gatewayGroups[protocol] statGroup := statGroups[protocol] identityRow := identityByProtocol[protocol] row := ProtocolFlowProtocol{Protocol: protocol} if gatewayGroup != nil { row.RawOK = gatewayGroup.rawOK row.RealtimeCandidates = gatewayRealtimeCandidates(gatewayGroup.gatewayPublishCounters) row.FieldsPublished = gatewayGroup.fieldsPublished if gatewayGroup.hasLatestCount { value := gatewayGroup.latestCount row.LatestFieldCount = &value } } row.IdentityTotal = identityRow.Total row.IdentitySkipped = identityRow.SkippedCount row.IdentityResolved = identityRow.ResolvedCount row.IdentityNonResolved = identityRow.NonResolvedCount row.IdentityNonResolvedRatio = identityRow.NonResolvedRatio if statGroup != nil { row.StatWritesOK = statGroup.samples.writesOK row.StatMileageCandidateWrites = statMileageCandidateWrites(statGroup.samples) row.StatSamplesFound = statGroup.samples.found row.StatSamplesWritten = statGroup.samples.written row.StatSamplesActionableSkipped = statActionableSkips(statGroup.samples) row.StatSkippedMissingFields = statGroup.samples.skippedMissingFields row.StatSkippedMissingVIN = statGroup.samples.skippedMissingVIN row.StatSkippedMissingMileage = statGroup.samples.skippedMissingMileage row.StatSkippedNonMileageFrame = statGroup.samples.skippedNonMileageFrame row.StatSkippedNonPositiveMileage = statGroup.samples.skippedNonPositiveMileage row.StatSkippedMissingTime = statGroup.samples.skippedMissingTime row.StatEventTimeFutureAdjusted = statGroup.samples.futureAdjusted row.StatSkippedMissingSource = statGroup.samples.skippedMissingSource row.StatSkippedSameMileage = statGroup.samples.skippedSameMileage row.StatSourceWritten = statGroup.sources.written row.StatSourceThrottled = statGroup.sources.skippedThrottled row.StatSourceMissingEndpoint = statGroup.sources.skippedMissingEndpoint row.StatSourceSkippedUnmanaged = statGroup.sources.skippedUnmanaged row.StatProjectionAttempted = statGroup.projections.attempted row.StatProjectionWritten = statGroup.projections.written row.StatProjectionThrottled = statGroup.projections.skippedThrottled } row.Status, row.Reason = protocolFlowStatus(gatewayGroup, identityRow, statGroup, thresholds) out.Protocols = append(out.Protocols, row) } return out } func protocolFlowStatGroups(parsed parsedMetricSet) map[string]*protocolFlowStatCounters { groups := map[string]*protocolFlowStatCounters{} groupForSample := func(sample metricSample) *protocolFlowStatCounters { protocol := protocolForFieldsMetricSample(sample) group := groups[protocol] if group == nil { group = &protocolFlowStatCounters{} groups[protocol] = group } return group } for _, sample := range parsed.samples { switch sample.name { case "vehicle_stat_writes_total": if sample.labels["status"] == "ok" { groupForSample(sample).samples.writesOK += sample.value } case "vehicle_stat_samples_total": group := groupForSample(sample) switch sample.labels["status"] { case "found": group.samples.found += sample.value group.sources.found += sample.value case "written": group.samples.written += sample.value group.projections.samplesWritten += sample.value case "skipped_missing_fields": group.samples.skippedMissingFields += sample.value case "skipped_missing_vin": group.samples.skippedMissingVIN += sample.value case "skipped_missing_mileage": group.samples.skippedMissingMileage += sample.value case "skipped_non_mileage_frame": group.samples.skippedNonMileageFrame += sample.value case "skipped_non_positive_mileage": group.samples.skippedNonPositiveMileage += sample.value case "skipped_missing_time": group.samples.skippedMissingTime += sample.value case "event_time_future_adjusted": group.samples.futureAdjusted += sample.value case "skipped_same_mileage": group.samples.skippedSameMileage += sample.value case "skipped_missing_source": group.samples.skippedMissingSource += sample.value } case "vehicle_stat_sources_total": group := groupForSample(sample) switch sample.labels["status"] { case "attempted": group.sources.attempted += sample.value case "written": group.sources.written += sample.value case "skipped_throttled": group.sources.skippedThrottled += sample.value case "skipped_missing_endpoint": group.sources.skippedMissingEndpoint += sample.value case "skipped_unmanaged": group.sources.skippedUnmanaged += sample.value } case "vehicle_stat_projections_total": group := groupForSample(sample) switch sample.labels["status"] { case "attempted": group.projections.attempted += sample.value case "written": group.projections.written += sample.value case "skipped_throttled": group.projections.skippedThrottled += sample.value } } } return groups } func protocolForFieldsMetricSample(sample metricSample) string { if protocol := strings.TrimSpace(sample.labels["protocol"]); protocol != "" { return protocol } if protocol, ok := topics.ProtocolForKnownFieldsTopic(sample.labels["topic"]); ok { return protocol } return "unknown" } func statActionableSkips(group statSampleCounters) float64 { return group.skippedMissingFields + group.skippedMissingVIN + group.skippedMissingMileage + group.skippedNonPositiveMileage + group.skippedMissingTime + group.skippedMissingSource } func statOnlyMissingMileageSkips(group statSampleCounters) bool { return group.skippedMissingMileage > 0 && group.skippedMissingFields == 0 && group.skippedMissingVIN == 0 && group.skippedNonPositiveMileage == 0 && group.skippedMissingTime == 0 && group.skippedMissingSource == 0 } func statMileageCandidateWrites(group statSampleCounters) float64 { candidates := group.writesOK - group.skippedNonMileageFrame if candidates < 0 { return 0 } return candidates } func protocolFlowStatus(gatewayGroup *gatewayFieldFlowGroup, identityRow IdentityProtocolResolution, statGroup *protocolFlowStatCounters, thresholds Thresholds) (string, string) { if gatewayGroup != nil { realtimeCandidates := gatewayRealtimeCandidates(gatewayGroup.gatewayPublishCounters) status, reason := gatewayFieldDiagnosticsStatus(gatewayGroup, realtimeCandidates, thresholds) if status != "ok" { return status, "gateway_fields_" + reason } } if thresholds.GatewayIdentityMinSamples > 0 && thresholds.GatewayIdentityMaxUnresolvedRatio > 0 && float64(identityRow.Total) >= thresholds.GatewayIdentityMinSamples && identityRow.NonResolvedRatio > thresholds.GatewayIdentityMaxUnresolvedRatio { return "degraded", "identity_unresolved_ratio" } if statGroup == nil { return "ok", "" } if thresholds.StatSampleMinWrites > 0 && statMileageCandidateWrites(statGroup.samples) >= thresholds.StatSampleMinWrites { if thresholds.StatSampleFutureAdjustmentMax >= 0 && statGroup.samples.futureAdjusted > thresholds.StatSampleFutureAdjustmentMax { return "degraded", "stat_future_event_time_adjusted" } if statGroup.samples.found <= 0 { if statActionableSkips(statGroup.samples) <= 0 && statGroup.samples.skippedNonMileageFrame > 0 { return "ok", "" } if statOnlyMissingMileageSkips(statGroup.samples) { return "ok", "" } return "degraded", "stat_samples_missing" } if thresholds.StatSampleMaxActionableSkipRatio > 0 { ratio := statActionableSkips(statGroup.samples) / statMileageCandidateWrites(statGroup.samples) if ratio > thresholds.StatSampleMaxActionableSkipRatio { return "degraded", "stat_actionable_skip_ratio" } } if statGroup.samples.written <= 0 && statGroup.samples.skippedMissingSource > 0 { return "degraded", "stat_samples_missing_source" } } if thresholds.StatSourceMinSamples > 0 && statGroup.sources.found >= thresholds.StatSourceMinSamples { activeSourceTouches := statGroup.sources.written + statGroup.sources.skippedThrottled + statGroup.sources.skippedUnmanaged if activeSourceTouches <= 0 && statGroup.sources.skippedMissingEndpoint <= 0 { return "degraded", "stat_source_tracking_missing" } if thresholds.StatSourceMaxMissingRatio > 0 { ratio := statGroup.sources.skippedMissingEndpoint / statGroup.sources.found if ratio > thresholds.StatSourceMaxMissingRatio { return "degraded", "stat_source_missing_endpoint_ratio" } } } if thresholds.StatProjectionMinSampleWrites > 0 && statGroup.projections.samplesWritten >= thresholds.StatProjectionMinSampleWrites && statGroup.projections.written <= 0 { return "degraded", "stat_projection_missing" } return "ok", "" } func findingsForStatProjections(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.StatProjectionMinSampleWrites <= 0 { return nil } groups := map[string]*statProjectionCounters{} for _, sample := range parsed.samples { switch sample.name { case "vehicle_stat_samples_total": if sample.labels["status"] == "written" { statProjectionGroup(groups, sample.labels["topic"]).samplesWritten += sample.value } case "vehicle_stat_projections_total": group := statProjectionGroup(groups, sample.labels["topic"]) switch sample.labels["status"] { case "attempted": group.attempted += sample.value case "written": group.written += sample.value case "skipped_throttled": group.skippedThrottled += sample.value } } } var topics []string for topic := range groups { topics = append(topics, topic) } sort.Strings(topics) var findings []string for _, topic := range topics { group := groups[topic] if group.samplesWritten < thresholds.StatProjectionMinSampleWrites { continue } if group.written > 0 { continue } findings = append(findings, fmt.Sprintf( "stat final projection missing for %s: samples written %.0f, projections attempted %.0f, written %.0f, skipped throttled %.0f", topic, group.samplesWritten, group.attempted, group.written, group.skippedThrottled, )) } return findings } func statProjectionGroup(groups map[string]*statProjectionCounters, topic string) *statProjectionCounters { topic = strings.TrimSpace(topic) if topic == "" { topic = "unknown" } group := groups[topic] if group == nil { group = &statProjectionCounters{} groups[topic] = group } return group } func findingsForStatCache(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.StatCacheEntriesMax <= 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_stat_cache_entries" || sample.value <= thresholds.StatCacheEntriesMax { continue } cache := strings.TrimSpace(sample.labels["cache"]) if cache == "" { cache = "unknown" } findings = append(findings, fmt.Sprintf( "stat cache entries %.0f exceeds %.0f for cache %s", sample.value, thresholds.StatCacheEntriesMax, cache, )) } sort.Strings(findings) return findings } type realtimeThrottleCounters struct { passed float64 skippedInterval float64 skippedNonRealtime float64 skippedMissingVIN float64 } func findingsForRealtimeThrottle(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.RealtimeThrottleMinSamples <= 0 { return nil } groups := map[string]*realtimeThrottleCounters{} for _, sample := range parsed.samples { if sample.name != "vehicle_realtime_throttle_total" { continue } group := realtimeThrottleGroup(groups, sample.labels["store"], sample.labels["protocol"]) switch sample.labels["status"] { case "passed": group.passed += sample.value case "skipped_interval": group.skippedInterval += sample.value case "skipped_non_realtime": group.skippedNonRealtime += sample.value case "skipped_missing_vin": group.skippedMissingVIN += sample.value } } var keys []string for key := range groups { keys = append(keys, key) } sort.Strings(keys) var findings []string for _, key := range keys { group := groups[key] total := group.passed + group.skippedInterval + group.skippedNonRealtime + group.skippedMissingVIN if total < thresholds.RealtimeThrottleMinSamples || group.skippedInterval <= 0 || group.passed > 0 { continue } findings = append(findings, fmt.Sprintf( "realtime throttle passing updates missing for %s: total %.0f, passed %.0f, skipped interval %.0f, skipped non-realtime %.0f, skipped missing vin %.0f", key, total, group.passed, group.skippedInterval, group.skippedNonRealtime, group.skippedMissingVIN, )) } return findings } func findingsForRealtimeThrottleCache(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.RealtimeThrottleCacheEntriesMax <= 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_realtime_throttle_cache_entries" || sample.value <= thresholds.RealtimeThrottleCacheEntriesMax { continue } store := strings.TrimSpace(sample.labels["store"]) if store == "" { store = "unknown" } findings = append(findings, fmt.Sprintf( "realtime throttle cache entries %.0f exceeds %.0f for store %s", sample.value, thresholds.RealtimeThrottleCacheEntriesMax, store, )) } sort.Strings(findings) return findings } func findingsForRealtimePlateCache(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.RealtimePlateCacheEntriesMax <= 0 { return nil } value := parsed.totals["vehicle_realtime_plate_cache_entries"] if value <= thresholds.RealtimePlateCacheEntriesMax { return nil } return []string{fmt.Sprintf( "realtime plate cache entries %.0f exceeds %.0f", value, thresholds.RealtimePlateCacheEntriesMax, )} } func realtimeThrottleGroup(groups map[string]*realtimeThrottleCounters, store string, protocol string) *realtimeThrottleCounters { store = strings.TrimSpace(store) if store == "" { store = "unknown" } protocol = strings.TrimSpace(protocol) if protocol == "" { protocol = "unknown" } key := "store=" + store + ",protocol=" + protocol group := groups[key] if group == nil { group = &realtimeThrottleCounters{} groups[key] = group } return group } type fastWriterRedisFieldCounters struct { seen float64 written float64 skippedStale float64 } type fastWriterRedisEnvelopeCounters struct { seen float64 updated float64 skippedNonRealtime float64 skippedMissingVIN float64 skippedMissingVehicleKey float64 skippedMissingFields float64 } type fastWriterMessageCounters struct { received float64 ok float64 invalidJSON float64 } func findingsForFastWriterMessageCompletion(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.FastWriterMessageMinReceived <= 0 || thresholds.FastWriterMessageMaxUnackedRatio <= 0 { return nil } groups := map[string]*fastWriterMessageCounters{} for _, sample := range parsed.samples { if sample.name != "vehicle_fast_writer_messages_total" { continue } subject := strings.TrimSpace(sample.labels["subject"]) if subject == "" || subject == "mixed" || subject == "unknown" { continue } group := groups[subject] if group == nil { group = &fastWriterMessageCounters{} groups[subject] = group } switch sample.labels["status"] { case "received": group.received += sample.value case "ok": group.ok += sample.value case "invalid_json": group.invalidJSON += sample.value } } var subjects []string for subject := range groups { subjects = append(subjects, subject) } sort.Strings(subjects) var findings []string for _, subject := range subjects { group := groups[subject] if group.received < thresholds.FastWriterMessageMinReceived { continue } completed := group.ok + group.invalidJSON unacked := group.received - completed if unacked < 0 { unacked = 0 } ratio := unacked / group.received if ratio <= thresholds.FastWriterMessageMaxUnackedRatio { continue } findings = append(findings, fmt.Sprintf( "fast writer unacked message ratio %.2f exceeds %.2f for subject %s: received %.0f, ok %.0f, invalid_json %.0f, unacked %.0f", ratio, thresholds.FastWriterMessageMaxUnackedRatio, subject, group.received, group.ok, group.invalidJSON, unacked, )) } return findings } func findingsForFastWriterRedisEnvelopes(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.FastWriterRedisEnvelopeMinSeen <= 0 { return nil } groups := map[string]*fastWriterRedisEnvelopeCounters{} for _, sample := range parsed.samples { if sample.name != "vehicle_fast_writer_redis_envelopes_total" { continue } subject := strings.TrimSpace(sample.labels["subject"]) if subject == "" || subject == "mixed" || subject == "unknown" { continue } group := groups[subject] if group == nil { group = &fastWriterRedisEnvelopeCounters{} groups[subject] = group } switch sample.labels["status"] { case "seen": group.seen += sample.value case "updated": group.updated += sample.value case "skipped_non_realtime": group.skippedNonRealtime += sample.value case "skipped_missing_vin": group.skippedMissingVIN += sample.value case "skipped_missing_vehicle_key": group.skippedMissingVehicleKey += sample.value case "skipped_missing_fields": group.skippedMissingFields += sample.value } } var subjects []string for subject := range groups { subjects = append(subjects, subject) } sort.Strings(subjects) var findings []string for _, subject := range subjects { group := groups[subject] if group.seen < thresholds.FastWriterRedisEnvelopeMinSeen { continue } if group.updated <= 0 { findings = append(findings, fmt.Sprintf( "fast writer redis realtime updates missing for subject %s: envelopes seen %.0f, updated %.0f, skipped non-realtime %.0f, missing vin %.0f, missing vehicle key %.0f, missing parsed fields %.0f", subject, group.seen, group.updated, group.skippedNonRealtime, group.skippedMissingVIN, group.skippedMissingVehicleKey, group.skippedMissingFields, )) continue } if thresholds.FastWriterRedisEnvelopeMaxBadRatio <= 0 { continue } actionableSkips := group.skippedMissingVIN + group.skippedMissingVehicleKey + group.skippedMissingFields ratio := actionableSkips / group.seen if ratio <= thresholds.FastWriterRedisEnvelopeMaxBadRatio { continue } findings = append(findings, fmt.Sprintf( "fast writer redis envelope skip ratio %.2f exceeds %.2f for subject %s: envelopes seen %.0f, updated %.0f, missing vin %.0f, missing vehicle key %.0f, missing parsed fields %.0f", ratio, thresholds.FastWriterRedisEnvelopeMaxBadRatio, subject, group.seen, group.updated, group.skippedMissingVIN, group.skippedMissingVehicleKey, group.skippedMissingFields, )) } return findings } func findingsForFastWriterRedisFields(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.FastWriterRedisFieldMinSeen <= 0 || thresholds.FastWriterRedisFieldMaxStaleRatio <= 0 { return nil } groups := map[string]*fastWriterRedisFieldCounters{} for _, sample := range parsed.samples { if sample.name != "vehicle_fast_writer_redis_fields_total" { continue } subject := strings.TrimSpace(sample.labels["subject"]) if subject == "" || subject == "mixed" || subject == "unknown" { continue } group := groups[subject] if group == nil { group = &fastWriterRedisFieldCounters{} groups[subject] = group } switch sample.labels["status"] { case "seen": group.seen += sample.value case "written": group.written += sample.value case "skipped_stale": group.skippedStale += sample.value } } var subjects []string for subject := range groups { subjects = append(subjects, subject) } sort.Strings(subjects) var findings []string for _, subject := range subjects { group := groups[subject] if group.seen < thresholds.FastWriterRedisFieldMinSeen || group.seen <= 0 { continue } ratio := group.skippedStale / group.seen if ratio <= thresholds.FastWriterRedisFieldMaxStaleRatio { continue } findings = append(findings, fmt.Sprintf( "fast writer redis stale field ratio %.2f exceeds %.2f for subject %s (seen %.0f, written %.0f, skipped_stale %.0f)", ratio, thresholds.FastWriterRedisFieldMaxStaleRatio, subject, group.seen, group.written, group.skippedStale, )) } return findings } func findingsForHistoryLocationErrors(parsed parsedMetricSet, thresholds Thresholds, checkedAt time.Time) []string { if thresholds.HistoryLocationErrorMax < 0 { return nil } lastErrorByTopic := map[string]float64{} if thresholds.HistoryLocationErrorRecentSec > 0 { for _, sample := range parsed.samples { if sample.name != "vehicle_history_last_location_write_unix_seconds" || sample.labels["status"] != "error" || sample.value <= 0 { continue } topic := strings.TrimSpace(sample.labels["topic"]) if sample.value > lastErrorByTopic[topic] { lastErrorByTopic[topic] = sample.value } } } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_history_location_writes_total" || sample.labels["status"] != "error" { continue } if sample.value <= thresholds.HistoryLocationErrorMax { continue } topic := strings.TrimSpace(sample.labels["topic"]) if topic == "" { topic = "unknown" } recency := "" if thresholds.HistoryLocationErrorRecentSec > 0 { if lastError, ok := lastErrorByTopic[topic]; ok { ageSec := checkedAt.Sub(time.Unix(int64(lastError), 0).UTC()).Seconds() if ageSec < 0 { ageSec = 0 } if ageSec > thresholds.HistoryLocationErrorRecentSec { continue } recency = fmt.Sprintf(", last error %.0fs ago", ageSec) } else { recency = ", last error activity unavailable" } } findings = append(findings, fmt.Sprintf( "history location writes error %.0f exceeds %.0f for topic %s%s", sample.value, thresholds.HistoryLocationErrorMax, topic, recency, )) } sort.Strings(findings) return findings } func findingsForHistoryLocationAccounting(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.HistoryLocationAccountingMaxRatio <= 0 { return nil } writesOKByTopic := map[string]float64{} locationStatusByTopic := map[string]float64{} for _, sample := range parsed.samples { topic := strings.TrimSpace(sample.labels["topic"]) if topic == "" || topic == "mixed" || topic == "unknown" { continue } switch { case sample.name == "vehicle_history_writes_total" && sample.labels["status"] == "ok": writesOKByTopic[topic] += sample.value case sample.name == "vehicle_history_location_writes_total": locationStatusByTopic[topic] += sample.value } } var topics []string for topic := range writesOKByTopic { topics = append(topics, topic) } sort.Strings(topics) var findings []string for _, topic := range topics { writesOK := writesOKByTopic[topic] if writesOK < thresholds.KafkaConsumerMessageMinReceived { continue } accounted := locationStatusByTopic[topic] diff := math.Abs(writesOK - accounted) ratio := diff / writesOK if ratio <= thresholds.HistoryLocationAccountingMaxRatio { continue } findings = append(findings, fmt.Sprintf( "history location accounting mismatch ratio %.2f exceeds %.2f for topic %s: history writes ok %.0f, location statuses %.0f, diff %.0f", ratio, thresholds.HistoryLocationAccountingMaxRatio, topic, writesOK, accounted, diff, )) } return findings } func findingsForHistoryFallbackErrors(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.HistoryFallbackErrorMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_history_batch_fallback_total" || sample.labels["status"] != "error" { continue } if sample.value <= thresholds.HistoryFallbackErrorMax { continue } findings = append(findings, fmt.Sprintf( "history batch fallback errors %.0f exceeds %.0f", sample.value, thresholds.HistoryFallbackErrorMax, )) } sort.Strings(findings) return findings } func findingsForFastWriterFallbackErrors(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.FastWriterFallbackErrorMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_fast_writer_batch_fallback_total" || sample.labels["status"] != "error" { continue } if sample.value <= thresholds.FastWriterFallbackErrorMax { continue } stage := strings.TrimSpace(sample.labels["stage"]) if stage == "" { stage = "unknown" } findings = append(findings, fmt.Sprintf( "fast writer batch fallback errors %.0f exceeds %.0f for stage %s", sample.value, thresholds.FastWriterFallbackErrorMax, stage, )) } sort.Strings(findings) return findings } func findingsForFastWriterDecoupledUpdateErrors(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.FastWriterDecoupledUpdateErrorMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_fast_writer_decoupled_updates_total" || sample.labels["status"] != "error" { continue } if sample.value <= thresholds.FastWriterDecoupledUpdateErrorMax { continue } reason := strings.TrimSpace(sample.labels["reason"]) if reason == "" { reason = "unknown" } findings = append(findings, fmt.Sprintf( "fast writer decoupled update errors %.0f exceeds %.0f for reason %s", sample.value, thresholds.FastWriterDecoupledUpdateErrorMax, reason, )) } sort.Strings(findings) return findings } func findingsForWriteRetries(parsed parsedMetricSet, thresholds Thresholds) []string { var findings []string findings = append(findings, writeRetryFindings(parsed, "vehicle_history_write_retries_total", "history", thresholds.HistoryRetryMax, thresholds.HistoryRetryExhaustedMax)...) findings = append(findings, writeRetryFindings(parsed, "vehicle_stat_write_retries_total", "stat", thresholds.StatRetryMax, thresholds.StatRetryExhaustedMax)...) sort.Strings(findings) return findings } func writeRetryFindings(parsed parsedMetricSet, metricName string, serviceLabel string, retryMax float64, exhaustedMax float64) []string { var findings []string for _, sample := range parsed.samples { if sample.name != metricName { continue } status := strings.TrimSpace(sample.labels["status"]) operation := strings.TrimSpace(sample.labels["operation"]) if operation == "" { operation = "unknown" } switch status { case "retry": if retryMax < 0 || sample.value <= retryMax { continue } findings = append(findings, fmt.Sprintf( "%s write retries %.0f exceeds %.0f for operation %s", serviceLabel, sample.value, retryMax, operation, )) case "exhausted": if exhaustedMax < 0 || sample.value <= exhaustedMax { continue } findings = append(findings, fmt.Sprintf( "%s write retry exhausted %.0f exceeds %.0f for operation %s", serviceLabel, sample.value, exhaustedMax, operation, )) } } return findings } func findingsForStatInvalidJSON(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.StatInvalidJSONMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_stat_kafka_messages_total" || sample.labels["status"] != "invalid_json" { continue } if sample.value <= thresholds.StatInvalidJSONMax { continue } topic := strings.TrimSpace(sample.labels["topic"]) if topic == "" { topic = "unknown" } findings = append(findings, fmt.Sprintf( "stat invalid json %.0f exceeds %.0f for topic %s", sample.value, thresholds.StatInvalidJSONMax, topic, )) } sort.Strings(findings) return findings } func findingsForStatProtocolTopicMismatches(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.StatProtocolTopicMismatchMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_stat_kafka_messages_total" || sample.labels["status"] != "protocol_topic_mismatch" { continue } if sample.value <= thresholds.StatProtocolTopicMismatchMax { continue } topic := strings.TrimSpace(sample.labels["topic"]) if topic == "" { topic = "unknown" } findings = append(findings, fmt.Sprintf( "stat protocol topic mismatches %.0f exceeds %.0f for topic %s", sample.value, thresholds.StatProtocolTopicMismatchMax, topic, )) } findings = append(findings, findingsForStatMetricProtocolTopicMismatches(parsed, thresholds)...) sort.Strings(findings) return findings } func findingsForStatMetricProtocolTopicMismatches(parsed parsedMetricSet, thresholds Thresholds) []string { var findings []string for _, sample := range parsed.samples { switch sample.name { case "vehicle_stat_samples_total", "vehicle_stat_sources_total", "vehicle_stat_projections_total": default: continue } topic := strings.TrimSpace(sample.labels["topic"]) expectedProtocol, ok := topics.ProtocolForKnownFieldsTopic(topic) if !ok { continue } protocol := strings.TrimSpace(sample.labels["protocol"]) if sample.value <= thresholds.StatProtocolTopicMismatchMax { continue } status := strings.TrimSpace(sample.labels["status"]) if status == "" { status = "unknown" } if protocol == "" { findings = append(findings, fmt.Sprintf( "stat metric protocol label missing %.0f exceeds %.0f for %s topic %s (status %s)", sample.value, thresholds.StatProtocolTopicMismatchMax, sample.name, topic, status, )) continue } if protocol == expectedProtocol { continue } findings = append(findings, fmt.Sprintf( "stat metric protocol topic mismatches %.0f exceeds %.0f for %s topic %s: expected %s, got %s (status %s)", sample.value, thresholds.StatProtocolTopicMismatchMax, sample.name, topic, expectedProtocol, protocol, status, )) } return findings } func findingsForFastWriterInvalidJSON(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.FastWriterInvalidJSONMax < 0 { return nil } var findings []string for _, sample := range parsed.samples { if sample.name != "vehicle_fast_writer_messages_total" || sample.labels["status"] != "invalid_json" { continue } if sample.value <= thresholds.FastWriterInvalidJSONMax { continue } subject := strings.TrimSpace(sample.labels["subject"]) if subject == "" { subject = "unknown" } findings = append(findings, fmt.Sprintf( "fast writer invalid json %.0f exceeds %.0f for subject %s", sample.value, thresholds.FastWriterInvalidJSONMax, subject, )) } sort.Strings(findings) return findings } func findingsForStatSamples(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.StatSampleMinWrites <= 0 { return nil } groups := map[string]*statSampleCounters{} for _, sample := range parsed.samples { switch sample.name { case "vehicle_stat_writes_total": if sample.labels["status"] != "ok" { continue } statSampleGroup(groups, sample.labels["topic"]).writesOK += sample.value case "vehicle_stat_samples_total": group := statSampleGroup(groups, sample.labels["topic"]) switch sample.labels["status"] { case "found": group.found += sample.value case "written": group.written += sample.value case "skipped_missing_fields": group.skippedMissingFields += sample.value case "skipped_missing_vin": group.skippedMissingVIN += sample.value case "skipped_missing_mileage": group.skippedMissingMileage += sample.value case "skipped_non_mileage_frame": group.skippedNonMileageFrame += sample.value case "skipped_non_positive_mileage": group.skippedNonPositiveMileage += sample.value case "skipped_missing_time": group.skippedMissingTime += sample.value case "event_time_future_adjusted": group.futureAdjusted += sample.value case "skipped_same_mileage": group.skippedSameMileage += sample.value case "skipped_missing_source": group.skippedMissingSource += sample.value } } } var topics []string for topic := range groups { topics = append(topics, topic) } sort.Strings(topics) var findings []string for _, topic := range topics { group := groups[topic] mileageCandidates := statMileageCandidateWrites(*group) if mileageCandidates < thresholds.StatSampleMinWrites { continue } if thresholds.StatSampleFutureAdjustmentMax >= 0 && group.futureAdjusted > thresholds.StatSampleFutureAdjustmentMax { findings = append(findings, fmt.Sprintf( "stat future event time adjustments %.0f exceeds %.0f for %s: writes ok %.0f, mileage candidates %.0f, found %.0f, written %.0f", group.futureAdjusted, thresholds.StatSampleFutureAdjustmentMax, topic, group.writesOK, mileageCandidates, group.found, group.written, )) } actionableSkips := statActionableSkips(*group) if group.found <= 0 { if actionableSkips <= 0 && group.skippedNonMileageFrame > 0 { continue } if statOnlyMissingMileageSkips(*group) { continue } findings = append(findings, fmt.Sprintf( "stat samples missing for %s: writes ok %.0f, mileage candidates %.0f, missing fields %.0f, skipped missing vin %.0f, missing mileage %.0f, non-mileage frames %.0f, non-positive mileage %.0f, missing time %.0f, missing source %.0f", topic, group.writesOK, mileageCandidates, group.skippedMissingFields, group.skippedMissingVIN, group.skippedMissingMileage, group.skippedNonMileageFrame, group.skippedNonPositiveMileage, group.skippedMissingTime, group.skippedMissingSource, )) continue } if thresholds.StatSampleMaxActionableSkipRatio > 0 { ratio := actionableSkips / mileageCandidates if ratio > thresholds.StatSampleMaxActionableSkipRatio { findings = append(findings, fmt.Sprintf( "stat actionable skip ratio %.2f exceeds %.2f for %s: writes ok %.0f, mileage candidates %.0f, found %.0f, written %.0f, missing fields %.0f, skipped missing vin %.0f, missing mileage %.0f, non-mileage frames %.0f, non-positive mileage %.0f, missing time %.0f, missing source %.0f, same mileage %.0f", ratio, thresholds.StatSampleMaxActionableSkipRatio, topic, group.writesOK, mileageCandidates, group.found, group.written, group.skippedMissingFields, group.skippedMissingVIN, group.skippedMissingMileage, group.skippedNonMileageFrame, group.skippedNonPositiveMileage, group.skippedMissingTime, group.skippedMissingSource, group.skippedSameMileage, )) } } if group.written <= 0 && group.skippedMissingSource > 0 { findings = append(findings, fmt.Sprintf( "stat samples not written for %s: found %.0f, skipped missing source %.0f, same mileage %.0f", topic, group.found, group.skippedMissingSource, group.skippedSameMileage, )) } } return findings } func findingsForStatSources(parsed parsedMetricSet, thresholds Thresholds) []string { if thresholds.StatSourceMinSamples <= 0 { return nil } groups := map[string]*statSourceCounters{} for _, sample := range parsed.samples { switch sample.name { case "vehicle_stat_samples_total": if sample.labels["status"] == "found" { statSourceGroup(groups, sample.labels["topic"]).found += sample.value } case "vehicle_stat_sources_total": group := statSourceGroup(groups, sample.labels["topic"]) switch sample.labels["status"] { case "attempted": group.attempted += sample.value case "written": group.written += sample.value case "skipped_throttled": group.skippedThrottled += sample.value case "skipped_missing_endpoint": group.skippedMissingEndpoint += sample.value case "skipped_unmanaged": group.skippedUnmanaged += sample.value } } } var topics []string for topic := range groups { topics = append(topics, topic) } sort.Strings(topics) var findings []string for _, topic := range topics { group := groups[topic] if group.found < thresholds.StatSourceMinSamples || group.found <= 0 { continue } activeSourceTouches := group.written + group.skippedThrottled + group.skippedUnmanaged if activeSourceTouches <= 0 && group.skippedMissingEndpoint <= 0 { findings = append(findings, fmt.Sprintf( "stat source tracking missing for %s: samples found %.0f, source written %.0f, source throttled %.0f, source unmanaged %.0f, missing endpoint %.0f", topic, group.found, group.written, group.skippedThrottled, group.skippedUnmanaged, group.skippedMissingEndpoint, )) continue } if thresholds.StatSourceMaxMissingRatio <= 0 { continue } ratio := group.skippedMissingEndpoint / group.found if ratio > thresholds.StatSourceMaxMissingRatio { findings = append(findings, fmt.Sprintf( "stat source missing endpoint ratio %.2f exceeds %.2f for %s: samples found %.0f, source written %.0f, source throttled %.0f, source unmanaged %.0f, missing endpoint %.0f", ratio, thresholds.StatSourceMaxMissingRatio, topic, group.found, group.written, group.skippedThrottled, group.skippedUnmanaged, group.skippedMissingEndpoint, )) } } return findings } func statSampleGroup(groups map[string]*statSampleCounters, topic string) *statSampleCounters { topic = strings.TrimSpace(topic) if topic == "" { topic = "unknown" } group := groups[topic] if group == nil { group = &statSampleCounters{} groups[topic] = group } return group } func statSourceGroup(groups map[string]*statSourceCounters, topic string) *statSourceCounters { topic = strings.TrimSpace(topic) if topic == "" { topic = "unknown" } group := groups[topic] if group == nil { group = &statSourceCounters{} groups[topic] = group } return group } func histogramP99Findings(parsed parsedMetricSet, histogramName string, label string, thresholdMS float64, minSamples float64) []string { if thresholdMS <= 0 { return nil } groups := histogramGroups(parsed.samples, histogramName) var findings []string var keys []string for key := range groups { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { estimate, count, ok := histogramQuantile(groups[key], 0.99) if !ok || count < minSamples || estimate <= thresholdMS { continue } findings = append(findings, fmt.Sprintf("%s p99 %.0fms exceeds %.0fms (%s, samples %.0f)", label, estimate, thresholdMS, key, count)) } return findings } func recentP99GaugeFindings(parsed parsedMetricSet, metricName string, samplesMetricName string, label string, thresholdMS float64, minSamples float64) []string { if thresholdMS <= 0 { return nil } samplesByKey := map[string]float64{} for _, sample := range parsed.samples { if sample.name != samplesMetricName { continue } samplesByKey[metricLabelsKey(sample.labels)] = sample.value } var findings []string for _, sample := range parsed.samples { if sample.name != metricName { continue } key := metricLabelsKey(sample.labels) samples := samplesByKey[key] if samples < minSamples || sample.value <= thresholdMS { continue } findings = append(findings, fmt.Sprintf("%s recent p99 %.0fms exceeds %.0fms (%s, samples %.0f)", label, sample.value, thresholdMS, key, samples)) } sort.Strings(findings) return findings } func parseMetrics(text string) map[string]float64 { return parseMetricSet(text).totals } type parsedMetricSet struct { totals map[string]float64 samples []metricSample } type metricSample struct { name string labels map[string]string value float64 } func parseMetricSet(text string) parsedMetricSet { out := map[string]float64{} var samples []metricSample scanner := bufio.NewScanner(strings.NewReader(text)) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, "#") { continue } sample, ok := parseMetricSample(line) if !ok { continue } if aggregateMetricByMax(sample.name) { if current, ok := out[sample.name]; !ok || sample.value > current { out[sample.name] = sample.value } } else { out[sample.name] += sample.value } samples = append(samples, sample) } return parsedMetricSet{totals: out, samples: samples} } func aggregateMetricByMax(name string) bool { return isLastActivityUnixMetric(name) || strings.HasSuffix(name, "_p99_ms") } func isLastActivityUnixMetric(name string) bool { return strings.Contains(name, "_last_") && strings.HasSuffix(name, "_unix_seconds") } func addLastActivityAges(metrics map[string]float64, checkedAt time.Time) { if len(metrics) == 0 { return } for name, value := range metrics { if !isLastActivityUnixMetric(name) || value <= 0 { continue } age := checkedAt.Sub(time.Unix(int64(value), 0).UTC()).Seconds() if age < 0 { age = 0 } metrics[strings.TrimSuffix(name, "_unix_seconds")+"_age_seconds"] = age } } func parseMetricLine(line string) (string, float64, bool) { sample, ok := parseMetricSample(line) return sample.name, sample.value, ok } func parseMetricSample(line string) (metricSample, bool) { parts := strings.Fields(line) if len(parts) < 2 { return metricSample{}, false } name, labels := parseMetricNameAndLabels(parts[0]) value, err := strconv.ParseFloat(parts[1], 64) if err != nil { return metricSample{}, false } return metricSample{name: name, labels: labels, value: value}, true } func parseMetricNameAndLabels(value string) (string, map[string]string) { index := strings.IndexByte(value, '{') if index < 0 { return value, nil } end := strings.LastIndexByte(value, '}') if end < index { return value[:index], nil } return value[:index], parseMetricLabels(value[index+1 : end]) } func parseMetricLabels(value string) map[string]string { labels := map[string]string{} for len(value) > 0 { value = strings.TrimLeft(value, " \t,") if value == "" { break } index := strings.IndexByte(value, '=') if index <= 0 { break } key := strings.TrimSpace(value[:index]) value = strings.TrimLeft(value[index+1:], " \t") if key == "" || value == "" { break } if value[0] != '"' { next := strings.IndexByte(value, ',') if next < 0 { labels[key] = strings.TrimSpace(value) break } labels[key] = strings.TrimSpace(value[:next]) value = value[next+1:] continue } labelValue, rest, ok := readQuotedLabelValue(value) if !ok { break } labels[key] = labelValue value = rest } if len(labels) == 0 { return nil } return labels } func readQuotedLabelValue(value string) (string, string, bool) { if value == "" || value[0] != '"' { return "", value, false } var b strings.Builder escaped := false for index := 1; index < len(value); index++ { ch := value[index] if escaped { switch ch { case 'n': b.WriteByte('\n') default: b.WriteByte(ch) } escaped = false continue } if ch == '\\' { escaped = true continue } if ch == '"' { return b.String(), value[index+1:], true } b.WriteByte(ch) } return "", value, false } type histogramBucket struct { upperBound float64 count float64 } func histogramGroups(samples []metricSample, histogramName string) map[string][]histogramBucket { groups := map[string][]histogramBucket{} for _, sample := range samples { if sample.name != histogramName+"_bucket" { continue } upperBound, ok := parseBucketUpperBound(sample.labels["le"]) if !ok { continue } key := metricLabelsKey(sample.labels, "le") groups[key] = append(groups[key], histogramBucket{upperBound: upperBound, count: sample.value}) } return groups } func parseBucketUpperBound(value string) (float64, bool) { if value == "+Inf" { return math.Inf(1), true } parsed, err := strconv.ParseFloat(value, 64) return parsed, err == nil } func histogramQuantile(buckets []histogramBucket, quantile float64) (float64, float64, bool) { if len(buckets) == 0 || quantile <= 0 || quantile > 1 { return 0, 0, false } sort.Slice(buckets, func(i, j int) bool { return buckets[i].upperBound < buckets[j].upperBound }) total := buckets[len(buckets)-1].count if total <= 0 { return 0, 0, false } target := total * quantile for _, bucket := range buckets { if bucket.count >= target { return bucket.upperBound, total, true } } return buckets[len(buckets)-1].upperBound, total, true } func metricLabelsKey(labels map[string]string, exclude ...string) string { if len(labels) == 0 { return "all" } excluded := map[string]struct{}{} for _, key := range exclude { excluded[key] = struct{}{} } keys := make([]string, 0, len(labels)) for key := range labels { if _, skip := excluded[key]; skip { continue } keys = append(keys, key) } if len(keys) == 0 { return "all" } sort.Strings(keys) parts := make([]string, 0, len(keys)) for _, key := range keys { parts = append(parts, key+"="+labels[key]) } return strings.Join(parts, ",") }