shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ops): recover stale history error health

Browse Source
This commit is contained in:
lingniu
2026-07-16 06:48:47 +08:00
parent e99087f329
commit ec3071d59b
5 changed files with 72 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
go/vehicle-gateway/internal/capacity/check.go
View File

@@ -563,6 +563,7 @@ type Thresholds struct {
HistoryWorkersMin float64
HistoryInvalidJSONMax float64
HistoryLocationErrorMax float64
HistoryLocationErrorRecentSec float64
HistoryLocationAccountingMaxRatio float64
HistoryFallbackErrorMax float64
HistoryRetryMax float64
@@ -690,6 +691,7 @@ func DefaultThresholds() Thresholds {
HistoryWorkersMin: 3,
HistoryInvalidJSONMax: 0,
HistoryLocationErrorMax: 0,
HistoryLocationErrorRecentSec: 300,
HistoryLocationAccountingMaxRatio: 0.02,
HistoryFallbackErrorMax: 0,
HistoryRetryMax: 100,
@@ -870,7 +872,7 @@ func EvaluateWithThresholds(metricsByService map[string]string, thresholds Thres
report.Findings = append(report.Findings, findingsForBridgeMessageCompletion(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForFieldsProjector(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistoryInvalidJSON(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistoryLocationErrors(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistoryLocationErrors(parsed, thresholds, report.CheckedAt)...)
report.Findings = append(report.Findings, findingsForHistoryLocationAccounting(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistoryFallbackErrors(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForWriteRetries(parsed, thresholds)...)
@@ -4079,10 +4081,22 @@ func findingsForFastWriterRedisFields(parsed parsedMetricSet, thresholds Thresho
return findings
}
func findingsForHistoryLocationErrors(parsed parsedMetricSet, thresholds Thresholds) []string {
func findingsForHistoryLocationErrors(parsed parsedMetricSet, thresholds Thresholds, checkedAt time.Time) []string {
if thresholds.HistoryLocationErrorMax < 0 {
return nil
}
lastErrorByTopic := map[string]float64{}
if thresholds.HistoryLocationErrorRecentSec > 0 {
for _, sample := range parsed.samples {
if sample.name != "vehicle_history_last_location_write_unix_seconds" || sample.labels["status"] != "error" || sample.value <= 0 {
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if sample.value > lastErrorByTopic[topic] {
lastErrorByTopic[topic] = sample.value
}
}
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_history_location_writes_total" || sample.labels["status"] != "error" {
@@ -4095,11 +4109,27 @@ func findingsForHistoryLocationErrors(parsed parsedMetricSet, thresholds Thresho
if topic == "" {
topic = "unknown"
}
recency := ""
if thresholds.HistoryLocationErrorRecentSec > 0 {
if lastError, ok := lastErrorByTopic[topic]; ok {
ageSec := checkedAt.Sub(time.Unix(int64(lastError), 0).UTC()).Seconds()
if ageSec < 0 {
ageSec = 0
}
if ageSec > thresholds.HistoryLocationErrorRecentSec {
continue
}
recency = fmt.Sprintf(", last error %.0fs ago", ageSec)
} else {
recency = ", last error activity unavailable"
}
}
findings = append(findings, fmt.Sprintf(
"history location writes error %.0f exceeds %.0f for topic %s",
"history location writes error %.0f exceeds %.0f for topic %s%s",
sample.value,
thresholds.HistoryLocationErrorMax,
topic,
recency,
))
}
sort.Strings(findings)