shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ops): recover stale history error health

Browse Source
This commit is contained in:
lingniu
2026-07-16 06:48:47 +08:00
parent e99087f329
commit ec3071d59b
5 changed files with 72 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
go/vehicle-gateway/cmd/capacity-check/main.go
View File

@@ -121,6 +121,7 @@ func main() {
flag.Float64Var(&thresholds.HistoryWriteRecentE2EP99MS, "history-write-recent-e2e-p99-ms", thresholds.HistoryWriteRecentE2EP99MS, "degrade when recent gateway received_at to history TDengine write p99 exceeds this many milliseconds; <=0 disables")
flag.Float64Var(&thresholds.HistoryWorkersMin, "history-workers-min", thresholds.HistoryWorkersMin, "degrade when history-writer runtime worker count is below this value; <=0 disables")
flag.Float64Var(&thresholds.HistoryLocationErrorMax, "history-location-error-max", thresholds.HistoryLocationErrorMax, "degrade when history-writer location derived write errors exceed this value; <0 disables")
flag.Float64Var(&thresholds.HistoryLocationErrorRecentSec, "history-location-error-recent-seconds", thresholds.HistoryLocationErrorRecentSec, "only degrade on history location write errors observed within this many seconds; <=0 evaluates the lifetime counter")
flag.Float64Var(&thresholds.HistoryLocationAccountingMaxRatio, "history-location-accounting-max-mismatch-ratio", thresholds.HistoryLocationAccountingMaxRatio, "degrade when location derived status counters do not account for successful history raw writes by more than this ratio; <=0 disables")
flag.Float64Var(&thresholds.HistoryFallbackErrorMax, "history-fallback-error-max", thresholds.HistoryFallbackErrorMax, "degrade when history-writer batch fallback single writes still fail; <0 disables")
flag.Float64Var(&thresholds.HistoryRetryMax, "history-retry-max", thresholds.HistoryRetryMax, "degrade when history-writer transient write retries exceed this value; <0 disables")

36
go/vehicle-gateway/internal/capacity/check.go
View File

@@ -563,6 +563,7 @@ type Thresholds struct {
HistoryWorkersMin float64
HistoryInvalidJSONMax float64
HistoryLocationErrorMax float64
HistoryLocationErrorRecentSec float64
HistoryLocationAccountingMaxRatio float64
HistoryFallbackErrorMax float64
HistoryRetryMax float64
@@ -690,6 +691,7 @@ func DefaultThresholds() Thresholds {
HistoryWorkersMin: 3,
HistoryInvalidJSONMax: 0,
HistoryLocationErrorMax: 0,
HistoryLocationErrorRecentSec: 300,
HistoryLocationAccountingMaxRatio: 0.02,
HistoryFallbackErrorMax: 0,
HistoryRetryMax: 100,
@@ -870,7 +872,7 @@ func EvaluateWithThresholds(metricsByService map[string]string, thresholds Thres
report.Findings = append(report.Findings, findingsForBridgeMessageCompletion(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForFieldsProjector(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistoryInvalidJSON(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistoryLocationErrors(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistoryLocationErrors(parsed, thresholds, report.CheckedAt)...)
report.Findings = append(report.Findings, findingsForHistoryLocationAccounting(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForHistoryFallbackErrors(parsed, thresholds)...)
report.Findings = append(report.Findings, findingsForWriteRetries(parsed, thresholds)...)
@@ -4079,10 +4081,22 @@ func findingsForFastWriterRedisFields(parsed parsedMetricSet, thresholds Thresho
return findings
}
func findingsForHistoryLocationErrors(parsed parsedMetricSet, thresholds Thresholds) []string {
func findingsForHistoryLocationErrors(parsed parsedMetricSet, thresholds Thresholds, checkedAt time.Time) []string {
if thresholds.HistoryLocationErrorMax < 0 {
return nil
}
lastErrorByTopic := map[string]float64{}
if thresholds.HistoryLocationErrorRecentSec > 0 {
for _, sample := range parsed.samples {
if sample.name != "vehicle_history_last_location_write_unix_seconds" || sample.labels["status"] != "error" || sample.value <= 0 {
continue
}
topic := strings.TrimSpace(sample.labels["topic"])
if sample.value > lastErrorByTopic[topic] {
lastErrorByTopic[topic] = sample.value
}
}
}
var findings []string
for _, sample := range parsed.samples {
if sample.name != "vehicle_history_location_writes_total" || sample.labels["status"] != "error" {
@@ -4095,11 +4109,27 @@ func findingsForHistoryLocationErrors(parsed parsedMetricSet, thresholds Thresho
if topic == "" {
topic = "unknown"
}
recency := ""
if thresholds.HistoryLocationErrorRecentSec > 0 {
if lastError, ok := lastErrorByTopic[topic]; ok {
ageSec := checkedAt.Sub(time.Unix(int64(lastError), 0).UTC()).Seconds()
if ageSec < 0 {
ageSec = 0
}
if ageSec > thresholds.HistoryLocationErrorRecentSec {
continue
}
recency = fmt.Sprintf(", last error %.0fs ago", ageSec)
} else {
recency = ", last error activity unavailable"
}
}
findings = append(findings, fmt.Sprintf(
"history location writes error %.0f exceeds %.0f for topic %s",
"history location writes error %.0f exceeds %.0f for topic %s%s",
sample.value,
thresholds.HistoryLocationErrorMax,
topic,
recency,
))
}
sort.Strings(findings)

38
go/vehicle-gateway/internal/capacity/check_test.go
View File

@@ -1173,12 +1173,14 @@ vehicle_history_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go
}
func TestEvaluateReportsHistoryLocationWriteErrors(t *testing.T) {
recentError := time.Now().Add(-time.Minute).Unix()
report := EvaluateWithThresholds(map[string]string{
"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 2`,
}, Thresholds{HistoryLocationErrorMax: 0})
vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 2
vehicle_history_last_location_write_unix_seconds{status="error",topic="vehicle.raw.go.jt808.v1"} ` + strconv.FormatInt(recentError, 10),
}, Thresholds{HistoryLocationErrorMax: 0, HistoryLocationErrorRecentSec: 300})
if report.Status != StatusDegraded {
t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
@@ -1187,6 +1189,7 @@ vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808
for _, want := range []string{
"history location writes error 2 exceeds 0",
"vehicle.raw.go.jt808.v1",
"last error",
} {
if !strings.Contains(joined, want) {
t.Fatalf("finding missing %q in:\n%s", want, joined)
@@ -1194,6 +1197,37 @@ vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808
}
}
func TestEvaluateIgnoresRecoveredHistoryLocationWriteErrors(t *testing.T) {
staleError := time.Now().Add(-10 * time.Minute).Unix()
report := EvaluateWithThresholds(map[string]string{
"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 2
vehicle_history_last_location_write_unix_seconds{status="error",topic="vehicle.raw.go.jt808.v1"} ` + strconv.FormatInt(staleError, 10),
}, Thresholds{HistoryLocationErrorMax: 0, HistoryLocationErrorRecentSec: 300})
if report.Status != StatusOK {
t.Fatalf("status = %s, want ok after the error window recovered: %#v", report.Status, report)
}
if len(report.Findings) != 0 {
t.Fatalf("findings = %#v, want none after the error window recovered", report.Findings)
}
}
func TestEvaluateKeepsHistoryLocationErrorWhenLastActivityIsUnavailable(t *testing.T) {
report := EvaluateWithThresholds(map[string]string{
"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 2`,
}, Thresholds{HistoryLocationErrorMax: 0, HistoryLocationErrorRecentSec: 300})
if report.Status != StatusDegraded || !strings.Contains(strings.Join(report.Findings, "\n"), "last error activity unavailable") {
t.Fatalf("missing timestamp must remain degraded, got %#v", report)
}
}
func TestEvaluateCanDisableHistoryLocationWriteErrorLimit(t *testing.T) {
report := EvaluateWithThresholds(map[string]string{
"history": `vehicle_service_info{service="vehicle-history-writer"} 1