shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ops): recover stale history error health

Browse Source
This commit is contained in:
lingniu
2026-07-16 06:48:47 +08:00
parent e99087f329
commit ec3071d59b
5 changed files with 72 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
go/vehicle-gateway/internal/capacity/check_test.go
View File

@@ -1173,12 +1173,14 @@ vehicle_history_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go
}
func TestEvaluateReportsHistoryLocationWriteErrors(t *testing.T) {
recentError := time.Now().Add(-time.Minute).Unix()
report := EvaluateWithThresholds(map[string]string{
"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 2`,
}, Thresholds{HistoryLocationErrorMax: 0})
vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 2
vehicle_history_last_location_write_unix_seconds{status="error",topic="vehicle.raw.go.jt808.v1"} ` + strconv.FormatInt(recentError, 10),
}, Thresholds{HistoryLocationErrorMax: 0, HistoryLocationErrorRecentSec: 300})
if report.Status != StatusDegraded {
t.Fatalf("status = %s, want degraded: %#v", report.Status, report)
@@ -1187,6 +1189,7 @@ vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808
for _, want := range []string{
"history location writes error 2 exceeds 0",
"vehicle.raw.go.jt808.v1",
"last error",
} {
if !strings.Contains(joined, want) {
t.Fatalf("finding missing %q in:\n%s", want, joined)
@@ -1194,6 +1197,37 @@ vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808
}
}
func TestEvaluateIgnoresRecoveredHistoryLocationWriteErrors(t *testing.T) {
staleError := time.Now().Add(-10 * time.Minute).Unix()
report := EvaluateWithThresholds(map[string]string{
"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 2
vehicle_history_last_location_write_unix_seconds{status="error",topic="vehicle.raw.go.jt808.v1"} ` + strconv.FormatInt(staleError, 10),
}, Thresholds{HistoryLocationErrorMax: 0, HistoryLocationErrorRecentSec: 300})
if report.Status != StatusOK {
t.Fatalf("status = %s, want ok after the error window recovered: %#v", report.Status, report)
}
if len(report.Findings) != 0 {
t.Fatalf("findings = %#v, want none after the error window recovered", report.Findings)
}
}
func TestEvaluateKeepsHistoryLocationErrorWhenLastActivityIsUnavailable(t *testing.T) {
report := EvaluateWithThresholds(map[string]string{
"history": `vehicle_service_info{service="vehicle-history-writer"} 1
vehicle_history_config{setting="workers"} 3
vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1
vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 2`,
}, Thresholds{HistoryLocationErrorMax: 0, HistoryLocationErrorRecentSec: 300})
if report.Status != StatusDegraded || !strings.Contains(strings.Join(report.Findings, "\n"), "last error activity unavailable") {
t.Fatalf("missing timestamp must remain degraded, got %#v", report)
}
}
func TestEvaluateCanDisableHistoryLocationWriteErrorLimit(t *testing.T) {
report := EvaluateWithThresholds(map[string]string{
"history": `vehicle_service_info{service="vehicle-history-writer"} 1