fix(stats): recover stale odometer mileage
This commit is contained in:
32
docs/ops/2026-07-20-daily-mileage-track-reconciliation.md
Normal file
32
docs/ops/2026-07-20-daily-mileage-track-reconciliation.md
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
# 2026-07-20 日里程与轨迹里程核对
|
||||||
|
|
||||||
|
## 事件范围
|
||||||
|
|
||||||
|
用户在跨自然日后反馈沪A01559F、沪A62013F“今日里程未计算但轨迹回放有里程”。核对按 `Asia/Shanghai` 自然日执行;由于反馈发生在 2026-07-20 00:00 之后,而车辆最后活动发生在前一日,本次事实窗口固定为 2026-07-19 00:00:00 至 2026-07-20 00:00:00。
|
||||||
|
|
||||||
|
## 生产事实
|
||||||
|
|
||||||
|
| 车牌 | VIN | 轨迹事实 | 日里程结果 | 结论 |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| 沪A01559F | `LMRKH9AC7R1004120` | 主来源 JT808;2,793 个有效点;协议总里程差 769.1 km | 宇通 809 km、JT808 769.1 km;宇通按来源优先级进入业务统计 | 当日曾存在统计恢复延迟,结果已恢复 |
|
||||||
|
| 沪A62013F | `LMRKH9AC9R1004099` | 宇通 520 个源点、519 个有效点;最高速度 0;轨迹汇总 0 km | 宇通 0 km;累计总里程 24,245 km | 0 km 正确;轨迹点展示的是累计总里程,不得补成正向日里程 |
|
||||||
|
|
||||||
|
沪A01559F 的 2026-07-19 最终日里程在 2026-07-20 00:32 恢复。沪A62013F 的宇通总里程事件时间仍为 2026-07-18 16:48,2026-07-19 只有静止位置证据,因此只能保留显式 0 km。
|
||||||
|
|
||||||
|
## 根因与修复
|
||||||
|
|
||||||
|
stat-writer 已支持在累计里程缺失时,用有速度证据的相邻 GPS 坐标生成低优先级日里程候选;但累计里程字段存在且持续重复时,流程会提前进入 `skipped_same_mileage`,没有执行 GPS 累计。这会影响“车辆真实移动、仪表累计值停止刷新”的 GB32960 或宇通来源。
|
||||||
|
|
||||||
|
修复后:
|
||||||
|
|
||||||
|
- 重复累计里程仍保留为 `skipped_same_mileage` 证据;
|
||||||
|
- GB32960、宇通只有在设备速度大于 0 且坐标有效时才进入 GPS 累计;
|
||||||
|
- GPS 状态仍按 10 秒节流,并拒绝超过 10 分钟、时间倒序或隐含速度超过 220 km/h 的线段;
|
||||||
|
- 正向仪表累计里程优先于 GPS 估算,真实仪表值恢复后会自动覆盖估算;
|
||||||
|
- 静止车辆不会因 GPS 漂移产生虚假里程。
|
||||||
|
|
||||||
|
## 同类数据审计与补录边界
|
||||||
|
|
||||||
|
生产 `stats-backfill` 对 2026-07-19 全部 19 台有宇通日活动的车辆执行只读扫描,未发现除现有正向仪表里程之外、可由有效移动线段恢复的 GPS 候选。沪A01559F 的结果已经恢复,无需重复改写;沪A62013F 缺少移动证据,禁止补录正数。
|
||||||
|
|
||||||
|
后续历史补录继续使用同一条安全规则:只有存在可用移动线段时才生成 `gps_coordinate_accumulation` 候选,不按累计总里程、位置点数量或页面展示值推断日里程。
|
||||||
@@ -511,7 +511,7 @@ BACKFILL_DATE_TO=2026-07-12 \
|
|||||||
|
|
||||||
宇通协议会把定位、车况和仪表里程拆成不同的稀疏帧。当天只有明确 `meter_speed=0` 的定位帧、但没有新的仪表总里程帧时,stat-writer 会用同一来源最近的历史总里程生成一条显式 `0 km` 记录,`quality_reason=stationary_carry_forward`;后续收到真实总里程后仍会正常覆盖增长。`vehicle_stat_samples_total{status="recovered_stationary"}` 用于观察该恢复路径。历史补录只能在轨迹核验确认为 `0 km` 后,临时为单次 `stats-backfill` 增加 `BACKFILL_STATIONARY_CARRY_FORWARD=true`;该开关默认关闭,禁止写入 timer,避免仅凭最新静止状态推断整日未行驶。
|
宇通协议会把定位、车况和仪表里程拆成不同的稀疏帧。当天只有明确 `meter_speed=0` 的定位帧、但没有新的仪表总里程帧时,stat-writer 会用同一来源最近的历史总里程生成一条显式 `0 km` 记录,`quality_reason=stationary_carry_forward`;后续收到真实总里程后仍会正常覆盖增长。`vehicle_stat_samples_total{status="recovered_stationary"}` 用于观察该恢复路径。历史补录只能在轨迹核验确认为 `0 km` 后,临时为单次 `stats-backfill` 增加 `BACKFILL_STATIONARY_CARRY_FORWARD=true`;该开关默认关闭,禁止写入 timer,避免仅凭最新静止状态推断整日未行驶。
|
||||||
|
|
||||||
三类协议都优先采用终端上报的累计里程字段。部分终端只有坐标、速度而累计里程稀疏或停止刷新时,stat-writer 会按同一 VIN、自然日、协议和物理来源累计相邻坐标距离,状态持久化在 `vehicle_daily_gps_mileage_state`,并以 `quality_reason=gps_coordinate_accumulation` 写入低优先级候选。JT808 兼容没有速度字段的存量终端;GB32960、宇通 MQTT 必须同时满足设备速度大于 0,且实时状态写入按 10 秒节流,避免静止 GPS 漂移和 1 秒级宇通上报放大 MySQL 压力。相邻点超过 10 分钟、时间倒序或隐含速度超过 220 km/h 的线段不累计。选举顺序为“正向累计里程 > 正向 GPS 轨迹估算 > 累计里程零值占位”,因此稀疏累计里程造成的错误 0 km 会被已有移动轨迹纠正,真实正向累计里程恢复后仍会自动覆盖估算值;禁止把估算值混入平台累计里程归并。`vehicle_stat_samples_total{status="recovered_gps_coordinate"}` 和 `vehicle_stat_cache_entries{cache="gps_accumulation"}` 用于观察实时恢复量及节流缓存。`stats-backfill` 默认用 TDengine `vehicle_locations` 为缺少正向累计里程但存在可用移动线段的车辆补建同口径候选;可临时设置 `BACKFILL_GPS_FALLBACK=false` 禁用全部协议的降级路径,旧环境变量 `BACKFILL_JT808_GPS_FALLBACK` 继续兼容。
|
三类协议都优先采用终端上报的累计里程字段。部分终端只有坐标、速度而累计里程稀疏或停止刷新时,stat-writer 会按同一 VIN、自然日、协议和物理来源累计相邻坐标距离,状态持久化在 `vehicle_daily_gps_mileage_state`,并以 `quality_reason=gps_coordinate_accumulation` 写入低优先级候选。该路径既覆盖总里程字段缺失,也覆盖字段存在但数值与上一帧相同;重复总里程仍计入 `skipped_same_mileage` 证据,只有同时具备有效移动点时才增加 GPS 候选。JT808 兼容没有速度字段的存量终端;GB32960、宇通 MQTT 必须同时满足设备速度大于 0,且实时状态写入按 10 秒节流,避免静止 GPS 漂移和 1 秒级宇通上报放大 MySQL 压力。相邻点超过 10 分钟、时间倒序或隐含速度超过 220 km/h 的线段不累计。选举顺序为“正向累计里程 > 正向 GPS 轨迹估算 > 累计里程零值占位”,因此稀疏累计里程造成的错误 0 km 会被已有移动轨迹纠正,真实正向累计里程恢复后仍会自动覆盖估算值;禁止把估算值混入平台累计里程归并。`vehicle_stat_samples_total{status="recovered_gps_coordinate"}` 和 `vehicle_stat_cache_entries{cache="gps_accumulation"}` 用于观察实时恢复量及节流缓存。`stats-backfill` 默认用 TDengine `vehicle_locations` 为缺少正向累计里程但存在可用移动线段的车辆补建同口径候选;可临时设置 `BACKFILL_GPS_FALLBACK=false` 禁用全部协议的降级路径,旧环境变量 `BACKFILL_JT808_GPS_FALLBACK` 继续兼容。
|
||||||
|
|
||||||
生产定时补算使用 systemd timer,默认每天 01:30 补算昨天往前 3 天,覆盖上游晚到或断传后恢复的总里程字段:
|
生产定时补算使用 systemd timer,默认每天 01:30 补算昨天往前 3 天,覆盖上游晚到或断传后恢复的总里程字段:
|
||||||
|
|
||||||
|
|||||||
@@ -361,6 +361,24 @@ func (w *Writer) AppendWithResult(ctx context.Context, env envelope.FrameEnvelop
|
|||||||
}
|
}
|
||||||
if w.seenSameMileage(sample) {
|
if w.seenSameMileage(sample) {
|
||||||
result.SamplesSkippedSameMileage++
|
result.SamplesSkippedSameMileage++
|
||||||
|
// A repeated cumulative odometer is usually a harmless duplicate,
|
||||||
|
// but some upstream platforms keep reporting positions and speed
|
||||||
|
// while their instrument mileage remains stale for hours. Preserve
|
||||||
|
// the duplicate evidence and accumulate a speed-backed GPS
|
||||||
|
// candidate so real movement does not disappear from daily mileage.
|
||||||
|
if point, ok := GPSMileagePointFromEnvelope(env, identity, w.loc); ok && w.shouldAccumulateGPS(point) {
|
||||||
|
_, recovered, recoverErr := AccumulateGPSMileage(ctx, w.exec, point)
|
||||||
|
if recoverErr != nil {
|
||||||
|
return result, recoverErr
|
||||||
|
}
|
||||||
|
w.markGPSAccumulated(point)
|
||||||
|
if recovered {
|
||||||
|
result.SamplesWritten++
|
||||||
|
result.SamplesRecoveredGPSCoordinate++
|
||||||
|
result.ProjectionsAttempted++
|
||||||
|
result.ProjectionsWritten++
|
||||||
|
}
|
||||||
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
candidate := SourceMileageSampleFromMetric(sample, identity)
|
candidate := SourceMileageSampleFromMetric(sample, identity)
|
||||||
|
|||||||
@@ -2109,6 +2109,78 @@ func TestWriterAppendWithResultReportsDuplicateMileage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestWriterDuplicateYutongOdometerStillEntersGPSFallback(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("sqlmock.New() error = %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||||
|
writer := NewWriter(db, loc)
|
||||||
|
eventTime := time.Date(2026, 7, 19, 8, 0, 0, 0, loc)
|
||||||
|
event := managedTestSource(envelope.FrameEnvelope{
|
||||||
|
EventID: "evt-yutong-stale-odometer",
|
||||||
|
Protocol: envelope.ProtocolYutongMQTT,
|
||||||
|
VIN: "LMRKH9AC9R1004099",
|
||||||
|
DeviceID: "LMRKH9AC9R1004099",
|
||||||
|
SourceEndpoint: "mqtt://yutong/ytforward/shln/2",
|
||||||
|
EventTimeMS: eventTime.UnixMilli(),
|
||||||
|
Fields: map[string]any{
|
||||||
|
"yutong_mqtt.data.total_mileage": 24245000,
|
||||||
|
"yutong_mqtt.data.longitude": 114.435672,
|
||||||
|
"yutong_mqtt.data.latitude": 30.645438,
|
||||||
|
"yutong_mqtt.data.meter_speed": 18,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
samples, _, err := samplesFromEnvelopeWithResult(event, loc)
|
||||||
|
if err != nil || len(samples) != 1 {
|
||||||
|
t.Fatalf("samplesFromEnvelopeWithResult() samples=%+v err=%v", samples, err)
|
||||||
|
}
|
||||||
|
identity, ok := NewSourceIdentityFromEnvelope(event)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("expected managed Yutong source identity")
|
||||||
|
}
|
||||||
|
writer.markSourceTouched(identity, eventTime)
|
||||||
|
writer.markMileageWritten(samples[0])
|
||||||
|
|
||||||
|
mock.ExpectBegin()
|
||||||
|
mock.ExpectQuery(`SELECT first_event_time, latest_event_time`).
|
||||||
|
WillReturnRows(sqlmock.NewRows([]string{
|
||||||
|
"first_event_time", "latest_event_time", "latest_event_id",
|
||||||
|
"latest_longitude", "latest_latitude", "daily_mileage_km",
|
||||||
|
"point_count", "usable_segment_count", "bad_jump_count",
|
||||||
|
"long_gap_count", "out_of_order_count",
|
||||||
|
}).AddRow(eventTime.Add(-20*time.Second), eventTime.Add(-20*time.Second), "evt-before", 114.434672, 30.645438, 0, 1, 0, 0, 0, 0))
|
||||||
|
mock.ExpectExec(`UPDATE vehicle_daily_gps_mileage_state`).
|
||||||
|
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||||
|
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`).
|
||||||
|
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||||
|
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`).
|
||||||
|
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||||
|
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source`).
|
||||||
|
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||||
|
mock.ExpectExec(`DELETE FROM vehicle_daily_mileage`).
|
||||||
|
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||||
|
mock.ExpectCommit()
|
||||||
|
|
||||||
|
result, err := writer.AppendWithResult(context.Background(), event)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("AppendWithResult() error = %v", err)
|
||||||
|
}
|
||||||
|
if result.SamplesFound != 1 || result.SamplesSkippedSameMileage != 1 ||
|
||||||
|
result.SamplesRecoveredGPSCoordinate != 1 || result.SamplesWritten != 1 ||
|
||||||
|
result.ProjectionsAttempted != 1 || result.ProjectionsWritten != 1 {
|
||||||
|
t.Fatalf("duplicate odometer evidence = %+v", result)
|
||||||
|
}
|
||||||
|
if got := writer.CacheStats().GPSAccumulationEntries; got != 1 {
|
||||||
|
t.Fatalf("GPS accumulation entries = %d, want stale moving odometer to enter fallback", got)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("unmet SQL expectations: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestWriterDoesNotCacheMileageWhenUpsertFails(t *testing.T) {
|
func TestWriterDoesNotCacheMileageWhenUpsertFails(t *testing.T) {
|
||||||
exec := &recordingExec{errs: []error{errors.New("mysql unavailable"), nil}}
|
exec := &recordingExec{errs: []error{errors.New("mysql unavailable"), nil}}
|
||||||
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))
|
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||||
|
|||||||
Reference in New Issue
Block a user