shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: materialize stationary yutong daily mileage

Browse Source
This commit is contained in:
lingniu
2026-07-19 13:23:33 +08:00
parent dffe559160
commit 4da26ad3ba
7 changed files with 292 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
go/vehicle-gateway/cmd/stat-writer/main.go
View File

@@ -687,6 +687,7 @@ func recordStatSampleMetrics(registry *metrics.Registry, message kafka.Message,
addStatSampleMetric(registry, message, protocol, "skipped_missing_fields", result.SamplesSkippedMissingFields)
addStatSampleMetric(registry, message, protocol, "skipped_missing_vin", result.SamplesSkippedMissingVIN)
addStatSampleMetric(registry, message, protocol, "skipped_missing_mileage", result.SamplesSkippedMissingMileage)
addStatSampleMetric(registry, message, protocol, "recovered_stationary", result.SamplesRecoveredStationary)
addStatSampleMetric(registry, message, protocol, "skipped_non_mileage_frame", result.SamplesSkippedNonMileageFrame)
addStatSampleMetric(registry, message, protocol, "skipped_non_positive_mileage", result.SamplesSkippedNonPositiveMileage)
addStatSampleMetric(registry, message, protocol, "skipped_missing_time", result.SamplesSkippedMissingTime)

42
go/vehicle-gateway/cmd/stats-backfill/main.go
View File

@@ -35,6 +35,7 @@ type config struct {
Reset bool
Debug bool
EventTimeFullScan bool
StationaryCarry bool
ProgressEvery int64
BaselineLookback int
Location *time.Location
@@ -453,7 +454,8 @@ func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB,
return added, err
}
for vin, currentSources := range current {
for _, currentRow := range currentSources {
for i := range currentSources {
currentRow := currentSources[i]
key := vin + "|" + date + "|" + string(protocol) + "|" + currentRow.SourceKey
if _, exists := aggregates[key]; exists {
continue
@@ -462,8 +464,22 @@ func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB,
if err != nil {
return added, err
}
if cfg.StationaryCarry && !hasPrevious {
continue
}
if cfg.StationaryCarry {
// A sparse Yutong location frame can retain an older
// realtime odometer than the durable daily baseline.
// Stationary carry-forward represents an explicit zero
// day, so both ends must use the latest trusted baseline.
currentRow.FirstTotalKM = previousRow.TotalKM
currentRow.TotalKM = previousRow.TotalKM
currentSources[i] = currentRow
}
agg := aggregateFromDailySource(date, protocol, currentRow, previousRow, hasPrevious)
if hasPrevious {
if cfg.StationaryCarry {
agg.QualityReason = stats.QualityReasonStationaryCarry
} else if hasPrevious {
agg.QualityReason = "realtime_location_fallback_historical_baseline"
} else {
agg.QualityReason = "realtime_location_fallback_current_day_first_baseline"
@@ -716,15 +732,28 @@ func queryRealtimeLocationLastRows(ctx context.Context, db *sql.DB, cfg config,
if loc == nil {
loc = time.FixedZone("Asia/Shanghai", 8*3600)
}
sqlText := `SELECT l.vin, COALESCE(NULLIF(s.peer, ''), ''), l.total_mileage_event_time, l.total_mileage_km
eventTimeColumn := "l.total_mileage_event_time"
timePredicate := `l.total_mileage_event_time >= ?
AND l.total_mileage_event_time < DATE_ADD(?, INTERVAL 1 DAY)`
if cfg.StationaryCarry {
eventTimeColumn = "l.event_time"
timePredicate = `l.event_time >= ?
AND l.event_time < DATE_ADD(?, INTERVAL 1 DAY)
AND l.total_mileage_event_time < ?
AND COALESCE(l.speed_kmh, 0) = 0`
}
sqlText := `SELECT l.vin, COALESCE(NULLIF(s.peer, ''), ''), ` + eventTimeColumn + `, l.total_mileage_km
FROM vehicle_realtime_location l
LEFT JOIN vehicle_realtime_snapshot s ON s.protocol = l.protocol AND s.vin = l.vin
WHERE l.protocol = ?
AND l.vin IS NOT NULL AND l.vin <> ''
AND l.total_mileage_km IS NOT NULL AND l.total_mileage_km > 0
AND l.total_mileage_event_time >= ?
AND l.total_mileage_event_time < DATE_ADD(?, INTERVAL 1 DAY)`
rows, err := db.QueryContext(ctx, sqlText, string(protocol), date, date)
AND ` + timePredicate
args := []any{string(protocol), date, date}
if cfg.StationaryCarry {
args = append(args, date)
}
rows, err := db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
@@ -979,6 +1008,7 @@ func loadConfig() (config, error) {
Reset: envBool("BACKFILL_RESET", false),
Debug: envBool("BACKFILL_DEBUG", false),
EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false),
StationaryCarry: envBool("BACKFILL_STATIONARY_CARRY_FORWARD", false),
ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
BaselineLookback: envInt("BACKFILL_BASELINE_LOOKBACK_DAYS", 7),
Location: loc,

89
go/vehicle-gateway/cmd/stats-backfill/main_test.go
View File

@@ -323,6 +323,95 @@ func TestQueryRealtimeLocationLastRowsBuildsYutongSourceFromPeer(t *testing.T) {
}
}
func TestQueryRealtimeLocationLastRowsCanCarryStationaryYutongMileage(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
currentEventTime := time.Date(2026, 7, 19, 6, 5, 42, 0, loc)
mock.ExpectQuery("(?s)SELECT.*l\\.event_time.*l\\.total_mileage_event_time < \\?.*COALESCE\\(l\\.speed_kmh, 0\\) = 0").
WithArgs("YUTONG_MQTT", "2026-07-19", "2026-07-19", "2026-07-19").
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "event_time", "total_mileage_km"}).
AddRow("LMRKH9AC9R1004099", "mqtt://yutong/ytforward/shln/2", currentEventTime, 24245.0))
rows, err := queryRealtimeLocationLastRows(context.Background(), db, config{
Location: loc,
StationaryCarry: true,
}, envelope.ProtocolYutongMQTT, "2026-07-19")
if err != nil {
t.Fatalf("queryRealtimeLocationLastRows() error = %v", err)
}
sourceRows := rows["LMRKH9AC9R1004099"]
if len(sourceRows) != 1 {
t.Fatalf("rows = %d, want 1", len(sourceRows))
}
if !sourceRows[0].TS.Equal(currentEventTime) || sourceRows[0].TotalKM != 24245 {
t.Fatalf("stationary carry row = %#v", sourceRows[0])
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestAddRealtimeLocationFallbackStationaryCarryUsesTrustedHistoricalBaseline(t *testing.T) {
mysqlDB, mysqlMock, err := sqlmock.New()
if err != nil {
t.Fatalf("mysql sqlmock.New() error = %v", err)
}
defer mysqlDB.Close()
tdDB, tdMock, err := sqlmock.New()
if err != nil {
t.Fatalf("td sqlmock.New() error = %v", err)
}
defer tdDB.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
vin := "LMRKH9AC5R1004133"
currentTS := time.Date(2026, 7, 19, 10, 20, 20, 0, loc)
previousTS := time.Date(2026, 7, 17, 8, 49, 55, 0, loc)
sourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", vin, "mqtt", "PLATFORM", "yutong")
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l\\.event_time.*l\\.total_mileage_event_time < \\?").
WithArgs("YUTONG_MQTT", "2026-07-19", "2026-07-19", "2026-07-19").
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "event_time", "total_mileage_km"}).
AddRow(vin, "mqtt://yutong/ytforward/shln/4", currentTS, 55451.0))
mysqlMock.ExpectQuery("(?s)SELECT latest_total_mileage_km, latest_event_time.*FROM vehicle_daily_mileage_source").
WithArgs(vin, "2026-07-19", "YUTONG_MQTT", sourceKey).
WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}).AddRow(55466.0, previousTS))
aggregates := map[string]*metricAgg{}
added, err := addRealtimeLocationFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
TDengineDatabase: "lingniu_vehicle_ts",
DateFrom: "2026-07-19",
DateTo: "2026-07-19",
Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT},
Location: loc,
StationaryCarry: true,
}, aggregates)
if err != nil {
t.Fatalf("addRealtimeLocationFallbackAggregates() error = %v", err)
}
if added != 1 || len(aggregates) != 1 {
t.Fatalf("added=%d aggregates=%d", added, len(aggregates))
}
for _, agg := range aggregates {
if agg.FirstKM != 55466 || agg.LatestKM != 55466 {
t.Fatalf("stationary km range = %v -> %v", agg.FirstKM, agg.LatestKM)
}
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonStationaryCarry {
t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason)
}
}
if err := mysqlMock.ExpectationsWereMet(); err != nil {
t.Fatalf("mysql sql expectations: %v", err)
}
if err := tdMock.ExpectationsWereMet(); err != nil {
t.Fatalf("td sql expectations: %v", err)
}
}
func TestAddRealtimeLocationFallbackAggregatesUsesHistoricalBaseline(t *testing.T) {
mysqlDB, mysqlMock, err := sqlmock.New()
if err != nil {