shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(stats): recover jt808 mileage from safe gps tracks

Browse Source
This commit is contained in:
lingniu
2026-07-19 22:23:19 +08:00
parent 2f195e725a
commit e45daff528
11 changed files with 860 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
go/vehicle-gateway/cmd/stat-writer/main.go
View File

@@ -783,6 +783,7 @@ func recordStatSampleMetrics(registry *metrics.Registry, message kafka.Message,
addStatSampleMetric(registry, message, protocol, "skipped_missing_vin", result.SamplesSkippedMissingVIN)
addStatSampleMetric(registry, message, protocol, "skipped_missing_mileage", result.SamplesSkippedMissingMileage)
addStatSampleMetric(registry, message, protocol, "recovered_stationary", result.SamplesRecoveredStationary)
addStatSampleMetric(registry, message, protocol, "recovered_gps_coordinate", result.SamplesRecoveredGPSCoordinate)
addStatSampleMetric(registry, message, protocol, "skipped_non_mileage_frame", result.SamplesSkippedNonMileageFrame)
addStatSampleMetric(registry, message, protocol, "skipped_non_positive_mileage", result.SamplesSkippedNonPositiveMileage)
addStatSampleMetric(registry, message, protocol, "skipped_missing_time", result.SamplesSkippedMissingTime)

182
go/vehicle-gateway/cmd/stats-backfill/main.go
View File

@@ -36,6 +36,7 @@ type config struct {
Debug bool
EventTimeFullScan bool
StationaryCarry bool
JT808GPSFallback bool
ProgressEvery int64
BaselineLookback int
Location *time.Location
@@ -150,6 +151,10 @@ func main() {
if err != nil {
fail("build realtime-location fallback aggregates", err)
}
gpsFallbacks, err := addJT808GPSCoordinateFallbackAggregates(ctx, mysqlDB, td, cfg, aggregates)
if err != nil {
fail("build JT808 GPS-coordinate fallback aggregates", err)
}
var written int64
var normalized int
if !cfg.DryRun {
@@ -162,7 +167,7 @@ func main() {
fail("normalize historical platform sources", err)
}
}
slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "realtimeLocationFallbacks", fallbacks, "written", written, "platformSourcesNormalized", normalized)
slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "realtimeLocationFallbacks", fallbacks, "jt808GPSFallbacks", gpsFallbacks, "written", written, "platformSourcesNormalized", normalized)
return
}
@@ -322,8 +327,10 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
}
clearedTargets[target] = struct{}{}
}
if err := stats.UpsertDataSource(ctx, db, identity, agg.LatestEventTime); err != nil {
return written, err
if agg.QualityReason != stats.QualityReasonGPSCoordinate {
if err := stats.UpsertDataSource(ctx, db, identity, agg.LatestEventTime); err != nil {
return written, err
}
}
dailyKM := stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM)
candidate := stats.SourceMileageSample{
@@ -498,6 +505,174 @@ func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB,
return added, nil
}
type gpsCoordinateBackfillState struct {
previous stats.GPSMileagePoint
firstEventTime time.Time
latestEventTime time.Time
distanceKM float64
pointCount int64
usableSegments int64
badJumps int64
longGaps int64
}
func addJT808GPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
if !cfg.JT808GPSFallback || tdDB == nil || aggregates == nil || !containsProtocol(cfg.Protocols, envelope.ProtocolJT808) {
return 0, nil
}
existing, err := existingJT808OdometerTargets(ctx, mysqlDB, cfg)
if err != nil {
return 0, err
}
for _, agg := range aggregates {
if agg != nil && agg.Protocol == envelope.ProtocolJT808 && agg.QualityReason != stats.QualityReasonGPSCoordinate {
existing[agg.VIN+"|"+agg.Date] = struct{}{}
}
}
where := []string{
fmt.Sprintf("ts >= '%s 00:00:00'", quote(cfg.DateFrom)),
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(cfg.DateTo))),
"protocol = 'JT808'",
"vin IS NOT NULL",
"vin <> ''",
"longitude BETWEEN -180 AND 180",
"latitude BETWEEN -90 AND 90",
"NOT (longitude = 0 AND latitude = 0)",
}
sqlText := fmt.Sprintf(`SELECT vin, ts, longitude, latitude
FROM %s.vehicle_locations
WHERE %s
ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
rows, err := tdDB.QueryContext(ctx, sqlText)
if err != nil {
return 0, err
}
defer rows.Close()
states := map[string]*gpsCoordinateBackfillState{}
for rows.Next() {
var vin string
var eventTime time.Time
var longitude, latitude float64
if err := rows.Scan(&vin, &eventTime, &longitude, &latitude); err != nil {
return 0, err
}
vin = strings.TrimSpace(vin)
eventTime = eventTime.In(cfg.Location)
date := eventTime.Format("2006-01-02")
target := vin + "|" + date
if _, ok := existing[target]; ok {
continue
}
current := stats.GPSMileagePoint{
VIN: vin,
StatDate: date,
Protocol: envelope.ProtocolJT808,
EventTime: eventTime,
Longitude: longitude,
Latitude: latitude,
}
state := states[target]
if state == nil {
states[target] = &gpsCoordinateBackfillState{
previous: current,
firstEventTime: eventTime,
latestEventTime: eventTime,
pointCount: 1,
}
continue
}
segment := stats.CalculateGPSMileageSegment(state.previous, current)
if segment.OutOfOrder {
continue
}
state.previous = current
state.latestEventTime = eventTime
state.pointCount++
if segment.Usable {
state.distanceKM += segment.DistanceKM
state.usableSegments++
}
if segment.BadJump {
state.badJumps++
}
if segment.LongGap {
state.longGaps++
}
}
if err := rows.Err(); err != nil {
return 0, err
}
added := 0
for target, state := range states {
if state.usableSegments == 0 {
continue
}
parts := strings.SplitN(target, "|", 2)
if len(parts) != 2 {
continue
}
vin, date := parts[0], parts[1]
sourceKey := string(envelope.ProtocolJT808) + ":" + vin + "@GPS_COORDINATE"
key := vin + "|" + date + "|" + string(envelope.ProtocolJT808) + "|" + sourceKey
aggregates[key] = &metricAgg{
VIN: vin,
Date: date,
Protocol: envelope.ProtocolJT808,
FirstKM: 0,
LatestKM: state.distanceKM,
Count: state.pointCount,
SourceKey: sourceKey,
SourceEndpoint: "gps-coordinate",
PlatformName: "JT808 GPS轨迹估算",
SourceKind: "UNKNOWN",
FirstEventTime: state.firstEventTime,
LatestEventTime: state.latestEventTime,
QualityStatus: stats.QualityOK,
QualityReason: stats.QualityReasonGPSCoordinate,
}
added++
}
if len(states) > 0 {
slog.Info("JT808 GPS-coordinate fallbacks loaded", "states", len(states), "added", added)
}
return added, nil
}
func existingJT808OdometerTargets(ctx context.Context, db *sql.DB, cfg config) (map[string]struct{}, error) {
result := map[string]struct{}{}
if db == nil {
return result, nil
}
rows, err := db.QueryContext(ctx, `SELECT DISTINCT vin, DATE_FORMAT(stat_date, '%Y-%m-%d')
FROM vehicle_daily_mileage_source
WHERE protocol = 'JT808'
AND stat_date >= ? AND stat_date <= ?
AND quality_status = ?
AND COALESCE(quality_reason, '') <> ?
AND latest_total_mileage_km IS NOT NULL`, cfg.DateFrom, cfg.DateTo, stats.QualityOK, stats.QualityReasonGPSCoordinate)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var vin, date string
if err := rows.Scan(&vin, &date); err != nil {
return nil, err
}
result[strings.TrimSpace(vin)+"|"+strings.TrimSpace(date)] = struct{}{}
}
return result, rows.Err()
}
func containsProtocol(protocols []envelope.Protocol, target envelope.Protocol) bool {
for _, protocol := range protocols {
if protocol == target {
return true
}
}
return false
}
func indexAggregateSourceRowsByDate(aggregates map[string]*metricAgg, protocol envelope.Protocol) map[string]map[string][]dailySourceLast {
indexed := map[string]map[string][]dailySourceLast{}
for _, agg := range aggregates {
@@ -1009,6 +1184,7 @@ func loadConfig() (config, error) {
Debug: envBool("BACKFILL_DEBUG", false),
EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false),
StationaryCarry: envBool("BACKFILL_STATIONARY_CARRY_FORWARD", false),
JT808GPSFallback: envBool("BACKFILL_JT808_GPS_FALLBACK", true),
ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
BaselineLookback: envInt("BACKFILL_BASELINE_LOOKBACK_DAYS", 7),
Location: loc,

56
go/vehicle-gateway/cmd/stats-backfill/main_test.go
View File

@@ -990,6 +990,62 @@ func TestResolveBackfillDateRangeDefaultsToToday(t *testing.T) {
}
}
func TestAddJT808GPSCoordinateFallbackAggregatesOnlyMissingOdometerVehicles(t *testing.T) {
mysqlDB, mysqlMock, err := sqlmock.New()
if err != nil {
t.Fatalf("mysql sqlmock.New() error = %v", err)
}
defer mysqlDB.Close()
tdDB, tdMock, err := sqlmock.New()
if err != nil {
t.Fatalf("tdengine sqlmock.New() error = %v", err)
}
defer tdDB.Close()
mysqlMock.ExpectQuery("SELECT DISTINCT vin, DATE_FORMAT").
WithArgs("2026-07-19", "2026-07-19", stats.QualityOK, stats.QualityReasonGPSCoordinate).
WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date"}).
AddRow("VIN-WITH-ODOMETER", "2026-07-19"))
loc := time.FixedZone("Asia/Shanghai", 8*3600)
start := time.Date(2026, 7, 19, 8, 0, 0, 0, loc)
tdMock.ExpectQuery("SELECT vin, ts, longitude, latitude").
WillReturnRows(sqlmock.NewRows([]string{"vin", "ts", "longitude", "latitude"}).
AddRow("VIN-GPS-FALLBACK", start, 121.4737, 31.2304).
AddRow("VIN-WITH-ODOMETER", start, 121.4737, 31.2304).
AddRow("VIN-GPS-FALLBACK", start.Add(5*time.Minute), 121.4837, 31.2304).
AddRow("VIN-WITH-ODOMETER", start.Add(5*time.Minute), 121.4837, 31.2304))
aggregates := map[string]*metricAgg{}
added, err := addJT808GPSCoordinateFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
DateFrom: "2026-07-19",
DateTo: "2026-07-19",
Protocols: []envelope.Protocol{envelope.ProtocolJT808},
JT808GPSFallback: true,
Location: loc,
TDengineDatabase: "vehicle_ts",
}, aggregates)
if err != nil {
t.Fatalf("addJT808GPSCoordinateFallbackAggregates() error = %v", err)
}
if added != 1 || len(aggregates) != 1 {
t.Fatalf("added=%d aggregates=%d", added, len(aggregates))
}
for _, aggregate := range aggregates {
if aggregate.VIN != "VIN-GPS-FALLBACK" || aggregate.QualityReason != stats.QualityReasonGPSCoordinate {
t.Fatalf("aggregate = %+v", aggregate)
}
if aggregate.LatestKM < 0.9 || aggregate.LatestKM > 1.1 || aggregate.Count != 2 {
t.Fatalf("GPS aggregate distance/count = %.3f/%d", aggregate.LatestKM, aggregate.Count)
}
}
if err := mysqlMock.ExpectationsWereMet(); err != nil {
t.Fatalf("mysql expectations: %v", err)
}
if err := tdMock.ExpectationsWereMet(); err != nil {
t.Fatalf("tdengine expectations: %v", err)
}
}
func TestResolveBackfillDateRangeUsesRelativeWindow(t *testing.T) {
t.Setenv("BACKFILL_DATE_FROM", "")
t.Setenv("BACKFILL_DATE_TO", "")