fix(stats): recover jt808 mileage from safe gps tracks

2026-07-19 22:23:19 +08:00
parent 2f195e725a
commit e45daff528
11 changed files with 860 additions and 14 deletions
--- a/go/vehicle-gateway/cmd/stats-backfill/main.go
+++ b/go/vehicle-gateway/cmd/stats-backfill/main.go
@@ -36,6 +36,7 @@ type config struct {
 	Debug             bool
 	EventTimeFullScan bool
 	StationaryCarry   bool
+	JT808GPSFallback  bool
 	ProgressEvery     int64
 	BaselineLookback  int
 	Location          *time.Location
@@ -150,6 +151,10 @@ func main() {
 		if err != nil {
 			fail("build realtime-location fallback aggregates", err)
 		}
+		gpsFallbacks, err := addJT808GPSCoordinateFallbackAggregates(ctx, mysqlDB, td, cfg, aggregates)
+		if err != nil {
+			fail("build JT808 GPS-coordinate fallback aggregates", err)
+		}
 		var written int64
 		var normalized int
 		if !cfg.DryRun {
@@ -162,7 +167,7 @@ func main() {
 				fail("normalize historical platform sources", err)
 			}
 		}
-		slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "realtimeLocationFallbacks", fallbacks, "written", written, "platformSourcesNormalized", normalized)
+		slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "realtimeLocationFallbacks", fallbacks, "jt808GPSFallbacks", gpsFallbacks, "written", written, "platformSourcesNormalized", normalized)
 		return
 	}

@@ -322,8 +327,10 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
 			}
 			clearedTargets[target] = struct{}{}
 		}
-		if err := stats.UpsertDataSource(ctx, db, identity, agg.LatestEventTime); err != nil {
-			return written, err
+		if agg.QualityReason != stats.QualityReasonGPSCoordinate {
+			if err := stats.UpsertDataSource(ctx, db, identity, agg.LatestEventTime); err != nil {
+				return written, err
+			}
 		}
 		dailyKM := stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM)
 		candidate := stats.SourceMileageSample{
@@ -498,6 +505,174 @@ func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB,
 	return added, nil
 }

+type gpsCoordinateBackfillState struct {
+	previous        stats.GPSMileagePoint
+	firstEventTime  time.Time
+	latestEventTime time.Time
+	distanceKM      float64
+	pointCount      int64
+	usableSegments  int64
+	badJumps        int64
+	longGaps        int64
+}
+
+func addJT808GPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
+	if !cfg.JT808GPSFallback || tdDB == nil || aggregates == nil || !containsProtocol(cfg.Protocols, envelope.ProtocolJT808) {
+		return 0, nil
+	}
+	existing, err := existingJT808OdometerTargets(ctx, mysqlDB, cfg)
+	if err != nil {
+		return 0, err
+	}
+	for _, agg := range aggregates {
+		if agg != nil && agg.Protocol == envelope.ProtocolJT808 && agg.QualityReason != stats.QualityReasonGPSCoordinate {
+			existing[agg.VIN+"|"+agg.Date] = struct{}{}
+		}
+	}
+	where := []string{
+		fmt.Sprintf("ts >= '%s 00:00:00'", quote(cfg.DateFrom)),
+		fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(cfg.DateTo))),
+		"protocol = 'JT808'",
+		"vin IS NOT NULL",
+		"vin <> ''",
+		"longitude BETWEEN -180 AND 180",
+		"latitude BETWEEN -90 AND 90",
+		"NOT (longitude = 0 AND latitude = 0)",
+	}
+	sqlText := fmt.Sprintf(`SELECT vin, ts, longitude, latitude
+FROM %s.vehicle_locations
+WHERE %s
+ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
+	rows, err := tdDB.QueryContext(ctx, sqlText)
+	if err != nil {
+		return 0, err
+	}
+	defer rows.Close()
+	states := map[string]*gpsCoordinateBackfillState{}
+	for rows.Next() {
+		var vin string
+		var eventTime time.Time
+		var longitude, latitude float64
+		if err := rows.Scan(&vin, &eventTime, &longitude, &latitude); err != nil {
+			return 0, err
+		}
+		vin = strings.TrimSpace(vin)
+		eventTime = eventTime.In(cfg.Location)
+		date := eventTime.Format("2006-01-02")
+		target := vin + "|" + date
+		if _, ok := existing[target]; ok {
+			continue
+		}
+		current := stats.GPSMileagePoint{
+			VIN:       vin,
+			StatDate:  date,
+			Protocol:  envelope.ProtocolJT808,
+			EventTime: eventTime,
+			Longitude: longitude,
+			Latitude:  latitude,
+		}
+		state := states[target]
+		if state == nil {
+			states[target] = &gpsCoordinateBackfillState{
+				previous:        current,
+				firstEventTime:  eventTime,
+				latestEventTime: eventTime,
+				pointCount:      1,
+			}
+			continue
+		}
+		segment := stats.CalculateGPSMileageSegment(state.previous, current)
+		if segment.OutOfOrder {
+			continue
+		}
+		state.previous = current
+		state.latestEventTime = eventTime
+		state.pointCount++
+		if segment.Usable {
+			state.distanceKM += segment.DistanceKM
+			state.usableSegments++
+		}
+		if segment.BadJump {
+			state.badJumps++
+		}
+		if segment.LongGap {
+			state.longGaps++
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return 0, err
+	}
+	added := 0
+	for target, state := range states {
+		if state.usableSegments == 0 {
+			continue
+		}
+		parts := strings.SplitN(target, "|", 2)
+		if len(parts) != 2 {
+			continue
+		}
+		vin, date := parts[0], parts[1]
+		sourceKey := string(envelope.ProtocolJT808) + ":" + vin + "@GPS_COORDINATE"
+		key := vin + "|" + date + "|" + string(envelope.ProtocolJT808) + "|" + sourceKey
+		aggregates[key] = &metricAgg{
+			VIN:             vin,
+			Date:            date,
+			Protocol:        envelope.ProtocolJT808,
+			FirstKM:         0,
+			LatestKM:        state.distanceKM,
+			Count:           state.pointCount,
+			SourceKey:       sourceKey,
+			SourceEndpoint:  "gps-coordinate",
+			PlatformName:    "JT808 GPS轨迹估算",
+			SourceKind:      "UNKNOWN",
+			FirstEventTime:  state.firstEventTime,
+			LatestEventTime: state.latestEventTime,
+			QualityStatus:   stats.QualityOK,
+			QualityReason:   stats.QualityReasonGPSCoordinate,
+		}
+		added++
+	}
+	if len(states) > 0 {
+		slog.Info("JT808 GPS-coordinate fallbacks loaded", "states", len(states), "added", added)
+	}
+	return added, nil
+}
+
+func existingJT808OdometerTargets(ctx context.Context, db *sql.DB, cfg config) (map[string]struct{}, error) {
+	result := map[string]struct{}{}
+	if db == nil {
+		return result, nil
+	}
+	rows, err := db.QueryContext(ctx, `SELECT DISTINCT vin, DATE_FORMAT(stat_date, '%Y-%m-%d')
+FROM vehicle_daily_mileage_source
+WHERE protocol = 'JT808'
+  AND stat_date >= ? AND stat_date <= ?
+  AND quality_status = ?
+  AND COALESCE(quality_reason, '') <> ?
+  AND latest_total_mileage_km IS NOT NULL`, cfg.DateFrom, cfg.DateTo, stats.QualityOK, stats.QualityReasonGPSCoordinate)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var vin, date string
+		if err := rows.Scan(&vin, &date); err != nil {
+			return nil, err
+		}
+		result[strings.TrimSpace(vin)+"|"+strings.TrimSpace(date)] = struct{}{}
+	}
+	return result, rows.Err()
+}
+
+func containsProtocol(protocols []envelope.Protocol, target envelope.Protocol) bool {
+	for _, protocol := range protocols {
+		if protocol == target {
+			return true
+		}
+	}
+	return false
+}
+
 func indexAggregateSourceRowsByDate(aggregates map[string]*metricAgg, protocol envelope.Protocol) map[string]map[string][]dailySourceLast {
 	indexed := map[string]map[string][]dailySourceLast{}
 	for _, agg := range aggregates {
@@ -1009,6 +1184,7 @@ func loadConfig() (config, error) {
 		Debug:             envBool("BACKFILL_DEBUG", false),
 		EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false),
 		StationaryCarry:   envBool("BACKFILL_STATIONARY_CARRY_FORWARD", false),
+		JT808GPSFallback:  envBool("BACKFILL_JT808_GPS_FALLBACK", true),
 		ProgressEvery:     int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
 		BaselineLookback:  envInt("BACKFILL_BASELINE_LOOKBACK_DAYS", 7),
 		Location:          loc,