fix(mileage): recover sparse source daily distance

2026-07-19 22:57:31 +08:00
parent e45daff528
commit 4c64c4c738
11 changed files with 364 additions and 80 deletions
--- a/go/vehicle-gateway/cmd/stats-backfill/main.go
+++ b/go/vehicle-gateway/cmd/stats-backfill/main.go
@@ -36,7 +36,7 @@ type config struct {
 	Debug             bool
 	EventTimeFullScan bool
 	StationaryCarry   bool
-	JT808GPSFallback  bool
+	GPSFallback       bool
 	ProgressEvery     int64
 	BaselineLookback  int
 	Location          *time.Location
@@ -151,9 +151,9 @@ func main() {
 		if err != nil {
 			fail("build realtime-location fallback aggregates", err)
 		}
-		gpsFallbacks, err := addJT808GPSCoordinateFallbackAggregates(ctx, mysqlDB, td, cfg, aggregates)
+		gpsFallbacks, err := addGPSCoordinateFallbackAggregates(ctx, mysqlDB, td, cfg, aggregates)
 		if err != nil {
-			fail("build JT808 GPS-coordinate fallback aggregates", err)
+			fail("build GPS-coordinate fallback aggregates", err)
 		}
 		var written int64
 		var normalized int
@@ -516,29 +516,47 @@ type gpsCoordinateBackfillState struct {
 	longGaps        int64
 }

-func addJT808GPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
-	if !cfg.JT808GPSFallback || tdDB == nil || aggregates == nil || !containsProtocol(cfg.Protocols, envelope.ProtocolJT808) {
+func addGPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
+	if !cfg.GPSFallback || tdDB == nil || aggregates == nil {
 		return 0, nil
 	}
-	existing, err := existingJT808OdometerTargets(ctx, mysqlDB, cfg)
+	totalAdded := 0
+	for _, protocol := range cfg.Protocols {
+		added, err := addProtocolGPSCoordinateFallbackAggregates(ctx, mysqlDB, tdDB, cfg, aggregates, protocol)
+		if err != nil {
+			return totalAdded, err
+		}
+		totalAdded += added
+	}
+	return totalAdded, nil
+}
+
+func addProtocolGPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg, protocol envelope.Protocol) (int, error) {
+	if !supportsBackfillGPSFallback(protocol) {
+		return 0, nil
+	}
+	existing, err := existingPositiveOdometerTargets(ctx, mysqlDB, cfg, protocol)
 	if err != nil {
 		return 0, err
 	}
 	for _, agg := range aggregates {
-		if agg != nil && agg.Protocol == envelope.ProtocolJT808 && agg.QualityReason != stats.QualityReasonGPSCoordinate {
+		if agg != nil && agg.Protocol == protocol && agg.QualityReason != stats.QualityReasonGPSCoordinate && agg.LatestKM > agg.FirstKM {
 			existing[agg.VIN+"|"+agg.Date] = struct{}{}
 		}
 	}
 	where := []string{
 		fmt.Sprintf("ts >= '%s 00:00:00'", quote(cfg.DateFrom)),
 		fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(cfg.DateTo))),
-		"protocol = 'JT808'",
+		fmt.Sprintf("protocol = '%s'", quote(string(protocol))),
 		"vin IS NOT NULL",
 		"vin <> ''",
 		"longitude BETWEEN -180 AND 180",
 		"latitude BETWEEN -90 AND 90",
 		"NOT (longitude = 0 AND latitude = 0)",
 	}
+	if protocol != envelope.ProtocolJT808 {
+		where = append(where, "speed_kmh > 0")
+	}
 	sqlText := fmt.Sprintf(`SELECT vin, ts, longitude, latitude
 FROM %s.vehicle_locations
 WHERE %s
@@ -566,7 +584,7 @@ ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
 		current := stats.GPSMileagePoint{
 			VIN:       vin,
 			StatDate:  date,
-			Protocol:  envelope.ProtocolJT808,
+			Protocol:  protocol,
 			EventTime: eventTime,
 			Longitude: longitude,
 			Latitude:  latitude,
@@ -612,18 +630,18 @@ ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
 			continue
 		}
 		vin, date := parts[0], parts[1]
-		sourceKey := string(envelope.ProtocolJT808) + ":" + vin + "@GPS_COORDINATE"
-		key := vin + "|" + date + "|" + string(envelope.ProtocolJT808) + "|" + sourceKey
+		sourceKey := string(protocol) + ":" + vin + "@GPS_COORDINATE"
+		key := vin + "|" + date + "|" + string(protocol) + "|" + sourceKey
 		aggregates[key] = &metricAgg{
 			VIN:             vin,
 			Date:            date,
-			Protocol:        envelope.ProtocolJT808,
+			Protocol:        protocol,
 			FirstKM:         0,
 			LatestKM:        state.distanceKM,
 			Count:           state.pointCount,
 			SourceKey:       sourceKey,
 			SourceEndpoint:  "gps-coordinate",
-			PlatformName:    "JT808 GPS轨迹估算",
+			PlatformName:    backfillGPSPlatformName(protocol),
 			SourceKind:      "UNKNOWN",
 			FirstEventTime:  state.firstEventTime,
 			LatestEventTime: state.latestEventTime,
@@ -633,23 +651,44 @@ ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
 		added++
 	}
 	if len(states) > 0 {
-		slog.Info("JT808 GPS-coordinate fallbacks loaded", "states", len(states), "added", added)
+		slog.Info("GPS-coordinate fallbacks loaded", "protocol", protocol, "states", len(states), "added", added)
 	}
 	return added, nil
 }

-func existingJT808OdometerTargets(ctx context.Context, db *sql.DB, cfg config) (map[string]struct{}, error) {
+func supportsBackfillGPSFallback(protocol envelope.Protocol) bool {
+	switch protocol {
+	case envelope.ProtocolGB32960, envelope.ProtocolJT808, envelope.ProtocolYutongMQTT:
+		return true
+	default:
+		return false
+	}
+}
+
+func backfillGPSPlatformName(protocol envelope.Protocol) string {
+	switch protocol {
+	case envelope.ProtocolGB32960:
+		return "GB32960 GPS轨迹估算"
+	case envelope.ProtocolYutongMQTT:
+		return "宇通 GPS轨迹估算"
+	default:
+		return "JT808 GPS轨迹估算"
+	}
+}
+
+func existingPositiveOdometerTargets(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol) (map[string]struct{}, error) {
 	result := map[string]struct{}{}
 	if db == nil {
 		return result, nil
 	}
 	rows, err := db.QueryContext(ctx, `SELECT DISTINCT vin, DATE_FORMAT(stat_date, '%Y-%m-%d')
 FROM vehicle_daily_mileage_source
-WHERE protocol = 'JT808'
+WHERE protocol = ?
  AND stat_date >= ? AND stat_date <= ?
  AND quality_status = ?
  AND COALESCE(quality_reason, '') <> ?
-  AND latest_total_mileage_km IS NOT NULL`, cfg.DateFrom, cfg.DateTo, stats.QualityOK, stats.QualityReasonGPSCoordinate)
+  AND daily_mileage_km > 0
+  AND latest_total_mileage_km IS NOT NULL`, string(protocol), cfg.DateFrom, cfg.DateTo, stats.QualityOK, stats.QualityReasonGPSCoordinate)
 	if err != nil {
 		return nil, err
 	}
@@ -1184,7 +1223,7 @@ func loadConfig() (config, error) {
 		Debug:             envBool("BACKFILL_DEBUG", false),
 		EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false),
 		StationaryCarry:   envBool("BACKFILL_STATIONARY_CARRY_FORWARD", false),
-		JT808GPSFallback:  envBool("BACKFILL_JT808_GPS_FALLBACK", true),
+		GPSFallback:       envBool("BACKFILL_GPS_FALLBACK", envBool("BACKFILL_JT808_GPS_FALLBACK", true)),
 		ProgressEvery:     int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
 		BaselineLookback:  envInt("BACKFILL_BASELINE_LOOKBACK_DAYS", 7),
 		Location:          loc,
--- a/go/vehicle-gateway/cmd/stats-backfill/main_test.go
+++ b/go/vehicle-gateway/cmd/stats-backfill/main_test.go
@@ -990,7 +990,7 @@ func TestResolveBackfillDateRangeDefaultsToToday(t *testing.T) {
 	}
 }

-func TestAddJT808GPSCoordinateFallbackAggregatesOnlyMissingOdometerVehicles(t *testing.T) {
+func TestAddGPSCoordinateFallbackAggregatesOnlyMissingPositiveOdometerVehicles(t *testing.T) {
 	mysqlDB, mysqlMock, err := sqlmock.New()
 	if err != nil {
 		t.Fatalf("mysql sqlmock.New() error = %v", err)
@@ -1003,7 +1003,7 @@ func TestAddJT808GPSCoordinateFallbackAggregatesOnlyMissingOdometerVehicles(t *t
 	defer tdDB.Close()

 	mysqlMock.ExpectQuery("SELECT DISTINCT vin, DATE_FORMAT").
-		WithArgs("2026-07-19", "2026-07-19", stats.QualityOK, stats.QualityReasonGPSCoordinate).
+		WithArgs("JT808", "2026-07-19", "2026-07-19", stats.QualityOK, stats.QualityReasonGPSCoordinate).
 		WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date"}).
 			AddRow("VIN-WITH-ODOMETER", "2026-07-19"))
 	loc := time.FixedZone("Asia/Shanghai", 8*3600)
@@ -1016,16 +1016,16 @@ func TestAddJT808GPSCoordinateFallbackAggregatesOnlyMissingOdometerVehicles(t *t
 			AddRow("VIN-WITH-ODOMETER", start.Add(5*time.Minute), 121.4837, 31.2304))

 	aggregates := map[string]*metricAgg{}
-	added, err := addJT808GPSCoordinateFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
+	added, err := addGPSCoordinateFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
 		DateFrom:         "2026-07-19",
 		DateTo:           "2026-07-19",
 		Protocols:        []envelope.Protocol{envelope.ProtocolJT808},
-		JT808GPSFallback: true,
+		GPSFallback:      true,
 		Location:         loc,
 		TDengineDatabase: "vehicle_ts",
 	}, aggregates)
 	if err != nil {
-		t.Fatalf("addJT808GPSCoordinateFallbackAggregates() error = %v", err)
+		t.Fatalf("addGPSCoordinateFallbackAggregates() error = %v", err)
 	}
 	if added != 1 || len(aggregates) != 1 {
 		t.Fatalf("added=%d aggregates=%d", added, len(aggregates))
@@ -1046,6 +1046,58 @@ func TestAddJT808GPSCoordinateFallbackAggregatesOnlyMissingOdometerVehicles(t *t
 	}
 }

+func TestAddGPSCoordinateFallbackAggregatesUsesMovingYutongPoints(t *testing.T) {
+	mysqlDB, mysqlMock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("mysql sqlmock.New() error = %v", err)
+	}
+	defer mysqlDB.Close()
+	tdDB, tdMock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("tdengine sqlmock.New() error = %v", err)
+	}
+	defer tdDB.Close()
+
+	mysqlMock.ExpectQuery("SELECT DISTINCT vin, DATE_FORMAT").
+		WithArgs("YUTONG_MQTT", "2026-07-19", "2026-07-19", stats.QualityOK, stats.QualityReasonGPSCoordinate).
+		WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date"}))
+	loc := time.FixedZone("Asia/Shanghai", 8*3600)
+	start := time.Date(2026, 7, 19, 8, 0, 0, 0, loc)
+	tdMock.ExpectQuery("protocol = 'YUTONG_MQTT'.*speed_kmh > 0").
+		WillReturnRows(sqlmock.NewRows([]string{"vin", "ts", "longitude", "latitude"}).
+			AddRow("VIN-YUTONG-GPS", start, 121.4737, 31.2304).
+			AddRow("VIN-YUTONG-GPS", start.Add(time.Minute), 121.4837, 31.2304))
+
+	aggregates := map[string]*metricAgg{}
+	added, err := addGPSCoordinateFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
+		DateFrom:         "2026-07-19",
+		DateTo:           "2026-07-19",
+		Protocols:        []envelope.Protocol{envelope.ProtocolYutongMQTT},
+		GPSFallback:      true,
+		Location:         loc,
+		TDengineDatabase: "vehicle_ts",
+	}, aggregates)
+	if err != nil {
+		t.Fatalf("addGPSCoordinateFallbackAggregates() error = %v", err)
+	}
+	if added != 1 || len(aggregates) != 1 {
+		t.Fatalf("added=%d aggregates=%d", added, len(aggregates))
+	}
+	for _, aggregate := range aggregates {
+		if aggregate.Protocol != envelope.ProtocolYutongMQTT ||
+			aggregate.PlatformName != "宇通 GPS轨迹估算" ||
+			aggregate.QualityReason != stats.QualityReasonGPSCoordinate {
+			t.Fatalf("aggregate = %+v", aggregate)
+		}
+	}
+	if err := mysqlMock.ExpectationsWereMet(); err != nil {
+		t.Fatalf("mysql expectations: %v", err)
+	}
+	if err := tdMock.ExpectationsWereMet(); err != nil {
+		t.Fatalf("tdengine expectations: %v", err)
+	}
+}
+
 func TestResolveBackfillDateRangeUsesRelativeWindow(t *testing.T) {
 	t.Setenv("BACKFILL_DATE_FROM", "")
 	t.Setenv("BACKFILL_DATE_TO", "")