feat: build vehicle data platform and production pipeline

2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
--- a/go/vehicle-gateway/cmd/stats-backfill/main.go
+++ b/go/vehicle-gateway/cmd/stats-backfill/main.go
@@ -22,20 +22,21 @@ import (
 )

 type config struct {
-	MySQLDSN         string
-	TDengineDriver   string
-	TDengineDSN      string
-	TDengineDatabase string
-	DateFrom         string
-	DateTo           string
-	Protocols        []envelope.Protocol
-	Method           string
-	Limit            int
-	DryRun           bool
-	Reset            bool
-	Debug            bool
-	ProgressEvery    int64
-	Location         *time.Location
+	MySQLDSN          string
+	TDengineDriver    string
+	TDengineDSN       string
+	TDengineDatabase  string
+	DateFrom          string
+	DateTo            string
+	Protocols         []envelope.Protocol
+	Method            string
+	Limit             int
+	DryRun            bool
+	Reset             bool
+	Debug             bool
+	EventTimeFullScan bool
+	ProgressEvery     int64
+	Location          *time.Location
 }

 type rawFrameRow struct {
@@ -62,6 +63,9 @@ type metricAgg struct {
 	Phone           string
 	DeviceID        string
 	SourceEndpoint  string
+	SourceCode      string
+	PlatformName    string
+	SourceKind      string
 	FirstEventTime  time.Time
 	LatestEventTime time.Time
 	QualityStatus   string
@@ -74,6 +78,9 @@ type dailySourceLast struct {
 	Phone          string
 	DeviceID       string
 	SourceEndpoint string
+	SourceCode     string
+	PlatformName   string
+	SourceKind     string
 	FirstTS        time.Time
 	TS             time.Time
 	FirstTotalKM   float64
@@ -81,8 +88,19 @@ type dailySourceLast struct {
 	RawSampleCount int64
 }

+type sourceHistoryID struct {
+	VIN       string
+	SourceKey string
+}
+
+var defaultBackfillEnvFiles = []string{
+	"/opt/lingniu-go-native/env/base.env",
+	"/opt/lingniu-go-native/env/stat-writer.env",
+}
+
 func main() {
-	if err := loadEnvFiles(env("BACKFILL_ENV_FILES", os.Getenv("BACKFILL_ENV_FILE"))); err != nil {
+	envFiles := backfillEnvFiles(os.Getenv("BACKFILL_ENV_FILES"), os.Getenv("BACKFILL_ENV_FILE"), defaultBackfillEnvFiles)
+	if err := loadEnvFiles(envFiles); err != nil {
 		fail("load env file", err)
 	}
 	cfg, err := loadConfig()
@@ -121,18 +139,27 @@ func main() {
 		slog.Info("reset stats rows", "deleted", deleted)
 	}
 	if cfg.Method == "last_diff" {
-		aggregates, err := buildLastDiffAggregates(ctx, td, cfg)
+		aggregates, err := buildLastDiffAggregates(ctx, mysqlDB, td, cfg)
 		if err != nil {
 			fail("build last-diff aggregates", err)
 		}
+		fallbacks, err := addRealtimeLocationFallbackAggregates(ctx, mysqlDB, td, cfg, aggregates)
+		if err != nil {
+			fail("build realtime-location fallback aggregates", err)
+		}
 		var written int64
+		var normalized int
 		if !cfg.DryRun {
 			written, err = writeAggregates(ctx, mysqlDB, aggregates, 500)
 			if err != nil {
 				fail("write aggregates", err)
 			}
+			normalized, err = normalizeHistoricalPlatformSources(ctx, mysqlDB, cfg)
+			if err != nil {
+				fail("normalize historical platform sources", err)
+			}
 		}
-		slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "written", written)
+		slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "realtimeLocationFallbacks", fallbacks, "written", written, "platformSourcesNormalized", normalized)
 		return
 	}

@@ -234,6 +261,7 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample)
 				Phone:           sample.Phone,
 				DeviceID:        sample.DeviceID,
 				SourceEndpoint:  sample.SourceEndpoint,
+				PlatformName:    sample.PlatformName,
 				FirstEventTime:  sample.EventTime,
 				LatestEventTime: sample.EventTime,
 				QualityStatus:   stats.QualityOK,
@@ -255,6 +283,9 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample)
 			if strings.TrimSpace(sample.DeviceID) != "" {
 				agg.DeviceID = sample.DeviceID
 			}
+			if strings.TrimSpace(sample.PlatformName) != "" {
+				agg.PlatformName = sample.PlatformName
+			}
 		}
 		agg.Count++
 	}
@@ -271,6 +302,9 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
 			Protocol:       agg.Protocol,
 			SourceIP:       stats.NormalizeSourceIP(agg.SourceEndpoint),
 			SourceEndpoint: agg.SourceEndpoint,
+			SourceCode:     agg.SourceCode,
+			PlatformName:   agg.PlatformName,
+			SourceKind:     agg.SourceKind,
 		}
 		if strings.TrimSpace(identity.SourceIP) == "" {
 			continue
@@ -285,7 +319,7 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
 		if err := stats.UpsertDataSource(ctx, db, identity, agg.LatestEventTime); err != nil {
 			return written, err
 		}
-		dailyKM := agg.LatestKM - agg.FirstKM
+		dailyKM := stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM)
 		candidate := stats.SourceMileageSample{
 			VIN:             agg.VIN,
 			StatDate:        agg.Date,
@@ -295,6 +329,7 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
 			SourceEndpoint:  agg.SourceEndpoint,
 			Phone:           agg.Phone,
 			DeviceID:        agg.DeviceID,
+			PlatformName:    agg.PlatformName,
 			FirstTotalKM:    agg.FirstKM,
 			LatestTotalKM:   agg.LatestKM,
 			DailyKM:         dailyKM,
@@ -310,10 +345,7 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
 		if candidate.QualityReason == "" {
 			candidate.QualityReason = "same_source_previous_day"
 		}
-		if candidate.QualityStatus == stats.QualityOK && (candidate.DailyKM < 0 || candidate.DailyKM > maxTrustedDailyMileageKM) {
-			candidate.QualityStatus = stats.QualityInvalidDelta
-			candidate.QualityReason = "outside_daily_range"
-		}
+		stats.ApplyMileageQualityRules(&candidate)
 		if err := stats.UpsertSourceMileage(ctx, db, candidate); err != nil {
 			return written, err
 		}
@@ -325,6 +357,27 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
 	return written, nil
 }

+func normalizeHistoricalPlatformSources(ctx context.Context, db *sql.DB, cfg config) (int, error) {
+	dates, err := dateRange(cfg.DateFrom, cfg.DateTo)
+	if err != nil {
+		return 0, err
+	}
+	normalized := 0
+	for _, protocol := range cfg.Protocols {
+		if protocol != envelope.ProtocolJT808 {
+			continue
+		}
+		for _, date := range dates {
+			count, err := stats.NormalizePlatformSourceMileageForDate(ctx, db, date, protocol)
+			if err != nil {
+				return normalized, err
+			}
+			normalized += count
+		}
+	}
+	return normalized, nil
+}
+
 func clearBackfillTargetMileage(ctx context.Context, db *sql.DB, vin string, statDate string, protocol envelope.Protocol) error {
 	if db == nil || strings.TrimSpace(vin) == "" || strings.TrimSpace(statDate) == "" || strings.TrimSpace(string(protocol)) == "" {
 		return nil
@@ -337,47 +390,173 @@ func clearBackfillTargetMileage(ctx context.Context, db *sql.DB, vin string, sta
 	return err
 }

-func buildLastDiffAggregates(ctx context.Context, db *sql.DB, cfg config) (map[string]*metricAgg, error) {
+func buildLastDiffAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config) (map[string]*metricAgg, error) {
 	targetDates, err := dateRange(cfg.DateFrom, cfg.DateTo)
 	if err != nil {
 		return nil, err
 	}
 	aggregates := map[string]*metricAgg{}
 	for _, protocol := range cfg.Protocols {
+		latestHistory := map[sourceHistoryID]dailySourceLast{}
+		preWindow, err := queryPreviousLastSourceRows(ctx, tdDB, cfg, protocol, targetDates[0])
+		if err != nil {
+			return nil, err
+		}
+		rememberLatestSourceRows(latestHistory, preWindow)
+		slog.Info("pre-window baseline loaded", "protocol", protocol, "before_date", targetDates[0], "vehicles", len(preWindow))
 		for _, date := range targetDates {
-			current, err := queryDailyLastSourceRows(ctx, db, cfg, protocol, date)
+			current, err := queryDailyLastSourceRows(ctx, tdDB, cfg, protocol, date)
 			if err != nil {
 				return nil, err
 			}
-			previous, err := queryPreviousLastSourceRows(ctx, db, cfg, protocol, date)
-			if err != nil {
-				return nil, err
-			}
-			slog.Info("daily last loaded", "protocol", protocol, "date", date, "vehicles", len(current), "previousVehicles", len(previous))
+			slog.Info("daily last loaded", "protocol", protocol, "date", date, "vehicles", len(current), "historicalSources", len(latestHistory))
 			for vin, currentSources := range current {
-				previousBySource := map[string]dailySourceLast{}
-				for _, previousRow := range previous[vin] {
-					previousBySource[previousRow.SourceKey] = previousRow
-				}
 				for _, currentRow := range currentSources {
 					key := vin + "|" + date + "|" + string(protocol) + "|" + currentRow.SourceKey
-					previousRow, hasPrevious := previousBySource[currentRow.SourceKey]
+					previousRow, hasPrevious, err := resolvePreviousSourceRow(ctx, mysqlDB, date, protocol, latestHistory, currentRow)
+					if err != nil {
+						return nil, err
+					}
 					aggregates[key] = aggregateFromDailySource(date, protocol, currentRow, previousRow, hasPrevious)
 				}
 			}
+			// Current-day last values become eligible only for later target dates.
+			rememberLatestSourceRows(latestHistory, current)
 		}
 	}
 	return aggregates, nil
 }

+func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
+	if aggregates == nil {
+		return 0, fmt.Errorf("aggregates map is nil")
+	}
+	targetDates, err := dateRange(cfg.DateFrom, cfg.DateTo)
+	if err != nil {
+		return 0, err
+	}
+	var added int
+	for _, protocol := range cfg.Protocols {
+		if protocol != envelope.ProtocolYutongMQTT {
+			continue
+		}
+		latestHistory := map[sourceHistoryID]dailySourceLast{}
+		preWindow, err := queryPreviousLastSourceRows(ctx, tdDB, cfg, protocol, targetDates[0])
+		if err != nil {
+			return added, err
+		}
+		rememberLatestSourceRows(latestHistory, preWindow)
+		aggregateRowsByDate := indexAggregateSourceRowsByDate(aggregates, protocol)
+		for _, date := range targetDates {
+			current, err := queryRealtimeLocationLastRows(ctx, mysqlDB, cfg, protocol, date)
+			if err != nil {
+				return added, err
+			}
+			for vin, currentSources := range current {
+				for _, currentRow := range currentSources {
+					key := vin + "|" + date + "|" + string(protocol) + "|" + currentRow.SourceKey
+					if _, exists := aggregates[key]; exists {
+						continue
+					}
+					previousRow, hasPrevious, err := resolvePreviousSourceRow(ctx, mysqlDB, date, protocol, latestHistory, currentRow)
+					if err != nil {
+						return added, err
+					}
+					agg := aggregateFromDailySource(date, protocol, currentRow, previousRow, hasPrevious)
+					if hasPrevious {
+						agg.QualityReason = "realtime_location_fallback_historical_baseline"
+					} else {
+						agg.QualityReason = "realtime_location_fallback_current_day_first_baseline"
+					}
+					aggregates[key] = agg
+					added++
+				}
+			}
+			rememberLatestSourceRows(latestHistory, aggregateRowsByDate[date])
+			rememberLatestSourceRows(latestHistory, current)
+			if len(current) > 0 {
+				slog.Info("realtime location fallback loaded", "protocol", protocol, "date", date, "vehicles", len(current), "added", added)
+			}
+		}
+	}
+	return added, nil
+}
+
+func indexAggregateSourceRowsByDate(aggregates map[string]*metricAgg, protocol envelope.Protocol) map[string]map[string][]dailySourceLast {
+	indexed := map[string]map[string][]dailySourceLast{}
+	for _, agg := range aggregates {
+		if agg == nil || agg.Protocol != protocol || strings.TrimSpace(agg.Date) == "" {
+			continue
+		}
+		rows := indexed[agg.Date]
+		if rows == nil {
+			rows = map[string][]dailySourceLast{}
+			indexed[agg.Date] = rows
+		}
+		rows[agg.VIN] = append(rows[agg.VIN], dailySourceLast{
+			VIN:            agg.VIN,
+			SourceKey:      agg.SourceKey,
+			Phone:          agg.Phone,
+			DeviceID:       agg.DeviceID,
+			SourceEndpoint: agg.SourceEndpoint,
+			SourceCode:     agg.SourceCode,
+			PlatformName:   agg.PlatformName,
+			SourceKind:     agg.SourceKind,
+			FirstTS:        agg.LatestEventTime,
+			TS:             agg.LatestEventTime,
+			FirstTotalKM:   agg.LatestKM,
+			TotalKM:        agg.LatestKM,
+			RawSampleCount: agg.Count,
+		})
+	}
+	return indexed
+}
+
+func rememberLatestSourceRows(history map[sourceHistoryID]dailySourceLast, rows map[string][]dailySourceLast) {
+	for vin, sourceRows := range rows {
+		for _, row := range sourceRows {
+			rowVIN := strings.TrimSpace(row.VIN)
+			if rowVIN == "" {
+				rowVIN = strings.TrimSpace(vin)
+				row.VIN = rowVIN
+			}
+			id := sourceHistoryID{VIN: rowVIN, SourceKey: strings.TrimSpace(row.SourceKey)}
+			if id.VIN == "" || id.SourceKey == "" || row.TS.IsZero() {
+				continue
+			}
+			if existing, ok := history[id]; !ok || row.TS.After(existing.TS) {
+				history[id] = row
+			}
+		}
+	}
+}
+
+func latestHistoricalSourceRow(history map[sourceHistoryID]dailySourceLast, current dailySourceLast) (dailySourceLast, bool) {
+	id := sourceHistoryID{
+		VIN:       strings.TrimSpace(current.VIN),
+		SourceKey: strings.TrimSpace(current.SourceKey),
+	}
+	previous, ok := history[id]
+	return previous, ok && previous.TS.Before(current.TS)
+}
+
+func resolvePreviousSourceRow(ctx context.Context, db *sql.DB, date string, protocol envelope.Protocol, history map[sourceHistoryID]dailySourceLast, current dailySourceLast) (dailySourceLast, bool, error) {
+	if previous, ok := latestHistoricalSourceRow(history, current); ok {
+		return previous, true, nil
+	}
+	return queryDurablePreviousSourceRow(ctx, db, date, protocol, current)
+}
+
 func aggregateFromDailySource(date string, protocol envelope.Protocol, current dailySourceLast, previous dailySourceLast, hasPrevious bool) *metricAgg {
 	firstKM := current.FirstTotalKM
 	firstEventTime := current.FirstTS
-	qualityReason := "current_day_first_sample"
+	qualityStatus := stats.QualityOK
+	qualityReason := stats.QualityReasonCurrentDayFirst
 	if hasPrevious {
 		firstKM = previous.TotalKM
 		firstEventTime = previous.TS
-		qualityReason = "historical_source_baseline"
+		qualityStatus = stats.QualityOK
+		qualityReason = stats.QualityReasonHistorical
 	}
 	if firstEventTime.IsZero() {
 		firstEventTime = current.TS
@@ -386,7 +565,7 @@ func aggregateFromDailySource(date string, protocol envelope.Protocol, current d
 	if count <= 0 {
 		count = 1
 	}
-	return &metricAgg{
+	agg := &metricAgg{
 		VIN:             current.VIN,
 		Date:            date,
 		Protocol:        protocol,
@@ -397,11 +576,43 @@ func aggregateFromDailySource(date string, protocol envelope.Protocol, current d
 		Phone:           current.Phone,
 		DeviceID:        current.DeviceID,
 		SourceEndpoint:  current.SourceEndpoint,
+		SourceCode:      current.SourceCode,
+		PlatformName:    current.PlatformName,
+		SourceKind:      current.SourceKind,
 		FirstEventTime:  firstEventTime,
 		LatestEventTime: current.TS,
-		QualityStatus:   stats.QualityOK,
+		QualityStatus:   qualityStatus,
 		QualityReason:   qualityReason,
 	}
+	candidate := stats.SourceMileageSample{
+		FirstTotalKM:    agg.FirstKM,
+		LatestTotalKM:   agg.LatestKM,
+		DailyKM:         stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM),
+		FirstEventTime:  agg.FirstEventTime,
+		LatestEventTime: agg.LatestEventTime,
+		QualityStatus:   agg.QualityStatus,
+		QualityReason:   agg.QualityReason,
+	}
+	stats.ApplyMileageQualityRules(&candidate)
+	agg.QualityStatus = candidate.QualityStatus
+	agg.QualityReason = candidate.QualityReason
+	return agg
+}
+
+func queryDurablePreviousSourceRow(ctx context.Context, db *sql.DB, date string, protocol envelope.Protocol, current dailySourceLast) (dailySourceLast, bool, error) {
+	if db == nil {
+		return dailySourceLast{}, false, nil
+	}
+	totalKM, eventTime, found, err := stats.LookupLatestSourceBaselineBefore(ctx, db, current.VIN, date, protocol, current.SourceKey)
+	if err != nil || !found {
+		return dailySourceLast{}, found, err
+	}
+	previous := current
+	previous.FirstTS = eventTime
+	previous.TS = eventTime
+	previous.FirstTotalKM = totalKM
+	previous.TotalKM = totalKM
+	return previous, true, nil
 }

 type trustedChoice struct {
@@ -409,8 +620,6 @@ type trustedChoice struct {
 	previous dailySourceLast
 }

-const maxTrustedDailyMileageKM = 1000
-
 func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast) (trustedChoice, bool) {
 	previousBySource := map[string]dailySourceLast{}
 	for _, row := range previous {
@@ -423,8 +632,8 @@ func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast)
 		if !ok {
 			continue
 		}
-		delta := currentRow.TotalKM - previousRow.TotalKM
-		if delta < 0 || delta > maxTrustedDailyMileageKM {
+		delta, ok, _ := stats.NormalizeDailyMileageDeltaForWindow(stats.DailyMileageFromDayBoundary(previousRow.TotalKM, currentRow.TotalKM), previousRow.TS, currentRow.TS)
+		if !ok {
 			continue
 		}
 		if chosen.current.SourceKey == "" || delta < chosenDelta {
@@ -436,19 +645,18 @@ func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast)
 }

 func queryDailyLastSourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
-	where := []string{
-		fmt.Sprintf("ts >= '%s 00:00:00'", quote(date)),
-		fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(date))),
+	where := backfillTimePredicates(cfg, date, nextDate(date))
+	where = append(where,
 		"parse_status = 'OK'",
 		"vin IS NOT NULL",
 		"vin <> ''",
 		fmt.Sprintf("protocol = '%s'", quote(string(protocol))),
 		realtimeMileageFramePredicate(),
-	}
+	)
 	if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
 		where = append(where, predicate)
 	}
-	sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, FIRST(ts), FIRST(parsed_json), LAST(ts), LAST(parsed_json), COUNT(*)
+	sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, FIRST(event_time), FIRST(parsed_json), LAST(event_time), LAST(parsed_json), COUNT(*)
 FROM %s.raw_frames
 WHERE %s
 GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
@@ -456,24 +664,103 @@ GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), s
 }

 func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
-	where := []string{
-		fmt.Sprintf("ts < '%s 00:00:00'", quote(date)),
+	// The baseline is the nearest earlier sample, not necessarily yesterday's.
+	// LAST aggregates the complete pre-window history once per source so empty
+	// calendar days do not make a backfill fall back to the current day's first
+	// sample.
+	where := backfillBeforePredicates(date)
+	where = append(where,
 		"parse_status = 'OK'",
 		"vin IS NOT NULL",
 		"vin <> ''",
 		fmt.Sprintf("protocol = '%s'", quote(string(protocol))),
 		realtimeMileageFramePredicate(),
-	}
+	)
 	if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
 		where = append(where, predicate)
 	}
-	sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, LAST(ts), LAST(parsed_json), COUNT(*)
+	sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, LAST(event_time), LAST(parsed_json), COUNT(*)
 FROM %s.raw_frames
 WHERE %s
 GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
 	return querySourceRows(ctx, db, cfg, protocol, sqlText, false)
 }

+func backfillBeforePredicates(eventDateExclusive string) []string {
+	return []string{
+		fmt.Sprintf("event_time < '%s 00:00:00'", quote(eventDateExclusive)),
+	}
+}
+
+func queryRealtimeLocationLastRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
+	if db == nil {
+		return nil, nil
+	}
+	loc := cfg.Location
+	if loc == nil {
+		loc = time.FixedZone("Asia/Shanghai", 8*3600)
+	}
+	sqlText := `SELECT l.vin, COALESCE(NULLIF(s.peer, ''), ''), l.total_mileage_event_time, l.total_mileage_km
+FROM vehicle_realtime_location l
+LEFT JOIN vehicle_realtime_snapshot s ON s.protocol = l.protocol AND s.vin = l.vin
+WHERE l.protocol = ?
+  AND l.vin IS NOT NULL AND l.vin <> ''
+  AND l.total_mileage_km IS NOT NULL AND l.total_mileage_km > 0
+  AND l.total_mileage_event_time >= ?
+  AND l.total_mileage_event_time < DATE_ADD(?, INTERVAL 1 DAY)`
+	rows, err := db.QueryContext(ctx, sqlText, string(protocol), date, date)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[string][]dailySourceLast{}
+	for rows.Next() {
+		var vin string
+		var peer sql.NullString
+		var eventTime time.Time
+		var totalKM float64
+		if err := rows.Scan(&vin, &peer, &eventTime, &totalKM); err != nil {
+			return nil, err
+		}
+		vin = strings.TrimSpace(vin)
+		if vin == "" || totalKM <= 0 {
+			continue
+		}
+		sourceEndpoint := strings.TrimSpace(peer.String)
+		if sourceEndpoint == "" && protocol == envelope.ProtocolYutongMQTT {
+			sourceEndpoint = "mqtt://yutong/realtime-location"
+		}
+		deviceID := ""
+		if protocol == envelope.ProtocolYutongMQTT {
+			deviceID = vin
+		}
+		sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint)
+		sourceKey := normalizedSourceKeyForSource(string(protocol), "", deviceID, sourceEndpoint, sourceKind, sourceCode)
+		if sourceKey == "" {
+			continue
+		}
+		row := dailySourceLast{
+			VIN:            vin,
+			SourceKey:      sourceKey,
+			DeviceID:       deviceID,
+			SourceEndpoint: sourceEndpoint,
+			SourceCode:     sourceCode,
+			PlatformName:   platformName,
+			SourceKind:     sourceKind,
+			FirstTS:        eventTime.In(loc),
+			TS:             eventTime.In(loc),
+			FirstTotalKM:   totalKM,
+			TotalKM:        totalKM,
+			RawSampleCount: 1,
+		}
+		out[vin] = append(out[vin], row)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, sqlText string, includeFirst bool) (map[string][]dailySourceLast, error) {
 	rows, err := db.QueryContext(ctx, sqlText)
 	if err != nil {
@@ -512,7 +799,8 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
 				firstTotalKM = parsedFirst
 			}
 		}
-		sourceKey := normalizedSourceKey(string(protocol), phone, deviceID, sourceEndpoint)
+		sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint)
+		sourceKey := normalizedSourceKeyForSource(string(protocol), phone, deviceID, sourceEndpoint, sourceKind, sourceCode)
 		if sourceKey == "" {
 			continue
 		}
@@ -522,8 +810,11 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
 			Phone:          strings.TrimSpace(phone),
 			DeviceID:       strings.TrimSpace(deviceID),
 			SourceEndpoint: strings.TrimSpace(sourceEndpoint),
+			SourceCode:     sourceCode,
+			PlatformName:   platformName,
+			SourceKind:     sourceKind,
 			FirstTS:        firstTS.In(cfg.Location),
-			TS:             ts,
+			TS:             ts.In(cfg.Location),
 			FirstTotalKM:   firstTotalKM,
 			TotalKM:        latestTotalKM,
 			RawSampleCount: rawSampleCount,
@@ -562,8 +853,19 @@ func mileageFromParsed(protocol envelope.Protocol, vin string, parsedJSON string
 }

 func normalizedSourceKey(protocol string, phone string, deviceID string, endpoint string) string {
+	return normalizedSourceKeyForSource(protocol, phone, deviceID, endpoint, "", "")
+}
+
+func normalizedSourceKeyForSource(protocol string, phone string, deviceID string, endpoint string, sourceKind string, sourceCode string) string {
 	sourceIP := stats.NormalizeSourceIP(endpoint)
-	return stats.SourceKey(envelope.Protocol(protocol), phone, deviceID, sourceIP)
+	return stats.SourceKeyForSource(envelope.Protocol(protocol), phone, deviceID, sourceIP, sourceKind, sourceCode)
+}
+
+func knownPlatformSourceMetadata(protocol envelope.Protocol, endpoint string) (sourceCode string, platformName string, sourceKind string) {
+	if protocol == envelope.ProtocolYutongMQTT && stats.NormalizeSourceIP(endpoint) == "mqtt" {
+		return "yutong", "宇通", "PLATFORM"
+	}
+	return "", "", ""
 }

 func fieldsForStats(protocol envelope.Protocol, vin string, text string) map[string]any {
@@ -574,7 +876,7 @@ func fieldsForStats(protocol envelope.Protocol, vin string, text string) map[str
 	}
 	parsed := map[string]any{}
 	if err := json.Unmarshal([]byte(text), &parsed); err == nil && len(parsed) > 0 {
-		if flattened, _, ok := realtime.ParsedFieldsForEnvelope(envelope.FrameEnvelope{
+		if flattened, _, ok := realtime.ComputeParsedFieldsForEnvelope(envelope.FrameEnvelope{
 			Protocol: protocol,
 			VIN:      vin,
 			Parsed:   parsed,
@@ -604,7 +906,12 @@ func mileageKeys(protocol envelope.Protocol) []string {
 	case envelope.ProtocolJT808:
 		return []string{"jt808.location.total_mileage_km"}
 	case envelope.ProtocolYutongMQTT:
-		return []string{"yutong_mqtt.data.total_mileage", "yutong_mqtt.root.data.total_mileage"}
+		return []string{
+			"yutong_mqtt.data.total_mileage_km",
+			"yutong_mqtt.root.data.total_mileage_km",
+			"yutong_mqtt.data.total_mileage",
+			"yutong_mqtt.root.data.total_mileage",
+		}
 	default:
 		return nil
 	}
@@ -624,38 +931,67 @@ func loadConfig() (config, error) {
 	if err != nil {
 		loc = time.FixedZone("Asia/Shanghai", 8*3600)
 	}
-	dateTo := env("BACKFILL_DATE_TO", time.Now().In(loc).Format("2006-01-02"))
-	dateFrom := env("BACKFILL_DATE_FROM", dateTo)
+	dateFrom, dateTo := resolveBackfillDateRange(time.Now(), loc)
 	protocols, err := parseProtocols(env("BACKFILL_PROTOCOLS", "GB32960,JT808,YUTONG_MQTT"))
 	if err != nil {
 		return config{}, err
 	}
 	return config{
-		MySQLDSN:         env("MYSQL_DSN", ""),
-		TDengineDriver:   env("TDENGINE_DRIVER", "taosWS"),
-		TDengineDSN:      env("TDENGINE_DSN", ""),
-		TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
-		DateFrom:         dateFrom,
-		DateTo:           dateTo,
-		Protocols:        protocols,
-		Method:           env("BACKFILL_METHOD", "last_diff"),
-		Limit:            envInt("BACKFILL_LIMIT", 0),
-		DryRun:           envBool("BACKFILL_DRY_RUN", true),
-		Reset:            envBool("BACKFILL_RESET", false),
-		Debug:            envBool("BACKFILL_DEBUG", false),
-		ProgressEvery:    int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
-		Location:         loc,
+		MySQLDSN:          env("MYSQL_DSN", ""),
+		TDengineDriver:    env("TDENGINE_DRIVER", "taosWS"),
+		TDengineDSN:       env("TDENGINE_DSN", ""),
+		TDengineDatabase:  env("TDENGINE_DATABASE", history.DefaultDatabase),
+		DateFrom:          dateFrom,
+		DateTo:            dateTo,
+		Protocols:         protocols,
+		Method:            env("BACKFILL_METHOD", "last_diff"),
+		Limit:             envInt("BACKFILL_LIMIT", 0),
+		DryRun:            envBool("BACKFILL_DRY_RUN", true),
+		Reset:             envBool("BACKFILL_RESET", false),
+		Debug:             envBool("BACKFILL_DEBUG", false),
+		EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false),
+		ProgressEvery:     int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
+		Location:          loc,
 	}, nil
 }

+func resolveBackfillDateRange(now time.Time, loc *time.Location) (string, string) {
+	if loc == nil {
+		loc = time.FixedZone("Asia/Shanghai", 8*3600)
+	}
+	explicitFrom := strings.TrimSpace(os.Getenv("BACKFILL_DATE_FROM"))
+	explicitTo := strings.TrimSpace(os.Getenv("BACKFILL_DATE_TO"))
+	if explicitTo != "" || explicitFrom != "" {
+		dateTo := explicitTo
+		if dateTo == "" {
+			dateTo = now.In(loc).Format("2006-01-02")
+		}
+		dateFrom := explicitFrom
+		if dateFrom == "" {
+			dateFrom = dateTo
+		}
+		return dateFrom, dateTo
+	}
+	daysBack := envInt("BACKFILL_DAYS_BACK", 0)
+	if daysBack < 0 {
+		daysBack = 0
+	}
+	windowDays := envInt("BACKFILL_WINDOW_DAYS", 1)
+	if windowDays < 1 {
+		windowDays = 1
+	}
+	dateToTime := now.In(loc).AddDate(0, 0, -daysBack)
+	dateFromTime := dateToTime.AddDate(0, 0, -(windowDays - 1))
+	return dateFromTime.Format("2006-01-02"), dateToTime.Format("2006-01-02")
+}
+
 func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, error) {
-	where := []string{
-		fmt.Sprintf("ts >= '%s 00:00:00'", quote(cfg.DateFrom)),
-		fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(cfg.DateTo))),
+	where := backfillTimePredicates(cfg, cfg.DateFrom, nextDate(cfg.DateTo))
+	where = append(where,
 		"parse_status = 'OK'",
 		"vin IS NOT NULL",
 		"vin <> ''",
-	}
+	)
 	if len(cfg.Protocols) > 0 {
 		quoted := make([]string, 0, len(cfg.Protocols))
 		for _, protocol := range cfg.Protocols {
@@ -667,7 +1003,7 @@ func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, err
 	sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json
 FROM %s.raw_frames
 WHERE %s
-ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
+	ORDER BY event_time ASC, ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
 	if cfg.Limit > 0 {
 		sqlText += fmt.Sprintf(" LIMIT %d", cfg.Limit)
 	}
@@ -675,6 +1011,23 @@ ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
 	return db.QueryContext(ctx, sqlText)
 }

+func backfillTimePredicates(cfg config, eventDateFrom string, eventDateToExclusive string) []string {
+	where := make([]string, 0, 4)
+	if !cfg.EventTimeFullScan {
+		// ts is the TDengine primary timestamp and may be adjusted slightly to keep
+		// rows unique. Use it only as a broad index-friendly window; event_time is
+		// the protocol business boundary used for the final natural-day result.
+		where = append(where,
+			fmt.Sprintf("ts >= '%s 00:00:00'", quote(previousDate(eventDateFrom))),
+			fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(eventDateToExclusive))),
+		)
+	}
+	return append(where,
+		fmt.Sprintf("event_time >= '%s 00:00:00'", quote(eventDateFrom)),
+		fmt.Sprintf("event_time < '%s 00:00:00'", quote(eventDateToExclusive)),
+	)
+}
+
 func realtimeMileageFramePredicate() string {
 	return `(
  (protocol = 'GB32960' AND message_id IN (2,3))
@@ -684,17 +1037,39 @@ func realtimeMileageFramePredicate() string {
 }

 func mileageBearingFramePredicate(protocol envelope.Protocol) string {
-	keys := mileageKeys(protocol)
-	if len(keys) == 0 {
+	tokens := mileageSearchTokens(protocol)
+	if len(tokens) == 0 {
 		return ""
 	}
-	conditions := make([]string, 0, len(keys))
-	for _, key := range keys {
-		conditions = append(conditions, fmt.Sprintf("parsed_json LIKE '%%%s%%'", quote(key)))
+	conditions := make([]string, 0, len(tokens))
+	for _, token := range tokens {
+		conditions = append(conditions, fmt.Sprintf("parsed_json LIKE '%%%s%%'", quote(token)))
 	}
 	return "(" + strings.Join(conditions, " OR ") + ")"
 }

+func mileageSearchTokens(protocol envelope.Protocol) []string {
+	tokens := append([]string{}, mileageKeys(protocol)...)
+	switch protocol {
+	case envelope.ProtocolYutongMQTT:
+		tokens = append(tokens, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km")
+	}
+	seen := make(map[string]struct{}, len(tokens))
+	out := make([]string, 0, len(tokens))
+	for _, token := range tokens {
+		token = strings.TrimSpace(token)
+		if token == "" {
+			continue
+		}
+		if _, ok := seen[token]; ok {
+			continue
+		}
+		seen[token] = struct{}{}
+		out = append(out, token)
+	}
+	return out
+}
+
 func scanRawFrame(rows *sql.Rows) (rawFrameRow, error) {
 	var protocol, vin, phone, deviceID, sourceEndpoint, eventID, parsed string
 	var messageID int64
@@ -804,6 +1179,26 @@ func loadEnvFiles(paths string) error {
 	return nil
 }

+func backfillEnvFiles(explicitFiles string, explicitFile string, defaults []string) string {
+	if value := strings.TrimSpace(explicitFiles); value != "" {
+		return value
+	}
+	if value := strings.TrimSpace(explicitFile); value != "" {
+		return value
+	}
+	var existing []string
+	for _, path := range defaults {
+		path = strings.TrimSpace(path)
+		if path == "" {
+			continue
+		}
+		if _, err := os.Stat(path); err == nil {
+			existing = append(existing, path)
+		}
+	}
+	return strings.Join(existing, ",")
+}
+
 func loadEnvFile(path string) error {
 	path = strings.TrimSpace(path)
 	if path == "" {