feat: sync vehicle monitoring and accumulated platform updates

Add monitoring metrics, numeric sorting, report-time display, vehicle number labels, and map layout fixes. Include accumulated mileage, hydrogen, OneOS authentication, open platform, ingestion, deployment, and UI updates. Exclude temporary files, generated reports, and credentials. Validate frontend 632 tests and production build, API packages, vehicle-map tests, and release installer tests. Gateway suite has one pre-existing date-sensitive identity throttling test failure reproduced on the parent commit.
2026-10-26 18:05:59 +08:00
parent 8b32ae2988
commit d6b957dfc8
290 changed files with 28238 additions and 1647 deletions
@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log/slog"
 	"os"
@@ -43,6 +44,7 @@ type config struct {
 	ListHydrogenVINs  bool
 	ListActiveVINs    bool
 	Location          *time.Location
+	SourceCatalog     map[backfillSourceID]stats.SourceIdentity
 }

 type rawFrameRow struct {
@@ -150,9 +152,27 @@ func main() {
 		return
 	}

-	schemaWriter := stats.NewWriter(mysqlDB, cfg.Location)
-	if err := schemaWriter.EnsureSchema(ctx); err != nil {
-		fail("ensure schema", err)
+	if envBool("CUSTOMER_HYDROGEN_WORKER", false) {
+		if err := runCustomerHydrogenWorker(ctx, mysqlDB, td, cfg); err != nil {
+			fail("customer hydrogen worker", err)
+		}
+		return
+	}
+	if envBool("BACKFILL_HYDROGEN_ONLY", false) {
+		if err := runCustomerHydrogenBackfill(ctx, mysqlDB, td, cfg); err != nil {
+			fail("customer hydrogen backfill", err)
+		}
+		return
+	}
+	cfg.SourceCatalog, err = loadBackfillSourceCatalog(ctx, mysqlDB)
+	if err != nil {
+		fail("load backfill source catalog", err)
+	}
+	if !cfg.DryRun {
+		schemaWriter := stats.NewWriter(mysqlDB, cfg.Location)
+		if err := schemaWriter.EnsureSchema(ctx); err != nil {
+			fail("ensure schema", err)
+		}
 	}
 	if cfg.Reset && !cfg.DryRun {
 		deleted, err := resetStats(ctx, mysqlDB, cfg)
@@ -178,6 +198,9 @@ func main() {
 		if err != nil {
 			fail("cleanup sub-threshold GPS mileage candidates", err)
 		}
+		if cfg.Debug {
+			logBackfillCandidates(aggregates)
+		}
 		var written int64
 		var normalized int
 		if !cfg.DryRun {
@@ -250,6 +273,7 @@ func main() {
 			ParsedFields:   fields,
 			ParseStatus:    envelope.ParseOK,
 		}
+		env.SourceCode, env.PlatformName, env.SourceKind = backfillSourceMetadata(cfg, row.Protocol, row.SourceEndpoint)
 		samples, err := stats.SamplesFromEnvelope(env, cfg.Location)
 		if err != nil {
 			slog.Warn("skip invalid sample", "event_id", row.EventID, "error", err)
@@ -347,6 +371,23 @@ func activeModeSampleCount(sample stats.MetricSample) int64 {
 }

 func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*metricAgg, batchSize int) (int64, error) {
+	gps, other := map[string]*metricAgg{}, map[string]*metricAgg{}
+	for key, agg := range aggregates {
+		if agg.Protocol == envelope.ProtocolJT808 {
+			gps[key] = agg
+		} else {
+			other[key] = agg
+		}
+	}
+	written, err := writeProtectedGPSAggregates(ctx, db, gps)
+	if err != nil {
+		return written, err
+	}
+	n, err := writeLegacyAggregates(ctx, db, other, batchSize)
+	return written + n, err
+}
+
+func writeLegacyAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*metricAgg, batchSize int) (int64, error) {
 	if len(aggregates) == 0 {
 		return 0, nil
 	}
@@ -429,11 +470,25 @@ func normalizeHistoricalPlatformSources(ctx context.Context, db *sql.DB, cfg con
 			continue
 		}
 		for _, date := range dates {
-			count, err := stats.NormalizePlatformSourceMileageForDate(ctx, db, date, protocol)
+			if len(cfg.VINs) > 0 {
+				for _, vin := range cfg.VINs {
+					if err := normalizeProtectedGPSDay(ctx, db, vin, date); err != nil {
+						return normalized, err
+					}
+					normalized++
+				}
+				continue
+			}
+			vins, err := stats.PlatformSourceMileageLegacyVINs(ctx, db, date, protocol)
 			if err != nil {
 				return normalized, err
 			}
-			normalized += count
+			for _, vin := range vins {
+				if err := normalizeProtectedGPSDay(ctx, db, vin, date); err != nil {
+					return normalized, err
+				}
+				normalized++
+			}
 		}
 	}
 	return normalized, nil
@@ -592,7 +647,8 @@ FROM vehicle_daily_mileage_source
 WHERE stat_date >= ? AND stat_date <= ?
  AND quality_reason = ?
  AND daily_mileage_km >= 0 AND daily_mileage_km < ?
-  AND protocol IN (` + strings.Join(placeholders, ",") + `)
+  AND protocol IN (` + strings.Join(placeholders, ",") + `)`
+	query = appendBackfillVINScope(query, cfg, "vin") + `
 GROUP BY vin, stat_date, protocol
 ORDER BY stat_date, protocol, vin`
 	rows, err := db.QueryContext(ctx, query, args...)
@@ -697,6 +753,9 @@ func addProtocolGPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sq
 		"latitude BETWEEN -90 AND 90",
 		"NOT (longitude = 0 AND latitude = 0)",
 	}
+	if predicate := backfillVINPredicate(cfg, "vin"); predicate != "" {
+		where = append(where, predicate)
+	}
 	if protocol != envelope.ProtocolJT808 {
 		where = append(where, "speed_kmh > 0")
 	}
@@ -1060,13 +1119,17 @@ func queryDailyLastSourceRows(ctx context.Context, db *sql.DB, cfg config, proto
 	if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
 		where = append(where, predicate)
 	}
+	if predicate := backfillVINPredicate(cfg, "vin"); predicate != "" {
+		where = append(where, predicate)
+	}
 	sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
  FIRST(event_time), FIRST(parsed_json), FIRST(raw_text),
-  LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
+  LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*),
+  MIN(CAST(event_time AS BIGINT)), MAX(CAST(event_time AS BIGINT))
 FROM %s.raw_frames
 WHERE %s
 GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
-	return querySourceRows(ctx, db, cfg, protocol, sqlText, true)
+	return querySourceRows(ctx, db, cfg, protocol, sqlText, true, strings.Join(where, " AND "))
 }

 func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
@@ -1085,12 +1148,15 @@ func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, pr
 	if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
 		where = append(where, predicate)
 	}
+	if predicate := backfillVINPredicate(cfg, "vin"); predicate != "" {
+		where = append(where, predicate)
+	}
 	sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
-  LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
+  LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*), MAX(CAST(event_time AS BIGINT))
 FROM %s.raw_frames
 WHERE %s
 GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
-	return querySourceRows(ctx, db, cfg, protocol, sqlText, false)
+	return querySourceRows(ctx, db, cfg, protocol, sqlText, false, strings.Join(where, " AND "))
 }

 func backfillBeforePredicates(cfg config, eventDateExclusive string) []string {
@@ -1141,6 +1207,7 @@ WHERE l.protocol = ?
 	if cfg.StationaryCarry {
 		args = append(args, date)
 	}
+	sqlText = appendBackfillVINScope(sqlText, cfg, "l.vin")
 	rows, err := db.QueryContext(ctx, sqlText, args...)
 	if err != nil {
 		return nil, err
@@ -1167,7 +1234,7 @@ WHERE l.protocol = ?
 		if protocol == envelope.ProtocolYutongMQTT {
 			deviceID = vin
 		}
-		sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint)
+		sourceCode, platformName, sourceKind := backfillSourceMetadata(cfg, protocol, sourceEndpoint)
 		sourceKey := normalizedSourceKeyForSource(string(protocol), "", deviceID, sourceEndpoint, sourceKind, sourceCode)
 		if sourceKey == "" {
 			continue
@@ -1194,37 +1261,46 @@ WHERE l.protocol = ?
 	return out, nil
 }

-func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, sqlText string, includeFirst bool) (map[string][]dailySourceLast, error) {
+func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, sqlText string, includeFirst bool, boundaryWhere string) (map[string][]dailySourceLast, error) {
 	rows, err := db.QueryContext(ctx, sqlText)
 	if err != nil {
 		return nil, err
 	}
 	defer rows.Close()
-	latestBySource := map[string]dailySourceLast{}
+	var evidence []sourceBoundaryEvidence
 	for rows.Next() {
-		var vin string
-		var phone string
-		var deviceID string
-		var sourceEndpoint string
-		var firstTS time.Time
-		var firstParsedJSON string
-		var firstRawText string
-		var ts time.Time
-		var parsedJSON string
-		var rawText string
-		var rawSampleCount int64
+		var e sourceBoundaryEvidence
 		if includeFirst {
-			if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &firstTS, &firstParsedJSON, &firstRawText, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
+			if err := rows.Scan(&e.vin, &e.phone, &e.deviceID, &e.endpoint, &e.firstTS, &e.firstParsed, &e.firstRaw, &e.lastTS, &e.lastParsed, &e.lastRaw, &e.count, &e.minimumMS, &e.maximumMS); err != nil {
 				return nil, err
 			}
 		} else {
-			if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
+			if err := rows.Scan(&e.vin, &e.phone, &e.deviceID, &e.endpoint, &e.lastTS, &e.lastParsed, &e.lastRaw, &e.count, &e.maximumMS); err != nil {
 				return nil, err
 			}
-			firstTS = ts
-			firstParsedJSON = parsedJSON
-			firstRawText = rawText
+			e.minimumMS = e.maximumMS
 		}
+		evidence = append(evidence, e)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	// Release the aggregate cursor before resolving exceptional event-time
+	// boundaries; this also works with a single TDengine connection.
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	latestBySource := map[string]dailySourceLast{}
+	for _, e := range evidence {
+		if err := correctSourceEventBoundaries(ctx, db, cfg, protocol, boundaryWhere, includeFirst, &e); err != nil {
+			if errors.Is(err, errNoUsableSourceBoundary) {
+				continue
+			}
+			return nil, err
+		}
+		vin, phone, deviceID, sourceEndpoint := e.vin, e.phone, e.deviceID, e.endpoint
+		firstTS, firstParsedJSON, firstRawText := e.firstTS, e.firstParsed, e.firstRaw
+		ts, parsedJSON, rawText, rawSampleCount := e.lastTS, e.lastParsed, e.lastRaw, e.count
 		latestTotalKM, ok := mileageFromEvidence(protocol, vin, parsedJSON, rawText, ts, cfg.Location)
 		if !ok {
 			continue
@@ -1235,7 +1311,7 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
 				firstTotalKM = parsedFirst
 			}
 		}
-		sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint)
+		sourceCode, platformName, sourceKind := backfillSourceMetadata(cfg, protocol, sourceEndpoint)
 		sourceKey := normalizedSourceKeyForSource(string(protocol), phone, deviceID, sourceEndpoint, sourceKind, sourceCode)
 		if sourceKey == "" {
 			continue
@@ -1256,9 +1332,10 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
 			RawSampleCount: rawSampleCount,
 		}
 		key := row.VIN + "|" + row.SourceKey
-		if existing, ok := latestBySource[key]; !ok || row.TS.After(existing.TS) {
-			latestBySource[key] = row
+		if existing, ok := latestBySource[key]; ok {
+			row = mergeDailySourceRows(existing, row)
 		}
+		latestBySource[key] = row
 	}
 	if err := rows.Err(); err != nil {
 		return nil, err
@@ -1731,6 +1808,9 @@ func resetStats(ctx context.Context, db *sql.DB, cfg config) (int64, error) {
 		}
 		clauses = append(clauses, "protocol IN ("+strings.Join(placeholders, ",")+")")
 	}
+	if predicate := backfillVINPredicate(cfg, "vin"); predicate != "" {
+		clauses = append(clauses, predicate)
+	}
 	var deleted int64
 	for _, table := range []string{"vehicle_daily_mileage_source", "vehicle_daily_mileage"} {
 		result, err := db.ExecContext(ctx, "DELETE FROM "+table+" WHERE "+strings.Join(clauses, " AND "), args...)