feat: build vehicle data platform and production pipeline

go/vehicle-gateway/internal/realtime/snapshot_writer.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry"
 )
 
 type SnapshotExecer interface {
@@ -21,11 +22,20 @@ type PlateResolver interface {
 }
 
 type CachedPlateResolver struct {
-	delegate PlateResolver
-	ttl      time.Duration
-	now      func() time.Time
-	mu       sync.Mutex
-	entries  map[string]cachedPlateEntry
+	delegate   PlateResolver
+	ttl        time.Duration
+	maxEntries int
+	now        func() time.Time
+	observe    func(PlateCacheStats)
+	mu         sync.Mutex
+	entries    map[string]cachedPlateEntry
+	evictions  int
+}
+
+type PlateCacheStats struct {
+	Entries    int
+	MaxEntries int
+	Evictions  int
 }
 
 type cachedPlateEntry struct {
@@ -35,18 +45,38 @@ type cachedPlateEntry struct {
 }
 
 func NewCachedPlateResolver(delegate PlateResolver, ttl time.Duration) *CachedPlateResolver {
+	return NewCachedPlateResolverWithMaxEntries(delegate, ttl, 200000)
+}
+
+func NewCachedPlateResolverWithMaxEntries(delegate PlateResolver, ttl time.Duration, maxEntries int) *CachedPlateResolver {
 	if delegate == nil {
 		panic("cached plate resolver delegate must not be nil")
 	}
 	if ttl <= 0 {
 		ttl = 10 * time.Minute
 	}
-	return &CachedPlateResolver{
-		delegate: delegate,
-		ttl:      ttl,
-		now:      time.Now,
-		entries:  map[string]cachedPlateEntry{},
+	if maxEntries < 0 {
+		maxEntries = 0
 	}
+	return &CachedPlateResolver{
+		delegate:   delegate,
+		ttl:        ttl,
+		maxEntries: maxEntries,
+		now:        time.Now,
+		entries:    map[string]cachedPlateEntry{},
+	}
+}
+
+func (r *CachedPlateResolver) SetStatsObserver(observe func(PlateCacheStats)) {
+	r.mu.Lock()
+	r.observe = observe
+	r.mu.Unlock()
+}
+
+func (r *CachedPlateResolver) CacheStats() PlateCacheStats {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return PlateCacheStats{Entries: len(r.entries), MaxEntries: r.maxEntries, Evictions: r.evictions}
 }
 
 func (r *CachedPlateResolver) PlateByVIN(ctx context.Context, vin string) (string, error) {
@@ -76,13 +106,46 @@ func (r *CachedPlateResolver) PlateByVIN(ctx context.Context, vin string) (strin
 		notFound:  errors.Is(err, sql.ErrNoRows),
 		expiresAt: now.Add(r.ttl),
 	}
+	r.enforceLimitLocked(now)
+	stats := PlateCacheStats{Entries: len(r.entries), MaxEntries: r.maxEntries, Evictions: r.evictions}
+	observe := r.observe
 	r.mu.Unlock()
+	if observe != nil {
+		observe(stats)
+	}
 	if err != nil {
 		return "", err
 	}
 	return strings.TrimSpace(plate), nil
 }
 
+func (r *CachedPlateResolver) enforceLimitLocked(now time.Time) {
+	if r.maxEntries <= 0 || len(r.entries) <= r.maxEntries {
+		return
+	}
+	for vin, entry := range r.entries {
+		if now.After(entry.expiresAt) || now.Equal(entry.expiresAt) {
+			delete(r.entries, vin)
+			r.evictions++
+		}
+	}
+	for len(r.entries) > r.maxEntries {
+		victim := ""
+		var victimExpiresAt time.Time
+		for vin, entry := range r.entries {
+			if victim == "" || entry.expiresAt.Before(victimExpiresAt) {
+				victim = vin
+				victimExpiresAt = entry.expiresAt
+			}
+		}
+		if victim == "" {
+			return
+		}
+		delete(r.entries, victim)
+		r.evictions++
+	}
+}
+
 type SnapshotWriter struct {
 	exec          SnapshotExecer
 	plateResolver PlateResolver
@@ -111,6 +174,19 @@ func (w *SnapshotWriter) EnsureSchema(ctx context.Context) error {
 	if _, err := w.exec.ExecContext(ctx, realtimeLocationTableSQL); err != nil {
 		return err
 	}
+	for _, statement := range realtimeLocationCompatibilitySQL {
+		if _, err := w.exec.ExecContext(ctx, statement); err != nil && !isDuplicateColumnError(err) {
+			return err
+		}
+	}
+	for _, statement := range cleanupInvalidRealtimeTotalMileageSQL {
+		if _, err := w.exec.ExecContext(ctx, statement); err != nil {
+			return err
+		}
+	}
+	if _, err := w.exec.ExecContext(ctx, backfillRealtimeAccessProjectionSQL); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -129,13 +205,11 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
 	if err != nil {
 		return err
 	}
-	eventTime := nullableTime(env.EventTimeMS)
+	eventTimeMS, _ := envelope.NormalizedEventTimeMS(env)
+	eventTime := nullableTime(eventTimeMS)
 	receivedAt := nullableTime(env.ReceivedAtMS)
 	platformName := platformNameFromEnvelope(env)
-	parsed, err := w.snapshotFieldsForEnvelope(ctx, env, vin)
-	if err != nil {
-		return err
-	}
+	parsed := snapshotFieldsForEnvelope(env)
 	if _, err = w.exec.ExecContext(ctx, upsertRealtimeSnapshotSQL,
 		string(env.Protocol),
 		vin,
@@ -146,6 +220,9 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
 		eventTime,
 		receivedAt,
 		env.StableEventID(),
+		receivedAt,
+		receivedAt,
+		env.StableEventID(),
 	); err != nil {
 		return err
 	}
@@ -162,6 +239,7 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
 		location.Longitude,
 		location.SpeedKMH,
 		location.TotalMileageKM,
+		location.TotalMileageAt,
 		location.SOCPercent,
 		location.AltitudeM,
 		location.DirectionDeg,
@@ -173,87 +251,25 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
 	return err
 }
 
-func (w *SnapshotWriter) snapshotFieldsForEnvelope(ctx context.Context, env envelope.FrameEnvelope, vin string) (map[string]any, error) {
-	if len(env.Parsed) == 0 {
-		return nil, nil
-	}
-	parsed := cloneMap(env.Parsed)
-	incoming := realtimeSnapshotFlatFields(env, parsed)
-	if queryer, ok := w.exec.(Queryer); ok {
-		existing, err := realtimeSnapshotParsedJSON(ctx, queryer, env.Protocol, vin)
-		if err != nil {
-			return nil, err
-		}
-		if len(existing) > 0 {
-			if isStructuredSnapshotParsed(env.Protocol, existing) {
-				parsed = mergeParsedForProtocol(env.Protocol, existing, env.Parsed)
-				mergedEnv := env
-				mergedEnv.ParsedFields = nil
-				mergedEnv.ParsedFieldTypes = nil
-				return realtimeSnapshotFlatFields(mergedEnv, parsed), nil
-			}
-			return mergeRealtimeSnapshotFields(existing, incoming), nil
-		}
-	}
-	return incoming, nil
-}
-
-func realtimeSnapshotFlatFields(env envelope.FrameEnvelope, parsed map[string]any) map[string]any {
-	if len(env.ParsedFields) > 0 {
-		fields := cloneMap(env.ParsedFields)
-		if len(fields) > 0 {
-			return fields
-		}
-	}
-	rows := realtimeKVFields(env, parsed)
-	if len(rows) == 0 {
+func snapshotFieldsForEnvelope(env envelope.FrameEnvelope) map[string]any {
+	if len(env.ParsedFields) == 0 {
 		return nil
 	}
-	fields := make(map[string]any, len(rows))
-	for _, row := range rows {
-		key := realtimeKVFieldPath(row.Domain, row.Field)
-		if key == "" {
-			continue
-		}
-		fields[key] = row.Value
+	return realtimeSnapshotFlatFields(env)
+}
+
+func realtimeSnapshotFlatFields(env envelope.FrameEnvelope) map[string]any {
+	fields := cloneMap(env.ParsedFields)
+	filterInvalidRealtimeMeasurementFields(fields)
+	if len(fields) == 0 {
+		return nil
 	}
 	return fields
 }
 
-func mergeRealtimeSnapshotFields(existing map[string]any, incoming map[string]any) map[string]any {
-	if len(existing) == 0 {
-		return cloneMap(incoming)
-	}
-	merged := cloneMap(existing)
-	for key, value := range incoming {
-		merged[key] = value
-	}
-	return merged
-}
-
-func isStructuredSnapshotParsed(protocol envelope.Protocol, parsed map[string]any) bool {
-	if len(parsed) == 0 {
-		return false
-	}
-	if protocol == envelope.ProtocolGB32960 {
-		if _, ok := parsed["data_units"]; ok {
-			return true
-		}
-	}
-	mapping := realtimeMapping(protocol)
-	for key := range mapping.TopLevelName {
-		if _, ok := parsed[key]; ok {
-			return true
-		}
-	}
-	return false
-}
-
 func platformNameFromEnvelope(env envelope.FrameEnvelope) string {
-	if env.Fields != nil {
-		if value := strings.TrimSpace(stringValue(env.Fields["platform_account"])); value != "" {
-			return value
-		}
+	if value := strings.TrimSpace(env.PlatformName); value != "" {
+		return value
 	}
 	if env.Parsed != nil {
 		if value := strings.TrimSpace(stringValue(env.Parsed["platform_name"])); value != "" {
@@ -275,25 +291,6 @@ func stringValue(value any) string {
 	}
 }
 
-func realtimeSnapshotParsedJSON(ctx context.Context, queryer Queryer, protocol envelope.Protocol, vin string) (map[string]any, error) {
-	var raw sql.NullString
-	err := queryer.QueryRowContext(ctx, selectRealtimeSnapshotParsedJSONSQL, string(protocol), vin).Scan(&raw)
-	if err != nil {
-		if errors.Is(err, sql.ErrNoRows) {
-			return nil, nil
-		}
-		return nil, err
-	}
-	if !raw.Valid || strings.TrimSpace(raw.String) == "" {
-		return nil, nil
-	}
-	var parsed map[string]any
-	if err := json.Unmarshal([]byte(raw.String), &parsed); err != nil {
-		return nil, nil
-	}
-	return parsed, nil
-}
-
 func marshalParsedJSON(parsed map[string]any) any {
 	if len(parsed) == 0 {
 		return nil
@@ -335,6 +332,7 @@ type realtimeLocationRow struct {
 	Longitude      float64
 	SpeedKMH       any
 	TotalMileageKM any
+	TotalMileageAt any
 	SOCPercent     any
 	AltitudeM      any
 	DirectionDeg   any
@@ -345,71 +343,53 @@ type realtimeLocationRow struct {
 }
 
 func realtimeLocationFromEnvelope(env envelope.FrameEnvelope, vin string, plate string) (realtimeLocationRow, bool) {
-	latitude, okLat := numberField(env.Fields, envelope.FieldLatitude)
-	longitude, okLon := numberField(env.Fields, envelope.FieldLongitude)
-	if !okLat || !okLon {
+	location, ok := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields)
+	if !ok {
 		return realtimeLocationRow{}, false
 	}
+	eventTimeMS, _ := envelope.NormalizedEventTimeMS(env)
+	totalMileageKM, hasTotalMileage := telemetry.TotalMileageKM(env.Protocol, env.ParsedFields)
+	if totalMileageKM <= 0 {
+		hasTotalMileage = false
+	}
+	var totalMileageValue any
+	var totalMileageAt any
+	if hasTotalMileage {
+		totalMileageValue = totalMileageKM
+		totalMileageAt = nullableTime(eventTimeMS)
+	}
 	return realtimeLocationRow{
 		Protocol:       string(env.Protocol),
 		VIN:            vin,
 		Plate:          plate,
-		EventTime:      nullableTime(env.EventTimeMS),
-		Latitude:       latitude,
-		Longitude:      longitude,
-		SpeedKMH:       nullableNumberField(env.Fields, envelope.FieldSpeedKMH),
-		TotalMileageKM: nullableNumberField(env.Fields, envelope.FieldTotalMileageKM),
-		SOCPercent:     nullableNumberField(env.Fields, envelope.FieldSOCPercent),
-		AltitudeM:      nullableNumberField(env.Fields, "altitude_m"),
-		DirectionDeg:   nullableNumberField(env.Fields, "direction_deg"),
-		AlarmFlag:      nullableNumberField(env.Fields, "alarm_flag"),
-		StatusFlag:     nullableNumberField(env.Fields, "status_flag"),
+		EventTime:      nullableTime(eventTimeMS),
+		Latitude:       location.Latitude,
+		Longitude:      location.Longitude,
+		SpeedKMH:       optionalFloatValue(location.SpeedKMH),
+		TotalMileageKM: totalMileageValue,
+		TotalMileageAt: totalMileageAt,
+		SOCPercent:     optionalFloatValue(location.SOCPercent),
+		AltitudeM:      optionalFloatValue(location.AltitudeM),
+		DirectionDeg:   optionalFloatValue(location.DirectionDeg),
+		AlarmFlag:      optionalInt64Value(location.AlarmFlag),
+		StatusFlag:     optionalInt64Value(location.StatusFlag),
 		ReceivedAt:     nullableTime(env.ReceivedAtMS),
 		EventID:        env.StableEventID(),
 	}, true
 }
 
-func nullableNumberField(fields map[string]any, key string) any {
-	value, ok := numberField(fields, key)
-	if !ok {
+func optionalFloatValue(value *float64) any {
+	if value == nil {
 		return nil
 	}
-	return value
+	return *value
 }
 
-func numberField(fields map[string]any, key string) (float64, bool) {
-	value, ok := fields[key]
-	if !ok {
-		return 0, false
-	}
-	switch typed := value.(type) {
-	case float64:
-		return typed, true
-	case float32:
-		return float64(typed), true
-	case int:
-		return float64(typed), true
-	case int8:
-		return float64(typed), true
-	case int16:
-		return float64(typed), true
-	case int32:
-		return float64(typed), true
-	case int64:
-		return float64(typed), true
-	case uint:
-		return float64(typed), true
-	case uint8:
-		return float64(typed), true
-	case uint16:
-		return float64(typed), true
-	case uint32:
-		return float64(typed), true
-	case uint64:
-		return float64(typed), true
-	default:
-		return 0, false
+func optionalInt64Value(value *int64) any {
+	if value == nil {
+		return nil
 	}
+	return *value
 }
 
 type BindingPlateResolver struct {
@@ -469,30 +449,19 @@ func nullableTime(ms int64) any {
 func isRealtimeSnapshotEvent(env envelope.FrameEnvelope) bool {
 	switch env.Protocol {
 	case envelope.ProtocolGB32960:
-		return env.MessageID == "0x02" && hasGB32960RealtimeDataUnits(env)
+		return (env.MessageID == "0x02" || env.MessageID == "0x03") && telemetry.HasRealtimeFields(env.Protocol, env.ParsedFields)
 	case envelope.ProtocolJT808:
 		return env.MessageID == "0x0200" && hasLocationFields(env)
 	case envelope.ProtocolYutongMQTT:
-		return len(env.Fields) > 0
+		return telemetry.HasRealtimeFields(env.Protocol, env.ParsedFields)
 	default:
 		return false
 	}
 }
 
-func hasGB32960RealtimeDataUnits(env envelope.FrameEnvelope) bool {
-	if units, ok := env.Parsed["data_units"].([]any); ok && len(units) > 0 {
-		return true
-	}
-	if units, ok := env.Parsed["data_units"].([]map[string]any); ok && len(units) > 0 {
-		return true
-	}
-	return false
-}
-
 func hasLocationFields(env envelope.FrameEnvelope) bool {
-	_, okLat := numberField(env.Fields, envelope.FieldLatitude)
-	_, okLon := numberField(env.Fields, envelope.FieldLongitude)
-	return okLat && okLon
+	_, ok := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields)
+	return ok
 }
 
 const realtimeSnapshotTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_snapshot (
@@ -505,16 +474,31 @@ const realtimeSnapshotTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_sn
   event_time DATETIME(3) NULL,
   received_at DATETIME(3) NULL,
   event_id VARCHAR(64) NOT NULL DEFAULT '',
+  access_first_seen_at DATETIME(3) NULL,
+  access_previous_received_at DATETIME(3) NULL,
+  access_latest_received_at DATETIME(3) NULL,
+  access_report_interval_ms BIGINT UNSIGNED NULL,
+  access_sample_count BIGINT UNSIGNED NOT NULL DEFAULT 0,
+  access_latest_event_id VARCHAR(64) NOT NULL DEFAULT '',
+  access_first_seen_source VARCHAR(32) NOT NULL DEFAULT '',
   updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   PRIMARY KEY (protocol, vin),
   KEY idx_vin (vin),
-  KEY idx_protocol_updated (protocol, updated_at)
+  KEY idx_protocol_updated (protocol, updated_at),
+  KEY idx_access_latest (access_latest_received_at, vin)
 )`
 
 var realtimeSnapshotCompatibilitySQL = []string{
 	"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN platform_name VARCHAR(64) NOT NULL DEFAULT '' AFTER plate",
 	"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN peer VARCHAR(128) NOT NULL DEFAULT '' AFTER platform_name",
 	"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN parsed_json LONGTEXT NULL AFTER peer",
+	"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_first_seen_at DATETIME(3) NULL AFTER event_id",
+	"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_previous_received_at DATETIME(3) NULL AFTER access_first_seen_at",
+	"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_latest_received_at DATETIME(3) NULL AFTER access_previous_received_at",
+	"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_report_interval_ms BIGINT UNSIGNED NULL AFTER access_latest_received_at",
+	"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_sample_count BIGINT UNSIGNED NOT NULL DEFAULT 0 AFTER access_report_interval_ms",
+	"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_latest_event_id VARCHAR(64) NOT NULL DEFAULT '' AFTER access_sample_count",
+	"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_first_seen_source VARCHAR(32) NOT NULL DEFAULT '' AFTER access_latest_event_id",
 }
 
 func isDuplicateColumnError(err error) bool {
@@ -524,21 +508,27 @@ func isDuplicateColumnError(err error) bool {
 
 const upsertRealtimeSnapshotSQL = `
 INSERT INTO vehicle_realtime_snapshot
-  (protocol, vin, plate, platform_name, peer, parsed_json, event_time, received_at, event_id)
-VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+  (protocol, vin, plate, platform_name, peer, parsed_json, event_time, received_at, event_id,
+   access_first_seen_at, access_latest_received_at, access_sample_count, access_latest_event_id, access_first_seen_source)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, 'live_writer')
 ON DUPLICATE KEY UPDATE
   plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
   platform_name = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(platform_name), platform_name),
   peer = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(peer), peer),
-  parsed_json = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(parsed_json), parsed_json),
+  parsed_json = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time),
+    IF(VALUES(parsed_json) IS NULL OR VALUES(parsed_json) = '', parsed_json, JSON_MERGE_PATCH(COALESCE(NULLIF(vehicle_realtime_snapshot.parsed_json, ''), JSON_OBJECT()), VALUES(parsed_json))),
+    parsed_json),
   event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(event_time), event_time),
   received_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(received_at), received_at),
   event_id = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(event_id), event_id),
-  updated_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), CURRENT_TIMESTAMP, updated_at)
+  updated_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), CURRENT_TIMESTAMP, updated_at),
+  access_previous_received_at = IF(VALUES(access_latest_received_at) IS NOT NULL AND (vehicle_realtime_snapshot.access_latest_received_at IS NULL OR VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at) AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, vehicle_realtime_snapshot.access_latest_received_at, access_previous_received_at),
+  access_report_interval_ms = IF(VALUES(access_latest_received_at) IS NOT NULL AND vehicle_realtime_snapshot.access_latest_received_at IS NOT NULL AND VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, TIMESTAMPDIFF(MICROSECOND, vehicle_realtime_snapshot.access_latest_received_at, VALUES(access_latest_received_at)) DIV 1000, access_report_interval_ms),
+  access_sample_count = IF(VALUES(access_latest_received_at) IS NOT NULL AND (vehicle_realtime_snapshot.access_latest_received_at IS NULL OR VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at) AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, access_sample_count + 1, access_sample_count),
+  access_latest_received_at = IF(VALUES(access_latest_received_at) IS NOT NULL AND (vehicle_realtime_snapshot.access_latest_received_at IS NULL OR VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at) AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, VALUES(access_latest_received_at), access_latest_received_at),
+  access_latest_event_id = IF(VALUES(access_latest_received_at) IS NOT NULL AND VALUES(access_latest_received_at) = vehicle_realtime_snapshot.access_latest_received_at AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, VALUES(access_latest_event_id), access_latest_event_id)
 `
 
-const selectRealtimeSnapshotParsedJSONSQL = `SELECT parsed_json FROM vehicle_realtime_snapshot WHERE protocol = ? AND vin = ?`
-
 const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_location (
   protocol VARCHAR(32) NOT NULL,
   vin VARCHAR(32) NOT NULL DEFAULT '',
@@ -548,6 +538,7 @@ const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_lo
   longitude DECIMAL(12,6) NOT NULL,
   speed_kmh DECIMAL(10,3) NULL,
   total_mileage_km DECIMAL(18,3) NULL,
+  total_mileage_event_time DATETIME(3) NULL,
   soc_percent DECIMAL(6,2) NULL,
   altitude_m DECIMAL(10,3) NULL,
   direction_deg DECIMAL(10,3) NULL,
@@ -561,11 +552,50 @@ const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_lo
   KEY idx_protocol_updated (protocol, updated_at)
 )`
 
+var realtimeLocationCompatibilitySQL = []string{
+	"ALTER TABLE vehicle_realtime_location ADD COLUMN total_mileage_event_time DATETIME(3) NULL AFTER total_mileage_km",
+}
+
+var cleanupInvalidRealtimeTotalMileageSQL = []string{
+	`UPDATE vehicle_realtime_location
+SET total_mileage_km = NULL,
+    total_mileage_event_time = NULL
+WHERE total_mileage_km IS NOT NULL
+  AND total_mileage_km <= 0`,
+	`UPDATE vehicle_realtime_snapshot
+SET parsed_json = JSON_REMOVE(parsed_json, '$."jt808.location.total_mileage_km"')
+WHERE protocol = 'JT808'
+  AND parsed_json IS NOT NULL
+  AND CAST(JSON_UNQUOTE(JSON_EXTRACT(parsed_json, '$."jt808.location.total_mileage_km"')) AS DECIMAL(18,3)) <= 0`,
+	`UPDATE vehicle_realtime_snapshot
+SET parsed_json = JSON_REMOVE(parsed_json, '$."gb32960.vehicle.total_mileage_km"')
+WHERE protocol = 'GB32960'
+  AND parsed_json IS NOT NULL
+  AND CAST(JSON_UNQUOTE(JSON_EXTRACT(parsed_json, '$."gb32960.vehicle.total_mileage_km"')) AS DECIMAL(18,3)) <= 0`,
+	`UPDATE vehicle_realtime_snapshot
+SET parsed_json = JSON_REMOVE(parsed_json, '$."yutong_mqtt.data.total_mileage_km"')
+WHERE protocol = 'YUTONG_MQTT'
+  AND parsed_json IS NOT NULL
+	  AND CAST(JSON_UNQUOTE(JSON_EXTRACT(parsed_json, '$."yutong_mqtt.data.total_mileage_km"')) AS DECIMAL(18,3)) <= 0`,
+}
+
+const backfillRealtimeAccessProjectionSQL = `UPDATE vehicle_realtime_snapshot
+SET access_first_seen_at = COALESCE(access_first_seen_at, received_at, updated_at),
+    access_latest_received_at = COALESCE(access_latest_received_at, received_at, updated_at),
+    access_sample_count = IF(access_sample_count = 0, 1, access_sample_count),
+    access_latest_event_id = IF(access_latest_event_id = '', event_id, access_latest_event_id),
+    access_first_seen_source = IF(access_first_seen_source = '', 'snapshot_backfill', access_first_seen_source)
+WHERE access_first_seen_at IS NULL
+   OR access_latest_received_at IS NULL
+   OR access_sample_count = 0
+   OR access_latest_event_id = ''
+   OR access_first_seen_source = ''`
+
 const upsertRealtimeLocationSQL = `
 INSERT INTO vehicle_realtime_location
-  (protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km,
+  (protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, total_mileage_event_time,
    soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id)
-VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 ON DUPLICATE KEY UPDATE
   plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
   event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(event_time), event_time),
@@ -573,6 +603,7 @@ ON DUPLICATE KEY UPDATE
   longitude = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(longitude), longitude),
   speed_kmh = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(speed_kmh), speed_kmh), speed_kmh),
   total_mileage_km = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(total_mileage_km), total_mileage_km), total_mileage_km),
+  total_mileage_event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), IF(VALUES(total_mileage_km) IS NOT NULL, COALESCE(VALUES(total_mileage_event_time), total_mileage_event_time), total_mileage_event_time), total_mileage_event_time),
   soc_percent = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(soc_percent), soc_percent), soc_percent),
   altitude_m = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(altitude_m), altitude_m), altitude_m),
   direction_deg = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(direction_deg), direction_deg), direction_deg),