feat: build vehicle data platform and production pipeline

go/vehicle-gateway/internal/stats/daily_metric.go

@@ -3,7 +3,6 @@ package stats
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
 	"strconv"
 	"strings"
@@ -11,6 +10,15 @@ import (
 	"time"
 
 	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry"
+)
+
+const (
+	maxNegativeMileageJitterKM  = 1.0
+	defaultCacheRetention       = 72 * time.Hour
+	defaultCacheCleanupInterval = 10 * time.Minute
+	defaultBaselineMissTTL      = time.Minute
+	defaultMaxCacheEntries      = 1000000
 )
 
 type Execer interface {
@@ -18,14 +26,26 @@ type Execer interface {
 }
 
 type Writer struct {
-	exec                Execer
-	query               Queryer
-	loc                 *time.Location
-	sourceTouchInterval time.Duration
-	mu                  sync.Mutex
-	lastTotalMileage    map[string]float64
-	lastSourceSeen      map[string]time.Time
-	baselineCache       map[string]sourceBaselineCacheEntry
+	exec                   Execer
+	query                  Queryer
+	loc                    *time.Location
+	sourceTouchInterval    time.Duration
+	projectionInterval     time.Duration
+	cacheRetention         time.Duration
+	cacheCleanupInterval   time.Duration
+	baselineMissTTL        time.Duration
+	maxCacheEntries        int
+	lastCacheCleanup       time.Time
+	lastCacheCleanupStats  cacheCleanupStats
+	cacheEvictions         cacheCleanupStats
+	mu                     sync.Mutex
+	lastTotalMileage       map[string]float64
+	lastSourceSeen         map[string]time.Time
+	lastProjection         map[string]projectionCacheEntry
+	baselineCache          map[string]sourceBaselineCacheEntry
+	mileageKeysByPrefix    map[string]map[string]struct{}
+	projectionKeysByPrefix map[string]map[string]struct{}
+	baselineKeysByPrefix   map[string]map[string]struct{}
 }
 
 type MetricSample struct {
@@ -38,6 +58,53 @@ type MetricSample struct {
 	Phone          string
 	DeviceID       string
 	SourceEndpoint string
+	PlatformName   string
+}
+
+type AppendResult struct {
+	SamplesFound                     int
+	SamplesWritten                   int
+	SamplesSkippedMissingFields      int
+	SamplesSkippedMissingVIN         int
+	SamplesSkippedMissingMileage     int
+	SamplesSkippedNonMileageFrame    int
+	SamplesSkippedNonPositiveMileage int
+	SamplesSkippedMissingTime        int
+	SamplesSkippedSameMileage        int
+	SamplesSkippedMissingSource      int
+	SamplesAdjustedFutureEventTime   int
+	SourceTouchesAttempted           int
+	SourceTouchesWritten             int
+	SourceTouchesSkippedThrottled    int
+	SourceTouchesSkippedMissing      int
+	SourceTouchesSkippedUnmanaged    int
+	ProjectionsAttempted             int
+	ProjectionsWritten               int
+	ProjectionsSkippedThrottled      int
+}
+
+type CacheStats struct {
+	LastTotalMileageEntries int
+	LastSourceSeenEntries   int
+	LastProjectionEntries   int
+	BaselineEntries         int
+	MaxEntries              int
+	LastCleanupAt           time.Time
+	LastCleanupTotalMileage int
+	LastCleanupSourceSeen   int
+	LastCleanupProjection   int
+	LastCleanupBaseline     int
+	TotalMileageEvictions   int
+	SourceSeenEvictions     int
+	ProjectionEvictions     int
+	BaselineEvictions       int
+}
+
+type cacheCleanupStats struct {
+	totalMileage int
+	sourceSeen   int
+	projection   int
+	baseline     int
 }
 
 func NewWriter(exec Execer, loc *time.Location) *Writer {
@@ -48,12 +115,21 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
 		loc = time.FixedZone("Asia/Shanghai", 8*3600)
 	}
 	writer := &Writer{
-		exec:                exec,
-		loc:                 loc,
-		sourceTouchInterval: time.Minute,
-		lastTotalMileage:    map[string]float64{},
-		lastSourceSeen:      map[string]time.Time{},
-		baselineCache:       map[string]sourceBaselineCacheEntry{},
+		exec:                   exec,
+		loc:                    loc,
+		sourceTouchInterval:    time.Minute,
+		projectionInterval:     15 * time.Second,
+		cacheRetention:         defaultCacheRetention,
+		cacheCleanupInterval:   defaultCacheCleanupInterval,
+		baselineMissTTL:        defaultBaselineMissTTL,
+		maxCacheEntries:        defaultMaxCacheEntries,
+		lastTotalMileage:       map[string]float64{},
+		lastSourceSeen:         map[string]time.Time{},
+		lastProjection:         map[string]projectionCacheEntry{},
+		baselineCache:          map[string]sourceBaselineCacheEntry{},
+		mileageKeysByPrefix:    map[string]map[string]struct{}{},
+		projectionKeysByPrefix: map[string]map[string]struct{}{},
+		baselineKeysByPrefix:   map[string]map[string]struct{}{},
 	}
 	if query, ok := exec.(Queryer); ok {
 		writer.query = query
@@ -61,6 +137,82 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
 	return writer
 }
 
+func (w *Writer) SetSourceTouchInterval(interval time.Duration) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if interval < 0 {
+		interval = 0
+	}
+	w.sourceTouchInterval = interval
+}
+
+func (w *Writer) SetProjectionInterval(interval time.Duration) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if interval < 0 {
+		interval = 0
+	}
+	w.projectionInterval = interval
+}
+
+func (w *Writer) SetCacheRetention(retention time.Duration) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if retention < 0 {
+		retention = 0
+	}
+	w.cacheRetention = retention
+}
+
+func (w *Writer) SetCacheCleanupInterval(interval time.Duration) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if interval < 0 {
+		interval = 0
+	}
+	w.cacheCleanupInterval = interval
+}
+
+func (w *Writer) SetBaselineMissTTL(ttl time.Duration) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if ttl < 0 {
+		ttl = 0
+	}
+	w.baselineMissTTL = ttl
+}
+
+func (w *Writer) SetMaxCacheEntries(maxEntries int) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if maxEntries < 0 {
+		maxEntries = 0
+	}
+	w.maxCacheEntries = maxEntries
+	w.enforceCacheLimitsLocked()
+}
+
+func (w *Writer) CacheStats() CacheStats {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return CacheStats{
+		LastTotalMileageEntries: len(w.lastTotalMileage),
+		LastSourceSeenEntries:   len(w.lastSourceSeen),
+		LastProjectionEntries:   len(w.lastProjection),
+		BaselineEntries:         len(w.baselineCache),
+		MaxEntries:              w.maxCacheEntries,
+		LastCleanupAt:           w.lastCacheCleanup,
+		LastCleanupTotalMileage: w.lastCacheCleanupStats.totalMileage,
+		LastCleanupSourceSeen:   w.lastCacheCleanupStats.sourceSeen,
+		LastCleanupProjection:   w.lastCacheCleanupStats.projection,
+		LastCleanupBaseline:     w.lastCacheCleanupStats.baseline,
+		TotalMileageEvictions:   w.cacheEvictions.totalMileage,
+		SourceSeenEvictions:     w.cacheEvictions.sourceSeen,
+		ProjectionEvictions:     w.cacheEvictions.projection,
+		BaselineEvictions:       w.cacheEvictions.baseline,
+	}
+}
+
 func (w *Writer) EnsureSchema(ctx context.Context) error {
 	for _, statement := range []string{
 		DataSourceTableSQL,
@@ -80,48 +232,121 @@ func (w *Writer) EnsureSchema(ctx context.Context) error {
 }
 
 func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
-	identity, hasSource := NewSourceIdentity(env.Protocol, env.SourceEndpoint)
-	if hasSource {
-		seenAt := w.sourceSeenAt(env)
+	_, err := w.AppendWithResult(ctx, env)
+	return err
+}
+
+func (w *Writer) AppendWithResult(ctx context.Context, env envelope.FrameEnvelope) (AppendResult, error) {
+	var result AppendResult
+	if len(env.Fields) == 0 {
+		result.SamplesSkippedMissingFields = 1
+		return result, nil
+	}
+	seenAt := w.sourceSeenAt(env)
+	w.maybeCleanupCaches(seenAt)
+	identity, hasSource := NewSourceIdentityFromEnvelope(env)
+	var sourceResult AppendResult
+	if hasSource && ShouldManageDataSource(identity) {
 		if w.shouldTouchSource(identity, seenAt) {
+			sourceResult.SourceTouchesAttempted = 1
 			if err := UpsertDataSource(ctx, w.exec, identity, seenAt); err != nil {
-				return err
+				return sourceResult, err
 			}
 			w.markSourceTouched(identity, seenAt)
+			sourceResult.SourceTouchesWritten = 1
+		} else {
+			sourceResult.SourceTouchesSkippedThrottled = 1
 		}
+	} else if hasSource {
+		sourceResult.SourceTouchesSkippedUnmanaged = 1
 	}
-	samples, err := SamplesFromEnvelope(env, w.loc)
+	samples, extractionResult, err := samplesFromEnvelopeWithResult(env, w.loc)
+	result = extractionResult
+	result.SourceTouchesAttempted += sourceResult.SourceTouchesAttempted
+	result.SourceTouchesWritten += sourceResult.SourceTouchesWritten
+	result.SourceTouchesSkippedThrottled += sourceResult.SourceTouchesSkippedThrottled
+	result.SourceTouchesSkippedMissing += sourceResult.SourceTouchesSkippedMissing
+	result.SourceTouchesSkippedUnmanaged += sourceResult.SourceTouchesSkippedUnmanaged
 	if err != nil {
-		return err
+		return result, err
 	}
 	for _, sample := range samples {
-		if w.seenSameMileage(sample) {
+		if !hasSource {
+			result.SamplesSkippedMissingSource++
+			result.SourceTouchesSkippedMissing++
 			continue
 		}
-		if !hasSource {
+		if w.seenSameMileage(sample) {
+			result.SamplesSkippedSameMileage++
 			continue
 		}
 		candidate := SourceMileageSampleFromMetric(sample, identity)
 		if err := w.applyRealtimeBaseline(ctx, &candidate); err != nil {
-			return err
+			return result, err
 		}
-		if err := UpsertSourceMileage(ctx, w.exec, candidate); err != nil {
-			return err
+		projectDaily := w.shouldProjectDailyMileage(sample)
+		if projectDaily {
+			result.ProjectionsAttempted++
+		} else {
+			result.ProjectionsSkippedThrottled++
 		}
-		if err := ProjectDailyMileage(ctx, w.exec, sample.VIN, sample.StatDate, sample.Protocol); err != nil {
-			return err
+		if err := w.writeMileageSample(ctx, sample, candidate, projectDaily); err != nil {
+			return result, err
+		}
+		w.markBaselineWritten(sample, candidate)
+		if projectDaily {
+			w.markProjected(sample)
+			result.ProjectionsWritten++
 		}
 		w.markMileageWritten(sample)
+		result.SamplesWritten++
+	}
+	return result, nil
+}
+
+func (w *Writer) writeMileageSample(ctx context.Context, sample MetricSample, candidate SourceMileageSample, projectDaily bool) error {
+	if projectDaily {
+		if beginner, ok := w.exec.(txBeginner); ok {
+			tx, err := beginner.BeginTx(ctx, nil)
+			if err != nil {
+				return err
+			}
+			if err := UpsertSourceMileage(ctx, tx, candidate); err != nil {
+				_ = tx.Rollback()
+				return err
+			}
+			if ShouldNormalizePlatformSourceMileage(candidate) {
+				if err := NormalizePlatformSourceMileage(ctx, tx, sample.VIN, sample.StatDate, sample.Protocol); err != nil {
+					_ = tx.Rollback()
+					return err
+				}
+			}
+			if err := projectDailyMileageWithExec(ctx, tx, sample.VIN, sample.StatDate, sample.Protocol); err != nil {
+				_ = tx.Rollback()
+				return err
+			}
+			return tx.Commit()
+		}
+	}
+	if err := UpsertSourceMileage(ctx, w.exec, candidate); err != nil {
+		return err
+	}
+	if projectDaily {
+		if ShouldNormalizePlatformSourceMileage(candidate) {
+			if err := NormalizePlatformSourceMileage(ctx, w.exec, sample.VIN, sample.StatDate, sample.Protocol); err != nil {
+				return err
+			}
+		}
+		return ProjectDailyMileage(ctx, w.exec, sample.VIN, sample.StatDate, sample.Protocol)
 	}
 	return nil
 }
 
 func (w *Writer) sourceSeenAt(env envelope.FrameEnvelope) time.Time {
-	eventMS := env.EventTimeMS
-	if eventMS <= 0 {
-		eventMS = env.ReceivedAtMS
+	if env.ReceivedAtMS > 0 {
+		return time.UnixMilli(env.ReceivedAtMS).In(w.loc)
 	}
-	if eventMS > 0 {
+	if eventMS, ok := statEventTimeMS(env); ok {
 		return time.UnixMilli(eventMS).In(w.loc)
 	}
 	return time.Now().In(w.loc)
@@ -131,6 +356,9 @@ func (w *Writer) shouldTouchSource(identity SourceIdentity, seenAt time.Time) bo
 	key := string(identity.Protocol) + "|" + identity.SourceIP
 	w.mu.Lock()
 	defer w.mu.Unlock()
+	if w.sourceTouchInterval == 0 {
+		return true
+	}
 	last, ok := w.lastSourceSeen[key]
 	if ok && !seenAt.After(last.Add(w.sourceTouchInterval)) {
 		return false
@@ -142,9 +370,45 @@ func (w *Writer) markSourceTouched(identity SourceIdentity, seenAt time.Time) {
 	key := string(identity.Protocol) + "|" + identity.SourceIP
 	w.mu.Lock()
 	w.lastSourceSeen[key] = seenAt
+	w.enforceCacheLimitsLocked()
 	w.mu.Unlock()
 }
 
+func (w *Writer) shouldProjectDailyMileage(sample MetricSample) bool {
+	key := projectionCacheKey(sample)
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	interval := w.projectionInterval
+	if interval == 0 {
+		return true
+	}
+	entry, ok := w.lastProjection[key]
+	if !ok {
+		return true
+	}
+	if _, ok := entry.sourceKeys[sample.SourceKey]; !ok {
+		return true
+	}
+	return sample.EventTime.After(entry.projectedAt.Add(interval))
+}
+
+func (w *Writer) markProjected(sample MetricSample) {
+	key := projectionCacheKey(sample)
+	prefix := projectionCachePrefix(sample)
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.deleteProjectionPrefixExceptLocked(prefix, key)
+	entry := w.lastProjection[key]
+	if entry.sourceKeys == nil {
+		entry.sourceKeys = map[string]struct{}{}
+	}
+	entry.sourceKeys[sample.SourceKey] = struct{}{}
+	entry.projectedAt = sample.EventTime
+	w.lastProjection[key] = entry
+	w.addCacheKeyLocked(w.projectionKeysByPrefix, prefix, key)
+	w.enforceCacheLimitsLocked()
+}
+
 func (w *Writer) seenSameMileage(sample MetricSample) bool {
 	key := mileageCacheKey(sample)
 	w.mu.Lock()
@@ -160,17 +424,59 @@ func (w *Writer) markMileageWritten(sample MetricSample) {
 	key := mileageCacheKey(sample)
 	w.mu.Lock()
 	defer w.mu.Unlock()
-	for existing := range w.lastTotalMileage {
-		if strings.HasPrefix(existing, prefix) && existing != key {
-			delete(w.lastTotalMileage, existing)
-		}
-	}
-	for existing := range w.baselineCache {
-		if strings.HasPrefix(existing, prefix) && existing != key {
-			delete(w.baselineCache, existing)
-		}
-	}
+	w.deleteMileagePrefixExceptLocked(prefix, key)
+	w.deleteBaselinePrefixExceptLocked(prefix, key)
 	w.lastTotalMileage[key] = sample.TotalMileageKM
+	w.addCacheKeyLocked(w.mileageKeysByPrefix, prefix, key)
+	w.enforceCacheLimitsLocked()
+}
+
+func (w *Writer) maybeCleanupCaches(now time.Time) {
+	if now.IsZero() {
+		now = time.Now().In(w.loc)
+	}
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	retention := w.cacheRetention
+	if retention <= 0 {
+		w.enforceCacheLimitsLocked()
+		return
+	}
+	interval := w.cacheCleanupInterval
+	if interval > 0 && !w.lastCacheCleanup.IsZero() && !now.After(w.lastCacheCleanup.Add(interval)) {
+		w.enforceCacheLimitsLocked()
+		return
+	}
+	w.lastCacheCleanup = now
+	cleanupStats := cacheCleanupStats{}
+	cutoff := now.Add(-retention)
+	cutoffDate := cutoff.In(w.loc).Format("2006-01-02")
+	for key, seenAt := range w.lastSourceSeen {
+		if !seenAt.IsZero() && seenAt.Before(cutoff) {
+			delete(w.lastSourceSeen, key)
+			cleanupStats.sourceSeen++
+		}
+	}
+	for key, entry := range w.lastProjection {
+		if entry.projectedAt.IsZero() || entry.projectedAt.Before(cutoff) || cacheKeyDateBefore(key, cutoffDate) {
+			w.deleteProjectionKeyLocked(key)
+			cleanupStats.projection++
+		}
+	}
+	for key := range w.lastTotalMileage {
+		if cacheKeyDateBefore(key, cutoffDate) {
+			w.deleteMileageKeyLocked(key)
+			cleanupStats.totalMileage++
+		}
+	}
+	for key := range w.baselineCache {
+		if cacheKeyDateBefore(key, cutoffDate) {
+			w.deleteBaselineKeyLocked(key)
+			cleanupStats.baseline++
+		}
+	}
+	w.lastCacheCleanupStats = cleanupStats
+	w.enforceCacheLimitsLocked()
 }
 
 func (w *Writer) applyRealtimeBaseline(ctx context.Context, candidate *SourceMileageSample) error {
@@ -182,22 +488,22 @@ func (w *Writer) applyRealtimeBaseline(ctx context.Context, candidate *SourceMil
 		return err
 	}
 	if !found {
+		// If no earlier odometer exists at all, use the first current-day sample.
+		// Later samples retain that boundary through the in-memory baseline cache.
+		candidate.DailyKM = 0
 		candidate.QualityStatus = QualityOK
-		candidate.QualityReason = "current_day_first_sample"
+		candidate.QualityReason = QualityReasonCurrentDayFirst
 		return nil
 	}
 	candidate.FirstTotalKM = baseline.LatestTotalKM
 	candidate.FirstEventTime = baseline.LatestEventTime
-	candidate.DailyKM = candidate.LatestTotalKM - baseline.LatestTotalKM
+	candidate.DailyKM = DailyMileageFromDayBoundary(baseline.LatestTotalKM, candidate.LatestTotalKM)
 	candidate.QualityStatus = QualityOK
 	candidate.QualityReason = baseline.QualityReason
 	if candidate.QualityReason == "" {
-		candidate.QualityReason = "historical_source_baseline"
-	}
-	if candidate.DailyKM < 0 || candidate.DailyKM > maxSelectedDailyMileageKM {
-		candidate.QualityStatus = QualityInvalidDelta
-		candidate.QualityReason = "outside_daily_range"
+		candidate.QualityReason = QualityReasonHistorical
 	}
+	ApplyMileageQualityRules(candidate)
 	return nil
 }
 
@@ -208,10 +514,15 @@ func (w *Writer) previousBaseline(ctx context.Context, candidate SourceMileageSa
 		StatDate:  candidate.StatDate,
 		SourceKey: candidate.SourceKey,
 	})
+	now := time.Now()
 	w.mu.Lock()
 	if cached, ok := w.baselineCache[cacheKey]; ok {
-		w.mu.Unlock()
-		return cached.baseline, cached.found, nil
+		missExpired := !cached.found && (w.baselineMissTTL == 0 || cached.cachedAt.IsZero() || now.Sub(cached.cachedAt) >= w.baselineMissTTL)
+		if !missExpired {
+			w.mu.Unlock()
+			return cached.baseline, cached.found, nil
+		}
+		w.deleteBaselineKeyLocked(cacheKey)
 	}
 	w.mu.Unlock()
 
@@ -219,21 +530,165 @@ func (w *Writer) previousBaseline(ctx context.Context, candidate SourceMileageSa
 	if err != nil {
 		return sourceBaseline{}, false, err
 	}
-	if !found {
-		baseline, found, err = lookupCurrentSourceBaseline(ctx, w.query, candidate.VIN, candidate.StatDate, candidate.Protocol, candidate.SourceKey)
-		if err != nil {
-			return sourceBaseline{}, false, err
-		}
-	}
 
 	if found {
-		w.mu.Lock()
-		w.baselineCache[cacheKey] = sourceBaselineCacheEntry{baseline: baseline, found: true}
-		w.mu.Unlock()
+		w.cacheBaseline(candidate, cacheKey, sourceBaselineCacheEntry{baseline: baseline, found: true})
+	} else {
+		w.cacheBaseline(candidate, cacheKey, sourceBaselineCacheEntry{found: false})
 	}
 	return baseline, found, nil
 }
 
+func (w *Writer) markBaselineWritten(sample MetricSample, candidate SourceMileageSample) {
+	if candidate.QualityStatus != QualityOK {
+		return
+	}
+	baselineTotal := candidate.FirstTotalKM
+	if baselineTotal <= 0 {
+		baselineTotal = candidate.LatestTotalKM
+	}
+	baselineTime := candidate.FirstEventTime
+	if baselineTime.IsZero() {
+		baselineTime = candidate.LatestEventTime
+	}
+	if baselineTotal <= 0 || baselineTime.IsZero() {
+		return
+	}
+	cacheKey := mileageCacheKey(sample)
+	w.cacheBaseline(candidate, cacheKey, sourceBaselineCacheEntry{
+		baseline: sourceBaseline{
+			LatestTotalKM:   baselineTotal,
+			LatestEventTime: baselineTime,
+			QualityReason:   candidate.QualityReason,
+		},
+		found: true,
+	})
+}
+
+func (w *Writer) cacheBaseline(candidate SourceMileageSample, cacheKey string, entry sourceBaselineCacheEntry) {
+	if cacheKey == "" {
+		return
+	}
+	w.mu.Lock()
+	if entry.cachedAt.IsZero() {
+		entry.cachedAt = time.Now()
+	}
+	w.baselineCache[cacheKey] = entry
+	w.addCacheKeyLocked(w.baselineKeysByPrefix, mileageCachePrefix(MetricSample{
+		VIN:       candidate.VIN,
+		Protocol:  candidate.Protocol,
+		SourceKey: candidate.SourceKey,
+	}), cacheKey)
+	w.enforceCacheLimitsLocked()
+	w.mu.Unlock()
+}
+
+func (w *Writer) addCacheKeyLocked(index map[string]map[string]struct{}, prefix string, key string) {
+	if prefix == "" || key == "" {
+		return
+	}
+	keys := index[prefix]
+	if keys == nil {
+		keys = map[string]struct{}{}
+		index[prefix] = keys
+	}
+	keys[key] = struct{}{}
+}
+
+func (w *Writer) deleteMileagePrefixExceptLocked(prefix string, keep string) {
+	for key := range w.mileageKeysByPrefix[prefix] {
+		if key == keep {
+			continue
+		}
+		w.deleteMileageKeyLocked(key)
+	}
+}
+
+func (w *Writer) deleteBaselinePrefixExceptLocked(prefix string, keep string) {
+	for key := range w.baselineKeysByPrefix[prefix] {
+		if key == keep {
+			continue
+		}
+		w.deleteBaselineKeyLocked(key)
+	}
+}
+
+func (w *Writer) deleteProjectionPrefixExceptLocked(prefix string, keep string) {
+	for key := range w.projectionKeysByPrefix[prefix] {
+		if key == keep {
+			continue
+		}
+		w.deleteProjectionKeyLocked(key)
+	}
+}
+
+func (w *Writer) deleteMileageKeyLocked(key string) {
+	delete(w.lastTotalMileage, key)
+	w.deleteIndexedCacheKeyLocked(w.mileageKeysByPrefix, cacheKeyPrefix(key), key)
+}
+
+func (w *Writer) deleteBaselineKeyLocked(key string) {
+	delete(w.baselineCache, key)
+	w.deleteIndexedCacheKeyLocked(w.baselineKeysByPrefix, cacheKeyPrefix(key), key)
+}
+
+func (w *Writer) deleteProjectionKeyLocked(key string) {
+	delete(w.lastProjection, key)
+	w.deleteIndexedCacheKeyLocked(w.projectionKeysByPrefix, cacheKeyPrefix(key), key)
+}
+
+func (w *Writer) deleteIndexedCacheKeyLocked(index map[string]map[string]struct{}, prefix string, key string) {
+	if prefix == "" || key == "" {
+		return
+	}
+	keys := index[prefix]
+	if len(keys) == 0 {
+		return
+	}
+	delete(keys, key)
+	if len(keys) == 0 {
+		delete(index, prefix)
+	}
+}
+
+func (w *Writer) enforceCacheLimitsLocked() {
+	if w.maxCacheEntries <= 0 {
+		return
+	}
+	for len(w.lastTotalMileage) > w.maxCacheEntries {
+		victim := oldestCacheKeyByDate(w.lastTotalMileage)
+		if victim == "" {
+			break
+		}
+		w.deleteMileageKeyLocked(victim)
+		w.cacheEvictions.totalMileage++
+	}
+	for len(w.baselineCache) > w.maxCacheEntries {
+		victim := oldestCacheKeyByDate(w.baselineCache)
+		if victim == "" {
+			break
+		}
+		w.deleteBaselineKeyLocked(victim)
+		w.cacheEvictions.baseline++
+	}
+	for len(w.lastProjection) > w.maxCacheEntries {
+		victim := oldestProjectionCacheKey(w.lastProjection)
+		if victim == "" {
+			break
+		}
+		w.deleteProjectionKeyLocked(victim)
+		w.cacheEvictions.projection++
+	}
+	for len(w.lastSourceSeen) > w.maxCacheEntries {
+		victim := oldestSourceSeenKey(w.lastSourceSeen)
+		if victim == "" {
+			break
+		}
+		delete(w.lastSourceSeen, victim)
+		w.cacheEvictions.sourceSeen++
+	}
+}
+
 func mileageCachePrefix(sample MetricSample) string {
 	return fmt.Sprintf("%s|%s|%s|", sample.VIN, sample.Protocol, sample.SourceKey)
 }
@@ -242,34 +697,138 @@ func mileageCacheKey(sample MetricSample) string {
 	return mileageCachePrefix(sample) + sample.StatDate
 }
 
+func projectionCacheKey(sample MetricSample) string {
+	return projectionCachePrefix(sample) + sample.StatDate
+}
+
+func projectionCachePrefix(sample MetricSample) string {
+	return fmt.Sprintf("%s|%s|", sample.VIN, sample.Protocol)
+}
+
+func cacheKeyDateBefore(key string, cutoffDate string) bool {
+	date := cacheKeyDate(key)
+	if len(date) != len("2006-01-02") {
+		return false
+	}
+	return date < cutoffDate
+}
+
+func cacheKeyPrefix(key string) string {
+	index := strings.LastIndex(key, "|")
+	if index < 0 {
+		return ""
+	}
+	return key[:index+1]
+}
+
+func cacheKeyDate(key string) string {
+	index := strings.LastIndex(key, "|")
+	if index < 0 || index == len(key)-1 {
+		return ""
+	}
+	return key[index+1:]
+}
+
+func oldestCacheKeyByDate[T any](items map[string]T) string {
+	victim := ""
+	victimDate := ""
+	for key := range items {
+		date := cacheKeyDate(key)
+		if date == "" {
+			date = "9999-99-99"
+		}
+		if victim == "" || date < victimDate || (date == victimDate && key < victim) {
+			victim = key
+			victimDate = date
+		}
+	}
+	return victim
+}
+
+func oldestProjectionCacheKey(items map[string]projectionCacheEntry) string {
+	victim := ""
+	var victimTime time.Time
+	for key, entry := range items {
+		if victim == "" ||
+			(!entry.projectedAt.IsZero() && (victimTime.IsZero() || entry.projectedAt.Before(victimTime))) ||
+			(entry.projectedAt.Equal(victimTime) && key < victim) {
+			victim = key
+			victimTime = entry.projectedAt
+		}
+	}
+	if victim != "" {
+		return victim
+	}
+	return oldestCacheKeyByDate(items)
+}
+
+func oldestSourceSeenKey(items map[string]time.Time) string {
+	victim := ""
+	var victimTime time.Time
+	for key, seenAt := range items {
+		if victim == "" ||
+			(!seenAt.IsZero() && (victimTime.IsZero() || seenAt.Before(victimTime))) ||
+			(seenAt.Equal(victimTime) && key < victim) {
+			victim = key
+			victimTime = seenAt
+		}
+	}
+	return victim
+}
+
+type projectionCacheEntry struct {
+	projectedAt time.Time
+	sourceKeys  map[string]struct{}
+}
+
 type sourceBaselineCacheEntry struct {
 	baseline sourceBaseline
 	found    bool
+	cachedAt time.Time
 }
 
 func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, error) {
+	samples, _, err := samplesFromEnvelopeWithResult(env, loc)
+	return samples, err
+}
+
+func samplesFromEnvelopeWithResult(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, AppendResult, error) {
+	var result AppendResult
+	if len(env.Fields) == 0 {
+		result.SamplesSkippedMissingFields = 1
+		return nil, result, nil
+	}
 	vin := strings.TrimSpace(env.VIN)
 	if vin == "" {
-		return nil, nil
+		result.SamplesSkippedMissingVIN = 1
+		return nil, result, nil
 	}
 	totalMileage, ok := totalMileageKMFromEnvelope(env)
 	if !ok {
-		return nil, nil
+		if isMileageCandidateEnvelope(env) {
+			result.SamplesSkippedMissingMileage = 1
+		} else {
+			result.SamplesSkippedNonMileageFrame = 1
+		}
+		return nil, result, nil
 	}
 	if totalMileage <= 0 {
-		return nil, nil
+		result.SamplesSkippedNonPositiveMileage = 1
+		return nil, result, nil
 	}
 	if loc == nil {
 		loc = time.FixedZone("Asia/Shanghai", 8*3600)
 	}
-	eventMS := env.EventTimeMS
-	if eventMS <= 0 {
-		eventMS = env.ReceivedAtMS
+	eventMS, reason, ok := envelope.NormalizedEventTimeMSWithReason(env)
+	if !ok {
+		result.SamplesSkippedMissingTime = 1
+		return nil, result, nil
 	}
-	if eventMS <= 0 {
-		return nil, errors.New("event or received time is required")
+	if reason == envelope.EventTimeReasonReceivedFutureEvent {
+		result.SamplesAdjustedFutureEventTime = 1
 	}
 	statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02")
+	result.SamplesFound = 1
 	return []MetricSample{{
 		VIN:            vin,
 		Protocol:       env.Protocol,
@@ -280,11 +839,16 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr
 		Phone:          strings.TrimSpace(env.Phone),
 		DeviceID:       strings.TrimSpace(env.DeviceID),
 		SourceEndpoint: strings.TrimSpace(env.SourceEndpoint),
-	}}, nil
+		PlatformName:   strings.TrimSpace(env.PlatformName),
+	}}, result, nil
+}
+
+func statEventTimeMS(env envelope.FrameEnvelope) (int64, bool) {
+	return envelope.NormalizedEventTimeMS(env)
 }
 
 func sourceKey(env envelope.FrameEnvelope) string {
-	return SourceKey(env.Protocol, env.Phone, env.DeviceID, NormalizeSourceIP(env.SourceEndpoint))
+	return SourceKeyForSource(env.Protocol, env.Phone, env.DeviceID, NormalizeSourceIP(env.SourceEndpoint), env.SourceKind, env.SourceCode)
 }
 
 func isIgnorableSchemaChangeError(err error) bool {
@@ -300,41 +864,59 @@ func isIgnorableSchemaChangeError(err error) bool {
 		strings.Contains(text, "1091")
 }
 
-type mileageFieldMapping struct {
-	key   string
-	scale float64
-}
+type mileageFieldMapping = telemetry.MileageFieldMapping
 
 func totalMileageKMFromEnvelope(env envelope.FrameEnvelope) (float64, bool) {
-	if value, ok := floatField(env, envelope.FieldTotalMileageKM); ok {
-		return value, true
-	}
-	for _, mapping := range mileageMappingsByProtocol(env.Protocol) {
-		value, ok := floatField(env, mapping.key)
-		if !ok {
-			continue
-		}
-		return value * mapping.scale, true
-	}
-	return 0, false
+	return telemetry.TotalMileageKM(env.Protocol, env.Fields)
 }
 
 func mileageMappingsByProtocol(protocol envelope.Protocol) []mileageFieldMapping {
-	switch protocol {
-	case envelope.ProtocolGB32960:
-		return []mileageFieldMapping{{key: "gb32960.vehicle.total_mileage_km", scale: 1}}
+	return telemetry.MileageFieldMappings(protocol)
+}
+
+func isMileageCandidateEnvelope(env envelope.FrameEnvelope) bool {
+	switch env.Protocol {
 	case envelope.ProtocolJT808:
-		return []mileageFieldMapping{{key: "jt808.location.total_mileage_km", scale: 1}}
+		messageID := strings.TrimSpace(env.MessageID)
+		return strings.EqualFold(messageID, "0x0200") ||
+			hasFieldPrefix(env.Fields, "jt808.location.") ||
+			hasAnyField(env.Fields, envelope.FieldLatitude, envelope.FieldLongitude, envelope.FieldSpeedKMH)
+	case envelope.ProtocolGB32960:
+		return hasFieldPrefix(env.Fields, "gb32960.vehicle.") ||
+			hasAnyField(env.Fields, envelope.FieldLatitude, envelope.FieldLongitude, envelope.FieldSpeedKMH, envelope.FieldSOCPercent)
 	case envelope.ProtocolYutongMQTT:
-		return []mileageFieldMapping{
-			{key: "yutong_mqtt.data.total_mileage", scale: 0.001},
-			{key: "yutong_mqtt.root.data.total_mileage", scale: 0.001},
-		}
+		return hasFieldPrefix(env.Fields, "yutong_mqtt.data.") ||
+			hasFieldPrefix(env.Fields, "yutong_mqtt.root.data.") ||
+			hasAnyField(env.Fields, envelope.FieldLatitude, envelope.FieldLongitude, envelope.FieldSpeedKMH, envelope.FieldSOCPercent)
 	default:
-		return nil
+		return true
 	}
 }
 
+func hasFieldPrefix(fields map[string]any, prefix string) bool {
+	if len(fields) == 0 || prefix == "" {
+		return false
+	}
+	for key := range fields {
+		if strings.HasPrefix(key, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+func hasAnyField(fields map[string]any, keys ...string) bool {
+	if len(fields) == 0 {
+		return false
+	}
+	for _, key := range keys {
+		if _, ok := fields[key]; ok {
+			return true
+		}
+	}
+	return false
+}
+
 func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
 	if env.Fields == nil {
 		return 0, false