lingniu-vehicle-ingest/go/vehicle-gateway/cmd/stats-backfill/protected_write.go

package main

import (
	"context"
	"database/sql"
	"log/slog"
	"sort"
	"strings"
	"time"

	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
)

// Rebuild each GPS vehicle-day atomically. Retained RAW is not guaranteed to
// contain every source or a later sample already accepted by the live writer.
// Preserve those durable candidates instead of deleting the complete day.
func writeProtectedGPSAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*metricAgg) (int64, error) {
	groups := map[string][]*metricAgg{}
	for _, agg := range aggregates {
		if strings.TrimSpace(stats.NormalizeSourceIP(agg.SourceEndpoint)) == "" {
			continue
		}
		key := agg.VIN + "|" + agg.Date
		groups[key] = append(groups[key], agg)
	}
	keys := make([]string, 0, len(groups))
	for key := range groups {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	var written int64
	for _, key := range keys {
		group := groups[key]
		sort.Slice(group, func(i, j int) bool { return group[i].SourceKey < group[j].SourceKey })
		n, err := writeProtectedGPSDay(ctx, db, group)
		if err != nil {
			return written, err
		}
		written += n
	}
	return written, nil
}

func writeProtectedGPSDay(ctx context.Context, db *sql.DB, group []*metricAgg) (int64, error) {
	first := group[0]
	tx, err := db.BeginTx(ctx, nil)
	if err != nil {
		return 0, err
	}
	defer tx.Rollback()
	existing, selectedManual, err := lockGPSCandidates(ctx, tx, first.VIN, first.Date)
	if err != nil {
		return 0, err
	}
	var written int64
	for _, agg := range group {
		candidate := backfillCandidate(agg)
		if old, ok := existing[candidate.SourceKey]; ok {
			if isManualGPSCandidate(old) {
				continue
			}
			candidate = mergeProtectedGPSCandidate(old, candidate)
		}
		identity := stats.SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: candidate.SourceIP,
			SourceEndpoint: candidate.SourceEndpoint, SourceCode: agg.SourceCode, PlatformName: agg.PlatformName, SourceKind: agg.SourceKind}
		if candidate.QualityReason != stats.QualityReasonGPSCoordinate {
			if err := stats.UpsertDataSource(ctx, tx, identity, candidate.LatestEventTime); err != nil {
				return 0, err
			}
		}
		if err := replaceBackfillCandidate(ctx, tx, candidate); err != nil {
			return 0, err
		}
		written++
	}
	// An explicitly selected imported report is a business override. Keep that
	// final result while still correcting the underlying automatic candidates.
	if !selectedManual {
		if err := stats.ProjectDailyMileage(ctx, tx, first.VIN, first.Date, envelope.ProtocolJT808); err != nil {
			return 0, err
		}
	}
	if err := tx.Commit(); err != nil {
		return 0, err
	}
	return written, nil
}

func normalizeProtectedGPSDay(ctx context.Context, db *sql.DB, vin, date string) error {
	tx, err := db.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	defer tx.Rollback()
	_, manual, err := lockGPSCandidates(ctx, tx, vin, date)
	if err != nil {
		return err
	}
	if manual {
		return tx.Commit()
	}
	if err := stats.NormalizePlatformSourceMileage(ctx, tx, vin, date, envelope.ProtocolJT808); err != nil {
		return err
	}
	if err := stats.ProjectDailyMileage(ctx, tx, vin, date, envelope.ProtocolJT808); err != nil {
		return err
	}
	return tx.Commit()
}

func isManualGPSCandidate(c stats.SourceMileageSample) bool {
	return strings.HasPrefix(c.QualityReason, "manual") || strings.HasPrefix(c.SourceIP, "manual") || strings.HasPrefix(c.SourceIP, "legacy-mysql")
}

func backfillCandidate(agg *metricAgg) stats.SourceMileageSample {
	c := stats.SourceMileageSample{
		VIN: agg.VIN, StatDate: agg.Date, Protocol: agg.Protocol, SourceKey: agg.SourceKey,
		SourceIP: stats.NormalizeSourceIP(agg.SourceEndpoint), SourceEndpoint: agg.SourceEndpoint,
		Phone: agg.Phone, DeviceID: agg.DeviceID, PlatformName: agg.PlatformName,
		FirstTotalKM: agg.FirstKM, LatestTotalKM: agg.LatestKM,
		DailyKM:               stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM),
		PureHydrogenMileageKM: agg.PureHydrogenMileageKM, SampleCount: agg.Count,
		PureHydrogenSampleCount: agg.PureHydrogenSampleCount,
		FirstEventTime:          agg.FirstEventTime, LatestEventTime: agg.LatestEventTime,
		LatestPureHydrogenActive: agg.LatestPureHydrogenActive, LatestPureHydrogenModeKnown: agg.LatestPureHydrogenModeKnown,
		QualityStatus: agg.QualityStatus, QualityReason: agg.QualityReason,
	}
	stats.ApplyMileageQualityRules(&c)
	return c
}

func mergeProtectedGPSCandidate(old, next stats.SourceMileageSample) stats.SourceMileageSample {
	if next.QualityReason == stats.QualityReasonGPSCoordinate || old.QualityReason == stats.QualityReasonGPSCoordinate {
		if old.LatestEventTime.After(next.LatestEventTime) {
			return old
		}
		return next
	}
	if old.LatestEventTime.After(next.LatestEventTime) {
		slog.Info("preserved newer durable GPS end", "vin", next.VIN, "date", next.StatDate,
			"source_key", next.SourceKey, "raw_end", next.LatestEventTime, "durable_end", old.LatestEventTime)
		next.LatestEventTime, next.LatestTotalKM = old.LatestEventTime, old.LatestTotalKM
		next.SourceEndpoint = old.SourceEndpoint
		next.LatestPureHydrogenActive, next.LatestPureHydrogenModeKnown = old.LatestPureHydrogenActive, old.LatestPureHydrogenModeKnown
	}
	// Prefer the newest prior-day boundary; otherwise retain the earliest
	// current-day sample. Always move the timestamp and odometer together.
	if preferDurableGPSFirst(old.FirstEventTime, next.FirstEventTime, next.StatDate) {
		next.FirstEventTime, next.FirstTotalKM = old.FirstEventTime, old.FirstTotalKM
	}
	if old.SampleCount > next.SampleCount {
		next.SampleCount = old.SampleCount
	}
	if old.PureHydrogenSampleCount > next.PureHydrogenSampleCount {
		next.PureHydrogenSampleCount = old.PureHydrogenSampleCount
	}
	if old.PureHydrogenMileageKM > next.PureHydrogenMileageKM {
		next.PureHydrogenMileageKM = old.PureHydrogenMileageKM
	}
	next.DailyKM = stats.DailyMileageFromDayBoundary(next.FirstTotalKM, next.LatestTotalKM)
	next.QualityStatus = stats.QualityOK
	if next.FirstEventTime.Format("2006-01-02") < next.StatDate {
		next.QualityReason = stats.QualityReasonHistorical
	} else {
		next.QualityReason = stats.QualityReasonCurrentDayFirst
	}
	stats.ApplyMileageQualityRules(&next)
	return next
}

func preferDurableGPSFirst(old, next time.Time, date string) bool {
	if old.IsZero() {
		return false
	}
	if next.IsZero() {
		return true
	}
	oldBefore, nextBefore := old.Format("2006-01-02") < date, next.Format("2006-01-02") < date
	if oldBefore != nextBefore {
		return oldBefore
	}
	if oldBefore {
		return old.After(next)
	}
	return old.Before(next)
}

func lockGPSCandidates(ctx context.Context, tx *sql.Tx, vin, date string) (map[string]stats.SourceMileageSample, bool, error) {
	rows, err := tx.QueryContext(ctx, `SELECT source_key,source_ip,COALESCE(source_endpoint,''),COALESCE(phone,''),COALESCE(device_id,''),COALESCE(platform_name,''),
first_total_mileage_km,latest_total_mileage_km,daily_mileage_km,pure_hydrogen_mileage_km,sample_count,pure_hydrogen_sample_count,
first_event_time,latest_event_time,latest_pure_hydrogen_active,latest_pure_hydrogen_mode_known,quality_status,COALESCE(quality_reason,''),is_selected
FROM vehicle_daily_mileage_source WHERE vin=? AND stat_date=? AND protocol='JT808' FOR UPDATE`, vin, date)
	if err != nil {
		return nil, false, err
	}
	defer rows.Close()
	out := map[string]stats.SourceMileageSample{}
	manual := false
	for rows.Next() {
		c := stats.SourceMileageSample{VIN: vin, StatDate: date, Protocol: envelope.ProtocolJT808}
		var first, last sql.NullTime
		var firstKM, lastKM sql.NullFloat64
		var selected bool
		if err := rows.Scan(&c.SourceKey, &c.SourceIP, &c.SourceEndpoint, &c.Phone, &c.DeviceID, &c.PlatformName,
			&firstKM, &lastKM, &c.DailyKM, &c.PureHydrogenMileageKM, &c.SampleCount, &c.PureHydrogenSampleCount,
			&first, &last, &c.LatestPureHydrogenActive, &c.LatestPureHydrogenModeKnown, &c.QualityStatus, &c.QualityReason, &selected); err != nil {
			return nil, false, err
		}
		c.FirstEventTime, c.LatestEventTime = first.Time, last.Time
		c.FirstTotalKM, c.LatestTotalKM = firstKM.Float64, lastKM.Float64
		out[c.SourceKey] = c
		if selected && isManualGPSCandidate(c) {
			manual = true
		}
	}
	return out, manual, rows.Err()
}

func replaceBackfillCandidate(ctx context.Context, tx *sql.Tx, c stats.SourceMileageSample) error {
	_, err := tx.ExecContext(ctx, `INSERT INTO vehicle_daily_mileage_source
(vin,stat_date,protocol,source_key,source_ip,source_endpoint,phone,device_id,platform_name,
first_total_mileage_km,latest_total_mileage_km,daily_mileage_km,pure_hydrogen_mileage_km,
sample_count,pure_hydrogen_sample_count,first_event_time,latest_event_time,
latest_pure_hydrogen_active,latest_pure_hydrogen_mode_known,quality_status,quality_reason)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
ON DUPLICATE KEY UPDATE source_ip=VALUES(source_ip),source_endpoint=VALUES(source_endpoint),phone=VALUES(phone),device_id=VALUES(device_id),
platform_name=VALUES(platform_name),first_total_mileage_km=VALUES(first_total_mileage_km),latest_total_mileage_km=VALUES(latest_total_mileage_km),
daily_mileage_km=VALUES(daily_mileage_km),pure_hydrogen_mileage_km=VALUES(pure_hydrogen_mileage_km),
sample_count=VALUES(sample_count),pure_hydrogen_sample_count=VALUES(pure_hydrogen_sample_count),
first_event_time=VALUES(first_event_time),latest_event_time=VALUES(latest_event_time),
latest_pure_hydrogen_active=VALUES(latest_pure_hydrogen_active),latest_pure_hydrogen_mode_known=VALUES(latest_pure_hydrogen_mode_known),
quality_status=VALUES(quality_status),quality_reason=VALUES(quality_reason),updated_at=CURRENT_TIMESTAMP`,
		c.VIN, c.StatDate, string(c.Protocol), c.SourceKey, c.SourceIP, c.SourceEndpoint, c.Phone, c.DeviceID, c.PlatformName,
		c.FirstTotalKM, c.LatestTotalKM, c.DailyKM, c.PureHydrogenMileageKM, c.SampleCount, c.PureHydrogenSampleCount,
		c.FirstEventTime.Truncate(time.Second), c.LatestEventTime.Truncate(time.Second),
		c.LatestPureHydrogenActive, c.LatestPureHydrogenModeKnown, c.QualityStatus, c.QualityReason)
	return err
}