package main import ( "context" "database/sql" "log/slog" "sort" "strings" "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats" ) // Rebuild each GPS vehicle-day atomically. Retained RAW is not guaranteed to // contain every source or a later sample already accepted by the live writer. // Preserve those durable candidates instead of deleting the complete day. func writeProtectedGPSAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*metricAgg) (int64, error) { groups := map[string][]*metricAgg{} for _, agg := range aggregates { if strings.TrimSpace(stats.NormalizeSourceIP(agg.SourceEndpoint)) == "" { continue } key := agg.VIN + "|" + agg.Date groups[key] = append(groups[key], agg) } keys := make([]string, 0, len(groups)) for key := range groups { keys = append(keys, key) } sort.Strings(keys) var written int64 for _, key := range keys { group := groups[key] sort.Slice(group, func(i, j int) bool { return group[i].SourceKey < group[j].SourceKey }) n, err := writeProtectedGPSDay(ctx, db, group) if err != nil { return written, err } written += n } return written, nil } func writeProtectedGPSDay(ctx context.Context, db *sql.DB, group []*metricAgg) (int64, error) { first := group[0] tx, err := db.BeginTx(ctx, nil) if err != nil { return 0, err } defer tx.Rollback() existing, selectedManual, err := lockGPSCandidates(ctx, tx, first.VIN, first.Date) if err != nil { return 0, err } var written int64 for _, agg := range group { candidate := backfillCandidate(agg) if old, ok := existing[candidate.SourceKey]; ok { if isManualGPSCandidate(old) { continue } candidate = mergeProtectedGPSCandidate(old, candidate) } identity := stats.SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: candidate.SourceIP, SourceEndpoint: candidate.SourceEndpoint, SourceCode: agg.SourceCode, PlatformName: agg.PlatformName, SourceKind: agg.SourceKind} if candidate.QualityReason != stats.QualityReasonGPSCoordinate { if err := stats.UpsertDataSource(ctx, tx, identity, candidate.LatestEventTime); err != nil { return 0, err } } if err := replaceBackfillCandidate(ctx, tx, candidate); err != nil { return 0, err } written++ } // An explicitly selected imported report is a business override. Keep that // final result while still correcting the underlying automatic candidates. if !selectedManual { if err := stats.ProjectDailyMileage(ctx, tx, first.VIN, first.Date, envelope.ProtocolJT808); err != nil { return 0, err } } if err := tx.Commit(); err != nil { return 0, err } return written, nil } func normalizeProtectedGPSDay(ctx context.Context, db *sql.DB, vin, date string) error { tx, err := db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() _, manual, err := lockGPSCandidates(ctx, tx, vin, date) if err != nil { return err } if manual { return tx.Commit() } if err := stats.NormalizePlatformSourceMileage(ctx, tx, vin, date, envelope.ProtocolJT808); err != nil { return err } if err := stats.ProjectDailyMileage(ctx, tx, vin, date, envelope.ProtocolJT808); err != nil { return err } return tx.Commit() } func isManualGPSCandidate(c stats.SourceMileageSample) bool { return strings.HasPrefix(c.QualityReason, "manual") || strings.HasPrefix(c.SourceIP, "manual") || strings.HasPrefix(c.SourceIP, "legacy-mysql") } func backfillCandidate(agg *metricAgg) stats.SourceMileageSample { c := stats.SourceMileageSample{ VIN: agg.VIN, StatDate: agg.Date, Protocol: agg.Protocol, SourceKey: agg.SourceKey, SourceIP: stats.NormalizeSourceIP(agg.SourceEndpoint), SourceEndpoint: agg.SourceEndpoint, Phone: agg.Phone, DeviceID: agg.DeviceID, PlatformName: agg.PlatformName, FirstTotalKM: agg.FirstKM, LatestTotalKM: agg.LatestKM, DailyKM: stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM), PureHydrogenMileageKM: agg.PureHydrogenMileageKM, SampleCount: agg.Count, PureHydrogenSampleCount: agg.PureHydrogenSampleCount, FirstEventTime: agg.FirstEventTime, LatestEventTime: agg.LatestEventTime, LatestPureHydrogenActive: agg.LatestPureHydrogenActive, LatestPureHydrogenModeKnown: agg.LatestPureHydrogenModeKnown, QualityStatus: agg.QualityStatus, QualityReason: agg.QualityReason, } stats.ApplyMileageQualityRules(&c) return c } func mergeProtectedGPSCandidate(old, next stats.SourceMileageSample) stats.SourceMileageSample { if next.QualityReason == stats.QualityReasonGPSCoordinate || old.QualityReason == stats.QualityReasonGPSCoordinate { if old.LatestEventTime.After(next.LatestEventTime) { return old } return next } if old.LatestEventTime.After(next.LatestEventTime) { slog.Info("preserved newer durable GPS end", "vin", next.VIN, "date", next.StatDate, "source_key", next.SourceKey, "raw_end", next.LatestEventTime, "durable_end", old.LatestEventTime) next.LatestEventTime, next.LatestTotalKM = old.LatestEventTime, old.LatestTotalKM next.SourceEndpoint = old.SourceEndpoint next.LatestPureHydrogenActive, next.LatestPureHydrogenModeKnown = old.LatestPureHydrogenActive, old.LatestPureHydrogenModeKnown } // Prefer the newest prior-day boundary; otherwise retain the earliest // current-day sample. Always move the timestamp and odometer together. if preferDurableGPSFirst(old.FirstEventTime, next.FirstEventTime, next.StatDate) { next.FirstEventTime, next.FirstTotalKM = old.FirstEventTime, old.FirstTotalKM } if old.SampleCount > next.SampleCount { next.SampleCount = old.SampleCount } if old.PureHydrogenSampleCount > next.PureHydrogenSampleCount { next.PureHydrogenSampleCount = old.PureHydrogenSampleCount } if old.PureHydrogenMileageKM > next.PureHydrogenMileageKM { next.PureHydrogenMileageKM = old.PureHydrogenMileageKM } next.DailyKM = stats.DailyMileageFromDayBoundary(next.FirstTotalKM, next.LatestTotalKM) next.QualityStatus = stats.QualityOK if next.FirstEventTime.Format("2006-01-02") < next.StatDate { next.QualityReason = stats.QualityReasonHistorical } else { next.QualityReason = stats.QualityReasonCurrentDayFirst } stats.ApplyMileageQualityRules(&next) return next } func preferDurableGPSFirst(old, next time.Time, date string) bool { if old.IsZero() { return false } if next.IsZero() { return true } oldBefore, nextBefore := old.Format("2006-01-02") < date, next.Format("2006-01-02") < date if oldBefore != nextBefore { return oldBefore } if oldBefore { return old.After(next) } return old.Before(next) } func lockGPSCandidates(ctx context.Context, tx *sql.Tx, vin, date string) (map[string]stats.SourceMileageSample, bool, error) { rows, err := tx.QueryContext(ctx, `SELECT source_key,source_ip,COALESCE(source_endpoint,''),COALESCE(phone,''),COALESCE(device_id,''),COALESCE(platform_name,''), first_total_mileage_km,latest_total_mileage_km,daily_mileage_km,pure_hydrogen_mileage_km,sample_count,pure_hydrogen_sample_count, first_event_time,latest_event_time,latest_pure_hydrogen_active,latest_pure_hydrogen_mode_known,quality_status,COALESCE(quality_reason,''),is_selected FROM vehicle_daily_mileage_source WHERE vin=? AND stat_date=? AND protocol='JT808' FOR UPDATE`, vin, date) if err != nil { return nil, false, err } defer rows.Close() out := map[string]stats.SourceMileageSample{} manual := false for rows.Next() { c := stats.SourceMileageSample{VIN: vin, StatDate: date, Protocol: envelope.ProtocolJT808} var first, last sql.NullTime var firstKM, lastKM sql.NullFloat64 var selected bool if err := rows.Scan(&c.SourceKey, &c.SourceIP, &c.SourceEndpoint, &c.Phone, &c.DeviceID, &c.PlatformName, &firstKM, &lastKM, &c.DailyKM, &c.PureHydrogenMileageKM, &c.SampleCount, &c.PureHydrogenSampleCount, &first, &last, &c.LatestPureHydrogenActive, &c.LatestPureHydrogenModeKnown, &c.QualityStatus, &c.QualityReason, &selected); err != nil { return nil, false, err } c.FirstEventTime, c.LatestEventTime = first.Time, last.Time c.FirstTotalKM, c.LatestTotalKM = firstKM.Float64, lastKM.Float64 out[c.SourceKey] = c if selected && isManualGPSCandidate(c) { manual = true } } return out, manual, rows.Err() } func replaceBackfillCandidate(ctx context.Context, tx *sql.Tx, c stats.SourceMileageSample) error { _, err := tx.ExecContext(ctx, `INSERT INTO vehicle_daily_mileage_source (vin,stat_date,protocol,source_key,source_ip,source_endpoint,phone,device_id,platform_name, first_total_mileage_km,latest_total_mileage_km,daily_mileage_km,pure_hydrogen_mileage_km, sample_count,pure_hydrogen_sample_count,first_event_time,latest_event_time, latest_pure_hydrogen_active,latest_pure_hydrogen_mode_known,quality_status,quality_reason) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE source_ip=VALUES(source_ip),source_endpoint=VALUES(source_endpoint),phone=VALUES(phone),device_id=VALUES(device_id), platform_name=VALUES(platform_name),first_total_mileage_km=VALUES(first_total_mileage_km),latest_total_mileage_km=VALUES(latest_total_mileage_km), daily_mileage_km=VALUES(daily_mileage_km),pure_hydrogen_mileage_km=VALUES(pure_hydrogen_mileage_km), sample_count=VALUES(sample_count),pure_hydrogen_sample_count=VALUES(pure_hydrogen_sample_count), first_event_time=VALUES(first_event_time),latest_event_time=VALUES(latest_event_time), latest_pure_hydrogen_active=VALUES(latest_pure_hydrogen_active),latest_pure_hydrogen_mode_known=VALUES(latest_pure_hydrogen_mode_known), quality_status=VALUES(quality_status),quality_reason=VALUES(quality_reason),updated_at=CURRENT_TIMESTAMP`, c.VIN, c.StatDate, string(c.Protocol), c.SourceKey, c.SourceIP, c.SourceEndpoint, c.Phone, c.DeviceID, c.PlatformName, c.FirstTotalKM, c.LatestTotalKM, c.DailyKM, c.PureHydrogenMileageKM, c.SampleCount, c.PureHydrogenSampleCount, c.FirstEventTime.Truncate(time.Second), c.LatestEventTime.Truncate(time.Second), c.LatestPureHydrogenActive, c.LatestPureHydrogenModeKnown, c.QualityStatus, c.QualityReason) return err }