package main import ( "context" "database/sql" "fmt" "log/slog" "sort" "strings" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats" ) type backfillSourceID struct { protocol envelope.Protocol ip string } // Resolve forwarding endpoints before grouping RAW or looking up durable daily // boundaries. Normalizing only after writing cannot repair an IP-key lookup // that has already missed the newer @PLATFORM boundary. func loadBackfillSourceCatalog(ctx context.Context, db *sql.DB) (map[backfillSourceID]stats.SourceIdentity, error) { rows, err := db.QueryContext(ctx, `SELECT protocol, source_ip, COALESCE(source_code, ''), COALESCE(platform_name, ''), source_kind FROM vehicle_data_source WHERE protocol = 'JT808'`) if err != nil { return nil, err } defer rows.Close() catalog := map[backfillSourceID]stats.SourceIdentity{} for rows.Next() { var protocol string var source stats.SourceIdentity if err := rows.Scan(&protocol, &source.SourceIP, &source.SourceCode, &source.PlatformName, &source.SourceKind); err != nil { return nil, err } source.Protocol = envelope.Protocol(strings.TrimSpace(protocol)) source.SourceIP = stats.NormalizeSourceIP(source.SourceIP) source.SourceCode = strings.TrimSpace(source.SourceCode) source.PlatformName = strings.TrimSpace(source.PlatformName) source.SourceKind = strings.ToUpper(strings.TrimSpace(source.SourceKind)) if source.SourceIP != "" { catalog[backfillSourceID{source.Protocol, source.SourceIP}] = source } } return catalog, rows.Err() } func backfillSourceMetadata(cfg config, protocol envelope.Protocol, endpoint string) (string, string, string) { if source, ok := cfg.SourceCatalog[backfillSourceID{protocol, stats.NormalizeSourceIP(endpoint)}]; ok { return source.SourceCode, source.PlatformName, source.SourceKind } return knownPlatformSourceMetadata(protocol, endpoint) } func mergeDailySourceRows(existing, incoming dailySourceLast) dailySourceLast { latest := existing if incoming.TS.After(existing.TS) { latest = incoming } if incoming.FirstTS.Before(existing.FirstTS) { latest.FirstTS, latest.FirstTotalKM = incoming.FirstTS, incoming.FirstTotalKM } else { latest.FirstTS, latest.FirstTotalKM = existing.FirstTS, existing.FirstTotalKM } latest.RawSampleCount = existing.RawSampleCount + incoming.RawSampleCount return latest } func backfillVINPredicate(cfg config, column string) string { if len(cfg.VINs) == 0 { return "" } values := make([]string, 0, len(cfg.VINs)) for _, vin := range cfg.VINs { values = append(values, "'"+quote(vin)+"'") } return column + " IN (" + strings.Join(values, ",") + ")" } func appendBackfillVINScope(query string, cfg config, column string) string { if predicate := backfillVINPredicate(cfg, column); predicate != "" { query += "\n AND " + predicate } return query } func logBackfillCandidates(aggregates map[string]*metricAgg) { keys := make([]string, 0, len(aggregates)) for key := range aggregates { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { a := aggregates[key] slog.Info("backfill candidate", "vin", a.VIN, "date", a.Date, "protocol", a.Protocol, "source_key", a.SourceKey, "platform", a.PlatformName, "baseline_time", a.FirstEventTime.Format("2006-01-02 15:04:05"), "baseline_km", a.FirstKM, "end_time", a.LatestEventTime.Format("2006-01-02 15:04:05"), "end_km", a.LatestKM, "daily_km", fmt.Sprintf("%.3f", stats.DailyMileageFromDayBoundary(a.FirstKM, a.LatestKM)), "samples", a.Count, "quality", a.QualityStatus) } }